packages feed

folly-clib 0.0 → 20250713.1537

raw patch · 1456 files changed

+348212/−67 lines, 1456 files

This diff is very large; some files are shown as “too large to diff”. Download the raw patch for the complete diff.

Files

+ fast_float-8.0.0/include/fast_float/ascii_number.h view
@@ -0,0 +1,588 @@+#ifndef FASTFLOAT_ASCII_NUMBER_H+#define FASTFLOAT_ASCII_NUMBER_H++#include <cctype>+#include <cstdint>+#include <cstring>+#include <iterator>+#include <limits>+#include <type_traits>++#include "float_common.h"++#ifdef FASTFLOAT_SSE2+#include <emmintrin.h>+#endif++#ifdef FASTFLOAT_NEON+#include <arm_neon.h>+#endif++namespace fast_float {++template <typename UC> fastfloat_really_inline constexpr bool has_simd_opt() {+#ifdef FASTFLOAT_HAS_SIMD+  return std::is_same<UC, char16_t>::value;+#else+  return false;+#endif+}++// Next function can be micro-optimized, but compilers are entirely+// able to optimize it well.+template <typename UC>+fastfloat_really_inline constexpr bool is_integer(UC c) noexcept {+  return !(c > UC('9') || c < UC('0'));+}++fastfloat_really_inline constexpr uint64_t byteswap(uint64_t val) {+  return (val & 0xFF00000000000000) >> 56 | (val & 0x00FF000000000000) >> 40 |+         (val & 0x0000FF0000000000) >> 24 | (val & 0x000000FF00000000) >> 8 |+         (val & 0x00000000FF000000) << 8 | (val & 0x0000000000FF0000) << 24 |+         (val & 0x000000000000FF00) << 40 | (val & 0x00000000000000FF) << 56;+}++// Read 8 UC into a u64. Truncates UC if not char.+template <typename UC>+fastfloat_really_inline FASTFLOAT_CONSTEXPR20 uint64_t+read8_to_u64(UC const *chars) {+  if (cpp20_and_in_constexpr() || !std::is_same<UC, char>::value) {+    uint64_t val = 0;+    for (int i = 0; i < 8; ++i) {+      val |= uint64_t(uint8_t(*chars)) << (i * 8);+      ++chars;+    }+    return val;+  }+  uint64_t val;+  ::memcpy(&val, chars, sizeof(uint64_t));+#if FASTFLOAT_IS_BIG_ENDIAN == 1+  // Need to read as-if the number was in little-endian order.+  val = byteswap(val);+#endif+  return val;+}++#ifdef FASTFLOAT_SSE2++fastfloat_really_inline uint64_t simd_read8_to_u64(__m128i const data) {+  FASTFLOAT_SIMD_DISABLE_WARNINGS+  __m128i const packed = _mm_packus_epi16(data, data);+#ifdef FASTFLOAT_64BIT+  return uint64_t(_mm_cvtsi128_si64(packed));+#else+  uint64_t value;+  // Visual Studio + older versions of GCC don't support _mm_storeu_si64+  _mm_storel_epi64(reinterpret_cast<__m128i *>(&value), packed);+  return value;+#endif+  FASTFLOAT_SIMD_RESTORE_WARNINGS+}++fastfloat_really_inline uint64_t simd_read8_to_u64(char16_t const *chars) {+  FASTFLOAT_SIMD_DISABLE_WARNINGS+  return simd_read8_to_u64(+      _mm_loadu_si128(reinterpret_cast<__m128i const *>(chars)));+  FASTFLOAT_SIMD_RESTORE_WARNINGS+}++#elif defined(FASTFLOAT_NEON)++fastfloat_really_inline uint64_t simd_read8_to_u64(uint16x8_t const data) {+  FASTFLOAT_SIMD_DISABLE_WARNINGS+  uint8x8_t utf8_packed = vmovn_u16(data);+  return vget_lane_u64(vreinterpret_u64_u8(utf8_packed), 0);+  FASTFLOAT_SIMD_RESTORE_WARNINGS+}++fastfloat_really_inline uint64_t simd_read8_to_u64(char16_t const *chars) {+  FASTFLOAT_SIMD_DISABLE_WARNINGS+  return simd_read8_to_u64(+      vld1q_u16(reinterpret_cast<uint16_t const *>(chars)));+  FASTFLOAT_SIMD_RESTORE_WARNINGS+}++#endif // FASTFLOAT_SSE2++// MSVC SFINAE is broken pre-VS2017+#if defined(_MSC_VER) && _MSC_VER <= 1900+template <typename UC>+#else+template <typename UC, FASTFLOAT_ENABLE_IF(!has_simd_opt<UC>()) = 0>+#endif+// dummy for compile+uint64_t simd_read8_to_u64(UC const *) {+  return 0;+}++// credit  @aqrit+fastfloat_really_inline FASTFLOAT_CONSTEXPR14 uint32_t+parse_eight_digits_unrolled(uint64_t val) {+  uint64_t const mask = 0x000000FF000000FF;+  uint64_t const mul1 = 0x000F424000000064; // 100 + (1000000ULL << 32)+  uint64_t const mul2 = 0x0000271000000001; // 1 + (10000ULL << 32)+  val -= 0x3030303030303030;+  val = (val * 10) + (val >> 8); // val = (val * 2561) >> 8;+  val = (((val & mask) * mul1) + (((val >> 16) & mask) * mul2)) >> 32;+  return uint32_t(val);+}++// Call this if chars are definitely 8 digits.+template <typename UC>+fastfloat_really_inline FASTFLOAT_CONSTEXPR20 uint32_t+parse_eight_digits_unrolled(UC const *chars) noexcept {+  if (cpp20_and_in_constexpr() || !has_simd_opt<UC>()) {+    return parse_eight_digits_unrolled(read8_to_u64(chars)); // truncation okay+  }+  return parse_eight_digits_unrolled(simd_read8_to_u64(chars));+}++// credit @aqrit+fastfloat_really_inline constexpr bool+is_made_of_eight_digits_fast(uint64_t val) noexcept {+  return !((((val + 0x4646464646464646) | (val - 0x3030303030303030)) &+            0x8080808080808080));+}++#ifdef FASTFLOAT_HAS_SIMD++// Call this if chars might not be 8 digits.+// Using this style (instead of is_made_of_eight_digits_fast() then+// parse_eight_digits_unrolled()) ensures we don't load SIMD registers twice.+fastfloat_really_inline FASTFLOAT_CONSTEXPR20 bool+simd_parse_if_eight_digits_unrolled(char16_t const *chars,+                                    uint64_t &i) noexcept {+  if (cpp20_and_in_constexpr()) {+    return false;+  }+#ifdef FASTFLOAT_SSE2+  FASTFLOAT_SIMD_DISABLE_WARNINGS+  __m128i const data =+      _mm_loadu_si128(reinterpret_cast<__m128i const *>(chars));++  // (x - '0') <= 9+  // http://0x80.pl/articles/simd-parsing-int-sequences.html+  __m128i const t0 = _mm_add_epi16(data, _mm_set1_epi16(32720));+  __m128i const t1 = _mm_cmpgt_epi16(t0, _mm_set1_epi16(-32759));++  if (_mm_movemask_epi8(t1) == 0) {+    i = i * 100000000 + parse_eight_digits_unrolled(simd_read8_to_u64(data));+    return true;+  } else+    return false;+  FASTFLOAT_SIMD_RESTORE_WARNINGS+#elif defined(FASTFLOAT_NEON)+  FASTFLOAT_SIMD_DISABLE_WARNINGS+  uint16x8_t const data = vld1q_u16(reinterpret_cast<uint16_t const *>(chars));++  // (x - '0') <= 9+  // http://0x80.pl/articles/simd-parsing-int-sequences.html+  uint16x8_t const t0 = vsubq_u16(data, vmovq_n_u16('0'));+  uint16x8_t const mask = vcltq_u16(t0, vmovq_n_u16('9' - '0' + 1));++  if (vminvq_u16(mask) == 0xFFFF) {+    i = i * 100000000 + parse_eight_digits_unrolled(simd_read8_to_u64(data));+    return true;+  } else+    return false;+  FASTFLOAT_SIMD_RESTORE_WARNINGS+#else+  (void)chars;+  (void)i;+  return false;+#endif // FASTFLOAT_SSE2+}++#endif // FASTFLOAT_HAS_SIMD++// MSVC SFINAE is broken pre-VS2017+#if defined(_MSC_VER) && _MSC_VER <= 1900+template <typename UC>+#else+template <typename UC, FASTFLOAT_ENABLE_IF(!has_simd_opt<UC>()) = 0>+#endif+// dummy for compile+bool simd_parse_if_eight_digits_unrolled(UC const *, uint64_t &) {+  return 0;+}++template <typename UC, FASTFLOAT_ENABLE_IF(!std::is_same<UC, char>::value) = 0>+fastfloat_really_inline FASTFLOAT_CONSTEXPR20 void+loop_parse_if_eight_digits(UC const *&p, UC const *const pend, uint64_t &i) {+  if (!has_simd_opt<UC>()) {+    return;+  }+  while ((std::distance(p, pend) >= 8) &&+         simd_parse_if_eight_digits_unrolled(+             p, i)) { // in rare cases, this will overflow, but that's ok+    p += 8;+  }+}++fastfloat_really_inline FASTFLOAT_CONSTEXPR20 void+loop_parse_if_eight_digits(char const *&p, char const *const pend,+                           uint64_t &i) {+  // optimizes better than parse_if_eight_digits_unrolled() for UC = char.+  while ((std::distance(p, pend) >= 8) &&+         is_made_of_eight_digits_fast(read8_to_u64(p))) {+    i = i * 100000000 ++        parse_eight_digits_unrolled(read8_to_u64(+            p)); // in rare cases, this will overflow, but that's ok+    p += 8;+  }+}++enum class parse_error {+  no_error,+  // [JSON-only] The minus sign must be followed by an integer.+  missing_integer_after_sign,+  // A sign must be followed by an integer or dot.+  missing_integer_or_dot_after_sign,+  // [JSON-only] The integer part must not have leading zeros.+  leading_zeros_in_integer_part,+  // [JSON-only] The integer part must have at least one digit.+  no_digits_in_integer_part,+  // [JSON-only] If there is a decimal point, there must be digits in the+  // fractional part.+  no_digits_in_fractional_part,+  // The mantissa must have at least one digit.+  no_digits_in_mantissa,+  // Scientific notation requires an exponential part.+  missing_exponential_part,+};++template <typename UC> struct parsed_number_string_t {+  int64_t exponent{0};+  uint64_t mantissa{0};+  UC const *lastmatch{nullptr};+  bool negative{false};+  bool valid{false};+  bool too_many_digits{false};+  // contains the range of the significant digits+  span<UC const> integer{};  // non-nullable+  span<UC const> fraction{}; // nullable+  parse_error error{parse_error::no_error};+};++using byte_span = span<char const>;+using parsed_number_string = parsed_number_string_t<char>;++template <typename UC>+fastfloat_really_inline FASTFLOAT_CONSTEXPR20 parsed_number_string_t<UC>+report_parse_error(UC const *p, parse_error error) {+  parsed_number_string_t<UC> answer;+  answer.valid = false;+  answer.lastmatch = p;+  answer.error = error;+  return answer;+}++// Assuming that you use no more than 19 digits, this will+// parse an ASCII string.+template <typename UC>+fastfloat_really_inline FASTFLOAT_CONSTEXPR20 parsed_number_string_t<UC>+parse_number_string(UC const *p, UC const *pend,+                    parse_options_t<UC> options) noexcept {+  chars_format const fmt = detail::adjust_for_feature_macros(options.format);+  UC const decimal_point = options.decimal_point;++  parsed_number_string_t<UC> answer;+  answer.valid = false;+  answer.too_many_digits = false;+  // assume p < pend, so dereference without checks;+  answer.negative = (*p == UC('-'));+  // C++17 20.19.3.(7.1) explicitly forbids '+' sign here+  if ((*p == UC('-')) ||+      (uint64_t(fmt & chars_format::allow_leading_plus) &&+       !uint64_t(fmt & detail::basic_json_fmt) && *p == UC('+'))) {+    ++p;+    if (p == pend) {+      return report_parse_error<UC>(+          p, parse_error::missing_integer_or_dot_after_sign);+    }+    if (uint64_t(fmt & detail::basic_json_fmt)) {+      if (!is_integer(*p)) { // a sign must be followed by an integer+        return report_parse_error<UC>(p,+                                      parse_error::missing_integer_after_sign);+      }+    } else {+      if (!is_integer(*p) &&+          (*p !=+           decimal_point)) { // a sign must be followed by an integer or the dot+        return report_parse_error<UC>(+            p, parse_error::missing_integer_or_dot_after_sign);+      }+    }+  }+  UC const *const start_digits = p;++  uint64_t i = 0; // an unsigned int avoids signed overflows (which are bad)++  while ((p != pend) && is_integer(*p)) {+    // a multiplication by 10 is cheaper than an arbitrary integer+    // multiplication+    i = 10 * i ++        uint64_t(*p -+                 UC('0')); // might overflow, we will handle the overflow later+    ++p;+  }+  UC const *const end_of_integer_part = p;+  int64_t digit_count = int64_t(end_of_integer_part - start_digits);+  answer.integer = span<UC const>(start_digits, size_t(digit_count));+  if (uint64_t(fmt & detail::basic_json_fmt)) {+    // at least 1 digit in integer part, without leading zeros+    if (digit_count == 0) {+      return report_parse_error<UC>(p, parse_error::no_digits_in_integer_part);+    }+    if ((start_digits[0] == UC('0') && digit_count > 1)) {+      return report_parse_error<UC>(start_digits,+                                    parse_error::leading_zeros_in_integer_part);+    }+  }++  int64_t exponent = 0;+  bool const has_decimal_point = (p != pend) && (*p == decimal_point);+  if (has_decimal_point) {+    ++p;+    UC const *before = p;+    // can occur at most twice without overflowing, but let it occur more, since+    // for integers with many digits, digit parsing is the primary bottleneck.+    loop_parse_if_eight_digits(p, pend, i);++    while ((p != pend) && is_integer(*p)) {+      uint8_t digit = uint8_t(*p - UC('0'));+      ++p;+      i = i * 10 + digit; // in rare cases, this will overflow, but that's ok+    }+    exponent = before - p;+    answer.fraction = span<UC const>(before, size_t(p - before));+    digit_count -= exponent;+  }+  if (uint64_t(fmt & detail::basic_json_fmt)) {+    // at least 1 digit in fractional part+    if (has_decimal_point && exponent == 0) {+      return report_parse_error<UC>(p,+                                    parse_error::no_digits_in_fractional_part);+    }+  } else if (digit_count ==+             0) { // we must have encountered at least one integer!+    return report_parse_error<UC>(p, parse_error::no_digits_in_mantissa);+  }+  int64_t exp_number = 0; // explicit exponential part+  if ((uint64_t(fmt & chars_format::scientific) && (p != pend) &&+       ((UC('e') == *p) || (UC('E') == *p))) ||+      (uint64_t(fmt & detail::basic_fortran_fmt) && (p != pend) &&+       ((UC('+') == *p) || (UC('-') == *p) || (UC('d') == *p) ||+        (UC('D') == *p)))) {+    UC const *location_of_e = p;+    if ((UC('e') == *p) || (UC('E') == *p) || (UC('d') == *p) ||+        (UC('D') == *p)) {+      ++p;+    }+    bool neg_exp = false;+    if ((p != pend) && (UC('-') == *p)) {+      neg_exp = true;+      ++p;+    } else if ((p != pend) &&+               (UC('+') ==+                *p)) { // '+' on exponent is allowed by C++17 20.19.3.(7.1)+      ++p;+    }+    if ((p == pend) || !is_integer(*p)) {+      if (!uint64_t(fmt & chars_format::fixed)) {+        // The exponential part is invalid for scientific notation, so it must+        // be a trailing token for fixed notation. However, fixed notation is+        // disabled, so report a scientific notation error.+        return report_parse_error<UC>(p, parse_error::missing_exponential_part);+      }+      // Otherwise, we will be ignoring the 'e'.+      p = location_of_e;+    } else {+      while ((p != pend) && is_integer(*p)) {+        uint8_t digit = uint8_t(*p - UC('0'));+        if (exp_number < 0x10000000) {+          exp_number = 10 * exp_number + digit;+        }+        ++p;+      }+      if (neg_exp) {+        exp_number = -exp_number;+      }+      exponent += exp_number;+    }+  } else {+    // If it scientific and not fixed, we have to bail out.+    if (uint64_t(fmt & chars_format::scientific) &&+        !uint64_t(fmt & chars_format::fixed)) {+      return report_parse_error<UC>(p, parse_error::missing_exponential_part);+    }+  }+  answer.lastmatch = p;+  answer.valid = true;++  // If we frequently had to deal with long strings of digits,+  // we could extend our code by using a 128-bit integer instead+  // of a 64-bit integer. However, this is uncommon.+  //+  // We can deal with up to 19 digits.+  if (digit_count > 19) { // this is uncommon+    // It is possible that the integer had an overflow.+    // We have to handle the case where we have 0.0000somenumber.+    // We need to be mindful of the case where we only have zeroes...+    // E.g., 0.000000000...000.+    UC const *start = start_digits;+    while ((start != pend) && (*start == UC('0') || *start == decimal_point)) {+      if (*start == UC('0')) {+        digit_count--;+      }+      start++;+    }++    if (digit_count > 19) {+      answer.too_many_digits = true;+      // Let us start again, this time, avoiding overflows.+      // We don't need to check if is_integer, since we use the+      // pre-tokenized spans from above.+      i = 0;+      p = answer.integer.ptr;+      UC const *int_end = p + answer.integer.len();+      uint64_t const minimal_nineteen_digit_integer{1000000000000000000};+      while ((i < minimal_nineteen_digit_integer) && (p != int_end)) {+        i = i * 10 + uint64_t(*p - UC('0'));+        ++p;+      }+      if (i >= minimal_nineteen_digit_integer) { // We have a big integers+        exponent = end_of_integer_part - p + exp_number;+      } else { // We have a value with a fractional component.+        p = answer.fraction.ptr;+        UC const *frac_end = p + answer.fraction.len();+        while ((i < minimal_nineteen_digit_integer) && (p != frac_end)) {+          i = i * 10 + uint64_t(*p - UC('0'));+          ++p;+        }+        exponent = answer.fraction.ptr - p + exp_number;+      }+      // We have now corrected both exponent and i, to a truncated value+    }+  }+  answer.exponent = exponent;+  answer.mantissa = i;+  return answer;+}++template <typename T, typename UC>+fastfloat_really_inline FASTFLOAT_CONSTEXPR20 from_chars_result_t<UC>+parse_int_string(UC const *p, UC const *pend, T &value,+                 parse_options_t<UC> options) {+  chars_format const fmt = detail::adjust_for_feature_macros(options.format);+  int const base = options.base;++  from_chars_result_t<UC> answer;++  UC const *const first = p;++  bool const negative = (*p == UC('-'));+#ifdef FASTFLOAT_VISUAL_STUDIO+#pragma warning(push)+#pragma warning(disable : 4127)+#endif+  if (!std::is_signed<T>::value && negative) {+#ifdef FASTFLOAT_VISUAL_STUDIO+#pragma warning(pop)+#endif+    answer.ec = std::errc::invalid_argument;+    answer.ptr = first;+    return answer;+  }+  if ((*p == UC('-')) ||+      (uint64_t(fmt & chars_format::allow_leading_plus) && (*p == UC('+')))) {+    ++p;+  }++  UC const *const start_num = p;++  while (p != pend && *p == UC('0')) {+    ++p;+  }++  bool const has_leading_zeros = p > start_num;++  UC const *const start_digits = p;++  uint64_t i = 0;+  if (base == 10) {+    loop_parse_if_eight_digits(p, pend, i); // use SIMD if possible+  }+  while (p != pend) {+    uint8_t digit = ch_to_digit(*p);+    if (digit >= base) {+      break;+    }+    i = uint64_t(base) * i + digit; // might overflow, check this later+    p++;+  }++  size_t digit_count = size_t(p - start_digits);++  if (digit_count == 0) {+    if (has_leading_zeros) {+      value = 0;+      answer.ec = std::errc();+      answer.ptr = p;+    } else {+      answer.ec = std::errc::invalid_argument;+      answer.ptr = first;+    }+    return answer;+  }++  answer.ptr = p;++  // check u64 overflow+  size_t max_digits = max_digits_u64(base);+  if (digit_count > max_digits) {+    answer.ec = std::errc::result_out_of_range;+    return answer;+  }+  // this check can be eliminated for all other types, but they will all require+  // a max_digits(base) equivalent+  if (digit_count == max_digits && i < min_safe_u64(base)) {+    answer.ec = std::errc::result_out_of_range;+    return answer;+  }++  // check other types overflow+  if (!std::is_same<T, uint64_t>::value) {+    if (i > uint64_t(std::numeric_limits<T>::max()) + uint64_t(negative)) {+      answer.ec = std::errc::result_out_of_range;+      return answer;+    }+  }++  if (negative) {+#ifdef FASTFLOAT_VISUAL_STUDIO+#pragma warning(push)+#pragma warning(disable : 4146)+#endif+    // this weird workaround is required because:+    // - converting unsigned to signed when its value is greater than signed max+    // is UB pre-C++23.+    // - reinterpret_casting (~i + 1) would work, but it is not constexpr+    // this is always optimized into a neg instruction (note: T is an integer+    // type)+    value = T(-std::numeric_limits<T>::max() -+              T(i - uint64_t(std::numeric_limits<T>::max())));+#ifdef FASTFLOAT_VISUAL_STUDIO+#pragma warning(pop)+#endif+  } else {+    value = T(i);+  }++  answer.ec = std::errc();+  return answer;+}++} // namespace fast_float++#endif
+ fast_float-8.0.0/include/fast_float/bigint.h view
@@ -0,0 +1,638 @@+#ifndef FASTFLOAT_BIGINT_H+#define FASTFLOAT_BIGINT_H++#include <algorithm>+#include <cstdint>+#include <climits>+#include <cstring>++#include "float_common.h"++namespace fast_float {++// the limb width: we want efficient multiplication of double the bits in+// limb, or for 64-bit limbs, at least 64-bit multiplication where we can+// extract the high and low parts efficiently. this is every 64-bit+// architecture except for sparc, which emulates 128-bit multiplication.+// we might have platforms where `CHAR_BIT` is not 8, so let's avoid+// doing `8 * sizeof(limb)`.+#if defined(FASTFLOAT_64BIT) && !defined(__sparc)+#define FASTFLOAT_64BIT_LIMB 1+typedef uint64_t limb;+constexpr size_t limb_bits = 64;+#else+#define FASTFLOAT_32BIT_LIMB+typedef uint32_t limb;+constexpr size_t limb_bits = 32;+#endif++typedef span<limb> limb_span;++// number of bits in a bigint. this needs to be at least the number+// of bits required to store the largest bigint, which is+// `log2(10**(digits + max_exp))`, or `log2(10**(767 + 342))`, or+// ~3600 bits, so we round to 4000.+constexpr size_t bigint_bits = 4000;+constexpr size_t bigint_limbs = bigint_bits / limb_bits;++// vector-like type that is allocated on the stack. the entire+// buffer is pre-allocated, and only the length changes.+template <uint16_t size> struct stackvec {+  limb data[size];+  // we never need more than 150 limbs+  uint16_t length{0};++  stackvec() = default;+  stackvec(stackvec const &) = delete;+  stackvec &operator=(stackvec const &) = delete;+  stackvec(stackvec &&) = delete;+  stackvec &operator=(stackvec &&other) = delete;++  // create stack vector from existing limb span.+  FASTFLOAT_CONSTEXPR20 stackvec(limb_span s) {+    FASTFLOAT_ASSERT(try_extend(s));+  }++  FASTFLOAT_CONSTEXPR14 limb &operator[](size_t index) noexcept {+    FASTFLOAT_DEBUG_ASSERT(index < length);+    return data[index];+  }++  FASTFLOAT_CONSTEXPR14 const limb &operator[](size_t index) const noexcept {+    FASTFLOAT_DEBUG_ASSERT(index < length);+    return data[index];+  }++  // index from the end of the container+  FASTFLOAT_CONSTEXPR14 const limb &rindex(size_t index) const noexcept {+    FASTFLOAT_DEBUG_ASSERT(index < length);+    size_t rindex = length - index - 1;+    return data[rindex];+  }++  // set the length, without bounds checking.+  FASTFLOAT_CONSTEXPR14 void set_len(size_t len) noexcept {+    length = uint16_t(len);+  }++  constexpr size_t len() const noexcept { return length; }++  constexpr bool is_empty() const noexcept { return length == 0; }++  constexpr size_t capacity() const noexcept { return size; }++  // append item to vector, without bounds checking+  FASTFLOAT_CONSTEXPR14 void push_unchecked(limb value) noexcept {+    data[length] = value;+    length++;+  }++  // append item to vector, returning if item was added+  FASTFLOAT_CONSTEXPR14 bool try_push(limb value) noexcept {+    if (len() < capacity()) {+      push_unchecked(value);+      return true;+    } else {+      return false;+    }+  }++  // add items to the vector, from a span, without bounds checking+  FASTFLOAT_CONSTEXPR20 void extend_unchecked(limb_span s) noexcept {+    limb *ptr = data + length;+    std::copy_n(s.ptr, s.len(), ptr);+    set_len(len() + s.len());+  }++  // try to add items to the vector, returning if items were added+  FASTFLOAT_CONSTEXPR20 bool try_extend(limb_span s) noexcept {+    if (len() + s.len() <= capacity()) {+      extend_unchecked(s);+      return true;+    } else {+      return false;+    }+  }++  // resize the vector, without bounds checking+  // if the new size is longer than the vector, assign value to each+  // appended item.+  FASTFLOAT_CONSTEXPR20+  void resize_unchecked(size_t new_len, limb value) noexcept {+    if (new_len > len()) {+      size_t count = new_len - len();+      limb *first = data + len();+      limb *last = first + count;+      ::std::fill(first, last, value);+      set_len(new_len);+    } else {+      set_len(new_len);+    }+  }++  // try to resize the vector, returning if the vector was resized.+  FASTFLOAT_CONSTEXPR20 bool try_resize(size_t new_len, limb value) noexcept {+    if (new_len > capacity()) {+      return false;+    } else {+      resize_unchecked(new_len, value);+      return true;+    }+  }++  // check if any limbs are non-zero after the given index.+  // this needs to be done in reverse order, since the index+  // is relative to the most significant limbs.+  FASTFLOAT_CONSTEXPR14 bool nonzero(size_t index) const noexcept {+    while (index < len()) {+      if (rindex(index) != 0) {+        return true;+      }+      index++;+    }+    return false;+  }++  // normalize the big integer, so most-significant zero limbs are removed.+  FASTFLOAT_CONSTEXPR14 void normalize() noexcept {+    while (len() > 0 && rindex(0) == 0) {+      length--;+    }+  }+};++fastfloat_really_inline FASTFLOAT_CONSTEXPR14 uint64_t+empty_hi64(bool &truncated) noexcept {+  truncated = false;+  return 0;+}++fastfloat_really_inline FASTFLOAT_CONSTEXPR20 uint64_t+uint64_hi64(uint64_t r0, bool &truncated) noexcept {+  truncated = false;+  int shl = leading_zeroes(r0);+  return r0 << shl;+}++fastfloat_really_inline FASTFLOAT_CONSTEXPR20 uint64_t+uint64_hi64(uint64_t r0, uint64_t r1, bool &truncated) noexcept {+  int shl = leading_zeroes(r0);+  if (shl == 0) {+    truncated = r1 != 0;+    return r0;+  } else {+    int shr = 64 - shl;+    truncated = (r1 << shl) != 0;+    return (r0 << shl) | (r1 >> shr);+  }+}++fastfloat_really_inline FASTFLOAT_CONSTEXPR20 uint64_t+uint32_hi64(uint32_t r0, bool &truncated) noexcept {+  return uint64_hi64(r0, truncated);+}++fastfloat_really_inline FASTFLOAT_CONSTEXPR20 uint64_t+uint32_hi64(uint32_t r0, uint32_t r1, bool &truncated) noexcept {+  uint64_t x0 = r0;+  uint64_t x1 = r1;+  return uint64_hi64((x0 << 32) | x1, truncated);+}++fastfloat_really_inline FASTFLOAT_CONSTEXPR20 uint64_t+uint32_hi64(uint32_t r0, uint32_t r1, uint32_t r2, bool &truncated) noexcept {+  uint64_t x0 = r0;+  uint64_t x1 = r1;+  uint64_t x2 = r2;+  return uint64_hi64(x0, (x1 << 32) | x2, truncated);+}++// add two small integers, checking for overflow.+// we want an efficient operation. for msvc, where+// we don't have built-in intrinsics, this is still+// pretty fast.+fastfloat_really_inline FASTFLOAT_CONSTEXPR20 limb+scalar_add(limb x, limb y, bool &overflow) noexcept {+  limb z;+// gcc and clang+#if defined(__has_builtin)+#if __has_builtin(__builtin_add_overflow)+  if (!cpp20_and_in_constexpr()) {+    overflow = __builtin_add_overflow(x, y, &z);+    return z;+  }+#endif+#endif++  // generic, this still optimizes correctly on MSVC.+  z = x + y;+  overflow = z < x;+  return z;+}++// multiply two small integers, getting both the high and low bits.+fastfloat_really_inline FASTFLOAT_CONSTEXPR20 limb+scalar_mul(limb x, limb y, limb &carry) noexcept {+#ifdef FASTFLOAT_64BIT_LIMB+#if defined(__SIZEOF_INT128__)+  // GCC and clang both define it as an extension.+  __uint128_t z = __uint128_t(x) * __uint128_t(y) + __uint128_t(carry);+  carry = limb(z >> limb_bits);+  return limb(z);+#else+  // fallback, no native 128-bit integer multiplication with carry.+  // on msvc, this optimizes identically, somehow.+  value128 z = full_multiplication(x, y);+  bool overflow;+  z.low = scalar_add(z.low, carry, overflow);+  z.high += uint64_t(overflow); // cannot overflow+  carry = z.high;+  return z.low;+#endif+#else+  uint64_t z = uint64_t(x) * uint64_t(y) + uint64_t(carry);+  carry = limb(z >> limb_bits);+  return limb(z);+#endif+}++// add scalar value to bigint starting from offset.+// used in grade school multiplication+template <uint16_t size>+inline FASTFLOAT_CONSTEXPR20 bool small_add_from(stackvec<size> &vec, limb y,+                                                 size_t start) noexcept {+  size_t index = start;+  limb carry = y;+  bool overflow;+  while (carry != 0 && index < vec.len()) {+    vec[index] = scalar_add(vec[index], carry, overflow);+    carry = limb(overflow);+    index += 1;+  }+  if (carry != 0) {+    FASTFLOAT_TRY(vec.try_push(carry));+  }+  return true;+}++// add scalar value to bigint.+template <uint16_t size>+fastfloat_really_inline FASTFLOAT_CONSTEXPR20 bool+small_add(stackvec<size> &vec, limb y) noexcept {+  return small_add_from(vec, y, 0);+}++// multiply bigint by scalar value.+template <uint16_t size>+inline FASTFLOAT_CONSTEXPR20 bool small_mul(stackvec<size> &vec,+                                            limb y) noexcept {+  limb carry = 0;+  for (size_t index = 0; index < vec.len(); index++) {+    vec[index] = scalar_mul(vec[index], y, carry);+  }+  if (carry != 0) {+    FASTFLOAT_TRY(vec.try_push(carry));+  }+  return true;+}++// add bigint to bigint starting from index.+// used in grade school multiplication+template <uint16_t size>+FASTFLOAT_CONSTEXPR20 bool large_add_from(stackvec<size> &x, limb_span y,+                                          size_t start) noexcept {+  // the effective x buffer is from `xstart..x.len()`, so exit early+  // if we can't get that current range.+  if (x.len() < start || y.len() > x.len() - start) {+    FASTFLOAT_TRY(x.try_resize(y.len() + start, 0));+  }++  bool carry = false;+  for (size_t index = 0; index < y.len(); index++) {+    limb xi = x[index + start];+    limb yi = y[index];+    bool c1 = false;+    bool c2 = false;+    xi = scalar_add(xi, yi, c1);+    if (carry) {+      xi = scalar_add(xi, 1, c2);+    }+    x[index + start] = xi;+    carry = c1 | c2;+  }++  // handle overflow+  if (carry) {+    FASTFLOAT_TRY(small_add_from(x, 1, y.len() + start));+  }+  return true;+}++// add bigint to bigint.+template <uint16_t size>+fastfloat_really_inline FASTFLOAT_CONSTEXPR20 bool+large_add_from(stackvec<size> &x, limb_span y) noexcept {+  return large_add_from(x, y, 0);+}++// grade-school multiplication algorithm+template <uint16_t size>+FASTFLOAT_CONSTEXPR20 bool long_mul(stackvec<size> &x, limb_span y) noexcept {+  limb_span xs = limb_span(x.data, x.len());+  stackvec<size> z(xs);+  limb_span zs = limb_span(z.data, z.len());++  if (y.len() != 0) {+    limb y0 = y[0];+    FASTFLOAT_TRY(small_mul(x, y0));+    for (size_t index = 1; index < y.len(); index++) {+      limb yi = y[index];+      stackvec<size> zi;+      if (yi != 0) {+        // re-use the same buffer throughout+        zi.set_len(0);+        FASTFLOAT_TRY(zi.try_extend(zs));+        FASTFLOAT_TRY(small_mul(zi, yi));+        limb_span zis = limb_span(zi.data, zi.len());+        FASTFLOAT_TRY(large_add_from(x, zis, index));+      }+    }+  }++  x.normalize();+  return true;+}++// grade-school multiplication algorithm+template <uint16_t size>+FASTFLOAT_CONSTEXPR20 bool large_mul(stackvec<size> &x, limb_span y) noexcept {+  if (y.len() == 1) {+    FASTFLOAT_TRY(small_mul(x, y[0]));+  } else {+    FASTFLOAT_TRY(long_mul(x, y));+  }+  return true;+}++template <typename = void> struct pow5_tables {+  static constexpr uint32_t large_step = 135;+  static constexpr uint64_t small_power_of_5[] = {+      1UL,+      5UL,+      25UL,+      125UL,+      625UL,+      3125UL,+      15625UL,+      78125UL,+      390625UL,+      1953125UL,+      9765625UL,+      48828125UL,+      244140625UL,+      1220703125UL,+      6103515625UL,+      30517578125UL,+      152587890625UL,+      762939453125UL,+      3814697265625UL,+      19073486328125UL,+      95367431640625UL,+      476837158203125UL,+      2384185791015625UL,+      11920928955078125UL,+      59604644775390625UL,+      298023223876953125UL,+      1490116119384765625UL,+      7450580596923828125UL,+  };+#ifdef FASTFLOAT_64BIT_LIMB+  constexpr static limb large_power_of_5[] = {+      1414648277510068013UL, 9180637584431281687UL, 4539964771860779200UL,+      10482974169319127550UL, 198276706040285095UL};+#else+  constexpr static limb large_power_of_5[] = {+      4279965485U, 329373468U,  4020270615U, 2137533757U, 4287402176U,+      1057042919U, 1071430142U, 2440757623U, 381945767U,  46164893U};+#endif+};++#if FASTFLOAT_DETAIL_MUST_DEFINE_CONSTEXPR_VARIABLE++template <typename T> constexpr uint32_t pow5_tables<T>::large_step;++template <typename T> constexpr uint64_t pow5_tables<T>::small_power_of_5[];++template <typename T> constexpr limb pow5_tables<T>::large_power_of_5[];++#endif++// big integer type. implements a small subset of big integer+// arithmetic, using simple algorithms since asymptotically+// faster algorithms are slower for a small number of limbs.+// all operations assume the big-integer is normalized.+struct bigint : pow5_tables<> {+  // storage of the limbs, in little-endian order.+  stackvec<bigint_limbs> vec;++  FASTFLOAT_CONSTEXPR20 bigint() : vec() {}++  bigint(bigint const &) = delete;+  bigint &operator=(bigint const &) = delete;+  bigint(bigint &&) = delete;+  bigint &operator=(bigint &&other) = delete;++  FASTFLOAT_CONSTEXPR20 bigint(uint64_t value) : vec() {+#ifdef FASTFLOAT_64BIT_LIMB+    vec.push_unchecked(value);+#else+    vec.push_unchecked(uint32_t(value));+    vec.push_unchecked(uint32_t(value >> 32));+#endif+    vec.normalize();+  }++  // get the high 64 bits from the vector, and if bits were truncated.+  // this is to get the significant digits for the float.+  FASTFLOAT_CONSTEXPR20 uint64_t hi64(bool &truncated) const noexcept {+#ifdef FASTFLOAT_64BIT_LIMB+    if (vec.len() == 0) {+      return empty_hi64(truncated);+    } else if (vec.len() == 1) {+      return uint64_hi64(vec.rindex(0), truncated);+    } else {+      uint64_t result = uint64_hi64(vec.rindex(0), vec.rindex(1), truncated);+      truncated |= vec.nonzero(2);+      return result;+    }+#else+    if (vec.len() == 0) {+      return empty_hi64(truncated);+    } else if (vec.len() == 1) {+      return uint32_hi64(vec.rindex(0), truncated);+    } else if (vec.len() == 2) {+      return uint32_hi64(vec.rindex(0), vec.rindex(1), truncated);+    } else {+      uint64_t result =+          uint32_hi64(vec.rindex(0), vec.rindex(1), vec.rindex(2), truncated);+      truncated |= vec.nonzero(3);+      return result;+    }+#endif+  }++  // compare two big integers, returning the large value.+  // assumes both are normalized. if the return value is+  // negative, other is larger, if the return value is+  // positive, this is larger, otherwise they are equal.+  // the limbs are stored in little-endian order, so we+  // must compare the limbs in ever order.+  FASTFLOAT_CONSTEXPR20 int compare(bigint const &other) const noexcept {+    if (vec.len() > other.vec.len()) {+      return 1;+    } else if (vec.len() < other.vec.len()) {+      return -1;+    } else {+      for (size_t index = vec.len(); index > 0; index--) {+        limb xi = vec[index - 1];+        limb yi = other.vec[index - 1];+        if (xi > yi) {+          return 1;+        } else if (xi < yi) {+          return -1;+        }+      }+      return 0;+    }+  }++  // shift left each limb n bits, carrying over to the new limb+  // returns true if we were able to shift all the digits.+  FASTFLOAT_CONSTEXPR20 bool shl_bits(size_t n) noexcept {+    // Internally, for each item, we shift left by n, and add the previous+    // right shifted limb-bits.+    // For example, we transform (for u8) shifted left 2, to:+    //      b10100100 b01000010+    //      b10 b10010001 b00001000+    FASTFLOAT_DEBUG_ASSERT(n != 0);+    FASTFLOAT_DEBUG_ASSERT(n < sizeof(limb) * 8);++    size_t shl = n;+    size_t shr = limb_bits - shl;+    limb prev = 0;+    for (size_t index = 0; index < vec.len(); index++) {+      limb xi = vec[index];+      vec[index] = (xi << shl) | (prev >> shr);+      prev = xi;+    }++    limb carry = prev >> shr;+    if (carry != 0) {+      return vec.try_push(carry);+    }+    return true;+  }++  // move the limbs left by `n` limbs.+  FASTFLOAT_CONSTEXPR20 bool shl_limbs(size_t n) noexcept {+    FASTFLOAT_DEBUG_ASSERT(n != 0);+    if (n + vec.len() > vec.capacity()) {+      return false;+    } else if (!vec.is_empty()) {+      // move limbs+      limb *dst = vec.data + n;+      limb const *src = vec.data;+      std::copy_backward(src, src + vec.len(), dst + vec.len());+      // fill in empty limbs+      limb *first = vec.data;+      limb *last = first + n;+      ::std::fill(first, last, 0);+      vec.set_len(n + vec.len());+      return true;+    } else {+      return true;+    }+  }++  // move the limbs left by `n` bits.+  FASTFLOAT_CONSTEXPR20 bool shl(size_t n) noexcept {+    size_t rem = n % limb_bits;+    size_t div = n / limb_bits;+    if (rem != 0) {+      FASTFLOAT_TRY(shl_bits(rem));+    }+    if (div != 0) {+      FASTFLOAT_TRY(shl_limbs(div));+    }+    return true;+  }++  // get the number of leading zeros in the bigint.+  FASTFLOAT_CONSTEXPR20 int ctlz() const noexcept {+    if (vec.is_empty()) {+      return 0;+    } else {+#ifdef FASTFLOAT_64BIT_LIMB+      return leading_zeroes(vec.rindex(0));+#else+      // no use defining a specialized leading_zeroes for a 32-bit type.+      uint64_t r0 = vec.rindex(0);+      return leading_zeroes(r0 << 32);+#endif+    }+  }++  // get the number of bits in the bigint.+  FASTFLOAT_CONSTEXPR20 int bit_length() const noexcept {+    int lz = ctlz();+    return int(limb_bits * vec.len()) - lz;+  }++  FASTFLOAT_CONSTEXPR20 bool mul(limb y) noexcept { return small_mul(vec, y); }++  FASTFLOAT_CONSTEXPR20 bool add(limb y) noexcept { return small_add(vec, y); }++  // multiply as if by 2 raised to a power.+  FASTFLOAT_CONSTEXPR20 bool pow2(uint32_t exp) noexcept { return shl(exp); }++  // multiply as if by 5 raised to a power.+  FASTFLOAT_CONSTEXPR20 bool pow5(uint32_t exp) noexcept {+    // multiply by a power of 5+    size_t large_length = sizeof(large_power_of_5) / sizeof(limb);+    limb_span large = limb_span(large_power_of_5, large_length);+    while (exp >= large_step) {+      FASTFLOAT_TRY(large_mul(vec, large));+      exp -= large_step;+    }+#ifdef FASTFLOAT_64BIT_LIMB+    uint32_t small_step = 27;+    limb max_native = 7450580596923828125UL;+#else+    uint32_t small_step = 13;+    limb max_native = 1220703125U;+#endif+    while (exp >= small_step) {+      FASTFLOAT_TRY(small_mul(vec, max_native));+      exp -= small_step;+    }+    if (exp != 0) {+      // Work around clang bug https://godbolt.org/z/zedh7rrhc+      // This is similar to https://github.com/llvm/llvm-project/issues/47746,+      // except the workaround described there don't work here+      FASTFLOAT_TRY(small_mul(+          vec, limb(((void)small_power_of_5[0], small_power_of_5[exp]))));+    }++    return true;+  }++  // multiply as if by 10 raised to a power.+  FASTFLOAT_CONSTEXPR20 bool pow10(uint32_t exp) noexcept {+    FASTFLOAT_TRY(pow5(exp));+    return pow2(exp);+  }+};++} // namespace fast_float++#endif
+ fast_float-8.0.0/include/fast_float/constexpr_feature_detect.h view
@@ -0,0 +1,46 @@+#ifndef FASTFLOAT_CONSTEXPR_FEATURE_DETECT_H+#define FASTFLOAT_CONSTEXPR_FEATURE_DETECT_H++#ifdef __has_include+#if __has_include(<version>)+#include <version>+#endif+#endif++// Testing for https://wg21.link/N3652, adopted in C++14+#if __cpp_constexpr >= 201304+#define FASTFLOAT_CONSTEXPR14 constexpr+#else+#define FASTFLOAT_CONSTEXPR14+#endif++#if defined(__cpp_lib_bit_cast) && __cpp_lib_bit_cast >= 201806L+#define FASTFLOAT_HAS_BIT_CAST 1+#else+#define FASTFLOAT_HAS_BIT_CAST 0+#endif++#if defined(__cpp_lib_is_constant_evaluated) &&                                \+    __cpp_lib_is_constant_evaluated >= 201811L+#define FASTFLOAT_HAS_IS_CONSTANT_EVALUATED 1+#else+#define FASTFLOAT_HAS_IS_CONSTANT_EVALUATED 0+#endif++// Testing for relevant C++20 constexpr library features+#if FASTFLOAT_HAS_IS_CONSTANT_EVALUATED && FASTFLOAT_HAS_BIT_CAST &&           \+    __cpp_lib_constexpr_algorithms >= 201806L /*For std::copy and std::fill*/+#define FASTFLOAT_CONSTEXPR20 constexpr+#define FASTFLOAT_IS_CONSTEXPR 1+#else+#define FASTFLOAT_CONSTEXPR20+#define FASTFLOAT_IS_CONSTEXPR 0+#endif++#if __cplusplus >= 201703L || (defined(_MSVC_LANG) && _MSVC_LANG >= 201703L)+#define FASTFLOAT_DETAIL_MUST_DEFINE_CONSTEXPR_VARIABLE 0+#else+#define FASTFLOAT_DETAIL_MUST_DEFINE_CONSTEXPR_VARIABLE 1+#endif++#endif // FASTFLOAT_CONSTEXPR_FEATURE_DETECT_H
+ fast_float-8.0.0/include/fast_float/decimal_to_binary.h view
@@ -0,0 +1,212 @@+#ifndef FASTFLOAT_DECIMAL_TO_BINARY_H+#define FASTFLOAT_DECIMAL_TO_BINARY_H++#include "float_common.h"+#include "fast_table.h"+#include <cfloat>+#include <cinttypes>+#include <cmath>+#include <cstdint>+#include <cstdlib>+#include <cstring>++namespace fast_float {++// This will compute or rather approximate w * 5**q and return a pair of 64-bit+// words approximating the result, with the "high" part corresponding to the+// most significant bits and the low part corresponding to the least significant+// bits.+//+template <int bit_precision>+fastfloat_really_inline FASTFLOAT_CONSTEXPR20 value128+compute_product_approximation(int64_t q, uint64_t w) {+  int const index = 2 * int(q - powers::smallest_power_of_five);+  // For small values of q, e.g., q in [0,27], the answer is always exact+  // because The line value128 firstproduct = full_multiplication(w,+  // power_of_five_128[index]); gives the exact answer.+  value128 firstproduct =+      full_multiplication(w, powers::power_of_five_128[index]);+  static_assert((bit_precision >= 0) && (bit_precision <= 64),+                " precision should  be in (0,64]");+  constexpr uint64_t precision_mask =+      (bit_precision < 64) ? (uint64_t(0xFFFFFFFFFFFFFFFF) >> bit_precision)+                           : uint64_t(0xFFFFFFFFFFFFFFFF);+  if ((firstproduct.high & precision_mask) ==+      precision_mask) { // could further guard with  (lower + w < lower)+    // regarding the second product, we only need secondproduct.high, but our+    // expectation is that the compiler will optimize this extra work away if+    // needed.+    value128 secondproduct =+        full_multiplication(w, powers::power_of_five_128[index + 1]);+    firstproduct.low += secondproduct.high;+    if (secondproduct.high > firstproduct.low) {+      firstproduct.high++;+    }+  }+  return firstproduct;+}++namespace detail {+/**+ * For q in (0,350), we have that+ *  f = (((152170 + 65536) * q ) >> 16);+ * is equal to+ *   floor(p) + q+ * where+ *   p = log(5**q)/log(2) = q * log(5)/log(2)+ *+ * For negative values of q in (-400,0), we have that+ *  f = (((152170 + 65536) * q ) >> 16);+ * is equal to+ *   -ceil(p) + q+ * where+ *   p = log(5**-q)/log(2) = -q * log(5)/log(2)+ */+constexpr fastfloat_really_inline int32_t power(int32_t q) noexcept {+  return (((152170 + 65536) * q) >> 16) + 63;+}+} // namespace detail++// create an adjusted mantissa, biased by the invalid power2+// for significant digits already multiplied by 10 ** q.+template <typename binary>+fastfloat_really_inline FASTFLOAT_CONSTEXPR14 adjusted_mantissa+compute_error_scaled(int64_t q, uint64_t w, int lz) noexcept {+  int hilz = int(w >> 63) ^ 1;+  adjusted_mantissa answer;+  answer.mantissa = w << hilz;+  int bias = binary::mantissa_explicit_bits() - binary::minimum_exponent();+  answer.power2 = int32_t(detail::power(int32_t(q)) + bias - hilz - lz - 62 ++                          invalid_am_bias);+  return answer;+}++// w * 10 ** q, without rounding the representation up.+// the power2 in the exponent will be adjusted by invalid_am_bias.+template <typename binary>+fastfloat_really_inline FASTFLOAT_CONSTEXPR20 adjusted_mantissa+compute_error(int64_t q, uint64_t w) noexcept {+  int lz = leading_zeroes(w);+  w <<= lz;+  value128 product =+      compute_product_approximation<binary::mantissa_explicit_bits() + 3>(q, w);+  return compute_error_scaled<binary>(q, product.high, lz);+}++// Computers w * 10 ** q.+// The returned value should be a valid number that simply needs to be+// packed. However, in some very rare cases, the computation will fail. In such+// cases, we return an adjusted_mantissa with a negative power of 2: the caller+// should recompute in such cases.+template <typename binary>+fastfloat_really_inline FASTFLOAT_CONSTEXPR20 adjusted_mantissa+compute_float(int64_t q, uint64_t w) noexcept {+  adjusted_mantissa answer;+  if ((w == 0) || (q < binary::smallest_power_of_ten())) {+    answer.power2 = 0;+    answer.mantissa = 0;+    // result should be zero+    return answer;+  }+  if (q > binary::largest_power_of_ten()) {+    // we want to get infinity:+    answer.power2 = binary::infinite_power();+    answer.mantissa = 0;+    return answer;+  }+  // At this point in time q is in [powers::smallest_power_of_five,+  // powers::largest_power_of_five].++  // We want the most significant bit of i to be 1. Shift if needed.+  int lz = leading_zeroes(w);+  w <<= lz;++  // The required precision is binary::mantissa_explicit_bits() + 3 because+  // 1. We need the implicit bit+  // 2. We need an extra bit for rounding purposes+  // 3. We might lose a bit due to the "upperbit" routine (result too small,+  // requiring a shift)++  value128 product =+      compute_product_approximation<binary::mantissa_explicit_bits() + 3>(q, w);+  // The computed 'product' is always sufficient.+  // Mathematical proof:+  // Noble Mushtak and Daniel Lemire, Fast Number Parsing Without Fallback (to+  // appear) See script/mushtak_lemire.py++  // The "compute_product_approximation" function can be slightly slower than a+  // branchless approach: value128 product = compute_product(q, w); but in+  // practice, we can win big with the compute_product_approximation if its+  // additional branch is easily predicted. Which is best is data specific.+  int upperbit = int(product.high >> 63);+  int shift = upperbit + 64 - binary::mantissa_explicit_bits() - 3;++  answer.mantissa = product.high >> shift;++  answer.power2 = int32_t(detail::power(int32_t(q)) + upperbit - lz -+                          binary::minimum_exponent());+  if (answer.power2 <= 0) { // we have a subnormal?+    // Here have that answer.power2 <= 0 so -answer.power2 >= 0+    if (-answer.power2 + 1 >=+        64) { // if we have more than 64 bits below the minimum exponent, you+              // have a zero for sure.+      answer.power2 = 0;+      answer.mantissa = 0;+      // result should be zero+      return answer;+    }+    // next line is safe because -answer.power2 + 1 < 64+    answer.mantissa >>= -answer.power2 + 1;+    // Thankfully, we can't have both "round-to-even" and subnormals because+    // "round-to-even" only occurs for powers close to 0 in the 32-bit and+    // and 64-bit case (with no more than 19 digits).+    answer.mantissa += (answer.mantissa & 1); // round up+    answer.mantissa >>= 1;+    // There is a weird scenario where we don't have a subnormal but just.+    // Suppose we start with 2.2250738585072013e-308, we end up+    // with 0x3fffffffffffff x 2^-1023-53 which is technically subnormal+    // whereas 0x40000000000000 x 2^-1023-53  is normal. Now, we need to round+    // up 0x3fffffffffffff x 2^-1023-53  and once we do, we are no longer+    // subnormal, but we can only know this after rounding.+    // So we only declare a subnormal if we are smaller than the threshold.+    answer.power2 =+        (answer.mantissa < (uint64_t(1) << binary::mantissa_explicit_bits()))+            ? 0+            : 1;+    return answer;+  }++  // usually, we round *up*, but if we fall right in between and and we have an+  // even basis, we need to round down+  // We are only concerned with the cases where 5**q fits in single 64-bit word.+  if ((product.low <= 1) && (q >= binary::min_exponent_round_to_even()) &&+      (q <= binary::max_exponent_round_to_even()) &&+      ((answer.mantissa & 3) == 1)) { // we may fall between two floats!+    // To be in-between two floats we need that in doing+    //   answer.mantissa = product.high >> (upperbit + 64 -+    //   binary::mantissa_explicit_bits() - 3);+    // ... we dropped out only zeroes. But if this happened, then we can go+    // back!!!+    if ((answer.mantissa << shift) == product.high) {+      answer.mantissa &= ~uint64_t(1); // flip it so that we do not round up+    }+  }++  answer.mantissa += (answer.mantissa & 1); // round up+  answer.mantissa >>= 1;+  if (answer.mantissa >= (uint64_t(2) << binary::mantissa_explicit_bits())) {+    answer.mantissa = (uint64_t(1) << binary::mantissa_explicit_bits());+    answer.power2++; // undo previous addition+  }++  answer.mantissa &= ~(uint64_t(1) << binary::mantissa_explicit_bits());+  if (answer.power2 >= binary::infinite_power()) { // infinity+    answer.power2 = binary::infinite_power();+    answer.mantissa = 0;+  }+  return answer;+}++} // namespace fast_float++#endif
+ fast_float-8.0.0/include/fast_float/digit_comparison.h view
@@ -0,0 +1,457 @@+#ifndef FASTFLOAT_DIGIT_COMPARISON_H+#define FASTFLOAT_DIGIT_COMPARISON_H++#include <algorithm>+#include <cstdint>+#include <cstring>+#include <iterator>++#include "float_common.h"+#include "bigint.h"+#include "ascii_number.h"++namespace fast_float {++// 1e0 to 1e19+constexpr static uint64_t powers_of_ten_uint64[] = {1UL,+                                                    10UL,+                                                    100UL,+                                                    1000UL,+                                                    10000UL,+                                                    100000UL,+                                                    1000000UL,+                                                    10000000UL,+                                                    100000000UL,+                                                    1000000000UL,+                                                    10000000000UL,+                                                    100000000000UL,+                                                    1000000000000UL,+                                                    10000000000000UL,+                                                    100000000000000UL,+                                                    1000000000000000UL,+                                                    10000000000000000UL,+                                                    100000000000000000UL,+                                                    1000000000000000000UL,+                                                    10000000000000000000UL};++// calculate the exponent, in scientific notation, of the number.+// this algorithm is not even close to optimized, but it has no practical+// effect on performance: in order to have a faster algorithm, we'd need+// to slow down performance for faster algorithms, and this is still fast.+template <typename UC>+fastfloat_really_inline FASTFLOAT_CONSTEXPR14 int32_t+scientific_exponent(parsed_number_string_t<UC> &num) noexcept {+  uint64_t mantissa = num.mantissa;+  int32_t exponent = int32_t(num.exponent);+  while (mantissa >= 10000) {+    mantissa /= 10000;+    exponent += 4;+  }+  while (mantissa >= 100) {+    mantissa /= 100;+    exponent += 2;+  }+  while (mantissa >= 10) {+    mantissa /= 10;+    exponent += 1;+  }+  return exponent;+}++// this converts a native floating-point number to an extended-precision float.+template <typename T>+fastfloat_really_inline FASTFLOAT_CONSTEXPR20 adjusted_mantissa+to_extended(T value) noexcept {+  using equiv_uint = equiv_uint_t<T>;+  constexpr equiv_uint exponent_mask = binary_format<T>::exponent_mask();+  constexpr equiv_uint mantissa_mask = binary_format<T>::mantissa_mask();+  constexpr equiv_uint hidden_bit_mask = binary_format<T>::hidden_bit_mask();++  adjusted_mantissa am;+  int32_t bias = binary_format<T>::mantissa_explicit_bits() -+                 binary_format<T>::minimum_exponent();+  equiv_uint bits;+#if FASTFLOAT_HAS_BIT_CAST+  bits = std::bit_cast<equiv_uint>(value);+#else+  ::memcpy(&bits, &value, sizeof(T));+#endif+  if ((bits & exponent_mask) == 0) {+    // denormal+    am.power2 = 1 - bias;+    am.mantissa = bits & mantissa_mask;+  } else {+    // normal+    am.power2 = int32_t((bits & exponent_mask) >>+                        binary_format<T>::mantissa_explicit_bits());+    am.power2 -= bias;+    am.mantissa = (bits & mantissa_mask) | hidden_bit_mask;+  }++  return am;+}++// get the extended precision value of the halfway point between b and b+u.+// we are given a native float that represents b, so we need to adjust it+// halfway between b and b+u.+template <typename T>+fastfloat_really_inline FASTFLOAT_CONSTEXPR20 adjusted_mantissa+to_extended_halfway(T value) noexcept {+  adjusted_mantissa am = to_extended(value);+  am.mantissa <<= 1;+  am.mantissa += 1;+  am.power2 -= 1;+  return am;+}++// round an extended-precision float to the nearest machine float.+template <typename T, typename callback>+fastfloat_really_inline FASTFLOAT_CONSTEXPR14 void round(adjusted_mantissa &am,+                                                         callback cb) noexcept {+  int32_t mantissa_shift = 64 - binary_format<T>::mantissa_explicit_bits() - 1;+  if (-am.power2 >= mantissa_shift) {+    // have a denormal float+    int32_t shift = -am.power2 + 1;+    cb(am, std::min<int32_t>(shift, 64));+    // check for round-up: if rounding-nearest carried us to the hidden bit.+    am.power2 = (am.mantissa <+                 (uint64_t(1) << binary_format<T>::mantissa_explicit_bits()))+                    ? 0+                    : 1;+    return;+  }++  // have a normal float, use the default shift.+  cb(am, mantissa_shift);++  // check for carry+  if (am.mantissa >=+      (uint64_t(2) << binary_format<T>::mantissa_explicit_bits())) {+    am.mantissa = (uint64_t(1) << binary_format<T>::mantissa_explicit_bits());+    am.power2++;+  }++  // check for infinite: we could have carried to an infinite power+  am.mantissa &= ~(uint64_t(1) << binary_format<T>::mantissa_explicit_bits());+  if (am.power2 >= binary_format<T>::infinite_power()) {+    am.power2 = binary_format<T>::infinite_power();+    am.mantissa = 0;+  }+}++template <typename callback>+fastfloat_really_inline FASTFLOAT_CONSTEXPR14 void+round_nearest_tie_even(adjusted_mantissa &am, int32_t shift,+                       callback cb) noexcept {+  uint64_t const mask = (shift == 64) ? UINT64_MAX : (uint64_t(1) << shift) - 1;+  uint64_t const halfway = (shift == 0) ? 0 : uint64_t(1) << (shift - 1);+  uint64_t truncated_bits = am.mantissa & mask;+  bool is_above = truncated_bits > halfway;+  bool is_halfway = truncated_bits == halfway;++  // shift digits into position+  if (shift == 64) {+    am.mantissa = 0;+  } else {+    am.mantissa >>= shift;+  }+  am.power2 += shift;++  bool is_odd = (am.mantissa & 1) == 1;+  am.mantissa += uint64_t(cb(is_odd, is_halfway, is_above));+}++fastfloat_really_inline FASTFLOAT_CONSTEXPR14 void+round_down(adjusted_mantissa &am, int32_t shift) noexcept {+  if (shift == 64) {+    am.mantissa = 0;+  } else {+    am.mantissa >>= shift;+  }+  am.power2 += shift;+}++template <typename UC>+fastfloat_really_inline FASTFLOAT_CONSTEXPR20 void+skip_zeros(UC const *&first, UC const *last) noexcept {+  uint64_t val;+  while (!cpp20_and_in_constexpr() &&+         std::distance(first, last) >= int_cmp_len<UC>()) {+    ::memcpy(&val, first, sizeof(uint64_t));+    if (val != int_cmp_zeros<UC>()) {+      break;+    }+    first += int_cmp_len<UC>();+  }+  while (first != last) {+    if (*first != UC('0')) {+      break;+    }+    first++;+  }+}++// determine if any non-zero digits were truncated.+// all characters must be valid digits.+template <typename UC>+fastfloat_really_inline FASTFLOAT_CONSTEXPR20 bool+is_truncated(UC const *first, UC const *last) noexcept {+  // do 8-bit optimizations, can just compare to 8 literal 0s.+  uint64_t val;+  while (!cpp20_and_in_constexpr() &&+         std::distance(first, last) >= int_cmp_len<UC>()) {+    ::memcpy(&val, first, sizeof(uint64_t));+    if (val != int_cmp_zeros<UC>()) {+      return true;+    }+    first += int_cmp_len<UC>();+  }+  while (first != last) {+    if (*first != UC('0')) {+      return true;+    }+    ++first;+  }+  return false;+}++template <typename UC>+fastfloat_really_inline FASTFLOAT_CONSTEXPR20 bool+is_truncated(span<UC const> s) noexcept {+  return is_truncated(s.ptr, s.ptr + s.len());+}++template <typename UC>+fastfloat_really_inline FASTFLOAT_CONSTEXPR20 void+parse_eight_digits(UC const *&p, limb &value, size_t &counter,+                   size_t &count) noexcept {+  value = value * 100000000 + parse_eight_digits_unrolled(p);+  p += 8;+  counter += 8;+  count += 8;+}++template <typename UC>+fastfloat_really_inline FASTFLOAT_CONSTEXPR14 void+parse_one_digit(UC const *&p, limb &value, size_t &counter,+                size_t &count) noexcept {+  value = value * 10 + limb(*p - UC('0'));+  p++;+  counter++;+  count++;+}++fastfloat_really_inline FASTFLOAT_CONSTEXPR20 void+add_native(bigint &big, limb power, limb value) noexcept {+  big.mul(power);+  big.add(value);+}++fastfloat_really_inline FASTFLOAT_CONSTEXPR20 void+round_up_bigint(bigint &big, size_t &count) noexcept {+  // need to round-up the digits, but need to avoid rounding+  // ....9999 to ...10000, which could cause a false halfway point.+  add_native(big, 10, 1);+  count++;+}++// parse the significant digits into a big integer+template <typename UC>+inline FASTFLOAT_CONSTEXPR20 void+parse_mantissa(bigint &result, parsed_number_string_t<UC> &num,+               size_t max_digits, size_t &digits) noexcept {+  // try to minimize the number of big integer and scalar multiplication.+  // therefore, try to parse 8 digits at a time, and multiply by the largest+  // scalar value (9 or 19 digits) for each step.+  size_t counter = 0;+  digits = 0;+  limb value = 0;+#ifdef FASTFLOAT_64BIT_LIMB+  size_t step = 19;+#else+  size_t step = 9;+#endif++  // process all integer digits.+  UC const *p = num.integer.ptr;+  UC const *pend = p + num.integer.len();+  skip_zeros(p, pend);+  // process all digits, in increments of step per loop+  while (p != pend) {+    while ((std::distance(p, pend) >= 8) && (step - counter >= 8) &&+           (max_digits - digits >= 8)) {+      parse_eight_digits(p, value, counter, digits);+    }+    while (counter < step && p != pend && digits < max_digits) {+      parse_one_digit(p, value, counter, digits);+    }+    if (digits == max_digits) {+      // add the temporary value, then check if we've truncated any digits+      add_native(result, limb(powers_of_ten_uint64[counter]), value);+      bool truncated = is_truncated(p, pend);+      if (num.fraction.ptr != nullptr) {+        truncated |= is_truncated(num.fraction);+      }+      if (truncated) {+        round_up_bigint(result, digits);+      }+      return;+    } else {+      add_native(result, limb(powers_of_ten_uint64[counter]), value);+      counter = 0;+      value = 0;+    }+  }++  // add our fraction digits, if they're available.+  if (num.fraction.ptr != nullptr) {+    p = num.fraction.ptr;+    pend = p + num.fraction.len();+    if (digits == 0) {+      skip_zeros(p, pend);+    }+    // process all digits, in increments of step per loop+    while (p != pend) {+      while ((std::distance(p, pend) >= 8) && (step - counter >= 8) &&+             (max_digits - digits >= 8)) {+        parse_eight_digits(p, value, counter, digits);+      }+      while (counter < step && p != pend && digits < max_digits) {+        parse_one_digit(p, value, counter, digits);+      }+      if (digits == max_digits) {+        // add the temporary value, then check if we've truncated any digits+        add_native(result, limb(powers_of_ten_uint64[counter]), value);+        bool truncated = is_truncated(p, pend);+        if (truncated) {+          round_up_bigint(result, digits);+        }+        return;+      } else {+        add_native(result, limb(powers_of_ten_uint64[counter]), value);+        counter = 0;+        value = 0;+      }+    }+  }++  if (counter != 0) {+    add_native(result, limb(powers_of_ten_uint64[counter]), value);+  }+}++template <typename T>+inline FASTFLOAT_CONSTEXPR20 adjusted_mantissa+positive_digit_comp(bigint &bigmant, int32_t exponent) noexcept {+  FASTFLOAT_ASSERT(bigmant.pow10(uint32_t(exponent)));+  adjusted_mantissa answer;+  bool truncated;+  answer.mantissa = bigmant.hi64(truncated);+  int bias = binary_format<T>::mantissa_explicit_bits() -+             binary_format<T>::minimum_exponent();+  answer.power2 = bigmant.bit_length() - 64 + bias;++  round<T>(answer, [truncated](adjusted_mantissa &a, int32_t shift) {+    round_nearest_tie_even(+        a, shift,+        [truncated](bool is_odd, bool is_halfway, bool is_above) -> bool {+          return is_above || (is_halfway && truncated) ||+                 (is_odd && is_halfway);+        });+  });++  return answer;+}++// the scaling here is quite simple: we have, for the real digits `m * 10^e`,+// and for the theoretical digits `n * 2^f`. Since `e` is always negative,+// to scale them identically, we do `n * 2^f * 5^-f`, so we now have `m * 2^e`.+// we then need to scale by `2^(f- e)`, and then the two significant digits+// are of the same magnitude.+template <typename T>+inline FASTFLOAT_CONSTEXPR20 adjusted_mantissa negative_digit_comp(+    bigint &bigmant, adjusted_mantissa am, int32_t exponent) noexcept {+  bigint &real_digits = bigmant;+  int32_t real_exp = exponent;++  // get the value of `b`, rounded down, and get a bigint representation of b+h+  adjusted_mantissa am_b = am;+  // gcc7 buf: use a lambda to remove the noexcept qualifier bug with+  // -Wnoexcept-type.+  round<T>(am_b,+           [](adjusted_mantissa &a, int32_t shift) { round_down(a, shift); });+  T b;+  to_float(false, am_b, b);+  adjusted_mantissa theor = to_extended_halfway(b);+  bigint theor_digits(theor.mantissa);+  int32_t theor_exp = theor.power2;++  // scale real digits and theor digits to be same power.+  int32_t pow2_exp = theor_exp - real_exp;+  uint32_t pow5_exp = uint32_t(-real_exp);+  if (pow5_exp != 0) {+    FASTFLOAT_ASSERT(theor_digits.pow5(pow5_exp));+  }+  if (pow2_exp > 0) {+    FASTFLOAT_ASSERT(theor_digits.pow2(uint32_t(pow2_exp)));+  } else if (pow2_exp < 0) {+    FASTFLOAT_ASSERT(real_digits.pow2(uint32_t(-pow2_exp)));+  }++  // compare digits, and use it to director rounding+  int ord = real_digits.compare(theor_digits);+  adjusted_mantissa answer = am;+  round<T>(answer, [ord](adjusted_mantissa &a, int32_t shift) {+    round_nearest_tie_even(+        a, shift, [ord](bool is_odd, bool _, bool __) -> bool {+          (void)_;  // not needed, since we've done our comparison+          (void)__; // not needed, since we've done our comparison+          if (ord > 0) {+            return true;+          } else if (ord < 0) {+            return false;+          } else {+            return is_odd;+          }+        });+  });++  return answer;+}++// parse the significant digits as a big integer to unambiguously round the+// the significant digits. here, we are trying to determine how to round+// an extended float representation close to `b+h`, halfway between `b`+// (the float rounded-down) and `b+u`, the next positive float. this+// algorithm is always correct, and uses one of two approaches. when+// the exponent is positive relative to the significant digits (such as+// 1234), we create a big-integer representation, get the high 64-bits,+// determine if any lower bits are truncated, and use that to direct+// rounding. in case of a negative exponent relative to the significant+// digits (such as 1.2345), we create a theoretical representation of+// `b` as a big-integer type, scaled to the same binary exponent as+// the actual digits. we then compare the big integer representations+// of both, and use that to direct rounding.+template <typename T, typename UC>+inline FASTFLOAT_CONSTEXPR20 adjusted_mantissa+digit_comp(parsed_number_string_t<UC> &num, adjusted_mantissa am) noexcept {+  // remove the invalid exponent bias+  am.power2 -= invalid_am_bias;++  int32_t sci_exp = scientific_exponent(num);+  size_t max_digits = binary_format<T>::max_digits();+  size_t digits = 0;+  bigint bigmant;+  parse_mantissa(bigmant, num, max_digits, digits);+  // can't underflow, since digits is at most max_digits.+  int32_t exponent = sci_exp + 1 - int32_t(digits);+  if (exponent >= 0) {+    return positive_digit_comp<T>(bigmant, exponent);+  } else {+    return negative_digit_comp<T>(bigmant, am, exponent);+  }+}++} // namespace fast_float++#endif
+ fast_float-8.0.0/include/fast_float/fast_float.h view
@@ -0,0 +1,59 @@++#ifndef FASTFLOAT_FAST_FLOAT_H+#define FASTFLOAT_FAST_FLOAT_H++#include "float_common.h"++namespace fast_float {+/**+ * This function parses the character sequence [first,last) for a number. It+ * parses floating-point numbers expecting a locale-indepent format equivalent+ * to what is used by std::strtod in the default ("C") locale. The resulting+ * floating-point value is the closest floating-point values (using either float+ * or double), using the "round to even" convention for values that would+ * otherwise fall right in-between two values. That is, we provide exact parsing+ * according to the IEEE standard.+ *+ * Given a successful parse, the pointer (`ptr`) in the returned value is set to+ * point right after the parsed number, and the `value` referenced is set to the+ * parsed value. In case of error, the returned `ec` contains a representative+ * error, otherwise the default (`std::errc()`) value is stored.+ *+ * The implementation does not throw and does not allocate memory (e.g., with+ * `new` or `malloc`).+ *+ * Like the C++17 standard, the `fast_float::from_chars` functions take an+ * optional last argument of the type `fast_float::chars_format`. It is a bitset+ * value: we check whether `fmt & fast_float::chars_format::fixed` and `fmt &+ * fast_float::chars_format::scientific` are set to determine whether we allow+ * the fixed point and scientific notation respectively. The default is+ * `fast_float::chars_format::general` which allows both `fixed` and+ * `scientific`.+ */+template <typename T, typename UC = char,+          typename = FASTFLOAT_ENABLE_IF(is_supported_float_type<T>::value)>+FASTFLOAT_CONSTEXPR20 from_chars_result_t<UC>+from_chars(UC const *first, UC const *last, T &value,+           chars_format fmt = chars_format::general) noexcept;++/**+ * Like from_chars, but accepts an `options` argument to govern number parsing.+ * Both for floating-point types and integer types.+ */+template <typename T, typename UC = char>+FASTFLOAT_CONSTEXPR20 from_chars_result_t<UC>+from_chars_advanced(UC const *first, UC const *last, T &value,+                    parse_options_t<UC> options) noexcept;++/**+ * from_chars for integer types.+ */+template <typename T, typename UC = char,+          typename = FASTFLOAT_ENABLE_IF(is_supported_integer_type<T>::value)>+FASTFLOAT_CONSTEXPR20 from_chars_result_t<UC>+from_chars(UC const *first, UC const *last, T &value, int base = 10) noexcept;++} // namespace fast_float++#include "parse_number.h"+#endif // FASTFLOAT_FAST_FLOAT_H
+ fast_float-8.0.0/include/fast_float/fast_table.h view
@@ -0,0 +1,708 @@+#ifndef FASTFLOAT_FAST_TABLE_H+#define FASTFLOAT_FAST_TABLE_H++#include <cstdint>++namespace fast_float {++/**+ * When mapping numbers from decimal to binary,+ * we go from w * 10^q to m * 2^p but we have+ * 10^q = 5^q * 2^q, so effectively+ * we are trying to match+ * w * 2^q * 5^q to m * 2^p. Thus the powers of two+ * are not a concern since they can be represented+ * exactly using the binary notation, only the powers of five+ * affect the binary significand.+ */++/**+ * The smallest non-zero float (binary64) is 2^-1074.+ * We take as input numbers of the form w x 10^q where w < 2^64.+ * We have that w * 10^-343  <  2^(64-344) 5^-343 < 2^-1076.+ * However, we have that+ * (2^64-1) * 10^-342 =  (2^64-1) * 2^-342 * 5^-342 > 2^-1074.+ * Thus it is possible for a number of the form w * 10^-342 where+ * w is a 64-bit value to be a non-zero floating-point number.+ *********+ * Any number of form w * 10^309 where w>= 1 is going to be+ * infinite in binary64 so we never need to worry about powers+ * of 5 greater than 308.+ */+template <class unused = void> struct powers_template {++  constexpr static int smallest_power_of_five =+      binary_format<double>::smallest_power_of_ten();+  constexpr static int largest_power_of_five =+      binary_format<double>::largest_power_of_ten();+  constexpr static int number_of_entries =+      2 * (largest_power_of_five - smallest_power_of_five + 1);+  // Powers of five from 5^-342 all the way to 5^308 rounded toward one.+  constexpr static uint64_t power_of_five_128[number_of_entries] = {+      0xeef453d6923bd65a, 0x113faa2906a13b3f,+      0x9558b4661b6565f8, 0x4ac7ca59a424c507,+      0xbaaee17fa23ebf76, 0x5d79bcf00d2df649,+      0xe95a99df8ace6f53, 0xf4d82c2c107973dc,+      0x91d8a02bb6c10594, 0x79071b9b8a4be869,+      0xb64ec836a47146f9, 0x9748e2826cdee284,+      0xe3e27a444d8d98b7, 0xfd1b1b2308169b25,+      0x8e6d8c6ab0787f72, 0xfe30f0f5e50e20f7,+      0xb208ef855c969f4f, 0xbdbd2d335e51a935,+      0xde8b2b66b3bc4723, 0xad2c788035e61382,+      0x8b16fb203055ac76, 0x4c3bcb5021afcc31,+      0xaddcb9e83c6b1793, 0xdf4abe242a1bbf3d,+      0xd953e8624b85dd78, 0xd71d6dad34a2af0d,+      0x87d4713d6f33aa6b, 0x8672648c40e5ad68,+      0xa9c98d8ccb009506, 0x680efdaf511f18c2,+      0xd43bf0effdc0ba48, 0x212bd1b2566def2,+      0x84a57695fe98746d, 0x14bb630f7604b57,+      0xa5ced43b7e3e9188, 0x419ea3bd35385e2d,+      0xcf42894a5dce35ea, 0x52064cac828675b9,+      0x818995ce7aa0e1b2, 0x7343efebd1940993,+      0xa1ebfb4219491a1f, 0x1014ebe6c5f90bf8,+      0xca66fa129f9b60a6, 0xd41a26e077774ef6,+      0xfd00b897478238d0, 0x8920b098955522b4,+      0x9e20735e8cb16382, 0x55b46e5f5d5535b0,+      0xc5a890362fddbc62, 0xeb2189f734aa831d,+      0xf712b443bbd52b7b, 0xa5e9ec7501d523e4,+      0x9a6bb0aa55653b2d, 0x47b233c92125366e,+      0xc1069cd4eabe89f8, 0x999ec0bb696e840a,+      0xf148440a256e2c76, 0xc00670ea43ca250d,+      0x96cd2a865764dbca, 0x380406926a5e5728,+      0xbc807527ed3e12bc, 0xc605083704f5ecf2,+      0xeba09271e88d976b, 0xf7864a44c633682e,+      0x93445b8731587ea3, 0x7ab3ee6afbe0211d,+      0xb8157268fdae9e4c, 0x5960ea05bad82964,+      0xe61acf033d1a45df, 0x6fb92487298e33bd,+      0x8fd0c16206306bab, 0xa5d3b6d479f8e056,+      0xb3c4f1ba87bc8696, 0x8f48a4899877186c,+      0xe0b62e2929aba83c, 0x331acdabfe94de87,+      0x8c71dcd9ba0b4925, 0x9ff0c08b7f1d0b14,+      0xaf8e5410288e1b6f, 0x7ecf0ae5ee44dd9,+      0xdb71e91432b1a24a, 0xc9e82cd9f69d6150,+      0x892731ac9faf056e, 0xbe311c083a225cd2,+      0xab70fe17c79ac6ca, 0x6dbd630a48aaf406,+      0xd64d3d9db981787d, 0x92cbbccdad5b108,+      0x85f0468293f0eb4e, 0x25bbf56008c58ea5,+      0xa76c582338ed2621, 0xaf2af2b80af6f24e,+      0xd1476e2c07286faa, 0x1af5af660db4aee1,+      0x82cca4db847945ca, 0x50d98d9fc890ed4d,+      0xa37fce126597973c, 0xe50ff107bab528a0,+      0xcc5fc196fefd7d0c, 0x1e53ed49a96272c8,+      0xff77b1fcbebcdc4f, 0x25e8e89c13bb0f7a,+      0x9faacf3df73609b1, 0x77b191618c54e9ac,+      0xc795830d75038c1d, 0xd59df5b9ef6a2417,+      0xf97ae3d0d2446f25, 0x4b0573286b44ad1d,+      0x9becce62836ac577, 0x4ee367f9430aec32,+      0xc2e801fb244576d5, 0x229c41f793cda73f,+      0xf3a20279ed56d48a, 0x6b43527578c1110f,+      0x9845418c345644d6, 0x830a13896b78aaa9,+      0xbe5691ef416bd60c, 0x23cc986bc656d553,+      0xedec366b11c6cb8f, 0x2cbfbe86b7ec8aa8,+      0x94b3a202eb1c3f39, 0x7bf7d71432f3d6a9,+      0xb9e08a83a5e34f07, 0xdaf5ccd93fb0cc53,+      0xe858ad248f5c22c9, 0xd1b3400f8f9cff68,+      0x91376c36d99995be, 0x23100809b9c21fa1,+      0xb58547448ffffb2d, 0xabd40a0c2832a78a,+      0xe2e69915b3fff9f9, 0x16c90c8f323f516c,+      0x8dd01fad907ffc3b, 0xae3da7d97f6792e3,+      0xb1442798f49ffb4a, 0x99cd11cfdf41779c,+      0xdd95317f31c7fa1d, 0x40405643d711d583,+      0x8a7d3eef7f1cfc52, 0x482835ea666b2572,+      0xad1c8eab5ee43b66, 0xda3243650005eecf,+      0xd863b256369d4a40, 0x90bed43e40076a82,+      0x873e4f75e2224e68, 0x5a7744a6e804a291,+      0xa90de3535aaae202, 0x711515d0a205cb36,+      0xd3515c2831559a83, 0xd5a5b44ca873e03,+      0x8412d9991ed58091, 0xe858790afe9486c2,+      0xa5178fff668ae0b6, 0x626e974dbe39a872,+      0xce5d73ff402d98e3, 0xfb0a3d212dc8128f,+      0x80fa687f881c7f8e, 0x7ce66634bc9d0b99,+      0xa139029f6a239f72, 0x1c1fffc1ebc44e80,+      0xc987434744ac874e, 0xa327ffb266b56220,+      0xfbe9141915d7a922, 0x4bf1ff9f0062baa8,+      0x9d71ac8fada6c9b5, 0x6f773fc3603db4a9,+      0xc4ce17b399107c22, 0xcb550fb4384d21d3,+      0xf6019da07f549b2b, 0x7e2a53a146606a48,+      0x99c102844f94e0fb, 0x2eda7444cbfc426d,+      0xc0314325637a1939, 0xfa911155fefb5308,+      0xf03d93eebc589f88, 0x793555ab7eba27ca,+      0x96267c7535b763b5, 0x4bc1558b2f3458de,+      0xbbb01b9283253ca2, 0x9eb1aaedfb016f16,+      0xea9c227723ee8bcb, 0x465e15a979c1cadc,+      0x92a1958a7675175f, 0xbfacd89ec191ec9,+      0xb749faed14125d36, 0xcef980ec671f667b,+      0xe51c79a85916f484, 0x82b7e12780e7401a,+      0x8f31cc0937ae58d2, 0xd1b2ecb8b0908810,+      0xb2fe3f0b8599ef07, 0x861fa7e6dcb4aa15,+      0xdfbdcece67006ac9, 0x67a791e093e1d49a,+      0x8bd6a141006042bd, 0xe0c8bb2c5c6d24e0,+      0xaecc49914078536d, 0x58fae9f773886e18,+      0xda7f5bf590966848, 0xaf39a475506a899e,+      0x888f99797a5e012d, 0x6d8406c952429603,+      0xaab37fd7d8f58178, 0xc8e5087ba6d33b83,+      0xd5605fcdcf32e1d6, 0xfb1e4a9a90880a64,+      0x855c3be0a17fcd26, 0x5cf2eea09a55067f,+      0xa6b34ad8c9dfc06f, 0xf42faa48c0ea481e,+      0xd0601d8efc57b08b, 0xf13b94daf124da26,+      0x823c12795db6ce57, 0x76c53d08d6b70858,+      0xa2cb1717b52481ed, 0x54768c4b0c64ca6e,+      0xcb7ddcdda26da268, 0xa9942f5dcf7dfd09,+      0xfe5d54150b090b02, 0xd3f93b35435d7c4c,+      0x9efa548d26e5a6e1, 0xc47bc5014a1a6daf,+      0xc6b8e9b0709f109a, 0x359ab6419ca1091b,+      0xf867241c8cc6d4c0, 0xc30163d203c94b62,+      0x9b407691d7fc44f8, 0x79e0de63425dcf1d,+      0xc21094364dfb5636, 0x985915fc12f542e4,+      0xf294b943e17a2bc4, 0x3e6f5b7b17b2939d,+      0x979cf3ca6cec5b5a, 0xa705992ceecf9c42,+      0xbd8430bd08277231, 0x50c6ff782a838353,+      0xece53cec4a314ebd, 0xa4f8bf5635246428,+      0x940f4613ae5ed136, 0x871b7795e136be99,+      0xb913179899f68584, 0x28e2557b59846e3f,+      0xe757dd7ec07426e5, 0x331aeada2fe589cf,+      0x9096ea6f3848984f, 0x3ff0d2c85def7621,+      0xb4bca50b065abe63, 0xfed077a756b53a9,+      0xe1ebce4dc7f16dfb, 0xd3e8495912c62894,+      0x8d3360f09cf6e4bd, 0x64712dd7abbbd95c,+      0xb080392cc4349dec, 0xbd8d794d96aacfb3,+      0xdca04777f541c567, 0xecf0d7a0fc5583a0,+      0x89e42caaf9491b60, 0xf41686c49db57244,+      0xac5d37d5b79b6239, 0x311c2875c522ced5,+      0xd77485cb25823ac7, 0x7d633293366b828b,+      0x86a8d39ef77164bc, 0xae5dff9c02033197,+      0xa8530886b54dbdeb, 0xd9f57f830283fdfc,+      0xd267caa862a12d66, 0xd072df63c324fd7b,+      0x8380dea93da4bc60, 0x4247cb9e59f71e6d,+      0xa46116538d0deb78, 0x52d9be85f074e608,+      0xcd795be870516656, 0x67902e276c921f8b,+      0x806bd9714632dff6, 0xba1cd8a3db53b6,+      0xa086cfcd97bf97f3, 0x80e8a40eccd228a4,+      0xc8a883c0fdaf7df0, 0x6122cd128006b2cd,+      0xfad2a4b13d1b5d6c, 0x796b805720085f81,+      0x9cc3a6eec6311a63, 0xcbe3303674053bb0,+      0xc3f490aa77bd60fc, 0xbedbfc4411068a9c,+      0xf4f1b4d515acb93b, 0xee92fb5515482d44,+      0x991711052d8bf3c5, 0x751bdd152d4d1c4a,+      0xbf5cd54678eef0b6, 0xd262d45a78a0635d,+      0xef340a98172aace4, 0x86fb897116c87c34,+      0x9580869f0e7aac0e, 0xd45d35e6ae3d4da0,+      0xbae0a846d2195712, 0x8974836059cca109,+      0xe998d258869facd7, 0x2bd1a438703fc94b,+      0x91ff83775423cc06, 0x7b6306a34627ddcf,+      0xb67f6455292cbf08, 0x1a3bc84c17b1d542,+      0xe41f3d6a7377eeca, 0x20caba5f1d9e4a93,+      0x8e938662882af53e, 0x547eb47b7282ee9c,+      0xb23867fb2a35b28d, 0xe99e619a4f23aa43,+      0xdec681f9f4c31f31, 0x6405fa00e2ec94d4,+      0x8b3c113c38f9f37e, 0xde83bc408dd3dd04,+      0xae0b158b4738705e, 0x9624ab50b148d445,+      0xd98ddaee19068c76, 0x3badd624dd9b0957,+      0x87f8a8d4cfa417c9, 0xe54ca5d70a80e5d6,+      0xa9f6d30a038d1dbc, 0x5e9fcf4ccd211f4c,+      0xd47487cc8470652b, 0x7647c3200069671f,+      0x84c8d4dfd2c63f3b, 0x29ecd9f40041e073,+      0xa5fb0a17c777cf09, 0xf468107100525890,+      0xcf79cc9db955c2cc, 0x7182148d4066eeb4,+      0x81ac1fe293d599bf, 0xc6f14cd848405530,+      0xa21727db38cb002f, 0xb8ada00e5a506a7c,+      0xca9cf1d206fdc03b, 0xa6d90811f0e4851c,+      0xfd442e4688bd304a, 0x908f4a166d1da663,+      0x9e4a9cec15763e2e, 0x9a598e4e043287fe,+      0xc5dd44271ad3cdba, 0x40eff1e1853f29fd,+      0xf7549530e188c128, 0xd12bee59e68ef47c,+      0x9a94dd3e8cf578b9, 0x82bb74f8301958ce,+      0xc13a148e3032d6e7, 0xe36a52363c1faf01,+      0xf18899b1bc3f8ca1, 0xdc44e6c3cb279ac1,+      0x96f5600f15a7b7e5, 0x29ab103a5ef8c0b9,+      0xbcb2b812db11a5de, 0x7415d448f6b6f0e7,+      0xebdf661791d60f56, 0x111b495b3464ad21,+      0x936b9fcebb25c995, 0xcab10dd900beec34,+      0xb84687c269ef3bfb, 0x3d5d514f40eea742,+      0xe65829b3046b0afa, 0xcb4a5a3112a5112,+      0x8ff71a0fe2c2e6dc, 0x47f0e785eaba72ab,+      0xb3f4e093db73a093, 0x59ed216765690f56,+      0xe0f218b8d25088b8, 0x306869c13ec3532c,+      0x8c974f7383725573, 0x1e414218c73a13fb,+      0xafbd2350644eeacf, 0xe5d1929ef90898fa,+      0xdbac6c247d62a583, 0xdf45f746b74abf39,+      0x894bc396ce5da772, 0x6b8bba8c328eb783,+      0xab9eb47c81f5114f, 0x66ea92f3f326564,+      0xd686619ba27255a2, 0xc80a537b0efefebd,+      0x8613fd0145877585, 0xbd06742ce95f5f36,+      0xa798fc4196e952e7, 0x2c48113823b73704,+      0xd17f3b51fca3a7a0, 0xf75a15862ca504c5,+      0x82ef85133de648c4, 0x9a984d73dbe722fb,+      0xa3ab66580d5fdaf5, 0xc13e60d0d2e0ebba,+      0xcc963fee10b7d1b3, 0x318df905079926a8,+      0xffbbcfe994e5c61f, 0xfdf17746497f7052,+      0x9fd561f1fd0f9bd3, 0xfeb6ea8bedefa633,+      0xc7caba6e7c5382c8, 0xfe64a52ee96b8fc0,+      0xf9bd690a1b68637b, 0x3dfdce7aa3c673b0,+      0x9c1661a651213e2d, 0x6bea10ca65c084e,+      0xc31bfa0fe5698db8, 0x486e494fcff30a62,+      0xf3e2f893dec3f126, 0x5a89dba3c3efccfa,+      0x986ddb5c6b3a76b7, 0xf89629465a75e01c,+      0xbe89523386091465, 0xf6bbb397f1135823,+      0xee2ba6c0678b597f, 0x746aa07ded582e2c,+      0x94db483840b717ef, 0xa8c2a44eb4571cdc,+      0xba121a4650e4ddeb, 0x92f34d62616ce413,+      0xe896a0d7e51e1566, 0x77b020baf9c81d17,+      0x915e2486ef32cd60, 0xace1474dc1d122e,+      0xb5b5ada8aaff80b8, 0xd819992132456ba,+      0xe3231912d5bf60e6, 0x10e1fff697ed6c69,+      0x8df5efabc5979c8f, 0xca8d3ffa1ef463c1,+      0xb1736b96b6fd83b3, 0xbd308ff8a6b17cb2,+      0xddd0467c64bce4a0, 0xac7cb3f6d05ddbde,+      0x8aa22c0dbef60ee4, 0x6bcdf07a423aa96b,+      0xad4ab7112eb3929d, 0x86c16c98d2c953c6,+      0xd89d64d57a607744, 0xe871c7bf077ba8b7,+      0x87625f056c7c4a8b, 0x11471cd764ad4972,+      0xa93af6c6c79b5d2d, 0xd598e40d3dd89bcf,+      0xd389b47879823479, 0x4aff1d108d4ec2c3,+      0x843610cb4bf160cb, 0xcedf722a585139ba,+      0xa54394fe1eedb8fe, 0xc2974eb4ee658828,+      0xce947a3da6a9273e, 0x733d226229feea32,+      0x811ccc668829b887, 0x806357d5a3f525f,+      0xa163ff802a3426a8, 0xca07c2dcb0cf26f7,+      0xc9bcff6034c13052, 0xfc89b393dd02f0b5,+      0xfc2c3f3841f17c67, 0xbbac2078d443ace2,+      0x9d9ba7832936edc0, 0xd54b944b84aa4c0d,+      0xc5029163f384a931, 0xa9e795e65d4df11,+      0xf64335bcf065d37d, 0x4d4617b5ff4a16d5,+      0x99ea0196163fa42e, 0x504bced1bf8e4e45,+      0xc06481fb9bcf8d39, 0xe45ec2862f71e1d6,+      0xf07da27a82c37088, 0x5d767327bb4e5a4c,+      0x964e858c91ba2655, 0x3a6a07f8d510f86f,+      0xbbe226efb628afea, 0x890489f70a55368b,+      0xeadab0aba3b2dbe5, 0x2b45ac74ccea842e,+      0x92c8ae6b464fc96f, 0x3b0b8bc90012929d,+      0xb77ada0617e3bbcb, 0x9ce6ebb40173744,+      0xe55990879ddcaabd, 0xcc420a6a101d0515,+      0x8f57fa54c2a9eab6, 0x9fa946824a12232d,+      0xb32df8e9f3546564, 0x47939822dc96abf9,+      0xdff9772470297ebd, 0x59787e2b93bc56f7,+      0x8bfbea76c619ef36, 0x57eb4edb3c55b65a,+      0xaefae51477a06b03, 0xede622920b6b23f1,+      0xdab99e59958885c4, 0xe95fab368e45eced,+      0x88b402f7fd75539b, 0x11dbcb0218ebb414,+      0xaae103b5fcd2a881, 0xd652bdc29f26a119,+      0xd59944a37c0752a2, 0x4be76d3346f0495f,+      0x857fcae62d8493a5, 0x6f70a4400c562ddb,+      0xa6dfbd9fb8e5b88e, 0xcb4ccd500f6bb952,+      0xd097ad07a71f26b2, 0x7e2000a41346a7a7,+      0x825ecc24c873782f, 0x8ed400668c0c28c8,+      0xa2f67f2dfa90563b, 0x728900802f0f32fa,+      0xcbb41ef979346bca, 0x4f2b40a03ad2ffb9,+      0xfea126b7d78186bc, 0xe2f610c84987bfa8,+      0x9f24b832e6b0f436, 0xdd9ca7d2df4d7c9,+      0xc6ede63fa05d3143, 0x91503d1c79720dbb,+      0xf8a95fcf88747d94, 0x75a44c6397ce912a,+      0x9b69dbe1b548ce7c, 0xc986afbe3ee11aba,+      0xc24452da229b021b, 0xfbe85badce996168,+      0xf2d56790ab41c2a2, 0xfae27299423fb9c3,+      0x97c560ba6b0919a5, 0xdccd879fc967d41a,+      0xbdb6b8e905cb600f, 0x5400e987bbc1c920,+      0xed246723473e3813, 0x290123e9aab23b68,+      0x9436c0760c86e30b, 0xf9a0b6720aaf6521,+      0xb94470938fa89bce, 0xf808e40e8d5b3e69,+      0xe7958cb87392c2c2, 0xb60b1d1230b20e04,+      0x90bd77f3483bb9b9, 0xb1c6f22b5e6f48c2,+      0xb4ecd5f01a4aa828, 0x1e38aeb6360b1af3,+      0xe2280b6c20dd5232, 0x25c6da63c38de1b0,+      0x8d590723948a535f, 0x579c487e5a38ad0e,+      0xb0af48ec79ace837, 0x2d835a9df0c6d851,+      0xdcdb1b2798182244, 0xf8e431456cf88e65,+      0x8a08f0f8bf0f156b, 0x1b8e9ecb641b58ff,+      0xac8b2d36eed2dac5, 0xe272467e3d222f3f,+      0xd7adf884aa879177, 0x5b0ed81dcc6abb0f,+      0x86ccbb52ea94baea, 0x98e947129fc2b4e9,+      0xa87fea27a539e9a5, 0x3f2398d747b36224,+      0xd29fe4b18e88640e, 0x8eec7f0d19a03aad,+      0x83a3eeeef9153e89, 0x1953cf68300424ac,+      0xa48ceaaab75a8e2b, 0x5fa8c3423c052dd7,+      0xcdb02555653131b6, 0x3792f412cb06794d,+      0x808e17555f3ebf11, 0xe2bbd88bbee40bd0,+      0xa0b19d2ab70e6ed6, 0x5b6aceaeae9d0ec4,+      0xc8de047564d20a8b, 0xf245825a5a445275,+      0xfb158592be068d2e, 0xeed6e2f0f0d56712,+      0x9ced737bb6c4183d, 0x55464dd69685606b,+      0xc428d05aa4751e4c, 0xaa97e14c3c26b886,+      0xf53304714d9265df, 0xd53dd99f4b3066a8,+      0x993fe2c6d07b7fab, 0xe546a8038efe4029,+      0xbf8fdb78849a5f96, 0xde98520472bdd033,+      0xef73d256a5c0f77c, 0x963e66858f6d4440,+      0x95a8637627989aad, 0xdde7001379a44aa8,+      0xbb127c53b17ec159, 0x5560c018580d5d52,+      0xe9d71b689dde71af, 0xaab8f01e6e10b4a6,+      0x9226712162ab070d, 0xcab3961304ca70e8,+      0xb6b00d69bb55c8d1, 0x3d607b97c5fd0d22,+      0xe45c10c42a2b3b05, 0x8cb89a7db77c506a,+      0x8eb98a7a9a5b04e3, 0x77f3608e92adb242,+      0xb267ed1940f1c61c, 0x55f038b237591ed3,+      0xdf01e85f912e37a3, 0x6b6c46dec52f6688,+      0x8b61313bbabce2c6, 0x2323ac4b3b3da015,+      0xae397d8aa96c1b77, 0xabec975e0a0d081a,+      0xd9c7dced53c72255, 0x96e7bd358c904a21,+      0x881cea14545c7575, 0x7e50d64177da2e54,+      0xaa242499697392d2, 0xdde50bd1d5d0b9e9,+      0xd4ad2dbfc3d07787, 0x955e4ec64b44e864,+      0x84ec3c97da624ab4, 0xbd5af13bef0b113e,+      0xa6274bbdd0fadd61, 0xecb1ad8aeacdd58e,+      0xcfb11ead453994ba, 0x67de18eda5814af2,+      0x81ceb32c4b43fcf4, 0x80eacf948770ced7,+      0xa2425ff75e14fc31, 0xa1258379a94d028d,+      0xcad2f7f5359a3b3e, 0x96ee45813a04330,+      0xfd87b5f28300ca0d, 0x8bca9d6e188853fc,+      0x9e74d1b791e07e48, 0x775ea264cf55347e,+      0xc612062576589dda, 0x95364afe032a819e,+      0xf79687aed3eec551, 0x3a83ddbd83f52205,+      0x9abe14cd44753b52, 0xc4926a9672793543,+      0xc16d9a0095928a27, 0x75b7053c0f178294,+      0xf1c90080baf72cb1, 0x5324c68b12dd6339,+      0x971da05074da7bee, 0xd3f6fc16ebca5e04,+      0xbce5086492111aea, 0x88f4bb1ca6bcf585,+      0xec1e4a7db69561a5, 0x2b31e9e3d06c32e6,+      0x9392ee8e921d5d07, 0x3aff322e62439fd0,+      0xb877aa3236a4b449, 0x9befeb9fad487c3,+      0xe69594bec44de15b, 0x4c2ebe687989a9b4,+      0x901d7cf73ab0acd9, 0xf9d37014bf60a11,+      0xb424dc35095cd80f, 0x538484c19ef38c95,+      0xe12e13424bb40e13, 0x2865a5f206b06fba,+      0x8cbccc096f5088cb, 0xf93f87b7442e45d4,+      0xafebff0bcb24aafe, 0xf78f69a51539d749,+      0xdbe6fecebdedd5be, 0xb573440e5a884d1c,+      0x89705f4136b4a597, 0x31680a88f8953031,+      0xabcc77118461cefc, 0xfdc20d2b36ba7c3e,+      0xd6bf94d5e57a42bc, 0x3d32907604691b4d,+      0x8637bd05af6c69b5, 0xa63f9a49c2c1b110,+      0xa7c5ac471b478423, 0xfcf80dc33721d54,+      0xd1b71758e219652b, 0xd3c36113404ea4a9,+      0x83126e978d4fdf3b, 0x645a1cac083126ea,+      0xa3d70a3d70a3d70a, 0x3d70a3d70a3d70a4,+      0xcccccccccccccccc, 0xcccccccccccccccd,+      0x8000000000000000, 0x0,+      0xa000000000000000, 0x0,+      0xc800000000000000, 0x0,+      0xfa00000000000000, 0x0,+      0x9c40000000000000, 0x0,+      0xc350000000000000, 0x0,+      0xf424000000000000, 0x0,+      0x9896800000000000, 0x0,+      0xbebc200000000000, 0x0,+      0xee6b280000000000, 0x0,+      0x9502f90000000000, 0x0,+      0xba43b74000000000, 0x0,+      0xe8d4a51000000000, 0x0,+      0x9184e72a00000000, 0x0,+      0xb5e620f480000000, 0x0,+      0xe35fa931a0000000, 0x0,+      0x8e1bc9bf04000000, 0x0,+      0xb1a2bc2ec5000000, 0x0,+      0xde0b6b3a76400000, 0x0,+      0x8ac7230489e80000, 0x0,+      0xad78ebc5ac620000, 0x0,+      0xd8d726b7177a8000, 0x0,+      0x878678326eac9000, 0x0,+      0xa968163f0a57b400, 0x0,+      0xd3c21bcecceda100, 0x0,+      0x84595161401484a0, 0x0,+      0xa56fa5b99019a5c8, 0x0,+      0xcecb8f27f4200f3a, 0x0,+      0x813f3978f8940984, 0x4000000000000000,+      0xa18f07d736b90be5, 0x5000000000000000,+      0xc9f2c9cd04674ede, 0xa400000000000000,+      0xfc6f7c4045812296, 0x4d00000000000000,+      0x9dc5ada82b70b59d, 0xf020000000000000,+      0xc5371912364ce305, 0x6c28000000000000,+      0xf684df56c3e01bc6, 0xc732000000000000,+      0x9a130b963a6c115c, 0x3c7f400000000000,+      0xc097ce7bc90715b3, 0x4b9f100000000000,+      0xf0bdc21abb48db20, 0x1e86d40000000000,+      0x96769950b50d88f4, 0x1314448000000000,+      0xbc143fa4e250eb31, 0x17d955a000000000,+      0xeb194f8e1ae525fd, 0x5dcfab0800000000,+      0x92efd1b8d0cf37be, 0x5aa1cae500000000,+      0xb7abc627050305ad, 0xf14a3d9e40000000,+      0xe596b7b0c643c719, 0x6d9ccd05d0000000,+      0x8f7e32ce7bea5c6f, 0xe4820023a2000000,+      0xb35dbf821ae4f38b, 0xdda2802c8a800000,+      0xe0352f62a19e306e, 0xd50b2037ad200000,+      0x8c213d9da502de45, 0x4526f422cc340000,+      0xaf298d050e4395d6, 0x9670b12b7f410000,+      0xdaf3f04651d47b4c, 0x3c0cdd765f114000,+      0x88d8762bf324cd0f, 0xa5880a69fb6ac800,+      0xab0e93b6efee0053, 0x8eea0d047a457a00,+      0xd5d238a4abe98068, 0x72a4904598d6d880,+      0x85a36366eb71f041, 0x47a6da2b7f864750,+      0xa70c3c40a64e6c51, 0x999090b65f67d924,+      0xd0cf4b50cfe20765, 0xfff4b4e3f741cf6d,+      0x82818f1281ed449f, 0xbff8f10e7a8921a4,+      0xa321f2d7226895c7, 0xaff72d52192b6a0d,+      0xcbea6f8ceb02bb39, 0x9bf4f8a69f764490,+      0xfee50b7025c36a08, 0x2f236d04753d5b4,+      0x9f4f2726179a2245, 0x1d762422c946590,+      0xc722f0ef9d80aad6, 0x424d3ad2b7b97ef5,+      0xf8ebad2b84e0d58b, 0xd2e0898765a7deb2,+      0x9b934c3b330c8577, 0x63cc55f49f88eb2f,+      0xc2781f49ffcfa6d5, 0x3cbf6b71c76b25fb,+      0xf316271c7fc3908a, 0x8bef464e3945ef7a,+      0x97edd871cfda3a56, 0x97758bf0e3cbb5ac,+      0xbde94e8e43d0c8ec, 0x3d52eeed1cbea317,+      0xed63a231d4c4fb27, 0x4ca7aaa863ee4bdd,+      0x945e455f24fb1cf8, 0x8fe8caa93e74ef6a,+      0xb975d6b6ee39e436, 0xb3e2fd538e122b44,+      0xe7d34c64a9c85d44, 0x60dbbca87196b616,+      0x90e40fbeea1d3a4a, 0xbc8955e946fe31cd,+      0xb51d13aea4a488dd, 0x6babab6398bdbe41,+      0xe264589a4dcdab14, 0xc696963c7eed2dd1,+      0x8d7eb76070a08aec, 0xfc1e1de5cf543ca2,+      0xb0de65388cc8ada8, 0x3b25a55f43294bcb,+      0xdd15fe86affad912, 0x49ef0eb713f39ebe,+      0x8a2dbf142dfcc7ab, 0x6e3569326c784337,+      0xacb92ed9397bf996, 0x49c2c37f07965404,+      0xd7e77a8f87daf7fb, 0xdc33745ec97be906,+      0x86f0ac99b4e8dafd, 0x69a028bb3ded71a3,+      0xa8acd7c0222311bc, 0xc40832ea0d68ce0c,+      0xd2d80db02aabd62b, 0xf50a3fa490c30190,+      0x83c7088e1aab65db, 0x792667c6da79e0fa,+      0xa4b8cab1a1563f52, 0x577001b891185938,+      0xcde6fd5e09abcf26, 0xed4c0226b55e6f86,+      0x80b05e5ac60b6178, 0x544f8158315b05b4,+      0xa0dc75f1778e39d6, 0x696361ae3db1c721,+      0xc913936dd571c84c, 0x3bc3a19cd1e38e9,+      0xfb5878494ace3a5f, 0x4ab48a04065c723,+      0x9d174b2dcec0e47b, 0x62eb0d64283f9c76,+      0xc45d1df942711d9a, 0x3ba5d0bd324f8394,+      0xf5746577930d6500, 0xca8f44ec7ee36479,+      0x9968bf6abbe85f20, 0x7e998b13cf4e1ecb,+      0xbfc2ef456ae276e8, 0x9e3fedd8c321a67e,+      0xefb3ab16c59b14a2, 0xc5cfe94ef3ea101e,+      0x95d04aee3b80ece5, 0xbba1f1d158724a12,+      0xbb445da9ca61281f, 0x2a8a6e45ae8edc97,+      0xea1575143cf97226, 0xf52d09d71a3293bd,+      0x924d692ca61be758, 0x593c2626705f9c56,+      0xb6e0c377cfa2e12e, 0x6f8b2fb00c77836c,+      0xe498f455c38b997a, 0xb6dfb9c0f956447,+      0x8edf98b59a373fec, 0x4724bd4189bd5eac,+      0xb2977ee300c50fe7, 0x58edec91ec2cb657,+      0xdf3d5e9bc0f653e1, 0x2f2967b66737e3ed,+      0x8b865b215899f46c, 0xbd79e0d20082ee74,+      0xae67f1e9aec07187, 0xecd8590680a3aa11,+      0xda01ee641a708de9, 0xe80e6f4820cc9495,+      0x884134fe908658b2, 0x3109058d147fdcdd,+      0xaa51823e34a7eede, 0xbd4b46f0599fd415,+      0xd4e5e2cdc1d1ea96, 0x6c9e18ac7007c91a,+      0x850fadc09923329e, 0x3e2cf6bc604ddb0,+      0xa6539930bf6bff45, 0x84db8346b786151c,+      0xcfe87f7cef46ff16, 0xe612641865679a63,+      0x81f14fae158c5f6e, 0x4fcb7e8f3f60c07e,+      0xa26da3999aef7749, 0xe3be5e330f38f09d,+      0xcb090c8001ab551c, 0x5cadf5bfd3072cc5,+      0xfdcb4fa002162a63, 0x73d9732fc7c8f7f6,+      0x9e9f11c4014dda7e, 0x2867e7fddcdd9afa,+      0xc646d63501a1511d, 0xb281e1fd541501b8,+      0xf7d88bc24209a565, 0x1f225a7ca91a4226,+      0x9ae757596946075f, 0x3375788de9b06958,+      0xc1a12d2fc3978937, 0x52d6b1641c83ae,+      0xf209787bb47d6b84, 0xc0678c5dbd23a49a,+      0x9745eb4d50ce6332, 0xf840b7ba963646e0,+      0xbd176620a501fbff, 0xb650e5a93bc3d898,+      0xec5d3fa8ce427aff, 0xa3e51f138ab4cebe,+      0x93ba47c980e98cdf, 0xc66f336c36b10137,+      0xb8a8d9bbe123f017, 0xb80b0047445d4184,+      0xe6d3102ad96cec1d, 0xa60dc059157491e5,+      0x9043ea1ac7e41392, 0x87c89837ad68db2f,+      0xb454e4a179dd1877, 0x29babe4598c311fb,+      0xe16a1dc9d8545e94, 0xf4296dd6fef3d67a,+      0x8ce2529e2734bb1d, 0x1899e4a65f58660c,+      0xb01ae745b101e9e4, 0x5ec05dcff72e7f8f,+      0xdc21a1171d42645d, 0x76707543f4fa1f73,+      0x899504ae72497eba, 0x6a06494a791c53a8,+      0xabfa45da0edbde69, 0x487db9d17636892,+      0xd6f8d7509292d603, 0x45a9d2845d3c42b6,+      0x865b86925b9bc5c2, 0xb8a2392ba45a9b2,+      0xa7f26836f282b732, 0x8e6cac7768d7141e,+      0xd1ef0244af2364ff, 0x3207d795430cd926,+      0x8335616aed761f1f, 0x7f44e6bd49e807b8,+      0xa402b9c5a8d3a6e7, 0x5f16206c9c6209a6,+      0xcd036837130890a1, 0x36dba887c37a8c0f,+      0x802221226be55a64, 0xc2494954da2c9789,+      0xa02aa96b06deb0fd, 0xf2db9baa10b7bd6c,+      0xc83553c5c8965d3d, 0x6f92829494e5acc7,+      0xfa42a8b73abbf48c, 0xcb772339ba1f17f9,+      0x9c69a97284b578d7, 0xff2a760414536efb,+      0xc38413cf25e2d70d, 0xfef5138519684aba,+      0xf46518c2ef5b8cd1, 0x7eb258665fc25d69,+      0x98bf2f79d5993802, 0xef2f773ffbd97a61,+      0xbeeefb584aff8603, 0xaafb550ffacfd8fa,+      0xeeaaba2e5dbf6784, 0x95ba2a53f983cf38,+      0x952ab45cfa97a0b2, 0xdd945a747bf26183,+      0xba756174393d88df, 0x94f971119aeef9e4,+      0xe912b9d1478ceb17, 0x7a37cd5601aab85d,+      0x91abb422ccb812ee, 0xac62e055c10ab33a,+      0xb616a12b7fe617aa, 0x577b986b314d6009,+      0xe39c49765fdf9d94, 0xed5a7e85fda0b80b,+      0x8e41ade9fbebc27d, 0x14588f13be847307,+      0xb1d219647ae6b31c, 0x596eb2d8ae258fc8,+      0xde469fbd99a05fe3, 0x6fca5f8ed9aef3bb,+      0x8aec23d680043bee, 0x25de7bb9480d5854,+      0xada72ccc20054ae9, 0xaf561aa79a10ae6a,+      0xd910f7ff28069da4, 0x1b2ba1518094da04,+      0x87aa9aff79042286, 0x90fb44d2f05d0842,+      0xa99541bf57452b28, 0x353a1607ac744a53,+      0xd3fa922f2d1675f2, 0x42889b8997915ce8,+      0x847c9b5d7c2e09b7, 0x69956135febada11,+      0xa59bc234db398c25, 0x43fab9837e699095,+      0xcf02b2c21207ef2e, 0x94f967e45e03f4bb,+      0x8161afb94b44f57d, 0x1d1be0eebac278f5,+      0xa1ba1ba79e1632dc, 0x6462d92a69731732,+      0xca28a291859bbf93, 0x7d7b8f7503cfdcfe,+      0xfcb2cb35e702af78, 0x5cda735244c3d43e,+      0x9defbf01b061adab, 0x3a0888136afa64a7,+      0xc56baec21c7a1916, 0x88aaa1845b8fdd0,+      0xf6c69a72a3989f5b, 0x8aad549e57273d45,+      0x9a3c2087a63f6399, 0x36ac54e2f678864b,+      0xc0cb28a98fcf3c7f, 0x84576a1bb416a7dd,+      0xf0fdf2d3f3c30b9f, 0x656d44a2a11c51d5,+      0x969eb7c47859e743, 0x9f644ae5a4b1b325,+      0xbc4665b596706114, 0x873d5d9f0dde1fee,+      0xeb57ff22fc0c7959, 0xa90cb506d155a7ea,+      0x9316ff75dd87cbd8, 0x9a7f12442d588f2,+      0xb7dcbf5354e9bece, 0xc11ed6d538aeb2f,+      0xe5d3ef282a242e81, 0x8f1668c8a86da5fa,+      0x8fa475791a569d10, 0xf96e017d694487bc,+      0xb38d92d760ec4455, 0x37c981dcc395a9ac,+      0xe070f78d3927556a, 0x85bbe253f47b1417,+      0x8c469ab843b89562, 0x93956d7478ccec8e,+      0xaf58416654a6babb, 0x387ac8d1970027b2,+      0xdb2e51bfe9d0696a, 0x6997b05fcc0319e,+      0x88fcf317f22241e2, 0x441fece3bdf81f03,+      0xab3c2fddeeaad25a, 0xd527e81cad7626c3,+      0xd60b3bd56a5586f1, 0x8a71e223d8d3b074,+      0x85c7056562757456, 0xf6872d5667844e49,+      0xa738c6bebb12d16c, 0xb428f8ac016561db,+      0xd106f86e69d785c7, 0xe13336d701beba52,+      0x82a45b450226b39c, 0xecc0024661173473,+      0xa34d721642b06084, 0x27f002d7f95d0190,+      0xcc20ce9bd35c78a5, 0x31ec038df7b441f4,+      0xff290242c83396ce, 0x7e67047175a15271,+      0x9f79a169bd203e41, 0xf0062c6e984d386,+      0xc75809c42c684dd1, 0x52c07b78a3e60868,+      0xf92e0c3537826145, 0xa7709a56ccdf8a82,+      0x9bbcc7a142b17ccb, 0x88a66076400bb691,+      0xc2abf989935ddbfe, 0x6acff893d00ea435,+      0xf356f7ebf83552fe, 0x583f6b8c4124d43,+      0x98165af37b2153de, 0xc3727a337a8b704a,+      0xbe1bf1b059e9a8d6, 0x744f18c0592e4c5c,+      0xeda2ee1c7064130c, 0x1162def06f79df73,+      0x9485d4d1c63e8be7, 0x8addcb5645ac2ba8,+      0xb9a74a0637ce2ee1, 0x6d953e2bd7173692,+      0xe8111c87c5c1ba99, 0xc8fa8db6ccdd0437,+      0x910ab1d4db9914a0, 0x1d9c9892400a22a2,+      0xb54d5e4a127f59c8, 0x2503beb6d00cab4b,+      0xe2a0b5dc971f303a, 0x2e44ae64840fd61d,+      0x8da471a9de737e24, 0x5ceaecfed289e5d2,+      0xb10d8e1456105dad, 0x7425a83e872c5f47,+      0xdd50f1996b947518, 0xd12f124e28f77719,+      0x8a5296ffe33cc92f, 0x82bd6b70d99aaa6f,+      0xace73cbfdc0bfb7b, 0x636cc64d1001550b,+      0xd8210befd30efa5a, 0x3c47f7e05401aa4e,+      0x8714a775e3e95c78, 0x65acfaec34810a71,+      0xa8d9d1535ce3b396, 0x7f1839a741a14d0d,+      0xd31045a8341ca07c, 0x1ede48111209a050,+      0x83ea2b892091e44d, 0x934aed0aab460432,+      0xa4e4b66b68b65d60, 0xf81da84d5617853f,+      0xce1de40642e3f4b9, 0x36251260ab9d668e,+      0x80d2ae83e9ce78f3, 0xc1d72b7c6b426019,+      0xa1075a24e4421730, 0xb24cf65b8612f81f,+      0xc94930ae1d529cfc, 0xdee033f26797b627,+      0xfb9b7cd9a4a7443c, 0x169840ef017da3b1,+      0x9d412e0806e88aa5, 0x8e1f289560ee864e,+      0xc491798a08a2ad4e, 0xf1a6f2bab92a27e2,+      0xf5b5d7ec8acb58a2, 0xae10af696774b1db,+      0x9991a6f3d6bf1765, 0xacca6da1e0a8ef29,+      0xbff610b0cc6edd3f, 0x17fd090a58d32af3,+      0xeff394dcff8a948e, 0xddfc4b4cef07f5b0,+      0x95f83d0a1fb69cd9, 0x4abdaf101564f98e,+      0xbb764c4ca7a4440f, 0x9d6d1ad41abe37f1,+      0xea53df5fd18d5513, 0x84c86189216dc5ed,+      0x92746b9be2f8552c, 0x32fd3cf5b4e49bb4,+      0xb7118682dbb66a77, 0x3fbc8c33221dc2a1,+      0xe4d5e82392a40515, 0xfabaf3feaa5334a,+      0x8f05b1163ba6832d, 0x29cb4d87f2a7400e,+      0xb2c71d5bca9023f8, 0x743e20e9ef511012,+      0xdf78e4b2bd342cf6, 0x914da9246b255416,+      0x8bab8eefb6409c1a, 0x1ad089b6c2f7548e,+      0xae9672aba3d0c320, 0xa184ac2473b529b1,+      0xda3c0f568cc4f3e8, 0xc9e5d72d90a2741e,+      0x8865899617fb1871, 0x7e2fa67c7a658892,+      0xaa7eebfb9df9de8d, 0xddbb901b98feeab7,+      0xd51ea6fa85785631, 0x552a74227f3ea565,+      0x8533285c936b35de, 0xd53a88958f87275f,+      0xa67ff273b8460356, 0x8a892abaf368f137,+      0xd01fef10a657842c, 0x2d2b7569b0432d85,+      0x8213f56a67f6b29b, 0x9c3b29620e29fc73,+      0xa298f2c501f45f42, 0x8349f3ba91b47b8f,+      0xcb3f2f7642717713, 0x241c70a936219a73,+      0xfe0efb53d30dd4d7, 0xed238cd383aa0110,+      0x9ec95d1463e8a506, 0xf4363804324a40aa,+      0xc67bb4597ce2ce48, 0xb143c6053edcd0d5,+      0xf81aa16fdc1b81da, 0xdd94b7868e94050a,+      0x9b10a4e5e9913128, 0xca7cf2b4191c8326,+      0xc1d4ce1f63f57d72, 0xfd1c2f611f63a3f0,+      0xf24a01a73cf2dccf, 0xbc633b39673c8cec,+      0x976e41088617ca01, 0xd5be0503e085d813,+      0xbd49d14aa79dbc82, 0x4b2d8644d8a74e18,+      0xec9c459d51852ba2, 0xddf8e7d60ed1219e,+      0x93e1ab8252f33b45, 0xcabb90e5c942b503,+      0xb8da1662e7b00a17, 0x3d6a751f3b936243,+      0xe7109bfba19c0c9d, 0xcc512670a783ad4,+      0x906a617d450187e2, 0x27fb2b80668b24c5,+      0xb484f9dc9641e9da, 0xb1f9f660802dedf6,+      0xe1a63853bbd26451, 0x5e7873f8a0396973,+      0x8d07e33455637eb2, 0xdb0b487b6423e1e8,+      0xb049dc016abc5e5f, 0x91ce1a9a3d2cda62,+      0xdc5c5301c56b75f7, 0x7641a140cc7810fb,+      0x89b9b3e11b6329ba, 0xa9e904c87fcb0a9d,+      0xac2820d9623bf429, 0x546345fa9fbdcd44,+      0xd732290fbacaf133, 0xa97c177947ad4095,+      0x867f59a9d4bed6c0, 0x49ed8eabcccc485d,+      0xa81f301449ee8c70, 0x5c68f256bfff5a74,+      0xd226fc195c6a2f8c, 0x73832eec6fff3111,+      0x83585d8fd9c25db7, 0xc831fd53c5ff7eab,+      0xa42e74f3d032f525, 0xba3e7ca8b77f5e55,+      0xcd3a1230c43fb26f, 0x28ce1bd2e55f35eb,+      0x80444b5e7aa7cf85, 0x7980d163cf5b81b3,+      0xa0555e361951c366, 0xd7e105bcc332621f,+      0xc86ab5c39fa63440, 0x8dd9472bf3fefaa7,+      0xfa856334878fc150, 0xb14f98f6f0feb951,+      0x9c935e00d4b9d8d2, 0x6ed1bf9a569f33d3,+      0xc3b8358109e84f07, 0xa862f80ec4700c8,+      0xf4a642e14c6262c8, 0xcd27bb612758c0fa,+      0x98e7e9cccfbd7dbd, 0x8038d51cb897789c,+      0xbf21e44003acdd2c, 0xe0470a63e6bd56c3,+      0xeeea5d5004981478, 0x1858ccfce06cac74,+      0x95527a5202df0ccb, 0xf37801e0c43ebc8,+      0xbaa718e68396cffd, 0xd30560258f54e6ba,+      0xe950df20247c83fd, 0x47c6b82ef32a2069,+      0x91d28b7416cdd27e, 0x4cdc331d57fa5441,+      0xb6472e511c81471d, 0xe0133fe4adf8e952,+      0xe3d8f9e563a198e5, 0x58180fddd97723a6,+      0x8e679c2f5e44ff8f, 0x570f09eaa7ea7648,+  };+};++#if FASTFLOAT_DETAIL_MUST_DEFINE_CONSTEXPR_VARIABLE++template <class unused>+constexpr uint64_t+    powers_template<unused>::power_of_five_128[number_of_entries];++#endif++using powers = powers_template<>;++} // namespace fast_float++#endif
+ fast_float-8.0.0/include/fast_float/float_common.h view
@@ -0,0 +1,1240 @@+#ifndef FASTFLOAT_FLOAT_COMMON_H+#define FASTFLOAT_FLOAT_COMMON_H++#include <cfloat>+#include <cstdint>+#include <cassert>+#include <cstring>+#include <limits>+#include <type_traits>+#include <system_error>+#ifdef __has_include+#if __has_include(<stdfloat>) && (__cplusplus > 202002L || _MSVC_LANG > 202002L)+#include <stdfloat>+#endif+#endif+#include "constexpr_feature_detect.h"++#define FASTFLOAT_VERSION_MAJOR 8+#define FASTFLOAT_VERSION_MINOR 0+#define FASTFLOAT_VERSION_PATCH 0++#define FASTFLOAT_STRINGIZE_IMPL(x) #x+#define FASTFLOAT_STRINGIZE(x) FASTFLOAT_STRINGIZE_IMPL(x)++#define FASTFLOAT_VERSION_STR                                                  \+  FASTFLOAT_STRINGIZE(FASTFLOAT_VERSION_MAJOR)                                 \+  "." FASTFLOAT_STRINGIZE(FASTFLOAT_VERSION_MINOR) "." FASTFLOAT_STRINGIZE(    \+      FASTFLOAT_VERSION_PATCH)++#define FASTFLOAT_VERSION                                                      \+  (FASTFLOAT_VERSION_MAJOR * 10000 + FASTFLOAT_VERSION_MINOR * 100 +           \+   FASTFLOAT_VERSION_PATCH)++namespace fast_float {++enum class chars_format : uint64_t;++namespace detail {+constexpr chars_format basic_json_fmt = chars_format(1 << 5);+constexpr chars_format basic_fortran_fmt = chars_format(1 << 6);+} // namespace detail++enum class chars_format : uint64_t {+  scientific = 1 << 0,+  fixed = 1 << 2,+  hex = 1 << 3,+  no_infnan = 1 << 4,+  // RFC 8259: https://datatracker.ietf.org/doc/html/rfc8259#section-6+  json = uint64_t(detail::basic_json_fmt) | fixed | scientific | no_infnan,+  // Extension of RFC 8259 where, e.g., "inf" and "nan" are allowed.+  json_or_infnan = uint64_t(detail::basic_json_fmt) | fixed | scientific,+  fortran = uint64_t(detail::basic_fortran_fmt) | fixed | scientific,+  general = fixed | scientific,+  allow_leading_plus = 1 << 7,+  skip_white_space = 1 << 8,+};++template <typename UC> struct from_chars_result_t {+  UC const *ptr;+  std::errc ec;+};++using from_chars_result = from_chars_result_t<char>;++template <typename UC> struct parse_options_t {+  constexpr explicit parse_options_t(chars_format fmt = chars_format::general,+                                     UC dot = UC('.'), int b = 10)+      : format(fmt), decimal_point(dot), base(b) {}++  /** Which number formats are accepted */+  chars_format format;+  /** The character used as decimal point */+  UC decimal_point;+  /** The base used for integers */+  int base;+};++using parse_options = parse_options_t<char>;++} // namespace fast_float++#if FASTFLOAT_HAS_BIT_CAST+#include <bit>+#endif++#if (defined(__x86_64) || defined(__x86_64__) || defined(_M_X64) ||            \+     defined(__amd64) || defined(__aarch64__) || defined(_M_ARM64) ||          \+     defined(__MINGW64__) || defined(__s390x__) ||                             \+     (defined(__ppc64__) || defined(__PPC64__) || defined(__ppc64le__) ||      \+      defined(__PPC64LE__)) ||                                                 \+     defined(__loongarch64))+#define FASTFLOAT_64BIT 1+#elif (defined(__i386) || defined(__i386__) || defined(_M_IX86) ||             \+       defined(__arm__) || defined(_M_ARM) || defined(__ppc__) ||              \+       defined(__MINGW32__) || defined(__EMSCRIPTEN__))+#define FASTFLOAT_32BIT 1+#else+  // Need to check incrementally, since SIZE_MAX is a size_t, avoid overflow.+// We can never tell the register width, but the SIZE_MAX is a good+// approximation. UINTPTR_MAX and INTPTR_MAX are optional, so avoid them for max+// portability.+#if SIZE_MAX == 0xffff+#error Unknown platform (16-bit, unsupported)+#elif SIZE_MAX == 0xffffffff+#define FASTFLOAT_32BIT 1+#elif SIZE_MAX == 0xffffffffffffffff+#define FASTFLOAT_64BIT 1+#else+#error Unknown platform (not 32-bit, not 64-bit?)+#endif+#endif++#if ((defined(_WIN32) || defined(_WIN64)) && !defined(__clang__)) ||           \+    (defined(_M_ARM64) && !defined(__MINGW32__))+#include <intrin.h>+#endif++#if defined(_MSC_VER) && !defined(__clang__)+#define FASTFLOAT_VISUAL_STUDIO 1+#endif++#if defined __BYTE_ORDER__ && defined __ORDER_BIG_ENDIAN__+#define FASTFLOAT_IS_BIG_ENDIAN (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)+#elif defined _WIN32+#define FASTFLOAT_IS_BIG_ENDIAN 0+#else+#if defined(__APPLE__) || defined(__FreeBSD__)+#include <machine/endian.h>+#elif defined(sun) || defined(__sun)+#include <sys/byteorder.h>+#elif defined(__MVS__)+#include <sys/endian.h>+#else+#ifdef __has_include+#if __has_include(<endian.h>)+#include <endian.h>+#endif //__has_include(<endian.h>)+#endif //__has_include+#endif+#+#ifndef __BYTE_ORDER__+// safe choice+#define FASTFLOAT_IS_BIG_ENDIAN 0+#endif+#+#ifndef __ORDER_LITTLE_ENDIAN__+// safe choice+#define FASTFLOAT_IS_BIG_ENDIAN 0+#endif+#+#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__+#define FASTFLOAT_IS_BIG_ENDIAN 0+#else+#define FASTFLOAT_IS_BIG_ENDIAN 1+#endif+#endif++#if defined(__SSE2__) || (defined(FASTFLOAT_VISUAL_STUDIO) &&                  \+                          (defined(_M_AMD64) || defined(_M_X64) ||             \+                           (defined(_M_IX86_FP) && _M_IX86_FP == 2)))+#define FASTFLOAT_SSE2 1+#endif++#if defined(__aarch64__) || defined(_M_ARM64)+#define FASTFLOAT_NEON 1+#endif++#if defined(FASTFLOAT_SSE2) || defined(FASTFLOAT_NEON)+#define FASTFLOAT_HAS_SIMD 1+#endif++#if defined(__GNUC__)+// disable -Wcast-align=strict (GCC only)+#define FASTFLOAT_SIMD_DISABLE_WARNINGS                                        \+  _Pragma("GCC diagnostic push")                                               \+      _Pragma("GCC diagnostic ignored \"-Wcast-align\"")+#else+#define FASTFLOAT_SIMD_DISABLE_WARNINGS+#endif++#if defined(__GNUC__)+#define FASTFLOAT_SIMD_RESTORE_WARNINGS _Pragma("GCC diagnostic pop")+#else+#define FASTFLOAT_SIMD_RESTORE_WARNINGS+#endif++#ifdef FASTFLOAT_VISUAL_STUDIO+#define fastfloat_really_inline __forceinline+#else+#define fastfloat_really_inline inline __attribute__((always_inline))+#endif++#ifndef FASTFLOAT_ASSERT+#define FASTFLOAT_ASSERT(x)                                                    \+  { ((void)(x)); }+#endif++#ifndef FASTFLOAT_DEBUG_ASSERT+#define FASTFLOAT_DEBUG_ASSERT(x)                                              \+  { ((void)(x)); }+#endif++// rust style `try!()` macro, or `?` operator+#define FASTFLOAT_TRY(x)                                                       \+  {                                                                            \+    if (!(x))                                                                  \+      return false;                                                            \+  }++#define FASTFLOAT_ENABLE_IF(...)                                               \+  typename std::enable_if<(__VA_ARGS__), int>::type++namespace fast_float {++fastfloat_really_inline constexpr bool cpp20_and_in_constexpr() {+#if FASTFLOAT_HAS_IS_CONSTANT_EVALUATED+  return std::is_constant_evaluated();+#else+  return false;+#endif+}++template <typename T>+struct is_supported_float_type+    : std::integral_constant<+          bool, std::is_same<T, double>::value || std::is_same<T, float>::value+#ifdef __STDCPP_FLOAT64_T__+                    || std::is_same<T, std::float64_t>::value+#endif+#ifdef __STDCPP_FLOAT32_T__+                    || std::is_same<T, std::float32_t>::value+#endif+#ifdef __STDCPP_FLOAT16_T__+                    || std::is_same<T, std::float16_t>::value+#endif+#ifdef __STDCPP_BFLOAT16_T__+                    || std::is_same<T, std::bfloat16_t>::value+#endif+          > {+};++template <typename T>+using equiv_uint_t = typename std::conditional<+    sizeof(T) == 1, uint8_t,+    typename std::conditional<+        sizeof(T) == 2, uint16_t,+        typename std::conditional<sizeof(T) == 4, uint32_t,+                                  uint64_t>::type>::type>::type;++template <typename T> struct is_supported_integer_type : std::is_integral<T> {};++template <typename UC>+struct is_supported_char_type+    : std::integral_constant<bool, std::is_same<UC, char>::value ||+                                       std::is_same<UC, wchar_t>::value ||+                                       std::is_same<UC, char16_t>::value ||+                                       std::is_same<UC, char32_t>::value+#ifdef __cpp_char8_t+                                       || std::is_same<UC, char8_t>::value+#endif+                             > {+};++// Compares two ASCII strings in a case insensitive manner.+template <typename UC>+inline FASTFLOAT_CONSTEXPR14 bool+fastfloat_strncasecmp(UC const *actual_mixedcase, UC const *expected_lowercase,+                      size_t length) {+  for (size_t i = 0; i < length; ++i) {+    UC const actual = actual_mixedcase[i];+    if ((actual < 256 ? actual | 32 : actual) != expected_lowercase[i]) {+      return false;+    }+  }+  return true;+}++#ifndef FLT_EVAL_METHOD+#error "FLT_EVAL_METHOD should be defined, please include cfloat."+#endif++// a pointer and a length to a contiguous block of memory+template <typename T> struct span {+  T const *ptr;+  size_t length;++  constexpr span(T const *_ptr, size_t _length) : ptr(_ptr), length(_length) {}++  constexpr span() : ptr(nullptr), length(0) {}++  constexpr size_t len() const noexcept { return length; }++  FASTFLOAT_CONSTEXPR14 const T &operator[](size_t index) const noexcept {+    FASTFLOAT_DEBUG_ASSERT(index < length);+    return ptr[index];+  }+};++struct value128 {+  uint64_t low;+  uint64_t high;++  constexpr value128(uint64_t _low, uint64_t _high) : low(_low), high(_high) {}++  constexpr value128() : low(0), high(0) {}+};++/* Helper C++14 constexpr generic implementation of leading_zeroes */+fastfloat_really_inline FASTFLOAT_CONSTEXPR14 int+leading_zeroes_generic(uint64_t input_num, int last_bit = 0) {+  if (input_num & uint64_t(0xffffffff00000000)) {+    input_num >>= 32;+    last_bit |= 32;+  }+  if (input_num & uint64_t(0xffff0000)) {+    input_num >>= 16;+    last_bit |= 16;+  }+  if (input_num & uint64_t(0xff00)) {+    input_num >>= 8;+    last_bit |= 8;+  }+  if (input_num & uint64_t(0xf0)) {+    input_num >>= 4;+    last_bit |= 4;+  }+  if (input_num & uint64_t(0xc)) {+    input_num >>= 2;+    last_bit |= 2;+  }+  if (input_num & uint64_t(0x2)) { /* input_num >>=  1; */+    last_bit |= 1;+  }+  return 63 - last_bit;+}++/* result might be undefined when input_num is zero */+fastfloat_really_inline FASTFLOAT_CONSTEXPR20 int+leading_zeroes(uint64_t input_num) {+  assert(input_num > 0);+  if (cpp20_and_in_constexpr()) {+    return leading_zeroes_generic(input_num);+  }+#ifdef FASTFLOAT_VISUAL_STUDIO+#if defined(_M_X64) || defined(_M_ARM64)+  unsigned long leading_zero = 0;+  // Search the mask data from most significant bit (MSB)+  // to least significant bit (LSB) for a set bit (1).+  _BitScanReverse64(&leading_zero, input_num);+  return (int)(63 - leading_zero);+#else+  return leading_zeroes_generic(input_num);+#endif+#else+  return __builtin_clzll(input_num);+#endif+}++// slow emulation routine for 32-bit+fastfloat_really_inline constexpr uint64_t emulu(uint32_t x, uint32_t y) {+  return x * (uint64_t)y;+}++fastfloat_really_inline FASTFLOAT_CONSTEXPR14 uint64_t+umul128_generic(uint64_t ab, uint64_t cd, uint64_t *hi) {+  uint64_t ad = emulu((uint32_t)(ab >> 32), (uint32_t)cd);+  uint64_t bd = emulu((uint32_t)ab, (uint32_t)cd);+  uint64_t adbc = ad + emulu((uint32_t)ab, (uint32_t)(cd >> 32));+  uint64_t adbc_carry = (uint64_t)(adbc < ad);+  uint64_t lo = bd + (adbc << 32);+  *hi = emulu((uint32_t)(ab >> 32), (uint32_t)(cd >> 32)) + (adbc >> 32) ++        (adbc_carry << 32) + (uint64_t)(lo < bd);+  return lo;+}++#ifdef FASTFLOAT_32BIT++// slow emulation routine for 32-bit+#if !defined(__MINGW64__)+fastfloat_really_inline FASTFLOAT_CONSTEXPR14 uint64_t _umul128(uint64_t ab,+                                                                uint64_t cd,+                                                                uint64_t *hi) {+  return umul128_generic(ab, cd, hi);+}+#endif // !__MINGW64__++#endif // FASTFLOAT_32BIT++// compute 64-bit a*b+fastfloat_really_inline FASTFLOAT_CONSTEXPR20 value128+full_multiplication(uint64_t a, uint64_t b) {+  if (cpp20_and_in_constexpr()) {+    value128 answer;+    answer.low = umul128_generic(a, b, &answer.high);+    return answer;+  }+  value128 answer;+#if defined(_M_ARM64) && !defined(__MINGW32__)+  // ARM64 has native support for 64-bit multiplications, no need to emulate+  // But MinGW on ARM64 doesn't have native support for 64-bit multiplications+  answer.high = __umulh(a, b);+  answer.low = a * b;+#elif defined(FASTFLOAT_32BIT) ||                                              \+    (defined(_WIN64) && !defined(__clang__) && !defined(_M_ARM64))+  answer.low = _umul128(a, b, &answer.high); // _umul128 not available on ARM64+#elif defined(FASTFLOAT_64BIT) && defined(__SIZEOF_INT128__)+  __uint128_t r = ((__uint128_t)a) * b;+  answer.low = uint64_t(r);+  answer.high = uint64_t(r >> 64);+#else+  answer.low = umul128_generic(a, b, &answer.high);+#endif+  return answer;+}++struct adjusted_mantissa {+  uint64_t mantissa{0};+  int32_t power2{0}; // a negative value indicates an invalid result+  adjusted_mantissa() = default;++  constexpr bool operator==(adjusted_mantissa const &o) const {+    return mantissa == o.mantissa && power2 == o.power2;+  }++  constexpr bool operator!=(adjusted_mantissa const &o) const {+    return mantissa != o.mantissa || power2 != o.power2;+  }+};++// Bias so we can get the real exponent with an invalid adjusted_mantissa.+constexpr static int32_t invalid_am_bias = -0x8000;++// used for binary_format_lookup_tables<T>::max_mantissa+constexpr uint64_t constant_55555 = 5 * 5 * 5 * 5 * 5;++template <typename T, typename U = void> struct binary_format_lookup_tables;++template <typename T> struct binary_format : binary_format_lookup_tables<T> {+  using equiv_uint = equiv_uint_t<T>;++  static constexpr int mantissa_explicit_bits();+  static constexpr int minimum_exponent();+  static constexpr int infinite_power();+  static constexpr int sign_index();+  static constexpr int+  min_exponent_fast_path(); // used when fegetround() == FE_TONEAREST+  static constexpr int max_exponent_fast_path();+  static constexpr int max_exponent_round_to_even();+  static constexpr int min_exponent_round_to_even();+  static constexpr uint64_t max_mantissa_fast_path(int64_t power);+  static constexpr uint64_t+  max_mantissa_fast_path(); // used when fegetround() == FE_TONEAREST+  static constexpr int largest_power_of_ten();+  static constexpr int smallest_power_of_ten();+  static constexpr T exact_power_of_ten(int64_t power);+  static constexpr size_t max_digits();+  static constexpr equiv_uint exponent_mask();+  static constexpr equiv_uint mantissa_mask();+  static constexpr equiv_uint hidden_bit_mask();+};++template <typename U> struct binary_format_lookup_tables<double, U> {+  static constexpr double powers_of_ten[] = {+      1e0,  1e1,  1e2,  1e3,  1e4,  1e5,  1e6,  1e7,  1e8,  1e9,  1e10, 1e11,+      1e12, 1e13, 1e14, 1e15, 1e16, 1e17, 1e18, 1e19, 1e20, 1e21, 1e22};++  // Largest integer value v so that (5**index * v) <= 1<<53.+  // 0x20000000000000 == 1 << 53+  static constexpr uint64_t max_mantissa[] = {+      0x20000000000000,+      0x20000000000000 / 5,+      0x20000000000000 / (5 * 5),+      0x20000000000000 / (5 * 5 * 5),+      0x20000000000000 / (5 * 5 * 5 * 5),+      0x20000000000000 / (constant_55555),+      0x20000000000000 / (constant_55555 * 5),+      0x20000000000000 / (constant_55555 * 5 * 5),+      0x20000000000000 / (constant_55555 * 5 * 5 * 5),+      0x20000000000000 / (constant_55555 * 5 * 5 * 5 * 5),+      0x20000000000000 / (constant_55555 * constant_55555),+      0x20000000000000 / (constant_55555 * constant_55555 * 5),+      0x20000000000000 / (constant_55555 * constant_55555 * 5 * 5),+      0x20000000000000 / (constant_55555 * constant_55555 * 5 * 5 * 5),+      0x20000000000000 / (constant_55555 * constant_55555 * constant_55555),+      0x20000000000000 / (constant_55555 * constant_55555 * constant_55555 * 5),+      0x20000000000000 /+          (constant_55555 * constant_55555 * constant_55555 * 5 * 5),+      0x20000000000000 /+          (constant_55555 * constant_55555 * constant_55555 * 5 * 5 * 5),+      0x20000000000000 /+          (constant_55555 * constant_55555 * constant_55555 * 5 * 5 * 5 * 5),+      0x20000000000000 /+          (constant_55555 * constant_55555 * constant_55555 * constant_55555),+      0x20000000000000 / (constant_55555 * constant_55555 * constant_55555 *+                          constant_55555 * 5),+      0x20000000000000 / (constant_55555 * constant_55555 * constant_55555 *+                          constant_55555 * 5 * 5),+      0x20000000000000 / (constant_55555 * constant_55555 * constant_55555 *+                          constant_55555 * 5 * 5 * 5),+      0x20000000000000 / (constant_55555 * constant_55555 * constant_55555 *+                          constant_55555 * 5 * 5 * 5 * 5)};+};++#if FASTFLOAT_DETAIL_MUST_DEFINE_CONSTEXPR_VARIABLE++template <typename U>+constexpr double binary_format_lookup_tables<double, U>::powers_of_ten[];++template <typename U>+constexpr uint64_t binary_format_lookup_tables<double, U>::max_mantissa[];++#endif++template <typename U> struct binary_format_lookup_tables<float, U> {+  static constexpr float powers_of_ten[] = {1e0f, 1e1f, 1e2f, 1e3f, 1e4f, 1e5f,+                                            1e6f, 1e7f, 1e8f, 1e9f, 1e10f};++  // Largest integer value v so that (5**index * v) <= 1<<24.+  // 0x1000000 == 1<<24+  static constexpr uint64_t max_mantissa[] = {+      0x1000000,+      0x1000000 / 5,+      0x1000000 / (5 * 5),+      0x1000000 / (5 * 5 * 5),+      0x1000000 / (5 * 5 * 5 * 5),+      0x1000000 / (constant_55555),+      0x1000000 / (constant_55555 * 5),+      0x1000000 / (constant_55555 * 5 * 5),+      0x1000000 / (constant_55555 * 5 * 5 * 5),+      0x1000000 / (constant_55555 * 5 * 5 * 5 * 5),+      0x1000000 / (constant_55555 * constant_55555),+      0x1000000 / (constant_55555 * constant_55555 * 5)};+};++#if FASTFLOAT_DETAIL_MUST_DEFINE_CONSTEXPR_VARIABLE++template <typename U>+constexpr float binary_format_lookup_tables<float, U>::powers_of_ten[];++template <typename U>+constexpr uint64_t binary_format_lookup_tables<float, U>::max_mantissa[];++#endif++template <>+inline constexpr int binary_format<double>::min_exponent_fast_path() {+#if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0)+  return 0;+#else+  return -22;+#endif+}++template <>+inline constexpr int binary_format<float>::min_exponent_fast_path() {+#if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0)+  return 0;+#else+  return -10;+#endif+}++template <>+inline constexpr int binary_format<double>::mantissa_explicit_bits() {+  return 52;+}++template <>+inline constexpr int binary_format<float>::mantissa_explicit_bits() {+  return 23;+}++template <>+inline constexpr int binary_format<double>::max_exponent_round_to_even() {+  return 23;+}++template <>+inline constexpr int binary_format<float>::max_exponent_round_to_even() {+  return 10;+}++template <>+inline constexpr int binary_format<double>::min_exponent_round_to_even() {+  return -4;+}++template <>+inline constexpr int binary_format<float>::min_exponent_round_to_even() {+  return -17;+}++template <> inline constexpr int binary_format<double>::minimum_exponent() {+  return -1023;+}++template <> inline constexpr int binary_format<float>::minimum_exponent() {+  return -127;+}++template <> inline constexpr int binary_format<double>::infinite_power() {+  return 0x7FF;+}++template <> inline constexpr int binary_format<float>::infinite_power() {+  return 0xFF;+}++template <> inline constexpr int binary_format<double>::sign_index() {+  return 63;+}++template <> inline constexpr int binary_format<float>::sign_index() {+  return 31;+}++template <>+inline constexpr int binary_format<double>::max_exponent_fast_path() {+  return 22;+}++template <>+inline constexpr int binary_format<float>::max_exponent_fast_path() {+  return 10;+}++template <>+inline constexpr uint64_t binary_format<double>::max_mantissa_fast_path() {+  return uint64_t(2) << mantissa_explicit_bits();+}++template <>+inline constexpr uint64_t binary_format<float>::max_mantissa_fast_path() {+  return uint64_t(2) << mantissa_explicit_bits();+}++// credit: Jakub Jelínek+#ifdef __STDCPP_FLOAT16_T__+template <typename U> struct binary_format_lookup_tables<std::float16_t, U> {+  static constexpr std::float16_t powers_of_ten[] = {1e0f16, 1e1f16, 1e2f16,+                                                     1e3f16, 1e4f16};++  // Largest integer value v so that (5**index * v) <= 1<<11.+  // 0x800 == 1<<11+  static constexpr uint64_t max_mantissa[] = {0x800,+                                              0x800 / 5,+                                              0x800 / (5 * 5),+                                              0x800 / (5 * 5 * 5),+                                              0x800 / (5 * 5 * 5 * 5),+                                              0x800 / (constant_55555)};+};++#if FASTFLOAT_DETAIL_MUST_DEFINE_CONSTEXPR_VARIABLE++template <typename U>+constexpr std::float16_t+    binary_format_lookup_tables<std::float16_t, U>::powers_of_ten[];++template <typename U>+constexpr uint64_t+    binary_format_lookup_tables<std::float16_t, U>::max_mantissa[];++#endif++template <>+inline constexpr std::float16_t+binary_format<std::float16_t>::exact_power_of_ten(int64_t power) {+  // Work around clang bug https://godbolt.org/z/zedh7rrhc+  return (void)powers_of_ten[0], powers_of_ten[power];+}++template <>+inline constexpr binary_format<std::float16_t>::equiv_uint+binary_format<std::float16_t>::exponent_mask() {+  return 0x7C00;+}++template <>+inline constexpr binary_format<std::float16_t>::equiv_uint+binary_format<std::float16_t>::mantissa_mask() {+  return 0x03FF;+}++template <>+inline constexpr binary_format<std::float16_t>::equiv_uint+binary_format<std::float16_t>::hidden_bit_mask() {+  return 0x0400;+}++template <>+inline constexpr int binary_format<std::float16_t>::max_exponent_fast_path() {+  return 4;+}++template <>+inline constexpr int binary_format<std::float16_t>::mantissa_explicit_bits() {+  return 10;+}++template <>+inline constexpr uint64_t+binary_format<std::float16_t>::max_mantissa_fast_path() {+  return uint64_t(2) << mantissa_explicit_bits();+}++template <>+inline constexpr uint64_t+binary_format<std::float16_t>::max_mantissa_fast_path(int64_t power) {+  // caller is responsible to ensure that+  // power >= 0 && power <= 4+  //+  // Work around clang bug https://godbolt.org/z/zedh7rrhc+  return (void)max_mantissa[0], max_mantissa[power];+}++template <>+inline constexpr int binary_format<std::float16_t>::min_exponent_fast_path() {+  return 0;+}++template <>+inline constexpr int+binary_format<std::float16_t>::max_exponent_round_to_even() {+  return 5;+}++template <>+inline constexpr int+binary_format<std::float16_t>::min_exponent_round_to_even() {+  return -22;+}++template <>+inline constexpr int binary_format<std::float16_t>::minimum_exponent() {+  return -15;+}++template <>+inline constexpr int binary_format<std::float16_t>::infinite_power() {+  return 0x1F;+}++template <> inline constexpr int binary_format<std::float16_t>::sign_index() {+  return 15;+}++template <>+inline constexpr int binary_format<std::float16_t>::largest_power_of_ten() {+  return 4;+}++template <>+inline constexpr int binary_format<std::float16_t>::smallest_power_of_ten() {+  return -27;+}++template <>+inline constexpr size_t binary_format<std::float16_t>::max_digits() {+  return 22;+}+#endif // __STDCPP_FLOAT16_T__++// credit: Jakub Jelínek+#ifdef __STDCPP_BFLOAT16_T__+template <typename U> struct binary_format_lookup_tables<std::bfloat16_t, U> {+  static constexpr std::bfloat16_t powers_of_ten[] = {1e0bf16, 1e1bf16, 1e2bf16,+                                                      1e3bf16};++  // Largest integer value v so that (5**index * v) <= 1<<8.+  // 0x100 == 1<<8+  static constexpr uint64_t max_mantissa[] = {0x100, 0x100 / 5, 0x100 / (5 * 5),+                                              0x100 / (5 * 5 * 5),+                                              0x100 / (5 * 5 * 5 * 5)};+};++#if FASTFLOAT_DETAIL_MUST_DEFINE_CONSTEXPR_VARIABLE++template <typename U>+constexpr std::bfloat16_t+    binary_format_lookup_tables<std::bfloat16_t, U>::powers_of_ten[];++template <typename U>+constexpr uint64_t+    binary_format_lookup_tables<std::bfloat16_t, U>::max_mantissa[];++#endif++template <>+inline constexpr std::bfloat16_t+binary_format<std::bfloat16_t>::exact_power_of_ten(int64_t power) {+  // Work around clang bug https://godbolt.org/z/zedh7rrhc+  return (void)powers_of_ten[0], powers_of_ten[power];+}++template <>+inline constexpr int binary_format<std::bfloat16_t>::max_exponent_fast_path() {+  return 3;+}++template <>+inline constexpr binary_format<std::bfloat16_t>::equiv_uint+binary_format<std::bfloat16_t>::exponent_mask() {+  return 0x7F80;+}++template <>+inline constexpr binary_format<std::bfloat16_t>::equiv_uint+binary_format<std::bfloat16_t>::mantissa_mask() {+  return 0x007F;+}++template <>+inline constexpr binary_format<std::bfloat16_t>::equiv_uint+binary_format<std::bfloat16_t>::hidden_bit_mask() {+  return 0x0080;+}++template <>+inline constexpr int binary_format<std::bfloat16_t>::mantissa_explicit_bits() {+  return 7;+}++template <>+inline constexpr uint64_t+binary_format<std::bfloat16_t>::max_mantissa_fast_path() {+  return uint64_t(2) << mantissa_explicit_bits();+}++template <>+inline constexpr uint64_t+binary_format<std::bfloat16_t>::max_mantissa_fast_path(int64_t power) {+  // caller is responsible to ensure that+  // power >= 0 && power <= 3+  //+  // Work around clang bug https://godbolt.org/z/zedh7rrhc+  return (void)max_mantissa[0], max_mantissa[power];+}++template <>+inline constexpr int binary_format<std::bfloat16_t>::min_exponent_fast_path() {+  return 0;+}++template <>+inline constexpr int+binary_format<std::bfloat16_t>::max_exponent_round_to_even() {+  return 3;+}++template <>+inline constexpr int+binary_format<std::bfloat16_t>::min_exponent_round_to_even() {+  return -24;+}++template <>+inline constexpr int binary_format<std::bfloat16_t>::minimum_exponent() {+  return -127;+}++template <>+inline constexpr int binary_format<std::bfloat16_t>::infinite_power() {+  return 0xFF;+}++template <> inline constexpr int binary_format<std::bfloat16_t>::sign_index() {+  return 15;+}++template <>+inline constexpr int binary_format<std::bfloat16_t>::largest_power_of_ten() {+  return 38;+}++template <>+inline constexpr int binary_format<std::bfloat16_t>::smallest_power_of_ten() {+  return -60;+}++template <>+inline constexpr size_t binary_format<std::bfloat16_t>::max_digits() {+  return 98;+}+#endif // __STDCPP_BFLOAT16_T__++template <>+inline constexpr uint64_t+binary_format<double>::max_mantissa_fast_path(int64_t power) {+  // caller is responsible to ensure that+  // power >= 0 && power <= 22+  //+  // Work around clang bug https://godbolt.org/z/zedh7rrhc+  return (void)max_mantissa[0], max_mantissa[power];+}++template <>+inline constexpr uint64_t+binary_format<float>::max_mantissa_fast_path(int64_t power) {+  // caller is responsible to ensure that+  // power >= 0 && power <= 10+  //+  // Work around clang bug https://godbolt.org/z/zedh7rrhc+  return (void)max_mantissa[0], max_mantissa[power];+}++template <>+inline constexpr double+binary_format<double>::exact_power_of_ten(int64_t power) {+  // Work around clang bug https://godbolt.org/z/zedh7rrhc+  return (void)powers_of_ten[0], powers_of_ten[power];+}++template <>+inline constexpr float binary_format<float>::exact_power_of_ten(int64_t power) {+  // Work around clang bug https://godbolt.org/z/zedh7rrhc+  return (void)powers_of_ten[0], powers_of_ten[power];+}++template <> inline constexpr int binary_format<double>::largest_power_of_ten() {+  return 308;+}++template <> inline constexpr int binary_format<float>::largest_power_of_ten() {+  return 38;+}++template <>+inline constexpr int binary_format<double>::smallest_power_of_ten() {+  return -342;+}++template <> inline constexpr int binary_format<float>::smallest_power_of_ten() {+  return -64;+}++template <> inline constexpr size_t binary_format<double>::max_digits() {+  return 769;+}++template <> inline constexpr size_t binary_format<float>::max_digits() {+  return 114;+}++template <>+inline constexpr binary_format<float>::equiv_uint+binary_format<float>::exponent_mask() {+  return 0x7F800000;+}++template <>+inline constexpr binary_format<double>::equiv_uint+binary_format<double>::exponent_mask() {+  return 0x7FF0000000000000;+}++template <>+inline constexpr binary_format<float>::equiv_uint+binary_format<float>::mantissa_mask() {+  return 0x007FFFFF;+}++template <>+inline constexpr binary_format<double>::equiv_uint+binary_format<double>::mantissa_mask() {+  return 0x000FFFFFFFFFFFFF;+}++template <>+inline constexpr binary_format<float>::equiv_uint+binary_format<float>::hidden_bit_mask() {+  return 0x00800000;+}++template <>+inline constexpr binary_format<double>::equiv_uint+binary_format<double>::hidden_bit_mask() {+  return 0x0010000000000000;+}++template <typename T>+fastfloat_really_inline FASTFLOAT_CONSTEXPR20 void+to_float(bool negative, adjusted_mantissa am, T &value) {+  using equiv_uint = equiv_uint_t<T>;+  equiv_uint word = equiv_uint(am.mantissa);+  word = equiv_uint(word | equiv_uint(am.power2)+                               << binary_format<T>::mantissa_explicit_bits());+  word =+      equiv_uint(word | equiv_uint(negative) << binary_format<T>::sign_index());+#if FASTFLOAT_HAS_BIT_CAST+  value = std::bit_cast<T>(word);+#else+  ::memcpy(&value, &word, sizeof(T));+#endif+}++template <typename = void> struct space_lut {+  static constexpr bool value[] = {+      0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,+      0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,+      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,+      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,+      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,+      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,+      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,+      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,+      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,+      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,+      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};+};++#if FASTFLOAT_DETAIL_MUST_DEFINE_CONSTEXPR_VARIABLE++template <typename T> constexpr bool space_lut<T>::value[];++#endif++template <typename UC> constexpr bool is_space(UC c) {+  return c < 256 && space_lut<>::value[uint8_t(c)];+}++template <typename UC> static constexpr uint64_t int_cmp_zeros() {+  static_assert((sizeof(UC) == 1) || (sizeof(UC) == 2) || (sizeof(UC) == 4),+                "Unsupported character size");+  return (sizeof(UC) == 1) ? 0x3030303030303030+         : (sizeof(UC) == 2)+             ? (uint64_t(UC('0')) << 48 | uint64_t(UC('0')) << 32 |+                uint64_t(UC('0')) << 16 | UC('0'))+             : (uint64_t(UC('0')) << 32 | UC('0'));+}++template <typename UC> static constexpr int int_cmp_len() {+  return sizeof(uint64_t) / sizeof(UC);+}++template <typename UC> constexpr UC const *str_const_nan();++template <> constexpr char const *str_const_nan<char>() { return "nan"; }++template <> constexpr wchar_t const *str_const_nan<wchar_t>() { return L"nan"; }++template <> constexpr char16_t const *str_const_nan<char16_t>() {+  return u"nan";+}++template <> constexpr char32_t const *str_const_nan<char32_t>() {+  return U"nan";+}++#ifdef __cpp_char8_t+template <> constexpr char8_t const *str_const_nan<char8_t>() {+  return u8"nan";+}+#endif++template <typename UC> constexpr UC const *str_const_inf();++template <> constexpr char const *str_const_inf<char>() { return "infinity"; }++template <> constexpr wchar_t const *str_const_inf<wchar_t>() {+  return L"infinity";+}++template <> constexpr char16_t const *str_const_inf<char16_t>() {+  return u"infinity";+}++template <> constexpr char32_t const *str_const_inf<char32_t>() {+  return U"infinity";+}++#ifdef __cpp_char8_t+template <> constexpr char8_t const *str_const_inf<char8_t>() {+  return u8"infinity";+}+#endif++template <typename = void> struct int_luts {+  static constexpr uint8_t chdigit[] = {+      255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,+      255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,+      255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,+      255, 255, 255, 0,   1,   2,   3,   4,   5,   6,   7,   8,   9,   255, 255,+      255, 255, 255, 255, 255, 10,  11,  12,  13,  14,  15,  16,  17,  18,  19,+      20,  21,  22,  23,  24,  25,  26,  27,  28,  29,  30,  31,  32,  33,  34,+      35,  255, 255, 255, 255, 255, 255, 10,  11,  12,  13,  14,  15,  16,  17,+      18,  19,  20,  21,  22,  23,  24,  25,  26,  27,  28,  29,  30,  31,  32,+      33,  34,  35,  255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,+      255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,+      255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,+      255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,+      255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,+      255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,+      255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,+      255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,+      255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,+      255};++  static constexpr size_t maxdigits_u64[] = {+      64, 41, 32, 28, 25, 23, 22, 21, 20, 19, 18, 18, 17, 17, 16, 16, 16, 16,+      15, 15, 15, 15, 14, 14, 14, 14, 14, 14, 14, 13, 13, 13, 13, 13, 13};++  static constexpr uint64_t min_safe_u64[] = {+      9223372036854775808ull,  12157665459056928801ull, 4611686018427387904,+      7450580596923828125,     4738381338321616896,     3909821048582988049,+      9223372036854775808ull,  12157665459056928801ull, 10000000000000000000ull,+      5559917313492231481,     2218611106740436992,     8650415919381337933,+      2177953337809371136,     6568408355712890625,     1152921504606846976,+      2862423051509815793,     6746640616477458432,     15181127029874798299ull,+      1638400000000000000,     3243919932521508681,     6221821273427820544,+      11592836324538749809ull, 876488338465357824,      1490116119384765625,+      2481152873203736576,     4052555153018976267,     6502111422497947648,+      10260628712958602189ull, 15943230000000000000ull, 787662783788549761,+      1152921504606846976,     1667889514952984961,     2386420683693101056,+      3379220508056640625,     4738381338321616896};+};++#if FASTFLOAT_DETAIL_MUST_DEFINE_CONSTEXPR_VARIABLE++template <typename T> constexpr uint8_t int_luts<T>::chdigit[];++template <typename T> constexpr size_t int_luts<T>::maxdigits_u64[];++template <typename T> constexpr uint64_t int_luts<T>::min_safe_u64[];++#endif++template <typename UC>+fastfloat_really_inline constexpr uint8_t ch_to_digit(UC c) {+  return int_luts<>::chdigit[static_cast<unsigned char>(c)];+}++fastfloat_really_inline constexpr size_t max_digits_u64(int base) {+  return int_luts<>::maxdigits_u64[base - 2];+}++// If a u64 is exactly max_digits_u64() in length, this is+// the value below which it has definitely overflowed.+fastfloat_really_inline constexpr uint64_t min_safe_u64(int base) {+  return int_luts<>::min_safe_u64[base - 2];+}++static_assert(std::is_same<equiv_uint_t<double>, uint64_t>::value,+              "equiv_uint should be uint64_t for double");+static_assert(std::numeric_limits<double>::is_iec559,+              "double must fulfill the requirements of IEC 559 (IEEE 754)");++static_assert(std::is_same<equiv_uint_t<float>, uint32_t>::value,+              "equiv_uint should be uint32_t for float");+static_assert(std::numeric_limits<float>::is_iec559,+              "float must fulfill the requirements of IEC 559 (IEEE 754)");++#ifdef __STDCPP_FLOAT64_T__+static_assert(std::is_same<equiv_uint_t<std::float64_t>, uint64_t>::value,+              "equiv_uint should be uint64_t for std::float64_t");+static_assert(+    std::numeric_limits<std::float64_t>::is_iec559,+    "std::float64_t must fulfill the requirements of IEC 559 (IEEE 754)");+#endif // __STDCPP_FLOAT64_T__++#ifdef __STDCPP_FLOAT32_T__+static_assert(std::is_same<equiv_uint_t<std::float32_t>, uint32_t>::value,+              "equiv_uint should be uint32_t for std::float32_t");+static_assert(+    std::numeric_limits<std::float32_t>::is_iec559,+    "std::float32_t must fulfill the requirements of IEC 559 (IEEE 754)");+#endif // __STDCPP_FLOAT32_T__++#ifdef __STDCPP_FLOAT16_T__+static_assert(+    std::is_same<binary_format<std::float16_t>::equiv_uint, uint16_t>::value,+    "equiv_uint should be uint16_t for std::float16_t");+static_assert(+    std::numeric_limits<std::float16_t>::is_iec559,+    "std::float16_t must fulfill the requirements of IEC 559 (IEEE 754)");+#endif // __STDCPP_FLOAT16_T__++#ifdef __STDCPP_BFLOAT16_T__+static_assert(+    std::is_same<binary_format<std::bfloat16_t>::equiv_uint, uint16_t>::value,+    "equiv_uint should be uint16_t for std::bfloat16_t");+static_assert(+    std::numeric_limits<std::bfloat16_t>::is_iec559,+    "std::bfloat16_t must fulfill the requirements of IEC 559 (IEEE 754)");+#endif // __STDCPP_BFLOAT16_T__++constexpr chars_format operator~(chars_format rhs) noexcept {+  using int_type = std::underlying_type<chars_format>::type;+  return static_cast<chars_format>(~static_cast<int_type>(rhs));+}++constexpr chars_format operator&(chars_format lhs, chars_format rhs) noexcept {+  using int_type = std::underlying_type<chars_format>::type;+  return static_cast<chars_format>(static_cast<int_type>(lhs) &+                                   static_cast<int_type>(rhs));+}++constexpr chars_format operator|(chars_format lhs, chars_format rhs) noexcept {+  using int_type = std::underlying_type<chars_format>::type;+  return static_cast<chars_format>(static_cast<int_type>(lhs) |+                                   static_cast<int_type>(rhs));+}++constexpr chars_format operator^(chars_format lhs, chars_format rhs) noexcept {+  using int_type = std::underlying_type<chars_format>::type;+  return static_cast<chars_format>(static_cast<int_type>(lhs) ^+                                   static_cast<int_type>(rhs));+}++fastfloat_really_inline FASTFLOAT_CONSTEXPR14 chars_format &+operator&=(chars_format &lhs, chars_format rhs) noexcept {+  return lhs = (lhs & rhs);+}++fastfloat_really_inline FASTFLOAT_CONSTEXPR14 chars_format &+operator|=(chars_format &lhs, chars_format rhs) noexcept {+  return lhs = (lhs | rhs);+}++fastfloat_really_inline FASTFLOAT_CONSTEXPR14 chars_format &+operator^=(chars_format &lhs, chars_format rhs) noexcept {+  return lhs = (lhs ^ rhs);+}++namespace detail {+// adjust for deprecated feature macros+constexpr chars_format adjust_for_feature_macros(chars_format fmt) {+  return fmt+#ifdef FASTFLOAT_ALLOWS_LEADING_PLUS+         | chars_format::allow_leading_plus+#endif+#ifdef FASTFLOAT_SKIP_WHITE_SPACE+         | chars_format::skip_white_space+#endif+      ;+}+} // namespace detail++} // namespace fast_float++#endif
+ fast_float-8.0.0/include/fast_float/parse_number.h view
@@ -0,0 +1,399 @@+#ifndef FASTFLOAT_PARSE_NUMBER_H+#define FASTFLOAT_PARSE_NUMBER_H++#include "ascii_number.h"+#include "decimal_to_binary.h"+#include "digit_comparison.h"+#include "float_common.h"++#include <cmath>+#include <cstring>+#include <limits>+#include <system_error>++namespace fast_float {++namespace detail {+/**+ * Special case +inf, -inf, nan, infinity, -infinity.+ * The case comparisons could be made much faster given that we know that the+ * strings a null-free and fixed.+ **/+template <typename T, typename UC>+from_chars_result_t<UC>+    FASTFLOAT_CONSTEXPR14 parse_infnan(UC const *first, UC const *last,+                                       T &value, chars_format fmt) noexcept {+  from_chars_result_t<UC> answer{};+  answer.ptr = first;+  answer.ec = std::errc(); // be optimistic+  // assume first < last, so dereference without checks;+  bool const minusSign = (*first == UC('-'));+  // C++17 20.19.3.(7.1) explicitly forbids '+' sign here+  if ((*first == UC('-')) ||+      (uint64_t(fmt & chars_format::allow_leading_plus) &&+       (*first == UC('+')))) {+    ++first;+  }+  if (last - first >= 3) {+    if (fastfloat_strncasecmp(first, str_const_nan<UC>(), 3)) {+      answer.ptr = (first += 3);+      value = minusSign ? -std::numeric_limits<T>::quiet_NaN()+                        : std::numeric_limits<T>::quiet_NaN();+      // Check for possible nan(n-char-seq-opt), C++17 20.19.3.7,+      // C11 7.20.1.3.3. At least MSVC produces nan(ind) and nan(snan).+      if (first != last && *first == UC('(')) {+        for (UC const *ptr = first + 1; ptr != last; ++ptr) {+          if (*ptr == UC(')')) {+            answer.ptr = ptr + 1; // valid nan(n-char-seq-opt)+            break;+          } else if (!((UC('a') <= *ptr && *ptr <= UC('z')) ||+                       (UC('A') <= *ptr && *ptr <= UC('Z')) ||+                       (UC('0') <= *ptr && *ptr <= UC('9')) || *ptr == UC('_')))+            break; // forbidden char, not nan(n-char-seq-opt)+        }+      }+      return answer;+    }+    if (fastfloat_strncasecmp(first, str_const_inf<UC>(), 3)) {+      if ((last - first >= 8) &&+          fastfloat_strncasecmp(first + 3, str_const_inf<UC>() + 3, 5)) {+        answer.ptr = first + 8;+      } else {+        answer.ptr = first + 3;+      }+      value = minusSign ? -std::numeric_limits<T>::infinity()+                        : std::numeric_limits<T>::infinity();+      return answer;+    }+  }+  answer.ec = std::errc::invalid_argument;+  return answer;+}++/**+ * Returns true if the floating-pointing rounding mode is to 'nearest'.+ * It is the default on most system. This function is meant to be inexpensive.+ * Credit : @mwalcott3+ */+fastfloat_really_inline bool rounds_to_nearest() noexcept {+  // https://lemire.me/blog/2020/06/26/gcc-not-nearest/+#if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0)+  return false;+#endif+  // See+  // A fast function to check your floating-point rounding mode+  // https://lemire.me/blog/2022/11/16/a-fast-function-to-check-your-floating-point-rounding-mode/+  //+  // This function is meant to be equivalent to :+  // prior: #include <cfenv>+  //  return fegetround() == FE_TONEAREST;+  // However, it is expected to be much faster than the fegetround()+  // function call.+  //+  // The volatile keyword prevents the compiler from computing the function+  // at compile-time.+  // There might be other ways to prevent compile-time optimizations (e.g.,+  // asm). The value does not need to be std::numeric_limits<float>::min(), any+  // small value so that 1 + x should round to 1 would do (after accounting for+  // excess precision, as in 387 instructions).+  static float volatile fmin = std::numeric_limits<float>::min();+  float fmini = fmin; // we copy it so that it gets loaded at most once.+//+// Explanation:+// Only when fegetround() == FE_TONEAREST do we have that+// fmin + 1.0f == 1.0f - fmin.+//+// FE_UPWARD:+//  fmin + 1.0f > 1+//  1.0f - fmin == 1+//+// FE_DOWNWARD or  FE_TOWARDZERO:+//  fmin + 1.0f == 1+//  1.0f - fmin < 1+//+// Note: This may fail to be accurate if fast-math has been+// enabled, as rounding conventions may not apply.+#ifdef FASTFLOAT_VISUAL_STUDIO+#pragma warning(push)+//  todo: is there a VS warning?+//  see+//  https://stackoverflow.com/questions/46079446/is-there-a-warning-for-floating-point-equality-checking-in-visual-studio-2013+#elif defined(__clang__)+#pragma clang diagnostic push+#pragma clang diagnostic ignored "-Wfloat-equal"+#elif defined(__GNUC__)+#pragma GCC diagnostic push+#pragma GCC diagnostic ignored "-Wfloat-equal"+#endif+  return (fmini + 1.0f == 1.0f - fmini);+#ifdef FASTFLOAT_VISUAL_STUDIO+#pragma warning(pop)+#elif defined(__clang__)+#pragma clang diagnostic pop+#elif defined(__GNUC__)+#pragma GCC diagnostic pop+#endif+}++} // namespace detail++template <typename T> struct from_chars_caller {+  template <typename UC>+  FASTFLOAT_CONSTEXPR20 static from_chars_result_t<UC>+  call(UC const *first, UC const *last, T &value,+       parse_options_t<UC> options) noexcept {+    return from_chars_advanced(first, last, value, options);+  }+};++#ifdef __STDCPP_FLOAT32_T__+template <> struct from_chars_caller<std::float32_t> {+  template <typename UC>+  FASTFLOAT_CONSTEXPR20 static from_chars_result_t<UC>+  call(UC const *first, UC const *last, std::float32_t &value,+       parse_options_t<UC> options) noexcept {+    // if std::float32_t is defined, and we are in C++23 mode; macro set for+    // float32; set value to float due to equivalence between float and+    // float32_t+    float val;+    auto ret = from_chars_advanced(first, last, val, options);+    value = val;+    return ret;+  }+};+#endif++#ifdef __STDCPP_FLOAT64_T__+template <> struct from_chars_caller<std::float64_t> {+  template <typename UC>+  FASTFLOAT_CONSTEXPR20 static from_chars_result_t<UC>+  call(UC const *first, UC const *last, std::float64_t &value,+       parse_options_t<UC> options) noexcept {+    // if std::float64_t is defined, and we are in C++23 mode; macro set for+    // float64; set value as double due to equivalence between double and+    // float64_t+    double val;+    auto ret = from_chars_advanced(first, last, val, options);+    value = val;+    return ret;+  }+};+#endif++template <typename T, typename UC, typename>+FASTFLOAT_CONSTEXPR20 from_chars_result_t<UC>+from_chars(UC const *first, UC const *last, T &value,+           chars_format fmt /*= chars_format::general*/) noexcept {+  return from_chars_caller<T>::call(first, last, value,+                                    parse_options_t<UC>(fmt));+}++/**+ * This function overload takes parsed_number_string_t structure that is created+ * and populated either by from_chars_advanced function taking chars range and+ * parsing options or other parsing custom function implemented by user.+ */+template <typename T, typename UC>+FASTFLOAT_CONSTEXPR20 from_chars_result_t<UC>+from_chars_advanced(parsed_number_string_t<UC> &pns, T &value) noexcept {++  static_assert(is_supported_float_type<T>::value,+                "only some floating-point types are supported");+  static_assert(is_supported_char_type<UC>::value,+                "only char, wchar_t, char16_t and char32_t are supported");++  from_chars_result_t<UC> answer;++  answer.ec = std::errc(); // be optimistic+  answer.ptr = pns.lastmatch;+  // The implementation of the Clinger's fast path is convoluted because+  // we want round-to-nearest in all cases, irrespective of the rounding mode+  // selected on the thread.+  // We proceed optimistically, assuming that detail::rounds_to_nearest()+  // returns true.+  if (binary_format<T>::min_exponent_fast_path() <= pns.exponent &&+      pns.exponent <= binary_format<T>::max_exponent_fast_path() &&+      !pns.too_many_digits) {+    // Unfortunately, the conventional Clinger's fast path is only possible+    // when the system rounds to the nearest float.+    //+    // We expect the next branch to almost always be selected.+    // We could check it first (before the previous branch), but+    // there might be performance advantages at having the check+    // be last.+    if (!cpp20_and_in_constexpr() && detail::rounds_to_nearest()) {+      // We have that fegetround() == FE_TONEAREST.+      // Next is Clinger's fast path.+      if (pns.mantissa <= binary_format<T>::max_mantissa_fast_path()) {+        value = T(pns.mantissa);+        if (pns.exponent < 0) {+          value = value / binary_format<T>::exact_power_of_ten(-pns.exponent);+        } else {+          value = value * binary_format<T>::exact_power_of_ten(pns.exponent);+        }+        if (pns.negative) {+          value = -value;+        }+        return answer;+      }+    } else {+      // We do not have that fegetround() == FE_TONEAREST.+      // Next is a modified Clinger's fast path, inspired by Jakub Jelínek's+      // proposal+      if (pns.exponent >= 0 &&+          pns.mantissa <=+              binary_format<T>::max_mantissa_fast_path(pns.exponent)) {+#if defined(__clang__) || defined(FASTFLOAT_32BIT)+        // Clang may map 0 to -0.0 when fegetround() == FE_DOWNWARD+        if (pns.mantissa == 0) {+          value = pns.negative ? T(-0.) : T(0.);+          return answer;+        }+#endif+        value = T(pns.mantissa) *+                binary_format<T>::exact_power_of_ten(pns.exponent);+        if (pns.negative) {+          value = -value;+        }+        return answer;+      }+    }+  }+  adjusted_mantissa am =+      compute_float<binary_format<T>>(pns.exponent, pns.mantissa);+  if (pns.too_many_digits && am.power2 >= 0) {+    if (am != compute_float<binary_format<T>>(pns.exponent, pns.mantissa + 1)) {+      am = compute_error<binary_format<T>>(pns.exponent, pns.mantissa);+    }+  }+  // If we called compute_float<binary_format<T>>(pns.exponent, pns.mantissa)+  // and we have an invalid power (am.power2 < 0), then we need to go the long+  // way around again. This is very uncommon.+  if (am.power2 < 0) {+    am = digit_comp<T>(pns, am);+  }+  to_float(pns.negative, am, value);+  // Test for over/underflow.+  if ((pns.mantissa != 0 && am.mantissa == 0 && am.power2 == 0) ||+      am.power2 == binary_format<T>::infinite_power()) {+    answer.ec = std::errc::result_out_of_range;+  }+  return answer;+}++template <typename T, typename UC>+FASTFLOAT_CONSTEXPR20 from_chars_result_t<UC>+from_chars_float_advanced(UC const *first, UC const *last, T &value,+                          parse_options_t<UC> options) noexcept {++  static_assert(is_supported_float_type<T>::value,+                "only some floating-point types are supported");+  static_assert(is_supported_char_type<UC>::value,+                "only char, wchar_t, char16_t and char32_t are supported");++  chars_format const fmt = detail::adjust_for_feature_macros(options.format);++  from_chars_result_t<UC> answer;+  if (uint64_t(fmt & chars_format::skip_white_space)) {+    while ((first != last) && fast_float::is_space(*first)) {+      first++;+    }+  }+  if (first == last) {+    answer.ec = std::errc::invalid_argument;+    answer.ptr = first;+    return answer;+  }+  parsed_number_string_t<UC> pns =+      parse_number_string<UC>(first, last, options);+  if (!pns.valid) {+    if (uint64_t(fmt & chars_format::no_infnan)) {+      answer.ec = std::errc::invalid_argument;+      answer.ptr = first;+      return answer;+    } else {+      return detail::parse_infnan(first, last, value, fmt);+    }+  }++  // call overload that takes parsed_number_string_t directly.+  return from_chars_advanced(pns, value);+}++template <typename T, typename UC, typename>+FASTFLOAT_CONSTEXPR20 from_chars_result_t<UC>+from_chars(UC const *first, UC const *last, T &value, int base) noexcept {++  static_assert(is_supported_integer_type<T>::value,+                "only integer types are supported");+  static_assert(is_supported_char_type<UC>::value,+                "only char, wchar_t, char16_t and char32_t are supported");++  parse_options_t<UC> options;+  options.base = base;+  return from_chars_advanced(first, last, value, options);+}++template <typename T, typename UC>+FASTFLOAT_CONSTEXPR20 from_chars_result_t<UC>+from_chars_int_advanced(UC const *first, UC const *last, T &value,+                        parse_options_t<UC> options) noexcept {++  static_assert(is_supported_integer_type<T>::value,+                "only integer types are supported");+  static_assert(is_supported_char_type<UC>::value,+                "only char, wchar_t, char16_t and char32_t are supported");++  chars_format const fmt = detail::adjust_for_feature_macros(options.format);+  int const base = options.base;++  from_chars_result_t<UC> answer;+  if (uint64_t(fmt & chars_format::skip_white_space)) {+    while ((first != last) && fast_float::is_space(*first)) {+      first++;+    }+  }+  if (first == last || base < 2 || base > 36) {+    answer.ec = std::errc::invalid_argument;+    answer.ptr = first;+    return answer;+  }++  return parse_int_string(first, last, value, options);+}++template <size_t TypeIx> struct from_chars_advanced_caller {+  static_assert(TypeIx > 0, "unsupported type");+};++template <> struct from_chars_advanced_caller<1> {+  template <typename T, typename UC>+  FASTFLOAT_CONSTEXPR20 static from_chars_result_t<UC>+  call(UC const *first, UC const *last, T &value,+       parse_options_t<UC> options) noexcept {+    return from_chars_float_advanced(first, last, value, options);+  }+};++template <> struct from_chars_advanced_caller<2> {+  template <typename T, typename UC>+  FASTFLOAT_CONSTEXPR20 static from_chars_result_t<UC>+  call(UC const *first, UC const *last, T &value,+       parse_options_t<UC> options) noexcept {+    return from_chars_int_advanced(first, last, value, options);+  }+};++template <typename T, typename UC>+FASTFLOAT_CONSTEXPR20 from_chars_result_t<UC>+from_chars_advanced(UC const *first, UC const *last, T &value,+                    parse_options_t<UC> options) noexcept {+  return from_chars_advanced_caller<+      size_t(is_supported_float_type<T>::value) ++      2 * size_t(is_supported_integer_type<T>::value)>::call(first, last, value,+                                                             options);+}++} // namespace fast_float++#endif
folly-clib.cabal view
@@ -1,69 +1,1525 @@ cabal-version:      3.4 name:               folly-clib-version:            0.0-synopsis:           The folly C++ library from Meta-author:             Simon Marlow-maintainer:         marlowsd@gmail.com-copyright:          Copyright (c) Meta Platforms, Inc. and affiliates.-homepage:           https://github.com/facebook/folly-bug-reports:        https://github.com/facebook/folly/issues-license:            Apache-2.0-license-files:-  folly/LICENSE-  fast_float-8.0.0/LICENSE-MIT-  fast_float-8.0.0/LICENSE-BOOST-  fast_float-8.0.0/LICENSE-APACHE-build-type:         Simple-extra-doc-files:    CHANGELOG.md--description:-  The folly C++ library from Meta, wrapped in a Cabal package so that-  it can be easily depended on by other packages. Having a Cabal-  package also means that we can version the library, which is useful-  as there are no versioned upstream releases.--  Also included is @fast_float-8.0.0@ because it is a dependency of-  folly and this version is not widely available as a distro package-  yet.--source-repository head-  type: git-  location: https://github.com/facebook/folly.git--common fb-cpp-  cxx-options: -std=c++17-  if !flag(clang)-     cxx-options: -fcoroutines-  if arch(x86_64)-     cxx-options: -march=haswell-  if flag(opt)-     cxx-options: -O3--flag opt-  default: False--flag clang-  default: False---- If False, we just depend on folly from pkg-config. This is to support the--- original build setup using getdeps.py, used in hsthrift's CI.-flag bundled-folly-  manual: True-  default: True--library-    import: fb-cpp-    default-language: Haskell2010-    if !flag(bundled-folly)-        pkgconfig-depends: libfolly-    else-        extra-libraries: stdc++, boost_filesystem, boost_program_options, boost_context-        pkgconfig-depends: fmt, libglog, openssl, snappy, libunwind--        -- The contents of cxx-sources and install-includes get spliced-        -- in by running 'make setup-folly'-        cxx-sources:-            --            -+version:            20250713.1537+synopsis:           The folly C++ library from Meta+author:             Simon Marlow+maintainer:         marlowsd@gmail.com+copyright:          Copyright (c) Meta Platforms, Inc. and affiliates.+homepage:           https://github.com/facebook/folly+bug-reports:        https://github.com/facebook/folly/issues+license:            Apache-2.0+license-files:+  folly/LICENSE+  fast_float-8.0.0/LICENSE-MIT+  fast_float-8.0.0/LICENSE-BOOST+  fast_float-8.0.0/LICENSE-APACHE+build-type:         Simple+extra-doc-files:    CHANGELOG.md++description:+  The folly C++ library from Meta, wrapped in a Cabal package so that+  it can be easily depended on by other packages. Having a Cabal+  package also means that we can version the library, which is useful+  as there are no versioned upstream releases.++  Also included is @fast_float-8.0.0@ because it is a dependency of+  folly and this version is not widely available as a distro package+  yet.++source-repository head+  type: git+  location: https://github.com/facebook/folly.git++common fb-cpp+  cxx-options: -std=c++17+  if !flag(clang)+     cxx-options: -fcoroutines+  if arch(x86_64)+     cxx-options: -march=haswell+  if flag(opt)+     cxx-options: -O3++flag opt+  default: False++flag clang+  default: False++-- If False, we just depend on folly from pkg-config. This is to support the+-- original build setup using getdeps.py, used in hsthrift's CI.+flag bundled-folly+  manual: True+  default: True++library+    import: fb-cpp+    default-language: Haskell2010+    if !flag(bundled-folly)+        pkgconfig-depends: libfolly+    else+        include-dirs: folly folly/_build fast_float-8.0.0/include+        extra-libraries: stdc++, boost_filesystem, boost_program_options, boost_context+        pkgconfig-depends: fmt, libglog, openssl, snappy, libunwind++        -- The contents of cxx-sources and install-includes get spliced+        -- in by running 'make setup-folly'+        cxx-sources:+            folly/folly/CancellationToken.cpp +            folly/folly/ClockGettimeWrappers.cpp +            folly/folly/Conv.cpp +            folly/folly/Demangle.cpp +            folly/folly/ExceptionString.cpp +            folly/folly/ExceptionWrapper.cpp +            folly/folly/Executor.cpp +            folly/folly/File.cpp +            folly/folly/FileUtil.cpp +            folly/folly/Fingerprint.cpp +            folly/folly/FmtUtility.cpp +            folly/folly/FollyMemcpy.cpp +            folly/folly/FollyMemset.cpp +            folly/folly/Format.cpp +            folly/folly/GroupVarint.cpp +            folly/folly/IPAddress.cpp +            folly/folly/IPAddressV4.cpp +            folly/folly/IPAddressV6.cpp +            folly/folly/MacAddress.cpp +            folly/folly/MicroLock.cpp +            folly/folly/Random.cpp +            folly/folly/ScopeGuard.cpp +            folly/folly/SharedMutex.cpp +            folly/folly/Singleton.cpp +            folly/folly/SingletonThreadLocal.cpp +            folly/folly/SocketAddress.cpp +            folly/folly/String.cpp +            folly/folly/Subprocess.cpp +            folly/folly/TimeoutQueue.cpp +            folly/folly/Try.cpp +            folly/folly/Unicode.cpp +            folly/folly/Uri.cpp +            folly/folly/memcpy_select_aarch64.cpp +            folly/folly/memset_select_aarch64.cpp +            folly/folly/algorithm/simd/Contains.cpp +            folly/folly/channels/MaxConcurrentRateLimiter.cpp +            folly/folly/cli/NestedCommandLineApp.cpp +            folly/folly/cli/ProgramOptions.cpp +            folly/folly/compression/Compression.cpp +            folly/folly/compression/CompressionContextPoolSingletons.cpp +            folly/folly/compression/QuotientMultiSet.cpp +            folly/folly/compression/Select64.cpp +            folly/folly/compression/Zlib.cpp +            folly/folly/compression/Zstd.cpp +            folly/folly/concurrency/CacheLocality.cpp +            folly/folly/concurrency/DeadlockDetector.cpp +            folly/folly/concurrency/ProcessLocalUniqueId.cpp +            folly/folly/concurrency/memory/AtomicReadMostlyMainPtr.cpp +            folly/folly/container/RegexMatchCache.cpp +            folly/folly/container/detail/F14Table.cpp +            folly/folly/coro/Baton.cpp +            folly/folly/coro/Mutex.cpp +            folly/folly/coro/SerialQueueRunner.cpp +            folly/folly/coro/SharedMutex.cpp +            folly/folly/coro/detail/Malloc.cpp +            folly/folly/crypto/Blake2xb.cpp +            folly/folly/crypto/LtHash.cpp +            folly/folly/crypto/detail/MathOperation_AVX2.cpp +            folly/folly/crypto/detail/MathOperation_SSE2.cpp +            folly/folly/crypto/detail/MathOperation_Simple.cpp +            folly/folly/debugging/exception_tracer/ExceptionCounterLib.cpp +            folly/folly/debugging/exception_tracer/ExceptionStackTraceLib.cpp +            folly/folly/debugging/exception_tracer/ExceptionTracer.cpp +            folly/folly/debugging/exception_tracer/ExceptionTracerLib.cpp +            folly/folly/debugging/exception_tracer/SmartExceptionStackTraceHooks.cpp +            folly/folly/debugging/exception_tracer/SmartExceptionTracer.cpp +            folly/folly/debugging/exception_tracer/SmartExceptionTracerSingleton.cpp +            folly/folly/debugging/exception_tracer/StackTrace.cpp +            folly/folly/debugging/symbolizer/Dwarf.cpp +            folly/folly/debugging/symbolizer/DwarfImpl.cpp +            folly/folly/debugging/symbolizer/DwarfLineNumberVM.cpp +            folly/folly/debugging/symbolizer/DwarfSection.cpp +            folly/folly/debugging/symbolizer/DwarfUtil.cpp +            folly/folly/debugging/symbolizer/Elf.cpp +            folly/folly/debugging/symbolizer/ElfCache.cpp +            folly/folly/debugging/symbolizer/LineReader.cpp +            folly/folly/debugging/symbolizer/SignalHandler.cpp +            folly/folly/debugging/symbolizer/StackTrace.cpp +            folly/folly/debugging/symbolizer/SymbolizePrinter.cpp +            folly/folly/debugging/symbolizer/SymbolizedFrame.cpp +            folly/folly/debugging/symbolizer/Symbolizer.cpp +            folly/folly/debugging/symbolizer/detail/Debug.cpp +            folly/folly/detail/AsyncTrace.cpp +            folly/folly/detail/FileUtilDetail.cpp +            folly/folly/detail/Futex.cpp +            folly/folly/detail/IPAddress.cpp +            folly/folly/detail/MemoryIdler.cpp +            folly/folly/detail/PerfScoped.cpp +            folly/folly/detail/RangeCommon.cpp +            folly/folly/detail/RangeSimd.cpp +            folly/folly/detail/RangeSse42.cpp +            folly/folly/detail/SimpleSimdStringUtils.cpp +            folly/folly/detail/SocketFastOpen.cpp +            folly/folly/detail/SplitStringSimd.cpp +            folly/folly/detail/Sse.cpp +            folly/folly/detail/StaticSingletonManager.cpp +            folly/folly/detail/ThreadLocalDetail.cpp +            folly/folly/detail/TrapOnAvx512.cpp +            folly/folly/detail/UniqueInstance.cpp +            folly/folly/detail/base64_detail/Base64Api.cpp +            folly/folly/detail/base64_detail/Base64SWAR.cpp +            folly/folly/detail/base64_detail/Base64_SSE4_2.cpp +            folly/folly/detail/thread_local_globals.cpp +            folly/folly/executors/CPUThreadPoolExecutor.cpp +            folly/folly/executors/Codel.cpp +            folly/folly/executors/EDFThreadPoolExecutor.cpp +            folly/folly/executors/ExecutionObserver.cpp +            folly/folly/executors/ExecutorWithPriority.cpp +            folly/folly/executors/FunctionScheduler.cpp +            folly/folly/executors/GlobalExecutor.cpp +            folly/folly/executors/GlobalThreadPoolList.cpp +            folly/folly/executors/IOThreadPoolDeadlockDetectorObserver.cpp +            folly/folly/executors/IOThreadPoolExecutor.cpp +            folly/folly/executors/InlineExecutor.cpp +            folly/folly/executors/ManualExecutor.cpp +            folly/folly/executors/QueueObserver.cpp +            folly/folly/executors/QueuedImmediateExecutor.cpp +            folly/folly/executors/SoftRealTimeExecutor.cpp +            folly/folly/executors/StrandExecutor.cpp +            folly/folly/executors/ThreadPoolExecutor.cpp +            folly/folly/executors/ThreadedExecutor.cpp +            folly/folly/executors/ThreadedRepeatingFunctionRunner.cpp +            folly/folly/executors/TimedDrivableExecutor.cpp +            folly/folly/executors/TimekeeperScheduledExecutor.cpp +            folly/folly/executors/thread_factory/PriorityThreadFactory.cpp +            folly/folly/ext/buck2/test_ext.cpp +            folly/folly/ext/test_ext.cpp +            folly/folly/external/farmhash/farmhash.cpp +            folly/folly/external/fast-crc32/avx512_crc32c_v8s3x4.cpp +            folly/folly/external/fast-crc32/neon_crc32c_v3s4x2e_v2.cpp +            folly/folly/external/fast-crc32/neon_eor3_crc32_v9s3x2e_s3.cpp +            folly/folly/external/fast-crc32/neon_eor3_crc32c_v8s2x4_s3.cpp +            folly/folly/external/fast-crc32/sse_crc32c_v8s3x3.cpp +            folly/folly/external/nvidia/detail/RangeSve2.cpp +            folly/folly/external/nvidia/hash/Checksum.cpp +            folly/folly/external/nvidia/hash/detail/Crc32cDetail.cpp +            folly/folly/fibers/BatchSemaphore.cpp +            folly/folly/fibers/Baton.cpp +            folly/folly/fibers/Fiber.cpp +            folly/folly/fibers/FiberManager.cpp +            folly/folly/fibers/GuardPageAllocator.cpp +            folly/folly/fibers/Semaphore.cpp +            folly/folly/fibers/SemaphoreBase.cpp +            folly/folly/fibers/SimpleLoopController.cpp +            folly/folly/fibers/async/Async.cpp +            folly/folly/fibers/detail/AtomicBatchDispatcher.cpp +            folly/folly/futures/Barrier.cpp +            folly/folly/futures/Future.cpp +            folly/folly/futures/HeapTimekeeper.cpp +            folly/folly/futures/ManualTimekeeper.cpp +            folly/folly/futures/Promise.cpp +            folly/folly/futures/SharedPromise.cpp +            folly/folly/futures/ThreadWheelTimekeeper.cpp +            folly/folly/futures/detail/Core.cpp +            folly/folly/hash/Checksum.cpp +            folly/folly/hash/SpookyHashV1.cpp +            folly/folly/hash/SpookyHashV2.cpp +            folly/folly/hash/detail/ChecksumDetail.cpp +            folly/folly/hash/detail/Crc32CombineDetail.cpp +            folly/folly/hash/detail/Crc32cDetail.cpp +            folly/folly/init/Init.cpp +            folly/folly/init/Phase.cpp +            folly/folly/io/Cursor.cpp +            folly/folly/io/FsUtil.cpp +            folly/folly/io/GlobalShutdownSocketSet.cpp +            folly/folly/io/HugePages.cpp +            folly/folly/io/IOBuf.cpp +            folly/folly/io/IOBufIovecBuilder.cpp +            folly/folly/io/IOBufQueue.cpp +            folly/folly/io/RecordIO.cpp +            folly/folly/io/ShutdownSocketSet.cpp +            folly/folly/io/SocketOptionMap.cpp +            folly/folly/io/SocketOptionValue.cpp +            folly/folly/io/async/AsyncBase.cpp +            folly/folly/io/async/AsyncIO.cpp +            folly/folly/io/async/AsyncIoUringSocket.cpp +            folly/folly/io/async/AsyncPipe.cpp +            folly/folly/io/async/AsyncSSLSocket.cpp +            folly/folly/io/async/AsyncServerSocket.cpp +            folly/folly/io/async/AsyncSignalHandler.cpp +            folly/folly/io/async/AsyncSocket.cpp +            folly/folly/io/async/AsyncSocketException.cpp +            folly/folly/io/async/AsyncSocketTransport.cpp +            folly/folly/io/async/AsyncTimeout.cpp +            folly/folly/io/async/AsyncUDPSocket.cpp +            folly/folly/io/async/DelayedDestruction.cpp +            folly/folly/io/async/EpollBackend.cpp +            folly/folly/io/async/EventBase.cpp +            folly/folly/io/async/EventBaseBackendBase.cpp +            folly/folly/io/async/EventBaseLocal.cpp +            folly/folly/io/async/EventBaseManager.cpp +            folly/folly/io/async/EventBasePoller.cpp +            folly/folly/io/async/EventBaseThread.cpp +            folly/folly/io/async/EventHandler.cpp +            folly/folly/io/async/HHWheelTimer.cpp +            folly/folly/io/async/IoUring.cpp +            folly/folly/io/async/IoUringBackend.cpp +            folly/folly/io/async/IoUringEvent.cpp +            folly/folly/io/async/IoUringEventBaseLocal.cpp +            folly/folly/io/async/IoUringProvidedBufferRing.cpp +            folly/folly/io/async/IoUringZeroCopyBufferPool.cpp +            folly/folly/io/async/MuxIOThreadPoolExecutor.cpp +            folly/folly/io/async/PasswordInFile.cpp +            folly/folly/io/async/Request.cpp +            folly/folly/io/async/SSLContext.cpp +            folly/folly/io/async/SSLOptions.cpp +            folly/folly/io/async/STTimerFDTimeoutManager.cpp +            folly/folly/io/async/ScopedEventBaseThread.cpp +            folly/folly/io/async/SimpleAsyncIO.cpp +            folly/folly/io/async/TerminateCancellationToken.cpp +            folly/folly/io/async/TimeoutManager.cpp +            folly/folly/io/async/TimerFD.cpp +            folly/folly/io/async/TimerFDTimeoutManager.cpp +            folly/folly/io/async/VirtualEventBase.cpp +            folly/folly/io/async/fdsock/AsyncFdSocket.cpp +            folly/folly/io/async/fdsock/SocketFds.cpp +            folly/folly/io/async/ssl/OpenSSLUtils.cpp +            folly/folly/io/async/ssl/SSLErrors.cpp +            folly/folly/io/coro/ServerSocket.cpp +            folly/folly/io/coro/Transport.cpp +            folly/folly/json/DynamicParser.cpp +            folly/folly/json/JSONSchema.cpp +            folly/folly/json/JsonTestUtil.cpp +            folly/folly/json/bser/Dump.cpp +            folly/folly/json/bser/Load.cpp +            folly/folly/json/dynamic.cpp +            folly/folly/json/json.cpp +            folly/folly/json/json_patch.cpp +            folly/folly/json/json_pointer.cpp +            folly/folly/lang/CString.cpp +            folly/folly/lang/Exception.cpp +            folly/folly/lang/SafeAssert.cpp +            folly/folly/lang/ToAscii.cpp +            folly/folly/lang/UncaughtExceptions.cpp +            folly/folly/logging/AsyncFileWriter.cpp +            folly/folly/logging/AsyncLogWriter.cpp +            folly/folly/logging/BridgeFromGoogleLogging.cpp +            folly/folly/logging/CustomLogFormatter.cpp +            folly/folly/logging/FileHandlerFactory.cpp +            folly/folly/logging/FileWriterFactory.cpp +            folly/folly/logging/GlogStyleFormatter.cpp +            folly/folly/logging/ImmediateFileWriter.cpp +            folly/folly/logging/Init.cpp +            folly/folly/logging/InitWeak.cpp +            folly/folly/logging/LogCategory.cpp +            folly/folly/logging/LogCategoryConfig.cpp +            folly/folly/logging/LogConfig.cpp +            folly/folly/logging/LogConfigParser.cpp +            folly/folly/logging/LogHandlerConfig.cpp +            folly/folly/logging/LogLevel.cpp +            folly/folly/logging/LogMessage.cpp +            folly/folly/logging/LogName.cpp +            folly/folly/logging/LogStream.cpp +            folly/folly/logging/LogStreamProcessor.cpp +            folly/folly/logging/Logger.cpp +            folly/folly/logging/LoggerDB.cpp +            folly/folly/logging/ObjectToString.cpp +            folly/folly/logging/RateLimiter.cpp +            folly/folly/logging/StandardLogHandler.cpp +            folly/folly/logging/StandardLogHandlerFactory.cpp +            folly/folly/logging/StreamHandlerFactory.cpp +            folly/folly/logging/xlog.cpp +            folly/folly/memory/JemallocHugePageAllocator.cpp +            folly/folly/memory/JemallocNodumpAllocator.cpp +            folly/folly/memory/MallctlHelper.cpp +            folly/folly/memory/ReentrantAllocator.cpp +            folly/folly/memory/SanitizeAddress.cpp +            folly/folly/memory/SanitizeLeak.cpp +            folly/folly/memory/ThreadCachedArena.cpp +            folly/folly/memory/detail/MallocImpl.cpp +            folly/folly/net/NetOps.cpp +            folly/folly/net/NetOpsDispatcher.cpp +            folly/folly/net/TcpInfo.cpp +            folly/folly/net/TcpInfoDispatcher.cpp +            folly/folly/net/detail/SocketFileDescriptorMap.cpp +            folly/folly/observer/detail/Core.cpp +            folly/folly/observer/detail/ObserverManager.cpp +            folly/folly/portability/Builtins.cpp +            folly/folly/portability/Dirent.cpp +            folly/folly/portability/Fcntl.cpp +            folly/folly/portability/Filesystem.cpp +            folly/folly/portability/GFlags.cpp +            folly/folly/portability/Libgen.cpp +            folly/folly/portability/Malloc.cpp +            folly/folly/portability/OpenSSL.cpp +            folly/folly/portability/PThread.cpp +            folly/folly/portability/Sched.cpp +            folly/folly/portability/Sockets.cpp +            folly/folly/portability/Stdio.cpp +            folly/folly/portability/Stdlib.cpp +            folly/folly/portability/String.cpp +            folly/folly/portability/SysFile.cpp +            folly/folly/portability/SysMembarrier.cpp +            folly/folly/portability/SysMman.cpp +            folly/folly/portability/SysResource.cpp +            folly/folly/portability/SysStat.cpp +            folly/folly/portability/SysTime.cpp +            folly/folly/portability/SysUio.cpp +            folly/folly/portability/Time.cpp +            folly/folly/portability/Unistd.cpp +            folly/folly/result/result.cpp +            folly/folly/settings/CommandLineParser.cpp +            folly/folly/settings/Immutables.cpp +            folly/folly/settings/Settings.cpp +            folly/folly/settings/SettingsAccessorProxy.cpp +            folly/folly/settings/Types.cpp +            folly/folly/ssl/OpenSSLCertUtils.cpp +            folly/folly/ssl/OpenSSLHash.cpp +            folly/folly/ssl/OpenSSLKeyUtils.cpp +            folly/folly/ssl/PasswordCollector.cpp +            folly/folly/ssl/SSLSessionManager.cpp +            folly/folly/ssl/detail/OpenSSLSession.cpp +            folly/folly/stats/QuantileEstimator.cpp +            folly/folly/stats/TDigest.cpp +            folly/folly/stats/detail/DoubleRadixSort.cpp +            folly/folly/synchronization/AsymmetricThreadFence.cpp +            folly/folly/synchronization/AtomicNotification.cpp +            folly/folly/synchronization/DistributedMutex.cpp +            folly/folly/synchronization/Hazptr.cpp +            folly/folly/synchronization/HazptrDomain.cpp +            folly/folly/synchronization/HazptrThreadPoolExecutor.cpp +            folly/folly/synchronization/ParkingLot.cpp +            folly/folly/synchronization/Rcu.cpp +            folly/folly/synchronization/SanitizeThread.cpp +            folly/folly/synchronization/WaitOptions.cpp +            folly/folly/synchronization/detail/Hardware.cpp +            folly/folly/synchronization/detail/Sleeper.cpp +            folly/folly/system/AtFork.cpp +            folly/folly/system/EnvUtil.cpp +            folly/folly/system/HardwareConcurrency.cpp +            folly/folly/system/MemoryMapping.cpp +            folly/folly/system/Pid.cpp +            folly/folly/system/Shell.cpp +            folly/folly/system/ThreadId.cpp +            folly/folly/system/ThreadName.cpp +            folly/folly/testing/TestUtil.cpp +            folly/folly/tracing/AsyncStack.cpp +            folly/folly/io/async/test/ScopedBoundPort.cpp +            folly/folly/io/async/test/SocketPair.cpp +            folly/folly/io/async/test/TimeUtil.cpp++        install-includes:+            folly/AtomicHashArray-inl.h +            folly/AtomicHashArray.h +            folly/AtomicHashMap-inl.h +            folly/AtomicHashMap.h +            folly/AtomicIntrusiveLinkedList.h +            folly/AtomicLinkedList.h +            folly/AtomicUnorderedMap.h +            folly/Benchmark.h +            folly/BenchmarkUtil.h +            folly/Bits.h +            folly/CPortability.h +            folly/CancellationToken-inl.h +            folly/CancellationToken.h +            folly/Chrono.h +            folly/ClockGettimeWrappers.h +            folly/ConcurrentBitSet.h +            folly/ConcurrentLazy.h +            folly/ConcurrentSkipList-inl.h +            folly/ConcurrentSkipList.h +            folly/ConstexprMath.h +            folly/ConstructorCallbackList.h +            folly/Conv.h +            folly/CppAttributes.h +            folly/CpuId.h +            folly/DefaultKeepAliveExecutor.h +            folly/Demangle.h +            folly/DiscriminatedPtr.h +            folly/DynamicConverter.h +            folly/Exception.h +            folly/ExceptionString.h +            folly/ExceptionWrapper-inl.h +            folly/ExceptionWrapper.h +            folly/Executor.h +            folly/Expected.h +            folly/FBString.h +            folly/FBVector.h +            folly/File.h +            folly/FileUtil.h +            folly/Fingerprint.h +            folly/FixedString.h +            folly/FmtUtility.h +            folly/FollyMemcpy.h +            folly/FollyMemset.h +            folly/Format-inl.h +            folly/Format.h +            folly/FormatArg.h +            folly/FormatTraits.h +            folly/Function.h +            folly/GLog.h +            folly/GroupVarint.h +            folly/Hash.h +            folly/IPAddress.h +            folly/IPAddressException.h +            folly/IPAddressV4.h +            folly/IPAddressV6.h +            folly/Indestructible.h +            folly/IndexedMemPool.h +            folly/IntrusiveList.h +            folly/Lazy.h +            folly/Likely.h +            folly/MPMCPipeline.h +            folly/MPMCQueue.h +            folly/MacAddress.h +            folly/MapUtil.h +            folly/Math.h +            folly/MaybeManagedPtr.h +            folly/Memory.h +            folly/MicroLock.h +            folly/MicroSpinLock.h +            folly/MoveWrapper.h +            folly/ObserverContainer.h +            folly/OperationCancelled.h +            folly/Optional.h +            folly/Overload.h +            folly/PackedSyncPtr.h +            folly/Padded.h +            folly/Poly-inl.h +            folly/Poly.h +            folly/PolyException.h +            folly/Portability.h +            folly/Preprocessor.h +            folly/ProducerConsumerQueue.h +            folly/RWSpinLock.h +            folly/Random-inl.h +            folly/Random.h +            folly/Range.h +            folly/Replaceable.h +            folly/ScopeGuard.h +            folly/SharedMutex.h +            folly/Singleton-inl.h +            folly/Singleton.h +            folly/SingletonThreadLocal.h +            folly/SocketAddress.h +            folly/SpinLock.h +            folly/String-inl.h +            folly/String.h +            folly/Subprocess.h +            folly/Synchronized.h +            folly/SynchronizedPtr.h +            folly/ThreadCachedInt.h +            folly/ThreadLocal.h +            folly/TimeoutQueue.h +            folly/TokenBucket.h +            folly/Traits.h +            folly/Try-inl.h +            folly/Try.h +            folly/UTF8String.h +            folly/Unicode.h +            folly/Unit.h +            folly/Uri-inl.h +            folly/Uri.h +            folly/Utility.h +            folly/Varint.h +            folly/VirtualExecutor.h +            folly/base64.h +            folly/dynamic-inl.h +            folly/dynamic.h +            folly/json.h +            folly/json_patch.h +            folly/json_pointer.h +            folly/small_vector.h +            folly/sorted_vector_types.h +            folly/stop_watch.h +            folly/algorithm/BinaryHeap.h +            folly/algorithm/simd/Contains.h +            folly/algorithm/simd/FindFixed.h +            folly/algorithm/simd/Ignore.h +            folly/algorithm/simd/Movemask.h +            folly/algorithm/simd/detail/ContainsImpl.h +            folly/algorithm/simd/detail/SimdAnyOf.h +            folly/algorithm/simd/detail/SimdForEach.h +            folly/algorithm/simd/detail/SimdPlatform.h +            folly/algorithm/simd/detail/Traits.h +            folly/algorithm/simd/detail/UnrollUtils.h +            folly/algorithm/simd/find_first_of.h +            folly/algorithm/simd/find_first_of_extra.h +            folly/channels/Channel-fwd.h +            folly/channels/Channel-inl.h +            folly/channels/Channel.h +            folly/channels/ChannelCallbackHandle.h +            folly/channels/ChannelProcessor-inl.h +            folly/channels/ChannelProcessor.h +            folly/channels/ConsumeChannel-inl.h +            folly/channels/ConsumeChannel.h +            folly/channels/FanoutChannel-inl.h +            folly/channels/FanoutChannel.h +            folly/channels/FanoutSender-inl.h +            folly/channels/FanoutSender.h +            folly/channels/MaxConcurrentRateLimiter.h +            folly/channels/Merge-inl.h +            folly/channels/Merge.h +            folly/channels/MergeChannel-inl.h +            folly/channels/MergeChannel.h +            folly/channels/MultiplexChannel-inl.h +            folly/channels/MultiplexChannel.h +            folly/channels/OnClosedException.h +            folly/channels/Producer-inl.h +            folly/channels/Producer.h +            folly/channels/ProxyChannel-inl.h +            folly/channels/ProxyChannel.h +            folly/channels/RateLimiter.h +            folly/channels/Transform-inl.h +            folly/channels/Transform.h +            folly/channels/detail/AtomicQueue.h +            folly/channels/detail/ChannelBridge.h +            folly/channels/detail/IntrusivePtr.h +            folly/channels/detail/MultiplexerTraits.h +            folly/channels/detail/PointerVariant.h +            folly/channels/detail/Utility.h +            folly/chrono/Clock.h +            folly/chrono/Conv.h +            folly/chrono/Hardware.h +            folly/cli/NestedCommandLineApp.h +            folly/cli/ProgramOptions.h +            folly/codec/Uuid.h +            folly/codec/hex.h +            folly/compression/Compression.h +            folly/compression/CompressionContextPool.h +            folly/compression/CompressionContextPoolSingletons.h +            folly/compression/CompressionCoreLocalContextPool.h +            folly/compression/Instructions.h +            folly/compression/QuotientMultiSet-inl.h +            folly/compression/QuotientMultiSet.h +            folly/compression/Select64.h +            folly/compression/Utils.h +            folly/compression/Zlib.h +            folly/compression/Zstd.h +            folly/compression/elias_fano/BitVectorCoding.h +            folly/compression/elias_fano/CodingDetail.h +            folly/compression/elias_fano/EliasFanoCoding.h +            folly/concurrency/AtomicSharedPtr.h +            folly/concurrency/CacheLocality.h +            folly/concurrency/ConcurrentHashMap.h +            folly/concurrency/CoreCachedSharedPtr.h +            folly/concurrency/DeadlockDetector.h +            folly/concurrency/DynamicBoundedQueue.h +            folly/concurrency/PriorityUnboundedQueueSet.h +            folly/concurrency/ProcessLocalUniqueId.h +            folly/concurrency/SingletonRelaxedCounter.h +            folly/concurrency/ThreadCachedSynchronized.h +            folly/concurrency/UnboundedQueue.h +            folly/concurrency/container/FlatCombiningPriorityQueue.h +            folly/concurrency/container/LockFreeRingBuffer.h +            folly/concurrency/container/RelaxedConcurrentPriorityQueue.h +            folly/concurrency/container/SingleWriterFixedHashMap.h +            folly/concurrency/container/atomic_grow_array.h +            folly/concurrency/detail/AtomicSharedPtr-detail.h +            folly/concurrency/detail/ConcurrentHashMap-detail.h +            folly/concurrency/memory/AtomicReadMostlyMainPtr.h +            folly/concurrency/memory/PrimaryPtr.h +            folly/concurrency/memory/ReadMostlySharedPtr.h +            folly/concurrency/memory/TLRefCount.h +            folly/container/Access.h +            folly/container/Array.h +            folly/container/BitIterator.h +            folly/container/Enumerate.h +            folly/container/EvictingCacheMap.h +            folly/container/F14Map-fwd.h +            folly/container/F14Map.h +            folly/container/F14Set-fwd.h +            folly/container/F14Set.h +            folly/container/FBVector.h +            folly/container/Foreach-inl.h +            folly/container/Foreach.h +            folly/container/HeterogeneousAccess-fwd.h +            folly/container/HeterogeneousAccess.h +            folly/container/IntrusiveHeap.h +            folly/container/IntrusiveList.h +            folly/container/Iterator.h +            folly/container/MapUtil.h +            folly/container/Merge.h +            folly/container/RegexMatchCache.h +            folly/container/Reserve.h +            folly/container/SparseByteSet.h +            folly/container/StdBitset.h +            folly/container/View.h +            folly/container/WeightedEvictingCacheMap.h +            folly/container/detail/BitIteratorDetail.h +            folly/container/detail/BoolWrapper.h +            folly/container/detail/F14Defaults.h +            folly/container/detail/F14IntrinsicsAvailability.h +            folly/container/detail/F14MapFallback.h +            folly/container/detail/F14Mask.h +            folly/container/detail/F14Policy.h +            folly/container/detail/F14SetFallback.h +            folly/container/detail/F14Table.h +            folly/container/detail/Util.h +            folly/container/detail/tape_detail.h +            folly/container/heap_vector_types.h +            folly/container/range_traits.h +            folly/container/small_vector.h +            folly/container/sorted_vector_types.h +            folly/container/span.h +            folly/container/tape.h +            folly/container/vector_bool.h +            folly/coro/Accumulate-inl.h +            folly/coro/Accumulate.h +            folly/coro/AsyncGenerator.h +            folly/coro/AsyncPipe.h +            folly/coro/AsyncScope.h +            folly/coro/AsyncStack.h +            folly/coro/AutoCleanup-fwd.h +            folly/coro/AutoCleanup.h +            folly/coro/AwaitImmediately.h +            folly/coro/AwaitResult.h +            folly/coro/Baton.h +            folly/coro/BlockingWait.h +            folly/coro/BoundedQueue.h +            folly/coro/Cleanup.h +            folly/coro/Collect-inl.h +            folly/coro/Collect.h +            folly/coro/Concat-inl.h +            folly/coro/Concat.h +            folly/coro/Coroutine.h +            folly/coro/CurrentExecutor.h +            folly/coro/DetachOnCancel.h +            folly/coro/Filter-inl.h +            folly/coro/Filter.h +            folly/coro/FutureUtil.h +            folly/coro/Generator.h +            folly/coro/GmockHelpers.h +            folly/coro/GtestHelpers.h +            folly/coro/Invoke.h +            folly/coro/Merge-inl.h +            folly/coro/Merge.h +            folly/coro/Mutex.h +            folly/coro/Noexcept.h +            folly/coro/Promise.h +            folly/coro/Ready.h +            folly/coro/Result.h +            folly/coro/Retry.h +            folly/coro/RustAdaptors.h +            folly/coro/ScopeExit.h +            folly/coro/SerialQueueRunner.h +            folly/coro/SharedLock.h +            folly/coro/SharedMutex.h +            folly/coro/SharedPromise.h +            folly/coro/Sleep-inl.h +            folly/coro/Sleep.h +            folly/coro/SmallUnboundedQueue.h +            folly/coro/Synchronized.h +            folly/coro/Task.h +            folly/coro/TaskWrapper.h +            folly/coro/TimedWait.h +            folly/coro/Timeout-inl.h +            folly/coro/Timeout.h +            folly/coro/Traits.h +            folly/coro/Transform-inl.h +            folly/coro/Transform.h +            folly/coro/UnboundedQueue.h +            folly/coro/ViaIfAsync.h +            folly/coro/WithAsyncStack.h +            folly/coro/WithCancellation.h +            folly/coro/detail/Barrier.h +            folly/coro/detail/BarrierTask.h +            folly/coro/detail/CurrentAsyncFrame.h +            folly/coro/detail/Helpers.h +            folly/coro/detail/InlineTask.h +            folly/coro/detail/Malloc.h +            folly/coro/detail/ManualLifetime.h +            folly/coro/detail/PickTaskWrapper.h +            folly/coro/detail/Traits.h +            folly/coro/safe/AsyncClosure-fwd.h +            folly/coro/safe/AsyncClosure.h +            folly/coro/safe/Captures.h +            folly/coro/safe/NowTask.h +            folly/coro/safe/SafeAlias.h +            folly/coro/safe/SafeTask.h +            folly/coro/safe/detail/AsyncClosure.h +            folly/coro/safe/detail/AsyncClosureBindings.h +            folly/coro/safe/detail/DefineMovableDeepConstLrefCopyable.h +            folly/crypto/Blake2xb.h +            folly/crypto/LtHash-inl.h +            folly/crypto/LtHash.h +            folly/crypto/detail/LtHashInternal.h +            folly/debugging/exception_tracer/Compatibility.h +            folly/debugging/exception_tracer/ExceptionAbi.h +            folly/debugging/exception_tracer/ExceptionCounterLib.h +            folly/debugging/exception_tracer/ExceptionTracer.h +            folly/debugging/exception_tracer/ExceptionTracerLib.h +            folly/debugging/exception_tracer/SmartExceptionTracer.h +            folly/debugging/exception_tracer/SmartExceptionTracerSingleton.h +            folly/debugging/exception_tracer/StackTrace.h +            folly/debugging/symbolizer/Dwarf.h +            folly/debugging/symbolizer/DwarfImpl.h +            folly/debugging/symbolizer/DwarfLineNumberVM.h +            folly/debugging/symbolizer/DwarfSection.h +            folly/debugging/symbolizer/DwarfUtil.h +            folly/debugging/symbolizer/Elf-inl.h +            folly/debugging/symbolizer/Elf.h +            folly/debugging/symbolizer/ElfCache.h +            folly/debugging/symbolizer/LineReader.h +            folly/debugging/symbolizer/SignalHandler.h +            folly/debugging/symbolizer/StackTrace.h +            folly/debugging/symbolizer/SymbolizePrinter.h +            folly/debugging/symbolizer/SymbolizedFrame.h +            folly/debugging/symbolizer/Symbolizer.h +            folly/debugging/symbolizer/detail/Debug.h +            folly/detail/AsyncTrace.h +            folly/detail/AtomicHashUtils.h +            folly/detail/AtomicUnorderedMapUtils.h +            folly/detail/DiscriminatedPtrDetail.h +            folly/detail/FileUtilDetail.h +            folly/detail/FileUtilVectorDetail.h +            folly/detail/FingerprintPolynomial.h +            folly/detail/Futex-inl.h +            folly/detail/Futex.h +            folly/detail/GroupVarintDetail.h +            folly/detail/IPAddress.h +            folly/detail/IPAddressSource.h +            folly/detail/Iterators.h +            folly/detail/MPMCPipelineDetail.h +            folly/detail/MemoryIdler.h +            folly/detail/PerfScoped.h +            folly/detail/PolyDetail.h +            folly/detail/RangeCommon.h +            folly/detail/RangeSimd.h +            folly/detail/RangeSse42.h +            folly/detail/SimpleSimdStringUtils.h +            folly/detail/SimpleSimdStringUtilsImpl.h +            folly/detail/Singleton.h +            folly/detail/SlowFingerprint.h +            folly/detail/SocketFastOpen.h +            folly/detail/SplitStringSimd.h +            folly/detail/SplitStringSimdImpl.h +            folly/detail/Sse.h +            folly/detail/StaticSingletonManager.h +            folly/detail/ThreadLocalDetail.h +            folly/detail/TrapOnAvx512.h +            folly/detail/TurnSequencer.h +            folly/detail/TypeList.h +            folly/detail/UniqueInstance.h +            folly/detail/base64_detail/Base64Api.h +            folly/detail/base64_detail/Base64Common.h +            folly/detail/base64_detail/Base64Constants.h +            folly/detail/base64_detail/Base64HiddenConstants.h +            folly/detail/base64_detail/Base64SWAR.h +            folly/detail/base64_detail/Base64Scalar.h +            folly/detail/base64_detail/Base64Simd.h +            folly/detail/base64_detail/Base64_SSE4_2.h +            folly/detail/base64_detail/Base64_SSE4_2_Platform.h +            folly/detail/thread_local_globals.h +            folly/detail/tuple.h +            folly/executors/Async.h +            folly/executors/CPUThreadPoolExecutor.h +            folly/executors/Codel.h +            folly/executors/DrivableExecutor.h +            folly/executors/EDFThreadPoolExecutor.h +            folly/executors/ExecutionObserver.h +            folly/executors/ExecutorWithPriority-inl.h +            folly/executors/ExecutorWithPriority.h +            folly/executors/FiberIOExecutor.h +            folly/executors/FunctionScheduler.h +            folly/executors/FutureExecutor.h +            folly/executors/GlobalExecutor.h +            folly/executors/GlobalThreadPoolList.h +            folly/executors/IOExecutor.h +            folly/executors/IOObjectCache.h +            folly/executors/IOThreadPoolDeadlockDetectorObserver.h +            folly/executors/IOThreadPoolExecutor.h +            folly/executors/InlineExecutor.h +            folly/executors/ManualExecutor.h +            folly/executors/MeteredExecutor-inl.h +            folly/executors/MeteredExecutor.h +            folly/executors/QueueObserver.h +            folly/executors/QueuedImmediateExecutor.h +            folly/executors/ScheduledExecutor.h +            folly/executors/SequencedExecutor.h +            folly/executors/SerialExecutor-inl.h +            folly/executors/SerialExecutor.h +            folly/executors/SerializedExecutor.h +            folly/executors/SoftRealTimeExecutor.h +            folly/executors/StrandExecutor.h +            folly/executors/ThreadPoolExecutor.h +            folly/executors/ThreadedExecutor.h +            folly/executors/ThreadedRepeatingFunctionRunner.h +            folly/executors/TimedDrivableExecutor.h +            folly/executors/TimekeeperScheduledExecutor.h +            folly/executors/VirtualExecutor.h +            folly/executors/task_queue/BlockingQueue.h +            folly/executors/task_queue/LifoSemMPMCQueue.h +            folly/executors/task_queue/PriorityLifoSemMPMCQueue.h +            folly/executors/task_queue/PriorityUnboundedBlockingQueue.h +            folly/executors/task_queue/UnboundedBlockingQueue.h +            folly/executors/thread_factory/InitThreadFactory.h +            folly/executors/thread_factory/NamedThreadFactory.h +            folly/executors/thread_factory/PriorityThreadFactory.h +            folly/executors/thread_factory/ThreadFactory.h +            folly/experimental/EventCount.h +            folly/experimental/FlatCombiningPriorityQueue.h +            folly/experimental/FunctionScheduler.h +            folly/experimental/TestUtil.h +            folly/experimental/ThreadedRepeatingFunctionRunner.h +            folly/experimental/channels/Channel-fwd.h +            folly/experimental/channels/Channel-inl.h +            folly/experimental/channels/Channel.h +            folly/experimental/channels/ChannelCallbackHandle.h +            folly/experimental/channels/ChannelProcessor-inl.h +            folly/experimental/channels/ChannelProcessor.h +            folly/experimental/channels/ConsumeChannel-inl.h +            folly/experimental/channels/ConsumeChannel.h +            folly/experimental/channels/FanoutChannel-inl.h +            folly/experimental/channels/FanoutChannel.h +            folly/experimental/channels/FanoutSender-inl.h +            folly/experimental/channels/FanoutSender.h +            folly/experimental/channels/MaxConcurrentRateLimiter.h +            folly/experimental/channels/Merge-inl.h +            folly/experimental/channels/Merge.h +            folly/experimental/channels/MergeChannel-inl.h +            folly/experimental/channels/MergeChannel.h +            folly/experimental/channels/MultiplexChannel-inl.h +            folly/experimental/channels/MultiplexChannel.h +            folly/experimental/channels/OnClosedException.h +            folly/experimental/channels/Producer-inl.h +            folly/experimental/channels/Producer.h +            folly/experimental/channels/ProxyChannel-inl.h +            folly/experimental/channels/ProxyChannel.h +            folly/experimental/channels/RateLimiter.h +            folly/experimental/channels/Transform-inl.h +            folly/experimental/channels/Transform.h +            folly/experimental/channels/detail/AtomicQueue.h +            folly/experimental/channels/detail/ChannelBridge.h +            folly/experimental/channels/detail/FunctionTraits.h +            folly/experimental/channels/detail/IntrusivePtr.h +            folly/experimental/channels/detail/MultiplexerTraits.h +            folly/experimental/channels/detail/PointerVariant.h +            folly/experimental/channels/detail/Utility.h +            folly/experimental/coro/AsyncGenerator.h +            folly/experimental/coro/AsyncPipe.h +            folly/experimental/coro/AsyncScope.h +            folly/experimental/coro/AsyncStack.h +            folly/experimental/coro/AutoCleanup-fwd.h +            folly/experimental/coro/AutoCleanup.h +            folly/experimental/coro/Baton.h +            folly/experimental/coro/BlockingWait.h +            folly/experimental/coro/BoundedQueue.h +            folly/experimental/coro/Cleanup.h +            folly/experimental/coro/Collect-inl.h +            folly/experimental/coro/Collect.h +            folly/experimental/coro/Concat-inl.h +            folly/experimental/coro/Concat.h +            folly/experimental/coro/Coroutine.h +            folly/experimental/coro/CurrentExecutor.h +            folly/experimental/coro/DetachOnCancel.h +            folly/experimental/coro/Filter-inl.h +            folly/experimental/coro/Filter.h +            folly/experimental/coro/FutureUtil.h +            folly/experimental/coro/Generator.h +            folly/experimental/coro/GmockHelpers.h +            folly/experimental/coro/GtestHelpers.h +            folly/experimental/coro/Invoke.h +            folly/experimental/coro/Merge-inl.h +            folly/experimental/coro/Merge.h +            folly/experimental/coro/Mutex.h +            folly/experimental/coro/Promise.h +            folly/experimental/coro/Result.h +            folly/experimental/coro/Retry.h +            folly/experimental/coro/RustAdaptors.h +            folly/experimental/coro/ScopeExit.h +            folly/experimental/coro/SharedLock.h +            folly/experimental/coro/SharedMutex.h +            folly/experimental/coro/SharedPromise.h +            folly/experimental/coro/Sleep-inl.h +            folly/experimental/coro/Sleep.h +            folly/experimental/coro/SmallUnboundedQueue.h +            folly/experimental/coro/Task.h +            folly/experimental/coro/TimedWait.h +            folly/experimental/coro/Timeout-inl.h +            folly/experimental/coro/Timeout.h +            folly/experimental/coro/Traits.h +            folly/experimental/coro/Transform-inl.h +            folly/experimental/coro/Transform.h +            folly/experimental/coro/UnboundedQueue.h +            folly/experimental/coro/ViaIfAsync.h +            folly/experimental/coro/WithAsyncStack.h +            folly/experimental/coro/WithCancellation.h +            folly/experimental/coro/detail/Barrier.h +            folly/experimental/coro/detail/BarrierTask.h +            folly/experimental/coro/detail/CurrentAsyncFrame.h +            folly/experimental/coro/detail/Helpers.h +            folly/experimental/coro/detail/InlineTask.h +            folly/experimental/coro/detail/Malloc.h +            folly/experimental/coro/detail/ManualLifetime.h +            folly/experimental/coro/detail/Traits.h +            folly/experimental/crypto/Blake2xb.h +            folly/experimental/crypto/LtHash.h +            folly/experimental/exception_tracer/ExceptionAbi.h +            folly/experimental/exception_tracer/ExceptionCounterLib.h +            folly/experimental/exception_tracer/ExceptionTracer.h +            folly/experimental/exception_tracer/ExceptionTracerLib.h +            folly/experimental/exception_tracer/SmartExceptionTracer.h +            folly/experimental/exception_tracer/SmartExceptionTracerSingleton.h +            folly/experimental/exception_tracer/StackTrace.h +            folly/experimental/flat_combining/FlatCombining.h +            folly/experimental/io/AsyncBase.h +            folly/experimental/io/AsyncIO.h +            folly/experimental/io/AsyncIoUringSocket.h +            folly/experimental/io/AsyncIoUringSocketFactory.h +            folly/experimental/io/Epoll.h +            folly/experimental/io/EpollBackend.h +            folly/experimental/io/EventBasePoller.h +            folly/experimental/io/FsUtil.h +            folly/experimental/io/HugePages.h +            folly/experimental/io/IoUring.h +            folly/experimental/io/IoUringBackend.h +            folly/experimental/io/IoUringBase.h +            folly/experimental/io/IoUringEvent.h +            folly/experimental/io/IoUringEventBaseLocal.h +            folly/experimental/io/IoUringProvidedBufferRing.h +            folly/experimental/io/Liburing.h +            folly/experimental/io/MuxIOThreadPoolExecutor.h +            folly/experimental/io/SimpleAsyncIO.h +            folly/experimental/observer/CoreCachedObserver.h +            folly/experimental/observer/HazptrObserver.h +            folly/experimental/observer/Observable-inl.h +            folly/experimental/observer/Observable.h +            folly/experimental/observer/Observer-inl.h +            folly/experimental/observer/Observer-pre.h +            folly/experimental/observer/Observer.h +            folly/experimental/observer/ReadMostlyTLObserver.h +            folly/experimental/observer/SimpleObservable-inl.h +            folly/experimental/observer/SimpleObservable.h +            folly/experimental/observer/WithJitter-inl.h +            folly/experimental/observer/WithJitter.h +            folly/experimental/observer/detail/Core.h +            folly/experimental/observer/detail/GraphCycleDetector.h +            folly/experimental/observer/detail/ObserverManager.h +            folly/experimental/settings/Immutables.h +            folly/experimental/settings/Settings.h +            folly/experimental/settings/Types.h +            folly/experimental/settings/detail/SettingsImpl.h +            folly/experimental/symbolizer/Dwarf.h +            folly/experimental/symbolizer/DwarfImpl.h +            folly/experimental/symbolizer/DwarfLineNumberVM.h +            folly/experimental/symbolizer/DwarfSection.h +            folly/experimental/symbolizer/DwarfUtil.h +            folly/experimental/symbolizer/Elf-inl.h +            folly/experimental/symbolizer/Elf.h +            folly/experimental/symbolizer/ElfCache.h +            folly/experimental/symbolizer/LineReader.h +            folly/experimental/symbolizer/SignalHandler.h +            folly/experimental/symbolizer/StackTrace.h +            folly/experimental/symbolizer/SymbolizePrinter.h +            folly/experimental/symbolizer/SymbolizedFrame.h +            folly/experimental/symbolizer/Symbolizer.h +            folly/experimental/symbolizer/detail/Debug.h +            folly/ext/test_ext.h +            folly/external/aor/asmdefs.h +            folly/external/farmhash/farmhash.h +            folly/external/fast-crc32/avx512_crc32c_v8s3x4.h +            folly/external/fast-crc32/neon_crc32c_v3s4x2e_v2.h +            folly/external/fast-crc32/neon_eor3_crc32_v9s3x2e_s3.h +            folly/external/fast-crc32/neon_eor3_crc32c_v8s2x4_s3.h +            folly/external/fast-crc32/sse_crc32c_v8s3x3.h +            folly/external/nvidia/detail/RangeSve2.h +            folly/external/nvidia/hash/detail/Crc32cCombineDetail.h +            folly/external/rapidhash/rapidhash.h +            folly/fibers/AddTasks-inl.h +            folly/fibers/AddTasks.h +            folly/fibers/AtomicBatchDispatcher-inl.h +            folly/fibers/AtomicBatchDispatcher.h +            folly/fibers/BatchDispatcher.h +            folly/fibers/BatchSemaphore.h +            folly/fibers/Baton-inl.h +            folly/fibers/Baton.h +            folly/fibers/BoostContextCompatibility.h +            folly/fibers/CallOnce.h +            folly/fibers/EventBaseLoopController-inl.h +            folly/fibers/EventBaseLoopController.h +            folly/fibers/ExecutorBasedLoopController.h +            folly/fibers/ExecutorLoopController-inl.h +            folly/fibers/ExecutorLoopController.h +            folly/fibers/Fiber-inl.h +            folly/fibers/Fiber.h +            folly/fibers/FiberManager-inl.h +            folly/fibers/FiberManager.h +            folly/fibers/FiberManagerInternal-inl.h +            folly/fibers/FiberManagerInternal.h +            folly/fibers/FiberManagerMap-inl.h +            folly/fibers/FiberManagerMap.h +            folly/fibers/ForEach-inl.h +            folly/fibers/ForEach.h +            folly/fibers/GenericBaton.h +            folly/fibers/GuardPageAllocator.h +            folly/fibers/LoopController.h +            folly/fibers/Promise-inl.h +            folly/fibers/Promise.h +            folly/fibers/Semaphore.h +            folly/fibers/SemaphoreBase.h +            folly/fibers/SimpleLoopController.h +            folly/fibers/TimedMutex-inl.h +            folly/fibers/TimedMutex.h +            folly/fibers/WhenN-inl.h +            folly/fibers/WhenN.h +            folly/fibers/async/Async.h +            folly/fibers/async/AsyncStack.h +            folly/fibers/async/Baton.h +            folly/fibers/async/Collect-inl.h +            folly/fibers/async/Collect.h +            folly/fibers/async/FiberManager.h +            folly/fibers/async/Future.h +            folly/fibers/async/Promise.h +            folly/fibers/async/Task.h +            folly/fibers/async/WaitUtils.h +            folly/fibers/detail/AtomicBatchDispatcher.h +            folly/fibers/traits.h +            folly/functional/ApplyTuple.h +            folly/functional/Invoke.h +            folly/functional/Partial.h +            folly/functional/protocol.h +            folly/functional/traits.h +            folly/futures/Barrier.h +            folly/futures/Cleanup.h +            folly/futures/Future-inl.h +            folly/futures/Future-pre.h +            folly/futures/Future.h +            folly/futures/FutureSplitter.h +            folly/futures/HeapTimekeeper.h +            folly/futures/ManualTimekeeper.h +            folly/futures/Portability.h +            folly/futures/Promise-inl.h +            folly/futures/Promise.h +            folly/futures/Retrying.h +            folly/futures/SharedPromise-inl.h +            folly/futures/SharedPromise.h +            folly/futures/ThreadWheelTimekeeper.h +            folly/futures/WTCallback.h +            folly/futures/detail/Core.h +            folly/futures/detail/Types.h +            folly/gen/Base-inl.h +            folly/gen/Base.h +            folly/gen/Combine-inl.h +            folly/gen/Combine.h +            folly/gen/Core-inl.h +            folly/gen/Core.h +            folly/gen/File-inl.h +            folly/gen/File.h +            folly/gen/IStream.h +            folly/gen/Parallel-inl.h +            folly/gen/Parallel.h +            folly/gen/ParallelMap-inl.h +            folly/gen/ParallelMap.h +            folly/gen/String-inl.h +            folly/gen/String.h +            folly/hash/Checksum.h +            folly/hash/FarmHash.h +            folly/hash/Hash.h +            folly/hash/MurmurHash.h +            folly/hash/SpookyHashV1.h +            folly/hash/SpookyHashV2.h +            folly/hash/detail/ChecksumDetail.h +            folly/hash/rapidhash.h +            folly/hash/traits.h +            folly/init/Init.h +            folly/init/Phase.h +            folly/io/Cursor-inl.h +            folly/io/Cursor.h +            folly/io/FsUtil.h +            folly/io/GlobalShutdownSocketSet.h +            folly/io/HugePages.h +            folly/io/IOBuf.h +            folly/io/IOBufIovecBuilder.h +            folly/io/IOBufQueue.h +            folly/io/RecordIO-inl.h +            folly/io/RecordIO.h +            folly/io/ShutdownSocketSet.h +            folly/io/SocketOptionMap.h +            folly/io/SocketOptionValue.h +            folly/io/TypedIOBuf.h +            folly/io/async/AsyncBase.h +            folly/io/async/AsyncIO.h +            folly/io/async/AsyncIoUringSocket.h +            folly/io/async/AsyncIoUringSocketFactory.h +            folly/io/async/AsyncPipe.h +            folly/io/async/AsyncSSLSocket.h +            folly/io/async/AsyncServerSocket.h +            folly/io/async/AsyncSignalHandler.h +            folly/io/async/AsyncSocket.h +            folly/io/async/AsyncSocketBase.h +            folly/io/async/AsyncSocketException.h +            folly/io/async/AsyncSocketTransport.h +            folly/io/async/AsyncTimeout.h +            folly/io/async/AsyncTransport.h +            folly/io/async/AsyncTransportCertificate.h +            folly/io/async/AsyncUDPServerSocket.h +            folly/io/async/AsyncUDPSocket.h +            folly/io/async/AtomicNotificationQueue-inl.h +            folly/io/async/AtomicNotificationQueue.h +            folly/io/async/CertificateIdentityVerifier.h +            folly/io/async/DecoratedAsyncTransportWrapper.h +            folly/io/async/DelayedDestruction.h +            folly/io/async/DelayedDestructionBase.h +            folly/io/async/DestructorCheck.h +            folly/io/async/Epoll.h +            folly/io/async/EpollBackend.h +            folly/io/async/EventBase.h +            folly/io/async/EventBaseAtomicNotificationQueue-inl.h +            folly/io/async/EventBaseAtomicNotificationQueue.h +            folly/io/async/EventBaseBackendBase.h +            folly/io/async/EventBaseLocal.h +            folly/io/async/EventBaseManager.h +            folly/io/async/EventBasePoller.h +            folly/io/async/EventBaseThread.h +            folly/io/async/EventHandler.h +            folly/io/async/EventUtil.h +            folly/io/async/HHWheelTimer-fwd.h +            folly/io/async/HHWheelTimer.h +            folly/io/async/IoUring.h +            folly/io/async/IoUringBackend.h +            folly/io/async/IoUringBase.h +            folly/io/async/IoUringEvent.h +            folly/io/async/IoUringEventBaseLocal.h +            folly/io/async/IoUringProvidedBufferRing.h +            folly/io/async/IoUringZeroCopyBufferPool.h +            folly/io/async/Liburing.h +            folly/io/async/MuxIOThreadPoolExecutor.h +            folly/io/async/NotificationQueue.h +            folly/io/async/PasswordInFile.h +            folly/io/async/Request.h +            folly/io/async/SSLContext.h +            folly/io/async/SSLOptions.h +            folly/io/async/STTimerFDTimeoutManager.h +            folly/io/async/ScopedEventBaseThread.h +            folly/io/async/SimpleAsyncIO.h +            folly/io/async/TerminateCancellationToken.h +            folly/io/async/TimeoutManager.h +            folly/io/async/TimerFD.h +            folly/io/async/TimerFDTimeoutManager.h +            folly/io/async/VirtualEventBase.h +            folly/io/async/WriteChainAsyncTransportWrapper.h +            folly/io/async/WriteFlags.h +            folly/io/async/fdsock/AsyncFdSocket.h +            folly/io/async/fdsock/SocketFds.h +            folly/io/async/observer/AsyncSocketObserverContainer.h +            folly/io/async/observer/AsyncSocketObserverInterface.h +            folly/io/async/ssl/BasicTransportCertificate.h +            folly/io/async/ssl/OpenSSLTransportCertificate.h +            folly/io/async/ssl/OpenSSLUtils.h +            folly/io/async/ssl/SSLErrors.h +            folly/io/async/ssl/TLSDefinitions.h +            folly/io/coro/ServerSocket.h +            folly/io/coro/Transport.h +            folly/io/coro/TransportCallbackBase.h +            folly/io/coro/TransportCallbacks.h +            folly/json/DynamicConverter.h +            folly/json/DynamicParser-inl.h +            folly/json/DynamicParser.h +            folly/json/JSONSchema.h +            folly/json/JsonMockUtil.h +            folly/json/JsonTestUtil.h +            folly/json/bser/Bser.h +            folly/json/dynamic-inl.h +            folly/json/dynamic.h +            folly/json/json.h +            folly/json/json_patch.h +            folly/json/json_pointer.h +            folly/lang/Access.h +            folly/lang/Align.h +            folly/lang/Aligned.h +            folly/lang/Assume.h +            folly/lang/Badge.h +            folly/lang/Bindings.h +            folly/lang/Bits.h +            folly/lang/BitsClass.h +            folly/lang/Builtin.h +            folly/lang/CArray.h +            folly/lang/CString.h +            folly/lang/Cast.h +            folly/lang/CheckedMath.h +            folly/lang/CustomizationPoint.h +            folly/lang/Exception.h +            folly/lang/Extern.h +            folly/lang/Hint-inl.h +            folly/lang/Hint.h +            folly/lang/Keep.h +            folly/lang/New.h +            folly/lang/Ordering.h +            folly/lang/Pretty.h +            folly/lang/PropagateConst.h +            folly/lang/RValueReferenceWrapper.h +            folly/lang/SafeAlias-fwd.h +            folly/lang/SafeAssert.h +            folly/lang/StaticConst.h +            folly/lang/Switch.h +            folly/lang/Thunk.h +            folly/lang/ToAscii.h +            folly/lang/TypeInfo.h +            folly/lang/UncaughtExceptions.h +            folly/lang/VectorTraits.h +            folly/lang/named/Bindings.h +            folly/logging/AsyncFileWriter.h +            folly/logging/AsyncLogWriter.h +            folly/logging/AutoTimer.h +            folly/logging/BridgeFromGoogleLogging.h +            folly/logging/CustomLogFormatter.h +            folly/logging/FileHandlerFactory.h +            folly/logging/FileWriterFactory.h +            folly/logging/GlogStyleFormatter.h +            folly/logging/ImmediateFileWriter.h +            folly/logging/Init.h +            folly/logging/LogCategory.h +            folly/logging/LogCategoryConfig.h +            folly/logging/LogConfig.h +            folly/logging/LogConfigParser.h +            folly/logging/LogFormatter.h +            folly/logging/LogHandler.h +            folly/logging/LogHandlerConfig.h +            folly/logging/LogHandlerFactory.h +            folly/logging/LogLevel.h +            folly/logging/LogMessage.h +            folly/logging/LogName.h +            folly/logging/LogStream.h +            folly/logging/LogStreamProcessor.h +            folly/logging/LogWriter.h +            folly/logging/Logger.h +            folly/logging/LoggerDB.h +            folly/logging/ObjectToString.h +            folly/logging/RateLimiter.h +            folly/logging/StandardLogHandler.h +            folly/logging/StandardLogHandlerFactory.h +            folly/logging/StreamHandlerFactory.h +            folly/logging/xlog.h +            folly/memory/Arena-inl.h +            folly/memory/Arena.h +            folly/memory/JemallocHugePageAllocator.h +            folly/memory/JemallocNodumpAllocator.h +            folly/memory/MallctlHelper.h +            folly/memory/Malloc.h +            folly/memory/MemoryResource.h +            folly/memory/ReentrantAllocator.h +            folly/memory/SanitizeAddress.h +            folly/memory/SanitizeLeak.h +            folly/memory/ThreadCachedArena.h +            folly/memory/UninitializedMemoryHacks.h +            folly/memory/detail/MallocImpl.h +            folly/memory/not_null-inl.h +            folly/memory/not_null.h +            folly/memory/shared_from_this_ptr.h +            folly/net/NetOps.h +            folly/net/NetOpsDispatcher.h +            folly/net/NetworkSocket.h +            folly/net/TcpInfo.h +            folly/net/TcpInfoDispatcher.h +            folly/net/TcpInfoTypes.h +            folly/net/detail/SocketFileDescriptorMap.h +            folly/observer/CoreCachedObserver.h +            folly/observer/HazptrObserver.h +            folly/observer/Observable-inl.h +            folly/observer/Observable.h +            folly/observer/Observer-inl.h +            folly/observer/Observer-pre.h +            folly/observer/Observer.h +            folly/observer/ReadMostlyTLObserver.h +            folly/observer/SimpleObservable-inl.h +            folly/observer/SimpleObservable.h +            folly/observer/WithJitter-inl.h +            folly/observer/WithJitter.h +            folly/observer/detail/Core.h +            folly/observer/detail/GraphCycleDetector.h +            folly/observer/detail/ObserverManager.h +            folly/poly/Nullable.h +            folly/poly/Regular.h +            folly/portability/Asm.h +            folly/portability/Atomic.h +            folly/portability/Builtins.h +            folly/portability/Config.h +            folly/portability/Constexpr.h +            folly/portability/Dirent.h +            folly/portability/Event.h +            folly/portability/Fcntl.h +            folly/portability/Filesystem.h +            folly/portability/FmtCompile.h +            folly/portability/GFlags.h +            folly/portability/GMock.h +            folly/portability/GTest.h +            folly/portability/GTestProd.h +            folly/portability/IOVec.h +            folly/portability/Libgen.h +            folly/portability/Libunwind.h +            folly/portability/Malloc.h +            folly/portability/Math.h +            folly/portability/Memory.h +            folly/portability/OpenSSL.h +            folly/portability/PThread.h +            folly/portability/Sched.h +            folly/portability/Sockets.h +            folly/portability/SourceLocation.h +            folly/portability/Stdio.h +            folly/portability/Stdlib.h +            folly/portability/String.h +            folly/portability/SysFile.h +            folly/portability/SysMembarrier.h +            folly/portability/SysMman.h +            folly/portability/SysResource.h +            folly/portability/SysStat.h +            folly/portability/SysSyscall.h +            folly/portability/SysTime.h +            folly/portability/SysTypes.h +            folly/portability/SysUio.h +            folly/portability/Syslog.h +            folly/portability/Time.h +            folly/portability/Unistd.h +            folly/portability/Windows.h +            folly/portability/openat2.h +            folly/python/AsyncioExecutor.h +            folly/python/Weak.h +            folly/python/async_generator.h +            folly/python/coro.h +            folly/python/error.h +            folly/python/executor.h +            folly/python/futures.h +            folly/python/import.h +            folly/python/iobuf.h +            folly/random/xoshiro256pp.h +            folly/result/gtest_helpers.h +            folly/result/result.h +            folly/result/try.h +            folly/settings/CommandLineParser.h +            folly/settings/Immutables.h +            folly/settings/Observer.h +            folly/settings/Settings.h +            folly/settings/SettingsAccessorProxy.h +            folly/settings/Types.h +            folly/settings/detail/SettingsImpl.h +            folly/ssl/OpenSSLCertUtils.h +            folly/ssl/OpenSSLHash.h +            folly/ssl/OpenSSLKeyUtils.h +            folly/ssl/OpenSSLPtrTypes.h +            folly/ssl/OpenSSLTicketHandler.h +            folly/ssl/OpenSSLVersionFinder.h +            folly/ssl/PasswordCollector.h +            folly/ssl/SSLSession.h +            folly/ssl/SSLSessionManager.h +            folly/ssl/detail/OpenSSLSession.h +            folly/stats/BucketedTimeSeries-inl.h +            folly/stats/BucketedTimeSeries.h +            folly/stats/DigestBuilder-inl.h +            folly/stats/DigestBuilder.h +            folly/stats/Histogram-inl.h +            folly/stats/Histogram.h +            folly/stats/MultiLevelTimeSeries-inl.h +            folly/stats/MultiLevelTimeSeries.h +            folly/stats/QuantileEstimator-inl.h +            folly/stats/QuantileEstimator.h +            folly/stats/QuantileHistogram-inl.h +            folly/stats/QuantileHistogram.h +            folly/stats/StreamingStats.h +            folly/stats/TDigest.h +            folly/stats/TimeseriesHistogram-inl.h +            folly/stats/TimeseriesHistogram.h +            folly/stats/detail/Bucket.h +            folly/stats/detail/BufferedStat-inl.h +            folly/stats/detail/BufferedStat.h +            folly/stats/detail/DoubleRadixSort.h +            folly/stats/detail/SlidingWindow-inl.h +            folly/stats/detail/SlidingWindow.h +            folly/synchronization/AsymmetricThreadFence.h +            folly/synchronization/AtomicNotification-inl.h +            folly/synchronization/AtomicNotification.h +            folly/synchronization/AtomicRef.h +            folly/synchronization/AtomicStruct.h +            folly/synchronization/AtomicUtil-inl.h +            folly/synchronization/AtomicUtil.h +            folly/synchronization/Baton.h +            folly/synchronization/CallOnce.h +            folly/synchronization/DelayedInit.h +            folly/synchronization/DistributedMutex-inl.h +            folly/synchronization/DistributedMutex.h +            folly/synchronization/EventCount.h +            folly/synchronization/FlatCombining.h +            folly/synchronization/Hazptr-fwd.h +            folly/synchronization/Hazptr.h +            folly/synchronization/HazptrDomain.h +            folly/synchronization/HazptrHolder.h +            folly/synchronization/HazptrObj.h +            folly/synchronization/HazptrObjLinked.h +            folly/synchronization/HazptrRec.h +            folly/synchronization/HazptrThrLocal.h +            folly/synchronization/HazptrThreadPoolExecutor.h +            folly/synchronization/Latch.h +            folly/synchronization/LifoSem.h +            folly/synchronization/Lock.h +            folly/synchronization/MicroSpinLock.h +            folly/synchronization/NativeSemaphore.h +            folly/synchronization/ParkingLot.h +            folly/synchronization/PicoSpinLock.h +            folly/synchronization/RWSpinLock.h +            folly/synchronization/Rcu.h +            folly/synchronization/RelaxedAtomic.h +            folly/synchronization/SanitizeThread.h +            folly/synchronization/SaturatingSemaphore.h +            folly/synchronization/SmallLocks.h +            folly/synchronization/ThrottledLifoSem.h +            folly/synchronization/WaitOptions.h +            folly/synchronization/detail/AtomicUtils.h +            folly/synchronization/detail/Hardware.h +            folly/synchronization/detail/HazptrUtils.h +            folly/synchronization/detail/InlineFunctionRef.h +            folly/synchronization/detail/Sleeper.h +            folly/synchronization/detail/Spin.h +            folly/synchronization/detail/ThreadCachedLists.h +            folly/synchronization/detail/ThreadCachedReaders.h +            folly/synchronization/detail/ThreadCachedTag.h +            folly/synchronization/example/HazptrLockFreeLIFO.h +            folly/synchronization/example/HazptrSWMRSet.h +            folly/synchronization/example/HazptrWideCAS.h +            folly/system/AtFork.h +            folly/system/AuxVector.h +            folly/system/EnvUtil.h +            folly/system/HardwareConcurrency.h +            folly/system/MemoryMapping.h +            folly/system/Pid.h +            folly/system/Shell.h +            folly/system/ThreadId.h +            folly/system/ThreadName.h +            folly/testing/TestUtil.h +            folly/tracing/AsyncStack-inl.h +            folly/tracing/AsyncStack.h +            folly/tracing/ScopedTraceSection.h +            folly/tracing/StaticTracepoint-ELF.h +            folly/tracing/StaticTracepoint.h +            folly/container/test/F14TestUtil.h +            folly/container/test/TrackingTypes.h +            folly/io/async/test/AsyncSSLSocketTest.h +            folly/io/async/test/AsyncSocketTest.h +            folly/io/async/test/AsyncSocketTest2.h +            folly/io/async/test/BlockingSocket.h +            folly/io/async/test/CallbackStateEnum.h +            folly/io/async/test/ConnCallback.h +            folly/io/async/test/MockAsyncSocket.h +            folly/io/async/test/MockAsyncServerSocket.h +            folly/io/async/test/MockAsyncSSLSocket.h +            folly/io/async/test/MockAsyncTransport.h +            folly/io/async/test/MockAsyncUDPSocket.h +            folly/io/async/test/MockTimeoutManager.h +            folly/io/async/test/ScopedBoundPort.h +            folly/io/async/test/SocketPair.h +            folly/io/async/test/TFOUtil.h +            folly/io/async/test/TestSSLServer.h +            folly/io/async/test/TimeUtil.h +            folly/io/async/test/UndelayedDestruction.h +            folly/io/async/test/Util.h +            folly/synchronization/test/Semaphore.h +            folly/test/DeterministicSchedule.h +            folly/test/TestUtils.h++        install-includes:+            folly/folly-config.h+            fast_float/ascii_number.h+            fast_float/float_common.h+            fast_float/constexpr_feature_detect.h+            fast_float/fast_table.h+            fast_float/decimal_to_binary.h+            fast_float/digit_comparison.h+            fast_float/bigint.h+            fast_float/fast_float.h+            fast_float/parse_number.h
+ folly/_build/folly/folly-config.h view
@@ -0,0 +1,87 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#ifdef __APPLE__+#include <TargetConditionals.h> // @manual+#endif++#if !defined(FOLLY_MOBILE)+#if defined(__ANDROID__) || \+    (defined(__APPLE__) &&  \+     (TARGET_IPHONE_SIMULATOR || TARGET_OS_SIMULATOR || TARGET_OS_IPHONE))+#define FOLLY_MOBILE 1+#else+#define FOLLY_MOBILE 0+#endif+#endif // FOLLY_MOBILE++#define FOLLY_HAVE_PTHREAD 1+#define FOLLY_HAVE_PTHREAD_ATFORK 1++#define FOLLY_HAVE_LIBGFLAGS 1++#define FOLLY_HAVE_LIBGLOG 1++#define FOLLY_USE_JEMALLOC 1++#if __has_include(<features.h>)+#include <features.h>+#endif++#define FOLLY_HAVE_ACCEPT4 1+#define FOLLY_HAVE_GETRANDOM 1+#define FOLLY_HAVE_PREADV 1+#define FOLLY_HAVE_PWRITEV 1+#define FOLLY_HAVE_CLOCK_GETTIME 1+#define FOLLY_HAVE_PIPE2 1++#define FOLLY_HAVE_IFUNC 1+#define FOLLY_HAVE_UNALIGNED_ACCESS 1+#define FOLLY_HAVE_VLA 1+#define FOLLY_HAVE_WEAK_SYMBOLS 1+#define FOLLY_HAVE_LINUX_VDSO 1+#define FOLLY_HAVE_MALLOC_USABLE_SIZE 1+/* #undef FOLLY_HAVE_INT128_T */+#define FOLLY_HAVE_WCHAR_SUPPORT 1+#define FOLLY_HAVE_EXTRANDOM_SFMT19937 1+#define HAVE_VSNPRINTF_ERRORS 1++#define FOLLY_HAVE_LIBUNWIND 1+/* #undef FOLLY_HAVE_DWARF */+#define FOLLY_HAVE_ELF 1+#define FOLLY_HAVE_SWAPCONTEXT 1+#define FOLLY_HAVE_BACKTRACE 1+#define FOLLY_USE_SYMBOLIZER 1+#define FOLLY_DEMANGLE_MAX_SYMBOL_SIZE 1024++#define FOLLY_HAVE_SHADOW_LOCAL_WARNINGS 1++#define FOLLY_HAVE_LIBLZ4 1+#define FOLLY_HAVE_LIBLZMA 1+#define FOLLY_HAVE_LIBSNAPPY 1+#define FOLLY_HAVE_LIBZ 1+#define FOLLY_HAVE_LIBZSTD 1+#define FOLLY_HAVE_LIBBZ2 1++#define FOLLY_LIBRARY_SANITIZE_ADDRESS 0++/* #undef FOLLY_SUPPORT_SHARED_LIBRARY */++#define FOLLY_HAVE_LIBRT 0++#define FOLLY_HAVE_VSOCK 0
+ folly/folly/AtomicHashArray-inl.h view
@@ -0,0 +1,547 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#ifndef FOLLY_ATOMICHASHARRAY_H_+#error "This should only be included by AtomicHashArray.h"+#endif++#include <type_traits>++#include <folly/detail/AtomicHashUtils.h>+#include <folly/detail/Iterators.h>+#include <folly/lang/Bits.h>+#include <folly/lang/Exception.h>++namespace folly {++// AtomicHashArray private constructor --+template <+    class KeyT,+    class ValueT,+    class HashFcn,+    class EqualFcn,+    class Allocator,+    class ProbeFcn,+    class KeyConvertFcn>+AtomicHashArray<+    KeyT,+    ValueT,+    HashFcn,+    EqualFcn,+    Allocator,+    ProbeFcn,+    KeyConvertFcn>::+    AtomicHashArray(+        size_t capacity,+        KeyT emptyKey,+        KeyT lockedKey,+        KeyT erasedKey,+        double _maxLoadFactor,+        uint32_t cacheSize)+    : capacity_(capacity),+      maxEntries_(size_t(_maxLoadFactor * capacity_ + 0.5)),+      kEmptyKey_(emptyKey),+      kLockedKey_(lockedKey),+      kErasedKey_(erasedKey),+      kAnchorMask_(nextPowTwo(capacity_) - 1),+      numEntries_(0, cacheSize),+      numPendingEntries_(0, cacheSize),+      isFull_(0),+      numErases_(0) {+  if (capacity == 0) {+    throw_exception<std::invalid_argument>("capacity");+  }+}++/*+ * findInternal --+ *+ *   Sets ret.second to value found and ret.index to index+ *   of key and returns true, or if key does not exist returns false and+ *   ret.index is set to capacity_.+ */+template <+    class KeyT,+    class ValueT,+    class HashFcn,+    class EqualFcn,+    class Allocator,+    class ProbeFcn,+    class KeyConvertFcn>+template <class LookupKeyT, class LookupHashFcn, class LookupEqualFcn>+typename AtomicHashArray<+    KeyT,+    ValueT,+    HashFcn,+    EqualFcn,+    Allocator,+    ProbeFcn,+    KeyConvertFcn>::SimpleRetT+AtomicHashArray<+    KeyT,+    ValueT,+    HashFcn,+    EqualFcn,+    Allocator,+    ProbeFcn,+    KeyConvertFcn>::findInternal(const LookupKeyT key_in) {+  checkLegalKeyIfKey<LookupKeyT>(key_in);++  for (size_t idx = keyToAnchorIdx<LookupKeyT, LookupHashFcn>(key_in),+              numProbes = 0;+       ;+       idx = ProbeFcn()(idx, numProbes, capacity_)) {+    const KeyT key = acquireLoadKey(cells_[idx]);+    if (FOLLY_LIKELY(LookupEqualFcn()(key, key_in))) {+      return SimpleRetT(idx, true);+    }+    if (FOLLY_UNLIKELY(key == kEmptyKey_)) {+      // if we hit an empty element, this key does not exist+      return SimpleRetT(capacity_, false);+    }+    // NOTE: the way we count numProbes must be same in find(), insert(),+    // and erase(). Otherwise it may break probing.+    ++numProbes;+    if (FOLLY_UNLIKELY(numProbes >= capacity_)) {+      // probed every cell...fail+      return SimpleRetT(capacity_, false);+    }+  }+}++/*+ * insertInternal --+ *+ *   Returns false on failure due to key collision or full.+ *   Also sets ret.index to the index of the key.  If the map is full, sets+ *   ret.index = capacity_.  Also sets ret.second to cell value, thus if insert+ *   successful this will be what we just inserted, if there is a key collision+ *   this will be the previously inserted value, and if the map is full it is+ *   default.+ */+template <+    class KeyT,+    class ValueT,+    class HashFcn,+    class EqualFcn,+    class Allocator,+    class ProbeFcn,+    class KeyConvertFcn>+template <+    typename LookupKeyT,+    typename LookupHashFcn,+    typename LookupEqualFcn,+    typename LookupKeyToKeyFcn,+    typename... ArgTs>+typename AtomicHashArray<+    KeyT,+    ValueT,+    HashFcn,+    EqualFcn,+    Allocator,+    ProbeFcn,+    KeyConvertFcn>::SimpleRetT+AtomicHashArray<+    KeyT,+    ValueT,+    HashFcn,+    EqualFcn,+    Allocator,+    ProbeFcn,+    KeyConvertFcn>::insertInternal(LookupKeyT key_in, ArgTs&&... vCtorArgs) {+  const short NO_NEW_INSERTS = 1;+  const short NO_PENDING_INSERTS = 2;+  checkLegalKeyIfKey<LookupKeyT>(key_in);++  size_t idx = keyToAnchorIdx<LookupKeyT, LookupHashFcn>(key_in);+  size_t numProbes = 0;+  for (;;) {+    DCHECK_LT(idx, capacity_);+    value_type* cell = &cells_[idx];+    if (relaxedLoadKey(*cell) == kEmptyKey_) {+      // NOTE: isFull_ is set based on numEntries_.readFast(), so it's+      // possible to insert more than maxEntries_ entries. However, it's not+      // possible to insert past capacity_.+      ++numPendingEntries_;+      if (isFull_.load(std::memory_order_acquire)) {+        --numPendingEntries_;++        // Before deciding whether this insert succeeded, this thread needs to+        // wait until no other thread can add a new entry.++        // Correctness assumes isFull_ is true at this point. If+        // another thread now does ++numPendingEntries_, we expect it+        // to pass the isFull_.load() test above. (It shouldn't insert+        // a new entry.)+        detail::atomic_hash_spin_wait([&] {+          return (isFull_.load(std::memory_order_acquire) !=+                  NO_PENDING_INSERTS) &&+              (numPendingEntries_.readFull() != 0);+        });+        isFull_.store(NO_PENDING_INSERTS, std::memory_order_release);++        if (relaxedLoadKey(*cell) == kEmptyKey_) {+          // Don't insert past max load factor+          return SimpleRetT(capacity_, false);+        }+      } else {+        // An unallocated cell. Try once to lock it. If we succeed, insert here.+        // If we fail, fall through to comparison below; maybe the insert that+        // just beat us was for this very key....+        if (tryLockCell(cell)) {+          KeyT key_new;+          // Write the value - done before unlocking+          try {+            key_new = LookupKeyToKeyFcn()(key_in);+            typedef+                typename std::remove_const<LookupKeyT>::type LookupKeyTNoConst;+            constexpr bool kAlreadyChecked =+                std::is_same<KeyT, LookupKeyTNoConst>::value;+            if (!kAlreadyChecked) {+              checkLegalKeyIfKey(key_new);+            }+            DCHECK(relaxedLoadKey(*cell) == kLockedKey_);+            // A const mapped_type is only constant once constructed, so cast+            // away any const for the placement new here.+            using mapped = typename std::remove_const<mapped_type>::type;+            new (const_cast<mapped*>(&cell->second))+                ValueT(std::forward<ArgTs>(vCtorArgs)...);+            unlockCell(cell, key_new); // Sets the new key+          } catch (...) {+            // Transition back to empty key---requires handling+            // locked->empty below.+            unlockCell(cell, kEmptyKey_);+            --numPendingEntries_;+            throw;+          }+          // An erase() can race here and delete right after our insertion+          // Direct comparison rather than EqualFcn ok here+          // (we just inserted it)+          DCHECK(+              relaxedLoadKey(*cell) == key_new ||+              relaxedLoadKey(*cell) == kErasedKey_);+          --numPendingEntries_;+          ++numEntries_; // This is a thread cached atomic increment :)+          if (numEntries_.readFast() >= maxEntries_) {+            isFull_.store(NO_NEW_INSERTS, std::memory_order_relaxed);+          }+          return SimpleRetT(idx, true);+        }+        --numPendingEntries_;+      }+    }+    DCHECK(relaxedLoadKey(*cell) != kEmptyKey_);+    if (kLockedKey_ == acquireLoadKey(*cell)) {+      detail::atomic_hash_spin_wait([&] {+        return kLockedKey_ == acquireLoadKey(*cell);+      });+    }++    const KeyT thisKey = acquireLoadKey(*cell);+    if (LookupEqualFcn()(thisKey, key_in)) {+      // Found an existing entry for our key, but we don't overwrite the+      // previous value.+      return SimpleRetT(idx, false);+    } else if (thisKey == kEmptyKey_ || thisKey == kLockedKey_) {+      // We need to try again (i.e., don't increment numProbes or+      // advance idx): this case can happen if the constructor for+      // ValueT threw for this very cell (the rethrow block above).+      continue;+    }++    // NOTE: the way we count numProbes must be same in find(),+    // insert(), and erase(). Otherwise it may break probing.+    ++numProbes;+    if (FOLLY_UNLIKELY(numProbes >= capacity_)) {+      // probed every cell...fail+      return SimpleRetT(capacity_, false);+    }++    idx = ProbeFcn()(idx, numProbes, capacity_);+  }+}++/*+ * erase --+ *+ *   This will attempt to erase the given key key_in if the key is found. It+ *   returns 1 iff the key was located and marked as erased, and 0 otherwise.+ *+ *   Memory is not freed or reclaimed by erase, i.e. the cell containing the+ *   erased key will never be reused. If there's an associated value, we won't+ *   touch it either.+ */+template <+    class KeyT,+    class ValueT,+    class HashFcn,+    class EqualFcn,+    class Allocator,+    class ProbeFcn,+    class KeyConvertFcn>+size_t AtomicHashArray<+    KeyT,+    ValueT,+    HashFcn,+    EqualFcn,+    Allocator,+    ProbeFcn,+    KeyConvertFcn>::erase(KeyT key_in) {+  CHECK_NE(key_in, kEmptyKey_);+  CHECK_NE(key_in, kLockedKey_);+  CHECK_NE(key_in, kErasedKey_);++  for (size_t idx = keyToAnchorIdx(key_in), numProbes = 0;;+       idx = ProbeFcn()(idx, numProbes, capacity_)) {+    DCHECK_LT(idx, capacity_);+    value_type* cell = &cells_[idx];+    KeyT currentKey = acquireLoadKey(*cell);+    if (currentKey == kEmptyKey_ || currentKey == kLockedKey_) {+      // If we hit an empty (or locked) element, this key does not exist. This+      // is similar to how it's handled in find().+      return 0;+    }+    if (EqualFcn()(currentKey, key_in)) {+      // Found an existing entry for our key, attempt to mark it erased.+      // Some other thread may have erased our key, but this is ok.+      KeyT expect = currentKey;+      if (cellKeyPtr(*cell)->compare_exchange_strong(expect, kErasedKey_)) {+        numErases_.fetch_add(1, std::memory_order_relaxed);++        // Even if there's a value in the cell, we won't delete (or even+        // default construct) it because some other thread may be accessing it.+        // Locking it meanwhile won't work either since another thread may be+        // holding a pointer to it.++        // We found the key and successfully erased it.+        return 1;+      }+      // If another thread succeeds in erasing our key, we'll stop our search.+      return 0;+    }++    // NOTE: the way we count numProbes must be same in find(), insert(),+    // and erase(). Otherwise it may break probing.+    ++numProbes;+    if (FOLLY_UNLIKELY(numProbes >= capacity_)) {+      // probed every cell...fail+      return 0;+    }+  }+}++template <+    class KeyT,+    class ValueT,+    class HashFcn,+    class EqualFcn,+    class Allocator,+    class ProbeFcn,+    class KeyConvertFcn>+typename AtomicHashArray<+    KeyT,+    ValueT,+    HashFcn,+    EqualFcn,+    Allocator,+    ProbeFcn,+    KeyConvertFcn>::SmartPtr+AtomicHashArray<+    KeyT,+    ValueT,+    HashFcn,+    EqualFcn,+    Allocator,+    ProbeFcn,+    KeyConvertFcn>::create(size_t maxSize, const Config& c) {+  CHECK_LE(c.maxLoadFactor, 1.0);+  CHECK_GT(c.maxLoadFactor, 0.0);+  CHECK_NE(c.emptyKey, c.lockedKey);+  size_t capacity = size_t(maxSize / c.maxLoadFactor);+  size_t sz = sizeof(AtomicHashArray) + sizeof(value_type) * capacity;++  auto const mem = Allocator().allocate(sz);+  try {+    new (mem) AtomicHashArray(+        capacity,+        c.emptyKey,+        c.lockedKey,+        c.erasedKey,+        c.maxLoadFactor,+        c.entryCountThreadCacheSize);+  } catch (...) {+    Allocator().deallocate(mem, sz);+    throw;+  }++  SmartPtr map(static_cast<AtomicHashArray*>((void*)mem));++  /*+   * Mark all cells as empty.+   *+   * Note: we're bending the rules a little here accessing the key+   * element in our cells even though the cell object has not been+   * constructed, and casting them to atomic objects (see cellKeyPtr).+   * (Also, in fact we never actually invoke the value_type+   * constructor.)  This is in order to avoid needing to default+   * construct a bunch of value_type when we first start up: if you+   * have an expensive default constructor for the value type this can+   * noticeably speed construction time for an AHA.+   */+  FOR_EACH_RANGE (i, 0, map->capacity_) {+    cellKeyPtr(map->cells_[i])+        ->store(map->kEmptyKey_, std::memory_order_relaxed);+  }+  return map;+}++template <+    class KeyT,+    class ValueT,+    class HashFcn,+    class EqualFcn,+    class Allocator,+    class ProbeFcn,+    class KeyConvertFcn>+void AtomicHashArray<+    KeyT,+    ValueT,+    HashFcn,+    EqualFcn,+    Allocator,+    ProbeFcn,+    KeyConvertFcn>::destroy(AtomicHashArray* p) {+  assert(p);++  size_t sz = sizeof(AtomicHashArray) + sizeof(value_type) * p->capacity_;++  FOR_EACH_RANGE (i, 0, p->capacity_) {+    if (p->cells_[i].first != p->kEmptyKey_) {+      p->cells_[i].~value_type();+    }+  }+  p->~AtomicHashArray();++  Allocator().deallocate((char*)p, sz);+}++// clear -- clears all keys and values in the map and resets all counters+template <+    class KeyT,+    class ValueT,+    class HashFcn,+    class EqualFcn,+    class Allocator,+    class ProbeFcn,+    class KeyConvertFcn>+void AtomicHashArray<+    KeyT,+    ValueT,+    HashFcn,+    EqualFcn,+    Allocator,+    ProbeFcn,+    KeyConvertFcn>::clear() {+  FOR_EACH_RANGE (i, 0, capacity_) {+    if (cells_[i].first != kEmptyKey_) {+      cells_[i].~value_type();+      *const_cast<KeyT*>(&cells_[i].first) = kEmptyKey_;+    }+    CHECK(cells_[i].first == kEmptyKey_);+  }+  numEntries_.set(0);+  numPendingEntries_.set(0);+  isFull_.store(0, std::memory_order_relaxed);+  numErases_.store(0, std::memory_order_relaxed);+}++// Iterator implementation++template <+    class KeyT,+    class ValueT,+    class HashFcn,+    class EqualFcn,+    class Allocator,+    class ProbeFcn,+    class KeyConvertFcn>+template <class ContT, class IterVal>+struct AtomicHashArray<+    KeyT,+    ValueT,+    HashFcn,+    EqualFcn,+    Allocator,+    ProbeFcn,+    KeyConvertFcn>::aha_iterator+    : detail::IteratorFacade<+          aha_iterator<ContT, IterVal>,+          IterVal,+          std::forward_iterator_tag> {+  explicit aha_iterator() : aha_(nullptr) {}++  // Conversion ctor for interoperability between const_iterator and+  // iterator.  The enable_if<> magic keeps us well-behaved for+  // is_convertible<> (v. the iterator_facade documentation).+  template <class OtherContT, class OtherVal>+  aha_iterator(+      const aha_iterator<OtherContT, OtherVal>& o,+      typename std::enable_if<+          std::is_convertible<OtherVal*, IterVal*>::value>::type* = nullptr)+      : aha_(o.aha_), offset_(o.offset_) {}++  explicit aha_iterator(ContT* array, size_t offset)+      : aha_(array), offset_(offset) {}++  // Returns unique index that can be used with findAt().+  // WARNING: The following function will fail silently for hashtable+  // with capacity > 2^32+  uint32_t getIndex() const { return offset_; }++  void advancePastEmpty() {+    while (offset_ < aha_->capacity_ && !isValid()) {+      ++offset_;+    }+  }++ private:+  friend class AtomicHashArray;+  friend class detail::+      IteratorFacade<aha_iterator, IterVal, std::forward_iterator_tag>;++  void increment() {+    ++offset_;+    advancePastEmpty();+  }++  bool equal(const aha_iterator& o) const {+    return aha_ == o.aha_ && offset_ == o.offset_;+  }++  IterVal& dereference() const { return aha_->cells_[offset_]; }++  bool isValid() const {+    KeyT key = acquireLoadKey(aha_->cells_[offset_]);+    return key != aha_->kEmptyKey_ && key != aha_->kLockedKey_ &&+        key != aha_->kErasedKey_;+  }++ private:+  ContT* aha_;+  size_t offset_;+}; // aha_iterator++} // namespace folly
+ folly/folly/AtomicHashArray.h view
@@ -0,0 +1,431 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++/**+ *  AtomicHashArray is the building block for AtomicHashMap.  It provides the+ *  core lock-free functionality, but is limited by the fact that it cannot+ *  grow past its initialization size and is a little more awkward (no public+ *  constructor, for example).  If you're confident that you won't run out of+ *  space, don't mind the awkardness, and really need bare-metal performance,+ *  feel free to use AHA directly.+ *+ *  Check out AtomicHashMap.h for more thorough documentation on perf and+ *  general pros and cons relative to other hash maps.+ *+ */++#pragma once+#define FOLLY_ATOMICHASHARRAY_H_++#include <atomic>++#include <folly/ThreadCachedInt.h>+#include <folly/Utility.h>+#include <folly/hash/Hash.h>++namespace folly {++struct AtomicHashArrayLinearProbeFcn {+  inline size_t operator()(+      size_t idx, size_t /* numProbes */, size_t capacity) const {+    idx += 1; // linear probing++    // Avoid modulus because it's slow+    return FOLLY_LIKELY(idx < capacity) ? idx : (idx - capacity);+  }+};++struct AtomicHashArrayQuadraticProbeFcn {+  inline size_t operator()(+      size_t idx, size_t numProbes, size_t capacity) const {+    idx += numProbes; // quadratic probing++    // Avoid modulus because it's slow+    return FOLLY_LIKELY(idx < capacity) ? idx : (idx - capacity);+  }+};++// Enables specializing checkLegalKey without specializing its class.+namespace detail {+template <typename NotKeyT, typename KeyT>+inline void checkLegalKeyIfKeyTImpl(+    NotKeyT /* ignored */,+    KeyT /* emptyKey */,+    KeyT /* lockedKey */,+    KeyT /* erasedKey */) {}++template <typename KeyT>+inline void checkLegalKeyIfKeyTImpl(+    KeyT key_in, KeyT emptyKey, KeyT lockedKey, KeyT erasedKey) {+  DCHECK_NE(key_in, emptyKey);+  DCHECK_NE(key_in, lockedKey);+  DCHECK_NE(key_in, erasedKey);+}+} // namespace detail++template <+    class KeyT,+    class ValueT,+    class HashFcn = std::hash<KeyT>,+    class EqualFcn = std::equal_to<KeyT>,+    class Allocator = std::allocator<char>,+    class ProbeFcn = AtomicHashArrayLinearProbeFcn,+    class KeyConvertFcn = Identity>+class AtomicHashMap;++template <+    class KeyT,+    class ValueT,+    class HashFcn = std::hash<KeyT>,+    class EqualFcn = std::equal_to<KeyT>,+    class Allocator = std::allocator<char>,+    class ProbeFcn = AtomicHashArrayLinearProbeFcn,+    class KeyConvertFcn = Identity>+class AtomicHashArray {+  static_assert(+      (std::is_convertible<KeyT, int32_t>::value ||+       std::is_convertible<KeyT, int64_t>::value ||+       std::is_convertible<KeyT, const void*>::value),+      "You are trying to use AtomicHashArray with disallowed key "+      "types.  You must use atomically compare-and-swappable integer "+      "keys, or a different container class.");++ public:+  typedef KeyT key_type;+  typedef ValueT mapped_type;+  typedef HashFcn hasher;+  typedef EqualFcn key_equal;+  typedef KeyConvertFcn key_convert;+  typedef std::pair<const KeyT, ValueT> value_type;+  typedef std::size_t size_type;+  typedef std::ptrdiff_t difference_type;+  typedef value_type& reference;+  typedef const value_type& const_reference;+  typedef value_type* pointer;+  typedef const value_type* const_pointer;++  const size_t capacity_;+  const size_t maxEntries_;+  const KeyT kEmptyKey_;+  const KeyT kLockedKey_;+  const KeyT kErasedKey_;++  template <class ContT, class IterVal>+  struct aha_iterator;++  typedef aha_iterator<const AtomicHashArray, const value_type> const_iterator;+  typedef aha_iterator<AtomicHashArray, value_type> iterator;++  // You really shouldn't need this if you use the SmartPtr provided by create,+  // but if you really want to do something crazy like stick the released+  // pointer into a DescriminatedPtr or something, you'll need this to clean up+  // after yourself.+  static void destroy(AtomicHashArray*);++ private:+  const size_t kAnchorMask_;++  struct Deleter {+    void operator()(AtomicHashArray* ptr) { AtomicHashArray::destroy(ptr); }+  };++ public:+  typedef std::unique_ptr<AtomicHashArray, Deleter> SmartPtr;++  /*+   * create --+   *+   *   Creates AtomicHashArray objects.  Use instead of constructor/destructor.+   *+   *   We do things this way in order to avoid the perf penalty of a second+   *   pointer indirection when composing these into AtomicHashMap, which needs+   *   to store an array of pointers so that it can perform atomic operations on+   *   them when growing.+   *+   *   Instead of a mess of arguments, we take a max size and a Config struct to+   *   simulate named ctor parameters.  The Config struct has sensible defaults+   *   for everything, but is overloaded - if you specify a positive capacity,+   *   that will be used directly instead of computing it based on+   *   maxLoadFactor.+   *+   *   Create returns an AHA::SmartPtr which is a unique_ptr with a custom+   *   deleter to make sure everything is cleaned up properly.+   */+  struct Config {+    KeyT emptyKey;+    KeyT lockedKey;+    KeyT erasedKey;+    double maxLoadFactor;+    double growthFactor;+    uint32_t entryCountThreadCacheSize;+    size_t capacity; // if positive, overrides maxLoadFactor++    //  Cannot have constexpr ctor because some compilers rightly complain.+    Config()+        : emptyKey((KeyT)-1),+          lockedKey((KeyT)-2),+          erasedKey((KeyT)-3),+          maxLoadFactor(0.8),+          growthFactor(-1),+          entryCountThreadCacheSize(1000),+          capacity(0) {}+  };++  //  Cannot have pre-instantiated const Config instance because of SIOF.+  static SmartPtr create(size_t maxSize, const Config& c = Config());++  /*+   * find --+   *+   *+   *   Returns the iterator to the element if found, otherwise end().+   *+   *   As an optional feature, the type of the key to look up (LookupKeyT) is+   *   allowed to be different from the type of keys actually stored (KeyT).+   *+   *   This enables use cases where materializing the key is costly and usually+   *   redundant, e.g., canonicalizing/interning a set of strings and being able+   *   to look up by StringPiece. To use this feature, LookupHashFcn must take+   *   a LookupKeyT, and LookupEqualFcn must take KeyT and LookupKeyT as first+   *   and second parameter, respectively.+   *+   *   See folly/test/ArrayHashArrayTest.cpp for sample usage.+   */+  template <+      typename LookupKeyT = key_type,+      typename LookupHashFcn = hasher,+      typename LookupEqualFcn = key_equal>+  iterator find(LookupKeyT k) {+    return iterator(+        this, findInternal<LookupKeyT, LookupHashFcn, LookupEqualFcn>(k).idx);+  }++  template <+      typename LookupKeyT = key_type,+      typename LookupHashFcn = hasher,+      typename LookupEqualFcn = key_equal>+  const_iterator find(LookupKeyT k) const {+    return const_cast<AtomicHashArray*>(this)+        ->find<LookupKeyT, LookupHashFcn, LookupEqualFcn>(k);+  }++  /*+   * insert --+   *+   *   Returns a pair with iterator to the element at r.first and bool success.+   *   Retrieve the index with ret.first.getIndex().+   *+   *   Fails on key collision (does not overwrite) or if map becomes+   *   full, at which point no element is inserted, iterator is set to end(),+   *   and success is set false.  On collisions, success is set false, but the+   *   iterator is set to the existing entry.+   */+  std::pair<iterator, bool> insert(const value_type& r) {+    return emplace(r.first, r.second);+  }+  std::pair<iterator, bool> insert(value_type&& r) {+    return emplace(r.first, std::move(r.second));+  }++  /*+   * emplace --+   *+   *   Same contract as insert(), but performs in-place construction+   *   of the value type using the specified arguments.+   *+   *   Also, like find(), this method optionally allows 'key_in' to have a type+   *   different from that stored in the table; see find(). If and only if no+   *   equal key is already present, this method converts 'key_in' to a key of+   *   type KeyT using the provided LookupKeyToKeyFcn.+   */+  template <+      typename LookupKeyT = key_type,+      typename LookupHashFcn = hasher,+      typename LookupEqualFcn = key_equal,+      typename LookupKeyToKeyFcn = key_convert,+      typename... ArgTs>+  std::pair<iterator, bool> emplace(LookupKeyT key_in, ArgTs&&... vCtorArgs) {+    SimpleRetT ret = insertInternal<+        LookupKeyT,+        LookupHashFcn,+        LookupEqualFcn,+        LookupKeyToKeyFcn>(key_in, std::forward<ArgTs>(vCtorArgs)...);+    return std::make_pair(iterator(this, ret.idx), ret.success);+  }++  // returns the number of elements erased - should never exceed 1+  size_t erase(KeyT k);++  // clears all keys and values in the map and resets all counters.  Not thread+  // safe.+  void clear();++  // Exact number of elements in the map - note that readFull() acquires a+  // mutex.  See folly/ThreadCachedInt.h for more details.+  size_t size() const {+    return numEntries_.readFull() - numErases_.load(std::memory_order_relaxed);+  }++  bool empty() const { return size() == 0; }++  iterator begin() {+    iterator it(this, 0);+    it.advancePastEmpty();+    return it;+  }+  const_iterator begin() const {+    const_iterator it(this, 0);+    it.advancePastEmpty();+    return it;+  }++  iterator end() { return iterator(this, capacity_); }+  const_iterator end() const { return const_iterator(this, capacity_); }++  // See AtomicHashMap::findAt - access elements directly+  // WARNING: The following 2 functions will fail silently for hashtable+  // with capacity > 2^32+  iterator findAt(uint32_t idx) {+    DCHECK_LT(idx, capacity_);+    return iterator(this, idx);+  }+  const_iterator findAt(uint32_t idx) const {+    return const_cast<AtomicHashArray*>(this)->findAt(idx);+  }++  iterator makeIter(size_t idx) { return iterator(this, idx); }+  const_iterator makeIter(size_t idx) const {+    return const_iterator(this, idx);+  }++  // The max load factor allowed for this map+  double maxLoadFactor() const { return ((double)maxEntries_) / capacity_; }++  void setEntryCountThreadCacheSize(uint32_t newSize) {+    numEntries_.setCacheSize(newSize);+    numPendingEntries_.setCacheSize(newSize);+  }++  uint32_t getEntryCountThreadCacheSize() const {+    return numEntries_.getCacheSize();+  }++  /* Private data and helper functions... */++ private:+  friend class AtomicHashMap<+      KeyT,+      ValueT,+      HashFcn,+      EqualFcn,+      Allocator,+      ProbeFcn>;++  struct SimpleRetT {+    size_t idx;+    bool success;+    SimpleRetT(size_t i, bool s) : idx(i), success(s) {}+    SimpleRetT() = default;+  };++  template <+      typename LookupKeyT = key_type,+      typename LookupHashFcn = hasher,+      typename LookupEqualFcn = key_equal,+      typename LookupKeyToKeyFcn = Identity,+      typename... ArgTs>+  SimpleRetT insertInternal(LookupKeyT key, ArgTs&&... vCtorArgs);++  template <+      typename LookupKeyT = key_type,+      typename LookupHashFcn = hasher,+      typename LookupEqualFcn = key_equal>+  SimpleRetT findInternal(const LookupKeyT key);++  template <typename MaybeKeyT>+  void checkLegalKeyIfKey(MaybeKeyT key) {+    detail::checkLegalKeyIfKeyTImpl(key, kEmptyKey_, kLockedKey_, kErasedKey_);+  }++  static std::atomic<KeyT>* cellKeyPtr(const value_type& r) {+    // We need some illegal casting here in order to actually store+    // our value_type as a std::pair<const,>.  But a little bit of+    // undefined behavior never hurt anyone ...+    static_assert(+        sizeof(std::atomic<KeyT>) == sizeof(KeyT),+        "std::atomic is implemented in an unexpected way for AHM");+    return const_cast<std::atomic<KeyT>*>(+        reinterpret_cast<std::atomic<KeyT> const*>(&r.first));+  }++  static KeyT relaxedLoadKey(const value_type& r) {+    return cellKeyPtr(r)->load(std::memory_order_relaxed);+  }++  static KeyT acquireLoadKey(const value_type& r) {+    return cellKeyPtr(r)->load(std::memory_order_acquire);+  }++  // Fun with thread local storage - atomic increment is expensive+  // (relatively), so we accumulate in the thread cache and periodically+  // flush to the actual variable, and walk through the unflushed counts when+  // reading the value, so be careful of calling size() too frequently.  This+  // increases insertion throughput several times over while keeping the count+  // accurate.+  ThreadCachedInt<uint64_t> numEntries_; // Successful key inserts+  ThreadCachedInt<uint64_t> numPendingEntries_; // Used by insertInternal+  std::atomic<int64_t> isFull_; // Used by insertInternal+  std::atomic<int64_t> numErases_; // Successful key erases++  value_type cells_[0]; // This must be the last field of this class++  // Force constructor/destructor private since create/destroy should be+  // used externally instead+  AtomicHashArray(+      size_t capacity,+      KeyT emptyKey,+      KeyT lockedKey,+      KeyT erasedKey,+      double maxLoadFactor,+      uint32_t cacheSize);++  AtomicHashArray(const AtomicHashArray&) = delete;+  AtomicHashArray& operator=(const AtomicHashArray&) = delete;++  ~AtomicHashArray() = default;++  inline void unlockCell(value_type* const cell, KeyT newKey) {+    cellKeyPtr(*cell)->store(newKey, std::memory_order_release);+  }++  inline bool tryLockCell(value_type* const cell) {+    KeyT expect = kEmptyKey_;+    return cellKeyPtr(*cell)->compare_exchange_strong(+        expect, kLockedKey_, std::memory_order_acq_rel);+  }++  template <class LookupKeyT = key_type, class LookupHashFcn = hasher>+  inline size_t keyToAnchorIdx(const LookupKeyT k) const {+    const size_t hashVal = LookupHashFcn()(k);+    const size_t probe = hashVal & kAnchorMask_;+    return FOLLY_LIKELY(probe < capacity_) ? probe : hashVal % capacity_;+  }++}; // AtomicHashArray++} // namespace folly++#include <folly/AtomicHashArray-inl.h>
+ folly/folly/AtomicHashMap-inl.h view
@@ -0,0 +1,654 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#ifndef FOLLY_ATOMICHASHMAP_H_+#error "This should only be included by AtomicHashMap.h"+#endif++#include <folly/detail/AtomicHashUtils.h>+#include <folly/detail/Iterators.h>++#include <type_traits>++namespace folly {++// AtomicHashMap constructor -- Atomic wrapper that allows growth+// This class has a lot of overhead (184 Bytes) so only use for big maps+template <+    typename KeyT,+    typename ValueT,+    typename HashFcn,+    typename EqualFcn,+    typename Allocator,+    typename ProbeFcn,+    typename KeyConvertFcn>+AtomicHashMap<+    KeyT,+    ValueT,+    HashFcn,+    EqualFcn,+    Allocator,+    ProbeFcn,+    KeyConvertFcn>::AtomicHashMap(size_t finalSizeEst, const Config& config)+    : kGrowthFrac_(+          config.growthFactor < 0+              ? 1.0f - config.maxLoadFactor+              : config.growthFactor) {+  CHECK(config.maxLoadFactor > 0.0f && config.maxLoadFactor < 1.0f);+  subMaps_[0].store(+      SubMap::create(finalSizeEst, config).release(),+      std::memory_order_relaxed);+  auto subMapCount = kNumSubMaps_;+  FOR_EACH_RANGE (i, 1, subMapCount) {+    subMaps_[i].store(nullptr, std::memory_order_relaxed);+  }+  numMapsAllocated_.store(1, std::memory_order_relaxed);+}++// emplace --+template <+    typename KeyT,+    typename ValueT,+    typename HashFcn,+    typename EqualFcn,+    typename Allocator,+    typename ProbeFcn,+    typename KeyConvertFcn>+template <+    typename LookupKeyT,+    typename LookupHashFcn,+    typename LookupEqualFcn,+    typename LookupKeyToKeyFcn,+    typename... ArgTs>+std::pair<+    typename AtomicHashMap<+        KeyT,+        ValueT,+        HashFcn,+        EqualFcn,+        Allocator,+        ProbeFcn,+        KeyConvertFcn>::iterator,+    bool>+AtomicHashMap<+    KeyT,+    ValueT,+    HashFcn,+    EqualFcn,+    Allocator,+    ProbeFcn,+    KeyConvertFcn>::emplace(LookupKeyT k, ArgTs&&... vCtorArgs) {+  SimpleRetT ret = insertInternal<+      LookupKeyT,+      LookupHashFcn,+      LookupEqualFcn,+      LookupKeyToKeyFcn>(k, std::forward<ArgTs>(vCtorArgs)...);+  SubMap* subMap = subMaps_[ret.i].load(std::memory_order_relaxed);+  return std::make_pair(+      iterator(this, ret.i, subMap->makeIter(ret.j)), ret.success);+}++// insertInternal -- Allocates new sub maps as existing ones fill up.+template <+    typename KeyT,+    typename ValueT,+    typename HashFcn,+    typename EqualFcn,+    typename Allocator,+    typename ProbeFcn,+    typename KeyConvertFcn>+template <+    typename LookupKeyT,+    typename LookupHashFcn,+    typename LookupEqualFcn,+    typename LookupKeyToKeyFcn,+    typename... ArgTs>+typename AtomicHashMap<+    KeyT,+    ValueT,+    HashFcn,+    EqualFcn,+    Allocator,+    ProbeFcn,+    KeyConvertFcn>::SimpleRetT+AtomicHashMap<+    KeyT,+    ValueT,+    HashFcn,+    EqualFcn,+    Allocator,+    ProbeFcn,+    KeyConvertFcn>::insertInternal(LookupKeyT key, ArgTs&&... vCtorArgs) {+beginInsertInternal:+  auto nextMapIdx = // this maintains our state+      numMapsAllocated_.load(std::memory_order_acquire);+  typename SubMap::SimpleRetT ret;+  FOR_EACH_RANGE (i, 0, nextMapIdx) {+    // insert in each map successively.  If one succeeds, we're done!+    SubMap* subMap = subMaps_[i].load(std::memory_order_relaxed);+    ret = subMap->template insertInternal<+        LookupKeyT,+        LookupHashFcn,+        LookupEqualFcn,+        LookupKeyToKeyFcn>(key, std::forward<ArgTs>(vCtorArgs)...);+    if (ret.idx == subMap->capacity_) {+      continue; // map is full, so try the next one+    }+    // Either collision or success - insert in either case+    return SimpleRetT(i, ret.idx, ret.success);+  }++  // If we made it this far, all maps are full and we need to try to allocate+  // the next one.++  SubMap* primarySubMap = subMaps_[0].load(std::memory_order_relaxed);+  if (nextMapIdx >= kNumSubMaps_ ||+      primarySubMap->capacity_ * kGrowthFrac_ < 1.0) {+    // Can't allocate any more sub maps.+    throw AtomicHashMapFullError();+  }++  if (tryLockMap(nextMapIdx)) {+    // Alloc a new map and shove it in.  We can change whatever+    // we want because other threads are waiting on us...+    size_t numCellsAllocated =+        (size_t)(primarySubMap->capacity_ *+                 std::pow(1.0 + kGrowthFrac_, nextMapIdx - 1));+    size_t newSize = size_t(numCellsAllocated * kGrowthFrac_);+    DCHECK(+        subMaps_[nextMapIdx].load(std::memory_order_relaxed) ==+        (SubMap*)kLockedPtr_);+    // create a new map using the settings stored in the first map++    Config config;+    config.emptyKey = primarySubMap->kEmptyKey_;+    config.lockedKey = primarySubMap->kLockedKey_;+    config.erasedKey = primarySubMap->kErasedKey_;+    config.maxLoadFactor = primarySubMap->maxLoadFactor();+    config.entryCountThreadCacheSize =+        primarySubMap->getEntryCountThreadCacheSize();+    subMaps_[nextMapIdx].store(+        SubMap::create(newSize, config).release(), std::memory_order_relaxed);++    // Publish the map to other threads.+    numMapsAllocated_.fetch_add(1, std::memory_order_release);+    DCHECK_EQ(+        nextMapIdx + 1, numMapsAllocated_.load(std::memory_order_relaxed));+  } else {+    // If we lost the race, we'll have to wait for the next map to get+    // allocated before doing any insertion here.+    detail::atomic_hash_spin_wait([&] {+      return nextMapIdx >= numMapsAllocated_.load(std::memory_order_acquire);+    });+  }++  // Relaxed is ok here because either we just created this map, or we+  // just did a spin wait with an acquire load on numMapsAllocated_.+  SubMap* loadedMap = subMaps_[nextMapIdx].load(std::memory_order_relaxed);+  DCHECK(loadedMap && loadedMap != (SubMap*)kLockedPtr_);+  ret = loadedMap->insertInternal(key, std::forward<ArgTs>(vCtorArgs)...);+  if (ret.idx != loadedMap->capacity_) {+    return SimpleRetT(nextMapIdx, ret.idx, ret.success);+  }+  // We took way too long and the new map is already full...try again from+  // the top (this should pretty much never happen).+  goto beginInsertInternal;+}++// find --+template <+    typename KeyT,+    typename ValueT,+    typename HashFcn,+    typename EqualFcn,+    typename Allocator,+    typename ProbeFcn,+    typename KeyConvertFcn>+template <class LookupKeyT, class LookupHashFcn, class LookupEqualFcn>+typename AtomicHashMap<+    KeyT,+    ValueT,+    HashFcn,+    EqualFcn,+    Allocator,+    ProbeFcn,+    KeyConvertFcn>::iterator+AtomicHashMap<+    KeyT,+    ValueT,+    HashFcn,+    EqualFcn,+    Allocator,+    ProbeFcn,+    KeyConvertFcn>::find(LookupKeyT k) {+  SimpleRetT ret = findInternal<LookupKeyT, LookupHashFcn, LookupEqualFcn>(k);+  if (!ret.success) {+    return end();+  }+  SubMap* subMap = subMaps_[ret.i].load(std::memory_order_relaxed);+  return iterator(this, ret.i, subMap->makeIter(ret.j));+}++template <+    typename KeyT,+    typename ValueT,+    typename HashFcn,+    typename EqualFcn,+    typename Allocator,+    typename ProbeFcn,+    typename KeyConvertFcn>+template <class LookupKeyT, class LookupHashFcn, class LookupEqualFcn>+typename AtomicHashMap<+    KeyT,+    ValueT,+    HashFcn,+    EqualFcn,+    Allocator,+    ProbeFcn,+    KeyConvertFcn>::const_iterator+AtomicHashMap<+    KeyT,+    ValueT,+    HashFcn,+    EqualFcn,+    Allocator,+    ProbeFcn,+    KeyConvertFcn>::find(LookupKeyT k) const {+  return const_cast<AtomicHashMap*>(this)+      ->find<LookupKeyT, LookupHashFcn, LookupEqualFcn>(k);+}++// findInternal --+template <+    typename KeyT,+    typename ValueT,+    typename HashFcn,+    typename EqualFcn,+    typename Allocator,+    typename ProbeFcn,+    typename KeyConvertFcn>+template <class LookupKeyT, class LookupHashFcn, class LookupEqualFcn>+typename AtomicHashMap<+    KeyT,+    ValueT,+    HashFcn,+    EqualFcn,+    Allocator,+    ProbeFcn,+    KeyConvertFcn>::SimpleRetT+AtomicHashMap<+    KeyT,+    ValueT,+    HashFcn,+    EqualFcn,+    Allocator,+    ProbeFcn,+    KeyConvertFcn>::findInternal(const LookupKeyT k) const {+  SubMap* const primaryMap = subMaps_[0].load(std::memory_order_relaxed);+  typename SubMap::SimpleRetT ret =+      primaryMap+          ->template findInternal<LookupKeyT, LookupHashFcn, LookupEqualFcn>(k);+  if (FOLLY_LIKELY(ret.idx != primaryMap->capacity_)) {+    return SimpleRetT(0, ret.idx, ret.success);+  }+  const unsigned int numMaps =+      numMapsAllocated_.load(std::memory_order_acquire);+  FOR_EACH_RANGE (i, 1, numMaps) {+    // Check each map successively.  If one succeeds, we're done!+    SubMap* thisMap = subMaps_[i].load(std::memory_order_relaxed);+    ret =+        thisMap+            ->template findInternal<LookupKeyT, LookupHashFcn, LookupEqualFcn>(+                k);+    if (FOLLY_LIKELY(ret.idx != thisMap->capacity_)) {+      return SimpleRetT(i, ret.idx, ret.success);+    }+  }+  // Didn't find our key...+  return SimpleRetT(numMaps, 0, false);+}++// findAtInternal -- see encodeIndex() for details.+template <+    typename KeyT,+    typename ValueT,+    typename HashFcn,+    typename EqualFcn,+    typename Allocator,+    typename ProbeFcn,+    typename KeyConvertFcn>+typename AtomicHashMap<+    KeyT,+    ValueT,+    HashFcn,+    EqualFcn,+    Allocator,+    ProbeFcn,+    KeyConvertFcn>::SimpleRetT+AtomicHashMap<+    KeyT,+    ValueT,+    HashFcn,+    EqualFcn,+    Allocator,+    ProbeFcn,+    KeyConvertFcn>::findAtInternal(uint32_t idx) const {+  uint32_t subMapIdx, subMapOffset;+  if (idx & kSecondaryMapBit_) {+    // idx falls in a secondary map+    idx &= ~kSecondaryMapBit_; // unset secondary bit+    subMapIdx = idx >> kSubMapIndexShift_;+    DCHECK_LT(subMapIdx, numMapsAllocated_.load(std::memory_order_relaxed));+    subMapOffset = idx & kSubMapIndexMask_;+  } else {+    // idx falls in primary map+    subMapIdx = 0;+    subMapOffset = idx;+  }+  return SimpleRetT(subMapIdx, subMapOffset, true);+}++// erase --+template <+    typename KeyT,+    typename ValueT,+    typename HashFcn,+    typename EqualFcn,+    typename Allocator,+    typename ProbeFcn,+    typename KeyConvertFcn>+typename AtomicHashMap<+    KeyT,+    ValueT,+    HashFcn,+    EqualFcn,+    Allocator,+    ProbeFcn,+    KeyConvertFcn>::size_type+AtomicHashMap<+    KeyT,+    ValueT,+    HashFcn,+    EqualFcn,+    Allocator,+    ProbeFcn,+    KeyConvertFcn>::erase(const KeyT k) {+  int const numMaps = numMapsAllocated_.load(std::memory_order_acquire);+  FOR_EACH_RANGE (i, 0, numMaps) {+    // Check each map successively.  If one succeeds, we're done!+    if (subMaps_[i].load(std::memory_order_relaxed)->erase(k)) {+      return 1;+    }+  }+  // Didn't find our key...+  return 0;+}++// capacity -- summation of capacities of all submaps+template <+    typename KeyT,+    typename ValueT,+    typename HashFcn,+    typename EqualFcn,+    typename Allocator,+    typename ProbeFcn,+    typename KeyConvertFcn>+size_t AtomicHashMap<+    KeyT,+    ValueT,+    HashFcn,+    EqualFcn,+    Allocator,+    ProbeFcn,+    KeyConvertFcn>::capacity() const {+  size_t totalCap(0);+  int const numMaps = numMapsAllocated_.load(std::memory_order_acquire);+  FOR_EACH_RANGE (i, 0, numMaps) {+    totalCap += subMaps_[i].load(std::memory_order_relaxed)->capacity_;+  }+  return totalCap;+}++// spaceRemaining --+// number of new insertions until current submaps are all at max load+template <+    typename KeyT,+    typename ValueT,+    typename HashFcn,+    typename EqualFcn,+    typename Allocator,+    typename ProbeFcn,+    typename KeyConvertFcn>+size_t AtomicHashMap<+    KeyT,+    ValueT,+    HashFcn,+    EqualFcn,+    Allocator,+    ProbeFcn,+    KeyConvertFcn>::spaceRemaining() const {+  size_t spaceRem(0);+  int const numMaps = numMapsAllocated_.load(std::memory_order_acquire);+  FOR_EACH_RANGE (i, 0, numMaps) {+    SubMap* thisMap = subMaps_[i].load(std::memory_order_relaxed);+    spaceRem +=+        std::max(0, thisMap->maxEntries_ - &thisMap->numEntries_.readFull());+  }+  return spaceRem;+}++// clear -- Wipes all keys and values from primary map and destroys+// all secondary maps.  Not thread safe.+template <+    typename KeyT,+    typename ValueT,+    typename HashFcn,+    typename EqualFcn,+    typename Allocator,+    typename ProbeFcn,+    typename KeyConvertFcn>+void AtomicHashMap<+    KeyT,+    ValueT,+    HashFcn,+    EqualFcn,+    Allocator,+    ProbeFcn,+    KeyConvertFcn>::clear() {+  subMaps_[0].load(std::memory_order_relaxed)->clear();+  int const numMaps = numMapsAllocated_.load(std::memory_order_relaxed);+  FOR_EACH_RANGE (i, 1, numMaps) {+    SubMap* thisMap = subMaps_[i].load(std::memory_order_relaxed);+    DCHECK(thisMap);+    SubMap::destroy(thisMap);+    subMaps_[i].store(nullptr, std::memory_order_relaxed);+  }+  numMapsAllocated_.store(1, std::memory_order_relaxed);+}++// size --+template <+    typename KeyT,+    typename ValueT,+    typename HashFcn,+    typename EqualFcn,+    typename Allocator,+    typename ProbeFcn,+    typename KeyConvertFcn>+size_t AtomicHashMap<+    KeyT,+    ValueT,+    HashFcn,+    EqualFcn,+    Allocator,+    ProbeFcn,+    KeyConvertFcn>::size() const {+  size_t totalSize(0);+  int const numMaps = numMapsAllocated_.load(std::memory_order_acquire);+  FOR_EACH_RANGE (i, 0, numMaps) {+    totalSize += subMaps_[i].load(std::memory_order_relaxed)->size();+  }+  return totalSize;+}++// encodeIndex -- Encode the submap index and offset into return.+// index_ret must be pre-populated with the submap offset.+//+// We leave index_ret untouched when referring to the primary map+// so it can be as large as possible (31 data bits).  Max size of+// secondary maps is limited by what can fit in the low 27 bits.+//+// Returns the following bit-encoded data in index_ret:+//   if subMap == 0 (primary map) =>+//     bit(s)          value+//         31              0+//       0-30  submap offset (index_ret input)+//+//   if subMap > 0 (secondary maps) =>+//     bit(s)          value+//         31              1+//      27-30   which subMap+//       0-26  subMap offset (index_ret input)+template <+    typename KeyT,+    typename ValueT,+    typename HashFcn,+    typename EqualFcn,+    typename Allocator,+    typename ProbeFcn,+    typename KeyConvertFcn>+inline uint32_t AtomicHashMap<+    KeyT,+    ValueT,+    HashFcn,+    EqualFcn,+    Allocator,+    ProbeFcn,+    KeyConvertFcn>::encodeIndex(uint32_t subMap, uint32_t offset) {+  DCHECK_EQ(offset & kSecondaryMapBit_, 0); // offset can't be too big+  if (subMap == 0) {+    return offset;+  }+  // Make sure subMap isn't too big+  DCHECK_EQ(subMap >> kNumSubMapBits_, 0);+  // Make sure subMap bits of offset are clear+  DCHECK_EQ(offset & (~kSubMapIndexMask_ | kSecondaryMapBit_), 0);++  // Set high-order bits to encode which submap this index belongs to+  return offset | (subMap << kSubMapIndexShift_) | kSecondaryMapBit_;+}++// Iterator implementation++template <+    typename KeyT,+    typename ValueT,+    typename HashFcn,+    typename EqualFcn,+    typename Allocator,+    typename ProbeFcn,+    typename KeyConvertFcn>+template <class ContT, class IterVal, class SubIt>+struct AtomicHashMap<+    KeyT,+    ValueT,+    HashFcn,+    EqualFcn,+    Allocator,+    ProbeFcn,+    KeyConvertFcn>::ahm_iterator+    : detail::IteratorFacade<+          ahm_iterator<ContT, IterVal, SubIt>,+          IterVal,+          std::forward_iterator_tag> {+  explicit ahm_iterator() : ahm_(nullptr) {}++  // Conversion ctor for interoperability between const_iterator and+  // iterator.  The enable_if<> magic keeps us well-behaved for+  // is_convertible<> (v. the iterator_facade documentation).+  template <class OtherContT, class OtherVal, class OtherSubIt>+  ahm_iterator(+      const ahm_iterator<OtherContT, OtherVal, OtherSubIt>& o,+      typename std::enable_if<+          std::is_convertible<OtherSubIt, SubIt>::value>::type* = nullptr)+      : ahm_(o.ahm_), subMap_(o.subMap_), subIt_(o.subIt_) {}++  /*+   * Returns the unique index that can be used for access directly+   * into the data storage.+   */+  uint32_t getIndex() const {+    CHECK(!isEnd());+    return ahm_->encodeIndex(subMap_, subIt_.getIndex());+  }++ private:+  friend class AtomicHashMap;+  explicit ahm_iterator(ContT* ahm, uint32_t subMap, const SubIt& subIt)+      : ahm_(ahm), subMap_(subMap), subIt_(subIt) {}++  friend class detail::+      IteratorFacade<ahm_iterator, IterVal, std::forward_iterator_tag>;++  void increment() {+    CHECK(!isEnd());+    ++subIt_;+    checkAdvanceToNextSubmap();+  }++  bool equal(const ahm_iterator& other) const {+    if (ahm_ != other.ahm_) {+      return false;+    }++    if (isEnd() || other.isEnd()) {+      return isEnd() == other.isEnd();+    }++    return subMap_ == other.subMap_ && subIt_ == other.subIt_;+  }++  IterVal& dereference() const { return *subIt_; }++  bool isEnd() const { return ahm_ == nullptr; }++  void checkAdvanceToNextSubmap() {+    if (isEnd()) {+      return;+    }++    SubMap* thisMap = ahm_->subMaps_[subMap_].load(std::memory_order_relaxed);+    while (subIt_ == thisMap->end()) {+      // This sub iterator is done, advance to next one+      if (subMap_ + 1 <+          ahm_->numMapsAllocated_.load(std::memory_order_acquire)) {+        ++subMap_;+        thisMap = ahm_->subMaps_[subMap_].load(std::memory_order_relaxed);+        subIt_ = thisMap->begin();+      } else {+        ahm_ = nullptr;+        return;+      }+    }+  }++ private:+  ContT* ahm_;+  uint32_t subMap_;+  SubIt subIt_;+}; // ahm_iterator++} // namespace folly
+ folly/folly/AtomicHashMap.h view
@@ -0,0 +1,485 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++/*+ * AtomicHashMap --+ *+ * A high-performance concurrent hash map with int32_t or int64_t keys. Supports+ * insert, find(key), findAt(index), erase(key), size, and more.  Memory cannot+ * be freed or reclaimed by erase.  Can grow to a maximum of about 18 times the+ * initial capacity, but performance degrades linearly with growth. Can also be+ * used as an object store with unique 32-bit references directly into the+ * internal storage (retrieved with iterator::getIndex()).+ *+ * Advantages:+ *    - High-performance (~2-4x tbb::concurrent_hash_map in heavily+ *      multi-threaded environments).+ *    - Efficient memory usage if initial capacity is not over estimated+ *      (especially for small keys and values).+ *    - Good fragmentation properties (only allocates in large slabs which can+ *      be reused with clear() and never move).+ *    - Can generate unique, long-lived 32-bit references for efficient lookup+ *      (see findAt()).+ *+ * Disadvantages:+ *    - Keys must be native int32_t or int64_t, or explicitly converted.+ *    - Must be able to specify unique empty, locked, and erased keys+ *    - Performance degrades linearly as size grows beyond initialization+ *      capacity.+ *    - Max size limit of ~18x initial size (dependent on max load factor).+ *    - Memory is not freed or reclaimed by erase.+ *+ * Usage and Operation Details:+ *   Simple performance/memory tradeoff with maxLoadFactor.  Higher load factors+ *   give better memory utilization but probe lengths increase, reducing+ *   performance.+ *+ * Implementation and Performance Details:+ *   AHArray is a fixed size contiguous block of value_type cells.  When+ *   writing a cell, the key is locked while the rest of the record is+ *   written.  Once done, the cell is unlocked by setting the key.  find()+ *   is completely wait-free and doesn't require any non-relaxed atomic+ *   operations.  AHA cannot grow beyond initialization capacity, but is+ *   faster because of reduced data indirection.+ *+ *   AHMap is a wrapper around AHArray sub-maps that allows growth and provides+ *   an interface closer to the STL UnorderedAssociativeContainer concept. These+ *   sub-maps are allocated on the fly and are processed in series, so the more+ *   there are (from growing past initial capacity), the worse the performance.+ *+ *   Insert returns false if there is a key collision and throws if the max size+ *   of the map is exceeded.+ *+ *   Benchmark performance with 8 simultaneous threads processing 1 million+ *   unique <int64_t, int64_t> entries on a 4-core, 2.5 GHz machine:+ *+ *     Load Factor   Mem Efficiency   usec/Insert   usec/Find+ *         50%             50%           0.19         0.05+ *         85%             85%           0.20         0.06+ *         90%             90%           0.23         0.08+ *         95%             95%           0.27         0.10+ *+ *   See folly/tests/AtomicHashMapTest.cpp for more benchmarks.+ */++#pragma once+#define FOLLY_ATOMICHASHMAP_H_++#include <atomic>+#include <functional>+#include <stdexcept>++#include <folly/AtomicHashArray.h>+#include <folly/CPortability.h>+#include <folly/Likely.h>+#include <folly/ThreadCachedInt.h>+#include <folly/container/Foreach.h>+#include <folly/hash/Hash.h>++namespace folly {++/*+ * AtomicHashMap provides an interface somewhat similar to the+ * UnorderedAssociativeContainer concept in C++.  This does not+ * exactly match this concept (or even the basic Container concept),+ * because of some restrictions imposed by our datastructure.+ *+ * Specific differences (there are quite a few):+ *+ * - Efficiently thread safe for inserts (main point of this stuff),+ *   wait-free for lookups.+ *+ * - You can erase from this container, but the cell containing the key will+ *   not be free or reclaimed.+ *+ * - You can erase everything by calling clear() (and you must guarantee only+ *   one thread can be using the container to do that).+ *+ * - We aren't DefaultConstructible, CopyConstructible, Assignable, or+ *   EqualityComparable.  (Most of these are probably not something+ *   you actually want to do with this anyway.)+ *+ * - We don't support the various bucket functions, rehash(),+ *   reserve(), or equal_range().  Also no constructors taking+ *   iterators, although this could change.+ *+ * - Several insertion functions, notably operator[], are not+ *   implemented.  It is a little too easy to misuse these functions+ *   with this container, where part of the point is that when an+ *   insertion happens for a new key, it will atomically have the+ *   desired value.+ *+ * - The map has no templated insert() taking an iterator range, but+ *   we do provide an insert(key, value).  The latter seems more+ *   frequently useful for this container (to avoid sprinkling+ *   make_pair everywhere), and providing both can lead to some gross+ *   template error messages.+ *+ * - The Allocator must not be stateful (a new instance will be spun up for+ *   each allocation), and its allocate() method must take a raw number of+ *   bytes.+ *+ * - KeyT must be a 32 bit or 64 bit atomic integer type, and you must+ *   define special 'locked' and 'empty' key values in the ctor+ *+ * - We don't take the Hash function object as an instance in the+ *   constructor.+ *+ */++// Thrown when insertion fails due to running out of space for+// submaps.+struct FOLLY_EXPORT AtomicHashMapFullError : std::runtime_error {+  explicit AtomicHashMapFullError()+      : std::runtime_error("AtomicHashMap is full") {}+};++template <+    class KeyT,+    class ValueT,+    class HashFcn,+    class EqualFcn,+    class Allocator,+    class ProbeFcn,+    class KeyConvertFcn>+class AtomicHashMap {+  typedef AtomicHashArray<+      KeyT,+      ValueT,+      HashFcn,+      EqualFcn,+      Allocator,+      ProbeFcn,+      KeyConvertFcn>+      SubMap;++ public:+  typedef KeyT key_type;+  typedef ValueT mapped_type;+  typedef std::pair<const KeyT, ValueT> value_type;+  typedef HashFcn hasher;+  typedef EqualFcn key_equal;+  typedef KeyConvertFcn key_convert;+  typedef value_type* pointer;+  typedef value_type& reference;+  typedef const value_type& const_reference;+  typedef std::ptrdiff_t difference_type;+  typedef std::size_t size_type;+  typedef typename SubMap::Config Config;++  template <class ContT, class IterVal, class SubIt>+  struct ahm_iterator;++  typedef ahm_iterator<+      const AtomicHashMap,+      const value_type,+      typename SubMap::const_iterator>+      const_iterator;+  typedef ahm_iterator<AtomicHashMap, value_type, typename SubMap::iterator>+      iterator;++ public:+  const float kGrowthFrac_; // How much to grow when we run out of capacity.++  // The constructor takes a finalSizeEst which is the optimal+  // number of elements to maximize space utilization and performance,+  // and a Config object to specify more advanced options.+  explicit AtomicHashMap(size_t finalSizeEst, const Config& c = Config());++  AtomicHashMap(const AtomicHashMap&) = delete;+  AtomicHashMap& operator=(const AtomicHashMap&) = delete;++  ~AtomicHashMap() {+    const unsigned int numMaps =+        numMapsAllocated_.load(std::memory_order_relaxed);+    FOR_EACH_RANGE (i, 0, numMaps) {+      SubMap* thisMap = subMaps_[i].load(std::memory_order_relaxed);+      DCHECK(thisMap);+      SubMap::destroy(thisMap);+    }+  }++  key_equal key_eq() const { return key_equal(); }+  hasher hash_function() const { return hasher(); }++  /*+   * insert --+   *+   *   Returns a pair with iterator to the element at r.first and+   *   success.  Retrieve the index with ret.first.getIndex().+   *+   *   Does not overwrite on key collision, but returns an iterator to+   *   the existing element (since this could due to a race with+   *   another thread, it is often important to check this return+   *   value).+   *+   *   Allocates new sub maps as the existing ones become full.  If+   *   all sub maps are full, no element is inserted, and+   *   AtomicHashMapFullError is thrown.+   */+  std::pair<iterator, bool> insert(const value_type& r) {+    return emplace(r.first, r.second);+  }+  std::pair<iterator, bool> insert(key_type k, const mapped_type& v) {+    return emplace(k, v);+  }+  std::pair<iterator, bool> insert(value_type&& r) {+    return emplace(r.first, std::move(r.second));+  }+  std::pair<iterator, bool> insert(key_type k, mapped_type&& v) {+    return emplace(k, std::move(v));+  }++  /*+   * emplace --+   *+   *   Same contract as insert(), but performs in-place construction+   *   of the value type using the specified arguments.+   *+   *   Also, like find(), this method optionally allows 'key_in' to have a type+   *   different from that stored in the table; see find(). If and only if no+   *   equal key is already present, this method converts 'key_in' to a key of+   *   type KeyT using the provided LookupKeyToKeyFcn.+   */+  template <+      typename LookupKeyT = key_type,+      typename LookupHashFcn = hasher,+      typename LookupEqualFcn = key_equal,+      typename LookupKeyToKeyFcn = key_convert,+      typename... ArgTs>+  std::pair<iterator, bool> emplace(LookupKeyT k, ArgTs&&... vCtorArg);++  /*+   * find --+   *+   *   Returns the iterator to the element if found, otherwise end().+   *+   *   As an optional feature, the type of the key to look up (LookupKeyT) is+   *   allowed to be different from the type of keys actually stored (KeyT).+   *+   *   This enables use cases where materializing the key is costly and usually+   *   redundant, e.g., canonicalizing/interning a set of strings and being able+   *   to look up by StringPiece. To use this feature, LookupHashFcn must take+   *   a LookupKeyT, and LookupEqualFcn must take KeyT and LookupKeyT as first+   *   and second parameter, respectively.+   *+   *   See folly/test/ArrayHashMapTest.cpp for sample usage.+   */+  template <+      typename LookupKeyT = key_type,+      typename LookupHashFcn = hasher,+      typename LookupEqualFcn = key_equal>+  iterator find(LookupKeyT k);++  template <+      typename LookupKeyT = key_type,+      typename LookupHashFcn = hasher,+      typename LookupEqualFcn = key_equal>+  const_iterator find(LookupKeyT k) const;++  /*+   * erase --+   *+   *   Erases key k from the map+   *+   *   Returns 1 iff the key is found and erased, and 0 otherwise.+   */+  size_type erase(key_type k);++  /*+   * clear --+   *+   *   Wipes all keys and values from primary map and destroys all secondary+   *   maps.  Primary map remains allocated and thus the memory can be reused+   *   in place.  Not thread safe.+   *+   */+  void clear();++  /*+   * size --+   *+   *  Returns the exact size of the map.  Note this is not as cheap as typical+   *  size() implementations because, for each AtomicHashArray in this AHM, we+   *  need to grab a lock and accumulate the values from all the thread local+   *  counters.  See folly/ThreadCachedInt.h for more details.+   */+  size_t size() const;++  bool empty() const { return size() == 0; }++  size_type count(key_type k) const { return find(k) == end() ? 0 : 1; }++  /*+   * findAt --+   *+   *   Returns an iterator into the map.+   *+   *   idx should only be an unmodified value returned by calling getIndex() on+   *   a valid iterator returned by find() or insert(). If idx is invalid you+   *   have a bug and the process aborts.+   */+  iterator findAt(uint32_t idx) {+    SimpleRetT ret = findAtInternal(idx);+    DCHECK_LT(ret.i, numSubMaps());+    return iterator(+        this,+        ret.i,+        subMaps_[ret.i].load(std::memory_order_relaxed)->makeIter(ret.j));+  }+  const_iterator findAt(uint32_t idx) const {+    return const_cast<AtomicHashMap*>(this)->findAt(idx);+  }++  // Total capacity - summation of capacities of all submaps.+  size_t capacity() const;++  // Number of new insertions until current submaps are all at max load factor.+  size_t spaceRemaining() const;++  void setEntryCountThreadCacheSize(int32_t newSize) {+    const int numMaps = numMapsAllocated_.load(std::memory_order_acquire);+    for (int i = 0; i < numMaps; ++i) {+      SubMap* map = subMaps_[i].load(std::memory_order_relaxed);+      map->setEntryCountThreadCacheSize(newSize);+    }+  }++  // Number of sub maps allocated so far to implement this map.  The more there+  // are, the worse the performance.+  int numSubMaps() const {+    return numMapsAllocated_.load(std::memory_order_acquire);+  }++  iterator begin() {+    iterator it(this, 0, subMaps_[0].load(std::memory_order_relaxed)->begin());+    it.checkAdvanceToNextSubmap();+    return it;+  }++  const_iterator begin() const {+    const_iterator it(+        this, 0, subMaps_[0].load(std::memory_order_relaxed)->begin());+    it.checkAdvanceToNextSubmap();+    return it;+  }++  iterator end() { return iterator(); }++  const_iterator end() const { return const_iterator(); }++  /* Advanced functions for direct access: */++  inline uint32_t recToIdx(const value_type& r, bool mayInsert = true) {+    SimpleRetT ret =+        mayInsert ? insertInternal(r.first, r.second) : findInternal(r.first);+    return encodeIndex(ret.i, ret.j);+  }++  inline uint32_t recToIdx(value_type&& r, bool mayInsert = true) {+    SimpleRetT ret = mayInsert+        ? insertInternal(r.first, std::move(r.second))+        : findInternal(r.first);+    return encodeIndex(ret.i, ret.j);+  }++  inline uint32_t recToIdx(+      key_type k, const mapped_type& v, bool mayInsert = true) {+    SimpleRetT ret = mayInsert ? insertInternal(k, v) : findInternal(k);+    return encodeIndex(ret.i, ret.j);+  }++  inline uint32_t recToIdx(key_type k, mapped_type&& v, bool mayInsert = true) {+    SimpleRetT ret =+        mayInsert ? insertInternal(k, std::move(v)) : findInternal(k);+    return encodeIndex(ret.i, ret.j);+  }++  inline uint32_t keyToIdx(const KeyT k, bool mayInsert = false) {+    return recToIdx(value_type(k), mayInsert);+  }++  inline const value_type& idxToRec(uint32_t idx) const {+    SimpleRetT ret = findAtInternal(idx);+    return subMaps_[ret.i].load(std::memory_order_relaxed)->idxToRec(ret.j);+  }++  /* Private data and helper functions... */++ private:+  // This limits primary submap size to 2^31 ~= 2 billion, secondary submap+  // size to 2^(32 - kNumSubMapBits_ - 1) = 2^27 ~= 130 million, and num subMaps+  // to 2^kNumSubMapBits_ = 16.+  static const uint32_t kNumSubMapBits_ = 4;+  static const uint32_t kSecondaryMapBit_ = 1u << 31; // Highest bit+  static const uint32_t kSubMapIndexShift_ = 32 - kNumSubMapBits_ - 1;+  static const uint32_t kSubMapIndexMask_ = (1 << kSubMapIndexShift_) - 1;+  static const uint32_t kNumSubMaps_ = 1 << kNumSubMapBits_;+  static const uintptr_t kLockedPtr_ = 0x88ULL << 48; // invalid pointer++  struct SimpleRetT {+    uint32_t i;+    size_t j;+    bool success;+    SimpleRetT(uint32_t ii, size_t jj, bool s) : i(ii), j(jj), success(s) {}+    SimpleRetT() = default;+  };++  template <+      typename LookupKeyT = key_type,+      typename LookupHashFcn = hasher,+      typename LookupEqualFcn = key_equal,+      typename LookupKeyToKeyFcn = key_convert,+      typename... ArgTs>+  SimpleRetT insertInternal(LookupKeyT key, ArgTs&&... value);++  template <+      typename LookupKeyT = key_type,+      typename LookupHashFcn = hasher,+      typename LookupEqualFcn = key_equal>+  SimpleRetT findInternal(const LookupKeyT k) const;++  SimpleRetT findAtInternal(uint32_t idx) const;++  std::atomic<SubMap*> subMaps_[kNumSubMaps_];+  std::atomic<uint32_t> numMapsAllocated_;++  inline bool tryLockMap(unsigned int idx) {+    SubMap* val = nullptr;+    return subMaps_[idx].compare_exchange_strong(+        val, (SubMap*)kLockedPtr_, std::memory_order_acquire);+  }++  static inline uint32_t encodeIndex(uint32_t subMap, uint32_t subMapIdx);++}; // AtomicHashMap++template <+    class KeyT,+    class ValueT,+    class HashFcn = std::hash<KeyT>,+    class EqualFcn = std::equal_to<KeyT>,+    class Allocator = std::allocator<char>>+using QuadraticProbingAtomicHashMap = AtomicHashMap<+    KeyT,+    ValueT,+    HashFcn,+    EqualFcn,+    Allocator,+    AtomicHashArrayQuadraticProbeFcn>;+} // namespace folly++#include <folly/AtomicHashMap-inl.h>
+ folly/folly/AtomicIntrusiveLinkedList.h view
@@ -0,0 +1,206 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <atomic>+#include <cassert>+#include <utility>++namespace folly {++/**+ * A very simple atomic single-linked list primitive.+ *+ * Usage:+ *+ * class MyClass {+ *   AtomicIntrusiveLinkedListHook<MyClass> hook_;+ * }+ *+ * AtomicIntrusiveLinkedList<MyClass, &MyClass::hook_> list;+ * list.insert(&a);+ * list.sweep([] (MyClass* c) { doSomething(c); }+ */+template <class T>+struct AtomicIntrusiveLinkedListHook {+  T* next{nullptr};+};++template <class T, AtomicIntrusiveLinkedListHook<T> T::*HookMember>+class AtomicIntrusiveLinkedList {+ public:+  AtomicIntrusiveLinkedList() {}++  AtomicIntrusiveLinkedList(const AtomicIntrusiveLinkedList&) = delete;+  AtomicIntrusiveLinkedList& operator=(const AtomicIntrusiveLinkedList&) =+      delete;++  AtomicIntrusiveLinkedList(AtomicIntrusiveLinkedList&& other) noexcept+      : head_(other.head_.exchange(nullptr, std::memory_order_acq_rel)) {}++  // Absent because would be too error-prone to use correctly because of+  // the requirement that lists are empty upon destruction.+  AtomicIntrusiveLinkedList& operator=(+      AtomicIntrusiveLinkedList&& other) noexcept = delete;++  /**+   * Move the currently held elements to a new list.+   * The current list becomes empty, but concurrent threads+   * might still add new elements to it.+   *+   * Equivalent to calling a move constructor, but more linter-friendly+   * in case you still need the old list.+   */+  AtomicIntrusiveLinkedList spliceAll() { return std::move(*this); }++  /**+   * Move-assign the current list to `other`, then reverse-sweep+   * the old list with the provided callback `func`.+   *+   * A safe replacement for the move assignment operator, which is absent+   * because of the resource leak concerns.+   */+  template <typename F>+  void reverseSweepAndAssign(AtomicIntrusiveLinkedList&& other, F&& func) {+    auto otherHead = other.head_.exchange(nullptr, std::memory_order_acq_rel);+    auto head = head_.exchange(otherHead, std::memory_order_acq_rel);+    unlinkAll(head, std::forward<F>(func));+  }++  /**+   * Note: The list must be empty on destruction.+   */+  ~AtomicIntrusiveLinkedList() { assert(empty()); }++  /**+   * Returns the current head of the list.+   *+   * WARNING: The returned pointer might not be valid if the list+   * is modified concurrently!+   */+  T* unsafeHead() const { return head_.load(std::memory_order_acquire); }++  /**+   * Returns true if the list is empty.+   *+   * WARNING: This method's return value is only valid for a snapshot+   * of the state, it might become stale as soon as it's returned.+   */+  bool empty() const { return unsafeHead() == nullptr; }++  /**+   * Atomically insert t at the head of the list.+   * @return True if the inserted element is the only one in the list+   *         after the call.+   */+  bool insertHead(T* t) {+    assert(next(t) == nullptr);++    auto oldHead = head_.load(std::memory_order_relaxed);+    do {+      next(t) = oldHead;+      /* oldHead is updated by the call below.++         NOTE: we don't use next(t) instead of oldHead directly due to+         compiler bugs (GCC prior to 4.8.3 (bug 60272), clang (bug 18899),+         MSVC (bug 819819); source:+         http://en.cppreference.com/w/cpp/atomic/atomic/compare_exchange */+    } while (!head_.compare_exchange_weak(+        oldHead, t, std::memory_order_release, std::memory_order_relaxed));++    return oldHead == nullptr;+  }++  /**+   * Replaces the head with nullptr,+   * and calls func() on the removed elements in the order from tail to head.+   * Returns false if the list was empty.+   */+  template <typename F>+  bool sweepOnce(F&& func) {+    if (auto head = head_.exchange(nullptr, std::memory_order_acq_rel)) {+      auto rhead = reverse(head);+      unlinkAll(rhead, std::forward<F>(func));+      return true;+    }+    return false;+  }++  /**+   * Repeatedly replaces the head with nullptr,+   * and calls func() on the removed elements in the order from tail to head.+   * Stops when the list is empty.+   */+  template <typename F>+  void sweep(F&& func) {+    while (sweepOnce(func)) {+    }+  }++  /**+   * Similar to sweep() but calls func() on elements in LIFO order.+   *+   * func() is called for all elements in the list at the moment+   * reverseSweep() is called.  Unlike sweep() it does not loop to ensure the+   * list is empty at some point after the last invocation.  This way callers+   * can reason about the ordering: elements inserted since the last call to+   * reverseSweep() will be provided in LIFO order.+   *+   * Example: if elements are inserted in the order 1-2-3, the callback is+   * invoked 3-2-1.  If the callback moves elements onto a stack, popping off+   * the stack will produce the original insertion order 1-2-3.+   */+  template <typename F>+  void reverseSweep(F&& func) {+    // We don't loop like sweep() does because the overall order of callbacks+    // would be strand-wise LIFO which is meaningless to callers.+    auto head = head_.exchange(nullptr, std::memory_order_acq_rel);+    unlinkAll(head, std::forward<F>(func));+  }++ private:+  std::atomic<T*> head_{nullptr};++  static T*& next(T* t) { return (t->*HookMember).next; }++  /* Reverses a linked list, returning the pointer to the new head+     (old tail) */+  static T* reverse(T* head) {+    T* rhead = nullptr;+    while (head != nullptr) {+      auto t = head;+      head = next(t);+      next(t) = rhead;+      rhead = t;+    }+    return rhead;+  }++  /* Unlinks all elements in the linked list fragment pointed to by `head',+   * calling func() on every element */+  template <typename F>+  static void unlinkAll(T* head, F&& func) {+    while (head != nullptr) {+      auto t = head;+      head = next(t);+      next(t) = nullptr;+      func(t);+    }+  }+};++} // namespace folly
+ folly/folly/AtomicLinkedList.h view
@@ -0,0 +1,128 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <folly/AtomicIntrusiveLinkedList.h>+#include <folly/Memory.h>++namespace folly {++/**+ * A very simple atomic single-linked list primitive.+ *+ * Usage:+ *+ * AtomicLinkedList<MyClass> list;+ * list.insert(a);+ * list.sweep([] (MyClass& c) { doSomething(c); }+ */++template <class T>+class AtomicLinkedList {+ public:+  AtomicLinkedList() {}+  AtomicLinkedList(const AtomicLinkedList&) = delete;+  AtomicLinkedList& operator=(const AtomicLinkedList&) = delete;+  AtomicLinkedList(AtomicLinkedList&& other) noexcept = default;+  AtomicLinkedList& operator=(AtomicLinkedList&& other) noexcept {+    list_.reverseSweepAndAssign(std::move(other.list_), [](Wrapper* node) {+      delete node;+    });+    return *this;+  }++  ~AtomicLinkedList() {+    sweep([](T&&) {});+  }++  bool empty() const { return list_.empty(); }++  /**+   * Atomically insert t at the head of the list.+   * @return True if the inserted element is the only one in the list+   *         after the call.+   */+  bool insertHead(T t) {+    auto wrapper = std::make_unique<Wrapper>(std::move(t));++    return list_.insertHead(wrapper.release());+  }++  /**+   * Repeatedly pops element from head,+   * and calls func() on the removed elements in the order from tail to head.+   * Stops when the list is empty.+   */+  template <typename F>+  void sweep(F&& func) {+    list_.sweep([&](Wrapper* wrapperPtr) mutable {+      std::unique_ptr<Wrapper> wrapper(wrapperPtr);++      func(std::move(wrapper->data));+    });+  }++  /**+   * Sweeps the list a single time, as a single point in time swap with the+   * current contents of the list.+   *+   * Unlike sweep() it does not loop to ensure the list is empty at some point+   * after the last invocation.+   *+   * Returns false if the list is empty.+   */+  template <typename F>+  bool sweepOnce(F&& func) {+    return list_.sweepOnce([&](Wrapper* wrappedPtr) {+      std::unique_ptr<Wrapper> wrapper(wrappedPtr);+      func(std::move(wrapper->data));+    });+  }++  /**+   * Similar to sweep() but calls func() on elements in LIFO order.+   *+   * func() is called for all elements in the list at the moment+   * reverseSweep() is called.  Unlike sweep() it does not loop to ensure the+   * list is empty at some point after the last invocation.  This way callers+   * can reason about the ordering: elements inserted since the last call to+   * reverseSweep() will be provided in LIFO order.+   *+   * Example: if elements are inserted in the order 1-2-3, the callback is+   * invoked 3-2-1.  If the callback moves elements onto a stack, popping off+   * the stack will produce the original insertion order 1-2-3.+   */+  template <typename F>+  void reverseSweep(F&& func) {+    list_.reverseSweep([&](Wrapper* wrapperPtr) mutable {+      std::unique_ptr<Wrapper> wrapper(wrapperPtr);++      func(std::move(wrapper->data));+    });+  }++ private:+  struct Wrapper {+    explicit Wrapper(T&& t) : data(std::move(t)) {}++    AtomicIntrusiveLinkedListHook<Wrapper> hook;+    T data;+  };+  AtomicIntrusiveLinkedList<Wrapper, &Wrapper::hook> list_;+};++} // namespace folly
+ folly/folly/AtomicUnorderedMap.h view
@@ -0,0 +1,521 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <atomic>+#include <cstdint>+#include <functional>+#include <limits>+#include <stdexcept>+#include <system_error>+#include <type_traits>++#include <folly/Conv.h>+#include <folly/Likely.h>+#include <folly/Random.h>+#include <folly/ScopeGuard.h>+#include <folly/Traits.h>+#include <folly/detail/AtomicUnorderedMapUtils.h>+#include <folly/lang/Bits.h>+#include <folly/portability/SysMman.h>+#include <folly/portability/Unistd.h>++namespace folly {++/// You're probably reading this because you are looking for an+/// AtomicUnorderedMap<K,V> that is fully general, highly concurrent (for+/// reads, writes, and iteration), and makes no performance compromises.+/// We haven't figured that one out yet.  What you will find here is a+/// hash table implementation that sacrifices generality so that it can+/// give you all of the other things.+///+/// LIMITATIONS:+///+/// * Insert only (*) - the only write operation supported directly by+///   AtomicUnorderedInsertMap is findOrConstruct.  There is a (*) because+///   values aren't moved, so you can roll your own concurrency control for+///   in-place updates of values (see MutableData and MutableAtom below),+///   but the hash table itself doesn't help you.+///+/// * No resizing - you must specify the capacity up front, and once+///   the hash map gets full you won't be able to insert.  Insert+///   performance will degrade once the load factor is high.  Insert is+///   O(1/(1-actual_load_factor)).  Note that this is a pretty strong+///   limitation, because you can't remove existing keys.+///+/// * 2^30 maximum default capacity - by default AtomicUnorderedInsertMap+///   uses uint32_t internal indexes (and steals 2 bits), limiting you+///   to about a billion entries.  If you need more you can fill in all+///   of the template params so you change IndexType to uint64_t, or you+///   can use AtomicUnorderedInsertMap64.  64-bit indexes will increase+///   the space over of the map, of course.+///+/// WHAT YOU GET IN EXCHANGE:+///+/// * Arbitrary key and value types - any K and V that can be used in a+///   std::unordered_map can be used here.  In fact, the key and value+///   types don't even have to be copyable or moveable!+///+/// * Keys and values in the map won't be moved - it is safe to keep+///   pointers or references to the keys and values in the map, because+///   they are never moved or destroyed (until the map itself is destroyed).+///+/// * Iterators are never invalidated - writes don't invalidate iterators,+///   so you can scan and insert in parallel.+///+/// * Fast wait-free reads - reads are usually only a single cache miss,+///   even when the hash table is very large.  Wait-freedom means that+///   you won't see latency outliers even in the face of concurrent writes.+///+/// * Lock-free insert - writes proceed in parallel.  If a thread in the+///   middle of a write is unlucky and gets suspended, it doesn't block+///   anybody else.+///+/// COMMENTS ON INSERT-ONLY+///+/// This map provides wait-free linearizable reads and lock-free+/// linearizable inserts.  Inserted values won't be moved, but no+/// concurrency control is provided for safely updating them.  To remind+/// you of that fact they are only provided in const form.  This is the+/// only simple safe thing to do while preserving something like the normal+/// std::map iteration form, which requires that iteration be exposed+/// via std::pair (and prevents encapsulation of access to the value).+///+/// There are a couple of reasonable policies for doing in-place+/// concurrency control on the values.  I am hoping that the policy can+/// be injected via the value type or an extra template param, to keep+/// the core AtomicUnorderedInsertMap insert-only:+///+///   CONST: this is the currently implemented strategy, which is simple,+///   performant, and not that expressive.  You can always put in a value+///   with a mutable field (see MutableAtom below), but that doesn't look+///   as pretty as it should.+///+///   ATOMIC: for integers and integer-size trivially copyable structs+///   (via an adapter like tao/queues/AtomicStruct) the value can be a+///   std::atomic and read and written atomically.+///+///   SEQ-LOCK: attach a counter incremented before and after write.+///   Writers serialize by using CAS to make an even->odd transition,+///   then odd->even after the write.  Readers grab the value with memcpy,+///   checking sequence value before and after.  Readers retry until they+///   see an even sequence number that doesn't change.  This works for+///   larger structs, but still requires memcpy to be equivalent to copy+///   assignment, and it is no longer lock-free.  It scales very well,+///   because the readers are still invisible (no cache line writes).+///+///   LOCK: folly's SharedMutex would be a good choice here.+///+/// MEMORY ALLOCATION+///+/// Underlying memory is allocated as a big anonymous mmap chunk, which+/// might be cheaper than calloc() and is certainly not more expensive+/// for large maps.  If the SkipKeyValueDeletion template param is true+/// then deletion of the map consists of unmapping the backing memory,+/// which is much faster than destructing all of the keys and values.+/// Feel free to override if std::is_trivial_destructor isn't recognizing+/// the triviality of your destructors.+template <+    typename Key,+    typename Value,+    typename Hash = std::hash<Key>,+    typename KeyEqual = std::equal_to<Key>,+    bool SkipKeyValueDeletion =+        (std::is_trivially_destructible<Key>::value &&+         std::is_trivially_destructible<Value>::value),+    template <typename> class Atom = std::atomic,+    typename IndexType = uint32_t,+    typename Allocator = folly::detail::MMapAlloc>++struct AtomicUnorderedInsertMap {+  using key_type = Key;+  using mapped_type = Value;+  using value_type = std::pair<Key, Value>;+  using size_type = std::size_t;+  using difference_type = std::ptrdiff_t;+  using hasher = Hash;+  using key_equal = KeyEqual;+  using const_reference = const value_type&;++  struct ConstIterator {+    ConstIterator(const AtomicUnorderedInsertMap& owner, IndexType slot)+        : owner_(owner), slot_(slot) {}++    ConstIterator(const ConstIterator&) = default;+    ConstIterator& operator=(const ConstIterator&) = default;++    const value_type& operator*() const {+      return *owner_.slots_[slot_].keyValue();+    }++    const value_type* operator->() const {+      return owner_.slots_[slot_].keyValue();+    }++    // pre-increment+    const ConstIterator& operator++() {+      while (slot_ > 0) {+        --slot_;+        if (owner_.slots_[slot_].state() == LINKED) {+          break;+        }+      }+      return *this;+    }++    // post-increment+    ConstIterator operator++(int /* dummy */) {+      auto prev = *this;+      ++*this;+      return prev;+    }++    bool operator==(const ConstIterator& rhs) const {+      return slot_ == rhs.slot_;+    }+    bool operator!=(const ConstIterator& rhs) const { return !(*this == rhs); }++   private:+    const AtomicUnorderedInsertMap& owner_;+    IndexType slot_;+  };+  using const_iterator = ConstIterator;++  friend ConstIterator;++  /// Constructs a map that will support the insertion of maxSize key-value+  /// pairs without exceeding the max load factor.  Load factors of greater+  /// than 1 are not supported, and once the actual load factor of the+  /// map approaches 1 the insert performance will suffer.  The capacity+  /// is limited to 2^30 (about a billion) for the default IndexType,+  /// beyond which we will throw invalid_argument.+  explicit AtomicUnorderedInsertMap(+      size_t maxSize,+      float maxLoadFactor = 0.8f,+      const Allocator& alloc = Allocator())+      : allocator_(alloc) {+    size_t capacity = size_t(maxSize / std::min(1.0f, maxLoadFactor) + 128);+    size_t avail = size_t{1} << (8 * sizeof(IndexType) - 2);+    if (capacity > avail && maxSize < avail) {+      // we'll do our best+      capacity = avail;+    }+    if (capacity < maxSize || capacity > avail) {+      throw std::invalid_argument(+          "AtomicUnorderedInsertMap capacity must fit in IndexType with 2 bits "+          "left over");+    }++    numSlots_ = capacity;+    slotMask_ = folly::nextPowTwo(capacity * 4) - 1;+    mmapRequested_ = sizeof(Slot) * capacity;+    slots_ = reinterpret_cast<Slot*>(allocator_.allocate(mmapRequested_));+    zeroFillSlots();+    // mark the zero-th slot as in-use but not valid, since that happens+    // to be our nil value+    slots_[0].stateUpdate(EMPTY, CONSTRUCTING);+  }++  ~AtomicUnorderedInsertMap() {+    if (!SkipKeyValueDeletion) {+      for (size_t i = 1; i < numSlots_; ++i) {+        slots_[i].~Slot();+      }+    }+    allocator_.deallocate(reinterpret_cast<char*>(slots_), mmapRequested_);+  }++  /// Searches for the key, returning (iter,false) if it is found.+  /// If it is not found calls the functor Func with a void* argument+  /// that is raw storage suitable for placement construction of a Value+  /// (see raw_value_type), then returns (iter,true).  May call Func and+  /// then return (iter,false) if there are other concurrent writes, in+  /// which case the newly constructed value will be immediately destroyed.+  ///+  /// This function does not block other readers or writers.  If there+  /// are other concurrent writes, many parallel calls to func may happen+  /// and only the first one to complete will win.  The values constructed+  /// by the other calls to func will be destroyed.+  ///+  /// Usage:+  ///+  ///  AtomicUnorderedInsertMap<std::string,std::string> memo;+  ///+  ///  auto value = memo.findOrConstruct(key, [=](void* raw) {+  ///    new (raw) std::string(computation(key));+  ///  })->first;+  template <typename Func>+  std::pair<const_iterator, bool> findOrConstruct(const Key& key, Func&& func) {+    auto const slot = keyToSlotIdx(key);+    auto prev = slots_[slot].headAndState_.load(std::memory_order_acquire);++    auto existing = find(key, slot);+    if (existing != 0) {+      return std::make_pair(ConstIterator(*this, existing), false);+    }++    // The copying of key and the calling of func and find can throw exceptions.+    // Nothing else in this function can throw an exception. In the event of an+    // exception, deallocate as if the KV was beaten in a concurrent addition.+    const auto idx = allocateNear(slot);+    auto guardSlot = folly::makeGuard([&] {+      slots_[idx].stateUpdate(CONSTRUCTING, EMPTY);+    });+    value_type* addr = slots_[idx].keyValue();+    new (static_cast<void*>(std::addressof(addr->first))) Key(key);+    auto guardKey = folly::makeGuard([&] { addr->first.~Key(); });+    new (static_cast<void*>(std::addressof(addr->second))) Value(func());+    auto guardMapped = folly::makeGuard([&] { addr->second.~Value(); });++    while (true) {+      slots_[idx].next_ = prev >> 2;++      // we can merge the head update and the CONSTRUCTING -> LINKED update+      // into a single CAS if slot == idx (which should happen often)+      auto after = idx << 2;+      if (slot == idx) {+        after += LINKED;+      } else {+        after += (prev & 3);+      }++      if (slots_[slot].headAndState_.compare_exchange_strong(prev, after)) {+        // success+        if (idx != slot) {+          slots_[idx].stateUpdate(CONSTRUCTING, LINKED);+        }+        guardMapped.dismiss();+        guardKey.dismiss();+        guardSlot.dismiss();+        return std::make_pair(ConstIterator(*this, idx), true);+      }+      // compare_exchange_strong updates its first arg on failure, so+      // there is no need to reread prev++      existing = find(key, slot);+      if (existing != 0) {+        // our allocated key and value are no longer needed+        // and so the guards expire and invoke the cleanups+        return std::make_pair(ConstIterator(*this, existing), false);+      }+    }+  }++  /// This isn't really emplace, but it is what we need to test.+  /// Eventually we can duplicate all of the std::pair constructor+  /// forms, including a recursive tuple forwarding template+  /// http://functionalcpp.wordpress.com/2013/08/28/tuple-forwarding/).+  template <class K, class V>+  std::pair<const_iterator, bool> emplace(const K& key, V&& value) {+    return findOrConstruct(key, [&] { return Value(std::forward<V>(value)); });+  }++  const_iterator find(const Key& key) const {+    return ConstIterator(*this, find(key, keyToSlotIdx(key)));+  }++  const_iterator cbegin() const {+    IndexType slot = numSlots_ - 1;+    while (slot > 0 && slots_[slot].state() != LINKED) {+      --slot;+    }+    return ConstIterator(*this, slot);+  }+  const_iterator begin() const { return cbegin(); }++  const_iterator cend() const { return ConstIterator(*this, 0); }+  const_iterator end() const { return cend(); }++ private:+  enum : IndexType {+    kMaxAllocationTries = 1000, // after this we throw+  };++  enum BucketState : IndexType {+    EMPTY = 0,+    CONSTRUCTING = 1,+    LINKED = 2,+  };++  /// Lock-free insertion is easiest by prepending to collision chains.+  /// A large chaining hash table takes two cache misses instead of+  /// one, however.  Our solution is to colocate the bucket storage and+  /// the head storage, so that even though we are traversing chains we+  /// are likely to stay within the same cache line.  Just make sure to+  /// traverse head before looking at any keys.  This strategy gives us+  /// 32 bit pointers and fast iteration.+  struct Slot {+    /// The bottom two bits are the BucketState, the rest is the index+    /// of the first bucket for the chain whose keys map to this slot.+    /// When things are going well the head usually links to this slot,+    /// but that doesn't always have to happen.+    Atom<IndexType> headAndState_;++    /// The next bucket in the chain+    IndexType next_;++    /// Key and Value+    aligned_storage_for_t<value_type> raw_;++    ~Slot() {+      auto s = state();+      assert(s == EMPTY || s == LINKED);+      if (s == LINKED) {+        keyValue()->second.~Value();+        keyValue()->first.~Key();+      }+    }++    BucketState state() const {+      return BucketState(headAndState_.load(std::memory_order_acquire) & 3);+    }++    void stateUpdate(BucketState before, BucketState after) {+      assert(state() == before);+      headAndState_ += (after - before);+    }++    value_type* keyValue() {+      assert(state() != EMPTY);+      return static_cast<value_type*>(static_cast<void*>(&raw_));+    }++    const value_type* keyValue() const {+      assert(state() != EMPTY);+      return static_cast<const value_type*>(static_cast<const void*>(&raw_));+    }+  };++  // We manually manage the slot memory so we can bypass initialization+  // (by getting a zero-filled mmap chunk) and optionally destruction of+  // the slots++  size_t mmapRequested_;+  size_t numSlots_;++  /// tricky, see keyToSlotIdx+  size_t slotMask_;++  Allocator allocator_;+  Slot* slots_;++  IndexType keyToSlotIdx(const Key& key) const {+    size_t h = hasher()(key);+    h &= slotMask_;+    while (h >= numSlots_) {+      h -= numSlots_;+    }+    return h;+  }++  IndexType find(const Key& key, IndexType slot) const {+    KeyEqual ke = {};+    auto hs = slots_[slot].headAndState_.load(std::memory_order_acquire);+    for (slot = hs >> 2; slot != 0; slot = slots_[slot].next_) {+      if (ke(key, slots_[slot].keyValue()->first)) {+        return slot;+      }+    }+    return 0;+  }++  /// Allocates a slot and returns its index.  Tries to put it near+  /// slots_[start].+  IndexType allocateNear(IndexType start) {+    for (IndexType tries = 0; tries < kMaxAllocationTries; ++tries) {+      auto slot = allocationAttempt(start, tries);+      auto prev = slots_[slot].headAndState_.load(std::memory_order_acquire);+      if ((prev & 3) == EMPTY &&+          slots_[slot].headAndState_.compare_exchange_strong(+              prev, prev + CONSTRUCTING - EMPTY)) {+        return slot;+      }+    }+    throw std::bad_alloc();+  }++  /// Returns the slot we should attempt to allocate after tries failed+  /// tries, starting from the specified slot.  This is pulled out so we+  /// can specialize it differently during deterministic testing+  IndexType allocationAttempt(IndexType start, IndexType tries) const {+    if (FOLLY_LIKELY(tries < 8 && start + tries < numSlots_)) {+      return IndexType(start + tries);+    } else {+      IndexType rv;+      if (sizeof(IndexType) <= 4) {+        rv = IndexType(folly::Random::rand32(numSlots_));+      } else {+        rv = IndexType(folly::Random::rand64(numSlots_));+      }+      assert(rv < numSlots_);+      return rv;+    }+  }++  void zeroFillSlots() {+    using folly::detail::GivesZeroFilledMemory;+    if (!GivesZeroFilledMemory<Allocator>::value) {+      memset(static_cast<void*>(slots_), 0, mmapRequested_);+    }+  }+};++/// AtomicUnorderedInsertMap64 is just a type alias that makes it easier+/// to select a 64 bit slot index type.  Use this if you need a capacity+/// bigger than 2^30 (about a billion).  This increases memory overheads,+/// obviously.+template <+    typename Key,+    typename Value,+    typename Hash = std::hash<Key>,+    typename KeyEqual = std::equal_to<Key>,+    bool SkipKeyValueDeletion =+        (std::is_trivially_destructible<Key>::value &&+         std::is_trivially_destructible<Value>::value),+    template <typename> class Atom = std::atomic,+    typename Allocator = folly::detail::MMapAlloc>+using AtomicUnorderedInsertMap64 = AtomicUnorderedInsertMap<+    Key,+    Value,+    Hash,+    KeyEqual,+    SkipKeyValueDeletion,+    Atom,+    uint64_t,+    Allocator>;++/// MutableAtom is a tiny wrapper that gives you the option of atomically+/// updating values inserted into an AtomicUnorderedInsertMap<K,+/// MutableAtom<V>>.  This relies on AtomicUnorderedInsertMap's guarantee+/// that it doesn't move values.+template <typename T, template <typename> class Atom = std::atomic>+struct MutableAtom {+  mutable Atom<T> data;++  explicit MutableAtom(const T& init) : data(init) {}+};++/// MutableData is a tiny wrapper that gives you the option of using an+/// external concurrency control mechanism to updating values inserted+/// into an AtomicUnorderedInsertMap.+template <typename T>+struct MutableData {+  mutable T data;+  explicit MutableData(const T& init) : data(init) {}+};++} // namespace folly
+ folly/folly/Benchmark.h view
@@ -0,0 +1,693 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <folly/BenchmarkUtil.h>+#include <folly/Portability.h>+#include <folly/Preprocessor.h> // for FB_ANONYMOUS_VARIABLE+#include <folly/Range.h>+#include <folly/ScopeGuard.h>+#include <folly/Traits.h>+#include <folly/functional/Invoke.h>+#include <folly/lang/Hint.h>+#include <folly/portability/GFlags.h>++#include <cassert>+#include <chrono>+#include <functional>+#include <iosfwd>+#include <limits>+#include <mutex>+#include <set>+#include <type_traits>+#include <unordered_map>+#include <variant>++#include <boost/function_types/function_arity.hpp>+#include <glog/logging.h>++FOLLY_GFLAGS_DECLARE_bool(benchmark);+FOLLY_GFLAGS_DECLARE_uint32(bm_result_width_chars);++namespace folly {++/**+ * Runs all benchmarks defined. Usually put in main().+ */+void runBenchmarks();++/**+ * Runs all benchmarks defined if and only if the --benchmark flag has+ * been passed to the program. Usually put in main().+ */+inline bool runBenchmarksOnFlag() {+  if (FLAGS_benchmark) {+    runBenchmarks();+  }+  return FLAGS_benchmark;+}++class UserMetric {+ public:+  enum class Type { CUSTOM, TIME, METRIC };+  std::variant<int64_t, double> value;+  Type type{Type::CUSTOM};++  UserMetric() = default;+  /* implicit */ UserMetric(int64_t val, Type typ = Type::CUSTOM)+      : value(val), type(typ) {}++  // Allow users to provide precision values+  template <+      typename T,+      typename = std::enable_if_t<std::is_floating_point_v<T>>>+  explicit UserMetric(T precision_val, Type typ = Type::CUSTOM)+      : value(convert_helper(precision_val)), type(typ) {}++  friend bool operator==(const UserMetric& x, const UserMetric& y) {+    return x.value == y.value && x.type == y.type;+  }+  friend bool operator!=(const UserMetric& x, const UserMetric& y) {+    return !(x == y);+  }++ private:+  double convert_helper(double val) { return val; }+};++using UserCounters = std::unordered_map<std::string, UserMetric>;++namespace detail {+struct TimeIterData {+  std::chrono::high_resolution_clock::duration duration;+  unsigned int niter;+  UserCounters userCounters;+};++using BenchmarkFun = std::function<TimeIterData(unsigned int)>;++struct BenchmarkRegistration {+  std::string file;+  std::string name;+  BenchmarkFun func;+  bool useCounter = false;+};++struct BenchmarkResult {+  std::string file;+  std::string name;+  double timeInNs;+  UserCounters counters;++  friend std::ostream& operator<<(std::ostream&, const BenchmarkResult&);++  friend bool operator==(const BenchmarkResult&, const BenchmarkResult&);+  friend bool operator!=(const BenchmarkResult& x, const BenchmarkResult& y) {+    return !(x == y);+  }+};++struct BenchmarkSuspenderBase {+  /**+   * Accumulates time spent outside benchmark.+   */+  static std::chrono::high_resolution_clock::duration timeSpent;+  static std::chrono::high_resolution_clock::duration suspenderOverhead;+};++template <typename Clock>+struct BenchmarkSuspender : BenchmarkSuspenderBase {+  using TimePoint = std::chrono::high_resolution_clock::time_point;+  using Duration = std::chrono::high_resolution_clock::duration;++  struct DismissedTag {};+  static inline constexpr DismissedTag Dismissed{};++  BenchmarkSuspender() : start(Clock::now()) {}++  explicit BenchmarkSuspender(DismissedTag) : start(TimePoint{}) {}++  BenchmarkSuspender(const BenchmarkSuspender&) = delete;+  BenchmarkSuspender(BenchmarkSuspender&& rhs) noexcept {+    start = rhs.start;+    rhs.start = {};+  }++  BenchmarkSuspender& operator=(const BenchmarkSuspender&) = delete;+  BenchmarkSuspender& operator=(BenchmarkSuspender&& rhs) noexcept {+    if (start != TimePoint{}) {+      tally();+    }+    start = rhs.start;+    rhs.start = {};+    return *this;+  }++  ~BenchmarkSuspender() {+    if (start != TimePoint{}) {+      tally();+    }+  }++  void dismiss() {+    assert(start != TimePoint{});+    tally();+    start = {};+  }++  void rehire() {+    assert(start == TimePoint{});+    start = Clock::now();+  }++  template <class F>+  auto dismissing(F f) -> invoke_result_t<F> {+    SCOPE_EXIT {+      rehire();+    };+    dismiss();+    return f();+  }++  /**+   * This is for use inside of if-conditions, used in BENCHMARK macros.+   * If-conditions bypass the explicit on operator bool.+   */+  explicit operator bool() const { return false; }++ private:+  void tally() {+    auto end = Clock::now();+    timeSpent += (end - start) + suspenderOverhead;+    start = end;+  }++  TimePoint start;+};++class PerfScoped;++class BenchmarkingStateBase {+ public:+  template <typename Printer>+  std::pair<std::set<std::string>, std::vector<BenchmarkResult>>+  runBenchmarksWithPrinter(Printer* printer) const;++  std::vector<BenchmarkResult> runBenchmarksWithResults() const;++  static folly::StringPiece getGlobalBaselineNameForTests();+  static folly::StringPiece getGlobalSuspenderBaselineNameForTests();++  bool useCounters() const;++  void addBenchmarkImpl(+      const char* file, StringPiece name, BenchmarkFun, bool useCounter);++  std::vector<std::string> getBenchmarkList();++ protected:+  // There is no need for this virtual but we overcome a check+  virtual ~BenchmarkingStateBase() = default;++  PerfScoped setUpPerfScoped() const;++  // virtual for purely testing purposes.+  virtual PerfScoped doSetUpPerfScoped(+      const std::vector<std::string>& args) const;++  mutable std::mutex mutex_;+  std::vector<BenchmarkRegistration> benchmarks_;+};++template <typename Clock>+class BenchmarkingState : public BenchmarkingStateBase {+ public:+  template <typename Lambda>+  typename std::enable_if<folly::is_invocable_v<Lambda, unsigned>>::type+  addBenchmark(const char* file, StringPiece name, Lambda&& lambda) {+    auto execute = [=](unsigned int times) {+      BenchmarkSuspender<Clock>::timeSpent = {};+      unsigned int niter;++      // CORE MEASUREMENT STARTS+      auto start = Clock::now();+      niter = lambda(times);+      auto end = Clock::now();+      // CORE MEASUREMENT ENDS+      return detail::TimeIterData{+          (end - start) - BenchmarkSuspender<Clock>::timeSpent,+          niter,+          UserCounters{}};+    };++    this->addBenchmarkImpl(file, name, detail::BenchmarkFun(execute), false);+  }++  template <typename Lambda>+  typename std::enable_if<folly::is_invocable_v<Lambda>>::type addBenchmark(+      const char* file, StringPiece name, Lambda&& lambda) {+    addBenchmark(file, name, [=](unsigned int times) {+      unsigned int niter = 0;+      while (times-- > 0) {+        niter += lambda();+      }+      return niter;+    });+  }++  template <typename Lambda>+  typename std::enable_if<+      folly::is_invocable_v<Lambda, UserCounters&, unsigned>>::type+  addBenchmark(const char* file, StringPiece name, Lambda&& lambda) {+    auto execute = [=](unsigned int times) {+      BenchmarkSuspender<Clock>::timeSpent = {};+      unsigned int niter;++      // CORE MEASUREMENT STARTS+      auto start = std::chrono::high_resolution_clock::now();+      UserCounters counters;+      niter = lambda(counters, times);+      auto end = std::chrono::high_resolution_clock::now();+      // CORE MEASUREMENT ENDS+      return detail::TimeIterData{+          (end - start) - BenchmarkSuspender<Clock>::timeSpent,+          niter,+          counters};+    };++    this->addBenchmarkImpl(+        file,+        name,+        std::function<detail::TimeIterData(unsigned int)>(execute),+        true);+  }++  template <typename Lambda>+  typename std::enable_if<folly::is_invocable_v<Lambda, UserCounters&>>::type+  addBenchmark(const char* file, StringPiece name, Lambda&& lambda) {+    addBenchmark(file, name, [=](UserCounters& counters, unsigned int times) {+      unsigned int niter = 0;+      while (times-- > 0) {+        niter += lambda(counters);+      }+      return niter;+    });+  }+};++BenchmarkingState<std::chrono::high_resolution_clock>& globalBenchmarkState();++/**+ * Runs all benchmarks defined in the program, doesn't print by default.+ * Usually used when customized printing of results is desired.+ */+std::vector<BenchmarkResult> runBenchmarksWithResults();++/**+ * Adds a benchmark wrapped in a std::function.+ * Was designed to only be used internally but, unfortunately,+ * is not.+ */+inline void addBenchmarkImpl(+    const char* file, StringPiece name, BenchmarkFun f, bool useCounter) {+  globalBenchmarkState().addBenchmarkImpl(file, name, std::move(f), useCounter);+}++} // namespace detail++/**+ * Supporting type for BENCHMARK_SUSPEND defined below.+ */+struct BenchmarkSuspender+    : detail::BenchmarkSuspender<std::chrono::high_resolution_clock> {+  using Impl = detail::BenchmarkSuspender<std::chrono::high_resolution_clock>;+  using Impl::Impl;+};++/**+ * Adds a benchmark. Usually not called directly but instead through+ * the macro BENCHMARK defined below.+ * The lambda function involved can have one of the following forms:+ *  * take zero parameters, and the benchmark calls it repeatedly+ *  * take exactly one parameter of type unsigned, and the benchmark+ *    uses it with counter semantics (iteration occurs inside the+ *    function).+ *  * 2 versions of the above cases but also accept UserCounters& as+ *    as their first parameter.+ */+template <typename Lambda>+void addBenchmark(const char* file, StringPiece name, Lambda&& lambda) {+  detail::globalBenchmarkState().addBenchmark(file, name, lambda);+}++struct dynamic;++void benchmarkResultsToDynamic(+    const std::vector<detail::BenchmarkResult>& data, dynamic&);++void benchmarkResultsFromDynamic(+    const dynamic&, std::vector<detail::BenchmarkResult>&);++void printResultComparison(+    const std::vector<detail::BenchmarkResult>& base,+    const std::vector<detail::BenchmarkResult>& test);++} // namespace folly++/**+ * Introduces a benchmark function. Used internally, see BENCHMARK and+ * friends below.+ */++#define BENCHMARK_IMPL(funName, stringName, rv, paramType, paramName)        \+  static void funName(paramType);                                            \+  [[maybe_unused]] static bool FB_ANONYMOUS_VARIABLE(follyBenchmarkUnused) = \+      (::folly::addBenchmark(                                                \+           __FILE__,                                                         \+           stringName,                                                       \+           [](paramType paramName) -> unsigned {                             \+             funName(paramName);                                             \+             return rv;                                                      \+           }),                                                               \+       true);                                                                \+  static void funName(paramType paramName)++#define BENCHMARK_IMPL_COUNTERS(                                             \+    funName, stringName, counters, rv, paramType, paramName)                 \+  static void funName(                                                       \+      ::folly::UserCounters& FOLLY_PP_DETAIL_APPEND_VA_ARG(paramType));      \+  [[maybe_unused]] static bool FB_ANONYMOUS_VARIABLE(follyBenchmarkUnused) = \+      (::folly::addBenchmark(                                                \+           __FILE__,                                                         \+           stringName,                                                       \+           [](::folly::UserCounters& counters FOLLY_PP_DETAIL_APPEND_VA_ARG( \+               paramType paramName)) -> unsigned {                           \+             funName(counters FOLLY_PP_DETAIL_APPEND_VA_ARG(paramName));     \+             return rv;                                                      \+           }),                                                               \+       true);                                                                \+  static void funName(                                                       \+      [[maybe_unused]] ::folly::UserCounters& counters                       \+          FOLLY_PP_DETAIL_APPEND_VA_ARG(paramType paramName))++/**+ * Introduces a benchmark function with support for returning the actual+ * number of iterations. Used internally, see BENCHMARK_MULTI and friends+ * below.+ */+#define BENCHMARK_MULTI_IMPL(funName, stringName, paramType, paramName)      \+  static unsigned funName(paramType);                                        \+  [[maybe_unused]] static bool FB_ANONYMOUS_VARIABLE(follyBenchmarkUnused) = \+      (::folly::addBenchmark(                                                \+           __FILE__,                                                         \+           stringName,                                                       \+           [](paramType paramName) { return funName(paramName); }),          \+       true);                                                                \+  static unsigned funName(paramType paramName)++/**+ * Introduces a benchmark function. Use with either one or two arguments.+ * The first is the name of the benchmark. Use something descriptive, such+ * as insertVectorBegin. The second argument may be missing, or could be a+ * symbolic counter. The counter dictates how many internal iteration the+ * benchmark does. Example:+ *+ * BENCHMARK(vectorPushBack) {+ *   vector<int> v;+ *   v.push_back(42);+ * }+ *+ * BENCHMARK(insertVectorBegin, iters) {+ *   vector<int> v;+ *   for (unsigned int i = 0; i < iters; ++i) {+ *     v.insert(v.begin(), 42);+ *   }+ * }+ */+#define BENCHMARK(name, ...)                   \+  BENCHMARK_IMPL(                              \+      name,                                    \+      FOLLY_PP_STRINGIZE(name),                \+      FB_ARG_2_OR_1(1, ##__VA_ARGS__),         \+      FB_ONE_OR_NONE(unsigned, ##__VA_ARGS__), \+      __VA_ARGS__)++/**+ * Allow users to record customized counter during benchmarking,+ * there will be one extra column showing in the output result for each counter+ *+ * BENCHMARK_COUNTERS(insertVectorBegin, counters, iters) {+ *   vector<int> v;+ *   for (unsigned int i = 0; i < iters; ++i) {+ *     v.insert(v.begin(), 42);+ *   }+ *   BENCHMARK_SUSPEND {+ *      counters["foo"] = 10;+ *   }+ * }+ */+#define BENCHMARK_COUNTERS(name, counters, ...) \+  BENCHMARK_IMPL_COUNTERS(                      \+      name,                                     \+      FOLLY_PP_STRINGIZE(name),                 \+      counters,                                 \+      FB_ARG_2_OR_1(1, ##__VA_ARGS__),          \+      FB_ONE_OR_NONE(unsigned, ##__VA_ARGS__),  \+      __VA_ARGS__)+/**+ * Like BENCHMARK above, but allows the user to return the actual+ * number of iterations executed in the function body. This can be+ * useful if the benchmark function doesn't know upfront how many+ * iterations it's going to run or if it runs through a certain+ * number of test cases, e.g.:+ *+ * BENCHMARK_MULTI(benchmarkSomething) {+ *   std::vector<int> testCases { 0, 1, 1, 2, 3, 5 };+ *   for (int c : testCases) {+ *     doSomething(c);+ *   }+ *   return testCases.size();+ * }+ */+#define BENCHMARK_MULTI(name, ...)             \+  BENCHMARK_MULTI_IMPL(                        \+      name,                                    \+      FOLLY_PP_STRINGIZE(name),                \+      FB_ONE_OR_NONE(unsigned, ##__VA_ARGS__), \+      __VA_ARGS__)++/**+ * Defines a benchmark that passes a parameter to another one. This is+ * common for benchmarks that need a "problem size" in addition to+ * "number of iterations". Consider:+ *+ * void pushBack(uint32_t n, size_t initialSize) {+ *   vector<int> v;+ *   BENCHMARK_SUSPEND {+ *     v.resize(initialSize);+ *   }+ *   for (uint32_t i = 0; i < n; ++i) {+ *    v.push_back(i);+ *   }+ * }+ * BENCHMARK_PARAM(pushBack, 0)+ * BENCHMARK_PARAM(pushBack, 1000)+ * BENCHMARK_PARAM(pushBack, 1000000)+ *+ * The benchmark above estimates the speed of push_back at different+ * initial sizes of the vector. The framework will pass 0, 1000, and+ * 1000000 for initialSize, and the iteration count for n.+ */+#define BENCHMARK_PARAM(name, param) BENCHMARK_NAMED_PARAM(name, param, param)++/**+ * Same as BENCHMARK_PARAM, but allows one to return the actual number of+ * iterations that have been run.+ */+#define BENCHMARK_PARAM_MULTI(name, param) \+  BENCHMARK_NAMED_PARAM_MULTI(name, param, param)++/*+ * Like BENCHMARK_PARAM(), but allows a custom name to be specified for each+ * parameter, rather than using the parameter value.+ *+ * Useful when the parameter value is not a valid token for string pasting,+ * of when you want to specify multiple parameter arguments.+ *+ * For example:+ *+ * void addValue(uint32_t n, int64_t bucketSize, int64_t min, int64_t max) {+ *   Histogram<int64_t> hist(bucketSize, min, max);+ *   int64_t num = min;+ *   for (uint32_t i = 0; i < n; ++i) {+ *     hist.addValue(num);+ *     ++num;+ *     if (num > max) { num = min; }+ *   }+ * }+ *+ * BENCHMARK_NAMED_PARAM(addValue, 0_to_100, 1, 0, 100)+ * BENCHMARK_NAMED_PARAM(addValue, 0_to_1000, 10, 0, 1000)+ * BENCHMARK_NAMED_PARAM(addValue, 5k_to_20k, 250, 5000, 20000)+ */+#define BENCHMARK_NAMED_PARAM(name, param_name, ...)                   \+  BENCHMARK_IMPL(                                                      \+      FB_CONCATENATE(name, FB_CONCATENATE(_, param_name)),             \+      FOLLY_PP_STRINGIZE(name) "(" FOLLY_PP_STRINGIZE(param_name) ")", \+      iters,                                                           \+      unsigned,                                                        \+      iters) {                                                         \+    name(iters, ##__VA_ARGS__);                                        \+  }++/**+ * Same as BENCHMARK_NAMED_PARAM, but allows one to return the actual number+ * of iterations that have been run.+ */+#define BENCHMARK_NAMED_PARAM_MULTI(name, param_name, ...)             \+  BENCHMARK_MULTI_IMPL(                                                \+      FB_CONCATENATE(name, FB_CONCATENATE(_, param_name)),             \+      FOLLY_PP_STRINGIZE(name) "(" FOLLY_PP_STRINGIZE(param_name) ")", \+      unsigned,                                                        \+      iters) {                                                         \+    return name(iters, ##__VA_ARGS__);                                 \+  }++/**+ * Just like BENCHMARK, but prints the time relative to a+ * baseline. The baseline is the most recent BENCHMARK() seen in+ * the current scope. Example:+ *+ * // This is the baseline+ * BENCHMARK(insertVectorBegin, n) {+ *   vector<int> v;+ *   for (unsigned int i = 0; i < n; ++i) {+ *     v.insert(v.begin(), 42);+ *   }+ * }+ *+ * BENCHMARK_RELATIVE(insertListBegin, n) {+ *   list<int> s;+ *   for (unsigned int i = 0; i < n; ++i) {+ *     s.insert(s.begin(), 42);+ *   }+ * }+ *+ * Any number of relative benchmark can be associated with a+ * baseline. Another BENCHMARK() occurrence effectively establishes a+ * new baseline.+ */+#define BENCHMARK_RELATIVE(name, ...)          \+  BENCHMARK_IMPL(                              \+      name,                                    \+      "%" FOLLY_PP_STRINGIZE(name),            \+      FB_ARG_2_OR_1(1, ##__VA_ARGS__),         \+      FB_ONE_OR_NONE(unsigned, ##__VA_ARGS__), \+      __VA_ARGS__)++#define BENCHMARK_COUNTERS_RELATIVE(name, counters, ...) \+  BENCHMARK_IMPL_COUNTERS(                               \+      name,                                              \+      "%" FOLLY_PP_STRINGIZE(name),                      \+      counters,                                          \+      FB_ARG_2_OR_1(1, ##__VA_ARGS__),                   \+      FB_ONE_OR_NONE(unsigned, ##__VA_ARGS__),           \+      __VA_ARGS__)+/**+ * Same as BENCHMARK_RELATIVE, but allows one to return the actual number+ * of iterations that have been run.+ */+#define BENCHMARK_RELATIVE_MULTI(name, ...)    \+  BENCHMARK_MULTI_IMPL(                        \+      name,                                    \+      "%" FOLLY_PP_STRINGIZE(name),            \+      FB_ONE_OR_NONE(unsigned, ##__VA_ARGS__), \+      __VA_ARGS__)++/**+ * A combination of BENCHMARK_RELATIVE and BENCHMARK_PARAM.+ */+#define BENCHMARK_RELATIVE_PARAM(name, param) \+  BENCHMARK_RELATIVE_NAMED_PARAM(name, param, param)++/**+ * Same as BENCHMARK_RELATIVE_PARAM, but allows one to return the actual+ * number of iterations that have been run.+ */+#define BENCHMARK_RELATIVE_PARAM_MULTI(name, param) \+  BENCHMARK_RELATIVE_NAMED_PARAM_MULTI(name, param, param)++/**+ * A combination of BENCHMARK_RELATIVE and BENCHMARK_NAMED_PARAM.+ */+#define BENCHMARK_RELATIVE_NAMED_PARAM(name, param_name, ...)              \+  BENCHMARK_IMPL(                                                          \+      FB_CONCATENATE(name, FB_CONCATENATE(_, param_name)),                 \+      "%" FOLLY_PP_STRINGIZE(name) "(" FOLLY_PP_STRINGIZE(param_name) ")", \+      iters,                                                               \+      unsigned,                                                            \+      iters) {                                                             \+    name(iters, ##__VA_ARGS__);                                            \+  }++/**+ * Same as BENCHMARK_RELATIVE_NAMED_PARAM, but allows one to return the+ * actual number of iterations that have been run.+ */+#define BENCHMARK_RELATIVE_NAMED_PARAM_MULTI(name, param_name, ...)        \+  BENCHMARK_MULTI_IMPL(                                                    \+      FB_CONCATENATE(name, FB_CONCATENATE(_, param_name)),                 \+      "%" FOLLY_PP_STRINGIZE(name) "(" FOLLY_PP_STRINGIZE(param_name) ")", \+      unsigned,                                                            \+      iters) {                                                             \+    return name(iters, ##__VA_ARGS__);                                     \+  }++/**+ * Draws a line of dashes.+ */+#define BENCHMARK_DRAW_LINE()                                                \+  [[maybe_unused]] static bool FB_ANONYMOUS_VARIABLE(follyBenchmarkUnused) = \+      (::folly::addBenchmark(__FILE__, "-", []() -> unsigned { return 0; }), \+       true)++/**+ * Prints arbitrary text.+ */+#define BENCHMARK_DRAW_TEXT(text)                                              \+  [[maybe_unused]] static bool FB_ANONYMOUS_VARIABLE(follyBenchmarkUnused) =   \+      (::folly::addBenchmark(__FILE__, #text, []() -> unsigned { return 0; }), \+       true)++/**+ * Allows execution of code that doesn't count torward the benchmark's+ * time budget. Example:+ *+ * BENCHMARK_START_GROUP(insertVectorBegin, n) {+ *   vector<int> v;+ *   BENCHMARK_SUSPEND {+ *     v.reserve(n);+ *   }+ *   for (unsigned int i = 0; i < n; ++i) {+ *     v.insert(v.begin(), 42);+ *   }+ * }+ */+#define BENCHMARK_SUSPEND                             \+  if (auto FB_ANONYMOUS_VARIABLE(BENCHMARK_SUSPEND) = \+          ::folly::BenchmarkSuspender()) {            \+  } else
+ folly/folly/BenchmarkUtil.h view
@@ -0,0 +1,47 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <folly/Portability.h>+#include <folly/lang/Hint.h>++namespace folly {++/**+ * Call doNotOptimizeAway(var) to ensure that var will be computed even+ * post-optimization.  Use it for variables that are computed during+ * benchmarking but otherwise are useless. The compiler tends to do a+ * good job at eliminating unused variables, and this function fools it+ * into thinking var is in fact needed.+ *+ * Call makeUnpredictable(var) when you don't want the optimizer to use+ * its knowledge of var to shape the following code.  This is useful+ * when constant propagation or power reduction is possible during your+ * benchmark but not in real use cases.+ */++template <class T>+FOLLY_ALWAYS_INLINE void doNotOptimizeAway(const T& datum) {+  compiler_must_not_elide(datum);+}++template <typename T>+FOLLY_ALWAYS_INLINE void makeUnpredictable(T& datum) {+  compiler_must_not_predict(datum);+}++} // namespace folly
+ folly/folly/Bits.h view
@@ -0,0 +1,17 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#include <folly/lang/Bits.h> // @shim
+ folly/folly/CPortability.h view
@@ -0,0 +1,376 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++/* These definitions are in a separate file so that they+ * may be included from C- as well as C++-based projects. */++#include <folly/portability/Config.h>++/**+ * Portable version check.+ */+#ifndef __GNUC_PREREQ+#if defined __GNUC__ && defined __GNUC_MINOR__+/* nolint */+#define __GNUC_PREREQ(maj, min) \+  ((__GNUC__ << 16) + __GNUC_MINOR__ >= ((maj) << 16) + (min))+#else+/* nolint */+#define __GNUC_PREREQ(maj, min) 0+#endif+#endif++// portable version check for clang+#ifndef __CLANG_PREREQ+#if defined __clang__ && defined __clang_major__ && defined __clang_minor__+/* nolint */+#define __CLANG_PREREQ(maj, min) \+  ((__clang_major__ << 16) + __clang_minor__ >= ((maj) << 16) + (min))+#else+/* nolint */+#define __CLANG_PREREQ(maj, min) 0+#endif+#endif++#if defined(__has_builtin)+#define FOLLY_HAS_BUILTIN(...) __has_builtin(__VA_ARGS__)+#else+#define FOLLY_HAS_BUILTIN(...) 0+#endif++#if defined(__has_feature)+#define FOLLY_HAS_FEATURE(...) __has_feature(__VA_ARGS__)+#else+#define FOLLY_HAS_FEATURE(...) 0+#endif++#if defined(__has_warning)+#define FOLLY_HAS_WARNING(...) __has_warning(__VA_ARGS__)+#else+#define FOLLY_HAS_WARNING(...) 0+#endif++/* FOLLY_SANITIZE_ADDRESS is defined to 1 if the current compilation unit+ * is being compiled with ASAN or HWASAN enabled.+ *+ * Beware when using this macro in a header file: this macro may change values+ * across compilation units if some libraries are built with ASAN/HWASAN enabled+ * and some built with ASAN/HWSAN disabled. For instance, this may occur, if+ * folly itself was compiled without ASAN/HWSAN but a downstream project that+ * uses folly is compiling with ASAN/HWSAN enabled.+ *+ * Use FOLLY_LIBRARY_SANITIZE_ADDRESS (defined in folly-config.h) to check if+ * folly itself was compiled with ASAN enabled.+ */+#ifndef FOLLY_SANITIZE_ADDRESS+#if FOLLY_HAS_FEATURE(address_sanitizer) || defined(__SANITIZE_ADDRESS__) || \+    FOLLY_HAS_FEATURE(hwaddress_sanitizer)+#define FOLLY_SANITIZE_ADDRESS 1+#endif+#endif++/* Define attribute wrapper for function attribute used to disable+ * address sanitizer instrumentation. Unfortunately, this attribute+ * has issues when inlining is used, so disable that as well. */+#ifdef FOLLY_SANITIZE_ADDRESS+#if defined(__clang__)+#if __has_attribute(__no_sanitize__)+#define FOLLY_DISABLE_ADDRESS_SANITIZER                     \+  __attribute__((__no_sanitize__("address"), __noinline__)) \+  __attribute__((__no_sanitize__("hwaddress"), __noinline__))+#elif __has_attribute(__no_address_safety_analysis__)+#define FOLLY_DISABLE_ADDRESS_SANITIZER \+  __attribute__((__no_address_safety_analysis__, __noinline__))+#elif __has_attribute(__no_sanitize_address__)+#define FOLLY_DISABLE_ADDRESS_SANITIZER \+  __attribute__((__no_sanitize_address__, __noinline__))+#endif+#elif defined(__GNUC__)+#define FOLLY_DISABLE_ADDRESS_SANITIZER \+  __attribute__((__no_address_safety_analysis__, __noinline__))+#elif defined(_MSC_VER)+#define FOLLY_DISABLE_ADDRESS_SANITIZER __declspec(no_sanitize_address)+#endif+#endif+#ifndef FOLLY_DISABLE_ADDRESS_SANITIZER+#define FOLLY_DISABLE_ADDRESS_SANITIZER+#endif++/* Define a convenience macro to test when thread sanitizer is being used+ * across the different compilers (e.g. clang, gcc) */+#ifndef FOLLY_SANITIZE_THREAD+#if FOLLY_HAS_FEATURE(thread_sanitizer) || defined(__SANITIZE_THREAD__)+#define FOLLY_SANITIZE_THREAD 1+#endif+#endif++#ifdef FOLLY_SANITIZE_THREAD+#define FOLLY_DISABLE_THREAD_SANITIZER \+  __attribute__((no_sanitize_thread, noinline))+#else+#define FOLLY_DISABLE_THREAD_SANITIZER+#endif++/**+ * Define a convenience macro to test when memory sanitizer is being used+ * across the different compilers (e.g. clang, gcc)+ */+#ifndef FOLLY_SANITIZE_MEMORY+#if FOLLY_HAS_FEATURE(memory_sanitizer) || defined(__SANITIZE_MEMORY__)+#define FOLLY_SANITIZE_MEMORY 1+#endif+#endif++#ifdef FOLLY_SANITIZE_MEMORY+#define FOLLY_DISABLE_MEMORY_SANITIZER \+  __attribute__((no_sanitize_memory, noinline))+#else+#define FOLLY_DISABLE_MEMORY_SANITIZER+#endif++/**+ * Define a convenience macro to test when dataflow sanitizer is being used+ * across the different compilers (e.g. clang, gcc)+ */+#ifndef FOLLY_SANITIZE_DATAFLOW+#if FOLLY_HAS_FEATURE(dataflow_sanitizer) || defined(__SANITIZE_DATAFLOW__)+#define FOLLY_SANITIZE_DATAFLOW 1+#endif+#endif++#ifdef FOLLY_SANITIZE_DATAFLOW+#define FOLLY_DISABLE_DATAFLOW_SANITIZER \+  __attribute__((no_sanitize_dataflow, noinline))+#else+#define FOLLY_DISABLE_DATAFLOW_SANITIZER+#endif++/**+ * Define a convenience macro to test when undefined-behavior sanitizer is being+ * used across the different compilers (e.g. clang, gcc)+ */+#ifndef FOLLY_SANITIZE_UNDEFINED_BEHAVIOR+#if FOLLY_HAS_FEATURE(undefined_behavior_sanitizer) || \+    defined(__SANITIZER_UNDEFINED__)+#define FOLLY_SANITIZE_UNDEFINED_BEHAVIOR(...) 1+#endif+#endif++#ifdef FOLLY_SANITIZE_UNDEFINED_BEHAVIOR+#define FOLLY_DISABLE_UNDEFINED_BEHAVIOR_SANITIZER(...) \+  __attribute__((no_sanitize(__VA_ARGS__)))+#else+#define FOLLY_DISABLE_UNDEFINED_BEHAVIOR_SANITIZER(...)+#endif++/**+ * Define a convenience macro to test when ASAN, UBSAN, TSAN or MSAN sanitizer+ * are being used+ */+#ifndef FOLLY_SANITIZE+#if defined(FOLLY_SANITIZE_ADDRESS) || defined(FOLLY_SANITIZE_THREAD) ||  \+    defined(FOLLY_SANITIZE_MEMORY) || defined(FOLLY_SANITIZE_DATAFLOW) || \+    defined(FOLLY_SANITIZE_UNDEFINED_BEHAVIOR)+#define FOLLY_SANITIZE 1+#endif+#endif++#define FOLLY_DISABLE_SANITIZERS  \+  FOLLY_DISABLE_ADDRESS_SANITIZER \+  FOLLY_DISABLE_THREAD_SANITIZER  \+  FOLLY_DISABLE_MEMORY_SANITIZER  \+  FOLLY_DISABLE_UNDEFINED_BEHAVIOR_SANITIZER("undefined")++/**+ * Macro for marking functions as having public visibility.+ */+#if defined(__GNUC__)+#define FOLLY_EXPORT __attribute__((__visibility__("default")))+#else+#define FOLLY_EXPORT+#endif++// noinline+#ifdef _MSC_VER+#define FOLLY_NOINLINE __declspec(noinline)+#elif defined(__HIP_PLATFORM_HCC__)+// HIP software stack defines its own __noinline__ macro.+#define FOLLY_NOINLINE __attribute__((noinline))+#elif defined(__GNUC__)+#define FOLLY_NOINLINE __attribute__((__noinline__))+#else+#define FOLLY_NOINLINE+#endif++// always inline+#ifdef _MSC_VER+#define FOLLY_ALWAYS_INLINE __forceinline+#elif defined(__GNUC__)+#define FOLLY_ALWAYS_INLINE inline __attribute__((__always_inline__))+#else+#define FOLLY_ALWAYS_INLINE inline+#endif++// attribute hidden+#if defined(_MSC_VER)+#define FOLLY_ATTR_VISIBILITY_HIDDEN+#elif defined(__GNUC__)+#define FOLLY_ATTR_VISIBILITY_HIDDEN __attribute__((__visibility__("hidden")))+#else+#define FOLLY_ATTR_VISIBILITY_HIDDEN+#endif++// An attribute for marking symbols as weak, if supported+#if FOLLY_HAVE_WEAK_SYMBOLS+#define FOLLY_ATTR_WEAK __attribute__((__weak__))+#else+#define FOLLY_ATTR_WEAK+#endif++#if defined(__has_attribute)+#if __has_attribute(weak)+#define FOLLY_ATTR_WEAK_SYMBOLS_COMPILE_TIME __attribute__((__weak__))+#else+#define FOLLY_ATTR_WEAK_SYMBOLS_COMPILE_TIME+#endif+#else+#define FOLLY_ATTR_WEAK_SYMBOLS_COMPILE_TIME+#endif++// Microsoft ABI version (can be overridden manually if necessary)+#ifndef FOLLY_MICROSOFT_ABI_VER+#ifdef _MSC_VER+#define FOLLY_MICROSOFT_ABI_VER _MSC_VER+#endif+#endif++// FOLLY_NAME_RESOLVABLE+//+// An attribute that marks a function or variable as needing to be resolvable+// by name. This generally is needed if inline assembly refers to the variable+// by string name.+#ifdef __roar__+#define FOLLY_NAME_RESOLVABLE __attribute__((roar_resolvable_by_name))+#else+#define FOLLY_NAME_RESOLVABLE+#endif++//  FOLLY_ERASE+//+//  A conceptual attribute/syntax combo for erasing a function from the build+//  artifacts and forcing all call-sites to inline the callee, at least as far+//  as each compiler supports.+//+//  Semantically includes the inline specifier.+#define FOLLY_ERASE FOLLY_ALWAYS_INLINE FOLLY_ATTR_VISIBILITY_HIDDEN++//  FOLLY_ERASE_NOINLINE+//+//  Like FOLLY_ERASE, but also noinline. The naming similarity with FOLLY_ERASE+//  is specifically desirable.+#define FOLLY_ERASE_NOINLINE FOLLY_NOINLINE FOLLY_ATTR_VISIBILITY_HIDDEN++//  FOLLY_ERASE_HACK_GCC+//+//  Equivalent to FOLLY_ERASE, but without hiding under gcc. Useful when applied+//  to a function which may sometimes be hidden separately, for example by being+//  declared in an anonymous namespace, since in such cases with -Wattributes+//  enabled, gcc would emit: 'visibility' attribute ignored.+//+//  Semantically includes the inline specifier.+#if defined(__GNUC__) && !defined(__clang__)+#define FOLLY_ERASE_HACK_GCC FOLLY_ALWAYS_INLINE+#else+#define FOLLY_ERASE_HACK_GCC FOLLY_ERASE+#endif++//  FOLLY_ERASE_TRYCATCH+//+//  Equivalent to FOLLY_ERASE, but for code which might contain explicit+//  exception handling. Has the effect of FOLLY_ERASE, except under MSVC which+//  warns about __forceinline when functions contain exception handling.+//+//  Semantically includes the inline specifier.+#ifdef _MSC_VER+#define FOLLY_ERASE_TRYCATCH inline+#else+#define FOLLY_ERASE_TRYCATCH FOLLY_ERASE+#endif++// Generalize warning push/pop.+#if defined(__GNUC__) || defined(__clang__)+// Clang & GCC+#define FOLLY_PUSH_WARNING _Pragma("GCC diagnostic push")+#define FOLLY_POP_WARNING _Pragma("GCC diagnostic pop")+#define FOLLY_GNU_ENABLE_WARNING_INTERNAL2(warningName) #warningName+#define FOLLY_GNU_DISABLE_WARNING_INTERNAL2(warningName) #warningName+#define FOLLY_GNU_ENABLE_ERROR_INTERNAL2(warningName) #warningName+#define FOLLY_GNU_DISABLE_WARNING(warningName) \+  _Pragma(                                     \+      FOLLY_GNU_DISABLE_WARNING_INTERNAL2(GCC diagnostic ignored warningName))+#define FOLLY_GNU_ENABLE_WARNING(warningName) \+  _Pragma(                                    \+      FOLLY_GNU_ENABLE_WARNING_INTERNAL2(GCC diagnostic warning warningName))+#define FOLLY_GNU_ENABLE_ERROR(warningName) \+  _Pragma(FOLLY_GNU_ENABLE_ERROR_INTERNAL2(GCC diagnostic error warningName))+#ifdef __clang__+#define FOLLY_CLANG_DISABLE_WARNING(warningName) \+  FOLLY_GNU_DISABLE_WARNING(warningName)+#define FOLLY_GCC_DISABLE_WARNING(warningName)+#else+#define FOLLY_CLANG_DISABLE_WARNING(warningName)+#define FOLLY_GCC_DISABLE_WARNING(warningName) \+  FOLLY_GNU_DISABLE_WARNING(warningName)+#endif+#define FOLLY_MSVC_DISABLE_WARNING(warningNumber)+#elif defined(_MSC_VER)+#define FOLLY_PUSH_WARNING __pragma(warning(push))+#define FOLLY_POP_WARNING __pragma(warning(pop))+// Disable the GCC warnings.+#define FOLLY_GNU_ENABLE_WARNING(warningName)+#define FOLLY_GNU_DISABLE_WARNING(warningName)+#define FOLLY_GNU_ENABLE_ERROR(warningName)+#define FOLLY_GCC_DISABLE_WARNING(warningName)+#define FOLLY_CLANG_DISABLE_WARNING(warningName)+#define FOLLY_MSVC_DISABLE_WARNING(warningNumber) \+  __pragma(warning(disable : warningNumber))+#else+#define FOLLY_PUSH_WARNING+#define FOLLY_POP_WARNING+#define FOLLY_GNU_ENABLE_WARNING(warningName)+#define FOLLY_GNU_DISABLE_WARNING(warningName)+#define FOLLY_GNU_ENABLE_ERROR(warningName)+#define FOLLY_GCC_DISABLE_WARNING(warningName)+#define FOLLY_CLANG_DISABLE_WARNING(warningName)+#define FOLLY_MSVC_DISABLE_WARNING(warningNumber)+#endif++#ifdef FOLLY_HAVE_SHADOW_LOCAL_WARNINGS+#define FOLLY_GCC_DISABLE_NEW_SHADOW_WARNINGS            \+  FOLLY_GNU_DISABLE_WARNING("-Wshadow-compatible-local") \+  FOLLY_GNU_DISABLE_WARNING("-Wshadow-local")            \+  FOLLY_GNU_DISABLE_WARNING("-Wshadow")+#else+#define FOLLY_GCC_DISABLE_NEW_SHADOW_WARNINGS /* empty */+#endif++#if defined(_MSC_VER)+#define FOLLY_MSVC_DECLSPEC(...) __declspec(__VA_ARGS__)+#else+#define FOLLY_MSVC_DECLSPEC(...)+#endif
+ folly/folly/CancellationToken-inl.h view
@@ -0,0 +1,504 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#include <algorithm>+#include <array>+#include <cstdint>+#include <limits>+#include <tuple>+#include <utility>++#include <glog/logging.h>++namespace folly {++namespace detail {++struct MergingCancellationStateTag {};++// Internal cancellation state object.+class CancellationState {+ public:+  FOLLY_NODISCARD static CancellationStateSourcePtr create();++ protected:+  // Constructed initially with a CancellationSource reference count of 1.+  CancellationState() noexcept;+  // Constructed initially with a CancellationToken reference count of 1.+  explicit CancellationState(MergingCancellationStateTag) noexcept;++  virtual ~CancellationState();++  friend struct CancellationStateTokenDeleter;+  friend struct CancellationStateSourceDeleter;++  void removeTokenReference() noexcept;+  void removeSourceReference() noexcept;++ public:+  FOLLY_NODISCARD CancellationStateTokenPtr addTokenReference() noexcept;++  FOLLY_NODISCARD CancellationStateSourcePtr addSourceReference() noexcept;++  bool tryAddCallback(+      CancellationCallback* callback,+      bool incrementRefCountIfSuccessful) noexcept;++  void removeCallback(CancellationCallback* callback) noexcept;++  bool isCancellationRequested() const noexcept;+  bool canBeCancelled() const noexcept;++  // Request cancellation.+  // Return 'true' if cancellation had already been requested.+  // Return 'false' if this was the first thread to request+  // cancellation.+  bool requestCancellation() noexcept;++ private:+  void lock() noexcept;+  void unlock() noexcept;+  void unlockAndIncrementTokenCount() noexcept;+  void unlockAndDecrementTokenCount() noexcept;+  bool tryLockAndCancelUnlessCancelled() noexcept;++  template <typename Predicate>+  bool tryLock(Predicate predicate) noexcept;++  static bool canBeCancelled(std::uint64_t state) noexcept;+  static bool isCancellationRequested(std::uint64_t state) noexcept;+  static bool isLocked(std::uint64_t state) noexcept;++  static constexpr std::uint64_t kCancellationRequestedFlag = 1;+  static constexpr std::uint64_t kLockedFlag = 2;+  static constexpr std::uint64_t kMergingFlag = 4;+  static constexpr std::uint64_t kTokenReferenceCountIncrement = 8;+  static constexpr std::uint64_t kSourceReferenceCountIncrement =+      std::uint64_t(1) << 34u;+  static constexpr std::uint64_t kTokenReferenceCountMask =+      (kSourceReferenceCountIncrement - 1u) -+      (kTokenReferenceCountIncrement - 1u);+  static constexpr std::uint64_t kSourceReferenceCountMask =+      std::numeric_limits<std::uint64_t>::max() -+      (kSourceReferenceCountIncrement - 1u);++  // Bit 0 - Cancellation Requested+  // Bit 1 - Locked Flag+  // Bit 2 - MergingCancellationState Flag+  // Bits 3-33  - Token reference count (max ~2 billion)+  // Bits 34-63 - Source reference count (max ~1 billion)+  std::atomic<std::uint64_t> state_;+  CancellationCallback* head_{nullptr};+  std::thread::id signallingThreadId_;+};++template <typename... Data>+class CancellationStateWithData : public CancellationState {+  template <typename... Args>+  CancellationStateWithData(Args&&... data);++ public:+  template <typename... Args>+  FOLLY_NODISCARD static std::+      pair<CancellationStateSourcePtr, std::tuple<Data...>*>+      create(Args&&... data);++ private:+  std::tuple<Data...> data_;+};++inline void CancellationStateTokenDeleter::operator()(+    CancellationState* state) noexcept {+  state->removeTokenReference();+}++inline void CancellationStateSourceDeleter::operator()(+    CancellationState* state) noexcept {+  state->removeSourceReference();+}++} // namespace detail++inline CancellationToken::CancellationToken(+    const CancellationToken& other) noexcept+    : state_() {+  if (other.state_) {+    state_ = other.state_->addTokenReference();+  }+}++inline CancellationToken::CancellationToken(CancellationToken&& other) noexcept+    : state_(std::move(other.state_)) {}++inline CancellationToken& CancellationToken::operator=(+    const CancellationToken& other) noexcept {+  if (state_ != other.state_) {+    CancellationToken temp{other};+    swap(temp);+  }+  return *this;+}++inline CancellationToken& CancellationToken::operator=(+    CancellationToken&& other) noexcept {+  state_ = std::move(other.state_);+  return *this;+}++inline bool CancellationToken::isCancellationRequested() const noexcept {+  return state_ != nullptr && state_->isCancellationRequested();+}++inline bool CancellationToken::canBeCancelled() const noexcept {+  return state_ != nullptr && state_->canBeCancelled();+}++inline void CancellationToken::swap(CancellationToken& other) noexcept {+  std::swap(state_, other.state_);+}++inline CancellationToken::CancellationToken(+    detail::CancellationStateTokenPtr state) noexcept+    : state_(std::move(state)) {}++inline bool operator==(+    const CancellationToken& a, const CancellationToken& b) noexcept {+  return a.state_ == b.state_;+}++inline bool operator!=(+    const CancellationToken& a, const CancellationToken& b) noexcept {+  return !(a == b);+}++inline CancellationSource::CancellationSource()+    : state_(detail::CancellationState::create()) {}++inline CancellationSource::CancellationSource(+    const CancellationSource& other) noexcept+    : state_() {+  if (other.state_) {+    state_ = other.state_->addSourceReference();+  }+}++inline CancellationSource::CancellationSource(+    CancellationSource&& other) noexcept+    : state_(std::move(other.state_)) {}++inline CancellationSource& CancellationSource::operator=(+    const CancellationSource& other) noexcept {+  if (state_ != other.state_) {+    CancellationSource temp{other};+    swap(temp);+  }+  return *this;+}++inline CancellationSource& CancellationSource::operator=(+    CancellationSource&& other) noexcept {+  state_ = std::move(other.state_);+  return *this;+}++inline CancellationSource CancellationSource::invalid() noexcept {+  return CancellationSource{detail::CancellationStateSourcePtr{}};+}++inline bool CancellationSource::isCancellationRequested() const noexcept {+  return state_ != nullptr && state_->isCancellationRequested();+}++inline bool CancellationSource::canBeCancelled() const noexcept {+  return state_ != nullptr;+}++inline CancellationToken CancellationSource::getToken() const noexcept {+  if (state_ != nullptr) {+    return CancellationToken{state_->addTokenReference()};+  }+  return CancellationToken{};+}++inline bool CancellationSource::requestCancellation() const noexcept {+  if (state_ != nullptr) {+    return state_->requestCancellation();+  }+  return false;+}++inline void CancellationSource::swap(CancellationSource& other) noexcept {+  std::swap(state_, other.state_);+}++inline CancellationSource::CancellationSource(+    detail::CancellationStateSourcePtr&& state) noexcept+    : state_(std::move(state)) {}++template <+    typename Callable,+    std::enable_if_t<+        std::is_constructible<CancellationCallback::VoidFunction, Callable>::+            value,+        int>>+inline CancellationCallback::CancellationCallback(+    CancellationToken&& ct, Callable&& callable)+    : next_(nullptr),+      prevNext_(nullptr),+      state_(nullptr),+      callback_(static_cast<Callable&&>(callable)),+      destructorHasRunInsideCallback_(nullptr),+      callbackCompleted_(false) {+  if (ct.state_ != nullptr && ct.state_->tryAddCallback(this, false)) {+    state_ = ct.state_.release();+  }+}++template <+    typename Callable,+    std::enable_if_t<+        std::is_constructible<CancellationCallback::VoidFunction, Callable>::+            value,+        int>>+inline CancellationCallback::CancellationCallback(+    const CancellationToken& ct, Callable&& callable)+    : next_(nullptr),+      prevNext_(nullptr),+      state_(nullptr),+      callback_(static_cast<Callable&&>(callable)),+      destructorHasRunInsideCallback_(nullptr),+      callbackCompleted_(false) {+  if (ct.state_ != nullptr && ct.state_->tryAddCallback(this, true)) {+    state_ = ct.state_.get();+  }+}++inline CancellationCallback::~CancellationCallback() {+  if (state_ != nullptr) {+    state_->removeCallback(this);+  }+}++inline void CancellationCallback::invokeCallback() noexcept {+  // Invoke within a noexcept context so that we std::terminate() if it throws.+  callback_();+}++namespace detail {++inline CancellationStateSourcePtr CancellationState::create() {+  return CancellationStateSourcePtr{new CancellationState()};+}++inline CancellationState::CancellationState() noexcept+    : state_(kSourceReferenceCountIncrement) {}+inline CancellationState::CancellationState(+    MergingCancellationStateTag) noexcept+    : state_(kTokenReferenceCountIncrement | kMergingFlag) {}++inline CancellationStateTokenPtr+CancellationState::addTokenReference() noexcept {+  state_.fetch_add(kTokenReferenceCountIncrement, std::memory_order_relaxed);+  return CancellationStateTokenPtr{this};+}++// This `alignas()` is here because "create" will going to allocate this+// back-to-back with our `CancellationCallback` array.+class alignas(CancellationCallback) MergingCancellationState+    : public CancellationState {+  // There's a noticeable perf gain from specializing the constructors+  struct CopyTag {};+  struct MoveTag {};+  struct CopyMoveTag {};++  explicit MergingCancellationState();++  CancellationCallback* callbackEnd_; // byte after the last callback++ public:+  // Ctors are a private implementation detail of the create*() factory funcs+  explicit MergingCancellationState(+      CopyTag, size_t nCopy, const CancellationToken** copyToks);+  explicit MergingCancellationState(+      MoveTag, size_t nMove, CancellationToken** moveToks);+  explicit MergingCancellationState(+      CopyMoveTag,+      size_t nCopy,+      const CancellationToken** copyToks,+      size_t nMove,+      CancellationToken** moveToks);+  ~MergingCancellationState() override;++  static CancellationStateTokenPtr createCopy(+      size_t nCopy, const CancellationToken** copyToks);+  static CancellationStateTokenPtr createMove(+      size_t nMove, CancellationToken** moveToks);+  static CancellationStateTokenPtr createCopyMove(+      size_t nCopy,+      const CancellationToken** copyToks,+      size_t nMove,+      CancellationToken** moveToks);+  void destroy() noexcept;+};++inline void CancellationState::removeTokenReference() noexcept {+  const auto oldState = state_.fetch_sub(+      kTokenReferenceCountIncrement, std::memory_order_acq_rel);+  DCHECK(+      (oldState & kTokenReferenceCountMask) >= kTokenReferenceCountIncrement);+  if (oldState < (2 * kTokenReferenceCountIncrement)) {+    if (oldState & kMergingFlag) {+      static_cast<MergingCancellationState*>(this)->destroy();+    } else {+      delete this;+    }+  }+}++inline CancellationStateSourcePtr+CancellationState::addSourceReference() noexcept {+  state_.fetch_add(kSourceReferenceCountIncrement, std::memory_order_relaxed);+  return CancellationStateSourcePtr{this};+}++inline void CancellationState::removeSourceReference() noexcept {+  const auto oldState = state_.fetch_sub(+      kSourceReferenceCountIncrement, std::memory_order_acq_rel);+  DCHECK(+      (oldState & kSourceReferenceCountMask) >= kSourceReferenceCountIncrement);+  if (oldState <+      (kSourceReferenceCountIncrement + kTokenReferenceCountIncrement)) {+    // No "free()" branch because "merging" state has no source pointers.+    DCHECK(!(oldState & kMergingFlag));+    delete this;+  }+}++inline bool CancellationState::isCancellationRequested() const noexcept {+  return isCancellationRequested(state_.load(std::memory_order_acquire));+}++inline bool CancellationState::canBeCancelled() const noexcept {+  return canBeCancelled(state_.load(std::memory_order_acquire));+}++inline bool CancellationState::canBeCancelled(std::uint64_t state) noexcept {+  // Can be cancelled if there is at least one CancellationSource ref-count+  // or if cancellation has been requested.+  return (state >= kSourceReferenceCountIncrement) ||+      (state & kMergingFlag) != 0 || isCancellationRequested(state);+}++inline bool CancellationState::isCancellationRequested(+    std::uint64_t state) noexcept {+  return (state & kCancellationRequestedFlag) != 0;+}++inline bool CancellationState::isLocked(std::uint64_t state) noexcept {+  return (state & kLockedFlag) != 0;+}++template <typename... Data>+struct WithDataTag {};++template <typename... Data>+template <typename... Args>+CancellationStateWithData<Data...>::CancellationStateWithData(Args&&... data)+    : data_(std::forward<Args>(data)...) {}++template <typename... Data>+template <typename... Args>+std::pair<CancellationStateSourcePtr, std::tuple<Data...>*>+CancellationStateWithData<Data...>::create(Args&&... data) {+  auto* state =+      new CancellationStateWithData<Data...>(std::forward<Args>(data)...);+  return {CancellationStateSourcePtr{state}, &state->data_};+}++} // namespace detail++template <typename... Data, typename... Args>+std::pair<CancellationSource, std::tuple<Data...>*> CancellationSource::create(+    detail::WithDataTag<Data...>, Args&&... data) {+  auto cancellationStateWithData =+      detail::CancellationStateWithData<Data...>::create(+          std::forward<Args>(data)...);+  return {+      CancellationSource{std::move(cancellationStateWithData.first)},+      cancellationStateWithData.second};+}++template <typename... Ts>+inline CancellationToken CancellationToken::merge(Ts&&... tokens) {+  if constexpr (sizeof...(Ts) == 0) {+    return CancellationToken();+  } else if constexpr (sizeof...(Ts) == 1) {+    return (tokens, ...);+  } else {+    constexpr size_t N = sizeof...(Ts);+    constexpr size_t NCopy =+        ((std::is_reference_v<Ts> || std::is_const_v<Ts>)+...);+    std::array<const CancellationToken*, NCopy> copyToks;+    std::array<CancellationToken*, N - NCopy> moveToks;+    const detail::CancellationState* prevState = nullptr;+    size_t copyIdx = 0, moveIdx = 0;+    (+        [&] {+          constexpr bool mustCopy =+              std::is_reference_v<Ts> || std::is_const_v<Ts>;+          auto* state = tokens.state_.get();+          if (!state) {+            return; // Omit empties+          }+          if (state == prevState) {+            if constexpr (!mustCopy) {+              // Move out the input token, although we didn't use it.  The+              // goal is to make deduplication non-observable by the user.+              std::exchange(tokens, {});+            }+            return; // Omit adjacent duplicates+          }+          prevState = state;+          if constexpr (mustCopy) {+            copyToks[copyIdx++] = &tokens;+          } else {+            moveToks[moveIdx++] = &tokens;+          }+        }(),+        ...);++    size_t n = copyIdx + moveIdx;+    if (n == 0) {+      return CancellationToken();+    } else if (n == 1) {+      if (moveIdx) { // A ternary would have type `const CT*` and NOT move!+        return std::move(*moveToks[0]);+      }+      return *copyToks[0];+    } else if constexpr (NCopy == N) {+      return CancellationToken(detail::MergingCancellationState::createCopy(+          copyIdx, copyToks.data()));+    } else if constexpr (NCopy == 0) {+      return CancellationToken(detail::MergingCancellationState::createMove(+          moveIdx, moveToks.data()));+    } else {+      return CancellationToken(detail::MergingCancellationState::createCopyMove(+          copyIdx, copyToks.data(), moveIdx, moveToks.data()));+    }+  }+}++} // namespace folly
+ folly/folly/CancellationToken.cpp view
@@ -0,0 +1,356 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#include <folly/CancellationToken.h>+#include <folly/Optional.h>+#include <folly/ScopeGuard.h>+#include <folly/lang/New.h>+#include <folly/portability/Memory.h>+#include <folly/synchronization/detail/Sleeper.h>++#include <glog/logging.h>++#include <new>+#include <thread>+#include <tuple>++namespace folly {+namespace detail {++CancellationState::~CancellationState() {+  DCHECK(head_ == nullptr);+  DCHECK(!isLocked(state_.load(std::memory_order_relaxed)));+  DCHECK(+      state_.load(std::memory_order_relaxed) < kTokenReferenceCountIncrement);+}++bool CancellationState::tryAddCallback(+    CancellationCallback* callback,+    bool incrementRefCountIfSuccessful) noexcept {+  // Try to acquire the lock, but abandon trying to acquire the lock if+  // cancellation has already been requested (we can just immediately invoke+  // the callback) or if cancellation can never be requested (we can just+  // skip registration).+  if (!tryLock([callback](std::uint64_t oldState) noexcept {+        if (isCancellationRequested(oldState)) {+          callback->invokeCallback();+          return false;+        }+        return canBeCancelled(oldState);+      })) {+    return false;+  }++  // We've acquired the lock and cancellation has not yet been requested.+  // Push this callback onto the head of the list.+  if (head_ != nullptr) {+    head_->prevNext_ = &callback->next_;+  }+  callback->next_ = head_;+  callback->prevNext_ = &head_;+  head_ = callback;++  if (incrementRefCountIfSuccessful) {+    // Combine multiple atomic operations into a single atomic operation.+    unlockAndIncrementTokenCount();+  } else {+    unlock();+  }++  // Successfully added the callback.+  return true;+}++void CancellationState::removeCallback(+    CancellationCallback* callback) noexcept {+  DCHECK(callback != nullptr);++  lock();++  if (callback->prevNext_ != nullptr) {+    // Still registered in the list => not yet executed.+    // Just remove it from the list.+    *callback->prevNext_ = callback->next_;+    if (callback->next_ != nullptr) {+      callback->next_->prevNext_ = callback->prevNext_;+    }++    unlockAndDecrementTokenCount();+    return;+  }++  unlock();++  // Callback has either already executed or is executing concurrently on+  // another thread.++  if (signallingThreadId_ == std::this_thread::get_id()) {+    // Callback executed on this thread or is still currently executing+    // and is deregistering itself from within the callback.+    if (callback->destructorHasRunInsideCallback_ != nullptr) {+      // Currently inside the callback, let the requestCancellation() method+      // know the object is about to be destructed and that it should+      // not try to access the object when the callback returns.+      *callback->destructorHasRunInsideCallback_ = true;+    }+  } else {+    // Callback is currently executing on another thread, block until it+    // finishes executing.+    folly::detail::Sleeper sleeper;+    while (!callback->callbackCompleted_.load(std::memory_order_acquire)) {+      sleeper.wait();+    }+  }++  removeTokenReference();+}++bool CancellationState::requestCancellation() noexcept {+  if (!tryLockAndCancelUnlessCancelled()) {+    // Was already marked as cancelled+    return true;+  }++  // This thread marked as cancelled and acquired the lock++  signallingThreadId_ = std::this_thread::get_id();++  while (head_ != nullptr) {+    // Dequeue the first item on the queue.+    CancellationCallback* callback = head_;+    head_ = callback->next_;+    const bool anyMore = head_ != nullptr;+    if (anyMore) {+      head_->prevNext_ = &head_;+    }+    // Mark this item as removed from the list.+    callback->prevNext_ = nullptr;++    // Don't hold the lock while executing the callback+    // as we don't want to block other threads from+    // deregistering callbacks.+    unlock();++    // TRICKY: Need to store a flag on the stack here that the callback+    // can use to signal that the destructor was executed inline+    // during the call.+    // If the destructor was executed inline then it's not safe to+    // dereference 'callback' after 'invokeCallback()' returns.+    // If the destructor runs on some other thread then the other+    // thread will block waiting for this thread to signal that the+    // callback has finished executing.+    bool destructorHasRunInsideCallback = false;+    callback->destructorHasRunInsideCallback_ = &destructorHasRunInsideCallback;++    callback->invokeCallback();++    if (!destructorHasRunInsideCallback) {+      callback->destructorHasRunInsideCallback_ = nullptr;+      callback->callbackCompleted_.store(true, std::memory_order_release);+    }++    if (!anyMore) {+      // This was the last item in the queue when we dequeued it.+      // No more items should be added to the queue after we have+      // marked the state as cancelled, only removed from the queue.+      // Avoid acquiring/releasing the lock in this case.+      return false;+    }++    lock();+  }++  unlock();++  return false;+}++void CancellationState::lock() noexcept {+  folly::detail::Sleeper sleeper;+  std::uint64_t oldState = state_.load(std::memory_order_relaxed);+  do {+    while (isLocked(oldState)) {+      sleeper.wait();+      oldState = state_.load(std::memory_order_relaxed);+    }+  } while (!state_.compare_exchange_weak(+      oldState,+      oldState | kLockedFlag,+      std::memory_order_acquire,+      std::memory_order_relaxed));+}++void CancellationState::unlock() noexcept {+  state_.fetch_sub(kLockedFlag, std::memory_order_release);+}++void CancellationState::unlockAndIncrementTokenCount() noexcept {+  state_.fetch_sub(+      kLockedFlag - kTokenReferenceCountIncrement, std::memory_order_release);+}++void CancellationState::unlockAndDecrementTokenCount() noexcept {+  auto oldState = state_.fetch_sub(+      kLockedFlag + kTokenReferenceCountIncrement, std::memory_order_acq_rel);+  if (oldState < (kLockedFlag + 2 * kTokenReferenceCountIncrement)) {+    // `MergedTokenDestroyedViaCallback` shows how this is triggered.+    if (UNLIKELY(oldState & kMergingFlag)) {+      static_cast<MergingCancellationState*>(this)->destroy();+    } else {+      delete this;+    }+  }+}++bool CancellationState::tryLockAndCancelUnlessCancelled() noexcept {+  folly::detail::Sleeper sleeper;+  std::uint64_t oldState = state_.load(std::memory_order_acquire);+  while (true) {+    if (isCancellationRequested(oldState)) {+      return false;+    } else if (isLocked(oldState)) {+      sleeper.wait();+      oldState = state_.load(std::memory_order_acquire);+    } else if (state_.compare_exchange_weak(+                   oldState,+                   oldState | kLockedFlag | kCancellationRequestedFlag,+                   std::memory_order_acq_rel,+                   std::memory_order_acquire)) {+      return true;+    }+  }+}++template <typename Predicate>+bool CancellationState::tryLock(Predicate predicate) noexcept {+  folly::detail::Sleeper sleeper;+  std::uint64_t oldState = state_.load(std::memory_order_acquire);+  while (true) {+    if (!predicate(oldState)) {+      return false;+    } else if (isLocked(oldState)) {+      sleeper.wait();+      oldState = state_.load(std::memory_order_acquire);+    } else if (state_.compare_exchange_weak(+                   oldState,+                   oldState | kLockedFlag,+                   std::memory_order_acquire)) {+      return true;+    }+  }+}++// CTOR EXCEPTION SAFETY: In case the `CancellationCallback` ctors below+// throw, we increment `callbackEnd_` as we go.  This ensures that the dtor+// unwinds only the ctors that succeeded.++MergingCancellationState::MergingCancellationState()+    : CancellationState(MergingCancellationStateTag{}),+      callbackEnd_(reinterpret_cast<CancellationCallback*>(this + 1)) {}++MergingCancellationState::MergingCancellationState(+    CopyTag, size_t nCopy, const CancellationToken** copyToks)+    : MergingCancellationState() {+  for (size_t i = 0; i < nCopy; ++i, ++callbackEnd_) {+    new (callbackEnd_) CancellationCallback(*copyToks[i], [this] {+      requestCancellation();+    });+  }+}++MergingCancellationState::MergingCancellationState(+    MoveTag, size_t nMove, CancellationToken** moveToks)+    : MergingCancellationState() {+  for (size_t i = 0; i < nMove; ++i, ++callbackEnd_) {+    new (callbackEnd_) CancellationCallback(std::move(*moveToks[i]), [this] {+      requestCancellation();+    });+  }+}++MergingCancellationState::MergingCancellationState(+    CopyMoveTag,+    size_t nCopy,+    const CancellationToken** copyToks,+    size_t nMove,+    CancellationToken** moveToks)+    : MergingCancellationState(CopyTag{}, nCopy, copyToks) {+  for (size_t i = 0; i < nMove; ++i, ++callbackEnd_) {+    new (callbackEnd_) CancellationCallback(std::move(*moveToks[i]), [this] {+      requestCancellation();+    });+  }+}++namespace {+template <typename... Args>+auto allocAndConstructMergingState(size_t n, Args&&... ctorArgs) {+  DCHECK_GE(n, 2); // `unlockAndDecrementTokenCount` assumes this+  // The merging state uses `alignas` -- this makes the offset math easier.+  static_assert(+      alignof(MergingCancellationState) >= alignof(CancellationCallback));+  // Future: If either type needs extended alignment, you must (1) use aligned+  // `folly::operator_new`, (2) update the alignment math here and in `destroy`.+  static_assert(alignof(MergingCancellationState) <= alignof(std::max_align_t));+  static_assert(alignof(CancellationCallback) <= alignof(std::max_align_t));+  void* p = operator_new( // fundamental alignment suffices per above+      sizeof(MergingCancellationState) + n * sizeof(CancellationCallback));+  // Free memory if the ctor throws. NB: Sized `delete` isn't worth it here.+  auto guard = makeGuard(std::bind(operator_delete, p));+  auto res = CancellationStateTokenPtr{+      new (p) MergingCancellationState(std::forward<Args>(ctorArgs)...)};+  guard.dismiss();+  return res;+}+} // namespace++CancellationStateTokenPtr MergingCancellationState::createCopy(+    size_t nCopy, const CancellationToken** copyToks) {+  return allocAndConstructMergingState(nCopy, CopyTag{}, nCopy, copyToks);+}+CancellationStateTokenPtr MergingCancellationState::createMove(+    size_t nMove, CancellationToken** moveToks) {+  return allocAndConstructMergingState(nMove, MoveTag{}, nMove, moveToks);+}+CancellationStateTokenPtr MergingCancellationState::createCopyMove(+    size_t nCopy,+    const CancellationToken** copyToks,+    size_t nMove,+    CancellationToken** moveToks) {+  return allocAndConstructMergingState(+      nCopy + nMove, CopyMoveTag{}, nCopy, copyToks, nMove, moveToks);+}++MergingCancellationState::~MergingCancellationState() {+  // Arrays are expected to be destroyed in reverse order (although today's+  // `MergeCancellationState` specific ctor does not require it).+  auto callbackStart = reinterpret_cast<CancellationCallback*>(this + 1);+  while (callbackEnd_ > callbackStart) {+    (--callbackEnd_)->~CancellationCallback();+  }+}++void MergingCancellationState::destroy() noexcept {+  // `MergingCancellationState::create` used `operator_new` + in-place `new`+  auto allocSize = reinterpret_cast<std::byte*>(callbackEnd_) -+      reinterpret_cast<std::byte*>(this);+  this->~MergingCancellationState();+  operator_delete(this, allocSize); // Sized `delete` might be 1-2ns faster+}++} // namespace detail++} // namespace folly
+ folly/folly/CancellationToken.h view
@@ -0,0 +1,361 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <folly/CppAttributes.h>+#include <folly/Function.h>+#include <folly/OperationCancelled.h>++#include <atomic>+#include <memory>+#include <thread>+#include <type_traits>++namespace folly {++class CancellationCallback;+class CancellationSource;++namespace detail {+class CancellationState;+struct CancellationStateTokenDeleter {+  void operator()(CancellationState*) noexcept;+};+struct CancellationStateSourceDeleter {+  void operator()(CancellationState*) noexcept;+};+using CancellationStateTokenPtr =+    std::unique_ptr<CancellationState, CancellationStateTokenDeleter>;+using CancellationStateSourcePtr =+    std::unique_ptr<CancellationState, CancellationStateSourceDeleter>;+template <typename...>+struct WithDataTag;+} // namespace detail++/**+ * A CancellationToken is an object that can be passed into an function or+ * operation that allows the caller to later request that the operation be+ * cancelled.+ *+ * A CancellationToken object can be obtained by calling the .getToken()+ * method on a CancellationSource or by copying another CancellationToken+ * object. All CancellationToken objects obtained from the same original+ * CancellationSource object all reference the same underlying cancellation+ * state and will all be cancelled together.+ *+ * If your function needs to be cancellable but does not need to request+ * cancellation then you should take a CancellationToken as a parameter.+ * If your function needs to be able to request cancellation then you+ * should instead take a CancellationSource as a parameter.+ *+ * @refcode folly/docs/examples/folly/CancellationToken.cpp+ * @class folly::CancellationToken+ */+class CancellationToken {+ public:+  /**+   * Constructs to a token that can never be cancelled.+   *+   * Pass a default-constructed CancellationToken into an operation that+   * you never intend to cancel. These objects are very cheap to create.+   */+  CancellationToken() noexcept = default;++  /// Construct a copy of the token that shares the same underlying state.+  CancellationToken(const CancellationToken& other) noexcept;++  /// Construct a token by moving the underlying state+  CancellationToken(CancellationToken&& other) noexcept;++  CancellationToken& operator=(const CancellationToken& other) noexcept;+  CancellationToken& operator=(CancellationToken&& other) noexcept;++  /**+   * Query whether someone has called .requestCancellation() on an instance+   * of CancellationSource object associated with this CancellationToken.+   */+  bool isCancellationRequested() const noexcept;++  /**+   * Query whether this CancellationToken can ever have cancellation requested+   * on it.+   *+   * This will return false if the CancellationToken is not associated with a+   * CancellationSource object. eg. because the CancellationToken was+   * default-constructed, has been moved-from or because the last+   * CancellationSource object associated with the underlying cancellation state+   * has been destroyed and the operation has not yet been cancelled and so+   * never will be.+   *+   * Implementations of operations may be able to take more efficient code-paths+   * if they know they can never be cancelled.+   */+  bool canBeCancelled() const noexcept;++  /**+   * Obtain a CancellationToken linked to any number of other+   * CancellationTokens.+   *+   * This token will have cancellation requested when any of the passed-in+   * tokens do.+   * This token is cancellable if any of the passed-in tokens are at the time of+   * construction.+   */+  template <typename... Ts>+  static CancellationToken merge(Ts&&... tokens);++  /**+   * Swaps the underlying state of the cancellation token with the token that is+   * passed-in.+   */+  void swap(CancellationToken& other) noexcept;++  friend bool operator==(+      const CancellationToken& a, const CancellationToken& b) noexcept;++ private:+  friend class CancellationCallback;+  friend class CancellationSource;++  explicit CancellationToken(detail::CancellationStateTokenPtr state) noexcept;++  detail::CancellationStateTokenPtr state_;+};++bool operator==(+    const CancellationToken& a, const CancellationToken& b) noexcept;+bool operator!=(+    const CancellationToken& a, const CancellationToken& b) noexcept;++/**+ * A CancellationSource object provides the ability to request cancellation of+ * operations that an associated CancellationToken was passed to.+ *+ * @refcode folly/docs/examples/folly/CancellationSource.cpp+ * @class folly::CancellationSource+ */+// Example usage:+//   CancellationSource cs;+//   Future<void> f = startSomeOperation(cs.getToken());+//+//   // Later...+//   cs.requestCancellation();+class CancellationSource {+ public:+  /// Construct to a new, independent cancellation source.+  CancellationSource();++  /**+   * Construct a new reference to the same underlying cancellation state.+   *+   * Either the original or the new copy can be used to request cancellation+   * of associated work.+   */+  CancellationSource(const CancellationSource& other) noexcept;++  /**+   * This leaves 'other' in an empty state where 'requestCancellation()' is a+   * no-op and 'canBeCancelled()' returns false.+   */+  CancellationSource(CancellationSource&& other) noexcept;++  CancellationSource& operator=(const CancellationSource& other) noexcept;+  CancellationSource& operator=(CancellationSource&& other) noexcept;++  /**+   * Construct a CancellationSource that cannot be cancelled.+   *+   * This factory function can be used to obtain a CancellationSource that+   * is equivalent to a moved-from CancellationSource object without needing+   * to allocate any shared-state.+   */+  static CancellationSource invalid() noexcept;++  /**+   * Query if cancellation has already been requested on this CancellationSource+   * or any other CancellationSource object copied from the same original+   * CancellationSource object.+   */+  bool isCancellationRequested() const noexcept;++  /**+   * Query if cancellation can be requested through this CancellationSource+   * object. This will only return false if the CancellationSource object has+   * been moved-from.+   */+  bool canBeCancelled() const noexcept;++  /**+   * Obtain a CancellationToken linked to this CancellationSource.+   *+   * This token can be passed into cancellable operations to allow the caller+   * to later request cancellation of that operation.+   */+  CancellationToken getToken() const noexcept;++  /**+   * Request cancellation of work associated with this CancellationSource.+   *+   * This will ensure subsequent calls to isCancellationRequested() on any+   * CancellationSource or CancellationToken object associated with the same+   * underlying cancellation state to return true.+   *+   * If this is the first call to requestCancellation() on any+   * CancellationSource object with the same underlying state then this call+   * will also execute the callbacks associated with any CancellationCallback+   * objects that were constructed with an associated CancellationToken.+   *+   * Note that it is possible that another thread may be concurrently+   * registering a callback with CancellationCallback. This method guarantees+   * that either this thread will see the callback registration and will+   * ensure that the callback is called, or the CancellationCallback constructor+   * will see the cancellation-requested signal and will execute the callback+   * inline inside the constructor.+   *+   * Returns the previous state of 'isCancellationRequested()'. i.e.+   *  - 'true' if cancellation had previously been requested.+   *  - 'false' if this was the first call to request cancellation.+   */+  bool requestCancellation() const noexcept;++  /**+   * Swaps the underlying state of the cancellation source with the source that+   * is passed-in.+   *+   * @param other The other cancellation source to copy the underlying state+   * from.+   */+  void swap(CancellationSource& other) noexcept;++  friend bool operator==(+      const CancellationSource& a, const CancellationSource& b) noexcept;++  /**+   * Returns a pair of <CancellationSource, Data> where the underlying state is+   * created using the arguments that is passed-in.+   */+  template <typename... Data, typename... Args>+  static std::pair<CancellationSource, std::tuple<Data...>*> create(+      detail::WithDataTag<Data...>, Args&&...);++ private:+  explicit CancellationSource(+      detail::CancellationStateSourcePtr&& state) noexcept;++  detail::CancellationStateSourcePtr state_;+};++bool operator==(+    const CancellationSource& a, const CancellationSource& b) noexcept;+bool operator!=(+    const CancellationSource& a, const CancellationSource& b) noexcept;++/**+ * A CancellationCallback object registers the callback with the specified+ * CancellationToken such that the callback will be+ * executed if the corresponding CancellationSource object has the+ * requestCancellation() method called on it.+ *+ * If the CancellationToken object already had cancellation requested+ * then the callback will be executed inline on the current thread before+ * the constructor returns. Otherwise, the callback will be executed on+ * in the execution context of the first thread to call requestCancellation()+ * on a corresponding CancellationSource.+ *+ * The callback object must not throw any unhandled exceptions. Doing so+ * will result in the program terminating via std::terminate().+ *+ * A CancellationCallback object is neither copyable nor movable.+ *+ * @refcode folly/docs/examples/folly/CancellationCallback.cpp+ * @class folly::CancellationCallback+ */+class CancellationCallback {+  using VoidFunction = folly::Function<void()>;++ public:+  template <+      typename Callable,+      std::enable_if_t<+          std::is_constructible<VoidFunction, Callable>::value,+          int> = 0>+  CancellationCallback(CancellationToken&& ct, Callable&& callable);+  template <+      typename Callable,+      std::enable_if_t<+          std::is_constructible<VoidFunction, Callable>::value,+          int> = 0>+  CancellationCallback(const CancellationToken& ct, Callable&& callable);++  /**+   * Deregisters the callback from the CancellationToken.+   *+   * If cancellation has been requested concurrently on another thread and the+   * callback is currently executing then the destructor will block until after+   * the callback has returned (otherwise it might be left with a dangling+   * reference).+   *+   * You should generally try to implement your callback functions to be lock+   * free to avoid deadlocks between the callback executing and the+   * CancellationCallback destructor trying to deregister the callback.+   *+   * If the callback has not started executing yet then the callback will be+   * deregistered from the CancellationToken before the destructor completes.+   *+   * Once the destructor returns you can be guaranteed that the callback will+   * not be called by a subsequent call to 'requestCancellation()' on a+   * CancellationSource associated with the CancellationToken passed to the+   * constructor.+   */+  ~CancellationCallback();++  // Not copyable/movable+  CancellationCallback(const CancellationCallback&) = delete;+  CancellationCallback(CancellationCallback&&) = delete;+  CancellationCallback& operator=(const CancellationCallback&) = delete;+  CancellationCallback& operator=(CancellationCallback&&) = delete;++ private:+  friend class detail::CancellationState;++  void invokeCallback() noexcept;++  CancellationCallback* next_;++  // Pointer to the pointer that points to this node in the linked list.+  // This could be the 'next_' of a previous CancellationCallback or could+  // be the 'head_' pointer of the CancellationState.+  // If this node is inserted in the list then this will be non-null.+  CancellationCallback** prevNext_;++  detail::CancellationState* state_;+  VoidFunction callback_;++  // Pointer to a flag stored on the stack of the caller to invokeCallback()+  // that is used to indicate to the caller of invokeCallback() that the+  // destructor has run and it is no longer valid to access the callback+  // object.+  bool* destructorHasRunInsideCallback_;++  // Flag used to signal that the callback has completed executing on another+  // thread and it is now safe to exit the destructor.+  std::atomic<bool> callbackCompleted_;+};++} // namespace folly++#include <folly/CancellationToken-inl.h>
+ folly/folly/Chrono.h view
@@ -0,0 +1,132 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <chrono>+#include <ctime>+#include <stdexcept>+#include <type_traits>++#include <folly/Portability.h>+#include <folly/lang/Exception.h>+#include <folly/portability/Time.h>++namespace folly {+namespace chrono {++/* using override */ using std::chrono::abs;+/* using override */ using std::chrono::ceil;+/* using override */ using std::chrono::floor;+/* using override */ using std::chrono::round;++//  steady_clock_spec+//+//  All clocks with this spec share epoch and tick rate.+struct steady_clock_spec {};++//  system_clock_spec+//+//  All clocks with this spec share epoch and tick rate.+struct system_clock_spec {};++//  clock_traits+//+//  Detects and reexports per-clock traits.+//+//  Specializeable for clocks for which trait detection fails..+template <typename Clock>+struct clock_traits {+ private:+  template <typename C>+  using detect_spec_ = typename C::folly_spec;++ public:+  using spec = detected_or_t<void, detect_spec_, Clock>;+};++template <>+struct clock_traits<std::chrono::steady_clock> {+  using spec = steady_clock_spec;+};+template <>+struct clock_traits<std::chrono::system_clock> {+  using spec = system_clock_spec;+};++struct coarse_steady_clock {+  using folly_spec = steady_clock_spec;++  using duration = std::chrono::steady_clock::duration;+  using rep = duration::rep;+  using period = duration::period;+  using time_point = std::chrono::time_point<coarse_steady_clock>;+  constexpr static bool is_steady = true;++  static time_point now() noexcept {+#ifndef CLOCK_MONOTONIC_COARSE+    auto time = std::chrono::steady_clock::now().time_since_epoch();+#else+    timespec ts;+    int ret = clock_gettime(CLOCK_MONOTONIC_COARSE, &ts);+    if (kIsDebug && (ret != 0)) {+      throw_exception<std::runtime_error>(+          "Error using CLOCK_MONOTONIC_COARSE.");+    }+    auto time =+        std::chrono::seconds(ts.tv_sec) + std::chrono::nanoseconds(ts.tv_nsec);+#endif+    return time_point(std::chrono::duration_cast<duration>(time));+  }+};++struct coarse_system_clock {+  using folly_spec = system_clock_spec;++  using duration = std::chrono::system_clock::duration;+  using rep = duration::rep;+  using period = duration::period;+  using time_point = std::chrono::time_point<coarse_system_clock>;+  constexpr static bool is_steady = false;++  static time_point now() noexcept {+#ifndef CLOCK_REALTIME_COARSE+    auto time = std::chrono::system_clock::now().time_since_epoch();+#else+    timespec ts;+    int ret = clock_gettime(CLOCK_REALTIME_COARSE, &ts);+    if (kIsDebug && (ret != 0)) {+      throw_exception<std::runtime_error>("Error using CLOCK_REALTIME_COARSE.");+    }+    auto time =+        std::chrono::seconds(ts.tv_sec) + std::chrono::nanoseconds(ts.tv_nsec);+#endif+    return time_point(std::chrono::duration_cast<duration>(time));+  }++  static std::time_t to_time_t(const time_point& t) noexcept {+    auto d = t.time_since_epoch();+    return std::chrono::duration_cast<std::chrono::seconds>(d).count();+  }++  static time_point from_time_t(std::time_t t) noexcept {+    return time_point(+        std::chrono::duration_cast<duration>(std::chrono::seconds(t)));+  }+};++} // namespace chrono+} // namespace folly
+ folly/folly/ClockGettimeWrappers.cpp view
@@ -0,0 +1,92 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#include <folly/ClockGettimeWrappers.h>++#include <folly/Likely.h>+#include <folly/portability/Time.h>++#include <chrono>++#include <ctime>++#ifndef _WIN32+#define _GNU_SOURCE 1+#include <dlfcn.h>+#endif++namespace folly {+namespace chrono {++static int64_t clock_gettime_ns_fallback(clockid_t clock) {+  struct timespec ts;+  int r = clock_gettime(clock, &ts);+  if (FOLLY_UNLIKELY(r != 0)) {+    // Mimic what __clock_gettime_ns does (even though this can be a legit+    // value).+    return -1;+  }+  std::chrono::nanoseconds result =+      std::chrono::seconds(ts.tv_sec) + std::chrono::nanoseconds(ts.tv_nsec);+  return result.count();+}++// Initialize with default behavior, which we might override on Linux hosts+// with VDSO support.+int (*clock_gettime)(clockid_t, timespec* ts) = &::clock_gettime;+int64_t (*clock_gettime_ns)(clockid_t) = &clock_gettime_ns_fallback;++// In MSAN mode use glibc's versions as they are intercepted by the MSAN+// runtime which properly tracks memory initialization.+#if defined(FOLLY_HAVE_LINUX_VDSO) && !defined(FOLLY_SANITIZE_MEMORY)++namespace {++struct VdsoInitializer {+  VdsoInitializer() {+    m_handle = dlopen("linux-vdso.so.1", RTLD_LAZY | RTLD_LOCAL | RTLD_NOLOAD);+    if (!m_handle) {+      return;+    }++    void* p = dlsym(m_handle, "__vdso_clock_gettime");+    if (p) {+      folly::chrono::clock_gettime = (int (*)(clockid_t, timespec*))p;+    }+    p = dlsym(m_handle, "__vdso_clock_gettime_ns");+    if (p) {+      folly::chrono::clock_gettime_ns = (int64_t(*)(clockid_t))p;+    }+  }++  ~VdsoInitializer() {+    if (m_handle) {+      clock_gettime = &::clock_gettime;+      clock_gettime_ns = &clock_gettime_ns_fallback;+      dlclose(m_handle);+    }+  }++ private:+  void* m_handle;+};++const VdsoInitializer vdso_initializer;+} // namespace++#endif+} // namespace chrono+} // namespace folly
+ folly/folly/ClockGettimeWrappers.h view
@@ -0,0 +1,29 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <folly/portability/Time.h>++#include <time.h>++namespace folly {+namespace chrono {++extern int (*clock_gettime)(clockid_t, timespec* ts);+extern int64_t (*clock_gettime_ns)(clockid_t);+} // namespace chrono+} // namespace folly
+ folly/folly/ConcurrentBitSet.h view
@@ -0,0 +1,157 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <array>+#include <atomic>+#include <cassert>+#include <cstddef>+#include <limits>++#include <folly/Portability.h>++namespace folly {++/**+ * An atomic bitset of fixed size (specified at compile time).+ *+ * Formerly known as AtomicBitSet. It was renamed while fixing a bug+ * to avoid any silent breakages during run time.+ */+template <size_t N>+class ConcurrentBitSet {+ public:+  /**+   * Construct a ConcurrentBitSet; all bits are initially false.+   */+  ConcurrentBitSet();++  ConcurrentBitSet(const ConcurrentBitSet&) = delete;+  ConcurrentBitSet& operator=(const ConcurrentBitSet&) = delete;++  /**+   * Set bit idx to true, using the given memory order. Returns the+   * previous value of the bit.+   *+   * Note that the operation is a read-modify-write operation due to the use+   * of fetch_or.+   */+  bool set(size_t idx, std::memory_order order = std::memory_order_seq_cst);++  /**+   * Set bit idx to false, using the given memory order. Returns the+   * previous value of the bit.+   *+   * Note that the operation is a read-modify-write operation due to the use+   * of fetch_and.+   */+  bool reset(size_t idx, std::memory_order order = std::memory_order_seq_cst);++  /**+   * Set bit idx to the given value, using the given memory order. Returns+   * the previous value of the bit.+   *+   * Note that the operation is a read-modify-write operation due to the use+   * of fetch_and or fetch_or.+   *+   * Yes, this is an overload of set(), to keep as close to std::bitset's+   * interface as possible.+   */+  bool set(+      size_t idx,+      bool value,+      std::memory_order order = std::memory_order_seq_cst);++  /**+   * Read bit idx.+   */+  bool test(+      size_t idx, std::memory_order order = std::memory_order_seq_cst) const;++  /**+   * Same as test() with the default memory order.+   */+  bool operator[](size_t idx) const;++  /**+   * Return the size of the bitset.+   */+  constexpr size_t size() const { return N; }++ private:+  // Pick the largest lock-free type available+#if (ATOMIC_LLONG_LOCK_FREE == 2)+  typedef unsigned long long BlockType;+#elif (ATOMIC_LONG_LOCK_FREE == 2)+  typedef unsigned long BlockType;+#else+  // Even if not lock free, what can we do?+  typedef unsigned int BlockType;+#endif+  typedef std::atomic<BlockType> AtomicBlockType;++  static constexpr size_t kBitsPerBlock =+      std::numeric_limits<BlockType>::digits;++  static constexpr size_t blockIndex(size_t bit) { return bit / kBitsPerBlock; }++  static constexpr size_t bitOffset(size_t bit) { return bit % kBitsPerBlock; }++  // avoid casts+  static constexpr BlockType kOne = 1;+  static constexpr size_t kNumBlocks = (N + kBitsPerBlock - 1) / kBitsPerBlock;+  std::array<AtomicBlockType, kNumBlocks> data_;+};++// value-initialize to zero+template <size_t N>+inline ConcurrentBitSet<N>::ConcurrentBitSet() : data_() {}++template <size_t N>+inline bool ConcurrentBitSet<N>::set(size_t idx, std::memory_order order) {+  assert(idx < N);+  BlockType mask = kOne << bitOffset(idx);+  return data_[blockIndex(idx)].fetch_or(mask, order) & mask;+}++template <size_t N>+inline bool ConcurrentBitSet<N>::reset(size_t idx, std::memory_order order) {+  assert(idx < N);+  BlockType mask = kOne << bitOffset(idx);+  return data_[blockIndex(idx)].fetch_and(~mask, order) & mask;+}++template <size_t N>+inline bool ConcurrentBitSet<N>::set(+    size_t idx, bool value, std::memory_order order) {+  return value ? set(idx, order) : reset(idx, order);+}++template <size_t N>+inline bool ConcurrentBitSet<N>::test(+    size_t idx, std::memory_order order) const {+  assert(idx < N);+  BlockType mask = kOne << bitOffset(idx);+  return data_[blockIndex(idx)].load(order) & mask;+}++template <size_t N>+inline bool ConcurrentBitSet<N>::operator[](size_t idx) const {+  return test(idx);+}++} // namespace folly
+ folly/folly/ConcurrentLazy.h view
@@ -0,0 +1,74 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <type_traits>+#include <utility>++#include <folly/functional/Invoke.h>+#include <folly/synchronization/DelayedInit.h>++namespace folly {++/*+ * ConcurrentLazy is for thread-safe, delayed initialization of a value. This+ * combines the benefits of both `folly::Lazy` and `folly::DelayedInit` to+ * compute the value, once, at access time.+ *+ * There are a few differences between the non-concurrent Lazy, most notably:+ *+ *   - this only safely initializes the value; thread-safety of the underlying+ *     value is left up to the caller.+ *   - the underlying types are not copyable or moveable, which means that this+ *     type is also not copyable or moveable.+ *+ * Otherwise, all design considerations from `folly::Lazy` are reflected here.+ */++template <class Ctor>+struct ConcurrentLazy {+  using result_type = invoke_result_t<Ctor>;++  static_assert(+      !std::is_const<Ctor>::value, "Func should not be a const-qualified type");+  static_assert(+      !std::is_reference<Ctor>::value, "Func should not be a reference type");++  template <+      typename F,+      std::enable_if_t<std::is_constructible_v<Ctor, F>, int> = 0>+  explicit ConcurrentLazy(F&& f) noexcept(+      std::is_nothrow_constructible_v<Ctor, F>)+      : ctor_(static_cast<F&&>(f)) {}++  const result_type& operator()() const {+    return value_.try_emplace_with(std::ref(ctor_));+  }++  result_type& operator()() { return value_.try_emplace_with(std::ref(ctor_)); }++ private:+  mutable folly::DelayedInit<result_type> value_;+  mutable Ctor ctor_;+};++template <class Func>+ConcurrentLazy<remove_cvref_t<Func>> concurrent_lazy(Func&& func) {+  return ConcurrentLazy<remove_cvref_t<Func>>(static_cast<Func&&>(func));+}++} // namespace folly
+ folly/folly/ConcurrentSkipList-inl.h view
@@ -0,0 +1,342 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <algorithm>+#include <atomic>+#include <climits>+#include <cmath>+#include <memory>+#include <mutex>+#include <type_traits>+#include <vector>++#include <boost/random.hpp>+#include <glog/logging.h>++#include <folly/ConstexprMath.h>+#include <folly/Memory.h>+#include <folly/ThreadLocal.h>+#include <folly/synchronization/MicroSpinLock.h>++namespace folly {+namespace detail {++template <typename ValT, typename NodeT>+class csl_iterator;++template <typename T>+class SkipListNode {+  enum : uint16_t {+    IS_HEAD_NODE = 1,+    MARKED_FOR_REMOVAL = (1 << 1),+    FULLY_LINKED = (1 << 2),+  };++ public:+  typedef T value_type;++  SkipListNode(const SkipListNode&) = delete;+  SkipListNode& operator=(const SkipListNode&) = delete;++  template <+      typename NodeAlloc,+      typename U,+      typename =+          typename std::enable_if<std::is_convertible<U, T>::value>::type>+  static SkipListNode* create(+      NodeAlloc& alloc, int height, U&& data, bool isHead = false) {+    DCHECK(height >= 1 && height < 64) << height;++    size_t size =+        sizeof(SkipListNode) + height * sizeof(std::atomic<SkipListNode*>);+    auto storage = std::allocator_traits<NodeAlloc>::allocate(alloc, size);+    // do placement new+    return new (storage)+        SkipListNode(uint8_t(height), std::forward<U>(data), isHead);+  }++  template <typename NodeAlloc>+  static void destroy(NodeAlloc& alloc, SkipListNode* node) {+    size_t size = sizeof(SkipListNode) ++        node->height_ * sizeof(std::atomic<SkipListNode*>);+    node->~SkipListNode();+    std::allocator_traits<NodeAlloc>::deallocate(+        alloc, typename std::allocator_traits<NodeAlloc>::pointer(node), size);+  }++  template <typename NodeAlloc>+  struct DestroyIsNoOp+      : StrictConjunction<+            AllocatorHasTrivialDeallocate<NodeAlloc>,+            std::is_trivially_destructible<SkipListNode>> {};++  // copy the head node to a new head node assuming lock acquired+  SkipListNode* copyHead(SkipListNode* node) {+    DCHECK(node != nullptr && height_ > node->height_);+    setFlags(node->getFlags());+    for (uint8_t i = 0; i < node->height_; ++i) {+      setSkip(i, node->skip(i));+    }+    return this;+  }++  inline SkipListNode* skip(int layer) const {+    DCHECK_LT(layer, height_);+    return skip_[layer].load(std::memory_order_acquire);+  }++  // next valid node as in the linked list+  SkipListNode* next() {+    SkipListNode* node;+    for (node = skip(0); (node != nullptr && node->markedForRemoval());+         node = node->skip(0)) {+    }+    return node;+  }++  void setSkip(uint8_t h, SkipListNode* next) {+    DCHECK_LT(h, height_);+    skip_[h].store(next, std::memory_order_release);+  }++  value_type& data() { return data_; }+  const value_type& data() const { return data_; }+  int maxLayer() const { return height_ - 1; }+  int height() const { return height_; }++  std::unique_lock<MicroSpinLock> acquireGuard() {+    return std::unique_lock<MicroSpinLock>(spinLock_);+  }++  bool fullyLinked() const { return getFlags() & FULLY_LINKED; }+  bool markedForRemoval() const { return getFlags() & MARKED_FOR_REMOVAL; }+  bool isHeadNode() const { return getFlags() & IS_HEAD_NODE; }++  void setIsHeadNode() { setFlags(uint16_t(getFlags() | IS_HEAD_NODE)); }+  void setFullyLinked() { setFlags(uint16_t(getFlags() | FULLY_LINKED)); }+  void setMarkedForRemoval() {+    setFlags(uint16_t(getFlags() | MARKED_FOR_REMOVAL));+  }++ private:+  // Note! this can only be called from create() as a placement new.+  template <typename U>+  SkipListNode(uint8_t height, U&& data, bool isHead)+      : height_(height), data_(std::forward<U>(data)) {+    spinLock_.init();+    setFlags(0);+    if (isHead) {+      setIsHeadNode();+    }+    // need to explicitly init the dynamic atomic pointer array+    for (uint8_t i = 0; i < height_; ++i) {+      new (&skip_[i]) std::atomic<SkipListNode*>(nullptr);+    }+  }++  ~SkipListNode() {+    for (uint8_t i = 0; i < height_; ++i) {+      skip_[i].~atomic();+    }+  }++  uint16_t getFlags() const { return flags_.load(std::memory_order_acquire); }+  void setFlags(uint16_t flags) {+    flags_.store(flags, std::memory_order_release);+  }++  // TODO(xliu): on x86_64, it's possible to squeeze these into+  // skip_[0] to maybe save 8 bytes depending on the data alignments.+  // NOTE: currently this is x86_64 only anyway, due to the+  // MicroSpinLock.+  std::atomic<uint16_t> flags_;+  const uint8_t height_;+  MicroSpinLock spinLock_;++  value_type data_;++  std::atomic<SkipListNode*> skip_[0];+};++class SkipListRandomHeight {+  enum { kMaxHeight = 64 };++ public:+  // make it a singleton.+  static SkipListRandomHeight* instance() {+    static SkipListRandomHeight instance_;+    return &instance_;+  }++  int getHeight(int maxHeight) const {+    DCHECK_LE(maxHeight, kMaxHeight) << "max height too big!";+    double p = randomProb();+    for (int i = 0; i < maxHeight; ++i) {+      if (p < lookupTable_[i]) {+        return i + 1;+      }+    }+    return maxHeight;+  }++  size_t getSizeLimit(int height) const {+    DCHECK_LT(height, kMaxHeight);+    return sizeLimitTable_[height];+  }++ private:+  SkipListRandomHeight() { initLookupTable(); }++  void initLookupTable() {+    // set skip prob = 1/E+    static const double kProbInv = exp(1);+    static const double kProb = 1.0 / kProbInv;+    static const size_t kMaxSizeLimit = std::numeric_limits<size_t>::max();++    double sizeLimit = 1;+    double p = lookupTable_[0] = (1 - kProb);+    sizeLimitTable_[0] = 1;+    for (int i = 1; i < kMaxHeight - 1; ++i) {+      p *= kProb;+      sizeLimit *= kProbInv;+      lookupTable_[i] = lookupTable_[i - 1] + p;+      sizeLimitTable_[i] = folly::constexpr_clamp_cast<size_t>(sizeLimit);+    }+    lookupTable_[kMaxHeight - 1] = 1;+    sizeLimitTable_[kMaxHeight - 1] = kMaxSizeLimit;+  }++  static double randomProb() {+    static ThreadLocal<boost::lagged_fibonacci2281> rng_;+    return (*rng_)();+  }++  double lookupTable_[kMaxHeight];+  size_t sizeLimitTable_[kMaxHeight];+};++template <typename NodeType, typename NodeAlloc, typename = void>+class NodeRecycler;++template <typename NodeType, typename NodeAlloc>+class NodeRecycler<+    NodeType,+    NodeAlloc,+    typename std::enable_if<+        !NodeType::template DestroyIsNoOp<NodeAlloc>::value>::type> {+ public:+  explicit NodeRecycler(const NodeAlloc& alloc)+      : refs_(0), dirty_(false), alloc_(alloc) {+    lock_.init();+  }++  explicit NodeRecycler() : refs_(0), dirty_(false) { lock_.init(); }++  ~NodeRecycler() {+    CHECK_EQ(refs(), 0);+    if (nodes_) {+      for (auto& node : *nodes_) {+        NodeType::destroy(alloc_, node);+      }+    }+  }++  void add(NodeType* node) {+    std::lock_guard g(lock_);+    if (nodes_.get() == nullptr) {+      nodes_ = std::make_unique<std::vector<NodeType*>>(1, node);+    } else {+      nodes_->push_back(node);+    }+    DCHECK_GT(refs(), 0);+    dirty_.store(true, std::memory_order_relaxed);+  }++  int addRef() { return refs_.fetch_add(1, std::memory_order_acq_rel); }++  int releaseRef() {+    // This if statement is purely an optimization. It's possible that this+    // misses an opportunity to delete, but that's OK, we'll try again at+    // the next opportunity. It does not harm the thread safety. For this+    // reason, we can use relaxed loads to make the decision.+    if (!dirty_.load(std::memory_order_relaxed) || refs() > 1) {+      return refs_.fetch_add(-1, std::memory_order_acq_rel);+    }++    std::unique_ptr<std::vector<NodeType*>> newNodes;+    int ret;+    {+      // The order at which we lock, add, swap, is very important for+      // correctness.+      std::lock_guard g(lock_);+      ret = refs_.fetch_add(-1, std::memory_order_acq_rel);+      if (ret == 1) {+        // When releasing the last reference, it is safe to remove all the+        // current nodes in the recycler, as we already acquired the lock here+        // so no more new nodes can be added, even though new accessors may be+        // added after this.+        newNodes.swap(nodes_);+        dirty_.store(false, std::memory_order_relaxed);+      }+    }+    // TODO(xliu) should we spawn a thread to do this when there are large+    // number of nodes in the recycler?+    if (newNodes) {+      for (auto& node : *newNodes) {+        NodeType::destroy(alloc_, node);+      }+    }+    return ret;+  }++  NodeAlloc& alloc() { return alloc_; }++ private:+  int refs() const { return refs_.load(std::memory_order_relaxed); }++  std::unique_ptr<std::vector<NodeType*>> nodes_;+  std::atomic<int32_t> refs_; // current number of visitors to the list+  std::atomic<bool> dirty_; // whether *nodes_ is non-empty+  MicroSpinLock lock_; // protects access to *nodes_+  NodeAlloc alloc_;+};++// In case of arena allocator, no recycling is necessary, and it's possible+// to save on ConcurrentSkipList size.+template <typename NodeType, typename NodeAlloc>+class NodeRecycler<+    NodeType,+    NodeAlloc,+    typename std::enable_if<+        NodeType::template DestroyIsNoOp<NodeAlloc>::value>::type> {+ public:+  explicit NodeRecycler(const NodeAlloc& alloc) : alloc_(alloc) {}++  void addRef() {}+  void releaseRef() {}++  void add(NodeType* /* node */) {}++  NodeAlloc& alloc() { return alloc_; }++ private:+  NodeAlloc alloc_;+};++} // namespace detail+} // namespace folly
+ folly/folly/ConcurrentSkipList.h view
@@ -0,0 +1,828 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++// A concurrent skip list (CSL) implementation.+// Ref: http://www.cs.tau.ac.il/~shanir/nir-pubs-web/Papers/OPODIS2006-BA.pdf++/*++This implements a sorted associative container that supports only+unique keys.  (Similar to std::set.)++Features:++  1. Small memory overhead: ~40% less memory overhead compared with+     std::set (1.6 words per node versus 3). It has an minimum of 4+     words (7 words if there nodes got deleted) per-list overhead+     though.++  2. Read accesses (count, find iterator, skipper) are lock-free and+     mostly wait-free (the only wait a reader may need to do is when+     the node it is visiting is in a pending stage, i.e. deleting,+     adding and not fully linked).  Write accesses (remove, add) need+     to acquire locks, but locks are local to the predecessor nodes+     and/or successor nodes.++  3. Good high contention performance, comparable single-thread+     performance.  In the multithreaded case (12 workers), CSL tested+     10x faster than a RWSpinLocked std::set for an averaged sized+     list (1K - 1M nodes).++     Comparable read performance to std::set when single threaded,+     especially when the list size is large, and scales better to+     larger lists: when the size is small, CSL can be 20-50% slower on+     find()/contains().  As the size gets large (> 1M elements),+     find()/contains() can be 30% faster.++     Iterating through a skiplist is similar to iterating through a+     linked list, thus is much (2-6x) faster than on a std::set+     (tree-based).  This is especially true for short lists due to+     better cache locality.  Based on that, it's also faster to+     intersect two skiplists.++  4. Lazy removal with GC support.  The removed nodes get deleted when+     the last Accessor to the skiplist is destroyed.++Caveats:++  1. Write operations are usually 30% slower than std::set in a single+     threaded environment.++  2. Need to have a head node for each list, which has a 4 word+     overhead.++  3. When the list is quite small (< 1000 elements), single threaded+     benchmarks show CSL can be 10x slower than std:set.++  4. The interface requires using an Accessor to access the skiplist.+    (See below.)++  5. Currently x64 only, due to use of MicroSpinLock.++  6. Freed nodes will not be reclaimed as long as there are ongoing+     uses of the list.++Sample usage:++     typedef ConcurrentSkipList<int> SkipListT;+     shared_ptr<SkipListT> sl(SkipListT::createInstance(init_head_height);+     {+       // It's usually good practice to hold an accessor only during+       // its necessary life cycle (but not in a tight loop as+       // Accessor creation incurs ref-counting overhead).+       //+       // Holding it longer delays garbage-collecting the deleted+       // nodes in the list.+       SkipListT::Accessor accessor(sl);+       accessor.insert(23);+       accessor.erase(2);+       for (auto &elem : accessor) {+         // use elem to access data+       }+       ... ...+     }++ Another useful type is the Skipper accessor.  This is useful if you+ want to skip to locations in the way std::lower_bound() works,+ i.e. it can be used for going through the list by skipping to the+ node no less than a specified key.  The Skipper keeps its location as+ state, which makes it convenient for things like implementing+ intersection of two sets efficiently, as it can start from the last+ visited position.++     {+       SkipListT::Accessor accessor(sl);+       SkipListT::Skipper skipper(accessor);+       skipper.to(30);+       if (skipper) {+         CHECK_LE(30, *skipper);+       }+       ...  ...+       // GC may happen when the accessor gets destructed.+     }+*/++#pragma once++#include <algorithm>+#include <atomic>+#include <limits>+#include <memory>+#include <type_traits>++#include <glog/logging.h>++#include <folly/ConcurrentSkipList-inl.h>+#include <folly/Likely.h>+#include <folly/Memory.h>+#include <folly/detail/Iterators.h>+#include <folly/synchronization/MicroSpinLock.h>++namespace folly {++template <+    typename T,+    typename Comp = std::less<T>,+    // All nodes are allocated using provided SysAllocator,+    // it should be thread-safe.+    typename NodeAlloc = SysAllocator<char>,+    int MAX_HEIGHT = 24>+class ConcurrentSkipList {+  // MAX_HEIGHT needs to be at least 2 to suppress compiler+  // warnings/errors (Werror=uninitialized triggered due to preds_[1]+  // being treated as a scalar in the compiler).+  static_assert(+      MAX_HEIGHT >= 2 && MAX_HEIGHT < 64,+      "MAX_HEIGHT can only be in the range of [2, 64)");+  typedef std::unique_lock<folly::MicroSpinLock> ScopedLocker;+  typedef ConcurrentSkipList<T, Comp, NodeAlloc, MAX_HEIGHT> SkipListType;++ public:+  typedef detail::SkipListNode<T> NodeType;+  typedef T value_type;+  typedef T key_type;++  typedef detail::csl_iterator<value_type, NodeType> iterator;+  typedef detail::csl_iterator<const value_type, NodeType> const_iterator;++  class Accessor;+  class Skipper;++  explicit ConcurrentSkipList(int height, const NodeAlloc& alloc)+      : recycler_(alloc),+        head_(NodeType::create(recycler_.alloc(), height, value_type(), true)) {+  }++  explicit ConcurrentSkipList(int height)+      : recycler_(),+        head_(NodeType::create(recycler_.alloc(), height, value_type(), true)) {+  }++  // Convenience function to get an Accessor to a new instance.+  static Accessor create(int height, const NodeAlloc& alloc) {+    return Accessor(createInstance(height, alloc));+  }++  static Accessor create(int height = 1) {+    return Accessor(createInstance(height));+  }++  // Create a shared_ptr skiplist object with initial head height.+  static std::shared_ptr<SkipListType> createInstance(+      int height, const NodeAlloc& alloc) {+    return std::make_shared<ConcurrentSkipList>(height, alloc);+  }++  static std::shared_ptr<SkipListType> createInstance(int height = 1) {+    return std::make_shared<ConcurrentSkipList>(height);+  }++  size_t size() const { return size_.load(std::memory_order_relaxed); }+  bool empty() const { return size() == 0; }++  //===================================================================+  // Below are implementation details.+  // Please see ConcurrentSkipList::Accessor for stdlib-like APIs.+  //===================================================================++  ~ConcurrentSkipList() {+    if constexpr (NodeType::template DestroyIsNoOp<NodeAlloc>::value) {+      // Avoid traversing the list if using arena allocator.+      return;+    }+    for (NodeType* current = head_.load(std::memory_order_relaxed); current;) {+      NodeType* tmp = current->skip(0);+      NodeType::destroy(recycler_.alloc(), current);+      current = tmp;+    }+  }++ private:+  static bool greater(const value_type& data, const NodeType* node) {+    return node && Comp()(node->data(), data);+  }++  static bool less(const value_type& data, const NodeType* node) {+    return (node == nullptr) || Comp()(data, node->data());+  }++  static int findInsertionPoint(+      NodeType* cur,+      int cur_layer,+      const value_type& data,+      NodeType* preds[],+      NodeType* succs[]) {+    int foundLayer = -1;+    NodeType* pred = cur;+    NodeType* foundNode = nullptr;+    for (int layer = cur_layer; layer >= 0; --layer) {+      NodeType* node = pred->skip(layer);+      while (greater(data, node)) {+        pred = node;+        node = node->skip(layer);+      }+      if (foundLayer == -1 && !less(data, node)) { // the two keys equal+        foundLayer = layer;+        foundNode = node;+      }+      preds[layer] = pred;++      // if found, succs[0..foundLayer] need to point to the cached foundNode,+      // as foundNode might be deleted at the same time thus pred->skip() can+      // return nullptr or another node.+      succs[layer] = foundNode ? foundNode : node;+    }+    return foundLayer;+  }++  int height() const { return head_.load(std::memory_order_acquire)->height(); }++  int maxLayer() const { return height() - 1; }++  size_t incrementSize(int delta) {+    return size_.fetch_add(delta, std::memory_order_relaxed) + delta;+  }++  // Returns the node if found, nullptr otherwise.+  NodeType* find(const value_type& data) {+    auto ret = findNode(data);+    if (ret.second && !ret.first->markedForRemoval()) {+      return ret.first;+    }+    return nullptr;+  }++  // lock all the necessary nodes for changing (adding or removing) the list.+  // returns true if all the lock acquired successfully and the related nodes+  // are all validate (not in certain pending states), false otherwise.+  bool lockNodesForChange(+      int nodeHeight,+      ScopedLocker guards[MAX_HEIGHT],+      NodeType* preds[MAX_HEIGHT],+      NodeType* succs[MAX_HEIGHT],+      bool adding = true) {+    NodeType *pred, *succ, *prevPred = nullptr;+    bool valid = true;+    for (int layer = 0; valid && layer < nodeHeight; ++layer) {+      pred = preds[layer];+      DCHECK(pred != nullptr)+          << "layer=" << layer << " height=" << height()+          << " nodeheight=" << nodeHeight;+      succ = succs[layer];+      if (pred != prevPred) {+        guards[layer] = pred->acquireGuard();+        prevPred = pred;+      }+      valid = !pred->markedForRemoval() &&+          pred->skip(layer) == succ; // check again after locking++      if (adding) { // when adding a node, the succ shouldn't be going away+        valid = valid && (succ == nullptr || !succ->markedForRemoval());+      }+    }++    return valid;+  }++  // Returns a paired value:+  //   pair.first always stores the pointer to the node with the same input key.+  //     It could be either the newly added data, or the existed data in the+  //     list with the same key.+  //   pair.second stores whether the data is added successfully:+  //     0 means not added, otherwise returns the new size.+  template <typename U>+  std::pair<NodeType*, size_t> addOrGetData(U&& data) {+    NodeType *preds[MAX_HEIGHT], *succs[MAX_HEIGHT];+    NodeType* newNode;+    size_t newSize;+    while (true) {+      int max_layer = 0;+      int layer = findInsertionPointGetMaxLayer(data, preds, succs, &max_layer);++      if (layer >= 0) {+        NodeType* nodeFound = succs[layer];+        DCHECK(nodeFound != nullptr);+        if (nodeFound->markedForRemoval()) {+          continue; // if it's getting deleted retry finding node.+        }+        // wait until fully linked.+        while (FOLLY_UNLIKELY(!nodeFound->fullyLinked())) {+        }+        return std::make_pair(nodeFound, 0);+      }++      // need to capped at the original height -- the real height may have grown+      int nodeHeight =+          detail::SkipListRandomHeight::instance()->getHeight(max_layer + 1);++      ScopedLocker guards[MAX_HEIGHT];+      if (!lockNodesForChange(nodeHeight, guards, preds, succs)) {+        continue; // give up the locks and retry until all valid+      }++      // locks acquired and all valid, need to modify the links under the locks.+      newNode = NodeType::create(+          recycler_.alloc(), nodeHeight, std::forward<U>(data));+      for (int k = 0; k < nodeHeight; ++k) {+        newNode->setSkip(k, succs[k]);+        preds[k]->setSkip(k, newNode);+      }++      newNode->setFullyLinked();+      newSize = incrementSize(1);+      break;+    }++    int hgt = height();+    size_t sizeLimit =+        detail::SkipListRandomHeight::instance()->getSizeLimit(hgt);++    if (hgt < MAX_HEIGHT && newSize > sizeLimit) {+      growHeight(hgt + 1);+    }+    CHECK_GT(newSize, 0);+    return std::make_pair(newNode, newSize);+  }++  bool remove(const value_type& data) {+    NodeType* nodeToDelete = nullptr;+    ScopedLocker nodeGuard;+    bool isMarked = false;+    int nodeHeight = 0;+    NodeType *preds[MAX_HEIGHT], *succs[MAX_HEIGHT];++    while (true) {+      int max_layer = 0;+      int layer = findInsertionPointGetMaxLayer(data, preds, succs, &max_layer);+      if (!isMarked && (layer < 0 || !okToDelete(succs[layer], layer))) {+        return false;+      }++      if (!isMarked) {+        nodeToDelete = succs[layer];+        nodeHeight = nodeToDelete->height();+        nodeGuard = nodeToDelete->acquireGuard();+        if (nodeToDelete->markedForRemoval()) {+          return false;+        }+        nodeToDelete->setMarkedForRemoval();+        isMarked = true;+      }++      // acquire pred locks from bottom layer up+      ScopedLocker guards[MAX_HEIGHT];+      if (!lockNodesForChange(nodeHeight, guards, preds, succs, false)) {+        continue; // this will unlock all the locks+      }++      for (int k = nodeHeight - 1; k >= 0; --k) {+        preds[k]->setSkip(k, nodeToDelete->skip(k));+      }++      incrementSize(-1);+      break;+    }+    recycle(nodeToDelete);+    return true;+  }++  const value_type* first() const {+    auto node = head_.load(std::memory_order_acquire)->skip(0);+    return node ? &node->data() : nullptr;+  }++  const value_type* last() const {+    NodeType* pred = head_.load(std::memory_order_acquire);+    NodeType* node = nullptr;+    for (int layer = maxLayer(); layer >= 0; --layer) {+      do {+        node = pred->skip(layer);+        if (node) {+          pred = node;+        }+      } while (node != nullptr);+    }+    return pred == head_.load(std::memory_order_relaxed)+        ? nullptr+        : &pred->data();+  }++  static bool okToDelete(NodeType* candidate, int layer) {+    DCHECK(candidate != nullptr);+    return candidate->fullyLinked() && candidate->maxLayer() == layer &&+        !candidate->markedForRemoval();+  }++  // find node for insertion/deleting+  int findInsertionPointGetMaxLayer(+      const value_type& data,+      NodeType* preds[],+      NodeType* succs[],+      int* max_layer) const {+    *max_layer = maxLayer();+    return findInsertionPoint(+        head_.load(std::memory_order_acquire), *max_layer, data, preds, succs);+  }++  // Find node for access. Returns a paired values:+  // pair.first = the first node that no-less than data value+  // pair.second = 1 when the data value is founded, or 0 otherwise.+  // This is like lower_bound, but not exact: we could have the node marked for+  // removal so still need to check that.+  std::pair<NodeType*, int> findNode(const value_type& data) const {+    return findNodeDownRight(data);+  }++  // Find node by first stepping down then stepping right. Based on benchmark+  // results, this is slightly faster than findNodeRightDown for better+  // locality on the skipping pointers.+  std::pair<NodeType*, int> findNodeDownRight(const value_type& data) const {+    NodeType* pred = head_.load(std::memory_order_acquire);+    int ht = pred->height();+    NodeType* node = nullptr;++    bool found = false;+    while (!found) {+      // stepping down+      for (; ht > 0 && less(data, node = pred->skip(ht - 1)); --ht) {+      }+      if (ht == 0) {+        return std::make_pair(node, 0); // not found+      }+      // node <= data now, but we need to fix up ht+      --ht;++      // stepping right+      while (greater(data, node)) {+        pred = node;+        node = node->skip(ht);+      }+      found = !less(data, node);+    }+    return std::make_pair(node, found);+  }++  // find node by first stepping right then stepping down.+  // We still keep this for reference purposes.+  std::pair<NodeType*, int> findNodeRightDown(const value_type& data) const {+    NodeType* pred = head_.load(std::memory_order_acquire);+    NodeType* node = nullptr;+    auto top = maxLayer();+    int found = 0;+    for (int layer = top; !found && layer >= 0; --layer) {+      node = pred->skip(layer);+      while (greater(data, node)) {+        pred = node;+        node = node->skip(layer);+      }+      found = !less(data, node);+    }+    return std::make_pair(node, found);+  }++  NodeType* lower_bound(const value_type& data) const {+    auto node = findNode(data).first;+    while (node != nullptr && node->markedForRemoval()) {+      node = node->skip(0);+    }+    return node;+  }++  void growHeight(int height) {+    NodeType* oldHead = head_.load(std::memory_order_acquire);+    if (oldHead->height() >= height) { // someone else already did this+      return;+    }++    NodeType* newHead =+        NodeType::create(recycler_.alloc(), height, value_type(), true);++    { // need to guard the head node in case others are adding/removing+      // nodes linked to the head.+      ScopedLocker g = oldHead->acquireGuard();+      newHead->copyHead(oldHead);+      NodeType* expected = oldHead;+      if (!head_.compare_exchange_strong(+              expected, newHead, std::memory_order_release)) {+        // if someone has already done the swap, just return.+        NodeType::destroy(recycler_.alloc(), newHead);+        return;+      }+      oldHead->setMarkedForRemoval();+    }+    recycle(oldHead);+  }++  void recycle(NodeType* node) { recycler_.add(node); }++  detail::NodeRecycler<NodeType, NodeAlloc> recycler_;+  std::atomic<NodeType*> head_;+  std::atomic<size_t> size_{0};+};++template <typename T, typename Comp, typename NodeAlloc, int MAX_HEIGHT>+class ConcurrentSkipList<T, Comp, NodeAlloc, MAX_HEIGHT>::Accessor {+  typedef detail::SkipListNode<T> NodeType;+  typedef ConcurrentSkipList<T, Comp, NodeAlloc, MAX_HEIGHT> SkipListType;++ public:+  typedef T value_type;+  typedef T key_type;+  typedef T& reference;+  typedef T* pointer;+  typedef const T& const_reference;+  typedef const T* const_pointer;+  typedef size_t size_type;+  typedef Comp key_compare;+  typedef Comp value_compare;++  typedef typename SkipListType::iterator iterator;+  typedef typename SkipListType::const_iterator const_iterator;+  typedef typename SkipListType::Skipper Skipper;++  explicit Accessor(std::shared_ptr<ConcurrentSkipList> skip_list)+      : slHolder_(std::move(skip_list)) {+    sl_ = slHolder_.get();+    DCHECK(sl_ != nullptr);+    sl_->recycler_.addRef();+  }++  // Unsafe initializer: the caller assumes the responsibility to keep+  // skip_list valid during the whole life cycle of the Accessor.+  explicit Accessor(ConcurrentSkipList* skip_list) : sl_(skip_list) {+    DCHECK(sl_ != nullptr);+    sl_->recycler_.addRef();+  }++  Accessor(const Accessor& accessor)+      : sl_(accessor.sl_), slHolder_(accessor.slHolder_) {+    sl_->recycler_.addRef();+  }++  Accessor& operator=(const Accessor& accessor) {+    if (this != &accessor) {+      slHolder_ = accessor.slHolder_;+      sl_->recycler_.releaseRef();+      sl_ = accessor.sl_;+      sl_->recycler_.addRef();+    }+    return *this;+  }++  ~Accessor() { sl_->recycler_.releaseRef(); }++  bool empty() const { return sl_->size() == 0; }+  size_t size() const { return sl_->size(); }+  size_type max_size() const { return std::numeric_limits<size_type>::max(); }++  // returns end() if the value is not in the list, otherwise returns an+  // iterator pointing to the data, and it's guaranteed that the data is valid+  // as far as the Accessor is hold.+  iterator find(const key_type& value) { return iterator(sl_->find(value)); }+  const_iterator find(const key_type& value) const {+    return iterator(sl_->find(value));+  }+  size_type count(const key_type& data) const { return contains(data); }++  iterator begin() const {+    NodeType* head = sl_->head_.load(std::memory_order_acquire);+    return iterator(head->next());+  }+  iterator end() const { return iterator(nullptr); }+  const_iterator cbegin() const { return begin(); }+  const_iterator cend() const { return end(); }++  template <+      typename U,+      typename =+          typename std::enable_if<std::is_convertible<U, T>::value>::type>+  std::pair<iterator, bool> insert(U&& data) {+    auto ret = sl_->addOrGetData(std::forward<U>(data));+    return std::make_pair(iterator(ret.first), ret.second);+  }+  size_t erase(const key_type& data) { return remove(data); }++  iterator lower_bound(const key_type& data) const {+    return iterator(sl_->lower_bound(data));+  }++  size_t height() const { return sl_->height(); }++  // first() returns pointer to the first element in the skiplist, or+  // nullptr if empty.+  //+  // last() returns the pointer to the last element in the skiplist,+  // nullptr if list is empty.+  //+  // Note: As concurrent writing can happen, first() is not+  //   guaranteed to be the min_element() in the list. Similarly+  //   last() is not guaranteed to be the max_element(), and both of them can+  //   be invalid (i.e. nullptr), so we name them differently from front() and+  //   tail() here.+  const key_type* first() const { return sl_->first(); }+  const key_type* last() const { return sl_->last(); }++  // Try to remove the last element in the skip list.+  //+  // Returns true if we removed it, false if either the list is empty+  // or a race condition happened (i.e. the used-to-be last element+  // was already removed by another thread).+  bool pop_back() {+    auto last = sl_->last();+    return last ? sl_->remove(*last) : false;+  }++  std::pair<key_type*, bool> addOrGetData(const key_type& data) {+    auto ret = sl_->addOrGetData(data);+    return std::make_pair(&ret.first->data(), ret.second);+  }++  SkipListType* skiplist() const { return sl_; }++  // legacy interfaces+  // TODO:(xliu) remove these.+  // Returns true if the node is added successfully, false if not, i.e. the+  // node with the same key already existed in the list.+  bool contains(const key_type& data) const { return sl_->find(data); }+  bool add(const key_type& data) { return sl_->addOrGetData(data).second; }+  bool remove(const key_type& data) { return sl_->remove(data); }++ private:+  SkipListType* sl_;+  std::shared_ptr<SkipListType> slHolder_;+};++// implements forward iterator concept.+template <typename ValT, typename NodeT>+class detail::csl_iterator+    : public detail::IteratorFacade<+          csl_iterator<ValT, NodeT>,+          ValT,+          std::forward_iterator_tag> {+ public:+  typedef ValT value_type;+  typedef value_type& reference;+  typedef value_type* pointer;+  typedef ptrdiff_t difference_type;++  explicit csl_iterator(NodeT* node = nullptr) : node_(node) {}++  template <typename OtherVal, typename OtherNode>+  csl_iterator(+      const csl_iterator<OtherVal, OtherNode>& other,+      typename std::enable_if<+          std::is_convertible<OtherVal*, ValT*>::value>::type* = nullptr)+      : node_(other.node_) {}++  size_t nodeSize() const {+    return node_ == nullptr+        ? 0+        : node_->height() * sizeof(NodeT*) + sizeof(*this);+  }++  bool good() const { return node_ != nullptr; }++ private:+  template <class, class>+  friend class csl_iterator;+  friend class detail::+      IteratorFacade<csl_iterator, ValT, std::forward_iterator_tag>;++  void increment() { node_ = node_->next(); }+  bool equal(const csl_iterator& other) const { return node_ == other.node_; }+  value_type& dereference() const { return node_->data(); }++  NodeT* node_;+};++// Skipper interface+template <typename T, typename Comp, typename NodeAlloc, int MAX_HEIGHT>+class ConcurrentSkipList<T, Comp, NodeAlloc, MAX_HEIGHT>::Skipper {+  typedef detail::SkipListNode<T> NodeType;+  typedef ConcurrentSkipList<T, Comp, NodeAlloc, MAX_HEIGHT> SkipListType;+  typedef typename SkipListType::Accessor Accessor;++ public:+  typedef T value_type;+  typedef T& reference;+  typedef T* pointer;+  typedef ptrdiff_t difference_type;++  Skipper(std::shared_ptr<SkipListType> skipList)+      : accessor_(std::move(skipList)) {+    init();+  }++  Skipper(const Accessor& accessor) : accessor_(accessor) { init(); }++  void init() {+    // need to cache the head node+    NodeType* head_node = head();+    headHeight_ = head_node->height();+    for (int i = 0; i < headHeight_; ++i) {+      preds_[i] = head_node;+      succs_[i] = head_node->skip(i);+    }+    int max_layer = maxLayer();+    for (int i = 0; i < max_layer; ++i) {+      hints_[i] = uint8_t(i + 1);+    }+    hints_[max_layer] = max_layer;+  }++  // advance to the next node in the list.+  Skipper& operator++() {+    preds_[0] = succs_[0];+    succs_[0] = preds_[0]->skip(0);+    int height = curHeight();+    for (int i = 1; i < height && preds_[0] == succs_[i]; ++i) {+      preds_[i] = succs_[i];+      succs_[i] = preds_[i]->skip(i);+    }+    return *this;+  }++  Accessor& accessor() { return accessor_; }+  const Accessor& accessor() const { return accessor_; }++  bool good() const { return succs_[0] != nullptr; }++  int maxLayer() const { return headHeight_ - 1; }++  int curHeight() const {+    // need to cap the height to the cached head height, as the current node+    // might be some newly inserted node and also during the time period the+    // head height may have grown.+    return succs_[0] ? std::min(headHeight_, succs_[0]->height()) : 0;+  }++  const value_type& data() const {+    DCHECK(succs_[0] != nullptr);+    return succs_[0]->data();+  }++  value_type& operator*() const {+    DCHECK(succs_[0] != nullptr);+    return succs_[0]->data();+  }++  value_type* operator->() {+    DCHECK(succs_[0] != nullptr);+    return &succs_[0]->data();+  }++  /*+   * Skip to the position whose data is no less than the parameter.+   * (I.e. the lower_bound).+   *+   * Returns true if the data is found, false otherwise.+   */+  bool to(const value_type& data) {+    int layer = curHeight() - 1;+    if (layer < 0) {+      return false; // reaches the end of the list+    }++    int lyr = hints_[layer];+    int max_layer = maxLayer();+    while (SkipListType::greater(data, succs_[lyr]) && lyr < max_layer) {+      ++lyr;+    }+    hints_[layer] = lyr; // update the hint++    int foundLayer = SkipListType::findInsertionPoint(+        preds_[lyr], lyr, data, preds_, succs_);+    if (foundLayer < 0) {+      return false;+    }++    DCHECK(succs_[0] != nullptr)+        << "lyr=" << lyr << "; max_layer=" << max_layer;+    return !succs_[0]->markedForRemoval();+  }++ private:+  NodeType* head() const {+    return accessor_.skiplist()->head_.load(std::memory_order_acquire);+  }++  Accessor accessor_;+  int headHeight_;+  NodeType *succs_[MAX_HEIGHT], *preds_[MAX_HEIGHT];+  uint8_t hints_[MAX_HEIGHT];+};++} // namespace folly
+ folly/folly/ConstexprMath.h view
@@ -0,0 +1,977 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <cassert>+#include <cstddef>+#include <cstdint>+#include <functional>+#include <limits>+#include <type_traits>++#include <folly/Portability.h>+#include <folly/lang/CheckedMath.h>+#include <folly/portability/Constexpr.h>++namespace folly {++/// numbers+///+/// mimic: std::numbers, C++20 (partial)+namespace numbers {++namespace detail {+template <typename T>+using enable_if_floating_t =+    std::enable_if_t<std::is_floating_point<T>::value, T>;+}++/// e_v+///+/// mimic: std::numbers::e_v, C++20+template <typename T>+inline constexpr T e_v = detail::enable_if_floating_t<T>(+    2.71828182845904523536028747135266249775724709369995L);++/// ln2_v+///+/// mimic: std::numbers::ln2_v, C++20+template <typename T>+inline constexpr T ln2_v = detail::enable_if_floating_t<T>(+    0.69314718055994530941723212145817656807550013436025L);++/// e+///+/// mimic: std::numbers::e, C++20+inline constexpr double e = e_v<double>;++/// ln2+///+/// mimic: std::numbers::ln2, C++20+inline constexpr double ln2 = ln2_v<double>;++} // namespace numbers++/// floating_point_integral_constant+///+/// Like std::integral_constant but for floating-point types holding integral+/// values representable in an integral type.+template <typename T, typename S, S Value>+struct floating_point_integral_constant {+  using value_type = T;+  static constexpr value_type value = static_cast<value_type>(Value);+  constexpr operator value_type() const noexcept { return value; }+  constexpr value_type operator()() const noexcept { return value; }+};++//  ----++namespace detail {++template <typename T>+constexpr size_t constexpr_iterated_squares_desc_size_(T const base) {+  using lim = std::numeric_limits<T>;+  size_t s = 1;+  auto r = base;+  while (r <= lim::max() / r) {+    ++s;+    r *= r;+  }+  return s;+}++} // namespace detail++/// constexpr_iterated_squares_desc_size_v+///+/// Effectively calculates: floor(log(max_exponent)/log(base))+///+/// For use with constexpr_iterated_squares_desc below.+template <typename Base>+inline constexpr size_t constexpr_iterated_squares_desc_size_v =+    detail::constexpr_iterated_squares_desc_size_(Base::value);++/// constexpr_iterated_squares_desc+///+/// A constexpr scaling array of integer powers-of-powers-of-two, descending,+/// with the associated powers-of-two.+///+/// scaling = [..., {8, b^8}, {4, b^4}, {2, b^2}, {1, b^1}] for b = base+///+/// Includes select constexpr scaling algorithms based on the scaling array.+///+/// The scaling array and the scaling algorithms are general-purpose, if niche.+/// They may be used by other constexpr math functions (floating-point) either+/// to improve runtime performance or to improve numerical approximations.+///+/// Some compilers fail to support passing some types as non-type template+/// params. In particular, long double is not universally supported. Therefore,+/// this utility takes its base as a type rather than as a value. For floating-+/// point integral bases, that is, bases of floating-point type but of integral+/// value, floating_point_integral_constant is the easiest parameterization.+template <typename T, std::size_t Size>+struct constexpr_iterated_squares_desc {+  static_assert(Size > 0, "requires non-zero size");++  using size_type = decltype(Size);+  using base_type = T;++  struct item_type {+    size_type power;+    base_type scale;+  };++  static constexpr size_type size = Size;+  base_type base;+  item_type scaling[size];++ private:+  using lim = std::numeric_limits<base_type>;++  static_assert(+      lim::max_exponent < std::numeric_limits<size_type>::max(),+      "size_type too small for base_type");++ public:+  explicit constexpr constexpr_iterated_squares_desc(base_type r) noexcept+      : base{r}, scaling{} {+    assert(size <= detail::constexpr_iterated_squares_desc_size_(base));+    size_type i = 0;+    size_type p = 1;+    while (true) { // a for-loop might cause multiplication overflow below+      scaling[size - 1 - i] = {p, r};+      if (++i == size) {+        break;+      }+      p *= 2;+      r *= r;+    }+  }++  /// shrink+  ///+  /// Returns scaling params of the form:+  ///   item_type{power, scale} with scale = base ^ power+  /// With power the smallest nonnegative integer such that:+  ///   abs(num) / scale <= max+  constexpr item_type shrink(base_type const num, base_type const max) const {+    assert(max > base_type(0));+    auto const rmax = max / base;+    auto const snum = num < base_type(0) ? -num : num;+    auto power = size_type(0);+    auto scale = base_type(1);+    if (!(snum / scale <= max)) {+      for (auto const& i : scaling) {+        auto const next = scale * i.scale;+        auto const div = snum / next;+        if (div <= rmax) {+          continue;+        }+        power += i.power;+        scale = next;+        if (div <= max) {+          break;+        }+      }+    }+    assert(snum / scale <= max);+    return {power, scale};+  }++  /// growth+  ///+  /// Returns scaling params of the form:+  ///   item_type{power, scale} with scale = base ^ power+  /// With power the smallest nonnegative integer such that:+  ///   abs(num) * scale >= min+  constexpr item_type growth(base_type const num, base_type const min) const {+    assert(min > base_type(0));+    auto const rmin = min * base;+    auto const snum = num < base_type(0) ? -num : num;+    auto power = size_type(0);+    auto scale = base_type(1);+    if (!(snum * scale >= min)) {+      for (auto const& i : scaling) {+        auto const next = scale * i.scale;+        auto const mul = snum * next;+        if (mul >= rmin) {+          continue;+        }+        power += i.power;+        scale = next;+        if (mul >= min) {+          break;+        }+      }+    }+    assert(snum * scale >= min);+    return {power, scale};+  }+};++/// constexpr_iterated_squares_desc_v+///+/// An instance of constexpr_iterated_squares_desc of max size with the given+/// base.+template <typename Base>+inline constexpr auto constexpr_iterated_squares_desc_v =+    constexpr_iterated_squares_desc<+        typename Base::value_type,+        constexpr_iterated_squares_desc_size_v<Base>>{Base::value};++/// constexpr_iterated_squares_desc_2_v+///+/// An alias for constexpr_iterated_squares_desc_v with base 2, which is the+/// most common base to use with iterated-squares.+template <typename T>+constexpr auto& constexpr_iterated_squares_desc_2_v =+    constexpr_iterated_squares_desc_v<+        floating_point_integral_constant<T, int, 2>>;++// TLDR: Prefer using operator< for ordering. And when+// a and b are equivalent objects, we return b to make+// sorting stable.+// See http://stepanovpapers.com/notes.pdf for details.+template <typename T, typename... Ts>+constexpr T constexpr_max(T a, Ts... ts) {+  T list[] = {ts..., a}; // 0-length arrays are illegal+  for (auto i = 0u; i < sizeof...(Ts); ++i) {+    a = list[i] < a ? a : list[i];+  }+  return a;+}++// When a and b are equivalent objects, we return a to+// make sorting stable.+template <typename T, typename... Ts>+constexpr T constexpr_min(T a, Ts... ts) {+  T list[] = {ts..., a}; // 0-length arrays are illegal+  for (auto i = 0u; i < sizeof...(Ts); ++i) {+    a = list[i] < a ? list[i] : a;+  }+  return a;+}++template <typename T, typename Less>+constexpr T const& constexpr_clamp(+    T const& v, T const& lo, T const& hi, Less less) {+  T const& a = less(v, lo) ? lo : v;+  T const& b = less(hi, a) ? hi : a;+  return b;+}+template <typename T>+constexpr T const& constexpr_clamp(T const& v, T const& lo, T const& hi) {+  return constexpr_clamp(v, lo, hi, std::less<T>{});+}++template <typename T>+constexpr bool constexpr_isnan(T const t) {+  return t != t; // NOLINT+}++namespace detail {++template <typename T, typename = void>+struct constexpr_abs_helper {};++template <typename T>+struct constexpr_abs_helper<+    T,+    typename std::enable_if<std::is_floating_point<T>::value>::type> {+  static constexpr T go(T t) { return t < static_cast<T>(0) ? -t : t; }+};++template <typename T>+struct constexpr_abs_helper<+    T,+    typename std::enable_if<+        std::is_integral<T>::value && !std::is_same<T, bool>::value &&+        std::is_unsigned<T>::value>::type> {+  static constexpr T go(T t) { return t; }+};++template <typename T>+struct constexpr_abs_helper<+    T,+    typename std::enable_if<+        std::is_integral<T>::value && !std::is_same<T, bool>::value &&+        std::is_signed<T>::value>::type> {+  static constexpr typename std::make_unsigned<T>::type go(T t) {+    return typename std::make_unsigned<T>::type(t < static_cast<T>(0) ? -t : t);+  }+};++} // namespace detail++template <typename T>+constexpr auto constexpr_abs(T t)+    -> decltype(detail::constexpr_abs_helper<T>::go(t)) {+  return detail::constexpr_abs_helper<T>::go(t);+}++namespace detail {++template <typename T>+constexpr T constexpr_log2_(T a, T e) {+  return e == T(1) ? a : constexpr_log2_(a + T(1), e / T(2));+}++template <typename T>+constexpr T constexpr_log2_ceil_(T l2, T t) {+  return l2 + T(T(1) << l2 < t ? 1 : 0);+}++} // namespace detail++template <typename T>+constexpr T constexpr_log2(T t) {+  return detail::constexpr_log2_(T(0), t);+}++template <typename T>+constexpr T constexpr_log2_ceil(T t) {+  return detail::constexpr_log2_ceil_(constexpr_log2(t), t);+}++/// constexpr_trunc+///+/// mimic: std::trunc (C++23)+template <+    typename T,+    std::enable_if_t<std::is_floating_point<T>::value, int> = 0>+constexpr T constexpr_trunc(T const t) {+  using lim = std::numeric_limits<T>;+  using int_type = std::uintmax_t;+  using int_lim = std::numeric_limits<int_type>;+  static_assert(lim::radix == 2, "non-binary radix");+  static_assert(lim::digits <= int_lim::digits, "overwide mantissa");+  constexpr auto bound = static_cast<T>(std::uintmax_t(1) << (lim::digits - 1));+  auto const neg = !constexpr_isnan(t) && t < T(0);+  auto const s = neg ? -t : t;+  if (constexpr_isnan(t) || t == T(0) || !(s < bound)) {+    return t;+  }+  if (s < T(1)) {+    return neg ? -T(0) : T(0);+  }+  auto const r = static_cast<T>(static_cast<int_type>(s));+  return neg ? -r : r;+}++template <typename T, std::enable_if_t<std::is_integral<T>::value, int> = 0>+constexpr T constexpr_trunc(T const t) {+  return t;+}++/// constexpr_round+///+/// mimic: std::round (C++23)+template <typename T>+constexpr T constexpr_round(T const t) {+  constexpr auto half = T(1) / T(2);+  auto const same = constexpr_isnan(t) || t == T(0);+  return same ? t : constexpr_trunc(t < T(0) ? t - half : t + half);+}++/// constexpr_floor+///+/// mimic: std::floor (C++23)+template <typename T>+constexpr T constexpr_floor(T const t) {+  auto const s = constexpr_trunc(t);+  return t < s ? s - T(1) : s;+}++/// constexpr_ceil+///+/// mimic: std::ceil (C++23)+template <typename T>+constexpr T constexpr_ceil(T const t) {+  auto const s = constexpr_trunc(t);+  return s < t ? s + T(1) : s;+}++/// constexpr_ceil+///+/// The least integer at least t that round divides.+template <typename T>+constexpr T constexpr_ceil(T t, T round) {+  return round == T(0)+      ? t+      : ((t + (t <= T(0) ? T(0) : round - T(1))) / round) * round;+}++/// constexpr_mult+///+/// Multiply two values, allowing for constexpr floating-pooint overflow to+/// infinity.+template <typename T>+constexpr T constexpr_mult(T const a, T const b) {+  using lim = std::numeric_limits<T>;+  if (constexpr_isnan(a) || constexpr_isnan(b)) {+    return constexpr_isnan(a) ? a : b;+  }+  if (std::is_floating_point<T>::value) {+    constexpr auto inf = lim::infinity();+    auto const ax = constexpr_abs(a);+    auto const bx = constexpr_abs(b);+    if ((ax == T(0) && bx == inf) || (bx == T(0) && ax == inf)) {+      return lim::quiet_NaN();+    }+    // floating-point multiplication overflow, ie where multiplication of two+    // finite values overflows to infinity of either sign, is not constexpr per+    // gcc+    // floating-point division overflow, ie where division of two finite values+    // overflows to infinity of either sign, is not constexpr per gcc+    // floating-point division by zero is not constexpr per any compiler, but we+    // use it in the checks for the other two conditions+    if (ax != inf && bx != inf && T(1) < bx && lim::max() / bx < ax) {+      auto const a_neg = static_cast<bool>(a < T(0));+      auto const b_neg = static_cast<bool>(b < T(0));+      auto const sign = a_neg == b_neg ? T(1) : T(-1);+      return sign * inf;+    }+  }+  return a * b;+}++namespace detail {++template <+    typename T,+    typename E,+    std::enable_if_t<std::is_signed<E>::value, int> = 1>+constexpr T constexpr_ipow(T const base, E const exp) {+  if (std::is_floating_point<T>::value) {+    if (exp < E(0)) {+      return T(1) / constexpr_ipow(base, -exp);+    }+    if (exp == E(0)) {+      return T(1);+    }+    if (constexpr_isnan(base)) {+      return base;+    }+  }+  assert(!(exp < E(0)) && "negative exponent with integral base");+  if (exp == E(0)) {+    return T(1);+  }+  if (exp == E(1)) {+    return base;+  }+  auto const hexp = constexpr_trunc(exp / E(2));+  auto const div = constexpr_ipow(base, hexp);+  auto const rem = hexp * E(2) == exp ? T(1) : base;+  return constexpr_mult(constexpr_mult(div, div), rem);+}++template <+    typename T,+    typename E,+    std::enable_if_t<std::is_unsigned<E>::value, int> = 1>+constexpr T constexpr_ipow(T const base, E const exp) {+  if (std::is_floating_point<T>::value) {+    if (exp == E(0)) {+      return T(1);+    }+    if (constexpr_isnan(base)) {+      return base;+    }+  }+  if (exp == E(0)) {+    return T(1);+  }+  if (exp == E(1)) {+    return base;+  }+  auto const hexp = constexpr_trunc(exp / E(2));+  auto const div = constexpr_ipow(base, hexp);+  auto const rem = hexp * E(2) == exp ? T(1) : base;+  return constexpr_mult(constexpr_mult(div, div), rem);+}++} // namespace detail++/// constexpr_exp+///+/// Calculates an approximation of the mathematical function exp(num). Usable in+/// constant evaluations. Like std::exp, which becomes constexpr in C++26.+///+/// The integer overload uses iterated squaring and multiplication. The+/// floating-point overlaod naively evaluates the taylor series of exp(num)+/// until approximate convergence.+///+/// mimic: std::exp (C++23, C++26)+template <+    typename T,+    typename N,+    std::enable_if_t<+        std::is_floating_point<T>::value && std::is_integral<N>::value &&+            !std::is_same<N, bool>::value,+        int> = 0>+constexpr T constexpr_exp(N const power) {+  auto const npower = constexpr_abs(power);+  auto const result = detail::constexpr_ipow(numbers::e_v<T>, npower);+  return power < N(0) ? T(1) / result : result;+}+template <+    typename N,+    std::enable_if_t<+        std::is_integral<N>::value && !std::is_same<N, bool>::value,+        int> = 0>+constexpr double constexpr_exp(N const power) {+  return constexpr_exp<double>(power);+}+template <+    typename T,+    std::enable_if_t<std::is_floating_point<T>::value, int> = 0>+constexpr T constexpr_exp(T const power) {+  using lim = std::numeric_limits<T>;++  // edge cases+  if (constexpr_isnan(power)) {+    return power;+  }+  if (power == -lim::infinity()) {+    return +T(0);+  }+  if (power == +lim::infinity()) {+    return power;+  }++  // convergence works better with positive powers since signs do not alternate+  auto const abspower = constexpr_abs(power);+  // convergence must short-circuit when terms grow to floating-point infinity+  auto const bound = T(1) < abspower ? lim::max() / abspower : lim::infinity();++  // term #index = power * coeff+  auto index = size_t(0);+  auto term = T(1);+  // result = sum of terms+  auto result = T(1);+  // sum the terms until ~convergence+  while (!(constexpr_abs(term) < lim::epsilon())) {+    if (bound < term) {+      return power < T(0) ? T(0) : lim::infinity();+    }+    index += 1;+    term = term * abspower / index;+    result += term;+  }+  return power < T(0) ? T(1) / result : result;+}++/// constexpr_log+///+/// Calculates an approximation of the natural logarithm ln(num).+///+/// The implementation uses a quickly-converging, high-precision iterative+/// technique as described in:+///   https://en.wikipedia.org/wiki/Natural_logarithm#High_precision+///+/// The technique works best with numbers that are close enough to 1, so the+/// implementation uses a quick shrink/growth technique as described in:+///   https://en.wikipedia.org/wiki/Natural_logarithm#Efficient_computation+template <+    typename T,+    std::enable_if_t<std::is_floating_point<T>::value, int> = 0>+constexpr T constexpr_log(T const num) {+  using lim = std::numeric_limits<T>;+  constexpr auto& isq = constexpr_iterated_squares_desc_2_v<T>;++  // edge cases+  if (constexpr_isnan(num)) {+    return num;+  }+  if (num < T(0)) {+    return lim::quiet_NaN();+  }+  if (num == T(0)) {+    return -lim::infinity();+  }+  if (num == lim::infinity()) {+    return num;+  }++  // compression+  auto const shrink = isq.shrink(num, isq.base);+  auto const growth = isq.growth(num, T(1));+  auto const scaled = num * growth.scale / shrink.scale;+  assert(scaled <= isq.base);+  assert(scaled >= T(1));++  auto sum = T(0);+  auto delta = T(2);+  while (constexpr_abs(delta) >= lim::epsilon()) {+    auto expterm = constexpr_exp(sum);+    delta = T(2) * (scaled - expterm) / (scaled + expterm);+    sum += delta;+  }+  auto const ln2 = numbers::ln2_v<T>;+  return sum - growth.power * ln2 + shrink.power * ln2;+}++/// constexpr_pow+///+/// Calculates an approximation of the value of base raised to the exponent exp.+///+/// The implementation uses iterated squaring and multiplication for the integer+/// part of the exponent and uses the identity x^y = exp(y * log(x)) for the+/// fractional part of the exponent.+///+/// Notes:+/// * Forbids base of +0 or -0 with finite non-positive exponent: in part since+///   the plausible infinite result would be sensitive to the sign of the zero;+///   and in part since std::pow would be required or permitted to raise error+///   div-by-zero.+/// * Forbids finite negative base with finite non-integer exponent: in part+///   since std::pow would be required to raise error invalid.+///+/// mimic: std::pow (C++26)+template <+    typename T,+    typename E,+    std::enable_if_t<+        std::is_integral<E>::value && !std::is_same<E, bool>::value,+        int> = 0>+constexpr T constexpr_pow(T const base, E const exp) {+  return detail::constexpr_ipow(base, exp);+}+template <+    typename T,+    std::enable_if_t<std::is_floating_point<T>::value, int> = 0>+constexpr T constexpr_pow(T const base, T const exp) {+  using lim = std::numeric_limits<T>;++  // edge cases+  if (exp == T(0)) {+    return T(1);+  }+  if (constexpr_isnan(base)) {+    return base;+  }+  if (exp == lim::infinity() || exp == -lim::infinity()) {+    auto const abase = constexpr_abs(base);+    if (abase < T(1)) {+      return exp == lim::infinity() ? T(0) : lim::infinity();+    }+    if (T(1) < abase) {+      return exp == lim::infinity() ? lim::infinity() : T(0);+    }+    return T(1);+  }+  if (base == T(1)) {+    return base;+  }+  if (constexpr_isnan(exp)) {+    return exp;+  }+  assert(base != T(0) || exp > T(0)); // error div-by-zero+  if (base == lim::infinity()) {+    return exp < T(0) ? T(0) : lim::infinity();+  }+  if (base == -lim::infinity()) {+    auto const oddi = //+        exp == constexpr_trunc(exp) &&+        exp != constexpr_trunc(exp / T(2)) * T(2);+    return (oddi ? -T(1) : T(1)) * (exp < T(0) ? T(0) : lim::infinity());+  }+  if (base == T(0)) {+    auto const oddi = //+        exp == constexpr_trunc(exp) &&+        exp != constexpr_trunc(exp / T(2)) * T(2);+    return oddi ? base : T(0);+  }+  if (exp < T(0)) {+    return T(1) / constexpr_pow(base, -exp);+  }++  // as an identity: x^y = exp(y * log(x)); but calculation is imprecise ... so,+  // for better precision, split the calculation into its integral-power and its+  // fractional-power components+  // as a cost, the complexity of constexpr_ipow here is logarithmic in y, i.e.,+  // linear in the logarithm of y, which can be prohibitive+  auto const exp_trunc = constexpr_trunc(exp);+  assert(T(0) < base || exp == exp_trunc); // error invalid+  auto const exp_fract = exp - exp_trunc;+  auto const anyi = exp_fract == T(0);+  return constexpr_mult(+      detail::constexpr_ipow(base, exp_trunc),+      anyi ? T(1) : constexpr_exp(exp_fract * constexpr_log(base)));+}++/// constexpr_find_last_set+///+/// Return the 1-based index of the most significant bit which is set.+/// For x > 0, constexpr_find_last_set(x) == 1 + floor(log2(x)).+template <typename T>+constexpr std::size_t constexpr_find_last_set(T const t) {+  using U = std::make_unsigned_t<T>;+  return t == T(0) ? 0 : 1 + constexpr_log2(static_cast<U>(t));+}++namespace detail {+template <typename U>+constexpr std::size_t constexpr_find_first_set_(+    std::size_t s, std::size_t a, U const u) {+  return s == 0+      ? a+      : constexpr_find_first_set_(+            s / 2, a + s * bool((u >> a) % (U(1) << s) == U(0)), u);+}+} // namespace detail++/// constexpr_find_first_set+///+/// Return the 1-based index of the least significant bit which is set.+/// For x > 0, the exponent in the largest power of two which does not divide x.+template <typename T>+constexpr std::size_t constexpr_find_first_set(T t) {+  using U = std::make_unsigned_t<T>;+  using size = std::integral_constant<std::size_t, sizeof(T) * 4>;+  return t == T(0)+      ? 0+      : 1 + detail::constexpr_find_first_set_(size{}, 0, static_cast<U>(t));+}++template <typename T>+constexpr T constexpr_add_overflow_clamped(T a, T b) {+  using L = std::numeric_limits<T>;+  using M = std::intmax_t;+  static_assert(+      !std::is_integral<T>::value || sizeof(T) <= sizeof(M),+      "Integral type too large!");+  if (!folly::is_constant_evaluated_or(true)) {+    if constexpr (std::is_integral_v<T>) {+      T ret{};+      if (FOLLY_UNLIKELY(!checked_add(&ret, a, b))) {+        if constexpr (std::is_signed_v<T>) {+          // Could be either overflow or underflow for signed types.+          // Can only be underflow if both inputs are negative.+          if (a < 0 && b < 0) {+            return L::min();+          }+        }+        return L::max();+      }+      return ret;+    }+  }+  // clang-format off+  return+    // don't do anything special for non-integral types.+    !std::is_integral<T>::value ? a + b :+    // for narrow integral types, just convert to intmax_t.+    sizeof(T) < sizeof(M)+      ? T(constexpr_clamp(+          static_cast<M>(a) + static_cast<M>(b),+          static_cast<M>(L::min()),+          static_cast<M>(L::max()))) :+    // when a >= 0, cannot add more than `MAX - a` onto a.+    !(a < 0) ? a + constexpr_min(b, T(L::max() - a)) :+    // a < 0 && b >= 0, `a + b` will always be in valid range of type T.+    !(b < 0) ? a + b :+    // a < 0 && b < 0, keep the result >= MIN.+              a + constexpr_max(b, T(L::min() - a));+  // clang-format on+}++template <typename T>+constexpr T constexpr_sub_overflow_clamped(T a, T b) {+  using L = std::numeric_limits<T>;+  using M = std::intmax_t;+  static_assert(+      !std::is_integral<T>::value || sizeof(T) <= sizeof(M),+      "Integral type too large!");+  // clang-format off+  return+    // don't do anything special for non-integral types.+    !std::is_integral<T>::value ? a - b :+    // for unsigned type, keep result >= 0.+    std::is_unsigned<T>::value ? (a < b ? 0 : a - b) :+    // for narrow signed integral types, just convert to intmax_t.+    sizeof(T) < sizeof(M)+      ? T(constexpr_clamp(+          static_cast<M>(a) - static_cast<M>(b),+          static_cast<M>(L::min()),+          static_cast<M>(L::max()))) :+    // (a >= 0 && b >= 0) || (a < 0 && b < 0), `a - b` will always be valid.+    (a < 0) == (b < 0) ? a - b :+    // MIN < b, so `-b` should be in valid range (-MAX <= -b <= MAX),+    // convert subtraction to addition.+    L::min() < b ? constexpr_add_overflow_clamped(a, T(-b)) :+    // -b = -MIN = (MAX + 1) and a <= -1, result is in valid range.+    a < 0 ? a - b :+    // -b = -MIN = (MAX + 1) and a >= 0, result > MAX.+            L::max();+  // clang-format on+}++// clamp_cast<> provides sane numeric conversions from float point numbers to+// integral numbers, and between different types of integral numbers. It helps+// to avoid unexpected bugs introduced by bad conversion, and undefined behavior+// like overflow when casting float point numbers to integral numbers.+//+// When doing clamp_cast<Dst>(value), if `value` is in valid range of Dst,+// it will give correct result in Dst, equal to `value`.+//+// If `value` is outside the representable range of Dst, it will be clamped to+// MAX or MIN in Dst, instead of being undefined behavior.+//+// Float NaNs are converted to 0 in integral type.+//+// Here's some comparison with static_cast<>:+// (with FB-internal gcc-5-glibc-2.23 toolchain)+//+// static_cast<int32_t>(NaN) = 6+// clamp_cast<int32_t>(NaN) = 0+//+// static_cast<int32_t>(9999999999.0f) = -348639895+// clamp_cast<int32_t>(9999999999.0f) = 2147483647+//+// static_cast<int32_t>(2147483647.0f) = -348639895+// clamp_cast<int32_t>(2147483647.0f) = 2147483647+//+// static_cast<uint32_t>(4294967295.0f) = 0+// clamp_cast<uint32_t>(4294967295.0f) = 4294967295+//+// static_cast<uint32_t>(-1) = 4294967295+// clamp_cast<uint32_t>(-1) = 0+//+// static_cast<int16_t>(32768u) = -32768+// clamp_cast<int16_t>(32768u) = 32767++template <typename Dst, typename Src>+constexpr typename std::enable_if<std::is_integral<Src>::value, Dst>::type+constexpr_clamp_cast(Src src) {+  static_assert(+      std::is_integral<Dst>::value && sizeof(Dst) <= sizeof(int64_t),+      "constexpr_clamp_cast can only cast into integral type (up to 64bit)");++  using L = std::numeric_limits<Dst>;+  // clang-format off+  return+    // Check if Src and Dst have same signedness.+    std::is_signed<Src>::value == std::is_signed<Dst>::value+    ? (+      // Src and Dst have same signedness. If sizeof(Src) <= sizeof(Dst),+      // we can safely convert Src to Dst without any loss of accuracy.+      sizeof(Src) <= sizeof(Dst) ? Dst(src) :+      // If Src is larger in size, we need to clamp it to valid range in Dst.+      Dst(constexpr_clamp(src, Src(L::min()), Src(L::max()))))+    // Src and Dst have different signedness.+    // Check if it's signed -> unsigend cast.+    : std::is_signed<Src>::value && std::is_unsigned<Dst>::value+    ? (+      // If src < 0, the result should be 0.+      src < 0 ? Dst(0) :+      // Otherwise, src >= 0. If src can fit into Dst, we can safely cast it+      // without loss of accuracy.+      sizeof(Src) <= sizeof(Dst) ? Dst(src) :+      // If Src is larger in size than Dst, we need to ensure the result is+      // at most Dst MAX.+      Dst(constexpr_min(src, Src(L::max()))))+    // It's unsigned -> signed cast.+    : (+      // Since Src is unsigned, and Dst is signed, Src can fit into Dst only+      // when sizeof(Src) < sizeof(Dst).+      sizeof(Src) < sizeof(Dst) ? Dst(src) :+      // If Src does not fit into Dst, we need to ensure the result is at most+      // Dst MAX.+      Dst(constexpr_min(src, Src(L::max()))));+  // clang-format on+}++namespace detail {+// Upper/lower bound values that could be accurately represented in both+// integral and float point types.+constexpr double kClampCastLowerBoundDoubleToInt64F = -9223372036854774784.0;+constexpr double kClampCastUpperBoundDoubleToInt64F = 9223372036854774784.0;+constexpr double kClampCastUpperBoundDoubleToUInt64F = 18446744073709549568.0;++constexpr float kClampCastLowerBoundFloatToInt32F = -2147483520.0f;+constexpr float kClampCastUpperBoundFloatToInt32F = 2147483520.0f;+constexpr float kClampCastUpperBoundFloatToUInt32F = 4294967040.0f;++// This works the same as constexpr_clamp, but the comparison are done in Src+// to prevent any implicit promotions.+template <typename D, typename S>+constexpr D constexpr_clamp_cast_helper(S src, S sl, S su, D dl, D du) {+  return src < sl ? dl : (src > su ? du : D(src));+}+} // namespace detail++template <typename Dst, typename Src>+constexpr typename std::enable_if<std::is_floating_point<Src>::value, Dst>::type+constexpr_clamp_cast(Src src) {+  static_assert(+      std::is_integral<Dst>::value && sizeof(Dst) <= sizeof(int64_t),+      "constexpr_clamp_cast can only cast into integral type (up to 64bit)");++  using L = std::numeric_limits<Dst>;+  // clang-format off+  return+    // Special case: cast NaN into 0.+    constexpr_isnan(src) ? Dst(0) :+    // using `sizeof(Src) > sizeof(Dst)` as a heuristic that Dst can be+    // represented in Src without loss of accuracy.+    // see: https://en.wikipedia.org/wiki/Floating-point_arithmetic+    sizeof(Src) > sizeof(Dst) ?+      detail::constexpr_clamp_cast_helper(+          src, Src(L::min()), Src(L::max()), L::min(), L::max()) :+    // sizeof(Src) < sizeof(Dst) only happens when doing cast of+    // 32bit float -> u/int64_t.+    // Losslessly promote float into double, change into double -> u/int64_t.+    sizeof(Src) < sizeof(Dst) ? (+      src >= 0.0+      ? constexpr_clamp_cast<Dst>(+            constexpr_clamp_cast<std::uint64_t>(double(src)))+      : constexpr_clamp_cast<Dst>(+            constexpr_clamp_cast<std::int64_t>(double(src)))) :+    // The following are for sizeof(Src) == sizeof(Dst).+    std::is_same<Src, double>::value && std::is_same<Dst, int64_t>::value ?+      detail::constexpr_clamp_cast_helper(+          double(src),+          detail::kClampCastLowerBoundDoubleToInt64F,+          detail::kClampCastUpperBoundDoubleToInt64F,+          L::min(),+          L::max()) :+    std::is_same<Src, double>::value && std::is_same<Dst, uint64_t>::value ?+      detail::constexpr_clamp_cast_helper(+          double(src),+          0.0,+          detail::kClampCastUpperBoundDoubleToUInt64F,+          L::min(),+          L::max()) :+    std::is_same<Src, float>::value && std::is_same<Dst, int32_t>::value ?+      detail::constexpr_clamp_cast_helper(+          float(src),+          detail::kClampCastLowerBoundFloatToInt32F,+          detail::kClampCastUpperBoundFloatToInt32F,+          L::min(),+          L::max()) :+      detail::constexpr_clamp_cast_helper(+          float(src),+          0.0f,+          detail::kClampCastUpperBoundFloatToUInt32F,+          L::min(),+          L::max());+  // clang-format on+}++} // namespace folly
+ folly/folly/ConstructorCallbackList.h view
@@ -0,0 +1,159 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once+#include <array>+#include <atomic>+#include <iterator>+#include <memory>+#include <stdexcept>+#include <fmt/format.h>+#include <folly/detail/StaticSingletonManager.h>++#include <folly/Format.h>+#include <folly/Function.h>+#include <folly/SharedMutex.h>++namespace folly {++// A mixin to register and issue callbacks every time a class constructor is+// invoked+//+// For example:+// #include <folly/ConstructorCallbackList.h>+//+// class Foo {+//    ...+//   private:+//    ...+//    // add this member last to minimize partially constructed errors+//    ConstructorCallbackList<Foo> constructorCB_{this};+// }+//+// int main() {+//   auto cb = [](Foo * f) {+//    std::cout << "New Foo" << f << std::endl;+//   };+//   ConstructorCallbackList<Foo>::addCallback(cb);+//   Foo f{}; // will call callback, print to stdout+// }+//+// This code is designed to be light weight so as to mixin to many+// places with low overhead.+//+// NOTE: The callback is triggered with a *partially* constructed object.+// This implies that that callback code can only access members that are+// constructed *before* the ConstructorCallbackList object.  Also, at the time+// of the callback, none of the Foo() constructor code will have run.+// Per the example above,+// the best practice is to place the ConstructorCallbackList declaration last+// in the parent class.  This will minimize the amount of uninitialized+// data in the Foo instance, but will not eliminate it unless it has a trivial+// constructor.+//+// Implementation/Overhead Notes:+//+// By design, adding ConstructorCallbackList to an object should be very+// light weight.  From a memory context, this adds 1 byte of memory to the+// parent class. From a CPU/performance perspective, the constructor does a load+// of an atomic int and the cost of the actual callbacks themselves.  So if this+// is put in place and only used infrequently, e.g., during debugging,+// this cost should be quite small.+//+// A compile-time static array is used intentionally over a dynamic one for+// two reasons: (1) a dynamic array seems to require a proper lock in+// the constructor which would exceed our perf target, and (2) having a+// finite array provides some sanity checking on the number of callbacks+// that can be registered.++template <class T, std::size_t MaxCallbacks = 4>+class ConstructorCallbackList {+ public:+  static constexpr std::size_t kMaxCallbacks = MaxCallbacks;++  using This = ConstructorCallbackList<T, MaxCallbacks>;+  using Callback = folly::Function<void(T*)>;+  using CallbackArray = std::array<typename This::Callback, MaxCallbacks>;++  explicit ConstructorCallbackList(T* t) {+    // This code depends on the C++ standard where values that are+    // initialized to zero ("Zero Initiation") are initialized before any more+    // complex static pre-main() dynamic initialization - see+    // https://en.cppreference.com/w/cpp/language/initialization) for+    // more details.+    //+    // This assumption prevents a subtle initialization race condition+    // where something could call this code pre-main() before+    // numCallbacks_ was set to zero, and thus prevents issuing+    // callbacks on garbage data.++    auto nCBs = This::global().numCallbacks_.load(std::memory_order_acquire);++    // fire callbacks to inform listeners about the new constructor+    /****+     * We don't need the full lock here, just the atomic int to tell us+     * how far into the array to go/how many callbacks are registered+     *+     * NOTE that nCBs > 0 will always imply that callbacks_ is non-nullptr+     */+    for (size_t i = 0; i < nCBs; i++) {+      (This::global().callbacks_)[i](t);+    }+  }++  /**+   * Add a callback to the static class that will fire every time+   * someone creates a new one.+   *+   * Implement this as a static array of callbacks rather than a dynamic+   * vector to avoid nasty race conditions on resize, startup and shutdown.+   *+   * Implement this with functions rather than an observer pattern classes+   * to avoid race conditions on shutdown+   *+   * Intentionally don't implement removeConstructorCallbackList to simplify+   * implementation (e.g., just the counter is atomic rather than the whole+   * array) and thus reduce computational cost.+   *+   * @throw std::length_error() if this callback would exceed our max+   */+  static void addCallback(Callback cb) {+    // Ensure that a single callback is added at a time+    std::lock_guard g(This::global().mutex_);+    auto idx = This::global().numCallbacks_.load(std::memory_order_acquire);++    if (idx >= (This::global().callbacks_).size()) {+      throw std::length_error(+          fmt::format("Too many callbacks - max {}", MaxCallbacks));+    }+    (This::global().callbacks_)[idx] = std::move(cb);+    // Only increment numCallbacks_ after fully initializing the array+    // entry. This step makes the new array entry visible to other threads.+    This::global().numCallbacks_.store(idx + 1, std::memory_order_release);+  }++ private:+  // use createGlobal to avoid races on shutdown+  struct GlobalStorage {+    mutable folly::SharedMutex mutex_;+    This::CallbackArray callbacks_{};+    std::atomic<size_t> numCallbacks_{0};+  };+  static auto& global() {+    return folly::detail::createGlobal<GlobalStorage, void>();+  }+};+} // namespace folly
+ folly/folly/Conv.cpp view
@@ -0,0 +1,740 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#include <folly/Conv.h>++#include <array>+#include <istream>++#include <folly/lang/SafeAssert.h>++#include <fast_float/fast_float.h>++namespace folly {+namespace detail {++namespace {++/**+ * Finds the first non-digit in a string. The number of digits+ * searched depends on the precision of the Tgt integral. Assumes the+ * string starts with NO whitespace and NO sign.+ *+ * The semantics of the routine is:+ *   for (;; ++b) {+ *     if (b >= e || !isdigit(*b)) return b;+ *   }+ *+ *  Complete unrolling marks bottom-line (i.e. entire conversion)+ *  improvements of 20%.+ */+inline const char* findFirstNonDigit(const char* b, const char* e) {+  for (; b < e; ++b) {+    auto const c = static_cast<unsigned>(*b) - '0';+    if (c >= 10) {+      break;+    }+  }+  return b;+}++// Maximum value of number when represented as a string+template <class T>+struct MaxString {+  static const char* const value;+};++template <>+const char* const MaxString<uint8_t>::value = "255";+template <>+const char* const MaxString<uint16_t>::value = "65535";+template <>+const char* const MaxString<uint32_t>::value = "4294967295";+#if __SIZEOF_LONG__ == 4+template <>+const char* const MaxString<unsigned long>::value = "4294967295";+#else+template <>+const char* const MaxString<unsigned long>::value = "18446744073709551615";+#endif+static_assert(+    sizeof(unsigned long) >= 4,+    "Wrong value for MaxString<unsigned long>::value,"+    " please update.");+template <>+const char* const MaxString<unsigned long long>::value = "18446744073709551615";+static_assert(+    sizeof(unsigned long long) >= 8,+    "Wrong value for MaxString<unsigned long long>::value"+    ", please update.");++#if FOLLY_HAVE_INT128_T+template <>+const char* const MaxString<__uint128_t>::value =+    "340282366920938463463374607431768211455";+#endif++/*+ * Lookup tables that converts from a decimal character value to an integral+ * binary value, shifted by a decimal "shift" multiplier.+ * For all character values in the range '0'..'9', the table at those+ * index locations returns the actual decimal value shifted by the multiplier.+ * For all other values, the lookup table returns an invalid OOR value.+ */+// Out-of-range flag value, larger than the largest value that can fit in+// four decimal bytes (9999), but four of these added up together should+// still not overflow uint16_t.+constexpr int32_t OOR = 10000;++alignas(16) constexpr uint16_t shift1[] = {+    OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, // 0-9+    OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, //  10+    OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, //  20+    OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, //  30+    OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, 0,   1, //  40+    2,   3,   4,   5,   6,   7,   8,   9,   OOR, OOR,+    OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, //  60+    OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, //  70+    OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, //  80+    OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, //  90+    OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, // 100+    OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, // 110+    OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, // 120+    OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, // 130+    OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, // 140+    OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, // 150+    OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, // 160+    OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, // 170+    OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, // 180+    OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, // 190+    OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, // 200+    OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, // 210+    OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, // 220+    OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, // 230+    OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, // 240+    OOR, OOR, OOR, OOR, OOR, OOR // 250+};++alignas(16) constexpr uint16_t shift10[] = {+    OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, // 0-9+    OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, //  10+    OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, //  20+    OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, //  30+    OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, 0,   10, //  40+    20,  30,  40,  50,  60,  70,  80,  90,  OOR, OOR,+    OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, //  60+    OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, //  70+    OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, //  80+    OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, //  90+    OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, // 100+    OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, // 110+    OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, // 120+    OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, // 130+    OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, // 140+    OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, // 150+    OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, // 160+    OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, // 170+    OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, // 180+    OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, // 190+    OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, // 200+    OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, // 210+    OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, // 220+    OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, // 230+    OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, // 240+    OOR, OOR, OOR, OOR, OOR, OOR // 250+};++alignas(16) constexpr uint16_t shift100[] = {+    OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, // 0-9+    OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, //  10+    OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, //  20+    OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, //  30+    OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, 0,   100, //  40+    200, 300, 400, 500, 600, 700, 800, 900, OOR, OOR,+    OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, //  60+    OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, //  70+    OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, //  80+    OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, //  90+    OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, // 100+    OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, // 110+    OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, // 120+    OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, // 130+    OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, // 140+    OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, // 150+    OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, // 160+    OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, // 170+    OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, // 180+    OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, // 190+    OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, // 200+    OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, // 210+    OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, // 220+    OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, // 230+    OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, OOR, // 240+    OOR, OOR, OOR, OOR, OOR, OOR // 250+};++alignas(16) constexpr uint16_t shift1000[] = {+    OOR,  OOR,  OOR,  OOR,  OOR,  OOR,  OOR,  OOR,  OOR, OOR, // 0-9+    OOR,  OOR,  OOR,  OOR,  OOR,  OOR,  OOR,  OOR,  OOR, OOR, //  10+    OOR,  OOR,  OOR,  OOR,  OOR,  OOR,  OOR,  OOR,  OOR, OOR, //  20+    OOR,  OOR,  OOR,  OOR,  OOR,  OOR,  OOR,  OOR,  OOR, OOR, //  30+    OOR,  OOR,  OOR,  OOR,  OOR,  OOR,  OOR,  OOR,  0,   1000, //  40+    2000, 3000, 4000, 5000, 6000, 7000, 8000, 9000, OOR, OOR,+    OOR,  OOR,  OOR,  OOR,  OOR,  OOR,  OOR,  OOR,  OOR, OOR, //  60+    OOR,  OOR,  OOR,  OOR,  OOR,  OOR,  OOR,  OOR,  OOR, OOR, //  70+    OOR,  OOR,  OOR,  OOR,  OOR,  OOR,  OOR,  OOR,  OOR, OOR, //  80+    OOR,  OOR,  OOR,  OOR,  OOR,  OOR,  OOR,  OOR,  OOR, OOR, //  90+    OOR,  OOR,  OOR,  OOR,  OOR,  OOR,  OOR,  OOR,  OOR, OOR, // 100+    OOR,  OOR,  OOR,  OOR,  OOR,  OOR,  OOR,  OOR,  OOR, OOR, // 110+    OOR,  OOR,  OOR,  OOR,  OOR,  OOR,  OOR,  OOR,  OOR, OOR, // 120+    OOR,  OOR,  OOR,  OOR,  OOR,  OOR,  OOR,  OOR,  OOR, OOR, // 130+    OOR,  OOR,  OOR,  OOR,  OOR,  OOR,  OOR,  OOR,  OOR, OOR, // 140+    OOR,  OOR,  OOR,  OOR,  OOR,  OOR,  OOR,  OOR,  OOR, OOR, // 150+    OOR,  OOR,  OOR,  OOR,  OOR,  OOR,  OOR,  OOR,  OOR, OOR, // 160+    OOR,  OOR,  OOR,  OOR,  OOR,  OOR,  OOR,  OOR,  OOR, OOR, // 170+    OOR,  OOR,  OOR,  OOR,  OOR,  OOR,  OOR,  OOR,  OOR, OOR, // 180+    OOR,  OOR,  OOR,  OOR,  OOR,  OOR,  OOR,  OOR,  OOR, OOR, // 190+    OOR,  OOR,  OOR,  OOR,  OOR,  OOR,  OOR,  OOR,  OOR, OOR, // 200+    OOR,  OOR,  OOR,  OOR,  OOR,  OOR,  OOR,  OOR,  OOR, OOR, // 210+    OOR,  OOR,  OOR,  OOR,  OOR,  OOR,  OOR,  OOR,  OOR, OOR, // 220+    OOR,  OOR,  OOR,  OOR,  OOR,  OOR,  OOR,  OOR,  OOR, OOR, // 230+    OOR,  OOR,  OOR,  OOR,  OOR,  OOR,  OOR,  OOR,  OOR, OOR, // 240+    OOR,  OOR,  OOR,  OOR,  OOR,  OOR // 250+};++struct ErrorString {+  const char* string;+  bool quote;+};++// Keep this in sync with ConversionCode in Conv.h+constexpr const std::array<+    ErrorString,+    static_cast<std::size_t>(ConversionCode::NUM_ERROR_CODES)>+    kErrorStrings{{+        // SUCCESS+        {"Success", true},+        // EMPTY_INPUT_STRING+        {"Empty input string", true},+        // NO_DIGITS+        {"No digits found in input string", true},+        // BOOL_OVERFLOW+        {"Integer overflow when parsing bool (must be 0 or 1)", true},+        // BOOL_INVALID_VALUE+        {"Invalid value for bool", true},+        // NON_DIGIT_CHAR+        {"Non-digit character found", true},+        // INVALID_LEADING_CHAR+        {"Invalid leading character", true},+        // POSITIVE_OVERFLOW+        {"Overflow during conversion", true},+        // NEGATIVE_OVERFLOW+        {"Negative overflow during conversion", true},+        // STRING_TO_FLOAT_ERROR+        {"Unable to convert string to floating point value", true},+        // NON_WHITESPACE_AFTER_END+        {"Non-whitespace character found after end of conversion", true},+        // ARITH_POSITIVE_OVERFLOW+        {"Overflow during arithmetic conversion", false},+        // ARITH_NEGATIVE_OVERFLOW+        {"Negative overflow during arithmetic conversion", false},+        // ARITH_LOSS_OF_PRECISION+        {"Loss of precision during arithmetic conversion", false},+        // SPLIT_ERROR,+        {"Unexpected number of fields resulting from a split", true},+        // CUSTOM,+        {"Custom conversion failed", true},+    }};++// Check if ASCII is really ASCII+using IsAscii =+    std::bool_constant<'A' == 65 && 'Z' == 90 && 'a' == 97 && 'z' == 122>;++// The code in this file that uses tolower() really only cares about+// 7-bit ASCII characters, so we can take a nice shortcut here.+inline char tolower_ascii(char in) {+  return IsAscii::value ? in | 0x20 : char(std::tolower(in));+}++inline bool bool_str_cmp(const char** b, size_t len, const char* value) {+  // Can't use strncasecmp, since we want to ensure that the full value matches+  const char* p = *b;+  const char* e = *b + len;+  const char* v = value;+  while (*v != '\0') {+    if (p == e || tolower_ascii(*p) != *v) { // value is already lowercase+      return false;+    }+    ++p;+    ++v;+  }++  *b = p;+  return true;+}++} // namespace++Expected<bool, ConversionCode> str_to_bool(StringPiece* src) noexcept {+  auto b = src->begin(), e = src->end();+  for (;; ++b) {+    if (b >= e) {+      return makeUnexpected(ConversionCode::EMPTY_INPUT_STRING);+    }+    if ((*b < '\t' || *b > '\r') && *b != ' ') {+      break;+    }+  }++  bool result;+  auto len = size_t(e - b);+  switch (*b) {+    case '0':+    case '1': {+      result = false;+      for (; b < e && isdigit(*b); ++b) {+        if (result || (*b != '0' && *b != '1')) {+          return makeUnexpected(ConversionCode::BOOL_OVERFLOW);+        }+        result = (*b == '1');+      }+      break;+    }+    case 'y':+    case 'Y':+      result = true;+      if (!bool_str_cmp(&b, len, "yes")) {+        ++b; // accept the single 'y' character+      }+      break;+    case 'n':+    case 'N':+      result = false;+      if (!bool_str_cmp(&b, len, "no")) {+        ++b;+      }+      break;+    case 't':+    case 'T':+      result = true;+      if (!bool_str_cmp(&b, len, "true")) {+        ++b;+      }+      break;+    case 'f':+    case 'F':+      result = false;+      if (!bool_str_cmp(&b, len, "false")) {+        ++b;+      }+      break;+    case 'o':+    case 'O':+      if (bool_str_cmp(&b, len, "on")) {+        result = true;+      } else if (bool_str_cmp(&b, len, "off")) {+        result = false;+      } else {+        return makeUnexpected(ConversionCode::BOOL_INVALID_VALUE);+      }+      break;+    default:+      return makeUnexpected(ConversionCode::BOOL_INVALID_VALUE);+  }++  src->assign(b, e);++  return result;+}++/// Uses `fast_float::from_chars` to convert from string to an integer.+template <class Tgt>+Expected<Tgt, ConversionCode> str_to_floating_fast_float_from_chars(+    StringPiece* src) noexcept {+  if (src->empty()) {+    return makeUnexpected(ConversionCode::EMPTY_INPUT_STRING);+  }++  // move through leading whitespace characters+  auto* e = src->end();+  auto* b = std::find_if_not(src->begin(), e, [](char c) {+    return (c >= '\t' && c <= '\r') || c == ' ';+  });+  if (b == e) {+    return makeUnexpected(ConversionCode::EMPTY_INPUT_STRING);+  }++  Tgt result;+  fast_float::parse_options options{+      fast_float::chars_format::general |+      fast_float::chars_format::allow_leading_plus};+  auto [ptr, ec] = fast_float::from_chars_advanced(b, e, result, options);+  bool isOutOfRange{ec == std::errc::result_out_of_range};+  bool isOk{ec == std::errc()};+  if (!isOk && !isOutOfRange) {+    return makeUnexpected(ConversionCode::STRING_TO_FLOAT_ERROR);+  }++  auto numMatchedChars = ptr - src->data();+  src->advance(numMatchedChars);+  return result;+}++template Expected<float, ConversionCode>+str_to_floating_fast_float_from_chars<float>(StringPiece* src) noexcept;+template Expected<double, ConversionCode>+str_to_floating_fast_float_from_chars<double>(StringPiece* src) noexcept;++/**+ * StringPiece to double, with progress information. Alters the+ * StringPiece parameter to munch the already-parsed characters.+ */+template <class Tgt>+Expected<Tgt, ConversionCode> str_to_floating(StringPiece* src) noexcept {+  return detail::str_to_floating_fast_float_from_chars<Tgt>(src);+}++template Expected<float, ConversionCode> str_to_floating<float>(+    StringPiece* src) noexcept;+template Expected<double, ConversionCode> str_to_floating<double>(+    StringPiece* src) noexcept;++namespace {++/**+ * This class takes care of additional processing needed for signed values,+ * like leading sign character and overflow checks.+ */+template <typename T, bool IsSigned = is_signed_v<T>>+class SignedValueHandler;++template <typename T>+class SignedValueHandler<T, true> {+ public:+  ConversionCode init(const char*& b) {+    negative_ = false;+    if (!std::isdigit(*b)) {+      if (*b == '-') {+        negative_ = true;+      } else if (FOLLY_UNLIKELY(*b != '+')) {+        return ConversionCode::INVALID_LEADING_CHAR;+      }+      ++b;+    }+    return ConversionCode::SUCCESS;+  }++  ConversionCode overflow() {+    return negative_+        ? ConversionCode::NEGATIVE_OVERFLOW+        : ConversionCode::POSITIVE_OVERFLOW;+  }++  template <typename U>+  Expected<T, ConversionCode> finalize(U value) {+    T rv;+    if (negative_) {+      FOLLY_PUSH_WARNING+      FOLLY_MSVC_DISABLE_WARNING(4146)++      // unary minus operator applied to unsigned type, result still unsigned+      rv = T(-value);++      FOLLY_POP_WARNING++      if (FOLLY_UNLIKELY(rv > 0)) {+        return makeUnexpected(ConversionCode::NEGATIVE_OVERFLOW);+      }+    } else {+      rv = T(value);+      if (FOLLY_UNLIKELY(rv < 0)) {+        return makeUnexpected(ConversionCode::POSITIVE_OVERFLOW);+      }+    }+    return rv;+  }++ private:+  bool negative_;+};++// For unsigned types, we don't need any extra processing+template <typename T>+class SignedValueHandler<T, false> {+ public:+  ConversionCode init(const char*&) { return ConversionCode::SUCCESS; }++  ConversionCode overflow() { return ConversionCode::POSITIVE_OVERFLOW; }++  Expected<T, ConversionCode> finalize(T value) { return value; }+};++} // namespace++/**+ * String represented as a pair of pointers to char to signed/unsigned+ * integrals. Assumes NO whitespace before or after, and also that the+ * string is composed entirely of digits (and an optional sign only for+ * signed types). String may be empty, in which case digits_to returns+ * an appropriate error.+ */+template <class Tgt>+inline Expected<Tgt, ConversionCode> digits_to(+    const char* b, const char* const e) noexcept {+  using UT = make_unsigned_t<Tgt>;+  assert(b <= e);++  SignedValueHandler<Tgt> sgn;++  auto err = sgn.init(b);+  if (FOLLY_UNLIKELY(err != ConversionCode::SUCCESS)) {+    return makeUnexpected(err);+  }++  auto size = size_t(e - b);++  /* Although the string is entirely made of digits, we still need to+   * check for overflow.+   */+  if (size > std::numeric_limits<UT>::digits10) {+    // Leading zeros?+    if (b < e && *b == '0') {+      for (++b;; ++b) {+        if (b == e) {+          return Tgt(0); // just zeros, e.g. "0000"+        }+        if (*b != '0') {+          size = size_t(e - b);+          break;+        }+      }+    }+    if (size > std::numeric_limits<UT>::digits10 &&+        (size != std::numeric_limits<UT>::digits10 + 1 ||+         strncmp(b, MaxString<UT>::value, size) > 0)) {+      return makeUnexpected(sgn.overflow());+    }+  }++  // Here we know that the number won't overflow when+  // converted. Proceed without checks.++  UT result = 0;++  for (; e - b >= 4; b += 4) {+    result *= UT(10000);+    const int32_t r0 = shift1000[static_cast<size_t>(b[0])];+    const int32_t r1 = shift100[static_cast<size_t>(b[1])];+    const int32_t r2 = shift10[static_cast<size_t>(b[2])];+    const int32_t r3 = shift1[static_cast<size_t>(b[3])];+    const auto sum = r0 + r1 + r2 + r3;+    if (sum >= OOR) {+      goto outOfRange;+    }+    result += UT(sum);+  }++  switch (e - b) {+    case 3: {+      const int32_t r0 = shift100[static_cast<size_t>(b[0])];+      const int32_t r1 = shift10[static_cast<size_t>(b[1])];+      const int32_t r2 = shift1[static_cast<size_t>(b[2])];+      const auto sum = r0 + r1 + r2;+      if (sum >= OOR) {+        goto outOfRange;+      }+      result = UT(1000 * result + sum);+      break;+    }+    case 2: {+      const int32_t r0 = shift10[static_cast<size_t>(b[0])];+      const int32_t r1 = shift1[static_cast<size_t>(b[1])];+      const auto sum = r0 + r1;+      if (sum >= OOR) {+        goto outOfRange;+      }+      result = UT(100 * result + sum);+      break;+    }+    case 1: {+      const int32_t sum = shift1[static_cast<size_t>(b[0])];+      if (sum >= OOR) {+        goto outOfRange;+      }+      result = UT(10 * result + sum);+      break;+    }+    default:+      assert(b == e);+      if (size == 0) {+        return makeUnexpected(ConversionCode::NO_DIGITS);+      }+      break;+  }++  return sgn.finalize(result);++outOfRange:+  return makeUnexpected(ConversionCode::NON_DIGIT_CHAR);+}++template Expected<char, ConversionCode> digits_to<char>(+    const char*, const char*) noexcept;+template Expected<signed char, ConversionCode> digits_to<signed char>(+    const char*, const char*) noexcept;+template Expected<unsigned char, ConversionCode> digits_to<unsigned char>(+    const char*, const char*) noexcept;++template Expected<short, ConversionCode> digits_to<short>(+    const char*, const char*) noexcept;+template Expected<unsigned short, ConversionCode> digits_to<unsigned short>(+    const char*, const char*) noexcept;++template Expected<int, ConversionCode> digits_to<int>(+    const char*, const char*) noexcept;+template Expected<unsigned int, ConversionCode> digits_to<unsigned int>(+    const char*, const char*) noexcept;++template Expected<long, ConversionCode> digits_to<long>(+    const char*, const char*) noexcept;+template Expected<unsigned long, ConversionCode> digits_to<unsigned long>(+    const char*, const char*) noexcept;++template Expected<long long, ConversionCode> digits_to<long long>(+    const char*, const char*) noexcept;+template Expected<unsigned long long, ConversionCode>+digits_to<unsigned long long>(const char*, const char*) noexcept;++#if FOLLY_HAVE_INT128_T+template Expected<__int128, ConversionCode> digits_to<__int128>(+    const char*, const char*) noexcept;+template Expected<unsigned __int128, ConversionCode>+digits_to<unsigned __int128>(const char*, const char*) noexcept;+#endif++/**+ * StringPiece to integrals, with progress information. Alters the+ * StringPiece parameter to munch the already-parsed characters.+ */+template <class Tgt>+Expected<Tgt, ConversionCode> str_to_integral(StringPiece* src) noexcept {+  using UT = make_unsigned_t<Tgt>;++  auto b = src->data(), past = src->data() + src->size();++  for (;; ++b) {+    if (FOLLY_UNLIKELY(b >= past)) {+      return makeUnexpected(ConversionCode::EMPTY_INPUT_STRING);+    }+    if ((*b < '\t' || *b > '\r') && *b != ' ') {+      break;+    }+  }++  SignedValueHandler<Tgt> sgn;+  auto err = sgn.init(b);++  if (FOLLY_UNLIKELY(err != ConversionCode::SUCCESS)) {+    return makeUnexpected(err);+  }+  if (is_signed_v<Tgt> && FOLLY_UNLIKELY(b >= past)) {+    return makeUnexpected(ConversionCode::NO_DIGITS);+  }+  if (FOLLY_UNLIKELY(!isdigit(*b))) {+    return makeUnexpected(ConversionCode::NON_DIGIT_CHAR);+  }++  auto m = findFirstNonDigit(b + 1, past);++  auto tmp = digits_to<UT>(b, m);++  if (FOLLY_UNLIKELY(!tmp.hasValue())) {+    return makeUnexpected(+        tmp.error() == ConversionCode::POSITIVE_OVERFLOW+            ? sgn.overflow()+            : tmp.error());+  }++  auto res = sgn.finalize(tmp.value());++  if (res.hasValue()) {+    src->advance(size_t(m - src->data()));+  }++  return res;+}++template Expected<char, ConversionCode> str_to_integral<char>(+    StringPiece* src) noexcept;+template Expected<signed char, ConversionCode> str_to_integral<signed char>(+    StringPiece* src) noexcept;+template Expected<unsigned char, ConversionCode> str_to_integral<unsigned char>(+    StringPiece* src) noexcept;++template Expected<short, ConversionCode> str_to_integral<short>(+    StringPiece* src) noexcept;+template Expected<unsigned short, ConversionCode>+str_to_integral<unsigned short>(StringPiece* src) noexcept;++template Expected<int, ConversionCode> str_to_integral<int>(+    StringPiece* src) noexcept;+template Expected<unsigned int, ConversionCode> str_to_integral<unsigned int>(+    StringPiece* src) noexcept;++template Expected<long, ConversionCode> str_to_integral<long>(+    StringPiece* src) noexcept;+template Expected<unsigned long, ConversionCode> str_to_integral<unsigned long>(+    StringPiece* src) noexcept;++template Expected<long long, ConversionCode> str_to_integral<long long>(+    StringPiece* src) noexcept;+template Expected<unsigned long long, ConversionCode>+str_to_integral<unsigned long long>(StringPiece* src) noexcept;++#if FOLLY_HAVE_INT128_T+template Expected<__int128, ConversionCode> str_to_integral<__int128>(+    StringPiece* src) noexcept;+template Expected<unsigned __int128, ConversionCode>+str_to_integral<unsigned __int128>(StringPiece* src) noexcept;+#endif+} // namespace detail++ConversionError makeConversionError(ConversionCode code, StringPiece input) {+  using namespace detail;+  static_assert(+      std::is_unsigned<std::underlying_type<ConversionCode>::type>::value,+      "ConversionCode should be unsigned");+  auto index = static_cast<std::size_t>(code);+  FOLLY_SAFE_CHECK(index < kErrorStrings.size(), "code=", uint64_t(index));+  const ErrorString& err = kErrorStrings[index];+  if (code == ConversionCode::EMPTY_INPUT_STRING && input.empty()) {+    return {err.string, code};+  }+  std::string tmp(err.string);+  tmp.append(": ");+  if (err.quote) {+    tmp.append(1, '"');+  }+  if (!input.empty()) {+    tmp.append(input.data(), input.size());+  }+  if (err.quote) {+    tmp.append(1, '"');+  }+  return {tmp, code};+}++} // namespace folly
+ folly/folly/Conv.h view
@@ -0,0 +1,1738 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++//+// Docs: https://fburl.com/fbcref_conv+//++/**+ * Conv provides the ubiquitous method `to<TargetType>(source)`, along with+ * a few other generic interfaces for converting objects to and from+ * string-like types (std::string, fbstring, StringPiece), as well as+ * range-checked conversions between numeric and enum types. The mechanisms are+ * extensible, so that user-specified types can add folly::to support.+ *+ *     folly::to<std::string>(123)+ *     // "123"+ *+ *******************************************************************************+ * ## TYPE -> STRING CONVERSIONS+ *******************************************************************************+ * You can call the `to<std::string>` or `to<fbstring>`. These are variadic+ * functions that convert their arguments to strings, and concatenate them to+ * form a result. So, for example,+ *+ *     auto str = to<std::string>(123, "456", 789);+ *+ * Sets str to `"123456789"`.+ *+ * In addition to just concatenating the arguments, related functions can+ * delimit them with some string: `toDelim<std::string>(",", "123", 456, "789")`+ * will return the string `"123,456,789"`.+ *+ * toAppend does not return a string; instead, it takes a pointer to a string as+ * its last argument, and appends the result of the concatenation into it:+ *     std::string str = "123";+ *     toAppend(456, "789", &str); // Now str is "123456789".+ *+ * The toAppendFit function acts like toAppend, but it precalculates the size+ * required to perform the append operation, and reserves that space in the+ * output string before actually inserting its arguments. This can sometimes+ * save on string expansion, but beware: appending to the same string many times+ * with toAppendFit is likely a pessimization, since it will resize the string+ * once per append.+ *+ * The combination of the append and delim variants also exist: toAppendDelim+ * and toAppendDelimFit are defined, with the obvious semantics.+ *+ *******************************************************************************+ * ## STRING -> TYPE CONVERSIONS+ *******************************************************************************+ * Going in the other direction, and parsing a string into a C++ type, is also+ * supported:+ *     to<int>("123"); // Returns 123.+ *+ * Out of range (e.g. `to<std::uint8_t>("1000")`), or invalidly formatted (e.g.+ * `to<int>("four")`) inputs will throw. If throw-on-error is undesirable (for+ * instance: you're dealing with untrusted input, and want to protect yourself+ * from users sending you down a very slow exception-throwing path), you can use+ * `tryTo<T>`, which will return an `Expected<T, ConversionCode>`.+ *+ * There are overloads of to() and tryTo() that take a `StringPiece*`. These+ * parse out a type from the beginning of a string, and modify the passed-in+ * StringPiece to indicate the portion of the string not consumed.+ *+ *******************************************************************************+ * ## NUMERIC / ENUM CONVERSIONS+ *******************************************************************************+ * Conv also supports a `to<T>(S)` overload, where T and S are numeric or enum+ * types, that checks to see that the target type can represent its argument,+ * and will throw if it cannot. This includes cases where a floating point to+ * integral conversion is attempted on a value with a non-zero fractional+ * component, and integral to floating point conversions that would lose+ * precision. Enum conversions are range-checked for the underlying type of the+ * enum, but there is no check that the input value is a valid choice of enum+ * value.+ *+ *******************************************************************************+ * ## CUSTOM TYPE CONVERSIONS+ *******************************************************************************+ * Users may customize the string conversion functionality for their own data+ * types. The key functions you should implement are:+ *     // Two functions to allow conversion to your type from a string.+ *     Expected<StringPiece, ConversionCode> parseTo(folly::StringPiece in,+ *         YourType& out);+ *     YourErrorType makeConversionError(YourErrorType in, StringPiece in);+ *     // Two functions to allow conversion from your type to a string.+ *     template <class String>+ *   void toAppend(const YourType& in, String* out);+ *       size_t estimateSpaceNeeded(const YourType& in);+ *+ * These are documented below, inline.+ *+ * @file Conv.h+ */++#pragma once++#include <algorithm>+#include <cassert>+#include <cctype>+#include <climits>+#include <cmath>+#include <cstddef>+#include <limits>+#include <optional>+#include <stdexcept>+#include <string>+#include <system_error>+#include <tuple>+#include <type_traits>+#include <utility>++#if __has_include(<charconv>)+#include <charconv>+#endif++#include <double-conversion/double-conversion.h> // V8 JavaScript implementation++#include <folly/CPortability.h>++#include <folly/Demangle.h>+#include <folly/Expected.h>+#include <folly/FBString.h>+#include <folly/Likely.h>+#include <folly/Portability.h>+#include <folly/Range.h>+#include <folly/Traits.h>+#include <folly/Unit.h>+#include <folly/Utility.h>+#include <folly/lang/Exception.h>+#include <folly/lang/Pretty.h>+#include <folly/lang/ToAscii.h>+#include <folly/portability/Math.h>++namespace folly {++// Keep this in sync with kErrorStrings in Conv.cpp+enum class ConversionCode : unsigned char {+  SUCCESS,+  EMPTY_INPUT_STRING,+  NO_DIGITS,+  BOOL_OVERFLOW,+  BOOL_INVALID_VALUE,+  NON_DIGIT_CHAR,+  INVALID_LEADING_CHAR,+  POSITIVE_OVERFLOW,+  NEGATIVE_OVERFLOW,+  STRING_TO_FLOAT_ERROR,+  NON_WHITESPACE_AFTER_END,+  ARITH_POSITIVE_OVERFLOW,+  ARITH_NEGATIVE_OVERFLOW,+  ARITH_LOSS_OF_PRECISION,+  SPLIT_ERROR,+  CUSTOM,+  NUM_ERROR_CODES, // has to be the last entry+};++struct FOLLY_EXPORT ConversionErrorBase : std::range_error {+  using std::range_error::range_error;+};++class FOLLY_EXPORT ConversionError : public ConversionErrorBase {+ public:+  ConversionError(const std::string& str, ConversionCode code)+      : ConversionErrorBase(str), code_(code) {}++  ConversionError(const char* str, ConversionCode code)+      : ConversionErrorBase(str), code_(code) {}++  ConversionCode errorCode() const { return code_; }++ private:+  ConversionCode code_;+};++/**+ * Custom Error Translation+ *+ * Your overloaded parseTo() function can return a custom error code on failure.+ * ::folly::to() will call makeConversionError to translate that error code into+ * an object to throw. makeConversionError is found by argument-dependent+ * lookup. It should have this signature:+ *+ * namespace other_namespace {+ * enum YourErrorCode { BAD_ERROR, WORSE_ERROR };+ *+ * struct YourConversionError : ConversionErrorBase {+ *   YourConversionError(const char* what) : ConversionErrorBase(what) {}+ * };+ *+ * YourConversionError+ * makeConversionError(YourErrorCode code, ::folly::StringPiece sp) {+ *   ...+ *   return YourConversionError(messageString);+ * }+ */+ConversionError makeConversionError(ConversionCode code, StringPiece input);++namespace detail {+/**+ * Enforce that the suffix following a number is made up only of whitespace.+ */+inline ConversionCode enforceWhitespaceErr(StringPiece sp) {+  for (auto c : sp) {+    if (FOLLY_UNLIKELY(!std::isspace(c))) {+      return ConversionCode::NON_WHITESPACE_AFTER_END;+    }+  }+  return ConversionCode::SUCCESS;+}++/**+ * Keep this implementation around for prettyToDouble().+ */+inline void enforceWhitespace(StringPiece sp) {+  auto err = enforceWhitespaceErr(sp);+  if (err != ConversionCode::SUCCESS) {+    throw_exception(makeConversionError(err, sp));+  }+}+} // namespace detail++/**+ * @overloadbrief to, but return an Expected+ *+ * The identity conversion function.+ * tryTo<T>(T) returns itself for all types T.+ */+template <class Tgt, class Src>+typename std::enable_if<+    std::is_same<Tgt, typename std::decay<Src>::type>::value,+    Expected<Tgt, ConversionCode>>::type+tryTo(Src&& value) noexcept {+  return static_cast<Src&&>(value);+}++/**+ * @overloadbrief Convert from one type to another.+ */+template <class Tgt, class Src>+typename std::enable_if<+    std::is_same<Tgt, typename std::decay<Src>::type>::value,+    Tgt>::type+to(Src&& value) {+  return static_cast<Src&&>(value);+}++/**+ * Arithmetic to boolean+ */++/**+ * Unchecked conversion from arithmetic to boolean. This is different from the+ * other arithmetic conversions because we use the C convention of treating any+ * non-zero value as true, instead of range checking.+ */+template <class Tgt, class Src>+typename std::enable_if<+    is_arithmetic_v<Src> && !std::is_same<Tgt, Src>::value &&+        std::is_same<Tgt, bool>::value,+    Expected<Tgt, ConversionCode>>::type+tryTo(const Src& value) noexcept {+  return value != Src();+}++template <class Tgt, class Src>+typename std::enable_if<+    is_arithmetic_v<Src> && !std::is_same<Tgt, Src>::value &&+        std::is_same<Tgt, bool>::value,+    Tgt>::type+to(const Src& value) {+  return value != Src();+}++/**+ * Anything to string+ */++namespace detail {++template <class... T>+using LastElement = type_pack_element_t<sizeof...(T) - 1, T...>;++#ifdef _MSC_VER+// MSVC can't quite figure out the LastElementImpl::call() stuff+// in the base implementation, so we have to use tuples instead,+// which result in significantly more templates being compiled,+// though the runtime performance is the same.++template <typename... Ts, typename R = LastElement<Ts...>>+const R& getLastElement(const Ts&... ts) {+  return std::get<sizeof...(Ts) - 1>(std::forward_as_tuple(ts...));+}++inline void getLastElement() {}+#else+template <typename...>+struct LastElementImpl;+template <>+struct LastElementImpl<> {+  static void call() {}+};+template <typename Ign, typename... Igns>+struct LastElementImpl<Ign, Igns...> {+  template <typename Last>+  static const Last& call(Igns..., const Last& last) {+    return last;+  }+};++template <typename... Ts, typename R = LastElement<Ts...>>+const R& getLastElement(const Ts&... ts) {+  return LastElementImpl<Ignored<Ts>...>::call(ts...);+}+#endif++} // namespace detail++/**+ * Conversions from integral types to string types.+ */++#if FOLLY_HAVE_INT128_T+namespace detail {++template <typename IntegerType>+constexpr unsigned int digitsEnough() {+  // digits10 returns the number of decimal digits that this type can represent,+  // not the number of characters required for the max value, so we need to add+  // one. ex: char digits10 returns 2, because 256-999 cannot be represented,+  // but we need 3.+  auto const digits10 = std::numeric_limits<IntegerType>::digits10;+  return static_cast<unsigned int>(digits10) + 1;+}++inline size_t unsafeTelescope128(char* outb, char* oute, unsigned __int128 x) {+  using Usrc = unsigned __int128;++  // Decompose the input into at most 3 components using the largest power-of-10+  // base that fits in a 64-bit unsigned integer, and then convert the+  // components using 64-bit arithmetic and concatenate them.+  constexpr static auto kBase = UINT64_C(10'000'000'000'000'000'000);+  constexpr static size_t kBaseDigits = 19;++  size_t p = 0;+  const auto leading = [&](Usrc v) {+    assert(v >> 64 == 0);+    p = detail::to_ascii_with_route<10, to_ascii_alphabet_lower>(+        outb, oute, static_cast<uint64_t>(v));+  };+  const auto append = [&](uint64_t v) {+    assert(v < kBase);+    assert(outb + p + kBaseDigits <= oute);+    auto v64 = static_cast<uint64_t>(v);+    detail::to_ascii_with_route<10, to_ascii_alphabet_lower>(+        outb + p, kBaseDigits, v64);+    p += kBaseDigits;+  };++  if (x >> 64 > 0) {+    const auto rem = static_cast<uint64_t>(x % kBase);+    x /= kBase;++    if (x >> 64 > 0) {+      const auto rem2 = static_cast<uint64_t>(x % kBase);+      x /= kBase;++      leading(x);+      append(rem2);+      append(rem);+      return p;+    }++    leading(x);+    append(rem);+    return p;+  }++  leading(x);+  return p;+}++} // namespace detail+#endif++/**+ * @overloadbrief Appends conversion to string.+ *+ * A single char gets appended.+ */+template <class Tgt>+void toAppend(char value, Tgt* result) {+  *result += value;+}++/**+ * @overloadbrief Estimates the number of characters in a value's string+ * representation.+ */+template <class T>+constexpr typename std::enable_if<std::is_same<T, char>::value, size_t>::type+estimateSpaceNeeded(T) {+  return 1;+}++template <size_t N>+constexpr size_t estimateSpaceNeeded(const char (&)[N]) {+  return N;+}++/**+ * Everything implicitly convertible to const char* gets appended.+ */+template <class Tgt, class Src>+typename std::enable_if<+    std::is_convertible<Src, const char*>::value &&+    IsSomeString<Tgt>::value>::type+toAppend(Src value, Tgt* result) {+  // Treat null pointers like an empty string, as in:+  // operator<<(std::ostream&, const char*).+  const char* c = value;+  if (c) {+    result->append(value);+  }+}++template <class Src>+typename std::enable_if<std::is_convertible<Src, const char*>::value, size_t>::+    type+    estimateSpaceNeeded(Src value) {+  const char* c = value;+  return c ? std::strlen(c) : 0;+}++template <class Src>+typename std::enable_if<IsSomeString<Src>::value, size_t>::type+estimateSpaceNeeded(Src const& value) {+  return value.size();+}++template <class Src>+typename std::enable_if<+    std::is_convertible<Src, folly::StringPiece>::value &&+        !IsSomeString<Src>::value &&+        !std::is_convertible<Src, const char*>::value,+    size_t>::type+estimateSpaceNeeded(Src value) {+  return folly::StringPiece(value).size();+}++template <>+inline size_t estimateSpaceNeeded(std::nullptr_t /* value */) {+  return 0;+}++template <class Src>+typename std::enable_if<+    std::is_pointer<Src>::value &&+        IsSomeString<std::remove_pointer<Src>>::value,+    size_t>::type+estimateSpaceNeeded(Src value) {+  return value->size();+}++/**+ * Strings get appended, too.+ */+template <class Tgt, class Src>+typename std::enable_if<+    IsSomeString<Src>::value && IsSomeString<Tgt>::value>::type+toAppend(const Src& value, Tgt* result) {+  result->append(value);+}++/**+ * and StringPiece objects too+ */+template <class Tgt>+typename std::enable_if<IsSomeString<Tgt>::value>::type toAppend(+    StringPiece value, Tgt* result) {+  result->append(value.data(), value.size());+}++/**+ * There's no implicit conversion from fbstring to other string types,+ * so make a specialization.+ */+template <class Tgt>+typename std::enable_if<IsSomeString<Tgt>::value>::type toAppend(+    const fbstring& value, Tgt* result) {+  result->append(value.data(), value.size());+}++#if FOLLY_HAVE_INT128_T+/**+ * Special handling for 128 bit integers.+ */++template <class Tgt>+void toAppend(__int128 value, Tgt* result) {+  typedef unsigned __int128 Usrc;+  char buffer[detail::digitsEnough<unsigned __int128>() + 1];+  const auto oute = buffer + sizeof(buffer);+  size_t p;++  if (value < 0) {+    buffer[0] = '-';+    p = 1 + detail::unsafeTelescope128(buffer + 1, oute, -Usrc(value));+  } else {+    p = detail::unsafeTelescope128(buffer, oute, value);+  }++  result->append(buffer, p);+}++template <class Tgt>+void toAppend(unsigned __int128 value, Tgt* result) {+  char buffer[detail::digitsEnough<unsigned __int128>()];+  size_t p = detail::unsafeTelescope128(buffer, buffer + sizeof(buffer), value);+  result->append(buffer, p);+}++template <class T>+constexpr+    typename std::enable_if<std::is_same<T, __int128>::value, size_t>::type+    estimateSpaceNeeded(T) {+  return detail::digitsEnough<__int128>();+}++template <class T>+constexpr typename std::+    enable_if<std::is_same<T, unsigned __int128>::value, size_t>::type+    estimateSpaceNeeded(T) {+  return detail::digitsEnough<unsigned __int128>();+}++#endif++/**+ * int32_t and int64_t to string (by appending) go through here. The+ * result is APPENDED to a preexisting string passed as the second+ * parameter. This should be efficient with fbstring because fbstring+ * incurs no dynamic allocation below 23 bytes and no number has more+ * than 22 bytes in its textual representation (20 for digits, one for+ * sign, one for the terminating 0).+ */+template <class Tgt, class Src>+typename std::enable_if<+    is_integral_v<Src> && is_signed_v<Src> && IsSomeString<Tgt>::value &&+    sizeof(Src) >= 4>::type+toAppend(Src value, Tgt* result) {+  char buffer[to_ascii_size_max_decimal<uint64_t>];+  auto uvalue = value < 0+      ? ~static_cast<uint64_t>(value) + 1+      : static_cast<uint64_t>(value);+  if (value < 0) {+    result->push_back('-');+  }+  result->append(buffer, to_ascii_decimal(buffer, uvalue));+}++template <class Src>+typename std::enable_if<+    is_integral_v<Src> && is_signed_v<Src> && sizeof(Src) >= 4 &&+        sizeof(Src) < 16,+    size_t>::type+estimateSpaceNeeded(Src value) {+  auto uvalue = value < 0+      ? ~static_cast<uint64_t>(value) + 1+      : static_cast<uint64_t>(value);+  return size_t(value < 0) + to_ascii_size_decimal(uvalue);+}++/**+ * As above, but for uint32_t and uint64_t.+ */+template <class Tgt, class Src>+typename std::enable_if<+    is_integral_v<Src> && !is_signed_v<Src> && IsSomeString<Tgt>::value &&+    sizeof(Src) >= 4>::type+toAppend(Src value, Tgt* result) {+  char buffer[to_ascii_size_max_decimal<uint64_t>];+  result->append(buffer, to_ascii_decimal(buffer, value));+}++template <class Src>+typename std::enable_if<+    is_integral_v<Src> && !is_signed_v<Src> && sizeof(Src) >= 4 &&+        sizeof(Src) < 16,+    size_t>::type+estimateSpaceNeeded(Src value) {+  return to_ascii_size_decimal(value);+}++/**+ * All small signed and unsigned integers to string go through 32-bit+ * types int32_t and uint32_t, respectively.+ */+template <class Tgt, class Src>+typename std::enable_if<+    is_integral_v<Src> && IsSomeString<Tgt>::value && sizeof(Src) < 4>::type+toAppend(Src value, Tgt* result) {+  typedef typename std::conditional<is_signed_v<Src>, int64_t, uint64_t>::type+      Intermediate;+  toAppend<Tgt>(static_cast<Intermediate>(value), result);+}++template <class Src>+typename std::enable_if<+    is_integral_v<Src> && sizeof(Src) < 4 && !std::is_same<Src, char>::value,+    size_t>::type+estimateSpaceNeeded(Src value) {+  typedef typename std::conditional<is_signed_v<Src>, int64_t, uint64_t>::type+      Intermediate;+  return estimateSpaceNeeded(static_cast<Intermediate>(value));+}++/**+ * Enumerated values get appended as integers.+ */+template <class Tgt, class Src>+typename std::enable_if<+    std::is_enum<Src>::value && IsSomeString<Tgt>::value>::type+toAppend(Src value, Tgt* result) {+  toAppend(to_underlying(value), result);+}++template <class Src>+typename std::enable_if<std::is_enum<Src>::value, size_t>::type+estimateSpaceNeeded(Src value) {+  return estimateSpaceNeeded(to_underlying(value));+}++/**+ * Conversions from floating-point types to string types.+ */++/// Operating mode for the floating point type version of+/// `folly::ToAppend`. This is modeled after+/// `double_conversion::DoubleToStringConverter::DtoaMode`.+/// Dtoa is an acryonym for Double to ASCII.+enum class DtoaMode {+  /// Outputs the shortest representation of a `double`.+  /// The output is either in decimal or exponential notation; which ever is+  /// shortest.+  SHORTEST,+  /// Outputs the shortest representation of a `float`.+  /// This outputs in either decimal or exponential notation, which ever is+  /// shortest.+  SHORTEST_SINGLE,+  /// Outputs fixed precision after the decimal point. Similar to+  /// `printf`'s %f.+  /// The output is in decimal notation.+  /// Use the `numDigits` parameter to specify the precision.+  FIXED,+  /// Outputs with a precision that is independent of the decimal point.+  /// The outputs is either decimal or exponential notation, depending on the+  /// value and the precision.+  /// Similar to `printf`'s %g formating.+  /// Use the `numDigits` parameter to specify the precision.+  PRECISION,+};++/// Flags for the floating point type version of `folly::ToAppend`.+/// This is modeled after `double_conversion::DoubleToStringConverter::Flags`.+/// Dtoa is an acryonym for Double to ASCII.+/// This enum is used to store bit wise flags, so a variable of this type may be+/// a bitwise combination of these definitions.+enum class DtoaFlags {+  NO_FLAGS = 0,+  /// Emits a plus sign for positive exponents. e.g., 1.2e+3+  EMIT_POSITIVE_EXPONENT_SIGN = 1,+  /// Emits a trailing decimal point. e.g., 123.+  EMIT_TRAILING_DECIMAL_POINT = 2,+  /// Emits a trailing decimal point. e.g., 123.0+  /// Requires `EMIT_TRAILING_DECIMAL_POINT` to be set.+  EMIT_TRAILING_ZERO_AFTER_POINT = 4,+  /// -0.0 outputs as 0.0+  UNIQUE_ZERO = 8,+  /// Trailing zeros are removed from the fractional portion+  /// of the result in precision mode. Matches `printf`'s %g.+  /// When `EMIT_TRAILING_ZERO_AFTER_POINT` is also given, one trailing zero is+  /// preserved.+  NO_TRAILING_ZERO = 16,+};++constexpr DtoaFlags operator|(DtoaFlags a, DtoaFlags b) {+  return static_cast<DtoaFlags>(to_underlying(a) | to_underlying(b));+}++constexpr DtoaFlags operator&(DtoaFlags a, DtoaFlags b) {+  return static_cast<DtoaFlags>(to_underlying(a) & to_underlying(b));+}++namespace detail {+constexpr int kConvMaxDecimalInShortestLow = -6;+/// 10^kConvMaxDecimalInShortestLow. Replace with constexpr std::pow in C++26.+constexpr double kConvMaxDecimalInShortestLowValue = 0.000001;+constexpr int kConvMaxDecimalInShortestHigh = 21;+/// 10^kConvMaxDecimalInShortestHigh. Replace with constexpr std::pow in C++26.+constexpr double kConvMaxDecimalInShortestHighValue =+    1'000'000'000'000'000'000'000.0;+constexpr int kBase10MaximalLength = 17;++constexpr int kConvMaxFixedDigitsAfterPoint =+    double_conversion::DoubleToStringConverter::kMaxFixedDigitsAfterPoint;+constexpr int kConvMaxPrecisionDigits =+    double_conversion::DoubleToStringConverter::kMaxPrecisionDigits;++/// Converts `DtoaMode` to+/// `double_conversion::DoubleToStringConverter::DtoaMode`.+/// This is temporary until+/// `double_conversion::DoubleToStringConverter::DtoaMode` is removed.+constexpr double_conversion::DoubleToStringConverter::DtoaMode convert(+    DtoaMode mode) {+  switch (mode) {+    case DtoaMode::SHORTEST:+      return double_conversion::DoubleToStringConverter::SHORTEST;+    case DtoaMode::SHORTEST_SINGLE:+      return double_conversion::DoubleToStringConverter::SHORTEST_SINGLE;+    case DtoaMode::FIXED:+      return double_conversion::DoubleToStringConverter::FIXED;+    case DtoaMode::PRECISION:+      return double_conversion::DoubleToStringConverter::PRECISION;+    default: /* unexpected */+      assert(false);+      // Default to PRECISION per exising behavior.+      return double_conversion::DoubleToStringConverter::PRECISION;+  }+}++/// Converts `DtoaFlags` to+/// `double_conversion::DoubleToStringConverter::DtoaFlags`.+/// This is temporary until+/// `double_conversion::DoubleToStringConverter::DtoaFlags` is removed.+constexpr double_conversion::DoubleToStringConverter::Flags convert(+    DtoaFlags flags) {+  return static_cast<double_conversion::DoubleToStringConverter::Flags>(flags);+}+} // namespace detail++/**+ * `numDigits` is only used with `FIXED` && `PRECISION`.+ */+template <class Tgt, class Src>+typename std::enable_if<+    std::is_floating_point<Src>::value && IsSomeString<Tgt>::value>::type+toAppend(+    Src value,+    Tgt* result,+    DtoaMode mode,+    unsigned int numDigits,+    DtoaFlags flags = DtoaFlags::NO_FLAGS) {+  double_conversion::DoubleToStringConverter::Flags dcFlags =+      detail::convert(flags);+  double_conversion::DoubleToStringConverter conv(+      dcFlags,+      "Infinity",+      "NaN",+      'E',+      detail::kConvMaxDecimalInShortestLow,+      detail::kConvMaxDecimalInShortestHigh,+      6, // max leading padding zeros+      1); // max trailing padding zeros+  char buffer[256];+  double_conversion::StringBuilder builder(buffer, sizeof(buffer));+  double_conversion::DoubleToStringConverter::DtoaMode dcMode =+      detail::convert(mode);+  FOLLY_PUSH_WARNING+  FOLLY_CLANG_DISABLE_WARNING("-Wcovered-switch-default")+  switch (dcMode) {+    case double_conversion::DoubleToStringConverter::SHORTEST:+      conv.ToShortest(value, &builder);+      break;+    case double_conversion::DoubleToStringConverter::SHORTEST_SINGLE:+      conv.ToShortestSingle(static_cast<float>(value), &builder);+      break;+    case double_conversion::DoubleToStringConverter::FIXED:+      conv.ToFixed(value, int(numDigits), &builder);+      break;+    case double_conversion::DoubleToStringConverter::PRECISION:+    default:+      assert(dcMode == double_conversion::DoubleToStringConverter::PRECISION);+      conv.ToPrecision(value, int(numDigits), &builder);+      break;+  }+  FOLLY_POP_WARNING+  const size_t length = size_t(builder.position());+  builder.Finalize();+  result->append(buffer, length);+}++/**+ * As above, but for floating point+ */+template <class Tgt, class Src>+typename std::enable_if<+    std::is_floating_point<Src>::value && IsSomeString<Tgt>::value>::type+toAppend(Src value, Tgt* result) {+  toAppend(value, result, DtoaMode::SHORTEST, 0);+}++/**+ * Upper bound of the length of the output from+ * DoubleToStringConverter::ToShortest(double, StringBuilder*),+ * as used in toAppend(double, string*).+ */+template <class Src>+typename std::enable_if<std::is_floating_point<Src>::value, size_t>::type+estimateSpaceNeeded(Src value) {+  // kBase10MaximalLength is 17. We add 1 for decimal point,+  // e.g. 10.0/9 is 17 digits and 18 characters, including the decimal point.+  constexpr int kMaxMantissaSpace = detail::kBase10MaximalLength + 1;+  // strlen("E-") + digits10(numeric_limits<double>::max_exponent10)+  constexpr int kMaxExponentSpace = 2 + 3;+  static const int kMaxPositiveSpace = std::max({+      // E.g. 1.1111111111111111E-100.+      kMaxMantissaSpace + kMaxExponentSpace,+      // E.g. 0.000001.1111111111111111, if kConvMaxDecimalInShortestLow is -6.+      kMaxMantissaSpace - detail::kConvMaxDecimalInShortestLow,+      // If kConvMaxDecimalInShortestHigh is 21, then 1e21 is the smallest+      // number > 1 which ToShortest outputs in exponential notation,+      // so 21 is the longest non-exponential number > 1.+      detail::kConvMaxDecimalInShortestHigh,+  });+  return size_t(+      kMaxPositiveSpace ++      (value < 0 ? 1 : 0)); // +1 for minus sign, if negative+}++template <class Src>+constexpr typename std::enable_if<+    !std::is_fundamental<Src>::value &&+#if FOLLY_HAVE_INT128_T+        // On OSX 10.10, is_fundamental<__int128> is false :-O+        !std::is_same<__int128, Src>::value &&+        !std::is_same<unsigned __int128, Src>::value &&+#endif+        !IsSomeString<Src>::value &&+        !std::is_convertible<Src, const char*>::value &&+        !std::is_convertible<Src, StringPiece>::value &&+        !std::is_enum<Src>::value,+    size_t>::type+estimateSpaceNeeded(const Src&) {+  return sizeof(Src) + 1; // dumbest best effort ever?+}++#ifndef DOXYGEN_SHOULD_SKIP_THIS+namespace detail {++template <typename>+struct EstimateSpaceToReserveAll;+template <size_t... I>+struct EstimateSpaceToReserveAll<std::index_sequence<I...>> {+  template <bool Tag, typename T>+  FOLLY_ERASE static constexpr size_t one(const T& v) {+    if constexpr (!Tag) {+      return 0;+    } else {+      return estimateSpaceNeeded(v);+    }+  }++  template <class... T>+  static size_t call(const T&... v) {+    const size_t sizes[] = {one<(I + 1 < sizeof...(I))>(v)...};+    size_t size = 0;+    for (const auto s : sizes) {+      size += s;+    }+    return size;+  }+};++template <class O>+void reserveInTarget(const O& o) {+  (void)o;+}+template <class T, class O>+void reserveInTarget(const T& v, const O& o) {+  o->reserve(estimateSpaceNeeded(v));+}+template <class T0, class T1, class... Ts>+void reserveInTarget(const T0& v0, const T1& v1, const Ts&... vs) {+  using seq = std::index_sequence_for<T0, T1, Ts...>;+  getLastElement(vs...)->reserve(+      EstimateSpaceToReserveAll<seq>::call(v0, v1, vs...));+}++template <class Delimiter, class... Ts>+void reserveInTargetDelim(const Delimiter& d, const Ts&... vs) {+  static_assert(sizeof...(vs) >= 2, "Needs at least 2 args");+  using seq = std::index_sequence_for<Ts...>;+  size_t fordelim = (sizeof...(vs) - 2) * estimateSpaceNeeded(d);+  getLastElement(vs...)->reserve(+      fordelim + EstimateSpaceToReserveAll<seq>::call(vs...));+}++template <typename>+struct ToAppendStrImplAll;+template <size_t... I>+struct ToAppendStrImplAll<std::index_sequence<I...>> {+  template <bool Tag, class T, class Tgt>+  FOLLY_ERASE static void one(const T& v, Tgt* result) {+    if constexpr (Tag) {+      toAppend(v, result);+    }+  }++  template <class... T>+  static void call(const T&... v) {+    auto r = getLastElement(v...);+    ((one<I + 1 < sizeof...(T)>(v, r)), ...);+  }+};++template <typename>+struct ToAppendDelimStrImplAll;+template <size_t... I>+struct ToAppendDelimStrImplAll<std::index_sequence<I...>> {+  template <size_t Tag, class Delimiter, class T, class Tgt>+  FOLLY_ERASE static void one(const Delimiter& d, const T& v, Tgt* result) {+    if constexpr (Tag >= 1) {+      toAppend(v, result);+    }+    if constexpr (Tag >= 2) {+      toAppend(d, result);+    }+  }++  template <class Delimiter, class... T>+  static void call(const Delimiter& d, const T&... v) {+    static_assert(sizeof...(I) > 0);+    constexpr size_t N = sizeof...(I) - 1;+    auto r = detail::getLastElement(v...);+    ((one<(N - I < 2 ? N - I : 2)>(d, v, r)), ...);+  }+};+template <+    class Delimiter,+    class T,+    class... Ts,+    std::enable_if_t<+        sizeof...(Ts) >= 2 &&+            IsSomeString<typename std::remove_pointer<+                detail::LastElement<Ts...>>::type>::value,+        int> = 0>+void toAppendDelimStrImpl(const Delimiter& delim, const T& v, const Ts&... vs) {+  using seq = std::index_sequence_for<T, Ts...>;+  ToAppendDelimStrImplAll<seq>::call(delim, v, vs...);+}+} // namespace detail+#endif++/**+ * Variadic conversion to string. Appends each element in turn.+ * If we have two or more things to append, we will not reserve+ * the space for them and will depend on strings exponential growth.+ * If you just append once consider using toAppendFit which reserves+ * the space needed (but does not have exponential as a result).+ *+ * Custom implementations of toAppend() can be provided in the same namespace as+ * the type to customize printing. estimateSpaceNeed() may also be provided to+ * avoid reallocations in toAppendFit():+ *+ * namespace other_namespace {+ *+ * template <class String>+ * void toAppend(const OtherType&, String* out);+ *+ * // optional+ * size_t estimateSpaceNeeded(const OtherType&);+ *+ * }+ */+template <+    class... Ts,+    std::enable_if_t<+        sizeof...(Ts) >= 3 &&+            IsSomeString<typename std::remove_pointer<+                detail::LastElement<Ts...>>::type>::value,+        int> = 0>+void toAppend(const Ts&... vs) {+  using seq = std::index_sequence_for<Ts...>;+  detail::ToAppendStrImplAll<seq>::call(vs...);+}++/**+ * @overloadbrief toAppend, but pre-allocate the exact amount of space required.+ *+ * Special version of the call that preallocates exactly as much memory+ * as need for arguments to be stored in target. This means we are+ * not doing exponential growth when we append. If you are using it+ * in a loop you are aiming at your foot with a big perf-destroying+ * bazooka.+ * On the other hand if you are appending to a string once, this+ * will probably save a few calls to malloc.+ */+template <+    class... Ts,+    std::enable_if_t<+        IsSomeString<typename std::remove_pointer<+            detail::LastElement<Ts...>>::type>::value,+        int> = 0>+void toAppendFit(const Ts&... vs) {+  ::folly::detail::reserveInTarget(vs...);+  toAppend(vs...);+}++template <class Ts>+void toAppendFit(const Ts&) {}++/**+ * Variadic base case: do nothing.+ */+template <class Tgt>+typename std::enable_if<IsSomeString<Tgt>::value>::type toAppend(+    Tgt* /* result */) {}++/**+ * @overloadbrief Use a specified delimiter between appendees.+ *+ * Variadic base case: do nothing.+ */+template <class Delimiter, class Tgt>+typename std::enable_if<IsSomeString<Tgt>::value>::type toAppendDelim(+    const Delimiter& /* delim */, Tgt* /* result */) {}++/**+ * 1 element: same as toAppend.+ */+template <class Delimiter, class T, class Tgt>+typename std::enable_if<IsSomeString<Tgt>::value>::type toAppendDelim(+    const Delimiter& /* delim */, const T& v, Tgt* tgt) {+  toAppend(v, tgt);+}++/**+ * Append to string with a delimiter in between elements. Check out+ * comments for toAppend for details about memory allocation.+ */+template <+    class Delimiter,+    class... Ts,+    std::enable_if_t<+        sizeof...(Ts) >= 3 &&+            IsSomeString<typename std::remove_pointer<+                detail::LastElement<Ts...>>::type>::value,+        int> = 0>+void toAppendDelim(const Delimiter& delim, const Ts&... vs) {+  detail::toAppendDelimStrImpl(delim, vs...);+}++/**+ * @overloadbrief toAppend with custom delimiter and exact pre-allocation.+ *+ * Detail in comment for toAppendFit+ */+template <+    class Delimiter,+    class... Ts,+    std::enable_if_t<+        IsSomeString<typename std::remove_pointer<+            detail::LastElement<Ts...>>::type>::value,+        int> = 0>+void toAppendDelimFit(const Delimiter& delim, const Ts&... vs) {+  detail::reserveInTargetDelim(delim, vs...);+  toAppendDelim(delim, vs...);+}++template <class De, class Ts>+void toAppendDelimFit(const De&, const Ts&) {}++/**+ * to<SomeString>(v1, v2, ...) uses toAppend() (see below) as back-end+ * for all types.+ */+template <+    class Tgt,+    class... Ts,+    std::enable_if_t<+        IsSomeString<Tgt>::value &&+            (sizeof...(Ts) != 1 ||+             !std::is_same<Tgt, detail::LastElement<void, Ts...>>::value),+        int> = 0>+Tgt to(const Ts&... vs) {+  Tgt result;+  toAppendFit(vs..., &result);+  return result;+}++/**+ * Special version of to<SomeString> for floating point. When calling+ * folly::to<SomeString>(double), generic implementation above will+ * firstly reserve 24 (or 25 when negative value) bytes. This will+ * introduce a malloc call for most mainstream string implementations.+ *+ * But for most cases, a floating point doesn't need 24 (or 25) bytes to+ * be converted as a string.+ *+ * This special version will not do string reserve.+ */+template <class Tgt, class Src>+typename std::enable_if<+    IsSomeString<Tgt>::value && std::is_floating_point<Src>::value,+    Tgt>::type+to(Src value) {+  Tgt result;+  toAppend(value, &result);+  return result;+}++/**+ * @overloadbrief Like `to`, but uses a custom delimiter.+ *+ * toDelim<SomeString>(SomeString str) returns itself.+ */+template <class Tgt, class Delim, class Src>+typename std::enable_if<+    IsSomeString<Tgt>::value &&+        std::is_same<Tgt, typename std::decay<Src>::type>::value,+    Tgt>::type+toDelim(const Delim& /* delim */, Src&& value) {+  return static_cast<Src&&>(value);+}++/**+ * toDelim<SomeString>(delim, v1, v2, ...) uses toAppendDelim() as+ * back-end for all types.+ */+template <+    class Tgt,+    class Delim,+    class... Ts,+    std::enable_if_t<+        IsSomeString<Tgt>::value &&+            (sizeof...(Ts) != 1 ||+             !std::is_same<Tgt, detail::LastElement<void, Ts...>>::value),+        int> = 0>+Tgt toDelim(const Delim& delim, const Ts&... vs) {+  Tgt result;+  toAppendDelimFit(delim, vs..., &result);+  return result;+}++/**+ * Conversions from string types to integral types.+ */++namespace detail {++Expected<bool, ConversionCode> str_to_bool(StringPiece* src) noexcept;++template <typename T>+Expected<T, ConversionCode> str_to_floating(StringPiece* src) noexcept;++extern template Expected<float, ConversionCode> str_to_floating<float>(+    StringPiece* src) noexcept;+extern template Expected<double, ConversionCode> str_to_floating<double>(+    StringPiece* src) noexcept;++template <typename T>+Expected<T, ConversionCode> str_to_floating_fast_float_from_chars(+    StringPiece* src) noexcept;++extern template Expected<float, ConversionCode>+str_to_floating_fast_float_from_chars<float>(StringPiece* src) noexcept;+extern template Expected<double, ConversionCode>+str_to_floating_fast_float_from_chars<double>(StringPiece* src) noexcept;++template <class Tgt>+Expected<Tgt, ConversionCode> digits_to(const char* b, const char* e) noexcept;++extern template Expected<char, ConversionCode> digits_to<char>(+    const char*, const char*) noexcept;+extern template Expected<signed char, ConversionCode> digits_to<signed char>(+    const char*, const char*) noexcept;+extern template Expected<unsigned char, ConversionCode>+digits_to<unsigned char>(const char*, const char*) noexcept;++extern template Expected<short, ConversionCode> digits_to<short>(+    const char*, const char*) noexcept;+extern template Expected<unsigned short, ConversionCode>+digits_to<unsigned short>(const char*, const char*) noexcept;++extern template Expected<int, ConversionCode> digits_to<int>(+    const char*, const char*) noexcept;+extern template Expected<unsigned int, ConversionCode> digits_to<unsigned int>(+    const char*, const char*) noexcept;++extern template Expected<long, ConversionCode> digits_to<long>(+    const char*, const char*) noexcept;+extern template Expected<unsigned long, ConversionCode>+digits_to<unsigned long>(const char*, const char*) noexcept;++extern template Expected<long long, ConversionCode> digits_to<long long>(+    const char*, const char*) noexcept;+extern template Expected<unsigned long long, ConversionCode>+digits_to<unsigned long long>(const char*, const char*) noexcept;++#if FOLLY_HAVE_INT128_T+extern template Expected<__int128, ConversionCode> digits_to<__int128>(+    const char*, const char*) noexcept;+extern template Expected<unsigned __int128, ConversionCode>+digits_to<unsigned __int128>(const char*, const char*) noexcept;+#endif++template <class T>+Expected<T, ConversionCode> str_to_integral(StringPiece* src) noexcept;++extern template Expected<char, ConversionCode> str_to_integral<char>(+    StringPiece* src) noexcept;+extern template Expected<signed char, ConversionCode>+str_to_integral<signed char>(StringPiece* src) noexcept;+extern template Expected<unsigned char, ConversionCode>+str_to_integral<unsigned char>(StringPiece* src) noexcept;++extern template Expected<short, ConversionCode> str_to_integral<short>(+    StringPiece* src) noexcept;+extern template Expected<unsigned short, ConversionCode>+str_to_integral<unsigned short>(StringPiece* src) noexcept;++extern template Expected<int, ConversionCode> str_to_integral<int>(+    StringPiece* src) noexcept;+extern template Expected<unsigned int, ConversionCode>+str_to_integral<unsigned int>(StringPiece* src) noexcept;++extern template Expected<long, ConversionCode> str_to_integral<long>(+    StringPiece* src) noexcept;+extern template Expected<unsigned long, ConversionCode>+str_to_integral<unsigned long>(StringPiece* src) noexcept;++extern template Expected<long long, ConversionCode> str_to_integral<long long>(+    StringPiece* src) noexcept;+extern template Expected<unsigned long long, ConversionCode>+str_to_integral<unsigned long long>(StringPiece* src) noexcept;++#if FOLLY_HAVE_INT128_T+extern template Expected<__int128, ConversionCode> str_to_integral<__int128>(+    StringPiece* src) noexcept;+extern template Expected<unsigned __int128, ConversionCode>+str_to_integral<unsigned __int128>(StringPiece* src) noexcept;+#endif++template <typename T>+typename std::+    enable_if<std::is_same<T, bool>::value, Expected<T, ConversionCode>>::type+    convertTo(StringPiece* src) noexcept {+  return str_to_bool(src);+}++template <typename T>+typename std::enable_if<+    std::is_floating_point<T>::value,+    Expected<T, ConversionCode>>::type+convertTo(StringPiece* src) noexcept {+  return str_to_floating<T>(src);+}++template <typename T>+typename std::enable_if<+    is_integral_v<T> && !std::is_same<T, bool>::value,+    Expected<T, ConversionCode>>::type+convertTo(StringPiece* src) noexcept {+  return str_to_integral<T>(src);+}++} // namespace detail++/**+ * String represented as a pair of pointers to char to unsigned+ * integrals. Assumes NO whitespace before or after.+ */+template <typename Tgt>+typename std::enable_if<+    is_integral_v<Tgt> && !std::is_same<Tgt, bool>::value,+    Expected<Tgt, ConversionCode>>::type+tryTo(const char* b, const char* e) noexcept {+  return detail::digits_to<Tgt>(b, e);+}++template <typename Tgt>+typename std::enable_if< //+    is_integral_v<Tgt> && !std::is_same<Tgt, bool>::value,+    Tgt>::type+to(const char* b, const char* e) {+  return tryTo<Tgt>(b, e).thenOrThrow(identity, [=](ConversionCode code) {+    return makeConversionError(code, StringPiece(b, e));+  });+}++/**+ * Conversions from string types to arithmetic types.+ */++/**+ * Parsing strings to numeric types.+ */+template <typename Tgt>+FOLLY_NODISCARD inline typename std::enable_if< //+    is_arithmetic_v<Tgt>,+    Expected<StringPiece, ConversionCode>>::type+parseTo(StringPiece src, Tgt& out) {+  return detail::convertTo<Tgt>(&src).then([&](Tgt res) {+    return void(out = res), src;+  });+}++/**+ * Integral / Floating Point to integral / Floating Point+ */++namespace detail {++/**+ * Bool to integral/float doesn't need any special checks, and this+ * overload means we aren't trying to see if a bool is less than+ * an integer.+ */+template <class Tgt>+typename std::enable_if<+    !std::is_same<Tgt, bool>::value &&+        (is_integral_v<Tgt> || std::is_floating_point<Tgt>::value),+    Expected<Tgt, ConversionCode>>::type+convertTo(const bool& value) noexcept {+  return static_cast<Tgt>(value ? 1 : 0);+}++/**+ * Checked conversion from integral to integral. The checks are only+ * performed when meaningful, e.g. conversion from int to long goes+ * unchecked.+ */+template <class Tgt, class Src>+typename std::enable_if<+    is_integral_v<Src> && !std::is_same<Tgt, Src>::value &&+        !std::is_same<Tgt, bool>::value && is_integral_v<Tgt>,+    Expected<Tgt, ConversionCode>>::type+convertTo(const Src& value) noexcept {+  if constexpr (+      make_unsigned_t<Tgt>(std::numeric_limits<Tgt>::max()) <+      make_unsigned_t<Src>(std::numeric_limits<Src>::max())) {+    if (greater_than<Tgt, std::numeric_limits<Tgt>::max()>(value)) {+      return makeUnexpected(ConversionCode::ARITH_POSITIVE_OVERFLOW);+    }+  }+  if constexpr (+      is_signed_v<Src> && (!is_signed_v<Tgt> || sizeof(Src) > sizeof(Tgt))) {+    if (less_than<Tgt, std::numeric_limits<Tgt>::min()>(value)) {+      return makeUnexpected(ConversionCode::ARITH_NEGATIVE_OVERFLOW);+    }+  }+  return static_cast<Tgt>(value);+}++/**+ * Checked conversion from floating to floating. The checks are only+ * performed when meaningful, e.g. conversion from float to double goes+ * unchecked.+ */+template <class Tgt, class Src>+typename std::enable_if<+    std::is_floating_point<Tgt>::value && std::is_floating_point<Src>::value &&+        !std::is_same<Tgt, Src>::value,+    Expected<Tgt, ConversionCode>>::type+convertTo(const Src& value) noexcept {+  if (FOLLY_UNLIKELY(std::isinf(value))) {+    return static_cast<Tgt>(value);+  }+  if constexpr (+      std::numeric_limits<Tgt>::max() < std::numeric_limits<Src>::max()) {+    if (value > std::numeric_limits<Tgt>::max()) {+      return makeUnexpected(ConversionCode::ARITH_POSITIVE_OVERFLOW);+    }+    if (value < std::numeric_limits<Tgt>::lowest()) {+      return makeUnexpected(ConversionCode::ARITH_NEGATIVE_OVERFLOW);+    }+  }+  return static_cast<Tgt>(value);+}++/**+ * Check if a floating point value can safely be converted to an+ * integer value without triggering undefined behaviour.+ */+template <typename Tgt, typename Src>+inline typename std::enable_if<+    std::is_floating_point<Src>::value && is_integral_v<Tgt> &&+        !std::is_same<Tgt, bool>::value,+    bool>::type+checkConversion(const Src& value) {+  constexpr Src tgtMaxAsSrc = static_cast<Src>(std::numeric_limits<Tgt>::max());+  constexpr Src tgtMinAsSrc = static_cast<Src>(std::numeric_limits<Tgt>::min());+  // NOTE: The following two comparisons also handle the case where value is+  // NaN, as all comparisons with NaN are false.+  if (!(value < tgtMaxAsSrc)) {+    if (!(value <= tgtMaxAsSrc)) {+      return false;+    }+    const Src mmax = folly::nextafter(tgtMaxAsSrc, Src());+    if (static_cast<Tgt>(value - mmax) >+        std::numeric_limits<Tgt>::max() - static_cast<Tgt>(mmax)) {+      return false;+    }+  } else if (value <= tgtMinAsSrc) {+    if (value < tgtMinAsSrc) {+      return false;+    }+    const Src mmin = folly::nextafter(tgtMinAsSrc, Src());+    if (static_cast<Tgt>(value - mmin) <+        std::numeric_limits<Tgt>::min() - static_cast<Tgt>(mmin)) {+      return false;+    }+  }+  return true;+}++// Integers can always safely be converted to floating point values+template <typename Tgt, typename Src>+constexpr typename std::enable_if<+    is_integral_v<Src> && std::is_floating_point<Tgt>::value,+    bool>::type+checkConversion(const Src&) {+  return true;+}++// Also, floating point values can always be safely converted to bool+// Per the standard, any floating point value that is not zero will yield true+template <typename Tgt, typename Src>+constexpr typename std::enable_if<+    std::is_floating_point<Src>::value && std::is_same<Tgt, bool>::value,+    bool>::type+checkConversion(const Src&) {+  return true;+}++/**+ * Checked conversion from integral to floating point and back. The+ * result must be convertible back to the source type without loss of+ * precision. This seems Draconian but sometimes is what's needed, and+ * complements existing routines nicely. For various rounding+ * routines, see <math>.+ */+template <typename Tgt, typename Src>+typename std::enable_if<+    (is_integral_v<Src> && std::is_floating_point<Tgt>::value) ||+        (std::is_floating_point<Src>::value && is_integral_v<Tgt>),+    Expected<Tgt, ConversionCode>>::type+convertTo(const Src& value) noexcept {+  if (FOLLY_LIKELY(checkConversion<Tgt>(value))) {+    Tgt result = static_cast<Tgt>(value);+    if (FOLLY_LIKELY(checkConversion<Src>(result))) {+      Src witness = static_cast<Src>(result);+      if (FOLLY_LIKELY(value == witness)) {+        return result;+      }+    }+  }+  return makeUnexpected(ConversionCode::ARITH_LOSS_OF_PRECISION);+}++template <typename Tgt, typename Src>+inline std::string errorValue(const Src& value) {+  return to<std::string>("(", pretty_name<Tgt>(), ") ", value);+}++template <typename Tgt, typename Src>+using IsArithToArith = std::bool_constant<+    !std::is_same<Tgt, Src>::value && !std::is_same<Tgt, bool>::value &&+    is_arithmetic_v<Src> && is_arithmetic_v<Tgt>>;++} // namespace detail++template <typename Tgt, typename Src>+typename std::enable_if<+    detail::IsArithToArith<Tgt, Src>::value,+    Expected<Tgt, ConversionCode>>::type+tryTo(const Src& value) noexcept {+  return detail::convertTo<Tgt>(value);+}++template <typename Tgt, typename Src>+typename std::enable_if<detail::IsArithToArith<Tgt, Src>::value, Tgt>::type to(+    const Src& value) {+  return tryTo<Tgt>(value).thenOrThrow(identity, [&](ConversionCode e) {+    return makeConversionError(e, detail::errorValue<Tgt>(value));+  });+}++/**+ * Custom Conversions+ *+ * Any type can be used with folly::to by implementing parseTo. The+ * implementation should be provided in the namespace of the type to facilitate+ * argument-dependent lookup:+ *+ * namespace other_namespace {+ * ::folly::Expected<::folly::StringPiece, SomeErrorCode>+ *   parseTo(::folly::StringPiece, OtherType&) noexcept;+ * }+ */+template <class T>+FOLLY_NODISCARD typename std::enable_if<+    std::is_enum<T>::value,+    Expected<StringPiece, ConversionCode>>::type+parseTo(StringPiece in, T& out) noexcept {+  typename std::underlying_type<T>::type tmp{};+  auto restOrError = parseTo(in, tmp);+  out = static_cast<T>(tmp); // Harmless if parseTo fails+  return restOrError;+}++FOLLY_NODISCARD+inline Expected<StringPiece, ConversionCode> parseTo(+    StringPiece in, StringPiece& out) noexcept {+  out = in;+  return StringPiece{in.end(), in.end()};+}++namespace detail {++template <class Str>+FOLLY_ERASE Expected<StringPiece, ConversionCode> parseToStr(+    StringPiece in, Str& out) {+  out.clear();+  out.append(in.data(), in.size()); // TODO try/catch?+  return StringPiece{in.end(), in.end()};+}++} // namespace detail++FOLLY_NODISCARD+inline Expected<StringPiece, ConversionCode> parseTo(+    StringPiece in, std::string& out) {+  return detail::parseToStr(in, out);+}++FOLLY_NODISCARD+inline Expected<StringPiece, ConversionCode> parseTo(+    StringPiece in, std::string_view& out) {+  out = std::string_view(in.data(), in.size());+  return StringPiece{in.end(), in.end()};+}++FOLLY_NODISCARD+inline Expected<StringPiece, ConversionCode> parseTo(+    StringPiece in, fbstring& out) {+  return detail::parseToStr(in, out);+}++template <class Str>+FOLLY_NODISCARD inline typename std::enable_if<+    IsSomeString<Str>::value,+    Expected<StringPiece, ConversionCode>>::type+parseTo(StringPiece in, Str& out) {+  return detail::parseToStr(in, out);+}++namespace detail {+template <typename Tgt>+using ParseToResult = decltype(parseTo(StringPiece{}, std::declval<Tgt&>()));++struct CheckTrailingSpace {+  Expected<Unit, ConversionCode> operator()(StringPiece sp) const {+    auto e = enforceWhitespaceErr(sp);+    if (FOLLY_UNLIKELY(e != ConversionCode::SUCCESS)) {+      return makeUnexpected(e);+    }+    return unit;+  }+};++template <class Error>+struct ReturnUnit {+  template <class T>+  constexpr Expected<Unit, Error> operator()(T&&) const {+    return unit;+  }+};++// Older versions of the parseTo customization point threw on error and+// returned void. Handle that.+template <class Tgt>+inline typename std::enable_if<+    std::is_void<ParseToResult<Tgt>>::value,+    Expected<StringPiece, ConversionCode>>::type+parseToWrap(StringPiece sp, Tgt& out) {+  parseTo(sp, out);+  return StringPiece(sp.end(), sp.end());+}++template <class Tgt>+inline typename std::enable_if<+    !std::is_void<ParseToResult<Tgt>>::value,+    ParseToResult<Tgt>>::type+parseToWrap(StringPiece sp, Tgt& out) {+  return parseTo(sp, out);+}++template <typename Tgt>+using ParseToError = ExpectedErrorType<decltype(detail::parseToWrap(+    StringPiece{}, std::declval<Tgt&>()))>;++} // namespace detail++/**+ * String or StringPiece to target conversion. Accepts leading and trailing+ * whitespace, but no non-space trailing characters.+ */++template <class Tgt>+inline typename std::enable_if<+    !std::is_same<StringPiece, Tgt>::value,+    Expected<Tgt, detail::ParseToError<Tgt>>>::type+tryTo(StringPiece src) noexcept {+  Tgt result{};+  using Error = detail::ParseToError<Tgt>;+  using Check = typename std::conditional<+      is_arithmetic_v<Tgt>,+      detail::CheckTrailingSpace,+      detail::ReturnUnit<Error>>::type;+  return parseTo(src, result).then(Check(), [&](Unit) {+    return std::move(result);+  });+}++template <class Tgt, class Src>+inline typename std::enable_if<+    IsSomeString<Src>::value && !std::is_same<StringPiece, Tgt>::value,+    Tgt>::type+to(Src const& src) {+  return to<Tgt>(StringPiece(src.data(), src.size()));+}++template <class Tgt>+inline+    typename std::enable_if<!std::is_same<StringPiece, Tgt>::value, Tgt>::type+    to(StringPiece src) {+  Tgt result{};+  using Error = detail::ParseToError<Tgt>;+  using Check = typename std::conditional<+      is_arithmetic_v<Tgt>,+      detail::CheckTrailingSpace,+      detail::ReturnUnit<Error>>::type;+  auto tmp = detail::parseToWrap(src, result);+  return tmp+      .thenOrThrow(+          Check(),+          [&](Error e) { throw_exception(makeConversionError(e, src)); })+      .thenOrThrow(+          [&](Unit) { return std::move(result); },+          [&](Error e) {+            throw_exception(makeConversionError(e, tmp.value()));+          });+}++/**+ * tryTo/to that take the strings by pointer so the caller gets information+ * about how much of the string was consumed by the conversion. These do not+ * check for trailing whitespace.+ */+template <class Tgt>+Expected<Tgt, detail::ParseToError<Tgt>> tryTo(StringPiece* src) noexcept {+  Tgt result;+  return parseTo(*src, result).then([&, src](StringPiece sp) -> Tgt {+    *src = sp;+    return std::move(result);+  });+}++template <class Tgt>+Tgt to(StringPiece* src) {+  Tgt result{};+  using Error = detail::ParseToError<Tgt>;+  return parseTo(*src, result)+      .thenOrThrow(+          [&, src](StringPiece sp) -> Tgt {+            *src = sp;+            return std::move(result);+          },+          [=](Error e) { return makeConversionError(e, *src); });+}++/**+ * Enum to anything and back+ */++template <class Tgt, class Src>+typename std::enable_if<+    std::is_enum<Src>::value && !std::is_same<Src, Tgt>::value &&+        !std::is_convertible<Tgt, StringPiece>::value,+    Expected<Tgt, ConversionCode>>::type+tryTo(const Src& value) noexcept {+  return tryTo<Tgt>(to_underlying(value));+}++template <class Tgt, class Src>+typename std::enable_if<+    !std::is_convertible<Src, StringPiece>::value && std::is_enum<Tgt>::value &&+        !std::is_same<Src, Tgt>::value,+    Expected<Tgt, ConversionCode>>::type+tryTo(const Src& value) noexcept {+  using I = typename std::underlying_type<Tgt>::type;+  return tryTo<I>(value).then([](I i) { return static_cast<Tgt>(i); });+}++template <class Tgt, class Src>+typename std::enable_if<+    std::is_enum<Src>::value && !std::is_same<Src, Tgt>::value &&+        !std::is_convertible<Tgt, StringPiece>::value,+    Tgt>::type+to(const Src& value) {+  return to<Tgt>(to_underlying(value));+}++template <class Tgt, class Src>+typename std::enable_if<+    !std::is_convertible<Src, StringPiece>::value && std::is_enum<Tgt>::value &&+        !std::is_same<Src, Tgt>::value,+    Tgt>::type+to(const Src& value) {+  return static_cast<Tgt>(to<typename std::underlying_type<Tgt>::type>(value));+}++} // namespace folly
+ folly/folly/CppAttributes.h view
@@ -0,0 +1,197 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++/**+ * GCC compatible wrappers around clang attributes.+ */++#pragma once++#include <folly/Portability.h>++#ifndef __has_attribute+#define FOLLY_HAS_ATTRIBUTE(x) 0+#else+#define FOLLY_HAS_ATTRIBUTE(x) __has_attribute(x)+#endif++#ifndef __has_cpp_attribute+#define FOLLY_HAS_CPP_ATTRIBUTE(x) 0+#else+#define FOLLY_HAS_CPP_ATTRIBUTE(x) __has_cpp_attribute(x)+#endif++#ifndef __has_extension+#define FOLLY_HAS_EXTENSION(x) 0+#else+#define FOLLY_HAS_EXTENSION(x) __has_extension(x)+#endif++/**+ * Nullable indicates that a return value or a parameter may be a `nullptr`,+ * e.g.+ *+ * int* FOLLY_NULLABLE foo(int* a, int* FOLLY_NULLABLE b) {+ *   if (*a > 0) {  // safe dereference+ *     return nullptr;+ *   }+ *   if (*b < 0) {  // unsafe dereference+ *     return *a;+ *   }+ *   if (b != nullptr && *b == 1) {  // safe checked dereference+ *     return new int(1);+ *   }+ *   return nullptr;+ * }+ *+ * Ignores Clang's -Wnullability-extension since it correctly handles the case+ * where the extension is not present.+ */+#if FOLLY_HAS_EXTENSION(nullability)+#define FOLLY_NULLABLE                                   \+  FOLLY_PUSH_WARNING                                     \+  FOLLY_CLANG_DISABLE_WARNING("-Wnullability-extension") \+  _Nullable FOLLY_POP_WARNING+#define FOLLY_NONNULL                                    \+  FOLLY_PUSH_WARNING                                     \+  FOLLY_CLANG_DISABLE_WARNING("-Wnullability-extension") \+  _Nonnull FOLLY_POP_WARNING+#else+#define FOLLY_NULLABLE+#define FOLLY_NONNULL+#endif++/**+ * "Cold" indicates to the compiler that a function is only expected to be+ * called from unlikely code paths. It can affect decisions made by the+ * optimizer both when processing the function body and when analyzing+ * call-sites.+ */+#if FOLLY_HAS_CPP_ATTRIBUTE(gnu::cold)+#define FOLLY_ATTR_GNU_COLD gnu::cold+#else+#define FOLLY_ATTR_GNU_COLD+#endif++/// FOLLY_ATTR_MAYBE_UNUSED_IF_NDEBUG+///+/// When defined(NDEBUG), expands to maybe_unused; otherwise, expands to empty.+/// Useful for marking variables that are used, in the sense checked for by the+/// attribute maybe_unused, only in debug builds.+#if defined(NDEBUG)+#define FOLLY_ATTR_MAYBE_UNUSED_IF_NDEBUG maybe_unused+#else+#define FOLLY_ATTR_MAYBE_UNUSED_IF_NDEBUG+#endif++/**+ *  no_unique_address indicates that a member variable can be optimized to+ * occupy no space, rather than the minimum 1-byte used by default.+ *+ *  class Empty {};+ *+ *  class NonEmpty1 {+ *    [[FOLLY_ATTR_NO_UNIQUE_ADDRESS]] Empty e;+ *    int f;+ *  };+ *+ *  class NonEmpty2 {+ *    Empty e;+ *    int f;+ *  };+ *+ *  sizeof(NonEmpty1); // may be == sizeof(int)+ *  sizeof(NonEmpty2); // must be > sizeof(int)+ */+#if FOLLY_HAS_CPP_ATTRIBUTE(no_unique_address)+#define FOLLY_ATTR_NO_UNIQUE_ADDRESS no_unique_address+#elif FOLLY_HAS_CPP_ATTRIBUTE(msvc::no_unique_address)+#define FOLLY_ATTR_NO_UNIQUE_ADDRESS msvc::no_unique_address+#else+#define FOLLY_ATTR_NO_UNIQUE_ADDRESS+#endif++#if FOLLY_HAS_CPP_ATTRIBUTE(clang::no_destroy)+#define FOLLY_ATTR_CLANG_NO_DESTROY clang::no_destroy+#else+#define FOLLY_ATTR_CLANG_NO_DESTROY+#endif++#if FOLLY_HAS_CPP_ATTRIBUTE(clang::uninitialized)+#define FOLLY_ATTR_CLANG_UNINITIALIZED clang::uninitialized+#else+#define FOLLY_ATTR_CLANG_UNINITIALIZED+#endif++/**+ * Accesses to objects with types with this attribute are not subjected to+ * type-based alias analysis, but are instead assumed to be able to alias any+ * other type of objects, just like the char type.+ */+#if FOLLY_HAS_CPP_ATTRIBUTE(gnu::may_alias)+#define FOLLY_ATTR_GNU_MAY_ALIAS gnu::may_alias+#else+#define FOLLY_ATTR_GNU_MAY_ALIAS+#endif++#if FOLLY_HAS_CPP_ATTRIBUTE(gnu::pure)+#define FOLLY_ATTR_GNU_PURE gnu::pure+#else+#define FOLLY_ATTR_GNU_PURE+#endif++#if FOLLY_HAS_CPP_ATTRIBUTE(clang::preserve_most)+#define FOLLY_ATTR_CLANG_PRESERVE_MOST clang::preserve_most+#else+#define FOLLY_ATTR_CLANG_PRESERVE_MOST+#endif++#if FOLLY_HAS_CPP_ATTRIBUTE(clang::preserve_all)+#define FOLLY_ATTR_CLANG_PRESERVE_ALL clang::preserve_all+#else+#define FOLLY_ATTR_CLANG_PRESERVE_ALL+#endif++#if FOLLY_HAS_CPP_ATTRIBUTE(gnu::used)+#define FOLLY_ATTR_GNU_USED gnu::used+#else+#define FOLLY_ATTR_GNU_USED+#endif++#if FOLLY_HAS_CPP_ATTRIBUTE(gnu::retain)+#define FOLLY_ATTR_GNU_RETAIN gnu::retain+#else+#define FOLLY_ATTR_GNU_RETAIN+#endif++#if FOLLY_HAS_CPP_ATTRIBUTE(clang::lifetimebound)+#define FOLLY_ATTR_CLANG_LIFETIMEBOUND clang::lifetimebound+#else+#define FOLLY_ATTR_CLANG_LIFETIMEBOUND+#endif++#if FOLLY_HAS_CPP_ATTRIBUTE(clang::coro_await_elidable)+#define FOLLY_ATTR_CLANG_CORO_AWAIT_ELIDABLE clang::coro_await_elidable+#else+#define FOLLY_ATTR_CLANG_CORO_AWAIT_ELIDABLE+#endif++#if FOLLY_HAS_CPP_ATTRIBUTE(clang::coro_await_elidable_argument)+#define FOLLY_ATTR_CLANG_CORO_AWAIT_ELIDABLE_ARGUMENT \+  clang::coro_await_elidable_argument+#else+#define FOLLY_ATTR_CLANG_CORO_AWAIT_ELIDABLE_ARGUMENT+#endif
+ folly/folly/CpuId.h view
@@ -0,0 +1,297 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <cstdint>+#include <cstring>++#include <folly/Portability.h>++#ifdef _MSC_VER+#include <intrin.h>+#endif++namespace folly {++/**+ * Identification of an Intel CPU.+ * Supports CPUID feature flags (EAX=1) and extended features (EAX=7, ECX=0).+ * Values from+ * http://www.intel.com/content/www/us/en/processors/processor-identification-cpuid-instruction-note.html+ */+class CpuId {+ public:+  // Always inline in order for this to be usable from a __ifunc__.+  // In shared library mode, a __ifunc__ runs at relocation time, while the+  // PLT hasn't been fully populated yet; thus, ifuncs cannot use symbols+  // with potentially external linkage. (This issue is less likely in opt+  // mode since inlining happens more likely, and it doesn't happen for+  // statically linked binaries which don't depend on the PLT)+  FOLLY_ALWAYS_INLINE CpuId() {+#if defined(_MSC_VER) && (FOLLY_X64 || defined(_M_IX86))+#if !defined(__clang__)+    int reg[4];+    __cpuid(static_cast<int*>(reg), 0);+    vendor_[0] = (uint32_t)reg[1];+    vendor_[1] = (uint32_t)reg[3];+    vendor_[2] = (uint32_t)reg[2];+    const int n = reg[0];+    if (n >= 1) {+      __cpuid(static_cast<int*>(reg), 1);+      f1c_ = uint32_t(reg[2]);+      f1d_ = uint32_t(reg[3]);+    }+    if (n >= 7) {+      __cpuidex(static_cast<int*>(reg), 7, 0);+      f7b_ = uint32_t(reg[1]);+      f7c_ = uint32_t(reg[2]);+      f7d_ = uint32_t(reg[3]);+    }+#else+    // Clang compiler has a bug (fixed in https://reviews.llvm.org/D101338) in+    // which the `__cpuid` intrinsic does not save and restore `rbx` as it needs+    // to due to being a reserved register. So in that case, do the `cpuid`+    // ourselves. Clang supports inline assembly anyway.+    uint32_t n;+    uint32_t v0b, v0d, v0c;+    __asm__(+        "pushq %%rbx\n\t"+        "cpuid\n\t"+        "movl %%ebx, %1\n\t"+        "popq %%rbx\n\t"+        : "=a"(n), "=r"(v0b), "=d"(v0d), "=c"(v0c)+        : "a"(0));+    vendor_[0] = v0b;+    vendor_[1] = v0d;+    vendor_[2] = v0c;+    if (n >= 1) {+      uint32_t f1a;+      __asm__(+          "pushq %%rbx\n\t"+          "cpuid\n\t"+          "popq %%rbx\n\t"+          : "=a"(f1a), "=c"(f1c_), "=d"(f1d_)+          : "a"(1)+          :);+    }+    if (n >= 7) {+      __asm__(+          "pushq %%rbx\n\t"+          "cpuid\n\t"+          "movq %%rbx, %%rax\n\t"+          "popq %%rbx"+          : "=a"(f7b_), "=c"(f7c_), "=d"(f7d_)+          : "a"(7), "c"(0));+    }+#endif+#elif defined(__i386__) && defined(__PIC__) && !defined(__clang__) && \+    defined(__GNUC__)+    // The following block like the normal cpuid branch below, but gcc+    // reserves ebx for use of its pic register so we must specially+    // handle the save and restore to avoid clobbering the register+    uint32_t n;+    uint32_t v0b, v0d, v0c;+    __asm__(+        "pushl %%ebx\n\t"+        "cpuid\n\t"+        "movl %%ebx, %1\n\t"+        "popl %%ebx\n\t"+        : "=a"(n), "=r"(v0b), "=d"(v0d), "=c"(v0c)+        : "a"(0));+    vendor_[0] = v0b;+    vendor_[1] = v0d;+    vendor_[2] = v0c;+    if (n >= 1) {+      uint32_t f1a;+      __asm__(+          "pushl %%ebx\n\t"+          "cpuid\n\t"+          "popl %%ebx\n\t"+          : "=a"(f1a), "=c"(f1c_), "=d"(f1d_)+          : "a"(1)+          :);+    }+    if (n >= 7) {+      __asm__(+          "pushl %%ebx\n\t"+          "cpuid\n\t"+          "movl %%ebx, %%eax\n\t"+          "popl %%ebx"+          : "=a"(f7b_), "=c"(f7c_), "=d"(f7d_)+          : "a"(7), "c"(0));+    }+#elif FOLLY_X64 || defined(__i386__)+    uint32_t n;+    uint32_t v0b, v0d, v0c;+    __asm__("cpuid" : "=a"(n), "=b"(v0b), "=d"(v0d), "=c"(v0c) : "a"(0));+    vendor_[0] = v0b;+    vendor_[1] = v0d;+    vendor_[2] = v0c;+    if (n >= 1) {+      uint32_t f1a;+      __asm__("cpuid" : "=a"(f1a), "=c"(f1c_), "=d"(f1d_) : "a"(1) : "ebx");+    }+    if (n >= 7) {+      uint32_t f7a;+      __asm__("cpuid"+              : "=a"(f7a), "=b"(f7b_), "=c"(f7c_), "=d"(f7d_)+              : "a"(7), "c"(0));+    }+#endif+  }++#define FOLLY_DETAIL_CPUID_X(name, r, bit) \+  FOLLY_ALWAYS_INLINE bool name() const { return ((r) & (1U << bit)) != 0; }++// cpuid(1): Processor Info and Feature Bits.+#define FOLLY_DETAIL_CPUID_C(name, bit) FOLLY_DETAIL_CPUID_X(name, f1c_, bit)+  FOLLY_DETAIL_CPUID_C(sse3, 0)+  FOLLY_DETAIL_CPUID_C(pclmuldq, 1)+  FOLLY_DETAIL_CPUID_C(dtes64, 2)+  FOLLY_DETAIL_CPUID_C(monitor, 3)+  FOLLY_DETAIL_CPUID_C(dscpl, 4)+  FOLLY_DETAIL_CPUID_C(vmx, 5)+  FOLLY_DETAIL_CPUID_C(smx, 6)+  FOLLY_DETAIL_CPUID_C(eist, 7)+  FOLLY_DETAIL_CPUID_C(tm2, 8)+  FOLLY_DETAIL_CPUID_C(ssse3, 9)+  FOLLY_DETAIL_CPUID_C(cnxtid, 10)+  FOLLY_DETAIL_CPUID_C(fma, 12)+  FOLLY_DETAIL_CPUID_C(cx16, 13)+  FOLLY_DETAIL_CPUID_C(xtpr, 14)+  FOLLY_DETAIL_CPUID_C(pdcm, 15)+  FOLLY_DETAIL_CPUID_C(pcid, 17)+  FOLLY_DETAIL_CPUID_C(dca, 18)+  FOLLY_DETAIL_CPUID_C(sse41, 19)+  FOLLY_DETAIL_CPUID_C(sse42, 20)+  FOLLY_DETAIL_CPUID_C(x2apic, 21)+  FOLLY_DETAIL_CPUID_C(movbe, 22)+  FOLLY_DETAIL_CPUID_C(popcnt, 23)+  FOLLY_DETAIL_CPUID_C(tscdeadline, 24)+  FOLLY_DETAIL_CPUID_C(aes, 25)+  FOLLY_DETAIL_CPUID_C(xsave, 26)+  FOLLY_DETAIL_CPUID_C(osxsave, 27)+  FOLLY_DETAIL_CPUID_C(avx, 28)+  FOLLY_DETAIL_CPUID_C(f16c, 29)+  FOLLY_DETAIL_CPUID_C(rdrand, 30)+#undef FOLLY_DETAIL_CPUID_C+#define FOLLY_DETAIL_CPUID_D(name, bit) FOLLY_DETAIL_CPUID_X(name, f1d_, bit)+  FOLLY_DETAIL_CPUID_D(fpu, 0)+  FOLLY_DETAIL_CPUID_D(vme, 1)+  FOLLY_DETAIL_CPUID_D(de, 2)+  FOLLY_DETAIL_CPUID_D(pse, 3)+  FOLLY_DETAIL_CPUID_D(tsc, 4)+  FOLLY_DETAIL_CPUID_D(msr, 5)+  FOLLY_DETAIL_CPUID_D(pae, 6)+  FOLLY_DETAIL_CPUID_D(mce, 7)+  FOLLY_DETAIL_CPUID_D(cx8, 8)+  FOLLY_DETAIL_CPUID_D(apic, 9)+  FOLLY_DETAIL_CPUID_D(sep, 11)+  FOLLY_DETAIL_CPUID_D(mtrr, 12)+  FOLLY_DETAIL_CPUID_D(pge, 13)+  FOLLY_DETAIL_CPUID_D(mca, 14)+  FOLLY_DETAIL_CPUID_D(cmov, 15)+  FOLLY_DETAIL_CPUID_D(pat, 16)+  FOLLY_DETAIL_CPUID_D(pse36, 17)+  FOLLY_DETAIL_CPUID_D(psn, 18)+  FOLLY_DETAIL_CPUID_D(clfsh, 19)+  FOLLY_DETAIL_CPUID_D(ds, 21)+  FOLLY_DETAIL_CPUID_D(acpi, 22)+  FOLLY_DETAIL_CPUID_D(mmx, 23)+  FOLLY_DETAIL_CPUID_D(fxsr, 24)+  FOLLY_DETAIL_CPUID_D(sse, 25)+  FOLLY_DETAIL_CPUID_D(sse2, 26)+  FOLLY_DETAIL_CPUID_D(ss, 27)+  FOLLY_DETAIL_CPUID_D(htt, 28)+  FOLLY_DETAIL_CPUID_D(tm, 29)+  FOLLY_DETAIL_CPUID_D(pbe, 31)+#undef FOLLY_DETAIL_CPUID_D++  // cpuid(7): Extended Features.+#define FOLLY_DETAIL_CPUID_B(name, bit) FOLLY_DETAIL_CPUID_X(name, f7b_, bit)+  FOLLY_DETAIL_CPUID_B(bmi1, 3)+  FOLLY_DETAIL_CPUID_B(hle, 4)+  FOLLY_DETAIL_CPUID_B(avx2, 5)+  FOLLY_DETAIL_CPUID_B(smep, 7)+  FOLLY_DETAIL_CPUID_B(bmi2, 8)+  FOLLY_DETAIL_CPUID_B(erms, 9)+  FOLLY_DETAIL_CPUID_B(invpcid, 10)+  FOLLY_DETAIL_CPUID_B(rtm, 11)+  FOLLY_DETAIL_CPUID_B(mpx, 14)+  FOLLY_DETAIL_CPUID_B(avx512f, 16)+  FOLLY_DETAIL_CPUID_B(avx512dq, 17)+  FOLLY_DETAIL_CPUID_B(rdseed, 18)+  FOLLY_DETAIL_CPUID_B(adx, 19)+  FOLLY_DETAIL_CPUID_B(smap, 20)+  FOLLY_DETAIL_CPUID_B(avx512ifma, 21)+  FOLLY_DETAIL_CPUID_B(pcommit, 22)+  FOLLY_DETAIL_CPUID_B(clflushopt, 23)+  FOLLY_DETAIL_CPUID_B(clwb, 24)+  FOLLY_DETAIL_CPUID_B(avx512pf, 26)+  FOLLY_DETAIL_CPUID_B(avx512er, 27)+  FOLLY_DETAIL_CPUID_B(avx512cd, 28)+  FOLLY_DETAIL_CPUID_B(sha, 29)+  FOLLY_DETAIL_CPUID_B(avx512bw, 30)+  FOLLY_DETAIL_CPUID_B(avx512vl, 31)+#undef FOLLY_DETAIL_CPUID_B+#define FOLLY_DETAIL_CPUID_C(name, bit) FOLLY_DETAIL_CPUID_X(name, f7c_, bit)+  FOLLY_DETAIL_CPUID_C(prefetchwt1, 0)+  FOLLY_DETAIL_CPUID_C(avx512vbmi, 1)+  FOLLY_DETAIL_CPUID_C(avx512vbmi2, 6)+  FOLLY_DETAIL_CPUID_C(vaes, 9)+  FOLLY_DETAIL_CPUID_C(vpclmulqdq, 10)+  FOLLY_DETAIL_CPUID_C(avx512vnni, 11)+  FOLLY_DETAIL_CPUID_C(avx512bitalg, 12)+  FOLLY_DETAIL_CPUID_C(avx512vpopcntdq, 14)+  FOLLY_DETAIL_CPUID_C(rdpid, 22)+#undef FOLLY_DETAIL_CPUID_C+#define FOLLY_DETAIL_CPUID_D(name, bit) FOLLY_DETAIL_CPUID_X(name, f7d_, bit)+  FOLLY_DETAIL_CPUID_D(avx5124vnniw, 2)+  FOLLY_DETAIL_CPUID_D(avx5124fmaps, 3)+  FOLLY_DETAIL_CPUID_D(avx512vp2intersect, 8)+  FOLLY_DETAIL_CPUID_D(amxbf16, 22)+  FOLLY_DETAIL_CPUID_D(avx512fp16, 23)+  FOLLY_DETAIL_CPUID_D(amxtile, 24)+  FOLLY_DETAIL_CPUID_D(amxint8, 25)+#undef FOLLY_DETAIL_CPUID_D++#undef FOLLY_DETAIL_CPUID_X++#define FOLLY_DETAIL_VENDOR(name, str)                         \+  FOLLY_ALWAYS_INLINE bool vendor_##name() const {             \+    /* Size of str should be 12 + NUL terminator. */           \+    static_assert(sizeof(str) == 13, "Bad CPU Vendor string"); \+    /* Just as with the main CpuId call above, this can also   \+    still be in an __ifunc__, so no function calls :( */       \+    return memcmp(&vendor_[0], &str[0], 12) == 0;              \+  }++  FOLLY_DETAIL_VENDOR(intel, "GenuineIntel")+  FOLLY_DETAIL_VENDOR(amd, "AuthenticAMD")++#undef FOLLY_DETAIL_VENDOR++ private:+  uint32_t vendor_[3] = {0};+  uint32_t f1c_ = 0;+  uint32_t f1d_ = 0;+  uint32_t f7b_ = 0;+  uint32_t f7c_ = 0;+  uint32_t f7d_ = 0;+};++} // namespace folly
+ folly/folly/DefaultKeepAliveExecutor.h view
@@ -0,0 +1,174 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <future>++#include <glog/logging.h>++#include <folly/Executor.h>+#include <folly/executors/SequencedExecutor.h>+#include <folly/synchronization/Baton.h>++namespace folly {++/// An Executor accepts units of work with add(), which should be+/// threadsafe.+class DefaultKeepAliveExecutor : public virtual Executor {+ public:+  virtual ~DefaultKeepAliveExecutor() override { DCHECK(!keepAlive_); }++  template <typename ExecutorT>+  static auto getWeakRef(ExecutorT& executor) {+    static_assert(+        std::is_base_of<DefaultKeepAliveExecutor, ExecutorT>::value,+        "getWeakRef only works for folly::DefaultKeepAliveExecutor implementations.");+    using WeakRefExecutorType = std::conditional_t<+        std::is_base_of<SequencedExecutor, ExecutorT>::value,+        SequencedExecutor,+        Executor>;+    return WeakRef<WeakRefExecutorType>::create(+        executor.controlBlock_, &executor);+  }++  folly::Executor::KeepAlive<> weakRef() { return getWeakRef(*this); }++  class WeakRefExecutor : public virtual Executor {};++ protected:+  void joinKeepAlive() {+    DCHECK(keepAlive_);+    keepAlive_.reset();+    keepAliveReleaseBaton_.wait();+  }++  void joinAndResetKeepAlive() {+    joinKeepAlive();+    auto keepAliveCount =+        controlBlock_->keepAliveCount_.exchange(1, std::memory_order_relaxed);+    DCHECK_EQ(keepAliveCount, 0);+    keepAliveReleaseBaton_.reset();+    keepAlive_ = makeKeepAlive(this);+  }++ private:+  struct ControlBlock {+    std::atomic<ptrdiff_t> keepAliveCount_{1};+  };++  template <typename ExecutorT = Executor>+  class WeakRef : public virtual ExecutorT, public virtual WeakRefExecutor {+   public:+    static folly::Executor::KeepAlive<ExecutorT> create(+        std::shared_ptr<ControlBlock> controlBlock, ExecutorT* executor) {+      return makeKeepAlive(new WeakRef(std::move(controlBlock), executor));+    }++    void add(Func f) override {+      if (auto executor = lock()) {+        executor->add(std::move(f));+      }+    }++    void addWithPriority(Func f, int8_t priority) override {+      if (auto executor = lock()) {+        executor->addWithPriority(std::move(f), priority);+      }+    }++    virtual uint8_t getNumPriorities() const override { return numPriorities_; }++   private:+    WeakRef(std::shared_ptr<ControlBlock> controlBlock, ExecutorT* executor)+        : controlBlock_(std::move(controlBlock)),+          executor_(executor),+          numPriorities_(executor->getNumPriorities()) {}++    bool keepAliveAcquire() noexcept override {+      auto keepAliveCount =+          keepAliveCount_.fetch_add(1, std::memory_order_relaxed);+      // We should never increment from 0+      DCHECK(keepAliveCount > 0);+      return true;+    }++    void keepAliveRelease() noexcept override {+      auto keepAliveCount =+          keepAliveCount_.fetch_sub(1, std::memory_order_acq_rel);+      DCHECK(keepAliveCount >= 1);++      if (keepAliveCount == 1) {+        delete this;+      }+    }++    folly::Executor::KeepAlive<ExecutorT> lock() {+      auto controlBlock =+          controlBlock_->keepAliveCount_.load(std::memory_order_relaxed);+      do {+        if (controlBlock == 0) {+          return {};+        }+      } while (!controlBlock_->keepAliveCount_.compare_exchange_weak(+          controlBlock,+          controlBlock + 1,+          std::memory_order_release,+          std::memory_order_relaxed));++      return makeKeepAlive<ExecutorT>(executor_);+    }++    std::atomic<size_t> keepAliveCount_{1};++    std::shared_ptr<ControlBlock> controlBlock_;+    ExecutorT* executor_;++    uint8_t numPriorities_;+  };++  bool keepAliveAcquire() noexcept override {+    auto keepAliveCount =+        controlBlock_->keepAliveCount_.fetch_add(1, std::memory_order_relaxed);+    // We should never increment from 0+    DCHECK(keepAliveCount > 0);+    return true;+  }++  void keepAliveRelease() noexcept override {+    auto keepAliveCount =+        controlBlock_->keepAliveCount_.fetch_sub(1, std::memory_order_acq_rel);+    DCHECK(keepAliveCount >= 1);++    if (keepAliveCount == 1) {+      keepAliveReleaseBaton_.post(); // std::memory_order_release+    }+  }++  std::shared_ptr<ControlBlock> controlBlock_{std::make_shared<ControlBlock>()};+  Baton<> keepAliveReleaseBaton_;+  KeepAlive<DefaultKeepAliveExecutor> keepAlive_{makeKeepAlive(this)};+};++template <typename ExecutorT>+auto getWeakRef(ExecutorT& executor) {+  static_assert(+      std::is_base_of<DefaultKeepAliveExecutor, ExecutorT>::value,+      "getWeakRef only works for folly::DefaultKeepAliveExecutor implementations.");+  return DefaultKeepAliveExecutor::getWeakRef(executor);+}++} // namespace folly
+ folly/folly/Demangle.cpp view
@@ -0,0 +1,255 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#include <folly/Demangle.h>++#include <algorithm>+#include <cstring>++#include <folly/CPortability.h>+#include <folly/CppAttributes.h>+#include <folly/Utility.h>+#include <folly/functional/Invoke.h>+#include <folly/lang/CString.h>++#if __has_include(<cxxabi.h>)+#include <cxxabi.h>+#endif++//  The headers <libiberty.h> (binutils) and <string.h> (glibc) both declare the+//  symbol basename. Unfortunately, the declarations are different. So including+//  both headers in the same translation unit fails due to the two conflicting+//  declarations. Since <demangle.h> includes <libiberty.h> we must be careful.+#if __has_include(<demangle.h>)+#pragma push_macro("HAVE_DECL_BASENAME")+#define HAVE_DECL_BASENAME 1+#include <demangle.h> // @manual+#pragma pop_macro("HAVE_DECL_BASENAME")+#endif++//  try to find cxxabi demangle+//+//  prefer using a weakref++#if __has_include(<cxxabi.h>)++[[gnu::weakref("__cxa_demangle")]] static char* cxxabi_demangle(+    char const*, char*, size_t*, int*);++#else // __has_include(<cxxabi.h>)++static constexpr auto cxxabi_demangle = static_cast<char* (*)(...)>(nullptr);++#endif // __has_include(<cxxabi.h>)++//  try to find liberty demangle+//+//  cannot use a weak symbol since it may be the only referenced symbol in+//  liberty+//+//  in contrast with cxxabi, where there are certainly other referenced symbols+//+//  for rust_demangle_callback, detect its declaration in the header++#if __has_include(<demangle.h>)++namespace {+struct poison {};++[[maybe_unused]] FOLLY_ERASE void rust_demangle_callback(poison);++[[maybe_unused]] FOLLY_ERASE int rust_demangle_callback_fallback(+    const char*, int, demangle_callbackref, void*) {+  return 0;+}++FOLLY_CREATE_QUAL_INVOKER(+    invoke_rust_demangle_primary, ::rust_demangle_callback);+FOLLY_CREATE_QUAL_INVOKER(+    invoke_rust_demangle_fallback, rust_demangle_callback_fallback);++using invoke_rust_demangle_fn = folly::invoke_first_match<+    invoke_rust_demangle_primary,+    invoke_rust_demangle_fallback>;+constexpr invoke_rust_demangle_fn invoke_rust_demangle;++int call_rust_demangle_callback(+    const char* mangled, int options, demangle_callbackref cb, void* opaque) {+  return invoke_rust_demangle(mangled, options, cb, opaque);+}++} // namespace++using liberty_demangle_t = int(const char*, int, demangle_callbackref, void*);++static constexpr liberty_demangle_t* liberty_cplus_demangle =+    cplus_demangle_v3_callback;+static constexpr liberty_demangle_t* liberty_rust_demangle =+    call_rust_demangle_callback;++#if defined(DMGL_NO_RECURSE_LIMIT)+static constexpr auto liberty_demangle_options_no_recurse_limit =+    DMGL_NO_RECURSE_LIMIT;+#else+static constexpr auto liberty_demangle_options_no_recurse_limit = 0;+#endif++static constexpr auto liberty_demangle_options = //+    DMGL_PARAMS | DMGL_ANSI | DMGL_TYPES | //+    liberty_demangle_options_no_recurse_limit;++#else // __has_include(<demangle.h>)++using liberty_demangle_t = int(...);++static constexpr liberty_demangle_t* liberty_cplus_demangle = nullptr;+static constexpr liberty_demangle_t* liberty_rust_demangle = nullptr;+static constexpr auto liberty_demangle_options = 0;++#endif // __has_include(<demangle.h>)++//  implementations++namespace folly {++bool const demangle_build_has_cxxabi = cxxabi_demangle;+bool const demangle_build_has_liberty = //+    to_bool(liberty_cplus_demangle) && //+    to_bool(liberty_rust_demangle);++namespace {+void demangleStringCallback(const char* str, size_t size, void* p) {+  fbstring* demangle = static_cast<fbstring*>(p);++  demangle->append(str, size);+}+} // namespace++fbstring demangle(const char* name) {+  if (!name) {+    return fbstring();+  }++  if (demangle_max_symbol_size) {+    // GCC's __cxa_demangle() uses on-stack data structures for the+    // parser state which are linear in the number of components of the+    // symbol. For extremely long symbols, this can cause a stack+    // overflow. We set an arbitrary symbol length limit above which we+    // just return the mangled name.+    size_t mangledLen = strlen(name);+    if (mangledLen > demangle_max_symbol_size) {+      return fbstring(name, mangledLen);+    }+  }++  if (folly::demangle_build_has_liberty) {+    liberty_demangle_t* funcs[] = {+        liberty_rust_demangle,+        liberty_cplus_demangle,+    };++    for (auto func : funcs) {+      fbstring demangled;++      // Unlike most library functions, this returns 1 on success and 0 on+      // failure+      int success = func(+          name, liberty_demangle_options, demangleStringCallback, &demangled);+      if (success && !demangled.empty()) {+        return demangled;+      }+    }+  }++  if (cxxabi_demangle) {+    int status;+    size_t len = 0;+    // malloc() memory for the demangled type name+    char* demangled = cxxabi_demangle(name, nullptr, &len, &status);+    if (status != 0) {+      return name;+    }+    // len is the length of the buffer (including NUL terminator and maybe+    // other junk)+    return fbstring(+        demangled, strlen(demangled), len, AcquireMallocatedString());+  } else {+    return fbstring(name);+  }+}++namespace {++struct DemangleBuf {+  char* dest;+  size_t remaining;+  size_t total;+};++void demangleBufCallback(const char* str, size_t size, void* p) {+  DemangleBuf* buf = static_cast<DemangleBuf*>(p);+  size_t n = std::min(buf->remaining, size);+  memcpy(buf->dest, str, n);+  buf->dest += n;+  buf->remaining -= n;+  buf->total += size;+}++} // namespace++size_t demangle(const char* name, char* out, size_t outSize) {+  if (demangle_max_symbol_size) {+    size_t mangledLen = strlen(name);+    if (mangledLen > demangle_max_symbol_size) {+      if (outSize) {+        size_t n = std::min(mangledLen, outSize - 1);+        memcpy(out, name, n);+        out[n] = '\0';+      }+      return mangledLen;+    }+  }++  if (folly::demangle_build_has_liberty) {+    liberty_demangle_t* funcs[] = {+        liberty_rust_demangle,+        liberty_cplus_demangle,+    };++    for (auto func : funcs) {+      DemangleBuf dbuf;+      dbuf.dest = out;+      dbuf.remaining = outSize ? outSize - 1 : 0; // leave room for null term+      dbuf.total = 0;++      // Unlike most library functions, this returns 1 on success and 0 on+      // failure+      int success =+          func(name, liberty_demangle_options, demangleBufCallback, &dbuf);+      if (success) {+        if (outSize != 0) {+          *dbuf.dest = '\0';+        }+        return dbuf.total;+      }+    }+  }++  // fallback - just return original+  return folly::strlcpy(out, name, outSize);+}++} // namespace folly
+ folly/folly/Demangle.h view
@@ -0,0 +1,73 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <folly/FBString.h>+#include <folly/portability/Config.h>++namespace folly {++inline constexpr size_t demangle_max_symbol_size =+#if defined(FOLLY_DEMANGLE_MAX_SYMBOL_SIZE)+    FOLLY_DEMANGLE_MAX_SYMBOL_SIZE;+#else+    0;+#endif++extern bool const demangle_build_has_cxxabi;+extern bool const demangle_build_has_liberty;++/**+ * Return the demangled (prettified) version of a C++ type.+ *+ * This function tries to produce a human-readable type, but the type name will+ * be returned unchanged in case of error or if demangling isn't supported on+ * your system.+ *+ * Use for debugging -- do not rely on demangle() returning anything useful.+ *+ * This function may allocate memory (and therefore throw std::bad_alloc).+ */+fbstring demangle(const char* name);+inline fbstring demangle(const std::type_info& type) {+  return demangle(type.name());+}++/**+ * Return the demangled (prettified) version of a C++ type in a user-provided+ * buffer.+ *+ * The semantics are the same as for snprintf or strlcpy: bufSize is the size+ * of the buffer, the string is always null-terminated, and the return value is+ * the number of characters (not including the null terminator) that would have+ * been written if the buffer was big enough. (So a return value >= bufSize+ * indicates that the output was truncated)+ *+ * This function does not allocate memory and is async-signal-safe.+ *+ * Note that the underlying function for the fbstring-returning demangle is+ * somewhat standard (abi::__cxa_demangle, which uses malloc), the underlying+ * function for this version is less so (cplus_demangle_v3_callback from+ * libiberty), so it is possible for the fbstring version to work, while this+ * version returns the original, mangled name.+ */+size_t demangle(const char* name, char* out, size_t outSize);+inline size_t demangle(const std::type_info& type, char* buf, size_t bufSize) {+  return demangle(type.name(), buf, bufSize);+}++} // namespace folly
+ folly/folly/DiscriminatedPtr.h view
@@ -0,0 +1,241 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++/**+ * Discriminated pointer: Type-safe pointer to one of several types.+ *+ * Similar to std::variant, but has no space overhead over a raw pointer, as+ * it relies on the fact that (on x86_64) there are 16 unused bits in a+ * pointer.+ */++#pragma once++#include <limits>+#include <stdexcept>++#include <glog/logging.h>++#include <folly/Likely.h>+#include <folly/Portability.h>+#include <folly/detail/DiscriminatedPtrDetail.h>++#if !FOLLY_X64 && !FOLLY_AARCH64 && !FOLLY_PPC64 && !FOLLY_RISCV64+#error "DiscriminatedPtr is x64, arm64, ppc64 and riscv64 specific code."+#endif++namespace folly {++/**+ * Discriminated pointer.+ *+ * Given a list of types, a DiscriminatedPtr<Types...> may point to an object+ * of one of the given types, or may be empty.  DiscriminatedPtr is type-safe:+ * you may only get a pointer to the type that you put in, otherwise get+ * throws an exception (and get_nothrow returns nullptr)+ *+ * This pointer does not do any kind of lifetime management -- it's not a+ * "smart" pointer.  You are responsible for deallocating any memory used+ * to hold pointees, if necessary.+ */+template <typename... Types>+class DiscriminatedPtr {+  // <, not <=, as our indexes are 1-based (0 means "empty")+  static_assert(+      sizeof...(Types) < std::numeric_limits<uint16_t>::max(),+      "too many types");++ public:+  /**+   * Create an empty DiscriminatedPtr.+   */+  DiscriminatedPtr() : data_(0) {}++  /**+   * Create a DiscriminatedPtr that points to an object of type T.+   * Fails at compile time if T is not a valid type (listed in Types)+   */+  template <typename T>+  explicit DiscriminatedPtr(T* ptr) {+    set(ptr, typeIndex<T>());+  }++  /**+   * Set this DiscriminatedPtr to point to an object of type T.+   * Fails at compile time if T is not a valid type (listed in Types)+   */+  template <typename T>+  void set(T* ptr) {+    set(ptr, typeIndex<T>());+  }++  /**+   * Get a pointer to the object that this DiscriminatedPtr points to, if it is+   * of type T.  Fails at compile time if T is not a valid type (listed in+   * Types), and returns nullptr if this DiscriminatedPtr is empty or points to+   * an object of a different type.+   */+  template <typename T>+  T* get_nothrow() noexcept {+    void* p = FOLLY_LIKELY(hasType<T>()) ? ptr() : nullptr;+    return static_cast<T*>(p);+  }++  template <typename T>+  const T* get_nothrow() const noexcept {+    const void* p = FOLLY_LIKELY(hasType<T>()) ? ptr() : nullptr;+    return static_cast<const T*>(p);+  }++  /**+   * Get a pointer to the object that this DiscriminatedPtr points to, if it is+   * of type T.  Fails at compile time if T is not a valid type (listed in+   * Types), and throws std::invalid_argument if this DiscriminatedPtr is empty+   * or points to an object of a different type.+   */+  template <typename T>+  T* get() {+    if (FOLLY_UNLIKELY(!hasType<T>())) {+      throw std::invalid_argument("Invalid type");+    }+    return static_cast<T*>(ptr());+  }++  template <typename T>+  const T* get() const {+    if (FOLLY_UNLIKELY(!hasType<T>())) {+      throw std::invalid_argument("Invalid type");+    }+    return static_cast<const T*>(ptr());+  }++  /**+   * Return true iff this DiscriminatedPtr is empty.+   */+  bool empty() const { return index() == 0; }++  /**+   * Return true iff the object pointed by this DiscriminatedPtr has type T,+   * false otherwise.  Fails at compile time if T is not a valid type (listed+   * in Types...)+   */+  template <typename T>+  bool hasType() const {+    return index() == typeIndex<T>();+  }++  /**+   * Clear this DiscriminatedPtr, making it empty.+   */+  void clear() { data_ = 0; }++  /**+   * Assignment operator from a pointer of type T.+   */+  template <typename T>+  DiscriminatedPtr& operator=(T* ptr) {+    set(ptr);+    return *this;+  }++  /**+   * Apply a visitor to this object, calling the appropriate overload for+   * the type currently stored in DiscriminatedPtr.  Throws invalid_argument+   * if the DiscriminatedPtr is empty.+   *+   * The visitor must meet the following requirements:+   *+   * - The visitor must allow invocation as a function by overloading+   *   operator(), unambiguously accepting all values of type T* (or const T*)+   *   for all T in Types...+   * - All operations of the function object on T* (or const T*) must+   *   return the same type (or a static_assert will fire).+   */+  template <typename V>+  _t<dptr_detail::VisitorResult<V, Types...>> apply(V&& visitor) {+    size_t n = index();+    if (n == 0) {+      throw std::invalid_argument("Empty DiscriminatedPtr");+    }+    constexpr dptr_detail::ApplyVisitor<Types...> call;+    return call(n, visitor, ptr());+  }++  template <typename V>+  _t<dptr_detail::ConstVisitorResult<V, Types...>> apply(V&& visitor) const {+    size_t n = index();+    if (n == 0) {+      throw std::invalid_argument("Empty DiscriminatedPtr");+    }+    constexpr dptr_detail::ApplyConstVisitor<Types...> call;+    return call(n, visitor, ptr());+  }++  /**+   * Get the 1-based type index of the type currently stored in this pointer.+   * Returns 0 if the pointer is empty.+   */+  size_t index() const { return data_ >> 48; }++ private:+  /**+   * Get the 1-based type index of T in Types.+   */+  template <typename T>+  uint16_t typeIndex() const {+    constexpr auto idx = type_pack_find_v<T, Types...>;+    static_assert(idx < sizeof...(Types));+    return uint16_t(idx + 1);+  }+  void* ptr() const {+    return reinterpret_cast<void*>(data_ & ((1ULL << 48) - 1));+  }++  void set(void* p, uint16_t v) {+    uintptr_t ip = reinterpret_cast<uintptr_t>(p);+    CHECK(!(ip >> 48));+    ip |= static_cast<uintptr_t>(v) << 48;+    data_ = ip;+  }++  /**+   * We store a pointer in the least significant 48 bits of data_, and a type+   * index (0 = empty, or 1-based index in Types) in the most significant 16+   * bits.  We rely on the fact that pointers have their most significant 16+   * bits clear on x86_64.+   */+  uintptr_t data_;+};++template <typename Visitor, typename... Args>+decltype(auto) apply_visitor(+    Visitor&& visitor, const DiscriminatedPtr<Args...>& variant) {+  return variant.apply(std::forward<Visitor>(visitor));+}++template <typename Visitor, typename... Args>+decltype(auto) apply_visitor(+    Visitor&& visitor, DiscriminatedPtr<Args...>& variant) {+  return variant.apply(std::forward<Visitor>(visitor));+}++template <typename Visitor, typename... Args>+decltype(auto) apply_visitor(+    Visitor&& visitor, DiscriminatedPtr<Args...>&& variant) {+  return variant.apply(std::forward<Visitor>(visitor));+}++} // namespace folly
+ folly/folly/DynamicConverter.h view
@@ -0,0 +1,17 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#include <folly/json/DynamicConverter.h>
+ folly/folly/Exception.h view
@@ -0,0 +1,152 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <errno.h>++#include <cstdio>+#include <stdexcept>+#include <system_error>++#include <folly/Conv.h>+#include <folly/FBString.h>+#include <folly/Likely.h>+#include <folly/Portability.h>+#include <folly/portability/SysTypes.h>++namespace folly {++// Various helpers to throw appropriate std::system_error exceptions from C+// library errors (returned in errno, as positive return values (many POSIX+// functions), or as negative return values (Linux syscalls))+//+// The *Explicit functions take an explicit value for errno.++// On linux and similar platforms the value of `errno` is a mixture of+// POSIX-`errno`-domain error codes and per-OS extended error codes. So the+// most appropriate category to use is `system_category`.+//+// On Windows `system_category` means codes that can be returned by Win32 API+// `GetLastError` and codes from the `errno`-domain must be reported as+// `generic_category`.+inline const std::error_category& errorCategoryForErrnoDomain() noexcept {+  if (kIsWindows) {+    return std::generic_category();+  }+  return std::system_category();+}++inline std::system_error makeSystemErrorExplicit(int err, const char* msg) {+  return std::system_error(err, errorCategoryForErrnoDomain(), msg);+}++template <class... Args>+std::system_error makeSystemErrorExplicit(int err, Args&&... args) {+  return makeSystemErrorExplicit(+      err, to<fbstring>(std::forward<Args>(args)...).c_str());+}++inline std::system_error makeSystemError(const char* msg) {+  return makeSystemErrorExplicit(errno, msg);+}++template <class... Args>+std::system_error makeSystemError(Args&&... args) {+  return makeSystemErrorExplicit(errno, std::forward<Args>(args)...);+}++// Helper to throw std::system_error+[[noreturn]] inline void throwSystemErrorExplicit(int err, const char* msg) {+  throw_exception(makeSystemErrorExplicit(err, msg));+}++template <class... Args>+[[noreturn]] void throwSystemErrorExplicit(int err, Args&&... args) {+  throw_exception(makeSystemErrorExplicit(err, std::forward<Args>(args)...));+}++// Helper to throw std::system_error from errno and components of a string+template <class... Args>+[[noreturn]] void throwSystemError(Args&&... args) {+  throwSystemErrorExplicit(errno, std::forward<Args>(args)...);+}++// Check a Posix return code (0 on success, error number on error), throw+// on error.+template <class... Args>+void checkPosixError(int err, Args&&... args) {+  if (FOLLY_UNLIKELY(err != 0)) {+    throwSystemErrorExplicit(err, std::forward<Args>(args)...);+  }+}++// Check a Linux kernel-style return code (>= 0 on success, negative error+// number on error), throw on error.+template <class... Args>+void checkKernelError(ssize_t ret, Args&&... args) {+  if (FOLLY_UNLIKELY(ret < 0)) {+    throwSystemErrorExplicit(int(-ret), std::forward<Args>(args)...);+  }+}++// Check a traditional Unix return code (-1 and sets errno on error), throw+// on error.+template <class... Args>+void checkUnixError(ssize_t ret, Args&&... args) {+  if (FOLLY_UNLIKELY(ret == -1)) {+    throwSystemError(std::forward<Args>(args)...);+  }+}++template <class... Args>+void checkUnixErrorExplicit(ssize_t ret, int savedErrno, Args&&... args) {+  if (FOLLY_UNLIKELY(ret == -1)) {+    throwSystemErrorExplicit(savedErrno, std::forward<Args>(args)...);+  }+}++// Check the return code from a fopen-style function (returns a non-nullptr+// FILE* on success, nullptr on error, sets errno).  Works with fopen, fdopen,+// freopen, tmpfile, etc.+template <class... Args>+void checkFopenError(FILE* fp, Args&&... args) {+  if (FOLLY_UNLIKELY(!fp)) {+    throwSystemError(std::forward<Args>(args)...);+  }+}++template <class... Args>+void checkFopenErrorExplicit(FILE* fp, int savedErrno, Args&&... args) {+  if (FOLLY_UNLIKELY(!fp)) {+    throwSystemErrorExplicit(savedErrno, std::forward<Args>(args)...);+  }+}++/**+ * If cond is not true, raise an exception of type E.  E must have a ctor that+ * works with const char* (a description of the failure).+ */+#define CHECK_THROW(cond, E)                       \+  do {                                             \+    if (!(cond)) {                                 \+      folly::throw_exception<E>(                   \+          "Check failed: " #cond ", in " __FILE__  \+          ":" FOLLY_PP_STRINGIZE_MACRO(__LINE__)); \+    }                                              \+  } while (0)++} // namespace folly
+ folly/folly/ExceptionString.cpp view
@@ -0,0 +1,51 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#include <folly/ExceptionString.h>++#include <utility>++#include <folly/Demangle.h>+#include <folly/lang/Exception.h>+#include <folly/lang/TypeInfo.h>++namespace folly {++namespace {++fbstring exception_string_type(std::type_info const* ti) {+  return ti ? demangle(*ti) : "<unknown exception>";+}++} // namespace++/**+ * Debug string for an exception: include type and what(), if+ * defined.+ */+fbstring exceptionStr(std::exception const& e) {+  auto prefix = exception_string_type(folly::type_info_of(e));+  return std::move(prefix) + ": " + e.what();+}++fbstring exceptionStr(std::exception_ptr const& ep) {+  if (auto ex = exception_ptr_get_object<std::exception>(ep)) {+    return exceptionStr(*ex);+  }+  return exception_string_type(exception_ptr_get_type(ep));+}++} // namespace folly
+ folly/folly/ExceptionString.h view
@@ -0,0 +1,33 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <exception>++#include <folly/FBString.h>++namespace folly {++/**+ * Debug string for an exception: include type and what(), if+ * defined.+ */+fbstring exceptionStr(std::exception const& e);++fbstring exceptionStr(std::exception_ptr const& ep);++} // namespace folly
+ folly/folly/ExceptionWrapper-inl.h view
@@ -0,0 +1,254 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#include <folly/Portability.h>++namespace folly {++struct exception_wrapper::with_exception_from_fn_ {+  struct impl_var_ {+    template <typename>+    using apply = void;+  };+  struct impl_arg_ {+    template <typename F>+    using apply = function_arguments_element_t<0, F>;+  };+  struct impl_bye_;+  template <typename Sig, std::size_t NArgs = function_arguments_size_v<Sig>>+  using impl_ = conditional_t<+      function_is_variadic_v<Sig>,+      impl_var_,+      conditional_t<NArgs == 1, impl_arg_, impl_bye_>>;++  template <typename T>+  using member_ = typename member_pointer_traits<T>::member_type;++  template <typename Void, typename>+  struct arg_type_;+  template <class Sig>+  struct arg_type_<std::enable_if_t<std::is_function<Sig>::value>, Sig> {+    using type = typename impl_<Sig>::template apply<Sig>;+  };+  template <class Ptr>+  struct arg_type_<std::enable_if_t<std::is_pointer<Ptr>::value>, Ptr>+      : arg_type_<void, std::remove_pointer_t<Ptr>> {};+  template <class Obj>+  struct arg_type_<void_t<decltype(&Obj::operator())>, Obj>+      : arg_type_<void, member_<decltype(&Obj::operator())>> {};++  // void if Fn is a variadic callable; otherwise the first arg type+  template <typename, typename Fn>+  using apply = _t<arg_type_<void, Fn>>;+};++struct exception_wrapper::with_exception_from_ex_ {+  template <typename Ex, typename>+  using apply = Ex;+};++inline exception_wrapper::exception_wrapper(exception_wrapper&& that) noexcept+    : ptr_{detail::extract_exception_ptr(std::move(that.ptr_))} {}++inline exception_wrapper::exception_wrapper(+    std::exception_ptr const& ptr) noexcept+    : ptr_{ptr} {}++inline exception_wrapper::exception_wrapper(std::exception_ptr&& ptr) noexcept+    : ptr_{detail::extract_exception_ptr(std::move(ptr))} {}++template <+    class Ex,+    class Ex_,+    FOLLY_REQUIRES_DEF(Conjunction<+                       exception_wrapper::IsStdException<Ex_>,+                       exception_wrapper::IsRegularExceptionType<Ex_>>::value)>+inline exception_wrapper::exception_wrapper(Ex&& ex)+    : ptr_{make_exception_ptr_with(std::in_place, std::forward<Ex>(ex))} {}++template <+    class Ex,+    class Ex_,+    FOLLY_REQUIRES_DEF(exception_wrapper::IsRegularExceptionType<Ex_>::value)>+inline exception_wrapper::exception_wrapper(std::in_place_t, Ex&& ex)+    : ptr_{make_exception_ptr_with(std::in_place, std::forward<Ex>(ex))} {}++template <+    class Ex,+    typename... As,+    FOLLY_REQUIRES_DEF(exception_wrapper::IsRegularExceptionType<Ex>::value)>+inline exception_wrapper::exception_wrapper(+    std::in_place_type_t<Ex>, As&&... as)+    : ptr_{make_exception_ptr_with(+          std::in_place_type<Ex>, std::forward<As>(as)...)} {}++inline exception_wrapper& exception_wrapper::operator=(+    exception_wrapper&& that) noexcept {+  // assume relocatability on all platforms+  constexpr auto sz = sizeof(std::exception_ptr);+  std::exception_ptr tmp;+  std::memcpy(static_cast<void*>(&tmp), &ptr_, sz);+  std::memcpy(static_cast<void*>(&ptr_), &that.ptr_, sz);+  std::memset(static_cast<void*>(&that.ptr_), 0, sz);+  return *this;+}++inline void exception_wrapper::swap(exception_wrapper& that) noexcept {+  // assume relocatability on all platforms+  constexpr auto sz = sizeof(std::exception_ptr);+  aligned_storage_for_t<std::exception_ptr> storage;+  std::memcpy(&storage, &ptr_, sz);+  std::memcpy(static_cast<void*>(&ptr_), &that.ptr_, sz);+  std::memcpy(static_cast<void*>(&that.ptr_), &storage, sz);+}++inline exception_wrapper::operator bool() const noexcept {+  return !!ptr_;+}++inline bool exception_wrapper::operator!() const noexcept {+  return !ptr_;+}++inline void exception_wrapper::reset() {+  ptr_ = {};+}++inline bool exception_wrapper::has_exception_ptr() const noexcept {+  return !!ptr_;+}++inline std::exception* exception_wrapper::get_exception() noexcept {+  return exception_ptr_get_object<std::exception>(ptr_);+}+inline std::exception const* exception_wrapper::get_exception() const noexcept {+  return exception_ptr_get_object<std::exception>(ptr_);+}++template <typename Ex>+inline Ex* exception_wrapper::get_exception() noexcept {+  return exception_ptr_get_object_hint<Ex>(ptr_);+}++template <typename Ex>+inline Ex const* exception_wrapper::get_exception() const noexcept {+  return exception_ptr_get_object_hint<Ex>(ptr_);+}++inline std::exception_ptr exception_wrapper::to_exception_ptr()+    const& noexcept {+  return ptr_;+}++inline std::exception_ptr& exception_wrapper::exception_ptr() & noexcept {+  return ptr_;+}+inline std::exception_ptr const& exception_wrapper::exception_ptr()+    const& noexcept {+  return ptr_;+}+inline std::exception_ptr&& exception_wrapper::exception_ptr() && noexcept {+  return std::move(ptr_);+}+inline std::exception_ptr const&& exception_wrapper::exception_ptr()+    const&& noexcept {+  return std::move(ptr_);+}++inline std::type_info const* exception_wrapper::type() const noexcept {+  return exception_ptr_get_type(ptr_);+}++inline folly::fbstring exception_wrapper::what() const {+  if (auto e = get_exception()) {+    return class_name() + ": " + e->what();+  }+  return class_name();+}++inline folly::fbstring exception_wrapper::class_name() const {+  auto const* const ti = type();+  return !*this ? "" : !ti ? "<unknown>" : folly::demangle(*ti);+}++template <class Ex>+inline bool exception_wrapper::is_compatible_with() const noexcept {+  return exception_ptr_get_object<Ex>(ptr_);+}++[[noreturn]] inline void exception_wrapper::throw_exception() const {+  ptr_ ? std::rethrow_exception(ptr_) : onNoExceptionError(__func__);+}++template <class Ex>+[[noreturn]] inline void exception_wrapper::throw_with_nested(Ex&& ex) const {+  try {+    throw_exception();+  } catch (...) {+    std::throw_with_nested(std::forward<Ex>(ex));+  }+}++template <class This, class Fn>+inline bool exception_wrapper::with_exception_(This&, Fn fn_, tag_t<void>) {+  return void(fn_()), true;+}++template <class This, class Fn, typename Ex>+inline bool exception_wrapper::with_exception_(This& this_, Fn fn_, tag_t<Ex>) {+  auto ptr = this_.template get_exception<remove_cvref_t<Ex>>();+  return ptr && (void(fn_(static_cast<Ex&>(*ptr))), true);+}++template <class Ex, class This, class Fn>+inline bool exception_wrapper::with_exception_(This& this_, Fn fn_) {+  using from_fn = with_exception_from_fn_;+  using from_ex = with_exception_from_ex_;+  using from = conditional_t<std::is_void<Ex>::value, from_fn, from_ex>;+  using type = typename from::template apply<Ex, Fn>;+  return with_exception_(this_, std::move(fn_), tag<type>);+}++template <class This, class... CatchFns>+inline void exception_wrapper::handle_(+    This& this_, char const* name, CatchFns&... fns) {+  if (!this_) {+    onNoExceptionError(name);+  }+  if (!(with_exception_<void>(this_, fns) || ...)) {+    this_.throw_exception();+  }+}++template <class Ex, class Fn>+inline bool exception_wrapper::with_exception(Fn fn) {+  return with_exception_<Ex>(*this, std::move(fn));+}+template <class Ex, class Fn>+inline bool exception_wrapper::with_exception(Fn fn) const {+  return with_exception_<Ex const>(*this, std::move(fn));+}++template <class... CatchFns>+inline void exception_wrapper::handle(CatchFns... fns) {+  handle_(*this, __func__, fns...);+}+template <class... CatchFns>+inline void exception_wrapper::handle(CatchFns... fns) const {+  handle_(*this, __func__, fns...);+}++} // namespace folly
+ folly/folly/ExceptionWrapper.cpp view
@@ -0,0 +1,35 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#include <folly/ExceptionWrapper.h>++#include <iostream>++namespace folly {++[[noreturn]] void exception_wrapper::onNoExceptionError(+    char const* const name) {+  std::ios_base::Init ioinit_; // ensure std::cerr is alive+  std::cerr << "Cannot use `" << name+            << "` with an empty folly::exception_wrapper" << std::endl;+  std::terminate();+}++fbstring exceptionStr(exception_wrapper const& ew) {+  return ew.what();+}++} // namespace folly
+ folly/folly/ExceptionWrapper.h view
@@ -0,0 +1,403 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <cassert>+#include <cstdint>+#include <exception>+#include <iosfwd>+#include <memory>+#include <new>+#include <type_traits>+#include <typeinfo>+#include <utility>++#include <folly/CPortability.h>+#include <folly/CppAttributes.h>+#include <folly/Demangle.h>+#include <folly/ExceptionString.h>+#include <folly/FBString.h>+#include <folly/Portability.h>+#include <folly/Traits.h>+#include <folly/Utility.h>+#include <folly/functional/traits.h>+#include <folly/lang/Assume.h>+#include <folly/lang/Exception.h>++namespace folly {++#define FOLLY_REQUIRES_DEF(...) \+  std::enable_if_t<static_cast<bool>(__VA_ARGS__), long>++#define FOLLY_REQUIRES(...) FOLLY_REQUIRES_DEF(__VA_ARGS__) = __LINE__++//! Throwing exceptions can be a convenient way to handle errors. Storing+//! exceptions in an `exception_ptr` makes it easy to handle exceptions in a+//! different thread or at a later time. `exception_ptr` can also be used in a+//! very generic result/exception wrapper.+//!+//! However, inspecting exceptions through the `exception_ptr` interface, namely+//! through `rethrow_exception`, is expensive. This is a wrapper interface which+//! offers faster inspection.+//!+//! \par Example usage:+//! \code+//! exception_wrapper globalExceptionWrapper;+//!+//! // Thread1+//! void doSomethingCrazy() {+//!   int rc = doSomethingCrazyWithLameReturnCodes();+//!   if (rc == NAILED_IT) {+//!     globalExceptionWrapper = exception_wrapper();+//!   } else if (rc == FACE_PLANT) {+//!     globalExceptionWrapper = make_exception_wrapper<FacePlantException>();+//!   } else if (rc == FAIL_WHALE) {+//!     globalExceptionWrapper = make_exception_wrapper<FailWhaleException>();+//!   }+//! }+//!+//! // Thread2: Exceptions are ok!+//! void processResult() {+//!   try {+//!     globalExceptionWrapper.throw_exception();+//!   } catch (const FacePlantException& e) {+//!     LOG(ERROR) << "FACEPLANT!";+//!   } catch (const FailWhaleException& e) {+//!     LOG(ERROR) << "FAILWHALE!";+//!   }+//! }+//!+//! // Thread2: Exceptions are bad!+//! void processResult() {+//!   globalExceptionWrapper.handle(+//!       [&](FacePlantException& faceplant) {+//!         LOG(ERROR) << "FACEPLANT";+//!       },+//!       [&](FailWhaleException& failwhale) {+//!         LOG(ERROR) << "FAILWHALE!";+//!       },+//!       [](...) {+//!         LOG(FATAL) << "Unrecognized exception";+//!       });+//! }+//! \endcode+class exception_wrapper final {+ private:+  struct with_exception_from_fn_;+  struct with_exception_from_ex_;++  [[noreturn]] static void onNoExceptionError(char const* name);++  template <class Ex>+  using IsStdException = std::is_base_of<std::exception, std::decay_t<Ex>>;++  std::exception_ptr ptr_;++  template <class T>+  struct IsRegularExceptionType+      : StrictConjunction<+            std::is_copy_constructible<T>,+            Negation<std::is_base_of<exception_wrapper, T>>,+            Negation<std::is_abstract<T>>> {};++  template <class This, class Fn>+  static bool with_exception_(This& this_, Fn fn_, tag_t<void>);++  template <class This, class Fn, typename Ex>+  static bool with_exception_(This& this_, Fn fn_, tag_t<Ex>);++  template <class Ex, class This, class Fn>+  static bool with_exception_(This& this_, Fn fn_);++  template <class This, class... CatchFns>+  static void handle_(This& this_, char const* name, CatchFns&... fns);++ public:+  //! Default-constructs an empty `exception_wrapper`+  //! \post `type() == nullptr`+  exception_wrapper() noexcept {}++  //! Move-constructs an `exception_wrapper`+  //! \post `*this` contains the value of `that` prior to the move+  //! \post `that.type() == nullptr`+  exception_wrapper(exception_wrapper&& that) noexcept;++  //! Copy-constructs an `exception_wrapper`+  //! \post `*this` contains a copy of `that`, and `that` is unmodified+  //! \post `type() == that.type()`+  exception_wrapper(exception_wrapper const& that) = default;++  //! Move-assigns an `exception_wrapper`+  //! \pre `this != &that`+  //! \post `*this` contains the value of `that` prior to the move+  //! \post `that.type() == nullptr`+  exception_wrapper& operator=(exception_wrapper&& that) noexcept;++  //! Copy-assigns an `exception_wrapper`+  //! \post `*this` contains a copy of `that`, and `that` is unmodified+  //! \post `type() == that.type()`+  exception_wrapper& operator=(exception_wrapper const& that) = default;++  //! \post `!ptr || bool(*this)`+  explicit exception_wrapper(std::exception_ptr const& ptr) noexcept;+  explicit exception_wrapper(std::exception_ptr&& ptr) noexcept;++  //! \pre `typeid(ex) == typeid(typename decay<Ex>::type)`+  //! \post `bool(*this)`+  //! \post `type() == &typeid(ex)`+  //! \note Exceptions of types derived from `std::exception` can be implicitly+  //!     converted to an `exception_wrapper`.+  template <+      class Ex,+      class Ex_ = std::decay_t<Ex>,+      FOLLY_REQUIRES(+          Conjunction<IsStdException<Ex_>, IsRegularExceptionType<Ex_>>::value)>+  /* implicit */ exception_wrapper(Ex&& ex);++  //! \pre `typeid(ex) == typeid(typename decay<Ex>::type)`+  //! \post `bool(*this)`+  //! \post `type() == &typeid(ex)`+  //! \note Exceptions of types not derived from `std::exception` can still be+  //!     used to construct an `exception_wrapper`, but you must specify+  //!     `std::in_place` as the first parameter.+  template <+      class Ex,+      class Ex_ = std::decay_t<Ex>,+      FOLLY_REQUIRES(IsRegularExceptionType<Ex_>::value)>+  exception_wrapper(std::in_place_t, Ex&& ex);++  template <+      class Ex,+      typename... As,+      FOLLY_REQUIRES(IsRegularExceptionType<Ex>::value)>+  exception_wrapper(std::in_place_type_t<Ex>, As&&... as);++  //! Swaps the value of `*this` with the value of `that`+  void swap(exception_wrapper& that) noexcept;++  //! \return `true` if `*this` is holding an exception.+  explicit operator bool() const noexcept;++  //! \return `!bool(*this)`+  bool operator!() const noexcept;++  //! Make this `exception_wrapper` empty+  //! \post `!*this`+  void reset();++  //! \return `true` if this `exception_wrapper` holds an exception.+  bool has_exception_ptr() const noexcept;++  //! \return a pointer to the `std::exception` held by `*this`, if it holds+  //!     one; otherwise, returns `nullptr`.+  std::exception* get_exception() noexcept;+  //! \overload+  std::exception const* get_exception() const noexcept;++  //! \returns a pointer to the `Ex` held by `*this`, if it holds an object+  //!     whose type `From` permits `std::is_convertible<From*, Ex*>`;+  //!     otherwise, returns `nullptr`.+  //!+  //! `folly::get_exception<Ex>(ew)` is identical, but avoids the "dependent+  //! template" wart, and supports inspecting other types.+  //!+  //! This is most efficient when `Ex` matches the exact stored type, or when+  //! the type alias `Ex::folly_get_exception_hint_types` has a good hint.+  template <typename Ex>+  Ex* get_exception() noexcept;+  //! \overload+  template <typename Ex>+  Ex const* get_exception() const noexcept;++  //! \return A `std::exception_ptr` that references the exception held by+  //!     `*this`.+  std::exception_ptr to_exception_ptr() const& noexcept;+  // NB: Can add this back, if a good use-case arises.+  std::exception_ptr to_exception_ptr() && = delete;++  std::exception_ptr& exception_ptr() & noexcept;+  std::exception_ptr const& exception_ptr() const& noexcept;+  std::exception_ptr&& exception_ptr() && noexcept;+  std::exception_ptr const&& exception_ptr() const&& noexcept;++  //! \return `true` if the wrappers point to the same exception object+  friend inline bool operator==(+      exception_wrapper const& lhs, exception_wrapper const& rhs) noexcept {+    return lhs.ptr_ == rhs.ptr_;+  }++  //! Returns the `typeid` of the wrapped exception object. If there is no+  //!     wrapped exception object, returns `nullptr`.+  std::type_info const* type() const noexcept;++  //! \return If `get_exception() != nullptr`, `class_name() + ": " ++  //!     get_exception()->what()`; otherwise, `class_name()`.+  folly::fbstring what() const;++  //! \return If `!*this`, the empty string; otherwise, if `!type()`, text that+  //!     is not a class name; otherwise, the demangling of `type()->name()`.+  folly::fbstring class_name() const;++  //! \tparam Ex The expression type to check for compatibility with.+  //! \return `true` if and only if `*this` wraps an exception that would be+  //!     caught with a `catch(Ex const&)` clause.+  //! \note If `*this` is empty, this function returns `false`.+  template <class Ex>+  bool is_compatible_with() const noexcept;++  //! Throws the wrapped expression.+  //! \pre `bool(*this)`+  [[noreturn]] void throw_exception() const;++  //! Terminates the process with the wrapped expression.+  [[noreturn]] void terminate_with() const noexcept { throw_exception(); }++  //! Throws the wrapped expression nested into another exception.+  //! \pre `bool(*this)`+  //! \param ex Exception in *this will be thrown nested into ex;+  //      see std::throw_with_nested() for details on this semantic.+  template <class Ex>+  [[noreturn]] void throw_with_nested(Ex&& ex) const;++  //! Call `fn` with the wrapped exception (if any), if `fn` can accept it.+  //! \par Example+  //! \code+  //! exception_wrapper ew{std::runtime_error("goodbye cruel world")};+  //!+  //! assert( ew.with_exception([](std::runtime_error& e){/*...*/}) );+  //!+  //! assert( !ew.with_exception([](int& e){/*...*/}) );+  //!+  //! assert( !exception_wrapper{}.with_exception([](int& e){/*...*/}) );+  //! \endcode+  //! \tparam Ex Optionally, the type of the exception that `fn` accepts.+  //! \tparam Fn The type of a monomophic function object.+  //! \param fn A function object to call with the wrapped exception+  //! \return `true` if and only if `fn` was called.+  //! \note Optionally, you may explicitly specify the type of the exception+  //!     that `fn` expects, as in+  //! \code+  //! ew.with_exception<std::runtime_error>([](auto&& e) { /*...*/; });+  //! \endcode+  //! \note The handler is not invoked with an active exception.+  //!     **Do not try to rethrow the exception with `throw;` from within your+  //!     handler -- that is, a throw expression with no operand.** This may+  //!     cause your process to terminate. (It is perfectly ok to throw from+  //!     a handler so long as you specify the exception to throw, as in+  //!     `throw e;`.)+  template <class Ex = void const, class Fn>+  bool with_exception(Fn fn);+  //! \overload+  template <class Ex = void const, class Fn>+  bool with_exception(Fn fn) const;++  //! Handle the wrapped expression as if with a series of `catch` clauses,+  //!     propagating the exception if no handler matches.+  //! \par Example+  //! \code+  //! exception_wrapper ew{std::runtime_error("goodbye cruel world")};+  //!+  //! ew.handle(+  //!   [&](std::logic_error const& e) {+  //!      LOG(DFATAL) << "ruh roh";+  //!      ew.throw_exception(); // rethrow the active exception without+  //!                           // slicing it. Will not be caught by other+  //!                           // handlers in this call.+  //!   },+  //!   [&](std::exception const& e) {+  //!      LOG(ERROR) << ew.what();+  //!   });+  //! \endcode+  //! In the above example, any exception _not_ derived from `std::exception`+  //!     will be propagated. To specify a catch-all clause, pass a lambda that+  //!     takes a C-style ellipses, as in:+  //! \code+  //! ew.handle(/*...* /, [](...) { /* handle unknown exception */ } )+  //! \endcode+  //! \pre `!*this`+  //! \tparam CatchFns A pack of unary monomorphic function object types.+  //! \param fns A pack of unary monomorphic function objects to be treated as+  //!     an ordered list of potential exception handlers.+  //! \note The handlers are not invoked with an active exception.+  //!     **Do not try to rethrow the exception with `throw;` from within your+  //!     handler -- that is, a throw expression with no operand.** This may+  //!     cause your process to terminate. (It is perfectly ok to throw from+  //!     a handler so long as you specify the exception to throw, as in+  //!     `throw e;`.)+  template <class... CatchFns>+  void handle(CatchFns... fns);+  //! \overload+  template <class... CatchFns>+  void handle(CatchFns... fns) const;++  // Implement the `folly::get_exception<Ex>(ew)` protocol+  template <typename Ex>+  Ex const* get_exception(get_exception_tag_t) const noexcept {+    return get_exception<Ex>();+  }+  template <typename Ex>+  Ex* get_mutable_exception(get_exception_tag_t) noexcept {+    return get_exception<Ex>();+  }+};++/**+ * \return An `exception_wrapper` that wraps an instance of type `Ex`+ *     that has been constructed with arguments `std::forward<As>(as)...`.+ */+template <class Ex, typename... As>+exception_wrapper make_exception_wrapper(As&&... as) {+  return exception_wrapper{std::in_place_type<Ex>, std::forward<As>(as)...};+}++/**+ * Inserts `ew.what()` into the ostream `sout`.+ * \return `sout`+ */+template <class Ch>+std::basic_ostream<Ch>& operator<<(+    std::basic_ostream<Ch>& sout, exception_wrapper const& ew) {+  sout << ew.class_name();+  if (auto e = ew.get_exception()) {+    sout << ": " << e->what();+  }+  return sout;+}++/**+ * Swaps the value of `a` with the value of `b`.+ */+inline void swap(exception_wrapper& a, exception_wrapper& b) noexcept {+  a.swap(b);+}++// For consistency with exceptionStr() functions in ExceptionString.h+fbstring exceptionStr(exception_wrapper const& ew);++//! `try_and_catch` is a convenience for `try {} catch(...) {}`` that returns an+//! `exception_wrapper` with the thrown exception, if any.+template <typename F>+exception_wrapper try_and_catch(F&& fn) noexcept {+  auto x = [&] { return void(static_cast<F&&>(fn)()), std::exception_ptr{}; };+  return exception_wrapper{catch_exception(x, current_exception)};+}+} // namespace folly++#include <folly/ExceptionWrapper-inl.h>++#undef FOLLY_REQUIRES+#undef FOLLY_REQUIRES_DEF
+ folly/folly/Executor.cpp view
@@ -0,0 +1,111 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#include <folly/Executor.h>++#include <stdexcept>++#include <glog/logging.h>++#include <folly/ExceptionString.h>+#include <folly/Portability.h>+#include <folly/lang/Exception.h>++namespace folly {++void Executor::invokeCatchingExnsLog(char const* const prefix) noexcept {+  auto ep = current_exception();+  LOG(ERROR) << prefix << " threw unhandled " << exceptionStr(ep);+}++void Executor::addWithPriority(Func, int8_t /* priority */) {+  throw std::runtime_error(+      "addWithPriority() is not implemented for this Executor");+}++bool Executor::keepAliveAcquire() noexcept {+  return false;+}++void Executor::keepAliveRelease() noexcept {+  LOG(FATAL) << __func__ << "() should not be called for folly::Executor types "+             << "which do not override keepAliveAcquire()";+}++// Base case of permitting with no termination to avoid nullptr tests+static ExecutorBlockingList emptyList{nullptr, {false, false, nullptr, {}}};++thread_local ExecutorBlockingList* executor_blocking_list = &emptyList;++Optional<ExecutorBlockingContext> getExecutorBlockingContext() noexcept {+  return //+      kIsMobile || !executor_blocking_list->curr.forbid ? none : //+      make_optional(executor_blocking_list->curr);+}++ExecutorBlockingGuard::ExecutorBlockingGuard(PermitTag) noexcept {+  if (!kIsMobile) {+    list_ = *executor_blocking_list;+    list_.prev = executor_blocking_list;+    list_.curr.forbid = false;+    // Do not overwrite tag or executor pointer+    executor_blocking_list = &list_;+  }+}++ExecutorBlockingGuard::ExecutorBlockingGuard(+    TrackTag, Executor* ex, StringPiece tag) noexcept {+  if (!kIsMobile) {+    list_ = *executor_blocking_list;+    list_.prev = executor_blocking_list;+    list_.curr.forbid = true;+    list_.curr.ex = ex;+    // If no string was provided, maintain the parent string to keep some+    // information+    if (!tag.empty()) {+      list_.curr.tag = tag;+    }+    executor_blocking_list = &list_;+  }+}++ExecutorBlockingGuard::ExecutorBlockingGuard(+    ProhibitTag, Executor* ex, StringPiece tag) noexcept {+  if (!kIsMobile) {+    list_ = *executor_blocking_list;+    list_.prev = executor_blocking_list;+    list_.curr.forbid = true;+    list_.curr.ex = ex;+    list_.curr.allowTerminationOnBlocking = true;+    // If no string was provided, maintain the parent string to keep some+    // information+    if (!tag.empty()) {+      list_.curr.tag = tag;+    }+    executor_blocking_list = &list_;+  }+}++ExecutorBlockingGuard::~ExecutorBlockingGuard() {+  if (!kIsMobile) {+    if (executor_blocking_list != &list_) {+      terminate_with<std::logic_error>("dtor mismatch");+    }+    executor_blocking_list = list_.prev;+  }+}++} // namespace folly
+ folly/folly/Executor.h view
@@ -0,0 +1,351 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <cassert>+#include <climits>+#include <utility>++#include <folly/Function.h>+#include <folly/Optional.h>+#include <folly/Range.h>+#include <folly/Utility.h>+#include <folly/lang/Exception.h>++namespace folly {++using Func = Function<void()>;++namespace detail {++class ExecutorKeepAliveBase {+ public:+  //  A dummy keep-alive is a keep-alive to an executor which does not support+  //  the keep-alive mechanism.+  static constexpr uintptr_t kDummyFlag = uintptr_t(1) << 0;++  //  An alias keep-alive is a keep-alive to an executor to which there is+  //  known to be another keep-alive whose lifetime surrounds the lifetime of+  //  the alias.+  static constexpr uintptr_t kAliasFlag = uintptr_t(1) << 1;++  static constexpr uintptr_t kFlagMask = kDummyFlag | kAliasFlag;+  static constexpr uintptr_t kExecutorMask = ~kFlagMask;+};++} // namespace detail++/// An Executor accepts units of work with add(), which should be+/// threadsafe.+class Executor {+ public:+  virtual ~Executor() = default;++  /// Enqueue a function to be executed by this executor. This and all+  /// variants must be threadsafe.+  virtual void add(Func) = 0;++  /// Enqueue a function with a given priority, where 0 is the medium priority+  /// This is up to the implementation to enforce+  virtual void addWithPriority(Func, int8_t priority);++  virtual uint8_t getNumPriorities() const { return 1; }++  static constexpr int8_t LO_PRI = SCHAR_MIN;+  static constexpr int8_t MID_PRI = 0;+  static constexpr int8_t HI_PRI = SCHAR_MAX;++  /**+   * Executor::KeepAlive is a safe pointer to an Executor.+   * For any Executor that supports KeepAlive functionality, Executor's+   * destructor will block until all the KeepAlive objects associated with that+   * Executor are destroyed.+   * For Executors that don't support the KeepAlive functionality, KeepAlive+   * doesn't provide such protection.+   *+   * KeepAlive should *always* be used instead of Executor*. KeepAlive can be+   * implicitly constructed from Executor*. getKeepAliveToken() helper method+   * can be used to construct a KeepAlive in templated code if you need to+   * preserve the original Executor type.+   */+  template <typename ExecutorT = Executor>+  class KeepAlive : private detail::ExecutorKeepAliveBase {+   public:+    using KeepAliveFunc = Function<void(KeepAlive&&)>;++    KeepAlive() = default;++    ~KeepAlive() {+      static_assert(+          std::is_standard_layout<KeepAlive>::value, "standard-layout");+      static_assert(sizeof(KeepAlive) == sizeof(void*), "pointer size");+      static_assert(alignof(KeepAlive) == alignof(void*), "pointer align");++      reset();+    }++    KeepAlive(KeepAlive&& other) noexcept+        : storage_(std::exchange(other.storage_, 0)) {}++    KeepAlive(const KeepAlive& other) noexcept+        : KeepAlive(getKeepAliveToken(other.get())) {}++    template <+        typename OtherExecutor,+        typename = typename std::enable_if<+            std::is_convertible<OtherExecutor*, ExecutorT*>::value>::type>+    /* implicit */ KeepAlive(KeepAlive<OtherExecutor>&& other) noexcept+        : KeepAlive(other.get(), other.storage_ & kFlagMask) {+      other.storage_ = 0;+    }++    template <+        typename OtherExecutor,+        typename = typename std::enable_if<+            std::is_convertible<OtherExecutor*, ExecutorT*>::value>::type>+    /* implicit */ KeepAlive(const KeepAlive<OtherExecutor>& other) noexcept+        : KeepAlive(getKeepAliveToken(other.get())) {}++    /* implicit */ KeepAlive(ExecutorT* executor) {+      *this = getKeepAliveToken(executor);+    }++    KeepAlive& operator=(KeepAlive&& other) noexcept {+      reset();+      storage_ = std::exchange(other.storage_, 0);+      return *this;+    }++    KeepAlive& operator=(KeepAlive const& other) {+      return operator=(folly::copy(other));+    }++    template <+        typename OtherExecutor,+        typename = typename std::enable_if<+            std::is_convertible<OtherExecutor*, ExecutorT*>::value>::type>+    KeepAlive& operator=(KeepAlive<OtherExecutor>&& other) noexcept {+      return *this = KeepAlive(std::move(other));+    }++    template <+        typename OtherExecutor,+        typename = typename std::enable_if<+            std::is_convertible<OtherExecutor*, ExecutorT*>::value>::type>+    KeepAlive& operator=(const KeepAlive<OtherExecutor>& other) {+      return *this = KeepAlive(other);+    }++    void reset() noexcept {+      if (Executor* executor = get()) {+        auto const flags = std::exchange(storage_, 0) & kFlagMask;+        if (!(flags & (kDummyFlag | kAliasFlag))) {+          executor->keepAliveRelease();+        }+      }+    }++    explicit operator bool() const { return storage_; }++    ExecutorT* get() const {+      return reinterpret_cast<ExecutorT*>(storage_ & kExecutorMask);+    }++    ExecutorT& operator*() const { return *get(); }++    ExecutorT* operator->() const { return get(); }++    KeepAlive copy() const {+      return isKeepAliveDummy(*this) //+          ? makeKeepAliveDummy(get())+          : getKeepAliveToken(get());+    }++    KeepAlive get_alias() const { return KeepAlive(storage_ | kAliasFlag); }++    template <class KAF>+    void add(KAF&& f) && {+      static_assert(+          is_invocable<KAF, KeepAlive&&>::value,+          "Parameter to add must be void(KeepAlive&&)>");+      auto ex = get();+      ex->add([ka = std::move(*this), f_2 = std::forward<KAF>(f)]() mutable {+        f_2(std::move(ka));+      });+    }++   private:+    friend class Executor;+    template <typename OtherExecutor>+    friend class KeepAlive;++    KeepAlive(ExecutorT* executor, uintptr_t flags) noexcept+        : storage_(reinterpret_cast<uintptr_t>(executor) | flags) {+      assert(executor);+      assert(!(reinterpret_cast<uintptr_t>(executor) & ~kExecutorMask));+      assert(!(flags & kExecutorMask));+    }++    explicit KeepAlive(uintptr_t storage) noexcept : storage_(storage) {}++    //  Combined storage for the executor pointer and for all flags.+    uintptr_t storage_{reinterpret_cast<uintptr_t>(nullptr)};+  };++  template <typename ExecutorT>+  static KeepAlive<ExecutorT> getKeepAliveToken(ExecutorT* executor) {+    static_assert(+        std::is_base_of<Executor, ExecutorT>::value,+        "getKeepAliveToken only works for folly::Executor implementations.");+    if (!executor) {+      return {};+    }+    folly::Executor* executorPtr = executor;+    if (executorPtr->keepAliveAcquire()) {+      return makeKeepAlive<ExecutorT>(executor);+    }+    return makeKeepAliveDummy<ExecutorT>(executor);+  }++  template <typename ExecutorT>+  static KeepAlive<ExecutorT> getKeepAliveToken(ExecutorT& executor) {+    static_assert(+        std::is_base_of<Executor, ExecutorT>::value,+        "getKeepAliveToken only works for folly::Executor implementations.");+    return getKeepAliveToken(&executor);+  }++  template <typename F>+  FOLLY_ERASE static void invokeCatchingExns(char const* p, F f) noexcept {+    catch_exception(f, invokeCatchingExnsLog, p);+  }++ protected:+  /**+   * Returns true if the KeepAlive is constructed from an executor that does+   * not support the keep alive ref-counting functionality+   */+  template <typename ExecutorT>+  static bool isKeepAliveDummy(const KeepAlive<ExecutorT>& keepAlive) {+    return keepAlive.storage_ & KeepAlive<ExecutorT>::kDummyFlag;+  }++  static bool keepAliveAcquire(Executor* executor) {+    return executor->keepAliveAcquire();+  }+  static void keepAliveRelease(Executor* executor) {+    return executor->keepAliveRelease();+  }++  // Acquire a keep alive token. Should return false if keep-alive mechanism+  // is not supported.+  virtual bool keepAliveAcquire() noexcept;+  // Release a keep alive token previously acquired by keepAliveAcquire().+  // Will never be called if keepAliveAcquire() returns false.+  virtual void keepAliveRelease() noexcept;++  template <typename ExecutorT>+  static KeepAlive<ExecutorT> makeKeepAlive(ExecutorT* executor) {+    static_assert(+        std::is_base_of<Executor, ExecutorT>::value,+        "makeKeepAlive only works for folly::Executor implementations.");+    return KeepAlive<ExecutorT>{executor, uintptr_t(0)};+  }++ private:+  static void invokeCatchingExnsLog(char const* prefix) noexcept;++  template <typename ExecutorT>+  static KeepAlive<ExecutorT> makeKeepAliveDummy(ExecutorT* executor) {+    static_assert(+        std::is_base_of<Executor, ExecutorT>::value,+        "makeKeepAliveDummy only works for folly::Executor implementations.");+    return KeepAlive<ExecutorT>{executor, KeepAlive<ExecutorT>::kDummyFlag};+  }+};++/// Returns a keep-alive token which guarantees that Executor will keep+/// processing tasks until the token is released (if supported by Executor).+/// KeepAlive always contains a valid pointer to an Executor.+template <typename ExecutorT>+Executor::KeepAlive<ExecutorT> getKeepAliveToken(ExecutorT* executor) {+  static_assert(+      std::is_base_of<Executor, ExecutorT>::value,+      "getKeepAliveToken only works for folly::Executor implementations.");+  return Executor::getKeepAliveToken(executor);+}++template <typename ExecutorT>+Executor::KeepAlive<ExecutorT> getKeepAliveToken(ExecutorT& executor) {+  static_assert(+      std::is_base_of<Executor, ExecutorT>::value,+      "getKeepAliveToken only works for folly::Executor implementations.");+  return getKeepAliveToken(&executor);+}++template <typename ExecutorT>+Executor::KeepAlive<ExecutorT> getKeepAliveToken(+    Executor::KeepAlive<ExecutorT>& ka) {+  return ka.copy();+}++struct ExecutorBlockingContext {+  bool forbid;+  bool allowTerminationOnBlocking;+  Executor* ex = nullptr;+  StringPiece tag;+};+static_assert(+    std::is_standard_layout<ExecutorBlockingContext>::value,+    "non-standard layout");++struct ExecutorBlockingList {+  ExecutorBlockingList* prev;+  ExecutorBlockingContext curr;+};+static_assert(+    std::is_standard_layout<ExecutorBlockingList>::value,+    "non-standard layout");++class ExecutorBlockingGuard {+ public:+  struct PermitTag {};+  struct TrackTag {};+  struct ProhibitTag {};++  ~ExecutorBlockingGuard();+  ExecutorBlockingGuard() = delete;++  explicit ExecutorBlockingGuard(PermitTag) noexcept;+  explicit ExecutorBlockingGuard(+      TrackTag, Executor* ex, StringPiece tag) noexcept;+  explicit ExecutorBlockingGuard(+      ProhibitTag, Executor* ex, StringPiece tag) noexcept;++  ExecutorBlockingGuard(ExecutorBlockingGuard&&) = delete;+  ExecutorBlockingGuard(ExecutorBlockingGuard const&) = delete;++  ExecutorBlockingGuard& operator=(ExecutorBlockingGuard const&) = delete;+  ExecutorBlockingGuard& operator=(ExecutorBlockingGuard&&) = delete;++ private:+  ExecutorBlockingList list_;+};++Optional<ExecutorBlockingContext> getExecutorBlockingContext() noexcept;++} // namespace folly
+ folly/folly/Expected.h view
@@ -0,0 +1,1732 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++/**+ * Like folly::Optional, but can store a value *or* an error.+ */++#pragma once++#include <cassert>+#include <cstddef>+#include <initializer_list>+#include <new>+#include <stdexcept>+#include <type_traits>+#include <utility>++#include <folly/CPortability.h>+#include <folly/CppAttributes.h>+#include <folly/Likely.h>+#include <folly/Portability.h>+#include <folly/Preprocessor.h>+#include <folly/Traits.h>+#include <folly/Unit.h>+#include <folly/Utility.h>+#include <folly/lang/Exception.h>+#include <folly/lang/Hint.h>++#define FOLLY_EXPECTED_ID(X) FB_CONCATENATE(FB_CONCATENATE(Folly, X), __LINE__)++#define FOLLY_REQUIRES_IMPL(...)                                            \+  bool FOLLY_EXPECTED_ID(Requires) = false,                                 \+       typename std::enable_if<                                             \+           (FOLLY_EXPECTED_ID(Requires) || static_cast<bool>(__VA_ARGS__)), \+           int>::type = 0++#define FOLLY_REQUIRES_TRAILING(...) , FOLLY_REQUIRES_IMPL(__VA_ARGS__)++#define FOLLY_REQUIRES(...) template <FOLLY_REQUIRES_IMPL(__VA_ARGS__)>++namespace folly {++namespace expected_detail {+namespace expected_detail_ExpectedHelper {+struct ExpectedHelper;+}+/* using override */ using expected_detail_ExpectedHelper::ExpectedHelper;+} // namespace expected_detail++/**+ * Unexpected - a helper type used to disambiguate the construction of+ * Expected objects in the error state.+ */+template <class Error>+class FOLLY_NODISCARD Unexpected final {+  template <class E>+  friend class Unexpected;+  template <class V, class E>+  friend class Expected;+  friend struct expected_detail::ExpectedHelper;++ public:+  /**+   * Constructors+   */+  Unexpected() = default;+  Unexpected(const Unexpected&) = default;+  Unexpected(Unexpected&&) = default;+  Unexpected& operator=(const Unexpected&) = default;+  Unexpected& operator=(Unexpected&&) = default;+  [[FOLLY_ATTR_GNU_COLD]] constexpr /* implicit */ Unexpected(const Error& err)+      : error_(err) {}+  [[FOLLY_ATTR_GNU_COLD]] constexpr /* implicit */ Unexpected(Error&& err)+      : error_(std::move(err)) {}++  template <class Other FOLLY_REQUIRES_TRAILING(+      std::is_constructible<Error, Other&&>::value)>+  constexpr /* implicit */ Unexpected(Unexpected<Other> that)+      : error_(std::move(that.error())) {}++  /**+   * Assignment+   */+  template <class Other FOLLY_REQUIRES_TRAILING(+      std::is_assignable<Error&, Other&&>::value)>+  Unexpected& operator=(Unexpected<Other> that) {+    error_ = std::move(that.error());+  }++  /**+   * Observers+   */+  Error& error() & { return error_; }+  const Error& error() const& { return error_; }+  Error&& error() && { return std::move(error_); }+  const Error&& error() const&& { return std::move(error_); }++ private:+  Error error_;+};++template <+    class Error FOLLY_REQUIRES_TRAILING(IsEqualityComparable<Error>::value)>+inline bool operator==(+    const Unexpected<Error>& lhs, const Unexpected<Error>& rhs) {+  return lhs.error() == rhs.error();+}++template <+    class Error FOLLY_REQUIRES_TRAILING(IsEqualityComparable<Error>::value)>+inline bool operator!=(+    const Unexpected<Error>& lhs, const Unexpected<Error>& rhs) {+  return !(lhs == rhs);+}++/**+ * For constructing an Unexpected object from an error code. Unexpected objects+ * are implicitly convertible to Expected object in the error state. Usage is+ * as follows:+ *+ * enum class MyErrorCode { BAD_ERROR, WORSE_ERROR };+ * Expected<int, MyErrorCode> myAPI() {+ *   int i = // ...;+ *   return i ? makeExpected<MyErrorCode>(i)+ *            : makeUnexpected(MyErrorCode::BAD_ERROR);+ * }+ */+template <class Error>+constexpr Unexpected<typename std::decay<Error>::type> makeUnexpected(+    Error&& err) {+  return Unexpected<typename std::decay<Error>::type>{+      static_cast<Error&&>(err)};+}++template <class Error>+class FOLLY_EXPORT BadExpectedAccess;++/**+ * A common base type for exceptions thrown by Expected when the caller+ * erroneously requests a value.+ */+template <>+class FOLLY_EXPORT BadExpectedAccess<void> : public std::exception {+ public:+  explicit BadExpectedAccess() noexcept = default;+  BadExpectedAccess(BadExpectedAccess const&) {}+  BadExpectedAccess& operator=(BadExpectedAccess const&) { return *this; }++  char const* what() const noexcept override { return "bad expected access"; }+};++/**+ * An exception type thrown by Expected on catastrophic logic errors, i.e., when+ * the caller tries to access the value within an Expected but when the Expected+ * instead contains an error.+ */+template <class Error>+class FOLLY_EXPORT BadExpectedAccess : public BadExpectedAccess<void> {+ public:+  explicit BadExpectedAccess(Error error)+      : error_{static_cast<Error&&>(error)} {}++  /**+   * The error code that was held by the Expected object when the caller+   * erroneously requested the value.+   */+  Error& error() & { return error_; }+  Error const& error() const& { return error_; }+  Error&& error() && { return static_cast<Error&&>(error_); }+  Error const&& error() const&& { return static_cast<Error const&&>(error_); }++ private:+  Error error_;+};++/**+ * Forward declarations+ */+template <class Value, class Error>+class Expected;++template <class Error, class Value>+FOLLY_NODISCARD constexpr Expected<typename std::decay<Value>::type, Error>+makeExpected(Value&&);++/**+ * Alias for an Expected type's associated value_type+ */+template <class Expected>+using ExpectedValueType =+    typename std::remove_reference<Expected>::type::value_type;++/**+ * Alias for an Expected type's associated error_type+ */+template <class Expected>+using ExpectedErrorType =+    typename std::remove_reference<Expected>::type::error_type;++// Details...+namespace expected_detail {++template <typename Value, typename Error>+struct Promise;+template <typename Value, typename Error>+struct PromiseReturn;++template <template <class...> class Trait, class... Ts>+using StrictAllOf = StrictConjunction<Trait<Ts>...>;++template <class T>+using IsCopyable = StrictConjunction<+    std::is_copy_constructible<T>,+    std::is_copy_assignable<T>>;++template <class T>+using IsMovable = StrictConjunction<+    std::is_move_constructible<T>,+    std::is_move_assignable<T>>;++template <class T>+using IsNothrowCopyable = StrictConjunction<+    std::is_nothrow_copy_constructible<T>,+    std::is_nothrow_copy_assignable<T>>;++template <class T>+using IsNothrowMovable = StrictConjunction<+    std::is_nothrow_move_constructible<T>,+    std::is_nothrow_move_assignable<T>>;++template <class From, class To>+using IsConvertible = StrictConjunction<+    std::is_constructible<To, From>,+    std::is_assignable<To&, From>>;++template <class T, class U>+auto doEmplaceAssign(int, T& t, U&& u)+    -> decltype(void(t = static_cast<U&&>(u))) {+  t = static_cast<U&&>(u);+}++template <class T, class U>+auto doEmplaceAssign(long, T& t, U&& u)+    -> decltype(void(T(static_cast<U&&>(u)))) {+  auto addr = const_cast<void*>(static_cast<void const*>(std::addressof(t)));+  t.~T();+  ::new (addr) T(static_cast<U&&>(u));+}++template <class T, class... Us>+auto doEmplaceAssign(int, T& t, Us&&... us)+    -> decltype(void(t = T(static_cast<Us&&>(us)...))) {+  t = T(static_cast<Us&&>(us)...);+}++template <class T, class... Us>+auto doEmplaceAssign(long, T& t, Us&&... us)+    -> decltype(void(T(static_cast<Us&&>(us)...))) {+  auto addr = const_cast<void*>(static_cast<void const*>(std::addressof(t)));+  t.~T();+  ::new (addr) T(static_cast<Us&&>(us)...);+}++struct EmptyTag {};+struct ValueTag {};+struct ErrorTag {};+enum class Which : unsigned char { eEmpty, eValue, eError };+enum class StorageType { ePODStruct, ePODUnion, eUnion };++template <class Value, class Error>+constexpr StorageType getStorageType() {+  return StrictAllOf<std::is_trivially_copyable, Value, Error>::value+      ? (sizeof(std::pair<Value, Error>) <= sizeof(void* [2]) &&+                 StrictAllOf<std::is_trivial, Value, Error>::value+             ? StorageType::ePODStruct+             : StorageType::ePODUnion)+      : StorageType::eUnion;+}++template <+    class Value,+    class Error,+    StorageType = expected_detail::getStorageType<Value, Error>()> // ePODUnion+struct ExpectedStorage {+  using value_type = Value;+  using error_type = Error;+  union {+    Value value_;+    Error error_;+    char ch_;+  };+  Which which_;++  template <class E = Error, class = decltype(E{})>+  constexpr ExpectedStorage() noexcept(noexcept(E{}))+      : error_{}, which_(Which::eError) {}+  explicit constexpr ExpectedStorage(EmptyTag) noexcept+      : ch_{unsafe_default_initialized}, which_(Which::eEmpty) {}+  template <class... Vs>+  explicit constexpr ExpectedStorage(ValueTag, Vs&&... vs) noexcept(+      noexcept(Value(static_cast<Vs&&>(vs)...)))+      : value_(static_cast<Vs&&>(vs)...), which_(Which::eValue) {}+  template <class... Es>+  explicit constexpr ExpectedStorage(ErrorTag, Es&&... es) noexcept(+      noexcept(Error(static_cast<Es&&>(es)...)))+      : error_(static_cast<Es&&>(es)...), which_(Which::eError) {}+  void clear() noexcept {}+  static constexpr bool uninitializedByException() noexcept {+    // Although which_ may temporarily be eEmpty during construction, it+    // is always either eValue or eError for a fully-constructed Expected.+    return false;+  }+  template <class... Vs>+  void assignValue(Vs&&... vs) {+    expected_detail::doEmplaceAssign(0, value_, static_cast<Vs&&>(vs)...);+    which_ = Which::eValue;+  }+  template <class... Es>+  void assignError(Es&&... es) {+    expected_detail::doEmplaceAssign(0, error_, static_cast<Es&&>(es)...);+    which_ = Which::eError;+  }+  template <class Other>+  void assign(Other&& that) {+    switch (that.which_) {+      case Which::eValue:+        this->assignValue(static_cast<Other&&>(that).value());+        break;+      case Which::eError:+        this->assignError(static_cast<Other&&>(that).error());+        break;+      case Which::eEmpty:+      default:+        this->clear();+        break;+    }+  }+  Value& value() & { return value_; }+  const Value& value() const& { return value_; }+  Value&& value() && { return std::move(value_); }+  const Value&& value() const&& { return std::move(value_); }+  Error& error() & { return error_; }+  const Error& error() const& { return error_; }+  Error&& error() && { return std::move(error_); }+  const Error&& error() const&& { return std::move(error_); }+};++template <class Value, class Error>+struct ExpectedUnion {+  union {+    Value value_;+    Error error_;+    char ch_ = unsafe_default_initialized;+  };+  Which which_ = Which::eEmpty;++  explicit constexpr ExpectedUnion(EmptyTag) noexcept {}+  template <class... Vs>+  explicit constexpr ExpectedUnion(ValueTag, Vs&&... vs) noexcept(+      noexcept(Value(static_cast<Vs&&>(vs)...)))+      : value_(static_cast<Vs&&>(vs)...), which_(Which::eValue) {}+  template <class... Es>+  explicit constexpr ExpectedUnion(ErrorTag, Es&&... es) noexcept(+      noexcept(Error(static_cast<Es&&>(es)...)))+      : error_(static_cast<Es&&>(es)...), which_(Which::eError) {}+  ExpectedUnion(const ExpectedUnion&) {}+  ExpectedUnion(ExpectedUnion&&) noexcept {}+  ExpectedUnion& operator=(const ExpectedUnion&) { return *this; }+  ExpectedUnion& operator=(ExpectedUnion&&) noexcept { return *this; }+  ~ExpectedUnion() {}+  Value& value() & { return value_; }+  const Value& value() const& { return value_; }+  Value&& value() && { return std::move(value_); }+  const Value&& value() const&& { return std::move(value_); }+  Error& error() & { return error_; }+  const Error& error() const& { return error_; }+  Error&& error() && { return std::move(error_); }+  const Error&& error() const&& { return std::move(error_); }+};++template <class Derived, bool, bool Noexcept>+struct CopyConstructible {+  constexpr CopyConstructible() = default;+  CopyConstructible(const CopyConstructible& that) noexcept(Noexcept) {+    static_cast<Derived*>(this)->assign(static_cast<const Derived&>(that));+  }+  constexpr CopyConstructible(CopyConstructible&&) = default;+  CopyConstructible& operator=(const CopyConstructible&) = default;+  CopyConstructible& operator=(CopyConstructible&&) = default;+};++template <class Derived, bool Noexcept>+struct CopyConstructible<Derived, false, Noexcept> {+  constexpr CopyConstructible() = default;+  CopyConstructible(const CopyConstructible&) = delete;+  constexpr CopyConstructible(CopyConstructible&&) = default;+  CopyConstructible& operator=(const CopyConstructible&) = default;+  CopyConstructible& operator=(CopyConstructible&&) = default;+};++template <class Derived, bool, bool Noexcept>+struct MoveConstructible {+  constexpr MoveConstructible() = default;+  constexpr MoveConstructible(const MoveConstructible&) = default;+  MoveConstructible(MoveConstructible&& that) noexcept(Noexcept) {+    static_cast<Derived*>(this)->assign(std::move(static_cast<Derived&>(that)));+  }+  MoveConstructible& operator=(const MoveConstructible&) = default;+  MoveConstructible& operator=(MoveConstructible&&) = default;+};++template <class Derived, bool Noexcept>+struct MoveConstructible<Derived, false, Noexcept> {+  constexpr MoveConstructible() = default;+  constexpr MoveConstructible(const MoveConstructible&) = default;+  MoveConstructible(MoveConstructible&&) = delete;+  MoveConstructible& operator=(const MoveConstructible&) = default;+  MoveConstructible& operator=(MoveConstructible&&) = default;+};++template <class Derived, bool, bool Noexcept>+struct CopyAssignable {+  constexpr CopyAssignable() = default;+  constexpr CopyAssignable(const CopyAssignable&) = default;+  constexpr CopyAssignable(CopyAssignable&&) = default;+  CopyAssignable& operator=(const CopyAssignable& that) noexcept(Noexcept) {+    static_cast<Derived*>(this)->assign(static_cast<const Derived&>(that));+    return *this;+  }+  CopyAssignable& operator=(CopyAssignable&&) = default;+};++template <class Derived, bool Noexcept>+struct CopyAssignable<Derived, false, Noexcept> {+  constexpr CopyAssignable() = default;+  constexpr CopyAssignable(const CopyAssignable&) = default;+  constexpr CopyAssignable(CopyAssignable&&) = default;+  CopyAssignable& operator=(const CopyAssignable&) = delete;+  CopyAssignable& operator=(CopyAssignable&&) = default;+};++template <class Derived, bool, bool Noexcept>+struct MoveAssignable {+  constexpr MoveAssignable() = default;+  constexpr MoveAssignable(const MoveAssignable&) = default;+  constexpr MoveAssignable(MoveAssignable&&) = default;+  MoveAssignable& operator=(const MoveAssignable&) = default;+  MoveAssignable& operator=(MoveAssignable&& that) noexcept(Noexcept) {+    static_cast<Derived*>(this)->assign(std::move(static_cast<Derived&>(that)));+    return *this;+  }+};++template <class Derived, bool Noexcept>+struct MoveAssignable<Derived, false, Noexcept> {+  constexpr MoveAssignable() = default;+  constexpr MoveAssignable(const MoveAssignable&) = default;+  constexpr MoveAssignable(MoveAssignable&&) = default;+  MoveAssignable& operator=(const MoveAssignable&) = default;+  MoveAssignable& operator=(MoveAssignable&& that) = delete;+};++template <class Value, class Error>+struct ExpectedStorage<Value, Error, StorageType::eUnion>+    : ExpectedUnion<Value, Error>,+      CopyConstructible<+          ExpectedStorage<Value, Error, StorageType::eUnion>,+          StrictAllOf<std::is_copy_constructible, Value, Error>::value,+          StrictAllOf<std::is_nothrow_copy_constructible, Value, Error>::value>,+      MoveConstructible<+          ExpectedStorage<Value, Error, StorageType::eUnion>,+          StrictAllOf<std::is_move_constructible, Value, Error>::value,+          StrictAllOf<std::is_nothrow_move_constructible, Value, Error>::value>,+      CopyAssignable<+          ExpectedStorage<Value, Error, StorageType::eUnion>,+          StrictAllOf<IsCopyable, Value, Error>::value,+          StrictAllOf<IsNothrowCopyable, Value, Error>::value>,+      MoveAssignable<+          ExpectedStorage<Value, Error, StorageType::eUnion>,+          StrictAllOf<IsMovable, Value, Error>::value,+          StrictAllOf<IsNothrowMovable, Value, Error>::value> {+  using value_type = Value;+  using error_type = Error;+  using Base = ExpectedUnion<Value, Error>;+  template <class E = Error, class = decltype(E{})>+  constexpr ExpectedStorage() noexcept(noexcept(E{})) : Base{ErrorTag{}} {}+  ExpectedStorage(const ExpectedStorage&) = default;+  ExpectedStorage(ExpectedStorage&&) = default;+  ExpectedStorage& operator=(const ExpectedStorage&) = default;+  ExpectedStorage& operator=(ExpectedStorage&&) = default;+  using ExpectedUnion<Value, Error>::ExpectedUnion;+  ~ExpectedStorage() { clear(); }+  void clear() noexcept {+    switch (this->which_) {+      case Which::eValue:+        this->value().~Value();+        break;+      case Which::eError:+        this->error().~Error();+        break;+      case Which::eEmpty:+        break;+    }+    this->which_ = Which::eEmpty;+  }+  bool uninitializedByException() const noexcept {+    return this->which_ == Which::eEmpty;+  }+  template <class... Vs>+  void assignValue(Vs&&... vs) {+    auto& val = this->value();+    if (this->which_ == Which::eValue) {+      expected_detail::doEmplaceAssign(0, val, static_cast<Vs&&>(vs)...);+    } else {+      this->clear();+      auto addr =+          const_cast<void*>(static_cast<void const*>(std::addressof(val)));+      ::new (addr) Value(static_cast<Vs&&>(vs)...);+      this->which_ = Which::eValue;+    }+  }+  template <class... Es>+  void assignError(Es&&... es) {+    if (this->which_ == Which::eError) {+      expected_detail::doEmplaceAssign(+          0, this->error(), static_cast<Es&&>(es)...);+    } else {+      this->clear();+      ::new ((void*)std::addressof(this->error()))+          Error(static_cast<Es&&>(es)...);+      this->which_ = Which::eError;+    }+  }+  bool isSelfAssign(const ExpectedStorage* that) const { return this == that; }+  constexpr bool isSelfAssign(const void*) const { return false; }+  template <class Other>+  void assign(Other&& that) {+    if (isSelfAssign(&that)) {+      return;+    }+    FOLLY_PUSH_WARNING+    FOLLY_CLANG_DISABLE_WARNING("-Wcovered-switch-default")+    switch (that.which_) {+      case Which::eValue:+        this->assignValue(static_cast<Other&&>(that).value());+        break;+      case Which::eError:+        this->assignError(static_cast<Other&&>(that).error());+        break;+      case Which::eEmpty:+      default:+        this->clear();+        break;+    }+    FOLLY_POP_WARNING+  }+};++// For small (pointer-sized) trivial types, a struct is faster than a union.+template <class Value, class Error>+struct ExpectedStorage<Value, Error, StorageType::ePODStruct> {+  using value_type = Value;+  using error_type = Error;+  Which which_;+  Error error_;+  Value value_;++  constexpr ExpectedStorage() noexcept+      : which_(Which::eError), error_{}, value_{} {}+  explicit constexpr ExpectedStorage(EmptyTag) noexcept+      : which_(Which::eEmpty), error_{}, value_{} {}+  template <class... Vs>+  explicit constexpr ExpectedStorage(ValueTag, Vs&&... vs) noexcept(+      noexcept(Value(static_cast<Vs&&>(vs)...)))+      : which_(Which::eValue), error_{}, value_(static_cast<Vs&&>(vs)...) {}+  template <class... Es>+  explicit constexpr ExpectedStorage(ErrorTag, Es&&... es) noexcept(+      noexcept(Error(static_cast<Es&&>(es)...)))+      : which_(Which::eError), error_(static_cast<Es&&>(es)...), value_{} {}+  void clear() noexcept {}+  constexpr static bool uninitializedByException() noexcept { return false; }+  template <class... Vs>+  void assignValue(Vs&&... vs) {+    expected_detail::doEmplaceAssign(0, value_, static_cast<Vs&&>(vs)...);+    which_ = Which::eValue;+  }+  template <class... Es>+  void assignError(Es&&... es) {+    expected_detail::doEmplaceAssign(0, error_, static_cast<Es&&>(es)...);+    which_ = Which::eError;+  }+  template <class Other>+  void assign(Other&& that) {+    switch (that.which_) {+      case Which::eValue:+        this->assignValue(static_cast<Other&&>(that).value());+        break;+      case Which::eError:+        this->assignError(static_cast<Other&&>(that).error());+        break;+      case Which::eEmpty:+      default:+        this->clear();+        break;+    }+  }+  Value& value() & { return value_; }+  const Value& value() const& { return value_; }+  Value&& value() && { return std::move(value_); }+  const Value&& value() const&& { return std::move(value_); }+  Error& error() & { return error_; }+  const Error& error() const& { return error_; }+  Error&& error() && { return std::move(error_); }+  const Error&& error() const&& { return std::move(error_); }+};++namespace expected_detail_ExpectedHelper {+// Tricky hack so that Expected::then can handle lambdas that return void+template <class T>+inline T&& operator,(T&& t, Unit) noexcept {+  return static_cast<T&&>(t);+}++struct ExpectedHelper {+  template <typename V, typename E>+  FOLLY_ERASE static void assume_empty(Expected<V, E> const& x) {+    compiler_may_unsafely_assume(x.which_ == Which::eEmpty);+  }++  template <class Error, class T>+  static constexpr Expected<typename std::decay<T>::type, Error> return_(+      T&& t) {+    return folly::makeExpected<Error>(static_cast<T&&>(t));+  }++  template <+      class Error,+      class T,+      class U FOLLY_REQUIRES_TRAILING(+          expected_detail::IsConvertible<U&&, Error>::value)>+  static constexpr Expected<T, Error> return_(Expected<T, U>&& t) {+    return Expected<T, Error>(static_cast<Expected<T, U>&&>(t));+  }++  template <class This>+  static typename std::decay<This>::type then_(This&& ex) {+    return static_cast<This&&>(ex);+  }++  FOLLY_PUSH_WARNING+  // Don't warn about not using the overloaded comma operator.+  FOLLY_MSVC_DISABLE_WARNING(4913)+  // On MSVC in optimized builds, the following functions can throw warning 4702+  // Unreachable Code which will block builds which treat warnings as error.+  FOLLY_MSVC_DISABLE_WARNING(4702)+  template <+      class This,+      class Fn,+      class... Fns,+      class E = ExpectedErrorType<This>,+      class T = ExpectedHelper>+  static auto then_(This&& ex, Fn&& fn, Fns&&... fns)+      -> decltype(T::then_(+          T::template return_<E>(+              (std::declval<Fn>()(std::declval<This>().value()), unit)),+          std::declval<Fns>()...)) {+    if (FOLLY_LIKELY(ex.which_ == expected_detail::Which::eValue)) {+      return T::then_(+          T::template return_<E>(+              // Uses the comma operator defined above IFF the lambda+              // returns non-void.+              (static_cast<Fn&&>(fn)(static_cast<This&&>(ex).value()), unit)),+          static_cast<Fns&&>(fns)...);+    }+    return makeUnexpected(static_cast<This&&>(ex).error());+  }++  template <+      class This,+      class Yes,+      class No,+      class Ret = decltype(std::declval<Yes>()(std::declval<This>().value())),+      class Err = decltype(std::declval<No>()(std::declval<This>().error()))+          FOLLY_REQUIRES_TRAILING(!std::is_void<Err>::value)>+  static Ret thenOrThrow_(This&& ex, Yes&& yes, No&& no) {+    if (FOLLY_LIKELY(ex.which_ == expected_detail::Which::eValue)) {+      return Ret(static_cast<Yes&&>(yes)(static_cast<This&&>(ex).value()));+    }+    throw_exception(static_cast<No&&>(no)(static_cast<This&&>(ex).error()));+  }++  template <+      class This,+      class Yes,+      class No,+      class Ret = decltype(std::declval<Yes>()(std::declval<This>().value())),+      class Err = decltype(std::declval<No>()(std::declval<This&>().error()))+          FOLLY_REQUIRES_TRAILING(std::is_void<Err>::value)>+  static Ret thenOrThrow_(This&& ex, Yes&& yes, No&& no) {+    if (FOLLY_LIKELY(ex.which_ == expected_detail::Which::eValue)) {+      return Ret(static_cast<Yes&&>(yes)(static_cast<This&&>(ex).value()));+    }+    static_cast<No&&>(no)(ex.error());+    throw_exception<BadExpectedAccess<ExpectedErrorType<This>>>(+        static_cast<This&&>(ex).error());+  }++  /**+   * Note: the condition for specialization is easy to miss here - this is for+   * where Err fails is_void, AND the else chain is not void returning. Invoked+   * when orElse handles a chain.+   */+  template <+      class This,+      class No,+      class... AndFns,+      class E = ExpectedErrorType<This>,+      class V = ExpectedValueType<This>,+      class T = ExpectedHelper,+      class RetValue =+          decltype(T::then_(+                       T::template return_<E>(+                           (std::declval<No>()(std::declval<This>().error()))),+                       std::declval<AndFns>()...)+                       .value()),+      typename std::enable_if<!std::is_same<+          std::remove_reference<RetValue>,+          std::remove_reference<folly::Unit&&>>::value>::type* = nullptr,+      class Err = decltype(std::declval<No>()(std::declval<This&>().error()))+          FOLLY_REQUIRES_TRAILING(!std::is_void<Err>::value)>+  static auto orElse_(This&& ex, No&& no, AndFns&&... fns) -> Expected<V, E> {+    // Note - this basically decays into then_ once the first type (No) is+    // called for the error.+    if (FOLLY_LIKELY(ex.which_ == expected_detail::Which::eValue)) {+      return T::template return_<E>((T::then_(T::template return_<E>(+          // Uses the comma operator defined above IFF the lambda+          // returns non-void.+          static_cast<decltype(ex)&&>(ex).value()))));+    }+    return T::then_(+        T::template return_<E>(+            (static_cast<No&&>(no)(static_cast<This&&>(ex).error()))),+        static_cast<AndFns&&>(fns)...);+  }++  /**+   * Note: the condition for specialization is easy to miss here - this is for+   * where Err fails is_void AND the else chain is void returning. Invoked when+   * orElse handles a chain.+   */+  template <+      class This,+      class No,+      class... AndFns,+      class E = ExpectedErrorType<This>,+      class T = ExpectedHelper,+      class RetValue =+          decltype(T::then_(+                       T::template return_<E>(+                           (std::declval<No>()(std::declval<This>().error()))),+                       std::declval<AndFns>()...)+                       .value()),+      typename std::enable_if<std::is_same<+          std::remove_reference<RetValue>,+          std::remove_reference<folly::Unit&&>>::value>::type* = nullptr,+      class Err = decltype(std::declval<No>()(std::declval<This&>().error()))+          FOLLY_REQUIRES_TRAILING(!std::is_void<Err>::value)>+  static auto orElse_(This&& ex, No&& no, AndFns&&... fns)+      -> Expected<folly::Unit, E> {+    // Note - this basically decays into then_ once the first type (No) is+    // called for the error.+    if (std::forward<This>(ex).which_ == expected_detail::Which::eValue) {+      // Void returning method on else must either throw, or be replaced with a+      // chain that ends in a valid result.""+      throw_exception<BadExpectedAccess<void>>();+    }+    return T::then_(+        T::template return_<E>(+            (static_cast<No&&>(no)(static_cast<This&&>(ex).error()))),+        static_cast<AndFns&&>(fns)...);+  }++  /**+   * Note: the condition for specialization is easy to miss here - this is for+   * where Err passes is_void. Invoked when orElse handles a void returning+   * func.+   */+  template <+      class This,+      class No,+      class... AndFns,+      class E = ExpectedErrorType<This>,+      class V = ExpectedValueType<This>,+      class T = ExpectedHelper,+      class Err = decltype(std::declval<No>()(std::declval<This&>().error()))+          FOLLY_REQUIRES_TRAILING(std::is_void<Err>::value)>+  static auto orElse_(This&& ex, No&& no, AndFns&&...) -> Expected<V, E> {+    if (FOLLY_LIKELY(ex.which_ == expected_detail::Which::eValue)) {+      return return_<E>(static_cast<decltype(ex)&&>(ex).value());+    }+    static_cast<No&&>(no)(static_cast<This&&>(ex).error());+    return makeUnexpected(static_cast<decltype(ex)&&>(ex).error());+  }++  template <+      class This,+      class OnError,+      class Err =+          decltype(std::declval<OnError>()(std::declval<This>().error()))+              FOLLY_REQUIRES_TRAILING(std::is_void<Err>::value)>+  static This onError_(This&& ex, OnError&& onError) {+    if (UNLIKELY(ex.which_ == expected_detail::Which::eError)) {+      static_cast<OnError&&>(onError)(static_cast<This&&>(ex).error());+    }+    return ex;+  }++  FOLLY_POP_WARNING+};+} // namespace expected_detail_ExpectedHelper++struct UnexpectedTag {};++} // namespace expected_detail++using unexpected_t =+    expected_detail::UnexpectedTag (&)(expected_detail::UnexpectedTag);++inline expected_detail::UnexpectedTag unexpected(+    expected_detail::UnexpectedTag = {}) {+  return {};+}++namespace expected_detail {+// empty+} // namespace expected_detail++/**+ * Expected - For holding a value or an error. Useful as an alternative to+ * exceptions, for APIs where throwing on failure would be too expensive.+ *+ * Expected<Value, Error> is a variant over the types Value and Error.+ *+ * Expected does not offer support for references. Use+ * Expected<std::reference_wrapper<T>, Error> if your API needs to return a+ * reference or an error.+ *+ * Expected offers a continuation-based interface to reduce the boilerplate+ * of checking error codes. The Expected::then member function takes a lambda+ * that is to execute should the Expected object contain a value. The return+ * value of the lambda is wrapped in an Expected and returned. If the lambda is+ * not executed because the Expected contains an error, the error is returned+ * immediately in a new Expected object.+ *+ * Expected<int, Error> funcTheFirst();+ * Expected<std::string, Error> funcTheSecond() {+ *   return funcTheFirst().then([](int i) { return std::to_string(i); });+ * }+ *+ * The above line of code could more verbosely written as:+ *+ * Expected<std::string, Error> funcTheSecond() {+ *   if (auto ex = funcTheFirst()) {+ *     return std::to_string(*ex);+ *   }+ *   return makeUnexpected(ex.error());+ * }+ *+ * Continuations can chain, like:+ *+ * Expected<D, Error> maybeD = someFunc()+ *     .then([](A a){return B(a);})+ *     .then([](B b){return C(b);})+ *     .then([](C c){return D(c);});+ *+ * To avoid the redundant error checking that would happen if a call at the+ * front of the chain returns an error, these call chains can be collaped into+ * a single call to .then:+ *+ * Expected<D, Error> maybeD = someFunc()+ *     .then([](A a){return B(a);},+ *           [](B b){return C(b);},+ *           [](C c){return D(c);});+ *+ * The result of .then() is wrapped into Expected< ~, Error > if it isn't+ * of that form already. Consider the following code:+ *+ * extern Expected<std::string, Error> readLineFromIO();+ * extern Expected<int, Error> parseInt(std::string);+ * extern int increment(int);+ *+ * Expected<int, Error> x = readLineFromIO().then(parseInt).then(increment);+ *+ * From the code above, we see that .then() works both with functions that+ * return an Expected< ~, Error > (like parseInt) and with ones that return+ * a plain value (like increment). In the case of parseInt, .then() returns+ * the result of parseInt as-is. In the case of increment, it wraps the int+ * that increment returns into an Expected< int, Error >.+ *+ * Sometimes when using a continuation you would prefer an exception to be+ * thrown for a value-less Expected. For that you can use .thenOrThrow, as+ * follows:+ *+ * B b = someFunc()+ *     .thenOrThrow([](A a){return B(a);});+ *+ * The above call to thenOrThrow will invoke the lambda if the Expected returned+ * by someFunc() contains a value. Otherwise, it will throw an exception of type+ * Unexpected<Error>::BadExpectedAccess. If you prefer it throw an exception of+ * a different type, you can pass a second lambda to thenOrThrow:+ *+ * B b = someFunc()+ *     .thenOrThrow([](A a){return B(a);},+ *                  [](Error e) {throw MyException(e);});+ *+ * Like C++17's std::variant, Expected offers the almost-never-empty guarantee;+ * that is, an Expected<Value, Error> almost always contains either a Value or+ * and Error. Partially-formed Expected objects occur when an assignment to+ * an Expected object that would change the type of the contained object (Value-+ * to-Error or vice versa) throws. Trying to access either the contained value+ * or error object causes Expected to throw folly::BadExpectedAccess.+ *+ * Expected models OptionalPointee, so calling 'get_pointer(ex)' will return a+ * pointer to nullptr if the 'ex' is in the error state, and a pointer to the+ * value otherwise:+ *+ *  Expected<int, Error> maybeInt = ...;+ *  if (int* v = get_pointer(maybeInt)) {+ *    cout << *v << endl;+ *  }+ */+template <class Value, class Error>+class Expected final : expected_detail::ExpectedStorage<Value, Error> {+  template <class, class>+  friend class Expected;+  template <class, class, expected_detail::StorageType>+  friend struct expected_detail::ExpectedStorage;+  friend struct expected_detail::ExpectedHelper;+  using Base = expected_detail::ExpectedStorage<Value, Error>;+  Base& base() & { return *this; }+  const Base& base() const& { return *this; }+  Base&& base() && { return std::move(*this); }++  struct MakeBadExpectedAccess {+    template <class E>+    auto operator()(E&& e) {+      return BadExpectedAccess<Error>(static_cast<E&&>(e));+    }+  };++ public:+  using value_type = Value;+  using error_type = Error;++  template <class U>+  using rebind = Expected<U, Error>;++  using promise_type = expected_detail::Promise<Value, Error>;++  static_assert(+      !std::is_reference<Value>::value,+      "Expected may not be used with reference types");+  static_assert(+      !std::is_abstract<Value>::value,+      "Expected may not be used with abstract types");++  /*+   * Constructors+   */+  template <class B = Base, class = decltype(B{})>+  Expected() noexcept(noexcept(B{})) : Base{} {}+  Expected(const Expected& that) = default;+  Expected(Expected&& that) = default;++  template <+      class V,+      class E FOLLY_REQUIRES_TRAILING(+          !std::is_same<Expected<V, E>, Expected>::value &&+          std::is_constructible<Value, V&&>::value &&+          std::is_constructible<Error, E&&>::value)>+  Expected(Expected<V, E> that) : Base{expected_detail::EmptyTag{}} {+    this->assign(std::move(that));+  }++  FOLLY_REQUIRES(std::is_copy_constructible<Value>::value)+  constexpr /* implicit */ Expected(const Value& val) noexcept(+      noexcept(Value(val)))+      : Base{expected_detail::ValueTag{}, val} {}++  FOLLY_REQUIRES(std::is_move_constructible<Value>::value)+  constexpr /* implicit */ Expected(Value&& val) noexcept(+      noexcept(Value(std::move(val))))+      : Base{expected_detail::ValueTag{}, std::move(val)} {}++  template <class T FOLLY_REQUIRES_TRAILING(+      std::is_convertible<T, Value>::value &&+      !std::is_convertible<T, Error>::value)>+  constexpr /* implicit */ Expected(T&& val) noexcept(+      noexcept(Value(static_cast<T&&>(val))))+      : Base{expected_detail::ValueTag{}, static_cast<T&&>(val)} {}++  template <class... Ts FOLLY_REQUIRES_TRAILING(+      std::is_constructible<Value, Ts&&...>::value)>+  explicit constexpr Expected(std::in_place_t, Ts&&... ts) noexcept(+      noexcept(Value(std::declval<Ts>()...)))+      : Base{expected_detail::ValueTag{}, static_cast<Ts&&>(ts)...} {}++  template <+      class U,+      class... Ts FOLLY_REQUIRES_TRAILING(+          std::is_constructible<Value, std::initializer_list<U>&, Ts&&...>::+              value)>+  explicit constexpr Expected(+      std::in_place_t,+      std::initializer_list<U> il,+      Ts&&... ts) noexcept(noexcept(Value(std::declval<Ts>()...)))+      : Base{expected_detail::ValueTag{}, il, static_cast<Ts&&>(ts)...} {}++  // If overload resolution selects one of these deleted functions, that+  // means you need to use makeUnexpected+  /* implicit */ Expected(const Error&) = delete;+  /* implicit */ Expected(Error&&) = delete;++  FOLLY_REQUIRES(std::is_copy_constructible<Error>::value)+  constexpr Expected(unexpected_t, const Error& err) noexcept(+      noexcept(Error(err)))+      : Base{expected_detail::ErrorTag{}, err} {}++  FOLLY_REQUIRES(std::is_move_constructible<Error>::value)+  constexpr Expected(unexpected_t, Error&& err) noexcept(+      noexcept(Error(std::move(err))))+      : Base{expected_detail::ErrorTag{}, std::move(err)} {}++  FOLLY_REQUIRES(std::is_copy_constructible<Error>::value)+  constexpr /* implicit */ Expected(const Unexpected<Error>& err) noexcept(+      noexcept(Error(err.error())))+      : Base{expected_detail::ErrorTag{}, err.error()} {}++  FOLLY_REQUIRES(std::is_move_constructible<Error>::value)+  constexpr /* implicit */ Expected(Unexpected<Error>&& err) noexcept(+      noexcept(Error(std::move(err.error()))))+      : Base{expected_detail::ErrorTag{}, std::move(err.error())} {}++  template <class OtherError FOLLY_REQUIRES_TRAILING(+      std::is_convertible<const OtherError&, Error>::value)>+  constexpr /* implicit */ Expected(const Unexpected<OtherError>& err) noexcept(+      noexcept(Error(err.error())))+      : Base{expected_detail::ErrorTag{}, Error(err.error())} {}++  template <class OtherError FOLLY_REQUIRES_TRAILING(+      std::is_convertible<OtherError&&, Error>::value)>+  constexpr /* implicit  */ Expected(Unexpected<OtherError>&& err) noexcept(+      noexcept(Error(std::move(err.error()))))+      : Base{expected_detail::ErrorTag{}, Error(std::move(err.error()))} {}++  /*+   * Assignment operators+   */+  Expected& operator=(const Expected& that) = default;+  Expected& operator=(Expected&& that) = default;++  template <+      class V,+      class E FOLLY_REQUIRES_TRAILING(+          !std::is_same<Expected<V, E>, Expected>::value &&+          expected_detail::IsConvertible<V&&, Value>::value &&+          expected_detail::IsConvertible<E&&, Error>::value)>+  Expected& operator=(Expected<V, E> that) {+    this->assign(std::move(that));+    return *this;+  }++  FOLLY_REQUIRES(expected_detail::IsCopyable<Value>::value)+  Expected& operator=(const Value& val) noexcept(+      expected_detail::IsNothrowCopyable<Value>::value) {+    this->assignValue(val);+    return *this;+  }++  FOLLY_REQUIRES(expected_detail::IsMovable<Value>::value)+  Expected& operator=(Value&& val) noexcept(+      expected_detail::IsNothrowMovable<Value>::value) {+    this->assignValue(std::move(val));+    return *this;+  }++  template <class T FOLLY_REQUIRES_TRAILING(+      std::is_convertible<T, Value>::value &&+      !std::is_convertible<T, Error>::value)>+  Expected& operator=(T&& val) {+    this->assignValue(static_cast<T&&>(val));+    return *this;+  }++  FOLLY_REQUIRES(expected_detail::IsCopyable<Error>::value)+  Expected& operator=(const Unexpected<Error>& err) noexcept(+      expected_detail::IsNothrowCopyable<Error>::value) {+    this->assignError(err.error());+    return *this;+  }++  FOLLY_REQUIRES(expected_detail::IsMovable<Error>::value)+  Expected& operator=(Unexpected<Error>&& err) noexcept(+      expected_detail::IsNothrowMovable<Error>::value) {+    this->assignError(std::move(err.error()));+    return *this;+  }++  template <class... Ts FOLLY_REQUIRES_TRAILING(+      std::is_constructible<Value, Ts&&...>::value)>+  void emplace(Ts&&... ts) {+    this->assignValue(static_cast<Ts&&>(ts)...);+  }++  /**+   * swap+   */+  void swap(Expected& that) noexcept(+      std::is_nothrow_swappable_v<Value> &&+      std::is_nothrow_swappable_v<Error>) {+    if (this->uninitializedByException() || that.uninitializedByException()) {+      throw_exception<BadExpectedAccess<void>>();+    }+    using std::swap;+    if (*this) {+      if (that) {+        swap(this->value_, that.value_);+      } else {+        Error e(std::move(that.error_));+        that.assignValue(std::move(this->value_));+        this->assignError(std::move(e));+      }+    } else {+      if (!that) {+        swap(this->error_, that.error_);+      } else {+        Error e(std::move(this->error_));+        this->assignValue(std::move(that.value_));+        that.assignError(std::move(e));+      }+    }+  }++  // If overload resolution selects one of these deleted functions, that+  // means you need to use makeUnexpected+  /* implicit */ Expected& operator=(const Error&) = delete;+  /* implicit */ Expected& operator=(Error&&) = delete;++  /**+   * Relational Operators+   */+  template <class Val, class Err>+  friend typename std::enable_if<IsEqualityComparable<Val>::value, bool>::type+  operator==(const Expected<Val, Err>& lhs, const Expected<Val, Err>& rhs);+  template <class Val, class Err>+  friend typename std::enable_if<IsLessThanComparable<Val>::value, bool>::type+  operator<(const Expected<Val, Err>& lhs, const Expected<Val, Err>& rhs);++  /*+   * Accessors+   */+  constexpr bool hasValue() const noexcept {+    return FOLLY_LIKELY(expected_detail::Which::eValue == this->which_);+  }++  constexpr bool hasError() const noexcept {+    return FOLLY_UNLIKELY(expected_detail::Which::eError == this->which_);+  }++  using Base::uninitializedByException;++  const Value& value() const& {+    requireValue();+    return this->Base::value();+  }++  Value& value() & {+    requireValue();+    return this->Base::value();+  }++  const Value&& value() const&& {+    requireValueMove();+    return std::move(this->Base::value());+  }++  Value&& value() && {+    requireValueMove();+    return std::move(this->Base::value());+  }++  const Error& error() const& {+    requireError();+    return this->Base::error();+  }++  Error& error() & {+    requireError();+    return this->Base::error();+  }++  const Error&& error() const&& {+    requireError();+    return std::move(this->Base::error());+  }++  Error&& error() && {+    requireError();+    return std::move(this->Base::error());+  }++  // Return a copy of the value if set, or a given default if not.+  template <class U>+  Value value_or(U&& dflt) const& {+    if (FOLLY_LIKELY(this->which_ == expected_detail::Which::eValue)) {+      return this->value_;+    }+    return static_cast<U&&>(dflt);+  }++  template <class U>+  Value value_or(U&& dflt) && {+    if (FOLLY_LIKELY(this->which_ == expected_detail::Which::eValue)) {+      return std::move(this->value_);+    }+    return static_cast<U&&>(dflt);+  }++  explicit constexpr operator bool() const noexcept { return hasValue(); }++  const Value& operator*() const& { return this->value(); }++  Value& operator*() & { return this->value(); }++  Value&& operator*() && { return std::move(std::move(*this).value()); }++  const Value* operator->() const { return std::addressof(this->value()); }++  Value* operator->() { return std::addressof(this->value()); }++  const Value* get_pointer() const& noexcept {+    return hasValue() ? std::addressof(this->value_) : nullptr;+  }++  Value* get_pointer() & noexcept {+    return hasValue() ? std::addressof(this->value_) : nullptr;+  }++  Value* get_pointer() && = delete;++  /**+   * then+   */+  template <class... Fns FOLLY_REQUIRES_TRAILING(sizeof...(Fns) >= 1)>+  auto then(Fns&&... fns)+      const& -> decltype(expected_detail::ExpectedHelper::then_(+                 std::declval<const Base&>(), std::declval<Fns>()...)) {+    if (this->uninitializedByException()) {+      throw_exception<BadExpectedAccess<void>>();+    }+    return expected_detail::ExpectedHelper::then_(+        base(), static_cast<Fns&&>(fns)...);+  }++  template <class... Fns FOLLY_REQUIRES_TRAILING(sizeof...(Fns) >= 1)>+  auto+  then(Fns&&... fns) & -> decltype(expected_detail::ExpectedHelper::then_(+                           std::declval<Base&>(), std::declval<Fns>()...)) {+    if (this->uninitializedByException()) {+      throw_exception<BadExpectedAccess<void>>();+    }+    return expected_detail::ExpectedHelper::then_(+        base(), static_cast<Fns&&>(fns)...);+  }++  template <class... Fns FOLLY_REQUIRES_TRAILING(sizeof...(Fns) >= 1)>+  auto+  then(Fns&&... fns) && -> decltype(expected_detail::ExpectedHelper::then_(+                            std::declval<Base&&>(), std::declval<Fns>()...)) {+    if (this->uninitializedByException()) {+      throw_exception<BadExpectedAccess<void>>();+    }+    return expected_detail::ExpectedHelper::then_(+        std::move(base()), static_cast<Fns&&>(fns)...);+  }++  /**+   * orElse - returns if it has a value, otherwise it calls a function with the+   * error type+   */+  template <class... Fns FOLLY_REQUIRES_TRAILING(sizeof...(Fns) >= 1)>+  auto orElse(Fns&&... fns)+      const& -> decltype(expected_detail::ExpectedHelper::orElse_(+                 std::declval<const Base&>(), std::declval<Fns>()...)) {+    if (this->uninitializedByException()) {+      throw_exception<BadExpectedAccess<void>>();+    }+    return expected_detail::ExpectedHelper::orElse_(+        base(), static_cast<Fns&&>(fns)...);+  }++  template <class... Fns FOLLY_REQUIRES_TRAILING(sizeof...(Fns) >= 1)>+  auto+  orElse(Fns&&... fns) & -> decltype(expected_detail::ExpectedHelper::orElse_(+                             std::declval<Base&>(), std::declval<Fns>()...)) {+    if (this->uninitializedByException()) {+      throw_exception<BadExpectedAccess<void>>();+    }+    return expected_detail::ExpectedHelper::orElse_(+        base(), static_cast<Fns&&>(fns)...);+  }++  template <class... Fns FOLLY_REQUIRES_TRAILING(sizeof...(Fns) >= 1)>+  auto+  orElse(Fns&&... fns) && -> decltype(expected_detail::ExpectedHelper::orElse_(+                              std::declval<Base&&>(), std::declval<Fns>()...)) {+    if (this->uninitializedByException()) {+      throw_exception<BadExpectedAccess<void>>();+    }+    return expected_detail::ExpectedHelper::orElse_(+        std::move(base()), static_cast<Fns&&>(fns)...);+  }++  /**+   * thenOrThrow+   */+  template <class Yes, class No = MakeBadExpectedAccess>+  auto thenOrThrow(Yes&& yes, No&& no = No{})+      const& -> decltype(std::declval<Yes>()(std::declval<const Value&>())) {+    using Ret = decltype(std::declval<Yes>()(std::declval<const Value&>()));+    if (this->uninitializedByException()) {+      throw_exception<BadExpectedAccess<void>>();+    }+    return Ret(expected_detail::ExpectedHelper::thenOrThrow_(+        base(), static_cast<Yes&&>(yes), static_cast<No&&>(no)));+  }++  template <class Yes, class No = MakeBadExpectedAccess>+  auto thenOrThrow(+      Yes&& yes,+      No&& no =+          No{}) & -> decltype(std::declval<Yes>()(std::declval<Value&>())) {+    using Ret = decltype(std::declval<Yes>()(std::declval<Value&>()));+    if (this->uninitializedByException()) {+      throw_exception<BadExpectedAccess<void>>();+    }+    return Ret(expected_detail::ExpectedHelper::thenOrThrow_(+        base(), static_cast<Yes&&>(yes), static_cast<No&&>(no)));+  }++  template <class Yes, class No = MakeBadExpectedAccess>+  auto thenOrThrow(+      Yes&& yes,+      No&& no =+          No{}) && -> decltype(std::declval<Yes>()(std::declval<Value&&>())) {+    using Ret = decltype(std::declval<Yes>()(std::declval<Value&&>()));+    if (this->uninitializedByException()) {+      throw_exception<BadExpectedAccess<void>>();+    }+    return Ret(expected_detail::ExpectedHelper::thenOrThrow_(+        std::move(base()), static_cast<Yes&&>(yes), static_cast<No&&>(no)));+  }++  /**+   * onError+   */+  template <class OnError>+  auto onError(OnError&& onError) const& {+    if (this->uninitializedByException()) {+      throw_exception<BadExpectedAccess<void>>();+    }++    return expected_detail::ExpectedHelper::onError_(+        *this, static_cast<OnError&&>(onError));+  }++  template <class OnError>+  auto onError(OnError&& onError) & {+    if (this->uninitializedByException()) {+      throw_exception<BadExpectedAccess<void>>();+    }+    return expected_detail::ExpectedHelper::onError_(+        *this, static_cast<OnError&&>(onError));+  }++  template <class OnError>+  auto onError(OnError&& onError) && {+    if (this->uninitializedByException()) {+      throw_exception<BadExpectedAccess<void>>();+    }+    return expected_detail::ExpectedHelper::onError_(+        std::move(*this), static_cast<OnError&&>(onError));+  }++  // Quasi-private, exposed only for implementing efficient short-circuiting+  // coroutines on top of `Expected`.  Do NOT use this instead of+  // `optional<Expected<>>`, for these reasons:+  //   - This public ctor again become private in the future.+  //   - It is incompatible with upcoming `std::expected`.+  //   - It violates `folly::Expected`'s almost-never-empty guarantee.+  //   - There's no native `empty()` test -- `uninitializedByException()` is+  //     always `false` for some value types, and `!hasValue() && !hasError()`+  //     is less efficient.+  explicit Expected(expected_detail::EmptyTag tag) noexcept : Base{tag} {}++ private:+  friend struct expected_detail::PromiseReturn<Value, Error>;++  // for when coroutine promise return-object conversion is eager+  Expected(expected_detail::EmptyTag tag, Expected*& pointer) noexcept+      : Base{tag} {+    pointer = this;+  }++  void requireValue() const {+    if (FOLLY_UNLIKELY(!hasValue())) {+      if (FOLLY_LIKELY(hasError())) {+        throw_exception<BadExpectedAccess<Error>>(this->error_);+      }+      throw_exception<BadExpectedAccess<void>>();+    }+  }++  template <typename Self>+  static void requireValueMove(Self& self) {+    if (FOLLY_UNLIKELY(!self.hasValue())) {+      if (FOLLY_LIKELY(self.hasError())) {+        throw_exception<BadExpectedAccess<Error>>(std::move(self.error_));+      }+      throw_exception<BadExpectedAccess<void>>();+    }+  }++  void requireValueMove() { return requireValueMove(*this); }+  void requireValueMove() const { return requireValueMove(*this); }++  void requireError() const {+    if (FOLLY_UNLIKELY(!hasError())) {+      throw_exception<BadExpectedAccess<void>>();+    }+  }++  expected_detail::Which which() const noexcept { return this->which_; }+};++template <class Value, class Error>+inline typename std::enable_if<IsEqualityComparable<Value>::value, bool>::type+operator==(+    const Expected<Value, Error>& lhs, const Expected<Value, Error>& rhs) {+  if (FOLLY_UNLIKELY(lhs.uninitializedByException())) {+    throw_exception<BadExpectedAccess<void>>();+  }+  if (FOLLY_UNLIKELY(lhs.which_ != rhs.which_)) {+    return false;+  }+  if (FOLLY_UNLIKELY(lhs.hasError())) {+    return true; // All error states are considered equal+  }+  return lhs.value_ == rhs.value_;+}++template <+    class Value,+    class Error FOLLY_REQUIRES_TRAILING(IsEqualityComparable<Value>::value)>+inline bool operator!=(+    const Expected<Value, Error>& lhs, const Expected<Value, Error>& rhs) {+  return !(rhs == lhs);+}++template <class Value, class Error>+inline typename std::enable_if<IsLessThanComparable<Value>::value, bool>::type+operator<(+    const Expected<Value, Error>& lhs, const Expected<Value, Error>& rhs) {+  if (FOLLY_UNLIKELY(+          lhs.uninitializedByException() || rhs.uninitializedByException())) {+    throw_exception<BadExpectedAccess<void>>();+  }+  if (FOLLY_UNLIKELY(lhs.hasError())) {+    return !rhs.hasError();+  }+  if (FOLLY_UNLIKELY(rhs.hasError())) {+    return false;+  }+  return lhs.value_ < rhs.value_;+}++template <+    class Value,+    class Error FOLLY_REQUIRES_TRAILING(IsLessThanComparable<Value>::value)>+inline bool operator<=(+    const Expected<Value, Error>& lhs, const Expected<Value, Error>& rhs) {+  return !(rhs < lhs);+}++template <+    class Value,+    class Error FOLLY_REQUIRES_TRAILING(IsLessThanComparable<Value>::value)>+inline bool operator>(+    const Expected<Value, Error>& lhs, const Expected<Value, Error>& rhs) {+  return rhs < lhs;+}++template <+    class Value,+    class Error FOLLY_REQUIRES_TRAILING(IsLessThanComparable<Value>::value)>+inline bool operator>=(+    const Expected<Value, Error>& lhs, const Expected<Value, Error>& rhs) {+  return !(lhs < rhs);+}++/**+ * swap Expected values+ */+template <class Value, class Error>+void swap(Expected<Value, Error>& lhs, Expected<Value, Error>& rhs) noexcept(+    std::is_nothrow_swappable_v<Value> && std::is_nothrow_swappable_v<Error>) {+  lhs.swap(rhs);+}++template <class Value, class Error>+const Value* get_pointer(const Expected<Value, Error>& ex) noexcept {+  return ex.get_pointer();+}++template <class Value, class Error>+Value* get_pointer(Expected<Value, Error>& ex) noexcept {+  return ex.get_pointer();+}++/**+ * For constructing an Expected object from a value, with the specified+ * Error type. Usage is as follows:+ *+ * enum MyErrorCode { BAD_ERROR, WORSE_ERROR };+ * Expected<int, MyErrorCode> myAPI() {+ *   int i = // ...;+ *   return i ? makeExpected<MyErrorCode>(i) : makeUnexpected(BAD_ERROR);+ * }+ */+template <class Error, class Value>+FOLLY_NODISCARD constexpr Expected<typename std::decay<Value>::type, Error>+makeExpected(Value&& val) {+  return Expected<typename std::decay<Value>::type, Error>{+      std::in_place, static_cast<Value&&>(val)};+}++// Suppress comparability of Expected<T> with T, despite implicit conversion.+template <class Value, class Error>+bool operator==(const Expected<Value, Error>&, const Value& other) = delete;+template <class Value, class Error>+bool operator!=(const Expected<Value, Error>&, const Value& other) = delete;+template <class Value, class Error>+bool operator<(const Expected<Value, Error>&, const Value& other) = delete;+template <class Value, class Error>+bool operator<=(const Expected<Value, Error>&, const Value& other) = delete;+template <class Value, class Error>+bool operator>=(const Expected<Value, Error>&, const Value& other) = delete;+template <class Value, class Error>+bool operator>(const Expected<Value, Error>&, const Value& other) = delete;+template <class Value, class Error>+bool operator==(const Value& other, const Expected<Value, Error>&) = delete;+template <class Value, class Error>+bool operator!=(const Value& other, const Expected<Value, Error>&) = delete;+template <class Value, class Error>+bool operator<(const Value& other, const Expected<Value, Error>&) = delete;+template <class Value, class Error>+bool operator<=(const Value& other, const Expected<Value, Error>&) = delete;+template <class Value, class Error>+bool operator>=(const Value& other, const Expected<Value, Error>&) = delete;+template <class Value, class Error>+bool operator>(const Value& other, const Expected<Value, Error>&) = delete;++} // namespace folly++#undef FOLLY_REQUIRES+#undef FOLLY_REQUIRES_TRAILING++// Enable the use of folly::Expected with `co_await`+// Inspired by https://github.com/toby-allsopp/coroutine_monad+#if FOLLY_HAS_COROUTINES+#include <folly/coro/Coroutine.h>++namespace folly {+namespace expected_detail {+template <typename Value, typename Error>+struct PromiseBase;++template <typename Value, typename Error>+struct PromiseReturn {+  Expected<Value, Error> storage_{EmptyTag{}};+  Expected<Value, Error>*& pointer_;++  /* implicit */ PromiseReturn(PromiseBase<Value, Error>& p) noexcept+      : pointer_{p.value_} {+    pointer_ = &storage_;+  }+  PromiseReturn(PromiseReturn const&) = delete;+  // letting dtor be trivial makes the coroutine crash+  // TODO: fix clang/llvm codegen+  ~PromiseReturn() {}+  /* implicit */ operator Expected<Value, Error>() {+    // handle both deferred and eager return-object conversion behaviors+    // see docs for detect_promise_return_object_eager_conversion+    if (folly::coro::detect_promise_return_object_eager_conversion()) {+      assert(storage_.which_ == expected_detail::Which::eEmpty);+      return Expected<Value, Error>{EmptyTag{}, pointer_}; // eager+    } else {+      assert(storage_.which_ != expected_detail::Which::eEmpty);+      return std::move(storage_); // deferred+    }+  }+};++template <typename Value, typename Error>+struct PromiseBase {+  Expected<Value, Error>* value_ = nullptr;++  PromiseBase() = default;+  PromiseBase(PromiseBase const&) = delete;+  void operator=(PromiseBase const&) = delete;++  [[nodiscard]] coro::suspend_never initial_suspend() const noexcept {+    return {};+  }+  [[nodiscard]] coro::suspend_never final_suspend() const noexcept {+    return {};+  }+  [[noreturn]] void unhandled_exception() {+    // Technically, throwing from unhandled_exception is underspecified:+    // https://github.com/GorNishanov/CoroutineWording/issues/17+    rethrow_current_exception();+  }++  PromiseReturn<Value, Error> get_return_object() noexcept { return *this; }+};++template <typename Value>+inline constexpr bool ReturnsVoid =+    std::is_trivial_v<Value> && std::is_empty_v<Value>;++template <typename Value, typename Error>+struct PromiseReturnsValue : public PromiseBase<Value, Error> {+  template <typename U = Value>+  void return_value(U&& u) {+    auto& v = *this->value_;+    ExpectedHelper::assume_empty(v);+    v = static_cast<U&&>(u);+  }+};++template <typename Value, typename Error>+struct PromiseReturnsVoid : public PromiseBase<Value, Error> {+  // When the coroutine uses `return;` you can fail via `co_await err`.+  void return_void() { this->value_->emplace(Value{}); }+};++template <typename Value, typename Error>+struct Promise //+    : conditional_t<+          ReturnsVoid<Value>,+          PromiseReturnsVoid<Value, Error>,+          PromiseReturnsValue<Value, Error>> {};++template <typename Error>+struct UnexpectedAwaitable {+  Unexpected<Error> o_;++  explicit UnexpectedAwaitable(Unexpected<Error> o) : o_(std::move(o)) {}++  constexpr std::false_type await_ready() const noexcept { return {}; }+  void await_resume() { compiler_may_unsafely_assume_unreachable(); }++  template <typename U>+  FOLLY_ALWAYS_INLINE void await_suspend(+      coro::coroutine_handle<Promise<U, Error>> h) {+    auto& v = *h.promise().value_;+    ExpectedHelper::assume_empty(v);+    v = std::move(o_);+    h.destroy();+  }+};++template <typename Value, typename Error>+struct ExpectedAwaitable {+  Expected<Value, Error> o_;++  explicit ExpectedAwaitable(Expected<Value, Error> o) : o_(std::move(o)) {}++  bool await_ready() const noexcept { return o_.hasValue(); }+  Value await_resume() { return std::move(o_.value()); }++  // Explicitly only allow suspension into a Promise+  template <typename U>+  FOLLY_ALWAYS_INLINE void await_suspend(+      coro::coroutine_handle<Promise<U, Error>> h) {+    auto& v = *h.promise().value_;+    ExpectedHelper::assume_empty(v);+    v = makeUnexpected(std::move(o_.error()));+    // Abort the rest of the coroutine. resume() is not going to be called+    h.destroy();+  }+};++} // namespace expected_detail++template <typename Error>+expected_detail::UnexpectedAwaitable<Error>+/* implicit */ operator co_await(Unexpected<Error> o) {+  return expected_detail::UnexpectedAwaitable<Error>{std::move(o)};+}++template <typename Value, typename Error>+expected_detail::ExpectedAwaitable<Value, Error>+/* implicit */ operator co_await(Expected<Value, Error> o) {+  return expected_detail::ExpectedAwaitable<Value, Error>{std::move(o)};+}++} // namespace folly+#endif // FOLLY_HAS_COROUTINES
+ folly/folly/FBString.h view
@@ -0,0 +1,2872 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++// String type.++#pragma once++#include <algorithm>+#include <atomic>+#include <cassert>+#include <cstddef>+#include <cstring>+#include <iosfwd>+#include <limits>+#include <stdexcept>+#include <string>+#include <string_view>+#include <type_traits>+#include <utility>++#include <fmt/format.h>+#include <folly/CPortability.h>+#include <folly/CppAttributes.h>+#include <folly/Likely.h>+#include <folly/Portability.h>+#include <folly/Traits.h>+#include <folly/hash/Hash.h>+#include <folly/lang/Assume.h>+#include <folly/lang/CheckedMath.h>+#include <folly/lang/Exception.h>+#include <folly/memory/Malloc.h>++#if FOLLY_CPLUSPLUS >= 202002L+#include <compare>+#endif++FOLLY_PUSH_WARNING+// Ignore shadowing warnings within this file, so includers can use -Wshadow.+FOLLY_GNU_DISABLE_WARNING("-Wshadow")++namespace folly {++// When compiling with ASan, always heap-allocate the string even if+// it would fit in-situ, so that ASan can detect access to the string+// buffer after it has been invalidated (destroyed, resized, etc.).+// Note that this flag doesn't remove support for in-situ strings, as+// that would break ABI-compatibility and wouldn't allow linking code+// compiled with this flag with code compiled without.+#ifdef FOLLY_SANITIZE_ADDRESS+#define FBSTRING_DISABLE_SSO true+#else+#define FBSTRING_DISABLE_SSO false+#endif++namespace fbstring_detail {++template <class InIt, class OutIt>+inline std::pair<InIt, OutIt> copy_n(+    InIt b, typename std::iterator_traits<InIt>::difference_type n, OutIt d) {+  for (; n != 0; --n, ++b, ++d) {+    *d = *b;+  }+  return std::make_pair(b, d);+}++template <class Pod, class T>+inline void podFill(Pod* b, Pod* e, T c) {+  assert(b && e && b <= e);+  constexpr auto kUseMemset = sizeof(T) == 1;+  if constexpr (kUseMemset) {+    memset(b, c, size_t(e - b));+  } else {+    auto const ee = b + ((e - b) & ~7u);+    for (; b != ee; b += 8) {+      b[0] = c;+      b[1] = c;+      b[2] = c;+      b[3] = c;+      b[4] = c;+      b[5] = c;+      b[6] = c;+      b[7] = c;+    }+    // Leftovers+    for (; b != e; ++b) {+      *b = c;+    }+  }+}++/*+ * Lightly structured memcpy, simplifies copying PODs and introduces+ * some asserts. Unfortunately using this function may cause+ * measurable overhead (presumably because it adjusts from a begin/end+ * convention to a pointer/size convention, so it does some extra+ * arithmetic even though the caller might have done the inverse+ * adaptation outside).+ */+template <class Pod>+inline void podCopy(const Pod* b, const Pod* e, Pod* d) {+  assert(b != nullptr);+  assert(e != nullptr);+  assert(d != nullptr);+  assert(e >= b);+  assert(d >= e || d + (e - b) <= b);+  memcpy(d, b, (e - b) * sizeof(Pod));+}++/*+ * Lightly structured memmove, simplifies copying PODs and introduces+ * some asserts+ */+template <class Pod>+inline void podMove(const Pod* b, const Pod* e, Pod* d) {+  assert(e >= b);+  memmove(d, b, (e - b) * sizeof(*b));+}+} // namespace fbstring_detail++/**+ * Defines a special acquisition method for constructing fbstring+ * objects. AcquireMallocatedString means that the user passes a+ * pointer to a malloc-allocated string that the fbstring object will+ * take into custody.+ */+enum class AcquireMallocatedString {};++/*+ * fbstring_core_model is a mock-up type that defines all required+ * signatures of a fbstring core. The fbstring class itself uses such+ * a core object to implement all of the numerous member functions+ * required by the standard.+ *+ * If you want to define a new core, copy the definition below and+ * implement the primitives. Then plug the core into basic_fbstring as+ * a template argument.++template <class Char>+class fbstring_core_model {+ public:+  fbstring_core_model();+  fbstring_core_model(const fbstring_core_model &);+  fbstring_core_model& operator=(const fbstring_core_model &) = delete;+  ~fbstring_core_model();+  // Returns a pointer to string's buffer (currently only contiguous+  // strings are supported). The pointer is guaranteed to be valid+  // until the next call to a non-const member function.+  const Char * data() const;+  // Much like data(), except the string is prepared to support+  // character-level changes. This call is a signal for+  // e.g. reference-counted implementation to fork the data. The+  // pointer is guaranteed to be valid until the next call to a+  // non-const member function.+  Char* mutableData();+  // Returns a pointer to string's buffer and guarantees that a+  // readable '\0' lies right after the buffer. The pointer is+  // guaranteed to be valid until the next call to a non-const member+  // function.+  const Char * c_str() const;+  // Shrinks the string by delta characters. Asserts that delta <=+  // size().+  void shrink(size_t delta);+  // Expands the string by delta characters (i.e. after this call+  // size() will report the old size() plus delta) but without+  // initializing the expanded region. The expanded region is+  // zero-terminated. Returns a pointer to the memory to be+  // initialized (the beginning of the expanded portion). The caller+  // is expected to fill the expanded area appropriately.+  // If expGrowth is true, exponential growth is guaranteed.+  // It is not guaranteed not to reallocate even if size() + delta <+  // capacity(), so all references to the buffer are invalidated.+  Char* expandNoinit(size_t delta, bool expGrowth);+  // Expands the string by one character and sets the last character+  // to c.+  void push_back(Char c);+  // Returns the string's size.+  size_t size() const;+  // Returns the string's capacity, i.e. maximum size that the string+  // can grow to without reallocation. Note that for reference counted+  // strings that's technically a lie - even assigning characters+  // within the existing size would cause a reallocation.+  size_t capacity() const;+  // Returns true if the data underlying the string is actually shared+  // across multiple strings (in a refcounted fashion).+  bool isShared() const;+  // Makes sure that at least minCapacity characters are available for+  // the string without reallocation. For reference-counted strings,+  // it should fork the data even if minCapacity < size().+  void reserve(size_t minCapacity);+};+*/++/**+ * This is the core of the string. The code should work on 32- and+ * 64-bit and both big- and little-endianan architectures with any+ * Char size.+ *+ * The storage is selected as follows (assuming we store one-byte+ * characters on a 64-bit machine): (a) "small" strings between 0 and+ * 23 chars are stored in-situ without allocation (the rightmost byte+ * stores the size); (b) "medium" strings from 24 through 254 chars+ * are stored in malloc-allocated memory that is copied eagerly; (c)+ * "large" strings of 255 chars and above are stored in a similar+ * structure as medium arrays, except that the string is+ * reference-counted and copied lazily. the reference count is+ * allocated right before the character array.+ *+ * The discriminator between these three strategies sits in two+ * bits of the rightmost char of the storage:+ * - If neither is set, then the string is small. Its length is represented by+ *   the lower-order bits on little-endian or the high-order bits on big-endian+ *   of that rightmost character. The value of these six bits is+ *   `maxSmallSize - size`, so this quantity must be subtracted from+ *   `maxSmallSize` to compute the `size` of the string (see `smallSize()`).+ *   This scheme ensures that when `size == `maxSmallSize`, the last byte in the+ *   storage is \0. This way, storage will be a null-terminated sequence of+ *   bytes, even if all 23 bytes of data are used on a 64-bit architecture.+ *   This enables `c_str()` and `data()` to simply return a pointer to the+ *   storage.+ *+ * - If the MSb is set, the string is medium width.+ *+ * - If the second MSb is set, then the string is large. On little-endian,+ *   these 2 bits are the 2 MSbs of MediumLarge::capacity_, while on+ *   big-endian, these 2 bits are the 2 LSbs. This keeps both little-endian+ *   and big-endian fbstring_core equivalent with merely different ops used+ *   to extract capacity/category.+ */+template <class Char>+class fbstring_core {+ public:+  fbstring_core() noexcept { reset(); }++  fbstring_core(const fbstring_core& rhs) {+    assert(&rhs != this);+    switch (rhs.category()) {+      case Category::isSmall:+        copySmall(rhs);+        break;+      case Category::isMedium:+        copyMedium(rhs);+        break;+      case Category::isLarge:+        copyLarge(rhs);+        break;+      default:+        folly::assume_unreachable();+    }+    assert(size() == rhs.size());+    assert(memcmp(data(), rhs.data(), size() * sizeof(Char)) == 0);+  }++  fbstring_core& operator=(const fbstring_core& rhs) = delete;++  fbstring_core(fbstring_core&& goner) noexcept {+    // Take goner's guts+    ml_ = goner.ml_;+    // Clean goner's carcass+    goner.reset();+  }++  fbstring_core(+      const Char* const data,+      const size_t size,+      bool disableSSO = FBSTRING_DISABLE_SSO) {+    if (!disableSSO && size <= maxSmallSize) {+      initSmall(data, size);+    } else if (size <= maxMediumSize) {+      initMedium(data, size);+    } else {+      initLarge(data, size);+    }+    assert(this->size() == size);+    assert(size == 0 || memcmp(this->data(), data, size * sizeof(Char)) == 0);+  }++  ~fbstring_core() noexcept {+    if (category() == Category::isSmall) {+      return;+    }+    destroyMediumLarge();+  }++  // Snatches a previously mallocated string. The parameter "size"+  // is the size of the string, and the parameter "allocatedSize"+  // is the size of the mallocated block.  The string must be+  // \0-terminated, so allocatedSize >= size + 1 and data[size] == '\0'.+  //+  // So if you want a 2-character string, pass malloc(3) as "data",+  // pass 2 as "size", and pass 3 as "allocatedSize".+  fbstring_core(+      Char* const data,+      const size_t size,+      const size_t allocatedSize,+      AcquireMallocatedString) {+    if (size > 0) {+      assert(allocatedSize >= size + 1);+      assert(data[size] == '\0');+      // Use the medium string storage+      ml_.data_ = data;+      ml_.size_ = size;+      // Don't forget about null terminator+      ml_.setCapacity(allocatedSize - 1, Category::isMedium);+    } else {+      // No need for the memory+      free(data);+      reset();+    }+  }++  // swap below doesn't test whether &rhs == this (and instead+  // potentially does extra work) on the premise that the rarity of+  // that situation actually makes the check more expensive than is+  // worth.+  void swap(fbstring_core& rhs) {+    auto const t = ml_;+    ml_ = rhs.ml_;+    rhs.ml_ = t;+  }++  // In C++11 data() and c_str() are 100% equivalent.+  const Char* data() const { return c_str(); }++  Char* data() { return c_str(); }++  Char* mutableData() {+    switch (category()) {+      case Category::isSmall:+        return small_;+      case Category::isMedium:+        return ml_.data_;+      case Category::isLarge:+        return mutableDataLarge();+    }+    folly::assume_unreachable();+  }++  const Char* c_str() const {+    const Char* ptr = ml_.data_;+    // With this syntax, GCC and Clang generate a CMOV instead of a branch.+    ptr = (category() == Category::isSmall) ? small_ : ptr;+    return ptr;+  }++  void shrink(const size_t delta) {+    if (category() == Category::isSmall) {+      shrinkSmall(delta);+    } else if (+        category() == Category::isMedium || RefCounted::refs(ml_.data_) == 1) {+      shrinkMedium(delta);+    } else {+      shrinkLarge(delta);+    }+  }++  FOLLY_NOINLINE+  void reserve(size_t minCapacity, bool disableSSO = FBSTRING_DISABLE_SSO) {+    FOLLY_PUSH_WARNING+    FOLLY_CLANG_DISABLE_WARNING("-Wcovered-switch-default")+    switch (category()) {+      case Category::isSmall:+        reserveSmall(minCapacity, disableSSO);+        break;+      case Category::isMedium:+        reserveMedium(minCapacity);+        break;+      case Category::isLarge:+        reserveLarge(minCapacity);+        break;+      default:+        folly::assume_unreachable();+    }+    FOLLY_POP_WARNING+    assert(capacity() >= minCapacity);+  }++  Char* expandNoinit(+      const size_t delta,+      bool expGrowth = false,+      bool disableSSO = FBSTRING_DISABLE_SSO);++  void push_back(Char c) { *expandNoinit(1, /* expGrowth = */ true) = c; }++  size_t size() const {+    size_t ret = ml_.size_;+    if constexpr (kIsLittleEndian) {+      // We can save a couple instructions, because the category is+      // small iff the last char, as unsigned, is <= maxSmallSize.+      typedef typename std::make_unsigned<Char>::type UChar;+      auto maybeSmallSize = size_t(maxSmallSize) -+          size_t(static_cast<UChar>(small_[maxSmallSize]));+      // With this syntax, GCC and Clang generate a CMOV instead of a branch.+      ret =+          (static_cast<ptrdiff_t>(maybeSmallSize) >= 0) ? maybeSmallSize : ret;+    } else {+      ret = (category() == Category::isSmall) ? smallSize() : ret;+    }+    return ret;+  }++  size_t capacity() const {+    FOLLY_PUSH_WARNING+    FOLLY_CLANG_DISABLE_WARNING("-Wcovered-switch-default")+    switch (category()) {+      case Category::isSmall:+        return maxSmallSize;+      case Category::isLarge:+        // For large-sized strings, a multi-referenced chunk has no+        // available capacity. This is because any attempt to append+        // data would trigger a new allocation.+        if (RefCounted::refs(ml_.data_) > 1) {+          return ml_.size_;+        }+        break;+      case Category::isMedium:+      default:+        break;+    }+    FOLLY_POP_WARNING+    return ml_.capacity();+  }++  bool isShared() const {+    return category() == Category::isLarge && RefCounted::refs(ml_.data_) > 1;+  }++ private:+  Char* c_str() {+    Char* ptr = ml_.data_;+    // With this syntax, GCC and Clang generate a CMOV instead of a branch.+    ptr = (category() == Category::isSmall) ? small_ : ptr;+    return ptr;+  }++  void reset() { setSmallSize(0); }++  FOLLY_NOINLINE void destroyMediumLarge() noexcept {+    auto const c = category();+    assert(c != Category::isSmall);+    if (c == Category::isMedium) {+      free(ml_.data_);+    } else {+      RefCounted::decrementRefs(ml_.data_);+    }+  }++  struct RefCounted {+    std::atomic<size_t> refCount_;+    Char data_[1];++    constexpr static size_t getDataOffset() {+      return offsetof(RefCounted, data_);+    }++    static RefCounted* fromData(Char* p) {+      return static_cast<RefCounted*>(static_cast<void*>(+          static_cast<unsigned char*>(static_cast<void*>(p)) -+          getDataOffset()));+    }++    static size_t refs(Char* p) {+      return fromData(p)->refCount_.load(std::memory_order_acquire);+    }++    static void incrementRefs(Char* p) {+      fromData(p)->refCount_.fetch_add(1, std::memory_order_acq_rel);+    }++    static void decrementRefs(Char* p) {+      auto const dis = fromData(p);+      size_t oldcnt = dis->refCount_.fetch_sub(1, std::memory_order_acq_rel);+      assert(oldcnt > 0);+      if (oldcnt == 1) {+        ::free(dis);+      }+    }++    static RefCounted* create(size_t* size) {+      size_t capacityBytes;+      if (!folly::checked_add(&capacityBytes, *size, size_t(1))) {+        throw_exception(std::length_error(""));+      }+      if (!folly::checked_muladd(+              &capacityBytes, capacityBytes, sizeof(Char), getDataOffset())) {+        throw_exception(std::length_error(""));+      }+      const size_t allocSize = goodMallocSize(capacityBytes);+      auto result = static_cast<RefCounted*>(checkedMalloc(allocSize));+      result->refCount_.store(1, std::memory_order_release);+      *size = (allocSize - getDataOffset()) / sizeof(Char) - 1;+      return result;+    }++    static RefCounted* create(const Char* data, size_t* size) {+      const size_t effectiveSize = *size;+      auto result = create(size);+      if (FOLLY_LIKELY(effectiveSize > 0)) {+        fbstring_detail::podCopy(data, data + effectiveSize, result->data_);+      }+      return result;+    }++    static RefCounted* reallocate(+        Char* const data,+        const size_t currentSize,+        const size_t currentCapacity,+        size_t* newCapacity) {+      assert(*newCapacity > 0 && *newCapacity > currentSize);+      size_t capacityBytes;+      if (!folly::checked_add(&capacityBytes, *newCapacity, size_t(1))) {+        throw_exception(std::length_error(""));+      }+      if (!folly::checked_muladd(+              &capacityBytes, capacityBytes, sizeof(Char), getDataOffset())) {+        throw_exception(std::length_error(""));+      }+      const size_t allocNewCapacity = goodMallocSize(capacityBytes);+      auto const dis = fromData(data);+      assert(dis->refCount_.load(std::memory_order_acquire) == 1);+      auto result = static_cast<RefCounted*>(smartRealloc(+          dis,+          getDataOffset() + (currentSize + 1) * sizeof(Char),+          getDataOffset() + (currentCapacity + 1) * sizeof(Char),+          allocNewCapacity));+      assert(result->refCount_.load(std::memory_order_acquire) == 1);+      *newCapacity = (allocNewCapacity - getDataOffset()) / sizeof(Char) - 1;+      return result;+    }+  };++  typedef uint8_t category_type;++  enum class Category : category_type {+    isSmall = 0,+    isMedium = kIsLittleEndian ? 0x80 : 0x2,+    isLarge = kIsLittleEndian ? 0x40 : 0x1,+  };++  Category category() const {+    // works for both big-endian and little-endian+    return static_cast<Category>(bytes_[lastChar] & categoryExtractMask);+  }++  struct MediumLarge {+    Char* data_;+    size_t size_;+    size_t capacity_;++    size_t capacity() const {+      return kIsLittleEndian ? capacity_ & capacityExtractMask : capacity_ >> 2;+    }++    void setCapacity(size_t cap, Category cat) {+      capacity_ = kIsLittleEndian+          ? cap | (static_cast<size_t>(cat) << kCategoryShift)+          : (cap << 2) | static_cast<size_t>(cat);+    }+  };++  union {+    uint8_t bytes_[sizeof(MediumLarge)]; // For accessing the last byte.+    Char small_[sizeof(MediumLarge) / sizeof(Char)];+    MediumLarge ml_;+  };++  constexpr static size_t lastChar = sizeof(MediumLarge) - 1;+  constexpr static size_t maxSmallSize = lastChar / sizeof(Char);+  constexpr static size_t maxMediumSize = 254 / sizeof(Char);+  constexpr static uint8_t categoryExtractMask = kIsLittleEndian ? 0xC0 : 0x3;+  constexpr static size_t kCategoryShift = (sizeof(size_t) - 1) * 8;+  constexpr static size_t capacityExtractMask = kIsLittleEndian+      ? ~(size_t(categoryExtractMask) << kCategoryShift)+      : 0x0 /* unused */;++  static_assert(+      !(sizeof(MediumLarge) % sizeof(Char)),+      "Corrupt memory layout for fbstring.");++  size_t smallSize() const {+    assert(category() == Category::isSmall);+    constexpr auto shift = kIsLittleEndian ? 0 : 2;+    auto smallShifted = static_cast<size_t>(small_[maxSmallSize]) >> shift;+    assert(static_cast<size_t>(maxSmallSize) >= smallShifted);+    return static_cast<size_t>(maxSmallSize) - smallShifted;+  }++  void setSmallSize(size_t s) {+    // Warning: this should work with uninitialized strings too,+    // so don't assume anything about the previous value of+    // small_[maxSmallSize].+    assert(s <= maxSmallSize);+    constexpr auto shift = kIsLittleEndian ? 0 : 2;+    small_[maxSmallSize] = char((maxSmallSize - s) << shift);+    small_[s] = '\0';+    assert(category() == Category::isSmall && size() == s);+  }++  void copySmall(const fbstring_core&);+  void copyMedium(const fbstring_core&);+  void copyLarge(const fbstring_core&);++  void initSmall(const Char* data, size_t size);+  void initMedium(const Char* data, size_t size);+  void initLarge(const Char* data, size_t size);++  void reserveSmall(size_t minCapacity, bool disableSSO);+  void reserveMedium(size_t minCapacity);+  void reserveLarge(size_t minCapacity);++  void shrinkSmall(size_t delta);+  void shrinkMedium(size_t delta);+  void shrinkLarge(size_t delta);++  void unshare(size_t minCapacity = 0);+  Char* mutableDataLarge();+};++template <class Char>+inline void fbstring_core<Char>::copySmall(const fbstring_core& rhs) {+  static_assert(offsetof(MediumLarge, data_) == 0, "fbstring layout failure");+  static_assert(+      offsetof(MediumLarge, size_) == sizeof(ml_.data_),+      "fbstring layout failure");+  static_assert(+      offsetof(MediumLarge, capacity_) == 2 * sizeof(ml_.data_),+      "fbstring layout failure");+  // Just write the whole thing, don't look at details. In+  // particular we need to copy capacity anyway because we want+  // to set the size (don't forget that the last character,+  // which stores a short string's length, is shared with the+  // ml_.capacity field).+  ml_ = rhs.ml_;+  assert(category() == Category::isSmall && this->size() == rhs.size());+}++template <class Char>+FOLLY_NOINLINE void fbstring_core<Char>::copyMedium(const fbstring_core& rhs) {+  // Medium strings are copied eagerly. Don't forget to allocate+  // one extra Char for the null terminator.+  auto const allocSize = goodMallocSize((1 + rhs.ml_.size_) * sizeof(Char));+  ml_.data_ = static_cast<Char*>(checkedMalloc(allocSize));+  // Also copies terminator.+  fbstring_detail::podCopy(+      rhs.ml_.data_, rhs.ml_.data_ + rhs.ml_.size_ + 1, ml_.data_);+  ml_.size_ = rhs.ml_.size_;+  ml_.setCapacity(allocSize / sizeof(Char) - 1, Category::isMedium);+  assert(category() == Category::isMedium);+}++template <class Char>+FOLLY_NOINLINE void fbstring_core<Char>::copyLarge(const fbstring_core& rhs) {+  // Large strings are just refcounted+  ml_ = rhs.ml_;+  RefCounted::incrementRefs(ml_.data_);+  assert(category() == Category::isLarge && size() == rhs.size());+}++// Small strings are bitblitted+template <class Char>+inline void fbstring_core<Char>::initSmall(+    const Char* const data, const size_t size) {+  // Layout is: Char* data_, size_t size_, size_t capacity_+  static_assert(+      sizeof(*this) == sizeof(Char*) + 2 * sizeof(size_t),+      "fbstring has unexpected size");+  static_assert(+      sizeof(Char*) == sizeof(size_t), "fbstring size assumption violation");+  // sizeof(size_t) must be a power of 2+  static_assert(+      (sizeof(size_t) & (sizeof(size_t) - 1)) == 0,+      "fbstring size assumption violation");++  constexpr size_t kPageSize = 4096;++  const auto addr = reinterpret_cast<uintptr_t>(data);+  if (!kIsSanitize && // sanitizer would trap on over-reads+      size && (addr ^ (addr + sizeof(small_) - 1)) < kPageSize) {+    // the input data is all within one page so over-reads will not segfault+    std::memcpy(small_, data, sizeof(small_)); // lowers to a 4-insn sequence+  } else {+    if (size != 0) {+      fbstring_detail::podCopy(data, data + size, small_);+    }+  }+  setSmallSize(size);+}++template <class Char>+FOLLY_NOINLINE void fbstring_core<Char>::initMedium(+    const Char* const data, const size_t size) {+  // Medium strings are allocated normally. Don't forget to+  // allocate one extra Char for the terminating null.+  auto const allocSize = goodMallocSize((1 + size) * sizeof(Char));+  ml_.data_ = static_cast<Char*>(checkedMalloc(allocSize));+  if (FOLLY_LIKELY(size > 0)) {+    fbstring_detail::podCopy(data, data + size, ml_.data_);+  }+  ml_.size_ = size;+  ml_.setCapacity(allocSize / sizeof(Char) - 1, Category::isMedium);+  ml_.data_[size] = '\0';+}++template <class Char>+FOLLY_NOINLINE void fbstring_core<Char>::initLarge(+    const Char* const data, const size_t size) {+  // Large strings are allocated differently+  size_t effectiveCapacity = size;+  auto const newRC = RefCounted::create(data, &effectiveCapacity);+  ml_.data_ = newRC->data_;+  ml_.size_ = size;+  ml_.setCapacity(effectiveCapacity, Category::isLarge);+  ml_.data_[size] = '\0';+}++template <class Char>+FOLLY_NOINLINE void fbstring_core<Char>::unshare(size_t minCapacity) {+  assert(category() == Category::isLarge);+  size_t effectiveCapacity = std::max(minCapacity, ml_.capacity());+  auto const newRC = RefCounted::create(&effectiveCapacity);+  // If this fails, someone placed the wrong capacity in an+  // fbstring.+  assert(effectiveCapacity >= ml_.capacity());+  // Also copies terminator.+  fbstring_detail::podCopy(ml_.data_, ml_.data_ + ml_.size_ + 1, newRC->data_);+  RefCounted::decrementRefs(ml_.data_);+  ml_.data_ = newRC->data_;+  ml_.setCapacity(effectiveCapacity, Category::isLarge);+  // size_ remains unchanged.+}++template <class Char>+inline Char* fbstring_core<Char>::mutableDataLarge() {+  assert(category() == Category::isLarge);+  if (RefCounted::refs(ml_.data_) > 1) { // Ensure unique.+    unshare();+  }+  return ml_.data_;+}++template <class Char>+FOLLY_NOINLINE void fbstring_core<Char>::reserveLarge(size_t minCapacity) {+  assert(category() == Category::isLarge);+  if (RefCounted::refs(ml_.data_) > 1) { // Ensure unique+    // We must make it unique regardless; in-place reallocation is+    // useless if the string is shared. In order to not surprise+    // people, reserve the new block at current capacity or+    // more. That way, a string's capacity never shrinks after a+    // call to reserve.+    unshare(minCapacity);+  } else {+    // String is not shared, so let's try to realloc (if needed)+    if (minCapacity > ml_.capacity()) {+      // Asking for more memory+      auto const newRC = RefCounted::reallocate(+          ml_.data_, ml_.size_, ml_.capacity(), &minCapacity);+      ml_.data_ = newRC->data_;+      ml_.setCapacity(minCapacity, Category::isLarge);+    }+    assert(capacity() >= minCapacity);+  }+}++template <class Char>+FOLLY_NOINLINE void fbstring_core<Char>::reserveMedium(+    const size_t minCapacity) {+  assert(category() == Category::isMedium);+  // String is not shared+  if (minCapacity <= ml_.capacity()) {+    return; // nothing to do, there's enough room+  }+  if (minCapacity <= maxMediumSize) {+    // Keep the string at medium size. Don't forget to allocate+    // one extra Char for the terminating null.+    size_t capacityBytes = goodMallocSize((1 + minCapacity) * sizeof(Char));+    // Also copies terminator.+    ml_.data_ = static_cast<Char*>(smartRealloc(+        ml_.data_,+        (ml_.size_ + 1) * sizeof(Char),+        (ml_.capacity() + 1) * sizeof(Char),+        capacityBytes));+    ml_.setCapacity(capacityBytes / sizeof(Char) - 1, Category::isMedium);+  } else {+    // Conversion from medium to large string+    fbstring_core nascent;+    // Will recurse to another branch of this function+    nascent.reserve(minCapacity);+    nascent.ml_.size_ = ml_.size_;+    // Also copies terminator.+    fbstring_detail::podCopy(+        ml_.data_, ml_.data_ + ml_.size_ + 1, nascent.ml_.data_);+    nascent.swap(*this);+    assert(capacity() >= minCapacity);+  }+}++template <class Char>+FOLLY_NOINLINE void fbstring_core<Char>::reserveSmall(+    size_t minCapacity, const bool disableSSO) {+  assert(category() == Category::isSmall);+  if (!disableSSO && minCapacity <= maxSmallSize) {+    // small+    // Nothing to do, everything stays put+  } else if (minCapacity <= maxMediumSize) {+    // medium+    // Don't forget to allocate one extra Char for the terminating null+    auto const allocSizeBytes =+        goodMallocSize((1 + minCapacity) * sizeof(Char));+    auto const pData = static_cast<Char*>(checkedMalloc(allocSizeBytes));+    auto const size = smallSize();+    // Also copies terminator.+    fbstring_detail::podCopy(small_, small_ + size + 1, pData);+    ml_.data_ = pData;+    ml_.size_ = size;+    ml_.setCapacity(allocSizeBytes / sizeof(Char) - 1, Category::isMedium);+  } else {+    // large+    auto const newRC = RefCounted::create(&minCapacity);+    auto const size = smallSize();+    // Also copies terminator.+    fbstring_detail::podCopy(small_, small_ + size + 1, newRC->data_);+    ml_.data_ = newRC->data_;+    ml_.size_ = size;+    ml_.setCapacity(minCapacity, Category::isLarge);+    assert(capacity() >= minCapacity);+  }+}++template <class Char>+inline Char* fbstring_core<Char>::expandNoinit(+    const size_t delta,+    bool expGrowth, /* = false */+    bool disableSSO /* = FBSTRING_DISABLE_SSO */) {+  // Strategy is simple: make room, then change size+  assert(capacity() >= size());+  size_t sz, newSz;+  if (category() == Category::isSmall) {+    sz = smallSize();+    newSz = sz + delta;+    if (!disableSSO && FOLLY_LIKELY(newSz <= maxSmallSize)) {+      setSmallSize(newSz);+      return small_ + sz;+    }+    reserveSmall(+        expGrowth ? std::max(newSz, 2 * maxSmallSize) : newSz, disableSSO);+  } else {+    sz = ml_.size_;+    newSz = sz + delta;+    if (FOLLY_UNLIKELY(newSz > capacity())) {+      // ensures not shared+      reserve(expGrowth ? std::max(newSz, 1 + capacity() * 3 / 2) : newSz);+    }+  }+  assert(capacity() >= newSz);+  // Category can't be small - we took care of that above+  assert(category() == Category::isMedium || category() == Category::isLarge);+  ml_.size_ = newSz;+  ml_.data_[newSz] = '\0';+  assert(size() == newSz);+  return ml_.data_ + sz;+}++template <class Char>+inline void fbstring_core<Char>::shrinkSmall(const size_t delta) {+  // Check for underflow+  assert(delta <= smallSize());+  setSmallSize(smallSize() - delta);+}++template <class Char>+inline void fbstring_core<Char>::shrinkMedium(const size_t delta) {+  // Medium strings and unique large strings need no special+  // handling.+  assert(ml_.size_ >= delta);+  ml_.size_ -= delta;+  ml_.data_[ml_.size_] = '\0';+}++template <class Char>+inline void fbstring_core<Char>::shrinkLarge(const size_t delta) {+  assert(ml_.size_ >= delta);+  // Shared large string, must make unique. This is because of the+  // durn terminator must be written, which may trample the shared+  // data.+  if (delta) {+    fbstring_core(ml_.data_, ml_.size_ - delta).swap(*this);+  }+  // No need to write the terminator.+}++/**+ * Dummy fbstring core that uses an actual std::string. This doesn't+ * make any sense - it's just for testing purposes.+ */+template <class Char>+class dummy_fbstring_core {+ public:+  dummy_fbstring_core() {}+  dummy_fbstring_core(const dummy_fbstring_core& another)+      : backend_(another.backend_) {}+  dummy_fbstring_core(const Char* s, size_t n) : backend_(s, n) {}+  void swap(dummy_fbstring_core& rhs) { backend_.swap(rhs.backend_); }+  const Char* data() const { return backend_.data(); }+  Char* mutableData() { return const_cast<Char*>(backend_.data()); }+  void shrink(size_t delta) {+    assert(delta <= size());+    backend_.resize(size() - delta);+  }+  Char* expandNoinit(size_t delta) {+    auto const sz = size();+    backend_.resize(size() + delta);+    return backend_.data() + sz;+  }+  void push_back(Char c) { backend_.push_back(c); }+  size_t size() const { return backend_.size(); }+  size_t capacity() const { return backend_.capacity(); }+  bool isShared() const { return false; }+  void reserve(size_t minCapacity) { backend_.reserve(minCapacity); }++ private:+  std::basic_string<Char> backend_;+};++/**+ * This is the basic_string replacement. For conformity,+ * basic_fbstring takes the same template parameters, plus the last+ * one which is the core.+ */+template <+    typename E,+    class T = std::char_traits<E>,+    class A = std::allocator<E>,+    class Storage = fbstring_core<E>>+class basic_fbstring {+  static_assert(+      std::is_same<A, std::allocator<E>>::value,+      "fbstring ignores custom allocators");++  template <typename Ex, typename... Args>+  FOLLY_ALWAYS_INLINE static void enforce(bool condition, Args&&... args) {+    if (!condition) {+      throw_exception<Ex>(static_cast<Args&&>(args)...);+    }+  }++  bool isSane() const {+    return begin() <= end() && empty() == (size() == 0) &&+        empty() == (begin() == end()) && size() <= max_size() &&+        capacity() <= max_size() && size() <= capacity() &&+        begin()[size()] == '\0';+  }++  struct Invariant {+    Invariant& operator=(const Invariant&) = delete;+    explicit Invariant(const basic_fbstring& s) noexcept : s_(s) {+      assert(s_.isSane());+    }+    ~Invariant() noexcept { assert(s_.isSane()); }++   private:+    const basic_fbstring& s_;+  };++ public:+  // types+  typedef T traits_type;+  typedef typename traits_type::char_type value_type;+  typedef A allocator_type;+  typedef typename std::allocator_traits<A>::size_type size_type;+  typedef typename std::allocator_traits<A>::difference_type difference_type;++  typedef typename std::allocator_traits<A>::value_type& reference;+  typedef typename std::allocator_traits<A>::value_type const& const_reference;+  typedef typename std::allocator_traits<A>::pointer pointer;+  typedef typename std::allocator_traits<A>::const_pointer const_pointer;++  typedef E* iterator;+  typedef const E* const_iterator;+  typedef std::reverse_iterator<iterator> reverse_iterator;+  typedef std::reverse_iterator<const_iterator> const_reverse_iterator;++  static constexpr size_type npos = size_type(-1);+  typedef std::true_type IsRelocatable;++ private:+  using string_view_type = std::basic_string_view<value_type, traits_type>;++  template <typename StringViewLike>+  static inline constexpr bool is_string_view_like_v =+      std::is_convertible_v<StringViewLike const&, string_view_type> &&+      !std::is_convertible_v<StringViewLike const&, const_pointer>;++  template <typename StringViewLike, typename Dummy>+  using if_is_string_view_like_t =+      std::enable_if_t<is_string_view_like_v<StringViewLike>, Dummy>;++  static void procrustes(size_type& n, size_type nmax) {+    if (n > nmax) {+      n = nmax;+    }+  }++  static size_type traitsLength(const value_type* s);++  struct string_view_ctor {};+  FOLLY_NOINLINE basic_fbstring(+      string_view_type view, const A&, string_view_ctor)+      : store_(view.data(), view.size()) {}++ public:+  // C++11 21.4.2 construct/copy/destroy++  // Note: while the following two constructors can be (and previously were)+  // collapsed into one constructor written this way:+  //+  //   explicit basic_fbstring(const A& a = A()) noexcept { }+  //+  // This can cause Clang (at least version 3.7) to fail with the error:+  //   "chosen constructor is explicit in copy-initialization ...+  //   in implicit initialization of field '(x)' with omitted initializer"+  //+  // if used in a struct which is default-initialized.  Hence the split into+  // these two separate constructors.++  basic_fbstring() noexcept : basic_fbstring(A()) {}+  /* implicit */ basic_fbstring(std::nullptr_t) = delete;++  explicit basic_fbstring(const A&) noexcept {}++  basic_fbstring(const basic_fbstring& str) : store_(str.store_) {}++  // Move constructor+  basic_fbstring(basic_fbstring&& goner) noexcept+      : store_(std::move(goner.store_)) {}++  // This is defined for compatibility with std::string+  template <typename A2>+  /* implicit */ basic_fbstring(const std::basic_string<E, T, A2>& str)+      : store_(str.data(), str.size()) {}++  basic_fbstring(+      const basic_fbstring& str,+      size_type pos,+      size_type n = npos,+      const A& /* a */ = A()) {+    assign(str, pos, n);+  }++  FOLLY_NOINLINE+  /* implicit */ basic_fbstring(const value_type* s, const A& /*a*/ = A())+      : store_(s, traitsLength(s)) {}++  FOLLY_NOINLINE+  basic_fbstring(const value_type* s, size_type n, const A& /*a*/ = A())+      : store_(s, n) {}++  FOLLY_NOINLINE+  basic_fbstring(size_type n, value_type c, const A& /*a*/ = A()) {+    auto const pData = store_.expandNoinit(n);+    fbstring_detail::podFill(pData, pData + n, c);+  }++  template <class InIt>+  FOLLY_NOINLINE basic_fbstring(+      InIt begin,+      InIt end,+      typename std::enable_if<+          !std::is_same<InIt, value_type*>::value,+          const A>::type& /*a*/+      = A()) {+    assign(begin, end);+  }++  // Specialization for const char*, const char*+  FOLLY_NOINLINE+  basic_fbstring(const value_type* b, const value_type* e, const A& /*a*/ = A())+      : store_(b, size_type(e - b)) {}++  // Nonstandard constructor+  basic_fbstring(+      value_type* s, size_type n, size_type c, AcquireMallocatedString a)+      : store_(s, n, c, a) {}++  // Construction from initialization list+  FOLLY_NOINLINE+  basic_fbstring(std::initializer_list<value_type> il) {+    assign(il.begin(), il.end());+  }++  template <+      typename StringViewLike,+      if_is_string_view_like_t<StringViewLike, int> = 0>+  explicit basic_fbstring(const StringViewLike& view, const A& a = A())+      : basic_fbstring(string_view_type(view), a, string_view_ctor{}) {}++  template <+      typename StringViewLike,+      if_is_string_view_like_t<StringViewLike, int> = 0>+  basic_fbstring(+      const StringViewLike& view, size_type pos, size_type n, const A& a = A())+      : basic_fbstring(+            string_view_type(view).substr(pos, n), a, string_view_ctor{}) {}++  ~basic_fbstring() noexcept {}++  basic_fbstring& operator=(const basic_fbstring& lhs);++  // Move assignment+  basic_fbstring& operator=(basic_fbstring&& goner) noexcept;++  // Compatibility with std::string+  template <typename A2>+  basic_fbstring& operator=(const std::basic_string<E, T, A2>& rhs) {+    return assign(rhs.data(), rhs.size());+  }++  // Compatibility with std::string+  std::basic_string<E, T, A> toStdString() const {+    return std::basic_string<E, T, A>(data(), size());+  }++  basic_fbstring& operator=(std::nullptr_t) = delete;++  basic_fbstring& operator=(const value_type* s) { return assign(s); }++  basic_fbstring& operator=(value_type c);++  // This actually goes directly against the C++ spec, but the+  // value_type overload is dangerous, so we're explicitly deleting+  // any overloads of operator= that could implicitly convert to+  // value_type.+  // Note that we do need to explicitly specify the template types because+  // otherwise MSVC 2017 will aggressively pre-resolve value_type to+  // traits_type::char_type, which won't compare as equal when determining+  // which overload the implementation is referring to.+  template <typename TP>+  typename std::enable_if<+      std::is_convertible<+          TP,+          typename basic_fbstring<E, T, A, Storage>::value_type>::value &&+          !std::is_same<+              typename std::decay<TP>::type,+              typename basic_fbstring<E, T, A, Storage>::value_type>::value,+      basic_fbstring<E, T, A, Storage>&>::type+  operator=(TP c) = delete;++  basic_fbstring& operator=(std::initializer_list<value_type> il) {+    return assign(il.begin(), il.end());+  }++  operator string_view_type() const noexcept { return {data(), size()}; }++  // C++11 21.4.3 iterators:+  iterator begin() { return store_.mutableData(); }++  const_iterator begin() const { return store_.data(); }++  const_iterator cbegin() const { return begin(); }++  iterator end() { return store_.mutableData() + store_.size(); }++  const_iterator end() const { return store_.data() + store_.size(); }++  const_iterator cend() const { return end(); }++  reverse_iterator rbegin() { return reverse_iterator(end()); }++  const_reverse_iterator rbegin() const {+    return const_reverse_iterator(end());+  }++  const_reverse_iterator crbegin() const { return rbegin(); }++  reverse_iterator rend() { return reverse_iterator(begin()); }++  const_reverse_iterator rend() const {+    return const_reverse_iterator(begin());+  }++  const_reverse_iterator crend() const { return rend(); }++  // Added by C++11+  // C++11 21.4.5, element access:+  const value_type& front() const { return *begin(); }+  const value_type& back() const {+    assert(!empty());+    // Should be begin()[size() - 1], but that branches twice+    return *(end() - 1);+  }+  value_type& front() { return *begin(); }+  value_type& back() {+    assert(!empty());+    // Should be begin()[size() - 1], but that branches twice+    return *(end() - 1);+  }+  void pop_back() {+    assert(!empty());+    store_.shrink(1);+  }++  // C++11 21.4.4 capacity:+  size_type size() const { return store_.size(); }++  size_type length() const { return size(); }++  size_type max_size() const { return std::numeric_limits<size_type>::max(); }++  void resize(size_type n, value_type c = value_type());++  size_type capacity() const { return store_.capacity(); }++  void reserve(size_type res_arg = 0) {+    enforce<std::length_error>(res_arg <= max_size(), "");+    store_.reserve(res_arg);+  }++  void shrink_to_fit() {+    // Shrink only if slack memory is sufficiently large+    if (capacity() < size() * 3 / 2) {+      return;+    }+    basic_fbstring(cbegin(), cend()).swap(*this);+  }++  void clear() { resize(0); }++  bool empty() const { return size() == 0; }++  // C++11 21.4.5 element access:+  const_reference operator[](size_type pos) const { return *(begin() + pos); }++  reference operator[](size_type pos) { return *(begin() + pos); }++  const_reference at(size_type n) const {+    enforce<std::out_of_range>(n < size(), "");+    return (*this)[n];+  }++  reference at(size_type n) {+    enforce<std::out_of_range>(n < size(), "");+    return (*this)[n];+  }++  // C++11 21.4.6 modifiers:+  basic_fbstring& operator+=(const basic_fbstring& str) { return append(str); }++  basic_fbstring& operator+=(const value_type* s) { return append(s); }++  basic_fbstring& operator+=(const value_type c) {+    push_back(c);+    return *this;+  }++  basic_fbstring& operator+=(std::initializer_list<value_type> il) {+    append(il);+    return *this;+  }++  template <+      typename StringViewLike,+      if_is_string_view_like_t<StringViewLike, int> = 0>+  basic_fbstring& operator+=(const StringViewLike& like) {+    append(like);+    return *this;+  }++  basic_fbstring& append(const basic_fbstring& str);++  basic_fbstring& append(+      const basic_fbstring& str, const size_type pos, size_type n);++  basic_fbstring& append(const value_type* s, size_type n);++  basic_fbstring& append(const value_type* s) {+    return append(s, traitsLength(s));+  }++  basic_fbstring& append(size_type n, value_type c);++  template <class InputIterator>+  basic_fbstring& append(InputIterator first, InputIterator last) {+    insert(end(), first, last);+    return *this;+  }++  basic_fbstring& append(std::initializer_list<value_type> il) {+    return append(il.begin(), il.end());+  }++  template <+      typename StringViewLike,+      if_is_string_view_like_t<StringViewLike, int> = 0>+  basic_fbstring& append(const StringViewLike& like) {+    string_view_type view = like;+    return append(view.begin(), view.end());+  }++  void push_back(const value_type c) { // primitive+    store_.push_back(c);+  }++  basic_fbstring& assign(const basic_fbstring& str) {+    if (&str == this) {+      return *this;+    }+    return assign(str.data(), str.size());+  }++  basic_fbstring& assign(basic_fbstring&& str) {+    return *this = std::move(str);+  }++  basic_fbstring& assign(+      const basic_fbstring& str, const size_type pos, size_type n);++  basic_fbstring& assign(const value_type* s, const size_type n);++  basic_fbstring& assign(const value_type* s) {+    return assign(s, traitsLength(s));+  }++  basic_fbstring& assign(std::initializer_list<value_type> il) {+    return assign(il.begin(), il.end());+  }++  template <class ItOrLength, class ItOrChar>+  basic_fbstring& assign(ItOrLength first_or_n, ItOrChar last_or_c) {+    return replace(begin(), end(), first_or_n, last_or_c);+  }++  basic_fbstring& insert(size_type pos1, const basic_fbstring& str) {+    return insert(pos1, str.data(), str.size());+  }++  basic_fbstring& insert(+      size_type pos1, const basic_fbstring& str, size_type pos2, size_type n) {+    enforce<std::out_of_range>(pos2 <= str.length(), "");+    procrustes(n, str.length() - pos2);+    return insert(pos1, str.data() + pos2, n);+  }++  basic_fbstring& insert(size_type pos, const value_type* s, size_type n) {+    enforce<std::out_of_range>(pos <= length(), "");+    insert(begin() + pos, s, s + n);+    return *this;+  }++  basic_fbstring& insert(size_type pos, const value_type* s) {+    return insert(pos, s, traitsLength(s));+  }++  basic_fbstring& insert(size_type pos, size_type n, value_type c) {+    enforce<std::out_of_range>(pos <= length(), "");+    insert(begin() + pos, n, c);+    return *this;+  }++  iterator insert(const_iterator p, const value_type c) {+    const size_type pos = p - cbegin();+    insert(p, 1, c);+    return begin() + pos;+  }++ private:+  typedef std::basic_istream<value_type, traits_type> istream_type;+  istream_type& getlineImpl(istream_type& is, value_type delim);++ public:+  friend inline istream_type& getline(+      istream_type& is, basic_fbstring& str, value_type delim) {+    return str.getlineImpl(is, delim);+  }++  friend inline istream_type& getline(istream_type& is, basic_fbstring& str) {+    return getline(is, str, '\n');+  }++ private:+  iterator insertImplDiscr(+      const_iterator i, size_type n, value_type c, std::true_type);++  template <class InputIter>+  iterator insertImplDiscr(+      const_iterator i, InputIter b, InputIter e, std::false_type);++  template <class FwdIterator>+  iterator insertImpl(+      const_iterator i,+      FwdIterator s1,+      FwdIterator s2,+      std::forward_iterator_tag);++  template <class InputIterator>+  iterator insertImpl(+      const_iterator i,+      InputIterator b,+      InputIterator e,+      std::input_iterator_tag);++ public:+  template <class ItOrLength, class ItOrChar>+  iterator insert(const_iterator p, ItOrLength first_or_n, ItOrChar last_or_c) {+    using Sel =+        std::bool_constant<std::numeric_limits<ItOrLength>::is_specialized>;+    return insertImplDiscr(p, first_or_n, last_or_c, Sel());+  }++  iterator insert(const_iterator p, std::initializer_list<value_type> il) {+    return insert(p, il.begin(), il.end());+  }++  basic_fbstring& erase(size_type pos = 0, size_type n = npos) {+    Invariant checker(*this);++    enforce<std::out_of_range>(pos <= length(), "");+    procrustes(n, length() - pos);+    std::copy(begin() + pos + n, end(), begin() + pos);+    resize(length() - n);+    return *this;+  }++  iterator erase(iterator position) {+    const size_type pos(position - begin());+    enforce<std::out_of_range>(pos <= size(), "");+    erase(pos, 1);+    return begin() + pos;+  }++  iterator erase(iterator first, iterator last) {+    const size_type pos(first - begin());+    erase(pos, last - first);+    return begin() + pos;+  }++  // Replaces at most n1 chars of *this, starting with pos1 with the+  // content of str+  basic_fbstring& replace(+      size_type pos1, size_type n1, const basic_fbstring& str) {+    return replace(pos1, n1, str.data(), str.size());+  }++  // Replaces at most n1 chars of *this, starting with pos1,+  // with at most n2 chars of str starting with pos2+  basic_fbstring& replace(+      size_type pos1,+      size_type n1,+      const basic_fbstring& str,+      size_type pos2,+      size_type n2) {+    enforce<std::out_of_range>(pos2 <= str.length(), "");+    return replace(+        pos1, n1, str.data() + pos2, std::min(n2, str.size() - pos2));+  }++  // Replaces at most n1 chars of *this, starting with pos, with chars from s+  basic_fbstring& replace(size_type pos, size_type n1, const value_type* s) {+    return replace(pos, n1, s, traitsLength(s));+  }++  // Replaces at most n1 chars of *this, starting with pos, with n2+  // occurrences of c+  //+  // consolidated with+  //+  // Replaces at most n1 chars of *this, starting with pos, with at+  // most n2 chars of str.  str must have at least n2 chars.+  template <class StrOrLength, class NumOrChar>+  basic_fbstring& replace(+      size_type pos, size_type n1, StrOrLength s_or_n2, NumOrChar n_or_c) {+    Invariant checker(*this);++    enforce<std::out_of_range>(pos <= size(), "");+    procrustes(n1, length() - pos);+    const iterator b = begin() + pos;+    return replace(b, b + n1, s_or_n2, n_or_c);+  }++  basic_fbstring& replace(iterator i1, iterator i2, const basic_fbstring& str) {+    return replace(i1, i2, str.data(), str.length());+  }++  basic_fbstring& replace(iterator i1, iterator i2, const value_type* s) {+    return replace(i1, i2, s, traitsLength(s));+  }++ private:+  basic_fbstring& replaceImplDiscr(+      iterator i1,+      iterator i2,+      const value_type* s,+      size_type n,+      std::integral_constant<int, 2>);++  basic_fbstring& replaceImplDiscr(+      iterator i1,+      iterator i2,+      size_type n2,+      value_type c,+      std::integral_constant<int, 1>);++  template <class InputIter>+  basic_fbstring& replaceImplDiscr(+      iterator i1,+      iterator i2,+      InputIter b,+      InputIter e,+      std::integral_constant<int, 0>);++ private:+  template <class FwdIterator>+  bool replaceAliased(+      iterator /* i1 */,+      iterator /* i2 */,+      FwdIterator /* s1 */,+      FwdIterator /* s2 */,+      std::false_type) {+    return false;+  }++  template <class FwdIterator>+  bool replaceAliased(+      iterator i1, iterator i2, FwdIterator s1, FwdIterator s2, std::true_type);++  template <class FwdIterator>+  void replaceImpl(+      iterator i1,+      iterator i2,+      FwdIterator s1,+      FwdIterator s2,+      std::forward_iterator_tag);++  template <class InputIterator>+  void replaceImpl(+      iterator i1,+      iterator i2,+      InputIterator b,+      InputIterator e,+      std::input_iterator_tag);++ public:+  template <class T1, class T2>+  basic_fbstring& replace(+      iterator i1, iterator i2, T1 first_or_n_or_s, T2 last_or_c_or_n) {+    constexpr bool num1 = std::numeric_limits<T1>::is_specialized,+                   num2 = std::numeric_limits<T2>::is_specialized;+    using Sel =+        std::integral_constant<int, num1 ? (num2 ? 1 : -1) : (num2 ? 2 : 0)>;+    return replaceImplDiscr(i1, i2, first_or_n_or_s, last_or_c_or_n, Sel());+  }++  size_type copy(value_type* s, size_type n, size_type pos = 0) const {+    enforce<std::out_of_range>(pos <= size(), "");+    procrustes(n, size() - pos);++    if (n != 0) {+      fbstring_detail::podCopy(data() + pos, data() + pos + n, s);+    }+    return n;+  }++  void swap(basic_fbstring& rhs) { store_.swap(rhs.store_); }++  const value_type* c_str() const { return store_.c_str(); }++  const value_type* data() const { return c_str(); }++  value_type* data() { return store_.data(); }++  allocator_type get_allocator() const { return allocator_type(); }++  size_type find(const basic_fbstring& str, size_type pos = 0) const {+    return find(str.data(), pos, str.length());+  }++  size_type find(+      const value_type* needle, size_type pos, size_type nsize) const;++  size_type find(const value_type* s, size_type pos = 0) const {+    return find(s, pos, traitsLength(s));+  }++  size_type find(value_type c, size_type pos = 0) const {+    return find(&c, pos, 1);+  }++  size_type rfind(const basic_fbstring& str, size_type pos = npos) const {+    return rfind(str.data(), pos, str.length());+  }++  size_type rfind(const value_type* s, size_type pos, size_type n) const;++  size_type rfind(const value_type* s, size_type pos = npos) const {+    return rfind(s, pos, traitsLength(s));+  }++  size_type rfind(value_type c, size_type pos = npos) const {+    return rfind(&c, pos, 1);+  }++  size_type find_first_of(const basic_fbstring& str, size_type pos = 0) const {+    return find_first_of(str.data(), pos, str.length());+  }++  size_type find_first_of(+      const value_type* s, size_type pos, size_type n) const;++  size_type find_first_of(const value_type* s, size_type pos = 0) const {+    return find_first_of(s, pos, traitsLength(s));+  }++  size_type find_first_of(value_type c, size_type pos = 0) const {+    return find_first_of(&c, pos, 1);+  }++  size_type find_last_of(+      const basic_fbstring& str, size_type pos = npos) const {+    return find_last_of(str.data(), pos, str.length());+  }++  size_type find_last_of(const value_type* s, size_type pos, size_type n) const;++  size_type find_last_of(const value_type* s, size_type pos = npos) const {+    return find_last_of(s, pos, traitsLength(s));+  }++  size_type find_last_of(value_type c, size_type pos = npos) const {+    return find_last_of(&c, pos, 1);+  }++  size_type find_first_not_of(+      const basic_fbstring& str, size_type pos = 0) const {+    return find_first_not_of(str.data(), pos, str.size());+  }++  size_type find_first_not_of(+      const value_type* s, size_type pos, size_type n) const;++  size_type find_first_not_of(const value_type* s, size_type pos = 0) const {+    return find_first_not_of(s, pos, traitsLength(s));+  }++  size_type find_first_not_of(value_type c, size_type pos = 0) const {+    return find_first_not_of(&c, pos, 1);+  }++  size_type find_last_not_of(+      const basic_fbstring& str, size_type pos = npos) const {+    return find_last_not_of(str.data(), pos, str.length());+  }++  size_type find_last_not_of(+      const value_type* s, size_type pos, size_type n) const;++  size_type find_last_not_of(const value_type* s, size_type pos = npos) const {+    return find_last_not_of(s, pos, traitsLength(s));+  }++  size_type find_last_not_of(value_type c, size_type pos = npos) const {+    return find_last_not_of(&c, pos, 1);+  }++  basic_fbstring substr(size_type pos = 0, size_type n = npos) const& {+    enforce<std::out_of_range>(pos <= size(), "");+    return basic_fbstring(data() + pos, std::min(n, size() - pos));+  }++  basic_fbstring substr(size_type pos = 0, size_type n = npos) && {+    enforce<std::out_of_range>(pos <= size(), "");+    erase(0, pos);+    if (n < size()) {+      resize(n);+    }+    return std::move(*this);+  }++  int compare(const basic_fbstring& str) const {+    // FIX due to Goncalo N M de Carvalho July 18, 2005+    return compare(0, size(), str);+  }++  int compare(size_type pos1, size_type n1, const basic_fbstring& str) const {+    return compare(pos1, n1, str.data(), str.size());+  }++  int compare(size_type pos1, size_type n1, const value_type* s) const {+    return compare(pos1, n1, s, traitsLength(s));+  }++  int compare(+      size_type pos1, size_type n1, const value_type* s, size_type n2) const {+    enforce<std::out_of_range>(pos1 <= size(), "");+    procrustes(n1, size() - pos1);+    // The line below fixed by Jean-Francois Bastien, 04-23-2007. Thanks!+    const int r = traits_type::compare(pos1 + data(), s, std::min(n1, n2));+    return r != 0 ? r : n1 > n2 ? 1 : n1 < n2 ? -1 : 0;+  }++  int compare(+      size_type pos1,+      size_type n1,+      const basic_fbstring& str,+      size_type pos2,+      size_type n2) const {+    enforce<std::out_of_range>(pos2 <= str.size(), "");+    return compare(+        pos1, n1, str.data() + pos2, std::min(n2, str.size() - pos2));+  }++  // Code from Jean-Francois Bastien (03/26/2007)+  int compare(const value_type* s) const {+    // Could forward to compare(0, size(), s, traitsLength(s))+    // but that does two extra checks+    const size_type n1(size()), n2(traitsLength(s));+    const int r = traits_type::compare(data(), s, std::min(n1, n2));+    return r != 0 ? r : n1 > n2 ? 1 : n1 < n2 ? -1 : 0;+  }++#if FOLLY_CPLUSPLUS >= 202002L+  friend auto operator<=>(+      const basic_fbstring& lhs, const basic_fbstring& rhs) {+    return lhs.spaceship(rhs.data(), rhs.size());+  }+  friend auto operator<=>(const basic_fbstring& lhs, const char* rhs) {+    return lhs.spaceship(rhs, traitsLength(rhs));+  }+  template <typename A2>+  friend auto operator<=>(+      const basic_fbstring& lhs, const std::basic_string<E, T, A2>& rhs) {+    return lhs.spaceship(rhs.data(), rhs.size());+  }+#endif // FOLLY_CPLUSPLUS >= 202002L++ private:+#if FOLLY_CPLUSPLUS >= 202002L+  auto spaceship(const value_type* rhsData, size_type rhsSize) const {+    auto c = compare(0, size(), rhsData, rhsSize);+    if (c == 0) {+      return std::strong_ordering::equal;+    } else if (c < 0) {+      return std::strong_ordering::less;+    } else {+      return std::strong_ordering::greater;+    }+  }+#endif // FOLLY_CPLUSPLUS >= 202002L++  // Data+  Storage store_;+};++template <typename E, class T, class A, class S>+FOLLY_NOINLINE typename basic_fbstring<E, T, A, S>::size_type+basic_fbstring<E, T, A, S>::traitsLength(const value_type* s) {+  return s+      ? traits_type::length(s)+      : (throw_exception<std::logic_error>(+             "basic_fbstring: null pointer initializer not valid"),+         0);+}++template <typename E, class T, class A, class S>+inline basic_fbstring<E, T, A, S>& basic_fbstring<E, T, A, S>::operator=(+    const basic_fbstring& lhs) {+  Invariant checker(*this);++  if (FOLLY_UNLIKELY(&lhs == this)) {+    return *this;+  }++  return assign(lhs.data(), lhs.size());+}++// Move assignment+template <typename E, class T, class A, class S>+inline basic_fbstring<E, T, A, S>& basic_fbstring<E, T, A, S>::operator=(+    basic_fbstring&& goner) noexcept {+  if (FOLLY_UNLIKELY(&goner == this)) {+    // Compatibility with std::basic_string<>,+    // C++11 21.4.2 [string.cons] / 23 requires self-move-assignment support.+    return *this;+  }+  // No need of this anymore+  this->~basic_fbstring();+  // Move the goner into this+  new (&store_) S(std::move(goner.store_));+  return *this;+}++template <typename E, class T, class A, class S>+inline basic_fbstring<E, T, A, S>& basic_fbstring<E, T, A, S>::operator=(+    value_type c) {+  Invariant checker(*this);++  if (empty()) {+    store_.expandNoinit(1);+  } else if (store_.isShared()) {+    basic_fbstring(1, c).swap(*this);+    return *this;+  } else {+    store_.shrink(size() - 1);+  }+  front() = c;+  return *this;+}++template <typename E, class T, class A, class S>+inline void basic_fbstring<E, T, A, S>::resize(+    const size_type n, const value_type c /*= value_type()*/) {+  Invariant checker(*this);++  auto size = this->size();+  if (n <= size) {+    store_.shrink(size - n);+  } else {+    auto const delta = n - size;+    auto pData = store_.expandNoinit(delta);+    fbstring_detail::podFill(pData, pData + delta, c);+  }+  assert(this->size() == n);+}++template <typename E, class T, class A, class S>+inline basic_fbstring<E, T, A, S>& basic_fbstring<E, T, A, S>::append(+    const basic_fbstring& str) {+#ifndef NDEBUG+  auto desiredSize = size() + str.size();+#endif+  append(str.data(), str.size());+  assert(size() == desiredSize);+  return *this;+}++template <typename E, class T, class A, class S>+inline basic_fbstring<E, T, A, S>& basic_fbstring<E, T, A, S>::append(+    const basic_fbstring& str, const size_type pos, size_type n) {+  const size_type sz = str.size();+  enforce<std::out_of_range>(pos <= sz, "");+  procrustes(n, sz - pos);+  return append(str.data() + pos, n);+}++template <typename E, class T, class A, class S>+FOLLY_NOINLINE basic_fbstring<E, T, A, S>& basic_fbstring<E, T, A, S>::append(+    const value_type* s, size_type n) {+  Invariant checker(*this);++  if (FOLLY_UNLIKELY(!n)) {+    // Unlikely but must be done+    return *this;+  }+  auto const oldSize = size();+  auto const oldData = data();+  auto pData = store_.expandNoinit(n, /* expGrowth = */ true);++  // Check for aliasing (rare). We could use "<=" here but in theory+  // those do not work for pointers unless the pointers point to+  // elements in the same array. For that reason we use+  // std::less_equal, which is guaranteed to offer a total order+  // over pointers. See discussion at http://goo.gl/Cy2ya for more+  // info.+  std::less_equal<const value_type*> le;+  if (FOLLY_UNLIKELY(le(oldData, s) && !le(oldData + oldSize, s))) {+    assert(le(s + n, oldData + oldSize));+    // expandNoinit() could have moved the storage, restore the source.+    s = data() + (s - oldData);+    fbstring_detail::podMove(s, s + n, pData);+  } else {+    fbstring_detail::podCopy(s, s + n, pData);+  }++  assert(size() == oldSize + n);+  return *this;+}++template <typename E, class T, class A, class S>+inline basic_fbstring<E, T, A, S>& basic_fbstring<E, T, A, S>::append(+    size_type n, value_type c) {+  Invariant checker(*this);+  auto pData = store_.expandNoinit(n, /* expGrowth = */ true);+  fbstring_detail::podFill(pData, pData + n, c);+  return *this;+}++template <typename E, class T, class A, class S>+inline basic_fbstring<E, T, A, S>& basic_fbstring<E, T, A, S>::assign(+    const basic_fbstring& str, const size_type pos, size_type n) {+  const size_type sz = str.size();+  enforce<std::out_of_range>(pos <= sz, "");+  procrustes(n, sz - pos);+  return assign(str.data() + pos, n);+}++template <typename E, class T, class A, class S>+FOLLY_NOINLINE basic_fbstring<E, T, A, S>& basic_fbstring<E, T, A, S>::assign(+    const value_type* s, const size_type n) {+  Invariant checker(*this);++  if (n == 0) {+    resize(0);+  } else if (size() >= n) {+    // s can alias this, we need to use podMove.+    fbstring_detail::podMove(s, s + n, store_.mutableData());+    store_.shrink(size() - n);+    assert(size() == n);+  } else {+    // If n is larger than size(), s cannot alias this string's+    // storage.+    resize(0);+    // Do not use exponential growth here: assign() should be tight,+    // to mirror the behavior of the equivalent constructor.+    fbstring_detail::podCopy(s, s + n, store_.expandNoinit(n));+  }++  assert(size() == n);+  return *this;+}++template <typename E, class T, class A, class S>+inline typename basic_fbstring<E, T, A, S>::istream_type&+basic_fbstring<E, T, A, S>::getlineImpl(istream_type& is, value_type delim) {+  Invariant checker(*this);++  clear();+  size_t size = 0;+  while (true) {+    size_t avail = capacity() - size;+    // fbstring has 1 byte extra capacity for the null terminator,+    // and getline null-terminates the read string.+    is.getline(store_.expandNoinit(avail), avail + 1, delim);+    size += is.gcount();++    if (is.bad() || is.eof() || !is.fail()) {+      // Done by either failure, end of file, or normal read.+      if (!is.bad() && !is.eof()) {+        --size; // gcount() also accounts for the delimiter.+      }+      resize(size);+      break;+    }++    assert(size == this->size());+    assert(size == capacity());+    // Start at minimum allocation 63 + terminator = 64.+    reserve(std::max<size_t>(63, 3 * size / 2));+    // Clear the error so we can continue reading.+    is.clear();+  }+  return is;+}++template <typename E, class T, class A, class S>+inline typename basic_fbstring<E, T, A, S>::size_type+basic_fbstring<E, T, A, S>::find(+    const value_type* needle,+    const size_type pos,+    const size_type nsize) const {+  auto const size = this->size();+  // nsize + pos can overflow (eg pos == npos), guard against that by checking+  // that nsize + pos does not wrap around.+  if (nsize + pos > size || nsize + pos < pos) {+    return npos;+  }++  if (nsize == 0) {+    return pos;+  }+  // Don't use std::search, use a Boyer-Moore-like trick by comparing+  // the last characters first+  auto const haystack = data();+  auto const nsize_1 = nsize - 1;+  auto const lastNeedle = needle[nsize_1];++  // Boyer-Moore skip value for the last char in the needle. Zero is+  // not a valid value; skip will be computed the first time it's+  // needed.+  size_type skip = 0;++  const E* i = haystack + pos;+  auto iEnd = haystack + size - nsize_1;++  while (i < iEnd) {+    // Boyer-Moore: match the last element in the needle+    while (i[nsize_1] != lastNeedle) {+      if (++i == iEnd) {+        // not found+        return npos;+      }+    }+    // Here we know that the last char matches+    // Continue in pedestrian mode+    for (size_t j = 0;;) {+      assert(j < nsize);+      if (i[j] != needle[j]) {+        // Not found, we can skip+        // Compute the skip value lazily+        if (skip == 0) {+          skip = 1;+          while (skip <= nsize_1 && needle[nsize_1 - skip] != lastNeedle) {+            ++skip;+          }+        }+        i += skip;+        break;+      }+      // Check if done searching+      if (++j == nsize) {+        // Yay+        return i - haystack;+      }+    }+  }+  return npos;+}++template <typename E, class T, class A, class S>+inline typename basic_fbstring<E, T, A, S>::iterator+basic_fbstring<E, T, A, S>::insertImplDiscr(+    const_iterator i, size_type n, value_type c, std::true_type) {+  Invariant checker(*this);++  assert(i >= cbegin() && i <= cend());+  const size_type pos = i - cbegin();++  auto oldSize = size();+  store_.expandNoinit(n, /* expGrowth = */ true);+  auto b = begin();+  fbstring_detail::podMove(b + pos, b + oldSize, b + pos + n);+  fbstring_detail::podFill(b + pos, b + pos + n, c);++  return b + pos;+}++template <typename E, class T, class A, class S>+template <class InputIter>+inline typename basic_fbstring<E, T, A, S>::iterator+basic_fbstring<E, T, A, S>::insertImplDiscr(+    const_iterator i, InputIter b, InputIter e, std::false_type) {+  return insertImpl(+      i, b, e, typename std::iterator_traits<InputIter>::iterator_category());+}++template <typename E, class T, class A, class S>+template <class FwdIterator>+inline typename basic_fbstring<E, T, A, S>::iterator+basic_fbstring<E, T, A, S>::insertImpl(+    const_iterator i,+    FwdIterator s1,+    FwdIterator s2,+    std::forward_iterator_tag) {+  Invariant checker(*this);++  assert(i >= cbegin() && i <= cend());+  const size_type pos = i - cbegin();+  auto n = std::distance(s1, s2);+  assert(n >= 0);++  auto oldSize = size();+  store_.expandNoinit(n, /* expGrowth = */ true);+  auto b = begin();+  fbstring_detail::podMove(b + pos, b + oldSize, b + pos + n);+  std::copy(s1, s2, b + pos);++  return b + pos;+}++template <typename E, class T, class A, class S>+template <class InputIterator>+inline typename basic_fbstring<E, T, A, S>::iterator+basic_fbstring<E, T, A, S>::insertImpl(+    const_iterator i,+    InputIterator b,+    InputIterator e,+    std::input_iterator_tag) {+  const auto pos = i - cbegin();+  basic_fbstring temp(cbegin(), i);+  for (; b != e; ++b) {+    temp.push_back(*b);+  }+  temp.append(i, cend());+  swap(temp);+  return begin() + pos;+}++template <typename E, class T, class A, class S>+inline basic_fbstring<E, T, A, S>& basic_fbstring<E, T, A, S>::replaceImplDiscr(+    iterator i1,+    iterator i2,+    const value_type* s,+    size_type n,+    std::integral_constant<int, 2>) {+  assert(i1 <= i2);+  assert(begin() <= i1 && i1 <= end());+  assert(begin() <= i2 && i2 <= end());+  return replace(i1, i2, s, s + n);+}++template <typename E, class T, class A, class S>+inline basic_fbstring<E, T, A, S>& basic_fbstring<E, T, A, S>::replaceImplDiscr(+    iterator i1,+    iterator i2,+    size_type n2,+    value_type c,+    std::integral_constant<int, 1>) {+  const size_type n1 = i2 - i1;+  if (n1 > n2) {+    std::fill(i1, i1 + n2, c);+    erase(i1 + n2, i2);+  } else {+    std::fill(i1, i2, c);+    insert(i2, n2 - n1, c);+  }+  assert(isSane());+  return *this;+}++template <typename E, class T, class A, class S>+template <class InputIter>+inline basic_fbstring<E, T, A, S>& basic_fbstring<E, T, A, S>::replaceImplDiscr(+    iterator i1,+    iterator i2,+    InputIter b,+    InputIter e,+    std::integral_constant<int, 0>) {+  using Cat = typename std::iterator_traits<InputIter>::iterator_category;+  replaceImpl(i1, i2, b, e, Cat());+  return *this;+}++template <typename E, class T, class A, class S>+template <class FwdIterator>+inline bool basic_fbstring<E, T, A, S>::replaceAliased(+    iterator i1, iterator i2, FwdIterator s1, FwdIterator s2, std::true_type) {+  std::less_equal<const value_type*> le{};+  const bool aliased = le(&*begin(), &*s1) && le(&*s1, &*end());+  if (!aliased) {+    return false;+  }+  // Aliased replace, copy to new string+  basic_fbstring temp;+  temp.reserve(size() - (i2 - i1) + std::distance(s1, s2));+  temp.append(begin(), i1).append(s1, s2).append(i2, end());+  swap(temp);+  return true;+}++template <typename E, class T, class A, class S>+template <class FwdIterator>+inline void basic_fbstring<E, T, A, S>::replaceImpl(+    iterator i1,+    iterator i2,+    FwdIterator s1,+    FwdIterator s2,+    std::forward_iterator_tag) {+  Invariant checker(*this);++  // Handle aliased replace+  using Sel = std::bool_constant<+      std::is_same<FwdIterator, iterator>::value ||+      std::is_same<FwdIterator, const_iterator>::value>;+  if (replaceAliased(i1, i2, s1, s2, Sel())) {+    return;+  }++  auto const n1 = i2 - i1;+  assert(n1 >= 0);+  auto const n2 = std::distance(s1, s2);+  assert(n2 >= 0);++  if (n1 > n2) {+    // shrinks+    std::copy(s1, s2, i1);+    erase(i1 + n2, i2);+  } else {+    // grows+    s1 = fbstring_detail::copy_n(s1, n1, i1).first;+    insert(i2, s1, s2);+  }+  assert(isSane());+}++template <typename E, class T, class A, class S>+template <class InputIterator>+inline void basic_fbstring<E, T, A, S>::replaceImpl(+    iterator i1,+    iterator i2,+    InputIterator b,+    InputIterator e,+    std::input_iterator_tag) {+  basic_fbstring temp(begin(), i1);+  temp.append(b, e).append(i2, end());+  swap(temp);+}++template <typename E, class T, class A, class S>+inline typename basic_fbstring<E, T, A, S>::size_type+basic_fbstring<E, T, A, S>::rfind(+    const value_type* s, size_type pos, size_type n) const {+  if (n > length()) {+    return npos;+  }+  pos = std::min(pos, length() - n);+  if (n == 0) {+    return pos;+  }++  const_iterator i(begin() + pos);+  for (;; --i) {+    if (traits_type::eq(*i, *s) && traits_type::compare(&*i, s, n) == 0) {+      return i - begin();+    }+    if (i == begin()) {+      break;+    }+  }+  return npos;+}++template <typename E, class T, class A, class S>+inline typename basic_fbstring<E, T, A, S>::size_type+basic_fbstring<E, T, A, S>::find_first_of(+    const value_type* s, size_type pos, size_type n) const {+  if (pos > length() || n == 0) {+    return npos;+  }+  const_iterator i(begin() + pos), finish(end());+  for (; i != finish; ++i) {+    if (traits_type::find(s, n, *i) != nullptr) {+      return i - begin();+    }+  }+  return npos;+}++template <typename E, class T, class A, class S>+inline typename basic_fbstring<E, T, A, S>::size_type+basic_fbstring<E, T, A, S>::find_last_of(+    const value_type* s, size_type pos, size_type n) const {+  if (!empty() && n > 0) {+    pos = std::min(pos, length() - 1);+    const_iterator i(begin() + pos);+    for (;; --i) {+      if (traits_type::find(s, n, *i) != nullptr) {+        return i - begin();+      }+      if (i == begin()) {+        break;+      }+    }+  }+  return npos;+}++template <typename E, class T, class A, class S>+inline typename basic_fbstring<E, T, A, S>::size_type+basic_fbstring<E, T, A, S>::find_first_not_of(+    const value_type* s, size_type pos, size_type n) const {+  if (pos < length()) {+    const_iterator i(begin() + pos), finish(end());+    for (; i != finish; ++i) {+      if (traits_type::find(s, n, *i) == nullptr) {+        return i - begin();+      }+    }+  }+  return npos;+}++template <typename E, class T, class A, class S>+inline typename basic_fbstring<E, T, A, S>::size_type+basic_fbstring<E, T, A, S>::find_last_not_of(+    const value_type* s, size_type pos, size_type n) const {+  if (!this->empty()) {+    pos = std::min(pos, size() - 1);+    const_iterator i(begin() + pos);+    for (;; --i) {+      if (traits_type::find(s, n, *i) == nullptr) {+        return i - begin();+      }+      if (i == begin()) {+        break;+      }+    }+  }+  return npos;+}++// non-member functions+// C++11 21.4.8.1/1+template <typename E, class T, class A, class S>+inline basic_fbstring<E, T, A, S> operator+(+    const basic_fbstring<E, T, A, S>& lhs,+    const basic_fbstring<E, T, A, S>& rhs) {+  basic_fbstring<E, T, A, S> result;+  result.reserve(lhs.size() + rhs.size());+  result.append(lhs).append(rhs);+  return result;+}++// C++11 21.4.8.1/2+template <typename E, class T, class A, class S>+inline basic_fbstring<E, T, A, S> operator+(+    basic_fbstring<E, T, A, S>&& lhs, const basic_fbstring<E, T, A, S>& rhs) {+  return std::move(lhs.append(rhs));+}++// C++11 21.4.8.1/3+template <typename E, class T, class A, class S>+inline basic_fbstring<E, T, A, S> operator+(+    const basic_fbstring<E, T, A, S>& lhs, basic_fbstring<E, T, A, S>&& rhs) {+  if (rhs.capacity() >= lhs.size() + rhs.size()) {+    // Good, at least we don't need to reallocate+    return std::move(rhs.insert(0, lhs));+  }+  // Meh, no go. Forward to operator+(const&, const&).+  auto const& rhsC = rhs;+  return lhs + rhsC;+}++// C++11 21.4.8.1/4+template <typename E, class T, class A, class S>+inline basic_fbstring<E, T, A, S> operator+(+    basic_fbstring<E, T, A, S>&& lhs, basic_fbstring<E, T, A, S>&& rhs) {+  return std::move(lhs.append(rhs));+}++// C++11 21.4.8.1/5+template <typename E, class T, class A, class S>+inline basic_fbstring<E, T, A, S> operator+(+    const E* lhs, const basic_fbstring<E, T, A, S>& rhs) {+  //+  basic_fbstring<E, T, A, S> result;+  const auto len = basic_fbstring<E, T, A, S>::traits_type::length(lhs);+  result.reserve(len + rhs.size());+  result.append(lhs, len).append(rhs);+  return result;+}++// C++11 21.4.8.1/6+template <typename E, class T, class A, class S>+inline basic_fbstring<E, T, A, S> operator+(+    const E* lhs, basic_fbstring<E, T, A, S>&& rhs) {+  //+  const auto len = basic_fbstring<E, T, A, S>::traits_type::length(lhs);+  if (rhs.capacity() >= len + rhs.size()) {+    // Good, at least we don't need to reallocate+    rhs.insert(rhs.begin(), lhs, lhs + len);+    return std::move(rhs);+  }+  // Meh, no go. Do it by hand since we have len already.+  basic_fbstring<E, T, A, S> result;+  result.reserve(len + rhs.size());+  result.append(lhs, len).append(rhs);+  return result;+}++// C++11 21.4.8.1/7+template <typename E, class T, class A, class S>+inline basic_fbstring<E, T, A, S> operator+(+    E lhs, const basic_fbstring<E, T, A, S>& rhs) {+  basic_fbstring<E, T, A, S> result;+  result.reserve(1 + rhs.size());+  result.push_back(lhs);+  result.append(rhs);+  return result;+}++// C++11 21.4.8.1/8+template <typename E, class T, class A, class S>+inline basic_fbstring<E, T, A, S> operator+(+    E lhs, basic_fbstring<E, T, A, S>&& rhs) {+  //+  if (rhs.capacity() > rhs.size()) {+    // Good, at least we don't need to reallocate+    rhs.insert(rhs.begin(), lhs);+    return std::move(rhs);+  }+  // Meh, no go. Forward to operator+(E, const&).+  auto const& rhsC = rhs;+  return lhs + rhsC;+}++// C++11 21.4.8.1/9+template <typename E, class T, class A, class S>+inline basic_fbstring<E, T, A, S> operator+(+    const basic_fbstring<E, T, A, S>& lhs, const E* rhs) {+  typedef typename basic_fbstring<E, T, A, S>::size_type size_type;+  typedef typename basic_fbstring<E, T, A, S>::traits_type traits_type;++  basic_fbstring<E, T, A, S> result;+  const size_type len = traits_type::length(rhs);+  result.reserve(lhs.size() + len);+  result.append(lhs).append(rhs, len);+  return result;+}++// C++11 21.4.8.1/10+template <typename E, class T, class A, class S>+inline basic_fbstring<E, T, A, S> operator+(+    basic_fbstring<E, T, A, S>&& lhs, const E* rhs) {+  //+  return std::move(lhs += rhs);+}++// C++11 21.4.8.1/11+template <typename E, class T, class A, class S>+inline basic_fbstring<E, T, A, S> operator+(+    const basic_fbstring<E, T, A, S>& lhs, E rhs) {+  basic_fbstring<E, T, A, S> result;+  result.reserve(lhs.size() + 1);+  result.append(lhs);+  result.push_back(rhs);+  return result;+}++// C++11 21.4.8.1/12+template <typename E, class T, class A, class S>+inline basic_fbstring<E, T, A, S> operator+(+    basic_fbstring<E, T, A, S>&& lhs, E rhs) {+  //+  return std::move(lhs += rhs);+}++template <typename E, class T, class A, class S>+inline bool operator==(+    const basic_fbstring<E, T, A, S>& lhs,+    const basic_fbstring<E, T, A, S>& rhs) {+  return lhs.size() == rhs.size() && lhs.compare(rhs) == 0;+}++template <typename E, class T, class A, class S>+inline bool operator==(std::nullptr_t, const basic_fbstring<E, T, A, S>&) =+    delete;++template <typename E, class T, class A, class S>+inline bool operator==(+    const typename basic_fbstring<E, T, A, S>::value_type* lhs,+    const basic_fbstring<E, T, A, S>& rhs) {+  return rhs == lhs;+}++template <typename E, class T, class A, class S>+inline bool operator==(const basic_fbstring<E, T, A, S>&, std::nullptr_t) =+    delete;++template <typename E, class T, class A, class S>+inline bool operator==(+    const basic_fbstring<E, T, A, S>& lhs,+    const typename basic_fbstring<E, T, A, S>::value_type* rhs) {+  return lhs.compare(rhs) == 0;+}++template <typename E, class T, class A, class S>+inline bool operator!=(+    const basic_fbstring<E, T, A, S>& lhs,+    const basic_fbstring<E, T, A, S>& rhs) {+  return !(lhs == rhs);+}++template <typename E, class T, class A, class S>+inline bool operator!=(std::nullptr_t, const basic_fbstring<E, T, A, S>&) =+    delete;++template <typename E, class T, class A, class S>+inline bool operator!=(+    const typename basic_fbstring<E, T, A, S>::value_type* lhs,+    const basic_fbstring<E, T, A, S>& rhs) {+  return !(lhs == rhs);+}++template <typename E, class T, class A, class S>+inline bool operator!=(const basic_fbstring<E, T, A, S>&, std::nullptr_t) =+    delete;++template <typename E, class T, class A, class S>+inline bool operator!=(+    const basic_fbstring<E, T, A, S>& lhs,+    const typename basic_fbstring<E, T, A, S>::value_type* rhs) {+  return !(lhs == rhs);+}++template <typename E, class T, class A, class S>+inline bool operator<(+    const basic_fbstring<E, T, A, S>& lhs,+    const basic_fbstring<E, T, A, S>& rhs) {+  return lhs.compare(rhs) < 0;+}++template <typename E, class T, class A, class S>+inline bool operator<(const basic_fbstring<E, T, A, S>&, std::nullptr_t) =+    delete;++template <typename E, class T, class A, class S>+inline bool operator<(+    const basic_fbstring<E, T, A, S>& lhs,+    const typename basic_fbstring<E, T, A, S>::value_type* rhs) {+  return lhs.compare(rhs) < 0;+}++template <typename E, class T, class A, class S>+inline bool operator<(std::nullptr_t, const basic_fbstring<E, T, A, S>&) =+    delete;++template <typename E, class T, class A, class S>+inline bool operator<(+    const typename basic_fbstring<E, T, A, S>::value_type* lhs,+    const basic_fbstring<E, T, A, S>& rhs) {+  return rhs.compare(lhs) > 0;+}++template <typename E, class T, class A, class S>+inline bool operator>(+    const basic_fbstring<E, T, A, S>& lhs,+    const basic_fbstring<E, T, A, S>& rhs) {+  return rhs < lhs;+}++template <typename E, class T, class A, class S>+inline bool operator>(const basic_fbstring<E, T, A, S>&, std::nullptr_t) =+    delete;++template <typename E, class T, class A, class S>+inline bool operator>(+    const basic_fbstring<E, T, A, S>& lhs,+    const typename basic_fbstring<E, T, A, S>::value_type* rhs) {+  return rhs < lhs;+}++template <typename E, class T, class A, class S>+inline bool operator>(std::nullptr_t, const basic_fbstring<E, T, A, S>&) =+    delete;++template <typename E, class T, class A, class S>+inline bool operator>(+    const typename basic_fbstring<E, T, A, S>::value_type* lhs,+    const basic_fbstring<E, T, A, S>& rhs) {+  return rhs < lhs;+}++template <typename E, class T, class A, class S>+inline bool operator<=(+    const basic_fbstring<E, T, A, S>& lhs,+    const basic_fbstring<E, T, A, S>& rhs) {+  return !(rhs < lhs);+}++template <typename E, class T, class A, class S>+inline bool operator<=(const basic_fbstring<E, T, A, S>&, std::nullptr_t) =+    delete;++template <typename E, class T, class A, class S>+inline bool operator<=(+    const basic_fbstring<E, T, A, S>& lhs,+    const typename basic_fbstring<E, T, A, S>::value_type* rhs) {+  return !(rhs < lhs);+}++template <typename E, class T, class A, class S>+inline bool operator<=(std::nullptr_t, const basic_fbstring<E, T, A, S>&) =+    delete;++template <typename E, class T, class A, class S>+inline bool operator<=(+    const typename basic_fbstring<E, T, A, S>::value_type* lhs,+    const basic_fbstring<E, T, A, S>& rhs) {+  return !(rhs < lhs);+}++template <typename E, class T, class A, class S>+inline bool operator>=(+    const basic_fbstring<E, T, A, S>& lhs,+    const basic_fbstring<E, T, A, S>& rhs) {+  return !(lhs < rhs);+}++template <typename E, class T, class A, class S>+inline bool operator>=(const basic_fbstring<E, T, A, S>&, std::nullptr_t) =+    delete;++template <typename E, class T, class A, class S>+inline bool operator>=(+    const basic_fbstring<E, T, A, S>& lhs,+    const typename basic_fbstring<E, T, A, S>::value_type* rhs) {+  return !(lhs < rhs);+}++template <typename E, class T, class A, class S>+inline bool operator>=(std::nullptr_t, const basic_fbstring<E, T, A, S>&) =+    delete;++template <typename E, class T, class A, class S>+inline bool operator>=(+    const typename basic_fbstring<E, T, A, S>::value_type* lhs,+    const basic_fbstring<E, T, A, S>& rhs) {+  return !(lhs < rhs);+}++#if FOLLY_CPLUSPLUS >= 202002+template <typename E, class T, class A, class S>+inline bool operator<=>(std::nullptr_t, const basic_fbstring<E, T, A, S>&) =+    delete;++template <typename E, class T, class A, class S>+inline bool operator<=>(const basic_fbstring<E, T, A, S>&, std::nullptr_t) =+    delete;+#endif++// C++11 21.4.8.8+template <typename E, class T, class A, class S>+void swap(basic_fbstring<E, T, A, S>& lhs, basic_fbstring<E, T, A, S>& rhs) {+  lhs.swap(rhs);+}++// TODO: make this faster.+template <typename E, class T, class A, class S>+inline std::basic_istream<+    typename basic_fbstring<E, T, A, S>::value_type,+    typename basic_fbstring<E, T, A, S>::traits_type>&+operator>>(+    std::basic_istream<+        typename basic_fbstring<E, T, A, S>::value_type,+        typename basic_fbstring<E, T, A, S>::traits_type>& is,+    basic_fbstring<E, T, A, S>& str) {+  typedef std::basic_istream<+      typename basic_fbstring<E, T, A, S>::value_type,+      typename basic_fbstring<E, T, A, S>::traits_type>+      _istream_type;+  typename _istream_type::sentry sentry(is);+  size_t extracted = 0;+  typename _istream_type::iostate err = _istream_type::goodbit;+  if (sentry) {+    auto n = is.width();+    if (n <= 0) {+      n = str.max_size();+    }+    str.erase();+    for (auto got = is.rdbuf()->sgetc(); extracted != size_t(n); ++extracted) {+      if (got == T::eof()) {+        err |= _istream_type::eofbit;+        is.width(0);+        break;+      }+      if (isspace(got)) {+        break;+      }+      str.push_back(got);+      got = is.rdbuf()->snextc();+    }+  }+  if (!extracted) {+    err |= _istream_type::failbit;+  }+  if (err) {+    is.setstate(err);+  }+  return is;+}++template <typename E, class T, class A, class S>+inline std::basic_ostream<+    typename basic_fbstring<E, T, A, S>::value_type,+    typename basic_fbstring<E, T, A, S>::traits_type>&+operator<<(+    std::basic_ostream<+        typename basic_fbstring<E, T, A, S>::value_type,+        typename basic_fbstring<E, T, A, S>::traits_type>& os,+    const basic_fbstring<E, T, A, S>& str) {+#ifdef _LIBCPP_VERSION+  typedef std::basic_ostream<+      typename basic_fbstring<E, T, A, S>::value_type,+      typename basic_fbstring<E, T, A, S>::traits_type>+      _ostream_type;+  typename _ostream_type::sentry _s(os);+  if (_s) {+    typedef std::ostreambuf_iterator<+        typename basic_fbstring<E, T, A, S>::value_type,+        typename basic_fbstring<E, T, A, S>::traits_type>+        _Ip;+    size_t __len = str.size();+    bool __left =+        (os.flags() & _ostream_type::adjustfield) == _ostream_type::left;+    if (__pad_and_output(+            _Ip(os),+            str.data(),+            __left ? str.data() + __len : str.data(),+            str.data() + __len,+            os,+            os.fill())+            .failed()) {+      os.setstate(_ostream_type::badbit | _ostream_type::failbit);+    }+  }+#elif defined(_MSC_VER)+  typedef decltype(os.precision()) streamsize;+  // MSVC doesn't define __ostream_insert+  os.write(str.data(), static_cast<streamsize>(str.size()));+#else+  std::__ostream_insert(os, str.data(), str.size());+#endif+  return os;+}++// basic_string compatibility routines++template <typename E, class T, class A, class S, class A2>+inline bool operator==(+    const basic_fbstring<E, T, A, S>& lhs,+    const std::basic_string<E, T, A2>& rhs) {+  return lhs.compare(0, lhs.size(), rhs.data(), rhs.size()) == 0;+}++template <typename E, class T, class A, class S, class A2>+inline bool operator==(+    const std::basic_string<E, T, A2>& lhs,+    const basic_fbstring<E, T, A, S>& rhs) {+  return rhs == lhs;+}++template <typename E, class T, class A, class S, class A2>+inline bool operator!=(+    const basic_fbstring<E, T, A, S>& lhs,+    const std::basic_string<E, T, A2>& rhs) {+  return !(lhs == rhs);+}++template <typename E, class T, class A, class S, class A2>+inline bool operator!=(+    const std::basic_string<E, T, A2>& lhs,+    const basic_fbstring<E, T, A, S>& rhs) {+  return !(lhs == rhs);+}++template <typename E, class T, class A, class S, class A2>+inline bool operator<(+    const basic_fbstring<E, T, A, S>& lhs,+    const std::basic_string<E, T, A2>& rhs) {+  return lhs.compare(0, lhs.size(), rhs.data(), rhs.size()) < 0;+}++template <typename E, class T, class A, class S, class A2>+inline bool operator>(+    const basic_fbstring<E, T, A, S>& lhs,+    const std::basic_string<E, T, A2>& rhs) {+  return lhs.compare(0, lhs.size(), rhs.data(), rhs.size()) > 0;+}++template <typename E, class T, class A, class S, class A2>+inline bool operator<(+    const std::basic_string<E, T, A2>& lhs,+    const basic_fbstring<E, T, A, S>& rhs) {+  return rhs > lhs;+}++template <typename E, class T, class A, class S, class A2>+inline bool operator>(+    const std::basic_string<E, T, A2>& lhs,+    const basic_fbstring<E, T, A, S>& rhs) {+  return rhs < lhs;+}++template <typename E, class T, class A, class S, class A2>+inline bool operator<=(+    const basic_fbstring<E, T, A, S>& lhs,+    const std::basic_string<E, T, A2>& rhs) {+  return !(lhs > rhs);+}++template <typename E, class T, class A, class S, class A2>+inline bool operator>=(+    const basic_fbstring<E, T, A, S>& lhs,+    const std::basic_string<E, T, A2>& rhs) {+  return !(lhs < rhs);+}++template <typename E, class T, class A, class S, class A2>+inline bool operator<=(+    const std::basic_string<E, T, A2>& lhs,+    const basic_fbstring<E, T, A, S>& rhs) {+  return !(lhs > rhs);+}++template <typename E, class T, class A, class S, class A2>+inline bool operator>=(+    const std::basic_string<E, T, A2>& lhs,+    const basic_fbstring<E, T, A, S>& rhs) {+  return !(lhs < rhs);+}++typedef basic_fbstring<char> fbstring;++// fbstring is relocatable+template <class T, class R, class A, class S>+FOLLY_ASSUME_RELOCATABLE(basic_fbstring<T, R, A, S>);++// Compatibility function, to make sure toStdString(s) can be called+// to convert a std::string or fbstring variable s into type std::string+// with very little overhead if s was already std::string+inline std::string toStdString(const folly::fbstring& s) {+  return std::string(s.data(), s.size());+}++inline const std::string& toStdString(const std::string& s) {+  return s;+}++// If called with a temporary, the compiler will select this overload instead+// of the above, so we don't return a (lvalue) reference to a temporary.+inline std::string&& toStdString(std::string&& s) {+  return std::move(s);+}++} // namespace folly++// Hash functions to make fbstring usable with e.g. unordered_map++#define FOLLY_FBSTRING_HASH1(T)                                               \+  template <>                                                                 \+  struct hash<::folly::basic_fbstring<T>> {                                   \+    size_t operator()(const ::folly::basic_fbstring<T>& s) const {            \+      return ::folly::hash::fnv32_buf_BROKEN(s.data(), s.size() * sizeof(T)); \+    }                                                                         \+  };++// The C++11 standard says that these four are defined for basic_string+#define FOLLY_FBSTRING_HASH      \+  FOLLY_FBSTRING_HASH1(char)     \+  FOLLY_FBSTRING_HASH1(char16_t) \+  FOLLY_FBSTRING_HASH1(char32_t) \+  FOLLY_FBSTRING_HASH1(wchar_t)++namespace std {++FOLLY_FBSTRING_HASH++} // namespace std++#undef FOLLY_FBSTRING_HASH+#undef FOLLY_FBSTRING_HASH1++FOLLY_POP_WARNING++#undef FBSTRING_DISABLE_SSO++namespace folly {+template <class T>+struct IsSomeString;++template <>+struct IsSomeString<fbstring> : std::true_type {};+} // namespace folly++template <>+struct fmt::formatter<folly::fbstring> : private formatter<fmt::string_view> {+  using formatter<fmt::string_view>::parse;++  template <typename Context>+  typename Context::iterator format(+      const folly::fbstring& s, Context& ctx) const {+    return formatter<fmt::string_view>::format({s.data(), s.size()}, ctx);+  }+};
+ folly/folly/FBVector.h view
@@ -0,0 +1,17 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#include <folly/container/FBVector.h>
+ folly/folly/File.cpp view
@@ -0,0 +1,180 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#include <folly/File.h>++#include <folly/Exception.h>+#include <folly/FileUtil.h>+#include <folly/ScopeGuard.h>+#include <folly/portability/Fcntl.h>+#include <folly/portability/FmtCompile.h>+#include <folly/portability/SysFile.h>+#include <folly/portability/Unistd.h>++#include <system_error>++#include <glog/logging.h>++namespace folly {++File::File(int fd, bool ownsFd) noexcept : fd_(fd), ownsFd_(ownsFd) {+  CHECK_GE(fd, -1) << "fd must be -1 or non-negative";+  CHECK(fd != -1 || !ownsFd) << "cannot own -1";+}++File::File(const char* name, int flags, mode_t mode)+    : fd_(fileops::open(name, flags, mode)), ownsFd_(false) {+  if (fd_ == -1) {+    throwSystemError(fmt::format(+        FOLLY_FMT_COMPILE("open(\"{}\", {:#o}, 0{:#o}) failed"),+        name,+        flags,+        mode));+  }+  ownsFd_ = true;+}++File::File(const std::string& name, int flags, mode_t mode)+    : File(name.c_str(), flags, mode) {}++File::File(StringPiece name, int flags, mode_t mode)+    : File(name.str(), flags, mode) {}++File::File(File&& other) noexcept : fd_(other.fd_), ownsFd_(other.ownsFd_) {+  other.release();+}++File& File::operator=(File&& other) {+  closeNoThrow();+  swap(other);+  return *this;+}++File::~File() {+  auto fd = fd_;+  if (!closeNoThrow()) { // ignore most errors+    DCHECK_NE(errno, EBADF)+        << "closing fd " << fd << ", it may already "+        << "have been closed. Another time, this might close the wrong FD.";+  }+}++/* static */ File File::temporary() {+  // make a temp file with tmpfile(), dup the fd, then return it in a File.+  FILE* tmpFile = tmpfile();+  checkFopenError(tmpFile, "tmpfile() failed");+  SCOPE_EXIT {+    fclose(tmpFile);+  };++  // TODO(nga): consider setting close-on-exec for the resulting FD+  int fd = ::dup(fileno(tmpFile));+  checkUnixError(fd, "dup() failed");++  return File(fd, true);+}++int File::release() noexcept {+  int released = fd_;+  fd_ = -1;+  ownsFd_ = false;+  return released;+}++void File::swap(File& other) noexcept {+  using std::swap;+  swap(fd_, other.fd_);+  swap(ownsFd_, other.ownsFd_);+}++void swap(File& a, File& b) noexcept {+  a.swap(b);+}++File File::dup() const {+  if (fd_ != -1) {+    int fd = ::dup(fd_);+    checkUnixError(fd, "dup() failed");++    return File(fd, true);+  }++  return File();+}++File File::dupCloseOnExec() const {+  if (fd_ != -1) {+    int fd;+#ifdef _WIN32+    fd = ::dup(fd_);+#else+    fd = ::fcntl(fd_, F_DUPFD_CLOEXEC, 0);+#endif+    checkUnixError(fd, "dup() failed");++    return File(fd, true);+  }++  return File();+}++void File::close() {+  if (!closeNoThrow()) {+    throwSystemError("close() failed");+  }+}++bool File::closeNoThrow() {+  int r = ownsFd_ ? fileops::close(fd_) : 0;+  release();+  return r == 0;+}++void File::lock() {+  doLock(LOCK_EX);+}+bool File::try_lock() {+  return doTryLock(LOCK_EX);+}+void File::lock_shared() {+  doLock(LOCK_SH);+}+bool File::try_lock_shared() {+  return doTryLock(LOCK_SH);+}++void File::doLock(int op) {+  checkUnixError(flockNoInt(fd_, op), "flock() failed (lock)");+}++bool File::doTryLock(int op) {+  int r = flockNoInt(fd_, op | LOCK_NB);+  // flock returns EWOULDBLOCK if already locked+  if (r == -1 && errno == EWOULDBLOCK) {+    return false;+  }+  checkUnixError(r, "flock() failed (try_lock)");+  return true;+}++void File::unlock() {+  checkUnixError(flockNoInt(fd_, LOCK_UN), "flock() failed (unlock)");+}+void File::unlock_shared() {+  unlock();+}++} // namespace folly
+ folly/folly/File.h view
@@ -0,0 +1,177 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++//+// Docs: https://fburl.com/fbcref_file+//++#pragma once++#include <fcntl.h>+#include <sys/stat.h>+#include <sys/types.h>++#include <string>+#include <system_error>++#include <folly/ExceptionWrapper.h>+#include <folly/Expected.h>+#include <folly/Portability.h>+#include <folly/Range.h>+#include <folly/portability/Fcntl.h>+#include <folly/portability/Unistd.h>++namespace folly {++/**+ * A File represents an open file.+ * @class folly::File+ * @refcode folly/docs/examples/folly/File.cpp+ */+class File {+ public:+  /**+   * Creates an empty File object, for late initialization.+   */+  constexpr File() noexcept : fd_(-1), ownsFd_(false) {}++  /**+   * Create a File object from an existing file descriptor.+   *+   * @param fd Existing file descriptor+   * @param ownsFd Takes ownership of the file descriptor if ownsFd is true.+   */+  explicit File(int fd, bool ownsFd = false) noexcept;++  /**+   * Open and create a file object.  Throws on error.+   * Owns the file descriptor implicitly.+   */+  explicit File(const char* name, int flags = O_RDONLY, mode_t mode = 0666);+  explicit File(+      const std::string& name, int flags = O_RDONLY, mode_t mode = 0666);+  explicit File(StringPiece name, int flags = O_RDONLY, mode_t mode = 0666);++  /**+   * All the constructors that are not noexcept can throw std::system_error.+   * This is a helper method to use folly::Expected to chain a file open event+   * to something else you want to do with the open fd.+   */+  template <typename... Args>+  static Expected<File, exception_wrapper> makeFile(Args&&... args) noexcept {+    try {+      return File(std::forward<Args>(args)...);+    } catch (const std::system_error&) {+      return makeUnexpected(exception_wrapper(current_exception()));+    }+  }++  ~File();++  /**+   * Create and return a temporary, owned file (uses tmpfile()).+   */+  static File temporary();++  /**+   * Return the file descriptor, or -1 if the file was closed.+   */+  int fd() const { return fd_; }++  /**+   * Returns 'true' iff the file was successfully opened.+   */+  explicit operator bool() const { return fd_ != -1; }++  /**+   * Duplicate file descriptor and return File that owns it.+   *+   * Duplicated file descriptor does not have close-on-exec flag set,+   * so it is leaked to child processes. Consider using "dupCloseOnExec".+   */+  File dup() const;++  /**+   * Duplicate file descriptor and return File that owns it.+   *+   * This functions creates a descriptor with close-on-exec flag set+   * (where supported, otherwise it is equivalent to "dup").+   */+  File dupCloseOnExec() const;++  /**+   * If we own the file descriptor, close the file and throw on error.+   * Otherwise, do nothing.+   */+  void close();++  /**+   * Closes the file (if owned).  Returns true on success, false (and sets+   * errno) on error.+   */+  bool closeNoThrow();++  /**+   * Returns and releases the file descriptor; no longer owned by this File.+   * Returns -1 if the File object didn't wrap a file.+   */+  int release() noexcept;++  /**+   * Swap this File with another.+   */+  void swap(File& other) noexcept;++  // movable+  File(File&&) noexcept;+  File& operator=(File&&);++  /**+   * FLOCK (INTERPROCESS) LOCKS+   *+   * NOTE THAT THESE LOCKS ARE flock() LOCKS.  That is, they may only be used+   * for inter-process synchronization -- an attempt to acquire a second lock+   * on the same file descriptor from the same process may succeed.  Attempting+   * to acquire a second lock on a different file descriptor for the same file+   * should fail, but some systems might implement flock() using fcntl() locks,+   * in which case it will succeed.+   */+  void lock();+  bool try_lock();+  void unlock();++  void lock_shared();+  bool try_lock_shared();+  void unlock_shared();++ private:+  void doLock(int op);+  bool doTryLock(int op);++  // unique+  File(const File&) = delete;+  File& operator=(const File&) = delete;++  int fd_;+  bool ownsFd_;+};++/**+ * Swaps the file descriptors and ownership+ */+void swap(File& a, File& b) noexcept;++} // namespace folly
+ folly/folly/FileUtil.cpp view
@@ -0,0 +1,378 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#include <folly/FileUtil.h>++#include <cerrno>+#include <string>+#include <system_error>+#include <vector>++#include <folly/detail/FileUtilDetail.h>+#include <folly/detail/FileUtilVectorDetail.h>+#include <folly/net/NetOps.h>+#include <folly/portability/Fcntl.h>+#include <folly/portability/Sockets.h>+#include <folly/portability/Stdlib.h>+#include <folly/portability/SysFile.h>+#include <folly/portability/SysStat.h>++namespace folly {+namespace {+iovec getIOVecFor(ByteRange);+} // namespace++using namespace fileutil_detail;++int openNoInt(const char* name, int flags, mode_t mode) {+  // Android NDK bionic with FORTIFY has this definition:+  // https://android.googlesource.com/platform/bionic/+/9349b9e51b/libc/include/bits/fortify/fcntl.h+  // ```+  // __BIONIC_ERROR_FUNCTION_VISIBILITY+  // int open(const char* pathname, int flags, mode_t modes, ...) __overloadable+  //         __errorattr(__open_too_many_args_error);+  // ```+  // This is originally to prevent open() with incorrect parameters.+  //+  // However, combined with folly wrapNotInt, template deduction will fail.+  // In this case, we create a custom lambda to bypass the error.+  // The solution is referenced from+  // https://github.com/llvm/llvm-project/commit/0a0e411204a2baa520fd73a8d69b664f98b428ba+  //+  auto openWrapper = [&] { return fileops::open(name, flags, mode); };+  return int(wrapNoInt(openWrapper));+}++static int filterCloseReturn(int r) {+  // Ignore EINTR.  On Linux, close() may only return EINTR after the file+  // descriptor has been closed, so you must not retry close() on EINTR --+  // in the best case, you'll get EBADF, and in the worst case, you'll end up+  // closing a different file (one opened from another thread).+  //+  // Interestingly enough, the Single Unix Specification says that the state+  // of the file descriptor is unspecified if close returns EINTR.  In that+  // case, the safe thing to do is also not to retry close() -- leaking a file+  // descriptor is definitely better than closing the wrong file.+  if (r == -1 && errno == EINTR) {+    return 0;+  }+  return r;+}++int closeNoInt(int fd) {+  return filterCloseReturn(fileops::close(fd));+}++int closeNoInt(NetworkSocket fd) {+  return filterCloseReturn(netops::close(fd));+}++int fsyncNoInt(int fd) {+  return int(wrapNoInt(fsync, fd));+}++int dupNoInt(int fd) {+  return int(wrapNoInt(dup, fd));+}++int dup2NoInt(int oldFd, int newFd) {+  return int(wrapNoInt(dup2, oldFd, newFd));+}++int fdatasyncNoInt(int fd) {+#if defined(__APPLE__)+  return int(wrapNoInt(fcntl, fd, F_FULLFSYNC));+#elif defined(__FreeBSD__) || defined(_MSC_VER)+  return int(wrapNoInt(fsync, fd));+#else+  return int(wrapNoInt(fdatasync, fd));+#endif+}++int ftruncateNoInt(int fd, off_t len) {+  return int(wrapNoInt(ftruncate, fd, len));+}++int truncateNoInt(const char* path, off_t len) {+  return int(wrapNoInt(truncate, path, len));+}++int flockNoInt(int fd, int operation) {+  return int(wrapNoInt(flock, fd, operation));+}++int shutdownNoInt(NetworkSocket fd, int how) {+  return int(wrapNoInt(netops::shutdown, fd, how));+}++ssize_t readNoInt(int fd, void* buf, size_t count) {+  return wrapNoInt(folly::fileops::read, fd, buf, count);+}++ssize_t preadNoInt(int fd, void* buf, size_t count, off_t offset) {+  return wrapNoInt(pread, fd, buf, count, offset);+}++ssize_t readvNoInt(int fd, const iovec* iov, int count) {+  return wrapNoInt(readv, fd, iov, count);+}++ssize_t preadvNoInt(int fd, const iovec* iov, int count, off_t offset) {+  return wrapNoInt(preadv, fd, iov, count, offset);+}++ssize_t writeNoInt(int fd, const void* buf, size_t count) {+  return wrapNoInt(folly::fileops::write, fd, buf, count);+}++ssize_t pwriteNoInt(int fd, const void* buf, size_t count, off_t offset) {+  return wrapNoInt(pwrite, fd, buf, count, offset);+}++ssize_t writevNoInt(int fd, const iovec* iov, int count) {+  return wrapNoInt(writev, fd, iov, count);+}++ssize_t pwritevNoInt(int fd, const iovec* iov, int count, off_t offset) {+  return wrapNoInt(pwritev, fd, iov, count, offset);+}++ssize_t readFull(int fd, void* buf, size_t count) {+  return wrapFull(folly::fileops::read, fd, buf, count);+}++ssize_t preadFull(int fd, void* buf, size_t count, off_t offset) {+  return wrapFull(pread, fd, buf, count, offset);+}++ssize_t writeFull(int fd, const void* buf, size_t count) {+  return wrapFull(folly::fileops::write, fd, const_cast<void*>(buf), count);+}++ssize_t pwriteFull(int fd, const void* buf, size_t count, off_t offset) {+  return wrapFull(pwrite, fd, const_cast<void*>(buf), count, offset);+}++#ifndef _WIN32+ssize_t readvFull(int fd, iovec* iov, int count) {+  return wrapvFull(readv, fd, iov, count);+}++ssize_t preadvFull(int fd, iovec* iov, int count, off_t offset) {+  return wrapvFull(preadv, fd, iov, count, offset);+}++ssize_t writevFull(int fd, iovec* iov, int count) {+  return wrapvFull(writev, fd, iov, count);+}++ssize_t pwritevFull(int fd, iovec* iov, int count, off_t offset) {+  return wrapvFull(pwritev, fd, iov, count, offset);+}+#else // _WIN32++// On Windows, the *vFull() functions wrap the simple read/pread/write/pwrite+// functions.  While folly/portability/SysUio.cpp does define readv() and+// writev() implementations for Windows, these attempt to lock the file to+// provide atomicity.  The *vFull() functions do not provide any atomicity+// guarantees, so we can avoid the locking logic.++ssize_t readvFull(int fd, iovec* iov, int count) {+  return wrapvFull(folly::fileops::read, fd, iov, count);+}++ssize_t preadvFull(int fd, iovec* iov, int count, off_t offset) {+  return wrapvFull(pread, fd, iov, count, offset);+}++ssize_t writevFull(int fd, iovec* iov, int count) {+  return wrapvFull(folly::fileops::write, fd, iov, count);+}++ssize_t pwritevFull(int fd, iovec* iov, int count, off_t offset) {+  return wrapvFull(pwrite, fd, iov, count, offset);+}+#endif // _WIN32++WriteFileAtomicOptions& WriteFileAtomicOptions::setPermissions(+    mode_t _permissions) {+  permissions = _permissions;+  return *this;+}++WriteFileAtomicOptions& WriteFileAtomicOptions::setSyncType(+    SyncType _syncType) {+  syncType = _syncType;+  return *this;+}++WriteFileAtomicOptions& WriteFileAtomicOptions::setTemporaryDirectory(+    std::string _temporaryDirectory) {+  temporaryDirectory = std::move(_temporaryDirectory);+  return *this;+}++namespace {+void throwIfWriteFileAtomicFailed(+    StringPiece function, StringPiece filename, std::int64_t rc) {+  if (rc != 0) {+    auto msg =+        std::string{function} + "() failed to update " + std::string{filename};+    throw std::system_error(rc, std::generic_category(), msg);+  }+}++// We write the data to a temporary file name first, then atomically rename+// it into place.+//+// If SyncType::WITH_SYNC is used, this ensures that the file contents will+// always be valid, even if we crash or are killed partway through writing out+// data.+int writeFileAtomicNoThrowImpl(+    StringPiece filename,+    iovec* iov,+    int count,+    const WriteFileAtomicOptions& options) {+  // create a null-terminated version of the filename+  auto filePathString = std::string{filename};+  auto temporaryFilePathString = fileutil_detail::getTemporaryFilePathString(+      filePathString, options.temporaryDirectory);++  auto tmpFD = mkstemp(const_cast<char*>(temporaryFilePathString.data()));+  if (tmpFD == -1) {+    return errno;+  }+  bool success = false;+  SCOPE_EXIT {+    if (tmpFD != -1) {+      fileops::close(tmpFD);+    }+    if (!success) {+      unlink(temporaryFilePathString.c_str());+    }+  };++  auto rc = writevFull(tmpFD, iov, count);+  if (rc == -1) {+    return errno;+  }++  rc = fchmod(tmpFD, options.permissions);+  if (rc == -1) {+    return errno;+  }++  // To guarantee atomicity across power failues on POSIX file systems,+  // the temporary file must be explicitly sync'ed before the rename.+  if (options.syncType == SyncType::WITH_SYNC) {+    rc = fsyncNoInt(tmpFD);+    if (rc == -1) {+      return errno;+    }+  }++  // Close the file before renaming to make sure all data has+  // been successfully written.+  rc = fileops::close(tmpFD);+  tmpFD = -1;+  if (rc == -1) {+    return errno;+  }++  rc = rename(temporaryFilePathString.c_str(), filePathString.c_str());+  if (rc == -1) {+    return errno;+  }+  success = true;+  return 0;+}+} // namespace++int writeFileAtomicNoThrow(+    StringPiece filename,+    iovec* iov,+    int count,+    mode_t permissions,+    SyncType syncType) {+  return writeFileAtomicNoThrowImpl(+      filename,+      iov,+      count,+      WriteFileAtomicOptions{}+          .setPermissions(permissions)+          .setSyncType(syncType));+}++int writeFileAtomicNoThrow(+    StringPiece filename,+    StringPiece data,+    const WriteFileAtomicOptions& options) {+  auto iov = getIOVecFor(ByteRange{data});+  return writeFileAtomicNoThrowImpl(filename, &iov, 1, options);+}++void writeFileAtomic(+    StringPiece filename,+    iovec* iov,+    int count,+    mode_t permissions,+    SyncType syncType) {+  auto rc = writeFileAtomicNoThrowImpl(+      filename,+      iov,+      count,+      WriteFileAtomicOptions{}+          .setPermissions(permissions)+          .setSyncType(syncType));+  throwIfWriteFileAtomicFailed(__func__, filename, rc);+}++void writeFileAtomic(+    StringPiece filename,+    ByteRange data,+    mode_t permissions,+    SyncType syncType) {+  auto iov = getIOVecFor(data);+  writeFileAtomic(filename, &iov, 1, permissions, syncType);+}++void writeFileAtomic(+    StringPiece filename,+    StringPiece data,+    mode_t permissions,+    SyncType syncType) {+  writeFileAtomic(filename, ByteRange(data), permissions, syncType);+}++void writeFileAtomic(+    StringPiece filename,+    StringPiece data,+    const WriteFileAtomicOptions& options) {+  auto iov = getIOVecFor(ByteRange{data});+  auto rc = writeFileAtomicNoThrowImpl(filename, &iov, 1, options);++  throwIfWriteFileAtomicFailed(__func__, filename, rc);+}++namespace {+iovec getIOVecFor(ByteRange byteRange) {+  iovec iov;+  iov.iov_base = const_cast<unsigned char*>(byteRange.data());+  iov.iov_len = byteRange.size();+  return iov;+}+} // namespace+} // namespace folly
+ folly/folly/FileUtil.h view
@@ -0,0 +1,332 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <sys/stat.h>+#include <sys/types.h>++#include <cassert>+#include <limits>++#include <folly/Portability.h>+#include <folly/Range.h>+#include <folly/ScopeGuard.h>+#include <folly/net/NetworkSocket.h>+#include <folly/portability/Fcntl.h>+#include <folly/portability/SysUio.h>+#include <folly/portability/Unistd.h>++namespace folly {++/**+ * Convenience wrappers around some commonly used system calls.  The *NoInt+ * wrappers retry on EINTR.  The *Full wrappers retry on EINTR and also loop+ * until all data is written.  Note that *Full wrappers weaken the thread+ * semantics of underlying system calls.+ */+int openNoInt(const char* name, int flags, mode_t mode = 0666);+// Two overloads, as we may be closing either a file or a socket.+int closeNoInt(int fd);+int closeNoInt(NetworkSocket fd);+int dupNoInt(int fd);+int dup2NoInt(int oldFd, int newFd);+int fsyncNoInt(int fd);+int fdatasyncNoInt(int fd);+int ftruncateNoInt(int fd, off_t len);+int truncateNoInt(const char* path, off_t len);+int flockNoInt(int fd, int operation);+int shutdownNoInt(NetworkSocket fd, int how);++ssize_t readNoInt(int fd, void* buf, size_t count);+ssize_t preadNoInt(int fd, void* buf, size_t count, off_t offset);+ssize_t readvNoInt(int fd, const iovec* iov, int count);+ssize_t preadvNoInt(int fd, const iovec* iov, int count, off_t offset);++ssize_t writeNoInt(int fd, const void* buf, size_t count);+ssize_t pwriteNoInt(int fd, const void* buf, size_t count, off_t offset);+ssize_t writevNoInt(int fd, const iovec* iov, int count);+ssize_t pwritevNoInt(int fd, const iovec* iov, int count, off_t offset);++/**+ * Wrapper around read() (and pread()) that, in addition to retrying on+ * EINTR, will loop until all data is read.+ *+ * This wrapper is only useful for blocking file descriptors (for non-blocking+ * file descriptors, you have to be prepared to deal with incomplete reads+ * anyway), and only exists because POSIX allows read() to return an incomplete+ * read if interrupted by a signal (instead of returning -1 and setting errno+ * to EINTR).+ *+ * Note that this wrapper weakens the thread safety of read(): the file pointer+ * is shared between threads, but the system call is atomic.  If multiple+ * threads are reading from a file at the same time, you don't know where your+ * data came from in the file, but you do know that the returned bytes were+ * contiguous.  You can no longer make this assumption if using readFull().+ * You should probably use pread() when reading from the same file descriptor+ * from multiple threads simultaneously, anyway.+ *+ * Note that readvFull and preadvFull require iov to be non-const, unlike+ * readv and preadv.  The contents of iov after these functions return+ * is unspecified.+ */+FOLLY_NODISCARD ssize_t readFull(int fd, void* buf, size_t count);+FOLLY_NODISCARD ssize_t+preadFull(int fd, void* buf, size_t count, off_t offset);+FOLLY_NODISCARD ssize_t readvFull(int fd, iovec* iov, int count);+FOLLY_NODISCARD ssize_t preadvFull(int fd, iovec* iov, int count, off_t offset);++/**+ * Similar to readFull and preadFull above, wrappers around write() and+ * pwrite() that loop until all data is written.+ *+ * Generally, the write() / pwrite() system call may always write fewer bytes+ * than requested, just like read().  In certain cases (such as when writing to+ * a pipe), POSIX provides stronger guarantees, but not in the general case.+ * For example, Linux (even on a 64-bit platform) won't write more than 2GB in+ * one write() system call.+ *+ * Note that writevFull and pwritevFull require iov to be non-const, unlike+ * writev and pwritev.  The contents of iov after these functions return+ * is unspecified.+ *+ * These functions return -1 on error, or the total number of bytes written+ * (which is always the same as the number of requested bytes) on success.+ */+ssize_t writeFull(int fd, const void* buf, size_t count);+ssize_t pwriteFull(int fd, const void* buf, size_t count, off_t offset);+ssize_t writevFull(int fd, iovec* iov, int count);+ssize_t pwritevFull(int fd, iovec* iov, int count, off_t offset);++/**+ * Read entire file (if num_bytes is defaulted) or no more than+ * num_bytes (otherwise) into container *out. The container is assumed+ * to be contiguous, with element size equal to 1, and offer size(),+ * reserve(), and random access (e.g. std::vector<char>, std::string,+ * fbstring).+ *+ * Returns: true on success or false on failure. In the latter case+ * errno will be set appropriately by the failing system primitive.+ */+template <class Container>+bool readFile(+    int fd,+    Container& out,+    size_t num_bytes = std::numeric_limits<size_t>::max()) {+  static_assert(+      sizeof(out[0]) == 1,+      "readFile: only containers with byte-sized elements accepted");++  size_t soFar = 0; // amount of bytes successfully read+  SCOPE_EXIT {+    assert(out.size() >= soFar); // resize better doesn't throw+    out.resize(soFar);+  };++  // Obtain file size:+  struct stat buf;+  if (fstat(fd, &buf) == -1) {+    return false;+  }+  // Some files (notably under /proc and /sys on Linux) lie about+  // their size, so treat the size advertised by fstat under advise+  // but don't rely on it. In particular, if the size is zero, we+  // should attempt to read stuff. If not zero, we'll attempt to read+  // one extra byte.+  constexpr size_t initialAlloc = 1024 * 4;+  out.resize(std::min(+      buf.st_size > 0 ? (size_t(buf.st_size) + 1) : initialAlloc, num_bytes));++  while (soFar < out.size()) {+    const auto actual = readFull(fd, &out[soFar], out.size() - soFar);+    if (actual == -1) {+      return false;+    }+    soFar += actual;+    if (soFar < out.size()) {+      // File exhausted+      break;+    }+    // Ew, allocate more memory. Use exponential growth to avoid+    // quadratic behavior. Cap size to num_bytes.+    out.resize(std::min(out.size() * 3 / 2, num_bytes));+  }++  return true;+}++/**+ * Same as above, but takes in a file name instead of fd+ */+template <class Container>+bool readFile(+    const char* file_name,+    Container& out,+    size_t num_bytes = std::numeric_limits<size_t>::max()) {+  assert(file_name);++  const auto fd = openNoInt(file_name, O_RDONLY | O_CLOEXEC);+  if (fd == -1) {+    return false;+  }++  SCOPE_EXIT {+    // Ignore errors when closing the file+    closeNoInt(fd);+  };++  return readFile(fd, out, num_bytes);+}++/**+ * Writes container to file. The container is assumed to be+ * contiguous, with element size equal to 1, and offering STL-like+ * methods empty(), size(), and indexed access+ * (e.g. std::vector<char>, std::string, fbstring, StringPiece).+ *+ * "flags" dictates the open flags to use. Default is to create file+ * if it doesn't exist and truncate it.+ *+ * Returns: true on success or false on failure. In the latter case+ * errno will be set appropriately by the failing system primitive.+ *+ * Note that this function may leave the file in a partially written state on+ * failure.  Use writeFileAtomic() if you want to ensure that the existing file+ * state will be unchanged on error.+ */+template <class Container>+bool writeFile(+    const Container& data,+    const char* filename,+    int flags = O_WRONLY | O_CREAT | O_TRUNC,+    mode_t mode = 0666) {+  static_assert(+      sizeof(data[0]) == 1, "writeFile works with element size equal to 1");+  int fd = fileops::open(filename, flags, mode);+  if (fd == -1) {+    return false;+  }+  bool ok = data.empty() ||+      writeFull(fd, &data[0], data.size()) == static_cast<ssize_t>(data.size());+  return closeNoInt(fd) == 0 && ok;+}++/* For atomic writes, do we sync to guarantee ordering or not? */+enum class SyncType {+  WITH_SYNC,+  WITHOUT_SYNC,+};++class WriteFileAtomicOptions {+ public:+  WriteFileAtomicOptions() = default;++  mode_t permissions{0644};+  SyncType syncType{SyncType::WITHOUT_SYNC};+  std::string temporaryDirectory;++  // The mode bits used for the temporary file+  WriteFileAtomicOptions& setPermissions(mode_t);++  // The default implementation does not sync the data to storage before the+  // rename.  Therefore, the write is *not* atomic in the event of a power+  // failure or OS crash.  To guarantee atomicity in these cases, specify+  // syncType = WITH_SYNC, which will incur a performance cost of waiting for+  // the data to be persisted to storage.  Note that the return of the function+  // does not guarantee the directory modifications have been written to disk; a+  // further sync of the directory after the function returns is required to+  // ensure the modification is durable.+  WriteFileAtomicOptions& setSyncType(SyncType);++  // The implementation creates a temporary file as an implementation detail+  // within this directory.  The temporary filenames themselves are+  // implementation defined.+  WriteFileAtomicOptions& setTemporaryDirectory(std::string);+};++/*+ * writeFileAtomic() does not currently work on Windows.+ * Windows does not provide atomic file renames, which makes implementing this+ * tricky.  Windows does have a MoveFileTransactedA() API which could+ * potentially be used, but according to the Microsoft documentation this API is+ * discouraged and may be removed in a future version.+ *+ * In order to implement this properly on Windows we would probably need a pair+ * of functions: one for writing the file, and one for reading the contents,+ * where the two functions synchronize with each other.  We can probably only+ * provide atomic update behavior with cooperation from the reader.+ */+#ifndef _WIN32++/**+ * Write file contents "atomically".+ *+ * This writes the data to a temporary file in the destination directory, and+ * then renames it to the specified path.  This guarantees that the specified+ * file will be replaced the specified contents on success, or will not be+ * modified on failure.+ *+ * Note that on platforms that do not provide atomic filesystem rename+ * functionality (e.g., Windows) this behavior may not be truly atomic.+ *+ * The default implementation does not sync the data to storage before the+ * rename.  Therefore, the write is *not* atomic in the event of a power failure+ * or OS crash.  To guarantee atomicity in these cases, specify syncType =+ * WITH_SYNC, which will incur a performance cost of waiting for the data to be+ * persisted to storage.  Note that the return of the function does not+ * guarantee the directory modifications have been written to disk; a further+ * sync of the directory after the function returns is required to ensure the+ * modification is durable.+ */+void writeFileAtomic(+    StringPiece filePath,+    iovec* iov,+    int count,+    mode_t permissions = 0644,+    SyncType syncType = SyncType::WITHOUT_SYNC);+void writeFileAtomic(+    StringPiece filePath,+    ByteRange data,+    mode_t permissions = 0644,+    SyncType syncType = SyncType::WITHOUT_SYNC);+void writeFileAtomic(+    StringPiece filePath,+    StringPiece data,+    mode_t permissions = 0644,+    SyncType syncType = SyncType::WITHOUT_SYNC);++void writeFileAtomic(+    StringPiece filePath, StringPiece data, const WriteFileAtomicOptions&);++/**+ * A version of writeFileAtomic() that returns an errno value instead of+ * throwing on error.+ *+ * Returns 0 on success or an errno value on error.+ */+int writeFileAtomicNoThrow(+    StringPiece filePath,+    iovec* iov,+    int count,+    mode_t permissions = 0644,+    SyncType syncType = SyncType::WITHOUT_SYNC);++int writeFileAtomicNoThrow(+    StringPiece filePath, StringPiece data, const WriteFileAtomicOptions&);++#endif // !_WIN32++} // namespace folly
+ folly/folly/Fingerprint.cpp view
@@ -0,0 +1,135 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#include <folly/Fingerprint.h>++#include <folly/Portability.h>+#include <folly/Utility.h>+#include <folly/detail/FingerprintPolynomial.h>++#include <utility>++namespace folly {+namespace detail {++namespace {++// The polynomials below were generated by a separate program that requires the+// NTL (Number Theory Library) from http://www.shoup.net/ntl/+//+// Briefly: randomly generate a polynomial of degree D, test for+// irreducibility, repeat until you find an irreducible polynomial+// (roughly 1/D of all polynomials of degree D are irreducible, so+// this will succeed in D/2 tries on average; D is small (64..128) so+// this simple method works well)+//+// DO NOT REPLACE THE POLYNOMIALS USED, EVER, as that would change the value+// of every single fingerprint in existence.+template <size_t Deg>+struct FingerprintTablePoly;+template <>+struct FingerprintTablePoly<63> {+  static constexpr uint64_t data[1] = {0xbf3736b51869e9b7};+};+template <>+struct FingerprintTablePoly<95> {+  static constexpr uint64_t data[2] = {0x51555cb0aa8d39c3, 0xb679ec3700000000};+};+template <>+struct FingerprintTablePoly<127> {+  static constexpr uint64_t data[2] = {0xc91bff9b8768b51b, 0x8c5d5853bd77b0d3};+};++template <typename D, size_t S0, size_t... I0>+constexpr auto copy_table(D const (&table)[S0], std::index_sequence<I0...>) {+  using array = std::array<D, S0>;+  return array{{table[I0]...}};+}+template <typename D, size_t S0>+constexpr auto copy_table(D const (&table)[S0]) {+  return copy_table(table, std::make_index_sequence<S0>{});+}++template <typename D, size_t S0, size_t S1, size_t... I0>+constexpr auto copy_table(+    D const (&table)[S0][S1], std::index_sequence<I0...>) {+  using array = std::array<std::array<D, S1>, S0>;+  return array{{copy_table(table[I0])...}};+}+template <typename D, size_t S0, size_t S1>+constexpr auto copy_table(D const (&table)[S0][S1]) {+  return copy_table(table, std::make_index_sequence<S0>{});+}++template <typename D, size_t S0, size_t S1, size_t S2, size_t... I0>+constexpr auto copy_table(+    D const (&table)[S0][S1][S2], std::index_sequence<I0...>) {+  using array = std::array<std::array<std::array<D, S2>, S1>, S0>;+  return array{{copy_table(table[I0])...}};+}+template <typename D, size_t S0, size_t S1, size_t S2>+constexpr auto copy_table(D const (&table)[S0][S1][S2]) {+  return copy_table(table, std::make_index_sequence<S0>{});+}++template <size_t Deg>+constexpr poly_table<Deg> make_poly_table() {+  FingerprintPolynomial<Deg> poly(FingerprintTablePoly<Deg>::data);+  uint64_t table[8][256][poly_size(Deg)] = {};+  // table[i][q] is Q(X) * X^(k+8*i) mod P(X),+  // where k is the number of bits in the fingerprint (and deg(P)) and+  // Q(X) = q7*X^7 + q6*X^6 + ... + q1*X + q0 is a degree-7 polynomial+  // whose coefficients are the bits of q.+  for (uint16_t x = 0; x < 256; x++) {+    FingerprintPolynomial<Deg> t;+    t.setHigh8Bits(uint8_t(x));+    for (auto& entry : table) {+      t.mulXkmod(8, poly);+      for (size_t j = 0; j < poly_size(Deg); ++j) {+        entry[x][j] = t.get(j);+      }+    }+  }+  return copy_table(table);+}++// private global variables marked constexpr to enforce that make_poly_table is+// really invoked at constexpr time, which would not otherwise be guaranteed+FOLLY_STORAGE_CONSTEXPR auto const poly_table_63 = make_poly_table<63>();+FOLLY_STORAGE_CONSTEXPR auto const poly_table_95 = make_poly_table<95>();+FOLLY_STORAGE_CONSTEXPR auto const poly_table_127 = make_poly_table<127>();++} // namespace++template <>+const uint64_t FingerprintTable<64>::poly[poly_size(64)] = {+    FingerprintTablePoly<63>::data[0]};+template <>+const uint64_t FingerprintTable<96>::poly[poly_size(96)] = {+    FingerprintTablePoly<95>::data[0], FingerprintTablePoly<95>::data[1]};+template <>+const uint64_t FingerprintTable<128>::poly[poly_size(128)] = {+    FingerprintTablePoly<127>::data[0], FingerprintTablePoly<127>::data[1]};++template <>+const poly_table<64> FingerprintTable<64>::table = poly_table_63;+template <>+const poly_table<96> FingerprintTable<96>::table = poly_table_95;+template <>+const poly_table<128> FingerprintTable<128>::table = poly_table_127;++} // namespace detail+} // namespace folly
+ folly/folly/Fingerprint.h view
@@ -0,0 +1,289 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++/**+ * Compute 64-, 96-, and 128-bit Rabin fingerprints, as described in+ * Michael O. Rabin (1981)+ *   Fingerprinting by Random Polynomials+ *   Center for Research in Computing Technology, Harvard University+ *   Tech Report TR-CSE-03-01+ *+ * The implementation follows the optimization described in+ * Andrei Z. Broder (1993)+ *   Some applications of Rabin's fingerprinting method+ *+ * extended for fingerprints larger than 64 bits, and modified to use+ * 64-bit instead of 32-bit integers for computation.+ *+ * The precomputed tables are in Fingerprint.cpp.+ *+ * Benchmarked on 10/13/2009 on a 2.5GHz quad-core Xeon L5420,+ * - Fingerprint<64>::update64() takes about 12ns+ * - Fingerprint<96>::update64() takes about 30ns+ * - Fingerprint<128>::update128() takes about 30ns+ * (unsurprisingly, Fingerprint<96> and Fingerprint<128> take the+ * same amount of time, as they both use 128-bit operations; the least+ * significant 32 bits of Fingerprint<96> will always be 0)+ */++#pragma once++#include <array>+#include <cstdint>++#include <folly/Range.h>++namespace folly {++namespace detail {++constexpr size_t poly_size(size_t bits) {+  return 1 + (bits - 1) / 64;+}++template <size_t Deg>+using poly_table =+    std::array<std::array<std::array<uint64_t, poly_size(Deg)>, 256>, 8>;++template <int BITS>+struct FingerprintTable {+  static const uint64_t poly[poly_size(BITS)];+  static const poly_table<BITS> table;+};++template <int BITS>+const uint64_t FingerprintTable<BITS>::poly[poly_size(BITS)] = {};+template <int BITS>+const poly_table<BITS> FingerprintTable<BITS>::table = {};++#ifndef _MSC_VER+// MSVC as of 2017 can't handle these extern specialization declarations,+// but they aren't needed for things to work right, so we just don't+// declare them in the header for MSVC.++#define FOLLY_DECLARE_FINGERPRINT_TABLES(BITS)                  \+  template <>                                                   \+  const uint64_t FingerprintTable<BITS>::poly[poly_size(BITS)]; \+  template <>                                                   \+  const poly_table<BITS> FingerprintTable<BITS>::table++FOLLY_DECLARE_FINGERPRINT_TABLES(64);+FOLLY_DECLARE_FINGERPRINT_TABLES(96);+FOLLY_DECLARE_FINGERPRINT_TABLES(128);++#undef FOLLY_DECLARE_FINGERPRINT_TABLES+#endif++} // namespace detail++/**+ * Compute the Rabin fingerprint.+ *+ * TODO(tudorb): Extend this to allow removing values from the computed+ * fingerprint (so we can fingerprint a sliding window, as in the Rabin-Karp+ * string matching algorithm)+ *+ * update* methods return *this, so you can chain them together:+ * Fingerprint<96>().update8(x).update(str).update64(val).write(output);+ */+template <int BITS>+class Fingerprint {+ public:+  Fingerprint() {+    // Use a non-zero starting value. We'll use (1 << (BITS-1))+    fp_[0] = 1ULL << 63;+    for (int i = 1; i < size(); i++) {+      fp_[i] = 0;+    }+  }++  Fingerprint& update8(uint8_t v) {+    uint8_t out = shlor8(v);+    xortab(detail::FingerprintTable<BITS>::table[0][out]);+    return *this;+  }++  // update32 and update64 are convenience functions to update the fingerprint+  // with 4 and 8 bytes at a time.  They are faster than calling update8+  // in a loop.  They process the bytes in big-endian order.+  Fingerprint& update32(uint32_t v) {+    uint32_t out = shlor32(v);+    for (int i = 0; i < 4; i++) {+      xortab(detail::FingerprintTable<BITS>::table[i][out & 0xff]);+      out >>= 8;+    }+    return *this;+  }++  Fingerprint& update64(uint64_t v) {+    uint64_t out = shlor64(v);+    for (int i = 0; i < 8; i++) {+      xortab(detail::FingerprintTable<BITS>::table[i][out & 0xff]);+      out >>= 8;+    }+    return *this;+  }++  Fingerprint& update(StringPiece str) {+    // TODO(tudorb): We could be smart and do update64 or update32 if aligned+    for (auto c : str) {+      update8(uint8_t(c));+    }+    return *this;+  }++  /**+   * Return the number of uint64s needed to hold the fingerprint value.+   */+  constexpr static int size() { return detail::poly_size(BITS); }++  /**+   * Write the computed fingerprint to an array of size() uint64_t's.+   * For Fingerprint<64>,  size()==1; we write 64 bits in out[0]+   * For Fingerprint<96>,  size()==2; we write 64 bits in out[0] and+   *                                  the most significant 32 bits of out[1]+   * For Fingerprint<128>, size()==2; we write 64 bits in out[0] and+   *                                  64 bits in out[1].+   */+  void write(uint64_t* out) const {+    for (int i = 0; i < size(); i++) {+      out[i] = fp_[i];+    }+  }++ private:+  // XOR the fingerprint with a value from one of the tables.+  void xortab(std::array<uint64_t, detail::poly_size(BITS)> const& tab) {+    for (int i = 0; i < size(); i++) {+      fp_[i] ^= tab[i];+    }+  }++  // Helper functions: shift the fingerprint value left by 8/32/64 bits,+  // return the "out" value (the bits that were shifted out), and add "v"+  // in the bits on the right.+  uint8_t shlor8(uint8_t v);+  uint32_t shlor32(uint32_t v);+  uint64_t shlor64(uint64_t v);++  uint64_t fp_[detail::poly_size(BITS)];+};++// Convenience functions++/**+ * Return the 64-bit Rabin fingerprint of a string.+ */+inline uint64_t fingerprint64(StringPiece str) {+  uint64_t fp;+  Fingerprint<64>().update(str).write(&fp);+  return fp;+}++/**+ * Compute the 96-bit Rabin fingerprint of a string.+ * Return the 64 most significant bits in *msb, and the 32 least significant+ * bits in *lsb.+ */+inline void fingerprint96(StringPiece str, uint64_t* msb, uint32_t* lsb) {+  uint64_t fp[2];+  Fingerprint<96>().update(str).write(fp);+  *msb = fp[0];+  *lsb = (uint32_t)(fp[1] >> 32);+}++/**+ * Compute the 128-bit Rabin fingerprint of a string.+ * Return the 64 most significant bits in *msb, and the 64 least significant+ * bits in *lsb.+ */+inline void fingerprint128(StringPiece str, uint64_t* msb, uint64_t* lsb) {+  uint64_t fp[2];+  Fingerprint<128>().update(str).write(fp);+  *msb = fp[0];+  *lsb = fp[1];+}++template <>+inline uint8_t Fingerprint<64>::shlor8(uint8_t v) {+  uint8_t out = (uint8_t)(fp_[0] >> 56);+  fp_[0] = (fp_[0] << 8) | ((uint64_t)v);+  return out;+}++template <>+inline uint32_t Fingerprint<64>::shlor32(uint32_t v) {+  uint32_t out = (uint32_t)(fp_[0] >> 32);+  fp_[0] = (fp_[0] << 32) | ((uint64_t)v);+  return out;+}++template <>+inline uint64_t Fingerprint<64>::shlor64(uint64_t v) {+  uint64_t out = fp_[0];+  fp_[0] = v;+  return out;+}++template <>+inline uint8_t Fingerprint<96>::shlor8(uint8_t v) {+  uint8_t out = (uint8_t)(fp_[0] >> 56);+  fp_[0] = (fp_[0] << 8) | (fp_[1] >> 56);+  fp_[1] = (fp_[1] << 8) | ((uint64_t)v << 32);+  return out;+}++template <>+inline uint32_t Fingerprint<96>::shlor32(uint32_t v) {+  uint32_t out = (uint32_t)(fp_[0] >> 32);+  fp_[0] = (fp_[0] << 32) | (fp_[1] >> 32);+  fp_[1] = ((uint64_t)v << 32);+  return out;+}++template <>+inline uint64_t Fingerprint<96>::shlor64(uint64_t v) {+  uint64_t out = fp_[0];+  fp_[0] = fp_[1] | (v >> 32);+  fp_[1] = v << 32;+  return out;+}++template <>+inline uint8_t Fingerprint<128>::shlor8(uint8_t v) {+  uint8_t out = (uint8_t)(fp_[0] >> 56);+  fp_[0] = (fp_[0] << 8) | (fp_[1] >> 56);+  fp_[1] = (fp_[1] << 8) | ((uint64_t)v);+  return out;+}++template <>+inline uint32_t Fingerprint<128>::shlor32(uint32_t v) {+  uint32_t out = (uint32_t)(fp_[0] >> 32);+  fp_[0] = (fp_[0] << 32) | (fp_[1] >> 32);+  fp_[1] = (fp_[1] << 32) | ((uint64_t)v);+  return out;+}++template <>+inline uint64_t Fingerprint<128>::shlor64(uint64_t v) {+  uint64_t out = fp_[0];+  fp_[0] = fp_[1];+  fp_[1] = v;+  return out;+}++} // namespace folly
+ folly/folly/FixedString.h view
@@ -0,0 +1,2973 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++// Fixed-size string type, for constexpr string handling.++#pragma once++#include <cassert>+#include <cstddef>+#include <initializer_list>+#include <iosfwd>+#include <stdexcept>+#include <string>+#include <string_view>+#include <type_traits>+#include <utility>++#include <folly/ConstexprMath.h>+#include <folly/Portability.h>+#include <folly/Range.h>+#include <folly/Utility.h>+#include <folly/lang/Exception.h>+#include <folly/lang/Ordering.h>+#include <folly/portability/Constexpr.h>++namespace folly {++template <class Char, std::size_t N>+class BasicFixedString;++template <std::size_t N>+using FixedString = BasicFixedString<char, N>;++namespace detail {+namespace fixedstring {++// This is a template so that the class static npos can be defined in the+// header.+template <class = void>+struct FixedStringBase_ {+  static constexpr std::size_t npos = static_cast<std::size_t>(-1);+};++using FixedStringBase = FixedStringBase_<>;++// Intentionally NOT constexpr. By making this not constexpr, we make+// checkOverflow below ill-formed in a constexpr context when the condition+// it's testing for fails. In this way, precondition violations are reported+// at compile-time instead of at runtime.+[[noreturn]] inline void assertOutOfBounds() {+  assert(false && "Array index out of bounds in BasicFixedString");+  throw_exception<std::out_of_range>(+      "Array index out of bounds in BasicFixedString");+}++constexpr std::size_t checkOverflow(std::size_t i, std::size_t max) {+  return i <= max ? i : (void(assertOutOfBounds()), max);+}++constexpr std::size_t checkOverflowOrNpos(std::size_t i, std::size_t max) {+  return i == FixedStringBase::npos+      ? max+      : (i <= max ? i : (void(assertOutOfBounds()), max));+}++constexpr std::size_t checkOverflowIfDebug(std::size_t i, std::size_t size) {+  return kIsDebug ? checkOverflow(i, size) : i;+}++// Intentionally NOT constexpr. See note above for assertOutOfBounds+[[noreturn]] inline void assertNotNullTerminated() noexcept {+  assert(+      false &&+      "Non-null terminated string used to initialize a BasicFixedString");+  std::terminate(); // Fail hard, fail fast.+}++// Parsing help for human readers: the following is a constexpr noexcept+// function that accepts a reference to an array as a parameter and returns+// a reference to the same array.+template <class Char, std::size_t N>+constexpr const Char (&checkNullTerminated(const Char (&a)[N]) noexcept)[N] {+  // Strange decltype(a)(a) used to make MSVC happy.+  return a[N - 1u] == Char(0)+          // In Debug mode, guard against embedded nulls:+          && (!kIsDebug || N - 1u == folly::constexpr_strlen(a))+      ? decltype(a)(a)+      : (assertNotNullTerminated(), decltype(a)(a));+}++template <class Left, class Right>+constexpr ordering compare_(+    const Left& left,+    std::size_t left_pos,+    std::size_t left_size,+    const Right& right,+    std::size_t right_pos,+    std::size_t right_size) noexcept {+  return left_pos == left_size+      ? (right_pos == right_size ? ordering::eq : ordering::lt)+      : (right_pos == right_size+             ? ordering::gt+             : (left[left_pos] < right[right_pos]+                    ? ordering::lt+                    : (left[left_pos] > right[right_pos]+                           ? ordering::gt+                           : fixedstring::compare_(+                                 left,+                                 left_pos + 1u,+                                 left_size,+                                 right,+                                 right_pos + 1u,+                                 right_size))));+}++template <class Left, class Right>+constexpr bool equal_(+    const Left& left,+    std::size_t left_size,+    const Right& right,+    std::size_t right_size) noexcept {+  return left_size == right_size &&+      ordering::eq == compare_(left, 0u, left_size, right, 0u, right_size);+}++template <class Char, class Left, class Right>+constexpr Char char_at_(+    const Left& left,+    std::size_t left_count,+    const Right& right,+    std::size_t right_count,+    std::size_t i) noexcept {+  return i < left_count ? left[i]+      : i < (left_count + right_count)+      ? right[i - left_count]+      : Char(0);+}++template <class Char, class Left, class Right>+constexpr Char char_at_(+    const Left& left,+    std::size_t left_size,+    std::size_t left_pos,+    std::size_t left_count,+    const Right& right,+    std::size_t right_pos,+    std::size_t right_count,+    std::size_t i) noexcept {+  FOLLY_PUSH_WARNING+#if defined(__GNUC__) && !defined(__clang__) && __GNUC__ <= 13+#pragma GCC diagnostic ignored "-Warray-bounds"+#endif+  return i < left_pos+      ? left[i]+      : (i < right_count + left_pos+             ? right[i - left_pos + right_pos]+             : (i < left_size - left_count + right_count+                    ? left[i - right_count + left_count]+                    : Char(0)));+  FOLLY_POP_WARNING+}++template <class Left, class Right>+constexpr bool find_at_(+    const Left& left,+    const Right& right,+    std::size_t pos,+    std::size_t count) noexcept {+  return 0u == count ||+      (left[pos + count - 1u] == right[count - 1u] &&+       find_at_(left, right, pos, count - 1u));+}++template <class Char, class Right>+constexpr bool find_one_of_at_(+    Char ch, const Right& right, std::size_t pos) noexcept {+  return 0u != pos &&+      (ch == right[pos - 1u] || find_one_of_at_(ch, right, pos - 1u));+}++template <class Left, class Right>+constexpr std::size_t find_(+    const Left& left,+    std::size_t left_size,+    const Right& right,+    std::size_t pos,+    std::size_t count) noexcept {+  return find_at_(left, right, pos, count) ? pos+      : left_size <= pos + count+      ? FixedStringBase::npos+      : find_(left, left_size, right, pos + 1u, count);+}++template <class Left, class Right>+constexpr std::size_t rfind_(+    const Left& left,+    const Right& right,+    std::size_t pos,+    std::size_t count) noexcept {+  return find_at_(left, right, pos, count) ? pos+      : 0u == pos+      ? FixedStringBase::npos+      : rfind_(left, right, pos - 1u, count);+}++template <class Left, class Right>+constexpr std::size_t find_first_of_(+    const Left& left,+    std::size_t left_size,+    const Right& right,+    std::size_t pos,+    std::size_t count) noexcept {+  return find_one_of_at_(left[pos], right, count) ? pos+      : left_size <= pos + 1u+      ? FixedStringBase::npos+      : find_first_of_(left, left_size, right, pos + 1u, count);+}++template <class Left, class Right>+constexpr std::size_t find_first_not_of_(+    const Left& left,+    std::size_t left_size,+    const Right& right,+    std::size_t pos,+    std::size_t count) noexcept {+  return !find_one_of_at_(left[pos], right, count) ? pos+      : left_size <= pos + 1u+      ? FixedStringBase::npos+      : find_first_not_of_(left, left_size, right, pos + 1u, count);+}++template <class Left, class Right>+constexpr std::size_t find_last_of_(+    const Left& left,+    const Right& right,+    std::size_t pos,+    std::size_t count) noexcept {+  return find_one_of_at_(left[pos], right, count) ? pos+      : 0u == pos+      ? FixedStringBase::npos+      : find_last_of_(left, right, pos - 1u, count);+}++template <class Left, class Right>+constexpr std::size_t find_last_not_of_(+    const Left& left,+    const Right& right,+    std::size_t pos,+    std::size_t count) noexcept {+  return !find_one_of_at_(left[pos], right, count) ? pos+      : 0u == pos+      ? FixedStringBase::npos+      : find_last_not_of_(left, right, pos - 1u, count);+}++struct Helper {+  template <class Char, class Left, class Right, std::size_t... Is>+  static constexpr BasicFixedString<Char, sizeof...(Is)> concat_(+      const Left& left,+      std::size_t left_count,+      const Right& right,+      std::size_t right_count,+      std::index_sequence<Is...> is) noexcept {+    return {left, left_count, right, right_count, is};+  }++  template <class Char, class Left, class Right, std::size_t... Is>+  static constexpr BasicFixedString<Char, sizeof...(Is)> replace_(+      const Left& left,+      std::size_t left_size,+      std::size_t left_pos,+      std::size_t left_count,+      const Right& right,+      std::size_t right_pos,+      std::size_t right_count,+      std::index_sequence<Is...> is) noexcept {+    return {+        left,+        left_size,+        left_pos,+        left_count,+        right,+        right_pos,+        right_count,+        is};+  }++  template <class Char, std::size_t N>+  static constexpr const Char (+      &data_(const BasicFixedString<Char, N>& that) noexcept)[N + 1u] {+    return that.data_;+  }+};++template <class T>+constexpr void constexpr_swap(T& a, T& b) noexcept(+    noexcept(a = T(std::move(a)))) {+  T tmp((std::move(a)));+  a = std::move(b);+  b = std::move(tmp);+}++// For constexpr reverse iteration over a BasicFixedString+template <class T>+struct ReverseIterator {+ private:+  T* p_ = nullptr;+  struct dummy_ {+    T* p_ = nullptr;+  };+  using other = typename std::conditional<+      std::is_const<T>::value,+      ReverseIterator<typename std::remove_const<T>::type>,+      dummy_>::type;++ public:+  using value_type = typename std::remove_const<T>::type;+  using reference = T&;+  using pointer = T*;+  using difference_type = std::ptrdiff_t;+  using iterator_category = std::random_access_iterator_tag;++  constexpr ReverseIterator() = default;+  constexpr ReverseIterator(const ReverseIterator&) = default;+  constexpr ReverseIterator& operator=(const ReverseIterator&) = default;+  constexpr explicit ReverseIterator(T* p) noexcept : p_(p) {}+  constexpr /* implicit */ ReverseIterator(const other& that) noexcept+      : p_(that.p_) {}+  friend constexpr bool operator==(+      ReverseIterator a, ReverseIterator b) noexcept {+    return a.p_ == b.p_;+  }+  friend constexpr bool operator!=(+      ReverseIterator a, ReverseIterator b) noexcept {+    return !(a == b);+  }+  constexpr reference operator*() const { return *(p_ - 1); }+  constexpr ReverseIterator& operator++() noexcept {+    --p_;+    return *this;+  }+  constexpr ReverseIterator operator++(int) noexcept {+    auto tmp(*this);+    --p_;+    return tmp;+  }+  constexpr ReverseIterator& operator--() noexcept {+    ++p_;+    return *this;+  }+  constexpr ReverseIterator operator--(int) noexcept {+    auto tmp(*this);+    ++p_;+    return tmp;+  }+  constexpr ReverseIterator& operator+=(std::ptrdiff_t i) noexcept {+    p_ -= i;+    return *this;+  }+  friend constexpr ReverseIterator operator+(+      std::ptrdiff_t i, ReverseIterator that) noexcept {+    return ReverseIterator{that.p_ - i};+  }+  friend constexpr ReverseIterator operator+(+      ReverseIterator that, std::ptrdiff_t i) noexcept {+    return ReverseIterator{that.p_ - i};+  }+  constexpr ReverseIterator& operator-=(std::ptrdiff_t i) noexcept {+    p_ += i;+    return *this;+  }+  friend constexpr ReverseIterator operator-(+      ReverseIterator that, std::ptrdiff_t i) noexcept {+    return ReverseIterator{that.p_ + i};+  }+  friend constexpr std::ptrdiff_t operator-(+      ReverseIterator a, ReverseIterator b) noexcept {+    return b.p_ - a.p_;+  }+  constexpr reference operator[](std::ptrdiff_t i) const noexcept {+    return *(*this + i);+  }+};++} // namespace fixedstring+} // namespace detail++/** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** *+ * \class BasicFixedString+ *+ * \tparam Char The character type. Must be a scalar type.+ * \tparam N The capacity and max size of string instances of this type.+ *+ * \brief A class for holding up to `N` characters of type `Char` that is+ *        amenable to `constexpr` string manipulation. It is guaranteed to not+ *        perform any dynamic allocation.+ *+ * `BasicFixedString` is a `std::string` work-alike that stores characters in an+ * internal buffer. It has minor interface differences that make it easy to work+ * with strings in a `constexpr` context.+ *+ * \par Example:+ * \par+ * \code+ * constexpr auto hello = makeFixedString("hello");         // a FixedString<5>+ * constexpr auto world = makeFixedString("world");         // a FixedString<5>+ * constexpr auto hello_world = hello + ' ' + world + '!';  // a FixedString<12>+ * static_assert(hello_world == "hello world!", "neato!");+ * \endcode+ * \par+ * `FixedString<N>` is an alias for `BasicFixedString<char, N>`.+ *+ * \par Constexpr and In-place Mutation+ * \par+ * On a C++14 compiler, `BasicFixedString` supports the full `std::string`+ * interface as `constexpr` member functions. On a C++11 compiler, the mutating+ * members are not `constexpr`, but non-mutating alternatives, which create a+ * new string, can be used instead. For example, instead of this:+ * \par+ * \code+ * constexpr FixedString<10> replace_example_cpp14() {+ *   FixedString<10> test{"****"};+ *   test.replace(1, 2, "!!!!");+ *   return test; // returns "*!!!!*"+ * }+ * \endcode+ * \par+ * You might write this instead:+ * \par+ * \code+ * constexpr FixedString<10> replace_example_cpp11() {+ *   // GNU compilers have an extension that make it possible to create+ *   // FixedString objects with a `""_fs` user-defined literal.+ *   using namespace folly;+ *   return makeFixedString("****").creplace(1, 2, "!!!!"); // "*!!!!*"+ * }+ * \endcode+ *+ * \par User-defined Literals+ * Instead of using the `folly::makeFixedString` helper function, you can use+ * a user-defined literal to make `FixedString` instances. The UDL feature of+ * C++ has some limitations that make this less than ideal; you must tell the+ * compiler roughly how many characters are in the string. The suffixes `_fs4`,+ * `_fs8`, `_fs16`, `_fs32`, `_fs64`, and `_fs128` exist to create instances+ * of types `FixedString<4>`, `FixedString<8>`, etc. For example:+ * \par+ * \code+ * using namespace folly::string_literals;+ * constexpr auto hello = "hello"_fs8; // A FixedString<8> containing "hello"+ * \endcode+ * \par+ * See Error Handling below for what to expect when you try to exceed the+ * capacity of a `FixedString` by storing too many characters in it.+ * \par+ * If your compiler supports GNU extensions, there is one additional suffix you+ * can use: `_fs`. This suffix always creates `FixedString` objects of exactly+ * the right size. For example:+ * \par+ * \code+ * using namespace folly::string_literals;+ * // NOTE: Only works on compilers with GNU extensions enabled. Clang and+ * // gcc support this (-Wgnu-string-literal-operator-template):+ * constexpr auto hello = "hello"_fs; // A FixedString<5> containing "hello"+ * \endcode+ *+ * \par Error Handling:+ * The capacity of a `BasicFixedString` is set at compile time. When the user+ * asks the string to exceed its capacity, one of three things will happen,+ * depending on the context:+ *\par+ *  -# If the attempt is made while evaluating a constant expression, the+ *     program will fail to compile.+ *  -# Otherwise, if the program is being run in debug mode, it will `assert`.+ *  -# Otherwise, the failed operation will throw a `std::out_of_range`+ *     exception.+ *\par+ * This is also the case if an invalid offset is passed to any member function,+ * or if `pop_back` or `cpop_back` is called on an empty `BasicFixedString`.+ *+ * Member functions documented as having preconditions will assert in Debug+ * mode (`!defined(NDEBUG)`) on precondition failures. Those documented with+ * \b Throws clauses will throw the specified exception on failure. Those with+ * both a precondition and a \b Throws clause will assert in Debug and throw+ * in Release mode.+ */+template <class Char, std::size_t N>+class BasicFixedString : private detail::fixedstring::FixedStringBase {+ private:+  template <class, std::size_t>+  friend class BasicFixedString;+  friend struct detail::fixedstring::Helper;++  // FUTURE: use constexpr_log2 to fold instantiations of BasicFixedString+  // together. All BasicFixedString<C, N> instantiations could share the+  // implementation of BasicFixedString<C, M>, where M is the next highest power+  // of 2 after N.+  //+  // Also, because of alignment of the data_ and size_ members, N should never+  // be smaller than `(alignof(std::size_t)/sizeof(C))-1` (-1 because of the+  // null terminator). OR, create a specialization for BasicFixedString<C, 0u>+  // that does not have a size_ member, since it is unnecessary.+  Char data_[N + 1u]; // +1 for the null terminator+  std::size_t size_; // Nbr of chars, not incl. null terminator. size_ <= N.++  using Indices = std::make_index_sequence<N>;++  template <class That, std::size_t... Is>+  constexpr BasicFixedString(+      const That& that,+      std::size_t size,+      std::index_sequence<Is...>,+      std::size_t pos = 0,+      std::size_t count = npos) noexcept+      : data_{(Is < (size - pos) && Is < count ? that[Is + pos] : Char(0))..., Char(0)},+        size_{folly::constexpr_min(size - pos, count)} {}++  template <std::size_t... Is>+  constexpr BasicFixedString(+      std::size_t count, Char ch, std::index_sequence<Is...>) noexcept+      : data_{((Is < count) ? ch : Char(0))..., Char(0)}, size_{count} {}++  // Concatenation constructor+  template <class Left, class Right, std::size_t... Is>+  constexpr BasicFixedString(+      const Left& left,+      std::size_t left_size,+      const Right& right,+      std::size_t right_size,+      std::index_sequence<Is...>) noexcept+      : data_{detail::fixedstring::char_at_<Char>(left, left_size, right, right_size, Is)..., Char(0)},+        size_{left_size + right_size} {}++  // Replace constructor+  template <class Left, class Right, std::size_t... Is>+  constexpr BasicFixedString(+      const Left& left,+      std::size_t left_size,+      std::size_t left_pos,+      std::size_t left_count,+      const Right& right,+      std::size_t right_pos,+      std::size_t right_count,+      std::index_sequence<Is...>) noexcept+      : data_{detail::fixedstring::char_at_<Char>(left, left_size, left_pos, left_count, right, right_pos, right_count, Is)..., Char(0)},+        size_{left_size - left_count + right_count} {}++ public:+  using size_type = std::size_t;+  using difference_type = std::ptrdiff_t;+  using reference = Char&;+  using const_reference = const Char&;+  using pointer = Char*;+  using const_pointer = const Char*;+  using iterator = Char*;+  using const_iterator = const Char*;+  using reverse_iterator = detail::fixedstring::ReverseIterator<Char>;+  using const_reverse_iterator =+      detail::fixedstring::ReverseIterator<const Char>;++  using detail::fixedstring::FixedStringBase::npos;++  /** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** **+   * Default construct+   * \post `size() == 0`+   * \post `at(0) == Char(0)`+   */+  constexpr BasicFixedString() : data_{}, size_{} {}++  /** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** **+   * Copy construct+   * \post `size() == that.size()`+   * \post `0 == strncmp(data(), that.data(), size())`+   * \post `at(size()) == Char(0)`+   */+  constexpr BasicFixedString(const BasicFixedString& /*that*/) = default;++  /** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** **+   * Construct from a differently-sized BasicFixedString+   * \pre `that.size() <= N`+   * \post `size() == that.size()`+   * \post `0 == strncmp(data(), that.data(), size())`+   * \post `at(size()) == Char(0)`+   * \throw std::out_of_range when that.size() > N. When M <= N, this+   *   constructor will never throw.+   * \note Conversions from larger-capacity BasicFixedString objects to smaller+   *   ones (`M > N`) are allowed as long as the *size()* of the source string+   *   is small enough.+   */+  template <std::size_t M>+  constexpr /* implicit */ BasicFixedString(+      const BasicFixedString<Char, M>& that) noexcept(M <= N)+      : BasicFixedString{that, 0u, that.size_} {}++  // Why is this deleted? To avoid confusion with the constructor that takes+  // a const Char* and a count.+  template <std::size_t M>+  constexpr BasicFixedString(+      const BasicFixedString<Char, M>& that,+      std::size_t pos) noexcept(false) = delete;++  /** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** **+   * Construct from an BasicFixedString, an offset, and a count+   * \param that The source string+   * \param pos The starting position in `that`+   * \param count The number of characters to copy. If `npos`, `count` is taken+   *              to be `that.size()-pos`.+   * \pre `pos <= that.size()`+   * \pre `count <= that.size()-pos && count <= N`+   * \post `size() == count`+   * \post `0 == strncmp(data(), that.data()+pos, size())`+   * \post `at(size()) == Char(0)`+   * \throw std::out_of_range when pos+count > that.size(), or when+   *        `count > N`+   */+  template <std::size_t M>+  constexpr BasicFixedString(+      const BasicFixedString<Char, M>& that,+      std::size_t pos,+      std::size_t count) noexcept(false)+      : BasicFixedString{+            that.data_,+            that.size_,+            std::make_index_sequence<(M < N ? M : N)>{},+            pos,+            detail::fixedstring::checkOverflow(+                detail::fixedstring::checkOverflowOrNpos(+                    count,+                    that.size_ -+                        detail::fixedstring::checkOverflow(pos, that.size_)),+                N)} {}++  /** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** **+   * Construct from a string literal+   * \pre `M-1 <= N`+   * \pre `that[M-1] == Char(0)`+   * \post `0 == strncmp(data(), that, M-1)`+   * \post `size() == M-1`+   * \post `at(size()) == Char(0)`+   */+  template <std::size_t M, class = typename std::enable_if<(M - 1u <= N)>::type>+  constexpr /* implicit */ BasicFixedString(const Char (&that)[M]) noexcept+      : BasicFixedString{+            detail::fixedstring::checkNullTerminated(that),+            M - 1u,+            std::make_index_sequence<M - 1u>{}} {}++  /** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** **+   * Construct from a `const Char*` and count+   * \pre `that` points to an array of at least `count` characters.+   * \pre `count <= N`+   * \post `size() == count`+   * \post `0 == strncmp(data(), that, size())`+   * \post `at(size()) == Char(0)`+   * \throw std::out_of_range when count > N+   */+  constexpr BasicFixedString(const Char* that, std::size_t count) noexcept(+      false)+      : BasicFixedString{+            that, detail::fixedstring::checkOverflow(count, N), Indices{}} {}++  /** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** **+   * Construct from a `std::basic_string_view<Char>`+   * \param that The source basic_string_view+   * \pre `that.size() <= N`+   * \post `size() == that.size()`+   * \post `0 == strncmp(data(), that.begin(), size())`+   * \post `at(size()) == Char(0)`+   * \throw std::out_of_range when that.size() > N+   */+  constexpr /* implicit */ BasicFixedString(+      std::basic_string_view<Char> that) noexcept(false)+      : BasicFixedString{that.data(), that.size()} {}++  /** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** **+   * Construct an BasicFixedString that contains `count` characters, all+   *   of which are `ch`.+   * \pre `count <= N`+   * \post `size() == count`+   * \post `npos == find_first_not_of(ch)`+   * \post `at(size()) == Char(0)`+   * \throw std::out_of_range when count > N+   */+  constexpr BasicFixedString(std::size_t count, Char ch) noexcept(false)+      : BasicFixedString{+            detail::fixedstring::checkOverflow(count, N), ch, Indices{}} {}++  /** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** **+   * Construct an BasicFixedString from a `std::initializer_list` of+   *   characters.+   * \pre `il.size() <= N`+   * \post `size() == count`+   * \post `0 == strncmp(data(), il.begin(), size())`+   * \post `at(size()) == Char(0)`+   * \throw std::out_of_range when il.size() > N+   */+  constexpr BasicFixedString(std::initializer_list<Char> il) noexcept(false)+      : BasicFixedString{il.begin(), il.size()} {}++  constexpr BasicFixedString& operator=(const BasicFixedString&) noexcept =+      default;++  /** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** **+   * Assign from a `BasicFixedString<Char, M>`.+   * \pre `that.size() <= N`+   * \post `size() == that.size()`+   * \post `0 == strncmp(data(), that.begin(), size())`+   * \post `at(size()) == Char(0)`+   * \throw std::out_of_range when that.size() > N. When M <= N, this+   *   assignment operator will never throw.+   * \note Assignments from larger-capacity BasicFixedString objects to smaller+   *   ones (`M > N`) are allowed as long as the *size* of the source string is+   *   small enough.+   * \return `*this`+   */+  template <std::size_t M>+  constexpr BasicFixedString& operator=(+      const BasicFixedString<Char, M>& that) noexcept(M <= N) {+    detail::fixedstring::checkOverflow(that.size_, N);+    size_ = that.copy(data_, that.size_);+    data_[size_] = Char(0);+    return *this;+  }++  /** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** **+   * Assign from a null-terminated array of characters.+   * \pre `M < N`+   * \pre `that` has no embedded null characters+   * \pre `that[M-1]==Char(0)`+   * \post `size() == M-1`+   * \post `0 == strncmp(data(), that, size())`+   * \post `at(size()) == Char(0)`+   * \return `*this`+   */+  template <std::size_t M, class = typename std::enable_if<(M - 1u <= N)>::type>+  constexpr BasicFixedString& operator=(const Char (&that)[M]) noexcept {+    return assign(detail::fixedstring::checkNullTerminated(that), M - 1u);+  }++  /** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** **+   * Assign from an `initializer_list` of characters.+   * \pre `il.size() <= N`+   * \post `size() == il.size()`+   * \post `0 == strncmp(data(), il.begin(), size())`+   * \post `at(size()) == Char(0)`+   * \throw std::out_of_range when il.size() > N+   * \return `*this`+   */+  constexpr BasicFixedString& operator=(+      std::initializer_list<Char> il) noexcept(false) {+    detail::fixedstring::checkOverflow(il.size(), N);+    for (std::size_t i = 0u; i < il.size(); ++i) {+      data_[i] = il.begin()[i];+    }+    size_ = il.size();+    data_[size_] = Char(0);+    return *this;+  }++  /** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** **+   * Conversion to folly::Range+   * \return `Range<Char*>{begin(), end()}`+   */+  constexpr Range<Char*> toRange() noexcept { return {begin(), end()}; }++  /**+   * \overload+   */+  constexpr Range<const Char*> toRange() const noexcept {+    return {begin(), end()};+  }++  /** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** **+   * Conversion to std::basic_string<Char>+   * \return `std::basic_string<Char>{begin(), end()}`+   */+  /* implicit */ operator std::basic_string<Char>() const noexcept(false) {+    return std::basic_string<Char>{begin(), end()};+  }++  std::basic_string<Char> toStdString() const noexcept(false) {+    return std::basic_string<Char>{begin(), end()};+  }++  /** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** **+   * Conversion to std::basic_string_view<Char>+   * \return `std::basic_string_view<Char>{begin(), end()}`+   */+  /* implicit */ constexpr operator std::basic_string_view<Char>() const {+    return std::basic_string_view<Char>{begin(), size()};+  }++  // Think hard about whether this is a good idea. It's certainly better than+  // an implicit conversion to `const Char*` since `delete "hi"_fs` will fail+  // to compile. But it creates ambiguities when passing a FixedString to an+  // API that has overloads for `const char*` and `folly::Range`, for instance.+  // using ArrayType = Char[N];+  // constexpr /* implicit */ operator ArrayType&() noexcept {+  //   return data_;+  // }++  // using ConstArrayType = const Char[N];+  // constexpr /* implicit */ operator ConstArrayType&() const noexcept {+  //   return data_;+  // }++  /** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** **+   * Assigns a sequence of `count` characters of value `ch`.+   * \param count The count of characters.+   * \param ch+   * \pre `count <= N`+   * \post `size() == count`+   * \post `npos == find_first_not_of(ch)`+   * \post `at(size()) == Char(0)`+   * \throw std::out_of_range when count > N+   * \return `*this`+   */+  constexpr BasicFixedString& assign(std::size_t count, Char ch) noexcept(+      false) {+    detail::fixedstring::checkOverflow(count, N);+    for (std::size_t i = 0u; i < count; ++i) {+      data_[i] = ch;+    }+    size_ = count;+    data_[size_] = Char(0);+    return *this;+  }++  /** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** **+   * Assigns characters from an `BasicFixedString` to this object.+   * \note Equivalent to `assign(that, 0, that.size())`+   */+  template <std::size_t M>+  constexpr BasicFixedString& assign(+      const BasicFixedString<Char, M>& that) noexcept(M <= N) {+    return *this = that;+  }++  // Why is this overload deleted? So users aren't confused by the difference+  // between str.assign("foo", N) and str.assign("foo"_fs, N). In the former,+  // N is a count of characters. In the latter, it would be a position, which+  // totally changes the meaning of the code.+  template <std::size_t M>+  constexpr BasicFixedString& assign(+      const BasicFixedString<Char, M>& that,+      std::size_t pos) noexcept(false) = delete;++  /** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** **+   * Assigns `count` characters from an `BasicFixedString` to this object,+   *   starting at position `pos` in the source object.+   * \param that The source string.+   * \param pos The starting position in the source string.+   * \param count The number of characters to copy. If `npos`, `count` is taken+   *              to be `that.size()-pos`.+   * \pre `pos <= that.size()`+   * \pre `count <= that.size()-pos`+   * \pre `count <= N`+   * \post `size() == count`+   * \post `0 == strncmp(data(), that.begin() + pos, count)`+   * \post `at(size()) == Char(0)`+   * \throw std::out_of_range when pos > that.size() or count > that.size()-pos+   *        or count > N.+   * \return `*this`+   */+  template <std::size_t M>+  constexpr BasicFixedString& assign(+      const BasicFixedString<Char, M>& that,+      std::size_t pos,+      std::size_t count) noexcept(false) {+    detail::fixedstring::checkOverflow(pos, that.size_);+    return assign(+        that.data_ + pos,+        detail::fixedstring::checkOverflowOrNpos(count, that.size_ - pos));+  }++  /** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** **+   * Assigns characters from an `BasicFixedString` to this object.+   * \pre `that` contains no embedded nulls.+   * \pre `that[M-1] == Char(0)`+   * \note Equivalent to `assign(that, M - 1)`+   */+  template <std::size_t M, class = typename std::enable_if<(M - 1u <= N)>::type>+  constexpr BasicFixedString& assign(const Char (&that)[M]) noexcept {+    return assign(detail::fixedstring::checkNullTerminated(that), M - 1u);+  }++  /** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** **+   * Assigns `count` characters from a range of characters to this object.+   * \param that A pointer to a range of characters.+   * \param count The number of characters to copy.+   * \pre `that` points to at least `count` characters.+   * \pre `count <= N`+   * \post `size() == count`+   * \post `0 == strncmp(data(), that, count)`+   * \post `at(size()) == Char(0)`+   * \throw std::out_of_range when count > N+   * \return `*this`+   */+  constexpr BasicFixedString& assign(+      const Char* that, std::size_t count) noexcept(false) {+    detail::fixedstring::checkOverflow(count, N);+    for (std::size_t i = 0u; i < count; ++i) {+      data_[i] = that[i];+    }+    size_ = count;+    data_[size_] = Char(0);+    return *this;+  }++  /** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** **+   * Swap the contents of this string with `that`.+   */+  constexpr void swap(BasicFixedString& that) noexcept {+    // less-than-or-equal here to copy the null terminator:+    for (std::size_t i = 0u; i <= folly::constexpr_max(size_, that.size_);+         ++i) {+      detail::fixedstring::constexpr_swap(data_[i], that.data_[i]);+    }+    detail::fixedstring::constexpr_swap(size_, that.size_);+  }++  /** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** **+   * Return a pointer to a range of `size()+1` characters, the last of which+   * is `Char(0)`.+   */+  constexpr Char* data() noexcept { return data_; }++  /**+   * \overload+   */+  constexpr const Char* data() const noexcept { return data_; }++  /** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** **+   * \return `data()`.+   */+  constexpr const Char* c_str() const noexcept { return data_; }++  /** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** **+   * \return `data()`.+   */+  constexpr Char* begin() noexcept { return data_; }++  /**+   * \overload+   */+  constexpr const Char* begin() const noexcept { return data_; }++  /** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** **+   * \return `data()`.+   */+  constexpr const Char* cbegin() const noexcept { return begin(); }++  /** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** **+   * \return `data() + size()`.+   */+  constexpr Char* end() noexcept { return data_ + size_; }++  /**+   * \overload+   */+  constexpr const Char* end() const noexcept { return data_ + size_; }++  /** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** **+   * \return `data() + size()`.+   */+  constexpr const Char* cend() const noexcept { return end(); }++  /** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** **+   * Returns a reverse iterator to the first character of the reversed string.+   * It corresponds to the last + 1 character of the non-reversed string.+   */+  constexpr reverse_iterator rbegin() noexcept {+    return reverse_iterator{data_ + size_};+  }++  /**+   * \overload+   */+  constexpr const_reverse_iterator rbegin() const noexcept {+    return const_reverse_iterator{data_ + size_};+  }++  /**+   * \note Equivalent to `rbegin()` on a const-qualified reference to `*this`.+   */+  constexpr const_reverse_iterator crbegin() const noexcept { return rbegin(); }++  /** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** **+   * Returns a reverse iterator to the last + 1 character of the reversed+   * string. It corresponds to the first character of the non-reversed string.+   */+  constexpr reverse_iterator rend() noexcept { return reverse_iterator{data_}; }++  /**+   * \overload+   */+  constexpr const_reverse_iterator rend() const noexcept {+    return const_reverse_iterator{data_};+  }++  /**+   * \note Equivalent to `rend()` on a const-qualified reference to `*this`.+   */+  constexpr const_reverse_iterator crend() const noexcept { return rend(); }++  /**+   * \return The number of `Char` elements in the string.+   */+  constexpr std::size_t size() const noexcept { return size_; }++  /**+   * \return The number of `Char` elements in the string.+   */+  constexpr std::size_t length() const noexcept { return size_; }++  /**+   * \return True if and only if `size() == 0`.+   */+  constexpr bool empty() const noexcept { return 0u == size_; }++  /**+   * \return `N`.+   */+  static constexpr std::size_t capacity() noexcept { return N; }++  /**+   * \return `N`.+   */+  static constexpr std::size_t max_size() noexcept { return N; }++  /**+   * \note `at(size())` is allowed will return `Char(0)`.+   * \return `*(data() + i)`+   * \throw std::out_of_range when i > size()+   */+  constexpr Char& at(std::size_t i) noexcept(false) {+    return i <= size_+        ? data_[i]+        : (throw_exception<std::out_of_range>(+               "Out of range in BasicFixedString::at"),+           data_[size_]);+  }++  /**+   * \overload+   */+  constexpr const Char& at(std::size_t i) const noexcept(false) {+    return i <= size_+        ? data_[i]+        : (throw_exception<std::out_of_range>(+               "Out of range in BasicFixedString::at"),+           data_[size_]);+  }++  /**+   * \pre `i <= size()`+   * \note `(*this)[size()]` is allowed will return `Char(0)`.+   * \return `*(data() + i)`+   */+  constexpr Char& operator[](std::size_t i) noexcept {+    return data_[detail::fixedstring::checkOverflowIfDebug(i, size_)];+  }++  /**+   * \overload+   */+  constexpr const Char& operator[](std::size_t i) const noexcept {+    return data_[detail::fixedstring::checkOverflowIfDebug(i, size_)];+  }++  /**+   * \note Equivalent to `(*this)[0]`+   */+  constexpr Char& front() noexcept { return (*this)[0u]; }++  /**+   * \overload+   */+  constexpr const Char& front() const noexcept { return (*this)[0u]; }++  /**+   * \note Equivalent to `at(size()-1)`+   * \pre `!empty()`+   */+  constexpr Char& back() noexcept {+    return data_[size_ - detail::fixedstring::checkOverflowIfDebug(1u, size_)];+  }++  /**+   * \overload+   */+  constexpr const Char& back() const noexcept {+    return data_[size_ - detail::fixedstring::checkOverflowIfDebug(1u, size_)];+  }++  /**+   * Clears the contents of this string.+   * \post `size() == 0u`+   * \post `at(size()) == Char(0)`+   */+  constexpr void clear() noexcept {+    data_[0u] = Char(0);+    size_ = 0u;+  }++  /**+   * \note Equivalent to `append(1u, ch)`.+   */+  constexpr void push_back(Char ch) noexcept(false) {+    detail::fixedstring::checkOverflow(1u, N - size_);+    data_[size_] = ch;+    data_[++size_] = Char(0);+  }++  /**+   * \note Equivalent to `cappend(1u, ch)`.+   */+  constexpr BasicFixedString<Char, N + 1u> cpush_back(Char ch) const noexcept {+    return cappend(ch);+  }++  /**+   * Removes the last character from the string.+   * \pre `!empty()`+   * \post `size()` is one fewer than before calling `pop_back()`.+   * \post `at(size()) == Char(0)`+   * \post The characters in the half-open range `[0,size()-1)` are unmodified.+   * \throw std::out_of_range if empty().+   */+  constexpr void pop_back() noexcept(false) {+    detail::fixedstring::checkOverflow(1u, size_);+    --size_;+    data_[size_] = Char(0);+  }++  /**+   * Returns a new string with the first `size()-1` characters from this string.+   * \pre `!empty()`+   * \note Equivalent to `BasicFixedString<Char, N-1u>{*this, 0u, size()-1u}`+   * \throw std::out_of_range if empty().+   */+  constexpr BasicFixedString<Char, N - 1u> cpop_back() const noexcept(false) {+    return {*this, 0u, size_ - detail::fixedstring::checkOverflow(1u, size_)};+  }++  /** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** **+   * Appends `count` copies of `ch` to this string.+   * \pre `count + old_size <= N`+   * \post The first `old_size` characters of the string are unmodified.+   * \post `size() == old_size + count`+   * \throw std::out_of_range if count > N - size().+   */+  constexpr BasicFixedString& append(std::size_t count, Char ch) noexcept(+      false) {+    detail::fixedstring::checkOverflow(count, N - size_);+    for (std::size_t i = 0u; i < count; ++i) {+      data_[size_ + i] = ch;+    }+    size_ += count;+    data_[size_] = Char(0);+    return *this;+  }++  /**+   * \note Equivalent to `append(*this, 0, that.size())`.+   */+  template <std::size_t M>+  constexpr BasicFixedString& append(+      const BasicFixedString<Char, M>& that) noexcept(false) {+    return append(that, 0u, that.size_);+  }++  // Why is this overload deleted? So as not to get confused with+  // append("null-terminated", N), where N would be a count instead+  // of a position.+  template <std::size_t M>+  constexpr BasicFixedString& append(+      const BasicFixedString<Char, M>& that,+      std::size_t pos) noexcept(false) = delete;++  /**+   * Appends `count` characters from another string to this one, starting at a+   * given offset, `pos`.+   * \param that The source string.+   * \param pos The starting position in the source string.+   * \param count The number of characters to append. If `npos`, `count` is+   *              taken to be `that.size()-pos`.+   * \pre `pos <= that.size()`+   * \pre `count <= that.size() - pos`+   * \pre `old_size + count <= N`+   * \post The first `old_size` characters of the string are unmodified.+   * \post `size() == old_size + count`+   * \post `at(size()) == Char(0)`+   * \throw std::out_of_range if pos + count > that.size() or if+   *        `old_size + count > N`.+   */+  template <std::size_t M>+  constexpr BasicFixedString& append(+      const BasicFixedString<Char, M>& that,+      std::size_t pos,+      std::size_t count) noexcept(false) {+    detail::fixedstring::checkOverflow(pos, that.size_);+    count = detail::fixedstring::checkOverflowOrNpos(count, that.size_ - pos);+    detail::fixedstring::checkOverflow(count, N - size_);+    for (std::size_t i = 0u; i < count; ++i) {+      data_[size_ + i] = that.data_[pos + i];+    }+    size_ += count;+    data_[size_] = Char(0);+    return *this;+  }++  /**+   * \note Equivalent to `append(that, strlen(that))`.+   */+  constexpr BasicFixedString& append(const Char* that) noexcept(false) {+    return append(that, folly::constexpr_strlen(that));+  }++  /**+   * Appends `count` characters from the specified character array.+   * \pre `that` points to a range of at least `count` characters.+   * \pre `count + old_size <= N`+   * \post The first `old_size` characters of the string are unmodified.+   * \post `size() == old_size + count`+   * \post `at(size()) == Char(0)`+   * \throw std::out_of_range if old_size + count > N.+   */+  constexpr BasicFixedString& append(+      const Char* that, std::size_t count) noexcept(false) {+    detail::fixedstring::checkOverflow(count, N - size_);+    for (std::size_t i = 0u; i < count; ++i) {+      data_[size_ + i] = that[i];+    }+    size_ += count;+    data_[size_] = Char(0);+    return *this;+  }++  /** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** **+   * Creates a new string by appending a character to an existing string, which+   *   is left unmodified.+   * \note Equivalent to `*this + ch`+   */+  constexpr BasicFixedString<Char, N + 1u> cappend(Char ch) const noexcept {+    return *this + ch;+  }++  /**+   * Creates a new string by appending a string to an existing string, which+   *   is left unmodified.+   * \note Equivalent to `*this + ch`+   */+  template <std::size_t M>+  constexpr BasicFixedString<Char, N + M> cappend(+      const BasicFixedString<Char, M>& that) const noexcept {+    return *this + that;+  }++  // Deleted to avoid confusion with append("char*", N), where N is a count+  // instead of a position.+  template <std::size_t M>+  constexpr BasicFixedString<Char, N + M> cappend(+      const BasicFixedString<Char, M>& that, std::size_t pos) const+      noexcept(false) = delete;++  /**+   * Creates a new string by appending characters from one string to another,+   *   which is left unmodified.+   * \note Equivalent to `*this + that.substr(pos, count)`+   */+  template <std::size_t M>+  constexpr BasicFixedString<Char, N + M> cappend(+      const BasicFixedString<Char, M>& that,+      std::size_t pos,+      std::size_t count) const noexcept(false) {+    return creplace(size_, 0u, that, pos, count);+  }++  /**+   * Creates a new string by appending a string literal to a string,+   *   which is left unmodified.+   * \note Equivalent to `*this + that`+   */+  template <std::size_t M>+  constexpr BasicFixedString<Char, N + M - 1u> cappend(+      const Char (&that)[M]) const noexcept {+    return creplace(size_, 0u, that);+  }++  // Deleted to avoid confusion with append("char*", N), where N is a count+  // instead of a position+  template <std::size_t M>+  constexpr BasicFixedString<Char, N + M - 1u> cappend(+      const Char (&that)[M], std::size_t pos) const noexcept(false) = delete;++  /**+   * Creates a new string by appending characters from one string to another,+   *   which is left unmodified.+   * \note Equivalent to `*this + makeFixedString(that).substr(pos, count)`+   */+  template <std::size_t M>+  constexpr BasicFixedString<Char, N + M - 1u> cappend(+      const Char (&that)[M], std::size_t pos, std::size_t count) const+      noexcept(false) {+    return creplace(size_, 0u, that, pos, count);+  }++  /** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** **+   * Appends characters from a null-terminated string literal to this string.+   * \note Equivalent to `append(that)`.+   */+  constexpr BasicFixedString& operator+=(const Char* that) noexcept(false) {+    return append(that);+  }++  /**+   * Appends characters from another string to this one.+   * \note Equivalent to `append(that)`.+   */+  template <std::size_t M>+  constexpr BasicFixedString& operator+=(+      const BasicFixedString<Char, M>& that) noexcept(false) {+    return append(that, 0u, that.size_);+  }++  /**+   * Appends a character to this string.+   * \note Equivalent to `push_back(ch)`.+   */+  constexpr BasicFixedString& operator+=(Char ch) noexcept(false) {+    push_back(ch);+    return *this;+  }++  /**+   * Appends characters from an `initializer_list` to this string.+   * \note Equivalent to `append(il.begin(), il.size())`.+   */+  constexpr BasicFixedString& operator+=(+      std::initializer_list<Char> il) noexcept(false) {+    return append(il.begin(), il.size());+  }++  /** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** **+   * Erase all characters from this string.+   * \note Equivalent to `clear()`+   * \return *this;+   */+  constexpr BasicFixedString& erase() noexcept {+    clear();+    return *this;+  }++  /**+   * Erases `count` characters from position `pos`. If `count` is `npos`,+   *   erases from `pos` to the end of the string.+   * \pre `pos <= size()`+   * \pre `count <= size() - pos || count == npos`+   * \post `size() == old_size - min(count, old_size - pos)`+   * \post `at(size()) == Char(0)`+   * \return *this;+   * \throw std::out_of_range when pos > size().+   */+  constexpr BasicFixedString& erase(+      std::size_t pos, std::size_t count = npos) noexcept(false) {+    using A = const Char[1];+    constexpr A a{Char(0)};+    return replace(+        pos,+        detail::fixedstring::checkOverflowOrNpos(+            count, size_ - detail::fixedstring::checkOverflow(pos, size_)),+        a,+        0u);+  }++  /**+   * \note Equivalent to `erase(first - data(), 1)`+   * \return A pointer to the first character after the erased character.+   */+  constexpr Char* erase(const Char* first) noexcept(false) {+    erase(first - data_, 1u);+    return data_ + (first - data_);+  }++  /**+   * \note Equivalent to `erase(first - data(), last - first)`+   * \return A pointer to the first character after the erased characters.+   */+  constexpr Char* erase(const Char* first, const Char* last) noexcept(false) {+    erase(first - data_, last - first);+    return data_ + (first - data_);+  }++  /** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** **+   * Create a new string by erasing all the characters from this string.+   * \note Equivalent to `BasicFixedString<Char, 0>{}`+   */+  constexpr BasicFixedString<Char, 0u> cerase() const noexcept { return {}; }++  /**+   * Create a new string by erasing all the characters after position `pos` from+   *   this string.+   * \note Equivalent to `creplace(pos, min(count, pos - size()), "")`+   */+  constexpr BasicFixedString cerase(+      std::size_t pos, std::size_t count = npos) const noexcept(false) {+    using A = const Char[1];+    return creplace(+        pos,+        detail::fixedstring::checkOverflowOrNpos(+            count, size_ - detail::fixedstring::checkOverflow(pos, size_)),+        A{Char(0)});+  }++  /** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** **+   * Compare two strings for lexicographical ordering.+   * \note Equivalent to+   * `compare(0, size(), that.data(), that.size())`+   */+  template <std::size_t M>+  constexpr int compare(const BasicFixedString<Char, M>& that) const noexcept {+    return compare(0u, size_, that, 0u, that.size_);+  }++  /**+   * Compare two strings for lexicographical ordering.+   * \note Equivalent to+   * `compare(this_pos, this_count, that.data(), that.size())`+   */+  template <std::size_t M>+  constexpr int compare(+      std::size_t this_pos,+      std::size_t this_count,+      const BasicFixedString<Char, M>& that) const noexcept(false) {+    return compare(this_pos, this_count, that, 0u, that.size_);+  }++  /**+   * Compare two strings for lexicographical ordering.+   * \note Equivalent to+   * `compare(this_pos, this_count, that.data() + that_pos, that_count)`+   */+  template <std::size_t M>+  constexpr int compare(+      std::size_t this_pos,+      std::size_t this_count,+      const BasicFixedString<Char, M>& that,+      std::size_t that_pos,+      std::size_t that_count) const noexcept(false) {+    return static_cast<int>(detail::fixedstring::compare_(+        data_,+        detail::fixedstring::checkOverflow(this_pos, size_),+        detail::fixedstring::checkOverflow(this_count, size_ - this_pos) ++            this_pos,+        that.data_,+        detail::fixedstring::checkOverflow(that_pos, that.size_),+        detail::fixedstring::checkOverflow(that_count, that.size_ - that_pos) ++            that_pos));+  }++  /**+   * Compare two strings for lexicographical ordering.+   * \note Equivalent to `compare(0, size(), that, strlen(that))`+   */+  constexpr int compare(const Char* that) const noexcept {+    return compare(0u, size_, that, folly::constexpr_strlen(that));+  }++  /**+   * \overload+   */+  constexpr int compare(Range<const Char*> that) const noexcept {+    return compare(0u, size_, that.begin(), that.size());+  }++  /**+   * Compare two strings for lexicographical ordering.+   * \note Equivalent to+   *   `compare(this_pos, this_count, that, strlen(that))`+   */+  constexpr int compare(+      std::size_t this_pos, std::size_t this_count, const Char* that) const+      noexcept(false) {+    return compare(this_pos, this_count, that, folly::constexpr_strlen(that));+  }++  /**+   * \overload+   */+  constexpr int compare(+      std::size_t this_pos,+      std::size_t this_count,+      Range<const Char*> that) const noexcept(false) {+    return compare(this_pos, this_count, that.begin(), that.size());+  }++  /**+   * Compare two strings for lexicographical ordering.+   *+   * Let `A` be the+   *   character sequence {`(*this)[this_pos]`, ...+   *   `(*this)[this_pos + this_count - 1]`}. Let `B` be the character sequence+   *   {`that[0]`, ...`that[count - 1]`}. Then...+   *+   * \return+   *   - `< 0` if `A` is ordered before the `B`+   *   - `> 0` if `B` is ordered before `A`+   *   - `0` if `A` equals `B`.+   *+   * \throw std::out_of_range if this_pos + this_count > size().+   */+  constexpr int compare(+      std::size_t this_pos,+      std::size_t this_count,+      const Char* that,+      std::size_t that_count) const noexcept(false) {+    return static_cast<int>(detail::fixedstring::compare_(+        data_,+        detail::fixedstring::checkOverflow(this_pos, size_),+        detail::fixedstring::checkOverflowOrNpos(this_count, size_ - this_pos) ++            this_pos,+        that,+        0u,+        that_count));+  }++  constexpr int compare(+      std::size_t this_pos,+      std::size_t this_count,+      Range<const Char*> that,+      std::size_t that_count) const noexcept(false) {+    return compare(+        this_pos,+        this_count,+        that.begin(),+        detail::fixedstring::checkOverflow(that_count, that.size()));+  }++  /** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** **+   * Return a substring from `pos` to the end of the string.+   * \note Equivalent to `BasicFixedString{*this, pos}`+   */+  constexpr BasicFixedString substr(std::size_t pos) const noexcept(false) {+    return {*this, pos};+  }++  /**+   * Return a substring from `pos` to the end of the string.+   * \note Equivalent to `BasicFixedString{*this, pos, count}`+   */+  constexpr BasicFixedString substr(std::size_t pos, std::size_t count) const+      noexcept(false) {+    return {*this, pos, count};+  }++  /** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** **+   * Replace the characters in the range denoted by the half-open range+   *   [`first`, `last`) with the string `that`.+   * \pre `first` and `last` point to characters within this string (including+   *   the terminating null).+   * \note Equivalent to+   *   `replace(first - data(), last - first, that.data(), that.size())`+   */+  template <std::size_t M>+  constexpr BasicFixedString& replace(+      const Char* first,+      const Char* last,+      const BasicFixedString<Char, M>& that) noexcept(false) {+    return replace(first - data_, last - first, that, 0u, that.size_);+  }++  /**+   * Replace `this_count` characters starting from position `this_pos` with the+   *   characters from string `that` starting at position `that_pos`.+   * \pre `that_pos <= that.size()`+   * \note Equivalent to+   *   <tt>replace(this_pos, this_count, that.data() + that_pos,+   *   that.size() - that_pos)</tt>+   */+  template <std::size_t M>+  constexpr BasicFixedString& replace(+      std::size_t this_pos,+      std::size_t this_count,+      const BasicFixedString<Char, M>& that,+      std::size_t that_pos = 0u) noexcept(false) {+    return replace(this_pos, this_count, that, that_pos, that.size_ - that_pos);+  }++  /**+   * Replace `this_count` characters starting from position `this_pos` with+   *   `that_count` characters from string `that` starting at position+   *   `that_pos`.+   * \pre `that_pos <= that.size() && that_count <= that.size() - that_pos`+   * \note Equivalent to+   *   `replace(this_pos, this_count, that.data() + that_pos, that_count)`+   */+  template <std::size_t M>+  constexpr BasicFixedString& replace(+      std::size_t this_pos,+      std::size_t this_count,+      const BasicFixedString<Char, M>& that,+      std::size_t that_pos,+      std::size_t that_count) noexcept(false) {+    return *this = creplace(this_pos, this_count, that, that_pos, that_count);+  }++  /**+   * Replace `this_count` characters starting from position `this_pos` with+   *   the characters from the string literal `that`.+   * \note Equivalent to+   *   `replace(this_pos, this_count, that, strlen(that))`+   */+  constexpr BasicFixedString& replace(+      std::size_t this_pos,+      std::size_t this_count,+      const Char* that) noexcept(false) {+    return replace(this_pos, this_count, that, folly::constexpr_strlen(that));+  }++  /**+   * Replace the characters denoted by the half-open range [`first`,`last`) with+   *   the characters from the string literal `that`.+   * \pre `first` and `last` point to characters within this string (including+   *   the terminating null).+   * \note Equivalent to+   *   `replace(first - data(), last - first, that, strlen(that))`+   */+  constexpr BasicFixedString& replace(+      const Char* first, const Char* last, const Char* that) noexcept(false) {+    return replace(+        first - data_, last - first, that, folly::constexpr_strlen(that));+  }++  /**+   * Replace `this_count` characters starting from position `this_pos` with+   *   `that_count` characters from the character sequence pointed to by `that`.+   * \param this_pos The starting offset within `*this` of the first character+   *   to be replaced.+   * \param this_count The number of characters to be replaced. If `npos`,+   *   it is treated as if `this_count` were `size() - this_pos`.+   * \param that A pointer to the replacement string.+   * \param that_count The number of characters in the replacement string.+   * \pre `this_pos <= size() && this_count <= size() - this_pos`+   * \pre `that` points to a contiguous sequence of at least `that_count`+   *   characters+   * \throw std::out_of_range on any of the following conditions:+   *   - `this_pos > size()`+   *   - `this_count > size() - this_pos`+   *   - `size() - this_count + that_count > N`+   */+  constexpr BasicFixedString& replace(+      std::size_t this_pos,+      std::size_t this_count,+      const Char* that,+      std::size_t that_count) noexcept(false) {+    return *this = detail::fixedstring::Helper::replace_<Char>(+               data_,+               size_,+               detail::fixedstring::checkOverflow(this_pos, size_),+               detail::fixedstring::checkOverflowOrNpos(+                   this_count, size_ - this_pos),+               that,+               0u,+               that_count,+               Indices{});+  }++  /**+   * Replace `this_count` characters starting from position `this_pos` with+   *   `that_count` characters `ch`.+   * \note Equivalent to+   *   `replace(this_pos, this_count, BasicFixedString{that_count, ch})`+   */+  constexpr BasicFixedString& replace(+      std::size_t this_pos,+      std::size_t this_count,+      std::size_t that_count,+      Char ch) noexcept(false) {+    return replace(this_pos, this_count, BasicFixedString{that_count, ch});+  }++  /**+   * Replace the characters denoted by the half-open range [`first`,`last`)+   *   with `that_count` characters `ch`.+   * \note Equivalent to+   *   `replace(first - data(), last - first, BasicFixedString{that_count, ch})`+   */+  constexpr BasicFixedString& replace(+      const Char* first,+      const Char* last,+      std::size_t that_count,+      Char ch) noexcept(false) {+    return replace(+        first - data_, last - first, BasicFixedString{that_count, ch});+  }++  /**+   * Replace the characters denoted by the half-open range [`first`,`last`) with+   *   the characters from the string literal `that`.+   * \pre `first` and `last` point to characters within this string (including+   *   the terminating null).+   * \note Equivalent to+   *   `replace(this_pos, this_count, il.begin(), il.size())`+   */+  constexpr BasicFixedString& replace(+      const Char* first,+      const Char* last,+      std::initializer_list<Char> il) noexcept(false) {+    return replace(first - data_, last - first, il.begin(), il.size());+  }++  /** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** **+   * Construct a new string by replacing `this_count` characters starting from+   *   position `this_pos` within this string with the characters from string+   *   `that` starting at position `that_pos`.+   * \pre `that_pos <= that.size()`+   * \note Equivalent to+   *   <tt>creplace(this_pos, this_count, that, that_pos,+   *   that.size() - that_pos)</tt>+   */+  template <std::size_t M>+  constexpr BasicFixedString<Char, N + M> creplace(+      std::size_t this_pos,+      std::size_t this_count,+      const BasicFixedString<Char, M>& that,+      std::size_t that_pos = 0u) const noexcept(false) {+    return creplace(+        this_pos,+        this_count,+        that,+        that_pos,+        that.size_ - detail::fixedstring::checkOverflow(that_pos, that.size_));+  }++  /**+   * Construct a new string by replacing `this_count` characters starting from+   *   position `this_pos` within this string with `that_count` characters from+   *   string `that` starting at position `that_pos`.+   * \param this_pos The starting offset within `*this` of the first character+   *   to be replaced.+   * \param this_count The number of characters to be replaced. If `npos`,+   *   it is treated as if `this_count` were `size() - this_pos`.+   * \param that A string that contains the replacement string.+   * \param that_pos The offset to the first character in the replacement+   *   string.+   * \param that_count The number of characters in the replacement string.+   * \pre `this_pos <= size() && this_count <= size() - this_pos`+   * \pre `that_pos <= that.size() && that_count <= that.size() - that_pos`+   * \post The size of the returned string is `size() - this_count + that_count`+   * \note Equivalent to <tt>BasicFixedString<Char, N + M>{substr(0, this_pos) ++   *    that.substr(that_pos, that_count) + substr(this_pos + this_count)}</tt>+   * \throw std::out_of_range on any of the following conditions:+   *   - `this_pos > size()`+   *   - `this_count > size() - this_pos`+   *   - `that_pos > that.size()`+   *   - `that_count > that.size() - that_pos`+   */+  template <std::size_t M>+  constexpr BasicFixedString<Char, N + M> creplace(+      std::size_t this_pos,+      std::size_t this_count,+      const BasicFixedString<Char, M>& that,+      std::size_t that_pos,+      std::size_t that_count) const noexcept(false) {+    return detail::fixedstring::Helper::replace_<Char>(+        data_,+        size_,+        detail::fixedstring::checkOverflow(this_pos, size_),+        detail::fixedstring::checkOverflowOrNpos(this_count, size_ - this_pos),+        that.data_,+        detail::fixedstring::checkOverflow(that_pos, that.size_),+        detail::fixedstring::checkOverflowOrNpos(+            that_count, that.size_ - that_pos),+        std::make_index_sequence<N + M>{});+  }++  /**+   * Construct a new string by replacing the characters denoted by the half-open+   *   range [`first`,`last`) within this string with the characters from string+   *   `that` starting at position `that_pos`.+   * \pre `that_pos <= that.size()`+   * \note Equivalent to+   *   <tt>creplace(first - data(), last - first, that, that_pos,+   *   that.size() - that_pos)</tt>+   */+  template <std::size_t M>+  constexpr BasicFixedString<Char, N + M> creplace(+      const Char* first,+      const Char* last,+      const BasicFixedString<Char, M>& that,+      std::size_t that_pos = 0u) const noexcept(false) {+    return creplace(+        first - data_,+        last - first,+        that,+        that_pos,+        that.size_ - detail::fixedstring::checkOverflow(that_pos, that.size_));+  }++  /**+   * Construct a new string by replacing the characters denoted by the half-open+   *   range [`first`,`last`) within this string with the `that_count`+   *   characters from string `that` starting at position `that_pos`.+   * \note Equivalent to+   *   <tt>creplace(first - data(), last - first, that, that_pos,+   *   that_count)</tt>+   */+  template <std::size_t M>+  constexpr BasicFixedString<Char, N + M> creplace(+      const Char* first,+      const Char* last,+      const BasicFixedString<Char, M>& that,+      std::size_t that_pos,+      std::size_t that_count) const noexcept(false) {+    return creplace(first - data_, last - first, that, that_pos, that_count);+  }++  /**+   * Construct a new string by replacing `this_count` characters starting from+   *   position `this_pos` within this string with `M-1` characters from+   *   character array `that`.+   * \pre `strlen(that) == M-1`+   * \note Equivalent to+   *   <tt>creplace(this_pos, this_count, that, 0, M - 1)</tt>+   */+  template <std::size_t M>+  constexpr BasicFixedString<Char, N + M - 1u> creplace(+      std::size_t this_pos, std::size_t this_count, const Char (&that)[M]) const+      noexcept(false) {+    return creplace(this_pos, this_count, that, 0u, M - 1u);+  }++  /**+   * Replace `this_count` characters starting from position `this_pos` with+   *   `that_count` characters from the character array `that` starting at+   *   position `that_pos`.+   * \param this_pos The starting offset within `*this` of the first character+   *   to be replaced.+   * \param this_count The number of characters to be replaced. If `npos`,+   *   it is treated as if `this_count` were `size() - this_pos`.+   * \param that An array of characters containing the replacement string.+   * \param that_pos The starting offset of the replacement string.+   * \param that_count The number of characters in the replacement string.  If+   *   `npos`, it is treated as if `that_count` were `M - 1 - that_pos`+   * \pre `this_pos <= size() && this_count <= size() - this_pos`+   * \pre `that_pos <= M - 1 && that_count <= M - 1 - that_pos`+   * \post The size of the returned string is `size() - this_count + that_count`+   * \note Equivalent to <tt>BasicFixedString<Char, N + M - 1>{+   *    substr(0, this_pos) ++   *    makeFixedString(that).substr(that_pos, that_count) ++   *    substr(this_pos + this_count)}</tt>+   * \throw std::out_of_range on any of the following conditions:+   *   - `this_pos > size()`+   *   - `this_count > size() - this_pos`+   *   - `that_pos >= M`+   *   - `that_count >= M - that_pos`+   */+  template <std::size_t M>+  constexpr BasicFixedString<Char, N + M - 1u> creplace(+      std::size_t this_pos,+      std::size_t this_count,+      const Char (&that)[M],+      std::size_t that_pos,+      std::size_t that_count) const noexcept(false) {+    return detail::fixedstring::Helper::replace_<Char>(+        data_,+        size_,+        detail::fixedstring::checkOverflow(this_pos, size_),+        detail::fixedstring::checkOverflowOrNpos(this_count, size_ - this_pos),+        detail::fixedstring::checkNullTerminated(that),+        detail::fixedstring::checkOverflow(that_pos, M - 1u),+        detail::fixedstring::checkOverflowOrNpos(that_count, M - 1u - that_pos),+        std::make_index_sequence<N + M - 1u>{});+  }++  /**+   * Construct a new string by replacing the characters denoted by the half-open+   *   range [`first`,`last`) within this string with the first `M-1`+   *   characters from the character array `that`.+   * \pre `strlen(that) == M-1`+   * \note Equivalent to+   *   <tt>creplace(first - data(), last - first, that, 0, M-1)</tt>+   */+  template <std::size_t M>+  constexpr BasicFixedString<Char, N + M - 1u> creplace(+      const Char* first, const Char* last, const Char (&that)[M]) const+      noexcept(false) {+    return creplace(first - data_, last - first, that, 0u, M - 1u);+  }++  /**+   * Construct a new string by replacing the characters denoted by the half-open+   *   range [`first`,`last`) within this string with the `that_count`+   *   characters from the character array `that` starting at position+   *   `that_pos`.+   * \pre `strlen(that) == M-1`+   * \note Equivalent to+   *   `creplace(first - data(), last - first, that, that_pos, that_count)`+   */+  template <std::size_t M>+  constexpr BasicFixedString<Char, N + M - 1u> creplace(+      const Char* first,+      const Char* last,+      const Char (&that)[M],+      std::size_t that_pos,+      std::size_t that_count) const noexcept(false) {+    return creplace(first - data_, last - first, that, that_pos, that_count);+  }++  /**+   * Copies `min(count, size())` characters starting from offset `0`+   *   from this string into the buffer pointed to by `dest`.+   * \return The number of characters copied.+   */+  constexpr std::size_t copy(Char* dest, std::size_t count) const noexcept {+    return copy(dest, count, 0u);+  }++  /**+   * Copies `min(count, size() - pos)` characters starting from offset `pos`+   *   from this string into the buffer pointed to by `dest`.+   * \pre `pos <= size()`+   * \return The number of characters copied.+   * \throw std::out_of_range if `pos > size()`+   */+  constexpr std::size_t copy(+      Char* dest, std::size_t count, std::size_t pos) const noexcept(false) {+    detail::fixedstring::checkOverflow(pos, size_);+    for (std::size_t i = 0u; i < count; ++i) {+      if (i + pos == size_) {+        return size_;+      }+      dest[i] = data_[i + pos];+    }+    return count;+  }++  /** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** **+   * Resizes the current string.+   * \note Equivalent to `resize(count, Char(0))`+   */+  constexpr void resize(std::size_t count) noexcept(false) {+    resize(count, Char(0));+  }++  /**+   * Resizes the current string by setting the size to `count` and setting+   *   `data()[count]` to `Char(0)`. If `count > old_size`, the characters+   *   in the range [`old_size`,`count`) are set to `ch`.+   */+  constexpr void resize(std::size_t count, Char ch) noexcept(false) {+    detail::fixedstring::checkOverflow(count, N);+    if (count == size_) {+    } else if (count < size_) {+      size_ = count;+      data_[size_] = Char(0);+    } else {+      for (; size_ < count; ++size_) {+        data_[size_] = ch;+      }+      data_[size_] = Char(0);+    }+  }++  /** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** **+   * Finds the first occurrence of the character sequence `that` in this string.+   * \note Equivalent to `find(that.data(), 0, that.size())`+   */+  template <std::size_t M>+  constexpr std::size_t find(+      const BasicFixedString<Char, M>& that) const noexcept {+    return find(that, 0u);+  }++  /**+   * Finds the first occurrence of the character sequence `that` in this string,+   *   starting at offset `pos`.+   * \pre `pos <= size()`+   * \note Equivalent to `find(that.data(), pos, that.size())`+   */+  template <std::size_t M>+  constexpr std::size_t find(+      const BasicFixedString<Char, M>& that, std::size_t pos) const+      noexcept(false) {+    return that.size_ <= size_ - detail::fixedstring::checkOverflow(pos, size_)+        ? detail::fixedstring::find_(data_, size_, that.data_, pos, that.size_)+        : npos;+  }++  /**+   * Finds the first occurrence of the character sequence `that` in this string.+   * \note Equivalent to `find(that.data(), 0, strlen(that))`+   */+  constexpr std::size_t find(const Char* that) const noexcept {+    return find(that, 0u, folly::constexpr_strlen(that));+  }++  /**+   * Finds the first occurrence of the character sequence `that` in this string,+   *   starting at offset `pos`.+   * \pre `pos <= size()`+   * \note Equivalent to `find(that.data(), pos, strlen(that))`+   */+  constexpr std::size_t find(const Char* that, std::size_t pos) const+      noexcept(false) {+    return find(that, pos, folly::constexpr_strlen(that));+  }++  /**+   * Finds the first occurrence of the first `count` characters in the buffer+   *   pointed to by `that` in this string, starting at offset `pos`.+   * \pre `pos <= size()`+   * \pre `that` points to a buffer containing at least `count` contiguous+   *   characters.+   * \return The lowest offset `i` such that `i >= pos` and+   *   `0 == strncmp(data() + i, that, count)`; or `npos` if there is no such+   *   offset `i`.+   * \throw std::out_of_range when `pos > size()`+   */+  constexpr std::size_t find(+      const Char* that, std::size_t pos, std::size_t count) const+      noexcept(false) {+    return count <= size_ - detail::fixedstring::checkOverflow(pos, size_)+        ? detail::fixedstring::find_(data_, size_, that, pos, count)+        : npos;+  }++  /**+   * Finds the first occurrence of the character `ch` in this string.+   * \note Equivalent to `find(&ch, 0, 1)`+   */+  constexpr std::size_t find(Char ch) const noexcept { return find(ch, 0u); }++  /**+   * Finds the first occurrence of the character character `c` in this string,+   *   starting at offset `pos`.+   * \pre `pos <= size()`+   * \note Equivalent to `find(&ch, pos, 1)`+   */+  constexpr std::size_t find(Char ch, std::size_t pos) const noexcept(false) {+    using A = const Char[1u];+    return 0u == size_ - detail::fixedstring::checkOverflow(pos, size_)+        ? npos+        : detail::fixedstring::find_(data_, size_, A{ch}, pos, 1u);+  }++  /** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** **+   * Finds the last occurrence of characters in the string+   *   `that` in this string.+   * \note Equivalent to `rfind(that.data(), size(), that.size())`+   */+  template <std::size_t M>+  constexpr std::size_t rfind(+      const BasicFixedString<Char, M>& that) const noexcept {+    return rfind(that, size_);+  }++  /**+   * Finds the last occurrence of characters in the string+   *   `that` in this string, starting at offset `pos`.+   * \note Equivalent to `rfind(that.data(), pos, that.size())`+   */+  template <std::size_t M>+  constexpr std::size_t rfind(+      const BasicFixedString<Char, M>& that, std::size_t pos) const+      noexcept(false) {+    return that.size_ <= size_+        ? detail::fixedstring::rfind_(+              data_,+              that.data_,+              folly::constexpr_min(+                  detail::fixedstring::checkOverflow(pos, size_),+                  size_ - that.size_),+              that.size_)+        : npos;+  }++  /**+   * Finds the last occurrence of characters in the buffer+   *   pointed to by `that` in this string.+   * \note Equivalent to `rfind(that, size(), strlen(that))`+   */+  constexpr std::size_t rfind(const Char* that) const noexcept {+    return rfind(that, size_, folly::constexpr_strlen(that));+  }++  /**+   * Finds the last occurrence of characters in the buffer+   *   pointed to by `that` in this string, starting at offset `pos`.+   * \note Equivalent to `rfind(that, pos, strlen(that))`+   */+  constexpr std::size_t rfind(const Char* that, std::size_t pos) const+      noexcept(false) {+    return rfind(that, pos, folly::constexpr_strlen(that));+  }++  /**+   * Finds the last occurrence of the first `count` characters in the buffer+   *   pointed to by `that` in this string, starting at offset `pos`.+   * \pre `pos <= size()`+   * \pre `that` points to a buffer containing at least `count` contiguous+   *   characters.+   * \return The largest offset `i` such that `i <= pos` and+   *   `i + count <= size()` and `0 == strncmp(data() + i, that, count)`; or+   *   `npos` if there is no such offset `i`.+   * \throw std::out_of_range when `pos > size()`+   */+  constexpr std::size_t rfind(+      const Char* that, std::size_t pos, std::size_t count) const+      noexcept(false) {+    return count <= size_+        ? detail::fixedstring::rfind_(+              data_,+              that,+              folly::constexpr_min(+                  detail::fixedstring::checkOverflow(pos, size_),+                  size_ - count),+              count)+        : npos;+  }++  /**+   * Finds the last occurrence of the character character `ch` in this string.+   * \note Equivalent to `rfind(&ch, size(), 1)`+   */+  constexpr std::size_t rfind(Char ch) const noexcept {+    return rfind(ch, size_);+  }++  /**+   * Finds the last occurrence of the character character `ch` in this string,+   *   starting at offset `pos`.+   * \pre `pos <= size()`+   * \note Equivalent to `rfind(&ch, pos, 1)`+   */+  constexpr std::size_t rfind(Char ch, std::size_t pos) const noexcept(false) {+    using A = const Char[1u];+    return 0u == size_+        ? npos+        : detail::fixedstring::rfind_(+              data_,+              A{ch},+              folly::constexpr_min(+                  detail::fixedstring::checkOverflow(pos, size_), size_ - 1u),+              1u);+  }++  /** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** **+   * Finds the first occurrence of any character in `that` in this string.+   * \note Equivalent to `find_first_of(that.data(), 0, that.size())`+   */+  template <std::size_t M>+  constexpr std::size_t find_first_of(+      const BasicFixedString<Char, M>& that) const noexcept {+    return find_first_of(that, 0u);+  }++  /**+   * Finds the first occurrence of any character in `that` in this string,+   *   starting at offset `pos`+   * \note Equivalent to `find_first_of(that.data(), pos, that.size())`+   */+  template <std::size_t M>+  constexpr std::size_t find_first_of(+      const BasicFixedString<Char, M>& that, std::size_t pos) const+      noexcept(false) {+    return size_ == detail::fixedstring::checkOverflow(pos, size_)+        ? npos+        : detail::fixedstring::find_first_of_(+              data_, size_, that.data_, pos, that.size_);+  }++  /**+   * Finds the first occurrence of any character in the null-terminated+   *   character sequence pointed to by `that` in this string.+   * \note Equivalent to `find_first_of(that, 0, strlen(that))`+   */+  constexpr std::size_t find_first_of(const Char* that) const noexcept {+    return find_first_of(that, 0u, folly::constexpr_strlen(that));+  }++  /**+   * Finds the first occurrence of any character in the null-terminated+   *   character sequence pointed to by `that` in this string,+   *   starting at offset `pos`+   * \note Equivalent to `find_first_of(that, pos, strlen(that))`+   */+  constexpr std::size_t find_first_of(const Char* that, std::size_t pos) const+      noexcept(false) {+    return find_first_of(that, pos, folly::constexpr_strlen(that));+  }++  /**+   * Finds the first occurrence of any character in the first `count` characters+   *   in the buffer pointed to by `that` in this string, starting at offset+   *  `pos`.+   * \pre `pos <= size()`+   * \pre `that` points to a buffer containing at least `count` contiguous+   *   characters.+   * \return The smallest offset `i` such that `i >= pos` and+   *   `std::find(that, that+count, at(i)) != that+count`; or+   *   `npos` if there is no such offset `i`.+   * \throw std::out_of_range when `pos > size()`+   */+  constexpr std::size_t find_first_of(+      const Char* that, std::size_t pos, std::size_t count) const+      noexcept(false) {+    return size_ == detail::fixedstring::checkOverflow(pos, size_)+        ? npos+        : detail::fixedstring::find_first_of_(data_, size_, that, pos, count);+  }++  /**+   * Finds the first occurrence of `ch` in this string.+   * \note Equivalent to `find_first_of(&ch, 0, 1)`+   */+  constexpr std::size_t find_first_of(Char ch) const noexcept {+    return find_first_of(ch, 0u);+  }++  /**+   * Finds the first occurrence of `ch` in this string,+   *   starting at offset `pos`.+   * \note Equivalent to `find_first_of(&ch, pos, 1)`+   */+  constexpr std::size_t find_first_of(Char ch, std::size_t pos) const+      noexcept(false) {+    using A = const Char[1u];+    return size_ == detail::fixedstring::checkOverflow(pos, size_)+        ? npos+        : detail::fixedstring::find_first_of_(data_, size_, A{ch}, pos, 1u);+  }++  /** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** **+   * Finds the first occurrence of any character not in `that` in this string.+   * \note Equivalent to `find_first_not_of(that.data(), 0, that.size())`+   */+  template <std::size_t M>+  constexpr std::size_t find_first_not_of(+      const BasicFixedString<Char, M>& that) const noexcept {+    return find_first_not_of(that, 0u);+  }++  /** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** **+   * Finds the first occurrence of any character not in `that` in this string.+   * \note Equivalent to `find_first_not_of(that.data(), 0, that.size())`+   */+  template <std::size_t M>+  constexpr std::size_t find_first_not_of(+      const BasicFixedString<Char, M>& that, std::size_t pos) const+      noexcept(false) {+    return size_ == detail::fixedstring::checkOverflow(pos, size_)+        ? npos+        : detail::fixedstring::find_first_not_of_(+              data_, size_, that.data_, pos, that.size_);+  }++  /**+   * Finds the first occurrence of any character not in the null-terminated+   *   character sequence pointed to by `that` in this string.+   * \note Equivalent to `find_first_not_of(that, 0, strlen(that))`+   */+  constexpr std::size_t find_first_not_of(const Char* that) const noexcept {+    return find_first_not_of(that, 0u, folly::constexpr_strlen(that));+  }++  /**+   * Finds the first occurrence of any character not in the null-terminated+   *   character sequence pointed to by `that` in this string,+   *   starting at offset `pos`+   * \note Equivalent to `find_first_not_of(that, pos, strlen(that))`+   */+  constexpr std::size_t find_first_not_of(+      const Char* that, std::size_t pos) const noexcept(false) {+    return find_first_not_of(that, pos, folly::constexpr_strlen(that));+  }++  /**+   * Finds the first occurrence of any character not in the first `count`+   *   characters in the buffer pointed to by `that` in this string, starting at+   *   offset `pos`.+   * \pre `pos <= size()`+   * \pre `that` points to a buffer containing at least `count` contiguous+   *   characters.+   * \return The smallest offset `i` such that `i >= pos` and+   *   `std::find(that, that+count, at(i)) == that+count`; or+   *   `npos` if there is no such offset `i`.+   * \throw std::out_of_range when `pos > size()`+   */+  constexpr std::size_t find_first_not_of(+      const Char* that, std::size_t pos, std::size_t count) const+      noexcept(false) {+    return size_ == detail::fixedstring::checkOverflow(pos, size_)+        ? npos+        : detail::fixedstring::find_first_not_of_(+              data_, size_, that, pos, count);+  }++  /**+   * Finds the first occurrence of any character other than `ch` in this string.+   * \note Equivalent to `find_first_not_of(&ch, 0, 1)`+   */+  constexpr std::size_t find_first_not_of(Char ch) const noexcept {+    return find_first_not_of(ch, 0u);+  }++  /**+   * Finds the first occurrence of any character other than `ch` in this string,+   *   starting at offset `pos`.+   * \note Equivalent to `find_first_not_of(&ch, pos, 1)`+   */+  constexpr std::size_t find_first_not_of(Char ch, std::size_t pos) const+      noexcept(false) {+    using A = const Char[1u];+    return 1u <= size_ - detail::fixedstring::checkOverflow(pos, size_)+        ? detail::fixedstring::find_first_not_of_(data_, size_, A{ch}, pos, 1u)+        : npos;+  }++  /** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** **+   * Finds the last occurrence of any character in `that` in this string.+   * \note Equivalent to `find_last_of(that.data(), size(), that.size())`+   */+  template <std::size_t M>+  constexpr std::size_t find_last_of(+      const BasicFixedString<Char, M>& that) const noexcept {+    return find_last_of(that, size_);+  }++  /**+   * Finds the last occurrence of any character in `that` in this string,+   *   starting at offset `pos`+   * \note Equivalent to `find_last_of(that.data(), pos, that.size())`+   */+  template <std::size_t M>+  constexpr std::size_t find_last_of(+      const BasicFixedString<Char, M>& that, std::size_t pos) const+      noexcept(false) {+    return 0u == size_+        ? npos+        : detail::fixedstring::find_last_of_(+              data_,+              that.data_,+              folly::constexpr_min(+                  detail::fixedstring::checkOverflow(pos, size_), size_ - 1u),+              that.size_);+  }++  /**+   * Finds the last occurrence of any character in the null-terminated+   *   character sequence pointed to by `that` in this string.+   * \note Equivalent to `find_last_of(that, size(), strlen(that))`+   */+  constexpr std::size_t find_last_of(const Char* that) const noexcept {+    return find_last_of(that, size_, folly::constexpr_strlen(that));+  }++  /**+   * Finds the last occurrence of any character in the null-terminated+   *   character sequence pointed to by `that` in this string,+   *   starting at offset `pos`+   * \note Equivalent to `find_last_of(that, pos, strlen(that))`+   */+  constexpr std::size_t find_last_of(const Char* that, std::size_t pos) const+      noexcept(false) {+    return find_last_of(that, pos, folly::constexpr_strlen(that));+  }++  /**+   * Finds the last occurrence of any character in the first `count` characters+   *   in the buffer pointed to by `that` in this string, starting at offset+   *  `pos`.+   * \pre `pos <= size()`+   * \pre `that` points to a buffer containing at least `count` contiguous+   *   characters.+   * \return The largest offset `i` such that `i <= pos` and+   *   `i < size()` and `std::find(that, that+count, at(i)) != that+count`; or+   *   `npos` if there is no such offset `i`.+   * \throw std::out_of_range when `pos > size()`+   */+  constexpr std::size_t find_last_of(+      const Char* that, std::size_t pos, std::size_t count) const+      noexcept(false) {+    return 0u == size_+        ? npos+        : detail::fixedstring::find_last_of_(+              data_,+              that,+              folly::constexpr_min(+                  detail::fixedstring::checkOverflow(pos, size_), size_ - 1u),+              count);+  }++  /**+   * Finds the last occurrence of `ch` in this string.+   * \note Equivalent to `find_last_of(&ch, size(), 1)`+   */+  constexpr std::size_t find_last_of(Char ch) const noexcept {+    return find_last_of(ch, size_);+  }++  /**+   * Finds the last occurrence of `ch` in this string,+   *   starting at offset `pos`.+   * \note Equivalent to `find_last_of(&ch, pos, 1)`+   */+  constexpr std::size_t find_last_of(Char ch, std::size_t pos) const+      noexcept(false) {+    using A = const Char[1u];+    return 0u == size_+        ? npos+        : detail::fixedstring::find_last_of_(+              data_,+              A{ch},+              folly::constexpr_min(+                  detail::fixedstring::checkOverflow(pos, size_), size_ - 1u),+              1u);+  }++  /** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** **+   * Finds the last occurrence of any character not in `that` in this string.+   * \note Equivalent to `find_last_not_of(that.data(), size(), that.size())`+   */+  template <std::size_t M>+  constexpr std::size_t find_last_not_of(+      const BasicFixedString<Char, M>& that) const noexcept {+    return find_last_not_of(that, size_);+  }++  /**+   * Finds the last occurrence of any character not in `that` in this string,+   *   starting at offset `pos`+   * \note Equivalent to `find_last_not_of(that.data(), pos, that.size())`+   */+  template <std::size_t M>+  constexpr std::size_t find_last_not_of(+      const BasicFixedString<Char, M>& that, std::size_t pos) const+      noexcept(false) {+    return 0u == size_+        ? npos+        : detail::fixedstring::find_last_not_of_(+              data_,+              that.data_,+              folly::constexpr_min(+                  detail::fixedstring::checkOverflow(pos, size_), size_ - 1u),+              that.size_);+  }++  /**+   * Finds the last occurrence of any character not in the null-terminated+   *   character sequence pointed to by `that` in this string.+   * \note Equivalent to `find_last_not_of(that, size(), strlen(that))`+   */+  constexpr std::size_t find_last_not_of(const Char* that) const noexcept {+    return find_last_not_of(that, size_, folly::constexpr_strlen(that));+  }++  /**+   * Finds the last occurrence of any character not in the null-terminated+   *   character sequence pointed to by `that` in this string,+   *   starting at offset `pos`+   * \note Equivalent to `find_last_not_of(that, pos, strlen(that))`+   */+  constexpr std::size_t find_last_not_of(+      const Char* that, std::size_t pos) const noexcept(false) {+    return find_last_not_of(that, pos, folly::constexpr_strlen(that));+  }++  /**+   * Finds the last occurrence of any character not in the first `count`+   *   characters in the buffer pointed to by `that` in this string, starting at+   *   offset `pos`.+   * \pre `pos <= size()`+   * \pre `that` points to a buffer containing at least `count` contiguous+   *   characters.+   * \return The largest offset `i` such that `i <= pos` and+   *   `i < size()` and `std::find(that, that+count, at(i)) == that+count`; or+   *   `npos` if there is no such offset `i`.+   * \throw std::out_of_range when `pos > size()`+   */+  constexpr std::size_t find_last_not_of(+      const Char* that, std::size_t pos, std::size_t count) const+      noexcept(false) {+    return 0u == size_+        ? npos+        : detail::fixedstring::find_last_not_of_(+              data_,+              that,+              folly::constexpr_min(+                  detail::fixedstring::checkOverflow(pos, size_), size_ - 1u),+              count);+  }++  /**+   * Finds the last occurrence of any character other than `ch` in this string.+   * \note Equivalent to `find_last_not_of(&ch, size(), 1)`+   */+  constexpr std::size_t find_last_not_of(Char ch) const noexcept {+    return find_last_not_of(ch, size_);+  }++  /**+   * Finds the last occurrence of any character other than `ch` in this string,+   *   starting at offset `pos`.+   * \note Equivalent to `find_last_not_of(&ch, pos, 1)`+   */+  constexpr std::size_t find_last_not_of(Char ch, std::size_t pos) const+      noexcept(false) {+    using A = const Char[1u];+    return 0u == size_+        ? npos+        : detail::fixedstring::find_last_not_of_(+              data_,+              A{ch},+              folly::constexpr_min(+                  detail::fixedstring::checkOverflow(pos, size_), size_ - 1u),+              1u);+  }++  /** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** **+   * Asymmetric relational operators+   */+  friend constexpr bool operator==(+      const Char* a, const BasicFixedString& b) noexcept {+    return detail::fixedstring::equal_(+        a, folly::constexpr_strlen(a), b.data_, b.size_);+  }++  /**+   * \overload+   */+  friend constexpr bool operator==(+      const BasicFixedString& a, const Char* b) noexcept {+    return b == a;+  }++  /**+   * \overload+   */+  friend constexpr bool operator==(+      Range<const Char*> a, const BasicFixedString& b) noexcept {+    return detail::fixedstring::equal_(a.begin(), a.size(), b.data_, b.size_);+  }++  /**+   * \overload+   */+  friend constexpr bool operator==(+      const BasicFixedString& a, Range<const Char*> b) noexcept {+    return b == a;+  }++  friend constexpr bool operator!=(+      const Char* a, const BasicFixedString& b) noexcept {+    return !(a == b);+  }++  /**+   * \overload+   */+  friend constexpr bool operator!=(+      const BasicFixedString& a, const Char* b) noexcept {+    return !(b == a);+  }++  /**+   * \overload+   */+  friend constexpr bool operator!=(+      Range<const Char*> a, const BasicFixedString& b) noexcept {+    return !(a == b);+  }++  /**+   * \overload+   */+  friend constexpr bool operator!=(+      const BasicFixedString& a, Range<const Char*> b) noexcept {+    return !(a == b);+  }++  friend constexpr bool operator<(+      const Char* a, const BasicFixedString& b) noexcept {+    return ordering::lt ==+        detail::fixedstring::compare_(+               a, 0u, folly::constexpr_strlen(a), b.data_, 0u, b.size_);+  }++  /**+   * \overload+   */+  friend constexpr bool operator<(+      const BasicFixedString& a, const Char* b) noexcept {+    return ordering::lt ==+        detail::fixedstring::compare_(+               a.data_, 0u, a.size_, b, 0u, folly::constexpr_strlen(b));+  }++  /**+   * \overload+   */+  friend constexpr bool operator<(+      Range<const Char*> a, const BasicFixedString& b) noexcept {+    return ordering::lt ==+        detail::fixedstring::compare_(+               a.begin(), 0u, a.size(), b.data_, 0u, b.size_);+  }++  /**+   * \overload+   */+  friend constexpr bool operator<(+      const BasicFixedString& a, Range<const Char*> b) noexcept {+    return ordering::lt ==+        detail::fixedstring::compare_(+               a.data_, 0u, a.size_, b.begin(), 0u, b.size());+  }++  friend constexpr bool operator>(+      const Char* a, const BasicFixedString& b) noexcept {+    return b < a;+  }++  /**+   * \overload+   */+  friend constexpr bool operator>(+      const BasicFixedString& a, const Char* b) noexcept {+    return b < a;+  }++  /**+   * \overload+   */+  friend constexpr bool operator>(+      Range<const Char*> a, const BasicFixedString& b) noexcept {+    return b < a;+  }++  /**+   * \overload+   */+  friend constexpr bool operator>(+      const BasicFixedString& a, Range<const Char*> b) noexcept {+    return b < a;+  }++  friend constexpr bool operator<=(+      const Char* a, const BasicFixedString& b) noexcept {+    return !(b < a);+  }++  /**+   * \overload+   */+  friend constexpr bool operator<=(+      const BasicFixedString& a, const Char* b) noexcept {+    return !(b < a);+  }++  /**+   * \overload+   */+  friend constexpr bool operator<=(+      Range<const Char*> const& a, const BasicFixedString& b) noexcept {+    return !(b < a);+  }++  /**+   * \overload+   */+  friend constexpr bool operator<=(+      const BasicFixedString& a, Range<const Char*> b) noexcept {+    return !(b < a);+  }++  friend constexpr bool operator>=(+      const Char* a, const BasicFixedString& b) noexcept {+    return !(a < b);+  }++  /**+   * \overload+   */+  friend constexpr bool operator>=(+      const BasicFixedString& a, const Char* b) noexcept {+    return !(a < b);+  }++  /**+   * \overload+   */+  friend constexpr bool operator>=(+      Range<const Char*> a, const BasicFixedString& b) noexcept {+    return !(a < b);+  }++  /**+   * \overload+   */+  friend constexpr bool operator>=(+      const BasicFixedString& a, Range<const Char*> const& b) noexcept {+    return !(a < b);+  }++  /** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** **+   * Asymmetric concatenation+   */+  template <std::size_t M>+  friend constexpr BasicFixedString<Char, N + M - 1u> operator+(+      const Char (&a)[M], const BasicFixedString& b) noexcept {+    return detail::fixedstring::Helper::concat_<Char>(+        detail::fixedstring::checkNullTerminated(a),+        M - 1u,+        b.data_,+        b.size_,+        std::make_index_sequence<N + M - 1u>{});+  }++  /**+   * \overload+   */+  template <std::size_t M>+  friend constexpr BasicFixedString<Char, N + M - 1u> operator+(+      const BasicFixedString& a, const Char (&b)[M]) noexcept {+    return detail::fixedstring::Helper::concat_<Char>(+        a.data_,+        a.size_,+        detail::fixedstring::checkNullTerminated(b),+        M - 1u,+        std::make_index_sequence<N + M - 1u>{});+  }++  /**+   * \overload+   */+  friend constexpr BasicFixedString<Char, N + 1u> operator+(+      Char a, const BasicFixedString& b) noexcept {+    using A = const Char[2u];+    return detail::fixedstring::Helper::concat_<Char>(+        A{a, Char(0)},+        1u,+        b.data_,+        b.size_,+        std::make_index_sequence<N + 1u>{});+  }++  /**+   * \overload+   */+  friend constexpr BasicFixedString<Char, N + 1u> operator+(+      const BasicFixedString& a, Char b) noexcept {+    using A = const Char[2u];+    return detail::fixedstring::Helper::concat_<Char>(+        a.data_,+        a.size_,+        A{b, Char(0)},+        1u,+        std::make_index_sequence<N + 1u>{});+  }+};++template <class C, std::size_t N>+inline std::basic_ostream<C>& operator<<(+    std::basic_ostream<C>& os, const BasicFixedString<C, N>& string) {+  using StreamSize = decltype(os.width());+  os.write(string.begin(), static_cast<StreamSize>(string.size()));+  return os;+}++/** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** **+ * Symmetric relational operators+ */+template <class Char, std::size_t A, std::size_t B>+constexpr bool operator==(+    const BasicFixedString<Char, A>& a,+    const BasicFixedString<Char, B>& b) noexcept {+  return detail::fixedstring::equal_(+      detail::fixedstring::Helper::data_(a),+      a.size(),+      detail::fixedstring::Helper::data_(b),+      b.size());+}++template <class Char, std::size_t A, std::size_t B>+constexpr bool operator!=(+    const BasicFixedString<Char, A>& a, const BasicFixedString<Char, B>& b) {+  return !(a == b);+}++template <class Char, std::size_t A, std::size_t B>+constexpr bool operator<(+    const BasicFixedString<Char, A>& a,+    const BasicFixedString<Char, B>& b) noexcept {+  return ordering::lt ==+      detail::fixedstring::compare_(+             detail::fixedstring::Helper::data_(a),+             0u,+             a.size(),+             detail::fixedstring::Helper::data_(b),+             0u,+             b.size());+}++template <class Char, std::size_t A, std::size_t B>+constexpr bool operator>(+    const BasicFixedString<Char, A>& a,+    const BasicFixedString<Char, B>& b) noexcept {+  return b < a;+}++template <class Char, std::size_t A, std::size_t B>+constexpr bool operator<=(+    const BasicFixedString<Char, A>& a,+    const BasicFixedString<Char, B>& b) noexcept {+  return !(b < a);+}++template <class Char, std::size_t A, std::size_t B>+constexpr bool operator>=(+    const BasicFixedString<Char, A>& a,+    const BasicFixedString<Char, B>& b) noexcept {+  return !(a < b);+}++/** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** **+ * Symmetric concatenation+ */+template <class Char, std::size_t N, std::size_t M>+constexpr BasicFixedString<Char, N + M> operator+(+    const BasicFixedString<Char, N>& a,+    const BasicFixedString<Char, M>& b) noexcept {+  return detail::fixedstring::Helper::concat_<Char>(+      detail::fixedstring::Helper::data_(a),+      a.size(),+      detail::fixedstring::Helper::data_(b),+      b.size(),+      std::make_index_sequence<N + M>{});+}++/** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** **+ * Construct a `BasicFixedString` object from a null-terminated array of+ * characters. The capacity and size of the string will be equal to one less+ * than the size of the array.+ * \pre `a` contains no embedded null characters.+ * \pre `a[N-1] == Char(0)`+ * \post For a returned string `s`, `s[i]==a[i]` for every `i` in [`0`,`N-1`].+ */+template <class Char, std::size_t N>+constexpr BasicFixedString<Char, N - 1u> makeFixedString(+    const Char (&a)[N]) noexcept {+  return {a};+}++/** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** **+ * Swap function+ */+template <class Char, std::size_t N>+constexpr void swap(+    BasicFixedString<Char, N>& a, BasicFixedString<Char, N>& b) noexcept {+  a.swap(b);+}++inline namespace literals {+inline namespace string_literals {+inline namespace {+// "const std::size_t&" is so that folly::npos has the same address in every+// translation unit. This is to avoid potential violations of the ODR.+constexpr const std::size_t& npos = detail::fixedstring::FixedStringBase::npos;+} // namespace++#if defined(__GNUC__) && !defined(__ICC)+#pragma GCC diagnostic push+#pragma GCC diagnostic ignored "-Wpragmas"+#pragma GCC diagnostic ignored "-Wgnu-string-literal-operator-template"++/** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** *+ * User-defined literals for creating FixedString objects from string literals+ * on the compilers that support it.+ *+ * \par Example:+ * \par+ * \code+ * using namespace folly::string_literals;+ * constexpr auto hello = "hello world!"_fs;+ * \endcode+ *+ * \note This requires a GNU compiler extension+ *   (-Wgnu-string-literal-operator-template) supported by clang and gcc,+ *   proposed for standardization in+ *   <http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2016/p0424r0.pdf>.+ *   \par+ *   For portable code, prefer the suffixes `_fs4`, `_fs8`, `_fs16`, `_fs32`,+ *   `_fs64`, and `_fs128` for creating instances of types `FixedString<4>`,+ *   `FixedString<8>`, `FixedString<16>`, etc.+ */+template <class Char, Char... Cs>+constexpr BasicFixedString<Char, sizeof...(Cs)> operator""_fs() noexcept {+  const Char a[] = {Cs..., Char(0)};+  return {+a, sizeof...(Cs)};+}++#pragma GCC diagnostic pop+#endif++#ifndef NO_FIXED_STR_UDL+#define FOLLY_DEFINE_FIXED_STRING_UDL(N)                     \+  constexpr FixedString<N> operator""_fs##N(                 \+      const char* that, std::size_t count) noexcept(false) { \+    return {that, count};                                    \+  }                                                          \+/**/++// Define UDLs _fs4, _fs8, _fs16, etc for FixedString<[4, 8, 16, ...]>+FOLLY_DEFINE_FIXED_STRING_UDL(4)+FOLLY_DEFINE_FIXED_STRING_UDL(8)+FOLLY_DEFINE_FIXED_STRING_UDL(16)+FOLLY_DEFINE_FIXED_STRING_UDL(32)+FOLLY_DEFINE_FIXED_STRING_UDL(64)+FOLLY_DEFINE_FIXED_STRING_UDL(128)++#undef FOLLY_DEFINE_FIXED_STRING_UDL+#endif+} // namespace string_literals+} // namespace literals++// TODO:+// // numeric conversions:+// template <std::size_t N>+// constexpr int stoi(const FixedString<N>& str, int base = 10);+// template <std::size_t N>+// constexpr unsigned stou(const FixedString<N>& str, int base = 10);+// template <std::size_t N>+// constexpr long stol(const FixedString<N>& str, int base = 10);+// template <std::size_t N>+// constexpr unsigned long stoul(const FixedString<N>& str, int base = 10;+// template <std::size_t N>+// constexpr long long stoll(const FixedString<N>& str, int base = 10);+// template <std::size_t N>+// constexpr unsigned long long stoull(const FixedString<N>& str,+// int base = 10);+// template <std::size_t N>+// constexpr float stof(const FixedString<N>& str);+// template <std::size_t N>+// constexpr double stod(const FixedString<N>& str);+// template <std::size_t N>+// constexpr long double stold(const FixedString<N>& str);+// template <int val>+// constexpr FixedString</*...*/> to_fixed_string_i() noexcept;+// template <unsigned val>+// constexpr FixedString</*...*/> to_fixed_string_u() noexcept;+// template <long val>+// constexpr FixedString</*...*/> to_fixed_string_l() noexcept;+// template <unsigned long val>+// constexpr FixedString</*...*/> to_fixed_string_ul() noexcept;+// template <long long val>+// constexpr FixedString</*...*/> to_fixed_string_ll() noexcept+// template <unsigned long long val>+// constexpr FixedString</*...*/> to_fixed_string_ull() noexcept;+} // namespace folly
+ folly/folly/FmtUtility.cpp view
@@ -0,0 +1,68 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#include <folly/FmtUtility.h>++#include <folly/Range.h>+#include <folly/String.h>+#include <folly/ssl/OpenSSLHash.h>++namespace folly {++std::string fmt_vformat_mangle_name_fn::operator()(+    std::string_view const key) const {+  auto& self = *this;+  std::string out;+  self(out, key);+  return out;+}++void fmt_vformat_mangle_name_fn::operator()(+    std::string& out, std::string_view const key) const {+  auto const keyr = folly::ByteRange(folly::StringPiece(key));+  uint8_t enc[32];+  auto const encr = folly::MutableByteRange{std::begin(enc), std::end(enc)};+#if FOLLY_OPENSSL_HAS_BLAKE2B+  folly::ssl::OpenSSLHash::blake2s256(encr, keyr);+#else+  folly::ssl::OpenSSLHash::sha256(encr, keyr);+#endif+  out.push_back('_');+  folly::hexlify(encr, out, true);+}++std::string fmt_vformat_mangle_format_string_fn::operator()(+    std::string_view const str) const {+  return operator()(options{}, str);+}++std::string fmt_vformat_mangle_format_string_fn::operator()(+    options const& opts, std::string_view const str) const {+  auto const fe_opts =+      format_string_for_each_named_arg_options{} //+          .set_numeric_args_as_named(opts.numeric_args_as_named);+  std::string out;+  char const* pos = str.data();+  format_string_for_each_named_arg(fe_opts, str, [&](auto const arg) {+    out.append(pos, arg.data());+    fmt_vformat_mangle_name(out, arg);+    pos = arg.data() + arg.size();+  });+  out.append(pos, str.data() + str.size());+  return out;+}++} // namespace folly
+ folly/folly/FmtUtility.h view
@@ -0,0 +1,83 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <fmt/args.h>++#include <folly/CppAttributes.h>++namespace folly {++/// fmt_make_format_args_from_map_fn+/// fmt_make_format_args_from_map+///+/// A helper function-object type and variable for making a format-args object+/// from a map.+///+/// May be useful for transitioning from legacy folly::svformat to fmt::vformat.+struct fmt_make_format_args_from_map_fn {+  template <typename Map>+  fmt::dynamic_format_arg_store<fmt::format_context> operator()(+      [[FOLLY_ATTR_CLANG_LIFETIMEBOUND]] Map const& map) const {+    fmt::dynamic_format_arg_store<fmt::format_context> ret;+    ret.reserve(map.size(), map.size());+    for (auto const& [key, val] : map) {+      ret.push_back(fmt::arg(key.c_str(), std::cref(val)));+    }+    return ret;+  }+};+inline constexpr fmt_make_format_args_from_map_fn+    fmt_make_format_args_from_map{};++/// fmt_vformat_mangle_name_fn+/// fmt_vformat_mangle_name+///+/// A helper function-object type and variable for mangling vformat named-arg+/// names which fmt::vformat might not otherwise permit.+struct fmt_vformat_mangle_name_fn {+  std::string operator()(std::string_view const str) const;+  void operator()(std::string& out, std::string_view const str) const;+};+inline constexpr fmt_vformat_mangle_name_fn fmt_vformat_mangle_name{};++/// fmt_vformat_mangle_format_string_fn+/// fmt_vformat_mangle_format_string+///+/// A helper function-object type and variable for mangling the content of+/// vformat format-strings containing named-arg names which fmt::vformat might+/// not otherwise permit.+struct fmt_vformat_mangle_format_string_fn {+  struct options {+    bool numeric_args_as_named = false;++    options& set_numeric_args_as_named(bool value) noexcept {+      numeric_args_as_named = value;+      return *this;+    }+  };++  std::string operator()(std::string_view const str) const;+  std::string operator()(options const& opts, std::string_view const str) const;+};+inline constexpr fmt_vformat_mangle_format_string_fn+    fmt_vformat_mangle_format_string{};++using fmt_vformat_mangle_format_string_options =+    fmt_vformat_mangle_format_string_fn::options;++} // namespace folly
+ folly/folly/FollyMemcpy.cpp view
@@ -0,0 +1,30 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#include <cstring>++#if !defined(__AVX2__) && !(defined(__linux__) && defined(__aarch64__))+namespace folly {++extern "C" void* __folly_memcpy(void* dst, const void* src, std::size_t size) {+  if (size == 0)+    return dst;+  return std::memmove(dst, src, size);+}++} // namespace folly++#endif
+ folly/folly/FollyMemcpy.h view
@@ -0,0 +1,23 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#include <cstddef>++namespace folly {++extern "C" void* __folly_memcpy(void* dst, const void* src, std::size_t size);++} // namespace folly
+ folly/folly/FollyMemset.cpp view
@@ -0,0 +1,29 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#include <cstring>++#if !defined(__AVX2__) && !(defined(__linux__) && defined(__aarch64__))++namespace folly {++extern "C" void* __folly_memset(void* dest, int ch, std::size_t count) {+  return std::memset(dest, ch, count);+}++} // namespace folly++#endif
+ folly/folly/FollyMemset.h view
@@ -0,0 +1,25 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <cstddef>++namespace folly {++extern "C" void* __folly_memset(void* dest, int ch, std::size_t count);++} // namespace folly
+ folly/folly/Format-inl.h view
@@ -0,0 +1,1151 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#ifndef FOLLY_FORMAT_H_+#error This file may only be included from Format.h.+#endif++#include <array>+#include <cinttypes>+#include <deque>+#include <map>+#include <unordered_map>+#include <vector>++#include <folly/Exception.h>+#include <folly/FormatTraits.h>+#include <folly/MapUtil.h>+#include <folly/Portability.h>+#include <folly/Traits.h>+#include <folly/lang/Exception.h>+#include <folly/lang/ToAscii.h>+#include <folly/portability/Windows.h>++// Ignore -Wformat-nonliteral and -Wconversion warnings within this file+FOLLY_PUSH_WARNING+FOLLY_GNU_DISABLE_WARNING("-Wformat-nonliteral")+FOLLY_GNU_DISABLE_WARNING("-Wconversion")++namespace folly {++namespace detail {++// Updates the end of the buffer after the comma separators have been added.+void insertThousandsGroupingUnsafe(char* start_buffer, char** end_buffer);++extern const std::array<std::array<char, 2>, 256> formatHexUpper;+extern const std::array<std::array<char, 2>, 256> formatHexLower;+extern const std::array<std::array<char, 3>, 512> formatOctal;+extern const std::array<std::array<char, 8>, 256> formatBinary;++const size_t kMaxHexLength = 2 * sizeof(uintmax_t);+const size_t kMaxOctalLength = 3 * sizeof(uintmax_t);+const size_t kMaxBinaryLength = 8 * sizeof(uintmax_t);++/**+ * Convert an unsigned to hex, using repr (which maps from each possible+ * 2-hex-bytes value to the 2-character representation).+ *+ * Just like folly::detail::uintToBuffer in Conv.h, writes at the *end* of+ * the supplied buffer and returns the offset of the beginning of the string+ * from the start of the buffer.  The formatted string will be in range+ * [buf+begin, buf+bufLen).+ */+template <class Uint>+size_t uintToHex(+    char* buffer,+    size_t bufLen,+    Uint v,+    std::array<std::array<char, 2>, 256> const& repr) {+  // 'v >>= 7, v >>= 1' is no more than a work around to get rid of shift size+  // warning when Uint = uint8_t (it's false as v >= 256 implies sizeof(v) > 1).+  for (; !less_than<unsigned, 256>(v); v >>= 7, v >>= 1) {+    auto b = v & 0xff;+    bufLen -= 2;+    buffer[bufLen] = repr[b][0];+    buffer[bufLen + 1] = repr[b][1];+  }+  buffer[--bufLen] = repr[v][1];+  if (v >= 16) {+    buffer[--bufLen] = repr[v][0];+  }+  return bufLen;+}++/**+ * Convert an unsigned to hex, using lower-case letters for the digits+ * above 9.  See the comments for uintToHex.+ */+template <class Uint>+inline size_t uintToHexLower(char* buffer, size_t bufLen, Uint v) {+  return uintToHex(buffer, bufLen, v, formatHexLower);+}++/**+ * Convert an unsigned to hex, using upper-case letters for the digits+ * above 9.  See the comments for uintToHex.+ */+template <class Uint>+inline size_t uintToHexUpper(char* buffer, size_t bufLen, Uint v) {+  return uintToHex(buffer, bufLen, v, formatHexUpper);+}++/**+ * Convert an unsigned to octal.+ *+ * Just like folly::detail::uintToBuffer in Conv.h, writes at the *end* of+ * the supplied buffer and returns the offset of the beginning of the string+ * from the start of the buffer.  The formatted string will be in range+ * [buf+begin, buf+bufLen).+ */+template <class Uint>+size_t uintToOctal(char* buffer, size_t bufLen, Uint v) {+  auto& repr = formatOctal;+  // 'v >>= 7, v >>= 2' is no more than a work around to get rid of shift size+  // warning when Uint = uint8_t (it's false as v >= 512 implies sizeof(v) > 1).+  for (; !less_than<unsigned, 512>(v); v >>= 7, v >>= 2) {+    auto b = v & 0x1ff;+    bufLen -= 3;+    buffer[bufLen] = repr[b][0];+    buffer[bufLen + 1] = repr[b][1];+    buffer[bufLen + 2] = repr[b][2];+  }+  buffer[--bufLen] = repr[v][2];+  if (v >= 8) {+    buffer[--bufLen] = repr[v][1];+  }+  if (v >= 64) {+    buffer[--bufLen] = repr[v][0];+  }+  return bufLen;+}++/**+ * Convert an unsigned to binary.+ *+ * Just like folly::detail::uintToBuffer in Conv.h, writes at the *end* of+ * the supplied buffer and returns the offset of the beginning of the string+ * from the start of the buffer.  The formatted string will be in range+ * [buf+begin, buf+bufLen).+ */+template <class Uint>+size_t uintToBinary(char* buffer, size_t bufLen, Uint v) {+  auto& repr = formatBinary;+  if (v == 0) {+    buffer[--bufLen] = '0';+    return bufLen;+  }+  for (; v; v >>= 7, v >>= 1) {+    auto b = v & 0xff;+    bufLen -= 8;+    memcpy(buffer + bufLen, &(repr[b][0]), 8);+  }+  while (buffer[bufLen] == '0') {+    ++bufLen;+  }+  return bufLen;+}++template <bool containerMode, bool RecordUsedArg, class Output>+void baseFormatterCallImpl(+    Output& out,+    size_t nargs,+    const int widths[],+    std::bool_constant<RecordUsedArg>(used)(const BaseFormatterBase&, size_t),+    BaseFormatterBase::DoFormatFn<Output>* const funs[],+    const BaseFormatterBase& base) {+  // Copy raw string (without format specifiers) to output;+  // not as simple as we'd like, as we still need to translate "}}" to "}"+  // and throw if we see any lone "}"+  auto outputString = [&out](StringPiece s) {+    auto p = s.begin();+    auto end = s.end();+    while (p != end) {+      auto q = static_cast<const char*>(memchr(p, '}', size_t(end - p)));+      if (!q) {+        out(StringPiece(p, end));+        break;+      }+      ++q;+      out(StringPiece(p, q));+      p = q;++      if (p == end || *p != '}') {+        throw_exception<BadFormatArg>(+            "folly::format: single '}' in format string");+      }+      ++p;+    }+  };++  auto str_ = base.str_;+  auto p = str_.begin();+  auto end = str_.end();++  int nextArg = 0;+  bool hasDefaultArgIndex = false;+  bool hasExplicitArgIndex = false;+  while (p != end) {+    auto q = static_cast<const char*>(memchr(p, '{', size_t(end - p)));+    if (!q) {+      outputString(StringPiece(p, end));+      break;+    }+    outputString(StringPiece(p, q));+    p = q + 1;++    if (p == end) {+      throw_exception<BadFormatArg>(+          "folly::format: '}' at end of format string");+    }++    // "{{" -> "{"+    if (*p == '{') {+      out(StringPiece(p, 1));+      ++p;+      continue;+    }++    // Format string+    q = static_cast<const char*>(memchr(p, '}', size_t(end - p)));+    if (q == nullptr) {+      throw_exception<BadFormatArg>("folly::format: missing ending '}'");+    }+    FormatArg arg(StringPiece(p, q));+    p = q + 1;++    int argIndex = 0;+    auto piece = arg.splitKey<true>(); // empty key component is okay+    if constexpr (containerMode) {+      arg.enforce(+          arg.width != FormatArg::kDynamicWidth,+          "dynamic field width not supported in vformat()");+      if (piece.empty()) {+        arg.setNextIntKey(nextArg++);+        hasDefaultArgIndex = true;+      } else {+        arg.setNextKey(piece);+        hasExplicitArgIndex = true;+      }+    } else {+      if (piece.empty()) {+        if (arg.width == FormatArg::kDynamicWidth) {+          arg.enforce(+              arg.widthIndex == FormatArg::kNoIndex,+              "cannot provide width arg index without value arg index");+          auto sizeArg = size_t(nextArg++);+          detail::formatCheckIndex(sizeArg, arg, nargs);+          if (RecordUsedArg) {+            used(base, sizeArg);+          }+          auto w = widths[sizeArg];+          arg.enforce(w >= 0, "dynamic field width argument must be integral");+          arg.width = w;+        }++        argIndex = nextArg++;+        hasDefaultArgIndex = true;+      } else {+        if (arg.width == FormatArg::kDynamicWidth) {+          arg.enforce(+              arg.widthIndex != FormatArg::kNoIndex,+              "cannot provide value arg index without width arg index");+          auto sizeArg = size_t(arg.widthIndex);+          detail::formatCheckIndex(sizeArg, arg, nargs);+          if (RecordUsedArg) {+            used(base, sizeArg);+          }+          auto w = widths[sizeArg];+          arg.enforce(w >= 0, "dynamic field width argument must be integral");+          arg.width = w;+        }++        auto result = tryTo<int>(piece);+        arg.enforce(result, "argument index must be integer");+        argIndex = *result;+        arg.enforce(argIndex >= 0, "argument index must be non-negative");+        hasExplicitArgIndex = true;+      }+    }++    if (hasDefaultArgIndex && hasExplicitArgIndex) {+      throw_exception<BadFormatArg>(+          "folly::format: may not have both default and explicit arg indexes");+    }++    if (RecordUsedArg) {+      used(base, argIndex);+    } else {+      formatCheckIndex(argIndex, arg, nargs);+      funs[argIndex](base, arg, out);+    }+  }+}++} // namespace detail++template <class Derived, bool containerMode, size_t... I, class... Args>+template <class Output>+void BaseFormatterImpl<+    Derived,+    containerMode,+    std::index_sequence<I...>,+    Args...>::operator()(Output& out) const {+  constexpr size_t nargs = sizeof...(Args);+  using RecordUsedSizeArgs = decltype(Derived::recordUsedArg(*this, 0));+  constexpr auto used = Derived::recordUsedArg;+  static constexpr auto funs = getDoFormatFnArray<Output>();+  constexpr auto in = unsafe_default_initialized;+  int widths[nargs + 1] = {conditional_t<!alignof(Args), int, int>{in}..., in};+  getSizeArg(widths);+  detail::baseFormatterCallImpl<containerMode, RecordUsedSizeArgs::value>(+      out, nargs, widths, *used, funs.data, *this);+}++namespace format_value {++template <class FormatCallback>+void formatString(StringPiece val, FormatArg& arg, FormatCallback& cb) {+  if (arg.width != FormatArg::kDefaultWidth && arg.width < 0) {+    throw_exception<BadFormatArg>("folly::format: invalid width");+  }+  if (arg.precision != FormatArg::kDefaultPrecision && arg.precision < 0) {+    throw_exception<BadFormatArg>("folly::format: invalid precision");+  }++  if (arg.precision != FormatArg::kDefaultPrecision &&+      val.size() > static_cast<size_t>(arg.precision)) {+    val.reset(val.data(), static_cast<size_t>(arg.precision));+  }++  constexpr int padBufSize = 128;+  char padBuf[padBufSize];++  // Output padding, no more than padBufSize at once+  auto pad = [&padBuf, &cb, padBufSize](int chars) {+    while (chars) {+      int n = std::min(chars, padBufSize);+      cb(StringPiece(padBuf, size_t(n)));+      chars -= n;+    }+  };++  int padRemaining = 0;+  if (arg.width != FormatArg::kDefaultWidth &&+      val.size() < static_cast<size_t>(arg.width)) {+    char fill = arg.fill == FormatArg::kDefaultFill ? ' ' : arg.fill;+    int padChars = static_cast<int>(arg.width - val.size());+    memset(padBuf, fill, size_t(std::min(padBufSize, padChars)));++    FOLLY_PUSH_WARNING+    FOLLY_CLANG_DISABLE_WARNING("-Wcovered-switch-default")+    switch (arg.align) {+      case FormatArg::Align::DEFAULT:+      case FormatArg::Align::LEFT:+        padRemaining = padChars;+        break;+      case FormatArg::Align::CENTER:+        pad(padChars / 2);+        padRemaining = padChars - padChars / 2;+        break;+      case FormatArg::Align::RIGHT:+      case FormatArg::Align::PAD_AFTER_SIGN:+        pad(padChars);+        break;+      case FormatArg::Align::INVALID:+      default:+        abort();+        break;+    }+    FOLLY_POP_WARNING+  }++  cb(val);++  if (padRemaining) {+    pad(padRemaining);+  }+}++template <class FormatCallback>+void formatNumber(+    StringPiece val, int prefixLen, FormatArg& arg, FormatCallback& cb) {+  // precision means something different for numbers+  arg.precision = FormatArg::kDefaultPrecision;+  if (arg.align == FormatArg::Align::DEFAULT) {+    arg.align = FormatArg::Align::RIGHT;+  } else if (prefixLen && arg.align == FormatArg::Align::PAD_AFTER_SIGN) {+    // Split off the prefix, then do any padding if necessary+    cb(val.subpiece(0, size_t(prefixLen)));+    val.advance(size_t(prefixLen));+    arg.width = std::max(arg.width - prefixLen, 0);+  }+  format_value::formatString(val, arg, cb);+}++template <typename FormatCallback>+struct FormatFormatterFn {+  FormatArg& arg;+  FormatCallback& cb;+  void operator()(StringPiece sp) {+    int sz = static_cast<int>(sp.size());+    if (arg.precision != FormatArg::kDefaultPrecision) {+      sz = std::min(arg.precision, sz);+      sp.reset(sp.data(), size_t(sz));+      arg.precision -= sz;+    }+    if (!sp.empty()) {+      cb(sp);+      if (arg.width != FormatArg::kDefaultWidth) {+        arg.width = std::max(arg.width - sz, 0);+      }+    }+  }+};++template <class FormatCallback, bool containerMode, class... Args>+void formatFormatter(+    const Formatter<containerMode, Args...>& formatter,+    FormatArg& arg,+    FormatCallback& cb) {+  if (arg.width == FormatArg::kDefaultWidth &&+      arg.precision == FormatArg::kDefaultPrecision) {+    // nothing to do+    formatter(cb);+  } else if (+      arg.align != FormatArg::Align::LEFT &&+      arg.align != FormatArg::Align::DEFAULT) {+    // We can only avoid creating a temporary string if we align left,+    // as we'd need to know the size beforehand otherwise+    format_value::formatString(formatter.str(), arg, cb);+  } else {+    auto fn = FormatFormatterFn<FormatCallback>{arg, cb};+    formatter(fn);+    if (arg.width != FormatArg::kDefaultWidth && arg.width != 0) {+      // Rely on formatString to do appropriate padding+      format_value::formatString(StringPiece(), arg, cb);+    }+  }+}++} // namespace format_value++// Definitions for default FormatValue classes++// Integral types (except bool)+template <class T>+class FormatValue<+    T,+    typename std::enable_if<+        std::is_integral<T>::value && !std::is_same<T, bool>::value>::type> {+ public:+  explicit FormatValue(T val) : val_(val) {}++  T getValue() const { return val_; }++  template <class FormatCallback>+  void format(FormatArg& arg, FormatCallback& cb) const {+    arg.validate(FormatArg::Type::INTEGER);+    doFormat(arg, cb);+  }++  template <class FormatCallback>+  void doFormat(FormatArg& arg, FormatCallback& cb) const {+    char presentation = arg.presentation;+    if (presentation == FormatArg::kDefaultPresentation) {+      presentation = std::is_same<T, char>::value ? 'c' : 'd';+    }++    // Do all work as unsigned, we'll add the prefix ('0' or '0x' if necessary)+    // and sign ourselves.+    typedef typename std::make_unsigned<T>::type UT;+    UT uval;+    char sign;+    if constexpr (std::is_signed<T>::value) {+      if (folly::is_negative(val_)) {+        // avoid unary negation of unsigned types, which may be warned against+        // avoid ub signed integer overflow, which ubsan checks against+        uval = UT(0 - static_cast<UT>(val_));+        sign = '-';+      } else {+        uval = static_cast<UT>(val_);+        FOLLY_PUSH_WARNING+        FOLLY_CLANG_DISABLE_WARNING("-Wcovered-switch-default")+        switch (arg.sign) {+          case FormatArg::Sign::PLUS_OR_MINUS:+            sign = '+';+            break;+          case FormatArg::Sign::SPACE_OR_MINUS:+            sign = ' ';+            break;+          case FormatArg::Sign::DEFAULT:+          case FormatArg::Sign::MINUS:+          case FormatArg::Sign::INVALID:+          default:+            sign = '\0';+            break;+        }+        FOLLY_POP_WARNING+      }+    } else {+      uval = static_cast<UT>(val_);+      sign = '\0';++      arg.enforce(+          arg.sign == FormatArg::Sign::DEFAULT,+          "sign specifications not allowed for unsigned values");+    }++    // 1 byte for sign, plus max of:+    // #x: two byte "0x" prefix + kMaxHexLength+    // #o: one byte "0" prefix + kMaxOctalLength+    // #b: two byte "0b" prefix + kMaxBinaryLength+    // n: 19 bytes + 1 NUL+    // ,d: 26 bytes (including thousands separators!)+    //+    // Binary format must take the most space, so we use that.+    //+    // Note that we produce a StringPiece rather than NUL-terminating,+    // so we don't need an extra byte for a NUL.+    constexpr size_t valBufSize = 1 + 2 + detail::kMaxBinaryLength;+    char valBuf[valBufSize];+    char* valBufBegin = nullptr;+    char* valBufEnd = nullptr;++    int prefixLen = 0;+    switch (presentation) {+      case 'n': {+        arg.enforce(+            !arg.basePrefix,+            "base prefix not allowed with '",+            presentation,+            "' specifier");++        arg.enforce(+            !arg.thousandsSeparator,+            "cannot use ',' with the '",+            presentation,+            "' specifier");++        valBufBegin = valBuf + 1; // room for sign+#if defined(__ANDROID__)+        int len = snprintf(+            valBufBegin,+            (valBuf + valBufSize) - valBufBegin,+            "%" PRIuMAX,+            static_cast<uintmax_t>(uval));+#else+        int len = snprintf(+            valBufBegin,+            size_t((valBuf + valBufSize) - valBufBegin),+            "%ju",+            static_cast<uintmax_t>(uval));+#endif+        // valBufSize should always be big enough, so this should never+        // happen.+        assert(len < valBuf + valBufSize - valBufBegin);+        valBufEnd = valBufBegin + len;+        break;+      }+      case 'd':+        arg.enforce(+            !arg.basePrefix,+            "base prefix not allowed with '",+            presentation,+            "' specifier");+        valBufBegin = valBuf + 1; // room for sign++        // Use to_ascii_decimal, faster than sprintf+        valBufEnd = valBufBegin ++            to_ascii_decimal(valBufBegin, valBuf + sizeof(valBuf), uval);+        if (arg.thousandsSeparator) {+          detail::insertThousandsGroupingUnsafe(valBufBegin, &valBufEnd);+        }+        break;+      case 'c':+        arg.enforce(+            !arg.basePrefix,+            "base prefix not allowed with '",+            presentation,+            "' specifier");+        arg.enforce(+            !arg.thousandsSeparator,+            "thousands separator (',') not allowed with '",+            presentation,+            "' specifier");+        valBufBegin = valBuf + 1; // room for sign+        *valBufBegin = static_cast<char>(uval);+        valBufEnd = valBufBegin + 1;+        break;+      case 'o':+      case 'O':+        arg.enforce(+            !arg.thousandsSeparator,+            "thousands separator (',') not allowed with '",+            presentation,+            "' specifier");+        valBufEnd = valBuf + valBufSize;+        valBufBegin = &valBuf[detail::uintToOctal(valBuf, valBufSize, uval)];+        if (arg.basePrefix) {+          *--valBufBegin = '0';+          prefixLen = 1;+        }+        break;+      case 'x':+        arg.enforce(+            !arg.thousandsSeparator,+            "thousands separator (',') not allowed with '",+            presentation,+            "' specifier");+        valBufEnd = valBuf + valBufSize;+        valBufBegin = &valBuf[detail::uintToHexLower(valBuf, valBufSize, uval)];+        if (arg.basePrefix) {+          *--valBufBegin = 'x';+          *--valBufBegin = '0';+          prefixLen = 2;+        }+        break;+      case 'X':+        arg.enforce(+            !arg.thousandsSeparator,+            "thousands separator (',') not allowed with '",+            presentation,+            "' specifier");+        valBufEnd = valBuf + valBufSize;+        valBufBegin = &valBuf[detail::uintToHexUpper(valBuf, valBufSize, uval)];+        if (arg.basePrefix) {+          *--valBufBegin = 'X';+          *--valBufBegin = '0';+          prefixLen = 2;+        }+        break;+      case 'b':+      case 'B':+        arg.enforce(+            !arg.thousandsSeparator,+            "thousands separator (',') not allowed with '",+            presentation,+            "' specifier");+        valBufEnd = valBuf + valBufSize;+        valBufBegin = &valBuf[detail::uintToBinary(valBuf, valBufSize, uval)];+        if (arg.basePrefix) {+          *--valBufBegin = presentation; // 0b or 0B+          *--valBufBegin = '0';+          prefixLen = 2;+        }+        break;+      default:+        arg.error("invalid specifier '", presentation, "'");+    }++    if (sign) {+      *--valBufBegin = sign;+      ++prefixLen;+    }++    format_value::formatNumber(+        StringPiece(valBufBegin, valBufEnd), prefixLen, arg, cb);+  }++ private:+  T val_;+};++// Bool+template <>+class FormatValue<bool> {+ public:+  explicit FormatValue(bool val) : val_(val) {}++  template <class FormatCallback>+  void format(FormatArg& arg, FormatCallback& cb) const {+    if (arg.presentation == FormatArg::kDefaultPresentation) {+      arg.validate(FormatArg::Type::OTHER);+      format_value::formatString(val_ ? "true" : "false", arg, cb);+    } else { // number+      FormatValue<int>(val_).format(arg, cb);+    }+  }++ private:+  bool val_;+};++// double+template <>+class FormatValue<double> {+ public:+  explicit FormatValue(double val) : val_(val) {}++  template <class FormatCallback>+  void format(FormatArg& arg, FormatCallback& cb) const {+    fbstring piece;+    int prefixLen;+    formatHelper(piece, prefixLen, arg);+    format_value::formatNumber(piece, prefixLen, arg, cb);+  }++ private:+  void formatHelper(fbstring& piece, int& prefixLen, FormatArg& arg) const;++  double val_;+};++// float (defer to double)+template <>+class FormatValue<float> {+ public:+  explicit FormatValue(float val) : val_(val) {}++  template <class FormatCallback>+  void format(FormatArg& arg, FormatCallback& cb) const {+    FormatValue<double>(val_).format(arg, cb);+  }++ private:+  float val_;+};++// String-y types (implicitly convertible to StringPiece, except char*)+template <class T>+class FormatValue<+    T,+    typename std::enable_if<+        (!std::is_pointer<T>::value ||+         !std::is_same<+             char,+             typename std::decay<typename std::remove_pointer<T>::type>::type>::+             value) &&+        std::is_convertible<T, StringPiece>::value>::type> {+ public:+  explicit FormatValue(StringPiece val) : val_(val) {}++  template <class FormatCallback>+  void format(FormatArg& arg, FormatCallback& cb) const {+    if (arg.keyEmpty()) {+      arg.validate(FormatArg::Type::OTHER);+      arg.enforce(+          arg.presentation == FormatArg::kDefaultPresentation ||+              arg.presentation == 's',+          "invalid specifier '",+          arg.presentation,+          "'");+      format_value::formatString(val_, arg, cb);+    } else {+      FormatValue<char>(val_.at(size_t(arg.splitIntKey()))).format(arg, cb);+    }+  }++ private:+  StringPiece val_;+};++// Null+template <>+class FormatValue<std::nullptr_t> {+ public:+  explicit FormatValue(std::nullptr_t) {}++  template <class FormatCallback>+  void format(FormatArg& arg, FormatCallback& cb) const {+    arg.validate(FormatArg::Type::OTHER);+    arg.enforce(+        arg.presentation == FormatArg::kDefaultPresentation,+        "invalid specifier '",+        arg.presentation,+        "'");+    format_value::formatString("(null)", arg, cb);+  }+};++// Partial specialization of FormatValue for char*+template <class T>+class FormatValue<+    T*,+    typename std::enable_if<+        std::is_same<char, typename std::decay<T>::type>::value>::type> {+ public:+  explicit FormatValue(T* val) : val_(val) {}++  template <class FormatCallback>+  void format(FormatArg& arg, FormatCallback& cb) const {+    if (arg.keyEmpty()) {+      if (!val_) {+        FormatValue<std::nullptr_t>(nullptr).format(arg, cb);+      } else {+        FormatValue<StringPiece>(val_).format(arg, cb);+      }+    } else {+      FormatValue<typename std::decay<T>::type>(val_[arg.splitIntKey()])+          .format(arg, cb);+    }+  }++ private:+  T* val_;+};++// Partial specialization of FormatValue for void*+template <class T>+class FormatValue<+    T*,+    typename std::enable_if<+        std::is_same<void, typename std::decay<T>::type>::value>::type> {+ public:+  explicit FormatValue(T* val) : val_(val) {}++  template <class FormatCallback>+  void format(FormatArg& arg, FormatCallback& cb) const {+    if (!val_) {+      FormatValue<std::nullptr_t>(nullptr).format(arg, cb);+    } else {+      // Print as a pointer, in hex.+      arg.validate(FormatArg::Type::OTHER);+      arg.enforce(+          arg.presentation == FormatArg::kDefaultPresentation,+          "invalid specifier '",+          arg.presentation,+          "'");+      arg.basePrefix = true;+      arg.presentation = 'x';+      if (arg.align == FormatArg::Align::DEFAULT) {+        arg.align = FormatArg::Align::LEFT;+      }+      FormatValue<uintptr_t>(reinterpret_cast<uintptr_t>(val_))+          .doFormat(arg, cb);+    }+  }++ private:+  T* val_;+};++template <class T, class = void>+class TryFormatValue {+ public:+  template <class FormatCallback>+  static void formatOrFail(+      T& /* value */, FormatArg& arg, FormatCallback& /* cb */) {+    arg.error("No formatter available for this type");+  }+};++template <class T>+class TryFormatValue<+    T,+    typename std::enable_if<+        0 < sizeof(FormatValue<typename std::decay<T>::type>)>::type> {+ public:+  template <class FormatCallback>+  static void formatOrFail(T& value, FormatArg& arg, FormatCallback& cb) {+    FormatValue<typename std::decay<T>::type>(value).format(arg, cb);+  }+};++// Partial specialization of FormatValue for other pointers+template <class T>+class FormatValue<+    T*,+    typename std::enable_if<+        !std::is_same<char, typename std::decay<T>::type>::value &&+        !std::is_same<void, typename std::decay<T>::type>::value>::type> {+ public:+  explicit FormatValue(T* val) : val_(val) {}++  template <class FormatCallback>+  void format(FormatArg& arg, FormatCallback& cb) const {+    if (arg.keyEmpty()) {+      FormatValue<void*>((void*)val_).format(arg, cb);+    } else {+      TryFormatValue<T>::formatOrFail(val_[arg.splitIntKey()], arg, cb);+    }+  }++ private:+  T* val_;+};++namespace detail {++// std::array+template <class T, size_t N>+struct IndexableTraits<std::array<T, N>>+    : public IndexableTraitsSeq<std::array<T, N>> {};++// std::vector+template <class T, class A>+struct IndexableTraits<std::vector<T, A>>+    : public IndexableTraitsSeq<std::vector<T, A>> {};++// std::deque+template <class T, class A>+struct IndexableTraits<std::deque<T, A>>+    : public IndexableTraitsSeq<std::deque<T, A>> {};++// std::map with integral keys+template <class K, class T, class C, class A>+struct IndexableTraits<+    std::map<K, T, C, A>,+    typename std::enable_if<std::is_integral<K>::value>::type>+    : public IndexableTraitsAssoc<std::map<K, T, C, A>> {};++// std::unordered_map with integral keys+template <class K, class T, class H, class E, class A>+struct IndexableTraits<+    std::unordered_map<K, T, H, E, A>,+    typename std::enable_if<std::is_integral<K>::value>::type>+    : public IndexableTraitsAssoc<std::unordered_map<K, T, H, E, A>> {};++} // namespace detail++// Partial specialization of FormatValue for integer-indexable containers+template <class T>+class FormatValue<T, typename detail::IndexableTraits<T>::enabled> {+ public:+  explicit FormatValue(const T& val) : val_(val) {}++  template <class FormatCallback>+  void format(FormatArg& arg, FormatCallback& cb) const {+    FormatValue<typename std::decay<+        typename detail::IndexableTraits<T>::value_type>::type>(+        detail::IndexableTraits<T>::at(val_, arg.splitIntKey()))+        .format(arg, cb);+  }++ private:+  const T& val_;+};++template <class Container, class Value>+class FormatValue<+    detail::DefaultValueWrapper<Container, Value>,+    typename detail::IndexableTraits<Container>::enabled> {+ public:+  explicit FormatValue(const detail::DefaultValueWrapper<Container, Value>& val)+      : val_(val) {}++  template <class FormatCallback>+  void format(FormatArg& arg, FormatCallback& cb) const {+    FormatValue<typename std::decay<+        typename detail::IndexableTraits<Container>::value_type>::type>(+        detail::IndexableTraits<Container>::at(+            val_.container, arg.splitIntKey(), val_.defaultValue))+        .format(arg, cb);+  }++ private:+  const detail::DefaultValueWrapper<Container, Value>& val_;+};++namespace detail {++// Define enabled, key_type, convert from StringPiece to the key types+// that we support+template <class T>+struct KeyFromStringPiece;++// std::string+template <>+struct KeyFromStringPiece<std::string> : public FormatTraitsBase {+  typedef std::string key_type;+  static std::string convert(StringPiece s) { return s.toString(); }+  typedef void enabled;+};++// fbstring+template <>+struct KeyFromStringPiece<fbstring> : public FormatTraitsBase {+  typedef fbstring key_type;+  static fbstring convert(StringPiece s) { return s.to<fbstring>(); }+};++// StringPiece+template <>+struct KeyFromStringPiece<StringPiece> : public FormatTraitsBase {+  typedef StringPiece key_type;+  static StringPiece convert(StringPiece s) { return s; }+};++// Base class for associative types keyed by strings+template <class T>+struct KeyableTraitsAssoc : public FormatTraitsBase {+  typedef typename T::key_type key_type;+  typedef typename T::value_type::second_type value_type;+  static const value_type& at(const T& map, StringPiece key) {+    if (auto ptr = get_ptr(map, KeyFromStringPiece<key_type>::convert(key))) {+      return *ptr;+    }+    throw_exception<FormatKeyNotFoundException>(key);+  }+  static const value_type& at(+      const T& map, StringPiece key, const value_type& dflt) {+    auto pos = map.find(KeyFromStringPiece<key_type>::convert(key));+    return pos != map.end() ? pos->second : dflt;+  }+};++// Define enabled, key_type, value_type, at() for supported string-keyed+// types+template <class T, class Enabled = void>+struct KeyableTraits;++// std::map with string key+template <class K, class T, class C, class A>+struct KeyableTraits<+    std::map<K, T, C, A>,+    typename KeyFromStringPiece<K>::enabled>+    : public KeyableTraitsAssoc<std::map<K, T, C, A>> {};++// std::unordered_map with string key+template <class K, class T, class H, class E, class A>+struct KeyableTraits<+    std::unordered_map<K, T, H, E, A>,+    typename KeyFromStringPiece<K>::enabled>+    : public KeyableTraitsAssoc<std::unordered_map<K, T, H, E, A>> {};++} // namespace detail++// Partial specialization of FormatValue for string-keyed containers+template <class T>+class FormatValue<T, typename detail::KeyableTraits<T>::enabled> {+ public:+  explicit FormatValue(const T& val) : val_(val) {}++  template <class FormatCallback>+  void format(FormatArg& arg, FormatCallback& cb) const {+    FormatValue<typename std::decay<+        typename detail::KeyableTraits<T>::value_type>::type>(+        detail::KeyableTraits<T>::at(val_, arg.splitKey()))+        .format(arg, cb);+  }++ private:+  const T& val_;+};++template <class Container, class Value>+class FormatValue<+    detail::DefaultValueWrapper<Container, Value>,+    typename detail::KeyableTraits<Container>::enabled> {+ public:+  explicit FormatValue(const detail::DefaultValueWrapper<Container, Value>& val)+      : val_(val) {}++  template <class FormatCallback>+  void format(FormatArg& arg, FormatCallback& cb) const {+    FormatValue<typename std::decay<+        typename detail::KeyableTraits<Container>::value_type>::type>(+        detail::KeyableTraits<Container>::at(+            val_.container, arg.splitKey(), val_.defaultValue))+        .format(arg, cb);+  }++ private:+  const detail::DefaultValueWrapper<Container, Value>& val_;+};++// Partial specialization of FormatValue for pairs+template <class A, class B>+class FormatValue<std::pair<A, B>> {+ public:+  explicit FormatValue(const std::pair<A, B>& val) : val_(val) {}++  template <class FormatCallback>+  void format(FormatArg& arg, FormatCallback& cb) const {+    int key = arg.splitIntKey();+    switch (key) {+      case 0:+        FormatValue<typename std::decay<A>::type>(val_.first).format(arg, cb);+        break;+      case 1:+        FormatValue<typename std::decay<B>::type>(val_.second).format(arg, cb);+        break;+      default:+        arg.error("invalid index for pair");+    }+  }++ private:+  const std::pair<A, B>& val_;+};++// Partial specialization of FormatValue for tuples+template <class... Args>+class FormatValue<std::tuple<Args...>> {+  typedef std::tuple<Args...> Tuple;++ public:+  explicit FormatValue(const Tuple& val) : val_(val) {}++  template <class FormatCallback>+  void format(FormatArg& arg, FormatCallback& cb) const {+    int key = arg.splitIntKey();+    arg.enforce(key >= 0, "tuple index must be non-negative");+    doFormat(size_t(key), arg, cb);+  }++ private:+  template <size_t K>+  using FV = FormatValue<+      typename std::decay<typename std::tuple_element<K, Tuple>::type>::type>;++  template <class Callback, size_t... I>+  void doFormat(+      size_t i, FormatArg& arg, Callback& cb, std::index_sequence<I...>) const {+    detail::formatCheckIndex(i, arg, sizeof...(Args));+    ((i == I && (FV<I>(std::get<I>(val_)).format(arg, cb), 0)), ...);+  }+  template <class Callback>+  void doFormat(size_t i, FormatArg& arg, Callback& cb) const {+    return doFormat(i, arg, cb, std::index_sequence_for<Args...>{});+  }++  const Tuple& val_;+};++// Partial specialization of FormatValue for nested Formatters+template <bool containerMode, class... Args, template <bool, class...> class F>+class FormatValue<+    F<containerMode, Args...>,+    typename std::enable_if<+        detail::IsFormatter<F<containerMode, Args...>>::value>::type> {+  typedef F<containerMode, Args...> FormatterValue;++ public:+  explicit FormatValue(const FormatterValue& f) : f_(f) {}++  template <class FormatCallback>+  void format(FormatArg& arg, FormatCallback& cb) const {+    format_value::formatFormatter(f_, arg, cb);+  }++ private:+  const FormatterValue& f_;+};++/**+ * Formatter objects can be appended to strings, and therefore they're+ * compatible with folly::toAppend and folly::to.+ */+template <class Tgt, bool containerMode, class... Args>+typename std::enable_if<IsSomeString<Tgt>::value>::type toAppend(+    const Formatter<containerMode, Args...>& value, Tgt* result) {+  value.appendTo(*result);+}++} // namespace folly++FOLLY_POP_WARNING
+ folly/folly/Format.cpp view
@@ -0,0 +1,431 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#include <folly/Format.h>++#include <cassert>++#include <folly/ConstexprMath.h>+#include <folly/CppAttributes.h>+#include <folly/Portability.h>+#include <folly/container/Array.h>++#include <double-conversion/double-conversion.h>++namespace folly {+namespace detail {++//  ctor for items in the align table+struct format_table_align_make_item {+  static constexpr std::size_t size = 256;+  constexpr FormatArg::Align operator()(std::size_t index) const {+    // clang-format off+    return+        index == '<' ? FormatArg::Align::LEFT:+        index == '>' ? FormatArg::Align::RIGHT :+        index == '=' ? FormatArg::Align::PAD_AFTER_SIGN :+        index == '^' ? FormatArg::Align::CENTER :+        FormatArg::Align::INVALID;+    // clang-format on+  }+};++//  ctor for items in the conv tables for representing parts of nonnegative+//  integers into ascii digits of length Size, over a given base Base+template <std::size_t Base, std::size_t Size, bool Upper = false>+struct format_table_conv_make_item {+  static_assert(Base <= 36, "Base is unrepresentable");+  struct make_item {+    std::size_t index{};+    constexpr char alpha(std::size_t ord) const {+      return static_cast<char>(+          ord < 10 ? '0' + ord : (Upper ? 'A' : 'a') + (ord - 10));+    }+    constexpr char operator()(std::size_t offset) const {+      return alpha(index / constexpr_pow(Base, Size - offset - 1) % Base);+    }+  };+  constexpr std::array<char, Size> operator()(std::size_t index) const {+    return make_array_with<Size>(make_item{index});+  }+};++//  ctor for items in the sign table+struct format_table_sign_make_item {+  static constexpr std::size_t size = 256;+  constexpr FormatArg::Sign operator()(std::size_t index) const {+    // clang-format off+    return+        index == '+' ? FormatArg::Sign::PLUS_OR_MINUS :+        index == '-' ? FormatArg::Sign::MINUS :+        index == ' ' ? FormatArg::Sign::SPACE_OR_MINUS :+        FormatArg::Sign::INVALID;+    // clang-format on+  }+};++//  the tables+FOLLY_STORAGE_CONSTEXPR auto formatAlignTable =+    make_array_with<256>(format_table_align_make_item{});+FOLLY_STORAGE_CONSTEXPR auto formatSignTable =+    make_array_with<256>(format_table_sign_make_item{});+FOLLY_STORAGE_CONSTEXPR decltype(formatHexLower) formatHexLower =+    make_array_with<256>(format_table_conv_make_item<16, 2, false>{});+FOLLY_STORAGE_CONSTEXPR decltype(formatHexUpper) formatHexUpper =+    make_array_with<256>(format_table_conv_make_item<16, 2, true>{});+FOLLY_STORAGE_CONSTEXPR decltype(formatOctal) formatOctal =+    make_array_with<512>(format_table_conv_make_item<8, 3>{});+FOLLY_STORAGE_CONSTEXPR decltype(formatBinary) formatBinary =+    make_array_with<256>(format_table_conv_make_item<2, 8>{});++} // namespace detail++using namespace folly::detail;++void FormatValue<double>::formatHelper(+    fbstring& piece, int& prefixLen, FormatArg& arg) const {+  using ::double_conversion::DoubleToStringConverter;+  using ::double_conversion::StringBuilder;++  arg.validate(FormatArg::Type::FLOAT);++  if (arg.presentation == FormatArg::kDefaultPresentation) {+    arg.presentation = 'g';+  }++  const char* infinitySymbol = isupper(arg.presentation) ? "INF" : "inf";+  const char* nanSymbol = isupper(arg.presentation) ? "NAN" : "nan";+  char exponentSymbol = isupper(arg.presentation) ? 'E' : 'e';++  if (arg.precision == FormatArg::kDefaultPrecision) {+    arg.precision = 6;+  }++  // 2+: for null terminator and optional sign shenanigans.+  constexpr int bufLen =+      2 ++      constexpr_max(+          2 + DoubleToStringConverter::kMaxFixedDigitsBeforePoint ++              DoubleToStringConverter::kMaxFixedDigitsAfterPoint,+          constexpr_max(+              8 + DoubleToStringConverter::kMaxExponentialDigits,+              7 + DoubleToStringConverter::kMaxPrecisionDigits));+  char buf[bufLen];+  StringBuilder builder(buf + 1, bufLen - 1);++  char plusSign;+  switch (arg.sign) {+    case FormatArg::Sign::PLUS_OR_MINUS:+      plusSign = '+';+      break;+    case FormatArg::Sign::SPACE_OR_MINUS:+      plusSign = ' ';+      break;+    case FormatArg::Sign::DEFAULT:+    case FormatArg::Sign::MINUS:+    case FormatArg::Sign::INVALID:+    default:+      plusSign = '\0';+      break;+  }++  auto flags = DoubleToStringConverter::EMIT_POSITIVE_EXPONENT_SIGN |+      (arg.trailingDot ? DoubleToStringConverter::EMIT_TRAILING_DECIMAL_POINT+                       : 0);++  double val = val_;+  switch (arg.presentation) {+    case '%':+      val *= 100;+      [[fallthrough]];+    case 'f':+    case 'F': {+      if (arg.precision > DoubleToStringConverter::kMaxFixedDigitsAfterPoint) {+        arg.precision = DoubleToStringConverter::kMaxFixedDigitsAfterPoint;+      }+      DoubleToStringConverter conv(+          flags,+          infinitySymbol,+          nanSymbol,+          exponentSymbol,+          -4,+          arg.precision,+          0,+          0);+      arg.enforce(+          conv.ToFixed(val, arg.precision, &builder),+          "fixed double conversion failed");+      break;+    }+    case 'e':+    case 'E': {+      if (arg.precision > DoubleToStringConverter::kMaxExponentialDigits) {+        arg.precision = DoubleToStringConverter::kMaxExponentialDigits;+      }++      DoubleToStringConverter conv(+          flags,+          infinitySymbol,+          nanSymbol,+          exponentSymbol,+          -4,+          arg.precision,+          0,+          0);+      arg.enforce(conv.ToExponential(val, arg.precision, &builder));+      break;+    }+    case 'n': // should be locale-aware, but isn't+    case 'g':+    case 'G': {+      if (arg.precision < DoubleToStringConverter::kMinPrecisionDigits) {+        arg.precision = DoubleToStringConverter::kMinPrecisionDigits;+      } else if (arg.precision > DoubleToStringConverter::kMaxPrecisionDigits) {+        arg.precision = DoubleToStringConverter::kMaxPrecisionDigits;+      }+      DoubleToStringConverter conv(+          flags,+          infinitySymbol,+          nanSymbol,+          exponentSymbol,+          -4,+          arg.precision,+          0,+          0);+      arg.enforce(conv.ToShortest(val, &builder));+      break;+    }+    default:+      arg.error("invalid specifier '", arg.presentation, "'");+  }++  auto len = builder.position();+  builder.Finalize();+  assert(len > 0);++  // Add '+' or ' ' sign if needed+  char* p = buf + 1;+  // anything that's neither negative nor nan+  prefixLen = 0;+  if (plusSign && (*p != '-' && *p != 'n' && *p != 'N')) {+    *--p = plusSign;+    ++len;+    prefixLen = 1;+  } else if (*p == '-') {+    prefixLen = 1;+  }++  piece = fbstring(p, size_t(len));+}++void FormatArg::initSlow() {+  auto b = fullArgString.begin();+  auto end = fullArgString.end();++  // Parse key+  auto p = static_cast<const char*>(memchr(b, ':', size_t(end - b)));+  if (!p) {+    key_ = StringPiece(b, end);+    return;+  }+  key_ = StringPiece(b, p);++  if (*p == ':') {+    // parse format spec+    if (++p == end) {+      return;+    }++    // fill/align, or just align+    Align a;+    if (p + 1 != end &&+        (a = formatAlignTable[static_cast<unsigned char>(p[1])]) !=+            Align::INVALID) {+      fill = *p;+      align = a;+      p += 2;+      if (p == end) {+        return;+      }+    } else if (+        (a = formatAlignTable[static_cast<unsigned char>(*p)]) !=+        Align::INVALID) {+      align = a;+      if (++p == end) {+        return;+      }+    }++    Sign s;+    auto uSign = static_cast<unsigned char>(*p);+    if ((s = formatSignTable[uSign]) != Sign::INVALID) {+      sign = s;+      if (++p == end) {+        return;+      }+    }++    if (*p == '#') {+      basePrefix = true;+      if (++p == end) {+        return;+      }+    }++    if (*p == '0') {+      enforce(align == Align::DEFAULT, "alignment specified twice");+      fill = '0';+      align = Align::PAD_AFTER_SIGN;+      if (++p == end) {+        return;+      }+    }++    auto readInt = [&] {+      auto const c = p;+      do {+        ++p;+      } while (p != end && *p >= '0' && *p <= '9');+      return to<int>(StringPiece(c, p));+    };++    if (*p == '*') {+      width = kDynamicWidth;+      ++p;++      if (p == end) {+        return;+      }++      if (*p >= '0' && *p <= '9') {+        widthIndex = readInt();+      }++      if (p == end) {+        return;+      }+    } else if (*p >= '0' && *p <= '9') {+      width = readInt();++      if (p == end) {+        return;+      }+    }++    if (*p == ',') {+      thousandsSeparator = true;+      if (++p == end) {+        return;+      }+    }++    if (*p == '.') {+      auto d = ++p;+      while (p != end && *p >= '0' && *p <= '9') {+        ++p;+      }+      if (p != d) {+        precision = to<int>(StringPiece(d, p));+        if (p != end && *p == '.') {+          trailingDot = true;+          ++p;+        }+      } else {+        trailingDot = true;+      }++      if (p == end) {+        return;+      }+    }++    presentation = *p;+    if (++p == end) {+      return;+    }+  }++  error("extra characters in format string");+}++void FormatArg::validate(Type type) const {+  enforce(keyEmpty(), "index not allowed");+  switch (type) {+    case Type::INTEGER:+      enforce(+          precision == kDefaultPrecision, "precision not allowed on integers");+      break;+    case Type::FLOAT:+      enforce(+          !basePrefix, "base prefix ('#') specifier only allowed on integers");+      enforce(+          !thousandsSeparator,+          "thousands separator (',') only allowed on integers");+      break;+    case Type::OTHER:+      enforce(+          align != Align::PAD_AFTER_SIGN,+          "'='alignment only allowed on numbers");+      enforce(sign == Sign::DEFAULT, "sign specifier only allowed on numbers");+      enforce(+          !basePrefix, "base prefix ('#') specifier only allowed on integers");+      enforce(+          !thousandsSeparator,+          "thousands separator (',') only allowed on integers");+      break;+  }+}++namespace detail {+void insertThousandsGroupingUnsafe(char* start_buffer, char** end_buffer) {+  auto remaining_digits = uint32_t(*end_buffer - start_buffer);+  uint32_t separator_size = (remaining_digits - 1) / 3;+  uint32_t result_size = remaining_digits + separator_size;+  *end_buffer = *end_buffer + separator_size;++  // get the end of the new string with the separators+  uint32_t buffer_write_index = result_size - 1;+  uint32_t buffer_read_index = remaining_digits - 1;+  start_buffer[buffer_write_index + 1] = 0;++  bool done = false;+  uint32_t next_group_size = 3;++  while (!done) {+    uint32_t current_group_size = std::max<uint32_t>(+        1, std::min<uint32_t>(remaining_digits, next_group_size));++    // write out the current group's digits to the buffer index+    for (uint32_t i = 0; i < current_group_size; i++) {+      start_buffer[buffer_write_index--] = start_buffer[buffer_read_index--];+    }++    // if not finished, write the separator before the next group+    if (buffer_write_index < buffer_write_index + 1) {+      start_buffer[buffer_write_index--] = ',';+    } else {+      done = true;+    }++    remaining_digits -= current_group_size;+  }+}+} // namespace detail++FormatKeyNotFoundException::FormatKeyNotFoundException(StringPiece key)+    : std::out_of_range(kMessagePrefix.str() + key.str()) {}++} // namespace folly
+ folly/folly/Format.h view
@@ -0,0 +1,440 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++//+// Docs: https://fburl.com/fbcref_format+//++/**+ * folly::format has been superseded by+ * [fmt](https://fmt.dev/latest/index.html). `#include <fmt/core.h>`+ *+ * format() performs text-formatting, similar to Python's str.format. The full+ * specification is on github:+ * https://github.com/facebook/folly/blob/main/folly/docs/Format.md+ *+ * @refcode folly/docs/examples/folly/Format.cpp+ * @file Format.h+ */++#pragma once+#define FOLLY_FORMAT_H_++#include <cstdio>+#include <ios>+#include <stdexcept>+#include <tuple>+#include <type_traits>++#include <folly/CPortability.h>+#include <folly/Conv.h>+#include <folly/FormatArg.h>+#include <folly/Range.h>+#include <folly/String.h>+#include <folly/Traits.h>++// Ignore shadowing warnings within this file, so includers can use -Wshadow.+FOLLY_PUSH_WARNING+FOLLY_GNU_DISABLE_WARNING("-Wshadow")++namespace folly {++// forward declarations+template <bool containerMode, class... Args>+class Formatter;+template <class... Args>+Formatter<false, Args...> format(StringPiece fmt, Args&&... args);+template <class C>+std::string svformat(StringPiece fmt, C&& container) = delete;+template <class T, class Enable = void>+class FormatValue;++// meta-attribute to identify formatters in this sea of template weirdness+namespace detail {+class FormatterTag {};++struct BaseFormatterBase {+  template <class Callback>+  using DoFormatFn = void(const BaseFormatterBase&, FormatArg&, Callback&);++  StringPiece str_;++  static std::false_type recordUsedArg(const BaseFormatterBase&, size_t) {+    return {};+  }+};++// BaseFormatterTuple suffices and is faster to compile than is std::tuple+template <size_t I, typename A>+struct BaseFormatterTupleIndexedValue {+  A value;+};+template <typename, typename...>+struct BaseFormatterTuple;+template <size_t... I, typename... A>+struct BaseFormatterTuple<std::index_sequence<I...>, A...>+    : BaseFormatterTupleIndexedValue<I, A>... {+  explicit BaseFormatterTuple(std::in_place_t, A&&... a)+      : BaseFormatterTupleIndexedValue<I, A>{static_cast<A&&>(a)}... {}+};++template <typename Str>+struct BaseFormatterAppendToString {+  Str& str;+  void operator()(StringPiece s) const { str.append(s.data(), s.size()); }+};++inline void formatCheckIndex(size_t i, const FormatArg& arg, size_t max) {+  arg.enforce(i < max, "argument index out of range, max=", max);+}+} // namespace detail++/**+ * Formatter class.+ *+ * Note that this class is tricky, as it keeps *references* to its lvalue+ * arguments (while it takes ownership of the temporaries), and it doesn't+ * copy the passed-in format string. Thankfully, you can't use this+ * directly, you have to use format(...) below.+ */++/* BaseFormatter class.+ * Overridable behaviors:+ * You may override the actual formatting of positional parameters in+ * `doFormatArg`. The Formatter class provides the default implementation.+ *+ * You may also override `recordUsedArg`. This override point was added to+ * permit static analysis of format strings, when it is inconvenient or+ * impossible to instantiate a BaseFormatter with the correct storage. If+ * overriding, the return type must be std::true_type.+ */+template <class Derived, bool containerMode, class Indices, class... Args>+class BaseFormatterImpl;+template <class Derived, bool containerMode, size_t... I, class... Args>+class BaseFormatterImpl<+    Derived,+    containerMode,+    std::index_sequence<I...>,+    Args...> : public detail::BaseFormatterBase {+ public:+  /**+   * Append to output.  out(StringPiece sp) may be called (more than once)+   */+  template <class Output>+  void operator()(Output& out) const;++  /**+   * Append to a string.+   */+  template <class Str>+  typename std::enable_if<IsSomeString<Str>::value>::type appendTo(+      Str& str) const {+    detail::BaseFormatterAppendToString<Str> out{str};+    (*this)(out);+  }++  /**+   * Conversion to string+   */+  std::string str() const {+    std::string s;+    appendTo(s);+    return s;+  }++  /**+   * Metadata to identify generated children of BaseFormatter+   */+  typedef detail::FormatterTag IsFormatter;++ private:+  template <typename T, typename D = typename std::decay<T>::type>+  using IsSizeable = std::bool_constant<+      std::is_integral<D>::value && !std::is_same<D, bool>::value>;++  template <class Callback>+  static constexpr c_array<DoFormatFn<Callback>*, sizeof...(Args) + 1>+  getDoFormatFnArray() {+    return {{Derived::template doFormatArg<I, Callback>..., nullptr}};+  }++  template <size_t, typename>+  constexpr int getSizeArgAt(std::false_type) const {+    return -1;+  }+  template <size_t K, typename T>+  int getSizeArgAt(std::true_type) const {+    using V = detail::BaseFormatterTupleIndexedValue<K, T>;+    return static_cast<int>(static_cast<const V&>(values_).value);+  }+  void getSizeArg(int* out) const {+    ((out[I] = getSizeArgAt<I, Args>(IsSizeable<Args>{})), ...);+  }++ protected:+  explicit BaseFormatterImpl(StringPiece str, Args&&... args)+      : detail::BaseFormatterBase{str},+        values_(std::in_place, static_cast<Args&&>(args)...) {}++  // Not copyable+  BaseFormatterImpl(const BaseFormatterImpl&) = delete;+  BaseFormatterImpl& operator=(const BaseFormatterImpl&) = delete;++  // Movable, but the move constructor and assignment operator are private,+  // for the exclusive use of format() (below).  This way, you can't create+  // a Formatter object, but can handle references to it (for streaming,+  // conversion to string, etc) -- which is good, as Formatter objects are+  // dangerous (they may hold references).+  BaseFormatterImpl(BaseFormatterImpl&&) = default;+  BaseFormatterImpl& operator=(BaseFormatterImpl&&) = default;++  template <size_t K, typename T = type_pack_element_t<K, Args...>>+  FormatValue<typename std::decay<T>::type> getFormatValue() const {+    using V = detail::BaseFormatterTupleIndexedValue<K, T>;+    auto& v = static_cast<const V&>(values_);+    return FormatValue<typename std::decay<T>::type>(v.value);+  }++  detail::BaseFormatterTuple<std::index_sequence<I...>, Args...> values_;+};+template <class Derived, bool containerMode, class... Args>+using BaseFormatter = BaseFormatterImpl<+    Derived,+    containerMode,+    std::index_sequence_for<Args...>,+    Args...>;++template <bool containerMode, class... Args>+class Formatter+    : public BaseFormatter<+          Formatter<containerMode, Args...>,+          containerMode,+          Args...> {+  using self = Formatter<containerMode, Args...>;+  using base = BaseFormatter<self, containerMode, Args...>;++  static_assert(+      !containerMode || sizeof...(Args) == 1,+      "Exactly one argument required in container mode");++ private:+  using base::base;++  template <size_t K, class Callback>+  static void doFormatArg(+      const detail::BaseFormatterBase& obj, FormatArg& arg, Callback& cb) {+    auto& d = static_cast<const Formatter&>(obj);+    d.template getFormatValue<K>().format(arg, cb);+  }++  friend base;++  template <class... A>+  friend Formatter<false, A...> format(StringPiece fmt, A&&... arg);+  template <class Str, class... A>+  friend typename std::enable_if<IsSomeString<Str>::value>::type format(+      Str* out, StringPiece fmt, A&&... args);+  template <class... A>+  friend std::string sformat(StringPiece fmt, A&&... arg);+  template <class C>+  friend std::string svformat(StringPiece fmt, C&& container);+};++namespace detail {+template <typename Out>+struct FormatterOstreamInsertionWriterFn {+  Out& out;+  void operator()(StringPiece sp) const {+    out.write(sp.data(), std::streamsize(sp.size()));+  }+};+} // namespace detail++/**+ * Formatter objects can be written to streams.+ */+template <class C, class CT, bool containerMode, class... Args>+std::ostream& operator<<(+    std::basic_ostream<C, CT>& out,+    const Formatter<containerMode, Args...>& formatter) {+  using out_t = std::basic_ostream<C, CT>;+  auto writer = detail::FormatterOstreamInsertionWriterFn<out_t>{out};+  formatter(writer);+  return out;+}++/**+ * Create a formatter object.+ *+ * std::string formatted = format("{} {}", 23, 42).str();+ * LOG(INFO) << format("{} {}", 23, 42);+ * writeTo(stdout, format("{} {}", 23, 42));+ */+template <class... Args>+[[deprecated(+    "Use fmt::format instead of folly::format for better performance, build "+    "times and compatibility with std::format")]] //+Formatter<false, Args...>+format(StringPiece fmt, Args&&... args) {+  return Formatter<false, Args...>(fmt, static_cast<Args&&>(args)...);+}++/**+ * Like format(), but immediately returns the formatted string instead of an+ * intermediate format object.+ */+template <class... Args>+inline std::string sformat(StringPiece fmt, Args&&... args) {+  return Formatter<false, Args...>(fmt, static_cast<Args&&>(args)...).str();+}++/**+ * Exception class thrown when a format key is not found in the given+ * associative container keyed by strings. We inherit std::out_of_range for+ * compatibility with callers that expect exception to be thrown directly+ * by std::map or std::unordered_map.+ *+ * Having the key be at the end of the message string, we can access it by+ * simply adding its offset to what(). Not storing separate std::string key+ * makes the exception type small and noexcept-copyable like std::out_of_range,+ * and therefore able to fit in-situ in exception_wrapper.+ */+class FOLLY_EXPORT FormatKeyNotFoundException : public std::out_of_range {+ public:+  explicit FormatKeyNotFoundException(StringPiece key);++  char const* key() const noexcept { return what() + kMessagePrefix.size(); }++ private:+  static constexpr StringPiece const kMessagePrefix = "format key not found: ";+};++/**+ * Wrap a sequence or associative container so that out-of-range lookups+ * return a default value rather than throwing an exception.+ *+ * Usage:+ * format("[no_such_key"], defaulted(map, 42))  -> 42+ */+namespace detail {+template <class Container, class Value>+struct DefaultValueWrapper {+  DefaultValueWrapper(const Container& container, const Value& defaultValue)+      : container(container), defaultValue(defaultValue) {}++  const Container& container;+  const Value& defaultValue;+};+} // namespace detail++template <class Container, class Value>+detail::DefaultValueWrapper<Container, Value> defaulted(+    const Container& c, const Value& v) {+  return detail::DefaultValueWrapper<Container, Value>(c, v);+}++/**+ * Append formatted output to a string.+ *+ * std::string foo;+ * format(&foo, "{} {}", 42, 23);+ *+ * Shortcut for toAppend(format(...), &foo);+ */+template <class Str, class... Args>+typename std::enable_if<IsSomeString<Str>::value>::type format(+    Str* out, StringPiece fmt, Args&&... args) {+  Formatter<false, Args...>(fmt, static_cast<Args&&>(args)...).appendTo(*out);+}++/**+ * Utilities for all format value specializations.+ */+namespace format_value {++/**+ * Format a string in "val", obeying appropriate alignment, padding, width,+ * and precision.  Treats Align::DEFAULT as Align::LEFT, and+ * Align::PAD_AFTER_SIGN as Align::RIGHT; use formatNumber for+ * number-specific formatting.+ */+template <class FormatCallback>+void formatString(StringPiece val, FormatArg& arg, FormatCallback& cb);++/**+ * Format a number in "val"; the first prefixLen characters form the prefix+ * (sign, "0x" base prefix, etc) which must be left-aligned if the alignment+ * is Align::PAD_AFTER_SIGN.  Treats Align::DEFAULT as Align::LEFT.  Ignores+ * arg.precision, as that has a different meaning for numbers (not "maximum+ * field width")+ */+template <class FormatCallback>+void formatNumber(+    StringPiece val, int prefixLen, FormatArg& arg, FormatCallback& cb);++/**+ * Format a Formatter object recursively.  Behaves just like+ * formatString(fmt.str(), arg, cb); but avoids creating a temporary+ * string if possible.+ */+template <class FormatCallback, bool containerMode, class... Args>+void formatFormatter(+    const Formatter<containerMode, Args...>& formatter,+    FormatArg& arg,+    FormatCallback& cb);++} // namespace format_value++/*+ * Specialize folly::FormatValue for your type.+ *+ * FormatValue<T> is constructed with a (reference-collapsed) T&&, which is+ * guaranteed to stay alive until the FormatValue object is destroyed, so you+ * may keep a reference (or pointer) to it instead of making a copy.+ *+ * You must define+ *   template <class Callback>+ *   void format(FormatArg& arg, Callback& cb) const;+ * with the following semantics: format the value using the given argument.+ *+ * arg is given by non-const reference for convenience -- it won't be reused,+ * so feel free to modify it in place if necessary.  (For example, wrap an+ * existing conversion but change the default, or remove the "key" when+ * extracting an element from a container)+ *+ * Call the callback to append data to the output.  You may call the callback+ * as many times as you'd like (or not at all, if you want to output an+ * empty string)+ */++namespace detail {++template <class T, class Enable = void>+struct IsFormatter : public std::false_type {};++template <class T>+struct IsFormatter<+    T,+    typename std::enable_if<+        std::is_same<typename T::IsFormatter, detail::FormatterTag>::value>::+        type> : public std::true_type {};+} // namespace detail++} // namespace folly++#include <folly/Format-inl.h>++FOLLY_POP_WARNING
+ folly/folly/FormatArg.h view
@@ -0,0 +1,285 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <stdexcept>++#include <folly/CPortability.h>+#include <folly/Conv.h>+#include <folly/Likely.h>+#include <folly/Portability.h>+#include <folly/Range.h>+#include <folly/lang/Exception.h>++namespace folly {++struct FormatArg;++class FOLLY_EXPORT BadFormatArg : public std::invalid_argument {+ private:+  friend struct FormatArg;+  struct ErrorStrTag {};++  template <typename... A>+  static std::string str(StringPiece descr, A const&... a) {+    return to<std::string>(+        "invalid format argument {"_sp, descr, "}: "_sp, a...);+  }++ public:+  using invalid_argument::invalid_argument;+  template <typename... A>+  explicit BadFormatArg(ErrorStrTag, StringPiece descr, A const&... a)+      : invalid_argument(str(descr, a...)) {}+};++/**+ * Parsed format argument.+ */+struct FormatArg {+  /**+   * Parse a format argument from a string.  Keeps a reference to the+   * passed-in string -- does not copy the given characters.+   */+  explicit FormatArg(StringPiece sp)+      : fullArgString(sp),+        fill(kDefaultFill),+        align(Align::DEFAULT),+        sign(Sign::DEFAULT),+        basePrefix(false),+        thousandsSeparator(false),+        trailingDot(false),+        width(kDefaultWidth),+        widthIndex(kNoIndex),+        precision(kDefaultPrecision),+        presentation(kDefaultPresentation),+        nextKeyMode_(NextKeyMode::NONE) {+    if (!sp.empty()) {+      initSlow();+    }+  }++  enum class Type {+    INTEGER,+    FLOAT,+    OTHER,+  };+  /**+   * Validate the argument for the given type; throws on error.+   */+  void validate(Type type) const;++  /**+   * Throw an exception if the first argument is false.  The exception+   * message will contain the argument string as well as any passed-in+   * arguments to enforce, formatted using folly::to<std::string>.+   */+  template <typename Check, typename... Args>+  void enforce(Check const& v, Args&&... args) const {+    static_assert(std::is_constructible<bool, Check>::value, "not castable");+    if (FOLLY_UNLIKELY(!v)) {+      error(static_cast<Args&&>(args)...);+    }+  }++  template <typename... Args>+  [[noreturn]] void error(Args&&... args) const;++  /**+   * Full argument string, as passed in to the constructor.+   */+  StringPiece fullArgString;++  /**+   * Fill+   */+  static constexpr char kDefaultFill = '\0';+  char fill;++  /**+   * Alignment+   */+  enum class Align : uint8_t {+    DEFAULT,+    LEFT,+    RIGHT,+    PAD_AFTER_SIGN,+    CENTER,+    INVALID,+  };+  Align align;++  /**+   * Sign+   */+  enum class Sign : uint8_t {+    DEFAULT,+    PLUS_OR_MINUS,+    MINUS,+    SPACE_OR_MINUS,+    INVALID,+  };+  Sign sign;++  /**+   * Output base prefix (0 for octal, 0x for hex)+   */+  bool basePrefix;++  /**+   * Output thousands separator (comma)+   */+  bool thousandsSeparator;++  /**+   * Force a trailing decimal on doubles which could be rendered as ints+   */+  bool trailingDot;++  /**+   * Field width and optional argument index+   */+  static constexpr int kDefaultWidth = -1;+  static constexpr int kDynamicWidth = -2;+  static constexpr int kNoIndex = -1;+  int width;+  int widthIndex;++  /**+   * Precision+   */+  static constexpr int kDefaultPrecision = -1;+  int precision;++  /**+   * Presentation+   */+  static constexpr char kDefaultPresentation = '\0';+  char presentation;++  /**+   * Split a key component from "key", which must be non-empty (an exception+   * is thrown otherwise).+   */+  template <bool emptyOk = false>+  StringPiece splitKey();++  /**+   * Is the entire key empty?+   */+  bool keyEmpty() const {+    return nextKeyMode_ == NextKeyMode::NONE && key_.empty();+  }++  /**+   * Split an key component from "key", which must be non-empty and a valid+   * integer (an exception is thrown otherwise).+   */+  int splitIntKey();++  void setNextIntKey(int val) {+    assert(nextKeyMode_ == NextKeyMode::NONE);+    nextKeyMode_ = NextKeyMode::INT;+    nextIntKey_ = val;+  }++  void setNextKey(StringPiece val) {+    assert(nextKeyMode_ == NextKeyMode::NONE);+    nextKeyMode_ = NextKeyMode::STRING;+    nextKey_ = val;+  }++ private:+  void initSlow();+  template <bool emptyOk>+  StringPiece doSplitKey();++  StringPiece key_;+  int nextIntKey_;+  StringPiece nextKey_;+  enum class NextKeyMode {+    NONE,+    INT,+    STRING,+  };+  NextKeyMode nextKeyMode_;+};++template <typename... Args>+[[noreturn]] inline void FormatArg::error(Args&&... args) const {+  // take advantage of throw_exception decaying char const (&)[N} to char const*+  // as a special case of the facility+  throw_exception<BadFormatArg>(+      BadFormatArg::ErrorStrTag{}, fullArgString, static_cast<Args&&>(args)...);+}++template <bool emptyOk>+inline StringPiece FormatArg::splitKey() {+  enforce(nextKeyMode_ != NextKeyMode::INT, "integer key expected");+  return doSplitKey<emptyOk>();+}++template <bool emptyOk>+inline StringPiece FormatArg::doSplitKey() {+  if (nextKeyMode_ == NextKeyMode::STRING) {+    nextKeyMode_ = NextKeyMode::NONE;+    if (!emptyOk) { // static+      enforce(!nextKey_.empty(), "non-empty key required");+    }+    return nextKey_;+  }++  if (key_.empty()) {+    if (!emptyOk) { // static+      error("non-empty key required");+    }+    return StringPiece();+  }++  const char* b = key_.begin();+  const char* e = key_.end();+  const char* p;+  if (e[-1] == ']') {+    --e;+    p = static_cast<const char*>(memchr(b, '[', size_t(e - b)));+    enforce(p != nullptr, "unmatched ']'");+  } else {+    p = static_cast<const char*>(memchr(b, '.', size_t(e - b)));+  }+  if (p) {+    key_.assign(p + 1, e);+  } else {+    p = e;+    key_.clear();+  }+  if (!emptyOk) { // static+    enforce(b != p, "non-empty key required");+  }+  return StringPiece(b, p);+}++inline int FormatArg::splitIntKey() {+  if (nextKeyMode_ == NextKeyMode::INT) {+    nextKeyMode_ = NextKeyMode::NONE;+    return nextIntKey_;+  }+  auto result = tryTo<int>(doSplitKey<true>());+  enforce(result, "integer key required");+  return *result;+}++} // namespace folly
+ folly/folly/FormatTraits.h view
@@ -0,0 +1,65 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <cstddef>+#include <type_traits>++namespace folly {+namespace detail {++// Shortcut, so we don't have to use enable_if everywhere+struct FormatTraitsBase {+  typedef void enabled;+};++// Traits that define enabled, value_type, and at() for anything+// indexable with integral keys: pointers, arrays, vectors, and maps+// with integral keys+template <class T, class Enable = void>+struct IndexableTraits;++// Base class for sequences (vectors, deques)+template <class C>+struct IndexableTraitsSeq : public FormatTraitsBase {+  typedef C container_type;+  typedef typename C::value_type value_type;++  static const value_type& at(const C& c, int idx) { return c.at(idx); }++  static const value_type& at(const C& c, int idx, const value_type& dflt) {+    return (idx >= 0 && size_t(idx) < c.size()) ? c.at(idx) : dflt;+  }+};++// Base class for associative types (maps)+template <class C>+struct IndexableTraitsAssoc : public FormatTraitsBase {+  typedef typename C::value_type::second_type value_type;++  static const value_type& at(const C& c, int idx) {+    return c.at(static_cast<typename C::key_type>(idx));+  }++  static const value_type& at(const C& c, int idx, const value_type& dflt) {+    auto pos = c.find(static_cast<typename C::key_type>(idx));+    return pos != c.end() ? pos->second : dflt;+  }+};++} // namespace detail+} // namespace folly
+ folly/folly/Function.h view
@@ -0,0 +1,1169 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++//+// Docs: https://fburl.com/fbcref_function+//++/**+ * @class folly::Function+ * @refcode folly/docs/examples/folly/Function.cpp+ *+ * A polymorphic function wrapper that is not copyable and does not+ * require the wrapped function to be copy constructible.+ *+ * `folly::Function` is a polymorphic function wrapper, similar to+ * `std::function`. The template parameters of the `folly::Function` define+ * the parameter signature of the wrapped callable, but not the specific+ * type of the embedded callable. E.g. a `folly::Function<int(int)>`+ * can wrap callables that return an `int` when passed an `int`. This can be a+ * function pointer or any class object implementing one or both of+ *+ *     int operator(int);+ *     int operator(int) const;+ *+ * If both are defined, the non-const one takes precedence.+ *+ * Unlike `std::function`, a `folly::Function` can wrap objects that are not+ * copy constructible. As a consequence of this, `folly::Function` itself+ * is not copyable, either.+ *+ * Another difference is that, unlike `std::function`, `folly::Function` treats+ * const-ness of methods correctly. While a `std::function` allows to wrap+ * an object that only implements a non-const `operator()` and invoke+ * a const-reference of the `std::function`, `folly::Function` requires you to+ * declare a function type as const in order to be able to execute it on a+ * const-reference.+ *+ * For example:+ *+ *     class Foo {+ *      public:+ *       void operator()() {+ *         // mutates the Foo object+ *       }+ *     };+ *+ *     class Bar {+ *       std::function<void(void)> foo_; // wraps a Foo object+ *      public:+ *       void mutateFoo() const+ *       {+ *         foo_();+ *       }+ *     };+ *+ * Even though `mutateFoo` is a const-method, so it can only reference `foo_`+ * as const, it is able to call the non-const `operator()` of the Foo+ * object that is embedded in the foo_ function.+ *+ * `folly::Function` will not allow you to do that. You will have to decide+ * whether you need to invoke your wrapped callable from a const reference+ * (like in the example above), in which case it will only wrap a+ * `operator() const`. If your functor does not implement that,+ * compilation will fail. If you do not require to be able to invoke the+ * wrapped function in a const context, you can wrap any functor that+ * implements either or both of const and non-const `operator()`.+ *+ * The template parameter of `folly::Function`, the `FunctionType`, can be+ * const-qualified. Be aware that the const is part of the function signature.+ * It does not mean that the function type is a const type.+ *+ *   using FunctionType = R(Args...);+ *   using ConstFunctionType = R(Args...) const;+ *+ * In this example, `FunctionType` and `ConstFunctionType` are different+ * types. `ConstFunctionType` is not the same as `const FunctionType`.+ * As a matter of fact, trying to use the latter should emit a compiler+ * warning or error, because it has no defined meaning.+ *+ *     // This will not compile:+ *     folly::Function<void(void) const> func = Foo();+ *     // because Foo does not have a member function of the form:+ *     //   void operator()() const;+ *+ *     // This will compile just fine:+ *     folly::Function<void(void)> func = Foo();+ *     // and it will wrap the existing member function:+ *     //   void operator()();+ *+ * When should a const function type be used? As a matter of fact, you will+ * probably not need to use const function types very often. See the following+ * example:+ *+ *     class Bar {+ *       folly::Function<void()> func_;+ *       folly::Function<void() const> constFunc_;+ *+ *       void someMethod() {+ *         // Can call func_.+ *         func_();+ *         // Can call constFunc_.+ *         constFunc_();+ *       }+ *+ *       void someConstMethod() const {+ *         // Can call constFunc_.+ *         constFunc_();+ *         // However, cannot call func_ because a non-const method cannot+ *         // be called from a const one.+ *       }+ *     };+ *+ * As you can see, whether the `folly::Function`'s function type should+ * be declared const or not is identical to whether a corresponding method+ * would be declared const or not.+ *+ * You only require a `folly::Function` to hold a const function type, if you+ * intend to invoke it from within a const context. This is to ensure that+ * you cannot mutate its inner state when calling in a const context.+ *+ * This is how the const/non-const choice relates to lambda functions:+ *+ *     // Non-mutable lambdas: can be stored in a non-const...+ *     folly::Function<void(int)> print_number =+ *       [] (int number) { std::cout << number << std::endl; };+ *+ *     // ...as well as in a const folly::Function+ *     folly::Function<void(int) const> print_number_const =+ *       [] (int number) { std::cout << number << std::endl; };+ *+ *     // Mutable lambda: can only be stored in a non-const folly::Function:+ *     int number = 0;+ *     folly::Function<void()> print_number =+ *       [number] () mutable { std::cout << ++number << std::endl; };+ *     // Trying to store the above mutable lambda in a+ *     // `folly::Function<void() const>` would lead to a compiler error:+ *     // error: no viable conversion from '(lambda at ...)' to+ *     // 'folly::Function<void () const>'+ *+ * Casting between const and non-const `folly::Function`s:+ * conversion from const to non-const signatures happens implicitly. Any+ * function that takes a `folly::Function<R(Args...)>` can be passed+ * a `folly::Function<R(Args...) const>` without explicit conversion.+ * This is safe, because casting from const to non-const only entails giving+ * up the ability to invoke the function from a const context.+ * Casting from a non-const to a const signature is potentially dangerous,+ * as it means that a function that may change its inner state when invoked+ * is made possible to call from a const context. Therefore this cast does+ * not happen implicitly. The function `folly::constCastFunction` can+ * be used to perform the cast.+ *+ *     // Mutable lambda: can only be stored in a non-const folly::Function:+ *     int number = 0;+ *     folly::Function<void()> print_number =+ *       [number] () mutable { std::cout << ++number << std::endl; };+ *+ *     // const-cast to a const folly::Function:+ *     folly::Function<void() const> print_number_const =+ *       constCastFunction(std::move(print_number));+ *+ * When to use const function types?+ * Generally, only when you need them. When you use a `folly::Function` as a+ * member of a struct or class, only use a const function signature when you+ * need to invoke the function from const context.+ * When passing a `folly::Function` to a function, the function should accept+ * a non-const `folly::Function` whenever possible, i.e. when it does not+ * need to pass on or store a const `folly::Function`. This is the least+ * possible constraint: you can always pass a const `folly::Function` when+ * the function accepts a non-const one.+ *+ * How does the const behaviour compare to `std::function`?+ * `std::function` can wrap object with non-const invocation behaviour but+ * exposes them as const. The equivalent behaviour can be achieved with+ * `folly::Function` like so:+ *+ *     std::function<void(void)> stdfunc = someCallable;+ *+ *     folly::Function<void(void) const> uniqfunc = constCastFunction(+ *       folly::Function<void(void)>(someCallable)+ *     );+ *+ * You need to wrap the callable first in a non-const `folly::Function` to+ * select a non-const invoke operator (or the const one if no non-const one is+ * present), and then move it into a const `folly::Function` using+ * `constCastFunction`.+ */++#pragma once++#include <cstring>+#include <functional>+#include <memory>+#include <new>+#include <type_traits>+#include <utility>++#include <folly/CppAttributes.h>+#include <folly/Portability.h>+#include <folly/Traits.h>+#include <folly/functional/Invoke.h>+#include <folly/lang/Align.h>+#include <folly/lang/Exception.h>+#include <folly/lang/New.h>++namespace folly {++template <typename FunctionType>+class Function;++template <typename ReturnType, typename... Args>+Function<ReturnType(Args...) const> constCastFunction(+    Function<ReturnType(Args...)>&&) noexcept;++template <typename ReturnType, typename... Args>+Function<ReturnType(Args...) const noexcept> constCastFunction(+    Function<ReturnType(Args...) noexcept>&&) noexcept;++namespace detail {+namespace function {++enum class Op { MOVE, NUKE, HEAP };++union Data {+  struct BigTrivialLayout {+    void* big;+    std::size_t size;+    std::size_t align;+  };++  void* big;+  BigTrivialLayout bigt;+  std::aligned_storage<6 * sizeof(void*)>::type tiny;+};++struct CoerceTag {};++template <typename T>+using FunctionNullptrTest =+    decltype(static_cast<bool>(static_cast<T const&>(T(nullptr)) == nullptr));++template <typename T>+constexpr bool IsNullptrCompatible = is_detected_v<FunctionNullptrTest, T>;++template <typename T, std::enable_if_t<!IsNullptrCompatible<T>, int> = 0>+constexpr bool isEmptyFunction(T const&) {+  return false;+}+template <typename T, std::enable_if_t<IsNullptrCompatible<T>, int> = 0>+constexpr bool isEmptyFunction(T const& t) {+  return static_cast<bool>(t == nullptr);+}++template <typename F, typename... Args>+using CallableResult = decltype(FOLLY_DECLVAL(F&&)(FOLLY_DECLVAL(Args&&)...));++template <typename F, typename... Args>+constexpr bool CallableNoexcept =+    noexcept(FOLLY_DECLVAL(F&&)(FOLLY_DECLVAL(Args&&)...));++template <+    typename From,+    typename To,+    typename = typename std::enable_if<+        !std::is_reference<To>::value || std::is_reference<From>::value>::type>+using IfSafeResultImpl = decltype(void(static_cast<To>(FOLLY_DECLVAL(From))));++#if defined(_MSC_VER)+//  Need a workaround for MSVC to avoid the inscrutable error:+//+//      folly\function.h(...) : fatal error C1001: An internal error has+//          occurred in the compiler.+//      (compiler file 'f:\dd\vctools\compiler\utc\src\p2\main.c', line 258)+//      To work around this problem, try simplifying or changing the program+//          near the locations listed above.+template <typename T>+using CallArg = T&&;+#else+template <typename T>+using CallArg = conditional_t<is_register_pass_v<T>, T, T&&>;+#endif++template <typename F, bool Nx, typename R, typename... A>+class FunctionTraitsSharedProxy {+  std::shared_ptr<Function<F>> sp_;++ public:+  explicit FunctionTraitsSharedProxy(std::nullptr_t) noexcept {}+  explicit FunctionTraitsSharedProxy(Function<F>&& func)+      : sp_(func ? std::make_shared<Function<F>>(std::move(func))+                 : std::shared_ptr<Function<F>>()) {}+  R operator()(A... args) const noexcept(Nx) {+    if (!sp_) {+      throw_exception<std::bad_function_call>();+    }+    return (*sp_)(static_cast<A&&>(args)...);+  }++  explicit operator bool() const noexcept { return sp_ != nullptr; }++  friend bool operator==(+      FunctionTraitsSharedProxy const& proxy, std::nullptr_t) noexcept {+    return proxy.sp_ == nullptr;+  }+  friend bool operator!=(+      FunctionTraitsSharedProxy const& proxy, std::nullptr_t) noexcept {+    return proxy.sp_ != nullptr;+  }++  friend bool operator==(+      std::nullptr_t, FunctionTraitsSharedProxy const& proxy) noexcept {+    return proxy.sp_ == nullptr;+  }+  friend bool operator!=(+      std::nullptr_t, FunctionTraitsSharedProxy const& proxy) noexcept {+    return proxy.sp_ != nullptr;+  }+};++template <+    typename Fun,+    bool Small,+    bool Nx,+    typename ReturnType,+    typename... Args>+ReturnType call_(Args... args, Data& p) noexcept(Nx) {+  auto& fn = *static_cast<Fun*>(Small ? &p.tiny : p.big);+  if constexpr (std::is_void<ReturnType>::value) {+    fn(static_cast<Args&&>(args)...);+  } else {+    return fn(static_cast<Args&&>(args)...);+  }+}++template <typename FunctionType>+struct FunctionTraits;++template <typename ReturnType, typename... Args>+struct FunctionTraits<ReturnType(Args...)> {+  using Call = ReturnType (*)(CallArg<Args>..., Data&);+  using ConstSignature = ReturnType(Args...) const;+  using NonConstSignature = ReturnType(Args...);+  using OtherSignature = ConstSignature;++  template <typename F, typename R = CallableResult<F&, Args...>>+  using IfSafeResult = IfSafeResultImpl<R, ReturnType>;++  template <typename Fun, bool Small>+  static constexpr Call call =+      call_<Fun, Small, false, ReturnType, CallArg<Args>...>;++  static ReturnType uninitCall(CallArg<Args>..., Data&) {+    throw_exception<std::bad_function_call>();+  }++  ReturnType operator()(Args... args) {+    auto& fn = *static_cast<Function<NonConstSignature>*>(this);+    return fn.call_(static_cast<Args&&>(args)..., fn.data_);+  }++  using SharedProxy =+      FunctionTraitsSharedProxy<NonConstSignature, false, ReturnType, Args...>;+};++template <typename ReturnType, typename... Args>+struct FunctionTraits<ReturnType(Args...) const> {+  using Call = ReturnType (*)(CallArg<Args>..., Data&);+  using ConstSignature = ReturnType(Args...) const;+  using NonConstSignature = ReturnType(Args...);+  using OtherSignature = NonConstSignature;++  template <typename F, typename R = CallableResult<const F&, Args...>>+  using IfSafeResult = IfSafeResultImpl<R, ReturnType>;++  template <typename Fun, bool Small>+  static constexpr Call call =+      call_<const Fun, Small, false, ReturnType, CallArg<Args>...>;++  static ReturnType uninitCall(CallArg<Args>..., Data&) {+    throw_exception<std::bad_function_call>();+  }++  ReturnType operator()(Args... args) const {+    auto& fn = *static_cast<const Function<ConstSignature>*>(this);+    return fn.call_(static_cast<Args&&>(args)..., fn.data_);+  }++  using SharedProxy =+      FunctionTraitsSharedProxy<ConstSignature, false, ReturnType, Args...>;+};++template <typename ReturnType, typename... Args>+struct FunctionTraits<ReturnType(Args...) noexcept> {+  using Call = ReturnType (*)(CallArg<Args>..., Data&) noexcept;+  using ConstSignature = ReturnType(Args...) const noexcept;+  using NonConstSignature = ReturnType(Args...) noexcept;+  using OtherSignature = ConstSignature;++  template <+      typename F,+      typename R = CallableResult<F&, Args...>,+      std::enable_if_t<CallableNoexcept<F&, Args...>, int> = 0>+  using IfSafeResult = IfSafeResultImpl<R, ReturnType>;++  template <typename Fun, bool Small>+  static constexpr Call call =+      call_<Fun, Small, true, ReturnType, CallArg<Args>...>;++  static ReturnType uninitCall(CallArg<Args>..., Data&) noexcept {+    terminate_with<std::bad_function_call>();+  }++  ReturnType operator()(Args... args) noexcept {+    auto& fn = *static_cast<Function<NonConstSignature>*>(this);+    return fn.call_(static_cast<Args&&>(args)..., fn.data_);+  }++  using SharedProxy =+      FunctionTraitsSharedProxy<NonConstSignature, true, ReturnType, Args...>;+};++template <typename ReturnType, typename... Args>+struct FunctionTraits<ReturnType(Args...) const noexcept> {+  using Call = ReturnType (*)(CallArg<Args>..., Data&) noexcept;+  using ConstSignature = ReturnType(Args...) const noexcept;+  using NonConstSignature = ReturnType(Args...) noexcept;+  using OtherSignature = NonConstSignature;++  template <+      typename F,+      typename R = CallableResult<const F&, Args...>,+      std::enable_if_t<CallableNoexcept<const F&, Args...>, int> = 0>+  using IfSafeResult = IfSafeResultImpl<R, ReturnType>;++  template <typename Fun, bool Small>+  static constexpr Call call =+      call_<const Fun, Small, true, ReturnType, CallArg<Args>...>;++  static ReturnType uninitCall(CallArg<Args>..., Data&) noexcept {+    terminate_with<std::bad_function_call>();+  }++  ReturnType operator()(Args... args) const noexcept {+    auto& fn = *static_cast<const Function<ConstSignature>*>(this);+    return fn.call_(static_cast<Args&&>(args)..., fn.data_);+  }++  using SharedProxy =+      FunctionTraitsSharedProxy<ConstSignature, true, ReturnType, Args...>;+};++// These are control functions. They type-erase the operations of move-+// construction, destruction, and conversion to bool.+//+// The interface operations are noexcept, so the implementations are as well.+// Having the implementations be noexcept in the type permits callers to omit+// exception-handling machinery.+//+// This is intentionally instantiated per size rather than per function in order+// to minimize the number of instantiations. It would be safe to minimize+// instantiations even more by simply having a single non-template function that+// copies sizeof(Data) bytes rather than only copying sizeof(Fun) bytes, but+// then for small function types it would be likely to cross cache lines without+// need. But it is only necessary to handle those sizes which are multiples of+// the alignof(Data), and to round up other sizes.+struct DispatchSmallTrivial {+  static constexpr bool is_in_situ = true;+  static constexpr bool is_trivial = true;++  template <std::size_t Size>+  static std::size_t exec_(Op o, Data* src, Data* dst) noexcept {+    switch (o) {+      case Op::MOVE:+        std::memcpy(static_cast<void*>(dst), static_cast<void*>(src), Size);+        break;+      case Op::NUKE:+        break;+      case Op::HEAP:+        break;+      default: /* unexpected */+        abort();+    }+    return 0U;+  }+  template <std::size_t size, std::size_t adjust = alignof(Data) - 1>+  static constexpr std::size_t size_ = (size + adjust) & ~adjust;+  template <typename Fun>+  static constexpr auto exec = exec_<size_<sizeof(Fun)>>;+};++struct DispatchBigTrivial {+  static constexpr bool is_in_situ = false;+  static constexpr bool is_trivial = true;++  template <typename Fun, typename Base>+  static constexpr auto call = Base::template callBig<Fun>;++  static constexpr bool is_align_large(size_t align) {+    return align > __STDCPP_DEFAULT_NEW_ALIGNMENT__;+  }++  template <bool IsAlignLarge>+  static std::size_t exec_(Op o, Data* src, Data* dst) noexcept {+    switch (o) {+      case Op::MOVE:+        dst->bigt = src->bigt;+        src->bigt = {};+        break;+      case Op::NUKE:+        IsAlignLarge+            ? operator_delete(+                  src->big, src->bigt.size, std::align_val_t(src->bigt.align))+            : operator_delete(src->big, src->bigt.size);+        break;+      case Op::HEAP:+        break;+      default: /* unexpected */+        abort();+    }+    return src->bigt.size;+  }+  template <typename T>+  static constexpr auto exec = exec_<is_align_large(alignof(T))>;++  FOLLY_ALWAYS_INLINE static void ctor(+      Data& data,+      void const* fun,+      std::size_t size,+      std::size_t align) noexcept {+    // cannot use type-specific new since type-specific new is overrideable+    // in concert with type-specific delete+    data.bigt.big = is_align_large(align)+        ? operator_new(size, std::align_val_t(align))+        : operator_new(size);+    data.bigt.size = size;+    data.bigt.align = align;+    std::memcpy(data.bigt.big, fun, size);+  }+};++struct DispatchSmall {+  static constexpr bool is_in_situ = true;+  static constexpr bool is_trivial = false;++  template <typename Fun>+  static std::size_t exec(Op o, Data* src, Data* dst) noexcept {+    switch (o) {+      case Op::MOVE:+        ::new (static_cast<void*>(&dst->tiny)) Fun(static_cast<Fun&&>(+            *static_cast<Fun*>(static_cast<void*>(&src->tiny))));+        [[fallthrough]];+      case Op::NUKE:+        static_cast<Fun*>(static_cast<void*>(&src->tiny))->~Fun();+        break;+      case Op::HEAP:+        break;+      default: /* unexpected */+        abort();+    }+    return 0U;+  }+};++struct DispatchBig {+  static constexpr bool is_in_situ = false;+  static constexpr bool is_trivial = false;++  template <typename Fun>+  static std::size_t exec(Op o, Data* src, Data* dst) noexcept {+    switch (o) {+      case Op::MOVE:+        dst->big = src->big;+        src->big = nullptr;+        break;+      case Op::NUKE:+        delete static_cast<Fun*>(src->big);+        break;+      case Op::HEAP:+        break;+      default: /* unexpected */+        abort();+    }+    return sizeof(Fun);+  }+};++template <bool InSitu, bool IsTriv>+struct Dispatch;+template <>+struct Dispatch<true, true> : DispatchSmallTrivial {};+template <>+struct Dispatch<true, false> : DispatchSmall {};+template <>+struct Dispatch<false, true> : DispatchBigTrivial {};+template <>+struct Dispatch<false, false> : DispatchBig {};++template <+    typename Fun,+    bool InSituSize = sizeof(Fun) <= sizeof(Data),+    bool InSituAlign = alignof(Fun) <= alignof(Data),+    bool InSituNoexcept = noexcept(Fun(FOLLY_DECLVAL(Fun)))>+using DispatchOf = Dispatch<+    InSituSize && InSituAlign && InSituNoexcept,+    std::is_trivially_copyable_v<Fun>>;++// This cannot be done inseide `Function` class, because the word+// `Function` there refers to the instantion and not the template.+template <typename T>+constexpr bool is_instantiation_of_folly_function_v =+    is_instantiation_of_v<Function, T>;++} // namespace function+} // namespace detail++template <typename FunctionType>+class Function final : private detail::function::FunctionTraits<FunctionType> {+  // These utility types are defined outside of the template to reduce+  // the number of instantiations, and then imported in the class+  // namespace for convenience.+  using Data = detail::function::Data;+  using Op = detail::function::Op;+  using CoerceTag = detail::function::CoerceTag;++  using Traits = detail::function::FunctionTraits<FunctionType>;+  using Call = typename Traits::Call;+  using Exec = std::size_t (*)(Op, Data*, Data*) noexcept;++  // The `data_` member is mutable to allow `constCastFunction` to work without+  // invoking undefined behavior. Const-correctness is only violated when+  // `FunctionType` is a const function type (e.g., `int() const`) and `*this`+  // is the result of calling `constCastFunction`.+  mutable Data data_{unsafe_default_initialized};+  Call call_{&Traits::uninitCall};+  Exec exec_{nullptr};++  std::size_t exec(Op o, Data* src, Data* dst) const {+    if (!exec_) {+      return 0U;+    }+    return exec_(o, src, dst);+  }++  friend Traits;+  friend Function<typename Traits::ConstSignature> folly::constCastFunction<>(+      Function<typename Traits::NonConstSignature>&&) noexcept;+  friend class Function<typename Traits::OtherSignature>;++  template <typename Signature>+  Function(Function<Signature>&& fun, CoerceTag) {+    using Fun = Function<Signature>;+    if (fun) {+      data_.big = new Fun(static_cast<Fun&&>(fun));+      call_ = Traits::template call<Fun, false>;+      exec_ = Exec(detail::function::DispatchBig::exec<Fun>);+    }+  }++  Function(Function<typename Traits::OtherSignature>&& that, CoerceTag) noexcept+      : call_(that.call_), exec_(that.exec_) {+    that.call_ = &Traits::uninitCall;+    that.exec_ = nullptr;+    exec(Op::MOVE, &that.data_, &data_);+  }++ public:+  /**+   * Default constructor. Constructs an empty Function.+   */+  constexpr Function() = default;++  // not copyable+  Function(const Function&) = delete;++#ifdef __OBJC__+  // Make sure Objective C blocks are copied+  template <class ReturnType, class... Args>+  /*implicit*/ Function(ReturnType (^objCBlock)(Args... args))+      : Function([blockCopy = (ReturnType(^)(Args...))[objCBlock copy]](+                     Args... args) { return blockCopy(args...); }) {}+#endif++  /**+   * Move constructor+   */+  Function(Function&& that) noexcept : call_(that.call_), exec_(that.exec_) {+    // that must be uninitialized before exec() call in the case of self move+    that.call_ = &Traits::uninitCall;+    that.exec_ = nullptr;+    exec(Op::MOVE, &that.data_, &data_);+  }++  /**+   * Constructs an empty `Function`.+   */+  /* implicit */ constexpr Function(std::nullptr_t) noexcept {}++  /**+   * Constructs a new `Function` from any callable object that is _not_ a+   * `folly::Function`.+   *+   * \note `typename Traits::template IfSafeResult<Fun>` prevents this overload+   * from being selected by overload resolution when `fun` is not a compatible+   * function.+   *+   * \note The noexcept requires some explanation. `is_in_situ` is true when the+   * decayed type fits within the internal buffer and is noexcept-movable. But+   * this ctor might copy, not move. What we need here, if this ctor does a+   * copy, is that this ctor be noexcept when the copy is noexcept. That is not+   * checked in `is_in_situ`, and shouldn't be, because once the `Function` is+   * constructed, the contained object is never copied. This check is for this+   * ctor only, in the case that this ctor does a copy.+   *+   * @param fun function pointers, pointers to static+   *     member functions, `std::reference_wrapper` objects, `std::function`+   *     objects, and arbitrary objects that implement `operator()` if the+   * parameter signature matches (i.e. it returns an object convertible to `R`+   * when called with `Args...`).+   */+  template <+      typename Fun,+      typename = std::enable_if_t<+          !detail::function::is_instantiation_of_folly_function_v<Fun>>,+      typename = typename Traits::template IfSafeResult<Fun>>+  /* implicit */ constexpr Function(Fun fun) noexcept(+      detail::function::DispatchOf<Fun>::is_in_situ) {+    using Dispatch = detail::function::DispatchOf<Fun>;+    if constexpr (detail::function::IsNullptrCompatible<Fun>) {+      if (detail::function::isEmptyFunction(fun)) {+        return;+      }+    }+    if constexpr (Dispatch::is_in_situ) {+      if constexpr (+          !std::is_empty<Fun>::value || !std::is_trivially_copyable_v<Fun>) {+        ::new (&data_.tiny) Fun(static_cast<Fun&&>(fun));+      }+    } else {+      if constexpr (Dispatch::is_trivial) {+        Dispatch::ctor(data_, &fun, sizeof(Fun), alignof(Fun));+      } else {+        data_.big = new Fun(static_cast<Fun&&>(fun));+      }+    }+    call_ = Traits::template call<Fun, Dispatch::is_in_situ>;+    exec_ = Exec(Dispatch::template exec<Fun>);+  }++  /**+   * For move-constructing from a `folly::Function<X(Ys...) [const?]>`.+   *+   * For a `Function` with a `const` function type, the object must be+   * callable from a `const`-reference, i.e. implement `operator() const`.+   * For a `Function` with a non-`const` function type, the object will+   * be called from a non-const reference, which means that it will execute+   * a non-const `operator()` if it is defined, and falls back to+   * `operator() const` otherwise.+   */+  template <+      typename Signature,+      typename Fun = Function<Signature>,+      // prevent gcc from making this a better match than move-ctor+      typename = std::enable_if_t<!std::is_same<Function, Fun>::value>,+      typename = typename Traits::template IfSafeResult<Fun>>+  Function(Function<Signature>&& that) noexcept(+      noexcept(Function(std::move(that), CoerceTag{})))+      : Function(std::move(that), CoerceTag{}) {}++  /**+   * If `ptr` is null, constructs an empty `Function`. Otherwise,+   * this constructor is equivalent to `Function(std::mem_fn(ptr))`.+   */+  template <+      typename Member,+      typename Class,+      // Prevent this overload from being selected when `ptr` is not a+      // compatible member function pointer.+      typename = decltype(Function(std::mem_fn((Member Class::*)0)))>+  /* implicit */ Function(Member Class::*ptr) noexcept {+    if (ptr) {+      *this = std::mem_fn(ptr);+    }+  }++  ~Function() { exec(Op::NUKE, &data_, nullptr); }++  Function& operator=(const Function&) = delete;++#ifdef __OBJC__+  // Make sure Objective C blocks are copied+  template <class ReturnType, class... Args>+  /* implicit */ Function& operator=(ReturnType (^objCBlock)(Args... args)) {+    (*this) =+        [blockCopy = (ReturnType(^)(Args...))[objCBlock copy]](Args... args) {+          return blockCopy(args...);+        };+    return *this;+  }+#endif++  /**+   * Move assignment operator+   *+   * \note Leaves `that` in a valid but unspecified state. If `&that == this`+   * then `*this` is left in a valid but unspecified state.+   */+  Function& operator=(Function&& that) noexcept {+    exec(Op::NUKE, &data_, nullptr);+    if (FOLLY_LIKELY(this != &that)) {+      that.exec(Op::MOVE, &that.data_, &data_);+      exec_ = that.exec_;+      call_ = that.call_;+    }+    that.exec_ = nullptr;+    that.call_ = &Traits::uninitCall;+    return *this;+  }++  /**+   * Assigns a callable object to this `Function`. If the operation fails,+   * `*this` is left unmodified.+   *+   * \note `typename = decltype(Function(FOLLY_DECLVAL(Fun&&)))` prevents this+   * overload from being selected by overload resolution when `fun` is not a+   * compatible function.+   */+  template <+      typename Fun,+      typename...,+      bool Nx = noexcept(Function(FOLLY_DECLVAL(Fun&&)))>+  Function& operator=(Fun fun) noexcept(Nx) {+    // Doing this in place is more efficient when we can do so safely.+    if (Nx) {+      // Q: Why is is safe to destroy and reconstruct this object in place?+      // A: See the explanation in the move assignment operator.+      this->~Function();+      ::new (this) Function(static_cast<Fun&&>(fun));+    } else {+      // Construct a temporary and (nothrow) swap.+      Function(static_cast<Fun&&>(fun)).swap(*this);+    }+    return *this;+  }++  /**+   * For assigning from a `Function<X(Ys..) [const?]>`.+   */+  template <+      typename Signature,+      typename...,+      typename = typename Traits::template IfSafeResult<Function<Signature>>>+  Function& operator=(Function<Signature>&& that) noexcept(+      noexcept(Function(std::move(that)))) {+    return (*this = Function(std::move(that)));+  }++  /**+   * Clears this `Function`.+   */+  Function& operator=(std::nullptr_t) noexcept { return (*this = Function()); }++  /**+   * If `ptr` is null, clears this `Function`. Otherwise, this assignment+   * operator is equivalent to `*this = std::mem_fn(ptr)`.+   */+  template <typename Member, typename Class>+  auto operator=(Member Class::*ptr) noexcept+      // Prevent this overload from being selected when `ptr` is not a+      // compatible member function pointer.+      -> decltype(operator=(std::mem_fn(ptr))) {+    return ptr ? (*this = std::mem_fn(ptr)) : (*this = Function());+  }++  /**+   * Call the wrapped callable object with the specified arguments.+   */+  using Traits::operator();++  /**+   * Exchanges the callable objects of `*this` and `that`.+   *+   * @param that a folly::Function ref+   */+  void swap(Function& that) noexcept { std::swap(*this, that); }++  /**+   * Returns `true` if this `Function` contains a callable, i.e. is+   * non-empty.+   */+  explicit operator bool() const noexcept { return exec_ != nullptr; }++  /**+   * Returns the size of the allocation made to store the callable on the+   * heap. If `0` is returned, there has been no additional memory+   * allocation because the callable is stored within the `Function` object.+   */+  std::size_t heapAllocatedMemory() const noexcept {+    return exec(Op::HEAP, &data_, nullptr);+  }++  using typename Traits::SharedProxy;++  /**+   * Move this `Function` into a copyable callable object, of which all copies+   * share the state.+   */+  SharedProxy asSharedProxy() && { return SharedProxy{std::move(*this)}; }++  /**+   * Construct a `std::function` by moving in the contents of this `Function`.+   * Note that the returned `std::function` will share its state (i.e. captured+   * data) across all copies you make of it, so be very careful when copying.+   */+  std::function<typename Traits::NonConstSignature> asStdFunction() && {+    return std::move(*this).asSharedProxy();+  }+};++template <typename FunctionType>+void swap(Function<FunctionType>& lhs, Function<FunctionType>& rhs) noexcept {+  lhs.swap(rhs);+}++template <typename FunctionType>+bool operator==(const Function<FunctionType>& fn, std::nullptr_t) {+  return !fn;+}++template <typename FunctionType>+bool operator==(std::nullptr_t, const Function<FunctionType>& fn) {+  return !fn;+}++template <typename FunctionType>+bool operator!=(const Function<FunctionType>& fn, std::nullptr_t) {+  return !(fn == nullptr);+}++template <typename FunctionType>+bool operator!=(std::nullptr_t, const Function<FunctionType>& fn) {+  return !(nullptr == fn);+}++/**+ * Casts a `folly::Function` from non-const to a const signature.+ *+ * NOTE: The name of `constCastFunction` should warn you that something+ * potentially dangerous is happening. As a matter of fact, using+ * `std::function` always involves this potentially dangerous aspect, which+ * is why it is not considered fully const-safe or even const-correct.+ * However, in most of the cases you will not need the dangerous aspect at all.+ * Either you do not require invocation of the function from a const context,+ * in which case you do not need to use `constCastFunction` and just+ * use a non-const `folly::Function`. Or, you may need invocation from const,+ * but the callable you are wrapping does not mutate its state (e.g. it is a+ * class object and implements `operator() const`, or it is a normal,+ * non-mutable lambda), in which case you can wrap the callable in a const+ * `folly::Function` directly, without using `constCastFunction`.+ * Only if you require invocation from a const context of a callable that+ * may mutate itself when invoked you have to go through the above procedure.+ * However, in that case what you do is potentially dangerous and requires+ * the equivalent of a `const_cast`, hence you need to call+ * `constCastFunction`.+ *+ * @param that a non-const folly::Function.+ */+template <typename ReturnType, typename... Args>+Function<ReturnType(Args...) const> constCastFunction(+    Function<ReturnType(Args...)>&& that) noexcept {+  return Function<ReturnType(Args...) const>{+      std::move(that), detail::function::CoerceTag{}};+}++template <typename ReturnType, typename... Args>+Function<ReturnType(Args...) const> constCastFunction(+    Function<ReturnType(Args...) const>&& that) noexcept {+  return std::move(that);+}++template <typename ReturnType, typename... Args>+Function<ReturnType(Args...) const noexcept> constCastFunction(+    Function<ReturnType(Args...) noexcept>&& that) noexcept {+  return Function<ReturnType(Args...) const noexcept>{+      std::move(that), detail::function::CoerceTag{}};+}++template <typename ReturnType, typename... Args>+Function<ReturnType(Args...) const noexcept> constCastFunction(+    Function<ReturnType(Args...) const noexcept>&& that) noexcept {+  return std::move(that);+}++namespace detail {++template <typename Void, typename>+struct function_ctor_deduce_;++template <typename P>+struct function_ctor_deduce_<+    std::enable_if_t<std::is_function<std::remove_pointer_t<P>>::value>,+    P> {+  using type = std::remove_pointer_t<P>;+};++template <typename F>+struct function_ctor_deduce_<void_t<decltype(&F::operator())>, F> {+  using type =+      typename member_pointer_traits<decltype(&F::operator())>::member_type;+};++template <typename F>+using function_ctor_deduce_t = typename function_ctor_deduce_<void, F>::type;++} // namespace detail++template <typename F>+Function(F) -> Function<detail::function_ctor_deduce_t<F>>;++/**+ * @class folly::FunctionRef+ *+ * A reference wrapper for callable objects+ *+ * FunctionRef is similar to std::reference_wrapper, but the template parameter+ * is the function signature type rather than the type of the referenced object.+ * A folly::FunctionRef is cheap to construct as it contains only a pointer to+ * the referenced callable and a pointer to a function which invokes the+ * callable.+ *+ * The user of FunctionRef must be aware of the reference semantics: storing a+ * copy of a FunctionRef is potentially dangerous and should be avoided unless+ * the referenced object definitely outlives the FunctionRef object. Thus any+ * function that accepts a FunctionRef parameter should only use it to invoke+ * the referenced function and not store a copy of it. Knowing that FunctionRef+ * itself has reference semantics, it is generally okay to use it to reference+ * lambdas that capture by reference.+ */++template <typename FunctionType>+class FunctionRef;++template <typename ReturnType, typename... Args>+class FunctionRef<ReturnType(Args...)> final {+  template <typename Arg>+  using CallArg = detail::function::CallArg<Arg>;++  using Call = ReturnType (*)(CallArg<Args>..., void*);++  static ReturnType uninitCall(CallArg<Args>..., void*) {+    throw_exception<std::bad_function_call>();+  }++  template <+      typename Fun,+      std::enable_if_t<!std::is_pointer<Fun>::value, int> = 0>+  static ReturnType call(CallArg<Args>... args, void* object) {+    using Pointer = std::add_pointer_t<Fun>;+    return static_cast<ReturnType>(invoke(+        static_cast<Fun&&>(*static_cast<Pointer>(object)),+        static_cast<Args&&>(args)...));+  }+  template <+      typename Fun,+      std::enable_if_t<std::is_pointer<Fun>::value, int> = 0>+  static ReturnType call(CallArg<Args>... args, void* object) {+    return static_cast<ReturnType>(+        invoke(reinterpret_cast<Fun>(object), static_cast<Args&&>(args)...));+  }++  void* object_{nullptr};+  Call call_{&FunctionRef::uninitCall};++ public:+  /**+   * Default constructor. Constructs an empty FunctionRef.+   *+   * Invoking it will throw std::bad_function_call.+   */+  constexpr FunctionRef() = default;++  /**+   * Like default constructor. Constructs an empty FunctionRef.+   *+   * Invoking it will throw std::bad_function_call.+   */+  constexpr explicit FunctionRef(std::nullptr_t) noexcept {}++  /**+   * Construct a FunctionRef from a reference to a callable object. If the+   * callable is considered to be an empty callable, the FunctionRef will be+   * empty.+   */+  template <+      typename Fun,+      std::enable_if_t<+          Conjunction<+              Negation<std::is_same<FunctionRef, std::decay_t<Fun>>>,+              is_invocable_r<ReturnType, Fun&&, Args&&...>>::value,+          int> = 0>+  constexpr /* implicit */ FunctionRef(Fun&& fun) noexcept {+    // `Fun` may be a const type, in which case we have to do a const_cast+    // to store the address in a `void*`. This is safe because the `void*`+    // will be cast back to `Fun*` (which is a const pointer whenever `Fun`+    // is a const type) inside `FunctionRef::call`+    auto& ref = fun; // work around forwarding lint advice+    if constexpr ( //+        detail::function::IsNullptrCompatible<std::decay_t<Fun>>) {+      if (detail::function::isEmptyFunction(fun)) {+        return;+      }+    }+    auto ptr = std::addressof(ref);+    object_ = const_cast<void*>(static_cast<void const*>(ptr));+    call_ = &FunctionRef::template call<Fun>;+  }++  /**+   * Constructs a FunctionRef from a pointer to a function. If the+   * pointer is nullptr, the FunctionRef will be empty.+   */+  template <+      typename Fun,+      std::enable_if_t<std::is_function<Fun>::value, int> = 0,+      std::enable_if_t<is_invocable_r_v<ReturnType, Fun&, Args&&...>, int> = 0>+  constexpr /* implicit */ FunctionRef(Fun* fun) noexcept {+    if (fun) {+      object_ = const_cast<void*>(reinterpret_cast<void const*>(fun));+      call_ = &FunctionRef::template call<Fun*>;+    }+  }++  ReturnType operator()(Args... args) const {+    return call_(static_cast<Args&&>(args)..., object_);+  }++  constexpr explicit operator bool() const noexcept { return object_; }++  constexpr friend bool operator==(+      FunctionRef<ReturnType(Args...)> ref, std::nullptr_t) noexcept {+    return ref.object_ == nullptr;+  }+  constexpr friend bool operator!=(+      FunctionRef<ReturnType(Args...)> ref, std::nullptr_t) noexcept {+    return ref.object_ != nullptr;+  }++  constexpr friend bool operator==(+      std::nullptr_t, FunctionRef<ReturnType(Args...)> ref) noexcept {+    return ref.object_ == nullptr;+  }+  constexpr friend bool operator!=(+      std::nullptr_t, FunctionRef<ReturnType(Args...)> ref) noexcept {+    return ref.object_ != nullptr;+  }+};++} // namespace folly
+ folly/folly/GLog.h view
@@ -0,0 +1,86 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <folly/Likely.h>++#include <atomic>+#include <chrono>++#include <glog/logging.h>++#ifndef FB_LOG_EVERY_MS+/**+ * Issues a LOG(severity) no more often than every+ * milliseconds. Example:+ *+ * FB_LOG_EVERY_MS(INFO, 10000) << "At least ten seconds passed"+ *   " since you last saw this.";+ *+ * The implementation uses for statements to introduce variables in+ * a nice way that doesn't mess surrounding statements.  It is thread+ * safe.  Non-positive intervals will always log.+ */+#define FB_LOG_EVERY_MS(severity, milli_interval)                              \+  for (decltype(milli_interval)                                                \+           FB_LEM_once = 1,                                                    \+           FB_LEM_interval = (milli_interval);                                 \+       FB_LEM_once;)                                                           \+    for (::std::chrono::milliseconds::rep FB_LEM_prev,                         \+         FB_LEM_now = FB_LEM_interval <= 0                                     \+             ? 0                                                               \+             : ::std::chrono::duration_cast<::std::chrono::milliseconds>(      \+                   ::std::chrono::system_clock::now().time_since_epoch())      \+                   .count();                                                   \+         FB_LEM_once;)                                                         \+      for (static ::std::atomic<::std::chrono::milliseconds::rep> FB_LEM_hist; \+           FB_LEM_once;                                                        \+           FB_LEM_once = 0)                                                    \+        if (FB_LEM_interval > 0 &&                                             \+            (FB_LEM_now -                                                      \+                     (FB_LEM_prev =                                            \+                          FB_LEM_hist.load(std::memory_order_acquire)) <       \+                 FB_LEM_interval ||                                            \+             !FB_LEM_hist.compare_exchange_strong(FB_LEM_prev, FB_LEM_now))) { \+        } else                                                                 \+          LOG(severity)++#endif++/**+ * Issues a LOG(severity) once.+ *+ *   FB_LOG_ONCE(ERROR) << "Log this error only once";+ *+ * This macro is thread-safe and does not impact surrounding statements.+ *+ * NOTE: A load() is used in the fast-path scenario (steady state) in order to+ * avoid a locked RMW operation.+ */+#ifndef FB_LOG_ONCE+#define FB_LOG_ONCE(severity)                                                  \+  for (auto __folly_detail_glog_once = true; __folly_detail_glog_once;)        \+    for (static ::std::atomic_bool __folly_detail_glog_logged{false};          \+         __folly_detail_glog_once;                                             \+         __folly_detail_glog_once = false)                                     \+      if (FOLLY_LIKELY(                                                        \+              __folly_detail_glog_logged.load(::std::memory_order_relaxed)) || \+          __folly_detail_glog_logged.exchange(                                 \+              true, ::std::memory_order_relaxed)) {                            \+      } else                                                                   \+        LOG(severity)+#endif
+ folly/folly/GroupVarint.cpp view
@@ -0,0 +1,153 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#include <folly/GroupVarint.h>++#include <folly/container/Array.h>++#if FOLLY_HAVE_GROUP_VARINT+namespace folly {++const uint32_t GroupVarint32::kMask[] = {+    0xff,+    0xffff,+    0xffffff,+    0xffffffff,+};++const uint64_t GroupVarint64::kMask[] = {+    0xff,+    0xffff,+    0xffffff,+    0xffffffff,+    0xffffffffffULL,+    0xffffffffffffULL,+    0xffffffffffffffULL,+    0xffffffffffffffffULL,+};++namespace detail {++struct group_varint_table_base_make_item {+  constexpr std::size_t get_d(std::size_t index, std::size_t j) const {+    return 1u + ((index >> (2 * j)) & 3u);+  }+  constexpr std::size_t get_offset(std::size_t index, std::size_t j) const {+    // clang-format off+    return+        (j > 0 ? get_d(index, 0) : 0) ++        (j > 1 ? get_d(index, 1) : 0) ++        (j > 2 ? get_d(index, 2) : 0) ++        (j > 3 ? get_d(index, 3) : 0) ++        0;+    // clang-format on+  }+};++struct group_varint_table_length_make_item : group_varint_table_base_make_item {+  constexpr std::uint8_t operator()(std::size_t index) const {+    return 1u + get_offset(index, 4);+  }+};++//  Reference: http://www.stepanovpapers.com/CIKM_2011.pdf+//+//  From 17 encoded bytes, we may use between 5 and 17 bytes to encode 4+//  integers.  The first byte is a key that indicates how many bytes each of+//  the 4 integers takes:+//+//  bit 0..1: length-1 of first integer+//  bit 2..3: length-1 of second integer+//  bit 4..5: length-1 of third integer+//  bit 6..7: length-1 of fourth integer+//+//  The value of the first byte is used as the index in a table which returns+//  a mask value for the SSSE3 PSHUFB instruction, which takes an XMM register+//  (16 bytes) and shuffles bytes from it into a destination XMM register+//  (optionally setting some of them to 0)+//+//  For example, if the key has value 4, that means that the first integer+//  uses 1 byte, the second uses 2 bytes, the third and fourth use 1 byte each,+//  so we set the mask value so that+//+//  r[0] = a[0]+//  r[1] = 0+//  r[2] = 0+//  r[3] = 0+//+//  r[4] = a[1]+//  r[5] = a[2]+//  r[6] = 0+//  r[7] = 0+//+//  r[8] = a[3]+//  r[9] = 0+//  r[10] = 0+//  r[11] = 0+//+//  r[12] = a[4]+//  r[13] = 0+//  r[14] = 0+//  r[15] = 0++struct group_varint_table_sse_mask_make_item+    : group_varint_table_base_make_item {+  constexpr auto partial_item(+      std::size_t d, std::size_t offset, std::size_t k) const {+    // if k < d, the j'th integer uses d bytes, consume them+    // set remaining bytes in result to 0+    // 0xff: set corresponding byte in result to 0+    return std::uint32_t((k < d ? offset + k : std::size_t(0xff)) << (8 * k));+  }++  constexpr auto item_impl(std::size_t d, std::size_t offset) const {+    // clang-format off+    return+        partial_item(d, offset, 0) |+        partial_item(d, offset, 1) |+        partial_item(d, offset, 2) |+        partial_item(d, offset, 3) |+        0;+    // clang-format on+  }++  constexpr auto item(std::size_t index, std::size_t j) const {+    return item_impl(get_d(index, j), get_offset(index, j));+  }++  constexpr auto operator()(std::size_t index) const {+    return std::array<std::uint32_t, 4>{{+        item(index, 0),+        item(index, 1),+        item(index, 2),+        item(index, 3),+    }};+  }+};++#if FOLLY_SSE >= 4+alignas(16) FOLLY_STORAGE_CONSTEXPR+    decltype(groupVarintSSEMasks) groupVarintSSEMasks =+        make_array_with<256>(group_varint_table_sse_mask_make_item{});+#endif++FOLLY_STORAGE_CONSTEXPR decltype(groupVarintLengths) groupVarintLengths =+    make_array_with<256>(group_varint_table_length_make_item{});++} // namespace detail++} // namespace folly+#endif
+ folly/folly/GroupVarint.h view
@@ -0,0 +1,630 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <cstdint>+#include <limits>++#include <glog/logging.h>++#include <folly/Portability.h>+#include <folly/Range.h>+#include <folly/detail/GroupVarintDetail.h>+#include <folly/lang/Bits.h>+#include <folly/portability/Builtins.h>++#if !defined(__GNUC__) && !defined(_MSC_VER)+#error GroupVarint.h requires GCC or MSVC+#endif++#if FOLLY_X64 || defined(__i386__) || FOLLY_PPC64 || FOLLY_AARCH64 || \+    FOLLY_RISCV64+#define FOLLY_HAVE_GROUP_VARINT 1+#else+#define FOLLY_HAVE_GROUP_VARINT 0+#endif++#if FOLLY_HAVE_GROUP_VARINT++#if FOLLY_SSE >= 4+#include <nmmintrin.h>+namespace folly {+namespace detail {+extern const std::array<std::array<std::uint32_t, 4>, 256> groupVarintSSEMasks;+} // namespace detail+} // namespace folly+#endif++namespace folly {+namespace detail {+extern const std::array<std::uint8_t, 256> groupVarintLengths;+} // namespace detail+} // namespace folly++namespace folly {++template <typename T>+class GroupVarint;++/**+ * GroupVarint encoding for 32-bit values.+ *+ * Encodes 4 32-bit integers at once, each using 1-4 bytes depending on size.+ * There is one byte of overhead.  (The first byte contains the lengths of+ * the four integers encoded as two bits each; 00=1 byte .. 11=4 bytes)+ *+ * This implementation assumes little-endian and does unaligned 32-bit+ * accesses, so it's basically not portable outside of the x86[_64] world.+ */+template <>+class GroupVarint<uint32_t> : public detail::GroupVarintBase<uint32_t> {+ public:+  /**+   * Return the number of bytes used to encode these four values.+   */+  static size_t size(uint32_t a, uint32_t b, uint32_t c, uint32_t d) {+    return (size_t)kHeaderSize + (size_t)kGroupSize + key(a) + key(b) + key(c) ++        key(d);+  }++  /**+   * Return the number of bytes used to encode four uint32_t values stored+   * at consecutive positions in an array.+   */+  static size_t size(const uint32_t* p) { return size(p[0], p[1], p[2], p[3]); }++  /**+   * Return the number of bytes used to encode count (<= 4) values.+   * If you clip a buffer after these many bytes, you can still decode+   * the first "count" values correctly (if the remaining size() -+   * partialSize() bytes are filled with garbage).+   */+  static size_t partialSize(const type* p, size_t count) {+    DCHECK_LE(count, kGroupSize);+    size_t s = kHeaderSize + count;+    for (; count; --count, ++p) {+      s += key(*p);+    }+    return s;+  }++  /**+   * Return the number of values from *p that are valid from an encoded+   * buffer of size bytes.+   */+  static size_t partialCount(const char* p, size_t size) {+    uint8_t v = uint8_t(*p);+    size_t s = kHeaderSize;+    s += 1 + b0key(v);+    if (s > size) {+      return 0;+    }+    s += 1 + b1key(v);+    if (s > size) {+      return 1;+    }+    s += 1 + b2key(v);+    if (s > size) {+      return 2;+    }+    s += 1 + b3key(v);+    if (s > size) {+      return 3;+    }+    return 4;+  }++  /**+   * Given a pointer to the beginning of an GroupVarint32-encoded block,+   * return the number of bytes used by the encoding.+   */+  static size_t encodedSize(const char* p) {+    return (size_t)kHeaderSize + (size_t)kGroupSize + b0key(uint8_t(*p)) ++        b1key(uint8_t(*p)) + b2key(uint8_t(*p)) + b3key(uint8_t(*p));+  }++  /**+   * Encode four uint32_t values into the buffer pointed-to by p, and return+   * the next position in the buffer (that is, one character past the last+   * encoded byte).  p needs to have at least size()+4 bytes available.+   */+  static char* encode(char* p, uint32_t a, uint32_t b, uint32_t c, uint32_t d) {+    uint8_t b0key = key(a);+    uint8_t b1key = key(b);+    uint8_t b2key = key(c);+    uint8_t b3key = key(d);+    *p++ = (b3key << 6) | (b2key << 4) | (b1key << 2) | b0key;+    storeUnaligned(p, a);+    p += b0key + 1;+    storeUnaligned(p, b);+    p += b1key + 1;+    storeUnaligned(p, c);+    p += b2key + 1;+    storeUnaligned(p, d);+    p += b3key + 1;+    return p;+  }++  /**+   * Encode four uint32_t values from the array pointed-to by src into the+   * buffer pointed-to by p, similar to encode(p,a,b,c,d) above.+   */+  static char* encode(char* p, const uint32_t* src) {+    return encode(p, src[0], src[1], src[2], src[3]);+  }++  /**+   * Decode four uint32_t values from a buffer, and return the next position+   * in the buffer (that is, one character past the last encoded byte).+   * The buffer needs to have at least 3 extra bytes available (they+   * may be read but ignored).+   */+  static const char* decode_simple(+      const char* p, uint32_t* a, uint32_t* b, uint32_t* c, uint32_t* d) {+    size_t k = loadUnaligned<uint8_t>(p);+    const char* end = p + detail::groupVarintLengths[k];+    ++p;+    size_t k0 = b0key(k);+    *a = loadUnaligned<uint32_t>(p) & kMask[k0];+    p += k0 + 1;+    size_t k1 = b1key(k);+    *b = loadUnaligned<uint32_t>(p) & kMask[k1];+    p += k1 + 1;+    size_t k2 = b2key(k);+    *c = loadUnaligned<uint32_t>(p) & kMask[k2];+    p += k2 + 1;+    size_t k3 = b3key(k);+    *d = loadUnaligned<uint32_t>(p) & kMask[k3];+    // p += k3+1;+    return end;+  }++  /**+   * Decode four uint32_t values from a buffer and store them in the array+   * pointed-to by dest, similar to decode(p,a,b,c,d) above.+   */+  static const char* decode_simple(const char* p, uint32_t* dest) {+    return decode_simple(p, dest, dest + 1, dest + 2, dest + 3);+  }++#if FOLLY_SSE >= 4+  /**+   * Just like the non-SSSE3 decode below, but with the additional constraint+   * that we must be able to read at least 17 bytes from the input pointer, p.+   */+  static const char* decode(const char* p, uint32_t* dest) {+    uint8_t key = uint8_t(p[0]);+    __m128i val = _mm_loadu_si128((const __m128i*)(p + 1));+    __m128i mask =+        _mm_load_si128((const __m128i*)detail::groupVarintSSEMasks[key].data());+    __m128i r = _mm_shuffle_epi8(val, mask);+    _mm_storeu_si128((__m128i*)dest, r);+    return p + detail::groupVarintLengths[key];+  }++  /**+   * Just like decode_simple, but with the additional constraint that+   * we must be able to read at least 17 bytes from the input pointer, p.+   */+  static const char* decode(+      const char* p, uint32_t* a, uint32_t* b, uint32_t* c, uint32_t* d) {+    uint8_t key = uint8_t(p[0]);+    __m128i val = _mm_loadu_si128((const __m128i*)(p + 1));+    __m128i mask =+        _mm_load_si128((const __m128i*)detail::groupVarintSSEMasks[key].data());+    __m128i r = _mm_shuffle_epi8(val, mask);++    *a = uint32_t(_mm_extract_epi32(r, 0));+    *b = uint32_t(_mm_extract_epi32(r, 1));+    *c = uint32_t(_mm_extract_epi32(r, 2));+    *d = uint32_t(_mm_extract_epi32(r, 3));++    return p + detail::groupVarintLengths[key];+  }++#else // FOLLY_SSE >= 4+  static const char* decode(+      const char* p, uint32_t* a, uint32_t* b, uint32_t* c, uint32_t* d) {+    return decode_simple(p, a, b, c, d);+  }++  static const char* decode(const char* p, uint32_t* dest) {+    return decode_simple(p, dest);+  }+#endif // FOLLY_SSE >= 4++ private:+  static uint8_t key(uint32_t x) {+    // __builtin_clz is undefined for the x==0 case+    return uint8_t(3 - (__builtin_clz(x | 1) / 8));+  }+  static size_t b0key(size_t x) { return x & 3; }+  static size_t b1key(size_t x) { return (x >> 2) & 3; }+  static size_t b2key(size_t x) { return (x >> 4) & 3; }+  static size_t b3key(size_t x) { return (x >> 6) & 3; }++  static const uint32_t kMask[];+};++/**+ * GroupVarint encoding for 64-bit values.+ *+ * Encodes 5 64-bit integers at once, each using 1-8 bytes depending on size.+ * There are two bytes of overhead.  (The first two bytes contain the lengths+ * of the five integers encoded as three bits each; 000=1 byte .. 111 = 8 bytes)+ *+ * This implementation assumes little-endian and does unaligned 64-bit+ * accesses, so it's basically not portable outside of the x86[_64] world.+ */+template <>+class GroupVarint<uint64_t> : public detail::GroupVarintBase<uint64_t> {+ public:+  /**+   * Return the number of bytes used to encode these five values.+   */+  static size_t size(+      uint64_t a, uint64_t b, uint64_t c, uint64_t d, uint64_t e) {+    return (size_t)kHeaderSize + (size_t)kGroupSize + key(a) + key(b) + key(c) ++        key(d) + key(e);+  }++  /**+   * Return the number of bytes used to encode five uint64_t values stored+   * at consecutive positions in an array.+   */+  static size_t size(const uint64_t* p) {+    return size(p[0], p[1], p[2], p[3], p[4]);+  }++  /**+   * Return the number of bytes used to encode count (<= 4) values.+   * If you clip a buffer after these many bytes, you can still decode+   * the first "count" values correctly (if the remaining size() -+   * partialSize() bytes are filled with garbage).+   */+  static size_t partialSize(const type* p, size_t count) {+    DCHECK_LE(count, kGroupSize);+    size_t s = kHeaderSize + count;+    for (; count; --count, ++p) {+      s += key(*p);+    }+    return s;+  }++  /**+   * Return the number of values from *p that are valid from an encoded+   * buffer of size bytes.+   */+  static size_t partialCount(const char* p, size_t size) {+    uint16_t v = loadUnaligned<uint16_t>(p);+    size_t s = kHeaderSize;+    s += 1 + b0key(v);+    if (s > size) {+      return 0;+    }+    s += 1 + b1key(v);+    if (s > size) {+      return 1;+    }+    s += 1 + b2key(v);+    if (s > size) {+      return 2;+    }+    s += 1 + b3key(v);+    if (s > size) {+      return 3;+    }+    s += 1 + b4key(v);+    if (s > size) {+      return 4;+    }+    return 5;+  }++  /**+   * Given a pointer to the beginning of an GroupVarint64-encoded block,+   * return the number of bytes used by the encoding.+   */+  static size_t encodedSize(const char* p) {+    uint16_t n = loadUnaligned<uint16_t>(p);+    return (size_t)kHeaderSize + (size_t)kGroupSize + b0key(n) + b1key(n) ++        b2key(n) + b3key(n) + b4key(n);+  }++  /**+   * Encode five uint64_t values into the buffer pointed-to by p, and return+   * the next position in the buffer (that is, one character past the last+   * encoded byte).  p needs to have at least size()+8 bytes available.+   */+  static char* encode(+      char* p, uint64_t a, uint64_t b, uint64_t c, uint64_t d, uint64_t e) {+    uint16_t b0key = key(a);+    uint16_t b1key = key(b);+    uint16_t b2key = key(c);+    uint16_t b3key = key(d);+    uint16_t b4key = key(e);+    storeUnaligned<uint16_t>(+        p,+        uint16_t(+            (b4key << 12) | (b3key << 9) | (b2key << 6) | (b1key << 3) |+            b0key));+    p += 2;+    storeUnaligned(p, a);+    p += b0key + 1;+    storeUnaligned(p, b);+    p += b1key + 1;+    storeUnaligned(p, c);+    p += b2key + 1;+    storeUnaligned(p, d);+    p += b3key + 1;+    storeUnaligned(p, e);+    p += b4key + 1;+    return p;+  }++  /**+   * Encode five uint64_t values from the array pointed-to by src into the+   * buffer pointed-to by p, similar to encode(p,a,b,c,d,e) above.+   */+  static char* encode(char* p, const uint64_t* src) {+    return encode(p, src[0], src[1], src[2], src[3], src[4]);+  }++  /**+   * Decode five uint64_t values from a buffer, and return the next position+   * in the buffer (that is, one character past the last encoded byte).+   * The buffer needs to have at least 7 bytes available (they may be read+   * but ignored).+   */+  static const char* decode(+      const char* p,+      uint64_t* a,+      uint64_t* b,+      uint64_t* c,+      uint64_t* d,+      uint64_t* e) {+    uint16_t k = loadUnaligned<uint16_t>(p);+    p += 2;+    uint8_t k0 = b0key(k);+    *a = loadUnaligned<uint64_t>(p) & kMask[k0];+    p += k0 + 1;+    uint8_t k1 = b1key(k);+    *b = loadUnaligned<uint64_t>(p) & kMask[k1];+    p += k1 + 1;+    uint8_t k2 = b2key(k);+    *c = loadUnaligned<uint64_t>(p) & kMask[k2];+    p += k2 + 1;+    uint8_t k3 = b3key(k);+    *d = loadUnaligned<uint64_t>(p) & kMask[k3];+    p += k3 + 1;+    uint8_t k4 = b4key(k);+    *e = loadUnaligned<uint64_t>(p) & kMask[k4];+    p += k4 + 1;+    return p;+  }++  /**+   * Decode five uint64_t values from a buffer and store them in the array+   * pointed-to by dest, similar to decode(p,a,b,c,d,e) above.+   */+  static const char* decode(const char* p, uint64_t* dest) {+    return decode(p, dest, dest + 1, dest + 2, dest + 3, dest + 4);+  }++ private:+  enum { kHeaderBytes = 2 };++  static uint8_t key(uint64_t x) {+    // __builtin_clzll is undefined for the x==0 case+    return uint8_t(7 - (__builtin_clzll(x | 1) / 8));+  }++  static uint8_t b0key(uint16_t x) { return x & 7u; }+  static uint8_t b1key(uint16_t x) { return (x >> 3) & 7u; }+  static uint8_t b2key(uint16_t x) { return (x >> 6) & 7u; }+  static uint8_t b3key(uint16_t x) { return (x >> 9) & 7u; }+  static uint8_t b4key(uint16_t x) { return (x >> 12) & 7u; }++  static const uint64_t kMask[];+};++typedef GroupVarint<uint32_t> GroupVarint32;+typedef GroupVarint<uint64_t> GroupVarint64;++/**+ * Simplify use of GroupVarint* for the case where data is available one+ * entry at a time (instead of one group at a time).  Handles buffering+ * and an incomplete last chunk.+ *+ * Output is a function object that accepts character ranges:+ * out(StringPiece) appends the given character range to the output.+ */+template <class T, class Output>+class GroupVarintEncoder {+ public:+  typedef GroupVarint<T> Base;+  typedef T type;++  explicit GroupVarintEncoder(Output out) : out_(out), count_(0) {}++  ~GroupVarintEncoder() { finish(); }++  /**+   * Add a value to the encoder.+   */+  void add(type val) {+    buf_[count_++] = val;+    if (count_ == Base::kGroupSize) {+      char* p = Base::encode(tmp_, buf_);+      out_(StringPiece(tmp_, p));+      count_ = 0;+    }+  }++  /**+   * Finish encoding, flushing any buffered values if necessary.+   * After finish(), the encoder is immediately ready to encode more data+   * to the same output.+   */+  void finish() {+    if (count_) {+      // This is not strictly necessary, but it makes testing easy;+      // uninitialized bytes are guaranteed to be recorded as taking one byte+      // (not more).+      for (size_t i = count_; i < Base::kGroupSize; i++) {+        buf_[i] = 0;+      }+      Base::encode(tmp_, buf_);+      out_(StringPiece(tmp_, Base::partialSize(buf_, count_)));+      count_ = 0;+    }+  }++  /**+   * Return the appender that was used.+   */+  Output& output() { return out_; }+  const Output& output() const { return out_; }++  /**+   * Reset the encoder, disregarding any state (except what was already+   * flushed to the output, of course).+   */+  void clear() { count_ = 0; }++ private:+  Output out_;+  char tmp_[Base::kMaxSize];+  type buf_[Base::kGroupSize];+  size_t count_;+};++/**+ * Simplify use of GroupVarint* for the case where the last group in the+ * input may be incomplete (but the exact size of the input is known).+ * Allows for extracting values one at a time.+ */+template <typename T>+class GroupVarintDecoder {+ public:+  typedef GroupVarint<T> Base;+  typedef T type;++  GroupVarintDecoder() = default;++  explicit GroupVarintDecoder(StringPiece data, size_t maxCount = (size_t)-1)+      : rrest_(data.end()),+        p_(data.data()),+        end_(data.end()),+        limit_(end_),+        pos_(0),+        count_(0),+        remaining_(maxCount) {}++  void reset(StringPiece data, size_t maxCount = (size_t)-1) {+    rrest_ = data.end();+    p_ = data.data();+    end_ = data.end();+    limit_ = end_;+    pos_ = 0;+    count_ = 0;+    remaining_ = maxCount;+  }++  /**+   * Read and return the next value.+   */+  bool next(type* val) {+    if (pos_ == count_) {+      // refill+      size_t rem = size_t(end_ - p_);+      if (rem == 0 || remaining_ == 0) {+        return false;+      }+      // next() attempts to read one full group at a time, and so we must have+      // at least enough bytes readable after its end to handle the case if the+      // last group is full.+      //+      // The best way to ensure this is to ensure that data has at least+      // Base::kMaxSize - 1 bytes readable *after* the end, otherwise we'll copy+      // into a temporary buffer.+      if (limit_ - p_ < Base::kMaxSize) {+        memcpy(tmp_, p_, rem);+        p_ = tmp_;+        end_ = p_ + rem;+        limit_ = tmp_ + sizeof(tmp_);+      }+      pos_ = 0;+      const char* n = Base::decode(p_, buf_);+      if (n <= end_) {+        // Full group could be decoded+        if (remaining_ >= Base::kGroupSize) {+          remaining_ -= Base::kGroupSize;+          count_ = Base::kGroupSize;+          p_ = n;+        } else {+          count_ = remaining_;+          remaining_ = 0;+          p_ += Base::partialSize(buf_, count_);+        }+      } else {+        // Can't decode a full group+        count_ = Base::partialCount(p_, size_t(end_ - p_));+        if (remaining_ >= count_) {+          remaining_ -= count_;+          p_ = end_;+        } else {+          count_ = remaining_;+          remaining_ = 0;+          p_ += Base::partialSize(buf_, count_);+        }+        if (count_ == 0) {+          return false;+        }+      }+    }+    *val = buf_[pos_++];+    return true;+  }++  StringPiece rest() const {+    // This is only valid after next() returned false+    CHECK(pos_ == count_ && (p_ == end_ || remaining_ == 0));+    // p_ may point to the internal buffer (tmp_), but we want+    // to return subpiece of the original data+    size_t size = size_t(end_ - p_);+    return StringPiece(rrest_ - size, rrest_);+  }++ private:+  const char* rrest_;+  const char* p_;+  const char* end_;+  const char* limit_;+  char tmp_[2 * Base::kMaxSize];+  type buf_[Base::kGroupSize];+  size_t pos_;+  size_t count_;+  size_t remaining_;+};++typedef GroupVarintDecoder<uint32_t> GroupVarint32Decoder;+typedef GroupVarintDecoder<uint64_t> GroupVarint64Decoder;++} // namespace folly++#endif // FOLLY_HAVE_GROUP_VARINT
+ folly/folly/Hash.h view
@@ -0,0 +1,20 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++//  shims:+#include <folly/hash/Hash.h>
+ folly/folly/IPAddress.cpp view
@@ -0,0 +1,488 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#include <folly/IPAddress.h>++#include <limits>+#include <ostream>+#include <string>+#include <vector>++#include <fmt/core.h>++#include <folly/String.h>+#include <folly/detail/IPAddressSource.h>+#include <folly/small_vector.h>++using std::ostream;+using std::string;+using std::vector;++namespace folly {++// free functions+size_t hash_value(const IPAddress& addr) {+  return addr.hash();+}+ostream& operator<<(ostream& os, const IPAddress& addr) {+  os << addr.str();+  return os;+}+void toAppend(IPAddress addr, string* result) {+  result->append(addr.str());+}+void toAppend(IPAddress addr, fbstring* result) {+  result->append(addr.str());+}++bool IPAddress::validate(StringPiece ip) noexcept {+  return IPAddressV4::validate(ip) || IPAddressV6::validate(ip);+}++// public static+IPAddressV4 IPAddress::createIPv4(const IPAddress& addr) {+  if (addr.isV4()) {+    return addr.asV4();+  } else {+    return addr.asV6().createIPv4();+  }+}++// public static+IPAddressV6 IPAddress::createIPv6(const IPAddress& addr) {+  if (addr.isV6()) {+    return addr.asV6();+  } else {+    return addr.asV4().createIPv6();+  }+}++namespace {++auto splitIpSlashCidr(StringPiece ipSlashCidr) {+  folly::small_vector<folly::StringPiece, 2> vec;+  folly::split('/', ipSlashCidr, vec);+  return vec;+}++} // namespace++// public static+CIDRNetwork IPAddress::createNetwork(+    StringPiece ipSlashCidr,+    int defaultCidr, /* = -1 */+    bool applyMask /* = true */) {+  auto const ret =+      IPAddress::tryCreateNetwork(ipSlashCidr, defaultCidr, applyMask);++  if (ret.hasValue()) {+    return ret.value();+  }++  if (ret.error() == CIDRNetworkError::INVALID_DEFAULT_CIDR) {+    throw std::range_error("defaultCidr must be <= UINT8_MAX");+  }++  if (ret.error() == CIDRNetworkError::INVALID_IP_SLASH_CIDR) {+    throw IPAddressFormatException(fmt::format(+        "Invalid ipSlashCidr specified. Expected IP/CIDR format, got '{}'",+        ipSlashCidr));+  }++  // Handler the remaining error cases. We re-parse the ip/mask pair+  // to make error messages more meaningful+  auto const vec = splitIpSlashCidr(ipSlashCidr);++  switch (ret.error()) {+    case CIDRNetworkError::INVALID_IP:+      CHECK_GE(vec.size(), 1);+      throw IPAddressFormatException(+          fmt::format("Invalid IP address {}", vec.at(0)));+    case CIDRNetworkError::INVALID_CIDR:+      CHECK_GE(vec.size(), 2);+      throw IPAddressFormatException(+          fmt::format("Mask value '{}' not a valid mask", vec.at(1)));+    case CIDRNetworkError::CIDR_MISMATCH: {+      auto const subnet = IPAddress::tryFromString(vec.at(0)).value();+      auto cidr = static_cast<uint8_t>(+          (defaultCidr > -1) ? defaultCidr : (subnet.isV4() ? 32 : 128));++      throw IPAddressFormatException(fmt::format(+          "CIDR value '{}' is > network bit count '{}'",+          vec.size() == 2 ? vec.at(1) : to<string>(cidr),+          subnet.bitCount()));+    }+    case CIDRNetworkError::INVALID_DEFAULT_CIDR:+    case CIDRNetworkError::INVALID_IP_SLASH_CIDR:+    default:+      // unreachable+      break;+  }++  CHECK(0);+}++// public static+Expected<CIDRNetwork, CIDRNetworkError> IPAddress::tryCreateNetwork(+    StringPiece ipSlashCidr, int defaultCidr, bool applyMask) {+  if (defaultCidr > std::numeric_limits<uint8_t>::max()) {+    return makeUnexpected(CIDRNetworkError::INVALID_DEFAULT_CIDR);+  }++  auto const vec = splitIpSlashCidr(ipSlashCidr);+  auto const elemCount = vec.size();++  if (elemCount == 0 || // weird invalid string+      elemCount > 2) { // invalid string (IP/CIDR/extras)+    return makeUnexpected(CIDRNetworkError::INVALID_IP_SLASH_CIDR);+  }++  auto const subnet = IPAddress::tryFromString(vec.at(0));+  if (subnet.hasError()) {+    return makeUnexpected(CIDRNetworkError::INVALID_IP);+  }++  auto cidr = static_cast<uint8_t>(+      (defaultCidr > -1) ? defaultCidr : (subnet.value().isV4() ? 32 : 128));++  if (elemCount == 2) {+    auto const maybeCidr = tryTo<uint8_t>(vec.at(1));+    if (maybeCidr.hasError()) {+      return makeUnexpected(CIDRNetworkError::INVALID_CIDR);+    }+    cidr = maybeCidr.value();+  }++  if (cidr > subnet.value().bitCount()) {+    return makeUnexpected(CIDRNetworkError::CIDR_MISMATCH);+  }++  return std::make_pair(+      applyMask ? subnet.value().mask(cidr) : subnet.value(), cidr);+}++// public static+std::string IPAddress::networkToString(const CIDRNetwork& network) {+  return fmt::format("{}/{}", network.first.str(), network.second);+}++// public static+IPAddress IPAddress::fromBinary(ByteRange bytes) {+  if (bytes.size() == 4) {+    return IPAddress(IPAddressV4::fromBinary(bytes));+  } else if (bytes.size() == 16) {+    return IPAddress(IPAddressV6::fromBinary(bytes));+  } else {+    string hexval = detail::Bytes::toHex(bytes.data(), bytes.size());+    throw IPAddressFormatException(+        fmt::format("Invalid address with hex value '{}'", hexval));+  }+}++Expected<IPAddress, IPAddressFormatError> IPAddress::tryFromBinary(+    ByteRange bytes) noexcept {+  // Check IPv6 first since it's our main protocol.+  if (bytes.size() == 16) {+    return IPAddressV6::tryFromBinary(bytes);+  } else if (bytes.size() == 4) {+    return IPAddressV4::tryFromBinary(bytes);+  } else {+    return makeUnexpected(IPAddressFormatError::UNSUPPORTED_ADDR_FAMILY);+  }+}++// public static+IPAddress IPAddress::fromLong(uint32_t src) {+  return IPAddress(IPAddressV4::fromLong(src));+}+IPAddress IPAddress::fromLongHBO(uint32_t src) {+  return IPAddress(IPAddressV4::fromLongHBO(src));+}++// default constructor+IPAddress::IPAddress() : addr_(), family_(AF_UNSPEC) {}++// public string constructor+IPAddress::IPAddress(StringPiece str) : addr_(), family_(AF_UNSPEC) {+  auto maybeIp = tryFromString(str);+  if (maybeIp.hasError()) {+    throw IPAddressFormatException(+        to<std::string>("Invalid IP address '", str, "'"));+  }+  *this = maybeIp.value();+}++Expected<IPAddress, IPAddressFormatError> IPAddress::tryFromString(+    StringPiece str) noexcept {+  // need to check for V4 address second, since IPv4-mapped IPv6 addresses may+  // contain a period+  if (str.find(':') != string::npos) {+    return IPAddressV6::tryFromString(str);+  } else if (str.find('.') != string::npos) {+    return IPAddressV4::tryFromString(str);+  } else {+    return makeUnexpected(IPAddressFormatError::UNSUPPORTED_ADDR_FAMILY);+  }+}++// public sockaddr constructor+IPAddress::IPAddress(const sockaddr* addr) : addr_(), family_(AF_UNSPEC) {+  auto ip = tryFromSockAddr(addr);+  if (ip.hasError()) {+    switch (ip.error()) {+      case IPAddressFormatError::UNSUPPORTED_ADDR_FAMILY:+        throw InvalidAddressFamilyException(addr->sa_family);+      case IPAddressFormatError::NULL_SOCKADDR:+        throw IPAddressFormatException("sockaddr == nullptr");+      case IPAddressFormatError::INVALID_IP:+        throw IPAddressFormatException("Invalid IP");+    }+  }+  *this = ip.value();+}++folly::Expected<IPAddress, IPAddressFormatError> IPAddress::tryFromSockAddr(+    const sockaddr* addr) noexcept {+  if (addr == nullptr) {+    return makeUnexpected(IPAddressFormatError::NULL_SOCKADDR);+  }+  switch (addr->sa_family) {+    case AF_INET: {+      auto v4addr = reinterpret_cast<const sockaddr_in*>(addr);+      return IPAddressV4(v4addr->sin_addr);+    }+    case AF_INET6: {+      auto v6addr = reinterpret_cast<const sockaddr_in6*>(addr);+      return IPAddressV6(*v6addr);+    }+    default:+      return makeUnexpected(IPAddressFormatError::UNSUPPORTED_ADDR_FAMILY);+  }+}++// public ipv4 constructor+IPAddress::IPAddress(const IPAddressV4 ipV4Addr) noexcept+    : addr_(ipV4Addr), family_(AF_INET) {}++// public ipv4 constructor+IPAddress::IPAddress(const in_addr ipV4Addr) noexcept+    : addr_(IPAddressV4(ipV4Addr)), family_(AF_INET) {}++// public ipv6 constructor+IPAddress::IPAddress(const IPAddressV6& ipV6Addr) noexcept+    : addr_(ipV6Addr), family_(AF_INET6) {}++// public ipv6 constructor+IPAddress::IPAddress(const in6_addr& ipV6Addr) noexcept+    : addr_(IPAddressV6(ipV6Addr)), family_(AF_INET6) {}++// Assign from V4 address+IPAddress& IPAddress::operator=(const IPAddressV4& ipv4_addr) noexcept {+  addr_ = IPAddressV46(ipv4_addr);+  family_ = AF_INET;+  return *this;+}++// Assign from V6 address+IPAddress& IPAddress::operator=(const IPAddressV6& ipv6_addr) noexcept {+  addr_ = IPAddressV46(ipv6_addr);+  family_ = AF_INET6;+  return *this;+}++// public+bool IPAddress::inSubnet(StringPiece cidrNetwork) const {+  auto subnetInfo = IPAddress::createNetwork(cidrNetwork);+  return inSubnet(subnetInfo.first, subnetInfo.second);+}++// public+bool IPAddress::inSubnet(const IPAddress& subnet, uint8_t cidr) const {+  if (bitCount() == subnet.bitCount()) {+    if (isV4()) {+      return asV4().inSubnet(subnet.asV4(), cidr);+    } else {+      return asV6().inSubnet(subnet.asV6(), cidr);+    }+  }+  // an IPv4 address can never belong in a IPv6 subnet unless the IPv6 is a 6to4+  // address and vice-versa+  if (isV6()) {+    const IPAddressV6& v6addr = asV6();+    const IPAddressV4& v4subnet = subnet.asV4();+    if (v6addr.is6To4()) {+      return v6addr.getIPv4For6To4().inSubnet(v4subnet, cidr);+    }+  } else if (subnet.isV6()) {+    const IPAddressV6& v6subnet = subnet.asV6();+    const IPAddressV4& v4addr = asV4();+    if (v6subnet.is6To4()) {+      return v4addr.inSubnet(v6subnet.getIPv4For6To4(), cidr);+    }+  }+  return false;+}++// public+bool IPAddress::inSubnetWithMask(+    const IPAddress& subnet, ByteRange mask) const {+  auto mkByteArray4 = [&]() -> ByteArray4 {+    ByteArray4 ba{{0}};+    std::memcpy(ba.data(), mask.begin(), std::min<size_t>(mask.size(), 4));+    return ba;+  };++  if (bitCount() == subnet.bitCount()) {+    if (isV4()) {+      return asV4().inSubnetWithMask(subnet.asV4(), mkByteArray4());+    } else {+      ByteArray16 ba{{0}};+      std::memcpy(ba.data(), mask.begin(), std::min<size_t>(mask.size(), 16));+      return asV6().inSubnetWithMask(subnet.asV6(), ba);+    }+  }++  // an IPv4 address can never belong in a IPv6 subnet unless the IPv6 is a 6to4+  // address and vice-versa+  if (isV6()) {+    const IPAddressV6& v6addr = asV6();+    const IPAddressV4& v4subnet = subnet.asV4();+    if (v6addr.is6To4()) {+      return v6addr.getIPv4For6To4().inSubnetWithMask(v4subnet, mkByteArray4());+    }+  } else if (subnet.isV6()) {+    const IPAddressV6& v6subnet = subnet.asV6();+    const IPAddressV4& v4addr = asV4();+    if (v6subnet.is6To4()) {+      return v4addr.inSubnetWithMask(v6subnet.getIPv4For6To4(), mkByteArray4());+    }+  }+  return false;+}++uint8_t IPAddress::getNthMSByte(size_t byteIndex) const {+  const auto highestIndex = byteCount() - 1;+  if (byteIndex > highestIndex) {+    throw std::invalid_argument(fmt::format(+        "Byte index must be <= {} for addresses of type: {}",+        highestIndex,+        detail::familyNameStr(family())));+  }+  if (isV4()) {+    return asV4().bytes()[byteIndex];+  }+  return asV6().bytes()[byteIndex];+}++// public+bool operator==(const IPAddress& addr1, const IPAddress& addr2) {+  if (addr1.empty() || addr2.empty()) {+    return addr1.empty() == addr2.empty();+  }+  if (addr1.family() == addr2.family()) {+    if (addr1.isV6()) {+      return (addr1.asV6() == addr2.asV6());+    } else if (addr1.isV4()) {+      return (addr1.asV4() == addr2.asV4());+    } else {+      CHECK_EQ(addr1.family(), AF_UNSPEC);+      // Two default initialized AF_UNSPEC addresses should be considered equal.+      // AF_UNSPEC is the only other value for which an IPAddress can be+      // created, in the default constructor case.+      return true;+    }+  }+  // addr1 is v4 mapped v6 address, addr2 is v4+  if (addr1.isIPv4Mapped() && addr2.isV4()) {+    if (IPAddress::createIPv4(addr1) == addr2.asV4()) {+      return true;+    }+  }+  // addr2 is v4 mapped v6 address, addr1 is v4+  if (addr2.isIPv4Mapped() && addr1.isV4()) {+    if (IPAddress::createIPv4(addr2) == addr1.asV4()) {+      return true;+    }+  }+  // we only compare IPv4 and IPv6 addresses+  return false;+}++bool operator<(const IPAddress& addr1, const IPAddress& addr2) {+  if (addr1.empty() || addr2.empty()) {+    return addr1.empty() < addr2.empty();+  }+  if (addr1.family() == addr2.family()) {+    if (addr1.isV6()) {+      return (addr1.asV6() < addr2.asV6());+    } else if (addr1.isV4()) {+      return (addr1.asV4() < addr2.asV4());+    } else {+      CHECK_EQ(addr1.family(), AF_UNSPEC);+      // Two default initialized AF_UNSPEC addresses can not be less than each+      // other. AF_UNSPEC is the only other value for which an IPAddress can be+      // created, in the default constructor case.+      return false;+    }+  }+  if (addr1.isV6()) {+    // means addr2 is v4, convert it to a mapped v6 address and compare+    return addr1.asV6() < addr2.asV4().createIPv6();+  }+  if (addr2.isV6()) {+    // means addr2 is v6, convert addr1 to v4 mapped and compare+    return addr1.asV4().createIPv6() < addr2.asV6();+  }+  return false;+}++CIDRNetwork IPAddress::longestCommonPrefix(+    const CIDRNetwork& one, const CIDRNetwork& two) {+  if (one.first.family() != two.first.family()) {+    throw std::invalid_argument(fmt::format(+        "Can't compute longest common prefix between addresses of different"+        "families. Passed: {} and {}",+        detail::familyNameStr(one.first.family()),+        detail::familyNameStr(two.first.family())));+  }+  if (one.first.isV4()) {+    auto prefix = IPAddressV4::longestCommonPrefix(+        {one.first.asV4(), one.second}, {two.first.asV4(), two.second});+    return {IPAddress(prefix.first), prefix.second};+  } else if (one.first.isV6()) {+    auto prefix = IPAddressV6::longestCommonPrefix(+        {one.first.asV6(), one.second}, {two.first.asV6(), two.second});+    return {IPAddress(prefix.first), prefix.second};+  } else {+    throw std::invalid_argument("Unknown address family");+  }+}++// clang-format off+[[noreturn]] void IPAddress::asV4Throw() const {+  auto fam = detail::familyNameStr(family());+  throw InvalidAddressFamilyException(+      fmt::format("Can't convert address with family {} to AF_INET address", fam));+}++[[noreturn]] void IPAddress::asV6Throw() const {+  auto fam = detail::familyNameStr(family());+  throw InvalidAddressFamilyException(+      fmt::format("Can't convert address with family {} to AF_INET6 address", fam));+}+// clang-format on++} // namespace folly
+ folly/folly/IPAddress.h view
@@ -0,0 +1,667 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++/**+ * Provides a unified interface for IP addresses.+ *+ * @refcode folly/docs/examples/folly/ipaddress.cpp+ *+ * @class folly::IPAddress+ * @see IPAddressV6+ * @see IPAddressV4+ */++#pragma once++#include <functional>+#include <iosfwd>+#include <memory>+#include <string>+#include <type_traits>+#include <utility> // std::pair++#include <folly/ConstexprMath.h>+#include <folly/IPAddressException.h>+#include <folly/IPAddressV4.h>+#include <folly/IPAddressV6.h>+#include <folly/Range.h>+#include <folly/detail/IPAddress.h>+#include <folly/lang/Exception.h>++namespace folly {++class IPAddress;++/**+ * Pair of IPAddress, netmask+ */+typedef std::pair<IPAddress, uint8_t> CIDRNetwork;++class IPAddress {+ private:+  template <typename F>+  auto pick(F f) const {+    return isV4() ? f(asV4()) : isV6() ? f(asV6()) : f(asNone());+  }++  class IPAddressNone {+   public:+    bool isZero() const { return true; }+    size_t bitCount() const { return 0; }+    std::string toJson() const {+      return "{family:'AF_UNSPEC', addr:'', hash:0}";+    }+    std::size_t hash() const { return std::hash<uint64_t>{}(0); }+    bool isLoopback() const {+      throw_exception<InvalidAddressFamilyException>("empty address");+    }+    bool isLinkLocal() const {+      throw_exception<InvalidAddressFamilyException>("empty address");+    }+    bool isLinkLocalBroadcast() const {+      throw_exception<InvalidAddressFamilyException>("empty address");+    }+    bool isNonroutable() const {+      throw_exception<InvalidAddressFamilyException>("empty address");+    }+    bool isPrivate() const {+      throw_exception<InvalidAddressFamilyException>("empty address");+    }+    bool isMulticast() const {+      throw_exception<InvalidAddressFamilyException>("empty address");+    }+    IPAddress mask(uint8_t numBits) const {+      (void)numBits;+      return IPAddress();+    }+    std::string str() const { return ""; }+    std::string toFullyQualified() const { return ""; }+    void toFullyQualifiedAppend(std::string& out) const {+      (void)out;+      return;+    }+    uint8_t version() const { return 0; }+    const unsigned char* bytes() const { return nullptr; }+  };++  IPAddressNone const& asNone() const {+    if (!empty()) {+      throw_exception<InvalidAddressFamilyException>("not empty");+    }+    return addr_.ipNoneAddr;+  }++ public:+  /**+   * Returns true if the input string can be parsed as an IP address.+   */+  static bool validate(StringPiece ip) noexcept;+  /**+   * Return the IPAddressV4 representation of the address, converting+   * from V6 to V4 if needed.+   *+   *+   *+   * @throws IPAddressFormatException if the V6 Address is not IPv4 mappes+   */+  static IPAddressV4 createIPv4(const IPAddress& addr);++  /**+   * Return the address as a IPAddressV6, converting from V4 to V6 if+   * needed.+   */+  static IPAddressV6 createIPv6(const IPAddress& addr);++  /**+   * Create a network and mask from a CIDR formatted address string.+   *+   * @param [in] ipSlashCidr IP/CIDR formatted string to split+   * @param [in] defaultCidr default value if no /N specified (if defaultCidr+   *             is -1, will use /32 for IPv4 and /128 for IPv6)+   * @param [in] mask apply mask on the address or not,+   *             e.g. 192.168.13.46/24 => 192.168.13.0/24+   * @return either pair with IPAddress network and uint8_t mask or+   *         CIDRNetworkError+   */+  static Expected<CIDRNetwork, CIDRNetworkError> tryCreateNetwork(+      StringPiece ipSlashCidr, int defaultCidr = -1, bool mask = true);++  /**+   * Create a network and mask from a CIDR formatted address string.+   *+   * Same as tryCreateNetwork() but throws on error.+   *+   * @throws IPAddressFormatException+   * @return pair with IPAddress network and uint8_t mask+   */+  static CIDRNetwork createNetwork(+      StringPiece ipSlashCidr, int defaultCidr = -1, bool mask = true);++  /**+   * Return a string representation of a CIDR block created with+   * createNetwork().+   *+   * @param [in] network pair of address and cidr+   * @return string representing the netblock+   */+  static std::string networkToString(const CIDRNetwork& network);++  /**+   * Create a new IPAddress instance from the provided binary data+   * in network byte order.+   *+   * @throws IPAddressFormatException if the length of `bytes` is not 4 or 16.+   *+   */+  static IPAddress fromBinary(ByteRange bytes);++  /**+   * Non-throwing version of fromBinary().+   * On failure returns IPAddressFormatError.+   */+  static Expected<IPAddress, IPAddressFormatError> tryFromBinary(+      ByteRange bytes) noexcept;++  /**+   * Tries to create a new IPAddress instance from provided string.+   *+   * On failure, returns IPAddressFormatError.+   */+  static Expected<IPAddress, IPAddressFormatError> tryFromString(+      StringPiece str) noexcept;++  /**+   * Tries to create a new IPAddress instance from the provided sockaddr.+   *+   * On failure, returns IPAddressFormatError.+   */+  static Expected<IPAddress, IPAddressFormatError> tryFromSockAddr(+      const sockaddr* addr) noexcept;++  /**+   * Create an IPAddress from a `uint32_t`, using network byte order.+   *+   * @throws IPAddressFormatException if `src` does not represent a valid IP+   * Address.+   */+  static IPAddress fromLong(uint32_t src);++  /**+   * Create an IPAddress from a `uint32_t`, using host byte order.+   *+   * @throws IPAddressFormatException if `src` does not represent a valid IP+   * address.+   */+  static IPAddress fromLongHBO(uint32_t src);++  /**+   * Given 2 (IPAddress, mask) pairs, extract the longest common pair.+   */+  static CIDRNetwork longestCommonPrefix(+      const CIDRNetwork& one, const CIDRNetwork& two);++  /**+   * Constructs an uninitialized IPAddress.+   */+  IPAddress();++  /**+   * Parse an IPAddress from a string representation.+   *+   * Formats accepted are exactly the same as the ones accepted by+   * `inet_pton()`, using `AF_INET6` if the string contains colons, and `AF_INET+   * `otherwise; with the exception that the whole address can optionally be+   * enclosed in square brackets.+   *+   * @throws IPAddressFormatException if `str` is not a valid IP Address.+   */+  explicit IPAddress(StringPiece str);++  /**+   * Create an IPAddress from a `sockaddr` struct.+   *+   * @throws IPAddressFormatException if `addr` is a nullptr, or not `AF_INET`+   * or `AF_INET6`+   */+  explicit IPAddress(const sockaddr* addr);++  /**+   * Create an IPAddress from an IPAddressV4+   */+  /* implicit */ IPAddress(const IPAddressV4 ipV4Addr) noexcept;++  /**+   * Create an IPAddress from an `in_addr` representation of an IPV4 address+   */+  /* implicit */ IPAddress(const in_addr addr) noexcept;++  /**+   * Create an IPAddress from an IPAddressV6+   */+  /* implicit */ IPAddress(const IPAddressV6& ipV6Addr) noexcept;++  /**+   * Create an IPAddress from an `in6_addr` representation of an IPV6 address+   */+  /* implicit */ IPAddress(const in6_addr& addr) noexcept;++  /**+   * @overloadbrief Copy assignment from other IPAddress representations+   *+   * Copy assignment from an IPAddressV4+   */+  IPAddress& operator=(const IPAddressV4& ipv4_addr) noexcept;++  /**+   * Copy assignment from an IPAddressV6+   */+  IPAddress& operator=(const IPAddressV6& ipv6_addr) noexcept;++  /**+   * Converts an IPAddress to an IPAddressV4 instance.+   *+   * @note This is not some handy convenience wrapper to convert an IPv4 address+   *       to a mapped IPv6 address. If you want that use createIPv6()+   *+   * @throws InvalidAddressFamilyException if the IP Address is not currently+   * a valid V4 instance.+   */+  const IPAddressV4& asV4() const {+    if (FOLLY_UNLIKELY(!isV4())) {+      asV4Throw();+    }+    return addr_.ipV4Addr;+  }++  /**+   * Converts an IPAddress to an IPAddressV6 instance+   *+   * @throws InvalidAddressFamilyException if the IP Address is not currently+   * a valid V6 instance.+   */+  const IPAddressV6& asV6() const {+    if (FOLLY_UNLIKELY(!isV6())) {+      asV6Throw();+    }+    return addr_.ipV6Addr;+  }++  /**+   * Return then `sa_family_t` of the IP Address+   */+  sa_family_t family() const { return family_; }++  /**+   * Given a `sockaddr_storage` struct, populate it with an appropriate value.+   *+   * @param [out] dest The struct to populate+   * @param port The port number to put in the struct (defaults to 0)+   *+   */+  int toSockaddrStorage(sockaddr_storage* dest, uint16_t port = 0) const {+    if (dest == nullptr) {+      throw_exception<IPAddressFormatException>("dest must not be null");+    }+    memset(dest, 0, sizeof(sockaddr_storage));+    dest->ss_family = family();++    if (isV4()) {+      sockaddr_in* sin = reinterpret_cast<sockaddr_in*>(dest);+      sin->sin_addr = asV4().toAddr();+      sin->sin_port = port;+#if defined(__APPLE__)+      sin->sin_len = sizeof(*sin);+#endif+      return sizeof(*sin);+    } else if (isV6()) {+      sockaddr_in6* sin = reinterpret_cast<sockaddr_in6*>(dest);+      sin->sin6_addr = asV6().toAddr();+      sin->sin6_port = port;+      sin->sin6_scope_id = asV6().getScopeId();+#if defined(__APPLE__)+      sin->sin6_len = sizeof(*sin);+#endif+      return sizeof(*sin);+    } else {+      throw_exception<InvalidAddressFamilyException>(family());+    }+  }++  /**+   * @overloadbrief Check if the address is found in the specified CIDR+   * netblock.+   *+   * This will return false if the specified cidrNet is V4, but the address is+   * V6. It will also return false if the specified cidrNet is V6 but the+   * address is V4. This method will do the right thing in the case of a v6+   * mapped v4 address.+   *+   * @note This is slower than the below counterparts. If perf is important use+   *       one of the two argument variations below.+   * @param [in] cidrNetwork address in "192.168.1.0/24" format+   * @throws IPAddressFormatException if no /mask in cidrNetwork+   * @return true if address is part of specified subnet with cidr+   */+  bool inSubnet(StringPiece cidrNetwork) const;++  /**+   * Check if an IPAddress belongs to a subnet.+   * @param [in] subnet Subnet to check against (e.g. 192.168.1.0)+   * @param [in] cidr   CIDR for subnet (e.g. 24 for /24)+   * @return true if address is part of specified subnet with cidr+   */+  bool inSubnet(const IPAddress& subnet, uint8_t cidr) const;++  /**+   * Check if an IPAddress belongs to the subnet with the given mask.+   *+   * This is the same as inSubnet but the mask is provided instead of looked up+   * from the cidr.+   * @param [in] subnet Subnet to check against+   * @param [in] mask   The netmask for the subnet+   * @return true if address is part of the specified subnet with mask+   */+  bool inSubnetWithMask(const IPAddress& subnet, ByteRange mask) const;++  /**+   * Returns true if address is a v4 mapped address+   */+  bool isIPv4Mapped() const { return isV6() && asV6().isIPv4Mapped(); }++  /**+   * Returns true if address is uninitialised+   */+  bool empty() const { return family_ == AF_UNSPEC; }++  /**+   * Returns true if address is initalised+   */+  explicit operator bool() const { return !empty(); }++  /**+   * Returns true if this represents an IPv4 address+   */+  bool isV4() const { return family_ == AF_INET; }++  /**+   * Returns true if this represents an IPv6 address+   */+  bool isV6() const { return family_ == AF_INET6; }++  /**+   * Returns true if the address is all zeros+   */+  bool isZero() const {+    return pick([&](auto& _) { return _.isZero(); });+  }++  /**+   * The number of bits in the address representation+   */+  size_t bitCount() const {+    return pick([&](auto& _) { return _.bitCount(); });+  }++  /**+   * The number of bytes in the address representation+   */+  size_t byteCount() const { return bitCount() / 8; }++  /**+   * Get the nth most significant bit of the IP address (0-indexed).+   * @param bitIndex n+   */+  bool getNthMSBit(size_t bitIndex) const {+    return detail::getNthMSBitImpl(*this, bitIndex, family());+  }++  /**+   * Get the nth most significant byte of the IP address (0-indexed).+   * @param byteIndex n+   */+  uint8_t getNthMSByte(size_t byteIndex) const;++  /**+   * Get the nth bit of the IP address (0-indexed).+   * @param bitIndex n+   */+  bool getNthLSBit(size_t bitIndex) const {+    return getNthMSBit(bitCount() - bitIndex - 1);+  }++  /**+   * Get the nth byte of the IP address (0-indexed).+   * @param byteIndex n+   */+  uint8_t getNthLSByte(size_t byteIndex) const {+    return getNthMSByte(byteCount() - byteIndex - 1);+  }++  /**+   * Get a json representation of the IP address.+   *+   * This prints a string representation of the address, for human consumption+   * or logging. The string will take the form of a JSON object that looks like:+   *  `{family:'AF_INET|AF_INET6', addr:'address', hash:long}`.+   */+  std::string toJson() const {+    return pick([&](auto& _) { return _.toJson(); });+  }++  /**+   * Returns a hash of the IP address.+   */+  std::size_t hash() const {+    return pick([&](auto& _) { return _.hash(); });+  }++  /**+   * Return true if the IP address qualifies as localhost.+   */+  bool isLoopback() const {+    return pick([&](auto& _) { return _.isLoopback(); });+  }++  /**+   * Return true if the IP address qualifies as link local+   */+  bool isLinkLocal() const {+    return pick([&](auto& _) { return _.isLinkLocal(); });+  }++  /**+   * Return true if the IP address qualifies as broadcast.+   */+  bool isLinkLocalBroadcast() const {+    return pick([&](auto& _) { return _.isLinkLocalBroadcast(); });+  }++  /**+   * Return true if the IP address is a special purpose address, as defined per+   * RFC 6890 (i.e. 0.0.0.0).+   *+   * For V6, true if the address is not in one of global scope blocks:+   * 2000::/3, ffxe::/16.+   */+  bool isNonroutable() const {+    return pick([&](auto& _) { return _.isNonroutable(); });+  }++  /**+   * Return true if the IP address is private, as per RFC 1918 and RFC 4193.+   *+   * For example, 192.168.xxx.xxx or fc00::/7 addresses.+   */+  bool isPrivate() const {+    return pick([&](auto& _) { return _.isPrivate(); });+  }++  /**+   * Return true if the IP address is a multicast address.+   */+  bool isMulticast() const {+    return pick([&](auto& _) { return _.isMulticast(); });+  }++  /**+   * Creates an IPAddress instance with all but most significant numBits set to+   * 0.+   *+   * @throws IPAddressFormatException if numBits > bitCount()+   *+   * @param [in] numBits number of bits to mask+   * @return IPAddress instance with bits set to 0+   */+  IPAddress mask(uint8_t numBits) const {+    return pick([&](auto& _) { return IPAddress(_.mask(numBits)); });+  }++  /**+   * Provides a string representation of address.+   *+   * @throws IPAddressFormatException on `inet_ntop` error.+   *+   * @note The string representation is calculated on demand.+   */+  std::string str() const {+    return pick([&](auto& _) { return _.str(); });+  }++  /**+   * Return the fully qualified string representation of the address.+   *+   * For V4 addresses this is the same as calling str(). For V6 addresses+   * this is the hex representation with : characters inserted every 4 digits.+   */+  std::string toFullyQualified() const {+    return pick([&](auto& _) { return _.toFullyQualified(); });+  }++  /**+   * Same as toFullyQualified() but append to an output string.+   */+  void toFullyQualifiedAppend(std::string& out) const {+    return pick([&](auto& _) { return _.toFullyQualifiedAppend(out); });+  }++  /**+   * Returns the IP address version. 0 if empty, 4 or 6 if nonempty.+   */+  uint8_t version() const {+    return pick([&](auto& _) { return _.version(); });+  }++  /**+   * Returns a pointer to the to IP address bytes, in network byte order.+   */+  const unsigned char* bytes() const {+    return pick([&](auto& _) { return _.bytes(); });+  }++ private:+  [[noreturn]] void asV4Throw() const;+  [[noreturn]] void asV6Throw() const;++  typedef union IPAddressV46 {+    IPAddressNone ipNoneAddr;+    IPAddressV4 ipV4Addr;+    IPAddressV6 ipV6Addr;+    IPAddressV46() noexcept : ipNoneAddr() {}+    explicit IPAddressV46(const IPAddressV4& addr) noexcept : ipV4Addr(addr) {}+    explicit IPAddressV46(const IPAddressV6& addr) noexcept : ipV6Addr(addr) {}+  } IPAddressV46;+  IPAddressV46 addr_;+  sa_family_t family_;+};++/**+ * `boost::hash` uses hash_value(), so this allows `boost::hash` to work+ * automatically for IPAddress+ */+std::size_t hash_value(const IPAddress& addr);++/**+ * Appends a string representation of the IP address to the stream using str().+ */+std::ostream& operator<<(std::ostream& os, const IPAddress& addr);++/**+ * @overloadbrief Define toAppend() to allow IPAddress to be used with+ * `folly::to<string>`+ */+void toAppend(IPAddress addr, std::string* result);+void toAppend(IPAddress addr, fbstring* result);++/**+ * Return true if two addresses are equal.+ *+ * V4-to-V6-mapped addresses are compared as V4 addresses.+ *+ * @return true if the two addresses are equal.+ */+bool operator==(const IPAddress& addr1, const IPAddress& addr2);++/**+ * Return true if `addr1 < addr2`+ *+ * V4-to-V6-mapped addresses are compared as V4 addresses.+ */+bool operator<(const IPAddress& addr1, const IPAddress& addr2);++/**+ * Return true if two address are not equal+ *+ * V4-to-V6-mapped addresses are compared as V4 addresses.+ */+inline bool operator!=(const IPAddress& addr1, const IPAddress& addr2) {+  return !(addr1 == addr2);+}++/**+ * Return true if `addr1 > addr2`+ *+ * V4-to-V6-mapped addresses are compared as V4 addresses.+ */+inline bool operator>(const IPAddress& addr1, const IPAddress& addr2) {+  return addr2 < addr1;+}++/**+ * Return true if `addr1 <= addr2`+ *+ * V4-to-V6-mapped addresses are compared as V4 addresses.+ */+inline bool operator<=(const IPAddress& addr1, const IPAddress& addr2) {+  return !(addr1 > addr2);+}++/**+ * Return true if `addr1 >= addr2`+ *+ * V4-to-V6-mapped addresses are compared as V4 addresses.+ */+inline bool operator>=(const IPAddress& addr1, const IPAddress& addr2) {+  return !(addr1 < addr2);+}++} // namespace folly++namespace std {+template <>+struct hash<folly::IPAddress> {+  size_t operator()(const folly::IPAddress& addr) const { return addr.hash(); }+};+} // namespace std
+ folly/folly/IPAddressException.h view
@@ -0,0 +1,82 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++/**+ * Error enums and exceptions for indicating errors when dealing with IP+ * Addresses. Used in IPAddress, IPAddressV4, and IPAddressV6.+ *+ * @file IPAddressException.h+ */++#pragma once++#include <exception>+#include <string>+#include <utility>++#include <folly/CPortability.h>+#include <folly/detail/IPAddress.h>+#include <folly/lang/Exception.h>++namespace folly {++/**+ * Error codes for non-throwing interface of IPAddress family of functions.+ */+enum class IPAddressFormatError {+  INVALID_IP,+  UNSUPPORTED_ADDR_FAMILY,+  NULL_SOCKADDR,+};++/**+ * Wraps errors from parsing IP/MASK string+ */+enum class CIDRNetworkError {+  INVALID_DEFAULT_CIDR,+  INVALID_IP_SLASH_CIDR,+  INVALID_IP,+  INVALID_CIDR,+  CIDR_MISMATCH,+};++/**+ * Exception that is thrown when dealing with invalid IP addresses. A subclass+ * of `std::runtime_error`+ */+class FOLLY_EXPORT IPAddressFormatException : public std::runtime_error {+ public:+  using std::runtime_error::runtime_error;+};++/**+ * Exception that is thrown when an IP Address is not of the family expected+ * (ie, expected a V4 but is a V6). A subclass of IPAddressFormatException.+ */+class FOLLY_EXPORT InvalidAddressFamilyException+    : public IPAddressFormatException {+ public:+  explicit InvalidAddressFamilyException(const char* msg)+      : IPAddressFormatException{msg} {}+  explicit InvalidAddressFamilyException(const std::string& msg) noexcept+      : IPAddressFormatException{msg} {}+  explicit InvalidAddressFamilyException(sa_family_t family) noexcept+      : InvalidAddressFamilyException(+            "Address family " + detail::familyNameStr(family) ++            " is not AF_INET or AF_INET6") {}+};++} // namespace folly
+ folly/folly/IPAddressV4.cpp view
@@ -0,0 +1,306 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#include <folly/IPAddressV4.h>++#include <ostream>+#include <string>++#include <fmt/core.h>++#include <folly/Conv.h>+#include <folly/IPAddress.h>+#include <folly/IPAddressV6.h>+#include <folly/String.h>+#include <folly/detail/IPAddressSource.h>++using std::ostream;+using std::string;++namespace folly {++// free functions+size_t hash_value(const IPAddressV4& addr) {+  return addr.hash();+}+ostream& operator<<(ostream& os, const IPAddressV4& addr) {+  os << addr.str();+  return os;+}+void toAppend(IPAddressV4 addr, string* result) {+  result->append(addr.str());+}+void toAppend(IPAddressV4 addr, fbstring* result) {+  result->append(addr.str());+}++bool IPAddressV4::validate(StringPiece ip) noexcept {+  return tryFromString(ip).hasValue();+}++// public static+IPAddressV4 IPAddressV4::fromLong(uint32_t src) {+  in_addr addr;+  addr.s_addr = src;+  return IPAddressV4(addr);+}++IPAddressV4 IPAddressV4::fromLongHBO(uint32_t src) {+  in_addr addr;+  addr.s_addr = htonl(src);+  return IPAddressV4(addr);+}++// static public+uint32_t IPAddressV4::toLong(StringPiece ip) {+  auto str = ip.str();+  in_addr addr;+  if (inet_pton(AF_INET, str.c_str(), &addr) != 1) {+    throw IPAddressFormatException(+        fmt::format("Can't convert invalid IP '{}' to long", ip));+  }+  return addr.s_addr;+}++// static public+uint32_t IPAddressV4::toLongHBO(StringPiece ip) {+  return ntohl(IPAddressV4::toLong(ip));+}++// public default constructor+IPAddressV4::IPAddressV4() = default;++// ByteArray4 constructor+IPAddressV4::IPAddressV4(const ByteArray4& src) noexcept : addr_(src) {}++// public string constructor+IPAddressV4::IPAddressV4(StringPiece addr) : addr_() {+  auto maybeIp = tryFromString(addr);+  if (maybeIp.hasError()) {+    throw IPAddressFormatException(+        to<std::string>("Invalid IPv4 address '", addr, "'"));+  }+  *this = maybeIp.value();+}++Expected<IPAddressV4, IPAddressFormatError> IPAddressV4::tryFromString(+    StringPiece str) noexcept {+  struct in_addr inAddr;+  if (inet_pton(AF_INET, str.str().c_str(), &inAddr) != 1) {+    return makeUnexpected(IPAddressFormatError::INVALID_IP);+  }+  return IPAddressV4(inAddr);+}++// in_addr constructor+IPAddressV4::IPAddressV4(const in_addr src) noexcept : addr_(src) {}++IPAddressV4 IPAddressV4::fromBinary(ByteRange bytes) {+  auto maybeIp = tryFromBinary(bytes);+  if (maybeIp.hasError()) {+    throw IPAddressFormatException(to<std::string>(+        "Invalid IPv4 binary data: length must be 4 bytes, got ",+        bytes.size()));+  }+  return maybeIp.value();+}++Expected<IPAddressV4, IPAddressFormatError> IPAddressV4::tryFromBinary(+    ByteRange bytes) noexcept {+  IPAddressV4 addr;+  auto setResult = addr.trySetFromBinary(bytes);+  if (setResult.hasError()) {+    return makeUnexpected(setResult.error());+  }+  return addr;+}++Expected<Unit, IPAddressFormatError> IPAddressV4::trySetFromBinary(+    ByteRange bytes) noexcept {+  if (bytes.size() != 4) {+    return makeUnexpected(IPAddressFormatError::INVALID_IP);+  }+  memcpy(&addr_.inAddr_.s_addr, bytes.data(), sizeof(in_addr));+  return folly::unit;+}++// static+IPAddressV4 IPAddressV4::fromInverseArpaName(const std::string& arpaname) {+  auto piece = StringPiece(arpaname);+  // input must be something like 1.0.168.192.in-addr.arpa+  if (!piece.removeSuffix(".in-addr.arpa")) {+    throw IPAddressFormatException(+        fmt::format("input does not end with '.in-addr.arpa': '{}'", arpaname));+  }+  std::vector<StringPiece> pieces;+  split(".", piece, pieces);+  if (pieces.size() != 4) {+    throw IPAddressFormatException(fmt::format("Invalid input. Got {}", piece));+  }+  // reverse 1.0.168.192 -> 192.168.0.1+  return IPAddressV4(join(".", pieces.rbegin(), pieces.rend()));+}+IPAddressV6 IPAddressV4::createIPv6() const {+  ByteArray16 ba{};+  ba[10] = 0xff;+  ba[11] = 0xff;+  std::memcpy(&ba[12], bytes(), 4);+  return IPAddressV6(ba);+}++// public+IPAddressV6 IPAddressV4::getIPv6For6To4() const {+  ByteArray16 ba{};+  ba[0] = (uint8_t)((IPAddressV6::PREFIX_6TO4 & 0xFF00) >> 8);+  ba[1] = (uint8_t)(IPAddressV6::PREFIX_6TO4 & 0x00FF);+  std::memcpy(&ba[2], bytes(), 4);+  return IPAddressV6(ba);+}++// public+string IPAddressV4::toJson() const {+  return fmt::format("{{family:'AF_INET', addr:'{}', hash:{}}}", str(), hash());+}++// public+bool IPAddressV4::inSubnet(StringPiece cidrNetwork) const {+  auto subnetInfo = IPAddress::createNetwork(cidrNetwork);+  auto addr = subnetInfo.first;+  if (!addr.isV4()) {+    throw IPAddressFormatException(+        fmt::format("Address '{}' is not a V4 address", addr.toJson()));+  }+  return inSubnetWithMask(addr.asV4(), fetchMask(subnetInfo.second));+}++// public+bool IPAddressV4::inSubnetWithMask(+    const IPAddressV4& subnet, const ByteArray4 cidrMask) const {+  const auto mask = detail::Bytes::mask(toByteArray(), cidrMask);+  const auto subMask = detail::Bytes::mask(subnet.toByteArray(), cidrMask);+  return (mask == subMask);+}++// public+bool IPAddressV4::isLoopback() const {+  static IPAddressV4 loopback_addr("127.0.0.0");+  return inSubnetWithMask(loopback_addr, fetchMask(8));+}++// public+bool IPAddressV4::isLinkLocal() const {+  static IPAddressV4 linklocal_addr("169.254.0.0");+  return inSubnetWithMask(linklocal_addr, fetchMask(16));+}++// public+bool IPAddressV4::isNonroutable() const {+  auto ip = toLongHBO();+  FOLLY_PUSH_WARNING+  FOLLY_CLANG_DISABLE_WARNING("-Wtautological-type-limit-compare")+  return isPrivate() ||+      (/* align */ true && ip <= 0x00FFFFFF) || // 0.0.0.0-0.255.255.255+      (ip >= 0xC0000000 && ip <= 0xC00000FF) || // 192.0.0.0-192.0.0.255+      (ip >= 0xC0000200 && ip <= 0xC00002FF) || // 192.0.2.0-192.0.2.255+      (ip >= 0xC6120000 && ip <= 0xC613FFFF) || // 198.18.0.0-198.19.255.255+      (ip >= 0xC6336400 && ip <= 0xC63364FF) || // 198.51.100.0-198.51.100.255+      (ip >= 0xCB007100 && ip <= 0xCB0071FF) || // 203.0.113.0-203.0.113.255+      (ip >= 0xE0000000 && ip <= 0xFFFFFFFF) || // 224.0.0.0-255.255.255.255+      false;+  FOLLY_POP_WARNING+}++// public+bool IPAddressV4::isPrivate() const {+  auto ip = toLongHBO();+  return // some ranges below+      (ip >= 0x0A000000 && ip <= 0x0AFFFFFF) || // 10.0.0.0-10.255.255.255+      (ip >= 0x7F000000 && ip <= 0x7FFFFFFF) || // 127.0.0.0-127.255.255.255+      (ip >= 0xA9FE0000 && ip <= 0xA9FEFFFF) || // 169.254.0.0-169.254.255.255+      (ip >= 0xAC100000 && ip <= 0xAC1FFFFF) || // 172.16.0.0-172.31.255.255+      (ip >= 0xC0A80000 && ip <= 0xC0A8FFFF) || // 192.168.0.0-192.168.255.255+      (ip >= 0x64400000 && ip <= 0x647fffff) || // 100.64.0.0-100.127.255.255+      false;+}++// public+bool IPAddressV4::isMulticast() const {+  return (toLongHBO() & 0xf0000000) == 0xe0000000;+}++// public+IPAddressV4 IPAddressV4::mask(size_t numBits) const {+  static const auto bits = bitCount();+  if (numBits > bits) {+    throw IPAddressFormatException(+        fmt::format("numBits({}) > bitsCount({})", numBits, bits));+  }++  ByteArray4 ba = detail::Bytes::mask(fetchMask(numBits), addr_.bytes_);+  return IPAddressV4(ba);+}++// public+string IPAddressV4::str() const {+  return detail::fastIpv4ToString(addr_.inAddr_);+}++// public+void IPAddressV4::toFullyQualifiedAppend(std::string& out) const {+  detail::fastIpv4AppendToString(addr_.inAddr_, out);+}++// public+string IPAddressV4::toInverseArpaName() const {+  return fmt::format(+      "{}.{}.{}.{}.in-addr.arpa",+      addr_.bytes_[3],+      addr_.bytes_[2],+      addr_.bytes_[1],+      addr_.bytes_[0]);+}++// public+uint8_t IPAddressV4::getNthMSByte(size_t byteIndex) const {+  const auto highestIndex = byteCount() - 1;+  if (byteIndex > highestIndex) {+    throw std::invalid_argument(fmt::format(+        "Byte index must be <= {} for addresses of type: {}",+        highestIndex,+        detail::familyNameStr(AF_INET)));+  }+  return bytes()[byteIndex];+}+// protected+ByteArray4 IPAddressV4::fetchMask(size_t numBits) {+  static const size_t bits = bitCount();+  if (numBits > bits) {+    throw IPAddressFormatException("IPv4 addresses are 32 bits");+  }+  auto const val = Endian::big(uint32_t(~uint64_t(0) << (32 - numBits)));+  ByteArray4 arr;+  std::memcpy(arr.data(), &val, sizeof(val));+  return arr;+}+// public static+CIDRNetworkV4 IPAddressV4::longestCommonPrefix(+    const CIDRNetworkV4& one, const CIDRNetworkV4& two) {+  auto prefix = detail::Bytes::longestCommonPrefix(+      one.first.addr_.bytes_, one.second, two.first.addr_.bytes_, two.second);+  return {IPAddressV4(prefix.first), prefix.second};+}++} // namespace folly
+ folly/folly/IPAddressV4.h view
@@ -0,0 +1,495 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++/**+ * A representation of an IPv4 address+ *+ * @class folly::IPAddressV4+ * @see IPAddress+ * @see IPAddressV6+ */++#pragma once++#include <cstring>++#include <array>+#include <functional>+#include <iosfwd>++#include <folly/Expected.h>+#include <folly/FBString.h>+#include <folly/IPAddressException.h>+#include <folly/Range.h>+#include <folly/detail/IPAddress.h>+#include <folly/hash/Hash.h>++namespace folly {++class IPAddress;+class IPAddressV4;+class IPAddressV6;++/**+ * Pair of IPAddressV4, netmask+ */+typedef std::pair<IPAddressV4, uint8_t> CIDRNetworkV4;++/**+ * Specialization of `std::array` for IPv4 addresses+ */+typedef std::array<uint8_t, 4> ByteArray4;++class IPAddressV4 {+ public:+  /**+   * Max size of std::string returned by toFullyQualified()+   */+  static constexpr size_t kMaxToFullyQualifiedSize =+      4 /*words*/ * 3 /*max chars per word*/ + 3 /*separators*/;++  /**+   * Returns true if the input string can be parsed as an IP address.+   */+  static bool validate(StringPiece ip) noexcept;++  /**+   * Create an IPAddressV4 instance from a uint32_t, using network byte+   * order+   */+  static IPAddressV4 fromLong(uint32_t src);+  /**+   * Create an IPAddressV4 instance from a uint32_t, using host byte+   * order+   */+  static IPAddressV4 fromLongHBO(uint32_t src);++  /**+   * Create a new IPAddressV4 from the provided ByteRange.+   *+   * @throws IPAddressFormatException if the input length is not 4 bytes.+   */+  static IPAddressV4 fromBinary(ByteRange bytes);++  /**+   * Create a new IPAddressV4 from the provided ByteRange.+   *+   * Returns an IPAddressFormatError if the input length is not 4 bytes.+   */+  static Expected<IPAddressV4, IPAddressFormatError> tryFromBinary(+      ByteRange bytes) noexcept;++  /**+   * Create a new IPAddressV4 from the provided string.+   *+   * Returns an IPAddressFormatError if the string is not a valid IP.+   */+  static Expected<IPAddressV4, IPAddressFormatError> tryFromString(+      StringPiece str) noexcept;++  /**+   * Returns the address as a ByteRange.+   */+  ByteRange toBinary() const {+    return ByteRange((const unsigned char*)&addr_.inAddr_.s_addr, 4);+  }++  /**+   * Create a new IPAddressV4 from a `in-addr.arpa` representation of an IP+   * address.+   *+   * @throws IPAddressFormatException if the input is not a valid in-addr.arpa+   * representation+   */+  static IPAddressV4 fromInverseArpaName(const std::string& arpaname);++  /**+   * Convert a IPv4 address string to a long, in network byte order.+   */++  static uint32_t toLong(StringPiece ip);++  /**+   * Convert a IPv4 address string to a long, in host byte order.+   *+   * This is slightly slower than toLong()+   */+  static uint32_t toLongHBO(StringPiece ip);++  /**+   * Default constructor for IPAddressV4.+   *+   * The address value will be 0.0.0.0+   */+  IPAddressV4();++  /**+   * Construct an IPAddressV4 from a string.+   *+   * @throws IPAddressFormatException if the string is not a valid IPv4+   * address.+   */+  explicit IPAddressV4(StringPiece addr);++  /**+   * Construct an IPAddressV4 from a ByteArray4, in network byte order.+   */+  explicit IPAddressV4(const ByteArray4& src) noexcept;++  /**+   * Construct an IPAddressV4 from an `in_addr` representation of an IPV4+   * address+   */+  explicit IPAddressV4(const in_addr src) noexcept;++  /**+   * Return the IPV6 mapped representation of the address.+   */+  IPAddressV6 createIPv6() const;++  /**+   * Return an IPV6 address in the format of a 6To4 address.+   */+  IPAddressV6 getIPv6For6To4() const;++  /**+   * Return the uint32_t representation of the address, in network byte order.+   */+  uint32_t toLong() const { return toAddr().s_addr; }++  /**+   * Return the uint32_t representation of the address, in host byte order.+   */+  uint32_t toLongHBO() const { return ntohl(toLong()); }++  /**+   * Returns the number of bits in the IP address.+   *+   * @returns 32+   */+  static constexpr size_t bitCount() { return 32; }++  /**+   * Get a json representation of the IP address.+   *+   * This prints a string representation of the address, for human consumption+   * or logging. The string will take the form of a JSON object that looks like:+   *  `{family:'AF_INET', addr:'address', hash:long}`.+   */+  std::string toJson() const;++  /**+   * Returns a hash of the IP address.+   */+  size_t hash() const {+    static const uint32_t seed = AF_INET;+    uint32_t hashed = hash::fnv32_buf(&addr_, 4);+    return hash::hash_combine(seed, hashed);+  }++  /**+   * @overloadbrief Check if the IP address is found in the specified CIDR+   * netblock.+   *+   * @throws IPAddressFormatException if no /mask+   *+   * @note This is slower than the other inSubnet() overload. If perf is+   * important use the other overload, or inSubnetWithMask().+   * @param [in] cidrNetwork address in "192.168.1.0/24" format+   * @return true if address is part of specified subnet with cidr+   */+  bool inSubnet(StringPiece cidrNetwork) const;++  /**+   * Check if an IPAddressV4 belongs to a subnet.+   * @param [in] subnet Subnet to check against (e.g. 192.168.1.0)+   * @param [in] cidr   CIDR for subnet (e.g. 24 for /24)+   * @return true if address is part of specified subnet with cidr+   */+  bool inSubnet(const IPAddressV4& subnet, uint8_t cidr) const {+    return inSubnetWithMask(subnet, fetchMask(cidr));+  }++  /**+   * Check if an IPAddressV4 belongs to the subnet with the given mask.+   *+   * This is the same as inSubnet but the mask is provided instead of looked up+   * from the cidr.+   * @param [in] subnet Subnet to check against+   * @param [in] mask   The netmask for the subnet+   * @return true if address is part of the specified subnet with mask+   */+  bool inSubnetWithMask(const IPAddressV4& subnet, const ByteArray4 mask) const;++  /**+   * Return true if the IP address qualifies as localhost.+   */+  bool isLoopback() const;++  /**+   * Return true if the IP address qualifies as link local+   */+  bool isLinkLocal() const;++  /**+   * Return true if the IP address is a special purpose address, as defined per+   * RFC 6890 (i.e. 0.0.0.0).+   *+   */+  bool isNonroutable() const;+  /**+   * Return true if the IP address is private, as per RFC 1918 and RFC 4193.+   *+   * For example, 192.168.xxx.xxx+   */+  bool isPrivate() const;++  /**+   * Return true if the IP address is a multicast address.+   */+  bool isMulticast() const;++  /**+   * Returns true if the address is all zeros+   */+  bool isZero() const {+    constexpr auto zero = ByteArray4{{}};+    return 0 == std::memcmp(bytes(), zero.data(), zero.size());+  }++  /**+   * Return true if the IP address qualifies as broadcast.+   */+  bool isLinkLocalBroadcast() const {+    return (INADDR_BROADCAST == toLongHBO());+  }++  /**+   * Creates an IPAddressV4 with all but most significant numBits set to+   * 0.+   *+   * @throws IPAddressFormatException if numBits > bitCount()+   *+   * @param [in] numBits number of bits to mask+   * @return IPAddress instance with bits set to 0+   */+  IPAddressV4 mask(size_t numBits) const;++  /**+   * Provides a string representation of address.+   *+   * @throws if IPAddressFormatException on `inet_ntop` error.+   *+   * The string representation is calculated on demand.+   */+  std::string str() const;++  /**+   * Create the inverse arpa representation of the IP address.+   *+   */+  std::string toInverseArpaName() const;++  /**+   * Return the underlying `in_addr` structure+   */+  in_addr toAddr() const { return addr_.inAddr_; }++  /**+   * Return the IP address represented as a `sockaddr_in` struct+   *+   */+  sockaddr_in toSockAddr() const {+    sockaddr_in addr;+    memset(&addr, 0, sizeof(sockaddr_in));+    addr.sin_family = AF_INET;+    memcpy(&addr.sin_addr, &addr_.inAddr_, sizeof(in_addr));+    return addr;+  }++  /**+   * Return a ByteArray4 containing the bytes of the IP address.+   */+  ByteArray4 toByteArray() const {+    ByteArray4 ba{{0}};+    std::memcpy(ba.data(), bytes(), 4);+    return ba;+  }++  /**+   * Return the fully qualified string representation of the address.+   *+   * This is the same as calling str().+   */+  std::string toFullyQualified() const { return str(); }++  /**+   * Same as toFullyQualified() but append to an output string.+   */+  void toFullyQualifiedAppend(std::string& out) const;++  /**+   * Returns the version of the IP Address (4).+   */+  uint8_t version() const { return 4; }++  /**+   * Return the mask associated with the given number of bits.+   *+   * If for instance numBits was 24 (e.g. /24) then the V4 mask returned should+   * be {0xff, 0xff, 0xff, 0x00}.+   *+   * @param [in] numBits bitmask to retrieve+   * @throws abort if numBits == 0 or numBits > bitCount()+   * @return mask associated with numBits+   */+  static ByteArray4 fetchMask(size_t numBits);++  /**+   * Given 2 (IPAddressV4, mask) pairs extract the longest common (IPAddressV4,+   * mask) pair+   */+  static CIDRNetworkV4 longestCommonPrefix(+      const CIDRNetworkV4& one, const CIDRNetworkV4& two);++  /**+   * Return the number of bytes in the IP address.+   *+   * @returns 4+   */+  static size_t byteCount() { return 4; }++  /**+   * Get the nth most significant bit of the IP address (0-indexed).+   * @param bitIndex n+   */+  bool getNthMSBit(size_t bitIndex) const {+    return detail::getNthMSBitImpl(*this, bitIndex, AF_INET);+  }++  /**+   * Get the nth most significant byte of the IP address (0-indexed).+   * @param byteIndex n+   */+  uint8_t getNthMSByte(size_t byteIndex) const;++  /**+   * Get the nth bit of the IP address (0-indexed).+   * @param bitIndex n+   */+  bool getNthLSBit(size_t bitIndex) const {+    return getNthMSBit(bitCount() - bitIndex - 1);+  }++  /**+   * Get the nth byte of the IP address (0-indexed).+   * @param byteIndex n+   */+  uint8_t getNthLSByte(size_t byteIndex) const {+    return getNthMSByte(byteCount() - byteIndex - 1);+  }++  /**+   * Returns a pointer to the to IP address bytes, in network byte order.+   */+  const unsigned char* bytes() const { return addr_.bytes_.data(); }++ private:+  union AddressStorage {+    static_assert(+        sizeof(in_addr) == sizeof(ByteArray4),+        "size of in_addr and ByteArray4 are different");+    in_addr inAddr_;+    ByteArray4 bytes_;+    AddressStorage() { std::memset(this, 0, sizeof(AddressStorage)); }+    explicit AddressStorage(const ByteArray4 bytes) : bytes_(bytes) {}+    explicit AddressStorage(const in_addr addr) : inAddr_(addr) {}+  } addr_;++  /**+   * Set the current IPAddressV4 object to the address specified by the+   * ByteRange given, in network byte order.+   *+   * Returns IPAddressFormatError if bytes.size() is not 4.+   */+  Expected<Unit, IPAddressFormatError> trySetFromBinary(+      ByteRange bytes) noexcept;+};++/**+ * `boost::hash` uses hash_value() so this allows `boost::hash` to work+ * automatically for IPAddressV4+ */+size_t hash_value(const IPAddressV4& addr);++/**+ * Appends a string representation of the IP address to the stream using str().+ */+std::ostream& operator<<(std::ostream& os, const IPAddressV4& addr);++/**+ * @overloadbrief Define toAppend() to allow IPAddress to be used with+ * `folly::to<string>`+ */+void toAppend(IPAddressV4 addr, std::string* result);+void toAppend(IPAddressV4 addr, fbstring* result);++/**+ * Return true if two addresses are equal.+ */+inline bool operator==(const IPAddressV4& addr1, const IPAddressV4& addr2) {+  return (addr1.toLong() == addr2.toLong());+}++/**+ * Return true if addr1 < addr2.+ */+inline bool operator<(const IPAddressV4& addr1, const IPAddressV4& addr2) {+  return (addr1.toLongHBO() < addr2.toLongHBO());+}+/**+ * Return true if addr1 != addr2.+ */+inline bool operator!=(const IPAddressV4& addr1, const IPAddressV4& addr2) {+  return !(addr1 == addr2);+}+/**+ * Return true if addr1 > addr2.+ */+inline bool operator>(const IPAddressV4& addr1, const IPAddressV4& addr2) {+  return addr2 < addr1;+}+/**+ * Return true if addr1 <= addr2.+ */+inline bool operator<=(const IPAddressV4& addr1, const IPAddressV4& addr2) {+  return !(addr1 > addr2);+}+/**+ * Return true if addr1 >= addr2.+ */+inline bool operator>=(const IPAddressV4& addr1, const IPAddressV4& addr2) {+  return !(addr1 < addr2);+}++} // namespace folly++namespace std {+template <>+struct hash<folly::IPAddressV4> {+  size_t operator()(const folly::IPAddressV4 addr) const { return addr.hash(); }+};+} // namespace std
+ folly/folly/IPAddressV6.cpp view
@@ -0,0 +1,541 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#include <folly/IPAddressV6.h>++#include <algorithm>+#include <ostream>+#include <string>++#include <fmt/core.h>++#include <folly/IPAddress.h>+#include <folly/IPAddressV4.h>+#include <folly/MacAddress.h>+#include <folly/ScopeGuard.h>+#include <folly/String.h>+#include <folly/detail/IPAddressSource.h>++#ifdef _WIN32+// Because of the massive pain that is libnl, this can't go into the socket+// portability header as you can't include <linux/if.h> and <net/if.h> in+// the same translation unit without getting errors -_-...+#include <iphlpapi.h> // @manual+#include <ntddndis.h> // @manual++// Alias the max size of an interface name to what posix expects.+#define IFNAMSIZ IF_NAMESIZE+#else+#include <net/if.h>+#endif++using std::ostream;+using std::string;++namespace folly {++// public static const+const uint32_t IPAddressV6::PREFIX_TEREDO = 0x20010000;+const uint32_t IPAddressV6::PREFIX_6TO4 = 0x2002;++// free functions+size_t hash_value(const IPAddressV6& addr) {+  return addr.hash();+}+ostream& operator<<(ostream& os, const IPAddressV6& addr) {+  os << addr.str();+  return os;+}+void toAppend(IPAddressV6 addr, string* result) {+  result->append(addr.str());+}+void toAppend(IPAddressV6 addr, fbstring* result) {+  result->append(addr.str());+}++bool IPAddressV6::validate(StringPiece ip) noexcept {+  return tryFromString(ip).hasValue();+}++// public default constructor+IPAddressV6::IPAddressV6() = default;++// public string constructor+IPAddressV6::IPAddressV6(StringPiece addr) {+  auto maybeIp = tryFromString(addr);+  if (maybeIp.hasError()) {+    throw IPAddressFormatException(+        to<std::string>("Invalid IPv6 address '", addr, "'"));+  }+  *this = maybeIp.value();+}++Expected<IPAddressV6, IPAddressFormatError> IPAddressV6::tryFromString(+    StringPiece str) noexcept {+  constexpr size_t kMaxSize = 45;++  // Allow addresses surrounded in brackets+  if (str.size() < 2) {+    return makeUnexpected(IPAddressFormatError::INVALID_IP);+  }++  auto ip = str.front() == '[' && str.back() == ']'+      ? str.subpiece(1, std::min(str.size() - 2, kMaxSize))+      : str.subpiece(0, std::min(str.size(), kMaxSize));++  std::array<char, kMaxSize + 1> ipBuffer;+  std::copy(ip.begin(), ip.end(), ipBuffer.begin());+  ipBuffer[ip.size()] = '\0';++  struct addrinfo* result;+  struct addrinfo hints;+  memset(&hints, 0, sizeof(hints));+  hints.ai_family = AF_INET6;+  hints.ai_socktype = SOCK_STREAM;+  hints.ai_flags = AI_NUMERICHOST;+  if (::getaddrinfo(ipBuffer.data(), nullptr, &hints, &result) == 0) {+    SCOPE_EXIT {+      ::freeaddrinfo(result);+    };+    const struct sockaddr_in6* sa =+        reinterpret_cast<struct sockaddr_in6*>(result->ai_addr);+    return IPAddressV6(*sa);+  }+  return makeUnexpected(IPAddressFormatError::INVALID_IP);+}++// in6_addr constructor+IPAddressV6::IPAddressV6(const in6_addr& src) noexcept : addr_(src) {}++// sockaddr_in6 constructor+IPAddressV6::IPAddressV6(const sockaddr_in6& src) noexcept+    : addr_(src.sin6_addr), scope_(uint16_t(src.sin6_scope_id)) {}++// ByteArray16 constructor+IPAddressV6::IPAddressV6(const ByteArray16& src) noexcept : addr_(src) {}++// link-local constructor+IPAddressV6::IPAddressV6(LinkLocalTag, MacAddress mac) : addr_(mac) {}++IPAddressV6::AddressStorage::AddressStorage(MacAddress mac) {+  // The link-local address uses modified EUI-64 format,+  // See RFC 4291 sections 2.5.1, 2.5.6, and Appendix A+  const auto* macBytes = mac.bytes();+  memcpy(&bytes_.front(), "\xfe\x80\x00\x00\x00\x00\x00\x00", 8);+  bytes_[8] = uint8_t(macBytes[0] ^ 0x02);+  bytes_[9] = macBytes[1];+  bytes_[10] = macBytes[2];+  bytes_[11] = 0xff;+  bytes_[12] = 0xfe;+  bytes_[13] = macBytes[3];+  bytes_[14] = macBytes[4];+  bytes_[15] = macBytes[5];+}++Optional<MacAddress> IPAddressV6::getMacAddressFromLinkLocal() const {+  // Returned MacAddress must be constructed from a link-local IPv6 address.+  if (!isLinkLocal()) {+    return folly::none;+  }+  return getMacAddressFromEUI64();+}++Optional<MacAddress> IPAddressV6::getMacAddressFromEUI64() const {+  if (!(addr_.bytes_[11] == 0xff && addr_.bytes_[12] == 0xfe)) {+    return folly::none;+  }+  // The auto configured address uses modified EUI-64 format,+  // See RFC 4291 sections 2.5.1, 2.5.6, and Appendix A+  std::array<uint8_t, MacAddress::SIZE> bytes;+  // Step 1: first 8 bytes are network prefix, and can be stripped+  // Step 2: invert the universal/local (U/L) flag (bit 7)+  bytes[0] = addr_.bytes_[8] ^ 0x02;+  // Step 3: copy these bytes as they are+  bytes[1] = addr_.bytes_[9];+  bytes[2] = addr_.bytes_[10];+  // Step 4: strip bytes (0xfffe), which are bytes_[11] and bytes_[12]+  // Step 5: copy the rest.+  bytes[3] = addr_.bytes_[13];+  bytes[4] = addr_.bytes_[14];+  bytes[5] = addr_.bytes_[15];+  return Optional<MacAddress>(MacAddress::fromBinary(range(bytes)));+}++IPAddressV6 IPAddressV6::fromBinary(ByteRange bytes) {+  auto maybeIp = tryFromBinary(bytes);+  if (maybeIp.hasError()) {+    throw IPAddressFormatException(to<std::string>(+        "Invalid IPv6 binary data: length must be 16 bytes, got ",+        bytes.size()));+  }+  return maybeIp.value();+}++Expected<IPAddressV6, IPAddressFormatError> IPAddressV6::tryFromBinary(+    ByteRange bytes) noexcept {+  IPAddressV6 addr;+  auto setResult = addr.trySetFromBinary(bytes);+  if (setResult.hasError()) {+    return makeUnexpected(setResult.error());+  }+  return addr;+}++Expected<Unit, IPAddressFormatError> IPAddressV6::trySetFromBinary(+    ByteRange bytes) noexcept {+  if (bytes.size() != 16) {+    return makeUnexpected(IPAddressFormatError::INVALID_IP);+  }+  memcpy(&addr_.in6Addr_.s6_addr, bytes.data(), sizeof(in6_addr));+  scope_ = 0;+  return unit;+}++// static+IPAddressV6 IPAddressV6::fromInverseArpaName(const std::string& arpaname) {+  auto piece = StringPiece(arpaname);+  if (!piece.removeSuffix(".ip6.arpa")) {+    throw IPAddressFormatException(fmt::format(+        "Invalid input. Should end with 'ip6.arpa'. Got '{}'", arpaname));+  }+  std::vector<StringPiece> pieces;+  split(".", piece, pieces);+  if (pieces.size() != 32) {+    throw IPAddressFormatException(+        fmt::format("Invalid input. Got '{}'", piece));+  }+  std::array<char, IPAddressV6::kToFullyQualifiedSize> ip;+  size_t pos = 0;+  int count = 0;+  for (size_t i = 1; i <= pieces.size(); i++) {+    ip[pos] = pieces[pieces.size() - i][0];+    pos++;+    count++;+    // add ':' every 4 chars+    if (count == 4 && pos < ip.size()) {+      ip[pos++] = ':';+      count = 0;+    }+  }+  return IPAddressV6(folly::range(ip));+}++// public+IPAddressV4 IPAddressV6::createIPv4() const {+  if (!isIPv4Mapped()) {+    throw IPAddressFormatException("addr is not v4-to-v6-mapped");+  }+  const unsigned char* by = bytes();+  return IPAddressV4(detail::Bytes::mkAddress4(&by[12]));+}++// convert two uint8_t bytes into a uint16_t as hibyte.lobyte+static inline uint16_t unpack(uint8_t lobyte, uint8_t hibyte) {+  return uint16_t((uint16_t(hibyte) << 8) | lobyte);+}++// given a src string, unpack count*2 bytes into dest+// dest must have as much storage as count+static inline void unpackInto(+    const unsigned char* src, uint16_t* dest, size_t count) {+  for (size_t i = 0, hi = 1, lo = 0; i < count; i++) {+    dest[i] = unpack(src[hi], src[lo]);+    hi += 2;+    lo += 2;+  }+}++// public+IPAddressV4 IPAddressV6::getIPv4For6To4() const {+  if (!is6To4()) {+    throw IPAddressV6::TypeError(+        fmt::format("Invalid IP '{}': not a 6to4 address", str()));+  }+  // convert 16x8 bytes into first 4x16 bytes+  uint16_t ints[4] = {0, 0, 0, 0};+  unpackInto(bytes(), ints, 4);+  // repack into 4x8+  union {+    unsigned char bytes[4];+    in_addr addr;+  } ipv4;+  ipv4.bytes[0] = (uint8_t)((ints[1] & 0xFF00) >> 8);+  ipv4.bytes[1] = (uint8_t)(ints[1] & 0x00FF);+  ipv4.bytes[2] = (uint8_t)((ints[2] & 0xFF00) >> 8);+  ipv4.bytes[3] = (uint8_t)(ints[2] & 0x00FF);+  return IPAddressV4(ipv4.addr);+}++// public+bool IPAddressV6::isIPv4Mapped() const {+  // v4 mapped addresses have their first 10 bytes set to 0, the next 2 bytes+  // set to 255 (0xff);+  const unsigned char* by = bytes();++  // check if first 10 bytes are 0+  for (int i = 0; i < 10; i++) {+    if (by[i] != 0x00) {+      return false;+    }+  }+  // check if bytes 11 and 12 are 255+  return by[10] == 0xff && by[11] == 0xff;+}++// public+IPAddressV6::Type IPAddressV6::type() const {+  // convert 16x8 bytes into first 2x16 bytes+  uint16_t ints[2] = {0, 0};+  unpackInto(bytes(), ints, 2);++  if ((((uint32_t)ints[0] << 16) | ints[1]) == IPAddressV6::PREFIX_TEREDO) {+    return Type::TEREDO;+  }++  if ((uint32_t)ints[0] == IPAddressV6::PREFIX_6TO4) {+    return Type::T6TO4;+  }++  return Type::NORMAL;+}++// public+string IPAddressV6::toJson() const {+  return fmt::format(+      "{{family:'AF_INET6', addr:'{}', hash:{}}}", str(), hash());+}++// public+size_t IPAddressV6::hash() const {+  if (isIPv4Mapped()) {+    /* An IPAddress containing this object would be equal (i.e. operator==)+       to an IPAddress containing the corresponding IPv4.+       So we must make sure that the hash values are the same as well */+    return IPAddress::createIPv4(*this).hash();+  }++  static const uint64_t seed = AF_INET6;+  uint64_t hash1 = 0, hash2 = 0;+  hash::SpookyHashV2::Hash128(&addr_, 16, &hash1, &hash2);+  return hash::hash_combine(seed, hash1, hash2);+}++// public+bool IPAddressV6::inSubnet(StringPiece cidrNetwork) const {+  auto subnetInfo = IPAddress::createNetwork(cidrNetwork);+  auto addr = subnetInfo.first;+  if (!addr.isV6()) {+    throw IPAddressFormatException(+        fmt::format("Address '{}' is not a V6 address", addr.toJson()));+  }+  return inSubnetWithMask(addr.asV6(), fetchMask(subnetInfo.second));+}++// public+bool IPAddressV6::inSubnetWithMask(+    const IPAddressV6& subnet, const ByteArray16& cidrMask) const {+  const auto mask = detail::Bytes::mask(toByteArray(), cidrMask);+  const auto subMask = detail::Bytes::mask(subnet.toByteArray(), cidrMask);+  return (mask == subMask);+}++// public+bool IPAddressV6::isLoopback() const {+  // Check if v4 mapped is loopback+  if (isIPv4Mapped() && createIPv4().isLoopback()) {+    return true;+  }+  auto socka = toSockAddr();+  return IN6_IS_ADDR_LOOPBACK(&socka.sin6_addr);+}++bool IPAddressV6::isRoutable() const {+  return+      // 2000::/3 is the only assigned global unicast block+      inBinarySubnet({{0x20, 0x00}}, 3) ||+      // ffxe::/16 are global scope multicast addresses,+      // which are eligible to be routed over the internet+      (isMulticast() && getMulticastScope() == 0xe);+}++bool IPAddressV6::isLinkLocalBroadcast() const {+  static const IPAddressV6 kLinkLocalBroadcast("ff02::1");+  return *this == kLinkLocalBroadcast;+}++// public+bool IPAddressV6::isPrivate() const {+  // Check if mapped is private+  if (isIPv4Mapped() && createIPv4().isPrivate()) {+    return true;+  }+  return isLoopback() || inBinarySubnet({{0xfc, 0x00}}, 7) || isLinkLocal();+}++// public+bool IPAddressV6::isLinkLocal() const {+  return inBinarySubnet({{0xfe, 0x80}}, 10);+}++bool IPAddressV6::isMulticast() const {+  return addr_.bytes_[0] == 0xff;+}++uint8_t IPAddressV6::getMulticastFlags() const {+  DCHECK(isMulticast());+  return uint8_t((addr_.bytes_[1] >> 4) & 0xf);+}++uint8_t IPAddressV6::getMulticastScope() const {+  DCHECK(isMulticast());+  return uint8_t(addr_.bytes_[1] & 0xf);+}++IPAddressV6 IPAddressV6::getSolicitedNodeAddress() const {+  // Solicited node addresses must be constructed from unicast (or anycast)+  // addresses+  DCHECK(!isMulticast());++  uint8_t bytes[16] = {+      0xff,+      0x02,+      0x00,+      0x00,+      0x00,+      0x00,+      0x00,+      0x00,+      0x00,+      0x00,+      0x00,+      0x01,+      0xff,+      addr_.bytes_[13],+      addr_.bytes_[14],+      addr_.bytes_[15],+  };+  return IPAddressV6::fromBinary(ByteRange(bytes, 16));+}++// public+IPAddressV6 IPAddressV6::mask(size_t numBits) const {+  static const auto bits = bitCount();+  if (numBits > bits) {+    throw IPAddressFormatException(+        fmt::format("numBits({}) > bitCount({})", numBits, bits));+  }+  ByteArray16 ba = detail::Bytes::mask(fetchMask(numBits), addr_.bytes_);+  return IPAddressV6(ba);+}++// public+string IPAddressV6::str() const {+  char buffer[INET6_ADDRSTRLEN + IFNAMSIZ + 1];++  if (!inet_ntop(AF_INET6, toAddr().s6_addr, buffer, INET6_ADDRSTRLEN)) {+    throw IPAddressFormatException(fmt::format(+        "Invalid address with hex '{}' with error {}",+        detail::Bytes::toHex(bytes(), 16),+        errnoStr(errno)));+  }++  auto scopeId = getScopeId();+  if (scopeId != 0) {+    auto len = strlen(buffer);+    buffer[len] = '%';++    auto errsv = errno;+    if (!if_indextoname(scopeId, buffer + len + 1)) {+      // if we can't map the if because eg. it no longer exists,+      // append the if index instead+      snprintf(buffer + len + 1, IFNAMSIZ, "%u", scopeId);+    }+    errno = errsv;+  }++  return string(buffer);+}++// public+string IPAddressV6::toFullyQualified() const {+  return detail::fastIpv6ToString(addr_.in6Addr_);+}++// public+void IPAddressV6::toFullyQualifiedAppend(std::string& out) const {+  detail::fastIpv6AppendToString(addr_.in6Addr_, out);+}++// public+string IPAddressV6::toInverseArpaName() const {+  constexpr folly::StringPiece lut = "0123456789abcdef";+  std::array<char, 32> a;+  int j = 0;+  for (int i = 15; i >= 0; i--) {+    a[j] = (lut[bytes()[i] & 0xf]);+    a[j + 1] = (lut[bytes()[i] >> 4]);+    j += 2;+  }+  return fmt::format("{}.ip6.arpa", join(".", a));+}++// public+uint8_t IPAddressV6::getNthMSByte(size_t byteIndex) const {+  const auto highestIndex = byteCount() - 1;+  if (byteIndex > highestIndex) {+    throw std::invalid_argument(fmt::format(+        "Byte index must be <= {} for addresses of type: {}",+        highestIndex,+        detail::familyNameStr(AF_INET6)));+  }+  return bytes()[byteIndex];+}++// protected+ByteArray16 IPAddressV6::fetchMask(size_t numBits) {+  static const size_t bits = bitCount();+  if (numBits > bits) {+    throw IPAddressFormatException("IPv6 addresses are 128 bits.");+  }+  if (numBits == 0) {+    return {{0}};+  }+  constexpr auto _0s = uint64_t(0);+  constexpr auto _1s = ~_0s;+  auto const fragment = Endian::big(_1s << ((128 - numBits) % 64));+  auto const hi = numBits <= 64 ? fragment : _1s;+  auto const lo = numBits <= 64 ? _0s : fragment;+  uint64_t const parts[] = {hi, lo};+  ByteArray16 arr;+  std::memcpy(arr.data(), parts, sizeof(parts));+  return arr;+}++// public static+CIDRNetworkV6 IPAddressV6::longestCommonPrefix(+    const CIDRNetworkV6& one, const CIDRNetworkV6& two) {+  auto prefix = detail::Bytes::longestCommonPrefix(+      one.first.addr_.bytes_, one.second, two.first.addr_.bytes_, two.second);+  return {IPAddressV6(prefix.first), prefix.second};+}++// protected+bool IPAddressV6::inBinarySubnet(+    const std::array<uint8_t, 2> addr, size_t numBits) const {+  auto masked = mask(numBits);+  return (std::memcmp(addr.data(), masked.bytes(), 2) == 0);+}+} // namespace folly
+ folly/folly/IPAddressV6.h view
@@ -0,0 +1,627 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++/**+ * A representation of an IPv6 address+ *+ * @see IPAddress+ * @see IPAddressV4+ *+ * @class folly::IPAddressV6+ */++#pragma once++#include <cstring>++#include <array>+#include <functional>+#include <iosfwd>+#include <map>+#include <stdexcept>++#include <folly/Expected.h>+#include <folly/FBString.h>+#include <folly/IPAddressException.h>+#include <folly/Optional.h>+#include <folly/Range.h>+#include <folly/detail/IPAddress.h>+#include <folly/hash/Hash.h>++namespace folly {++class IPAddress;+class IPAddressV4;+class IPAddressV6;+class MacAddress;++/**+ * Pair of IPAddressV6, netmask+ */+typedef std::pair<IPAddressV6, uint8_t> CIDRNetworkV6;++/**+ * Specialization for `std::array` for IPv6 addresses+ */+typedef std::array<uint8_t, 16> ByteArray16;++class IPAddressV6 {+ public:+  /**+   * Represents the different types that IPv6 Addresses can be+   *+   */+  enum Type {+    TEREDO,+    T6TO4,+    NORMAL,+  };++  /**+   * A constructor parameter to indicate that we should create a link-local+   * IPAddressV6.+   */+  enum LinkLocalTag {+    LINK_LOCAL,+  };++  /**+   * Alias std::runtime_error, to be thrown when a type assertion fails+   */+  typedef std::runtime_error TypeError;++  /**+   * The binary prefix for Teredo networks+   */+  static const uint32_t PREFIX_TEREDO;++  /**+   * The binary prefix for Teredo networks+   */+  static const uint32_t PREFIX_6TO4;++  /**+   * The size of the std::string returned by toFullyQualified.+   */+  static constexpr size_t kToFullyQualifiedSize =+      8 /*words*/ * 4 /*hex chars per word*/ + 7 /*separators*/;++  /**+   * Return true if the input string can be parsed as an IPv6 addres+   */+  static bool validate(StringPiece ip) noexcept;++  /**+   * Create a new IPAddressV6 instance from the provided binary data, in network+   * byte order.+   *+   * @throws IPAddressFormatException if the input length is not 16 bytes.+   */+  static IPAddressV6 fromBinary(ByteRange bytes);++  /**+   * Create a new IPAddressV6 from the provided ByteRange.+   *+   * Returns an IPAddressFormatError if the input length is not 4 bytes.+   */+  static Expected<IPAddressV6, IPAddressFormatError> tryFromBinary(+      ByteRange bytes) noexcept;++  /**+   * Create a new IPAddressV6 from the provided string.+   *+   * Returns an IPAddressFormatError if the string is not a valid IP.+   */+  static Expected<IPAddressV6, IPAddressFormatError> tryFromString(+      StringPiece str) noexcept;++  /**+   * Create a new IPAddress instance from the ip6.arpa representation.+   * @throws IPAddressFormatException if the input is not a valid ip6.arpa+   * representation+   */+  static IPAddressV6 fromInverseArpaName(const std::string& arpaname);++  /**+   * Returns the address as a ByteRange.+   */+  ByteRange toBinary() const {+    return ByteRange((const unsigned char*)&addr_.in6Addr_.s6_addr, 16);+  }++  /**+   * Default constructor for IPAddressV6.+   *+   * The address value will be ::0+   */+  IPAddressV6();++  /**+   * Construct an IPAddressV6 from a string+   *+   * @throws IPAddressFormatException if the string is not a valid IPv6 address.+   */+  explicit IPAddressV6(StringPiece addr);++  /**+   * Construct an IPAddressV6 from a ByteArray16+   */+  explicit IPAddressV6(const ByteArray16& src) noexcept;++  /**+   * Construct an IPAddressV6 from an `in_addr` representation of an IPV6+   * address+   */+  explicit IPAddressV6(const in6_addr& src) noexcept;++  /**+   * Construct an IPAddressV6 from an `sockaddr_in6` representation of an IPV6+   * address+   */+  explicit IPAddressV6(const sockaddr_in6& src) noexcept;++  /**+   * Create a link-local IPAddressV6 from the specified ethernet MAC address.+   */+  IPAddressV6(LinkLocalTag tag, MacAddress mac);++  /**+   * Return the mapped IPAddressV4+   *+   * @throws IPAddressFormatException if the address is not IPv4 mapped+   */+  IPAddressV4 createIPv4() const;++  /**+   * Return a V4 address if this is a 6To4 address.+   * @throws TypeError if not a 6To4 address+   */+  IPAddressV4 getIPv4For6To4() const;++  /**+   * Return true if the address is a 6to4 address+   */+  bool is6To4() const { return type() == IPAddressV6::Type::T6TO4; }++  /**+   * Return true if the address is a Teredo address+   */+  bool isTeredo() const { return type() == IPAddressV6::Type::TEREDO; }++  /**+   * Return true if the adddress is IPv4 mapped+   */+  bool isIPv4Mapped() const;++  /**+   * Return what type of IPv6 address this is.+   *+   * @see Type+   */+  Type type() const;++  /**+   * Return the number of bits in the IP address representation+   *+   * @returns 128+   */+  static constexpr size_t bitCount() { return 128; }++  /**+   * Get a json representation of the IP address.+   *+   * This prints a string representation of the address, for human consumption+   * or logging. The string will take the form of a JSON object that looks like:+   *  `{family:'AF_INET6', addr:'address', hash:long}`.+   */+  std::string toJson() const;++  /**+   * Returns a hash of the IP address.+   */+  size_t hash() const;++  /**+   * @overloadbrief Check if the address is found in the specified CIDR+   * netblock.+   *+   * This will return false if the specified cidrNet is V4, but the address is+   * V6. It will also return false if the specified cidrNet is V6 but the+   * address is V4. This method will do the right thing in the case of a v6+   * mapped v4 address.+   *+   * @note This is slower than the below counterparts. If perf is important use+   *       one of the two argument variations below.+   * @param [in] cidrNetwork address in "192.168.1.0/24" format+   * @throws IPAddressFormatException if no /mask in cidrNetwork+   * @return true if address is part of specified subnet with cidr+   */+  bool inSubnet(StringPiece cidrNetwork) const;++  /**+   * Check if an IPAddress belongs to a subnet.+   *+   * @param [in] subnet Subnet to check against (e.g. 192.168.1.0)+   * @param [in] cidr   CIDR for subnet (e.g. 24 for /24)+   * @return true if address is part of specified subnet with cidr+   */+  bool inSubnet(const IPAddressV6& subnet, uint8_t cidr) const {+    return inSubnetWithMask(subnet, fetchMask(cidr));+  }++  /**+   * Check if an IPAddress belongs to the subnet with the given mask.+   *+   * This is the same as inSubnet but the mask is provided instead of looked up+   * from the cidr.+   * @param [in] subnet Subnet to check against+   * @param [in] mask   The netmask for the subnet+   * @return true if address is part of the specified subnet with mask+   */+  bool inSubnetWithMask(+      const IPAddressV6& subnet, const ByteArray16& mask) const;++  /**+   * Return true if the IP address qualifies as localhost.+   */+  bool isLoopback() const;++  /**+   * Return true if the IP address is a special purpose address, as defined per+   * RFC 6890.+   *+   */+  bool isNonroutable() const { return !isRoutable(); }++  /**+   * Return true if this address is routable.+   */+  bool isRoutable() const;++  /**+   * Return true if the IP address is private, as per RFC 1918 and RFC 4193.+   *+   * For example, 192.168.xxx.xxx or fc00::/7 addresses.+   */+  bool isPrivate() const;++  /**+   * Return true if this is a link-local IPv6 address.+   *+   * Note that this only returns true for addresses in the fe80::/10 range.+   * It returns false for the loopback address (::1), even though this address+   * is also effectively has link-local scope.  It also returns false for+   * link-scope and interface-scope multicast addresses.+   */+  bool isLinkLocal() const;++  /**+   * Return the mac address if this is a link-local IPv6 address.+   *+   * @return a `folly::Optional<MacAddress>` union representing the mac address.+   *+   * If the address is not a link-local one it will return an empty Optional.+   * You can use Optional::value() to check whether the mac address is not null.+   */+  Optional<MacAddress> getMacAddressFromLinkLocal() const;++  /**+   * Return the mac address if this is an auto-configured IPv6 address based on+   * EUI-64+   *+   * If the address is not based on EUI-64 it will return an empty+   * Optional. You can use Optional::value() to check whether the mac address is+   * not null.+   *+   * @return a `folly::Optional<MacAddress>` union representing the mac address.+   *+   */+  Optional<MacAddress> getMacAddressFromEUI64() const;++  /**+   * Return true if this is a multicast address.+   */+  bool isMulticast() const;++  /**+   * Return the flags for a multicast address.+   *+   * This method may only be called on multicast addresses.+   */+  uint8_t getMulticastFlags() const;++  /**+   * Return the scope for a multicast address.+   *+   * This method may only be called on multicast addresses.+   */+  uint8_t getMulticastScope() const;++  /**+   * Return true if the address is 0+   */+  bool isZero() const {+    constexpr auto zero = ByteArray16{{}};+    return 0 == std::memcmp(bytes(), zero.data(), zero.size());+  }++  /**+   * Return true if the IP address qualifies as broadcast.+   */+  bool isLinkLocalBroadcast() const;++  /**+   * Creates an IPAddressV6 instance with all but most significant numBits set+   * to 0.+   *+   * @throws IPAddressFormatException if `numBits > bitCount()`+   *+   * @param [in] numBits number of bits to mask+   * @return IPAddress instance with bits set to 0+   */+  IPAddressV6 mask(size_t numBits) const;++  /**+   * Return the underlying `in6_addr` structure+   */+  in6_addr toAddr() const { return addr_.in6Addr_; }++  /**+   * Return the link-local scope id.+   *+   * This should always be 0 for IP addresses that are *not* link-local.+   *+   */+  uint16_t getScopeId() const { return scope_; }+  /**+   * Set the link-local scope id.+   *+   * This should always be 0 for IP addresses that are *not* link-local.+   *+   */+  void setScopeId(uint16_t scope) { scope_ = scope; }++  /**+   * Return the IP address represented as a `sockaddr_in6` struct+   *+   */+  sockaddr_in6 toSockAddr() const {+    sockaddr_in6 addr;+    memset(&addr, 0, sizeof(sockaddr_in6));+    addr.sin6_family = AF_INET6;+    addr.sin6_scope_id = scope_;+    memcpy(&addr.sin6_addr, &addr_.in6Addr_, sizeof(in6_addr));+    return addr;+  }++  /**+   * Return a ByteArray16 containing the bytes of the IP address.+   */+  ByteArray16 toByteArray() const {+    ByteArray16 ba{{0}};+    std::memcpy(ba.data(), bytes(), 16);+    return ba;+  }++  /**+   * Return the fully qualified string representation of the address.+   *+   * This is the hex representation with : characters inserted every 4 digits.+   */+  std::string toFullyQualified() const;++  /**+   * Same as toFullyQualified() but append to an output string.+   */+  void toFullyQualifiedAppend(std::string& out) const;++  /**+   * Create the inverse arpa representation of the IP address.+   *+   */+  std::string toInverseArpaName() const;++  /**+   * Provides a string representation of address.+   *+   * Throws an IPAddressFormatException on `inet_ntop` error.+   *+   * The string representation is calculated on demand.+   */+  std::string str() const;++  /**+   * Returns the version of the IP Address.+   *+   * @returns 6+   */+  uint8_t version() const { return 6; }++  /**+   * Return the solicited-node multicast address for this address.+   */+  IPAddressV6 getSolicitedNodeAddress() const;++  /**+   * Return the mask associated with the given number of bits.+   *+   * If for instance numBits was 24 (e.g. /24) then the V4 mask returned should+   * be {0xff, 0xff, 0xff, 0x00}.+   * @param [in] numBits bitmask to retrieve+   * @throws abort if numBits == 0 or numBits > bitCount()+   * @return mask associated with numBits+   */++  static ByteArray16 fetchMask(size_t numBits);++  /**+   * Given 2 (IPAddressV6, mask) pairs extract the longest common (IPAddressV6,+   * mask) pair+   */+  static CIDRNetworkV6 longestCommonPrefix(+      const CIDRNetworkV6& one, const CIDRNetworkV6& two);++  /**+   * The number of bytes in the IP address+   *+   * @returns 16+   */+  static constexpr size_t byteCount() { return 16; }++  /**+   * Get the nth most significant bit of the IP address (0-indexed).+   * @param bitIndex n+   */+  bool getNthMSBit(size_t bitIndex) const {+    return detail::getNthMSBitImpl(*this, bitIndex, AF_INET6);+  }++  /**+   * Get the nth most significant byte of the IP address (0-indexed).+   * @param byteIndex n+   */+  uint8_t getNthMSByte(size_t byteIndex) const;++  /**+   * Get the nth bit of the IP address (0-indexed).+   * @param bitIndex n+   */+  bool getNthLSBit(size_t bitIndex) const {+    return getNthMSBit(bitCount() - bitIndex - 1);+  }++  /**+   * Get the nth byte of the IP address (0-indexed).+   * @param byteIndex n+   */+  uint8_t getNthLSByte(size_t byteIndex) const {+    return getNthMSByte(byteCount() - byteIndex - 1);+  }++  /**+   * Returns a pointer to the to IP address bytes, in network byte order.+   */+  const unsigned char* bytes() const { return addr_.in6Addr_.s6_addr; }++ protected:+  /**+   * Helper that returns true if the address is in the binary subnet specified+   * by addr.+   */+  bool inBinarySubnet(const std::array<uint8_t, 2> addr, size_t numBits) const;++ private:+  auto tie() const { return std::tie(addr_.bytes_, scope_); }++ public:+  /**+   * Return true if the two addresses are equal.+   */+  friend inline bool operator==(+      const IPAddressV6& addr1, const IPAddressV6& addr2) {+    return addr1.tie() == addr2.tie();+  }+  /**+   * Return true if the two addresses are not equal.+   */+  friend inline bool operator!=(+      const IPAddressV6& addr1, const IPAddressV6& addr2) {+    return addr1.tie() != addr2.tie();+  }++  /**+   * Return true if addr1 < addr2.+   */+  friend inline bool operator<(+      const IPAddressV6& addr1, const IPAddressV6& addr2) {+    return addr1.tie() < addr2.tie();+  }++  /**+   * Return true if addr1 > addr2.+   */+  friend inline bool operator>(+      const IPAddressV6& addr1, const IPAddressV6& addr2) {+    return addr1.tie() > addr2.tie();+  }++  /**+   * Return true if addr1 <= addr2.+   */+  friend inline bool operator<=(+      const IPAddressV6& addr1, const IPAddressV6& addr2) {+    return addr1.tie() <= addr2.tie();+  }++  /**+   * Return true if addr1 >= addr2.+   */+  friend inline bool operator>=(+      const IPAddressV6& addr1, const IPAddressV6& addr2) {+    return addr1.tie() >= addr2.tie();+  }++ private:+  union AddressStorage {+    in6_addr in6Addr_;+    ByteArray16 bytes_;+    AddressStorage() { std::memset(this, 0, sizeof(AddressStorage)); }+    explicit AddressStorage(const ByteArray16& bytes) : bytes_(bytes) {}+    explicit AddressStorage(const in6_addr& addr) : in6Addr_(addr) {}+    explicit AddressStorage(MacAddress mac);+  } addr_;++  // Link-local scope id.  This should always be 0 for IPAddresses that+  // are *not* link-local.+  uint16_t scope_{0};++  /**+   * Set the current IPAddressV6 object to have the address specified by bytes.+   * Returns IPAddressFormatError if bytes.size() is not 16.+   */+  Expected<Unit, IPAddressFormatError> trySetFromBinary(+      ByteRange bytes) noexcept;+};++/**+ * `boost::hash` uses hash_value(), so this allows `boost::hash` to work+ * automatically for IPAddressV4+ */+std::size_t hash_value(const IPAddressV6& addr);++/**+ * Appends a string representation of the IP address to the stream using str().+ */++std::ostream& operator<<(std::ostream& os, const IPAddressV6& addr);++/**+ * @overloadbrief Define toAppend() to allow IPAddress to be used with+ * `folly::to<string>`+ */+void toAppend(IPAddressV6 addr, std::string* result);+void toAppend(IPAddressV6 addr, fbstring* result);++} // namespace folly++namespace std {+template <>+struct hash<folly::IPAddressV6> {+  size_t operator()(const folly::IPAddressV6& addr) const {+    return addr.hash();+  }+};+} // namespace std
+ folly/folly/Indestructible.h view
@@ -0,0 +1,164 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <cassert>+#include <new>+#include <type_traits>+#include <utility>++#include <folly/Traits.h>+#include <folly/Utility.h>++namespace folly {++/***+ *  Indestructible+ *+ *  When you need a Meyers singleton that will not get destructed, even at+ *  shutdown, and you also want the object stored inline.+ *+ *  Use like:+ *+ *      void doSomethingWithExpensiveData();+ *+ *      void doSomethingWithExpensiveData() {+ *        static const Indestructible<map<string, int>> data{+ *          map<string, int>{{"key1", 17}, {"key2", 19}, {"key3", 23}},+ *        };+ *        callSomethingTakingAMapByRef(*data);+ *      }+ *+ *  This should be used only for Meyers singletons, and, even then, only when+ *  the instance does not need to be destructed ever.+ *+ *  This should not be used more generally, e.g., as member fields, etc.+ *+ *  This is designed as an alternative, but with one fewer allocation at+ *  construction time and one fewer pointer dereference at access time, to the+ *  Meyers singleton pattern of:+ *+ *    void doSomethingWithExpensiveData() {+ *      static const auto data =  // never `delete`d+ *          new map<string, int>{{"key1", 17}, {"key2", 19}, {"key3", 23}};+ *      callSomethingTakingAMapByRef(*data);+ *    }+ */++struct factory_constructor_t {+  explicit factory_constructor_t() = default;+};++constexpr factory_constructor_t factory_constructor{};++template <typename T>+class Indestructible final {+ public:+  template <typename S = T, typename = decltype(S())>+  constexpr Indestructible() noexcept(noexcept(T()))+      : storage_{std::in_place} {}++  /**+   * Constructor accepting a single argument by forwarding reference, this+   * allows using list initialization without the overhead of things like+   * std::in_place, etc and also works with std::initializer_list constructors+   * which can't be deduced, the default parameter helps there.+   *+   *    auto i = folly::Indestructible<std::map<int, int>>{{{1, 2}}};+   *+   * This provides convenience+   *+   * There are two versions of this constructor - one for when the element is+   * implicitly constructible from the given argument and one for when the+   * type is explicitly but not implicitly constructible from the given+   * argument.+   */+  template <+      typename U = T,+      std::enable_if_t<std::is_constructible<T, U&&>::value>* = nullptr,+      std::enable_if_t<+          !std::is_same<Indestructible<T>, remove_cvref_t<U>>::value>* =+          nullptr,+      std::enable_if_t<!std::is_convertible<U&&, T>::value>* = nullptr>+  explicit constexpr Indestructible(U&& u) noexcept(+      noexcept(T(std::declval<U>())))+      : storage_{std::in_place, std::forward<U>(u)} {}+  template <+      typename U = T,+      std::enable_if_t<std::is_constructible<T, U&&>::value>* = nullptr,+      std::enable_if_t<+          !std::is_same<Indestructible<T>, remove_cvref_t<U>>::value>* =+          nullptr,+      std::enable_if_t<std::is_convertible<U&&, T>::value>* = nullptr>+  /* implicit */ constexpr Indestructible(U&& u) noexcept(+      noexcept(T(std::declval<U>())))+      : storage_{std::in_place, std::forward<U>(u)} {}++  template <typename... Args, typename = decltype(T(std::declval<Args>()...))>+  explicit constexpr Indestructible(Args&&... args) noexcept(+      noexcept(T(std::declval<Args>()...)))+      : storage_{std::in_place, std::forward<Args>(args)...} {}+  template <+      typename U,+      typename... Args,+      typename = decltype(T(+          std::declval<std::initializer_list<U>&>(), std::declval<Args>()...))>+  explicit constexpr Indestructible(std::initializer_list<U> il, Args... args) noexcept(+      noexcept(T(+          std::declval<std::initializer_list<U>&>(), std::declval<Args>()...)))+      : storage_{std::in_place, il, std::forward<Args>(args)...} {}++  template <typename Factory>+  constexpr Indestructible(factory_constructor_t, Factory&& factory) noexcept(+      noexcept(factory()))+      : storage_(factory_constructor, std::forward<Factory>(factory)) {}++  Indestructible(Indestructible const&) = delete;+  Indestructible& operator=(Indestructible const&) = delete;++  T* get() noexcept { return reinterpret_cast<T*>(&storage_.bytes); }+  T const* get() const noexcept {+    return reinterpret_cast<T const*>(&storage_.bytes);+  }+  T& operator*() noexcept { return *get(); }+  T const& operator*() const noexcept { return *get(); }+  T* operator->() noexcept { return get(); }+  T const* operator->() const noexcept { return get(); }++  /* implicit */ operator T&() noexcept { return *get(); }+  /* implicit */ operator T const&() const noexcept { return *get(); }++ private:+  struct Storage {+    aligned_storage_for_t<T> bytes;++    template <typename... Args, typename = decltype(T(std::declval<Args>()...))>+    explicit constexpr Storage(std::in_place_t, Args&&... args) noexcept(+        noexcept(T(std::declval<Args>()...))) {+      ::new (&bytes) T(std::forward<Args>(args)...);+    }++    template <typename Factory>+    constexpr Storage(factory_constructor_t, Factory factory) noexcept(+        noexcept(factory())) {+      ::new (&bytes) T(factory());+    }+  };++  Storage storage_{};+};+} // namespace folly
+ folly/folly/IndexedMemPool.h view
@@ -0,0 +1,551 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <assert.h>+#include <errno.h>+#include <stdint.h>++#include <type_traits>++#include <folly/Portability.h>+#include <folly/concurrency/CacheLocality.h>+#include <folly/portability/SysMman.h>+#include <folly/portability/Unistd.h>+#include <folly/synchronization/AtomicStruct.h>++// Ignore shadowing warnings within this file, so includers can use -Wshadow.+FOLLY_PUSH_WARNING+FOLLY_GNU_DISABLE_WARNING("-Wshadow")++namespace folly {++namespace detail {+template <typename Pool>+struct IndexedMemPoolRecycler;+} // namespace detail++template <+    typename T,+    bool EagerRecycleWhenTrivial = false,+    bool EagerRecycleWhenNotTrivial = true>+struct IndexedMemPoolTraits {+  static constexpr bool eagerRecycle() {+    return std::is_trivial<T>::value+        ? EagerRecycleWhenTrivial+        : EagerRecycleWhenNotTrivial;+  }++  /// Called when the element pointed to by ptr is allocated for the+  /// first time.+  static void initialize(T* ptr) {+    if constexpr (!eagerRecycle()) {+      new (ptr) T();+    }+  }++  /// Called when the element pointed to by ptr is freed at the pool+  /// destruction time.+  static void cleanup(T* ptr) {+    if constexpr (!eagerRecycle()) {+      ptr->~T();+    }+  }++  /// Called when the element is allocated with the arguments forwarded from+  /// IndexedMemPool::allocElem.+  template <typename... Args>+  static void onAllocate(T* ptr, Args&&... args) {+    static_assert(+        sizeof...(Args) == 0 || eagerRecycle(),+        "emplace-style allocation requires eager recycle, "+        "which is defaulted only for non-trivial types");+    if (eagerRecycle()) {+      new (ptr) T(std::forward<Args>(args)...);+    }+  }++  /// Called when the element is recycled.+  static void onRecycle(T* ptr) {+    if (eagerRecycle()) {+      ptr->~T();+    }+  }+};++/// IndexedMemPool traits that implements the lazy lifecycle strategy. In this+/// strategy elements are default-constructed the first time they are allocated,+/// and destroyed when the pool itself is destroyed.+template <typename T>+using IndexedMemPoolTraitsLazyRecycle = IndexedMemPoolTraits<T, false, false>;++/// IndexedMemPool traits that implements the eager lifecycle strategy. In this+/// strategy elements are constructed when they are allocated from the pool and+/// destroyed when recycled.+template <typename T>+using IndexedMemPoolTraitsEagerRecycle = IndexedMemPoolTraits<T, true, true>;++/// Instances of IndexedMemPool dynamically allocate and then pool their+/// element type (T), returning 4-byte integer indices that can be passed+/// to the pool's operator[] method to access or obtain pointers to the+/// actual elements.  The memory backing items returned from the pool+/// will always be readable, even if items have been returned to the pool.+/// These two features are useful for lock-free algorithms.  The indexing+/// behavior makes it easy to build tagged pointer-like-things, since+/// a large number of elements can be managed using fewer bits than a+/// full pointer.  The access-after-free behavior makes it safe to read+/// from T-s even after they have been recycled, since it is guaranteed+/// that the memory won't have been returned to the OS and unmapped+/// (the algorithm must still use a mechanism to validate that the read+/// was correct, but it doesn't have to worry about page faults), and if+/// the elements use internal sequence numbers it can be guaranteed that+/// there won't be an ABA match due to the element being overwritten with+/// a different type that has the same bit pattern.+///+/// The object lifecycle strategy is controlled by the Traits parameter.+/// One strategy, implemented by IndexedMemPoolTraitsEagerRecycle, is to+/// construct objects when they are allocated from the pool and destroy+/// them when they are recycled.  In this mode allocIndex and allocElem+/// have emplace-like semantics.  In another strategy, implemented by+/// IndexedMemPoolTraitsLazyRecycle, objects are default-constructed the+/// first time they are removed from the pool, and deleted when the pool+/// itself is deleted.  By default the first mode is used for non-trivial+/// T, and the second is used for trivial T.  Clients can customize the+/// object lifecycle by providing their own Traits implementation.+/// See IndexedMemPoolTraits for a Traits example.+///+/// IMPORTANT: Space for extra elements is allocated to account for those+/// that are inaccessible because they are in other local lists, so the+/// actual number of items that can be allocated ranges from capacity to+/// capacity + (NumLocalLists_-1)*LocalListLimit_.  This is important if+/// you are trying to maximize the capacity of the pool while constraining+/// the bit size of the resulting pointers, because the pointers will+/// actually range up to the boosted capacity.  See maxIndexForCapacity+/// and capacityForMaxIndex.+///+/// To avoid contention, NumLocalLists_ free lists of limited (less than+/// or equal to LocalListLimit_) size are maintained, and each thread+/// retrieves and returns entries from its associated local list.  If the+/// local list becomes too large then elements are placed in bulk in a+/// global free list.  This allows items to be efficiently recirculated+/// from consumers to producers.  AccessSpreader is used to access the+/// local lists, so there is no performance advantage to having more+/// local lists than L1 caches.+///+/// The pool mmap-s the entire necessary address space when the pool is+/// constructed, but delays element construction.  This means that only+/// elements that are actually returned to the caller get paged into the+/// process's resident set (RSS).+template <+    typename T,+    uint32_t NumLocalLists_ = 32,+    uint32_t LocalListLimit_ = 200,+    template <typename> class Atom = std::atomic,+    typename Traits = IndexedMemPoolTraits<T>>+struct IndexedMemPool {+  typedef T value_type;++  typedef std::unique_ptr<T, detail::IndexedMemPoolRecycler<IndexedMemPool>>+      UniquePtr;++  IndexedMemPool(const IndexedMemPool&) = delete;+  IndexedMemPool& operator=(const IndexedMemPool&) = delete;++  static_assert(LocalListLimit_ <= 255, "LocalListLimit must fit in 8 bits");+  enum {+    NumLocalLists = NumLocalLists_,+    LocalListLimit = LocalListLimit_,+  };++  static_assert(+      std::is_nothrow_default_constructible<Atom<uint32_t>>::value,+      "Atom must be nothrow default constructible");++  // these are public because clients may need to reason about the number+  // of bits required to hold indices from a pool, given its capacity++  static constexpr uint32_t maxIndexForCapacity(uint32_t capacity) {+    // index of std::numeric_limits<uint32_t>::max() is reserved for isAllocated+    // tracking+    return uint32_t(std::min(+        uint64_t(capacity) + (NumLocalLists - 1) * LocalListLimit,+        uint64_t(std::numeric_limits<uint32_t>::max() - 1)));+  }++  static constexpr uint32_t capacityForMaxIndex(uint32_t maxIndex) {+    return maxIndex - (NumLocalLists - 1) * LocalListLimit;+  }++  /// Constructs a pool that can allocate at least _capacity_ elements,+  /// even if all the local lists are full+  explicit IndexedMemPool(uint32_t capacity)+      : actualCapacity_(maxIndexForCapacity(capacity)),+        size_(0),+        globalHead_(TaggedPtr{}) {+    const size_t needed = sizeof(Slot) * (actualCapacity_ + 1);+    size_t pagesize = size_t(sysconf(_SC_PAGESIZE));+    mmapLength_ = ((needed - 1) & ~(pagesize - 1)) + pagesize;+    assert(needed <= mmapLength_ && mmapLength_ < needed + pagesize);+    assert((mmapLength_ % pagesize) == 0);++    slots_ = static_cast<Slot*>(mmap(+        nullptr,+        mmapLength_,+        PROT_READ | PROT_WRITE,+        MAP_PRIVATE | MAP_ANONYMOUS,+        -1,+        0));+    if (slots_ == MAP_FAILED) {+      assert(errno == ENOMEM);+      throw std::bad_alloc();+    }+  }++  /// Destroys all of the contained elements+  ~IndexedMemPool() {+    using A = Atom<uint32_t>;+    for (uint32_t i = maxAllocatedIndex(); i > 0; --i) {+      Traits::cleanup(slots_[i].elemPtr());+      slots_[i].localNext.~A();+      slots_[i].globalNext.~A();+    }+    munmap(slots_, mmapLength_);+  }++  /// Returns a lower bound on the number of elements that may be+  /// simultaneously allocated and not yet recycled.  Because of the+  /// local lists it is possible that more elements than this are returned+  /// successfully+  uint32_t capacity() { return capacityForMaxIndex(actualCapacity_); }++  /// Returns the maximum index of elements ever allocated in this pool+  /// including elements that have been recycled.+  uint32_t maxAllocatedIndex() const {+    // Take the minimum since it is possible that size_ > actualCapacity_.+    // This can happen if there are multiple concurrent requests+    // when size_ == actualCapacity_ - 1.+    return std::min(uint32_t(size_), uint32_t(actualCapacity_));+  }++  /// Finds a slot with a non-zero index, emplaces a T there if we're+  /// using the eager recycle lifecycle mode, and returns the index,+  /// or returns 0 if no elements are available.  Passes a pointer to+  /// the element to Traits::onAllocate before the slot is marked as+  /// allocated.+  template <typename... Args>+  uint32_t allocIndex(Args&&... args) {+    auto idx = localPop(localHead());+    if (idx != 0) {+      Slot& s = slot(idx);+      Traits::onAllocate(s.elemPtr(), std::forward<Args>(args)...);+      markAllocated(s);+    }+    return idx;+  }++  /// If an element is available, returns a std::unique_ptr to it that will+  /// recycle the element to the pool when it is reclaimed, otherwise returns+  /// a null (falsy) std::unique_ptr.  Passes a pointer to the element to+  /// Traits::onAllocate before the slot is marked as allocated.+  template <typename... Args>+  UniquePtr allocElem(Args&&... args) {+    auto idx = allocIndex(std::forward<Args>(args)...);+    T* ptr = idx == 0 ? nullptr : slot(idx).elemPtr();+    return UniquePtr(ptr, typename UniquePtr::deleter_type(this));+  }++  /// Gives up ownership previously granted by alloc()+  void recycleIndex(uint32_t idx) {+    assert(isAllocated(idx));+    localPush(localHead(), idx);+  }++  /// Provides access to the pooled element referenced by idx+  T& operator[](uint32_t idx) { return *(slot(idx).elemPtr()); }++  /// Provides access to the pooled element referenced by idx+  const T& operator[](uint32_t idx) const { return *(slot(idx).elemPtr()); }++  /// If elem == &pool[idx], then pool.locateElem(elem) == idx.  Also,+  /// pool.locateElem(nullptr) == 0+  uint32_t locateElem(const T* elem) const {+    if (!elem) {+      return 0;+    }++    static_assert(std::is_standard_layout<Slot>::value, "offsetof needs POD");++    auto slot = reinterpret_cast<const Slot*>(+        reinterpret_cast<const char*>(elem) - offsetof(Slot, elemStorage));+    auto rv = uint32_t(slot - slots_);++    // this assert also tests that rv is in range+    assert(elem == &(*this)[rv]);+    return rv;+  }++  /// Returns true iff idx has been alloc()ed and not recycleIndex()ed+  bool isAllocated(uint32_t idx) const {+    return slot(idx).localNext.load(std::memory_order_acquire) == uint32_t(-1);+  }++ private:+  ///////////// types++  struct Slot {+    aligned_storage_for_t<T> elemStorage;+    Atom<uint32_t> localNext;+    Atom<uint32_t> globalNext;++    Slot() : localNext{}, globalNext{} {}+    T* elemPtr() { return std::launder(reinterpret_cast<T*>(&elemStorage)); }+    const T* elemPtr() const {+      return std::launder(reinterpret_cast<const T*>(&elemStorage));+    }+  };++  struct TaggedPtr {+    uint32_t idx;++    // size is bottom 8 bits, tag in top 24.  g++'s code generation for+    // bitfields seems to depend on the phase of the moon, plus we can+    // do better because we can rely on other checks to avoid masking+    uint32_t tagAndSize;++    enum : uint32_t {+      SizeBits = 8,+      SizeMask = (1U << SizeBits) - 1,+      TagIncr = 1U << SizeBits,+    };++    uint32_t size() const { return tagAndSize & SizeMask; }++    TaggedPtr withSize(uint32_t repl) const {+      assert(repl <= LocalListLimit);+      return TaggedPtr{idx, (tagAndSize & ~SizeMask) | repl};+    }++    TaggedPtr withSizeIncr() const {+      assert(size() < LocalListLimit);+      return TaggedPtr{idx, tagAndSize + 1};+    }++    TaggedPtr withSizeDecr() const {+      assert(size() > 0);+      return TaggedPtr{idx, tagAndSize - 1};+    }++    TaggedPtr withIdx(uint32_t repl) const {+      return TaggedPtr{repl, tagAndSize + TagIncr};+    }++    TaggedPtr withEmpty() const { return withIdx(0).withSize(0); }+  };++  struct alignas(hardware_destructive_interference_size) LocalList {+    AtomicStruct<TaggedPtr, Atom> head;++    LocalList() : head(TaggedPtr{}) {}+  };++  ////////// fields++  /// the number of bytes allocated from mmap, which is a multiple of+  /// the page size of the machine+  size_t mmapLength_;++  /// the actual number of slots that we will allocate, to guarantee+  /// that we will satisfy the capacity requested at construction time.+  /// They will be numbered 1..actualCapacity_ (note the 1-based counting),+  /// and occupy slots_[1..actualCapacity_].+  uint32_t actualCapacity_;++  /// this records the number of slots that have actually been constructed.+  /// To allow use of atomic ++ instead of CAS, we let this overflow.+  /// The actual number of constructed elements is min(actualCapacity_,+  /// size_)+  Atom<uint32_t> size_;++  /// raw storage, only 1..min(size_,actualCapacity_) (inclusive) are+  /// actually constructed.  Note that slots_[0] is not constructed or used+  alignas(hardware_destructive_interference_size) Slot* slots_;++  /// use AccessSpreader to find your list.  We use stripes instead of+  /// thread-local to avoid the need to grow or shrink on thread start+  /// or join.   These are heads of lists chained with localNext+  LocalList local_[NumLocalLists];++  /// this is the head of a list of node chained by globalNext, that are+  /// themselves each the head of a list chained by localNext+  alignas(hardware_destructive_interference_size)+      AtomicStruct<TaggedPtr, Atom> globalHead_;++  ///////////// private methods++  uint32_t slotIndex(uint32_t idx) const {+    assert(+        0 < idx && idx <= actualCapacity_ &&+        idx <= size_.load(std::memory_order_acquire));+    return idx;+  }++  Slot& slot(uint32_t idx) { return slots_[slotIndex(idx)]; }++  const Slot& slot(uint32_t idx) const { return slots_[slotIndex(idx)]; }++  // localHead references a full list chained by localNext.  s should+  // reference slot(localHead), it is passed as a micro-optimization+  void globalPush(Slot& s, uint32_t localHead) {+    while (true) {+      TaggedPtr gh = globalHead_.load(std::memory_order_acquire);+      s.globalNext.store(gh.idx, std::memory_order_relaxed);+      if (globalHead_.compare_exchange_strong(gh, gh.withIdx(localHead))) {+        // success+        return;+      }+    }+  }++  // idx references a single node+  void localPush(AtomicStruct<TaggedPtr, Atom>& head, uint32_t idx) {+    Slot& s = slot(idx);+    TaggedPtr h = head.load(std::memory_order_acquire);+    bool recycled = false;+    while (true) {+      s.localNext.store(h.idx, std::memory_order_release);+      if (!recycled) {+        Traits::onRecycle(slot(idx).elemPtr());+        recycled = true;+      }++      if (h.size() == LocalListLimit) {+        // push will overflow local list, steal it instead+        if (head.compare_exchange_strong(h, h.withEmpty())) {+          // steal was successful, put everything in the global list+          globalPush(s, idx);+          return;+        }+      } else {+        // local list has space+        if (head.compare_exchange_strong(h, h.withIdx(idx).withSizeIncr())) {+          // success+          return;+        }+      }+      // h was updated by failing CAS+    }+  }++  // returns 0 if empty+  uint32_t globalPop() {+    while (true) {+      TaggedPtr gh = globalHead_.load(std::memory_order_acquire);+      if (gh.idx == 0 ||+          globalHead_.compare_exchange_strong(+              gh,+              gh.withIdx(+                  slot(gh.idx).globalNext.load(std::memory_order_relaxed)))) {+        // global list is empty, or pop was successful+        return gh.idx;+      }+    }+  }++  // returns 0 if allocation failed+  uint32_t localPop(AtomicStruct<TaggedPtr, Atom>& head) {+    while (true) {+      TaggedPtr h = head.load(std::memory_order_acquire);+      if (h.idx != 0) {+        // local list is non-empty, try to pop+        Slot& s = slot(h.idx);+        auto next = s.localNext.load(std::memory_order_relaxed);+        if (head.compare_exchange_strong(h, h.withIdx(next).withSizeDecr())) {+          // success+          return h.idx;+        }+        continue;+      }++      uint32_t idx = globalPop();+      if (idx == 0) {+        // global list is empty, allocate and construct new slot+        if (size_.load(std::memory_order_relaxed) >= actualCapacity_ ||+            (idx = ++size_) > actualCapacity_) {+          // allocation failed+          return 0;+        }+        Slot& s = slot(idx);+        // Atom is enforced above to be nothrow-default-constructible+        // As an optimization, use default-initialization (no parens) rather+        // than direct-initialization (with parens): these locations are+        // stored-to before they are loaded-from+        new (&s.localNext) Atom<uint32_t>;+        new (&s.globalNext) Atom<uint32_t>;+        Traits::initialize(s.elemPtr());+        return idx;+      }++      Slot& s = slot(idx);+      auto next = s.localNext.load(std::memory_order_relaxed);+      if (head.compare_exchange_strong(+              h, h.withIdx(next).withSize(LocalListLimit))) {+        // global list moved to local list, keep head for us+        return idx;+      }+      // local bulk push failed, return idx to the global list and try again+      globalPush(s, idx);+    }+  }++  AtomicStruct<TaggedPtr, Atom>& localHead() {+    auto stripe = AccessSpreader<Atom>::current(NumLocalLists);+    return local_[stripe].head;+  }++  void markAllocated(Slot& slot) {+    slot.localNext.store(uint32_t(-1), std::memory_order_release);+  }++ public:+  static constexpr std::size_t kSlotSize = sizeof(Slot);+};++namespace detail {++/// This is a stateful Deleter functor, which allows std::unique_ptr+/// to track elements allocated from an IndexedMemPool by tracking the+/// associated pool.  See IndexedMemPool::allocElem.+template <typename Pool>+struct IndexedMemPoolRecycler {+  Pool* pool;++  explicit IndexedMemPoolRecycler(Pool* pool) : pool(pool) {}++  IndexedMemPoolRecycler(const IndexedMemPoolRecycler<Pool>& rhs) = default;+  IndexedMemPoolRecycler& operator=(const IndexedMemPoolRecycler<Pool>& rhs) =+      default;++  void operator()(typename Pool::value_type* elem) const {+    pool->recycleIndex(pool->locateElem(elem));+  }+};++} // namespace detail++} // namespace folly++FOLLY_POP_WARNING
+ folly/folly/IntrusiveList.h view
@@ -0,0 +1,17 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#include <folly/container/IntrusiveList.h>
+ folly/folly/Lazy.h view
@@ -0,0 +1,142 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <type_traits>+#include <utility>++#include <folly/Optional.h>+#include <folly/functional/Invoke.h>++namespace folly {++//////////////////////////////////////////////////////////////////////++/*+ * Lazy -- for delayed initialization of a value.  The value's+ * initialization will be computed on demand at its first use, but+ * will not be recomputed if its value is requested again.  The value+ * may still be mutated after its initialization if the lazy is not+ * declared const.+ *+ * The value is created using folly::lazy, usually with a lambda, and+ * its value is requested using operator().+ *+ * Note that the value is not safe for concurrent accesses by multiple+ * threads, even if you declare it const.  See note below.+ *+ *+ * Example Usage:+ *+ *   void foo() {+ *     auto const val = folly::lazy([&]{+ *       return something_expensive(blah());+ *     });+ *+ *     if (condition1) {+ *       use(val());+ *     }+ *     if (condition2) {+ *       useMaybeAgain(val());+ *     } else {+ *       // Unneeded in this branch.+ *     }+ *   }+ *+ *+ * Rationale:+ *+ *    - operator() is used to request the value instead of an implicit+ *      conversion because the slight syntactic overhead in common+ *      seems worth the increased clarity.+ *+ *    - Lazy values do not model CopyConstructible because it is+ *      unclear what semantics would be desirable.  Either copies+ *      should share the cached value (adding overhead to cases that+ *      don't need to support copies), or they could recompute the+ *      value unnecessarily.  Sharing with mutable lazies would also+ *      leave them with non-value semantics despite looking+ *      value-like.+ *+ *    - Not thread safe for const accesses.  Many use cases for lazy+ *      values are local variables on the stack, where multiple+ *      threads shouldn't even be able to reach the value.  It still+ *      is useful to indicate/check that the value doesn't change with+ *      const, particularly when it is captured by a large family of+ *      lambdas.  Adding internal synchronization seems like it would+ *      pessimize the most common use case in favor of less likely use+ *      cases.+ *+ */++//////////////////////////////////////////////////////////////////////++namespace detail {++template <class Func>+struct Lazy {+  typedef invoke_result_t<Func> result_type;++  static_assert(+      !std::is_const<Func>::value, "Func should not be a const-qualified type");+  static_assert(+      !std::is_reference<Func>::value, "Func should not be a reference type");++  explicit Lazy(Func&& f) : func_(std::move(f)) {}+  explicit Lazy(const Func& f) : func_(f) {}++  Lazy(Lazy&& o) : value_(std::move(o.value_)), func_(std::move(o.func_)) {}++  Lazy(const Lazy&) = delete;+  Lazy& operator=(const Lazy&) = delete;+  Lazy& operator=(Lazy&&) = delete;++  const result_type& operator()() const {+    ensure_initialized();++    return *value_;+  }++  result_type& operator()() {+    ensure_initialized();++    return *value_;+  }++ private:+  void ensure_initialized() const {+    if (!value_) {+      value_ = func_();+    }+  }++  mutable Optional<result_type> value_;+  mutable Func func_;+};++} // namespace detail++//////////////////////////////////////////////////////////////////////++template <class Func>+auto lazy(Func&& fun) {+  return detail::Lazy<remove_cvref_t<Func>>(std::forward<Func>(fun));+}++//////////////////////////////////////////////////////////////////////++} // namespace folly
+ folly/folly/Likely.h view
@@ -0,0 +1,76 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+/**+ * Likeliness annotations.+ *+ * Useful when the author has better knowledge than the compiler of whether+ * the branch condition is overwhelmingly likely to take a specific value.+ *+ * Useful when the author has better knowledge than the compiler of which code+ * paths are designed as the fast path and which are designed as the slow path,+ * and to force the compiler to optimize for the fast path, even when it is not+ * overwhelmingly likely.+ *+ * Notes:+ * * All supported compilers treat unconditionally-noreturn blocks as unlikely.+ *   This is true for blocks which unconditionally throw exceptions and for+ *   blocks which unconditionally call [[noreturn]]-annotated functions. Such+ *   cases do not require likeliness annotations.+ *+ * @file Likely.h+ * @refcode folly/docs/examples/folly/Likely.cpp+ */++#pragma once++#include <folly/lang/Builtin.h>++/**+ * Treat the condition as likely.+ *+ * @def FOLLY_LIKELY+ */+#define FOLLY_LIKELY(...) FOLLY_BUILTIN_EXPECT((__VA_ARGS__), 1)++/**+ * Treat the condition as unlikely.+ *+ * @def FOLLY_UNLIKELY+ */+#define FOLLY_UNLIKELY(...) FOLLY_BUILTIN_EXPECT((__VA_ARGS__), 0)++// Un-namespaced annotations++#undef LIKELY+#undef UNLIKELY++/**+ * Treat the condition as likely.+ *+ * @def LIKELY+ */+/**+ * Treat the condition as unlikely.+ *+ * @def UNLIKELY+ */+#if defined(__GNUC__)+#define LIKELY(x) (__builtin_expect((x), 1))+#define UNLIKELY(x) (__builtin_expect((x), 0))+#else+#define LIKELY(x) (x)+#define UNLIKELY(x) (x)+#endif
+ folly/folly/MPMCPipeline.h view
@@ -0,0 +1,280 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <utility>++#include <glog/logging.h>++#include <folly/Portability.h>+#include <folly/detail/MPMCPipelineDetail.h>++namespace folly {++/**+ * Helper tag template to use amplification > 1+ */+template <class T, size_t Amp>+class MPMCPipelineStage;++/**+ * Multi-Producer, Multi-Consumer pipeline.+ *+ * A N-stage pipeline is a combination of N+1 MPMC queues (see MPMCQueue.h).+ *+ * At each stage, you may dequeue the results from the previous stage (possibly+ * from multiple threads) and enqueue results to the next stage. Regardless of+ * the order of completion, data is delivered to the next stage in the original+ * order.  Each input is matched with a "ticket" which must be produced+ * when enqueueing to the next stage.+ *+ * A given stage must produce exactly K ("amplification factor", default K=1)+ * results for every input. This is enforced by requiring that each ticket+ * is used exactly K times.+ *+ * Usage:+ *+ * // arguments are queue sizes+ * MPMCPipeline<int, std::string, int> pipeline(10, 10, 10);+ *+ * pipeline.blockingWrite(42);+ *+ * {+ *   int val;+ *   auto ticket = pipeline.blockingReadStage<0>(val);+ *   pipeline.blockingWriteStage<0>(ticket, folly::to<std::string>(val));+ * }+ *+ * {+ *   std::string val;+ *   auto ticket = pipeline.blockingReadStage<1>(val);+ *   int ival = 0;+ *   try {+ *     ival = folly::to<int>(val);+ *   } catch (...) {+ *     // We must produce exactly 1 output even on exception!+ *   }+ *   pipeline.blockingWriteStage<1>(ticket, ival);+ * }+ *+ * int result;+ * pipeline.blockingRead(result);+ * // result == 42+ *+ * To specify amplification factors greater than 1, use+ * MPMCPipelineStage<T, amplification> instead of T in the declaration:+ *+ * MPMCPipeline<int,+ *              MPMCPipelineStage<std::string, 2>,+ *              MPMCPipelineStage<int, 4>>+ *+ * declares a two-stage pipeline: the first stage produces 2 strings+ * for each input int, the second stage produces 4 ints for each input string,+ * so, overall, the pipeline produces 2*4 = 8 ints for each input int.+ *+ * Implementation details: we use N+1 MPMCQueue objects; each intermediate+ * queue connects two adjacent stages.  The MPMCQueue implementation is abused;+ * instead of using it as a queue, we insert in the output queue at the+ * position determined by the input queue's popTicket_.  We guarantee that+ * all slots are filled (and therefore the queue doesn't freeze) because+ * we require that each step produces exactly K outputs for every input.+ */+template <class In, class... Stages>+class MPMCPipeline {+  typedef std::tuple<detail::PipelineStageInfo<Stages>...> StageInfos;+  typedef std::tuple<+      detail::MPMCPipelineStageImpl<In>,+      detail::MPMCPipelineStageImpl<+          typename detail::PipelineStageInfo<Stages>::value_type>...>+      StageTuple;+  static constexpr size_t kAmplification =+      detail::AmplificationProduct<StageInfos>::value;++  class TicketBaseDebug {+   public:+    TicketBaseDebug() noexcept : owner_(nullptr), value_(0xdeadbeeffaceb00c) {}+    TicketBaseDebug(TicketBaseDebug&& other) noexcept+        : owner_(std::exchange(other.owner_, nullptr)),+          value_(std::exchange(other.value_, 0xdeadbeeffaceb00c)) {}+    explicit TicketBaseDebug(MPMCPipeline* owner, uint64_t value) noexcept+        : owner_(owner), value_(value) {}+    void check_owner(MPMCPipeline* owner) const { CHECK(owner == owner_); }++    MPMCPipeline* owner_;+    uint64_t value_;+  };++  class TicketBaseNDebug {+   public:+    TicketBaseNDebug() = default;+    TicketBaseNDebug(TicketBaseNDebug&&) = default;+    explicit TicketBaseNDebug(MPMCPipeline*, uint64_t value) noexcept+        : value_(value) {}+    void check_owner(MPMCPipeline*) const {}++    uint64_t value_;+  };++  using TicketBase =+      std::conditional_t<kIsDebug, TicketBaseDebug, TicketBaseNDebug>;++ public:+  /**+   * Ticket, returned by blockingReadStage, must be given back to+   * blockingWriteStage. Tickets are not thread-safe.+   */+  template <size_t Stage>+  class Ticket : TicketBase {+   public:+    ~Ticket() noexcept {+      CHECK_EQ(remainingUses_, 0) << "All tickets must be completely used!";+    }++    Ticket() noexcept : remainingUses_(0) {}++    Ticket(Ticket&& other) noexcept+        : TicketBase(static_cast<TicketBase&&>(other)),+          remainingUses_(std::exchange(other.remainingUses_, 0)) {}++    Ticket& operator=(Ticket&& other) noexcept {+      if (this != &other) {+        this->~Ticket();+        new (this) Ticket(std::move(other));+      }+      return *this;+    }++   private:+    friend class MPMCPipeline;+    size_t remainingUses_;++    Ticket(MPMCPipeline* owner, size_t amplification, uint64_t value) noexcept+        : TicketBase(owner, value * amplification),+          remainingUses_(amplification) {}++    uint64_t use(MPMCPipeline* owner) {+      CHECK_GT(remainingUses_--, 0);+      TicketBase::check_owner(owner);+      return TicketBase::value_++;+    }+  };++  /**+   * Default-construct pipeline. Useful to move-assign later,+   * just like MPMCQueue, see MPMCQueue.h for more details.+   */+  MPMCPipeline() = default;++  /**+   * Construct a pipeline with N+1 queue sizes.+   */+  template <class... Sizes>+  explicit MPMCPipeline(Sizes... sizes) : stages_(sizes...) {}++  /**+   * Push an element into (the first stage of) the pipeline. Blocking.+   */+  template <class... Args>+  void blockingWrite(Args&&... args) {+    std::get<0>(stages_).blockingWrite(std::forward<Args>(args)...);+  }++  /**+   * Try to push an element into (the first stage of) the pipeline.+   * Non-blocking.+   */+  template <class... Args>+  bool write(Args&&... args) {+    return std::get<0>(stages_).write(std::forward<Args>(args)...);+  }++  /**+   * Read an element for stage Stage and obtain a ticket. Blocking.+   */+  template <size_t Stage>+  Ticket<Stage> blockingReadStage(+      typename std::tuple_element<Stage, StageTuple>::type::value_type& elem) {+    return Ticket<Stage>(+        this,+        std::tuple_element<Stage, StageInfos>::type::kAmplification,+        std::get<Stage>(stages_).blockingRead(elem));+  }++  /**+   * Try to read an element for stage Stage and obtain a ticket.+   * Non-blocking.+   */+  template <size_t Stage>+  bool readStage(+      Ticket<Stage>& ticket,+      typename std::tuple_element<Stage, StageTuple>::type::value_type& elem) {+    uint64_t tval;+    if (!std::get<Stage>(stages_).readAndGetTicket(tval, elem)) {+      return false;+    }+    ticket = Ticket<Stage>(+        this,+        std::tuple_element<Stage, StageInfos>::type::kAmplification,+        tval);+    return true;+  }++  /**+   * Complete an element in stage Stage (pushing it for stage Stage+1).+   * Blocking.+   */+  template <size_t Stage, class... Args>+  void blockingWriteStage(Ticket<Stage>& ticket, Args&&... args) {+    std::get<Stage + 1>(stages_).blockingWriteWithTicket(+        ticket.use(this), std::forward<Args>(args)...);+  }++  /**+   * Pop an element from (the final stage of) the pipeline. Blocking.+   */+  void blockingRead(typename std::tuple_element<sizeof...(Stages), StageTuple>::+                        type::value_type& elem) {+    std::get<sizeof...(Stages)>(stages_).blockingRead(elem);+  }++  /**+   * Try to pop an element from (the final stage of) the pipeline.+   * Non-blocking.+   */+  bool read(typename std::tuple_element<sizeof...(Stages), StageTuple>::type::+                value_type& elem) {+    return std::get<sizeof...(Stages)>(stages_).read(elem);+  }++  /**+   * Estimate queue size, measured as values from the last stage.+   * (so if the pipeline has an amplification factor > 1, pushing an element+   * into the first stage will cause sizeGuess() to be == amplification factor)+   * Elements "in flight" (currently processed as part of a stage, so not+   * in any queue) are also counted.+   */+  ssize_t sizeGuess() const noexcept {+    return ssize_t(+        std::get<0>(stages_).writeCount() * kAmplification -+        std::get<sizeof...(Stages)>(stages_).readCount());+  }++ private:+  StageTuple stages_;+};++} // namespace folly
+ folly/folly/MPMCQueue.h view
@@ -0,0 +1,1556 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <algorithm>+#include <atomic>+#include <cassert>+#include <cstring>+#include <limits>+#include <type_traits>++#include <folly/Traits.h>+#include <folly/concurrency/CacheLocality.h>+#include <folly/detail/TurnSequencer.h>+#include <folly/portability/Unistd.h>++namespace folly {++namespace detail {++template <typename T, template <typename> class Atom>+struct SingleElementQueue;++template <typename T>+class MPMCPipelineStageImpl;++/// MPMCQueue base CRTP template+template <typename>+class MPMCQueueBase;++} // namespace detail++/// MPMCQueue<T> is a high-performance bounded concurrent queue that+/// supports multiple producers, multiple consumers, and optional blocking.+/// The queue has a fixed capacity, for which all memory will be allocated+/// up front.  The bulk of the work of enqueuing and dequeuing can be+/// performed in parallel.+///+/// MPMCQueue is linearizable.  That means that if a call to write(A)+/// returns before a call to write(B) begins, then A will definitely end up+/// in the queue before B, and if a call to read(X) returns before a call+/// to read(Y) is started, that X will be something from earlier in the+/// queue than Y.  This also means that if a read call returns a value, you+/// can be sure that all previous elements of the queue have been assigned+/// a reader (that reader might not yet have returned, but it exists).+///+/// The underlying implementation uses a ticket dispenser for the head and+/// the tail, spreading accesses across N single-element queues to produce+/// a queue with capacity N.  The ticket dispensers use atomic increment,+/// which is more robust to contention than a CAS loop.  Each of the+/// single-element queues uses its own CAS to serialize access, with an+/// adaptive spin cutoff.  When spinning fails on a single-element queue+/// it uses futex()'s _BITSET operations to reduce unnecessary wakeups+/// even if multiple waiters are present on an individual queue (such as+/// when the MPMCQueue's capacity is smaller than the number of enqueuers+/// or dequeuers).+///+/// In benchmarks (contained in tao/queues/ConcurrentQueueTests)+/// it handles 1 to 1, 1 to N, N to 1, and N to M thread counts better+/// than any of the alternatives present in fbcode, for both small (~10)+/// and large capacities.  In these benchmarks it is also faster than+/// tbb::concurrent_bounded_queue for all configurations.  When there are+/// many more threads than cores, MPMCQueue is _much_ faster than the tbb+/// queue because it uses futex() to block and unblock waiting threads,+/// rather than spinning with sched_yield.+///+/// NOEXCEPT INTERACTION: tl;dr; If it compiles you're fine.  Ticket-based+/// queues separate the assignment of queue positions from the actual+/// construction of the in-queue elements, which means that the T+/// constructor used during enqueue must not throw an exception.  This is+/// enforced at compile time using type traits, which requires that T be+/// adorned with accurate noexcept information.  If your type does not+/// use noexcept, you will have to wrap it in something that provides+/// the guarantee.  We provide an alternate safe implementation for types+/// that don't use noexcept but that are marked folly::IsRelocatable+/// and std::is_nothrow_constructible, which is common for folly types.+/// In particular, if you can declare FOLLY_ASSUME_FBVECTOR_COMPATIBLE+/// then your type can be put in MPMCQueue.+///+/// If you have a pool of N queue consumers that you want to shut down+/// after the queue has drained, one way is to enqueue N sentinel values+/// to the queue.  If the producer doesn't know how many consumers there+/// are you can enqueue one sentinel and then have each consumer requeue+/// two sentinels after it receives it (by requeuing 2 the shutdown can+/// complete in O(log P) time instead of O(P)).+template <+    typename T,+    template <typename> class Atom = std::atomic,+    bool Dynamic = false,+    class Allocator = std::allocator<T>>+class MPMCQueue+    : public detail::MPMCQueueBase<MPMCQueue<T, Atom, Dynamic, Allocator>>,+      std::allocator_traits<Allocator>::template rebind_alloc<+          detail::SingleElementQueue<T, Atom>> {+  friend class detail::MPMCPipelineStageImpl<T>;+  using Base = detail::MPMCQueueBase<MPMCQueue<T, Atom, Dynamic, Allocator>>;+  using Slot = detail::SingleElementQueue<T, Atom>;+  using SlotAllocator =+      typename std::allocator_traits<Allocator>::template rebind_alloc<Slot>;+  using SlotAllocatorTraits = std::allocator_traits<SlotAllocator>;++ public:+  using typename Base::value_type;++  explicit MPMCQueue(size_t queueCapacity) : Base(queueCapacity) {+    initQueue(queueCapacity);+  }++  MPMCQueue(size_t queueCapacity, const SlotAllocator& alloc)+      : Base(queueCapacity), SlotAllocator(alloc) {+    initQueue(queueCapacity);+  }++  MPMCQueue() noexcept = default;+  MPMCQueue(MPMCQueue&&) noexcept = default;+  /// IMPORTANT: The move operator is here to make it easier to perform+  /// the initialization phase, it is not safe to use when there are any+  /// concurrent accesses (this is not checked).+  MPMCQueue const& operator=(MPMCQueue&& rhs) {+    if (this != &rhs) {+      this->~MPMCQueue();+      new (this) MPMCQueue(std::move(rhs));+    }+    return *this;+  }+  ~MPMCQueue() {+    if (kUsingStdAllocator) {+      delete[] this->slots_;+      this->slots_ = nullptr;+    } else {+      if (this->slots_) {+        size_t count = this->capacity_ + 2 * this->kSlotPadding;+        for (size_t i = 0; i < count; ++i) {+          SlotAllocatorTraits::destroy(*this, this->slots_ + i);+        }+        SlotAllocatorTraits::deallocate(*this, this->slots_, count);+        this->slots_ = nullptr;+      }+    }+  }++ private:+  static constexpr bool kUsingStdAllocator =+      is_instantiation_of_v<std::allocator, Allocator>;++  void initQueue(size_t queueCapacity) {+    this->stride_ = this->computeStride(queueCapacity);+    size_t count = queueCapacity + 2 * this->kSlotPadding;+    if (kUsingStdAllocator) {+      this->slots_ = new Slot[count];+    } else {+      this->slots_ = SlotAllocatorTraits::allocate(*this, count);+      for (size_t i = 0; i < count; ++i) {+        SlotAllocatorTraits::construct(*this, this->slots_ + i);+      }+    }+  }+};++/// *** The dynamic version of MPMCQueue is deprecated. ***+/// Use UnboundedQueue instead.++/// The dynamic version of MPMCQueue allows dynamic expansion of queue+/// capacity, such that a queue may start with a smaller capacity than+/// specified and expand only if needed. Users may optionally specify+/// the initial capacity and the expansion multiplier.+///+/// The design uses a seqlock to enforce mutual exclusion among+/// expansion attempts. Regular operations read up-to-date queue+/// information (slots array, capacity, stride) inside read-only+/// seqlock sections, which are unimpeded when no expansion is in+/// progress.+///+/// An expansion computes a new capacity, allocates a new slots array,+/// and updates stride. No information needs to be copied from the+/// current slots array to the new one. When this happens, new slots+/// will not have sequence numbers that match ticket numbers. The+/// expansion needs to compute a ticket offset such that operations+/// that use new arrays can adjust the calculations of slot indexes+/// and sequence numbers that take into account that the new slots+/// start with sequence numbers of zero. The current ticket offset is+/// packed with the seqlock in an atomic 64-bit integer. The initial+/// offset is zero.+///+/// Lagging write and read operations with tickets lower than the+/// ticket offset of the current slots array (i.e., the minimum ticket+/// number that can be served by the current array) must use earlier+/// closed arrays instead of the current one. Information about closed+/// slots arrays (array address, capacity, stride, and offset) is+/// maintained in a logarithmic-sized structure. Each entry in that+/// structure never needs to be changed once set. The number of closed+/// arrays is half the value of the seqlock (when unlocked).+///+/// The acquisition of the seqlock to perform an expansion does not+/// prevent the issuing of new push and pop tickets concurrently. The+/// expansion must set the new ticket offset to a value that couldn't+/// have been issued to an operation that has already gone through a+/// seqlock read-only section (and hence obtained information for+/// older closed arrays).+///+/// Note that the total queue capacity can temporarily exceed the+/// specified capacity when there are lagging consumers that haven't+/// yet consumed all the elements in closed arrays. Users should not+/// rely on the capacity of dynamic queues for synchronization, e.g.,+/// they should not expect that a thread will definitely block on a+/// call to blockingWrite() when the queue size is known to be equal+/// to its capacity.+///+/// Note that some writeIfNotFull() and tryWriteUntil() operations may+/// fail even if the size of the queue is less than its maximum+/// capacity and despite the success of expansion, if the operation+/// happens to acquire a ticket that belongs to a closed array. This+/// is a transient condition. Typically, one or two ticket values may+/// be subject to such condition per expansion.+///+/// The dynamic version is a partial specialization of MPMCQueue with+/// Dynamic == true.+template <typename T, template <typename> class Atom, class Allocator>+class MPMCQueue<T, Atom, true, Allocator>+    : public detail::MPMCQueueBase<MPMCQueue<T, Atom, true, Allocator>> {+  static_assert(+      is_instantiation_of_v<std::allocator, Allocator>,+      "The dynamic version of MPMCQueue does not support custom allocators");+  friend class detail::MPMCQueueBase<MPMCQueue<T, Atom, true, Allocator>>;+  using Slot = detail::SingleElementQueue<T, Atom>;+  using Base = detail::MPMCQueueBase<MPMCQueue<T, Atom, true, Allocator>>;+  struct ClosedArray {+    uint64_t offset_{0};+    Slot* slots_{nullptr};+    size_t capacity_{0};+    int stride_{0};+  };++ public:+  explicit MPMCQueue(size_t queueCapacity) : Base(queueCapacity) {+    size_t cap = std::min<size_t>(kDefaultMinDynamicCapacity, queueCapacity);+    initQueue(cap, kDefaultExpansionMultiplier);+  }++  explicit MPMCQueue(+      size_t queueCapacity, size_t minCapacity, size_t expansionMultiplier)+      : Base(queueCapacity) {+    minCapacity = std::max<size_t>(1, minCapacity);+    size_t cap = std::min<size_t>(minCapacity, queueCapacity);+    expansionMultiplier = std::max<size_t>(2, expansionMultiplier);+    initQueue(cap, expansionMultiplier);+  }++  MPMCQueue() noexcept {+    dmult_ = 0;+    closed_ = nullptr;+  }++  MPMCQueue(MPMCQueue<T, Atom, true>&& rhs) noexcept {+    this->capacity_ = rhs.capacity_;+    new (&this->dslots_)+        Atom<Slot*>(rhs.dslots_.load(std::memory_order_relaxed));+    new (&this->dstride_)+        Atom<int>(rhs.dstride_.load(std::memory_order_relaxed));+    this->dstate_.store(+        rhs.dstate_.load(std::memory_order_relaxed), std::memory_order_relaxed);+    this->dcapacity_.store(+        rhs.dcapacity_.load(std::memory_order_relaxed),+        std::memory_order_relaxed);+    this->pushTicket_.store(+        rhs.pushTicket_.load(std::memory_order_relaxed),+        std::memory_order_relaxed);+    this->popTicket_.store(+        rhs.popTicket_.load(std::memory_order_relaxed),+        std::memory_order_relaxed);+    this->pushSpinCutoff_.store(+        rhs.pushSpinCutoff_.load(std::memory_order_relaxed),+        std::memory_order_relaxed);+    this->popSpinCutoff_.store(+        rhs.popSpinCutoff_.load(std::memory_order_relaxed),+        std::memory_order_relaxed);+    dmult_ = rhs.dmult_;+    closed_ = rhs.closed_;++    rhs.capacity_ = 0;+    rhs.dslots_.store(nullptr, std::memory_order_relaxed);+    rhs.dstride_.store(0, std::memory_order_relaxed);+    rhs.dstate_.store(0, std::memory_order_relaxed);+    rhs.dcapacity_.store(0, std::memory_order_relaxed);+    rhs.pushTicket_.store(0, std::memory_order_relaxed);+    rhs.popTicket_.store(0, std::memory_order_relaxed);+    rhs.pushSpinCutoff_.store(0, std::memory_order_relaxed);+    rhs.popSpinCutoff_.store(0, std::memory_order_relaxed);+    rhs.dmult_ = 0;+    rhs.closed_ = nullptr;+  }++  /// IMPORTANT: The move operator is here to make it easier to perform+  /// the initialization phase, it is not safe to use when there are any+  /// concurrent accesses (this is not checked).+  MPMCQueue<T, Atom, true> const& operator=(MPMCQueue<T, Atom, true>&& rhs) {+    if (this != &rhs) {+      this->~MPMCQueue();+      new (this) MPMCQueue(std::move(rhs));+    }+    return *this;+  }++  ~MPMCQueue() {+    if (closed_ != nullptr) {+      for (int i = getNumClosed(this->dstate_.load()) - 1; i >= 0; --i) {+        delete[] closed_[i].slots_;+      }+      delete[] closed_;+    }+    using AtomInt = Atom<int>;+    this->dstride_.~AtomInt();+    using AtomSlot = Atom<Slot*>;+    // Sort of a hack to get ~MPMCQueueBase to free dslots_+    auto slots = this->dslots_.load();+    this->dslots_.~AtomSlot();+    this->slots_ = slots;+  }++  size_t allocatedCapacity() const noexcept {+    return this->dcapacity_.load(std::memory_order_relaxed);+  }++  template <typename... Args>+  void blockingWrite(Args&&... args) noexcept {+    uint64_t ticket = this->pushTicket_++;+    Slot* slots;+    size_t cap;+    int stride;+    uint64_t state;+    uint64_t offset;+    do {+      if (!trySeqlockReadSection(state, slots, cap, stride)) {+        asm_volatile_pause();+        continue;+      }+      if (maybeUpdateFromClosed(state, ticket, offset, slots, cap, stride)) {+        // There was an expansion after this ticket was issued.+        break;+      }+      if (slots[this->idx((ticket - offset), cap, stride)].mayEnqueue(+              this->turn(ticket - offset, cap))) {+        // A slot is ready. No need to expand.+        break;+      } else if (+          this->popTicket_.load(std::memory_order_relaxed) + cap > ticket) {+        // May block, but a pop is in progress. No need to expand.+        // Get seqlock read section info again in case an expansion+        // occurred with an equal or higher ticket.+        continue;+      } else {+        // May block. See if we can expand.+        if (tryExpand(state, cap)) {+          // This or another thread started an expansion. Get updated info.+          continue;+        } else {+          // Can't expand.+          break;+        }+      }+    } while (true);+    this->enqueueWithTicketBase(+        ticket - offset, slots, cap, stride, std::forward<Args>(args)...);+  }++  void blockingReadWithTicket(uint64_t& ticket, T& elem) noexcept {+    ticket = this->popTicket_++;+    Slot* slots;+    size_t cap;+    int stride;+    uint64_t state;+    uint64_t offset;+    while (!trySeqlockReadSection(state, slots, cap, stride)) {+      asm_volatile_pause();+    }+    // If there was an expansion after the corresponding push ticket+    // was issued, adjust accordingly+    maybeUpdateFromClosed(state, ticket, offset, slots, cap, stride);+    this->dequeueWithTicketBase(ticket - offset, slots, cap, stride, elem);+  }++ private:+  enum {+    kSeqlockBits = 6,+    kDefaultMinDynamicCapacity = 10,+    kDefaultExpansionMultiplier = 10,+  };++  size_t dmult_;++  //  Info about closed slots arrays for use by lagging operations+  ClosedArray* closed_;++  void initQueue(const size_t cap, const size_t mult) {+    new (&this->dstride_) Atom<int>(this->computeStride(cap));+    Slot* slots = new Slot[cap + 2 * this->kSlotPadding];+    new (&this->dslots_) Atom<Slot*>(slots);+    this->dstate_.store(0);+    this->dcapacity_.store(cap);+    dmult_ = mult;+    size_t maxClosed = 0;+    for (size_t expanded = cap; expanded < this->capacity_; expanded *= mult) {+      ++maxClosed;+    }+    closed_ = (maxClosed > 0) ? new ClosedArray[maxClosed] : nullptr;+  }++  bool tryObtainReadyPushTicket(+      uint64_t& ticket, Slot*& slots, size_t& cap, int& stride) noexcept {+    uint64_t state;+    do {+      ticket = this->pushTicket_.load(std::memory_order_acquire); // A+      if (!trySeqlockReadSection(state, slots, cap, stride)) {+        asm_volatile_pause();+        continue;+      }++      // If there was an expansion with offset greater than this ticket,+      // adjust accordingly+      uint64_t offset;+      maybeUpdateFromClosed(state, ticket, offset, slots, cap, stride);++      if (slots[this->idx((ticket - offset), cap, stride)].mayEnqueue(+              this->turn(ticket - offset, cap))) {+        // A slot is ready.+        if (this->pushTicket_.compare_exchange_strong(ticket, ticket + 1)) {+          // Adjust ticket+          ticket -= offset;+          return true;+        } else {+          continue;+        }+      } else {+        if (ticket != this->pushTicket_.load(std::memory_order_relaxed)) { // B+          // Try again. Ticket changed.+          continue;+        }+        // Likely to block.+        // Try to expand unless the ticket is for a closed array+        if (offset == getOffset(state)) {+          if (tryExpand(state, cap)) {+            // This or another thread started an expansion. Get up-to-date info.+            continue;+          }+        }+        return false;+      }+    } while (true);+  }++  bool tryObtainPromisedPushTicket(+      uint64_t& ticket, Slot*& slots, size_t& cap, int& stride) noexcept {+    uint64_t state;+    do {+      ticket = this->pushTicket_.load(std::memory_order_acquire);+      auto numPops = this->popTicket_.load(std::memory_order_acquire);+      if (!trySeqlockReadSection(state, slots, cap, stride)) {+        asm_volatile_pause();+        continue;+      }++      const auto curCap = cap;+      // If there was an expansion with offset greater than this ticket,+      // adjust accordingly+      uint64_t offset;+      maybeUpdateFromClosed(state, ticket, offset, slots, cap, stride);++      int64_t n = ticket - numPops;++      if (n >= static_cast<ssize_t>(cap)) {+        if ((cap == curCap) && tryExpand(state, cap)) {+          // This or another thread started an expansion. Start over.+          continue;+        }+        // Can't expand.+        ticket -= offset;+        return false;+      }++      if (this->pushTicket_.compare_exchange_strong(ticket, ticket + 1)) {+        // Adjust ticket+        ticket -= offset;+        return true;+      }+    } while (true);+  }++  bool tryObtainReadyPopTicket(+      uint64_t& ticket, Slot*& slots, size_t& cap, int& stride) noexcept {+    uint64_t state;+    do {+      ticket = this->popTicket_.load(std::memory_order_relaxed);+      if (!trySeqlockReadSection(state, slots, cap, stride)) {+        asm_volatile_pause();+        continue;+      }++      // If there was an expansion after the corresponding push ticket+      // was issued, adjust accordingly+      uint64_t offset;+      maybeUpdateFromClosed(state, ticket, offset, slots, cap, stride);++      if (slots[this->idx((ticket - offset), cap, stride)].mayDequeue(+              this->turn(ticket - offset, cap))) {+        if (this->popTicket_.compare_exchange_strong(ticket, ticket + 1)) {+          // Adjust ticket+          ticket -= offset;+          return true;+        }+      } else {+        return false;+      }+    } while (true);+  }++  bool tryObtainPromisedPopTicket(+      uint64_t& ticket, Slot*& slots, size_t& cap, int& stride) noexcept {+    uint64_t state;+    do {+      ticket = this->popTicket_.load(std::memory_order_acquire);+      auto numPushes = this->pushTicket_.load(std::memory_order_acquire);+      if (!trySeqlockReadSection(state, slots, cap, stride)) {+        asm_volatile_pause();+        continue;+      }++      uint64_t offset;+      // If there was an expansion after the corresponding push+      // ticket was issued, adjust accordingly+      maybeUpdateFromClosed(state, ticket, offset, slots, cap, stride);++      if (ticket >= numPushes) {+        ticket -= offset;+        return false;+      }+      if (this->popTicket_.compare_exchange_strong(ticket, ticket + 1)) {+        ticket -= offset;+        return true;+      }+    } while (true);+  }++  /// Enqueues an element with a specific ticket number+  template <typename... Args>+  void enqueueWithTicket(const uint64_t ticket, Args&&... args) noexcept {+    Slot* slots;+    size_t cap;+    int stride;+    uint64_t state;+    uint64_t offset;++    while (!trySeqlockReadSection(state, slots, cap, stride)) {+    }++    // If there was an expansion after this ticket was issued, adjust+    // accordingly+    maybeUpdateFromClosed(state, ticket, offset, slots, cap, stride);++    this->enqueueWithTicketBase(+        ticket - offset, slots, cap, stride, std::forward<Args>(args)...);+  }++  uint64_t getOffset(const uint64_t state) const noexcept {+    return state >> kSeqlockBits;+  }++  int getNumClosed(const uint64_t state) const noexcept {+    return (state & ((1 << kSeqlockBits) - 1)) >> 1;+  }++  /// Try to expand the queue. Returns true if this expansion was+  /// successful or a concurrent expansion is in progress. Returns+  /// false if the queue has reached its maximum capacity or+  /// allocation has failed.+  bool tryExpand(const uint64_t state, const size_t cap) noexcept {+    if (cap == this->capacity_) {+      return false;+    }+    // Acquire seqlock+    uint64_t oldval = state;+    assert((state & 1) == 0);+    if (this->dstate_.compare_exchange_strong(oldval, state + 1)) {+      assert(cap == this->dcapacity_.load());+      uint64_t ticket =+          1 + std::max(this->pushTicket_.load(), this->popTicket_.load());+      size_t newCapacity = std::min(dmult_ * cap, this->capacity_);+      Slot* newSlots =+          new (std::nothrow) Slot[newCapacity + 2 * this->kSlotPadding];+      if (newSlots == nullptr) {+        // Expansion failed. Restore the seqlock+        this->dstate_.store(state);+        return false;+      }+      // Successful expansion+      // calculate the current ticket offset+      uint64_t offset = getOffset(state);+      // calculate index in closed array+      int index = getNumClosed(state);+      assert((index << 1) < (1 << kSeqlockBits));+      // fill the info for the closed slots array+      closed_[index].offset_ = offset;+      closed_[index].slots_ = this->dslots_.load();+      closed_[index].capacity_ = cap;+      closed_[index].stride_ = this->dstride_.load();+      // update the new slots array info+      this->dslots_.store(newSlots);+      this->dcapacity_.store(newCapacity);+      this->dstride_.store(this->computeStride(newCapacity));+      // Release the seqlock and record the new ticket offset+      this->dstate_.store((ticket << kSeqlockBits) + (2 * (index + 1)));+      return true;+    } else { // failed to acquire seqlock+      // Someone acquired the seqlock. Go back to the caller and get+      // up-to-date info.+      return true;+    }+  }++  /// Seqlock read-only section+  bool trySeqlockReadSection(+      uint64_t& state, Slot*& slots, size_t& cap, int& stride) noexcept {+    state = this->dstate_.load(std::memory_order_acquire);+    if (state & 1) {+      // Locked.+      return false;+    }+    // Start read-only section.+    slots = this->dslots_.load(std::memory_order_relaxed);+    cap = this->dcapacity_.load(std::memory_order_relaxed);+    stride = this->dstride_.load(std::memory_order_relaxed);+    // End of read-only section. Validate seqlock.+    std::atomic_thread_fence(std::memory_order_acquire);+    return (state == this->dstate_.load(std::memory_order_relaxed));+  }++  /// If there was an expansion after ticket was issued, update local variables+  /// of the lagging operation using the most recent closed array with+  /// offset <= ticket and return true. Otherwise, return false;+  bool maybeUpdateFromClosed(+      const uint64_t state,+      const uint64_t ticket,+      uint64_t& offset,+      Slot*& slots,+      size_t& cap,+      int& stride) noexcept {+    offset = getOffset(state);+    if (ticket >= offset) {+      return false;+    }+    for (int i = getNumClosed(state) - 1; i >= 0; --i) {+      offset = closed_[i].offset_;+      if (offset <= ticket) {+        slots = closed_[i].slots_;+        cap = closed_[i].capacity_;+        stride = closed_[i].stride_;+        return true;+      }+    }+    // A closed array with offset <= ticket should have been found+    assert(false);+    return false;+  }+};++namespace detail {++/// CRTP specialization of MPMCQueueBase+template <+    template <+        typename T,+        template <typename>+        class Atom,+        bool Dynamic,+        class Allocator>++    class Derived,+    typename T,+    template <typename>+    class Atom,+    bool Dynamic,+    class Allocator>+class MPMCQueueBase<Derived<T, Atom, Dynamic, Allocator>> {+  // Note: Using CRTP static casts in several functions of this base+  // template instead of making called functions virtual or duplicating+  // the code of calling functions in the derived partially specialized+  // template+  using DerivedType = Derived<T, Atom, Dynamic, Allocator>;++  static_assert(+      std::is_nothrow_constructible<T, T&&>::value ||+          folly::IsRelocatable<T>::value,+      "T must be relocatable or have a noexcept move constructor");++ public:+  typedef T value_type;++  using Slot = detail::SingleElementQueue<T, Atom>;++  explicit MPMCQueueBase(size_t queueCapacity)+      : capacity_(queueCapacity),+        dstate_(0),+        dcapacity_(0),+        pushTicket_(0),+        popTicket_(0),+        pushSpinCutoff_(0),+        popSpinCutoff_(0) {+    if (queueCapacity == 0) {+      throw std::invalid_argument(+          "MPMCQueue with explicit capacity 0 is impossible"+          // Stride computation in derived classes would sigfpe if capacity is 0+      );+    }++    // ideally this would be a static assert, but g++ doesn't allow it+    assert(+        alignof(MPMCQueue<T, Atom>) >= hardware_destructive_interference_size);+    assert(+        static_cast<uint8_t*>(static_cast<void*>(&popTicket_)) -+            static_cast<uint8_t*>(static_cast<void*>(&pushTicket_)) >=+        static_cast<ptrdiff_t>(hardware_destructive_interference_size));+  }++  /// A default-constructed queue is useful because a usable (non-zero+  /// capacity) queue can be moved onto it or swapped with it+  MPMCQueueBase() noexcept+      : capacity_(0),+        slots_(nullptr),+        stride_(0),+        dstate_(0),+        dcapacity_(0),+        pushTicket_(0),+        popTicket_(0),+        pushSpinCutoff_(0),+        popSpinCutoff_(0) {}++  /// IMPORTANT: The move constructor is here to make it easier to perform+  /// the initialization phase, it is not safe to use when there are any+  /// concurrent accesses (this is not checked).+  MPMCQueueBase(MPMCQueueBase<DerivedType>&& rhs) noexcept+      : capacity_(rhs.capacity_),+        slots_(rhs.slots_),+        stride_(rhs.stride_),+        dstate_(rhs.dstate_.load(std::memory_order_relaxed)),+        dcapacity_(rhs.dcapacity_.load(std::memory_order_relaxed)),+        pushTicket_(rhs.pushTicket_.load(std::memory_order_relaxed)),+        popTicket_(rhs.popTicket_.load(std::memory_order_relaxed)),+        pushSpinCutoff_(rhs.pushSpinCutoff_.load(std::memory_order_relaxed)),+        popSpinCutoff_(rhs.popSpinCutoff_.load(std::memory_order_relaxed)) {+    // relaxed ops are okay for the previous reads, since rhs queue can't+    // be in concurrent use++    // zero out rhs+    rhs.capacity_ = 0;+    rhs.slots_ = nullptr;+    rhs.stride_ = 0;+    rhs.dstate_.store(0, std::memory_order_relaxed);+    rhs.dcapacity_.store(0, std::memory_order_relaxed);+    rhs.pushTicket_.store(0, std::memory_order_relaxed);+    rhs.popTicket_.store(0, std::memory_order_relaxed);+    rhs.pushSpinCutoff_.store(0, std::memory_order_relaxed);+    rhs.popSpinCutoff_.store(0, std::memory_order_relaxed);+  }++  MPMCQueueBase& operator=(MPMCQueueBase&& rhs) = delete;+  MPMCQueueBase(const MPMCQueueBase&) = delete;+  MPMCQueueBase& operator=(const MPMCQueueBase&) = delete;++  /// MPMCQueue can only be safely destroyed when there are no+  /// pending enqueuers or dequeuers (this is not checked).+  ~MPMCQueueBase() { delete[] slots_; }++  /// Returns the number of writes (including threads that are blocked waiting+  /// to write) minus the number of reads (including threads that are blocked+  /// waiting to read). So effectively, it becomes:+  /// elements in queue + pending(calls to write) - pending(calls to read).+  /// If nothing is pending, then the method returns the actual number of+  /// elements in the queue.+  /// The returned value can be negative if there are no writers and the queue+  /// is empty, but there is one reader that is blocked waiting to read (in+  /// which case, the returned size will be -1).+  ssize_t size() const noexcept {+    // since both pushes and pops increase monotonically, we can get a+    // consistent snapshot either by bracketing a read of popTicket_ with+    // two reads of pushTicket_ that return the same value, or the other+    // way around.  We maximize our chances by alternately attempting+    // both bracketings.+    uint64_t pushes = pushTicket_.load(std::memory_order_acquire); // A+    uint64_t pops = popTicket_.load(std::memory_order_acquire); // B+    while (true) {+      uint64_t nextPushes = pushTicket_.load(std::memory_order_acquire); // C+      if (pushes == nextPushes) {+        // pushTicket_ didn't change from A (or the previous C) to C,+        // so we can linearize at B (or D)+        return ssize_t(pushes - pops);+      }+      pushes = nextPushes;+      uint64_t nextPops = popTicket_.load(std::memory_order_acquire); // D+      if (pops == nextPops) {+        // popTicket_ didn't chance from B (or the previous D), so we+        // can linearize at C+        return ssize_t(pushes - pops);+      }+      pops = nextPops;+    }+  }++  /// Returns true if there are no items available for dequeue+  bool isEmpty() const noexcept { return size() <= 0; }++  /// Returns true if there is currently no empty space to enqueue+  bool isFull() const noexcept {+    // careful with signed -> unsigned promotion, since size can be negative+    return size() >= static_cast<ssize_t>(capacity_);+  }++  /// Returns is a guess at size() for contexts that don't need a precise+  /// value, such as stats. More specifically, it returns the number of writes+  /// minus the number of reads, but after reading the number of writes, more+  /// writers could have came before the number of reads was sampled,+  /// and this method doesn't protect against such case.+  /// The returned value can be negative.+  ssize_t sizeGuess() const noexcept { return writeCount() - readCount(); }++  /// Doesn't change+  size_t capacity() const noexcept { return capacity_; }++  /// Doesn't change for non-dynamic+  size_t allocatedCapacity() const noexcept { return capacity_; }++  /// Returns the total number of calls to blockingWrite or successful+  /// calls to write, including those blockingWrite calls that are+  /// currently blocking+  uint64_t writeCount() const noexcept {+    return pushTicket_.load(std::memory_order_acquire);+  }++  /// Returns the total number of calls to blockingRead or successful+  /// calls to read, including those blockingRead calls that are currently+  /// blocking+  uint64_t readCount() const noexcept {+    return popTicket_.load(std::memory_order_acquire);+  }++  /// Enqueues a T constructed from args, blocking until space is+  /// available.  Note that this method signature allows enqueue via+  /// move, if args is a T rvalue, via copy, if args is a T lvalue, or+  /// via emplacement if args is an initializer list that can be passed+  /// to a T constructor.+  template <typename... Args>+  void blockingWrite(Args&&... args) noexcept {+    enqueueWithTicketBase(+        pushTicket_++, slots_, capacity_, stride_, std::forward<Args>(args)...);+  }++  /// If an item can be enqueued with no blocking, does so and returns+  /// true, otherwise returns false.  This method is similar to+  /// writeIfNotFull, but if you don't have a specific need for that+  /// method you should use this one.+  ///+  /// One of the common usages of this method is to enqueue via the+  /// move constructor, something like q.write(std::move(x)).  If write+  /// returns false because the queue is full then x has not actually been+  /// consumed, which looks strange.  To understand why it is actually okay+  /// to use x afterward, remember that std::move is just a typecast that+  /// provides an rvalue reference that enables use of a move constructor+  /// or operator.  std::move doesn't actually move anything.  It could+  /// more accurately be called std::rvalue_cast or std::move_permission.+  template <typename... Args>+  bool write(Args&&... args) noexcept {+    uint64_t ticket;+    Slot* slots;+    size_t cap;+    int stride;+    if (static_cast<DerivedType*>(this)->tryObtainReadyPushTicket(++            ticket, slots, cap, stride)) {+      // we have pre-validated that the ticket won't block+      enqueueWithTicketBase(+          ticket, slots, cap, stride, std::forward<Args>(args)...);+      return true;+    } else {+      return false;+    }+  }++  template <class Clock, typename... Args>+  bool tryWriteUntil(+      const std::chrono::time_point<Clock>& when, Args&&... args) noexcept {+    uint64_t ticket;+    Slot* slots;+    size_t cap;+    int stride;+    if (tryObtainPromisedPushTicketUntil(ticket, slots, cap, stride, when)) {+      // we have pre-validated that the ticket won't block, or rather that+      // it won't block longer than it takes another thread to dequeue an+      // element from the slot it identifies.+      enqueueWithTicketBase(+          ticket, slots, cap, stride, std::forward<Args>(args)...);+      return true;+    } else {+      return false;+    }+  }++  /// If the queue is not full, enqueues and returns true, otherwise+  /// returns false.  Unlike write this method can be blocked by another+  /// thread, specifically a read that has linearized (been assigned+  /// a ticket) but not yet completed.  If you don't really need this+  /// function you should probably use write.+  ///+  /// MPMCQueue isn't lock-free, so just because a read operation has+  /// linearized (and isFull is false) doesn't mean that space has been+  /// made available for another write.  In this situation write will+  /// return false, but writeIfNotFull will wait for the dequeue to finish.+  /// This method is required if you are composing queues and managing+  /// your own wakeup, because it guarantees that after every successful+  /// write a readIfNotEmpty will succeed.+  template <typename... Args>+  bool writeIfNotFull(Args&&... args) noexcept {+    uint64_t ticket;+    Slot* slots;+    size_t cap;+    int stride;+    if (static_cast<DerivedType*>(this)->tryObtainPromisedPushTicket(+            ticket, slots, cap, stride)) {+      // some other thread is already dequeuing the slot into which we+      // are going to enqueue, but we might have to wait for them to finish+      enqueueWithTicketBase(+          ticket, slots, cap, stride, std::forward<Args>(args)...);+      return true;+    } else {+      return false;+    }+  }++  /// Moves a dequeued element onto elem, blocking until an element+  /// is available+  void blockingRead(T& elem) noexcept {+    uint64_t ticket;+    static_cast<DerivedType*>(this)->blockingReadWithTicket(ticket, elem);+  }++  /// Same as blockingRead() but also records the ticket nunmer+  void blockingReadWithTicket(uint64_t& ticket, T& elem) noexcept {+    assert(capacity_ != 0);+    ticket = popTicket_++;+    dequeueWithTicketBase(ticket, slots_, capacity_, stride_, elem);+  }++  /// If an item can be dequeued with no blocking, does so and returns true,+  /// otherwise returns false.+  ///+  /// Note that if the matching write is still in progress, this may return+  /// false even if writes that have been started later have already+  /// completed. If an external mechanism is used for counting completed writes+  /// (for example a semaphore) to determine when an element is ready to+  /// dequeue, readIfNotEmpty() should be used instead, which will wait for the+  /// write in progress.+  bool read(T& elem) noexcept {+    uint64_t ticket;+    return readAndGetTicket(ticket, elem);+  }++  /// Same as read() but also records the ticket nunmer+  bool readAndGetTicket(uint64_t& ticket, T& elem) noexcept {+    Slot* slots;+    size_t cap;+    int stride;+    if (static_cast<DerivedType*>(this)->tryObtainReadyPopTicket(++            ticket, slots, cap, stride)) {+      // the ticket has been pre-validated to not block+      dequeueWithTicketBase(ticket, slots, cap, stride, elem);+      return true;+    } else {+      return false;+    }+  }++  template <class Clock, typename... Args>+  bool tryReadUntil(+      const std::chrono::time_point<Clock>& when, T& elem) noexcept {+    uint64_t ticket;+    Slot* slots;+    size_t cap;+    int stride;+    if (tryObtainPromisedPopTicketUntil(ticket, slots, cap, stride, when)) {+      // we have pre-validated that the ticket won't block, or rather that+      // it won't block longer than it takes another thread to enqueue an+      // element on the slot it identifies.+      dequeueWithTicketBase(ticket, slots, cap, stride, elem);+      return true;+    } else {+      return false;+    }+  }++  /// If the queue is not empty, dequeues and returns true, otherwise+  /// returns false.  If the matching write is still in progress then this+  /// method may block waiting for it.  If you don't rely on being able+  /// to dequeue (such as by counting completed write) then you should+  /// prefer read.+  bool readIfNotEmpty(T& elem) noexcept {+    uint64_t ticket;+    Slot* slots;+    size_t cap;+    int stride;+    if (static_cast<DerivedType*>(this)->tryObtainPromisedPopTicket(+            ticket, slots, cap, stride)) {+      // the matching enqueue already has a ticket, but might not be done+      dequeueWithTicketBase(ticket, slots, cap, stride, elem);+      return true;+    } else {+      return false;+    }+  }++ protected:+  enum {+    /// Once every kAdaptationFreq we will spin longer, to try to estimate+    /// the proper spin backoff+    kAdaptationFreq = 128,++    /// To avoid false sharing in slots_ with neighboring memory+    /// allocations, we pad it with this many SingleElementQueue-s at+    /// each end+    kSlotPadding =+        (hardware_destructive_interference_size - 1) / sizeof(Slot) + 1+  };++  /// The maximum number of items in the queue at once+  alignas(hardware_destructive_interference_size) size_t capacity_;++  /// Anonymous union for use when Dynamic = false and true, respectively+  union {+    /// An array of capacity_ SingleElementQueue-s, each of which holds+    /// either 0 or 1 item.  We over-allocate by 2 * kSlotPadding and don't+    /// touch the slots at either end, to avoid false sharing+    Slot* slots_;+    /// Current dynamic slots array of dcapacity_ SingleElementQueue-s+    Atom<Slot*> dslots_;+  };++  /// Anonymous union for use when Dynamic = false and true, respectively+  union {+    /// The number of slots_ indices that we advance for each ticket, to+    /// avoid false sharing.  Ideally slots_[i] and slots_[i + stride_]+    /// aren't on the same cache line+    int stride_;+    /// Current stride+    Atom<int> dstride_;+  };++  /// The following two members are used by dynamic MPMCQueue.+  /// Ideally they should be in MPMCQueue<T,Atom,true>, but we get+  /// better cache locality if they are in the same cache line as+  /// dslots_ and dstride_.+  ///+  /// Dynamic state. A packed seqlock and ticket offset+  Atom<uint64_t> dstate_;+  /// Dynamic capacity+  Atom<size_t> dcapacity_;++  /// Enqueuers get tickets from here+  alignas(hardware_destructive_interference_size) Atom<uint64_t> pushTicket_;++  /// Dequeuers get tickets from here+  alignas(hardware_destructive_interference_size) Atom<uint64_t> popTicket_;++  /// This is how many times we will spin before using FUTEX_WAIT when+  /// the queue is full on enqueue, adaptively computed by occasionally+  /// spinning for longer and smoothing with an exponential moving average+  alignas(+      hardware_destructive_interference_size) Atom<uint32_t> pushSpinCutoff_;++  /// The adaptive spin cutoff when the queue is empty on dequeue+  alignas(hardware_destructive_interference_size) Atom<uint32_t> popSpinCutoff_;++  /// Alignment doesn't prevent false sharing at the end of the struct,+  /// so fill out the last cache line+  char pad_[hardware_destructive_interference_size - sizeof(Atom<uint32_t>)];++  /// We assign tickets in increasing order, but we don't want to+  /// access neighboring elements of slots_ because that will lead to+  /// false sharing (multiple cores accessing the same cache line even+  /// though they aren't accessing the same bytes in that cache line).+  /// To avoid this we advance by stride slots per ticket.+  ///+  /// We need gcd(capacity, stride) to be 1 so that we will use all+  /// of the slots.  We ensure this by only considering prime strides,+  /// which either have no common divisors with capacity or else have+  /// a zero remainder after dividing by capacity.  That is sufficient+  /// to guarantee correctness, but we also want to actually spread the+  /// accesses away from each other to avoid false sharing (consider a+  /// stride of 7 with a capacity of 8).  To that end we try a few taking+  /// care to observe that advancing by -1 is as bad as advancing by 1+  /// when in comes to false sharing.+  ///+  /// The simple way to avoid false sharing would be to pad each+  /// SingleElementQueue, but since we have capacity_ of them that could+  /// waste a lot of space.+  static int computeStride(size_t capacity) noexcept {+    static const int smallPrimes[] = {2, 3, 5, 7, 11, 13, 17, 19, 23};++    int bestStride = 1;+    size_t bestSep = 1;+    for (int stride : smallPrimes) {+      if ((stride % capacity) == 0 || (capacity % stride) == 0) {+        continue;+      }+      size_t sep = stride % capacity;+      sep = std::min(sep, capacity - sep);+      if (sep > bestSep) {+        bestStride = stride;+        bestSep = sep;+      }+    }+    return bestStride;+  }++  /// Returns the index into slots_ that should be used when enqueuing or+  /// dequeuing with the specified ticket+  size_t idx(uint64_t ticket, size_t cap, int stride) noexcept {+    return ((ticket * stride) % cap) + kSlotPadding;+  }++  /// Maps an enqueue or dequeue ticket to the turn should be used at the+  /// corresponding SingleElementQueue+  uint32_t turn(uint64_t ticket, size_t cap) noexcept {+    assert(cap != 0);+    return uint32_t(ticket / cap);+  }++  /// Tries to obtain a push ticket for which SingleElementQueue::enqueue+  /// won't block.  Returns true on immediate success, false on immediate+  /// failure.+  bool tryObtainReadyPushTicket(+      uint64_t& ticket, Slot*& slots, size_t& cap, int& stride) noexcept {+    ticket = pushTicket_.load(std::memory_order_acquire); // A+    slots = slots_;+    cap = capacity_;+    stride = stride_;+    while (true) {+      if (!slots[idx(ticket, cap, stride)].mayEnqueue(turn(ticket, cap))) {+        // if we call enqueue(ticket, ...) on the SingleElementQueue+        // right now it would block, but this might no longer be the next+        // ticket.  We can increase the chance of tryEnqueue success under+        // contention (without blocking) by rechecking the ticket dispenser+        auto prev = ticket;+        ticket = pushTicket_.load(std::memory_order_acquire); // B+        if (prev == ticket) {+          // mayEnqueue was bracketed by two reads (A or prev B or prev+          // failing CAS to B), so we are definitely unable to enqueue+          return false;+        }+      } else {+        // we will bracket the mayEnqueue check with a read (A or prev B+        // or prev failing CAS) and the following CAS.  If the CAS fails+        // it will effect a load of pushTicket_+        if (pushTicket_.compare_exchange_strong(ticket, ticket + 1)) {+          return true;+        }+      }+    }+  }++  /// Tries until when to obtain a push ticket for which+  /// SingleElementQueue::enqueue  won't block.  Returns true on success, false+  /// on failure.+  /// ticket is filled on success AND failure.+  template <class Clock>+  bool tryObtainPromisedPushTicketUntil(+      uint64_t& ticket,+      Slot*& slots,+      size_t& cap,+      int& stride,+      const std::chrono::time_point<Clock>& when) noexcept {+    bool deadlineReached = false;+    while (!deadlineReached) {+      if (static_cast<DerivedType*>(this)->tryObtainPromisedPushTicket(+              ticket, slots, cap, stride)) {+        return true;+      }+      // ticket is a blocking ticket until the preceding ticket has been+      // processed: wait until this ticket's turn arrives. We have not reserved+      // this ticket so we will have to re-attempt to get a non-blocking ticket+      // if we wake up before we time-out.+      deadlineReached =+          !slots[idx(ticket, cap, stride)].tryWaitForEnqueueTurnUntil(+              turn(ticket, cap),+              pushSpinCutoff_,+              (ticket % kAdaptationFreq) == 0,+              when);+    }+    return false;+  }++  /// Tries to obtain a push ticket which can be satisfied if all+  /// in-progress pops complete.  This function does not block, but+  /// blocking may be required when using the returned ticket if some+  /// other thread's pop is still in progress (ticket has been granted but+  /// pop has not yet completed).+  bool tryObtainPromisedPushTicket(+      uint64_t& ticket, Slot*& slots, size_t& cap, int& stride) noexcept {+    auto numPushes = pushTicket_.load(std::memory_order_acquire); // A+    slots = slots_;+    cap = capacity_;+    stride = stride_;+    while (true) {+      ticket = numPushes;+      const auto numPops = popTicket_.load(std::memory_order_acquire); // B+      // n will be negative if pops are pending+      const int64_t n = int64_t(numPushes - numPops);+      if (n >= static_cast<ssize_t>(capacity_)) {+        // Full, linearize at B.  We don't need to recheck the read we+        // performed at A, because if numPushes was stale at B then the+        // real numPushes value is even worse+        return false;+      }+      if (pushTicket_.compare_exchange_strong(numPushes, numPushes + 1)) {+        return true;+      }+    }+  }++  /// Tries to obtain a pop ticket for which SingleElementQueue::dequeue+  /// won't block.  Returns true on immediate success, false on immediate+  /// failure.+  bool tryObtainReadyPopTicket(+      uint64_t& ticket, Slot*& slots, size_t& cap, int& stride) noexcept {+    ticket = popTicket_.load(std::memory_order_acquire);+    slots = slots_;+    cap = capacity_;+    stride = stride_;+    while (true) {+      if (!slots[idx(ticket, cap, stride)].mayDequeue(turn(ticket, cap))) {+        auto prev = ticket;+        ticket = popTicket_.load(std::memory_order_acquire);+        if (prev == ticket) {+          return false;+        }+      } else {+        if (popTicket_.compare_exchange_strong(ticket, ticket + 1)) {+          return true;+        }+      }+    }+  }++  /// Tries until when to obtain a pop ticket for which+  /// SingleElementQueue::dequeue won't block.  Returns true on success, false+  /// on failure.+  /// ticket is filled on success AND failure.+  template <class Clock>+  bool tryObtainPromisedPopTicketUntil(+      uint64_t& ticket,+      Slot*& slots,+      size_t& cap,+      int& stride,+      const std::chrono::time_point<Clock>& when) noexcept {+    bool deadlineReached = false;+    while (!deadlineReached) {+      if (static_cast<DerivedType*>(this)->tryObtainPromisedPopTicket(+              ticket, slots, cap, stride)) {+        return true;+      }+      // ticket is a blocking ticket until the preceding ticket has been+      // processed: wait until this ticket's turn arrives. We have not reserved+      // this ticket so we will have to re-attempt to get a non-blocking ticket+      // if we wake up before we time-out.+      deadlineReached =+          !slots[idx(ticket, cap, stride)].tryWaitForDequeueTurnUntil(+              turn(ticket, cap),+              pushSpinCutoff_,+              (ticket % kAdaptationFreq) == 0,+              when);+    }+    return false;+  }++  /// Similar to tryObtainReadyPopTicket, but returns a pop ticket whose+  /// corresponding push ticket has already been handed out, rather than+  /// returning one whose corresponding push ticket has already been+  /// completed.  This means that there is a possibility that the caller+  /// will block when using the ticket, but it allows the user to rely on+  /// the fact that if enqueue has succeeded, tryObtainPromisedPopTicket+  /// will return true.  The "try" part of this is that we won't have+  /// to block waiting for someone to call enqueue, although we might+  /// have to block waiting for them to finish executing code inside the+  /// MPMCQueue itself.+  bool tryObtainPromisedPopTicket(+      uint64_t& ticket, Slot*& slots, size_t& cap, int& stride) noexcept {+    auto numPops = popTicket_.load(std::memory_order_acquire); // A+    slots = slots_;+    cap = capacity_;+    stride = stride_;+    while (true) {+      ticket = numPops;+      const auto numPushes = pushTicket_.load(std::memory_order_acquire); // B+      if (numPops >= numPushes) {+        // Empty, or empty with pending pops.  Linearize at B.  We don't+        // need to recheck the read we performed at A, because if numPops+        // is stale then the fresh value is larger and the >= is still true+        return false;+      }+      if (popTicket_.compare_exchange_strong(numPops, numPops + 1)) {+        return true;+      }+    }+  }++  // Given a ticket, constructs an enqueued item using args+  template <typename... Args>+  void enqueueWithTicketBase(+      uint64_t ticket,+      Slot* slots,+      size_t cap,+      int stride,+      Args&&... args) noexcept {+    slots[idx(ticket, cap, stride)].enqueue(+        turn(ticket, cap),+        pushSpinCutoff_,+        (ticket % kAdaptationFreq) == 0,+        std::forward<Args>(args)...);+  }++  // To support tracking ticket numbers in MPMCPipelineStageImpl+  template <typename... Args>+  void enqueueWithTicket(uint64_t ticket, Args&&... args) noexcept {+    enqueueWithTicketBase(+        ticket, slots_, capacity_, stride_, std::forward<Args>(args)...);+  }++  // Given a ticket, dequeues the corresponding element+  void dequeueWithTicketBase(+      uint64_t ticket, Slot* slots, size_t cap, int stride, T& elem) noexcept {+    assert(cap != 0);+    slots[idx(ticket, cap, stride)].dequeue(+        turn(ticket, cap),+        popSpinCutoff_,+        (ticket % kAdaptationFreq) == 0,+        elem);+  }+};++/// SingleElementQueue implements a blocking queue that holds at most one+/// item, and that requires its users to assign incrementing identifiers+/// (turns) to each enqueue and dequeue operation.  Note that the turns+/// used by SingleElementQueue are doubled inside the TurnSequencer+template <typename T, template <typename> class Atom>+struct SingleElementQueue {+  ~SingleElementQueue() noexcept {+    if ((sequencer_.uncompletedTurnLSB() & 1) == 1) {+      // we are pending a dequeue, so we have a constructed item+      destroyContents();+    }+  }++  /// enqueue using in-place noexcept construction+  template <+      typename... Args,+      typename = typename std::enable_if<+          std::is_nothrow_constructible<T, Args...>::value>::type>+  void enqueue(+      const uint32_t turn,+      Atom<uint32_t>& spinCutoff,+      const bool updateSpinCutoff,+      Args&&... args) noexcept {+    sequencer_.waitForTurn(turn * 2, spinCutoff, updateSpinCutoff);+    new (&contents_) T(std::forward<Args>(args)...);+    sequencer_.completeTurn(turn * 2);+  }++  /// enqueue using move construction, either real (if+  /// is_nothrow_move_constructible) or simulated using relocation and+  /// default construction (if IsRelocatable and is_nothrow_constructible)+  template <+      typename = typename std::enable_if<+          (folly::IsRelocatable<T>::value &&+           std::is_nothrow_constructible<T>::value) ||+          std::is_nothrow_constructible<T, T&&>::value>::type>+  void enqueue(+      const uint32_t turn,+      Atom<uint32_t>& spinCutoff,+      const bool updateSpinCutoff,+      T&& goner) noexcept {+    enqueueImpl(+        turn,+        spinCutoff,+        updateSpinCutoff,+        std::move(goner),+        typename std::conditional<+            std::is_nothrow_constructible<T, T&&>::value,+            ImplByMove,+            ImplByRelocation>::type());+  }++  /// Waits until either:+  /// 1: the dequeue turn preceding the given enqueue turn has arrived+  /// 2: the given deadline has arrived+  /// Case 1 returns true, case 2 returns false.+  template <class Clock>+  bool tryWaitForEnqueueTurnUntil(+      const uint32_t turn,+      Atom<uint32_t>& spinCutoff,+      const bool updateSpinCutoff,+      const std::chrono::time_point<Clock>& when) noexcept {+    return sequencer_.tryWaitForTurn(+               turn * 2, spinCutoff, updateSpinCutoff, &when) !=+        TurnSequencer<Atom>::TryWaitResult::TIMEDOUT;+  }++  bool mayEnqueue(const uint32_t turn) const noexcept {+    return sequencer_.isTurn(turn * 2);+  }++  void dequeue(+      uint32_t turn,+      Atom<uint32_t>& spinCutoff,+      const bool updateSpinCutoff,+      T& elem) noexcept {+    dequeueImpl(+        turn,+        spinCutoff,+        updateSpinCutoff,+        elem,+        typename std::conditional<+            folly::IsRelocatable<T>::value,+            ImplByRelocation,+            ImplByMove>::type());+  }++  /// Waits until either:+  /// 1: the enqueue turn preceding the given dequeue turn has arrived+  /// 2: the given deadline has arrived+  /// Case 1 returns true, case 2 returns false.+  template <class Clock>+  bool tryWaitForDequeueTurnUntil(+      const uint32_t turn,+      Atom<uint32_t>& spinCutoff,+      const bool updateSpinCutoff,+      const std::chrono::time_point<Clock>& when) noexcept {+    return sequencer_.tryWaitForTurn(+               turn * 2 + 1, spinCutoff, updateSpinCutoff, &when) !=+        TurnSequencer<Atom>::TryWaitResult::TIMEDOUT;+  }++  bool mayDequeue(const uint32_t turn) const noexcept {+    return sequencer_.isTurn(turn * 2 + 1);+  }++ private:+  /// Storage for a T constructed with placement new+  aligned_storage_for_t<T> contents_;++  /// Even turns are pushes, odd turns are pops+  TurnSequencer<Atom> sequencer_;++  T* ptr() noexcept { return static_cast<T*>(static_cast<void*>(&contents_)); }++  void destroyContents() noexcept {+    try {+      ptr()->~T();+    } catch (...) {+      // g++ doesn't seem to have std::is_nothrow_destructible yet+    }+    if (kIsDebug) {+      memset(&contents_, 'Q', sizeof(T));+    }+  }++  /// Tag classes for dispatching to enqueue/dequeue implementation.+  struct ImplByRelocation {};+  struct ImplByMove {};++  /// enqueue using nothrow move construction.+  void enqueueImpl(+      const uint32_t turn,+      Atom<uint32_t>& spinCutoff,+      const bool updateSpinCutoff,+      T&& goner,+      ImplByMove) noexcept {+    sequencer_.waitForTurn(turn * 2, spinCutoff, updateSpinCutoff);+    new (&contents_) T(std::move(goner));+    sequencer_.completeTurn(turn * 2);+  }++  /// enqueue by simulating nothrow move with relocation, followed by+  /// default construction to a noexcept relocation.+  void enqueueImpl(+      const uint32_t turn,+      Atom<uint32_t>& spinCutoff,+      const bool updateSpinCutoff,+      T&& goner,+      ImplByRelocation) noexcept {+    sequencer_.waitForTurn(turn * 2, spinCutoff, updateSpinCutoff);+    memcpy(+        static_cast<void*>(&contents_),+        static_cast<void const*>(&goner),+        sizeof(T));+    sequencer_.completeTurn(turn * 2);+    new (&goner) T();+  }++  /// dequeue by destructing followed by relocation.  This version is preferred,+  /// because as much work as possible can be done before waiting.+  void dequeueImpl(+      uint32_t turn,+      Atom<uint32_t>& spinCutoff,+      const bool updateSpinCutoff,+      T& elem,+      ImplByRelocation) noexcept {+    try {+      elem.~T();+    } catch (...) {+      // unlikely, but if we don't complete our turn the queue will die+    }+    sequencer_.waitForTurn(turn * 2 + 1, spinCutoff, updateSpinCutoff);+    memcpy(+        static_cast<void*>(&elem),+        static_cast<void const*>(&contents_),+        sizeof(T));+    sequencer_.completeTurn(turn * 2 + 1);+  }++  /// dequeue by nothrow move assignment.+  void dequeueImpl(+      uint32_t turn,+      Atom<uint32_t>& spinCutoff,+      const bool updateSpinCutoff,+      T& elem,+      ImplByMove) noexcept {+    sequencer_.waitForTurn(turn * 2 + 1, spinCutoff, updateSpinCutoff);+    elem = std::move(*ptr());+    destroyContents();+    sequencer_.completeTurn(turn * 2 + 1);+  }+};++} // namespace detail++} // namespace folly
+ folly/folly/MacAddress.cpp view
@@ -0,0 +1,188 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#include <folly/MacAddress.h>++#include <cassert>+#include <ostream>++#include <folly/Exception.h>+#include <folly/Format.h>+#include <folly/IPAddressV6.h>+#include <folly/String.h>++using std::invalid_argument;+using std::string;++namespace folly {++const MacAddress MacAddress::BROADCAST{Endian::big(uint64_t(0xffffffffffffU))};+const MacAddress MacAddress::ZERO;++MacAddress::MacAddress(StringPiece str) {+  memset(&bytes_, 0, 8);+  parse(str);+}++MacAddress MacAddress::createMulticast(IPAddressV6 v6addr) {+  // This method should only be used for multicast addresses.+  assert(v6addr.isMulticast());++  uint8_t bytes[SIZE];+  bytes[0] = 0x33;+  bytes[1] = 0x33;+  memcpy(bytes + 2, v6addr.bytes() + 12, 4);+  return fromBinary(ByteRange(bytes, SIZE));+}++string MacAddress::toString() const {+  static const char hexValues[] = "0123456789abcdef";+  string result;+  result.resize(17);+  result[0] = hexValues[getByte(0) >> 4];+  result[1] = hexValues[getByte(0) & 0xf];+  result[2] = ':';+  result[3] = hexValues[getByte(1) >> 4];+  result[4] = hexValues[getByte(1) & 0xf];+  result[5] = ':';+  result[6] = hexValues[getByte(2) >> 4];+  result[7] = hexValues[getByte(2) & 0xf];+  result[8] = ':';+  result[9] = hexValues[getByte(3) >> 4];+  result[10] = hexValues[getByte(3) & 0xf];+  result[11] = ':';+  result[12] = hexValues[getByte(4) >> 4];+  result[13] = hexValues[getByte(4) & 0xf];+  result[14] = ':';+  result[15] = hexValues[getByte(5) >> 4];+  result[16] = hexValues[getByte(5) & 0xf];+  return result;+}++Expected<Unit, MacAddressFormatError> MacAddress::trySetFromString(+    StringPiece value) {+  return setFromString(value, [](auto _, auto) { return makeUnexpected(_); });+}++void MacAddress::setFromString(StringPiece value) {+  setFromString(value, [](auto, auto _) { return _(), unit; });+}++Expected<Unit, MacAddressFormatError> MacAddress::trySetFromBinary(+    ByteRange value) {+  return setFromBinary(value, [](auto _, auto) { return makeUnexpected(_); });+}++void MacAddress::setFromBinary(ByteRange value) {+  setFromBinary(value, [](auto, auto _) { return _(), unit; });+}++template <typename OnError>+Expected<Unit, MacAddressFormatError> MacAddress::setFromString(+    StringPiece str, OnError err) {+  // Helper function to convert a single hex char into an integer+  auto isSeparatorChar = [](char c) { return c == ':' || c == '-'; };++  uint8_t parsed[SIZE];+  auto p = str.begin();+  for (unsigned int byteIndex = 0; byteIndex < SIZE; ++byteIndex) {+    if (p == str.end()) {+      return err(MacAddressFormatError::Invalid, [&] {+        throw invalid_argument(+            sformat("invalid MAC address '{}': not enough digits", str));+      });+    }++    // Skip over ':' or '-' separators between bytes+    if (byteIndex != 0 && isSeparatorChar(*p)) {+      ++p;+      if (p == str.end()) {+        return err(MacAddressFormatError::Invalid, [&] {+          throw invalid_argument(+              sformat("invalid MAC address '{}': not enough digits", str));+        });+      }+    }++    // Parse the upper nibble+    uint8_t upper = detail::hexTable[static_cast<uint8_t>(*p)];+    if (upper & 0x10) {+      return err(MacAddressFormatError::Invalid, [&] {+        throw invalid_argument(+            sformat("invalid MAC address '{}': contains non-hex digit", str));+      });+    }+    ++p;++    // Parse the lower nibble+    uint8_t lower;+    if (p == str.end()) {+      lower = upper;+      upper = 0;+    } else {+      lower = detail::hexTable[static_cast<uint8_t>(*p)];+      if (lower & 0x10) {+        // Also accept ':', '-', or '\0', to handle the case where one+        // of the bytes was represented by just a single digit.+        if (isSeparatorChar(*p)) {+          lower = upper;+          upper = 0;+        } else {+          return err(MacAddressFormatError::Invalid, [&] {+            throw invalid_argument(sformat(+                "invalid MAC address '{}': contains non-hex digit", str));+          });+        }+      }+      ++p;+    }++    // Update parsed with the newly parsed byte+    parsed[byteIndex] = (upper << 4) | lower;+  }++  if (p != str.end()) {+    // String is too long to be a MAC address+    return err(MacAddressFormatError::Invalid, [&] {+      throw invalid_argument(+          sformat("invalid MAC address '{}': found trailing characters", str));+    });+  }++  // Only update now that we have successfully parsed the entire+  // string.  This way we remain unchanged on error.+  return setFromBinary(ByteRange(parsed, SIZE), err);+}++template <typename OnError>+Expected<Unit, MacAddressFormatError> MacAddress::setFromBinary(+    ByteRange value, OnError err) {+  if (value.size() != SIZE) {+    return err(MacAddressFormatError::Invalid, [&] {+      throw invalid_argument(+          sformat("MAC address must be 6 bytes long, got ", value.size()));+    });+  }+  memcpy(bytes_ + 2, value.begin(), SIZE);+  return unit;+}++std::ostream& operator<<(std::ostream& os, MacAddress address) {+  os << address.toString();+  return os;+}++} // namespace folly
+ folly/folly/MacAddress.h view
@@ -0,0 +1,260 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <iosfwd>++#include <folly/Conv.h>+#include <folly/Expected.h>+#include <folly/Range.h>+#include <folly/Unit.h>+#include <folly/lang/Bits.h>++namespace folly {++class IPAddressV6;++enum class MacAddressFormatError {+  Invalid,+};++/*+ * MacAddress represents an IEEE 802 MAC address.+ */+class MacAddress {+ public:+  static constexpr size_t SIZE = 6;+  static const MacAddress BROADCAST;+  static const MacAddress ZERO;++  /*+   * Construct a zero-initialized MacAddress.+   */+  MacAddress() { memset(&bytes_, 0, 8); }++  /*+   * Parse a MacAddress from a human-readable string.+   * The string must contain 6 one- or two-digit hexadecimal+   * numbers, separated by dashes or colons.+   * Examples: 00:02:C9:C8:F9:68 or 0-2-c9-c8-f9-68+   */+  explicit MacAddress(StringPiece str);++  static Expected<MacAddress, MacAddressFormatError> tryFromString(+      StringPiece value) {+    MacAddress ret;+    auto ok = ret.trySetFromString(value);+    if (!ok) {+      return makeUnexpected(ok.error());+    }+    return ret;+  }+  static MacAddress fromString(StringPiece value) {+    MacAddress ret;+    ret.setFromString(value);+    return ret;+  }++  /*+   * Construct a MAC address from its 6-byte binary value+   */+  static Expected<MacAddress, MacAddressFormatError> tryFromBinary(+      ByteRange value) {+    MacAddress ret;+    auto ok = ret.trySetFromBinary(value);+    if (!ok) {+      return makeUnexpected(ok.error());+    }+    return ret;+  }+  static MacAddress fromBinary(ByteRange value) {+    MacAddress ret;+    ret.setFromBinary(value);+    return ret;+  }++  /*+   * Construct a MacAddress from a uint64_t in network byte order.+   *+   * The first two bytes are ignored, and the MAC address is taken from the+   * latter 6 bytes.+   *+   * This is a static method rather than a constructor to avoid confusion+   * between host and network byte order constructors.+   */+  static MacAddress fromNBO(uint64_t value) { return MacAddress(value); }++  /*+   * Construct a MacAddress from a uint64_t in host byte order.+   *+   * The most significant two bytes are ignored, and the MAC address is taken+   * from the least significant 6 bytes.+   *+   * This is a static method rather than a constructor to avoid confusion+   * between host and network byte order constructors.+   */+  static MacAddress fromHBO(uint64_t value) {+    return MacAddress(Endian::big(value));+  }++  /*+   * Construct the multicast MacAddress for the specified multicast IPv6+   * address.+   */+  static MacAddress createMulticast(IPAddressV6 addr);++  /*+   * Get a pointer to the MAC address' binary value.+   *+   * The returned value points to internal storage inside the MacAddress+   * object.  It is only valid as long as the MacAddress, and its contents may+   * change if the MacAddress is updated.+   */+  const uint8_t* bytes() const { return bytes_ + 2; }++  /*+   * Return the address as a uint64_t, in network byte order.+   *+   * The first two bytes will be 0, and the subsequent 6 bytes will contain+   * the address in network byte order.+   */+  uint64_t u64NBO() const { return packedBytes(); }++  /*+   * Return the address as a uint64_t, in host byte order.+   *+   * The two most significant bytes will be 0, and the remaining 6 bytes will+   * contain the address.  The most significant of these 6 bytes will contain+   * the first byte that appear on the wire, and the least significant byte+   * will contain the last byte.+   */+  uint64_t u64HBO() const {+    // Endian::big() does what we want here, even though we are converting+    // from big-endian to host byte order.  This swaps if and only if+    // the host byte order is little endian.+    return Endian::big(packedBytes());+  }++  /*+   * Return a human-readable representation of the MAC address.+   */+  std::string toString() const;++  /*+   * Update the current MacAddress object from a human-readable string.+   */+  Expected<Unit, MacAddressFormatError> trySetFromString(StringPiece value);+  void setFromString(StringPiece value);+  void parse(StringPiece str) { setFromString(str); }++  /*+   * Update the current MacAddress object from a 6-byte binary representation.+   */+  Expected<Unit, MacAddressFormatError> trySetFromBinary(ByteRange value);+  void setFromBinary(ByteRange value);++  bool isBroadcast() const { return *this == BROADCAST; }+  bool isMulticast() const { return getByte(0) & 0x1; }+  bool isUnicast() const { return !isMulticast(); }++  /*+   * Return true if this MAC address is locally administered.+   *+   * Locally administered addresses are assigned by the local network+   * administrator, and are not guaranteed to be globally unique.  (It is+   * similar to IPv4's private address space.)+   *+   * Note that isLocallyAdministered() will return true for the broadcast+   * address, since it has the locally administered bit set.+   */+  bool isLocallyAdministered() const { return getByte(0) & 0x2; }++  // Comparison operators.++  bool operator==(const MacAddress& other) const {+    // All constructors and modifying methods make sure padding is 0,+    // so we don't need to mask these bytes out when comparing here.+    return packedBytes() == other.packedBytes();+  }++  bool operator<(const MacAddress& other) const {+    return u64HBO() < other.u64HBO();+  }++  bool operator!=(const MacAddress& other) const { return !(*this == other); }++  bool operator>(const MacAddress& other) const { return other < *this; }++  bool operator>=(const MacAddress& other) const { return !(*this < other); }++  bool operator<=(const MacAddress& other) const { return !(*this > other); }++ private:+  explicit MacAddress(uint64_t valueNBO) {+    memcpy(&bytes_, &valueNBO, 8);+    // Set the pad bytes to 0.+    // This allows us to easily compare two MacAddresses,+    // without having to worry about differences in the padding.+    bytes_[0] = 0;+    bytes_[1] = 0;+  }++  template <typename OnError>+  Expected<Unit, MacAddressFormatError> setFromString(+      StringPiece value, OnError err);++  template <typename OnError>+  Expected<Unit, MacAddressFormatError> setFromBinary(+      ByteRange value, OnError err);++  /* We store the 6 bytes starting at bytes_[2] (most significant)+     through bytes_[7] (least).+     bytes_[0] and bytes_[1] are always equal to 0 to simplify comparisons.+  */+  unsigned char bytes_[8];++  inline uint64_t getByte(size_t index) const { return bytes_[index + 2]; }++  uint64_t packedBytes() const {+    uint64_t u64;+    memcpy(&u64, bytes_, 8);+    return u64;+  }+};++/* Define toAppend() so to<string> will work */+template <class Tgt>+typename std::enable_if<IsSomeString<Tgt>::value>::type toAppend(+    MacAddress address, Tgt* result) {+  toAppend(address.toString(), result);+}++std::ostream& operator<<(std::ostream& os, MacAddress address);++} // namespace folly++namespace std {++// Provide an implementation for std::hash<MacAddress>+template <>+struct hash<folly::MacAddress> {+  size_t operator()(const folly::MacAddress& address) const {+    return std::hash<uint64_t>()(address.u64HBO());+  }+};++} // namespace std
+ folly/folly/MapUtil.h view
@@ -0,0 +1,17 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#include <folly/container/MapUtil.h>
+ folly/folly/Math.h view
@@ -0,0 +1,259 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++/**+ * Some arithmetic functions that seem to pop up or get hand-rolled a lot.+ * So far they are all focused on integer division.+ */++#pragma once++#include <cmath>+#include <cstddef>+#include <cstdint>+#include <limits>+#include <type_traits>++namespace folly {++namespace detail {++template <typename T>+inline constexpr T divFloorBranchless(T num, T denom) {+  // floor != trunc when the answer isn't exact and truncation went the+  // wrong way (truncation went toward positive infinity).  That happens+  // when the true answer is negative, which happens when num and denom+  // have different signs.  The following code compiles branch-free on+  // many platforms.+  return (num / denom) ++      ((num % denom) != 0 ? 1 : 0) *+      (std::is_signed<T>::value && (num ^ denom) < 0 ? -1 : 0);+}++template <typename T>+inline constexpr T divFloorBranchful(T num, T denom) {+  // First case handles negative result by preconditioning numerator.+  // Preconditioning decreases the magnitude of the numerator, which is+  // itself sign-dependent.  Second case handles zero or positive rational+  // result, where trunc and floor are the same.+  return std::is_signed<T>::value && (num ^ denom) < 0 && num != 0+      ? (num + (num > 0 ? -1 : 1)) / denom - 1+      : num / denom;+}++template <typename T>+inline constexpr T divCeilBranchless(T num, T denom) {+  // ceil != trunc when the answer isn't exact (truncation occurred)+  // and truncation went away from positive infinity.  That happens when+  // the true answer is positive, which happens when num and denom have+  // the same sign.+  return (num / denom) ++      ((num % denom) != 0 ? 1 : 0) *+      (std::is_signed<T>::value && (num ^ denom) < 0 ? 0 : 1);+}++template <typename T>+inline constexpr T divCeilBranchful(T num, T denom) {+  // First case handles negative or zero rational result, where trunc and ceil+  // are the same.+  // Second case handles positive result by preconditioning numerator.+  // Preconditioning decreases the magnitude of the numerator, which is+  // itself sign-dependent.+  return (std::is_signed<T>::value && (num ^ denom) < 0) || num == 0+      ? num / denom+      : (num + (num > 0 ? -1 : 1)) / denom + 1;+}++template <typename T>+inline constexpr T divRoundAwayBranchless(T num, T denom) {+  // away != trunc whenever truncation actually occurred, which is when+  // there is a non-zero remainder.  If the unrounded result is negative+  // then fixup moves it toward negative infinity.  If the unrounded+  // result is positive then adjustment makes it larger.+  return (num / denom) ++      ((num % denom) != 0 ? 1 : 0) *+      (std::is_signed<T>::value && (num ^ denom) < 0 ? -1 : 1);+}++template <typename T>+inline constexpr T divRoundAwayBranchful(T num, T denom) {+  // First case of second ternary operator handles negative rational+  // result, which is the same as divFloor.  Second case of second ternary+  // operator handles positive result, which is the same as divCeil.+  // Zero case is separated for simplicity.+  return num == 0+      ? 0+      : (num + (num > 0 ? -1 : 1)) / denom ++          (std::is_signed<T>::value && (num ^ denom) < 0 ? -1 : 1);+}++template <typename N, typename D>+using IdivResultType = typename std::enable_if<+    std::is_integral<N>::value && std::is_integral<D>::value &&+        !std::is_same<N, bool>::value && !std::is_same<D, bool>::value,+    decltype(N{1} / D{1})>::type;+} // namespace detail++#if defined(__arm__) && !FOLLY_AARCH64+constexpr auto kIntegerDivisionGivesRemainder = false;+#else+constexpr auto kIntegerDivisionGivesRemainder = true;+#endif++/**+ * Returns num/denom, rounded toward negative infinity.  Put another way,+ * returns the largest integral value that is less than or equal to the+ * exact (not rounded) fraction num/denom.+ *+ * The matching remainder (num - divFloor(num, denom) * denom) can be+ * negative only if denom is negative, unlike in truncating division.+ * Note that for unsigned types this is the same as the normal integer+ * division operator.  divFloor is equivalent to python's integral division+ * operator //.+ *+ * This function undergoes the same integer promotion rules as a+ * built-in operator, except that we don't allow bool -> int promotion.+ * This function is undefined if denom == 0.  It is also undefined if the+ * result type T is a signed type, num is std::numeric_limits<T>::min(),+ * and denom is equal to -1 after conversion to the result type.+ */+template <typename N, typename D>+inline constexpr detail::IdivResultType<N, D> divFloor(N num, D denom) {+  using R = decltype(num / denom);+  return detail::IdivResultType<N, D>(+      kIntegerDivisionGivesRemainder && std::is_signed<R>::value+          ? detail::divFloorBranchless<R>(num, denom)+          : detail::divFloorBranchful<R>(num, denom));+}++/**+ * Returns num/denom, rounded toward positive infinity.  Put another way,+ * returns the smallest integral value that is greater than or equal to+ * the exact (not rounded) fraction num/denom.+ *+ * This function undergoes the same integer promotion rules as a+ * built-in operator, except that we don't allow bool -> int promotion.+ * This function is undefined if denom == 0.  It is also undefined if the+ * result type T is a signed type, num is std::numeric_limits<T>::min(),+ * and denom is equal to -1 after conversion to the result type.+ */+template <typename N, typename D>+inline constexpr detail::IdivResultType<N, D> divCeil(N num, D denom) {+  using R = decltype(num / denom);+  return detail::IdivResultType<N, D>(+      kIntegerDivisionGivesRemainder && std::is_signed<R>::value+          ? detail::divCeilBranchless<R>(num, denom)+          : detail::divCeilBranchful<R>(num, denom));+}++/**+ * Returns num/denom, rounded toward zero.  If num and denom are non-zero+ * and have different signs (so the unrounded fraction num/denom is+ * negative), returns divCeil, otherwise returns divFloor.  If T is an+ * unsigned type then this is always equal to divFloor.+ *+ * Note that this is the same as the normal integer division operator,+ * at least since C99 (before then the rounding for negative results was+ * implementation defined).  This function is here for completeness and+ * as a place to hang this comment.+ *+ * This function undergoes the same integer promotion rules as a+ * built-in operator, except that we don't allow bool -> int promotion.+ * This function is undefined if denom == 0.  It is also undefined if the+ * result type T is a signed type, num is std::numeric_limits<T>::min(),+ * and denom is equal to -1 after conversion to the result type.+ */+template <typename N, typename D>+inline constexpr detail::IdivResultType<N, D> divTrunc(N num, D denom) {+  return detail::IdivResultType<N, D>(num / denom);+}++/**+ * Returns num/denom, rounded away from zero.  If num and denom are+ * non-zero and have different signs (so the unrounded fraction num/denom+ * is negative), returns divFloor, otherwise returns divCeil.  If T is+ * an unsigned type then this is always equal to divCeil.+ *+ * This function undergoes the same integer promotion rules as a+ * built-in operator, except that we don't allow bool -> int promotion.+ * This function is undefined if denom == 0.  It is also undefined if the+ * result type T is a signed type, num is std::numeric_limits<T>::min(),+ * and denom is equal to -1 after conversion to the result type.+ */+template <typename N, typename D>+inline constexpr detail::IdivResultType<N, D> divRoundAway(N num, D denom) {+  using R = decltype(num / denom);+  return detail::IdivResultType<N, D>(+      kIntegerDivisionGivesRemainder && std::is_signed<R>::value+          ? detail::divRoundAwayBranchless<R>(num, denom)+          : detail::divRoundAwayBranchful<R>(num, denom));+}++// clang-format off+// Disabling clang-formatting for midpoint to retain 1:1 correlation+// with LLVM++//  midpoint+//+//  mimic: std::numeric::midpoint, C++20+//  from:+//  https://github.com/llvm/llvm-project/blob/llvmorg-11.0.0/libcxx/include/numeric,+//  Apache 2.0 with LLVM exceptions++template <class _Tp>+constexpr std::enable_if_t<+    std::is_integral<_Tp>::value && !std::is_same<bool, _Tp>::value &&+        !std::is_null_pointer<_Tp>::value,+    _Tp>+midpoint(_Tp __a, _Tp __b) noexcept {+  using _Up = std::make_unsigned_t<_Tp>;+  constexpr _Up __bitshift = std::numeric_limits<_Up>::digits - 1;++  _Up __diff = _Up(__b) - _Up(__a);+  _Up __sign_bit = __b < __a;++  _Up __half_diff = (__diff / 2) + (__sign_bit << __bitshift) + (__sign_bit & __diff);++  return __a + __half_diff;+}++template <class _TPtr>+constexpr std::enable_if_t<+    std::is_pointer<_TPtr>::value &&+        std::is_object<std::remove_pointer_t<_TPtr>>::value &&+        !std::is_void<std::remove_pointer_t<_TPtr>>::value &&+        (sizeof(std::remove_pointer_t<_TPtr>) > 0),+    _TPtr>+midpoint(_TPtr __a, _TPtr __b) noexcept {+  return __a + midpoint(std::ptrdiff_t(0), __b - __a);+}++template <class _Fp>+constexpr std::enable_if_t<std::is_floating_point<_Fp>::value, _Fp> midpoint(+    _Fp __a,+    _Fp __b) noexcept {+  constexpr _Fp __lo = std::numeric_limits<_Fp>::min()*2;+  constexpr _Fp __hi = std::numeric_limits<_Fp>::max()/2;+  return std::abs(__a) <= __hi && std::abs(__b) <= __hi ?  // typical case: overflow is impossible+    (__a + __b)/2 :                                        // always correctly rounded+    std::abs(__a) < __lo ? __a + __b/2 :                   // not safe to halve a+    std::abs(__b) < __lo ? __a/2 + __b :                   // not safe to halve b+    __a/2 + __b/2;                                         // otherwise correctly rounded+}++// clang-format on++} // namespace folly
+ folly/folly/MaybeManagedPtr.h view
@@ -0,0 +1,120 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <memory>++namespace folly {++/**+ * MaybeManagedPtr stores either a raw pointer or a shared_ptr. It provides+ * normal pointer operations on the underlying raw pointer/shared_ptr.+ *+ * When storing a raw pointer, MaybeManagedPtr does not manage the pointer's+ * lifetime, i.e. never calls `free` on the pointer.+ *+ * When storing a shared_ptr, MaybeManagedPtr will release the shared_ptr+ * upon its own destruction.+ */+template <typename T>+class MaybeManagedPtr {+ public:+  /* implicit */ MaybeManagedPtr(T* t)+      : t_(std::shared_ptr<T>(std::shared_ptr<void>(), t)) {}+  /* implicit */ MaybeManagedPtr(std::shared_ptr<T> t) : t_(t) {}++  /**+   * Get pointer to the element contained in MaybeManagedPtr.+   *+   * @return             Pointer to the element contained in MaybeManagedPtr.+   */+  [[nodiscard]] T* get() const { return t_.get(); }++  /**+   * Return use count of the underlying shared pointer.+   *+   * @return             Use count of the underlying shared pointer.+   */+  [[nodiscard]] long useCount() const { return t_.use_count(); }++  /**+   * Member of pointer operator+   *+   * @return             Pointer to the element contained in MaybeManagedPtr.+   */+  constexpr T* operator->() const { return t_.get(); }++  /**+   * Indirection operator+   *+   * @return             Reference to the element contained in MaybeManagedPtr.+   */+  constexpr T& operator*() const& { return *t_.get(); }++  /**+   * Boolean type conversion operator+   *+   * @return             Returns true if the underlying shared pointer is not+   * null.+   */+  operator bool() const { return (t_.get() != nullptr); }++  /**+   * Boolean equal to operator+   *+   * @return             Returns true if the underlying shared pointer is equal+   * to rhs.+   */+  bool operator==(T* rhs) const { return t_.get() == rhs; }++  /**+   * Boolean equal to operator+   *+   * @return             Returns true if the underlying shared pointer is equal+   * to rhs.+   */+  bool operator==(const std::shared_ptr<T>& rhs) const { return t_ == rhs; }++  /**+   * Boolean not equal to operator+   *+   * @return             Returns true if the underlying shared pointer is not+   * equal to rhs.+   */+  bool operator!=(T* rhs) const { return !(t_.get() == rhs); }++  /**+   * Boolean not equal to operator+   *+   * @return             Returns true if the underlying shared pointer is not+   * equal to rhs.+   */+  bool operator!=(const std::shared_ptr<T>& rhs) const { return !(t_ == rhs); }++  /**+   * Pointer type conversion operator+   *+   * @return             Returns a pointer to the element contained in+   * MaybeManagedPtr.+   */+  operator T*() const { return t_.get(); }++ private:+  std::shared_ptr<T> t_;+};++} // namespace folly
+ folly/folly/Memory.h view
@@ -0,0 +1,921 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <cassert>+#include <cerrno>+#include <cstddef>+#include <cstdlib>+#include <exception>+#include <limits>+#include <memory>+#include <stdexcept>+#include <type_traits>+#include <utility>++#include <folly/ConstexprMath.h>+#include <folly/Likely.h>+#include <folly/Portability.h>+#include <folly/Traits.h>+#include <folly/Utility.h>+#include <folly/functional/Invoke.h>+#include <folly/lang/Align.h>+#include <folly/lang/Exception.h>+#include <folly/lang/Thunk.h>+#include <folly/memory/Malloc.h>+#include <folly/portability/Config.h>+#include <folly/portability/Constexpr.h>+#include <folly/portability/Malloc.h>++namespace folly {++namespace access {++/// to_address_fn+/// to_address+///+/// mimic: std::to_address (C++20)+///+/// adapted from: https://en.cppreference.com/w/cpp/memory/to_address, CC-BY-SA+struct to_address_fn {+ private:+  template <template <typename...> typename T, typename A, typename... B>+  static tag_t<A> get_first_arg(tag_t<T<A, B...>>);+  template <typename T>+  using first_arg_of = type_list_element_t<0, decltype(get_first_arg(tag<T>))>;+  template <typename T>+  using detect_element_type = typename T::element_type;+  template <typename T>+  using element_type_of =+      detected_or_t<first_arg_of<T>, detect_element_type, T>;++  template <typename T>+  using detect_to_address =+      decltype(std::pointer_traits<T>::to_address(FOLLY_DECLVAL(T const&)));++  template <typename T>+  static inline constexpr bool use_pointer_traits_to_address = Conjunction<+      is_detected<element_type_of, T>,+      is_detected<detect_to_address, T>>::value;++ public:+  template <typename T>+  constexpr T* operator()(T* p) const noexcept {+    static_assert(!std::is_function_v<T>);+    return p;+  }++  template <typename T>+  constexpr auto operator()(T const& p) const noexcept {+    if constexpr (use_pointer_traits_to_address<T>) {+      static_assert(noexcept(std::pointer_traits<T>::to_address(p)));+      return std::pointer_traits<T>::to_address(p);+    } else {+      static_assert(noexcept(operator()(p.operator->())));+      return operator()(p.operator->());+    }+  }+};+inline constexpr to_address_fn to_address;++} // namespace access++#if (defined(_POSIX_C_SOURCE) && _POSIX_C_SOURCE >= 200112L) || \+    (defined(_XOPEN_SOURCE) && _XOPEN_SOURCE >= 600) ||         \+    (defined(__ANDROID__) && (__ANDROID_API__ > 16)) ||         \+    (defined(__APPLE__)) || defined(__FreeBSD__) || defined(__wasm32__)++inline void* aligned_malloc(size_t size, size_t align) {+  // use posix_memalign, but mimic the behaviour of memalign+  void* ptr = nullptr;+  int rc = posix_memalign(&ptr, align, size);+  return rc == 0 ? (errno = 0, ptr) : (errno = rc, nullptr);+}++inline void aligned_free(void* aligned_ptr) {+  free(aligned_ptr);+}++#elif defined(_WIN32)++inline void* aligned_malloc(size_t size, size_t align) {+  return _aligned_malloc(size, align);+}++inline void aligned_free(void* aligned_ptr) {+  _aligned_free(aligned_ptr);+}++#else++inline void* aligned_malloc(size_t size, size_t align) {+  return memalign(align, size);+}++inline void aligned_free(void* aligned_ptr) {+  free(aligned_ptr);+}++#endif++namespace detail {+template <typename Alloc, size_t kAlign, bool kAllocate>+void rawOverAlignedImpl(Alloc const& alloc, size_t n, void*& raw) {+  static_assert((kAlign & (kAlign - 1)) == 0, "Align must be a power of 2");++  using AllocTraits = std::allocator_traits<Alloc>;+  using T = typename AllocTraits::value_type;++  constexpr bool kCanBypass = std::is_same<Alloc, std::allocator<T>>::value;++  // BaseType is a type that gives us as much alignment as we need if+  // we can get it naturally, otherwise it is aligned as max_align_t.+  // kBaseAlign is both the alignment and size of this type.+  constexpr size_t kBaseAlign = constexpr_min(kAlign, alignof(max_align_t));+  using BaseType = std::aligned_storage_t<kBaseAlign, kBaseAlign>;+  using BaseAllocTraits =+      typename AllocTraits::template rebind_traits<BaseType>;+  using BaseAlloc = typename BaseAllocTraits::allocator_type;+  static_assert(+      sizeof(BaseType) == kBaseAlign && alignof(BaseType) == kBaseAlign);++#if defined(__cpp_sized_deallocation)+  if (kCanBypass && kAlign == kBaseAlign) {+    // until std::allocator uses sized deallocation, it is worth the+    // effort to bypass it when we are able+    if (kAllocate) {+      raw = ::operator new(n * sizeof(T));+    } else {+      ::operator delete(raw, n * sizeof(T));+    }+    return;+  }+#endif++  if (kCanBypass && kAlign > kBaseAlign) {+    // allocating as BaseType isn't sufficient to get alignment, but+    // since we can bypass Alloc we can use something like posix_memalign.+    if (kAllocate) {+      raw = aligned_malloc(n * sizeof(T), kAlign);+    } else {+      aligned_free(raw);+    }+    return;+  }++  // we're not allowed to bypass Alloc, or we don't want to+  BaseAlloc a(alloc);++  // allocation size is counted in sizeof(BaseType)+  size_t quanta = (n * sizeof(T) + kBaseAlign - 1) / sizeof(BaseType);+  if (kAlign <= kBaseAlign) {+    // rebinding Alloc to BaseType is sufficient to get us the alignment+    // we want, happy path+    if (kAllocate) {+      raw = static_cast<void*>(+          std::addressof(*BaseAllocTraits::allocate(a, quanta)));+    } else {+      BaseAllocTraits::deallocate(+          a,+          std::pointer_traits<typename BaseAllocTraits::pointer>::pointer_to(+              *static_cast<BaseType*>(raw)),+          quanta);+    }+    return;+  }++  // Overaligned and custom allocator, our only option is to+  // overallocate and store a delta to the actual allocation just+  // before the returned ptr.+  //+  // If we give ourselves kAlign extra bytes, then since+  // sizeof(BaseType) divides kAlign we can meet alignment while+  // getting a prefix of one BaseType.  If we happen to get a+  // kAlign-aligned block, then we can return a pointer to underlying+  // + kAlign, otherwise there will be at least kBaseAlign bytes in+  // the unused prefix of the first kAlign-aligned block.+  if (kAllocate) {+    char* base = reinterpret_cast<char*>(std::addressof(+        *BaseAllocTraits::allocate(a, quanta + kAlign / sizeof(BaseType))));+    size_t byteDelta =+        kAlign - (reinterpret_cast<uintptr_t>(base) & (kAlign - 1));+    raw = static_cast<void*>(base + byteDelta);+    static_cast<size_t*>(raw)[-1] = byteDelta;+  } else {+    size_t byteDelta = static_cast<size_t*>(raw)[-1];+    char* base = static_cast<char*>(raw) - byteDelta;+    BaseAllocTraits::deallocate(+        a,+        std::pointer_traits<typename BaseAllocTraits::pointer>::pointer_to(+            *reinterpret_cast<BaseType*>(base)),+        quanta + kAlign / sizeof(BaseType));+  }+}+} // namespace detail++// Works like std::allocator_traits<Alloc>::allocate, but handles+// over-aligned types.  Feel free to manually specify any power of two as+// the Align template arg.  Must be matched with deallocateOverAligned.+// allocationBytesForOverAligned will give you the number of bytes that+// this function actually requests.+template <+    typename Alloc,+    size_t kAlign = alignof(typename std::allocator_traits<Alloc>::value_type)>+typename std::allocator_traits<Alloc>::pointer allocateOverAligned(+    Alloc const& alloc, size_t n) {+  void* raw = nullptr;+  detail::rawOverAlignedImpl<Alloc, kAlign, true>(alloc, n, raw);+  return std::pointer_traits<typename std::allocator_traits<Alloc>::pointer>::+      pointer_to(+          *static_cast<typename std::allocator_traits<Alloc>::value_type*>(+              raw));+}++template <+    typename Alloc,+    size_t kAlign = alignof(typename std::allocator_traits<Alloc>::value_type)>+void deallocateOverAligned(+    Alloc const& alloc,+    typename std::allocator_traits<Alloc>::pointer ptr,+    size_t n) {+  void* raw = static_cast<void*>(std::addressof(*ptr));+  detail::rawOverAlignedImpl<Alloc, kAlign, false>(alloc, n, raw);+}++template <+    typename Alloc,+    size_t kAlign = alignof(typename std::allocator_traits<Alloc>::value_type)>+size_t allocationBytesForOverAligned(size_t n) {+  static_assert((kAlign & (kAlign - 1)) == 0, "Align must be a power of 2");++  using AllocTraits = std::allocator_traits<Alloc>;+  using T = typename AllocTraits::value_type;++  constexpr size_t kBaseAlign = constexpr_min(kAlign, alignof(max_align_t));++  if (kAlign > kBaseAlign && std::is_same<Alloc, std::allocator<T>>::value) {+    return n * sizeof(T);+  } else {+    size_t quanta = (n * sizeof(T) + kBaseAlign - 1) / kBaseAlign;+    if (kAlign > kBaseAlign) {+      quanta += kAlign / kBaseAlign;+    }+    return quanta * kBaseAlign;+  }+}++/**+ * static_function_deleter+ *+ * So you can write this:+ *+ *      using RSA_deleter = folly::static_function_deleter<RSA, &RSA_free>;+ *      auto rsa = std::unique_ptr<RSA, RSA_deleter>(RSA_new());+ *      RSA_generate_key_ex(rsa.get(), bits, exponent, nullptr);+ *      rsa = nullptr;  // calls RSA_free(rsa.get())+ *+ * This would be sweet as well for BIO, but unfortunately BIO_free has signature+ * int(BIO*) while we require signature void(BIO*). So you would need to make a+ * wrapper for it:+ *+ *      inline void BIO_free_fb(BIO* bio) { CHECK_EQ(1, BIO_free(bio)); }+ *      using BIO_deleter = folly::static_function_deleter<BIO, &BIO_free_fb>;+ *      auto buf = std::unique_ptr<BIO, BIO_deleter>(BIO_new(BIO_s_mem()));+ *      buf = nullptr;  // calls BIO_free(buf.get())+ */++template <typename T, void (*f)(T*)>+struct static_function_deleter {+  void operator()(T* t) const { f(t); }+};++/**+ *  to_shared_ptr+ *+ *  Convert unique_ptr to shared_ptr without specifying the template type+ *  parameter and letting the compiler deduce it.+ *+ *  So you can write this:+ *+ *      auto sptr = to_shared_ptr(getSomethingUnique<T>());+ *+ *  Instead of this:+ *+ *      auto sptr = shared_ptr<T>(getSomethingUnique<T>());+ *+ *  Useful when `T` is long, such as:+ *+ *      using T = foobar::FooBarAsyncClient;+ */+template <typename T, typename D>+std::shared_ptr<T> to_shared_ptr(std::unique_ptr<T, D>&& ptr) {+  return std::shared_ptr<T>(std::move(ptr));+}++/**+ *  to_shared_ptr_aliasing+ */+template <typename T, typename U>+std::shared_ptr<U> to_shared_ptr_aliasing(std::shared_ptr<T> const& r, U* ptr) {+  return std::shared_ptr<U>(r, ptr);+}++/**+ *  to_shared_ptr_non_owning+ */+template <typename U>+std::shared_ptr<U> to_shared_ptr_non_owning(U* ptr) {+  return std::shared_ptr<U>(std::shared_ptr<void>{}, ptr);+}++/**+ *  to_weak_ptr+ *+ *  Make a weak_ptr and return it from a shared_ptr without specifying the+ *  template type parameter and letting the compiler deduce it.+ *+ *  So you can write this:+ *+ *      auto wptr = to_weak_ptr(getSomethingShared<T>());+ *+ *  Instead of this:+ *+ *      auto wptr = weak_ptr<T>(getSomethingShared<T>());+ *+ *  Useful when `T` is long, such as:+ *+ *      using T = foobar::FooBarAsyncClient;+ */+template <typename T>+std::weak_ptr<T> to_weak_ptr(const std::shared_ptr<T>& ptr) {+  return ptr;+}++#if defined(__GLIBCXX__)+namespace detail {+void weak_ptr_set_stored_ptr(std::weak_ptr<void>& w, void* ptr);++template <typename Tag, void* std::__weak_ptr<void>::*WeakPtr_Ptr_Field>+struct GenerateWeakPtrInternalsAccessor {+  friend void weak_ptr_set_stored_ptr(std::weak_ptr<void>& w, void* ptr) {+    w.*WeakPtr_Ptr_Field = ptr;+  }+};++// Each template instantiation of GenerateWeakPtrInternalsAccessor must+// be a new type, to avoid ODR problems.  We do this by tagging it with+// a type from an anon namespace.+namespace {+struct MemoryAnonTag {};+} // namespace++template struct GenerateWeakPtrInternalsAccessor<+    MemoryAnonTag,+    &std::__weak_ptr<void>::_M_ptr>;+} // namespace detail+#endif++/**+ *  to_weak_ptr_aliasing+ *+ *  Like to_weak_ptr, but arranges that lock().get() on the returned+ *  pointer points to ptr rather than r.get().+ *+ *  Equivalent to:+ *+ *      to_weak_ptr(std::shared_ptr<U>(r, ptr))+ *+ *  For libstdc++, ABI-specific tricks are used to optimize the+ *  implementation.+ */+template <typename T, typename U>+std::weak_ptr<U> to_weak_ptr_aliasing(const std::shared_ptr<T>& r, U* ptr) {+#if defined(__GLIBCXX__)+  std::weak_ptr<void> wv(r);+  detail::weak_ptr_set_stored_ptr(wv, ptr);+  FOLLY_PUSH_WARNING+  FOLLY_GCC_DISABLE_WARNING("-Wstrict-aliasing")+  return reinterpret_cast<std::weak_ptr<U>&&>(wv);+  FOLLY_POP_WARNING+#else+  return std::shared_ptr<U>(r, ptr);+#endif+}++/**+ *  copy_to_unique_ptr+ *+ *  Move or copy the argument to the heap and return it owned by a unique_ptr.+ *+ *  Like std::make_unique, but deduces the type of the owned object.+ */+template <typename T>+std::unique_ptr<remove_cvref_t<T>> copy_to_unique_ptr(T&& t) {+  return std::make_unique<remove_cvref_t<T>>(static_cast<T&&>(t));+}++/**+ *  copy_to_shared_ptr+ *+ *  Move or copy the argument to the heap and return it owned by a shared_ptr.+ *+ *  Like make_shared, but deduces the type of the owned object.+ */+template <typename T>+std::shared_ptr<remove_cvref_t<T>> copy_to_shared_ptr(T&& t) {+  return std::make_shared<remove_cvref_t<T>>(static_cast<T&&>(t));+}++/**+ *  copy_through_unique_ptr+ *+ *  If the argument is nonnull, allocates a copy of its pointee.+ */+template <typename T>+std::unique_ptr<T> copy_through_unique_ptr(const std::unique_ptr<T>& t) {+  static_assert(+      !std::is_polymorphic<T>::value || std::is_final<T>::value,+      "possibly slicing");+  return t ? std::make_unique<T>(*t) : nullptr;+}++/**+ *  copy_through_shared_ptr+ *+ *  If the argument is nonnull, allocates a copy of its pointee.+ */+template <typename T>+std::shared_ptr<T> copy_through_shared_ptr(const std::shared_ptr<T>& t) {+  static_assert(+      !std::is_polymorphic<T>::value || std::is_final<T>::value,+      "possibly slicing");+  return t ? std::make_shared<T>(*t) : nullptr;+}++//  erased_unique_ptr+//+//  A type-erased smart-ptr with unique ownership to a heap-allocated object.+using erased_unique_ptr = std::unique_ptr<void, void (*)(void*)>;++namespace detail {+// for erased_unique_ptr with types that specialize default_delete+template <typename T>+void erased_unique_ptr_delete(void* ptr) {+  std::default_delete<T>()(static_cast<T*>(ptr));+}+} // namespace detail++//  to_erased_unique_ptr+//+//  Converts an owning pointer to an object to an erased_unique_ptr.+template <typename T>+erased_unique_ptr to_erased_unique_ptr(T* const ptr) noexcept {+  return {ptr, detail::erased_unique_ptr_delete<T>};+}++//  to_erased_unique_ptr+//+//  Converts an owning std::unique_ptr to an erased_unique_ptr.+template <typename T>+erased_unique_ptr to_erased_unique_ptr(std::unique_ptr<T> ptr) noexcept {+  return to_erased_unique_ptr(ptr.release());+}++//  make_erased_unique+//+//  Allocate an object of the T on the heap, constructed with a..., and return+//  an owning erased_unique_ptr to it.+template <typename T, typename... A>+erased_unique_ptr make_erased_unique(A&&... a) {+  return to_erased_unique_ptr(std::make_unique<T>(static_cast<A&&>(a)...));+}++//  copy_to_erased_unique_ptr+//+//  Copy an object to the heap and return an owning erased_unique_ptr to it.+template <typename T>+erased_unique_ptr copy_to_erased_unique_ptr(T&& obj) {+  return to_erased_unique_ptr(copy_to_unique_ptr(static_cast<T&&>(obj)));+}++//  empty_erased_unique_ptr+//+//  Return an empty erased_unique_ptr.+inline erased_unique_ptr empty_erased_unique_ptr() {+  return {nullptr, nullptr};+}++/**+ * SysAllocator+ *+ * Resembles std::allocator, the default Allocator, but wraps std::malloc and+ * std::free.+ */+template <typename T>+class SysAllocator {+ private:+  using Self = SysAllocator<T>;++ public:+  using value_type = T;++  constexpr SysAllocator() = default;++  constexpr SysAllocator(SysAllocator const&) = default;++  template <typename U, std::enable_if_t<!std::is_same<U, T>::value, int> = 0>+  constexpr SysAllocator(SysAllocator<U> const&) noexcept {}++  T* allocate(size_t count) {+    auto const p = std::malloc(sizeof(T) * count);+    if (!p) {+      throw_exception<std::bad_alloc>();+    }+    return static_cast<T*>(p);+  }+  void deallocate(T* p, size_t count) { sizedFree(p, count * sizeof(T)); }++  friend bool operator==(Self const&, Self const&) noexcept { return true; }+  friend bool operator!=(Self const&, Self const&) noexcept { return false; }+};++class DefaultAlign {+ private:+  using Self = DefaultAlign;+  std::size_t align_;++ public:+  explicit DefaultAlign(std::size_t align) noexcept : align_(align) {+    assert(!(align_ < sizeof(void*)) && bool("bad align: too small"));+    assert(!(align_ & (align_ - 1)) && bool("bad align: not power-of-two"));+  }+  std::size_t operator()() const noexcept { return align_; }++  friend bool operator==(Self const& a, Self const& b) noexcept {+    return a.align_ == b.align_;+  }+  friend bool operator!=(Self const& a, Self const& b) noexcept {+    return a.align_ != b.align_;+  }+};++template <std::size_t Align>+class FixedAlign {+ private:+  static_assert(!(Align < sizeof(void*)), "bad align: too small");+  static_assert(!(Align & (Align - 1)), "bad align: not power-of-two");+  using Self = FixedAlign<Align>;++ public:+  constexpr std::size_t operator()() const noexcept { return Align; }++  friend bool operator==(Self const&, Self const&) noexcept { return true; }+  friend bool operator!=(Self const&, Self const&) noexcept { return false; }+};++/**+ * AlignedSysAllocator+ *+ * Resembles std::allocator, the default Allocator, but wraps aligned_malloc and+ * aligned_free.+ *+ * Accepts a policy parameter for providing the alignment, which must:+ *   * be invocable as std::size_t(std::size_t) noexcept+ *     * taking the type alignment and returning the allocation alignment+ *   * be noexcept-copy-constructible+ *   * have noexcept operator==+ *   * have noexcept operator!=+ *   * not be final+ *+ * DefaultAlign and FixedAlign<std::size_t>, provided above, are valid policies.+ */+template <typename T, typename Align = DefaultAlign>+class AlignedSysAllocator : private Align {+ private:+  using Self = AlignedSysAllocator<T, Align>;++  template <typename, typename>+  friend class AlignedSysAllocator;++  constexpr Align const& align() const { return *this; }++ public:+  static_assert(std::is_nothrow_copy_constructible<Align>::value);+  static_assert(is_nothrow_invocable_r_v<std::size_t, Align>);++  using value_type = T;++  using propagate_on_container_copy_assignment = std::true_type;+  using propagate_on_container_move_assignment = std::true_type;+  using propagate_on_container_swap = std::true_type;++  using Align::Align;++  // TODO: remove this ctor, which is is no longer required as of under gcc7+  template <+      typename S = Align,+      std::enable_if_t<std::is_default_constructible<S>::value, int> = 0>+  constexpr AlignedSysAllocator() noexcept(noexcept(Align())) : Align() {}++  constexpr AlignedSysAllocator(AlignedSysAllocator const&) = default;++  template <typename U, std::enable_if_t<!std::is_same<U, T>::value, int> = 0>+  constexpr AlignedSysAllocator(+      AlignedSysAllocator<U, Align> const& other) noexcept+      : Align(other.align()) {}++  T* allocate(size_t count) {+    auto const a = align()() < alignof(T) ? alignof(T) : align()();+    auto const p = aligned_malloc(sizeof(T) * count, a);+    if (!p) {+      if (FOLLY_UNLIKELY(errno != ENOMEM)) {+        std::terminate();+      }+      throw_exception<std::bad_alloc>();+    }+    return static_cast<T*>(p);+  }+  void deallocate(T* p, size_t /* count */) { aligned_free(p); }++  friend bool operator==(Self const& a, Self const& b) noexcept {+    return a.align() == b.align();+  }+  friend bool operator!=(Self const& a, Self const& b) noexcept {+    return a.align() != b.align();+  }+};++/**+ * CxxAllocatorAdaptor+ *+ * A type conforming to C++ concept Allocator, delegating operations to an+ * unowned Inner which has this required interface:+ *+ *   void* allocate(std::size_t)+ *   void deallocate(void*, std::size_t)+ *+ * Note that Inner is *not* a C++ Allocator.+ */+template <typename T, class Inner, bool FallbackToStdAlloc = false>+class CxxAllocatorAdaptor : private std::allocator<T> {+ private:+  using Self = CxxAllocatorAdaptor<T, Inner, FallbackToStdAlloc>;++  template <typename U, typename UInner, bool UFallback>+  friend class CxxAllocatorAdaptor;++  Inner* inner_ = nullptr;++ public:+  using value_type = T;++  using propagate_on_container_copy_assignment = std::true_type;+  using propagate_on_container_move_assignment = std::true_type;+  using propagate_on_container_swap = std::true_type;++  template <bool X = FallbackToStdAlloc, std::enable_if_t<X, int> = 0>+  constexpr explicit CxxAllocatorAdaptor() {}++  constexpr explicit CxxAllocatorAdaptor(Inner& ref) : inner_(&ref) {}++  constexpr CxxAllocatorAdaptor(CxxAllocatorAdaptor const&) = default;++  template <typename U, std::enable_if_t<!std::is_same<U, T>::value, int> = 0>+  constexpr CxxAllocatorAdaptor(+      CxxAllocatorAdaptor<U, Inner, FallbackToStdAlloc> const& other)+      : inner_(other.inner_) {}++  CxxAllocatorAdaptor& operator=(CxxAllocatorAdaptor const& other) = default;++  template <typename U, std::enable_if_t<!std::is_same<U, T>::value, int> = 0>+  CxxAllocatorAdaptor& operator=(+      CxxAllocatorAdaptor<U, Inner, FallbackToStdAlloc> const& other) noexcept {+    inner_ = other.inner_;+    return *this;+  }++  T* allocate(std::size_t n) {+    if (FallbackToStdAlloc && inner_ == nullptr) {+      return std::allocator<T>::allocate(n);+    }+    return static_cast<T*>(inner_->allocate(sizeof(T) * n));+  }++  void deallocate(T* p, std::size_t n) {+    if (inner_ != nullptr) {+      inner_->deallocate(p, sizeof(T) * n);+    } else {+      assert(FallbackToStdAlloc);+      std::allocator<T>::deallocate(p, n);+    }+  }++  friend bool operator==(Self const& a, Self const& b) noexcept {+    return a.inner_ == b.inner_;+  }+  friend bool operator!=(Self const& a, Self const& b) noexcept {+    return a.inner_ != b.inner_;+  }++  template <typename U>+  struct rebind {+    using other = CxxAllocatorAdaptor<U, Inner, FallbackToStdAlloc>;+  };+};++/*+ * allocator_delete+ *+ * A deleter which automatically works with a given allocator.+ *+ * Derives from the allocator to take advantage of the empty base+ * optimization when possible.+ */+template <typename Alloc>+class allocator_delete : private std::remove_reference<Alloc>::type {+ private:+  using allocator_type = typename std::remove_reference<Alloc>::type;+  using allocator_traits = std::allocator_traits<allocator_type>;+  using value_type = typename allocator_traits::value_type;+  using pointer = typename allocator_traits::pointer;++ public:+  allocator_delete() = default;+  allocator_delete(allocator_delete const&) = default;+  allocator_delete(allocator_delete&&) = default;+  allocator_delete& operator=(allocator_delete const&) = default;+  allocator_delete& operator=(allocator_delete&&) = default;++  explicit allocator_delete(const allocator_type& alloc)+      : allocator_type(alloc) {}++  explicit allocator_delete(allocator_type&& alloc)+      : allocator_type(std::move(alloc)) {}++  template <typename U>+  allocator_delete(const allocator_delete<U>& other)+      : allocator_type(other.get_allocator()) {}++  allocator_type const& get_allocator() const { return *this; }++  void operator()(pointer p) const {+    auto alloc = get_allocator();+    allocator_traits::destroy(alloc, p);+    allocator_traits::deallocate(alloc, p, 1);+  }+};++/**+ * allocate_unique, like std::allocate_shared but for std::unique_ptr+ */+template <typename T, typename Alloc, typename... Args>+std::unique_ptr<+    T,+    allocator_delete<+        typename std::allocator_traits<Alloc>::template rebind_alloc<T>>>+allocate_unique(Alloc const& alloc, Args&&... args) {+  using TAlloc =+      typename std::allocator_traits<Alloc>::template rebind_alloc<T>;++  using traits = std::allocator_traits<TAlloc>;+  struct DeferCondDeallocate {+    bool& cond;+    TAlloc& copy;+    T* p;+    ~DeferCondDeallocate() {+      if (FOLLY_UNLIKELY(!cond)) {+        traits::deallocate(copy, p, 1);+      }+    }+  };+  auto copy = TAlloc(alloc);+  auto const p = traits::allocate(copy, 1);+  {+    bool constructed = false;+    DeferCondDeallocate handler{constructed, copy, p};+    traits::construct(copy, p, static_cast<Args&&>(args)...);+    constructed = true;+  }+  return {p, allocator_delete<TAlloc>(std::move(copy))};+}++struct SysBufferDeleter {+  void operator()(void* ptr) { std::free(ptr); }+};+using SysBufferUniquePtr = std::unique_ptr<void, SysBufferDeleter>;++inline SysBufferUniquePtr allocate_sys_buffer(std::size_t size) {+  auto p = std::malloc(size);+  if (!p) {+    throw_exception<std::bad_alloc>();+  }+  return {p, {}};+}++/**+ * AllocatorHasTrivialDeallocate+ *+ * Unambiguously inherits std::integral_constant<bool, V> for some bool V.+ *+ * Describes whether a C++ Aallocator has trivial, i.e. no-op, deallocate().+ *+ * Also may be used to describe types which may be used with+ * CxxAllocatorAdaptor.+ */+template <typename Alloc>+struct AllocatorHasTrivialDeallocate : std::false_type {};++template <typename T, class Alloc>+struct AllocatorHasTrivialDeallocate<CxxAllocatorAdaptor<T, Alloc>>+    : AllocatorHasTrivialDeallocate<Alloc> {};++namespace detail {+// note that construct and destroy here are methods, not short names for+// the constructor and destructor+FOLLY_CREATE_MEMBER_INVOKER(AllocatorConstruct_, construct);+FOLLY_CREATE_MEMBER_INVOKER(AllocatorDestroy_, destroy);++template <typename Void, typename Alloc, typename... Args>+struct AllocatorCustomizesConstruct_+    : folly::is_invocable<AllocatorConstruct_, Alloc, Args...> {};++template <typename Alloc, typename... Args>+struct AllocatorCustomizesConstruct_<+    void_t<typename Alloc::folly_has_default_object_construct>,+    Alloc,+    Args...> : Negation<typename Alloc::folly_has_default_object_construct> {};++template <typename Void, typename Alloc, typename... Args>+struct AllocatorCustomizesDestroy_+    : folly::is_invocable<AllocatorDestroy_, Alloc, Args...> {};++template <typename Alloc, typename... Args>+struct AllocatorCustomizesDestroy_<+    void_t<typename Alloc::folly_has_default_object_destroy>,+    Alloc,+    Args...> : Negation<typename Alloc::folly_has_default_object_destroy> {};+} // namespace detail++/**+ * AllocatorHasDefaultObjectConstruct+ *+ * AllocatorHasDefaultObjectConstruct<A, T, Args...> unambiguously+ * inherits std::integral_constant<bool, V>, where V will be true iff+ * the effect of std::allocator_traits<A>::construct(a, p, args...) is+ * the same as new (static_cast<void*>(p)) T(args...).  If true then+ * any optimizations applicable to object construction (relying on+ * std::is_trivially_copyable<T>, for example) can be applied to objects+ * in an allocator-aware container using an allocation of type A.+ *+ * Allocator types can override V by declaring a type alias for+ * folly_has_default_object_construct.  It is helpful to do this if you+ * define a custom allocator type that defines a construct method, but+ * that method doesn't do anything except call placement new.+ */+template <typename Alloc, typename T, typename... Args>+struct AllocatorHasDefaultObjectConstruct+    : Negation<+          detail::AllocatorCustomizesConstruct_<void, Alloc, T*, Args...>> {};++template <typename Value, typename T, typename... Args>+struct AllocatorHasDefaultObjectConstruct<std::allocator<Value>, T, Args...>+    : std::true_type {};++/**+ * AllocatorHasDefaultObjectDestroy+ *+ * AllocatorHasDefaultObjectDestroy<A, T> unambiguously inherits+ * std::integral_constant<bool, V>, where V will be true iff the effect+ * of std::allocator_traits<A>::destroy(a, p) is the same as p->~T().+ * If true then optimizations applicable to object destruction (relying+ * on std::is_trivially_destructible<T>, for example) can be applied to+ * objects in an allocator-aware container using an allocator of type A.+ *+ * Allocator types can override V by declaring a type alias for+ * folly_has_default_object_destroy.  It is helpful to do this if you+ * define a custom allocator type that defines a destroy method, but that+ * method doesn't do anything except call the object's destructor.+ */+template <typename Alloc, typename T>+struct AllocatorHasDefaultObjectDestroy+    : Negation<detail::AllocatorCustomizesDestroy_<void, Alloc, T*>> {};++template <typename Value, typename T>+struct AllocatorHasDefaultObjectDestroy<std::allocator<Value>, T>+    : std::true_type {};++} // namespace folly
+ folly/folly/MicroLock.cpp view
@@ -0,0 +1,72 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#include <folly/MicroLock.h>++#include <thread>++#include <folly/portability/Asm.h>++namespace folly {++uint8_t MicroLockCore::lockSlowPath(+    uint8_t oldWord, unsigned maxSpins, unsigned maxYields) noexcept {+  uint8_t newWord;+  unsigned spins = 0;+  uint8_t heldBit = 1;+  uint8_t waitBit = heldBit << 1;+  uint8_t needWaitBit = 0;++retry:+  if ((oldWord & heldBit) != 0) {+    ++spins;+    if (spins > maxSpins + maxYields) {+      // Somebody appears to have the lock.  Block waiting for the+      // holder to unlock the lock.  We set heldbit(slot) so that the+      // lock holder knows to FUTEX_WAKE us.+      newWord = oldWord | waitBit;+      if (newWord != oldWord) {+        if (!atomic_ref(lock_).compare_exchange_weak(+                oldWord,+                newWord,+                std::memory_order_relaxed,+                std::memory_order_relaxed)) {+          goto retry;+        }+      }+      atomic_wait(&atomic_ref(lock_).atomic(), newWord);+      needWaitBit = waitBit;+    } else if (spins > maxSpins) {+      // sched_yield(), but more portable+      std::this_thread::yield();+    } else {+      folly::asm_volatile_pause();+    }+    oldWord = atomic_ref(lock_).load(std::memory_order_relaxed);+    goto retry;+  }++  newWord = oldWord | heldBit | needWaitBit;+  if (!atomic_ref(lock_).compare_exchange_weak(+          oldWord,+          newWord,+          std::memory_order_acquire,+          std::memory_order_relaxed)) {+    goto retry;+  }+  return decodeDataFromWord(newWord);+}+} // namespace folly
+ folly/folly/MicroLock.h view
@@ -0,0 +1,308 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <cassert>+#include <climits>+#include <cstdint>+#include <utility>++#include <folly/Optional.h>+#include <folly/Portability.h>+#include <folly/Utility.h>+#include <folly/synchronization/AtomicNotification.h>+#include <folly/synchronization/AtomicRef.h>++namespace folly {++/**+ * Tiny exclusive lock that uses 2 bits. It is stored as 1 byte and+ * has APIs for using the remaining 6 bits for storing user data.+ *+ * You should zero-initialize the bits of a MicroLock that you intend+ * to use.+ *+ * If you're not space-constrained, prefer std::mutex, which will+ * likely be faster, since it has more than two bits of information to+ * work with.+ *+ * You are free to put a MicroLock in a union with some other object.+ * If, for example, you want to use the bottom two bits of a pointer+ * as a lock, you can put a MicroLock in a union with the pointer,+ * which will use the two least-significant bits in the bottom byte.+ *+ * (Note that such a union is safe only because MicroLock is based on+ * a character type, and even under a strict interpretation of C++'s+ * aliasing rules, character types may alias anything.)+ *+ * Unused bits in the lock can be used to store user data via+ * lockAndLoad() and unlockAndStore(), or LockGuardWithData.+ *+ * The MaxSpins template parameter controls the number of times we+ * spin trying to acquire the lock.  MaxYields controls the number of+ * times we call sched_yield; once we've tried to acquire the lock+ * MaxSpins + MaxYields times, we sleep on the lock futex.+ * By adjusting these parameters, you can make MicroLock behave as+ * much or as little like a conventional spinlock as you'd like.+ *+ * Performance+ * -----------+ *+ * With the default template options, the timings for uncontended+ * acquire-then-release come out as follows on Intel(R) Xeon(R) CPU+ * E5-2660 0 @ 2.20GHz, in @mode/opt, as of the master tree at Tue, 01+ * Mar 2016 19:48:15.+ *+ * ========================================================================+ * folly/test/SmallLocksBenchmark.cpp          relative  time/iter  iters/s+ * ========================================================================+ * MicroSpinLockUncontendedBenchmark                       13.46ns   74.28M+ * PicoSpinLockUncontendedBenchmark                        14.99ns   66.71M+ * MicroLockUncontendedBenchmark                           27.06ns   36.96M+ * StdMutexUncontendedBenchmark                            25.18ns   39.72M+ * VirtualFunctionCall                                      1.72ns  579.78M+ * ========================================================================+ *+ * (The virtual dispatch benchmark is provided for scale.)+ *+ * While the uncontended case for MicroLock is competitive with the+ * glibc 2.2.0 implementation of std::mutex, std::mutex is likely to be+ * faster in the contended case, because we need to wake up all waiters+ * when we release.+ *+ * Make sure to benchmark your particular workload.+ *+ */++class MicroLockCore {+ protected:+  uint8_t lock_{};+  /**+   * Mask for bit indicating that the flag is held.+   */+  unsigned heldBit() const noexcept;+  /**+   * Mask for bit indicating that there is a waiter that should be woken up.+   */+  unsigned waitBit() const noexcept;++  uint8_t lockSlowPath(+      uint8_t oldWord, unsigned maxSpins, unsigned maxYields) noexcept;++  static constexpr unsigned kNumLockBits = 2;+  static constexpr uint8_t kLockBits =+      static_cast<uint8_t>((1 << kNumLockBits) - 1);+  static constexpr uint8_t kDataBits = static_cast<uint8_t>(~kLockBits);+  /**+   * Decodes the value stored in the unused bits of the lock.+   */+  static constexpr uint8_t decodeDataFromByte(uint8_t lockByte) noexcept {+    return static_cast<uint8_t>(lockByte >> kNumLockBits);+  }+  /**+   * Encodes the value for the unused bits of the lock.+   */+  static constexpr uint8_t encodeDataToByte(uint8_t data) noexcept {+    return static_cast<uint8_t>(data << kNumLockBits);+  }++  static constexpr uint8_t decodeDataFromWord(uint8_t word) noexcept {+    return static_cast<uint8_t>(word >> kNumLockBits);+  }+  static constexpr uint8_t encodeDataToWord(+      uint8_t word, uint8_t value) noexcept {+    const uint8_t preservedBits = word & ~(kDataBits);+    const uint8_t newBits = encodeDataToByte(value);+    return preservedBits | newBits;+  }++  template <typename Func>+  void unlockAndStoreWithModifier(Func modifier) noexcept;++ public:+  /**+   * Loads the data stored in the unused bits of the lock atomically.+   */+  uint8_t load(+      std::memory_order order = std::memory_order_seq_cst) const noexcept {+    return decodeDataFromWord(atomic_ref(lock_).load(order));+  }++  /**+   * Stores the data in the unused bits of the lock atomically. Since 2 bits are+   * used by the lock, the most significant 2 bits of the provided value will be+   * ignored.+   */+  void store(+      uint8_t value,+      std::memory_order order = std::memory_order_seq_cst) noexcept;++  /**+   * Unlocks the lock and stores the bits of the provided value into the data+   * bits. Since 2 bits are used by the lock, the most significant 2 bits of the+   * provided value will be ignored.+   */+  void unlockAndStore(uint8_t value) noexcept;+  void unlock() noexcept;+};++inline unsigned MicroLockCore::heldBit() const noexcept {+  return 1U << 0;+}++inline unsigned MicroLockCore::waitBit() const noexcept {+  return 1U << 1;+}++inline void MicroLockCore::store(+    uint8_t value, std::memory_order order) noexcept {+  auto oldWord = atomic_ref(lock_).load(std::memory_order_relaxed);+  while (true) {+    auto newWord = encodeDataToWord(oldWord, value);+    if (atomic_ref(lock_).compare_exchange_weak(+            oldWord, newWord, order, std::memory_order_relaxed)) {+      break;+    }+  }+}++template <typename Func>+void MicroLockCore::unlockAndStoreWithModifier(Func modifier) noexcept {+  uint8_t oldWord;+  uint8_t newWord;++  oldWord = atomic_ref(lock_).load(std::memory_order_relaxed);+  do {+    assert(oldWord & heldBit());+    newWord = modifier(oldWord) & ~(heldBit() | waitBit());+  } while (!atomic_ref(lock_).compare_exchange_weak(+      oldWord, newWord, std::memory_order_release, std::memory_order_relaxed));++  if (oldWord & waitBit()) {+    atomic_notify_one(&atomic_ref(lock_).atomic());+  }+}++inline void MicroLockCore::unlockAndStore(uint8_t value) noexcept {+  unlockAndStoreWithModifier([value](uint8_t oldWord) {+    return encodeDataToWord(oldWord, value);+  });+}++inline void MicroLockCore::unlock() noexcept {+  unlockAndStoreWithModifier(identity);+}++template <unsigned MaxSpins = 1000, unsigned MaxYields = 0>+class MicroLockBase : public MicroLockCore {+ public:+  /**+   * Locks the lock and returns the data stored in the unused bits of the lock.+   * This is useful when you want to use the unused bits of the lock to store+   * data, in which case reading and locking should be done in one atomic+   * operation.+   */+  uint8_t lockAndLoad() noexcept;+  void lock() noexcept { lockAndLoad(); }+  bool try_lock() noexcept;++  /**+   * A lock guard which allows reading and writing to the unused bits of the+   * lock as data.+   */+  struct LockGuardWithData {+    explicit LockGuardWithData(MicroLockBase<MaxSpins, MaxYields>& lock)+        : lock_(lock) {+      loadedValue_ = lock_.lockAndLoad();+    }++    ~LockGuardWithData() noexcept {+      if (storedValue_) {+        lock_.unlockAndStore(*storedValue_);+      } else {+        lock_.unlock();+      }+    }++    /**+     * The stored data bits at the time of locking.+     */+    uint8_t loadedValue() const noexcept { return loadedValue_; }++    /**+     * The value that will be stored back into data bits when it is unlocked.+     */+    void storeValue(uint8_t value) noexcept { storedValue_ = value; }++   private:+    MicroLockBase<MaxSpins, MaxYields>& lock_;+    uint8_t loadedValue_;+    folly::Optional<uint8_t> storedValue_;+  };+};++template <unsigned MaxSpins, unsigned MaxYields>+bool MicroLockBase<MaxSpins, MaxYields>::try_lock() noexcept {+  // N.B. You might think that try_lock is just the fast path of lock,+  // but you'd be wrong.  Keep in mind that other parts of our host+  // word might be changing while we take the lock!  We're not allowed+  // to fail spuriously if the lock is in fact not held, even if other+  // people are concurrently modifying other parts of the word.+  //+  // We need to loop until we either see firm evidence that somebody+  // else has the lock (by looking at heldBit) or see our CAS succeed.+  // A failed CAS by itself does not indicate lock-acquire failure.++  uint8_t oldWord = atomic_ref(lock_).load(std::memory_order_relaxed);+  do {+    if (oldWord & heldBit()) {+      return false;+    }+  } while (!atomic_ref(lock_).compare_exchange_weak(+      oldWord,+      oldWord | heldBit(),+      std::memory_order_acquire,+      std::memory_order_relaxed));++  return true;+}++template <unsigned MaxSpins, unsigned MaxYields>+uint8_t MicroLockBase<MaxSpins, MaxYields>::lockAndLoad() noexcept {+  static_assert(MaxSpins + MaxYields < (unsigned)-1, "overflow");++  uint8_t oldWord;+  oldWord = atomic_ref(lock_).load(std::memory_order_relaxed);+  if ((oldWord & heldBit()) == 0 &&+      atomic_ref(lock_).compare_exchange_weak(+          oldWord,+          to_narrow(oldWord | heldBit()),+          std::memory_order_acquire,+          std::memory_order_relaxed)) {+    // Fast uncontended case: memory_order_acquire above is our barrier+    return decodeDataFromWord(to_narrow(oldWord | heldBit()));+  } else {+    // lockSlowPath doesn't call waitBit(); it just shifts the input bit.  Make+    // sure its shifting produces the same result a call to waitBit would.+    assert(heldBit() << 1 == waitBit());+    // lockSlowPath emits its own memory barrier+    return lockSlowPath(oldWord, MaxSpins, MaxYields);+  }+}++typedef MicroLockBase<> MicroLock;+} // namespace folly
+ folly/folly/MicroSpinLock.h view
@@ -0,0 +1,17 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#include <folly/synchronization/MicroSpinLock.h> // @shim
+ folly/folly/MoveWrapper.h view
@@ -0,0 +1,76 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <memory>++namespace folly {++/** C++11 closures don't support move-in capture. Nor does std::bind.+    facepalm.++    http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2013/n3610.html++    "[...] a work-around that should make people's stomach crawl:+    write a wrapper that performs move-on-copy, much like the deprecated+    auto_ptr"++    Unlike auto_ptr, this doesn't require a heap allocation.+    */+template <class T>+class MoveWrapper {+ public:+  /** If value can be default-constructed, why not?+      Then we don't have to move it in */+  MoveWrapper() = default;++  /// Move a value in.+  explicit MoveWrapper(T&& t) : value(std::move(t)) {}++  /// copy is move+  MoveWrapper(const MoveWrapper& other) : value(std::move(other.value)) {}++  /// move is also move+  MoveWrapper(MoveWrapper&& other) : value(std::move(other.value)) {}++  const T& operator*() const { return value; }+  T& operator*() { return value; }++  const T* operator->() const { return &value; }+  T* operator->() { return &value; }++  /// move the value out (sugar for std::move(*moveWrapper))+  T&& move() { return std::move(value); }++  // If you want these you're probably doing it wrong, though they'd be+  // easy enough to implement+  MoveWrapper& operator=(MoveWrapper const&) = delete;+  MoveWrapper& operator=(MoveWrapper&&) = delete;++ private:+  mutable T value;+};++/// Make a MoveWrapper from the argument. Because the name "makeMoveWrapper"+/// is already quite transparent in its intent, this will work for lvalues as+/// if you had wrapped them in std::move.+template <class T, class T0 = typename std::remove_reference<T>::type>+MoveWrapper<T0> makeMoveWrapper(T&& t) {+  return MoveWrapper<T0>(std::forward<T0>(t));+}++} // namespace folly
+ folly/folly/ObserverContainer.h view
@@ -0,0 +1,1027 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <bitset>+#include <vector>++#include <glog/logging.h>+#include <folly/ConstructorCallbackList.h>+#include <folly/Function.h>+#include <folly/Optional.h>+#include <folly/ScopeGuard.h>+#include <folly/io/async/DestructorCheck.h>+#include <folly/small_vector.h>++/**+ * Tooling that makes it easier to design observable objects and observers.+ */++namespace folly {++/**+ * Interface for store of pointers to observers.+ */+template <typename Observer>+class ObserverContainerStoreBase {+ public:+  using observer_type = Observer;++  // ObserverContainerStore stores shared_ptr<Observer> objects.+  //+  // To support observer objects that are NOT managed by a shared_ptr, the+  // encapsulating ObserverContainer wraps unmanaged pointers inside of+  // shared_ptrs, but sets an empty deleter for the shared_ptr, so that the+  // pointer remains unmanaged.+  //+  // As a result, it is possible for the shared_ptrs maintained by the store to+  // be "unmanaged". The type alias `MaybeManagedObserverPointer` is used to+  // ensure that this detail is apparent in other parts of the container code.+  using MaybeManagedObserverPointer = std::shared_ptr<Observer>;++  virtual ~ObserverContainerStoreBase() = default;++  /**+   * Add an observer pointer to the store.+   *+   * @param observer     Observer to add.+   * @return             Whether observer was added (not already present).+   */+  virtual bool add(MaybeManagedObserverPointer observer) = 0;++  /**+   * Remove an observer pointer from the store.+   *+   * @param observer     Observer to remove.+   * @return             Whether observer found and removed from store.+   */+  virtual bool remove(MaybeManagedObserverPointer observer) = 0;++  /**+   * Get number of observers in store.+   *+   * If called while the store is being iterated, the returned value may not+   * reflect changes that occurred (e.g., observers added or removed) during+   * iteration.+   *+   * @return             Number of observers in store.+   */+  virtual size_t size() const = 0;++  /**+   * Policy that determines how invokeForEachObserver handles mutations.+   */+  enum class InvokeWhileIteratingPolicy {+    InvokeAdded, // if observer added, invoke fn for it+    DoNotInvokeAdded, // if observer added, do not invoke fn for it+    CheckNoChange, // observers must not be added or removed during iteration+    CheckNoAdded // observers must not be added during iteration+  };++  /**+   * Invoke function for each observer in the store.+   *+   * @param fn           Function to call for each observer in store.+   * @param policy       InvokeWhileIteratingPolicy policy.+   */+  virtual void invokeForEachObserver(+      folly::Function<void(MaybeManagedObserverPointer&)>&& fn,+      const InvokeWhileIteratingPolicy policy) = 0;++  /**+   * Invoke function for each observer in the store.+   *+   * @param fn           Function to call for each observer in store.+   * @param policy       InvokeWhileIteratingPolicy policy.+   */+  virtual void invokeForEachObserver(+      folly::Function<void(Observer*)>&& fn,+      const InvokeWhileIteratingPolicy policy) {+    invokeForEachObserver(+        [fnL = std::move(fn)](MaybeManagedObserverPointer& observer) mutable {+          fnL(observer.get());+        },+        policy);+  }+};++/**+ * Policy for ObserverContainerStore.+ *+ * Defines the udnerlying container type and the default size.+ */+template <unsigned int ReserveElements = 2>+struct ObserverContainerStorePolicyDefault {+  template <typename Observer>+  using container = std::conditional_t<+      !kIsMobile,+      folly::small_vector<Observer, ReserveElements>,+      std::vector<Observer>>;+  const static unsigned int reserve_elements = ReserveElements;+};++/**+ * Policy-based implementation of ObserverContainerStoreBase.+ */+template <+    typename Observer,+    typename Policy = ObserverContainerStorePolicyDefault<>>+class ObserverContainerStore : public ObserverContainerStoreBase<Observer> {+ public:+  using Base = ObserverContainerStoreBase<Observer>;+  using InvokeWhileIteratingPolicy = typename Base::InvokeWhileIteratingPolicy;++  /**+   * Construct a new store, reserving as configured.+   */+  ObserverContainerStore() { observers_.reserve(Policy::reserve_elements); }++  /**+   * Add an observer pointer to the store.+   *+   * @param observer     Observer to add.+   * @return             Whether observer was added (not already present).+   */+  bool add(std::shared_ptr<Observer> observer) override {+    // attempts to add the same observer multiple times are rejected+    if (std::find(observers_.begin(), observers_.end(), observer) !=+        observers_.end()) {+      return false;+    }++    if (iterating_) {+      CHECK(maybeCurrentIterationPolicy_.has_value());+      const auto& policy = maybeCurrentIterationPolicy_.value();+      switch (policy) {+        case InvokeWhileIteratingPolicy::InvokeAdded:+        case InvokeWhileIteratingPolicy::DoNotInvokeAdded:+          break;+        case InvokeWhileIteratingPolicy::CheckNoChange:+          folly::terminate_with<std::runtime_error>(+              "Cannot add observers while iterating "+              "per current iteration policy (CheckNoChange)");+          break;+        case InvokeWhileIteratingPolicy::CheckNoAdded:+          folly::terminate_with<std::runtime_error>(+              "Cannot add observers while iterating "+              "per current iteration policy (CheckNoAdded)");+          break;+      }+    }+    observers_.insert(observers_.end(), observer);+    return true;+  }++  /**+   * Remove an observer pointer from the store.+   *+   * @param observer     Observer to remove.+   * @return             Whether observer found and removed from store.+   */+  bool remove(std::shared_ptr<Observer> observer) override {+    const auto it = std::find(observers_.begin(), observers_.end(), observer);+    if (it == observers_.end()) {+      return false;+    }++    // if store is currently being iterated, set this element to nullptr and it+    // will be cleaned up after iteration is completed, else erase immediately.+    if (iterating_) {+      CHECK(maybeCurrentIterationPolicy_.has_value());+      const auto& policy = maybeCurrentIterationPolicy_.value();+      switch (policy) {+        case InvokeWhileIteratingPolicy::InvokeAdded:+        case InvokeWhileIteratingPolicy::DoNotInvokeAdded:+          break;+        case InvokeWhileIteratingPolicy::CheckNoChange:+          folly::terminate_with<std::runtime_error>(+              "Cannot remove observers while iterating "+              "per current iteration policy (CheckNoChange)");+          break;+        case InvokeWhileIteratingPolicy::CheckNoAdded:+          break;+      }++      *it = nullptr;+      removalDuringIteration_ = true;+    } else {+      observers_.erase(it);+    }++    return true;+  }++  /**+   * Get number of observers in store.+   *+   * If called while the store is being iterated, the returned value may not+   * reflect changes that occurred (e.g., observers added or removed) during+   * iteration.+   *+   * @return             Number of observers in store.+   */+  size_t size() const override { return observers_.size(); }++  /**+   * Invoke function for each observer in the store.+   *+   * @param fn           Function to call for each observer in store.+   * @param policy       InvokeWhileIteratingPolicy policy.+   */+  void invokeForEachObserver(+      folly::Function<void(typename Base::MaybeManagedObserverPointer&)>&& fn,+      const typename Base::InvokeWhileIteratingPolicy policy) noexcept+      override {+    CHECK(!iterating_)+        << "Nested iteration of ObserverContainer is prohibited.";+    CHECK(!maybeCurrentIterationPolicy_.has_value())+        << "Nested iteration of ObserverContainer is prohibited.";+    iterating_ = true;+    maybeCurrentIterationPolicy_ = policy;+    SCOPE_EXIT {+      if (removalDuringIteration_) {+        // observers removed while we were iterating through container;+        // remove elements for which the element value is null+        observers_.erase(+            std::remove_if(+                observers_.begin(),+                observers_.end(),+                [](const auto& elem) { return elem == nullptr; }),+            observers_.end());+      }+      iterating_ = false;+      maybeCurrentIterationPolicy_ = folly::none;+      removalDuringIteration_ = false;+    };++    const auto numObserversAtStart = observers_.size();++    // iterate through the list using indexes, not iterators, so that the list+    // can mutate during iteration...+    for (typename container_type::size_type idx = 0;+         // observers_.size() cannot decrease during iteration, so it should be+         // insignificantly faster to check the single size in the common case.+         idx < numObserversAtStart ||+         (idx < observers_.size() &&+          policy == InvokeWhileIteratingPolicy::InvokeAdded);+         idx++) {+      auto& observer = observers_.at(idx);+      if (!observer) { // empty space in list caused by incomplete removal+        continue;+      }++      fn(observer);+    }+  }++  using Base::invokeForEachObserver;++ private:+  using container_type =+      typename Policy::template container<std::shared_ptr<Observer>>;++  // The actual list of observers.+  container_type observers_;++  // Whether we are actively iterating through the list of observers.+  bool iterating_{false};++  // If we are actively iterating, the corresponding InvokeWhileIteratingPolicy.+  folly::Optional<InvokeWhileIteratingPolicy> maybeCurrentIterationPolicy_;++  // Whether a removal or addition occurred while we iterating through the list.+  bool removalDuringIteration_{false};+};++/**+ * Policy for ObserverContainerBase.+ *+ * @tparam EventEnum    Enum of events that observers can subscribe to.+ *                      Each event must have a unique integer value greater+ *                      than zero.+ *+ * @tparam BitsetSize   Size of bitset, must be greater than or equal to the+ *                      number of events in EventEnum.+ */+template <typename EventEnum, size_t BitsetSize>+struct ObserverContainerBasePolicyDefault {+  static constexpr size_t bitset_size() { return BitsetSize; }+  using event_enum = EventEnum;+};++/**+ * Base ObserverContainer and definition of Observers.+ */+template <+    typename ObserverInterface,+    typename Observed,+    typename ContainerPolicy>+class ObserverContainerBase {+ public:+  using interface_type = ObserverInterface;+  using observed_type = Observed;+  using policy_type = ContainerPolicy;+  using EventEnum = typename ContainerPolicy::event_enum;+  using EventEnumIntT = std::underlying_type_t<EventEnum>;++  virtual ~ObserverContainerBase() = default;++  /**+   * EventSet is used to keep track of the observer events that are enabled.+   */+  class ObserverEventSet {+   public:+    ObserverEventSet() : bitset_(0) {}++    /**+     * Enables all events.+     */+    void enableAllEvents() { bitset_.set(); }++    /**+     * Enables the events passed in the initializer list.+     *+     * @param eventsEnums  Events to enable.++     */+    template <typename... EventEnums>+    void enable(EventEnums... eventEnums) {+      for (auto&& event : {eventEnums...}) {+        const auto eventAsInt = static_cast<EventEnumIntT>(event);+        bitset_.set(eventAsInt);+      }+    }++    /**+     * Returns whether the event passed in is enabled.+     *+     * @param event        Event to check.+     * @return             Whether the passed event is enabled.+     */+    bool isEnabled(const EventEnum event) const {+      const auto eventAsInt = static_cast<EventEnumIntT>(event);+      return bitset_.test(eventAsInt);+    }++    /**+     * Builder that makes it easier to pass EventSet to Observer constructor.+     */+    class Builder {+     public:+      explicit Builder() = default;++      /**+       * Enables all events.+       */+      Builder&& enableAllEvents() {+        set_.enableAllEvents();+        return std::move(*this);+      }++      /**+       * Enables the events passed in the intiailizer list.+       *+       * @param events       Events to enable.+       */+      template <typename... EventEnums>+      Builder&& enable(EventEnums... eventEnums) {+        set_.enable(eventEnums...);+        return std::move(*this);+      }++      /**+       * Returns the EventSet that has been built.+       */+      ObserverEventSet build() && { return set_; }++     private:+      ObserverEventSet set_;+    };++   private:+    std::bitset<ContainerPolicy::bitset_size()> bitset_{0};+  };++  /**+   * Observer base interface.+   *+   * This interface includes the events exposed by the subject's observer+   * interface and the set of events that are provided by the ObserverContainer+   * (attached/detached/moved/destoyed). It also defines how observers subscribe+   * to specific events made available by the subject.+   */+  class ObserverBase : public ObserverInterface, public DestructorCheck {+   public:+    using observed_type = Observed;+    using interface_type = ObserverInterface;++    using EventSet = ObserverEventSet;+    using EventSetBuilder = typename ObserverEventSet::Builder;++    ~ObserverBase() override = default;++    /**+     * Construct a new observer with no event subscriptions.+     */+    ObserverBase() {}++    /**+     * Construct a new observer subscribed to events in the passed EventSet.+     */+    explicit ObserverBase(EventSet eventSet) : eventSet_(eventSet) {}++    /**+     * Base class that can be used to pass context about move operation.+     */+    class MoveContext {};++    /**+     * Base class that can be used to pass context about why object destroyed.+     */+    class DestroyContext {};++    /**+     * Invoked when this observer is attached to an object.+     *+     * @param obj   Object that observer is now attached to.+     */+    virtual void attached(Observed* /* obj */) noexcept {}++    /**+     * Invoked if this observer is detached from an object.+     *+     * @param obj   Object that observer is no longer attached to.+     */+    virtual void detached(Observed* /* obj */) noexcept {}++    /**+     * Invoked when an observed object's destructor is invoked.+     *+     * Destruction of the observed object implicitly implies detached, and thus+     * detached will not be called if an object is destroyed.+     *+     * @param obj           Object being destroyed.+     * @param ctx           Additional info about what triggered destruction.+     *                      Not available unless provided by the implementation;+     *                      if not supported it is a nullptr.+     */+    virtual void destroyed(+        Observed* /* obj */, DestroyContext* /* ctx */) noexcept {}++    /**+     * Invoked when object being observed changes due to move construction.+     *+     * @param oldObj        Object previously being observed.+     * @param newObj        Object now being observed.+     * @param ctx           Additional info about what triggered the move.+     *                      Not available unless provided by the implementation;+     *                      if not supported it is a nullptr.+     */+    virtual void moved(+        Observed* /* oldObj */,+        Observed* /* newObj */,+        MoveContext* /* ctx */) noexcept {}++    /**+     * Proxy function used to invoke a method defined in the observer interface.+     *+     * Can be overridden to enable composition of observers, including event bus+     * architectures in which multiple handlers act on an event.+     *+     * Implementations can remove themselves and add/remove other observers from+     * the container when handling this call. If new observers are added to the+     * container, invokeInterfaceMethod will be called on those new observers+     * as well. If you want to avoid this in your observer implementation, delay+     * mutation of the container until postInvokeInterfaceMethod is called.+     *+     * @param obj           Object associated with observer event.+     * @param fn            Function that will invoke the method associated with+     *                      an observer event, passing any event context.+     * @param maybeEvent    The event enum associated with the invocation.+     */+    virtual void invokeInterfaceMethod(+        Observed* obj,+        folly::Function<void(ObserverBase*, Observed*)>& fn,+        folly::Optional<EventEnum> /* maybeEvent */) noexcept {+      fn(this, obj);+    }++    /**+     * Invoked after invokeInterfaceMethod has completed for all observers.+     *+     * Can be used to delay mutation of the container after processing of an+     * event has completed. Implementations can remove themselves and add/remove+     * other observers from the container when handling this call. However, this+     * function will only be called for the set of observers in the container+     * when the preceding call to invokeInterfaceMethod finished.+     *+     * @param obj           Object associated with observer event.+     */+    virtual void postInvokeInterfaceMethod(Observed* /* obj */) noexcept {}++    /**+     * Returns the EventSet containing the events the observer wants.+     */+    const EventSet& getEventSet() const noexcept { return eventSet_; }++   private:+    const EventSet eventSet_;+  };++  /**+   * Observer interface.+   *+   * The interface between an observer container and observers in the container.+   *+   * This interface includes methods that are called upon relevant changes to+   * the observer's status in a container (added/removedFromObserverContainer).+   *+   * An observer must not be destroyed while it is in a container. This can be+   * accomplished by removing the observer from the container on its destruction+   * or delaying destruction.+   *+   * Typical use cases should not attempt to implement this interface and should+   * instead use a specialization such as ManagedObserver.+   */+  class Observer : public ObserverBase {+   public:+    using observed_type = Observed;+    using interface_type = ObserverInterface;++    using EventSet = ObserverEventSet;+    using EventSetBuilder = typename ObserverEventSet::Builder;++    ~Observer() override = default;++    /**+     * Construct a new observer with no event subscriptions.+     */+    Observer() : ObserverBase() {}++    /**+     * Construct a new observer subscribed to events in the passed EventSet.+     */+    explicit Observer(EventSet eventSet) : ObserverBase(eventSet) {}++    /**+     * Invoked when this observer has been added to an observer container.+     *+     * For the typical observer container implementation a call to `attached`+     * will proceed a call to this method.+     *+     * The observer implementation must ensure that it remains alive as long as+     * it is in this container.+     *+     * @param ctr           Container observer has been added to.+     */+    virtual void addedToObserverContainer(+        ObserverContainerBase* ctr) noexcept = 0;++    /**+     * Invoked when this observer has been removed from an observer container.+     *+     * For the typical observer container implementation a call to `detached`+     * will have occurred before this method is called.+     *+     * @param ctr           Container observer has been removed from.+     */+    virtual void removedFromObserverContainer(+        ObserverContainerBase* ctr) noexcept = 0;++    /**+     * Invoked when this observer is moved from one container to another.+     *+     * Occurs in the case of move construction of a new object during which the+     * observers in the observer container are shifted from the old object to+     * the new object.+     *+     * @param oldCtr        Container observer has been removed from.+     * @param newCtr        Container observer has been added to.+     */+    virtual void movedToObserverContainer(+        ObserverContainerBase* oldCtr,+        ObserverContainerBase* newCtr) noexcept = 0;+  };++  /**+   * Returns the object associated with the container (e.g., observed object).+   *+   * @return             Return object associated with container or nullptr.+   */+  virtual Observed* getObject() const = 0;++  /**+   * Adds an observer to the container.+   *+   * If the observer is already in the container, this is a no-op.+   *+   * @param observer     Observer to add.+   */+  virtual void addObserver(std::shared_ptr<Observer> observer) = 0;++  /**+   * Adds an observer to the container.+   *+   * If the observer is already in the container, this is a no-op.+   *+   * @param observer     Observer to add.+   */+  virtual void addObserver(Observer* observer) {+    // create a shared_ptr holding an unmanaged ptr+    // this does not trigger control block allocation+    return addObserver(+        std::shared_ptr<Observer>(std::shared_ptr<void>(), observer));+  }++  /**+   * Removes an observer from the container.+   *+   * @param observer     Observer to remove.+   * @return             Whether the observer was found and removed.+   */+  virtual bool removeObserver(std::shared_ptr<Observer> observer) = 0;++  /**+   * Removes an observer from the container.+   *+   * @param observer     Observer to remove.+   * @return             Whether the observer was found and removed.+   */+  virtual bool removeObserver(Observer* observer) {+    // create a shared_ptr holding an unmanaged ptr+    // this does not trigger control block allocation+    return removeObserver(+        std::shared_ptr<Observer>(std::shared_ptr<void>(), observer));+  }++  /**+   * Get number of observers in container.+   *+   * @return             Number of observers in container.+   */+  size_t numObservers() const { return getStoreConst().size(); }++  /**+   * Get a list of observers in the container of type T.+   *+   * @tparam T           Type of observer to find.+   * @return             List of observers in the container of type T.+   */+  template <typename T = Observer>+  std::vector<T*> findObservers() {+    static_assert(+        std::is_base_of<Observer, T>::value,+        "T must derive from ObserverContainer::Observer");++    std::vector<T*> matchingObservers;+    using InvokeWhileIteratingPolicy = typename ObserverContainerStoreBase<+        Observer>::InvokeWhileIteratingPolicy;+    getStore().invokeForEachObserver(+        [&matchingObservers](Observer* observer) {+          auto castPtr = dynamic_cast<T*>(observer);+          if (castPtr) {+            matchingObservers.emplace_back(castPtr);+          }+        },+        InvokeWhileIteratingPolicy::CheckNoChange);++    return matchingObservers;+  }++  /**+   * Get all observers.+   *+   * @return             List of observers in the container.+   */+  std::vector<Observer*> getObservers() { return findObservers<Observer>(); }++  /**+   * Returns if any observer in the container is subscribed to a given event.+   *+   * TODO(bschlinker): The current implementation scans the entire container to+   * search for an observer subscribed to the requested event; we should cache+   * this information instead and update the cache on observer add / remove.+   *+   * @tparam event       Event in EventEnum.+   * @return             If there are observers subscribed to the given event.+   */+  template <EventEnum event>+  bool hasObserversForEvent() {+    bool foundObserverWithEvent = false;+    using InvokeWhileIteratingPolicy = typename ObserverContainerStoreBase<+        Observer>::InvokeWhileIteratingPolicy;+    getStore().invokeForEachObserver(+        [&foundObserverWithEvent](Observer* observer) {+          foundObserverWithEvent |= observer->getEventSet().isEnabled(event);+        },+        InvokeWhileIteratingPolicy::CheckNoChange);+    return foundObserverWithEvent;+  }++  /**+   * Helper class for observers that attach to single object / container.+   *+   * Does not have any thread safety, and thus can only be used if the observer+   * is driven exclusively by the same thread as the thread that controls the+   * object being observed.+   *+   * Tracks the container the observer is in (if any). If the observer's+   * destructor is triggered while it is in an container, it will be removed+   * from the container during the destruction process.+   */+  class ManagedObserver : public Observer {+   public:+    using Observer::Observer;+    using EventSet = typename Observer::EventSet;+    using EventSetBuilder = typename Observer::EventSetBuilder;++    ~ManagedObserver() override { detach(); }++    /**+     * Detach the observer (if currently attached).+     *+     * If the observer is not currently attached, this is a no-op.+     *+     * @return       If successfully detached.+     */+    bool detach() {+      if (!ctr_) {+        return false;+      }+      return ctr_->removeObserver(this);+    }++    /**+     * Return if the observer is observing an object.+     *+     * @return       If observer is observing an object.+     */+    bool isObserving() const {+      return ctr_ != nullptr && ctr_->getObject() != nullptr;+    }++    /**+     * Get the object that is being observed or nullptr.+     *+     * @return       Object being observed.+     */+    Observed* getObservedObject() const {+      if (!ctr_) {+        return nullptr;+      }+      return ctr_->getObject();+    }++   private:+    void addedToObserverContainer(+        ObserverContainerBase* ctr) noexcept override {+      CHECK(!ctr_);+      ctr_ = ctr;+    }++    void removedFromObserverContainer(+        ObserverContainerBase* ctr) noexcept override {+      CHECK_EQ(ctr_, ctr);+      ctr_ = nullptr;+    }++    void movedToObserverContainer(+        ObserverContainerBase* oldCtr,+        ObserverContainerBase* newCtr) noexcept override {+      CHECK_EQ(ctr_, oldCtr);+      CHECK_NE(ctr_, newCtr);+      ctr_ = newCtr;+    }++    // Container the observer is in (or nullptr).+    ObserverContainerBase* ctr_{nullptr};+  };++ protected:+  virtual ObserverContainerStoreBase<Observer>& getStore() = 0;++  virtual const ObserverContainerStoreBase<Observer>& getStoreConst() const = 0;++  void invokeInterfaceMethodImpl(+      Observed* observed,+      folly::Function<void(ObserverBase*, Observed*)>&& fn,+      const folly::Optional<EventEnum> maybeEvent = folly::none) noexcept {+    using InvokeWhileIteratingPolicy = typename ObserverContainerStoreBase<+        Observer>::InvokeWhileIteratingPolicy;+    getStore().invokeForEachObserver(+        [observed, maybeEvent, &fn](Observer* observer) {+          if (!maybeEvent.has_value() ||+              observer->getEventSet().isEnabled(maybeEvent.value())) {+            observer->invokeInterfaceMethod(observed, fn, maybeEvent);+          }+        },+        InvokeWhileIteratingPolicy::InvokeAdded);+    getStore().invokeForEachObserver(+        [observed, maybeEvent](ObserverBase* observer) {+          if (!maybeEvent.has_value() ||+              observer->getEventSet().isEnabled(maybeEvent.value())) {+            observer->postInvokeInterfaceMethod(observed);+          }+        },+        InvokeWhileIteratingPolicy::DoNotInvokeAdded);+  }+};++/**+ * Policy-based implementation of ObserverContainerBase.+ */+template <+    typename ObserverInterface,+    typename Observed,+    typename ContainerPolicy,+    typename StorePolicy = ObserverContainerStorePolicyDefault<>,+    std::size_t MaxConstructorCallbacks = 4>+class ObserverContainer+    : public ObserverContainerBase<+          ObserverInterface,+          Observed,+          ContainerPolicy> {+ public:+  using ContainerBase =+      ObserverContainerBase<ObserverInterface, Observed, ContainerPolicy>;+  using Observer = typename ContainerBase::Observer;+  using EventEnum = typename ContainerBase::EventEnum;+  using StoreBase = ObserverContainerStoreBase<Observer>;+  using ContainerConstructorCallbackList =+      ConstructorCallbackList<ObserverContainer, MaxConstructorCallbacks>;++  explicit ObserverContainer(Observed* obj)+      : obj_(CHECK_NOTNULL(obj)), constructorCallbackList_(this) {}++  ObserverContainer(Observed* obj, ObserverContainer&& observerContainer)+      : obj_(CHECK_NOTNULL(obj)), constructorCallbackList_(this) {+    using InvokeWhileIteratingPolicy =+        typename StoreBase::InvokeWhileIteratingPolicy;+    observerContainer.getStore().invokeForEachObserver(+        [this, &observerContainer](+            typename StoreBase::MaybeManagedObserverPointer& observer) {+          // observer may be a managed pointer (e.g., a shared_ptr), and+          // invokeForEachObserver passes a reference, so we need to copy the+          // observer object before calling remove so that it will not be+          // destroyed upon removal from the old ObserverContainer+          auto observerCopy = observer;+          CHECK_NOTNULL(observerCopy.get());++          // remove+          const bool removed = observerContainer.getStore().remove(observer);+          CHECK(removed);++          // add to new, operating solely on observerCopy+          const bool added = getStore().add(observerCopy);+          CHECK(added);+          observerCopy->movedToObserverContainer(&observerContainer, this);+          observerCopy->moved(+              observerContainer.getObject(), obj_, nullptr /* ctx */);+        },+        InvokeWhileIteratingPolicy::CheckNoAdded);+  }++  ~ObserverContainer() override {+    using InvokeWhileIteratingPolicy =+        typename StoreBase::InvokeWhileIteratingPolicy;+    getStore().invokeForEachObserver(+        [this](Observer* observer) {+          DestructorCheck::Safety dc(*observer);+          observer->destroyed(obj_, nullptr /* ctx */);+          if (!dc.destroyed()) {+            observer->removedFromObserverContainer(this);+          }+        },+        InvokeWhileIteratingPolicy::CheckNoAdded);+  }++  /**+   * Returns the object associated with the container (e.g., observed object).+   *+   * @return             Return object associated with container or nullptr.+   */+  Observed* getObject() const override { return obj_; }++  using ContainerBase::addObserver;+  using ContainerBase::removeObserver;++  /**+   * Adds an observer to the container.+   *+   * If the observer is already in the container, this is a no-op.+   *+   * @param observer     Observer to add.+   */+  void addObserver(std::shared_ptr<Observer> observer) override {+    CHECK_NOTNULL(observer.get());+    if (getStore().add(observer)) {+      DestructorCheck::Safety dc(*observer);+      observer->addedToObserverContainer(this);+      if (!dc.destroyed()) {+        observer->attached(obj_);+      }+    }+  }++  /**+   * Removes an observer from the container.+   *+   * @param observer     Observer to remove.+   * @return             Whether the observer was found and removed.+   */+  bool removeObserver(std::shared_ptr<Observer> observer) override {+    CHECK_NOTNULL(observer.get());+    if (getStore().remove(observer)) {+      DestructorCheck::Safety dc(*observer);+      observer->detached(obj_);+      if (!dc.destroyed()) {+        observer->removedFromObserverContainer(this);+      }+      return true;+    }+    return false;+  }++  /**+   * Add a callback fired each time obj of the observed type is constructed.+   *+   * Uses ConstructorCallbackList. Can be used to attach observers to all+   * objects of a given type.+   *+   * @throw std::length_error() if installing callback would exceed max allowed+   */+  static void addConstructorCallback(+      typename ContainerConstructorCallbackList::Callback cb) {+    ContainerConstructorCallbackList::addCallback(std::move(cb));+  }++  /**+   * Invokes an observer interface method on observers subscribed to an event.+   *+   * See instead `invokeInterfaceMethodAllObservers` to invoke an interface+   * method on all observers without filtering based on observer event+   * subscription.+   *+   * @tparam event       Associated event in EventEnum. The passed function will+   *                     only be called for observers subscribed to this event.+   * @param  fn          Function to call for each observer that takes a pointer+   *                     to the observer and invokes the interface method.+   */+  template <EventEnum event>+  void invokeInterfaceMethod(+      folly::Function<void(ObserverInterface*, Observed*)>&& fn) noexcept {+    this->invokeInterfaceMethodImpl(obj_, std::move(fn), event);+  }++  /**+   * Invokes an observer interface method on all observers.+   *+   * @param  fn          Function to call for each observer that takes a pointer+   *                     to the observer and invokes the interface method.+   */+  void invokeInterfaceMethodAllObservers(+      folly::Function<void(ObserverInterface*, Observed*)>&& fn) noexcept {+    this->invokeInterfaceMethodImpl(obj_, std::move(fn), folly::none);+  }++  ObserverContainer(const ObserverContainer&) = delete;+  ObserverContainer(ObserverContainer&&) = delete;+  ObserverContainer& operator=(const ObserverContainer&) = delete;+  ObserverContainer& operator=(ObserverContainer&& rhs) = delete;++ private:+  StoreBase& getStore() override { return store_; }+  const StoreBase& getStoreConst() const override { return store_; }++  // Object being observed.+  Observed* obj_{nullptr};++  // Store that contains the observers in the container.+  ObserverContainerStore<Observer, StorePolicy> store_;++  // Enables objects to register constructor callbacks.+  //+  // This can be used to enable observers to be attached to all objects of a+  // given type immediately upon object construction.+  //+  // Initialized last and in ObserverContainer instead of ObserverContainerBase+  // to ensure that the container has completed initialization and is ready to+  // add observers when any constructor callbacks are called.+  ContainerConstructorCallbackList constructorCallbackList_{this};+};++} // namespace folly
+ folly/folly/OperationCancelled.h view
@@ -0,0 +1,65 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <exception>++namespace folly {++/// IMPORTANT: `folly`-internal, do NOT use this in new user code. Instead:+///+///   - `co_yield coro::co_cancelled` to signal that a coro was cancelled.+///+///   - To check for cancellation in `folly::coro` coroutines, use one of:+///       // (1) default behavior+///       co_await coro::co_safe_point+///       // (2) custom behavior+///       auto& ctok = co_await coro::co_current_cancellation_token;+///       if (ctok.isCancellationRequested()) {+///         /* ... do stuff ... */+///         co_yield coro::co_cancelled;+///       }+///+///   - Store `stopped_result` to obtain a `result<T>` or `non_value_result`+///     in a stopped state. To check for it, use `res.has_stopped()`.+///+///   - To avoid depending on `OperationCancelled` in code that would do+///     `try-catch` in `folly::coro` coroutines, do this instead:+///+///       auto res = co_await coro::co_await_result(mightGetCancelled());+///       if (auto* ex = get_exception<MyErr>(res)) {+///         // HANDLE ERROR HERE+///       } else if (res.has_stopped()) {+///         // HANDLE CANCELLATION HERE+///         co_yield coro::co_cancelled;+///       } else { // get value, or propagate unhandled errors/cancellation+///         auto v = co_await coro::co_ready(std::move(res));+///       }+///+/// Rationale / purpose: For now, `folly` uses the `OperationCancelled`+/// exception to signal "this work was stopped".  However, as of C++26 (see+/// P2300, plus arguments in P1677), standard C++ differentiates between+/// "value", "error", and "stopped" completions.  Therefore, we ask end-user+/// code to use the above cancellation-specific constructs, WITHOUT assuming+/// that cancellation / stopping is implemented as an exception.+struct OperationCancelled final : public std::exception {+  const char* what() const noexcept override {+    return "coroutine operation cancelled";+  }+};++} // namespace folly
+ folly/folly/Optional.h view
@@ -0,0 +1,753 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++//+// Docs: https://fburl.com/fbcref_optional+//++#pragma once++/*+ * Optional - For conditional initialization of values, like boost::optional,+ * but with support for move semantics and emplacement.  Reference type support+ * has not been included due to limited use cases and potential confusion with+ * semantics of assignment: Assigning to an optional reference could quite+ * reasonably copy its value or redirect the reference.+ *+ * Optional can be useful when a variable might or might not be needed:+ *+ *  Optional<Logger> maybeLogger = ...;+ *  if (maybeLogger) {+ *    maybeLogger->log("hello");+ *  }+ *+ * Optional enables a 'null' value for types which do not otherwise have+ * nullability, especially useful for parameter passing:+ *+ * void testIterator(const unique_ptr<Iterator>& it,+ *                   initializer_list<int> idsExpected,+ *                   Optional<initializer_list<int>> ranksExpected = none) {+ *   for (int i = 0; it->next(); ++i) {+ *     EXPECT_EQ(it->doc().id(), idsExpected[i]);+ *     if (ranksExpected) {+ *       EXPECT_EQ(it->doc().rank(), (*ranksExpected)[i]);+ *     }+ *   }+ * }+ *+ * Optional models OptionalPointee, so calling 'get_pointer(opt)' will return a+ * pointer to nullptr if the 'opt' is empty, and a pointer to the value if it is+ * not:+ *+ *  Optional<int> maybeInt = ...;+ *  if (int* v = get_pointer(maybeInt)) {+ *    cout << *v << endl;+ *  }+ */++#include <cassert>+#include <cstddef>+#include <functional>+#include <new>+#include <optional>+#include <stdexcept>+#include <type_traits>+#include <utility>++#include <folly/Portability.h>+#include <folly/Traits.h>+#include <folly/Utility.h>+#include <folly/hash/traits.h>+#include <folly/lang/Exception.h>++namespace folly {++template <class Value>+class Optional;++namespace detail {+struct OptionalEmptyTag {};+template <class Value>+struct OptionalPromise;+template <class Value>+struct OptionalPromiseReturn;+} // namespace detail++struct None {+  enum class _secret { _token };++  /**+   * No default constructor to support both `op = {}` and `op = none`+   * as syntax for clearing an Optional, just like std::nullopt_t.+   */+  explicit constexpr None(_secret) {}+};+constexpr None none{None::_secret::_token};++class FOLLY_EXPORT OptionalEmptyException : public std::runtime_error {+ public:+  OptionalEmptyException()+      : std::runtime_error("Empty Optional cannot be unwrapped") {}+};++/**+ * Optional is superseded by std::optional. Now that the C++ has a standardized+ * implementation, Optional exists primarily for backward compatibility.+ */+template <class Value>+class Optional {+ public:+  typedef Value value_type;++  using promise_type = detail::OptionalPromise<Value>;++  static_assert(+      !std::is_reference<Value>::value,+      "Optional may not be used with reference types");+  static_assert(+      !std::is_abstract<Value>::value,+      "Optional may not be used with abstract types");++  /// Default-constructed Optionals are None.+  constexpr Optional() noexcept {}++  Optional(const Optional& src) noexcept(+      std::is_nothrow_copy_constructible<Value>::value) {+    if (src.hasValue()) {+      construct(src.value());+    }+  }++  Optional(Optional&& src) noexcept(+      std::is_nothrow_move_constructible<Value>::value) {+    if (src.hasValue()) {+      construct(std::move(src.value()));+      src.reset();+    }+  }++  constexpr /* implicit */ Optional(const None&) noexcept {}++  constexpr /* implicit */ Optional(Value&& newValue) noexcept(+      std::is_nothrow_move_constructible<Value>::value) {+    construct(std::move(newValue));+  }++  constexpr /* implicit */ Optional(const Value& newValue) noexcept(+      std::is_nothrow_copy_constructible<Value>::value) {+    construct(newValue);+  }++  /**+   * Creates an Optional with a value, where that value is constructed in-place.+   *+   * The std::in_place argument exists so that values can be default constructed+   * (i.e. have no arguments), since this would otherwise be confused with+   * default-constructing an Optional, which in turn results in None.+   */+  template <typename... Args>+  constexpr explicit Optional(std::in_place_t, Args&&... args) noexcept(+      std::is_nothrow_constructible<Value, Args...>::value)+      : Optional{PrivateConstructor{}, std::forward<Args>(args)...} {}++  template <typename U, typename... Args>+  constexpr explicit Optional(+      std::in_place_t,+      std::initializer_list<U> il,+      Args&&... args) noexcept(std::+                                   is_nothrow_constructible<+                                       Value,+                                       std::initializer_list<U>,+                                       Args...>::value)+      : Optional{PrivateConstructor{}, il, std::forward<Args>(args)...} {}++  // Used only when an Optional is used with coroutines on MSVC+  /* implicit */ Optional(const detail::OptionalPromiseReturn<Value>& p)+      : Optional{} {+    p.promise_->value_ = this;+  }++  // Conversions to ease migration to std::optional++  /// Allow construction of Optional from std::optional.+  template <+      typename U,+      typename = std::enable_if_t<std::is_same_v<U, std::optional<Value>>>>+  explicit Optional(U&& newValue) noexcept(+      std::is_nothrow_move_constructible<Value>::value) {+    if (newValue.has_value()) {+      construct(std::move(*newValue));+      newValue.reset();+    }+  }+  template <+      typename U,+      typename = std::enable_if_t<std::is_same_v<U, std::optional<Value>>>>+  explicit Optional(const U& newValue) noexcept(+      std::is_nothrow_copy_constructible<Value>::value) {+    if (newValue.has_value()) {+      construct(*newValue);+    }+  }+  /// Allow explicit cast to std::optional+  /// @methodset Migration+  explicit operator std::optional<Value>() && noexcept(+      std::is_nothrow_move_constructible<Value>::value) {+    std::optional<Value> ret = storage_.hasValue+        ? std::optional<Value>(std::move(storage_.value))+        : std::nullopt;+    reset();+    return ret;+  }+  explicit operator std::optional<Value>() const& noexcept(+      std::is_nothrow_copy_constructible<Value>::value) {+    return storage_.hasValue+        ? std::optional<Value>(storage_.value)+        : std::nullopt;+  }++  std::optional<Value> toStdOptional() && noexcept {+    return static_cast<std::optional<Value>>(std::move(*this));+  }++  std::optional<Value> toStdOptional() const& noexcept {+    return static_cast<std::optional<Value>>(*this);+  }++  /// Set the Optional+  /// @methodset Modifiers+  void assign(const None&) { reset(); }++  void assign(Optional&& src) {+    if (this != &src) {+      if (src.hasValue()) {+        assign(std::move(src.value()));+        src.reset();+      } else {+        reset();+      }+    }+  }++  void assign(const Optional& src) {+    if (src.hasValue()) {+      assign(src.value());+    } else {+      reset();+    }+  }++  void assign(Value&& newValue) {+    if (hasValue()) {+      FOLLY_PUSH_WARNING+      FOLLY_GCC_DISABLE_WARNING("-Wmaybe-uninitialized")+      storage_.value = std::move(newValue);+      FOLLY_POP_WARNING+    } else {+      construct(std::move(newValue));+    }+  }++  void assign(const Value& newValue) {+    if (hasValue()) {+      FOLLY_PUSH_WARNING+      FOLLY_GCC_DISABLE_WARNING("-Wmaybe-uninitialized")+      storage_.value = newValue;+      FOLLY_POP_WARNING+    } else {+      construct(newValue);+    }+  }++  /// @methodset Modifiers+  Optional& operator=(None) noexcept {+    reset();+    return *this;+  }++  template <class Arg>+  Optional& operator=(Arg&& arg) {+    assign(std::forward<Arg>(arg));+    return *this;+  }++  Optional& operator=(Optional&& other) noexcept(+      std::is_nothrow_move_assignable<Value>::value) {+    assign(std::move(other));+    return *this;+  }++  Optional& operator=(const Optional& other) noexcept(+      std::is_nothrow_copy_assignable<Value>::value) {+    assign(other);+    return *this;+  }++  /// Construct a new value in the Optional, in-place.+  /// @methodset Modifiers+  template <class... Args>+  Value& emplace(Args&&... args) {+    reset();+    construct(std::forward<Args>(args)...);+    return value();+  }++  template <class U, class... Args>+  typename std::enable_if<+      std::is_constructible<Value, std::initializer_list<U>&, Args&&...>::value,+      Value&>::type+  emplace(std::initializer_list<U> ilist, Args&&... args) {+    reset();+    construct(ilist, std::forward<Args>(args)...);+    return value();+  }++  /// Set the Optional to None+  /// @methodset Modifiers+  void reset() noexcept { storage_.clear(); }++  /// Set the Optional to None+  /// @methodset Modifiers+  void clear() noexcept { reset(); }++  /// @methodset Modifiers+  void swap(Optional& that) noexcept(std::is_nothrow_swappable_v<Value>) {+    if (hasValue() && that.hasValue()) {+      using std::swap;+      swap(value(), that.value());+    } else if (hasValue()) {+      that.emplace(std::move(value()));+      reset();+    } else if (that.hasValue()) {+      emplace(std::move(that.value()));+      that.reset();+    }+  }++  /// Get the value. Must have value.+  /// @methodset Getters+  constexpr const Value& value() const& {+    require_value();+    return storage_.value;+  }++  constexpr Value& value() & {+    require_value();+    return storage_.value;+  }++  constexpr Value&& value() && {+    require_value();+    return std::move(storage_.value);+  }++  constexpr const Value&& value() const&& {+    require_value();+    return std::move(storage_.value);+  }++  /// Get the value by pointer; nullptr if None.+  /// @methodset Getters+  const Value* get_pointer() const& {+    return storage_.hasValue ? &storage_.value : nullptr;+  }+  Value* get_pointer() & {+    return storage_.hasValue ? &storage_.value : nullptr;+  }+  Value* get_pointer() && = delete;++  /// Does this Optional have a value.+  /// @methodset Observers+  constexpr bool has_value() const noexcept { return storage_.hasValue; }++  /// Does this Optional have a value.+  /// @methodset Observers+  constexpr bool hasValue() const noexcept { return has_value(); }++  /// Does this Optional have a value.+  /// @methodset Observers+  constexpr explicit operator bool() const noexcept { return has_value(); }++  /// Get the value. Must have value.+  /// @methodset Getters+  constexpr const Value& operator*() const& { return value(); }+  constexpr Value& operator*() & { return value(); }+  constexpr const Value&& operator*() const&& { return std::move(value()); }+  constexpr Value&& operator*() && { return std::move(value()); }++  /// Get the value. Must have value.+  /// @methodset Getters+  constexpr const Value* operator->() const { return &value(); }+  constexpr Value* operator->() { return &value(); }++  /// Return a copy of the value if set, or a given default if not.+  /// @methodset Getters+  template <class U>+  constexpr Value value_or(U&& dflt) const& {+    if (storage_.hasValue) {+      return storage_.value;+    }++    return std::forward<U>(dflt);+  }++  template <class U>+  constexpr Value value_or(U&& dflt) && {+    if (storage_.hasValue) {+      return std::move(storage_.value);+    }++    return std::forward<U>(dflt);+  }++ private:+  friend struct detail::OptionalPromiseReturn<Value>;++  template <class T>+  friend constexpr Optional<std::decay_t<T>> make_optional(T&&);+  template <class T, class... Args>+  friend constexpr Optional<T> make_optional(Args&&... args);+  template <class T, class U, class... As>+  friend constexpr Optional<T> make_optional(std::initializer_list<U>, As&&...);++  /**+   * Construct the optional in place, this is duplicated as a non-explicit+   * constructor to allow returning values that are non-movable from+   * make_optional using list initialization.+   *+   * Until C++17, at which point this will become unnecessary because of+   * specified prvalue elision.+   */+  struct PrivateConstructor {+    explicit PrivateConstructor() = default;+  };+  template <typename... Args>+  constexpr Optional(PrivateConstructor, Args&&... args) noexcept(+      std::is_nothrow_constructible<Value, Args&&...>::value) {+    construct(std::forward<Args>(args)...);+  }+  // for when coroutine promise return-object conversion is eager+  explicit Optional(detail::OptionalEmptyTag, Optional*& pointer) noexcept {+    pointer = this;+  }++  void require_value() const {+    if (!storage_.hasValue) {+      throw_exception<OptionalEmptyException>();+    }+  }++  template <class... Args>+  void construct(Args&&... args) {+    const void* ptr = &storage_.value;+    // For supporting const types.+    new (const_cast<void*>(ptr)) Value(std::forward<Args>(args)...);+    storage_.hasValue = true;+  }++  struct StorageTriviallyDestructible {+    union {+      char emptyState;+      Value value;+    };+    bool hasValue;++    constexpr StorageTriviallyDestructible()+        : emptyState(unsafe_default_initialized), hasValue{false} {}+    void clear() { hasValue = false; }+  };++  struct StorageNonTriviallyDestructible {+    union {+      char emptyState;+      Value value;+    };+    bool hasValue;++    FOLLY_CXX20_CONSTEXPR StorageNonTriviallyDestructible() : hasValue{false} {}+    ~StorageNonTriviallyDestructible() { clear(); }++    void clear() {+      if (hasValue) {+        hasValue = false;+        FOLLY_PUSH_WARNING+        FOLLY_GCC_DISABLE_WARNING("-Wmaybe-uninitialized")+        value.~Value();+        FOLLY_POP_WARNING+      }+    }+  };++  using Storage = typename std::conditional<+      std::is_trivially_destructible<Value>::value,+      StorageTriviallyDestructible,+      StorageNonTriviallyDestructible>::type;++  Storage storage_;+};++template <class T>+const T* get_pointer(const Optional<T>& opt) {+  return opt.get_pointer();+}++template <class T>+T* get_pointer(Optional<T>& opt) {+  return opt.get_pointer();+}++template <class T>+void swap(Optional<T>& a, Optional<T>& b) noexcept(noexcept(a.swap(b))) {+  a.swap(b);+}++template <class T>+constexpr Optional<std::decay_t<T>> make_optional(T&& v) {+  using PrivateConstructor =+      typename folly::Optional<std::decay_t<T>>::PrivateConstructor;+  return {PrivateConstructor{}, std::forward<T>(v)};+}++template <class T, class... Args>+constexpr folly::Optional<T> make_optional(Args&&... args) {+  using PrivateConstructor = typename folly::Optional<T>::PrivateConstructor;+  return {PrivateConstructor{}, std::forward<Args>(args)...};+}++template <class T, class U, class... Args>+constexpr folly::Optional<T> make_optional(+    std::initializer_list<U> il, Args&&... args) {+  using PrivateConstructor = typename folly::Optional<T>::PrivateConstructor;+  return {PrivateConstructor{}, il, std::forward<Args>(args)...};+}++///////////////////////////////////////////////////////////////////////////////+// Comparisons.++template <class U, class V>+constexpr bool operator==(const Optional<U>& a, const V& b) {+  return a.hasValue() && a.value() == b;+}++template <class U, class V>+constexpr bool operator!=(const Optional<U>& a, const V& b) {+  return !(a == b);+}++template <class U, class V>+constexpr bool operator==(const U& a, const Optional<V>& b) {+  return b.hasValue() && b.value() == a;+}++template <class U, class V>+constexpr bool operator!=(const U& a, const Optional<V>& b) {+  return !(a == b);+}++template <class U, class V>+constexpr bool operator==(const Optional<U>& a, const Optional<V>& b) {+  if (a.hasValue() != b.hasValue()) {+    return false;+  }+  if (a.hasValue()) {+    return a.value() == b.value();+  }+  return true;+}++template <class U, class V>+constexpr bool operator!=(const Optional<U>& a, const Optional<V>& b) {+  return !(a == b);+}++template <class U, class V>+constexpr bool operator<(const Optional<U>& a, const Optional<V>& b) {+  if (a.hasValue() != b.hasValue()) {+    return a.hasValue() < b.hasValue();+  }+  if (a.hasValue()) {+    return a.value() < b.value();+  }+  return false;+}++template <class U, class V>+constexpr bool operator>(const Optional<U>& a, const Optional<V>& b) {+  return b < a;+}++template <class U, class V>+constexpr bool operator<=(const Optional<U>& a, const Optional<V>& b) {+  return !(b < a);+}++template <class U, class V>+constexpr bool operator>=(const Optional<U>& a, const Optional<V>& b) {+  return !(a < b);+}++// Suppress comparability of Optional<T> with T, despite implicit conversion.+template <class V>+bool operator<(const Optional<V>&, const V& other) = delete;+template <class V>+bool operator<=(const Optional<V>&, const V& other) = delete;+template <class V>+bool operator>=(const Optional<V>&, const V& other) = delete;+template <class V>+bool operator>(const Optional<V>&, const V& other) = delete;+template <class V>+bool operator<(const V& other, const Optional<V>&) = delete;+template <class V>+bool operator<=(const V& other, const Optional<V>&) = delete;+template <class V>+bool operator>=(const V& other, const Optional<V>&) = delete;+template <class V>+bool operator>(const V& other, const Optional<V>&) = delete;++// Comparisons with none+template <class V>+constexpr bool operator==(const Optional<V>& a, None) noexcept {+  return !a.hasValue();+}+template <class V>+constexpr bool operator==(None, const Optional<V>& a) noexcept {+  return !a.hasValue();+}+template <class V>+constexpr bool operator<(const Optional<V>&, None) noexcept {+  return false;+}+template <class V>+constexpr bool operator<(None, const Optional<V>& a) noexcept {+  return a.hasValue();+}+template <class V>+constexpr bool operator>(const Optional<V>& a, None) noexcept {+  return a.hasValue();+}+template <class V>+constexpr bool operator>(None, const Optional<V>&) noexcept {+  return false;+}+template <class V>+constexpr bool operator<=(None, const Optional<V>&) noexcept {+  return true;+}+template <class V>+constexpr bool operator<=(const Optional<V>& a, None) noexcept {+  return !a.hasValue();+}+template <class V>+constexpr bool operator>=(const Optional<V>&, None) noexcept {+  return true;+}+template <class V>+constexpr bool operator>=(None, const Optional<V>& a) noexcept {+  return !a.hasValue();+}++///////////////////////////////////////////////////////////////////////////////++} // namespace folly++// Allow usage of Optional<T> in std::unordered_map and std::unordered_set+FOLLY_NAMESPACE_STD_BEGIN+template <class T>+struct hash<+    folly::enable_std_hash_helper<folly::Optional<T>, remove_const_t<T>>> {+  size_t operator()(folly::Optional<T> const& obj) const {+    return static_cast<bool>(obj) ? hash<remove_const_t<T>>()(*obj) : 0;+  }+};+FOLLY_NAMESPACE_STD_END++// Enable the use of folly::Optional with `co_await`+// Inspired by https://github.com/toby-allsopp/coroutine_monad+#if FOLLY_HAS_COROUTINES+#include <folly/coro/Coroutine.h>++namespace folly {+namespace detail {+template <typename Value>+struct OptionalPromise;++template <typename Value>+struct OptionalPromiseReturn {+  Optional<Value> storage_;+  Optional<Value>*& pointer_;++  /* implicit */ OptionalPromiseReturn(OptionalPromise<Value>& p) noexcept+      : pointer_{p.value_} {+    pointer_ = &storage_;+  }+  OptionalPromiseReturn(OptionalPromiseReturn const&) = delete;+  // letting dtor be trivial makes the coroutine crash+  // TODO: fix clang/llvm codegen+  ~OptionalPromiseReturn() {}+  /* implicit */ operator Optional<Value>() {+    // handle both deferred and eager return-object conversion behaviors+    // see docs for detect_promise_return_object_eager_conversion+    if (folly::coro::detect_promise_return_object_eager_conversion()) {+      assert(!storage_.has_value());+      return Optional{OptionalEmptyTag{}, pointer_}; // eager+    } else {+      return std::move(storage_); // deferred+    }+  }+};++template <typename Value>+struct OptionalPromise {+  Optional<Value>* value_ = nullptr;+  OptionalPromise() = default;+  OptionalPromise(OptionalPromise const&) = delete;+  OptionalPromiseReturn<Value> get_return_object() noexcept { return *this; }+  coro::suspend_never initial_suspend() const noexcept { return {}; }+  coro::suspend_never final_suspend() const noexcept { return {}; }+  template <typename U = Value>+  void return_value(U&& u) {+    *value_ = static_cast<U&&>(u);+  }+  void unhandled_exception() {+    // Technically, throwing from unhandled_exception is underspecified:+    // https://github.com/GorNishanov/CoroutineWording/issues/17+    rethrow_current_exception();+  }+};++template <typename Value>+struct OptionalAwaitable {+  Optional<Value> o_;+  bool await_ready() const noexcept { return o_.hasValue(); }+  Value await_resume() { return std::move(o_.value()); }++  // Explicitly only allow suspension into an OptionalPromise+  template <typename U>+  void await_suspend(coro::coroutine_handle<OptionalPromise<U>> h) const {+    // Abort the rest of the coroutine. resume() is not going to be called+    h.destroy();+  }+};+} // namespace detail++template <typename Value>+detail::OptionalAwaitable<Value>+/* implicit */ operator co_await(Optional<Value> o) {+  return {std::move(o)};+}+} // namespace folly++#endif // FOLLY_HAS_COROUTINES
+ folly/folly/Overload.h view
@@ -0,0 +1,325 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <type_traits>+#include <utility>++#include <folly/Portability.h>+#include <folly/Traits.h>+#include <folly/functional/Invoke.h>++/**+ * folly implementation of `std::overload` like functionality+ *+ * Example:+ *  struct One {};+ *  struct Two {};+ *  boost::variant<One, Two> value;+ *+ *  variant_match(value,+ *    [] (const One& one) { ... },+ *    [] (const Two& two) { ... });+ */++namespace folly {++namespace detail {++// MSVC does not implement noexcept deduction https://godbolt.org/z/Mxdjao1q6+#if !defined(_MSC_VER)+#define FOLLY_DETAIL_NOEXCEPT_SPECIFICATION noexcept(Noexcept)+#define FOLLY_DETAIL_NOEXCEPT_DECLARATION bool Noexcept,+#else+#define FOLLY_DETAIL_NOEXCEPT_SPECIFICATION+#define FOLLY_DETAIL_NOEXCEPT_DECLARATION+#endif++template <typename T>+struct FunctionClassType {+  using type = T;+};++// You cannot derive from a pointer to function, so wrap it in a class++template <FOLLY_DETAIL_NOEXCEPT_DECLARATION typename Return, typename... Args>+struct FunctionClassType<Return (*)(+    Args...) FOLLY_DETAIL_NOEXCEPT_SPECIFICATION> {+  using Ptr = Return (*)(Args...) FOLLY_DETAIL_NOEXCEPT_SPECIFICATION;+  struct type {+    /* implicit */ constexpr type(Ptr function) noexcept+        : function_(function) {}+    constexpr auto operator()(Args... args) const+        FOLLY_DETAIL_NOEXCEPT_SPECIFICATION->Return {+      return function_(std::forward<Args>(args)...);+    }++   private:+    Ptr function_;+  };+};++// You cannot derive from a pointer to member function, so wrap it in a class.+// This cannot be implemented with+// `std::enable_if_t<std::is_member_pointer_v<T>>` because you don't get+// preferred overload resolution on the object type to match const / ref+// qualifiers.++template <+    FOLLY_DETAIL_NOEXCEPT_DECLARATION typename Return,+    typename Self,+    typename... Args>+struct FunctionClassType<Return (Self::*)(+    Args...) FOLLY_DETAIL_NOEXCEPT_SPECIFICATION> {+  using Ptr = Return (Self::*)(Args...) FOLLY_DETAIL_NOEXCEPT_SPECIFICATION;+  struct type {+    /* implicit */ constexpr type(Ptr memberPointer) noexcept+        : memberPointer_(memberPointer) {}+    constexpr auto operator()(Self& self, Args... args) const+        FOLLY_DETAIL_NOEXCEPT_SPECIFICATION->Return {+      return (self.*memberPointer_)(std::forward<Args>(args)...);+    }+    constexpr auto operator()(Self&& self, Args... args) const+        FOLLY_DETAIL_NOEXCEPT_SPECIFICATION->Return {+      return (self.*memberPointer_)(std::forward<Args>(args)...);+    }++   private:+    Ptr memberPointer_;+  };+};++template <+    FOLLY_DETAIL_NOEXCEPT_DECLARATION typename Return,+    typename Self,+    typename... Args>+struct FunctionClassType<Return (Self::*)(+    Args...) const FOLLY_DETAIL_NOEXCEPT_SPECIFICATION> {+  using Ptr =+      Return (Self::*)(Args...) const FOLLY_DETAIL_NOEXCEPT_SPECIFICATION;+  struct type {+    /* implicit */ constexpr type(Ptr memberPointer) noexcept+        : memberPointer_(memberPointer) {}+    constexpr auto operator()(const Self& self, Args... args) const+        FOLLY_DETAIL_NOEXCEPT_SPECIFICATION->Return {+      return (self.*memberPointer_)(std::forward<Args>(args)...);+    }++   private:+    Ptr memberPointer_;+  };+};++template <+    FOLLY_DETAIL_NOEXCEPT_DECLARATION typename Return,+    typename Self,+    typename... Args>+struct FunctionClassType<+    Return (Self::*)(Args...) & FOLLY_DETAIL_NOEXCEPT_SPECIFICATION> {+  using Ptr = Return (Self::*)(Args...) & FOLLY_DETAIL_NOEXCEPT_SPECIFICATION;+  struct type {+    /* implicit */ constexpr type(Ptr memberPointer) noexcept+        : memberPointer_(memberPointer) {}+    constexpr auto operator()(Self& self, Args&&... args) const+        FOLLY_DETAIL_NOEXCEPT_SPECIFICATION->Return {+      return (self.*memberPointer_)(std::forward<Args>(args)...);+    }++   private:+    Ptr memberPointer_;+  };+};++template <+    FOLLY_DETAIL_NOEXCEPT_DECLARATION typename Return,+    typename Self,+    typename... Args>+struct FunctionClassType<+    Return (Self::*)(Args...) const & FOLLY_DETAIL_NOEXCEPT_SPECIFICATION> {+  using Ptr =+      Return (Self::*)(Args...) const& FOLLY_DETAIL_NOEXCEPT_SPECIFICATION;+  struct type {+    /* implicit */ constexpr type(Ptr memberPointer) noexcept+        : memberPointer_(memberPointer) {}+    constexpr auto operator()(const Self& self, Args... args) const+        FOLLY_DETAIL_NOEXCEPT_SPECIFICATION->Return {+      return (self.*memberPointer_)(std::forward<Args>(args)...);+    }++   private:+    Ptr memberPointer_;+  };+};++template <+    FOLLY_DETAIL_NOEXCEPT_DECLARATION typename Return,+    typename Self,+    typename... Args>+struct FunctionClassType<+    Return (Self::*)(Args...) && FOLLY_DETAIL_NOEXCEPT_SPECIFICATION> {+  using Ptr = Return (Self::*)(Args...) && FOLLY_DETAIL_NOEXCEPT_SPECIFICATION;+  struct type {+    /* implicit */ constexpr type(Ptr memberPointer) noexcept+        : memberPointer_(memberPointer) {}+    constexpr auto operator()(Self&& self, Args... args) const+        FOLLY_DETAIL_NOEXCEPT_SPECIFICATION->Return {+      return (std::move(self).*memberPointer_)(std::forward<Args>(args)...);+    }++   private:+    Ptr memberPointer_;+  };+};++template <+    FOLLY_DETAIL_NOEXCEPT_DECLARATION typename Return,+    typename Self,+    typename... Args>+struct FunctionClassType<+    Return (Self::*)(Args...) const && FOLLY_DETAIL_NOEXCEPT_SPECIFICATION> {+  using Ptr =+      Return (Self::*)(Args...) const&& FOLLY_DETAIL_NOEXCEPT_SPECIFICATION;+  struct type {+    /* implicit */ constexpr type(Ptr memberPointer) noexcept+        : memberPointer_(memberPointer) {}+    constexpr auto operator()(const Self&& self, Args... args) const+        FOLLY_DETAIL_NOEXCEPT_SPECIFICATION->Return {+      return (std::move(self).*memberPointer_)(std::forward<Args>(args)...);+    }++   private:+    Ptr memberPointer_;+  };+};++template <typename T, typename Self>+struct FunctionClassType<T Self::*> {+  using Ptr = T Self::*;+  struct type {+    /* implicit */ constexpr type(Ptr memberPointer) noexcept+        : memberPointer_(memberPointer) {}+    constexpr auto operator()(Self& self) const noexcept -> T& {+      return self.*memberPointer_;+    }+    constexpr auto operator()(const Self& self) const noexcept -> const T& {+      return self.*memberPointer_;+    }+    constexpr auto operator()(Self&& self) const noexcept -> T&& {+      return std::move(self).*memberPointer_;+    }+    constexpr auto operator()(const Self&& self) const noexcept -> const T&& {+      return std::move(self).*memberPointer_;+    }++   private:+    Ptr memberPointer_;+  };+};++#undef FOLLY_DETAIL_NOEXCEPT_DECLARATION+#undef FOLLY_DETAIL_NOEXCEPT_SPECIFICATION++template <typename...>+struct Overload {};++template <typename Case, typename... Cases>+struct Overload<Case, Cases...> : Overload<Cases...>, Case {+  explicit constexpr Overload(Case c, Cases... cs)+      : Overload<Cases...>(std::move(cs)...), Case(std::move(c)) {}++  using Case::operator();+  using Overload<Cases...>::operator();+};++template <typename Case>+struct Overload<Case> : Case {+  explicit constexpr Overload(Case c) : Case(std::move(c)) {}++  using Case::operator();+};+} // namespace detail++/*+ * Combine multiple `Cases` in one function object+ *+ * Each element of `Cases` must be a class type with `operator()`, a pointer to+ * a function, a pointer to a member function, or a pointer to member data.+ * `final` types and pointers to `volatile`-qualified member functions are not+ * supported. If the `Case` type is a pointer to member, the first argument must+ * be a class type or reference to class type (pointer to class type is not+ * supported).+ */+template <typename... Cases>+constexpr decltype(auto) overload(Cases&&... cases) {+  return detail::Overload<typename detail::FunctionClassType<+      typename std::decay<Cases>::type>::type...>{+      std::forward<Cases>(cases)...};+}++namespace overload_detail {+FOLLY_CREATE_MEMBER_INVOKER(valueless_by_exception, valueless_by_exception);+FOLLY_PUSH_WARNING+FOLLY_MSVC_DISABLE_WARNING(4003) /* not enough arguments to macro */+FOLLY_CREATE_FREE_INVOKER(visit, visit);+FOLLY_CREATE_FREE_INVOKER(apply_visitor, apply_visitor);+FOLLY_POP_WARNING+} // namespace overload_detail++/*+ * Match `Variant` with one of the `Cases`+ *+ * Note: you can also use `[] (const auto&) {...}` as default case+ *+ * Selects `visit` if `v.valueless_by_exception()` available and the call to+ * `visit` is valid (e.g. `std::variant`). Otherwise, selects `apply_visitor`+ * (e.g. `boost::variant`, `folly::DiscriminatedPtr`).+ */+template <typename Variant, typename... Cases>+decltype(auto) variant_match(Variant&& variant, Cases&&... cases) {+  using invoker = std::conditional_t<+      folly::Conjunction<+          is_invocable<overload_detail::valueless_by_exception, Variant>,+          is_invocable<+              overload_detail::visit,+              decltype(overload(std::forward<Cases>(cases)...)),+              Variant>>::value,+      overload_detail::visit,+      overload_detail::apply_visitor>;+  return invoker{}(+      overload(std::forward<Cases>(cases)...), std::forward<Variant>(variant));+}++template <typename R, typename Variant, typename... Cases>+R variant_match(Variant&& variant, Cases&&... cases) {+  auto f = [&](auto&& v) -> R {+    if constexpr (std::is_void<R>::value) {+      overload(std::forward<Cases>(cases)...)(std::forward<decltype(v)>(v));+    } else {+      return overload(std::forward<Cases>(cases)...)(+          std::forward<decltype(v)>(v));+    }+  };+  using invoker = std::conditional_t<+      folly::Conjunction<+          is_invocable<overload_detail::valueless_by_exception, Variant>,+          is_invocable<overload_detail::visit, decltype(f), Variant>>::value,+      overload_detail::visit,+      overload_detail::apply_visitor>;+  return invoker{}(f, std::forward<Variant>(variant));+}++} // namespace folly
+ folly/folly/PackedSyncPtr.h view
@@ -0,0 +1,154 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <type_traits>++#include <glog/logging.h>++#include <folly/Portability.h>+#include <folly/synchronization/SmallLocks.h>++#if FOLLY_X64 || FOLLY_PPC64 || FOLLY_AARCH64+#define FOLLY_HAS_PACKED_SYNC_PTR 1+#else+#define FOLLY_HAS_PACKED_SYNC_PTR 0+#endif++#if FOLLY_HAS_PACKED_SYNC_PTR++/*+ * An 8-byte pointer with an integrated spin lock and 15-bit integer+ * (you can use this for a size of the allocation, if you want, or+ * something else, or nothing).+ *+ * This is using an x64-specific detail about the effective virtual+ * address space.  Long story short: the upper two bytes of all our+ * pointers will be zero in reality---and if you have a couple billion+ * such pointers in core, it makes pretty good sense to try to make+ * use of that memory.  The exact details can be perused here:+ *+ *    http://en.wikipedia.org/wiki/X86-64#Canonical_form_addresses+ *+ * This is not a "smart" pointer: nothing automagical is going on+ * here.  Locking is up to the user.  Resource deallocation is up to+ * the user.  Locks are never acquired or released outside explicit+ * calls to lock() and unlock().+ *+ * Change the value of the raw pointer with set(), but you must hold+ * the lock when calling this function if multiple threads could be+ * using this class.+ *+ * TODO(jdelong): should we use the low order bit for the lock, so we+ * get a whole 16-bits for our integer?  (There's also 2 more bits+ * down there if the pointer comes from malloc.)+ */++namespace folly {++template <class T>+class PackedSyncPtr {+  // This just allows using this class even with T=void.  Attempting+  // to use the operator* or operator[] on a PackedSyncPtr<void> will+  // still properly result in a compile error.+  typedef typename std::add_lvalue_reference<T>::type reference;++ public:+  /*+   * If you default construct one of these, you must call this init()+   * function before using it.+   *+   * (We are avoiding a constructor to ensure gcc allows us to put+   * this class in packed structures.)+   */+  void init(T* initialPtr = nullptr, uint16_t initialExtra = 0) {+    auto intPtr = reinterpret_cast<uintptr_t>(initialPtr);+    CHECK(!(intPtr >> 48));+    data_.init(intPtr << 16);+    setExtra(initialExtra);+  }++  /*+   * Sets a new pointer.  You must hold the lock when calling this+   * function, or else be able to guarantee no other threads could be+   * using this PackedSyncPtr<>.+   */+  void set(T* t) {+    auto intPtr = reinterpret_cast<uintptr_t>(t);+    CHECK(!(intPtr >> 48));+    data_.setData((intPtr << 16) | uintptr_t(extra()));+  }++  /*+   * Get the pointer.+   *+   * You can call any of these without holding the lock, with the+   * normal types of behavior you'll get on x64 from reading a pointer+   * without locking.+   */+  T* get() const { return reinterpret_cast<T*>(data_.getData() >> 16); }+  T* operator->() const { return get(); }+  reference operator*() const { return *get(); }+  reference operator[](std::ptrdiff_t i) const { return get()[i]; }++  // Synchronization (logically const, even though this mutates our+  // locked state: you can lock a const PackedSyncPtr<T> to read it).+  void lock() const { data_.lock(); }+  void unlock() const { data_.unlock(); }+  bool try_lock() const { return data_.try_lock(); }++  /*+   * Access extra data stored in unused bytes of the pointer.+   *+   * It is ok to call this without holding the lock.+   */+  uint16_t extra() const { return data_.getData() & 0xffff; }++  /*+   * Don't try to put anything into this that has the high bit set:+   * that's what we're using for the mutex.+   *+   * Don't call this without holding the lock.+   */+  void setExtra(uint16_t extra) {+    CHECK(!(extra & 0x8000));+    auto ptr = data_.getData();+    data_.setData(uintptr_t(extra) | (ptr & (-1ull << 16)));+  }++ private:+  PicoSpinLock<uintptr_t, 15> data_;+};++static_assert(+    std::is_standard_layout<PackedSyncPtr<void>>::value &&+        std::is_trivial<PackedSyncPtr<void>>::value,+    "PackedSyncPtr must be kept a POD type.");+static_assert(+    sizeof(PackedSyncPtr<void>) == 8,+    "PackedSyncPtr should be only 8 bytes---something is "+    "messed up");++template <typename T>+std::ostream& operator<<(std::ostream& os, const PackedSyncPtr<T>& ptr) {+  os << "PackedSyncPtr(" << ptr.get() << ", " << ptr.extra() << ")";+  return os;+}++} // namespace folly++#endif
+ folly/folly/Padded.h view
@@ -0,0 +1,464 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <algorithm>+#include <cassert>+#include <cstdint>+#include <cstring>+#include <functional>+#include <iterator>+#include <limits>+#include <type_traits>++#include <boost/iterator/iterator_adaptor.hpp>++#include <folly/Portability.h>+#include <folly/Traits.h>+#include <folly/functional/Invoke.h>+#include <folly/portability/SysTypes.h>++/**+ * Code that aids in storing data aligned on block (possibly cache-line)+ * boundaries, perhaps with padding.+ *+ * Class Node represents one block.  Given an iterator to a container of+ * Node, class Iterator encapsulates an iterator to the underlying elements.+ * Adaptor converts a sequence of Node into a sequence of underlying elements+ * (not fully compatible with STL container requirements, see comments+ * near the Node class declaration).+ */++namespace folly {+namespace padded {++/**+ * A Node is a fixed-size container of as many objects of type T as would+ * fit in a region of memory of size NS.  The last NS % sizeof(T)+ * bytes are ignored and uninitialized.+ *+ * Node only works for trivial types, which is usually not a concern.  This+ * is intentional: Node itself is trivial, which means that it can be+ * serialized / deserialized using a simple memcpy.+ */+template <class T, size_t NS>+class Node {+  static_assert(+      std::is_trivial_v<T> && sizeof(T) <= NS && NS % alignof(T) == 0);++ public:+  typedef T value_type;+  static constexpr size_t kNodeSize = NS;+  static constexpr size_t kElementCount = NS / sizeof(T);+  static constexpr size_t kPaddingBytes = NS % sizeof(T);++  T* data() { return storage_.data; }+  const T* data() const { return storage_.data; }++  bool operator==(const Node& other) const {+    return memcmp(data(), other.data(), sizeof(T) * kElementCount) == 0;+  }+  bool operator!=(const Node& other) const { return !(*this == other); }++  /**+   * Return the number of nodes needed to represent n values.  Rounds up.+   */+  static constexpr size_t nodeCount(size_t n) {+    return (n + kElementCount - 1) / kElementCount;+  }++  /**+   * Return the total byte size needed to represent n values, rounded up+   * to the nearest full node.+   */+  static constexpr size_t paddedByteSize(size_t n) { return nodeCount(n) * NS; }++  /**+   * Return the number of bytes used for padding n values.+   * Note that, even if n is a multiple of kElementCount, this may+   * return non-zero if kPaddingBytes != 0, as the padding at the end of+   * the last node is not included in the result.+   */+  static constexpr size_t paddingBytes(size_t n) {+    return (+        n ? (kPaddingBytes ++             (kElementCount - 1 - (n - 1) % kElementCount) * sizeof(T))+          : 0);+  }++  /**+   * Return the minimum byte size needed to represent n values.+   * Does not round up.  Even if n is a multiple of kElementCount, this+   * may be different from paddedByteSize() if kPaddingBytes != 0, as+   * the padding at the end of the last node is not included in the result.+   * Note that the calculation below works for n=0 correctly (returns 0).+   */+  static constexpr size_t unpaddedByteSize(size_t n) {+    return paddedByteSize(n) - paddingBytes(n);+  }++ private:+  union Storage {+    unsigned char bytes[NS];+    T data[kElementCount];+  } storage_;+};++// We must define kElementCount and kPaddingBytes to work around a bug+// in gtest that odr-uses them.+template <class T, size_t NS>+constexpr size_t Node<T, NS>::kNodeSize;+template <class T, size_t NS>+constexpr size_t Node<T, NS>::kElementCount;+template <class T, size_t NS>+constexpr size_t Node<T, NS>::kPaddingBytes;++template <class Iter>+class Iterator;++namespace detail {++FOLLY_CREATE_MEMBER_INVOKER(emplace_back, emplace_back);++// Helper class template to define a base class for Iterator (below) and save+// typing.+template <+    template <class>+    class Class,+    class Iter,+    class Traits = std::iterator_traits<Iter>,+    class Ref = typename Traits::reference,+    class Val = typename Traits::value_type::value_type>+using IteratorBase = boost::iterator_adaptor<+    Class<Iter>, // CRTC+    Iter, // Base iterator type+    Val, // Value type+    boost::use_default, // Category or traversal+    like_t<Ref, Val>>; // Reference type++} // namespace detail++/**+ * Wrapper around iterators to Node to return iterators to the underlying+ * node elements.+ */+template <class Iter>+class Iterator : public detail::IteratorBase<Iterator, Iter> {+  using Super = detail::IteratorBase<Iterator, Iter>;++ public:+  using Node = typename std::iterator_traits<Iter>::value_type;++  Iterator() : pos_(0) {}++  explicit Iterator(Iter base) : Super(base), pos_(0) {}++  // Return the current node and the position inside the node+  const Node& node() const { return *this->base_reference(); }+  size_t pos() const { return pos_; }++ private:+  typename Super::reference dereference() const {+    return (*this->base_reference()).data()[pos_];+  }++  bool equal(const Iterator& other) const {+    return (+        this->base_reference() == other.base_reference() && pos_ == other.pos_);+  }++  void advance(typename Super::difference_type n) {+    constexpr ssize_t elementCount = Node::kElementCount; // signed!+    ssize_t newPos = pos_ + n;+    if (newPos >= 0 && newPos < elementCount) {+      pos_ = newPos;+      return;+    }+    ssize_t nblocks = newPos / elementCount;+    newPos %= elementCount;+    if (newPos < 0) {+      --nblocks; // negative+      newPos += elementCount;+    }+    this->base_reference() += nblocks;+    pos_ = newPos;+  }++  void increment() {+    if (++pos_ == Node::kElementCount) {+      ++this->base_reference();+      pos_ = 0;+    }+  }++  void decrement() {+    if (--pos_ == -1) {+      --this->base_reference();+      pos_ = Node::kElementCount - 1;+    }+  }++  typename Super::difference_type distance_to(const Iterator& other) const {+    constexpr ssize_t elementCount = Node::kElementCount; // signed!+    ssize_t nblocks =+        std::distance(this->base_reference(), other.base_reference());+    return nblocks * elementCount + (other.pos_ - pos_);+  }++  friend class boost::iterator_core_access;+  ssize_t pos_; // signed for easier advance() implementation+};++/**+ * Given a container to Node, return iterators to the first element in+ * the first Node / one past the last element in the last Node.+ * Note that the last node is assumed to be full; if that's not the case,+ * subtract from end() as appropriate.+ */++template <class Container>+Iterator<typename Container::const_iterator> cbegin(const Container& c) {+  return Iterator<typename Container::const_iterator>(std::begin(c));+}++template <class Container>+Iterator<typename Container::const_iterator> cend(const Container& c) {+  return Iterator<typename Container::const_iterator>(std::end(c));+}++template <class Container>+Iterator<typename Container::const_iterator> begin(const Container& c) {+  return cbegin(c);+}++template <class Container>+Iterator<typename Container::const_iterator> end(const Container& c) {+  return cend(c);+}++template <class Container>+Iterator<typename Container::iterator> begin(Container& c) {+  return Iterator<typename Container::iterator>(std::begin(c));+}++template <class Container>+Iterator<typename Container::iterator> end(Container& c) {+  return Iterator<typename Container::iterator>(std::end(c));+}++/**+ * Adaptor around a STL sequence container.+ *+ * Converts a sequence of Node into a sequence of its underlying elements+ * (with enough functionality to make it useful, although it's not fully+ * compatible with the STL container requirements, see below).+ *+ * Provides iterators (of the same category as those of the underlying+ * container), size(), front(), back(), push_back(), pop_back(), and const /+ * non-const versions of operator[] (if the underlying container supports+ * them).  Does not provide push_front() / pop_front() or arbitrary insert /+ * emplace / erase.  Also provides reserve() / capacity() if supported by the+ * underlying container.+ *+ * Yes, it's called Adaptor, not Adapter, as that's the name used by the STL+ * and by boost.  Deal with it.+ *+ * Internally, we hold a container of Node and the number of elements in+ * the last block.  We don't keep empty blocks, so the number of elements in+ * the last block is always between 1 and Node::kElementCount (inclusive).+ * (this is true if the container is empty as well to make push_back() simpler,+ * see the implementation of the size() method for details).+ */+template <class Container>+class Adaptor {+ public:+  typedef typename Container::value_type Node;+  typedef typename Node::value_type value_type;+  typedef value_type& reference;+  typedef const value_type& const_reference;+  typedef Iterator<typename Container::iterator> iterator;+  typedef Iterator<typename Container::const_iterator> const_iterator;+  typedef typename const_iterator::difference_type difference_type;+  typedef typename Container::size_type size_type;++  static constexpr size_t kElementsPerNode = Node::kElementCount;+  // Constructors+  Adaptor() : lastCount_(Node::kElementCount) {}+  explicit Adaptor(Container c, size_t lastCount = Node::kElementCount)+      : c_(std::move(c)), lastCount_(lastCount) {}+  explicit Adaptor(size_t n, const value_type& value = value_type())+      : c_(Node::nodeCount(n), fullNode(value)) {+    const auto count = n % Node::kElementCount;+    lastCount_ = count != 0 ? count : Node::kElementCount;+  }++  Adaptor(const Adaptor&) = default;+  Adaptor& operator=(const Adaptor&) = default;+  Adaptor(Adaptor&& other) noexcept+      : c_(std::move(other.c_)), lastCount_(other.lastCount_) {+    other.lastCount_ = Node::kElementCount;+  }+  Adaptor& operator=(Adaptor&& other) {+    if (this != &other) {+      c_ = std::move(other.c_);+      lastCount_ = other.lastCount_;+      other.lastCount_ = Node::kElementCount;+    }+    return *this;+  }++  // Iterators+  const_iterator cbegin() const { return const_iterator(c_.begin()); }+  const_iterator cend() const {+    auto it = const_iterator(c_.end());+    if (lastCount_ != Node::kElementCount) {+      it -= (Node::kElementCount - lastCount_);+    }+    return it;+  }+  const_iterator begin() const { return cbegin(); }+  const_iterator end() const { return cend(); }+  iterator begin() { return iterator(c_.begin()); }+  iterator end() {+    auto it = iterator(c_.end());+    if (lastCount_ != Node::kElementCount) {+      it -= difference_type(Node::kElementCount - lastCount_);+    }+    return it;+  }+  void swap(Adaptor& other) {+    using std::swap;+    swap(c_, other.c_);+    swap(lastCount_, other.lastCount_);+  }+  bool empty() const { return c_.empty(); }+  size_type size() const {+    return (+        c_.empty() ? 0 : (c_.size() - 1) * Node::kElementCount + lastCount_);+  }+  size_type max_size() const {+    return (+        (c_.max_size() <=+         std::numeric_limits<size_type>::max() / Node::kElementCount)+            ? c_.max_size() * Node::kElementCount+            : std::numeric_limits<size_type>::max());+  }++  const value_type& front() const {+    assert(!empty());+    return c_.front().data()[0];+  }+  value_type& front() {+    assert(!empty());+    return c_.front().data()[0];+  }++  const value_type& back() const {+    assert(!empty());+    return c_.back().data()[lastCount_ - 1];+  }+  value_type& back() {+    assert(!empty());+    return c_.back().data()[lastCount_ - 1];+  }++  template <typename... Args>+  void emplace_back(Args&&... args) {+    new (allocate_back()) value_type(std::forward<Args>(args)...);+  }++  void push_back(value_type x) { emplace_back(std::move(x)); }++  void pop_back() {+    assert(!empty());+    if (--lastCount_ == 0) {+      c_.pop_back();+      lastCount_ = Node::kElementCount;+    }+  }++  void clear() {+    c_.clear();+    lastCount_ = Node::kElementCount;+  }++  void reserve(size_type n) {+    assert(n >= 0);+    c_.reserve(Node::nodeCount(n));+  }++  size_type capacity() const { return c_.capacity() * Node::kElementCount; }++  const value_type& operator[](size_type idx) const {+    return c_[idx / Node::kElementCount].data()[idx % Node::kElementCount];+  }+  value_type& operator[](size_type idx) {+    return c_[idx / Node::kElementCount].data()[idx % Node::kElementCount];+  }++  /**+   * Return the underlying container and number of elements in the last block,+   * and clear *this.  Useful when you want to process the data as Nodes+   * (again) and want to avoid copies.+   */+  std::pair<Container, size_t> move() {+    std::pair<Container, size_t> p(std::move(c_), lastCount_);+    lastCount_ = Node::kElementCount;+    return p;+  }++  /**+   * Return a const reference to the underlying container and the current+   * number of elements in the last block.+   */+  std::pair<const Container&, size_t> peek() const {+    return std::make_pair(std::cref(c_), lastCount_);+  }++  void padToFullNode(const value_type& padValue) {+    // the if is necessary because c_ may be empty so we can't call c_.back()+    if (lastCount_ != Node::kElementCount) {+      auto last = c_.back().data();+      std::fill(last + lastCount_, last + Node::kElementCount, padValue);+      lastCount_ = Node::kElementCount;+    }+  }++ private:+  value_type* allocate_back() {+    if (lastCount_ == Node::kElementCount) {+      if constexpr (is_invocable_v<detail::emplace_back, Container&>) {+        c_.emplace_back();+      } else {+        c_.push_back(typename Container::value_type());+      }+      lastCount_ = 0;+    }+    return &c_.back().data()[lastCount_++];+  }++  static Node fullNode(const value_type& value) {+    Node n;+    std::fill(n.data(), n.data() + kElementsPerNode, value);+    return n;+  }+  Container c_; // container of Nodes+  size_t lastCount_; // number of elements in last Node+};++} // namespace padded+} // namespace folly
+ folly/folly/Poly-inl.h view
@@ -0,0 +1,231 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++namespace folly {+namespace detail {++template <class I>+inline PolyVal<I>::PolyVal(PolyVal&& that) noexcept {+  that.vptr_->ops_(Op::eMove, &that, static_cast<Data*>(this));+  vptr_ = std::exchange(that.vptr_, vtable<I>());+}++template <class I>+inline PolyVal<I>::PolyVal(PolyOrNonesuch const& that) {+  that.vptr_->ops_(+      Op::eCopy, const_cast<Data*>(that._data_()), PolyAccess::data(*this));+  vptr_ = that.vptr_;+}++template <class I>+inline PolyVal<I>::~PolyVal() {+  vptr_->ops_(Op::eNuke, this, nullptr);+}++template <class I>+inline Poly<I>& PolyVal<I>::operator=(PolyVal that) noexcept {+  vptr_->ops_(Op::eNuke, _data_(), nullptr);+  that.vptr_->ops_(Op::eMove, that._data_(), _data_());+  vptr_ = std::exchange(that.vptr_, vtable<I>());+  return static_cast<Poly<I>&>(*this);+}++template <class I>+template <class T, std::enable_if_t<ModelsInterface<T, I>::value, int>>+inline PolyVal<I>::PolyVal(T&& t) {+  using U = std::decay_t<T>;+  static_assert(+      std::is_copy_constructible<U>::value || !Copyable::value,+      "This Poly<> requires copyability, and the source object is not "+      "copyable");+  // The static and dynamic types should match; otherwise, this will slice.+  assert(+      typeid(t) == typeid(std::decay_t<T>) &&+      "Dynamic and static exception types don't match. Object would "+      "be sliced when storing in Poly.");+  if (inSitu<U>()) {+    auto const buff = static_cast<void*>(&_data_()->buff_);+    ::new (buff) U(static_cast<T&&>(t));+  } else {+    _data_()->pobj_ = new U(static_cast<T&&>(t));+  }+  vptr_ = vtableFor<I, U>();+}++template <class I>+template <class I2, std::enable_if_t<ValueCompatible<I, I2>::value, int>>+inline PolyVal<I>::PolyVal(Poly<I2> that) {+  static_assert(+      !Copyable::value || std::is_copy_constructible<Poly<I2>>::value,+      "This Poly<> requires copyability, and the source object is not "+      "copyable");+  auto* that_vptr = PolyAccess::vtable(that);+  if (that_vptr->state_ != State::eEmpty) {+    that_vptr->ops_(Op::eMove, PolyAccess::data(that), _data_());+    vptr_ = &select<I>(*std::exchange(that_vptr, vtable<std::decay_t<I2>>()));+  }+}++template <class I>+template <class T, std::enable_if_t<ModelsInterface<T, I>::value, int>>+inline Poly<I>& PolyVal<I>::operator=(T&& t) {+  *this = PolyVal(static_cast<T&&>(t));+  return static_cast<Poly<I>&>(*this);+}++template <class I>+template <class I2, std::enable_if_t<ValueCompatible<I, I2>::value, int>>+inline Poly<I>& PolyVal<I>::operator=(Poly<I2> that) {+  *this = PolyVal(std::move(that));+  return static_cast<Poly<I>&>(*this);+}++template <class I>+inline void PolyVal<I>::swap(Poly<I>& that) noexcept {+  switch (vptr_->state_) {+    case State::eEmpty:+      *this = std::move(that);+      break;+    case State::eOnHeap:+      if (State::eOnHeap == that.vptr_->state_) {+        std::swap(_data_()->pobj_, that._data_()->pobj_);+        std::swap(vptr_, that.vptr_);+        return;+      }+      [[fallthrough]];+    case State::eInSitu:+      std::swap(+          *this, static_cast<PolyVal<I>&>(that)); // NOTE: qualified, not ADL+  }+}++template <class I>+inline AddCvrefOf<PolyRoot<I>, I>& PolyRef<I>::_polyRoot_() const noexcept {+  return const_cast<AddCvrefOf<PolyRoot<I>, I>&>(+      static_cast<PolyRoot<I> const&>(*this));+}++template <class I>+constexpr RefType PolyRef<I>::refType() noexcept {+  using J = std::remove_reference_t<I>;+  return std::is_rvalue_reference<I>::value ? RefType::eRvalue+      : std::is_const<J>::value+      ? RefType::eConstLvalue+      : RefType::eLvalue;+}++template <class I>+template <class That, class I2>+inline PolyRef<I>::PolyRef(That&& that, Type<I2>) {+  auto* that_vptr = PolyAccess::vtable(PolyAccess::root(that));+  detail::State const that_state = that_vptr->state_;+  if (that_state == State::eEmpty) {+    throw BadPolyAccess();+  }+  auto* that_data = PolyAccess::data(PolyAccess::root(that));+  _data_()->pobj_ = that_state == State::eInSitu+      ? const_cast<void*>(static_cast<void const*>(&that_data->buff_))+      : that_data->pobj_;+  this->vptr_ = &select<std::decay_t<I>>(+      *static_cast<VTable<std::decay_t<I2>> const*>(that_vptr->ops_(+          Op::eRefr, nullptr, reinterpret_cast<void*>(refType()))));+}++template <class I>+inline PolyRef<I>::PolyRef(PolyRef const& that) noexcept {+  _data_()->pobj_ = that._data_()->pobj_;+  this->vptr_ = that.vptr_;+}++template <class I>+inline Poly<I>& PolyRef<I>::operator=(PolyRef const& that) noexcept {+  _data_()->pobj_ = that._data_()->pobj_;+  this->vptr_ = that.vptr_;+  return static_cast<Poly<I>&>(*this);+}++template <class I>+template <class T, std::enable_if_t<ModelsInterface<T, I>::value, int>>+inline PolyRef<I>::PolyRef(T&& t) noexcept {+  _data_()->pobj_ =+      const_cast<void*>(static_cast<void const*>(std::addressof(t)));+  this->vptr_ = vtableFor<std::decay_t<I>, AddCvrefOf<std::decay_t<T>, I>>();+}++template <class I>+template <+    class I2,+    std::enable_if_t<ReferenceCompatible<I, I2, I2&&>::value, int>>+inline PolyRef<I>::PolyRef(Poly<I2>&& that) noexcept(+    std::is_reference<I2>::value)+    : PolyRef{that, Type<I2>{}} {+  static_assert(+      Disjunction<std::is_reference<I2>, std::is_rvalue_reference<I>>::value,+      "Attempting to construct a Poly that is a reference to a temporary. "+      "This is probably a mistake.");+}++template <class I>+template <class T, std::enable_if_t<ModelsInterface<T, I>::value, int>>+inline Poly<I>& PolyRef<I>::operator=(T&& t) noexcept {+  *this = PolyRef(static_cast<T&&>(t));+  return static_cast<Poly<I>&>(*this);+}++template <class I>+template <+    class I2,+    std::enable_if_t<ReferenceCompatible<I, I2, I2&&>::value, int>>+inline Poly<I>& PolyRef<I>::operator=(Poly<I2>&& that) noexcept(+    std::is_reference<I2>::value) {+  *this = PolyRef(std::move(that));+  return static_cast<Poly<I>&>(*this);+}++template <class I>+template <+    class I2,+    std::enable_if_t<ReferenceCompatible<I, I2, I2&>::value, int>>+inline Poly<I>& PolyRef<I>::operator=(Poly<I2>& that) noexcept(+    std::is_reference<I2>::value) {+  *this = PolyRef(that);+  return static_cast<Poly<I>&>(*this);+}++template <class I>+template <+    class I2,+    std::enable_if_t<ReferenceCompatible<I, I2, I2 const&>::value, int>>+inline Poly<I>& PolyRef<I>::operator=(Poly<I2> const& that) noexcept(+    std::is_reference<I2>::value) {+  *this = PolyRef(that);+  return static_cast<Poly<I>&>(*this);+}++template <class I>+inline void PolyRef<I>::swap(Poly<I>& that) noexcept {+  std::swap(_data_()->pobj_, that._data_()->pobj_);+  std::swap(this->vptr_, that.vptr_);+}++template <class I>+inline AddCvrefOf<PolyImpl<I>, I>& PolyRef<I>::get() const noexcept {+  return const_cast<AddCvrefOf<PolyImpl<I>, I>&>(+      static_cast<PolyImpl<I> const&>(*this));+}++} // namespace detail+} // namespace folly
+ folly/folly/Poly.h view
@@ -0,0 +1,1095 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++// TODO: [x] "cast" from Poly<C&> to Poly<C&&>+// TODO: [ ] copy/move from Poly<C&>/Poly<C&&> to Poly<C>+// TODO: [ ] copy-on-write?+// TODO: [ ] down- and cross-casting? (Possible?)+// TODO: [ ] shared ownership? (Dubious.)+// TODO: [ ] can games be played with making the VTable a member of a struct+//           with strange alignment such that the address of the VTable can+//           be used to tell whether the object is stored in-situ or not?++#pragma once++#include <cassert>+#include <new>+#include <type_traits>+#include <typeinfo>+#include <utility>++#include <folly/CPortability.h>+#include <folly/CppAttributes.h>+#include <folly/Traits.h>+#include <folly/detail/TypeList.h>+#include <folly/lang/Assume.h>++#include <folly/PolyException.h>+#include <folly/detail/PolyDetail.h>++namespace folly {+template <class I>+struct Poly;++// MSVC workaround+template <class Node, class Tfx, class Access>+struct PolySelf_ {+  using type = decltype(Access::template self_<Node, Tfx>());+};++/**+ * Within the definition of interface `I`, `PolySelf<Base>` is an alias for+ * the instance of `Poly` that is currently being instantiated. It is+ * one of: `Poly<J>`, `Poly<J&&>`, `Poly<J&>`, or `Poly<J const&>`; where+ * `J` is either `I` or some interface that extends `I`.+ *+ * It can be used within interface definitions to declare members that accept+ * other `Poly` objects of the same type as `*this`.+ *+ * The first parameter may optionally be cv- and/or reference-qualified, in+ * which case, the qualification is applies to the type of the interface in the+ * resulting `Poly<>` instance. The second template parameter controls whether+ * or not the interface is decayed before the cv-ref qualifiers of the first+ * argument are applied. For example, given the following:+ *+ *     struct Foo {+ *       template <class Base>+ *       struct Interface : Base {+ *         using A = PolySelf<Base>;+ *         using B = PolySelf<Base &>;+ *         using C = PolySelf<Base const &>;+ *         using X = PolySelf<Base, PolyDecay>;+ *         using Y = PolySelf<Base &, PolyDecay>;+ *         using Z = PolySelf<Base const &, PolyDecay>;+ *       };+ *       // ...+ *     };+ *     struct Bar : PolyExtends<Foo> {+ *       // ...+ *     };+ *+ * Then for `Poly<Bar>`, the typedefs are aliases for the following types:+ * - `A` is `Poly<Bar>`+ * - `B` is `Poly<Bar &>`+ * - `C` is `Poly<Bar const &>`+ * - `X` is `Poly<Bar>`+ * - `Y` is `Poly<Bar &>`+ * - `Z` is `Poly<Bar const &>`+ *+ * And for `Poly<Bar &>`, the typedefs are aliases for the following types:+ * - `A` is `Poly<Bar &>`+ * - `B` is `Poly<Bar &>`+ * - `C` is `Poly<Bar &>`+ * - `X` is `Poly<Bar>`+ * - `Y` is `Poly<Bar &>`+ * - `Z` is `Poly<Bar const &>`+ */+template <+    class Node,+    class Tfx = detail::MetaIdentity,+    class Access = detail::PolyAccess>+using PolySelf = _t<PolySelf_<Node, Tfx, Access>>;++/**+ * When used in conjunction with `PolySelf`, controls how to construct `Poly`+ * types related to the one currently being instantiated.+ *+ * \sa PolySelf+ */+using PolyDecay = detail::MetaQuote<std::decay_t>;++#define FOLLY_POLY_MEMBER(SIG, MEM) ::folly::sig<SIG>(MEM)+#define FOLLY_POLY_MEMBERS(...) ::folly::PolyMembers<__VA_ARGS__>++template <auto... Ps>+struct PolyMembers {};++/**+ * Used in the definition of a `Poly` interface to say that the current+ * interface is an extension of a set of zero or more interfaces.+ *+ * Example:+ *+ *   struct IFoo {+ *     template <class Base> struct Interface : Base {+ *       void foo() { folly::poly_call<0>(*this); }+ *     };+ *     template <class T> using Members = FOLLY_POLY_MEMBERS(&T::foo);+ *   }+ *   struct IBar : PolyExtends<IFoo> {+ *     template <class Base> struct Interface : Base {+ *       void bar(int i) { folly::poly_call<0>(*this, i); }+ *     };+ *     template <class T> using Members = FOLLY_POLY_MEMBERS(&T::bar);+ *   }+ */+template <class... I>+struct PolyExtends : virtual I... {+  using Subsumptions = detail::TypeList<I...>;++  template <class Base>+  struct Interface : Base {+    Interface() = default;+    using Base::Base;+  };++  template <class...>+  using Members = PolyMembers<>;+};++////////////////////////////////////////////////////////////////////////////////+/**+ * Call the N-th member of the currently-being-defined interface. When the+ * first parameter is an object of type `PolySelf<Base>` (as opposed to `*this`)+ * you must explicitly specify which interface through which to dispatch.+ * For instance:+ *+ *     struct IAddable {+ *       template <class Base>+ *       struct Interface : Base {+ *         friend folly::PolySelf<Base, folly::PolyDecay>+ *         operator+(+ *             folly::PolySelf<Base> const& a,+ *             folly::PolySelf<Base> const& b) {+ *           return folly::poly_call<0, IAddable>(a, b);+ *         }+ *       };+ *       template <class T>+ *       static auto plus_(T const& a, T const& b) -> decltype(a + b) {+ *         return a + b;+ *       }+ *       template <class T>+ *       using Members = FOLLY_POLY_MEMBERS(&plus_<std::decay_t<T>>);+ *     };+ *+ * \sa PolySelf+ */+template <std::size_t N, typename This, typename... As>+auto poly_call(This&& _this, As&&... as)+    -> decltype(detail::PolyAccess::call<N>(+        static_cast<This&&>(_this), static_cast<As&&>(as)...)) {+  return detail::PolyAccess::call<N>(+      static_cast<This&&>(_this), static_cast<As&&>(as)...);+}++/// \overload+template <std::size_t N, class I, class Tail, typename... As>+decltype(auto) poly_call(detail::PolyNode<I, Tail>&& _this, As&&... as) {+  using This = detail::InterfaceOf<I, detail::PolyNode<I, Tail>>;+  return detail::PolyAccess::call<N>(+      static_cast<This&&>(_this), static_cast<As&&>(as)...);+}++/// \overload+template <std::size_t N, class I, class Tail, typename... As>+decltype(auto) poly_call(detail::PolyNode<I, Tail>& _this, As&&... as) {+  using This = detail::InterfaceOf<I, detail::PolyNode<I, Tail>>;+  return detail::PolyAccess::call<N>(+      static_cast<This&>(_this), static_cast<As&&>(as)...);+}++/// \overload+template <std::size_t N, class I, class Tail, typename... As>+decltype(auto) poly_call(detail::PolyNode<I, Tail> const& _this, As&&... as) {+  using This = detail::InterfaceOf<I, detail::PolyNode<I, Tail>>;+  return detail::PolyAccess::call<N>(+      static_cast<This const&>(_this), static_cast<As&&>(as)...);+}++/// \overload+template <+    std::size_t N,+    class I,+    class Poly,+    typename... As,+    std::enable_if_t<detail::IsPoly<Poly>::value, int> = 0>+auto poly_call(Poly&& _this, As&&... as)+    -> decltype(poly_call<N, I>(+        static_cast<Poly&&>(_this).get(), static_cast<As&&>(as)...)) {+  return poly_call<N, I>(+      static_cast<Poly&&>(_this).get(), static_cast<As&&>(as)...);+}++/// \cond+/// \overload+template <std::size_t N, class I, typename... As>+[[noreturn]] detail::Bottom poly_call(detail::ArchetypeBase const&, As&&...) {+  assume_unreachable();+}+/// \endcond++////////////////////////////////////////////////////////////////////////////////+/**+ * Try to cast the `Poly` object to the requested type. If the `Poly` stores an+ * object of that type, return a reference to the object; otherwise, throw an+ * exception.+ * \tparam T The (unqualified) type to which to cast the `Poly` object.+ * \tparam Poly The type of the `Poly` object.+ * \param that The `Poly` object to be cast.+ * \return A reference to the `T` object stored in or referred to by `that`.+ * \throw BadPolyAccess if `that` is empty.+ * \throw BadPolyCast if `that` does not store or refer to an object of type+ *        `T`.+ */+template <class T, class I>+detail::AddCvrefOf<T, I>&& poly_cast(detail::PolyRoot<I>&& that) {+  return detail::PolyAccess::cast<T>(std::move(that));+}++/// \overload+template <class T, class I>+detail::AddCvrefOf<T, I>& poly_cast(detail::PolyRoot<I>& that) {+  return detail::PolyAccess::cast<T>(that);+}++/// \overload+template <class T, class I>+detail::AddCvrefOf<T, I> const& poly_cast(detail::PolyRoot<I> const& that) {+  return detail::PolyAccess::cast<T>(that);+}++/// \cond+/// \overload+template <class T, class I>+[[noreturn]] detail::AddCvrefOf<T, I>&& poly_cast(detail::ArchetypeRoot<I>&&) {+  assume_unreachable();+}++/// \overload+template <class T, class I>+[[noreturn]] detail::AddCvrefOf<T, I>& poly_cast(detail::ArchetypeRoot<I>&) {+  assume_unreachable();+}++/// \overload+template <class T, class I>+[[noreturn]] detail::AddCvrefOf<T, I> const& poly_cast(+    detail::ArchetypeRoot<I> const&) {+  assume_unreachable();+}+/// \endcond++/// \overload+template <+    class T,+    class Poly,+    std::enable_if_t<detail::IsPoly<Poly>::value, int> = 0>+constexpr auto poly_cast(Poly&& that)+    -> decltype(poly_cast<T>(std::declval<Poly>().get())) {+  return poly_cast<T>(static_cast<Poly&&>(that).get());+}++////////////////////////////////////////////////////////////////////////////////+/**+ * Returns a reference to the `std::type_info` object corresponding to the+ * object currently stored in `that`. If `that` is empty, returns+ * `typeid(void)`.+ */+template <class I>+std::type_info const& poly_type(detail::PolyRoot<I> const& that) noexcept {+  return detail::PolyAccess::type(that);+}++/// \cond+/// \overload+[[noreturn]] inline std::type_info const& poly_type(+    detail::ArchetypeBase const&) noexcept {+  assume_unreachable();+}+/// \endcond++/// \overload+template <class Poly, std::enable_if_t<detail::IsPoly<Poly>::value, int> = 0>+constexpr auto poly_type(Poly const& that) noexcept+    -> decltype(poly_type(that.get())) {+  return poly_type(that.get());+}++////////////////////////////////////////////////////////////////////////////////+/**+ * Returns `true` if `that` is not currently storing an object; `false`,+ * otherwise.+ */+template <class I>+bool poly_empty(detail::PolyRoot<I> const& that) noexcept {+  return detail::State::eEmpty == detail::PolyAccess::vtable(that)->state_;+}++/// \overload+template <class I>+constexpr bool poly_empty(detail::PolyRoot<I&&> const&) noexcept {+  return false;+}++/// \overload+template <class I>+constexpr bool poly_empty(detail::PolyRoot<I&> const&) noexcept {+  return false;+}++/// \overload+template <class I>+constexpr bool poly_empty(Poly<I&&> const&) noexcept {+  return false;+}++/// \overload+template <class I>+constexpr bool poly_empty(Poly<I&> const&) noexcept {+  return false;+}++/// \cond+[[noreturn]] inline bool poly_empty(detail::ArchetypeBase const&) noexcept {+  assume_unreachable();+}+/// \endcond++////////////////////////////////////////////////////////////////////////////////+/**+ * Given a `Poly<I&>`, return a `Poly<I&&>`. Otherwise, when `I` is not a+ * reference type, returns a `Poly<I>&&` when given a `Poly<I>&`, like+ * `std::move`.+ */+template <+    class I,+    std::enable_if_t<Negation<std::is_reference<I>>::value, int> = 0>+constexpr Poly<I>&& poly_move(detail::PolyRoot<I>& that) noexcept {+  return static_cast<Poly<I>&&>(static_cast<Poly<I>&>(that));+}++/// \overload+template <class I, std::enable_if_t<Negation<std::is_const<I>>::value, int> = 0>+Poly<I&&> poly_move(detail::PolyRoot<I&> const& that) noexcept {+  return detail::PolyAccess::move(that);+}++/// \overload+template <class I>+Poly<I const&> poly_move(detail::PolyRoot<I const&> const& that) noexcept {+  return detail::PolyAccess::move(that);+}++/// \cond+/// \overload+[[noreturn]] inline detail::ArchetypeBase poly_move(+    detail::ArchetypeBase const&) noexcept {+  assume_unreachable();+}+/// \endcond++/// \overload+template <class Poly, std::enable_if_t<detail::IsPoly<Poly>::value, int> = 0>+constexpr auto poly_move(Poly& that) noexcept+    -> decltype(poly_move(that.get())) {+  return poly_move(that.get());+}++/// \cond+namespace detail {+/**+ * The implementation for `Poly` for when the interface type is not+ * reference-like qualified, as in `Poly<SemiRegular>`.+ */+template <class I>+struct PolyVal : PolyImpl<I> {+ private:+  friend PolyAccess;++  struct NoneSuch {};+  using Copyable = std::is_copy_constructible<PolyImpl<I>>;+  using PolyOrNonesuch = If<Copyable::value, PolyVal, NoneSuch>;++  using PolyRoot<I>::vptr_;++  PolyRoot<I>& _polyRoot_() noexcept { return *this; }+  PolyRoot<I> const& _polyRoot_() const noexcept { return *this; }++  Data* _data_() noexcept { return PolyAccess::data(*this); }+  Data const* _data_() const noexcept { return PolyAccess::data(*this); }++ public:+  /**+   * Default constructor.+   * \post `poly_empty(*this) == true`+   */+  PolyVal() = default;+  /**+   * Move constructor.+   * \post `poly_empty(that) == true`+   */+  PolyVal(PolyVal&& that) noexcept;+  /**+   * A copy constructor if `I` is copyable; otherwise, a useless constructor+   * from a private, incomplete type.+   */+  /* implicit */ PolyVal(PolyOrNonesuch const& that);++  ~PolyVal();++  /**+   * Inherit any constructors defined by any of the interfaces.+   */+  using PolyImpl<I>::PolyImpl;++  /**+   * Copy assignment, destroys the object currently held (if any) and makes+   * `*this` equal to `that` by stealing its guts.+   */+  Poly<I>& operator=(PolyVal that) noexcept;++  /**+   * Construct a Poly<I> from a concrete type that satisfies the I concept+   */+  template <class T, std::enable_if_t<ModelsInterface<T, I>::value, int> = 0>+  /* implicit */ PolyVal(T&& t);++  /**+   * Construct a `Poly` from a compatible `Poly`. "Compatible" here means: the+   * other interface extends this one either directly or indirectly.+   */+  template <class I2, std::enable_if_t<ValueCompatible<I, I2>::value, int> = 0>+  /* implicit */ PolyVal(Poly<I2> that);++  /**+   * Assign to this `Poly<I>` from a concrete type that satisfies the `I`+   * concept.+   */+  template <class T, std::enable_if_t<ModelsInterface<T, I>::value, int> = 0>+  Poly<I>& operator=(T&& t);++  /**+   * Assign a compatible `Poly` to `*this`. "Compatible" here means: the+   * other interface extends this one either directly or indirectly.+   */+  template <class I2, std::enable_if_t<ValueCompatible<I, I2>::value, int> = 0>+  Poly<I>& operator=(Poly<I2> that);++  /**+   * Swaps the values of two `Poly` objects.+   */+  void swap(Poly<I>& that) noexcept;+};++////////////////////////////////////////////////////////////////////////////////+/**+ * The implementation of `Poly` for when the interface type is+ * reference-qualified, like `Poly<SemiRegular &>`.+ */+template <class I>+struct PolyRef : private PolyImpl<I> {+ private:+  friend PolyAccess;++  AddCvrefOf<PolyRoot<I>, I>& _polyRoot_() const noexcept;++  Data* _data_() noexcept { return PolyAccess::data(*this); }+  Data const* _data_() const noexcept { return PolyAccess::data(*this); }++  static constexpr RefType refType() noexcept;++ protected:+  template <class That, class I2>+  PolyRef(That&& that, Type<I2>);++ public:+  /**+   * Copy constructor+   * \post `&poly_cast<T>(*this) == &poly_cast<T>(that)`, where `T` is the+   *       type of the object held by `that`.+   */+  PolyRef(PolyRef const& that) noexcept;++  /**+   * Copy assignment+   * \post `&poly_cast<T>(*this) == &poly_cast<T>(that)`, where `T` is the+   *       type of the object held by `that`.+   */+  Poly<I>& operator=(PolyRef const& that) noexcept;++  /**+   * Construct a `Poly<I>` from a concrete type that satisfies concept `I`.+   * \post `!poly_empty(*this)`+   */+  template <class T, std::enable_if_t<ModelsInterface<T, I>::value, int> = 0>+  /* implicit */ PolyRef(T&& t) noexcept;++  /**+   * Construct a `Poly<I>` from a compatible `Poly<I2>`.+   */+  template <+      class I2,+      std::enable_if_t<ReferenceCompatible<I, I2, I2&&>::value, int> = 0>+  /* implicit */ PolyRef(Poly<I2>&& that) noexcept(+      std::is_reference<I2>::value);++  template <+      class I2,+      std::enable_if_t<ReferenceCompatible<I, I2, I2&>::value, int> = 0>+  /* implicit */ PolyRef(Poly<I2>& that) noexcept(std::is_reference<I2>::value)+      : PolyRef{that, Type<I2>{}} {}++  template <+      class I2,+      std::enable_if_t<ReferenceCompatible<I, I2, I2 const&>::value, int> = 0>+  /* implicit */ PolyRef(Poly<I2> const& that) noexcept(+      std::is_reference<I2>::value)+      : PolyRef{that, Type<I2>{}} {}++  /**+   * Assign to a `Poly<I>` from a concrete type that satisfies concept `I`.+   * \post `!poly_empty(*this)`+   */+  template <class T, std::enable_if_t<ModelsInterface<T, I>::value, int> = 0>+  Poly<I>& operator=(T&& t) noexcept;++  /**+   * Assign to `*this` from another compatible `Poly`.+   */+  template <+      class I2,+      std::enable_if_t<ReferenceCompatible<I, I2, I2&&>::value, int> = 0>+  Poly<I>& operator=(Poly<I2>&& that) noexcept(std::is_reference<I2>::value);++  /**+   * \overload+   */+  template <+      class I2,+      std::enable_if_t<ReferenceCompatible<I, I2, I2&>::value, int> = 0>+  Poly<I>& operator=(Poly<I2>& that) noexcept(std::is_reference<I2>::value);++  /**+   * \overload+   */+  template <+      class I2,+      std::enable_if_t<ReferenceCompatible<I, I2, I2 const&>::value, int> = 0>+  Poly<I>& operator=(Poly<I2> const& that) noexcept(+      std::is_reference<I2>::value);++  /**+   * Swap which object this `Poly` references ("shallow" swap).+   */+  void swap(Poly<I>& that) noexcept;++  /**+   * Get a reference to the interface, with correct `const`-ness applied.+   */+  AddCvrefOf<PolyImpl<I>, I>& get() const noexcept;++  /**+   * Get a reference to the interface, with correct `const`-ness applied.+   */+  AddCvrefOf<PolyImpl<I>, I>& operator*() const noexcept { return get(); }++  /**+   * Get a pointer to the interface, with correct `const`-ness applied.+   */+  auto operator->() const noexcept { return &get(); }+};++template <class I>+using PolyValOrRef = If<std::is_reference<I>::value, PolyRef<I>, PolyVal<I>>;+} // namespace detail+/// \endcond++/**+ * `Poly` is a class template that makes it relatively easy to define a+ * type-erasing polymorphic object wrapper.+ *+ * \par Type-erasure+ *+ * \par+ * `std::function` is one example of a type-erasing polymorphic object wrapper;+ * `folly::exception_wrapper` is another. Type-erasure is often used as an+ * alternative to dynamic polymorphism via inheritance-based virtual dispatch.+ * The distinguishing characteristic of type-erasing wrappers are:+ * \li **Duck typing:** Types do not need to inherit from an abstract base+ *     class in order to be assignable to a type-erasing wrapper; they merely+ *     need to satisfy a particular interface.+ * \li **Value semantics:** Type-erasing wrappers are objects that can be+ *     passed around _by value_. This is in contrast to abstract base classes+ *     which must be passed by reference or by pointer or else suffer from+ *     _slicing_, which causes them to lose their polymorphic behaviors.+ *     Reference semantics make it difficult to reason locally about code.+ * \li **Automatic memory management:** When dealing with inheritance-based+ *     dynamic polymorphism, it is often necessary to allocate and manage+ *     objects on the heap. This leads to a proliferation of `shared_ptr`s and+ *     `unique_ptr`s in APIs, complicating their point-of-use. APIs that take+ *     type-erasing wrappers, on the other hand, can often store small objects+ *     in-situ, with no dynamic allocation. The memory management, if any, is+ *     handled for you, and leads to cleaner APIs: consumers of your API don't+ *     need to pass `shared_ptr<AbstractBase>`; they can simply pass any object+ *     that satisfies the interface you require. (`std::function` is a+ *     particularly compelling example of this benefit. Far worse would be an+ *     inheritance-based callable solution like+ *     `shared_ptr<ICallable<void(int)>>`. )+ *+ * \par Example: Defining a type-erasing function wrapper with `folly::Poly`+ *+ * \par+ * Defining a polymorphic wrapper with `Poly` is a matter of defining two+ * things:+ * \li An *interface*, consisting of public member functions, and+ * \li A *mapping* from a concrete type to a set of member function bindings.+ *+ * Below is a (heavily commented) example of a simple implementation of a+ * `std::function`-like polymorphic wrapper. Its interface has only a single+ * member function: `operator()`+ *+ *     // An interface for a callable object of a particular signature, Fun+ *     // (most interfaces don't need to be templates, FWIW).+ *     template <class Fun>+ *     struct IFunction;+ *+ *     template <class R, class... As>+ *     struct IFunction<R(As...)> {+ *       // An interface is defined as a nested class template called+ *       // Interface that takes a single template parameter, Base, from+ *       // which it inherits.+ *       template <class Base>+ *       struct Interface : Base {+ *         // The Interface has public member functions. These become the+ *         // public interface of the resulting Poly instantiation.+ *         // (Implementation note: Poly<IFunction<Sig>> will publicly+ *         // inherit from this struct, which is what gives it the right+ *         // member functions.)+ *         R operator()(As... as) const {+ *           // The definition of each member function in your interface will+ *           // always consist of a single line dispatching to+ *           // folly::poly_call<N>. The "N" corresponds to the N-th member+ *           // function in the list of member function bindings, Members,+ *           // defined below. The first argument will always be *this, and the+ *           // rest of the arguments should simply forward (if necessary) the+ *           // member function's arguments.+ *           return static_cast<R>(+ *               folly::poly_call<0>(*this, std::forward<As>(as)...));+ *         }+ *       };+ *+ *       // The "Members" alias template is a comma-separated list of bound+ *       // member functions for a given concrete type "T". The+ *       // "FOLLY_POLY_MEMBERS" macro accepts a comma-separated list, and the+ *       // (optional) "FOLLY_POLY_MEMBER" macro lets you disambiguate overloads+ *       // by explicitly specifying the function signature the target member+ *       // function should have. In this case, we require "T" to have a+ *       // function call operator with the signature `R(As...) const`.+ *       //+ *       // If you are using a C++17-compatible compiler, you can do away with+ *       // the macros and write this as:+ *       //+ *       //   template <class T>+ *       //   using Members = folly::PolyMembers<+ *       //       folly::sig<R(As...) const>(&T::operator())>;+ *       //+ *       // And since `folly::sig` is only needed for disambiguation in case of+ *       // overloads, if you are not concerned about objects with overloaded+ *       // function call operators, it could be further simplified to:+ *       //+ *       //   template <class T>+ *       //   using Members = folly::PolyMembers<&T::operator()>;+ *       //+ *       template <class T>+ *       using Members = FOLLY_POLY_MEMBERS(+ *           FOLLY_POLY_MEMBER(R(As...) const, &T::operator()));+ *     };+ *+ *     // Now that we have defined the interface, we can pass it to Poly to+ *     // create our type-erasing wrapper:+ *     template <class Fun>+ *     using Function = Poly<IFunction<Fun>>;+ *+ * \par+ * Given the above definition of `Function`, users can now initialize instances+ * of (say) `Function<int(int, int)>` with function objects like+ * `std::plus<int>` and `std::multiplies<int>`, as below:+ *+ *     Function<int(int, int)> fun = std::plus<int>{};+ *     assert(5 == fun(2, 3));+ *     fun = std::multiplies<int>{};+ *     assert(6 = fun(2, 3));+ *+ * \par Defining an interface with C++17+ *+ * \par+ * With C++17, defining an interface to be used with `Poly` is fairly+ * straightforward. As in the `Function` example above, there is a struct with+ * a nested `Interface` class template and a nested `Members` alias template.+ * No macros are needed with C++17.+ * \par+ * Imagine we were defining something like a Java-style iterator. If we are+ * using a C++17 compiler, our interface would look something like this:+ *+ *     template <class Value>+ *     struct IJavaIterator {+ *       template <class Base>+ *       struct Interface : Base {+ *         bool Done() const { return folly::poly_call<0>(*this); }+ *         Value Current() const { return folly::poly_call<1>(*this); }+ *         void Next() { folly::poly_call<2>(*this); }+ *       };+ *       // NOTE: This works in C++17 only:+ *       template <class T>+ *       using Members = folly::PolyMembers<&T::Done, &T::Current, &T::Next>;+ *     };+ *+ *     template <class Value>+ *     using JavaIterator = Poly<IJavaIterator>;+ *+ * \par+ * Given the above definition, `JavaIterator<int>` can be used to hold instances+ * of any type that has `Done`, `Current`, and `Next` member functions with the+ * correct (or compatible) signatures.+ *+ * \par+ * The presence of overloaded member functions complicates this picture. Often,+ * property members are faked in C++ with `const` and non-`const` member+ * function overloads, like in the interface specified below:+ *+ *     struct IIntProperty {+ *       template <class Base>+ *       struct Interface : Base {+ *         int Value() const { return folly::poly_call<0>(*this); }+ *         void Value(int i) { folly::poly_call<1>(*this, i); }+ *       };+ *       // NOTE: This works in C++17 only:+ *       template <class T>+ *       using Members = folly::PolyMembers<+ *         folly::sig<int() const>(&T::Value),+ *         folly::sig<void(int)>(&T::Value)>;+ *     };+ *+ *     using IntProperty = Poly<IIntProperty>;+ *+ * \par+ * Now, any object that has `Value` members of compatible signatures can be+ * assigned to instances of `IntProperty` object. Note how `folly::sig` is used+ * to disambiguate the overloads of `&T::Value`.+ *+ * \par Defining an interface with C++14+ *+ * \par+ * In C++14, the nice syntax above doesn't work, so we have to resort to macros.+ * The two examples above would look like this:+ *+ *     template <class Value>+ *     struct IJavaIterator {+ *       template <class Base>+ *       struct Interface : Base {+ *         bool Done() const { return folly::poly_call<0>(*this); }+ *         Value Current() const { return folly::poly_call<1>(*this); }+ *         void Next() { folly::poly_call<2>(*this); }+ *       };+ *       // NOTE: This works in C++14 and C++17:+ *       template <class T>+ *       using Members = FOLLY_POLY_MEMBERS(&T::Done, &T::Current, &T::Next);+ *     };+ *+ *     template <class Value>+ *     using JavaIterator = Poly<IJavaIterator>;+ *+ * \par+ * and+ *+ *     struct IIntProperty {+ *       template <class Base>+ *       struct Interface : Base {+ *         int Value() const { return folly::poly_call<0>(*this); }+ *         void Value(int i) { return folly::poly_call<1>(*this, i); }+ *       };+ *       // NOTE: This works in C++14 and C++17:+ *       template <class T>+ *       using Members = FOLLY_POLY_MEMBERS(+ *         FOLLY_POLY_MEMBER(int() const, &T::Value),+ *         FOLLY_POLY_MEMBER(void(int), &T::Value));+ *     };+ *+ *     using IntProperty = Poly<IIntProperty>;+ *+ * \par Extending interfaces+ *+ * \par+ * One typical advantage of inheritance-based solutions to runtime polymorphism+ * is that one polymorphic interface could extend another through inheritance.+ * The same can be accomplished with type-erasing polymorphic wrappers. In+ * the `Poly` library, you can use `folly::PolyExtends` to say that one+ * interface extends another.+ *+ *     struct IFoo {+ *       template <class Base>+ *       struct Interface : Base {+ *         void Foo() const { return folly::poly_call<0>(*this); }+ *       };+ *       template <class T>+ *       using Members = FOLLY_POLY_MEMBERS(&T::Foo);+ *     };+ *+ *     // The IFooBar interface extends the IFoo interface+ *     struct IFooBar : PolyExtends<IFoo> {+ *       template <class Base>+ *       struct Interface : Base {+ *         void Bar() const { return folly::poly_call<0>(*this); }+ *       };+ *       template <class T>+ *       using Members = FOLLY_POLY_MEMBERS(&T::Bar);+ *     };+ *+ *     using FooBar = Poly<IFooBar>;+ *+ * \par+ * Given the above defintion, instances of type `FooBar` have both `Foo()` and+ * `Bar()` member functions.+ *+ * \par+ * The sensible conversions exist between a wrapped derived type and a wrapped+ * base type. For instance, assuming `IDerived` extends `IBase` with+ * `PolyExtends`:+ *+ *     Poly<IDerived> derived = ...;+ *     Poly<IBase> base = derived; // This conversion is OK.+ *+ * \par+ * As you would expect, there is no conversion in the other direction, and at+ * present there is no `Poly` equivalent to `dynamic_cast`.+ *+ * \par Type-erasing polymorphic reference wrappers+ *+ * \par+ * Sometimes you don't need to own a copy of an object; a reference will do. For+ * that you can use `Poly` to capture a _reference_ to an object satisfying an+ * interface rather than the whole object itself. The syntax is intuitive.+ *+ *     int i = 42;+ *     // Capture a mutable reference to an object of any IRegular type:+ *     Poly<IRegular &> intRef = i;+ *     assert(42 == folly::poly_cast<int>(intRef));+ *     // Assert that we captured the address of "i":+ *     assert(&i == &folly::poly_cast<int>(intRef));+ *+ * \par+ * A reference-like `Poly` has a different interface than a value-like `Poly`.+ * Rather than calling member functions with the `obj.fun()` syntax, you would+ * use the `obj->fun()` syntax. This is for the sake of `const`-correctness.+ * For example, consider the code below:+ *+ *     struct IFoo {+ *       template <class Base>+ *       struct Interface {+ *         void Foo() { folly::poly_call<0>(*this); }+ *       };+ *       template <class T>+ *       using Members = folly::PolyMembers<&T::Foo>;+ *     };+ *+ *     struct SomeFoo {+ *       void Foo() { std::printf("SomeFoo::Foo\n"); }+ *     };+ *+ *     SomeFoo foo;+ *     Poly<IFoo &> const anyFoo = foo;+ *     anyFoo->Foo(); // prints "SomeFoo::Foo"+ *+ * \par+ * Notice in the above code that the `Foo` member function is non-`const`.+ * Notice also that the `anyFoo` object is `const`. However, since it has+ * captured a non-`const` reference to the `foo` object, it should still be+ * possible to dispatch to the non-`const` `Foo` member function. When+ * instantiated with a reference type, `Poly` has an overloaded `operator->`+ * member that returns a pointer to the `IFoo` interface with the correct+ * `const`-ness, which makes this work.+ *+ * \par+ * The same mechanism also prevents users from calling non-`const` member+ * functions on `Poly` objects that have captured `const` references, which+ * would violate `const`-correctness.+ *+ * \par+ * Sensible conversions exist between non-reference and reference `Poly`s. For+ * instance:+ *+ *     Poly<IRegular> value = 42;+ *     Poly<IRegular &> mutable_ref = value;+ *     Poly<IRegular const &> const_ref = mutable_ref;+ *+ *     assert(&poly_cast<int>(value) == &poly_cast<int>(mutable_ref));+ *     assert(&poly_cast<int>(value) == &poly_cast<int>(const_ref));+ *+ * \par Non-member functions (C++17)+ *+ * \par+ * If you wanted to write the interface `ILogicallyNegatable`, which captures+ * all types that can be negated with unary `operator!`, you could do it+ * as we've shown above, by binding `&T::operator!` in the nested `Members`+ * alias template, but that has the problem that it won't work for types that+ * have defined unary `operator!` as a free function. To handle this case,+ * the `Poly` library lets you use a free function instead of a member function+ * when creating a binding.+ *+ * \par+ * With C++17 you may use a lambda to create a binding, as shown in the example+ * below:+ *+ *     struct ILogicallyNegatable {+ *       template <class Base>+ *       struct Interface : Base {+ *         bool operator!() const { return folly::poly_call<0>(*this); }+ *       };+ *       template <class T>+ *       using Members = folly::PolyMembers<+ *         +[](T const& t) -> decltype(!t) { return !t; }>;+ *     };+ *+ * \par+ * This requires some explanation. The unary `operator+` in front of the lambda+ * is necessary! It causes the lambda to decay to a C-style function pointer,+ * which is one of the types that `folly::PolyMembers` accepts. The `decltype`+ * in the lambda return type is also necessary. Through the magic of SFINAE, it+ * will cause `Poly<ILogicallyNegatable>` to reject any types that don't support+ * unary `operator!`.+ *+ * \par+ * If you are using a free function to create a binding, the first parameter is+ * implicitly the `this` parameter. It will receive the type-erased object.+ *+ * \par Non-member functions (C++14)+ *+ * \par+ * If you are using a C++14 compiler, the defintion of `ILogicallyNegatable`+ * above will fail because lambdas are not `constexpr`. We can get the same+ * effect by writing the lambda as a named free function, as show below:+ *+ *     struct ILogicallyNegatable {+ *       template <class Base>+ *       struct Interface : Base {+ *         bool operator!() const { return folly::poly_call<0>(*this); }+ *       };+ *+ *       template <class T>+ *       static auto negate(T const& t) -> decltype(!t) { return !t; }+ *+ *       template <class T>+ *       using Members = FOLLY_POLY_MEMBERS(&negate<T>);+ *     };+ *+ * \par+ * As with the example that uses the lambda in the preceding section, the first+ * parameter is implicitly the `this` parameter. It will receive the type-erased+ * object.+ *+ * \par Multi-dispatch+ *+ * \par+ * What if you want to create an `IAddable` interface for things that can be+ * added? Adding requires _two_ objects, both of which are type-erased. This+ * interface requires dispatching on both objects, doing the addition only+ * if the types are the same. For this we make use of the `PolySelf` template+ * alias to define an interface that takes more than one object of the+ * erased type.+ *+ *     struct IAddable {+ *       template <class Base>+ *       struct Interface : Base {+ *         friend PolySelf<Base, Decay>+ *         operator+(PolySelf<Base> const& a, PolySelf<Base> const& b) {+ *           return folly::poly_call<0, IAddable>(a, b);+ *         }+ *       };+ *+ *       template <class T>+ *       using Members = folly::PolyMembers<+ *         +[](T const& a, T const& b) -> decltype(a + b) { return a + b; }>;+ *     };+ *+ * \par+ * Given the above defintion of `IAddable` we would be able to do the following:+ *+ *     Poly<IAddable> a = 2, b = 3;+ *     Poly<IAddable> c = a + b;+ *     assert(poly_cast<int>(c) == 5);+ *+ * \par+ * If `a` and `b` stored objects of different types, a `BadPolyCast` exception+ * would be thrown.+ *+ * \par Move-only types+ *+ * \par+ * If you want to store move-only types, then your interface should extend the+ * `IMoveOnly` interface.+ *+ * \par Implementation notes+ * \par+ * `Poly` will store "small" objects in an internal buffer, avoiding the cost of+ * of dynamic allocations. At present, this size is not configurable; it is+ * pegged at the size of two `double`s.+ *+ * \par+ * `Poly` objects are always nothrow movable. If you store an object in one that+ * has a potentially throwing move contructor, the object will be stored on the+ * heap, even if it could fit in the internal storage of the `Poly` object.+ * (So be sure to give your objects nothrow move constructors!)+ *+ * \par+ * `Poly` implements type-erasure in a manner very similar to how the compiler+ * accomplishes virtual dispatch. Every `Poly` object contains a pointer to a+ * table of function pointers. Member function calls involve a double-+ * indirection: once through the v-pointer, and other indirect function call+ * through the function pointer.+ */+template <class I>+struct Poly final : detail::PolyValOrRef<I> {+  friend detail::PolyAccess;+  Poly() = default;+  using detail::PolyValOrRef<I>::PolyValOrRef;+  using detail::PolyValOrRef<I>::operator=;+};++/**+ * Swap two `Poly<I>` instances.+ */+template <class I>+void swap(Poly<I>& left, Poly<I>& right) noexcept {+  left.swap(right);+}++/**+ * Pseudo-function template handy for disambiguating function overloads.+ *+ * For example, given:+ *     struct S {+ *       int property() const;+ *       void property(int);+ *     };+ *+ * You can get a member function pointer to the first overload with:+ *     folly::sig<int()const>(&S::property);+ *+ * This is arguably a nicer syntax that using the built-in `static_cast`:+ *     static_cast<int (S::*)() const>(&S::property);+ *+ * `sig` is also more permissive than `static_cast` about `const`. For instance,+ * the following also works:+ *     folly::sig<int()>(&S::property);+ *+ * The above is permitted+ */+template <class Sig>+inline constexpr detail::Sig<Sig> const sig = {};++} // namespace folly++#include <folly/Poly-inl.h>
+ folly/folly/PolyException.h view
@@ -0,0 +1,43 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <exception>+#include <typeinfo>++#include <folly/CPortability.h>++namespace folly {++/**+ * Exception type that is thrown on invalid access of an empty `Poly` object.+ */+struct FOLLY_EXPORT BadPolyAccess : std::exception {+  BadPolyAccess() = default;+  char const* what() const noexcept override { return "BadPolyAccess"; }+};++/**+ * Exception type that is thrown when attempting to extract from a `Poly` a+ * value of the wrong type.+ */+struct FOLLY_EXPORT BadPolyCast : std::bad_cast {+  BadPolyCast() = default;+  char const* what() const noexcept override { return "BadPolyCast"; }+};++} // namespace folly
+ folly/folly/Portability.h view
@@ -0,0 +1,728 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <cstddef>++#include <folly/CPortability.h>+#include <folly/portability/Config.h>++#if defined(_MSC_VER)+#define FOLLY_CPLUSPLUS _MSVC_LANG+#else+#define FOLLY_CPLUSPLUS __cplusplus+#endif++// On MSVC an incorrect <version> header get's picked up+#if !defined(_MSC_VER) && __has_include(<version>)+#include <version>+#endif++static_assert(FOLLY_CPLUSPLUS >= 201703L, "__cplusplus >= 201703L");++#if defined(__GNUC__) && !defined(__clang__)+#if defined(FOLLY_CONFIG_TEMPORARY_DOWNGRADE_GCC)+static_assert(__GNUC__ >= 9, "__GNUC__ >= 9");+#else+static_assert(__GNUC__ >= 10, "__GNUC__ >= 10");+#endif+#endif++#if defined(_MSC_VER)+static_assert(_MSC_VER >= 1920);+#endif++#if defined(_MSC_VER) || defined(_CPPLIB_VER)+static_assert(FOLLY_CPLUSPLUS >= 201703L, "__cplusplus >= 201703L");+#endif++// Unaligned loads and stores+namespace folly {+#if defined(FOLLY_HAVE_UNALIGNED_ACCESS) && FOLLY_HAVE_UNALIGNED_ACCESS+constexpr bool kHasUnalignedAccess = true;+#else+constexpr bool kHasUnalignedAccess = false;+#endif+} // namespace folly++// compiler specific attribute translation+// msvc should come first, so if clang is in msvc mode it gets the right defines++// NOTE: this will only do checking in msvc with versions that support /analyze+#ifdef _MSC_VER+#ifdef _USE_ATTRIBUTES_FOR_SAL+#undef _USE_ATTRIBUTES_FOR_SAL+#endif+/* nolint */+#define _USE_ATTRIBUTES_FOR_SAL 1+#include <sal.h> // @manual+#define FOLLY_PRINTF_FORMAT _Printf_format_string_+#define FOLLY_PRINTF_FORMAT_ATTR(format_param, dots_param) /**/+#else+#define FOLLY_PRINTF_FORMAT /**/+#define FOLLY_PRINTF_FORMAT_ATTR(format_param, dots_param) \+  __attribute__((__format__(__printf__, format_param, dots_param)))+#endif++// warn unused result+#if defined(__has_cpp_attribute)+#if __has_cpp_attribute(nodiscard)+#if defined(__clang__) || defined(__GNUC__)+#if __clang_major__ >= 10 || __GNUC__ >= 10+// early clang and gcc both warn on [[nodiscard]] when applied to class ctors+// easiest option is just to avoid emitting [[nodiscard]] under early clang/gcc+#define FOLLY_NODISCARD [[nodiscard]]+#endif+#endif+#endif+#endif+#ifndef FOLLY_NODISCARD+#define FOLLY_NODISCARD+#endif++// older clang-format gets confused by [[deprecated(...)]] on class decls+#define FOLLY_DEPRECATED(...) [[deprecated(__VA_ARGS__)]]++// target+#ifdef _MSC_VER+#define FOLLY_TARGET_ATTRIBUTE(target)+#else+#define FOLLY_TARGET_ATTRIBUTE(target) __attribute__((__target__(target)))+#endif++// detection for 64 bit+#if defined(__x86_64__) || defined(_M_X64)+#define FOLLY_X64 1+#else+#define FOLLY_X64 0+#endif++#if defined(__arm__)+#define FOLLY_ARM 1+#else+#define FOLLY_ARM 0+#endif++#if defined(__aarch64__)+#define FOLLY_AARCH64 1+#else+#define FOLLY_AARCH64 0+#endif++#if defined(__powerpc64__)+#define FOLLY_PPC64 1+#else+#define FOLLY_PPC64 0+#endif++#if defined(__s390x__)+#define FOLLY_S390X 1+#else+#define FOLLY_S390X 0+#endif++#if defined(__riscv)+#define FOLLY_RISCV64 1+#else+#define FOLLY_RISCV64 0+#endif++#if defined(__wasm__)+#define FOLLY_WASM 1+#else+#define FOLLY_WASM 0+#endif++namespace folly {+constexpr bool kIsArchArm = FOLLY_ARM == 1;+constexpr bool kIsArchAmd64 = FOLLY_X64 == 1;+constexpr bool kIsArchAArch64 = FOLLY_AARCH64 == 1;+constexpr bool kIsArchPPC64 = FOLLY_PPC64 == 1;+constexpr bool kIsArchS390X = FOLLY_S390X == 1;+constexpr bool kIsArchRISCV64 = FOLLY_RISCV64 == 1;+constexpr bool kIsArchWasm = FOLLY_WASM == 1;+} // namespace folly++namespace folly {++/**+ * folly::kIsLibrarySanitizeAddress reports if folly was compiled with ASAN+ * enabled.  Note that for compilation units outside of folly that include+ * folly/Portability.h, the value of kIsLibrarySanitizeAddress may be different+ * from whether or not the current compilation unit is being compiled with ASAN.+ */+#if FOLLY_LIBRARY_SANITIZE_ADDRESS+constexpr bool kIsLibrarySanitizeAddress = true;+#else+constexpr bool kIsLibrarySanitizeAddress = false;+#endif++#ifdef FOLLY_SANITIZE_ADDRESS+constexpr bool kIsSanitizeAddress = true;+#else+constexpr bool kIsSanitizeAddress = false;+#endif++#ifdef FOLLY_SANITIZE_THREAD+constexpr bool kIsSanitizeThread = true;+#else+constexpr bool kIsSanitizeThread = false;+#endif++#ifdef FOLLY_SANITIZE_DATAFLOW+constexpr bool kIsSanitizeDataflow = true;+#else+constexpr bool kIsSanitizeDataflow = false;+#endif++#ifdef FOLLY_SANITIZE+constexpr bool kIsSanitize = true;+#else+constexpr bool kIsSanitize = false;+#endif++#if defined(__OPTIMIZE__)+constexpr bool kIsOptimize = true;+#else+constexpr bool kIsOptimize = false;+#endif++#if defined(__OPTIMIZE_SIZE__)+constexpr bool kIsOptimizeSize = true;+#else+constexpr bool kIsOptimizeSize = false;+#endif+} // namespace folly++// packing is very ugly in msvc+#ifdef _MSC_VER+#define FOLLY_PACK_ATTR /**/+#define FOLLY_PACK_PUSH __pragma(pack(push, 1))+#define FOLLY_PACK_POP __pragma(pack(pop))+#elif defined(__GNUC__)+#define FOLLY_PACK_ATTR __attribute__((__packed__))+#define FOLLY_PACK_PUSH /**/+#define FOLLY_PACK_POP /**/+#else+#define FOLLY_PACK_ATTR /**/+#define FOLLY_PACK_PUSH /**/+#define FOLLY_PACK_POP /**/+#endif++// It turns out that GNU libstdc++ and LLVM libc++ differ on how they implement+// the 'std' namespace; the latter uses inline namespaces. Wrap this decision+// up in a macro to make forward-declarations easier.+#if defined(_LIBCPP_VERSION)+#define FOLLY_NAMESPACE_STD_BEGIN _LIBCPP_BEGIN_NAMESPACE_STD+#define FOLLY_NAMESPACE_STD_END _LIBCPP_END_NAMESPACE_STD+#else+#define FOLLY_NAMESPACE_STD_BEGIN namespace std {+#define FOLLY_NAMESPACE_STD_END }+#endif++// If the new c++ ABI is used, __cxx11 inline namespace needs to be added to+// some types, e.g. std::list.+#if defined(_GLIBCXX_USE_CXX11_ABI) && _GLIBCXX_USE_CXX11_ABI+#define FOLLY_GLIBCXX_NAMESPACE_CXX11_BEGIN \+  inline _GLIBCXX_BEGIN_NAMESPACE_CXX11+#define FOLLY_GLIBCXX_NAMESPACE_CXX11_END _GLIBCXX_END_NAMESPACE_CXX11+#else+#define FOLLY_GLIBCXX_NAMESPACE_CXX11_BEGIN+#define FOLLY_GLIBCXX_NAMESPACE_CXX11_END+#endif++// MSVC specific defines+// mainly for posix compat+#ifdef _MSC_VER++// We have compiler support for the newest of the new, but+// MSVC doesn't tell us that.+//+// Clang pretends to be MSVC on Windows, but it refuses to compile+// SSE4.2 intrinsics unless -march argument is specified.+// So cannot unconditionally define __SSE4_2__ in clang.+#ifndef __clang__+#if !defined(_M_ARM) && !defined(_M_ARM64)+#define __SSE4_2__ 1+#endif // !defined(_M_ARM) && !defined(_M_ARM64)++// Hide a GCC specific thing that breaks MSVC if left alone.+#define __extension__++// compiler specific to compiler specific+// nolint+#define __PRETTY_FUNCTION__ __FUNCSIG__+#endif++#endif++// Define FOLLY_HAS_EXCEPTIONS+#if (defined(__cpp_exceptions) && __cpp_exceptions >= 199711) || \+    FOLLY_HAS_FEATURE(cxx_exceptions)+#define FOLLY_HAS_EXCEPTIONS 1+#elif __GNUC__+#if defined(__EXCEPTIONS) && __EXCEPTIONS+#define FOLLY_HAS_EXCEPTIONS 1+#else // __EXCEPTIONS+#define FOLLY_HAS_EXCEPTIONS 0+#endif // __EXCEPTIONS+#elif FOLLY_MICROSOFT_ABI_VER+#if _CPPUNWIND+#define FOLLY_HAS_EXCEPTIONS 1+#else // _CPPUNWIND+#define FOLLY_HAS_EXCEPTIONS 0+#endif // _CPPUNWIND+#else+#define FOLLY_HAS_EXCEPTIONS 1 // default assumption for unknown platforms+#endif++// Debug+namespace folly {+#ifdef NDEBUG+constexpr auto kIsDebug = false;+#else+constexpr auto kIsDebug = true;+#endif+} // namespace folly++// Exceptions+namespace folly {+#if FOLLY_HAS_EXCEPTIONS+constexpr auto kHasExceptions = true;+#else+constexpr auto kHasExceptions = false;+#endif+} // namespace folly++// Endianness+namespace folly {+#ifdef _MSC_VER+// It's MSVC, so we just have to guess ... and allow an override+#ifdef FOLLY_ENDIAN_BE+constexpr auto kIsLittleEndian = false;+#else+constexpr auto kIsLittleEndian = true;+#endif+#else+constexpr auto kIsLittleEndian = __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__;+#endif+constexpr auto kIsBigEndian = !kIsLittleEndian;+} // namespace folly++// Weak+namespace folly {+#if FOLLY_HAVE_WEAK_SYMBOLS+constexpr auto kHasWeakSymbols = true;+#else+constexpr auto kHasWeakSymbols = false;+#endif+} // namespace folly++#ifndef FOLLY_SSE+#if defined(__SSE4_2__)+#define FOLLY_SSE 4+#define FOLLY_SSE_MINOR 2+#elif defined(__SSE4_1__)+#define FOLLY_SSE 4+#define FOLLY_SSE_MINOR 1+#elif defined(__SSE4__)+#define FOLLY_SSE 4+#define FOLLY_SSE_MINOR 0+#elif defined(__SSE3__)+#define FOLLY_SSE 3+#define FOLLY_SSE_MINOR 0+#elif defined(__SSE2__)+#define FOLLY_SSE 2+#define FOLLY_SSE_MINOR 0+#elif defined(__SSE__)+#define FOLLY_SSE 1+#define FOLLY_SSE_MINOR 0+#else+#define FOLLY_SSE 0+#define FOLLY_SSE_MINOR 0+#endif+#endif++#ifndef FOLLY_SSSE+#if defined(__SSSE3__)+#define FOLLY_SSSE 3+#else+#define FOLLY_SSSE 0+#endif+#endif++#define FOLLY_SSE_PREREQ(major, minor) \+  (FOLLY_SSE > major || FOLLY_SSE == major && FOLLY_SSE_MINOR >= minor)++#ifndef FOLLY_NEON+#if (defined(__ARM_NEON) || defined(__ARM_NEON__)) && !defined(__CUDACC__)+#define FOLLY_NEON 1+#else+#define FOLLY_NEON 0+#endif+#endif++#ifndef FOLLY_ARM_FEATURE_CRC32+#ifdef __ARM_FEATURE_CRC32+#define FOLLY_ARM_FEATURE_CRC32 1+#else+#define FOLLY_ARM_FEATURE_CRC32 0+#endif+#endif++#ifndef FOLLY_ARM_FEATURE_CRYPTO+#ifdef __ARM_FEATURE_CRYPTO+#define FOLLY_ARM_FEATURE_CRYPTO 1+#else+#define FOLLY_ARM_FEATURE_CRYPTO 0+#endif+#endif++#ifndef FOLLY_ARM_FEATURE_AES+#ifdef __ARM_FEATURE_AES+#define FOLLY_ARM_FEATURE_AES 1+#else+#define FOLLY_ARM_FEATURE_AES 0+#endif+#endif++#ifndef FOLLY_ARM_FEATURE_SHA2+#ifdef __ARM_FEATURE_SHA2+#define FOLLY_ARM_FEATURE_SHA2 1+#else+#define FOLLY_ARM_FEATURE_SHA2 0+#endif+#endif++#ifndef FOLLY_ARM_FEATURE_SHA3+#ifdef __ARM_FEATURE_SHA3+#define FOLLY_ARM_FEATURE_SHA3 1+#else+#define FOLLY_ARM_FEATURE_SHA3 0+#endif+#endif++#ifndef FOLLY_ARM_FEATURE_SVE+#ifdef __ARM_FEATURE_SVE+#define FOLLY_ARM_FEATURE_SVE 1+#else+#define FOLLY_ARM_FEATURE_SVE 0+#endif+#endif++#ifndef FOLLY_ARM_FEATURE_SVE2+#ifdef __ARM_FEATURE_SVE2+#define FOLLY_ARM_FEATURE_SVE2 1+#else+#define FOLLY_ARM_FEATURE_SVE2 0+#endif+#endif++#ifndef FOLLY_ARM_FEATURE_NEON_SVE_BRIDGE+#if FOLLY_ARM_FEATURE_SVE && __has_include(<arm_neon_sve_bridge.h>)+#define FOLLY_ARM_FEATURE_NEON_SVE_BRIDGE 1+#else+#define FOLLY_ARM_FEATURE_NEON_SVE_BRIDGE 0+#endif+#endif++// RTTI may not be enabled for this compilation unit.+#if defined(__GXX_RTTI) || defined(__cpp_rtti) || \+    (defined(_MSC_VER) && defined(_CPPRTTI))+#define FOLLY_HAS_RTTI 1+#else+#define FOLLY_HAS_RTTI 0+#endif++namespace folly {+constexpr bool const kHasRtti = FOLLY_HAS_RTTI;+} // namespace folly++#if defined(__APPLE__) || defined(_MSC_VER)+#define FOLLY_STATIC_CTOR_PRIORITY_MAX+#else+// 101 is the highest priority allowed by the init_priority attribute.+// This priority is already used by JEMalloc and other memory allocators so+// we will take the next one.+#define FOLLY_STATIC_CTOR_PRIORITY_MAX __attribute__((__init_priority__(102)))+#endif++#if defined(__APPLE__) && TARGET_OS_IOS+#define FOLLY_APPLE_IOS 1+#else+#define FOLLY_APPLE_IOS 0+#endif++#if defined(__APPLE__) && TARGET_OS_OSX+#define FOLLY_APPLE_MACOS 1+#else+#define FOLLY_APPLE_MACOS 0+#endif++#if defined(__APPLE__) && TARGET_OS_TV+#define FOLLY_APPLE_TVOS 1+#else+#define FOLLY_APPLE_TVOS 0+#endif++#if defined(__APPLE__) && TARGET_OS_WATCH+#define FOLLY_APPLE_WATCHOS 1+#else+#define FOLLY_APPLE_WATCHOS 0+#endif++namespace folly {++#ifdef __OBJC__+constexpr auto kIsObjC = true;+#else+constexpr auto kIsObjC = false;+#endif++#if FOLLY_MOBILE+constexpr auto kIsMobile = true;+#else+constexpr auto kIsMobile = false;+#endif++#if defined(__linux__) && !FOLLY_MOBILE+constexpr auto kIsLinux = true;+#else+constexpr auto kIsLinux = false;+#endif++#if defined(__FreeBSD__)+constexpr auto kIsFreeBSD = true;+#else+constexpr auto kIsFreeBSD = false;+#endif++#if defined(_WIN32)+constexpr auto kIsWindows = true;+#else+constexpr auto kIsWindows = false;+#endif++#if defined(__APPLE__)+constexpr auto kIsApple = true;+#else+constexpr auto kIsApple = false;+#endif++constexpr bool kIsAppleIOS = FOLLY_APPLE_IOS == 1;+constexpr bool kIsAppleMacOS = FOLLY_APPLE_MACOS == 1;+constexpr bool kIsAppleTVOS = FOLLY_APPLE_TVOS == 1;+constexpr bool kIsAppleWatchOS = FOLLY_APPLE_WATCHOS == 1;++#if defined(__GLIBCXX__)+constexpr auto kIsGlibcxx = true;+#else+constexpr auto kIsGlibcxx = false;+#endif++#if defined(__GLIBCXX__) && _GLIBCXX_RELEASE // major version, 7++constexpr auto kGlibcxxVer = _GLIBCXX_RELEASE;+#else+constexpr auto kGlibcxxVer = 0;+#endif++#if defined(__GLIBCXX__) && defined(_GLIBCXX_ASSERTIONS)+constexpr auto kGlibcxxAssertions = true;+#else+constexpr auto kGlibcxxAssertions = false;+#endif++#ifdef _LIBCPP_VERSION+constexpr auto kIsLibcpp = true;+#else+constexpr auto kIsLibcpp = false;+#endif++#if defined(__GLIBCXX__)+constexpr auto kIsLibstdcpp = true;+#else+constexpr auto kIsLibstdcpp = false;+#endif++#ifdef _MSC_VER+constexpr auto kMscVer = _MSC_VER;+#else+constexpr auto kMscVer = 0;+#endif++#if defined(__GNUC__) && __GNUC__+constexpr auto kGnuc = __GNUC__;+#else+constexpr auto kGnuc = 0;+#endif++#if __clang__+constexpr auto kIsClang = true;+constexpr auto kClangVerMajor = __clang_major__;+#else+constexpr auto kIsClang = false;+constexpr auto kClangVerMajor = 0;+#endif++#ifdef FOLLY_MICROSOFT_ABI_VER+constexpr auto kMicrosoftAbiVer = FOLLY_MICROSOFT_ABI_VER;+#else+constexpr auto kMicrosoftAbiVer = 0;+#endif++// cpplib is an implementation of the standard library, and is the one typically+// used with the msvc compiler+#ifdef _CPPLIB_VER+constexpr auto kCpplibVer = _CPPLIB_VER;+#else+constexpr auto kCpplibVer = 0;+#endif+} // namespace folly++#define FOLLY_PRAGMA_DETAIL_STR(X) #X++#if defined(_MSC_VER)+#define FOLLY_PRAGMA_UNROLL_N(N)+#elif defined(__GNUC__)+#define FOLLY_PRAGMA_UNROLL_N(N) _Pragma(FOLLY_PRAGMA_DETAIL_STR(GCC unroll(N)))+#else+#define FOLLY_PRAGMA_UNROLL_N(N) _Pragma(FOLLY_PRAGMA_DETAIL_STR(unroll(N)))+#endif++//  MSVC does not permit:+//+//    extern int const num;+//    constexpr int const num = 3;+//+//  Instead:+//+//    extern int const num;+//    FOLLY_STORAGE_CONSTEXPR int const num = 3;+//+//  True as of MSVC 2017.+#ifdef _MSC_VER+#define FOLLY_STORAGE_CONSTEXPR+#else+#define FOLLY_STORAGE_CONSTEXPR constexpr+#endif++//  FOLLY_CXX20_CONSTEXPR+//+//  C++20 permits more cases to be marked constexpr, including constructors that+//  leave members uninitialized and virtual functions.+#if FOLLY_CPLUSPLUS >= 202002L+#define FOLLY_CXX20_CONSTEXPR constexpr+#else+#define FOLLY_CXX20_CONSTEXPR+#endif++//  FOLLY_CXX23_CONSTEXPR+//+//  C++23 permits more cases to be marked constexpr, including definitions of+//  variables of non-literal type in constexpr function as long as they are not+//  constant-evaluated.+#if FOLLY_CPLUSPLUS >= 202302L+#define FOLLY_CXX23_CONSTEXPR constexpr+#else+#define FOLLY_CXX23_CONSTEXPR+#endif++// C++20 constinit+#if defined(__cpp_constinit) && __cpp_constinit >= 201907L+#define FOLLY_CONSTINIT constinit+#else+#define FOLLY_CONSTINIT+#endif++#if defined(FOLLY_CFG_NO_COROUTINES)+#define FOLLY_HAS_COROUTINES 0+#define FOLLY_HAS_IMMOVABLE_COROUTINES 0+#else+// folly::coro requires C++17 support+#if defined(__NVCC__)+// For now, NVCC matches other compilers but does not offer coroutines.+#define FOLLY_HAS_COROUTINES 0+#elif defined(_WIN32) && defined(__clang__) && !defined(LLVM_COROUTINES) && \+    !defined(LLVM_COROUTINES_CPP20)+// LLVM and MSVC coroutines are ABI incompatible, so for the MSVC implementation+// of <experimental/coroutine> on Windows we *don't* have coroutines.+//+// LLVM_COROUTINES indicates that LLVM compatible header is added to include+// path and can be used.+//+// LLVM_COROUTINES_CPP20 indicates that an LLVM compatible header using+// <coroutine> is added to the include path and can be used.++//+// Worse, if we define FOLLY_HAS_COROUTINES 1 we will include+// <experimental/coroutine> which will conflict with anyone who wants to load+// the LLVM implementation of coroutines on Windows.+#define FOLLY_HAS_COROUTINES 0+#elif defined(_MSC_VER) && _MSC_VER && defined(_RESUMABLE_FUNCTIONS_SUPPORTED)+// NOTE: MSVC 2017 does not currently support the full Coroutines TS since it+// does not yet support symmetric-transfer.+#define FOLLY_HAS_COROUTINES 0+#elif (                                                                    \+    (defined(__cpp_coroutines) && __cpp_coroutines >= 201703L) ||          \+    (defined(__cpp_impl_coroutine) && __cpp_impl_coroutine >= 201902L)) && \+    (__has_include(<coroutine>) || __has_include(<experimental/coroutine>))+#define FOLLY_HAS_COROUTINES 1+// This is mainly to workaround bugs triggered by LTO, when stack allocated+// variables in await_suspend end up on a coroutine frame.+#define FOLLY_CORO_AWAIT_SUSPEND_NONTRIVIAL_ATTRIBUTES FOLLY_NOINLINE+#else+#define FOLLY_HAS_COROUTINES 0+#endif++// NB: The C++20 requirement could be relaxed, but there's no clear benefit as+// of right now.+#if !FOLLY_HAS_COROUTINES || FOLLY_CPLUSPLUS < 202002L+#define FOLLY_HAS_IMMOVABLE_COROUTINES 0+// This logic is written as "good until proven broken" because it's possible+// that there's a good compiler older than the oldest good version I checked.+#elif defined(__clang_major__) && __clang_major__ <= 14+//  - 12.0.1 is bad: https://godbolt.org/z/6s489xE8P+//  - 14 is still bad: https://godbolt.org/z/nW1W8cWvb+//  - 15.0.0 is good: https://godbolt.org/z/Tco4c9hbq and sEaKKTf8r+#define FOLLY_HAS_IMMOVABLE_COROUTINES 0+// BEWARE: Older versions of Clang pretend to be MSVC and define+// `_MSC_FULL_VER`, but fortunately none of clang 15, 16, 17, 18, 19 do this,+// so this branch should not result in a false-negative.+#elif defined(_MSC_FULL_VER) && _MSC_FULL_VER <= 192930040+//  - 192930040 is bad: https://godbolt.org/z/E797W8xTT+//  - 192930153 is good: https://godbolt.org/z/cM4nW5rTK+#define FOLLY_HAS_IMMOVABLE_COROUTINES 0+#else+#define FOLLY_HAS_IMMOVABLE_COROUTINES 1 // good until proven broken+#endif+#endif // FOLLY_CFG_NO_COROUTINES++// It'd be possible to relax this, by refactoring `folly/result` code down to+// C++17, and by only blocking the coroutine support for non-coro compiles.+// However, `result<T>` is primarily targeted at newer codebases.+#if FOLLY_CPLUSPLUS >= 202002L && FOLLY_HAS_COROUTINES+#define FOLLY_HAS_RESULT 1+#else+#define FOLLY_HAS_RESULT 0+#endif++// C++20 consteval+#if FOLLY_CPLUSPLUS >= 202002L+#define FOLLY_CONSTEVAL consteval+#else+#define FOLLY_CONSTEVAL constexpr+#endif
+ folly/folly/Preprocessor.h view
@@ -0,0 +1,243 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <folly/CPortability.h>++/**+ * Necessarily evil preprocessor-related amenities.+ */++// MSVC's preprocessor is a pain, so we have to+// forcefully expand the VA args in some places.+#define FB_VA_GLUE(a, b) a b++/**+ * FB_ONE_OR_NONE(hello, world) expands to hello and+ * FB_ONE_OR_NONE(hello) expands to nothing. This macro is used to+ * insert or eliminate text based on the presence of another argument.+ */+#define FB_ONE_OR_NONE(a, ...) FB_VA_GLUE(FB_THIRD, (a, ##__VA_ARGS__, a))+#define FB_THIRD(a, b, ...) __VA_ARGS__++/**+ * Helper macro that extracts the first argument out of a list of any+ * number of arguments.+ */+#define FB_ARG_1(a, ...) a++/**+ * Helper macro that extracts the second argument out of a list of any+ * number of arguments. If only one argument is given, it returns+ * that.+ */+#ifdef _MSC_VER+// GCC refuses to expand this correctly if this macro itself was+// called with FB_VA_GLUE :(+#define FB_ARG_2_OR_1(...) \+  FB_VA_GLUE(FB_ARG_2_OR_1_IMPL, (__VA_ARGS__, __VA_ARGS__))+#else+#define FB_ARG_2_OR_1(...) FB_ARG_2_OR_1_IMPL(__VA_ARGS__, __VA_ARGS__)+#endif+// Support macro for the above+#define FB_ARG_2_OR_1_IMPL(a, b, ...) b++/**+ * Helper macro that provides a way to pass argument with commas in it to+ * some other macro whose syntax doesn't allow using extra parentheses.+ * Example:+ *+ *   #define MACRO(type, name) type name+ *   MACRO(FB_SINGLE_ARG(std::pair<size_t, size_t>), x);+ *+ */+#define FB_SINGLE_ARG(...) __VA_ARGS__++#define FOLLY_PP_DETAIL_APPEND_VA_ARG(...) , ##__VA_ARGS__++/**+ * Helper macro that just ignores its parameters.+ */+#define FOLLY_IGNORE(...)++/**+ * Helper macro that just ignores its parameters and inserts a semicolon.+ */+#define FOLLY_SEMICOLON(...) ;++/**+ * FB_ANONYMOUS_VARIABLE(str) introduces an identifier starting with+ * str and ending with a number that varies with the line.+ */+#ifndef FB_ANONYMOUS_VARIABLE+#define FB_CONCATENATE_IMPL(s1, s2) s1##s2+#define FB_CONCATENATE(s1, s2) FB_CONCATENATE_IMPL(s1, s2)+#ifdef __COUNTER__+// Modular builds build each module with its own preprocessor state, meaning+// `__COUNTER__` no longer provides a unique number across a TU.  Instead of+// calling back to just `__LINE__`, use a mix of `__COUNTER__` and `__LINE__`+// to try provide as much uniqueness as possible.+#if FOLLY_HAS_FEATURE(modules)+#define FB_ANONYMOUS_VARIABLE(str) \+  FB_CONCATENATE(FB_CONCATENATE(FB_CONCATENATE(str, __COUNTER__), _), __LINE__)+#else+#define FB_ANONYMOUS_VARIABLE(str) FB_CONCATENATE(str, __COUNTER__)+#endif+#else+#define FB_ANONYMOUS_VARIABLE(str) FB_CONCATENATE(str, __LINE__)+#endif+// FB_ANONYMOUS_VARIABLE_ODR_SAFE doesn't rely on __COUNTER__ and is safe to use+// in headers that should not violate the one-definition rule (ODR). It is+// especially useful for C++ modules that check for ODR violations.+#define FB_ANONYMOUS_VARIABLE_ODR_SAFE(str) FB_CONCATENATE(str, __LINE__)+#endif++/**+ * Use FOLLY_PP_STRINGIZE(x) when you'd want to do what #x does inside+ * another macro expansion.+ */+#define FOLLY_PP_STRINGIZE(x) #x++/**+ * Use FOLLY_PP_STRINGIZE_MACRO(x) when you want the string representation+ * of a non-string c++ preprocessing macro value, ex+ * FOLLY_PP_STRINGIZE_MACRO(__LINE__).+ */+#define FOLLY_PP_STRINGIZE_MACRO(x) FOLLY_PP_STRINGIZE(x)++#define FOLLY_PP_DETAIL_NARGS_1( \+    dummy,                       \+    _15,                         \+    _14,                         \+    _13,                         \+    _12,                         \+    _11,                         \+    _10,                         \+    _9,                          \+    _8,                          \+    _7,                          \+    _6,                          \+    _5,                          \+    _4,                          \+    _3,                          \+    _2,                          \+    _1,                          \+    _0,                          \+    ...)                         \+  _0+#define FOLLY_PP_DETAIL_NARGS(...) \+  FOLLY_PP_DETAIL_NARGS_1(         \+      dummy,                       \+      ##__VA_ARGS__,               \+      15,                          \+      14,                          \+      13,                          \+      12,                          \+      11,                          \+      10,                          \+      9,                           \+      8,                           \+      7,                           \+      6,                           \+      5,                           \+      4,                           \+      3,                           \+      2,                           \+      1,                           \+      0)++#define FOLLY_PP_DETAIL_FOR_EACH_REC_0(fn, ...)+#define FOLLY_PP_DETAIL_FOR_EACH_REC_1(fn, a, ...) \+  fn(a) FOLLY_PP_DETAIL_FOR_EACH_REC_0(fn, __VA_ARGS__)+#define FOLLY_PP_DETAIL_FOR_EACH_REC_2(fn, a, ...) \+  fn(a) FOLLY_PP_DETAIL_FOR_EACH_REC_1(fn, __VA_ARGS__)+#define FOLLY_PP_DETAIL_FOR_EACH_REC_3(fn, a, ...) \+  fn(a) FOLLY_PP_DETAIL_FOR_EACH_REC_2(fn, __VA_ARGS__)+#define FOLLY_PP_DETAIL_FOR_EACH_REC_4(fn, a, ...) \+  fn(a) FOLLY_PP_DETAIL_FOR_EACH_REC_3(fn, __VA_ARGS__)+#define FOLLY_PP_DETAIL_FOR_EACH_REC_5(fn, a, ...) \+  fn(a) FOLLY_PP_DETAIL_FOR_EACH_REC_4(fn, __VA_ARGS__)+#define FOLLY_PP_DETAIL_FOR_EACH_REC_6(fn, a, ...) \+  fn(a) FOLLY_PP_DETAIL_FOR_EACH_REC_5(fn, __VA_ARGS__)+#define FOLLY_PP_DETAIL_FOR_EACH_REC_7(fn, a, ...) \+  fn(a) FOLLY_PP_DETAIL_FOR_EACH_REC_6(fn, __VA_ARGS__)+#define FOLLY_PP_DETAIL_FOR_EACH_REC_8(fn, a, ...) \+  fn(a) FOLLY_PP_DETAIL_FOR_EACH_REC_7(fn, __VA_ARGS__)+#define FOLLY_PP_DETAIL_FOR_EACH_REC_9(fn, a, ...) \+  fn(a) FOLLY_PP_DETAIL_FOR_EACH_REC_8(fn, __VA_ARGS__)+#define FOLLY_PP_DETAIL_FOR_EACH_REC_10(fn, a, ...) \+  fn(a) FOLLY_PP_DETAIL_FOR_EACH_REC_9(fn, __VA_ARGS__)+#define FOLLY_PP_DETAIL_FOR_EACH_REC_11(fn, a, ...) \+  fn(a) FOLLY_PP_DETAIL_FOR_EACH_REC_10(fn, __VA_ARGS__)+#define FOLLY_PP_DETAIL_FOR_EACH_REC_12(fn, a, ...) \+  fn(a) FOLLY_PP_DETAIL_FOR_EACH_REC_11(fn, __VA_ARGS__)+#define FOLLY_PP_DETAIL_FOR_EACH_REC_13(fn, a, ...) \+  fn(a) FOLLY_PP_DETAIL_FOR_EACH_REC_12(fn, __VA_ARGS__)+#define FOLLY_PP_DETAIL_FOR_EACH_REC_14(fn, a, ...) \+  fn(a) FOLLY_PP_DETAIL_FOR_EACH_REC_13(fn, __VA_ARGS__)+#define FOLLY_PP_DETAIL_FOR_EACH_REC_15(fn, a, ...) \+  fn(a) FOLLY_PP_DETAIL_FOR_EACH_REC_14(fn, __VA_ARGS__)++#define FOLLY_PP_DETAIL_FOR_EACH_2(fn, n, ...) \+  FOLLY_PP_DETAIL_FOR_EACH_REC_##n(fn, __VA_ARGS__)+#define FOLLY_PP_DETAIL_FOR_EACH_1(fn, n, ...) \+  FOLLY_PP_DETAIL_FOR_EACH_2(fn, n, __VA_ARGS__)++/**+ *  FOLLY_PP_FOR_EACH+ *+ *  Used to invoke a preprocessor macro, the name of which is passed as the+ *  first argument, once for each subsequent variadic argument.+ *+ *  At present, supports [0, 16) arguments.+ *+ *  This input:+ *+ *    #define DOIT(a) go_do_it(a);+ *    FOLLY_PP_FOR_EACH(DOIT, 3, 5, 7)+ *    #undef DOIT+ *+ *  Expands to this output (with whitespace adjusted for clarity):+ *+ *    go_do_it(3);+ *    go_do_it(5);+ *    go_do_it(7);+ */+#define FOLLY_PP_FOR_EACH(fn, ...) \+  FOLLY_PP_DETAIL_FOR_EACH_1(      \+      fn, FOLLY_PP_DETAIL_NARGS(__VA_ARGS__), __VA_ARGS__)++#if defined(U)+#error defined(U) // literal U is used below+#endif++//  FOLLY_PP_CONSTINIT_LINE_UNSIGNED+//+//  MSVC with /ZI has a special backing variable for __LINE__ which is not a+//  literal - but token-pasting __LINE__ suppresses this backing variable. This+//  is done in MSVC to support its edit-and-continue feature.+//+//  This macro evaluates to:+//    __LINE__ ## U+//+//  So this macro may be ill-suited to cases which need exactly __LINE__.+//+//  Documentation:+//    https://docs.microsoft.com/en-us/cpp/build/reference/z7-zi-zi-debug-information-format?view=msvc-170#zi-1+//  Workaround:+//    https://stackoverflow.com/questions/57137351/line-is-not-constexpr-in-msvc+#define FOLLY_PP_CONSTINIT_LINE_UNSIGNED FB_CONCATENATE(__LINE__, U)
+ folly/folly/ProducerConsumerQueue.h view
@@ -0,0 +1,182 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <atomic>+#include <cassert>+#include <cstdlib>+#include <memory>+#include <stdexcept>+#include <type_traits>+#include <utility>++#include <folly/concurrency/CacheLocality.h>++namespace folly {++/*+ * ProducerConsumerQueue is a one producer and one consumer queue+ * without locks.+ */+template <class T>+struct ProducerConsumerQueue {+  typedef T value_type;++  ProducerConsumerQueue(const ProducerConsumerQueue&) = delete;+  ProducerConsumerQueue& operator=(const ProducerConsumerQueue&) = delete;++  // size must be >= 2.+  //+  // Also, note that the number of usable slots in the queue at any+  // given time is actually (size-1), so if you start with an empty queue,+  // isFull() will return true after size-1 insertions.+  explicit ProducerConsumerQueue(uint32_t size)+      : size_(size),+        records_(static_cast<T*>(std::malloc(sizeof(T) * size))),+        readIndex_(0),+        writeIndex_(0) {+    assert(size >= 2);+    if (!records_) {+      throw std::bad_alloc();+    }+  }++  ~ProducerConsumerQueue() {+    // We need to destruct anything that may still exist in our queue.+    // (No real synchronization needed at destructor time: only one+    // thread can be doing this.)+    if (!std::is_trivially_destructible<T>::value) {+      size_t readIndex = readIndex_;+      size_t endIndex = writeIndex_;+      while (readIndex != endIndex) {+        records_[readIndex].~T();+        if (++readIndex == size_) {+          readIndex = 0;+        }+      }+    }++    std::free(records_);+  }++  template <class... Args>+  bool write(Args&&... recordArgs) {+    auto const currentWrite = writeIndex_.load(std::memory_order_relaxed);+    auto nextRecord = currentWrite + 1;+    if (nextRecord == size_) {+      nextRecord = 0;+    }+    if (nextRecord != readIndex_.load(std::memory_order_acquire)) {+      new (&records_[currentWrite]) T(std::forward<Args>(recordArgs)...);+      writeIndex_.store(nextRecord, std::memory_order_release);+      return true;+    }++    // queue is full+    return false;+  }++  // move (or copy) the value at the front of the queue to given variable+  bool read(T& record) {+    auto const currentRead = readIndex_.load(std::memory_order_relaxed);+    if (currentRead == writeIndex_.load(std::memory_order_acquire)) {+      // queue is empty+      return false;+    }++    auto nextRecord = currentRead + 1;+    if (nextRecord == size_) {+      nextRecord = 0;+    }+    record = std::move(records_[currentRead]);+    records_[currentRead].~T();+    readIndex_.store(nextRecord, std::memory_order_release);+    return true;+  }++  // pointer to the value at the front of the queue (for use in-place) or+  // nullptr if empty.+  T* frontPtr() {+    auto const currentRead = readIndex_.load(std::memory_order_relaxed);+    if (currentRead == writeIndex_.load(std::memory_order_acquire)) {+      // queue is empty+      return nullptr;+    }+    return &records_[currentRead];+  }++  // queue must not be empty+  void popFront() {+    auto const currentRead = readIndex_.load(std::memory_order_relaxed);+    assert(currentRead != writeIndex_.load(std::memory_order_acquire));++    auto nextRecord = currentRead + 1;+    if (nextRecord == size_) {+      nextRecord = 0;+    }+    records_[currentRead].~T();+    readIndex_.store(nextRecord, std::memory_order_release);+  }++  bool isEmpty() const {+    return readIndex_.load(std::memory_order_acquire) ==+        writeIndex_.load(std::memory_order_acquire);+  }++  bool isFull() const {+    auto nextRecord = writeIndex_.load(std::memory_order_acquire) + 1;+    if (nextRecord == size_) {+      nextRecord = 0;+    }+    if (nextRecord != readIndex_.load(std::memory_order_acquire)) {+      return false;+    }+    // queue is full+    return true;+  }++  // * If called by consumer, then true size may be more (because producer may+  //   be adding items concurrently).+  // * If called by producer, then true size may be less (because consumer may+  //   be removing items concurrently).+  // * It is undefined to call this from any other thread.+  size_t sizeGuess() const {+    int ret = writeIndex_.load(std::memory_order_acquire) -+        readIndex_.load(std::memory_order_acquire);+    if (ret < 0) {+      ret += size_;+    }+    return ret;+  }++  // maximum number of items in the queue.+  size_t capacity() const { return size_ - 1; }++ private:+  using AtomicIndex = std::atomic<unsigned int>;++  char pad0_[hardware_destructive_interference_size];+  const uint32_t size_;+  T* const records_;++  alignas(hardware_destructive_interference_size) AtomicIndex readIndex_;+  alignas(hardware_destructive_interference_size) AtomicIndex writeIndex_;++  char pad1_[hardware_destructive_interference_size - sizeof(AtomicIndex)];+};++} // namespace folly
+ folly/folly/RWSpinLock.h view
@@ -0,0 +1,17 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#include <folly/synchronization/RWSpinLock.h> // @shim
+ folly/folly/Random-inl.h view
@@ -0,0 +1,86 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#ifndef FOLLY_RANDOM_H_+#error This file may only be included from folly/Random.h+#endif++namespace folly {++namespace detail {++// Return the state size needed by RNG, expressed as a number of uint32_t+// integers. Specialized for all templates specified in the C++11 standard.+// For some (mersenne_twister_engine), this is exported as a state_size static+// data member; for others, the standard shows formulas.++template <class RNG, typename = void>+struct StateSize {+  // A sane default.+  using type = std::integral_constant<size_t, 512>;+};++template <class RNG>+struct StateSize<RNG, void_t<decltype(RNG::state_size)>> {+  using type = std::integral_constant<size_t, RNG::state_size>;+};++template <class UIntType, UIntType a, UIntType c, UIntType m>+struct StateSize<std::linear_congruential_engine<UIntType, a, c, m>> {+  // From the standard [rand.eng.lcong], this is ceil(log2(m) / 32) + 3,+  // which is the same as ceil(ceil(log2(m) / 32) + 3, and+  // ceil(log2(m)) <= std::numeric_limits<UIntType>::digits+  using type = std::integral_constant<+      size_t,+      (std::numeric_limits<UIntType>::digits + 31) / 32 + 3>;+};++template <class UIntType, size_t w, size_t s, size_t r>+struct StateSize<std::subtract_with_carry_engine<UIntType, w, s, r>> {+  // [rand.eng.sub]: r * ceil(w / 32)+  using type = std::integral_constant<size_t, r*((w + 31) / 32)>;+};++template <typename RNG>+using StateSizeT = _t<StateSize<RNG>>;++template <class RNG>+struct SeedData {+  SeedData() {+    Random::secureRandom(seedData.data(), seedData.size() * sizeof(uint32_t));+  }++  static constexpr size_t stateSize = StateSizeT<RNG>::value;+  std::array<uint32_t, stateSize> seedData;+};++} // namespace detail++template <class RNG, class /* EnableIf */>+void Random::seed(RNG& rng) {+  detail::SeedData<RNG> sd;+  std::seed_seq s(std::begin(sd.seedData), std::end(sd.seedData));+  rng.seed(s);+}++template <class RNG, class /* EnableIf */>+auto Random::create() -> RNG {+  detail::SeedData<RNG> sd;+  std::seed_seq s(std::begin(sd.seedData), std::end(sd.seedData));+  return RNG(s);+}++} // namespace folly
+ folly/folly/Random.cpp view
@@ -0,0 +1,184 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#include <folly/Random.h>++#include <array>+#include <mutex>+#include <random>++#include <glog/logging.h>+#include <folly/CppAttributes.h>+#include <folly/SingletonThreadLocal.h>+#include <folly/ThreadLocal.h>+#include <folly/detail/FileUtilDetail.h>+#include <folly/portability/Config.h>+#include <folly/portability/SysTime.h>+#include <folly/portability/Unistd.h>+#include <folly/synchronization/RelaxedAtomic.h>++#ifdef _WIN32+#include <wincrypt.h> // @manual+#pragma comment(lib, "advapi32.lib")+#else+#include <fcntl.h>+#endif++#if FOLLY_HAVE_GETRANDOM+#include <sys/random.h>+#endif++namespace folly {++namespace {++void readRandomDevice(void* data, size_t size) {+#ifdef _WIN32+  static auto const cryptoProv = [] {+    HCRYPTPROV prov;+    if (!CryptAcquireContext(+            &prov, nullptr, nullptr, PROV_RSA_FULL, CRYPT_VERIFYCONTEXT)) {+      if (GetLastError() == NTE_BAD_KEYSET) {+        // Mostly likely cause of this is that no key container+        // exists yet, so try to create one.+        PCHECK(CryptAcquireContext(+            &prov, nullptr, nullptr, PROV_RSA_FULL, CRYPT_NEWKEYSET));+      } else {+        LOG(FATAL) << "Failed to acquire the default crypto context.";+      }+    }+    return prov;+  }();+  CHECK(size <= std::numeric_limits<DWORD>::max());+  PCHECK(CryptGenRandom(cryptoProv, (DWORD)size, (BYTE*)data));+#else+  ssize_t bytesRead = 0;+  auto gen = [](int, void* buf, size_t buflen) {+#if FOLLY_HAVE_GETRANDOM+    auto flags = 0u;+    return ::getrandom(buf, buflen, flags);+#else+    [](...) {}(buf, buflen);+    errno = ENOSYS;+    return -1;+#endif+  };+  bytesRead = fileutil_detail::wrapFull(gen, -1, data, size);+  if (bytesRead == -1 && errno == ENOSYS) {+    // Keep the random device open for the duration of the program.+    static int randomFd = ::open("/dev/urandom", O_RDONLY | O_CLOEXEC);+    PCHECK(randomFd >= 0);+    bytesRead = fileutil_detail::wrapFull(::read, randomFd, data, size);+  }+  PCHECK(bytesRead >= 0);+  CHECK_EQ(size_t(bytesRead), size);+#endif+}++class BufferedRandomDevice {+ public:+  static constexpr size_t kDefaultBufferSize = 128;++  static void notifyNewGlobalEpoch() { ++globalEpoch_; }++  explicit BufferedRandomDevice(size_t bufferSize = kDefaultBufferSize);++  void get(void* data, size_t size) {+    if (FOLLY_LIKELY(epoch_ == globalEpoch_ && size <= remaining())) {+      memcpy(data, ptr_, size);+      ptr_ += size;+    } else {+      getSlow(static_cast<unsigned char*>(data), size);+    }+  }++ private:+  void getSlow(unsigned char* data, size_t size);++  inline size_t remaining() const {+    return size_t(buffer_.get() + bufferSize_ - ptr_);+  }++  static relaxed_atomic<size_t> globalEpoch_;++  size_t epoch_{size_t(-1)}; // refill on first use+  const size_t bufferSize_;+  std::unique_ptr<unsigned char[]> buffer_;+  unsigned char* ptr_;+};++relaxed_atomic<size_t> BufferedRandomDevice::globalEpoch_{0};+struct RandomTag {};++BufferedRandomDevice::BufferedRandomDevice(size_t bufferSize)+    : bufferSize_(bufferSize),+      buffer_(new unsigned char[bufferSize]),+      ptr_(buffer_.get() + bufferSize) { // refill on first use+  [[maybe_unused]] static auto const init = [] {+    AtFork::registerHandler(+        nullptr,+        /*prepare*/ []() { return true; },+        /*parent*/ []() {},+        /*child*/+        []() {+          // Ensure child and parent do not share same entropy pool.+          BufferedRandomDevice::notifyNewGlobalEpoch();+        });+    return 0;+  }();+}++void BufferedRandomDevice::getSlow(unsigned char* data, size_t size) {+  if (epoch_ != globalEpoch_) {+    epoch_ = globalEpoch_;+    ptr_ = buffer_.get() + bufferSize_;+  }++  DCHECK_GT(size, remaining());+  if (size >= bufferSize_) {+    // Just read directly.+    readRandomDevice(data, size);+    return;+  }++  size_t copied = remaining();+  memcpy(data, ptr_, copied);+  data += copied;+  size -= copied;++  // refill+  readRandomDevice(buffer_.get(), bufferSize_);+  ptr_ = buffer_.get();++  memcpy(data, ptr_, size);+  ptr_ += size;+}++} // namespace++void Random::secureRandom(void* data, size_t size) {+  using Single = SingletonThreadLocal<BufferedRandomDevice, RandomTag>;+  Single::get().get(data, size);+}++ThreadLocalPRNG::result_type ThreadLocalPRNG::operator()() {+  struct Wrapper {+    Generator object{Random::create()};+  };+  using Single = SingletonThreadLocal<Wrapper, RandomTag>;+  return Single::get().object();+}+} // namespace folly
+ folly/folly/Random.h view
@@ -0,0 +1,437 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++//+// Docs: https://fburl.com/fbvref_random+//++#pragma once+#define FOLLY_RANDOM_H_++#include <cstdint>+#include <random>+#include <type_traits>++#include <folly/Portability.h>+#include <folly/Traits.h>+#include <folly/functional/Invoke.h>+#include <folly/lang/Bits.h>+#include <folly/random/xoshiro256pp.h>++#if defined(FOLLY_HAVE_EXTRANDOM_SFMT19937) && FOLLY_HAVE_EXTRANDOM_SFMT19937+#include <ext/random>+#endif++namespace folly {++namespace detail {++using DefaultGenerator = folly::xoshiro256pp_32;++#if defined(FOLLY_HAVE_EXTRANDOM_SFMT19937) && FOLLY_HAVE_EXTRANDOM_SFMT19937+using LegacyGenerator = __gnu_cxx::sfmt19937;+#else+using LegacyGenerator = std::mt19937;+#endif++} // namespace detail++/**+ * A PRNG with one instance per thread. This PRNG uses a mersenne twister random+ * number generator and is seeded from /dev/urandom. It should not be used for+ * anything which requires security, only for statistical randomness.+ */+class ThreadLocalPRNG {+  using Generator = detail::DefaultGenerator;++ public:+  using result_type = Generator::result_type;++  result_type operator()();++  static constexpr result_type min() { return Generator::min(); }+  static constexpr result_type max() { return Generator::max(); }+};++class Random {+ private:+  template <class RNG>+  using ValidRNG = typename std::+      enable_if<std::is_unsigned<invoke_result_t<RNG&>>::value, RNG>::type;++  template <class T>+  class SecureRNG {+   public:+    using result_type = typename std::enable_if<+        std::is_integral<T>::value && !std::is_same<T, bool>::value,+        T>::type;++    result_type operator()() { return Random::secureRandom<result_type>(); }++    static constexpr result_type min() {+      return std::numeric_limits<result_type>::min();+    }++    static constexpr result_type max() {+      return std::numeric_limits<result_type>::max();+    }+  };++  // Whether RNG output is surjective and uniform when truncated to ResultType.+  template <class RNG, class ResultType>+  static constexpr bool UniformRNG =+      (std::is_unsigned<ResultType>::value &&+       std::is_unsigned<typename RNG::result_type>::value &&+       // RNG range covers ResultType.+       RNG::min() == 0 &&+       RNG::max() >= std::numeric_limits<ResultType>::max() &&+       // Truncating the output maintains uniformness.+       (~RNG::max() == 0 || isPowTwo(RNG::max() + 1)));++ public:+  using DefaultGenerator = detail::DefaultGenerator;+  using LegacyGenerator = detail::LegacyGenerator;++  /**+   * Get secure random bytes. (On Linux and OSX, this means /dev/urandom).+   */+  static void secureRandom(void* data, size_t size);++  /**+   * Shortcut to get a secure random value of integral type.+   */+  template <class T>+  static typename std::enable_if<+      std::is_integral<T>::value && !std::is_same<T, bool>::value,+      T>::type+  secureRandom() {+    T val;+    secureRandom(&val, sizeof(val));+    return val;+  }++  /**+   * Returns a secure random uint32_t+   */+  static uint32_t secureRand32() { return secureRandom<uint32_t>(); }++  /**+   * Returns a secure random uint32_t in [0, max). If max == 0, returns 0.+   */+  static uint32_t secureRand32(uint32_t max) {+    SecureRNG<uint32_t> srng;+    return rand32(max, srng);+  }++  /**+   * Returns a secure random uint32_t in [min, max). If min == max, returns min.+   */+  static uint32_t secureRand32(uint32_t min, uint32_t max) {+    SecureRNG<uint32_t> srng;+    return rand32(min, max, srng);+  }++  /**+   * Returns a secure random uint64_t+   */+  static uint64_t secureRand64() { return secureRandom<uint64_t>(); }++  /**+   * Returns a secure random uint64_t in [0, max). If max == 0, returns 0.+   */+  static uint64_t secureRand64(uint64_t max) {+    SecureRNG<uint64_t> srng;+    return rand64(max, srng);+  }++  /**+   * Returns a secure random uint64_t in [min, max). If min == max, returns min.+   */+  static uint64_t secureRand64(uint64_t min, uint64_t max) {+    SecureRNG<uint64_t> srng;+    return rand64(min, max, srng);+  }++  /**+   * Returns true 1/n of the time. If n == 0, always returns false+   */+  static bool secureOneIn(uint32_t n) {+    if (n < 2) {+      return n;+    }+    SecureRNG<uint32_t> srng;+    return rand32(0, n, srng) == 0;+  }++  /**+   * Returns true 1/n of the time. If n == 0, always returns false+   */+  static bool secureOneIn64(uint64_t n) {+    if (n < 2) {+      return n;+    }+    SecureRNG<uint64_t> srng;+    return rand64(0, n, srng) == 0;+  }++  /**+   * Returns a secure double in [0, 1)+   */+  static double secureRandDouble01() {+    SecureRNG<uint64_t> srng;+    return randDouble01(srng);+  }++  /**+   * Returns a secure double in [min, max), if min == max, returns min.+   */+  static double secureRandDouble(double min, double max) {+    SecureRNG<uint64_t> srng;+    return randDouble(min, max, srng);+  }++  /**+   * (Re-)Seed an existing RNG with a good seed.+   *+   * Note that you should usually use ThreadLocalPRNG unless you need+   * reproducibility (such as during a test), in which case you'd want+   * to create a RNG with a good seed in production, and seed it yourself+   * in test.+   */+  template <class RNG = DefaultGenerator, class /* EnableIf */ = ValidRNG<RNG>>+  static void seed(RNG& rng);++  /**+   * Create a new RNG, seeded with a good seed.+   *+   * Note that you should usually use ThreadLocalPRNG unless you need+   * reproducibility (such as during a test), in which case you'd want+   * to create a RNG with a good seed in production, and seed it yourself+   * in test.+   */+  template <class RNG = DefaultGenerator, class /* EnableIf */ = ValidRNG<RNG>>+  static RNG create();++  /**+   * Create a new RNG, which can be used for applications that require secure+   * randomness.+   *+   * The resulting RNG will have worse performance than one created with+   * create(), so use it if you need the security.+   */+  static SecureRNG<uint32_t> createSecure() { return SecureRNG<uint32_t>(); }++  /**+   * Returns a random uint32_t+   */+  static uint32_t rand32() { return rand32(ThreadLocalPRNG()); }++  /**+   * Returns a random uint32_t given a specific RNG+   */+  template <class RNG, class /* EnableIf */ = ValidRNG<RNG>>+  static uint32_t rand32(RNG&& rng) {+    if constexpr (UniformRNG<std::decay_t<RNG>, uint32_t>) {+      return static_cast<uint32_t>(rng());+    } else {+      return std::uniform_int_distribution<uint32_t>(+          0, std::numeric_limits<uint32_t>::max())(rng);+    }+  }++  /**+   * Returns a random uint32_t in [0, max). If max == 0, returns 0.+   */+  static uint32_t rand32(uint32_t max) {+    return rand32(0, max, ThreadLocalPRNG());+  }++  /**+   * Returns a random uint32_t in [0, max) given a specific RNG.+   * If max == 0, returns 0.+   */+  template <class RNG = ThreadLocalPRNG, class /* EnableIf */ = ValidRNG<RNG>>+  static uint32_t rand32(uint32_t max, RNG&& rng) {+    return rand32(0, max, rng);+  }++  /**+   * Returns a random uint32_t in [min, max). If min == max, returns min.+   */+  static uint32_t rand32(uint32_t min, uint32_t max) {+    return rand32(min, max, ThreadLocalPRNG());+  }++  /**+   * Returns a random uint32_t in [min, max) given a specific RNG.+   * If min == max, returns min.+   */+  template <class RNG = ThreadLocalPRNG, class /* EnableIf */ = ValidRNG<RNG>>+  static uint32_t rand32(uint32_t min, uint32_t max, RNG&& rng) {+    if (min == max) {+      return min;+    }+    return std::uniform_int_distribution<uint32_t>(min, max - 1)(rng);+  }++  /**+   * Returns a random uint64_t+   */+  static uint64_t rand64() { return rand64(ThreadLocalPRNG()); }++  /**+   * Returns a random uint64_t+   */+  template <class RNG = ThreadLocalPRNG, class /* EnableIf */ = ValidRNG<RNG>>+  static uint64_t rand64(RNG&& rng) {+    if constexpr (UniformRNG<std::decay_t<RNG>, uint64_t>) {+      return rng();+    } else if constexpr (UniformRNG<std::decay_t<RNG>, uint32_t>) {+      return (static_cast<uint64_t>(rng()) << 32) |+          static_cast<uint32_t>(rng());+    } else {+      return std::uniform_int_distribution<uint64_t>(+          0, std::numeric_limits<uint64_t>::max())(rng);+    }+  }++  /**+   * Returns a random uint64_t in [0, max). If max == 0, returns 0.+   */+  static uint64_t rand64(uint64_t max) {+    return rand64(0, max, ThreadLocalPRNG());+  }++  /**+   * Returns a random uint64_t in [0, max). If max == 0, returns 0.+   */+  template <class RNG = ThreadLocalPRNG, class /* EnableIf */ = ValidRNG<RNG>>+  static uint64_t rand64(uint64_t max, RNG&& rng) {+    return rand64(0, max, rng);+  }++  /**+   * Returns a random uint64_t in [min, max). If min == max, returns min.+   */+  static uint64_t rand64(uint64_t min, uint64_t max) {+    return rand64(min, max, ThreadLocalPRNG());+  }++  /**+   * Returns a random uint64_t in [min, max). If min == max, returns min.+   */+  template <class RNG = ThreadLocalPRNG, class /* EnableIf */ = ValidRNG<RNG>>+  static uint64_t rand64(uint64_t min, uint64_t max, RNG&& rng) {+    if (min == max) {+      return min;+    }+    return std::uniform_int_distribution<uint64_t>(min, max - 1)(rng);+  }++  /**+   * Returns true 1/n of the time. If n == 0, always returns false+   */+  static bool oneIn(uint32_t n) { return oneIn(n, ThreadLocalPRNG()); }++  /**+   * Returns true 1/n of the time. If n == 0, always returns false+   */+  static bool oneIn64(uint64_t n) { return oneIn64(n, ThreadLocalPRNG()); }++  /**+   * Returns true 1/n of the time. If n == 0, always returns false+   */+  template <class RNG = ThreadLocalPRNG, class /* EnableIf */ = ValidRNG<RNG>>+  static bool oneIn(uint32_t n, RNG&& rng) {+    if (n < 2) {+      return n;+    }+    return rand32(0, n, std::forward<RNG>(rng)) == 0;+  }++  template <class RNG = ThreadLocalPRNG, class /* EnableIf */ = ValidRNG<RNG>>+  static bool oneIn64(uint64_t n, RNG&& rng) {+    if (n < 2) {+      return n;+    }+    return rand64(0, n, std::forward<RNG>(rng)) == 0;+  }++  /**+   * Returns true with the probability of p, false otherwise+   */+  static bool randBool(double p) { return randBool(p, ThreadLocalPRNG()); }++  /**+   * Returns true with the probability of p, false otherwise+   */+  template <class RNG = ThreadLocalPRNG, class /* EnableIf */ = ValidRNG<RNG>>+  static bool randBool(double p, RNG&& rng) {+    return randDouble01(std::forward<RNG>(rng)) < p;+  }++  /**+   * Returns a double in [0, 1)+   */+  static double randDouble01() { return randDouble01(ThreadLocalPRNG()); }++  /**+   * Returns a double in [0, 1)+   */+  template <class RNG = ThreadLocalPRNG, class /* EnableIf */ = ValidRNG<RNG>>+  static double randDouble01(RNG&& rng) {+    // Assuming 64-bit IEEE754 doubles, numbers in the form k/2^53 can be+    // represented exactly, so we can sample uniformly in [0, 1) by sampling an+    // integer in [0, 2^53) and scaling accordingly. This is the highest+    // precision we can obtain if we want a symmetric output distribution.+    // See https://prng.di.unimi.it/#remarks for more details.+    static_assert(+        std::numeric_limits<double>::digits == 53, "Unsupported double type");+    return (rand64(std::forward<RNG>(rng)) >> 11) * 0x1.0p-53;+  }++  /**+   * Returns a double in [min, max), if min == max, returns min.+   */+  static double randDouble(double min, double max) {+    return randDouble(min, max, ThreadLocalPRNG());+  }++  /**+   * Returns a double in [min, max), if min == max, returns min.+   */+  template <class RNG = ThreadLocalPRNG, class /* EnableIf */ = ValidRNG<RNG>>+  static double randDouble(double min, double max, RNG&& rng) {+    if (std::fabs(max - min) < std::numeric_limits<double>::epsilon()) {+      return min;+    }+    return std::uniform_real_distribution<double>(min, max)(rng);+  }+};++/*+ * Return a good seed for a random number generator.+ * Note that this is a legacy function, as it returns a 32-bit value, which+ * is too small to be useful as a "real" RNG seed. Use the functions in class+ * Random instead.+ */+inline uint32_t randomNumberSeed() {+  return Random::rand32();+}++} // namespace folly++#include <folly/Random-inl.h>
+ folly/folly/Range.h view
@@ -0,0 +1,1771 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++//+// Docs: https://fburl.com/fbcref_range+//++/**+ * Range abstraction using a pair of iterators. It is not+ * similar to boost's range abstraction because an API identical+ * with the former StringPiece class is required, which is used alot+ * internally. This abstraction does fulfill the needs of boost's+ * range-oriented algorithms though.+ *+ * Note: (Keep memory lifetime in mind when using this class, since it+ * does not manage the data it refers to - just like an iterator+ * would not.)+ *+ * Additional documentation is in folly/docs/Range.md+ *+ * @refcode folly/docs/examples/folly/Range.h+ * @struct folly::range+ */++#pragma once++#include <folly/Portability.h>+#include <folly/hash/SpookyHashV2.h>+#include <folly/lang/CString.h>+#include <folly/lang/Exception.h>+#include <folly/portability/Constexpr.h>++#include <algorithm>+#include <array>+#include <cassert>+#include <climits>+#include <cstddef>+#include <cstring>+#include <iosfwd>+#include <iterator>+#include <stdexcept>+#include <string>+#include <string_view>+#include <type_traits>++#if defined(__cpp_lib_ranges)+#include <ranges>+#endif++#if __has_include(<fmt/format.h>)+#include <fmt/format.h>+#endif++#include <folly/CpuId.h>+#include <folly/Likely.h>+#include <folly/Traits.h>+#include <folly/detail/RangeCommon.h>+#include <folly/detail/RangeSimd.h>++// Ignore shadowing warnings within this file, so includers can use -Wshadow.+FOLLY_PUSH_WARNING+FOLLY_GNU_DISABLE_WARNING("-Wshadow")++namespace folly {++/**+ * Ubiquitous helper template for knowing what's a string.+ */+template <class T>+struct IsSomeString : std::false_type {};++template <typename Alloc>+struct IsSomeString<std::basic_string<char, std::char_traits<char>, Alloc>>+    : std::true_type {};++template <class Iter>+class Range;++/**+ * Finds the first occurrence of needle in haystack. The algorithm is on+ * average faster than O(haystack.size() * needle.size()) but not as fast+ * as Boyer-Moore. On the upside, it does not do any upfront+ * preprocessing and does not allocate memory.+ */+template <+    class Iter,+    class Comp = std::equal_to<typename Range<Iter>::value_type>>+inline size_t qfind(+    const Range<Iter>& haystack, const Range<Iter>& needle, Comp eq = Comp());++/**+ * Finds the first occurrence of needle in haystack. The result is the+ * offset reported to the beginning of haystack, or string::npos if+ * needle wasn't found.+ */+template <class Iter>+size_t qfind(+    const Range<Iter>& haystack,+    const typename Range<Iter>::value_type& needle);++/**+ * Finds the last occurrence of needle in haystack. The result is the+ * offset reported to the beginning of haystack, or string::npos if+ * needle wasn't found.+ */+template <class Iter>+size_t rfind(+    const Range<Iter>& haystack,+    const typename Range<Iter>::value_type& needle);++/**+ * Finds the first occurrence of any element of needle in+ * haystack. The algorithm is O(haystack.size() * needle.size()).+ */+template <class Iter>+inline size_t qfind_first_of(+    const Range<Iter>& haystack, const Range<Iter>& needle);++/**+ * Small internal helper - returns the value just before an iterator.+ */+namespace detail {++/*+ * Use IsCharPointer<T>::type to enable const char* or char*.+ * Use IsCharPointer<T>::const_type to enable only const char*.+ */+template <class T>+struct IsCharPointer {};++template <>+struct IsCharPointer<char*> {+  using type = int;+};++template <>+struct IsCharPointer<const char*> {+  using const_type = int;+  using type = int;+};++template <class T>+struct IsUnsignedCharPointer {};++template <>+struct IsUnsignedCharPointer<unsigned char*> {+  using type = int;+};++template <>+struct IsUnsignedCharPointer<const unsigned char*> {+  using const_type = int;+  using type = int;+};++void range_is_char_type_f_(char const*);+void range_is_char_type_f_(wchar_t const*);+#if (defined(__cpp_char8_t) && __cpp_char8_t >= 201811L) || \+    FOLLY_CPLUSPLUS >= 202002+void range_is_char_type_f_(char8_t const*);+#endif+void range_is_char_type_f_(char16_t const*);+void range_is_char_type_f_(char32_t const*);+template <typename Iter>+using range_is_char_type_d_ =+    decltype(folly::detail::range_is_char_type_f_(FOLLY_DECLVAL(Iter)));+template <typename Iter>+constexpr bool range_is_char_type_v_ =+    is_detected_v<range_is_char_type_d_, Iter>;++void range_is_byte_type_f_(unsigned char const*);+void range_is_byte_type_f_(signed char const*);+void range_is_byte_type_f_(std::byte const*);+template <typename Iter>+using range_is_byte_type_d_ =+    decltype(folly::detail::range_is_byte_type_f_(FOLLY_DECLVAL(Iter)));+template <typename Iter>+constexpr bool range_is_byte_type_v_ =+    is_detected_v<range_is_byte_type_d_, Iter>;++struct range_traits_char_ {+  template <typename Value>+  using apply = std::char_traits<Value>;+};+struct range_traits_byte_ {+  template <typename Value>+  struct apply {+    FOLLY_ERASE static constexpr bool eq(Value a, Value b) { return a == b; }++    FOLLY_ERASE static constexpr int compare(+        Value const* a, Value const* b, std::size_t c) {+      return !c ? 0 : std::memcmp(a, b, c);+    }+  };+};+struct range_traits_fbck_ {+  template <typename Value>+  struct apply {+    FOLLY_ERASE static constexpr bool eq(Value a, Value b) { return a == b; }++    FOLLY_ERASE static constexpr int compare(+        Value const* a, Value const* b, std::size_t c) {+      while (c--) {+        auto&& ai = *a++;+        auto&& bi = *b++;+        if (ai < bi) {+          return -1;+        }+        if (bi < ai) {+          return +1;+        }+      }+      return 0;+    }+  };+};++template <typename Iter>+using range_traits_c_ = conditional_t<+    range_is_char_type_v_<Iter>,+    range_traits_char_,+    conditional_t< //+        range_is_byte_type_v_<Iter>,+        range_traits_byte_,+        range_traits_fbck_>>;+template <typename Iter, typename Value>+using range_traits_t_ = typename range_traits_c_<Iter>::template apply<Value>;++} // namespace detail++template <class Iter>+class Range {+ private:+  using iter_traits = std::iterator_traits<Iter>;++  template <typename Alloc>+  using string = std::basic_string<char, std::char_traits<char>, Alloc>;++ public:+  using value_type = typename iter_traits::value_type;+  using size_type = std::size_t;+  using difference_type = typename iter_traits::difference_type;+  using iterator = Iter;+  using const_iterator = Iter;+  using reference = typename iter_traits::reference;+  using const_reference = conditional_t<+      std::is_lvalue_reference_v<reference>,+      std::add_lvalue_reference_t<+          std::add_const_t<std::remove_reference_t<reference>>>,+      conditional_t<+          std::is_rvalue_reference_v<reference>,+          std::add_rvalue_reference_t<+              std::add_const_t<std::remove_reference_t<reference>>>,+          reference>>;++  /*+   * For MutableStringPiece and MutableByteRange we define StringPiece+   * and ByteRange as const_range_type (for everything else its just+   * identity). We do that to enable operations such as find with+   * args which are const.+   */+  using const_range_type = typename std::conditional<+      std::is_same<Iter, char*>::value ||+          std::is_same<Iter, unsigned char*>::value,+      Range<const value_type*>,+      Range<Iter>>::type;++  using traits_type = detail::range_traits_t_<Iter, value_type>;++  static const size_type npos;++  /**+   * Works for all iterator+   *+   *+   * @methodset Range+   */+  constexpr Range() : b_(), e_() {}++  constexpr Range(const Range&) = default;+  constexpr Range(Range&&) = default;++ public:+  /**+   * Works for all iterators+   *+   *+   * @methodset Range+   */+  constexpr Range(Iter start, Iter end) : b_(start), e_(end) {}+  /**+   *  Works only for random-access iterators+   *+   *+   * @methodset Range+   */+  constexpr Range(Iter start, size_t size) : b_(start), e_(start + size) {}++  /* implicit */ Range(std::nullptr_t) = delete;++  constexpr /* implicit */ Range(Iter str)+      : b_(str), e_(str + constexpr_strlen(str)) {+    static_assert(+        std::is_same<int, typename detail::IsCharPointer<Iter>::type>::value,+        "This constructor is only available for character ranges");+  }++  template <+      class Alloc,+      class T = Iter,+      typename detail::IsCharPointer<T>::const_type = 0>+  /* implicit */ Range(const string<Alloc>& str)+      : b_(str.data()), e_(b_ + str.size()) {}++  template <+      class Alloc,+      class T = Iter,+      typename detail::IsCharPointer<T>::const_type = 0>+  Range(const string<Alloc>& str, typename string<Alloc>::size_type startFrom) {+    if (FOLLY_UNLIKELY(startFrom > str.size())) {+      throw_exception<std::out_of_range>("index out of range");+    }+    b_ = str.data() + startFrom;+    e_ = str.data() + str.size();+  }++  template <+      class Alloc,+      class T = Iter,+      typename detail::IsCharPointer<T>::const_type = 0>+  Range(+      const string<Alloc>& str,+      typename string<Alloc>::size_type startFrom,+      typename string<Alloc>::size_type size) {+    if (FOLLY_UNLIKELY(startFrom > str.size())) {+      throw_exception<std::out_of_range>("index out of range");+    }+    b_ = str.data() + startFrom;+    if (str.size() - startFrom < size) {+      e_ = str.data() + str.size();+    } else {+      e_ = b_ + size;+    }+  }++  Range(const Range& other, size_type first, size_type length = npos)+      : Range(other.subpiece(first, length)) {}++  template <+      class Container,+      class = typename std::enable_if<+          std::is_same<Iter, typename Container::const_pointer>::value>::type,+      class = decltype(+          Iter(std::declval<Container const&>().data()),+          Iter(+              std::declval<Container const&>().data() ++              std::declval<Container const&>().size()))>+  /* implicit */ constexpr Range(Container const& container)+      : Range(container.data(), container.size()) {}++  template <+      class Container,+      class = typename std::enable_if<+          std::is_same<Iter, typename Container::const_pointer>::value>::type,+      class = decltype(+          Iter(std::declval<Container const&>().data()),+          Iter(+              std::declval<Container const&>().data() ++              std::declval<Container const&>().size()))>+  Range(Container const& container, typename Container::size_type startFrom) {+    auto const cdata = container.data();+    auto const csize = container.size();+    if (FOLLY_UNLIKELY(startFrom > csize)) {+      throw_exception<std::out_of_range>("index out of range");+    }+    b_ = cdata + startFrom;+    e_ = cdata + csize;+  }++  template <+      class Container,+      class = typename std::enable_if<+          std::is_same<Iter, typename Container::const_pointer>::value>::type,+      class = decltype(+          Iter(std::declval<Container const&>().data()),+          Iter(+              std::declval<Container const&>().data() ++              std::declval<Container const&>().size()))>+  Range(+      Container const& container,+      typename Container::size_type startFrom,+      typename Container::size_type size) {+    auto const cdata = container.data();+    auto const csize = container.size();+    if (FOLLY_UNLIKELY(startFrom > csize)) {+      throw_exception<std::out_of_range>("index out of range");+    }+    b_ = cdata + startFrom;+    if (csize - startFrom < size) {+      e_ = cdata + csize;+    } else {+      e_ = b_ + size;+    }+  }++  /**+   * @brief Allow explicit construction of ByteRange from std::string_view or+   * std::string.++   * Given that we allow implicit construction of ByteRange from+   * StringPiece, it makes sense to allow this explicit construction, and avoids+   * callers having to say ByteRange{StringPiece{str}} when they want a+   * ByteRange pointing to data in a std::string.+   *+   *+   * @methodset Range+   */+  template <+      class Container,+      class T = Iter,+      typename detail::IsUnsignedCharPointer<T>::const_type = 0,+      class = typename std::enable_if<+            std::is_same<typename Container::const_pointer, const char*>::value+          >::type,+      class = decltype(+          Iter(std::declval<Container const&>().data()),+          Iter(+              std::declval<Container const&>().data() ++              std::declval<Container const&>().size()))>+  explicit Range(const Container& str)+      : b_(reinterpret_cast<Iter>(str.data())), e_(b_ + str.size()) {}+  /**+   * @brief Allow implicit conversion from Range<const char*> (aka StringPiece)+   to+   * Range<const unsigned char*> (aka ByteRange).++   * Give both are frequently+   * used to represent ranges of bytes.  Allow explicit conversion in the other+   * direction.+   *+   * @methodset Range+   */+  template <+      class OtherIter,+      typename std::enable_if<+          (std::is_same<Iter, const unsigned char*>::value &&+           (std::is_same<OtherIter, const char*>::value ||+            std::is_same<OtherIter, char*>::value)),+          int>::type = 0>+  /* implicit */ Range(const Range<OtherIter>& other)+      : b_(reinterpret_cast<const unsigned char*>(other.begin())),+        e_(reinterpret_cast<const unsigned char*>(other.end())) {}++  template <+      class OtherIter,+      typename std::enable_if<+          (std::is_same<Iter, unsigned char*>::value &&+           std::is_same<OtherIter, char*>::value),+          int>::type = 0>+  /* implicit */ Range(const Range<OtherIter>& other)+      : b_(reinterpret_cast<unsigned char*>(other.begin())),+        e_(reinterpret_cast<unsigned char*>(other.end())) {}++  template <+      class OtherIter,+      typename std::enable_if<+          (std::is_same<Iter, const char*>::value &&+           (std::is_same<OtherIter, const unsigned char*>::value ||+            std::is_same<OtherIter, unsigned char*>::value)),+          int>::type = 0>+  explicit Range(const Range<OtherIter>& other)+      : b_(reinterpret_cast<const char*>(other.begin())),+        e_(reinterpret_cast<const char*>(other.end())) {}++  template <+      class OtherIter,+      typename std::enable_if<+          (std::is_same<Iter, char*>::value &&+           std::is_same<OtherIter, unsigned char*>::value),+          int>::type = 0>+  explicit Range(const Range<OtherIter>& other)+      : b_(reinterpret_cast<char*>(other.begin())),+        e_(reinterpret_cast<char*>(other.end())) {}++  /**+   * Allow implicit conversion from Range<From> to Range<To> if From is+   * implicitly convertible to To.+   *+   * @methodset Range+   */+  template <+      class OtherIter,+      typename std::enable_if<+          (!std::is_same<Iter, OtherIter>::value &&+           std::is_convertible<OtherIter, Iter>::value),+          int>::type = 0>+  constexpr /* implicit */ Range(const Range<OtherIter>& other)+      : b_(other.begin()), e_(other.end()) {}+  /**+   * Allow explicit conversion from Range<From> to Range<To> if From is+   * explicitly convertible to To.+   *+   * @methodset Range+   */+  template <+      class OtherIter,+      typename std::enable_if<+          (!std::is_same<Iter, OtherIter>::value &&+           !std::is_convertible<OtherIter, Iter>::value &&+           std::is_constructible<Iter, const OtherIter&>::value),+          int>::type = 0>+  constexpr explicit Range(const Range<OtherIter>& other)+      : b_(other.begin()), e_(other.end()) {}++  /**+   * Allow explicit construction of Range() from a std::array of a+   * convertible type.+   *+   * For instance, this allows constructing StringPiece from a+   * std::array<char, N> or a std::array<const char, N>+   *+   * @methodset Range+   */+  template <+      class T,+      size_t N,+      typename = typename std::enable_if<+          std::is_convertible<const T*, Iter>::value>::type>+  constexpr explicit Range(const std::array<T, N>& array)+      : b_{array.empty() ? nullptr : &array.at(0)},+        e_{array.empty() ? nullptr : &array.at(0) + N} {}+  template <+      class T,+      size_t N,+      typename =+          typename std::enable_if<std::is_convertible<T*, Iter>::value>::type>+  constexpr explicit Range(std::array<T, N>& array)+      : b_{array.empty() ? nullptr : &array.at(0)},+        e_{array.empty() ? nullptr : &array.at(0) + N} {}++  Range& operator=(const Range& rhs) & = default;+  Range& operator=(Range&& rhs) & = default;++  template <+      class Alloc,+      class T = Iter,+      typename detail::IsCharPointer<T>::const_type = 0>+  Range& operator=(string<Alloc>&& rhs) = delete;++  /**+   * Clear start and end iterators+   */+  void clear() {+    b_ = Iter();+    e_ = Iter();+  }++  /**+   * Assign start and end iterators+   */+  void assign(Iter start, Iter end) {+    b_ = start;+    e_ = end;+  }++  /**+   * Reset start and end interator based on size+   */+  void reset(Iter start, size_type size) {+    b_ = start;+    e_ = start + size;+  }++  // Works only for Range<const char*>+  template <typename Alloc>+  void reset(const string<Alloc>& str) {+    reset(str.data(), str.size());+  }++  constexpr size_type size() const {+    assert(b_ <= e_);+    return size_type(e_ - b_);+  }+  constexpr size_type walk_size() const {+    return size_type(std::distance(b_, e_));+  }+  constexpr bool empty() const { return b_ == e_; }+  constexpr Iter data() const { return b_; }+  constexpr Iter start() const { return b_; }+  constexpr Iter begin() const { return b_; }+  constexpr Iter end() const { return e_; }+  constexpr Iter cbegin() const { return b_; }+  constexpr Iter cend() const { return e_; }+  reference front() {+    assert(b_ < e_);+    return *b_;+  }+  reference back() {+    assert(b_ < e_);+    return *std::prev(e_);+  }+  const_reference front() const {+    assert(b_ < e_);+    return *b_;+  }+  const_reference back() const {+    assert(b_ < e_);+    return *std::prev(e_);+  }++ private:+  // It would be nice to be able to implicit convert to any target type+  // T for which either an (Iter, Iter) or (Iter, size_type) noexcept+  // constructor was available, and explicitly convert to any target+  // type for which those signatures were available but not noexcept.+  // The problem is that this creates ambiguity when there is also a+  // T constructor that takes a type U that is implicitly convertible+  // from Range.+  //+  // To avoid ambiguity, we need to avoid having explicit operator T+  // and implicit operator U coexist when T is constructible from U.+  // U cannot be deduced when searching for operator T (and C++ won't+  // perform an existential search for it), so we must limit the implicit+  // target types to a finite set that we can enumerate.+  //+  // At the moment the set of implicit target types consists of just+  // std::string_view (when it is available).+  struct NotStringView {};+  struct StringViewTypeChar {+    template <typename ValueType>+    using apply = std::basic_string_view<ValueType>;+  };+  struct StringViewTypeNone {+    template <typename>+    using apply = NotStringView;+  };+  template <typename ValueType>+  using StringViewTypeFunc = std::conditional_t<+      detail::range_is_char_type_v_<Iter>,+      StringViewTypeChar,+      StringViewTypeNone>;+  template <typename ValueType>+  struct StringViewType {+    using type =+        typename StringViewTypeFunc<ValueType>::template apply<ValueType>;+  };++  template <typename Target>+  struct IsConstructibleViaStringView+      : Conjunction<+            std::is_constructible<+                _t<StringViewType<value_type>>,+                Iter const&,+                size_type>,+            std::is_constructible<Target, _t<StringViewType<value_type>>>> {};++ public:+  /// explicit operator conversion to any compatible type+  ///+  /// A compatible type is one which is constructible with an iterator and a+  /// size (preferred), or a pair of iterators (fallback), passed by const-ref.+  ///+  /// Participates in overload resolution precisely when the target type is+  /// compatible. This allows std::is_constructible compile-time checks to work.+  template <+      typename Tgt,+      std::enable_if_t<+          std::is_constructible<Tgt, Iter const&, size_type>::value &&+              !IsConstructibleViaStringView<Tgt>::value,+          int> = 0>+  constexpr explicit operator Tgt() const noexcept(+      std::is_nothrow_constructible<Tgt, Iter const&, size_type>::value) {+    return Tgt(b_, walk_size());+  }+  template <+      typename Tgt,+      std::enable_if_t<+          !std::is_constructible<Tgt, Iter const&, size_type>::value &&+              std::is_constructible<Tgt, Iter const&, Iter const&>::value &&+              !IsConstructibleViaStringView<Tgt>::value,+          int> = 0>+  constexpr explicit operator Tgt() const noexcept(+      std::is_nothrow_constructible<Tgt, Iter const&, Iter const&>::value) {+    return Tgt(b_, e_);+  }++  /// implicit operator conversion to std::string_view+  template <+      typename Tgt,+      typename ValueType = value_type,+      std::enable_if_t<+          StrictConjunction<+              std::is_same<Tgt, _t<StringViewType<ValueType>>>,+              std::is_constructible<+                  _t<StringViewType<ValueType>>,+                  Iter const&,+                  size_type>>::value,+          int> = 0>+  constexpr operator Tgt() const noexcept(+      std::is_nothrow_constructible<Tgt, Iter const&, size_type>::value) {+    return Tgt(b_, walk_size());+  }++#if FMT_VERSION < 100000+  template <+      typename IterType = Iter,+      std::enable_if_t<detail::range_is_char_type_v_<IterType>, int> = 0>+  constexpr operator fmt::basic_string_view<value_type>() const noexcept {+    return _t<StringViewType<value_type>>(*this);+  }+#endif++  /// explicit non-operator conversion to any compatible type+  ///+  /// A compatible type is one which is constructible with an iterator and a+  /// size (preferred), or a pair of iterators (fallback), passed by const-ref.+  ///+  /// Participates in overload resolution precisely when the target type is+  /// compatible. This allows is_invocable compile-time checks to work.+  ///+  /// Provided in addition to the explicit operator conversion to permit passing+  /// additional arguments to the target type constructor. A canonical example+  /// of an additional argument might be an allocator, where the target type is+  /// some specialization of std::vector or std::basic_string in a context which+  /// requires a non-default-constructed allocator.+  template <typename Tgt, typename... Args>+  constexpr std::enable_if_t<+      std::is_constructible<Tgt, Iter const&, size_type>::value,+      Tgt>+  to(Args&&... args) const noexcept(+      std::is_nothrow_constructible<Tgt, Iter const&, size_type, Args&&...>::+          value) {+    return Tgt(b_, walk_size(), static_cast<Args&&>(args)...);+  }+  template <typename Tgt, typename... Args>+  constexpr std::enable_if_t<+      !std::is_constructible<Tgt, Iter const&, size_type>::value &&+          std::is_constructible<Tgt, Iter const&, Iter const&>::value,+      Tgt>+  to(Args&&... args) const noexcept(+      std::is_nothrow_constructible<Tgt, Iter const&, Iter const&, Args&&...>::+          value) {+    return Tgt(b_, e_, static_cast<Args&&>(args)...);+  }++  // Works only for Range<const char*> and Range<char*>+  std::string str() const { return to<std::string>(); }+  std::string toString() const { return to<std::string>(); }++  const_range_type castToConst() const { return const_range_type(*this); }++  int compare(const const_range_type& o) const {+    const size_type tsize = this->size();+    const size_type osize = o.size();+    const size_type msize = std::min(tsize, osize);+    int r = traits_type::compare(data(), o.data(), msize);+    if (r == 0 && tsize != osize) {+      // We check the signed bit of the subtraction and bit shift it+      // to produce either 0 or 2. The subtraction yields the+      // comparison values of either -1 or 1.+      r = (static_cast<int>((osize - tsize) >> (CHAR_BIT * sizeof(size_t) - 1))+           << 1) -+          1;+    }+    return r;+  }++  reference operator[](size_t i) {+    assert(i < size());+    return b_[i];+  }++  const_reference operator[](size_t i) const {+    assert(i < size());+    return b_[i];+  }++  reference at(size_t i) {+    if (i >= size()) {+      throw_exception<std::out_of_range>("index out of range");+    }+    return b_[i];+  }++  const_reference at(size_t i) const {+    if (i >= size()) {+      throw_exception<std::out_of_range>("index out of range");+    }+    return b_[i];+  }++  void advance(size_type n) {+    if (FOLLY_UNLIKELY(n > size())) {+      throw_exception<std::out_of_range>("index out of range");+    }+    b_ += n;+  }++  void subtract(size_type n) {+    if (FOLLY_UNLIKELY(n > size())) {+      throw_exception<std::out_of_range>("index out of range");+    }+    e_ -= n;+  }++  // Returns a window into the current range, starting at first, and spans+  // length characters (or until the end of the current range, whichever comes+  // first). Throws if first is past the end of the current range.+  Range subpiece(size_type first, size_type length = npos) const {+    if (FOLLY_UNLIKELY(first > size())) {+      throw_exception<std::out_of_range>("index out of range");+    }++    return Range(b_ + first, std::min(length, size() - first));+  }++  template <+      typename...,+      typename T = Iter,+      std::enable_if_t<detail::range_is_char_type_v_<T>, int> = 0>+  Range substr(size_type first, size_type length = npos) const {+    return subpiece(first, length);+  }++  // unchecked versions+  void uncheckedAdvance(size_type n) {+    assert(n <= size());+    b_ += n;+  }++  void uncheckedSubtract(size_type n) {+    assert(n <= size());+    e_ -= n;+  }++  Range uncheckedSubpiece(size_type first, size_type length = npos) const {+    assert(first <= size());+    return Range(b_ + first, std::min(length, size() - first));+  }++  void pop_front() {+    assert(b_ < e_);+    ++b_;+  }++  void pop_back() {+    assert(b_ < e_);+    --e_;+  }++  // string work-alike functions+  size_type find(const_range_type str) const {+    return qfind(castToConst(), str);+  }++  size_type find(const_range_type str, size_t pos) const {+    if (pos > size()) {+      return std::string::npos;+    }+    size_t ret = qfind(castToConst().subpiece(pos), str);+    return ret == npos ? ret : ret + pos;+  }++  size_type find(Iter s, size_t pos, size_t n) const {+    if (pos > size()) {+      return std::string::npos;+    }+    auto forFinding = castToConst();+    size_t ret = qfind(+        pos ? forFinding.subpiece(pos) : forFinding, const_range_type(s, n));+    return ret == npos ? ret : ret + pos;+  }++  // Works only for Range<(const) (unsigned) char*> which have Range(Iter) ctor+  size_type find(const Iter s) const {+    return qfind(castToConst(), const_range_type(s));+  }++  // Works only for Range<(const) (unsigned) char*> which have Range(Iter) ctor+  size_type find(const Iter s, size_t pos) const {+    if (pos > size()) {+      return std::string::npos;+    }+    size_type ret = qfind(castToConst().subpiece(pos), const_range_type(s));+    return ret == npos ? ret : ret + pos;+  }++  size_type find(const value_type& c) const { return qfind(castToConst(), c); }++  size_type rfind(const value_type& c) const {+    return folly::rfind(castToConst(), c);+  }++  size_type find(const value_type& c, size_t pos) const {+    if (pos > size()) {+      return std::string::npos;+    }+    size_type ret = qfind(castToConst().subpiece(pos), c);+    return ret == npos ? ret : ret + pos;+  }++  size_type find_first_of(const_range_type needles) const {+    return qfind_first_of(castToConst(), needles);+  }++  size_type find_first_of(const_range_type needles, size_t pos) const {+    if (pos > size()) {+      return std::string::npos;+    }+    size_type ret = qfind_first_of(castToConst().subpiece(pos), needles);+    return ret == npos ? ret : ret + pos;+  }++  // Works only for Range<(const) (unsigned) char*> which have Range(Iter) ctor+  size_type find_first_of(Iter needles) const {+    return find_first_of(const_range_type(needles));+  }++  // Works only for Range<(const) (unsigned) char*> which have Range(Iter) ctor+  size_type find_first_of(Iter needles, size_t pos) const {+    return find_first_of(const_range_type(needles), pos);+  }++  size_type find_first_of(Iter needles, size_t pos, size_t n) const {+    return find_first_of(const_range_type(needles, n), pos);+  }++  size_type find_first_of(const value_type& c) const { return find(c); }++  size_type find_first_of(const value_type& c, size_t pos) const {+    return find(c, pos);+  }++  /**+   * Determine whether the range contains the given subrange or item.+   *+   * Note: Call find() directly if the index is needed.+   */+  bool contains(const const_range_type& other) const {+    return find(other) != std::string::npos;+  }++  bool contains(const value_type& other) const {+    return find(other) != std::string::npos;+  }++  void swap(Range& rhs) {+    std::swap(b_, rhs.b_);+    std::swap(e_, rhs.e_);+  }++  /**+   * Does this Range start with another range?+   */+  bool startsWith(const const_range_type& other) const {+    return size() >= other.size() &&+        castToConst().subpiece(0, other.size()) == other;+  }+  bool startsWith(const value_type& c) const {+    return !empty() && front() == c;+  }++  template <class Comp>+  bool startsWith(const const_range_type& other, Comp&& eq) const {+    if (size() < other.size()) {+      return false;+    }+    auto const trunc = subpiece(0, other.size());+    return std::equal(+        trunc.begin(), trunc.end(), other.begin(), std::forward<Comp>(eq));+  }++  bool starts_with(const_range_type other) const noexcept {+    return startsWith(other);+  }+  bool starts_with(const value_type& c) const noexcept { return startsWith(c); }+  template <+      typename...,+      typename T = Iter,+      std::enable_if_t<detail::range_is_char_type_v_<T>, int> = 0>+  bool starts_with(const value_type* other) const {+    return startsWith(other);+  }++  /**+   * Does this Range end with another range?+   */+  bool endsWith(const const_range_type& other) const {+    return size() >= other.size() &&+        castToConst().subpiece(size() - other.size()) == other;+  }+  bool endsWith(const value_type& c) const { return !empty() && back() == c; }++  template <class Comp>+  bool endsWith(const const_range_type& other, Comp&& eq) const {+    if (size() < other.size()) {+      return false;+    }+    auto const trunc = subpiece(size() - other.size());+    return std::equal(+        trunc.begin(), trunc.end(), other.begin(), std::forward<Comp>(eq));+  }++  template <class Comp>+  bool equals(const const_range_type& other, Comp&& eq) const {+    return size() == other.size() &&+        std::equal(begin(), end(), other.begin(), std::forward<Comp>(eq));+  }++  bool ends_with(const_range_type other) const noexcept {+    return endsWith(other);+  }+  bool ends_with(const value_type& c) const noexcept { return endsWith(c); }+  template <+      typename...,+      typename T = Iter,+      std::enable_if_t<detail::range_is_char_type_v_<T>, int> = 0>+  bool ends_with(const value_type* other) const {+    return endsWith(other);+  }++  /**+   * Remove the items in [b, e), as long as this subrange is at the beginning+   * or end of the Range.+   *+   * Required for boost::algorithm::trim()+   */+  void erase(Iter b, Iter e) {+    if (b == b_) {+      b_ = e;+    } else if (e == e_) {+      e_ = b;+    } else {+      throw_exception<std::out_of_range>("index out of range");+    }+  }++  /**+   * Remove the given prefix and return true if the range starts with the given+   * prefix; return false otherwise.+   */+  bool removePrefix(const const_range_type& prefix) {+    return startsWith(prefix) && (b_ += prefix.size(), true);+  }+  bool removePrefix(const value_type& prefix) {+    return startsWith(prefix) && (++b_, true);+  }++  /**+   * Remove the given suffix and return true if the range ends with the given+   * suffix; return false otherwise.+   */+  bool removeSuffix(const const_range_type& suffix) {+    return endsWith(suffix) && (e_ -= suffix.size(), true);+  }+  bool removeSuffix(const value_type& suffix) {+    return endsWith(suffix) && (--e_, true);+  }++  /**+   * Replaces the content of the range, starting at position 'pos', with+   * contents of 'replacement'. Entire 'replacement' must fit into the+   * range. Returns false if 'replacements' does not fit. Example use:+   *+   * char in[] = "buffer";+   * auto msp = MutableStringPiece(input);+   * EXPECT_TRUE(msp.replaceAt(2, "tt"));+   * EXPECT_EQ(msp, "butter");+   *+   * // not enough space+   * EXPECT_FALSE(msp.replace(msp.size() - 1, "rr"));+   * EXPECT_EQ(msp, "butter"); // unchanged+   */+  bool replaceAt(size_t pos, const_range_type replacement) {+    if (size() < pos + replacement.size()) {+      return false;+    }++    std::copy(replacement.begin(), replacement.end(), begin() + pos);++    return true;+  }++  /**+   * Replaces all occurrences of 'source' with 'dest'. Returns number+   * of replacements made. Source and dest have to have the same+   * length. Throws if the lengths are different. If 'source' is a+   * pattern that is overlapping with itself, we perform sequential+   * replacement: "aaaaaaa".replaceAll("aa", "ba") --> "bababaa"+   *+   * Example use:+   *+   * char in[] = "buffer";+   * auto msp = MutableStringPiece(input);+   * EXPECT_EQ(msp.replaceAll("ff","tt"), 1);+   * EXPECT_EQ(msp, "butter");+   */+  size_t replaceAll(const_range_type source, const_range_type dest) {+    if (source.size() != dest.size()) {+      throw_exception<std::invalid_argument>(+          "replacement must have the same size as source");+    }++    if (dest.empty()) {+      return 0;+    }++    size_t pos = 0;+    size_t num_replaced = 0;+    size_type found = std::string::npos;+    while ((found = find(source, pos)) != std::string::npos) {+      replaceAt(found, dest);+      pos += source.size();+      ++num_replaced;+    }++    return num_replaced;+  }++  /**+   * Splits this `Range` `[b, e)` in the position `i` dictated by the next+   * occurrence of `delimiter`.+   *+   * Returns a new `Range` `[b, i)` and adjusts this range to start right after+   * the delimiter's position. This range will be empty if the delimiter is not+   * found. If called on an empty `Range`, both this and the returned `Range`+   * will be empty.+   *+   * Example:+   *+   *  folly::StringPiece s("sample string for split_next");+   *  auto p = s.split_step(' ');+   *+   *  // prints "string for split_next"+   *  cout << s << endl;+   *+   *  // prints "sample"+   *  cout << p << endl;+   *+   * Example 2:+   *+   *  void tokenize(StringPiece s, char delimiter) {+   *    while (!s.empty()) {+   *      cout << s.split_step(delimiter);+   *    }+   *  }+   *+   */+  Range split_step(const value_type& delimiter) {+    auto i = find(delimiter);+    Range result(b_, i == std::string::npos ? size() : i);++    b_ = result.end() == e_ ? e_ : std::next(result.end());++    return result;+  }++  Range split_step(Range delimiter) {+    auto i = find(delimiter);+    Range result(b_, i == std::string::npos ? size() : i);++    b_ = result.end() == e_+        ? e_+        : std::next(result.end(), difference_type(delimiter.size()));++    return result;+  }++  /**+   * Convenience method that calls `split_step()` and passes the result to a+   * functor, returning whatever the functor does. Any additional arguments+   * `args` passed to this function are perfectly forwarded to the functor.+   *+   * Say you have a functor with this signature:+   *+   *  Foo fn(Range r) { }+   *+   * `split_step()`'s return type will be `Foo`. It works just like:+   *+   *  auto result = fn(myRange.split_step(' '));+   *+   * A functor returning `void` is also supported.+   *+   * Example:+   *+   *  void do_some_parsing(folly::StringPiece s) {+   *    auto version = s.split_step(' ', [&](folly::StringPiece x) {+   *      if (x.empty()) {+   *        throw std::invalid_argument("empty string");+   *      }+   *      return std::strtoull(x.begin(), x.end(), 16);+   *    });+   *+   *    // ...+   *  }+   *+   *  struct Foo {+   *    void parse(folly::StringPiece s) {+   *      s.split_step(' ', parse_field, bar, 10);+   *      s.split_step('\t', parse_field, baz, 20);+   *+   *      auto const kludge = [](folly::StringPiece x, int &out, int def) {+   *        if (x == "null") {+   *          out = 0;+   *        } else {+   *          parse_field(x, out, def);+   *        }+   *      };+   *+   *      s.split_step('\t', kludge, gaz);+   *      s.split_step(' ', kludge, foo);+   *    }+   *+   *  private:+   *    int bar;+   *    int baz;+   *    int gaz;+   *    int foo;+   *+   *    static parse_field(folly::StringPiece s, int &out, int def) {+   *      try {+   *        out = folly::to<int>(s);+   *      } catch (std::exception const &) {+   *        value = def;+   *      }+   *    }+   *  };+   *+   */+  template <typename TProcess, typename... Args>+  auto split_step(+      const value_type& delimiter, TProcess&& process, Args&&... args)+      -> decltype(process(std::declval<Range>(), std::forward<Args>(args)...)) {+    return process(split_step(delimiter), std::forward<Args>(args)...);+  }++  template <typename TProcess, typename... Args>+  auto split_step(Range delimiter, TProcess&& process, Args&&... args)+      -> decltype(process(std::declval<Range>(), std::forward<Args>(args)...)) {+    return process(split_step(delimiter), std::forward<Args>(args)...);+  }++ private:+  Iter b_;+  Iter e_;+};++template <class Iter>+const typename Range<Iter>::size_type Range<Iter>::npos = std::string::npos;++template <class Iter>+void swap(Range<Iter>& lhs, Range<Iter>& rhs) {+  lhs.swap(rhs);+}++/**+ * Create a range from two iterators, with type deduction.+ */+template <class Iter>+constexpr Range<Iter> range(Iter first, Iter last) {+  return Range<Iter>(first, last);+}++/*+ * Creates a range to reference the contents of a contiguous-storage container.+ */+// Use pointers for types with '.data()' member+template <class Collection>+constexpr auto range(Collection& v) -> Range<decltype(v.data())> {+  return Range<decltype(v.data())>(v.data(), v.data() + v.size());+}+template <class Collection>+constexpr auto range(Collection const& v) -> Range<decltype(v.data())> {+  return Range<decltype(v.data())>(v.data(), v.data() + v.size());+}+template <class Collection>+constexpr auto crange(Collection const& v) -> Range<decltype(v.data())> {+  return Range<decltype(v.data())>(v.data(), v.data() + v.size());+}++template <class T, size_t n>+constexpr Range<T*> range(T (&array)[n]) {+  return Range<T*>(array, array + n);+}+template <class T, size_t n>+constexpr Range<T const*> range(T const (&array)[n]) {+  return Range<T const*>(array, array + n);+}+template <class T, size_t n>+constexpr Range<T const*> crange(T const (&array)[n]) {+  return Range<T const*>(array, array + n);+}++template <class T, size_t n>+constexpr Range<T*> range(std::array<T, n>& array) {+  return Range<T*>{array};+}+template <class T, size_t n>+constexpr Range<T const*> range(std::array<T, n> const& array) {+  return Range<T const*>{array};+}+template <class T, size_t n>+constexpr Range<T const*> crange(std::array<T, n> const& array) {+  return Range<T const*>{array};+}++template <class T>+constexpr Range<T const*> range(std::initializer_list<T> ilist) {+  return Range<T const*>(ilist.begin(), ilist.end());+}++template <class T>+constexpr Range<T const*> crange(std::initializer_list<T> ilist) {+  return Range<T const*>(ilist.begin(), ilist.end());+}++using StringPiece = Range<const char*>;+using MutableStringPiece = Range<char*>;+using ByteRange = Range<const unsigned char*>;+using MutableByteRange = Range<unsigned char*>;++template <class C>+std::basic_ostream<C>& operator<<(+    std::basic_ostream<C>& os, Range<C const*> piece) {+  using StreamSize = decltype(os.width());+  os.write(piece.start(), static_cast<StreamSize>(piece.size()));+  return os;+}++template <class C>+std::basic_ostream<C>& operator<<(std::basic_ostream<C>& os, Range<C*> piece) {+  using StreamSize = decltype(os.width());+  os.write(piece.start(), static_cast<StreamSize>(piece.size()));+  return os;+}++/**+ * Templated comparison operators+ */++template <class Iter>+inline bool operator==(const Range<Iter>& lhs, const Range<Iter>& rhs) {+  using value_type = typename Range<Iter>::value_type;+  if (lhs.size() != rhs.size()) {+    return false;+  }+  if constexpr (+      std::is_pointer_v<Iter> &&+      (std::is_integral_v<value_type> || std::is_enum_v<value_type>)) {+    auto const size = lhs.size() * sizeof(value_type);+    return 0 == size || 0 == std::memcmp(lhs.data(), rhs.data(), size);+  } else {+    for (size_t i = 0; i < lhs.size(); ++i) {+      if (!Range<Iter>::traits_type::eq(lhs[i], rhs[i])) {+        return false;+      }+    }+    return true;+  }+}++template <class Iter>+inline bool operator!=(const Range<Iter>& lhs, const Range<Iter>& rhs) {+  return !(operator==(lhs, rhs));+}++template <class Iter>+inline bool operator<(const Range<Iter>& lhs, const Range<Iter>& rhs) {+  return lhs.compare(rhs) < 0;+}++template <class Iter>+inline bool operator<=(const Range<Iter>& lhs, const Range<Iter>& rhs) {+  return lhs.compare(rhs) <= 0;+}++template <class Iter>+inline bool operator>(const Range<Iter>& lhs, const Range<Iter>& rhs) {+  return lhs.compare(rhs) > 0;+}++template <class Iter>+inline bool operator>=(const Range<Iter>& lhs, const Range<Iter>& rhs) {+  return lhs.compare(rhs) >= 0;+}++/**+ * Specializations of comparison operators for StringPiece+ */++namespace detail {++template <class A, class B>+struct ComparableAsStringPiece {+  enum {+    value = (std::is_convertible<A, StringPiece>::value &&+             std::is_same<B, StringPiece>::value) ||+        (std::is_convertible<B, StringPiece>::value &&+         std::is_same<A, StringPiece>::value)+  };+};++} // namespace detail++/**+ * operator== through conversion for Range<const char*>+ */+template <class T, class U>+std::enable_if_t<detail::ComparableAsStringPiece<T, U>::value, bool> operator==(+    const T& lhs, const U& rhs) {+  return StringPiece(lhs) == StringPiece(rhs);+}++/**+ * operator!= through conversion for Range<const char*>+ */+template <class T, class U>+std::enable_if_t<detail::ComparableAsStringPiece<T, U>::value, bool> operator!=(+    const T& lhs, const U& rhs) {+  return StringPiece(lhs) != StringPiece(rhs);+}++/**+ * operator< through conversion for Range<const char*>+ */+template <class T, class U>+std::enable_if_t<detail::ComparableAsStringPiece<T, U>::value, bool> operator<(+    const T& lhs, const U& rhs) {+  return StringPiece(lhs) < StringPiece(rhs);+}++/**+ * operator> through conversion for Range<const char*>+ */+template <class T, class U>+std::enable_if_t<detail::ComparableAsStringPiece<T, U>::value, bool> operator>(+    const T& lhs, const U& rhs) {+  return StringPiece(lhs) > StringPiece(rhs);+}++/**+ * operator< through conversion for Range<const char*>+ */+template <class T, class U>+std::enable_if_t<detail::ComparableAsStringPiece<T, U>::value, bool> operator<=(+    const T& lhs, const U& rhs) {+  return StringPiece(lhs) <= StringPiece(rhs);+}++/**+ * operator> through conversion for Range<const char*>+ */+template <class T, class U>+std::enable_if_t<detail::ComparableAsStringPiece<T, U>::value, bool> operator>=(+    const T& lhs, const U& rhs) {+  return StringPiece(lhs) >= StringPiece(rhs);+}++/**+ * Finds substrings faster than brute force by borrowing from Boyer-Moore+ */+template <class Iter, class Comp>+size_t qfind(const Range<Iter>& haystack, const Range<Iter>& needle, Comp eq) {+  // Don't use std::search, use a Boyer-Moore-like trick by comparing+  // the last characters first+  auto const nsize = needle.size();+  if (haystack.size() < nsize) {+    return std::string::npos;+  }+  if (!nsize) {+    return 0;+  }+  auto const nsize_1 = nsize - 1;+  auto const lastNeedle = needle[nsize_1];++  // Boyer-Moore skip value for the last char in the needle. Zero is+  // not a valid value; skip will be computed the first time it's+  // needed.+  std::string::size_type skip = 0;++  auto i = haystack.begin();+  auto iEnd = haystack.end() - nsize_1;++  while (i < iEnd) {+    // Boyer-Moore: match the last element in the needle+    while (!eq(i[nsize_1], lastNeedle)) {+      if (++i == iEnd) {+        // not found+        return std::string::npos;+      }+    }+    // Here we know that the last char matches+    // Continue in pedestrian mode+    for (size_t j = 0;;) {+      assert(j < nsize);+      if (!eq(i[j], needle[j])) {+        // Not found, we can skip+        // Compute the skip value lazily+        if (skip == 0) {+          skip = 1;+          while (skip <= nsize_1 && !eq(needle[nsize_1 - skip], lastNeedle)) {+            ++skip;+          }+        }+        i += skip;+        break;+      }+      // Check if done searching+      if (++j == nsize) {+        // Yay+        return size_t(i - haystack.begin());+      }+    }+  }+  return std::string::npos;+}++namespace detail {++inline size_t qfind_first_byte_of(+    const StringPiece haystack, const StringPiece needles) {+  // Let's default to the SIMD implementation. Internally, if that's not+  // available, the _nosimd version gets picked instead.+  return qfind_first_byte_of_simd(haystack, needles);+}++} // namespace detail++template <class Iter, class Comp>+size_t qfind_first_of(+    const Range<Iter>& haystack, const Range<Iter>& needles, Comp eq) {+  auto ret = std::find_first_of(+      haystack.begin(), haystack.end(), needles.begin(), needles.end(), eq);+  return ret == haystack.end() ? std::string::npos : ret - haystack.begin();+}++struct AsciiCaseSensitive {+  bool operator()(char lhs, char rhs) const { return lhs == rhs; }+};++/**+ * Check if two ascii characters are case insensitive equal.+ * The difference between the lower/upper case characters are the 6-th bit.+ * We also check they are alpha chars, in case of xor = 32.+ */+struct AsciiCaseInsensitive {+  bool operator()(char lhs, char rhs) const {+    char k = lhs ^ rhs;+    if (k == 0) {+      return true;+    }+    if (k != 32) {+      return false;+    }+    k = lhs | rhs;+    return (k >= 'a' && k <= 'z');+  }+};++template <class Iter>+size_t qfind(+    const Range<Iter>& haystack,+    const typename Range<Iter>::value_type& needle) {+  auto pos = std::find(haystack.begin(), haystack.end(), needle);+  return pos == haystack.end() ? std::string::npos : pos - haystack.data();+}++template <class Iter>+size_t rfind(+    const Range<Iter>& haystack,+    const typename Range<Iter>::value_type& needle) {+  for (auto i = haystack.size(); i-- > 0;) {+    if (haystack[i] == needle) {+      return i;+    }+  }+  return std::string::npos;+}++// specialization for StringPiece+template <>+inline size_t qfind(const Range<const char*>& haystack, const char& needle) {+  // memchr expects a not-null pointer, early return if the range is empty.+  if (haystack.empty()) {+    return std::string::npos;+  }+  auto pos = static_cast<const char*>(+      ::memchr(haystack.data(), needle, haystack.size()));+  return pos == nullptr ? std::string::npos : pos - haystack.data();+}++template <>+inline size_t rfind(const Range<const char*>& haystack, const char& needle) {+  // memchr expects a not-null pointer, early return if the range is empty.+  if (haystack.empty()) {+    return std::string::npos;+  }+  auto pos = static_cast<const char*>(+      memrchr(haystack.data(), needle, haystack.size()));+  return pos == nullptr ? std::string::npos : pos - haystack.data();+}++// specialization for ByteRange+template <>+inline size_t qfind(+    const Range<const unsigned char*>& haystack, const unsigned char& needle) {+  // memchr expects a not-null pointer, early return if the range is empty.+  if (haystack.empty()) {+    return std::string::npos;+  }+  auto pos = static_cast<const unsigned char*>(+      ::memchr(haystack.data(), needle, haystack.size()));+  return pos == nullptr ? std::string::npos : pos - haystack.data();+}++template <>+inline size_t rfind(+    const Range<const unsigned char*>& haystack, const unsigned char& needle) {+  // memchr expects a not-null pointer, early return if the range is empty.+  if (haystack.empty()) {+    return std::string::npos;+  }+  auto pos = static_cast<const unsigned char*>(+      memrchr(haystack.data(), needle, haystack.size()));+  return pos == nullptr ? std::string::npos : pos - haystack.data();+}++template <class Iter>+size_t qfind_first_of(const Range<Iter>& haystack, const Range<Iter>& needles) {+  return qfind_first_of(haystack, needles, AsciiCaseSensitive());+}++// specialization for StringPiece+template <>+inline size_t qfind_first_of(+    const Range<const char*>& haystack, const Range<const char*>& needles) {+  return detail::qfind_first_byte_of(haystack, needles);+}++// specialization for ByteRange+template <>+inline size_t qfind_first_of(+    const Range<const unsigned char*>& haystack,+    const Range<const unsigned char*>& needles) {+  return detail::qfind_first_byte_of(+      StringPiece(haystack), StringPiece(needles));+}++template <class Key, class Enable>+struct hasher;++template <class T>+struct hasher<+    folly::Range<T*>,+    std::enable_if_t<std::is_integral<T>::value, void>> {+  using folly_is_avalanching = std::true_type;++  size_t operator()(folly::Range<T*> r) const {+    // std::is_integral<T> is too restrictive, but is sufficient to+    // guarantee we can just hash all of the underlying bytes to get a+    // suitable hash of T.  Something like absl::is_uniquely_represented<T>+    // would be better.  std::is_pod is not enough, because POD types+    // can contain pointers and padding.  Also, floating point numbers+    // may be == without being bit-identical.  size_t is less than 64+    // bits on some platforms.+    return static_cast<size_t>(+        hash::SpookyHashV2::Hash64(r.begin(), r.size() * sizeof(T), 0));+  }+};++/**+ * _sp is a user-defined literal suffix to make an appropriate Range+ * specialization from a literal string.+ *+ * Modeled after C++17's `sv` suffix.+ */+inline namespace literals {+inline namespace string_piece_literals {+constexpr Range<char const*> operator""_sp(+    char const* str, size_t len) noexcept {+  return Range<char const*>(str, len);+}++#if defined(__cpp_char8_t) && __cpp_char8_t >= 201811L+constexpr Range<char8_t const*> operator""_sp(+    char8_t const* str, size_t len) noexcept {+  return Range<char8_t const*>(str, len);+}+#endif++constexpr Range<char16_t const*> operator""_sp(+    char16_t const* str, size_t len) noexcept {+  return Range<char16_t const*>(str, len);+}++constexpr Range<char32_t const*> operator""_sp(+    char32_t const* str, size_t len) noexcept {+  return Range<char32_t const*>(str, len);+}++constexpr Range<wchar_t const*> operator""_sp(+    wchar_t const* str, size_t len) noexcept {+  return Range<wchar_t const*>(str, len);+}+} // namespace string_piece_literals+} // namespace literals++} // namespace folly++// Avoid ambiguity in older fmt versions due to StringPiece's conversions.+#if FMT_VERSION >= 70000+namespace fmt {+template <>+struct formatter<folly::StringPiece> : private formatter<string_view> {+  using formatter<string_view>::parse;++#if FMT_VERSION >= 80000+  template <typename Context>+  typename Context::iterator format(folly::StringPiece s, Context& ctx) const {+    return formatter<string_view>::format({s.data(), s.size()}, ctx);+  }+#else+  template <typename Context>+  typename Context::iterator format(folly::StringPiece s, Context& ctx) {+    return formatter<string_view>::format({s.data(), s.size()}, ctx);+  }+#endif+};+} // namespace fmt+#endif++FOLLY_POP_WARNING++FOLLY_ASSUME_FBVECTOR_COMPATIBLE_1(folly::Range)++// Unfortunately it is not possible to forward declare enable_view under+// MSVC 2019.8 due to compiler bugs, so we need to include the actual+// definition if available.+#if __has_include(<range/v3/range/concepts.hpp>) && defined(_MSC_VER)+#include <range/v3/range/concepts.hpp> // @manual+#else+namespace ranges {+template <class T>+extern const bool enable_view;+} // namespace ranges+#endif++// Tell the range-v3 library that this type should satisfy+// the view concept (a lightweight, non-owning range).+namespace ranges {+template <class Iter>+inline constexpr bool enable_view<::folly::Range<Iter>> = true;+} // namespace ranges++#if defined(__cpp_lib_ranges)+template <typename T>+constexpr bool std::ranges::enable_borrowed_range<folly::Range<T>> = true;+#endif
+ folly/folly/Replaceable.h view
@@ -0,0 +1,636 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <initializer_list>+#include <new>+#include <type_traits>+#include <utility>++#include <folly/Portability.h>+#include <folly/Traits.h>+#include <folly/Utility.h>++/**+ * An instance of `Replaceable<T>` wraps an instance of `T`.+ *+ * You access the inner `T` instance with `operator*` and `operator->` (as if+ * it were a smart pointer).+ *+ * `Replaceable<T>` adds no indirection cost and performs no allocations.+ *+ * `Replaceable<T>` has the same size and alignment as `T`.+ *+ * You can replace the `T` within a `Replaceable<T>` using the `emplace` method+ * (presuming that it is constructible and destructible without throwing+ * exceptions). If the destructor or constructor you're using could throw an+ * exception you should use `Optional<T>` instead, as it's not a logic error+ * for that to be empty.+ *+ * Frequently Asked Questions+ * ==========================+ *+ * Why does this need to be so complicated?+ * ----------------------------------------+ *+ * If a `T` instance contains `const`-qualified member variables or reference+ * member variables we can't safely replace a `T` instance by destructing it+ * manually and using placement new. This is because compilers are permitted to+ * assume that the `const` or reference members of a named, referenced, or+ * pointed-to object do not change.+ *+ * For pointed-to objects in allocated storage you can use the pointer returned+ * by placement new or use the `launder` function to get a pointer to the new+ * object.  Note that `launder` doesn't affect its argument, it's still+ * undefined behaviour to use the original pointer. And none of this helps if+ * the object is a local or a member variable because the destructor call will+ * not have been laundered. In summary, this is the only way to use placement+ * new that is both simple and safe:+ *+ *      T* pT = new T(...);+ *      pT->~T();+ *      pT = ::new (pT) T(...);+ *      delete pT;+ *+ * What are the other safe solutions to this problem?+ * --------------------------------------------------+ *+ * * Ask the designer of `T` to de-`const` and -`reference` the members of `T`.+ *  - Makes `T` harder to reason about+ *  - Can reduce the performance of `T` methods+ *  - They can refuse to make the change+ * * Put the `T` on the heap and use a raw/unique/shared pointer.+ *  - Adds a level of indirection, costing performance.+ *  - Harder to reason about your code as you need to check for nullptr.+ * * Put the `T` in an `Optional`.+ *  - Harder to reason about your code as you need to check for None.+ * * Pass the problem on, making the new code also not-replaceable+ *  - Contagion is not really a solution+ *+ * Are there downsides to this?+ * ----------------------------+ *+ * There is a potential performance penalty after converting `T` to+ * `Replaceable<T>` if you have non-`T`-member-function code which repeatedly+ * examines the value of a `const` or `reference` data member of `T`, because+ * the compiler now has to look at the value each time whereas previously it+ * was permitted to load it once up-front and presume that it could never+ * change.+ *+ * Usage notes+ * ===========+ *+ * Don't store a reference to the `T` within a `Replaceable<T>` unless you can+ * show that its lifetime does not cross an `emplace` call. For safety a+ * reasonable rule is to always use `operator*()` to get a fresh temporary each+ * time you need a `T&.+ *+ * If you store a pointer to the `T` within a `Replaceable<T>` you **must**+ * launder it after each call to `emplace` before using it. Again you can+ * reasonably choose to always use `operator->()` to get a fresh temporary each+ * time you need a `T*.+ *+ * Thus far I haven't thought of a good reason to use `Replaceable<T>` or+ * `Replaceable<T> const&` as a function parameter type.+ *+ * `Replaceable<T>&` can make sense to pass to a function that conditionally+ * replaces the `T`, where `T` has `const` or reference member variables.+ *+ * The main use of `Replaceable<T>` is as a class member type or a local type+ * in long-running functions.+ *+ * It's probably time to rethink your design choices if you end up with+ * `Replaceable<Replaceable<T>>`, `Optional<Replaceable<T>>`,+ * `Replaceable<Optional<T>>`, `unique_ptr<Replaceable<T>>` etc. except as a+ *  result of template expansion.+ */++namespace folly {++template <class T>+class Replaceable;++namespace replaceable_detail {+/* Mixin templates to give `replaceable<T>` the following properties:+ *+ * 1. Trivial destructor if `T` has a trivial destructor; user-provided+ *    otherwise+ * 2. Move constructor if `T` has a move constructor; deleted otherwise+ * 3. Move assignment operator if `T` has a move constructor; deleted+ *    otherwise+ * 4. Copy constructor if `T` has a copy constructor; deleted otherwise+ * 5. Copy assignment operator if `T` has a copy constructor; deleted+ *    otherwise+ *+ * Has to be done in this way because we can't `enable_if` them away+ */+template <+    class T,+    bool = std::is_destructible<T>::value,+    bool = std::is_trivially_destructible<T>::value>+struct dtor_mixin;++/* Destructible and trivially destructible */+template <class T>+struct dtor_mixin<T, true, true> {};++/* Destructible and not trivially destructible */+template <class T>+struct dtor_mixin<T, true, false> {+  dtor_mixin() = default;+  dtor_mixin(dtor_mixin&&) = default;+  dtor_mixin(dtor_mixin const&) = default;+  dtor_mixin& operator=(dtor_mixin&&) = default;+  dtor_mixin& operator=(dtor_mixin const&) = default;+  ~dtor_mixin() noexcept(std::is_nothrow_destructible<T>::value) {+    T* destruct_ptr = std::launder(reinterpret_cast<T*>(+        reinterpret_cast<Replaceable<T>*>(this)->storage_));+    destruct_ptr->~T();+  }+};++/* Not destructible */+template <class T, bool A>+struct dtor_mixin<T, false, A> {+  dtor_mixin() = default;+  dtor_mixin(dtor_mixin&&) = default;+  dtor_mixin(dtor_mixin const&) = default;+  dtor_mixin& operator=(dtor_mixin&&) = default;+  dtor_mixin& operator=(dtor_mixin const&) = default;+  ~dtor_mixin() = delete;+};++template <+    class T,+    bool = std::is_default_constructible<T>::value,+    bool = std::is_move_constructible<T>::value>+struct default_and_move_ctor_mixin;++/* Not default-constructible and not move-constructible */+template <class T>+struct default_and_move_ctor_mixin<T, false, false> {+  default_and_move_ctor_mixin() = delete;+  default_and_move_ctor_mixin(default_and_move_ctor_mixin&&) = delete;+  default_and_move_ctor_mixin(default_and_move_ctor_mixin const&) = default;+  default_and_move_ctor_mixin& operator=(default_and_move_ctor_mixin&&) =+      default;+  default_and_move_ctor_mixin& operator=(default_and_move_ctor_mixin const&) =+      default;++ protected:+  inline explicit default_and_move_ctor_mixin(int) {}+};++/* Default-constructible and move-constructible */+template <class T>+struct default_and_move_ctor_mixin<T, true, true> {+  inline default_and_move_ctor_mixin() noexcept(+      std::is_nothrow_constructible<T>::value) {+    ::new (reinterpret_cast<Replaceable<T>*>(this)->storage_) T();+  }+  inline default_and_move_ctor_mixin(+      default_and_move_ctor_mixin&&+          other) noexcept(std::is_nothrow_constructible<T, T&&>::value) {+    ::new (reinterpret_cast<Replaceable<T>*>(this)->storage_)+        T(*std::move(reinterpret_cast<Replaceable<T>&>(other)));+  }+  default_and_move_ctor_mixin(default_and_move_ctor_mixin const&) = default;+  default_and_move_ctor_mixin& operator=(default_and_move_ctor_mixin&&) =+      default;+  inline default_and_move_ctor_mixin& operator=(+      default_and_move_ctor_mixin const&) = default;++ protected:+  inline explicit default_and_move_ctor_mixin(int) {}+};++/* Default-constructible and not move-constructible */+template <class T>+struct default_and_move_ctor_mixin<T, true, false> {+  inline default_and_move_ctor_mixin() noexcept(+      std::is_nothrow_constructible<T>::value) {+    ::new (reinterpret_cast<Replaceable<T>*>(this)->storage_) T();+  }+  default_and_move_ctor_mixin(default_and_move_ctor_mixin&&) = delete;+  default_and_move_ctor_mixin(default_and_move_ctor_mixin const&) = default;+  default_and_move_ctor_mixin& operator=(default_and_move_ctor_mixin&&) =+      default;+  default_and_move_ctor_mixin& operator=(default_and_move_ctor_mixin const&) =+      default;++ protected:+  inline explicit default_and_move_ctor_mixin(int) {}+};++/* Not default-constructible but is move-constructible */+template <class T>+struct default_and_move_ctor_mixin<T, false, true> {+  default_and_move_ctor_mixin() = delete;+  inline default_and_move_ctor_mixin(+      default_and_move_ctor_mixin&&+          other) noexcept(std::is_nothrow_constructible<T, T&&>::value) {+    ::new (reinterpret_cast<Replaceable<T>*>(this)->storage_)+        T(*std::move(reinterpret_cast<Replaceable<T>&>(other)));+  }+  default_and_move_ctor_mixin(default_and_move_ctor_mixin const&) = default;+  default_and_move_ctor_mixin& operator=(default_and_move_ctor_mixin&&) =+      default;+  default_and_move_ctor_mixin& operator=(default_and_move_ctor_mixin const&) =+      default;++ protected:+  inline explicit default_and_move_ctor_mixin(int) {}+};++template <+    class T,+    bool = (std::is_destructible<T>::value) &&+        (std::is_move_constructible<T>::value)>+struct move_assignment_mixin;++/* Not (destructible and move-constructible) */+template <class T>+struct move_assignment_mixin<T, false> {+  move_assignment_mixin() = default;+  move_assignment_mixin(move_assignment_mixin&&) = default;+  move_assignment_mixin(move_assignment_mixin const&) = default;+  move_assignment_mixin& operator=(move_assignment_mixin&&) = delete;+  move_assignment_mixin& operator=(move_assignment_mixin const&) = default;+};++/* Both destructible and move-constructible */+template <class T>+struct move_assignment_mixin<T, true> {+  move_assignment_mixin() = default;+  move_assignment_mixin(move_assignment_mixin&&) = default;+  move_assignment_mixin(move_assignment_mixin const&) = default;+  inline move_assignment_mixin&+  operator=(move_assignment_mixin&& other) noexcept(+      std::is_nothrow_destructible<T>::value &&+      std::is_nothrow_move_constructible<T>::value) {+    T* destruct_ptr = std::launder(reinterpret_cast<T*>(+        reinterpret_cast<Replaceable<T>*>(this)->storage_));+    destruct_ptr->~T();+    ::new (reinterpret_cast<Replaceable<T>*>(this)->storage_)+        T(*std::move(reinterpret_cast<Replaceable<T>&>(other)));+    return *this;+  }+  move_assignment_mixin& operator=(move_assignment_mixin const&) = default;+};++template <class T, bool = std::is_copy_constructible<T>::value>+struct copy_ctor_mixin;++/* Not copy-constructible */+template <class T>+struct copy_ctor_mixin<T, false> {+  copy_ctor_mixin() = default;+  copy_ctor_mixin(copy_ctor_mixin&&) = default;+  copy_ctor_mixin(copy_ctor_mixin const&) = delete;+  copy_ctor_mixin& operator=(copy_ctor_mixin&&) = default;+  copy_ctor_mixin& operator=(copy_ctor_mixin const&) = delete;+};++/* Copy-constructible */+template <class T>+struct copy_ctor_mixin<T, true> {+  copy_ctor_mixin() = default;+  inline copy_ctor_mixin(copy_ctor_mixin const& other) noexcept(+      std::is_nothrow_constructible<T, T const&>::value) {+    ::new (reinterpret_cast<Replaceable<T>*>(this)->storage_)+        T(*reinterpret_cast<Replaceable<T> const&>(other));+  }+  copy_ctor_mixin(copy_ctor_mixin&&) = default;+  copy_ctor_mixin& operator=(copy_ctor_mixin&&) = default;+  copy_ctor_mixin& operator=(copy_ctor_mixin const&) = default;+};++template <+    class T,+    bool = (std::is_destructible<T>::value) &&+        (std::is_copy_constructible<T>::value)>+struct copy_assignment_mixin;++/* Not (destructible and copy-constructible) */+template <class T>+struct copy_assignment_mixin<T, false> {+  copy_assignment_mixin() = default;+  copy_assignment_mixin(copy_assignment_mixin&&) = default;+  copy_assignment_mixin(copy_assignment_mixin const&) = default;+  copy_assignment_mixin& operator=(copy_assignment_mixin&&) = default;+  copy_assignment_mixin& operator=(copy_assignment_mixin const&) = delete;+};++/* Both destructible and copy-constructible */+template <class T>+struct copy_assignment_mixin<T, true> {+  copy_assignment_mixin() = default;+  copy_assignment_mixin(copy_assignment_mixin&&) = default;+  copy_assignment_mixin(copy_assignment_mixin const&) = default;+  copy_assignment_mixin& operator=(copy_assignment_mixin&&) = default;+  inline copy_assignment_mixin&+  operator=(copy_assignment_mixin const& other) noexcept(+      std::is_nothrow_destructible<T>::value &&+      std::is_nothrow_copy_constructible<T>::value) {+    T* destruct_ptr = std::launder(reinterpret_cast<T*>(+        reinterpret_cast<Replaceable<T>*>(this)->storage_));+    destruct_ptr->~T();+    ::new (reinterpret_cast<Replaceable<T>*>(this)->storage_)+        T(*reinterpret_cast<Replaceable<T> const&>(other));+    return *this;+  }+};++template <typename T>+struct is_constructible_from_replaceable+    : std::bool_constant<+          std::is_constructible<T, Replaceable<T>&>::value ||+          std::is_constructible<T, Replaceable<T>&&>::value ||+          std::is_constructible<T, const Replaceable<T>&>::value ||+          std::is_constructible<T, const Replaceable<T>&&>::value> {};++template <typename T>+struct is_convertible_from_replaceable+    : std::bool_constant<+          std::is_convertible<Replaceable<T>&, T>::value ||+          std::is_convertible<Replaceable<T>&&, T>::value ||+          std::is_convertible<const Replaceable<T>&, T>::value ||+          std::is_convertible<const Replaceable<T>&&, T>::value> {};+} // namespace replaceable_detail++template <class T>+using is_replaceable = is_instantiation_of<Replaceable, T>;++// Function to make a Replaceable with a type deduced from its input+template <class T>+constexpr Replaceable<std::decay_t<T>> make_replaceable(T&& t) {+  return Replaceable<std::decay_t<T>>(std::forward<T>(t));+}++template <class T, class... Args>+constexpr Replaceable<T> make_replaceable(Args&&... args) {+  return Replaceable<T>(std::in_place, std::forward<Args>(args)...);+}++template <class T, class U, class... Args>+constexpr Replaceable<T> make_replaceable(+    std::initializer_list<U> il, Args&&... args) {+  return Replaceable<T>(std::in_place, il, std::forward<Args>(args)...);+}++template <class T>+class alignas(T) Replaceable+    : public replaceable_detail::dtor_mixin<T>,+      public replaceable_detail::default_and_move_ctor_mixin<T>,+      public replaceable_detail::copy_ctor_mixin<T>,+      public replaceable_detail::move_assignment_mixin<T>,+      public replaceable_detail::copy_assignment_mixin<T> {+  using ctor_base = replaceable_detail::default_and_move_ctor_mixin<T>;++ public:+  using value_type = T;++  /* Rule-of-zero default- copy- and move- constructors. The ugly code to make+   * these work are above, in namespace folly::replaceable_detail.+   */+  constexpr Replaceable() = default;+  constexpr Replaceable(const Replaceable&) = default;+  constexpr Replaceable(Replaceable&&) = default;++  /* Rule-of-zero copy- and move- assignment operators. The ugly code to make+   * these work are above, in namespace folly::replaceable_detail.+   *+   * Note - these destruct the `T` and then in-place construct a new one based+   * on what is in the other replaceable; they do not invoke the assignment+   * operator of `T`.+   */+  Replaceable& operator=(const Replaceable&) = default;+  Replaceable& operator=(Replaceable&&) = default;++  /* Rule-of-zero destructor. The ugly code to make this work is above, in+   * namespace folly::replaceable_detail.+   */+  ~Replaceable() = default;++  /**+   * Constructors; these are modeled very closely on the definition of+   * `std::optional` in C++17.+   */+  template <+      class... Args,+      std::enable_if_t<std::is_constructible<T, Args&&...>::value, int> = 0>+  constexpr explicit Replaceable(std::in_place_t, Args&&... args)+      // clang-format off+      noexcept(std::is_nothrow_constructible<T, Args&&...>::value)+      // clang-format on+      : ctor_base(0) {+    ::new (storage_) T(std::forward<Args>(args)...);+  }++  template <+      class U,+      class... Args,+      std::enable_if_t<+          std::is_constructible<T, std::initializer_list<U>, Args&&...>::value,+          int> = 0>+  constexpr explicit Replaceable(+      std::in_place_t, std::initializer_list<U> il, Args&&... args)+      // clang-format off+      noexcept(std::is_nothrow_constructible<+          T,+          std::initializer_list<U>,+          Args&&...>::value)+      // clang-format on+      : ctor_base(0) {+    ::new (storage_) T(il, std::forward<Args>(args)...);+  }++  template <+      class U = T,+      std::enable_if_t<+          std::is_constructible<T, U&&>::value &&+              !std::is_same<std::decay_t<U>, std::in_place_t>::value &&+              !std::is_same<Replaceable<T>, std::decay_t<U>>::value &&+              std::is_convertible<U&&, T>::value,+          int> = 0>+  constexpr /* implicit */ Replaceable(U&& other)+      // clang-format off+      noexcept(std::is_nothrow_constructible<T, U&&>::value)+      // clang-format on+      : ctor_base(0) {+    ::new (storage_) T(std::forward<U>(other));+  }++  template <+      class U = T,+      std::enable_if_t<+          std::is_constructible<T, U&&>::value &&+              !std::is_same<std::decay_t<U>, std::in_place_t>::value &&+              !std::is_same<Replaceable<T>, std::decay_t<U>>::value &&+              !std::is_convertible<U&&, T>::value,+          int> = 0>+  constexpr explicit Replaceable(U&& other)+      // clang-format off+      noexcept(std::is_nothrow_constructible<T, U&&>::value)+      // clang-format on+      : ctor_base(0) {+    ::new (storage_) T(std::forward<U>(other));+  }++  template <+      class U,+      std::enable_if_t<+          std::is_constructible<T, const U&>::value &&+              !replaceable_detail::is_constructible_from_replaceable<+                  T>::value &&+              !replaceable_detail::is_convertible_from_replaceable<T>::value &&+              std::is_convertible<const U&, T>::value,+          int> = 0>+  /* implicit */ Replaceable(const Replaceable<U>& other)+      // clang-format off+      noexcept(std::is_nothrow_constructible<T, U const&>::value)+      // clang-format on+      : ctor_base(0) {+    ::new (storage_) T(*other);+  }++  template <+      class U,+      std::enable_if_t<+          std::is_constructible<T, const U&>::value &&+              !replaceable_detail::is_constructible_from_replaceable<+                  T>::value &&+              !replaceable_detail::is_convertible_from_replaceable<T>::value &&+              !std::is_convertible<const U&, T>::value,+          int> = 0>+  explicit Replaceable(const Replaceable<U>& other)+      // clang-format off+      noexcept(std::is_nothrow_constructible<T, U const&>::value)+      // clang-format on+      : ctor_base(0) {+    ::new (storage_) T(*other);+  }++  template <+      class U,+      std::enable_if_t<+          std::is_constructible<T, U&&>::value &&+              !replaceable_detail::is_constructible_from_replaceable<+                  T>::value &&+              !replaceable_detail::is_convertible_from_replaceable<T>::value &&+              std::is_convertible<U&&, T>::value,+          int> = 0>+  /* implicit */ Replaceable(Replaceable<U>&& other)+      // clang-format off+      noexcept(std::is_nothrow_constructible<T, U&&>::value)+      // clang-format on+      : ctor_base(0) {+    ::new (storage_) T(std::move(*other));+  }++  template <+      class U,+      std::enable_if_t<+          std::is_constructible<T, U&&>::value &&+              !replaceable_detail::is_constructible_from_replaceable<+                  T>::value &&+              !replaceable_detail::is_convertible_from_replaceable<T>::value &&+              !std::is_convertible<U&&, T>::value,+          int> = 0>+  explicit Replaceable(Replaceable<U>&& other)+      // clang-format off+      noexcept(std::is_nothrow_constructible<T, U&&>::value)+      // clang-format on+      : ctor_base(0) {+    ::new (storage_) T(std::move(*other));+  }++  /**+   * `emplace` destructs the contained object and in-place constructs the+   * replacement.+   *+   * The destructor must not throw (as usual). The constructor must not throw+   * because that would violate the invariant that a `Replaceable<T>` always+   * contains a T instance.+   *+   * As these methods are `noexcept` the program will be terminated if an+   * exception is thrown. If you are encountering this issue you should look at+   * using `Optional` instead.+   */+  template <class... Args>+  T& emplace(Args&&... args) noexcept {+    T* destruct_ptr = std::launder(reinterpret_cast<T*>(storage_));+    destruct_ptr->~T();+    return *::new (storage_) T(std::forward<Args>(args)...);+  }++  template <class U, class... Args>+  T& emplace(std::initializer_list<U> il, Args&&... args) noexcept {+    T* destruct_ptr = std::launder(reinterpret_cast<T*>(storage_));+    destruct_ptr->~T();+    return *::new (storage_) T(il, std::forward<Args>(args)...);+  }++  /**+   * `swap` just calls `swap(T&, T&)`.+   */+  void swap(Replaceable& other) noexcept(+      std::is_nothrow_swappable_v<Replaceable>) {+    using std::swap;+    swap(*(*this), *other);+  }++  /**+   * Methods to access the contained object. Intended to be very unsurprising.+   */+  constexpr const T* operator->() const {+    return std::launder(reinterpret_cast<T const*>(storage_));+  }++  constexpr T* operator->() {+    return std::launder(reinterpret_cast<T*>(storage_));+  }++  constexpr const T& operator*() const& {+    return *std::launder(reinterpret_cast<T const*>(storage_));+  }++  constexpr T& operator*() & {+    return *std::launder(reinterpret_cast<T*>(storage_));+  }++  constexpr T&& operator*() && {+    return std::move(*std::launder(reinterpret_cast<T*>(storage_)));+  }++  constexpr const T&& operator*() const&& {+    return std::move(*std::launder(reinterpret_cast<T const*>(storage_)));+  }++ private:+  friend struct replaceable_detail::dtor_mixin<T>;+  friend struct replaceable_detail::default_and_move_ctor_mixin<T>;+  friend struct replaceable_detail::copy_ctor_mixin<T>;+  friend struct replaceable_detail::move_assignment_mixin<T>;+  friend struct replaceable_detail::copy_assignment_mixin<T>;+  aligned_storage_for_t<T> storage_[1];+};++template <class T>+Replaceable(T) -> Replaceable<T>;++} // namespace folly
+ folly/folly/ScopeGuard.cpp view
@@ -0,0 +1,29 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#include <folly/ScopeGuard.h>++#include <exception>+#include <iostream>++/*static*/ void folly::detail::ScopeGuardImplBase::terminate() noexcept {+  // Ensure the availability of std::cerr+  std::ios_base::Init ioInit;+  std::cerr+      << "This program will now terminate because a folly::ScopeGuard callback "+         "threw an \nexception.\n";+  std::rethrow_exception(current_exception());+}
+ folly/folly/ScopeGuard.h view
@@ -0,0 +1,418 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++//+// Docs: https://fburl.com/fbcref_scopeguard+//++/**+ * ScopeGuard is a general implementation of the "Initialization is+ * Resource Acquisition" idiom.  It guarantees that a function+ * is executed upon leaving the current scope.+ *+ * @file ScopeGuard.h+ * @refcode folly/docs/examples/folly/ScopeGuard.cpp+ */+/*+ * The makeGuard() function is used to create a new ScopeGuard object.+ * It can be instantiated with a lambda function, a std::function<void()>,+ * a functor, or a void(*)() function pointer.+ *+ *+ * Usage example: Add a friend to memory if and only if it is also added+ * to the db.+ *+ * void User::addFriend(User& newFriend) {+ *   // add the friend to memory+ *   friends_.push_back(&newFriend);+ *+ *   // If the db insertion that follows fails, we should+ *   // remove it from memory.+ *   auto guard = makeGuard([&] { friends_.pop_back(); });+ *+ *   // this will throw an exception upon error, which+ *   // makes the ScopeGuard execute UserCont::pop_back()+ *   // once the Guard's destructor is called.+ *   db_->addFriend(GetName(), newFriend.GetName());+ *+ *   // an exception was not thrown, so don't execute+ *   // the Guard.+ *   guard.dismiss();+ * }+ *+ * It is also possible to create a guard in dismissed state with+ * makeDismissedGuard(), and later rehire it with the rehire()+ * method.+ *+ * makeDismissedGuard() is not just syntactic sugar for creating a guard and+ * immediately dismissing it, but it has a subtle behavior difference if+ * move-construction of the passed function can throw: if it does, the function+ * will be called by makeGuard(), but not by makeDismissedGuard().+ *+ * Examine ScopeGuardTest.cpp for some more sample usage.+ *+ * Stolen from:+ *   Andrei's and Petru Marginean's CUJ article:+ *     http://drdobbs.com/184403758+ *   and the loki library:+ *     http://loki-lib.sourceforge.net/index.php?n=Idioms.ScopeGuardPointer+ *   and triendl.kj article:+ *     http://www.codeproject.com/KB/cpp/scope_guard.aspx+ */+#pragma once++#include <cstddef>+#include <cstdlib>+#include <functional>+#include <new>+#include <type_traits>+#include <utility>++#include <folly/Portability.h>+#include <folly/Preprocessor.h>+#include <folly/Utility.h>+#include <folly/lang/Exception.h>+#include <folly/lang/UncaughtExceptions.h>++namespace folly {++namespace detail {++struct ScopeGuardDismissed {};++class ScopeGuardImplBase {+ public:+  void dismiss() noexcept { dismissed_ = true; }+  void rehire() noexcept { dismissed_ = false; }++ protected:+  ScopeGuardImplBase(bool dismissed = false) noexcept : dismissed_(dismissed) {}++  [[noreturn]] static void terminate() noexcept;+  static ScopeGuardImplBase makeEmptyScopeGuard() noexcept {+    return ScopeGuardImplBase{};+  }++  bool dismissed_;+};++template <typename FunctionType, bool InvokeNoexcept>+class ScopeGuardImpl : public ScopeGuardImplBase {+ public:+  explicit ScopeGuardImpl(FunctionType& fn) noexcept(+      std::is_nothrow_copy_constructible_v<FunctionType>)+      : ScopeGuardImpl(+            std::as_const(fn),+            makeFailsafe(+                std::is_nothrow_copy_constructible<FunctionType>{}, &fn)) {}++  explicit ScopeGuardImpl(const FunctionType& fn) noexcept(+      std::is_nothrow_copy_constructible_v<FunctionType>)+      : ScopeGuardImpl(+            fn,+            makeFailsafe(+                std::is_nothrow_copy_constructible<FunctionType>{}, &fn)) {}++  explicit ScopeGuardImpl(FunctionType&& fn) noexcept(+      std::is_nothrow_move_constructible_v<FunctionType>)+      : ScopeGuardImpl(+            std::move_if_noexcept(fn),+            makeFailsafe(+                std::is_nothrow_move_constructible<FunctionType>{}, &fn)) {}++  explicit ScopeGuardImpl(FunctionType&& fn, ScopeGuardDismissed) noexcept(+      std::is_nothrow_move_constructible_v<FunctionType>)+      // No need for failsafe in this case, as the guard is dismissed.+      : ScopeGuardImplBase{true}, function_(std::forward<FunctionType>(fn)) {}++  ScopeGuardImpl(ScopeGuardImpl&& other) noexcept(+      std::is_nothrow_move_constructible_v<FunctionType>)+      : function_(std::move_if_noexcept(other.function_)) {+    // If the above line attempts a copy and the copy throws, other is+    // left owning the cleanup action and will execute it (or not) depending+    // on the value of other.dismissed_. The following lines only execute+    // if the move/copy succeeded, in which case *this assumes ownership of+    // the cleanup action and dismisses other.+    dismissed_ = std::exchange(other.dismissed_, true);+  }++  ~ScopeGuardImpl() noexcept(InvokeNoexcept) {+    if (!dismissed_) {+      execute();+    }+  }++ private:+  static ScopeGuardImplBase makeFailsafe(std::true_type, const void*) noexcept {+    return makeEmptyScopeGuard();+  }++  template <typename Fn>+  static auto makeFailsafe(std::false_type, Fn* fn) noexcept+      -> ScopeGuardImpl<decltype(std::ref(*fn)), InvokeNoexcept> {+    return ScopeGuardImpl<decltype(std::ref(*fn)), InvokeNoexcept>{+        std::ref(*fn)};+  }++  template <typename Fn>+  explicit ScopeGuardImpl(Fn&& fn, ScopeGuardImplBase&& failsafe)+      : ScopeGuardImplBase{}, function_(std::forward<Fn>(fn)) {+    failsafe.dismiss();+  }++  void* operator new(std::size_t) = delete;++  void execute() noexcept(InvokeNoexcept) {+    if constexpr (InvokeNoexcept) {+      static_assert(std::is_same_v<void, decltype(function_())>);+      catch_exception(function_, &terminate);+    } else {+      function_();+    }+  }++  FunctionType function_;+};++template <typename F, bool INE>+using ScopeGuardImplDecay = ScopeGuardImpl<std::decay_t<F>, INE>;++} // namespace detail++/**+ * Create a scope guard.+ *+ * The returned object has methods .dismiss() and .rehire(), which will+ * deactivate/reactivate the calling of the function upon destruction.+ *+ * The return value of this function must be captured. Otherwise, since it is a+ * temporary, it will be destroyed immediately, thus calling the function.+ *+ *     auto guard = makeGuard(...); // good+ *+ *     makeGuard(...); // bad+ *+ * @param f  The function to execute upon the guard's destruction.+ * @refcode folly/docs/examples/folly/ScopeGuard2.cpp+ */+template <typename F>+FOLLY_NODISCARD detail::ScopeGuardImplDecay<F, true> makeGuard(F&& f) noexcept(+    noexcept(detail::ScopeGuardImplDecay<F, true>(static_cast<F&&>(f)))) {+  return detail::ScopeGuardImplDecay<F, true>(static_cast<F&&>(f));+}++/**+ * Create a scope guard in the dismissed state.+ *+ * The guard can be enabled using .rehire().+ *+ * @see makeGuard+ * @refcode folly/docs/examples/folly/ScopeGuard2.cpp+ */+template <typename F>+FOLLY_NODISCARD detail::ScopeGuardImplDecay<F, true>+makeDismissedGuard(F&& f) noexcept(+    noexcept(detail::ScopeGuardImplDecay<F, true>(+        static_cast<F&&>(f), detail::ScopeGuardDismissed{}))) {+  return detail::ScopeGuardImplDecay<F, true>(+      static_cast<F&&>(f), detail::ScopeGuardDismissed{});+}++namespace detail {++/**+ * ScopeGuard used for executing a function when leaving the current scope+ * depending on the presence of a new uncaught exception.+ *+ * If the executeOnException template parameter is true, the function is+ * executed if a new uncaught exception is present at the end of the scope.+ * If the parameter is false, then the function is executed if no new uncaught+ * exceptions are present at the end of the scope.+ *+ * Used to implement SCOPE_FAIL and SCOPE_SUCCESS below.+ */+template <typename FunctionType, bool ExecuteOnException>+class ScopeGuardForNewException {+ public:+  explicit ScopeGuardForNewException(const FunctionType& fn) : guard_(fn) {}++  explicit ScopeGuardForNewException(FunctionType&& fn)+      : guard_(std::move(fn)) {}++  ScopeGuardForNewException(ScopeGuardForNewException&& other) = default;++  ~ScopeGuardForNewException() noexcept(ExecuteOnException) {+    if (ExecuteOnException != (exceptionCounter_ < uncaught_exceptions())) {+      guard_.dismiss();+    }+  }++ private:+  void* operator new(std::size_t) = delete;+  void operator delete(void*) = delete;++  ScopeGuardImpl<FunctionType, ExecuteOnException> guard_;+  int exceptionCounter_{uncaught_exceptions()};+};++/**+ * Internal use for the macro SCOPE_FAIL below+ */+enum class ScopeGuardOnFail {};++template <typename FunctionType>+ScopeGuardForNewException<std::decay_t<FunctionType>, true> operator+(+    detail::ScopeGuardOnFail, FunctionType&& fn) {+  return ScopeGuardForNewException<std::decay_t<FunctionType>, true>(+      std::forward<FunctionType>(fn));+}++/**+ * Internal use for the macro SCOPE_SUCCESS below+ */+enum class ScopeGuardOnSuccess {};++template <typename FunctionType>+ScopeGuardForNewException<std::decay_t<FunctionType>, false> operator+(+    ScopeGuardOnSuccess, FunctionType&& fn) {+  return ScopeGuardForNewException<std::decay_t<FunctionType>, false>(+      std::forward<FunctionType>(fn));+}++/**+ * Internal use for the macro SCOPE_EXIT below+ */+enum class ScopeGuardOnExit {};++template <typename FunctionType>+ScopeGuardImpl<std::decay_t<FunctionType>, true> operator+(+    detail::ScopeGuardOnExit, FunctionType&& fn) {+  return ScopeGuardImpl<std::decay_t<FunctionType>, true>(+      std::forward<FunctionType>(fn));+}+} // namespace detail++} // namespace folly++//  SCOPE_EXIT+//+//  Example:+//+//      /* open scope */ {+//+//        some_resource_t resource;+//        some_resource_init(resource);+//        SCOPE_EXIT { some_resource_fini(resource); };+//+//        if (!cond)+//          throw 0; // the cleanup happens at end of the scope+//        else+//          return; // the cleanup happens at end of the scope+//+//        use_some_resource(resource); // may throw; cleanup will happen+//+//      } /* close scope */+//+//  The code in the braces passed to SCOPE_EXIT executes at the end of the+//  containing scope as if the code is the content of the destructor of an+//  object instantiated at the point of the SCOPE_EXIT, where the destructor+//  reference-captures all local variables it uses.+//+//  The cleanup code - the code in the braces passed to SCOPE_EXIT - always+//  executes at the end of the scope, regardless of whether the scope exits+//  normally or erroneously as if via the throw statement.+//+//  Caution: Suitable for coroutine functions only when the cleanup code does+//  not use captured references to thread-local objects. Recall that there is+//  no assumption that coroutines resume from co-await, co-yield, or co-return+//  in the same thread as the one in which they suspend.+//+//  Caution: May not execute if the scope exits erroneously but stack unwinding+//  is skipped, or if the scope does not exit at all such as with std::abort or+//  setcontext, which fibers use.++/**+ * Capture code that shall be run when the current scope exits.+ *+ * The code within SCOPE_EXIT's braces shall execute as if the code was in the+ * destructor of an object instantiated at the point of SCOPE_EXIT.+ *+ * Variables used within SCOPE_EXIT are captured by reference.+ *+ * @def SCOPE_EXIT+ */+#define SCOPE_EXIT                                        \+  auto FB_ANONYMOUS_VARIABLE_ODR_SAFE(SCOPE_EXIT_STATE) = \+      ::folly::detail::ScopeGuardOnExit() + [&]() noexcept++//  SCOPE_FAIL+//+//  May be useful in situations where the caller requests a resource where+//  initializations of the resource is multi-step and may fail.+//+//  Example:+//+//      some_resource_t resource;+//      some_resource_init(resource);+//      SCOPE_FAIL { some_resource_fini(resource); };+//+//      if (do_throw)+//        throw 0; // the cleanup happens at the end of the scope+//      else+//        return resource; // the cleanup does not happen+//+//  Warning: Not suitable for coroutine functions.++/**+ * Capture code to run if the scope exits with an exception.+ *+ * Like SCOPE_EXIT, but only executes the code if the scope exited due to an+ * exception.+ *+ * @def SCOPE_FAIL+ */+#define SCOPE_FAIL                                        \+  auto FB_ANONYMOUS_VARIABLE_ODR_SAFE(SCOPE_FAIL_STATE) = \+      ::folly::detail::ScopeGuardOnFail() + [&]() noexcept++//  SCOPE_SUCCESS+//+//  In a sense, the opposite of SCOPE_FAIL.+//+//  Example:+//+//      folly::stop_watch<> watch;+//      SCOPE_FAIL { log_failure(watch.elapsed(); };+//      SCOPE_SUCCESS { log_success(watch.elapsed(); };+//+//      if (do_throw)+//        throw 0; // the cleanup does not happen; log failure+//      else+//        return; // the cleanup happens at the end of the scope; log success+//+//  Warning: Not suitable for coroutine functions.++/**+ * Capture code to run if the scope exits without an exception.+ *+ * Like SCOPE_EXIT, but does not execute the code if the scope exited due to an+ * exception.+ *+ * @def SCOPE_SUCCESS+ */+#define SCOPE_SUCCESS                                        \+  auto FB_ANONYMOUS_VARIABLE_ODR_SAFE(SCOPE_SUCCESS_STATE) = \+      ::folly::detail::ScopeGuardOnSuccess() + [&]()
+ folly/folly/SharedMutex.cpp view
@@ -0,0 +1,81 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#include <system_error>++#include <folly/SharedMutex.h>++#include <folly/Indestructible.h>+#include <folly/lang/Exception.h>+#include <folly/portability/SysResource.h>++namespace folly {+// Explicitly instantiate SharedMutex here:+template class SharedMutexImpl<true>;+template class SharedMutexImpl<false>;++namespace shared_mutex_detail {+std::unique_lock<std::mutex> annotationGuard(void* ptr) {+  if (folly::kIsSanitizeThread) {+    // On TSAN builds, we have an array of mutexes and index into them based on+    // the address. If the array is of prime size things will work out okay+    // without a complicated hash function.+    static constexpr std::size_t kNumAnnotationMutexes = 251;+    static Indestructible<std::array<std::mutex, kNumAnnotationMutexes>>+        kAnnotationMutexes;+    auto index = reinterpret_cast<uintptr_t>(ptr) % kNumAnnotationMutexes;+    return std::unique_lock<std::mutex>((*kAnnotationMutexes)[index]);+  } else {+    return std::unique_lock<std::mutex>();+  }+}++uint32_t getMaxDeferredReadersSlow(relaxed_atomic<uint32_t>& cache) {+  uint32_t maxDeferredReaders = std::min(+      static_cast<uint32_t>(+          folly::nextPowTwo(CacheLocality::system().numCpus) << 1),+      shared_mutex_detail::kMaxDeferredReadersAllocated);+  // maxDeferredReaders must be a power of 2+  assert(!(maxDeferredReaders & (maxDeferredReaders - 1)));+  cache = maxDeferredReaders;+  return maxDeferredReaders;+}++long getCurrentThreadInvoluntaryContextSwitchCount() {+#ifdef RUSAGE_THREAD+  struct rusage usage;+  if (getrusage(RUSAGE_THREAD, &usage)) {+    return 0;+  } else {+    return usage.ru_nivcsw;+  }+#else+  return 0;+#endif+}++[[noreturn]] void throwOperationNotPermitted() {+  folly::throw_exception<std::system_error>(+      std::make_error_code(std::errc::operation_not_permitted));+}++[[noreturn]] void throwDeadlockWouldOccur() {+  folly::throw_exception<std::system_error>(+      std::make_error_code(std::errc::resource_deadlock_would_occur));+}++} // namespace shared_mutex_detail+} // namespace folly
+ folly/folly/SharedMutex.h view
@@ -0,0 +1,1752 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <stdint.h>++#include <atomic>+#include <chrono>+#include <memory>+#include <mutex>+#include <shared_mutex>+#include <thread>+#include <type_traits>+#include <utility>++#include <folly/CPortability.h>+#include <folly/CppAttributes.h>+#include <folly/Likely.h>+#include <folly/chrono/Hardware.h>+#include <folly/concurrency/CacheLocality.h>+#include <folly/detail/Futex.h>+#include <folly/portability/Asm.h>+#include <folly/synchronization/Lock.h>+#include <folly/synchronization/RelaxedAtomic.h>+#include <folly/synchronization/SanitizeThread.h>+#include <folly/system/ThreadId.h>++// SharedMutex is a reader-writer lock.  It is small, very fast, scalable+// on multi-core, and suitable for use when readers or writers may block.+// Unlike most other reader-writer locks, its throughput with concurrent+// readers scales linearly; it is able to acquire and release the lock+// in shared mode without cache line ping-ponging.  It is suitable for+// a wide range of lock hold times because it starts with spinning,+// proceeds to using sched_yield with a preemption heuristic, and then+// waits using futex and precise wakeups.+//+// SharedMutex provides all of the methods of folly::RWSpinLock,+// boost::shared_mutex, boost::upgrade_mutex, and C++14's+// std::shared_timed_mutex.  All operations that can block are available+// in try, try-for, and try-until (system_clock or steady_clock) versions.+//+// SharedMutexReadPriority gives priority to readers,+// SharedMutexWritePriority gives priority to writers.  SharedMutex is an+// alias for SharedMutexWritePriority, because writer starvation is more+// likely than reader starvation for the read-heavy workloads targeted+// by SharedMutex.+//+// In my tests SharedMutex is as good or better than the other+// reader-writer locks in use at Facebook for almost all use cases,+// sometimes by a wide margin.  (If it is rare that there are actually+// concurrent readers then RWSpinLock can be a few nanoseconds faster.)+// I compared it to folly::RWSpinLock, folly::RWTicketSpinLock64,+// boost::shared_mutex, pthread_rwlock_t, and a RWLock that internally uses+// spinlocks to guard state and pthread_mutex_t+pthread_cond_t to block.+// (Thrift's ReadWriteMutex is based underneath on pthread_rwlock_t.)+// It is generally as good or better than the rest when evaluating size,+// speed, scalability, or latency outliers.  In the corner cases where+// it is not the fastest (such as single-threaded use or heavy write+// contention) it is never very much worse than the best.  See the bottom+// of folly/test/SharedMutexTest.cpp for lots of microbenchmark results.+//+// Comparison to folly::RWSpinLock:+//+//  * SharedMutex is faster than RWSpinLock when there are actually+//    concurrent read accesses (sometimes much faster), and ~5 nanoseconds+//    slower when there is not actually any contention.  SharedMutex is+//    faster in every (benchmarked) scenario where the shared mode of+//    the lock is actually useful.+//+//  * Concurrent shared access to SharedMutex scales linearly, while total+//    RWSpinLock throughput drops as more threads try to access the lock+//    in shared mode.  Under very heavy read contention SharedMutex can+//    be two orders of magnitude faster than RWSpinLock (or any reader+//    writer lock that doesn't use striping or deferral).+//+//  * SharedMutex can safely protect blocking calls, because after an+//    initial period of spinning it waits using futex().+//+//  * RWSpinLock prioritizes readers, SharedMutex has both reader- and+//    writer-priority variants, but defaults to write priority.+//+//  * RWSpinLock's upgradeable mode blocks new readers, while SharedMutex's+//    doesn't.  Both semantics are reasonable.  The boost documentation+//    doesn't explicitly talk about this behavior (except by omitting+//    any statement that those lock modes conflict), but the boost+//    implementations do allow new readers while the upgradeable mode+//    is held.  See https://github.com/boostorg/thread/blob/master/+//      include/boost/thread/pthread/shared_mutex.hpp+//+// Both SharedMutex and RWSpinLock provide "exclusive", "upgrade",+// and "shared" modes.  At all times num_threads_holding_exclusive ++// num_threads_holding_upgrade <= 1, and num_threads_holding_exclusive ==+// 0 || num_threads_holding_shared == 0.  RWSpinLock has the additional+// constraint that num_threads_holding_shared cannot increase while+// num_threads_holding_upgrade is non-zero.+//+// Comparison to the internal RWLock:+//+//  * SharedMutex doesn't allow a maximum reader count to be configured,+//    so it can't be used as a semaphore in the same way as RWLock.+//+//  * SharedMutex is 4 bytes, RWLock is 256.+//+//  * SharedMutex is as fast or faster than RWLock in all of my+//    microbenchmarks, and has positive rather than negative scalability.+//+//  * RWLock and SharedMutex are both writer priority locks.+//+//  * SharedMutex avoids latency outliers as well as RWLock.+//+//  * SharedMutex uses different names (t != 0 below):+//+//    RWLock::lock(0)    => SharedMutex::lock()+//+//    RWLock::lock(t)    => SharedMutex::try_lock_for(milliseconds(t))+//+//    RWLock::tryLock()  => SharedMutex::try_lock()+//+//    RWLock::unlock()   => SharedMutex::unlock()+//+//    RWLock::enter(0)   => SharedMutex::lock_shared()+//+//    RWLock::enter(t)   =>+//        SharedMutex::try_lock_shared_for(milliseconds(t))+//+//    RWLock::tryEnter() => SharedMutex::try_lock_shared()+//+//    RWLock::leave()    => SharedMutex::unlock_shared()+//+//  * RWLock allows the reader count to be adjusted by a value other+//    than 1 during enter() or leave(). SharedMutex doesn't currently+//    implement this feature.+//+//  * RWLock's methods are marked const, SharedMutex's aren't.+//+// Reader-writer locks have the potential to allow concurrent access+// to shared read-mostly data, but in practice they often provide no+// improvement over a mutex.  The problem is the cache coherence protocol+// of modern CPUs.  Coherence is provided by making sure that when a cache+// line is written it is present in only one core's cache.  Since a memory+// write is required to acquire a reader-writer lock in shared mode, the+// cache line holding the lock is invalidated in all of the other caches.+// This leads to cache misses when another thread wants to acquire or+// release the lock concurrently.  When the RWLock is colocated with the+// data it protects (common), cache misses can also continue occur when+// a thread that already holds the lock tries to read the protected data.+//+// Ideally, a reader-writer lock would allow multiple cores to acquire+// and release the lock in shared mode without incurring any cache misses.+// This requires that each core records its shared access in a cache line+// that isn't read or written by other read-locking cores.  (Writers will+// have to check all of the cache lines.)  Typical server hardware when+// this comment was written has 16 L1 caches and cache lines of 64 bytes,+// so a lock striped over all L1 caches would occupy a prohibitive 1024+// bytes.  Nothing says that we need a separate set of per-core memory+// locations for each lock, however.  Each SharedMutex instance is only+// 4 bytes, but all locks together share a 2K area in which they make a+// core-local record of lock acquisitions.+//+// SharedMutex's strategy of using a shared set of core-local stripes has+// a potential downside, because it means that acquisition of any lock in+// write mode can conflict with acquisition of any lock in shared mode.+// If a lock instance doesn't actually experience concurrency then this+// downside will outweight the upside of improved scalability for readers.+// To avoid this problem we dynamically detect concurrent accesses to+// SharedMutex, and don't start using the deferred mode unless we actually+// observe concurrency.  See kNumSharedToStartDeferring.+//+// It is explicitly allowed to call unlock_shared() from a different+// thread than lock_shared(), so long as they are properly paired.+// unlock_shared() needs to find the location at which lock_shared()+// recorded the lock, which might be in the lock itself or in any of+// the shared slots.  If you can conveniently pass state from lock+// acquisition to release then the fastest mechanism is to std::move+// the std::shared_lock instance or an SharedMutex::Token (using+// lock_shared(Token&) and unlock_shared(Token&)).  The guard or token+// will tell unlock_shared where in deferredReaders[] to look for the+// deferred lock.  The Token-less version of unlock_shared() works in all+// cases, but is optimized for the common (no inter-thread handoff) case.+//+// In both read- and write-priority mode, a waiting lock() (exclusive mode)+// only blocks readers after it has waited for an active upgrade lock to be+// released; until the upgrade lock is released (or upgraded or downgraded)+// readers will still be able to enter.  Preferences about lock acquisition+// are not guaranteed to be enforced perfectly (even if they were, there+// is theoretically the chance that a thread could be arbitrarily suspended+// between calling lock() and SharedMutex code actually getting executed).+//+// try_*_for methods always try at least once, even if the duration+// is zero or negative.  The duration type must be compatible with+// std::chrono::steady_clock.  try_*_until methods also always try at+// least once.  std::chrono::system_clock and std::chrono::steady_clock+// are supported.+//+// If you have observed by profiling that your SharedMutex-s are getting+// cache misses on deferredReaders[] due to another SharedMutex user, then+// you can use the tag type to create your own instantiation of the type.+// The contention threshold (see kNumSharedToStartDeferring) should make+// this unnecessary in all but the most extreme cases.  Make sure to check+// that the increased icache and dcache footprint of the tagged result is+// worth it.++// SharedMutex's use of thread local storage is an optimization, so+// for the case where thread local storage is not supported, define it+// away.++// Note about TSAN (ThreadSanitizer): the SharedMutexWritePriority version+// (the default) of this mutex is annotated appropriately so that TSAN can+// perform lock inversion analysis. However, the SharedMutexReadPriority version+// is not annotated.  This is because TSAN's lock order heuristic+// assumes that two calls to lock_shared must be ordered, which leads+// to too many false positives for the reader-priority case.+//+// Suppose thread A holds a SharedMutexWritePriority lock in shared mode and an+// independent thread B is waiting for exclusive access. Then a thread C's+// lock_shared can't proceed until A has released the lock. Discounting+// situations that never use exclusive mode (so no lock is necessary at all)+// this means that without higher-level reasoning it is not safe to ignore+// reader <-> reader interactions.+//+// This reasoning does not apply to SharedMutexReadPriority, because there are+// no actions by a thread B that can make C need to wait for A. Since the+// overwhelming majority of SharedMutex instances use write priority, we+// restrict the TSAN annotations to only SharedMutexWritePriority.++namespace folly {++struct SharedMutexToken {+  enum class State : uint16_t {+    Invalid = 0,+    LockedShared, // May be inline or deferred.+    LockedInlineShared,+    LockedDeferredShared,+  };++  State state_{};+  uint16_t slot_{};++  constexpr SharedMutexToken() = default;++  explicit operator bool() const { return state_ != State::Invalid; }+};++struct SharedMutexPolicyDefault {+  static constexpr uint64_t max_spin_cycles = 4000;+  static constexpr uint32_t max_soft_yield_count = 1;+  static constexpr bool track_thread_id = false;+  static constexpr bool skip_annotate_rwlock = false;+};++namespace shared_mutex_detail {++struct PolicyTracked : SharedMutexPolicyDefault {+  static constexpr bool track_thread_id = true;+};+struct PolicySuppressTSAN : SharedMutexPolicyDefault {+  static constexpr bool skip_annotate_rwlock = true;+};++// Returns a guard that gives permission for the current thread to+// annotate, and adjust the annotation bits in, the SharedMutex at ptr.+std::unique_lock<std::mutex> annotationGuard(void* ptr);++constexpr uint32_t kMaxDeferredReadersAllocated = 256 * 2;++[[FOLLY_ATTR_GNU_COLD]] uint32_t getMaxDeferredReadersSlow(+    relaxed_atomic<uint32_t>& cache);++long getCurrentThreadInvoluntaryContextSwitchCount();++// kMaxDeferredReaders+FOLLY_EXPORT FOLLY_ALWAYS_INLINE uint32_t getMaxDeferredReaders() {+  static relaxed_atomic<uint32_t> cache{0};+  uint32_t const value = cache;+  return FOLLY_LIKELY(!!value) ? value : getMaxDeferredReadersSlow(cache);+}++class NopOwnershipTracker {+ public:+  void beginThreadOwnership() {}++  void maybeBeginThreadOwnership(bool) {}++  void endThreadOwnership() {}+};++class ThreadIdOwnershipTracker {+ public:+  void beginThreadOwnership() {+    assert(ownerTid_ == 0);+    ownerTid_ = tid();+  }++  void maybeBeginThreadOwnership(bool own) {+    if (own) {+      beginThreadOwnership();+    }+  }++  void endThreadOwnership() {+    // if you want to check that unlock happens on the same thread as lock,+    // assert that ownerTid_ == tid() here+    ownerTid_ = 0;+  }++ private:+  static unsigned tid() {+    /* library-local */ static thread_local unsigned cached = 0;+    auto z = cached;+    if (z == 0) {+      z = static_cast<unsigned>(getOSThreadID());+      cached = z;+    }+    return z;+  }++ private:+  // gettid() of thread holding the lock in U or E mode+  unsigned ownerTid_ = 0;+};+} // namespace shared_mutex_detail++template <+    bool ReaderPriority,+    typename Tag_ = void,+    template <typename> class Atom = std::atomic,+    typename Policy = SharedMutexPolicyDefault>+class SharedMutexImpl+    : std::conditional_t<+          Policy::track_thread_id,+          shared_mutex_detail::ThreadIdOwnershipTracker,+          shared_mutex_detail::NopOwnershipTracker> {+ private:+  static constexpr bool AnnotateForThreadSanitizer =+      kIsSanitizeThread && !ReaderPriority && !Policy::skip_annotate_rwlock;++  typedef std::conditional_t<+      Policy::track_thread_id,+      shared_mutex_detail::ThreadIdOwnershipTracker,+      shared_mutex_detail::NopOwnershipTracker>+      OwnershipTrackerBase;++ public:+  static constexpr bool kReaderPriority = ReaderPriority;+  typedef Tag_ Tag;++  typedef SharedMutexToken Token;++  constexpr SharedMutexImpl() noexcept : state_(0) {}++  SharedMutexImpl(const SharedMutexImpl&) = delete;+  SharedMutexImpl(SharedMutexImpl&&) = delete;+  SharedMutexImpl& operator=(const SharedMutexImpl&) = delete;+  SharedMutexImpl& operator=(SharedMutexImpl&&) = delete;++  // It is an error to destroy an SharedMutex that still has+  // any outstanding locks.  This is checked if NDEBUG isn't defined.+  // SharedMutex's exclusive mode can be safely used to guard the lock's+  // own destruction.  If, for example, you acquire the lock in exclusive+  // mode and then observe that the object containing the lock is no longer+  // needed, you can unlock() and then immediately destroy the lock.+  // See https://sourceware.org/bugzilla/show_bug.cgi?id=13690 for a+  // description about why this property needs to be explicitly mentioned.+  ~SharedMutexImpl() {+    auto state = state_.load(std::memory_order_relaxed);+    if (FOLLY_UNLIKELY((state & kHasS) != 0)) {+      cleanupTokenlessSharedDeferred(state);+    }++    if (folly::kIsDebug) {+      // These asserts check that everybody has released the lock before it+      // is destroyed.  If you arrive here while debugging that is likely+      // the problem.  (You could also have general heap corruption.)++      // if a futexWait fails to go to sleep because the value has been+      // changed, we don't necessarily clean up the wait bits, so it is+      // possible they will be set here in a correct system+      assert((state & ~(kWaitingAny | kMayDefer | kAnnotationCreated)) == 0);+      if ((state & kMayDefer) != 0) {+        const uint32_t maxDeferredReaders =+            shared_mutex_detail::getMaxDeferredReaders();+        for (uint32_t slot = 0; slot < maxDeferredReaders; ++slot) {+          auto slotValue =+              deferredReader(slot)->load(std::memory_order_relaxed);+          assert(!slotValueIsThis(slotValue));+          (void)slotValue;+        }+      }+    }+    annotateDestroy();+  }++  // Checks if an exclusive lock could succeed so that lock elision could be+  // enabled. Different from the two eligible_for_lock_{upgrade|shared}_elision+  // functions, this is a conservative check since kMayDefer indicates+  // "may-existing" deferred readers.+  bool eligible_for_lock_elision() const {+    // We rely on the transaction for linearization.  Wait bits are+    // irrelevant because a successful transaction will be in and out+    // without affecting the wakeup.  kBegunE is also okay for a similar+    // reason.+    auto state = state_.load(std::memory_order_relaxed);+    return (state & (kHasS | kMayDefer | kHasE | kHasU)) == 0;+  }++  // Checks if an upgrade lock could succeed so that lock elision could be+  // enabled.+  bool eligible_for_lock_upgrade_elision() const {+    auto state = state_.load(std::memory_order_relaxed);+    return (state & (kHasE | kHasU)) == 0;+  }++  // Checks if a shared lock could succeed so that lock elision could be+  // enabled.+  bool eligible_for_lock_shared_elision() const {+    // No need to honor kBegunE because a transaction doesn't block anybody+    auto state = state_.load(std::memory_order_relaxed);+    return (state & kHasE) == 0;+  }++  void lock() {+    WaitForever ctx;+    (void)lockExclusiveImpl(kHasSolo, ctx);+    OwnershipTrackerBase::beginThreadOwnership();+    annotateAcquired(annotate_rwlock_level::wrlock);+  }++  bool try_lock() {+    WaitNever ctx;+    auto result = lockExclusiveImpl(kHasSolo, ctx);+    OwnershipTrackerBase::maybeBeginThreadOwnership(result);+    annotateTryAcquired(result, annotate_rwlock_level::wrlock);+    return result;+  }++  template <class Rep, class Period>+  bool try_lock_for(const std::chrono::duration<Rep, Period>& duration) {+    WaitForDuration<Rep, Period> ctx(duration);+    auto result = lockExclusiveImpl(kHasSolo, ctx);+    OwnershipTrackerBase::maybeBeginThreadOwnership(result);+    annotateTryAcquired(result, annotate_rwlock_level::wrlock);+    return result;+  }++  template <class Clock, class Duration>+  bool try_lock_until(+      const std::chrono::time_point<Clock, Duration>& absDeadline) {+    WaitUntilDeadline<Clock, Duration> ctx{absDeadline};+    auto result = lockExclusiveImpl(kHasSolo, ctx);+    OwnershipTrackerBase::maybeBeginThreadOwnership(result);+    annotateTryAcquired(result, annotate_rwlock_level::wrlock);+    return result;+  }++  void unlock() {+    annotateReleased(annotate_rwlock_level::wrlock);+    OwnershipTrackerBase::endThreadOwnership();+    // It is possible that we have a left-over kWaitingNotS if the last+    // unlock_shared() that let our matching lock() complete finished+    // releasing before lock()'s futexWait went to sleep.  Clean it up now+    auto state = (state_ &= ~(kWaitingNotS | kPrevDefer | kHasE));+    assert((state & ~(kWaitingAny | kAnnotationCreated)) == 0);+    wakeRegisteredWaiters(state, kWaitingE | kWaitingU | kWaitingS);+  }++  // Managing the token yourself makes unlock_shared a bit faster. If the+  // tokenful version of lock_shared() is used, then it is required to pair the+  // lock with the tokenful version of unlock_shared(); alternatively, the token+  // can be invalidated with release_token(), which allows to use the tokenless+  // unlock_shared().++  void lock_shared() {+    WaitForever ctx;+    (void)lockSharedImpl(nullptr, ctx);+    annotateAcquired(annotate_rwlock_level::rdlock);+  }++  void lock_shared(Token& token) {+    WaitForever ctx;+    (void)lockSharedImpl(&token, ctx);+    annotateAcquired(annotate_rwlock_level::rdlock);+  }++  bool try_lock_shared() {+    WaitNever ctx;+    auto result = lockSharedImpl(nullptr, ctx);+    annotateTryAcquired(result, annotate_rwlock_level::rdlock);+    return result;+  }++  bool try_lock_shared(Token& token) {+    WaitNever ctx;+    auto result = lockSharedImpl(&token, ctx);+    annotateTryAcquired(result, annotate_rwlock_level::rdlock);+    return result;+  }++  template <class Rep, class Period>+  bool try_lock_shared_for(const std::chrono::duration<Rep, Period>& duration) {+    WaitForDuration<Rep, Period> ctx(duration);+    auto result = lockSharedImpl(nullptr, ctx);+    annotateTryAcquired(result, annotate_rwlock_level::rdlock);+    return result;+  }++  template <class Rep, class Period>+  bool try_lock_shared_for(+      const std::chrono::duration<Rep, Period>& duration, Token& token) {+    WaitForDuration<Rep, Period> ctx(duration);+    auto result = lockSharedImpl(&token, ctx);+    annotateTryAcquired(result, annotate_rwlock_level::rdlock);+    return result;+  }++  template <class Clock, class Duration>+  bool try_lock_shared_until(+      const std::chrono::time_point<Clock, Duration>& absDeadline) {+    WaitUntilDeadline<Clock, Duration> ctx{absDeadline};+    auto result = lockSharedImpl(nullptr, ctx);+    annotateTryAcquired(result, annotate_rwlock_level::rdlock);+    return result;+  }++  template <class Clock, class Duration>+  bool try_lock_shared_until(+      const std::chrono::time_point<Clock, Duration>& absDeadline,+      Token& token) {+    WaitUntilDeadline<Clock, Duration> ctx{absDeadline};+    auto result = lockSharedImpl(&token, ctx);+    annotateTryAcquired(result, annotate_rwlock_level::rdlock);+    return result;+  }++  void unlock_shared() {+    annotateReleased(annotate_rwlock_level::rdlock);++    auto state = state_.load(std::memory_order_acquire);++    // kPrevDefer can only be set if HasE or BegunE is set+    assert((state & (kPrevDefer | kHasE | kBegunE)) != kPrevDefer);++    // lock() strips kMayDefer immediately, but then copies it to+    // kPrevDefer so we can tell if the pre-lock() lock_shared() might+    // have deferred+    if ((state & (kMayDefer | kPrevDefer)) == 0 ||+        !tryUnlockTokenlessSharedDeferred()) {+      // Matching lock_shared() couldn't have deferred, or the deferred+      // lock has already been inlined by applyDeferredReaders()+      unlockSharedInline();+    }+  }++  void unlock_shared(Token& token) {+    if (token.state_ == Token::State::LockedShared) {+      unlock_shared();+      if (folly::kIsDebug) {+        token.state_ = Token::State::Invalid;+      }+      return;+    }++    annotateReleased(annotate_rwlock_level::rdlock);++    assert(+        token.state_ == Token::State::LockedInlineShared ||+        token.state_ == Token::State::LockedDeferredShared);++    if (token.state_ != Token::State::LockedDeferredShared ||+        !tryUnlockSharedDeferred(token.slot_)) {+      unlockSharedInline();+    }+    if (folly::kIsDebug) {+      token.state_ = Token::State::Invalid;+    }+  }++  // Invalidates the given token so that the tokenless version of+  // unlock_shared() can be called for a lock that was obtained from a tokenful+  // lock_shared(). Note that this does not unlock the mutex at any point.+  void release_token(Token& token) {+    assert(token.state_ != Token::State::Invalid);+    if (token.state_ != Token::State::LockedDeferredShared) {+      return;+    }++    auto slot = token.slot_;+    assert(slot < shared_mutex_detail::getMaxDeferredReaders());+    auto slotValue = tokenfulSlotValue();+    // Lock may have been inlined, in which case this will return false. We+    // don't need to do anything in this case.+    deferredReader(slot)->compare_exchange_strong(+        slotValue, tokenlessSlotValue());++    if (folly::kIsDebug) {+      token.state_ = Token::State::Invalid;+    }+  }++  void unlock_and_lock_shared() {+    OwnershipTrackerBase::endThreadOwnership();+    annotateReleased(annotate_rwlock_level::wrlock);+    annotateAcquired(annotate_rwlock_level::rdlock);+    // We can't use state_ -=, because we need to clear 2 bits (1 of which+    // has an uncertain initial state) and set 1 other.  We might as well+    // clear the relevant wake bits at the same time.  Note that since S+    // doesn't block the beginning of a transition to E (writer priority+    // can cut off new S, reader priority grabs BegunE and blocks deferred+    // S) we need to wake E as well.+    auto state = state_.load(std::memory_order_acquire);+    do {+      assert(+          (state & ~(kWaitingAny | kPrevDefer | kAnnotationCreated)) == kHasE);+    } while (!state_.compare_exchange_strong(+        state, (state & ~(kWaitingAny | kPrevDefer | kHasE)) + kIncrHasS));+    if ((state & (kWaitingE | kWaitingU | kWaitingS)) != 0) {+      futexWakeAll(kWaitingE | kWaitingU | kWaitingS);+    }+  }++  void unlock_and_lock_shared(Token& token) {+    unlock_and_lock_shared();+    token.state_ = Token::State::LockedInlineShared;+  }++  void lock_upgrade() {+    WaitForever ctx;+    (void)lockUpgradeImpl(ctx);+    OwnershipTrackerBase::beginThreadOwnership();+    // For TSAN: treat upgrade locks as equivalent to read locks+    annotateAcquired(annotate_rwlock_level::rdlock);+  }++  bool try_lock_upgrade() {+    WaitNever ctx;+    auto result = lockUpgradeImpl(ctx);+    OwnershipTrackerBase::maybeBeginThreadOwnership(result);+    annotateTryAcquired(result, annotate_rwlock_level::rdlock);+    return result;+  }++  template <class Rep, class Period>+  bool try_lock_upgrade_for(+      const std::chrono::duration<Rep, Period>& duration) {+    WaitForDuration<Rep, Period> ctx(duration);+    auto result = lockUpgradeImpl(ctx);+    OwnershipTrackerBase::maybeBeginThreadOwnership(result);+    annotateTryAcquired(result, annotate_rwlock_level::rdlock);+    return result;+  }++  template <class Clock, class Duration>+  bool try_lock_upgrade_until(+      const std::chrono::time_point<Clock, Duration>& absDeadline) {+    WaitUntilDeadline<Clock, Duration> ctx{absDeadline};+    auto result = lockUpgradeImpl(ctx);+    OwnershipTrackerBase::maybeBeginThreadOwnership(result);+    annotateTryAcquired(result, annotate_rwlock_level::rdlock);+    return result;+  }++  void unlock_upgrade() {+    annotateReleased(annotate_rwlock_level::rdlock);+    OwnershipTrackerBase::endThreadOwnership();+    auto state = (state_ -= kHasU);+    assert((state & (kWaitingNotS | kHasSolo)) == 0);+    wakeRegisteredWaiters(state, kWaitingE | kWaitingU);+  }++  void unlock_upgrade_and_lock() {+    // no waiting necessary, so waitMask is empty+    WaitForever ctx;+    (void)lockExclusiveImpl(0, ctx);+    annotateReleased(annotate_rwlock_level::rdlock);+    annotateAcquired(annotate_rwlock_level::wrlock);+  }++  void unlock_upgrade_and_lock_shared() {+    // No need to annotate for TSAN here because we model upgrade and shared+    // locks as the same.+    OwnershipTrackerBase::endThreadOwnership();+    auto state = (state_ -= kHasU - kIncrHasS);+    assert((state & (kWaitingNotS | kHasSolo)) == 0);+    wakeRegisteredWaiters(state, kWaitingE | kWaitingU);+  }++  void unlock_upgrade_and_lock_shared(Token& token) {+    unlock_upgrade_and_lock_shared();+    token.state_ = Token::State::LockedInlineShared;+  }++  void unlock_and_lock_upgrade() {+    annotateReleased(annotate_rwlock_level::wrlock);+    annotateAcquired(annotate_rwlock_level::rdlock);+    // We can't use state_ -=, because we need to clear 2 bits (1 of+    // which has an uncertain initial state) and set 1 other.  We might+    // as well clear the relevant wake bits at the same time.+    auto state = state_.load(std::memory_order_acquire);+    while (true) {+      assert(+          (state & ~(kWaitingAny | kPrevDefer | kAnnotationCreated)) == kHasE);+      auto after =+          (state & ~(kWaitingNotS | kWaitingS | kPrevDefer | kHasE)) + kHasU;+      if (state_.compare_exchange_strong(state, after)) {+        if ((state & kWaitingS) != 0) {+          futexWakeAll(kWaitingS);+        }+        return;+      }+    }+  }++ private:+  typedef typename folly::detail::Futex<Atom> Futex;++  // Internally we use four kinds of wait contexts.  These are structs+  // that provide a doWait method that returns true if a futex wake+  // was issued that intersects with the waitMask, false if there was a+  // timeout and no more waiting should be performed.  Spinning occurs+  // before the wait context is invoked.++  struct WaitForever {+    bool canBlock() { return true; }+    bool canTimeOut() { return false; }+    bool shouldTimeOut() { return false; }++    bool doWait(Futex& futex, uint32_t expected, uint32_t waitMask) {+      detail::futexWait(&futex, expected, waitMask);+      return true;+    }+  };++  struct WaitNever {+    bool canBlock() { return false; }+    bool canTimeOut() { return true; }+    bool shouldTimeOut() { return true; }++    bool doWait(+        Futex& /* futex */, uint32_t /* expected */, uint32_t /* waitMask */) {+      return false;+    }+  };++  template <class Rep, class Period>+  struct WaitForDuration {+    std::chrono::duration<Rep, Period> duration_;+    bool deadlineComputed_;+    std::chrono::steady_clock::time_point deadline_;++    explicit WaitForDuration(const std::chrono::duration<Rep, Period>& duration)+        : duration_(duration), deadlineComputed_(false) {}++    std::chrono::steady_clock::time_point deadline() {+      if (!deadlineComputed_) {+        deadline_ = std::chrono::steady_clock::now() + duration_;+        deadlineComputed_ = true;+      }+      return deadline_;+    }++    bool canBlock() { return duration_.count() > 0; }+    bool canTimeOut() { return true; }++    bool shouldTimeOut() {+      return std::chrono::steady_clock::now() > deadline();+    }++    bool doWait(Futex& futex, uint32_t expected, uint32_t waitMask) {+      auto result =+          detail::futexWaitUntil(&futex, expected, deadline(), waitMask);+      return result != folly::detail::FutexResult::TIMEDOUT;+    }+  };++  template <class Clock, class Duration>+  struct WaitUntilDeadline {+    std::chrono::time_point<Clock, Duration> absDeadline_;++    bool canBlock() { return true; }+    bool canTimeOut() { return true; }+    bool shouldTimeOut() { return Clock::now() > absDeadline_; }++    bool doWait(Futex& futex, uint32_t expected, uint32_t waitMask) {+      auto result =+          detail::futexWaitUntil(&futex, expected, absDeadline_, waitMask);+      return result != folly::detail::FutexResult::TIMEDOUT;+    }+  };++  void annotateLazyCreate() {+    if (AnnotateForThreadSanitizer &&+        (state_.load() & kAnnotationCreated) == 0) {+      auto guard = shared_mutex_detail::annotationGuard(this);+      // check again+      if ((state_.load() & kAnnotationCreated) == 0) {+        state_.fetch_or(kAnnotationCreated);+        annotate_benign_race_sized(+            &state_, sizeof(state_), "init TSAN", __FILE__, __LINE__);+        annotate_rwlock_create(this, __FILE__, __LINE__);+      }+    }+  }++  void annotateDestroy() {+    if (AnnotateForThreadSanitizer) {+      // call destroy only if the annotation was created+      if (state_.load() & kAnnotationCreated) {+        annotate_rwlock_destroy(this, __FILE__, __LINE__);+      }+    }+  }++  void annotateAcquired(annotate_rwlock_level w) {+    if (AnnotateForThreadSanitizer) {+      annotateLazyCreate();+      annotate_rwlock_acquired(this, w, __FILE__, __LINE__);+    }+  }++  void annotateTryAcquired(bool result, annotate_rwlock_level w) {+    if (AnnotateForThreadSanitizer) {+      annotateLazyCreate();+      annotate_rwlock_try_acquired(this, w, result, __FILE__, __LINE__);+    }+  }++  void annotateReleased(annotate_rwlock_level w) {+    if (AnnotateForThreadSanitizer) {+      assert((state_.load() & kAnnotationCreated) != 0);+      annotate_rwlock_released(this, w, __FILE__, __LINE__);+    }+  }++  // 32 bits of state+  Futex state_{};++  // S count needs to be on the end, because we explicitly allow it to+  // underflow.  This can occur while we are in the middle of applying+  // deferred locks (we remove them from deferredReaders[] before+  // inlining them), or during token-less unlock_shared() if a racing+  // lock_shared();unlock_shared() moves the deferredReaders slot while+  // the first unlock_shared() is scanning.  The former case is cleaned+  // up before we finish applying the locks.  The latter case can persist+  // until destruction, when it is cleaned up.+  static constexpr uint32_t kIncrHasS = 1 << 11;+  static constexpr uint32_t kHasS = ~(kIncrHasS - 1);++  // Set if annotation has been completed for this instance.  That annotation+  // (and setting this bit afterward) must be guarded by one of the mutexes in+  // annotationCreationGuards.+  static constexpr uint32_t kAnnotationCreated = 1 << 10;++  // If false, then there are definitely no deferred read locks for this+  // instance.  Cleared after initialization and when exclusively locked.+  static constexpr uint32_t kMayDefer = 1 << 9;++  // lock() cleared kMayDefer as soon as it starts draining readers (so+  // that it doesn't have to do a second CAS once drain completes), but+  // unlock_shared() still needs to know whether to scan deferredReaders[]+  // or not.  We copy kMayDefer to kPrevDefer when setting kHasE or+  // kBegunE, and clear it when clearing those bits.+  static constexpr uint32_t kPrevDefer = 1 << 8;++  // Exclusive-locked blocks all read locks and write locks.  This bit+  // may be set before all readers have finished, but in that case the+  // thread that sets it won't return to the caller until all read locks+  // have been released.+  static constexpr uint32_t kHasE = 1 << 7;++  // Exclusive-draining means that lock() is waiting for existing readers+  // to leave, but that new readers may still acquire shared access.+  // This is only used in reader priority mode.  New readers during+  // drain must be inline.  The difference between this and kHasU is that+  // kBegunE prevents kMayDefer from being set.+  static constexpr uint32_t kBegunE = 1 << 6;++  // At most one thread may have either exclusive or upgrade lock+  // ownership.  Unlike exclusive mode, ownership of the lock in upgrade+  // mode doesn't preclude other threads holding the lock in shared mode.+  // boost's concept for this doesn't explicitly say whether new shared+  // locks can be acquired one lock_upgrade has succeeded, but doesn't+  // list that as disallowed.  RWSpinLock disallows new read locks after+  // lock_upgrade has been acquired, but the boost implementation doesn't.+  // We choose the latter.+  static constexpr uint32_t kHasU = 1 << 5;++  // There are three states that we consider to be "solo", in that they+  // cannot coexist with other solo states.  These are kHasE, kBegunE,+  // and kHasU.  Note that S doesn't conflict with any of these, because+  // setting the kHasE is only one of the two steps needed to actually+  // acquire the lock in exclusive mode (the other is draining the existing+  // S holders).+  static constexpr uint32_t kHasSolo = kHasE | kBegunE | kHasU;++  // Once a thread sets kHasE it needs to wait for the current readers+  // to exit the lock.  We give this a separate wait identity from the+  // waiting to set kHasE so that we can perform partial wakeups (wake+  // one instead of wake all).+  static constexpr uint32_t kWaitingNotS = 1 << 4;++  // When waking writers we can either wake them all, in which case we+  // can clear kWaitingE, or we can call futexWake(1).  futexWake tells+  // us if anybody woke up, but even if we detect that nobody woke up we+  // can't clear the bit after the fact without issuing another wakeup.+  // To avoid thundering herds when there are lots of pending lock()+  // without needing to call futexWake twice when there is only one+  // waiter, kWaitingE actually encodes if we have observed multiple+  // concurrent waiters.  Tricky: ABA issues on futexWait mean that when+  // we see kWaitingESingle we can't assume that there is only one.+  static constexpr uint32_t kWaitingESingle = 1 << 2;+  static constexpr uint32_t kWaitingEMultiple = 1 << 3;+  static constexpr uint32_t kWaitingE = kWaitingESingle | kWaitingEMultiple;++  // kWaitingU is essentially a 1 bit saturating counter.  It always+  // requires a wakeAll.+  static constexpr uint32_t kWaitingU = 1 << 1;++  // All blocked lock_shared() should be awoken, so it is correct (not+  // suboptimal) to wakeAll if there are any shared readers.+  static constexpr uint32_t kWaitingS = 1 << 0;++  // kWaitingAny is a mask of all of the bits that record the state of+  // threads, rather than the state of the lock.  It is convenient to be+  // able to mask them off during asserts.+  static constexpr uint32_t kWaitingAny =+      kWaitingNotS | kWaitingE | kWaitingU | kWaitingS;++  // The reader count at which a reader will attempt to use the lock+  // in deferred mode.  If this value is 2, then the second concurrent+  // reader will set kMayDefer and use deferredReaders[].  kMayDefer is+  // cleared during exclusive access, so this threshold must be reached+  // each time a lock is held in exclusive mode.+  static constexpr uint32_t kNumSharedToStartDeferring = 2;++  // Maximum time in cycles a thread will spin waiting for a state transition.+  static constexpr uint64_t kMaxSpinCycles = Policy::max_spin_cycles;++  // The maximum number of soft yields before falling back to futex.+  // If the preemption heuristic is activated we will fall back before+  // this.  A soft yield takes ~900 nanos (two sched_yield plus a call+  // to getrusage, with checks of the goal at each step).  Soft yields+  // aren't compatible with deterministic execution under test (unlike+  // futexWaitUntil, which has a capricious but deterministic back end).+  static constexpr uint32_t kMaxSoftYieldCount = Policy::max_soft_yield_count;++  // If AccessSpreader assigns indexes from 0..k*n-1 on a system where some+  // level of the memory hierarchy is symmetrically divided into k pieces+  // (NUMA nodes, last-level caches, L1 caches, ...), then slot indexes+  // that are the same after integer division by k share that resource.+  // Our strategy for deferred readers is to probe up to numSlots/4 slots,+  // using the full granularity of AccessSpreader for the start slot+  // and then search outward.  We can use AccessSpreader::current(n)+  // without managing our own spreader if kMaxDeferredReaders <=+  // AccessSpreader::kMaxCpus, which is currently 128.+  //+  // In order to give each L1 cache its own playground, we need+  // kMaxDeferredReaders >= #L1 caches. We double it, making it+  // essentially the number of cores, so it doesn't easily run+  // out of deferred reader slots and start inlining the readers.+  // We do not know the number of cores at compile time, as the code+  // can be compiled from different server types than the one running+  // the service. So we allocate the static storage large enough to+  // hold all the slots (256).+  //+  // On x86_64 each DeferredReaderSlot is 8 bytes, so we need+  // kMaxDeferredReaders+  // * kDeferredSeparationFactor >= 64 * #L1 caches / 8 == 128.  If+  // kDeferredSearchDistance * kDeferredSeparationFactor <=+  // 64 / 8 then we will search only within a single cache line, which+  // guarantees we won't have inter-L1 contention.+ public:+  static constexpr uint32_t kDeferredSearchDistance = 2;+  static constexpr uint32_t kDeferredSeparationFactor = 4;++ private:+  static_assert(+      !(kDeferredSearchDistance & (kDeferredSearchDistance - 1)),+      "kDeferredSearchDistance must be a power of 2");++  // We need to make sure that if there is a lock_shared()+  // and lock_shared(token) followed by unlock_shared() and+  // unlock_shared(token), the token-less unlock doesn't null+  // out deferredReaders[token.slot_].  If we allowed that, then+  // unlock_shared(token) wouldn't be able to assume that its lock+  // had been inlined by applyDeferredReaders when it finds that+  // deferredReaders[token.slot_] no longer points to this.  We accomplish+  // this by stealing bit 0 from the pointer to record that the slot's+  // element has no token, hence our use of uintptr_t in deferredReaders[].+  static constexpr uintptr_t kTokenless = 0x1;++  // This is the starting location for Token-less unlock_shared().+  FOLLY_EXPORT FOLLY_ALWAYS_INLINE static relaxed_atomic<uint32_t>&+  tls_lastTokenlessSlot() {+    static relaxed_atomic<uint32_t> non_tl{};+    static thread_local relaxed_atomic<uint32_t> tl{};+    return kIsMobile ? non_tl : tl;+  }++  // Last deferred reader slot used.+  FOLLY_EXPORT FOLLY_ALWAYS_INLINE static relaxed_atomic<uint32_t>&+  tls_lastDeferredReaderSlot() {+    static relaxed_atomic<uint32_t> non_tl{};+    static thread_local relaxed_atomic<uint32_t> tl{};+    return kIsMobile ? non_tl : tl;+  }++  // Only indexes divisible by kDeferredSeparationFactor are used.+  // If any of those elements points to a SharedMutexImpl, then it+  // should be considered that there is a shared lock on that instance.+  // See kTokenless.+ public:+  typedef Atom<uintptr_t> DeferredReaderSlot;++ private:+  alignas(hardware_destructive_interference_size) static DeferredReaderSlot+      deferredReaders+          [shared_mutex_detail::kMaxDeferredReadersAllocated *+           kDeferredSeparationFactor];++  // Performs an exclusive lock, waiting for state_ & waitMask to be+  // zero first+  template <class WaitContext>+  bool lockExclusiveImpl(uint32_t preconditionGoalMask, WaitContext& ctx) {+    uint32_t state = state_.load(std::memory_order_acquire);+    if (FOLLY_LIKELY(+            (state & (preconditionGoalMask | kMayDefer | kHasS)) == 0 &&+            state_.compare_exchange_strong(state, (state | kHasE) & ~kHasU))) {+      return true;+    } else {+      return lockExclusiveImpl(state, preconditionGoalMask, ctx);+    }+  }++  template <class WaitContext>+  bool lockExclusiveImpl(+      uint32_t& state, uint32_t preconditionGoalMask, WaitContext& ctx) {+    while (true) {+      if (FOLLY_UNLIKELY((state & preconditionGoalMask) != 0) &&+          !waitForZeroBits(state, preconditionGoalMask, kWaitingE, ctx) &&+          ctx.canTimeOut()) {+        return false;+      }++      uint32_t after = (state & kMayDefer) == 0 ? 0 : kPrevDefer;+      if (!kReaderPriority || (state & (kMayDefer | kHasS)) == 0) {+        // Block readers immediately, either because we are in write+        // priority mode or because we can acquire the lock in one+        // step.  Note that if state has kHasU, then we are doing an+        // unlock_upgrade_and_lock() and we should clear it (reader+        // priority branch also does this).+        after |= (state | kHasE) & ~(kHasU | kMayDefer);+      } else {+        after |= (state | kBegunE) & ~(kHasU | kMayDefer);+      }+      if (state_.compare_exchange_strong(state, after)) {+        auto before = state;+        state = after;++        // If we set kHasE (writer priority) then no new readers can+        // arrive.  If we set kBegunE then they can still enter, but+        // they must be inline.  Either way we need to either spin on+        // deferredReaders[] slots, or inline them so that we can wait on+        // kHasS to zero itself.  deferredReaders[] is pointers, which on+        // x86_64 are bigger than futex() can handle, so we inline the+        // deferred locks instead of trying to futexWait on each slot.+        // Readers are responsible for rechecking state_ after recording+        // a deferred read to avoid atomicity problems between the state_+        // CAS and applyDeferredReader's reads of deferredReaders[].+        if (FOLLY_UNLIKELY((before & kMayDefer) != 0)) {+          applyDeferredReaders(state, ctx);+        }+        while (true) {+          assert((state & (kHasE | kBegunE)) != 0 && (state & kHasU) == 0);+          if (FOLLY_UNLIKELY((state & kHasS) != 0) &&+              !waitForZeroBits(state, kHasS, kWaitingNotS, ctx) &&+              ctx.canTimeOut()) {+            // Ugh.  We blocked new readers and other writers for a while,+            // but were unable to complete.  Move on.  On the plus side+            // we can clear kWaitingNotS because nobody else can piggyback+            // on it.+            state = (state_ &= ~(kPrevDefer | kHasE | kBegunE | kWaitingNotS));+            wakeRegisteredWaiters(state, kWaitingE | kWaitingU | kWaitingS);+            return false;+          }++          if (kReaderPriority && (state & kHasE) == 0) {+            assert((state & kBegunE) != 0);+            if (!state_.compare_exchange_strong(+                    state, (state & ~kBegunE) | kHasE)) {+              continue;+            }+          }++          return true;+        }+      }+    }+  }++  template <class WaitContext>+  bool waitForZeroBits(+      uint32_t& state, uint32_t goal, uint32_t waitMask, WaitContext& ctx) {+    for (uint64_t start = hardware_timestamp();;) {+      state = state_.load(std::memory_order_acquire);+      if ((state & goal) == 0) {+        return true;+      }+      const uint64_t elapsed = hardware_timestamp() - start;+      // NOTE: This is also true if hardware_timestamp() goes back in time, as+      // elapsed underflows.+      if (FOLLY_UNLIKELY(elapsed >= kMaxSpinCycles)) {+        return ctx.canBlock() &&+            yieldWaitForZeroBits(state, goal, waitMask, ctx);+      }+      asm_volatile_pause();+    }+  }++  template <class WaitContext>+  bool yieldWaitForZeroBits(+      uint32_t& state, uint32_t goal, uint32_t waitMask, WaitContext& ctx) {+    long thread_nivcsw = 0;+    long before = -1;+    for (uint32_t yieldCount = 0; yieldCount < kMaxSoftYieldCount;+         ++yieldCount) {+      for (int softState = 0; softState < 3; ++softState) {+        if (softState < 2) {+          std::this_thread::yield();+        } else {+          thread_nivcsw = shared_mutex_detail::+              getCurrentThreadInvoluntaryContextSwitchCount();+        }+        if (((state = state_.load(std::memory_order_acquire)) & goal) == 0) {+          return true;+        }+        if (ctx.shouldTimeOut()) {+          return false;+        }+      }+      if (before >= 0 && thread_nivcsw >= before + 2) {+        // One involuntary csw might just be occasional background work,+        // but if we get two in a row then we guess that there is someone+        // else who can profitably use this CPU.  Fall back to futex+        break;+      }+      before = thread_nivcsw;+    }+    return futexWaitForZeroBits(state, goal, waitMask, ctx);+  }++  template <class WaitContext>+  bool futexWaitForZeroBits(+      uint32_t& state, uint32_t goal, uint32_t waitMask, WaitContext& ctx) {+    assert(+        waitMask == kWaitingNotS || waitMask == kWaitingE ||+        waitMask == kWaitingU || waitMask == kWaitingS);++    while (true) {+      state = state_.load(std::memory_order_acquire);+      if ((state & goal) == 0) {+        return true;+      }++      auto after = state;+      if (waitMask == kWaitingE) {+        if ((state & kWaitingESingle) != 0) {+          after |= kWaitingEMultiple;+        } else {+          after |= kWaitingESingle;+        }+      } else {+        after |= waitMask;+      }++      // CAS is better than atomic |= here, because it lets us avoid+      // setting the wait flag when the goal is concurrently achieved+      if (after != state && !state_.compare_exchange_strong(state, after)) {+        continue;+      }++      if (!ctx.doWait(state_, after, waitMask)) {+        // timed out+        return false;+      }+    }+  }++  // Wakes up waiters registered in state_ as appropriate, clearing the+  // awaiting bits for anybody that was awoken.  Tries to perform direct+  // single wakeup of an exclusive waiter if appropriate+  void wakeRegisteredWaiters(uint32_t& state, uint32_t wakeMask) {+    if (FOLLY_UNLIKELY((state & wakeMask) != 0)) {+      wakeRegisteredWaitersImpl(state, wakeMask);+    }+  }++  [[FOLLY_ATTR_GNU_USED]]+  void wakeRegisteredWaitersImpl(uint32_t& state, uint32_t wakeMask) {+    // If there are multiple lock() pending only one of them will actually+    // get to wake up, so issuing futexWakeAll will make a thundering herd.+    // There's nothing stopping us from issuing futexWake(1) instead,+    // so long as the wait bits are still an accurate reflection of+    // the waiters.  If we notice (via futexWake's return value) that+    // nobody woke up then we can try again with the normal wake-all path.+    // Note that we can't just clear the bits at that point; we need to+    // clear the bits and then issue another wakeup.+    //+    // It is possible that we wake an E waiter but an outside S grabs the+    // lock instead, at which point we should wake pending U and S waiters.+    // Rather than tracking state to make the failing E regenerate the+    // wakeup, we just disable the optimization in the case that there+    // are waiting U or S that we are eligible to wake.+    if ((wakeMask & kWaitingE) == kWaitingE &&+        (state & wakeMask) == kWaitingE &&+        detail::futexWake(&state_, 1, kWaitingE) > 0) {+      // somebody woke up, so leave state_ as is and clear it later+      return;+    }++    if ((state & wakeMask) != 0) {+      auto prev = state_.fetch_and(~wakeMask);+      if ((prev & wakeMask) != 0) {+        futexWakeAll(wakeMask);+      }+      state = prev & ~wakeMask;+    }+  }++  void futexWakeAll(uint32_t wakeMask) {+    detail::futexWake(&state_, std::numeric_limits<int>::max(), wakeMask);+  }++  DeferredReaderSlot* deferredReader(uint32_t slot) {+    return &deferredReaders[slot * kDeferredSeparationFactor];+  }++  uintptr_t tokenfulSlotValue() { return reinterpret_cast<uintptr_t>(this); }++  uintptr_t tokenlessSlotValue() { return tokenfulSlotValue() | kTokenless; }++  bool slotValueIsThis(uintptr_t slotValue) {+    return (slotValue & ~kTokenless) == tokenfulSlotValue();+  }++  // Clears any deferredReaders[] that point to this, adjusting the inline+  // shared lock count to compensate.  Does some spinning and yielding+  // to avoid the work.  Always finishes the application, even if ctx+  // times out.+  template <class WaitContext>+  void applyDeferredReaders(uint32_t& state, WaitContext& ctx) {+    uint32_t slot = 0;++    const uint32_t maxDeferredReaders =+        shared_mutex_detail::getMaxDeferredReaders();+    for (uint64_t start = hardware_timestamp();;) {+      while (!slotValueIsThis(+          deferredReader(slot)->load(std::memory_order_acquire))) {+        if (++slot == maxDeferredReaders) {+          return;+        }+      }+      const uint64_t elapsed = hardware_timestamp() - start;+      // NOTE: This is also true if hardware_timestamp() goes back in time, as+      // elapsed underflows.+      if (FOLLY_UNLIKELY(elapsed >= kMaxSpinCycles)) {+        applyDeferredReaders(state, ctx, slot);+        return;+      }+      asm_volatile_pause();+    }+  }++  template <class WaitContext>+  void applyDeferredReaders(uint32_t& state, WaitContext& ctx, uint32_t slot) {+    long thread_nivcsw = 0;+    long before = -1;+    const uint32_t maxDeferredReaders =+        shared_mutex_detail::getMaxDeferredReaders();+    for (uint32_t yieldCount = 0; yieldCount < kMaxSoftYieldCount;+         ++yieldCount) {+      for (int softState = 0; softState < 3; ++softState) {+        if (softState < 2) {+          std::this_thread::yield();+        } else {+          thread_nivcsw = shared_mutex_detail::+              getCurrentThreadInvoluntaryContextSwitchCount();+        }+        while (!slotValueIsThis(+            deferredReader(slot)->load(std::memory_order_acquire))) {+          if (++slot == maxDeferredReaders) {+            return;+          }+        }+        if (ctx.shouldTimeOut()) {+          // finish applying immediately on timeout+          break;+        }+      }+      if (before >= 0 && thread_nivcsw >= before + 2) {+        // heuristic says run queue is not empty+        break;+      }+      before = thread_nivcsw;+    }++    uint32_t movedSlotCount = 0;+    for (; slot < maxDeferredReaders; ++slot) {+      auto slotPtr = deferredReader(slot);+      auto slotValue = slotPtr->load(std::memory_order_acquire);+      if (slotValueIsThis(slotValue) &&+          slotPtr->compare_exchange_strong(slotValue, 0)) {+        ++movedSlotCount;+      }+    }++    if (movedSlotCount > 0) {+      state = (state_ += movedSlotCount * kIncrHasS);+    }+    assert((state & (kHasE | kBegunE)) != 0);++    // if state + kIncrHasS overflows (off the end of state) then either+    // we have 2^(32-9) readers (almost certainly an application bug)+    // or we had an underflow (also a bug)+    assert(state < state + kIncrHasS);+  }++  // It is straightforward to make a token-less lock_shared() and+  // unlock_shared() either by making the token-less version always use+  // LockedInlineShared mode or by removing the token version.  Supporting+  // deferred operation for both types is trickier than it appears, because+  // the purpose of the token it so that unlock_shared doesn't have to+  // look in other slots for its deferred lock.  Token-less unlock_shared+  // might place a deferred lock in one place and then release a different+  // slot that was originally used by the token-ful version.  If this was+  // important we could solve the problem by differentiating the deferred+  // locks so that cross-variety release wouldn't occur.  The best way+  // is probably to steal a bit from the pointer, making deferredLocks[]+  // an array of Atom<uintptr_t>.++  template <class WaitContext>+  bool lockSharedImpl(Token* token, WaitContext& ctx) {+    uint32_t state = state_.load(std::memory_order_relaxed);+    if ((state & (kHasS | kMayDefer | kHasE)) == 0 &&+        state_.compare_exchange_strong(state, state + kIncrHasS)) {+      if (token != nullptr) {+        token->state_ = Token::State::LockedInlineShared;+      }+      return true;+    }+    return lockSharedImpl(state, token, ctx);+  }++  template <class WaitContext>+  bool lockSharedImpl(uint32_t& state, Token* token, WaitContext& ctx);++  // Updates the state in/out argument as if the locks were made inline,+  // but does not update state_+  void cleanupTokenlessSharedDeferred(uint32_t& state) {+    const uint32_t maxDeferredReaders =+        shared_mutex_detail::getMaxDeferredReaders();+    for (uint32_t i = 0; i < maxDeferredReaders; ++i) {+      auto slotPtr = deferredReader(i);+      auto slotValue = slotPtr->load(std::memory_order_relaxed);+      if (slotValue == tokenlessSlotValue()) {+        slotPtr->store(0, std::memory_order_relaxed);+        state += kIncrHasS;+        if ((state & kHasS) == 0) {+          break;+        }+      }+    }+  }++  bool tryUnlockTokenlessSharedDeferred();++  bool tryUnlockSharedDeferred(uint32_t slot) {+    assert(slot < shared_mutex_detail::getMaxDeferredReaders());+    auto slotValue = tokenfulSlotValue();+    return deferredReader(slot)->compare_exchange_strong(slotValue, 0);+  }++  uint32_t unlockSharedInline() {+    uint32_t state = (state_ -= kIncrHasS);+    assert(+        (state & (kHasE | kBegunE | kMayDefer)) != 0 ||+        state < state + kIncrHasS);+    if ((state & kHasS) == 0) {+      // Only the second half of lock() can be blocked by a non-zero+      // reader count, so that's the only thing we need to wake+      wakeRegisteredWaiters(state, kWaitingNotS);+    }+    return state;+  }++  template <class WaitContext>+  bool lockUpgradeImpl(WaitContext& ctx) {+    uint32_t state;+    do {+      if (!waitForZeroBits(state, kHasSolo, kWaitingU, ctx)) {+        return false;+      }+    } while (!state_.compare_exchange_strong(state, state | kHasU));+    return true;+  }+};++using SharedMutexReadPriority = SharedMutexImpl<true>;+using SharedMutexWritePriority = SharedMutexImpl<false>;+using SharedMutex = SharedMutexWritePriority;+using SharedMutexTracked = SharedMutexImpl<+    false,+    void,+    std::atomic,+    shared_mutex_detail::PolicyTracked>;+using SharedMutexSuppressTSAN = SharedMutexImpl<+    false,+    void,+    std::atomic,+    shared_mutex_detail::PolicySuppressTSAN>;++// Prevent the compiler from instantiating these in other translation units.+// They are instantiated once in SharedMutex.cpp+extern template class SharedMutexImpl<true>;+extern template class SharedMutexImpl<false>;++template <+    bool ReaderPriority,+    typename Tag_,+    template <typename>+    class Atom,+    typename Policy>+alignas(hardware_destructive_interference_size)+    typename SharedMutexImpl<ReaderPriority, Tag_, Atom, Policy>::+        DeferredReaderSlot+    SharedMutexImpl<ReaderPriority, Tag_, Atom, Policy>::deferredReaders+        [shared_mutex_detail::kMaxDeferredReadersAllocated *+         kDeferredSeparationFactor] = {};++template <+    bool ReaderPriority,+    typename Tag_,+    template <typename>+    class Atom,+    typename Policy>+bool SharedMutexImpl<ReaderPriority, Tag_, Atom, Policy>::+    tryUnlockTokenlessSharedDeferred() {+  uint32_t bestSlot = tls_lastTokenlessSlot();+  // use do ... while to avoid calling+  // shared_mutex_detail::getMaxDeferredReaders() unless necessary+  uint32_t i = 0;+  do {+    auto slotPtr = deferredReader(bestSlot ^ i);+    auto slotValue = slotPtr->load(std::memory_order_relaxed);+    if (slotValue == tokenlessSlotValue() &&+        slotPtr->compare_exchange_strong(slotValue, 0)) {+      tls_lastTokenlessSlot() = bestSlot ^ i;+      return true;+    }+    ++i;+  } while (i < shared_mutex_detail::getMaxDeferredReaders());+  return false;+}++template <+    bool ReaderPriority,+    typename Tag_,+    template <typename>+    class Atom,+    typename Policy>+template <class WaitContext>+bool SharedMutexImpl<ReaderPriority, Tag_, Atom, Policy>::lockSharedImpl(+    uint32_t& state, Token* token, WaitContext& ctx) {+  const uint32_t maxDeferredReaders =+      shared_mutex_detail::getMaxDeferredReaders();+  while (true) {+    if (FOLLY_UNLIKELY((state & kHasE) != 0) &&+        !waitForZeroBits(state, kHasE, kWaitingS, ctx) && ctx.canTimeOut()) {+      return false;+    }++    uint32_t slot = tls_lastDeferredReaderSlot();+    uintptr_t slotValue = 1; // any non-zero value will do++    bool canAlreadyDefer = (state & kMayDefer) != 0;+    bool aboveDeferThreshold =+        (state & kHasS) >= (kNumSharedToStartDeferring - 1) * kIncrHasS;+    bool drainInProgress = ReaderPriority && (state & kBegunE) != 0;+    if (canAlreadyDefer || (aboveDeferThreshold && !drainInProgress)) {+      /* Try using the most recent slot first. */+      slotValue = deferredReader(slot)->load(std::memory_order_relaxed);+      if (slotValue != 0) {+        // starting point for our empty-slot search, can change after+        // calling waitForZeroBits+        uint32_t bestSlot =+            (uint32_t)folly::AccessSpreader<Atom>::current(maxDeferredReaders);++        // deferred readers are already enabled, or it is time to+        // enable them if we can find a slot+        for (uint32_t i = 0; i < kDeferredSearchDistance; ++i) {+          slot = bestSlot ^ i;+          assert(slot < maxDeferredReaders);+          slotValue = deferredReader(slot)->load(std::memory_order_relaxed);+          if (slotValue == 0) {+            // found empty slot+            tls_lastDeferredReaderSlot() = slot;+            break;+          }+        }+      }+    }++    if (slotValue != 0) {+      // not yet deferred, or no empty slots+      if (state_.compare_exchange_strong(state, state + kIncrHasS)) {+        // successfully recorded the read lock inline+        if (token != nullptr) {+          token->state_ = Token::State::LockedInlineShared;+        }+        return true;+      }+      // state is updated, try again+      continue;+    }++    // record that deferred readers might be in use if necessary+    if ((state & kMayDefer) == 0) {+      if (!state_.compare_exchange_strong(state, state | kMayDefer)) {+        // keep going if CAS failed because somebody else set the bit+        // for us+        if ((state & (kHasE | kMayDefer)) != kMayDefer) {+          continue;+        }+      }+      // state = state | kMayDefer;+    }++    // try to use the slot+    bool gotSlot = deferredReader(slot)->compare_exchange_strong(+        slotValue,+        token == nullptr ? tokenlessSlotValue() : tokenfulSlotValue());++    // If we got the slot, we need to verify that an exclusive lock+    // didn't happen since we last checked.  If we didn't get the slot we+    // need to recheck state_ anyway to make sure we don't waste too much+    // work.  It is also possible that since we checked state_ someone+    // has acquired and released the write lock, clearing kMayDefer.+    // Both cases are covered by looking for the readers-possible bit,+    // because it is off when the exclusive lock bit is set.+    state = state_.load(std::memory_order_acquire);++    if (!gotSlot) {+      continue;+    }++    if (token == nullptr) {+      tls_lastTokenlessSlot() = slot;+    }++    if ((state & kMayDefer) != 0) {+      assert((state & kHasE) == 0);+      // success+      if (token != nullptr) {+        token->state_ = Token::State::LockedDeferredShared;+        token->slot_ = (uint16_t)slot;+      }+      return true;+    }++    // release the slot before retrying+    if (token == nullptr) {+      // We can't rely on slot.  Token-less slot values can be freed by+      // any unlock_shared(), so we need to do the full deferredReader+      // search during unlock.  Unlike unlock_shared(), we can't trust+      // kPrevDefer here.  This deferred lock isn't visible to lock()+      // (that's the whole reason we're undoing it) so there might have+      // subsequently been an unlock() and lock() with no intervening+      // transition to deferred mode.+      if (!tryUnlockTokenlessSharedDeferred()) {+        unlockSharedInline();+      }+    } else {+      if (!tryUnlockSharedDeferred(slot)) {+        unlockSharedInline();+      }+    }++    // We got here not because the lock was unavailable, but because+    // we lost a compare-and-swap.  Try-lock is typically allowed to+    // have spurious failures, but there is no lock efficiency gain+    // from exploiting that freedom here.+  }+}++namespace shared_mutex_detail {++[[noreturn]] void throwOperationNotPermitted();++[[noreturn]] void throwDeadlockWouldOccur();++} // namespace shared_mutex_detail++} // namespace folly++// std::shared_lock specialization for folly::SharedMutex to leverage tokenful+// version of unlock_shared for faster unlocking.+namespace std {++template <+    bool ReaderPriority,+    typename Tag_,+    template <typename>+    class Atom,+    typename Policy>+class shared_lock<+    ::folly::SharedMutexImpl<ReaderPriority, Tag_, Atom, Policy>> {+ public:+  using mutex_type =+      ::folly::SharedMutexImpl<ReaderPriority, Tag_, Atom, Policy>;+  using token_type = typename mutex_type::Token;++  shared_lock() noexcept = default;++  FOLLY_NODISCARD explicit shared_lock(mutex_type& mutex)+      : mutex_(std::addressof(mutex)) {+    lock();+  }++  shared_lock(mutex_type& mutex, std::defer_lock_t) noexcept+      : mutex_(std::addressof(mutex)) {}++  FOLLY_NODISCARD shared_lock(mutex_type& mutex, std::try_to_lock_t)+      : mutex_(std::addressof(mutex)) {+    try_lock();+  }++  FOLLY_NODISCARD shared_lock(mutex_type& mutex, std::adopt_lock_t)+      : mutex_(std::addressof(mutex)) {+    token_.state_ = token_type::State::LockedShared;+  }++  template <typename Clock, typename Duration>+  FOLLY_NODISCARD shared_lock(+      mutex_type& mutex,+      const std::chrono::time_point<Clock, Duration>& deadline)+      : mutex_(std::addressof(mutex)) {+    try_lock_until(deadline);+  }++  template <typename Rep, typename Period>+  FOLLY_NODISCARD shared_lock(+      mutex_type& mutex, const std::chrono::duration<Rep, Period>& timeout)+      : mutex_(std::addressof(mutex)) {+    try_lock_for(timeout);+  }++  ~shared_lock() {+    if (owns_lock()) {+      mutex_->unlock_shared(token_);+    }+  }++  shared_lock(const shared_lock&) = delete;++  shared_lock& operator=(const shared_lock&) = delete;++  shared_lock(shared_lock&& other) noexcept : shared_lock() { swap(other); }++  shared_lock& operator=(shared_lock&& other) noexcept {+    shared_lock(std::move(other)).swap(*this);+    return *this;+  }++  void lock() {+    error_if_not_lockable();+    mutex_->lock_shared(token_);+  }++  bool try_lock() {+    error_if_not_lockable();+    return mutex_->try_lock_shared(token_);+  }++  template <typename Rep, typename Period>+  bool try_lock_for(const std::chrono::duration<Rep, Period>& timeout) {+    error_if_not_lockable();+    return mutex_->try_lock_shared_for(timeout, token_);+  }++  template <typename Clock, typename Duration>+  bool try_lock_until(+      const std::chrono::time_point<Clock, Duration>& deadline) {+    error_if_not_lockable();+    return mutex_->try_lock_shared_until(deadline, token_);+  }++  void unlock() {+    if (FOLLY_UNLIKELY(!owns_lock())) {+      ::folly::shared_mutex_detail::throwOperationNotPermitted();+    }+    mutex_->unlock_shared(token_);+    token_ = {};+  }++  void swap(shared_lock& other) noexcept {+    std::swap(mutex_, other.mutex_);+    std::swap(token_, other.token_);+  }++  mutex_type* release() noexcept {+    if (owns_lock()) {+      mutex_->release_token(token_);+      token_ = {};+    }+    return std::exchange(mutex_, nullptr);+  }++  FOLLY_NODISCARD bool owns_lock() const noexcept {+    return static_cast<bool>(token_);+  }++  explicit operator bool() const noexcept { return owns_lock(); }++  FOLLY_NODISCARD mutex_type* mutex() const noexcept { return mutex_; }++ private:+  void error_if_not_lockable() const {+    if (FOLLY_UNLIKELY(mutex_ == nullptr)) {+      ::folly::shared_mutex_detail::throwOperationNotPermitted();+    }+    if (FOLLY_UNLIKELY(owns_lock())) {+      ::folly::shared_mutex_detail::throwDeadlockWouldOccur();+    }+  }++  mutex_type* mutex_ = nullptr;+  token_type token_;+};++} // namespace std
+ folly/folly/Singleton-inl.h view
@@ -0,0 +1,332 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++namespace folly {++namespace detail {++template <typename T>+template <typename Tag, typename VaultTag>+struct SingletonHolder<T>::Impl : SingletonHolder<T> {+  Impl()+      : SingletonHolder<T>(+            {typeid(T), typeid(Tag)}, *SingletonVault::singleton<VaultTag>()) {}+};++template <typename T>+template <typename Tag, typename VaultTag>+inline SingletonHolder<T>& SingletonHolder<T>::singleton() {+  return detail::createGlobal<Impl<Tag, VaultTag>, void>();+}++[[noreturn]] void singletonWarnDoubleRegistrationAndAbort(+    const TypeDescriptor& type);++template <typename T>+void SingletonHolder<T>::registerSingleton(CreateFunc c, TeardownFunc t) {+  std::lock_guard entry_lock(mutex_);++  if (state_ != SingletonHolderState::NotRegistered) {+    /* Possible causes:+     *+     * You have two instances of the same+     * folly::Singleton<Class>. Probably because you define the+     * singleton in a header included in multiple places? In general,+     * folly::Singleton shouldn't be in the header, only off in some+     * anonymous namespace in a cpp file. Code needing the singleton+     * will find it when that code references folly::Singleton<Class>.+     *+     * Alternatively, you could have 2 singletons with the same type+     * defined with a different name in a .cpp (source) file. For+     * example:+     *+     * Singleton<int> a([] { return new int(3); });+     * Singleton<int> b([] { return new int(4); });+     *+     * Adding tags should fix this (see documentation in the header).+     *+     */+    singletonWarnDoubleRegistrationAndAbort(type());+  }++  create_ = std::move(c);+  teardown_ = std::move(t);++  state_ = SingletonHolderState::Dead;+}++template <typename T>+void SingletonHolder<T>::registerSingletonMock(CreateFunc c, TeardownFunc t) {+  if (state_ == SingletonHolderState::NotRegistered) {+    detail::singletonWarnRegisterMockEarlyAndAbort(type());+  }+  if (state_ == SingletonHolderState::Living ||+      state_ == SingletonHolderState::LivingInChildAfterFork) {+    destroyInstance();+  }++  {+    auto creationOrder = vault_.creationOrder_.wlock();++    auto it = std::find(creationOrder->begin(), creationOrder->end(), type());+    if (it != creationOrder->end()) {+      creationOrder->erase(it);+    }+  }++  std::lock_guard entry_lock(mutex_);++  create_ = std::move(c);+  teardown_ = std::move(t);+}++template <typename T>+T* SingletonHolder<T>::get() {+  if (FOLLY_LIKELY(+          state_.load(std::memory_order_acquire) ==+          SingletonHolderState::Living)) {+    return instance_ptr_;+  }+  createInstance();++  if (instance_weak_.expired()) {+    detail::singletonThrowGetInvokedAfterDestruction(type());+  }++  return instance_ptr_;+}++template <typename T>+std::weak_ptr<T> SingletonHolder<T>::get_weak() {+  if (FOLLY_UNLIKELY(+          state_.load(std::memory_order_acquire) !=+          SingletonHolderState::Living)) {+    createInstance();+  }++  return instance_weak_core_cached_.get();+}++template <typename T>+std::shared_ptr<T> SingletonHolder<T>::try_get() {+  if (FOLLY_UNLIKELY(+          state_.load(std::memory_order_acquire) !=+          SingletonHolderState::Living)) {+    createInstance();+  }++  return instance_weak_core_cached_.lock();+}++template <typename T>+folly::ReadMostlySharedPtr<T> SingletonHolder<T>::try_get_fast() {+  if (FOLLY_UNLIKELY(+          state_.load(std::memory_order_acquire) !=+          SingletonHolderState::Living)) {+    createInstance();+  }++  return instance_weak_fast_.lock();+}++template <typename T>+template <typename Func>+invoke_result_t<Func, T*> detail::SingletonHolder<T>::apply(Func f) {+  return f(try_get().get());+}++template <typename T>+void SingletonHolder<T>::vivify() {+  if (FOLLY_UNLIKELY(+          state_.load(std::memory_order_relaxed) !=+          SingletonHolderState::Living)) {+    createInstance();+  }+}++template <typename T>+bool SingletonHolder<T>::hasLiveInstance() {+  return !instance_weak_.expired();+}++template <typename T>+void SingletonHolder<T>::preDestroyInstance(+    ReadMostlyMainPtrDeleter<>& deleter) {+  instance_copy_ = instance_;+  deleter.add(std::move(instance_));+}++template <typename T>+void SingletonHolder<T>::destroyInstance() {+  if (state_.load(std::memory_order_relaxed) ==+      SingletonHolderState::LivingInChildAfterFork) {+    if (vault_.failOnUseAfterFork_) {+      LOG(DFATAL) << "Attempting to destroy singleton " << type().name()+                  << " in child process after fork";+    } else {+      LOG(ERROR) << "Attempting to destroy singleton " << type().name()+                 << " in child process after fork";+    }+  }+  state_ = SingletonHolderState::Dead;+  instance_core_cached_.reset();+  instance_.reset();+  instance_copy_.reset();+  if (destroy_baton_) {+    constexpr std::chrono::seconds kDestroyWaitTime{5};+    auto const wait_options =+        destroy_baton_->wait_options().logging_enabled(false);+    auto last_reference_released =+        destroy_baton_->try_wait_for(kDestroyWaitTime, wait_options);+    if (last_reference_released) {+      vault_.addToShutdownLog("Destroying " + type().name());+      teardown_(instance_ptr_);+      vault_.addToShutdownLog(type().name() + " destroyed.");+    } else {+      print_destructor_stack_trace_->store(true);+      detail::singletonWarnDestroyInstanceLeak(type(), instance_ptr_);+    }+  }+}++template <typename T>+void SingletonHolder<T>::inChildAfterFork() {+  auto expected = SingletonHolderState::Living;+  state_.compare_exchange_strong(+      expected,+      SingletonHolderState::LivingInChildAfterFork,+      std::memory_order_relaxed,+      std::memory_order_relaxed);+}++template <typename T>+SingletonHolder<T>::SingletonHolder(+    TypeDescriptor typeDesc, SingletonVault& vault) noexcept+    : SingletonHolderBase(typeDesc), vault_(vault) {}++template <typename T>+bool SingletonHolder<T>::creationStarted() {+  // If alive, then creation was of course started.+  // This is flipped after creating_thread_ was set, and before it was reset.+  if (state_.load(std::memory_order_acquire) == SingletonHolderState::Living) {+    return true;+  }++  // Not yet built.  Is it currently in progress?+  if (creating_thread_.load(std::memory_order_acquire) != std::thread::id()) {+    return true;+  }++  return false;+}++template <typename T>+void SingletonHolder<T>::createInstance() {+  if (creating_thread_.load(std::memory_order_acquire) ==+      std::this_thread::get_id()) {+    detail::singletonWarnCreateCircularDependencyAndAbort(type());+  }++  std::lock_guard entry_lock(mutex_);+  if (state_.load(std::memory_order_acquire) == SingletonHolderState::Living) {+    return;+  }+  if (state_.load(std::memory_order_relaxed) ==+      SingletonHolderState::LivingInChildAfterFork) {+    if (vault_.failOnUseAfterFork_) {+      LOG(DFATAL) << "Attempting to use singleton " << type().name()+                  << " in child process after fork";+    } else {+      LOG(ERROR) << "Attempting to use singleton " << type().name()+                 << " in child process after fork";+    }+    auto expected = SingletonHolderState::LivingInChildAfterFork;+    state_.compare_exchange_strong(+        expected,+        SingletonHolderState::Living,+        std::memory_order_relaxed,+        std::memory_order_relaxed);+    return;+  }+  if (state_.load(std::memory_order_acquire) ==+      SingletonHolderState::NotRegistered) {+    detail::singletonWarnCreateUnregisteredAndAbort(type());+  }++  if (state_.load(std::memory_order_acquire) == SingletonHolderState::Living) {+    return;+  }++  SCOPE_EXIT {+    // Clean up creator thread when complete, and also, in case of errors here,+    // so that subsequent attempts don't think this is still in the process of+    // being built.+    creating_thread_.store(std::thread::id(), std::memory_order_release);+  };++  creating_thread_.store(std::this_thread::get_id(), std::memory_order_release);++  auto state = vault_.state_.rlock();+  if (vault_.type_.load(std::memory_order_relaxed) !=+          SingletonVault::Type::Relaxed &&+      !state->registrationComplete) {+    detail::singletonWarnCreateBeforeRegistrationCompleteAndAbort(type());+  }+  if (state->state == detail::SingletonVaultState::Type::Quiescing) {+    return;+  }++  auto destroy_baton = std::make_shared<folly::Baton<>>();+  auto print_destructor_stack_trace =+      std::make_shared<std::atomic<bool>>(false);++  // Can't use make_shared -- no support for a custom deleter, sadly.+  std::shared_ptr<T> instance(+      create_(),+      [destroy_baton, print_destructor_stack_trace, type = type()](T*) mutable {+        destroy_baton->post();+        if (print_destructor_stack_trace->load()) {+          detail::singletonPrintDestructionStackTrace(type);+        }+      });++  // We should schedule destroyInstances() only after the singleton was+  // created. This will ensure it will be destroyed before singletons,+  // not managed by folly::Singleton, which were initialized in its+  // constructor+  SingletonVault::scheduleDestroyInstances();++  instance_weak_ = instance;+  instance_ptr_ = instance.get();+  instance_core_cached_.reset(instance);+  instance_.reset(std::move(instance));+  instance_weak_fast_ = instance_;+  instance_weak_core_cached_.reset(instance_core_cached_);++  destroy_baton_ = std::move(destroy_baton);+  print_destructor_stack_trace_ = std::move(print_destructor_stack_trace);++  // This has to be the last step, because once state is Living other threads+  // may access instance and instance_weak w/o synchronization.+  state_.store(SingletonHolderState::Living, std::memory_order_release);++  vault_.creationOrder_.wlock()->push_back(type());+  vault_.instantiatedAtLeastOnce_.wlock()->insert(type());+}++} // namespace detail++} // namespace folly
+ folly/folly/Singleton.cpp view
@@ -0,0 +1,518 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#include <folly/Singleton.h>++#ifndef _WIN32+#include <dlfcn.h>+#include <signal.h>+#include <time.h>+#endif++#include <atomic>+#include <chrono>+#include <cstdio>+#include <cstdlib>+#include <string>+#include <fmt/chrono.h>+#include <fmt/format.h>++#include <folly/Demangle.h>+#include <folly/ScopeGuard.h>+#include <folly/experimental/symbolizer/Symbolizer.h>+#include <folly/lang/SafeAssert.h>+#include <folly/portability/Config.h>+#include <folly/portability/FmtCompile.h>+// Before registrationComplete() we cannot assume that glog has been+// initialized, so we need to use RAW_LOG for any message that may be logged+// before that.+#include <glog/raw_logging.h>++#if !defined(_WIN32) && !defined(__APPLE__) && !defined(__ANDROID__)+#define FOLLY_SINGLETON_HAVE_DLSYM 1+#endif++namespace folly {++#if FOLLY_SINGLETON_HAVE_DLSYM+namespace detail {+static void singleton_hs_init_weak(int* argc, char** argv[])+    __attribute__((__weakref__("hs_init")));+} // namespace detail+#endif++SingletonVault::Type SingletonVault::defaultVaultType() {+#if FOLLY_SINGLETON_HAVE_DLSYM+  bool isPython = dlsym(RTLD_DEFAULT, "Py_Main");+  bool isHaskell =+      detail::singleton_hs_init_weak || dlsym(RTLD_DEFAULT, "hs_init");+  bool isJVM = dlsym(RTLD_DEFAULT, "JNI_GetCreatedJavaVMs");+  bool isD = dlsym(RTLD_DEFAULT, "_d_run_main");+  bool isCgo = dlsym(RTLD_DEFAULT, "_cgo_topofstack") ||+      dlsym(RTLD_DEFAULT, "_cgo_panic");++  return isPython || isHaskell || isJVM || isD || isCgo+      ? Type::Relaxed+      : Type::Strict;+#else+  return Type::Relaxed;+#endif+}++namespace detail {++std::string TypeDescriptor::name() const {+  auto ret = demangle(ti_.name());+  if (tag_ti_ != std::type_index(typeid(DefaultTag))) {+    ret += "/";+    ret += demangle(tag_ti_.name());+  }+  return ret.toStdString();+}++[[noreturn]] void singletonWarnDoubleRegistrationAndAbort(+    const TypeDescriptor& type) {+  FOLLY_SAFE_FATAL( // May happen before registrationComplete().+      fmt::format(+          fmt::runtime( //+              "Double registration of singletons of the same "+              "underlying type; check for multiple definitions "+              "of type folly::Singleton<{}>"),+          type.name())+          .c_str());+}++[[noreturn]] void singletonWarnLeakyDoubleRegistrationAndAbort(+    const TypeDescriptor& type) {+  FOLLY_SAFE_FATAL( // May happen before registrationComplete().+      fmt::format(+          fmt::runtime( //+              "Double registration of singletons of the same "+              "underlying type; check for multiple definitions "+              "of type folly::LeakySingleton<{}>"),+          type.name())+          .c_str());+}++[[noreturn]] void singletonWarnLeakyInstantiatingNotRegisteredAndAbort(+    const TypeDescriptor& type) {+  FOLLY_SAFE_FATAL( // May happen before registrationComplete().+      "Creating instance for unregistered singleton: ",+      type.name().c_str());+}++[[noreturn]] void singletonWarnRegisterMockEarlyAndAbort(+    const TypeDescriptor& type) {+  FOLLY_SAFE_FATAL( // May happen before registrationComplete().+      "Registering mock before singleton was registered: ",+      type.name().c_str());+}++void singletonWarnDestroyInstanceLeak(+    const TypeDescriptor& type, const void* ptr) {+  LOG(ERROR)+      << "Singleton of type " << type.name() << " has a "+      << "living reference at destroyInstances time; beware! Raw "+      << "pointer is " << ptr << ". It is very likely "+      << "that some other singleton is holding a shared_ptr to it. "+      << "This singleton will be leaked (even if a shared_ptr to it "+      << "is eventually released)."+      << "Make sure dependencies between these singletons are "+      << "properly defined.";+}++[[noreturn]] void singletonWarnCreateCircularDependencyAndAbort(+    const TypeDescriptor& type) {+  FOLLY_SAFE_FATAL("circular singleton dependency: ", type.name().c_str());+}++[[noreturn]] void singletonWarnCreateUnregisteredAndAbort(+    const TypeDescriptor& type) {+  FOLLY_SAFE_FATAL( // May happen before registrationComplete().+      "Creating instance for unregistered singleton: ",+      type.name().c_str());+}++[[noreturn]] void singletonWarnCreateBeforeRegistrationCompleteAndAbort(+    const TypeDescriptor& type) {+  FOLLY_SAFE_FATAL( // May happen before registrationComplete().+      fmt::format(+          fmt::runtime( //+              "Singleton {} requested before "+              "registrationComplete() call.\n"+              "This usually means that either main() never called "+              "folly::init, or singleton was requested before main() "+              "(which is not allowed)"),+          type.name())+          .c_str());+}++void singletonPrintDestructionStackTrace(const TypeDescriptor& type) {+  auto trace = symbolizer::getStackTraceStr();+  LOG(ERROR) << "Singleton " << type.name() << " was released.\n"+             << "Stacktrace:\n"+             << (!trace.empty() ? trace : "(not available)");+}++[[noreturn]] void singletonThrowNullCreator(const std::type_info& type) {+  auto const msg = fmt::format(+      FOLLY_FMT_COMPILE(+          "nullptr_t should be passed if you want {} to be default constructed"),+      folly::StringPiece(demangle(type)));+  throw std::logic_error(msg);+}++[[noreturn]] void singletonThrowGetInvokedAfterDestruction(+    const TypeDescriptor& type) {+  throw std::runtime_error(+      "Raw pointer to a singleton requested after its destruction."+      " Singleton type is: " ++      type.name());+}++} // namespace detail++namespace {++struct FatalHelper {+  ~FatalHelper() {+    if (!leakedSingletons_.empty()) {+      std::string leakedTypes;+      for (const auto& singleton : leakedSingletons_) {+        leakedTypes += "\t" + singleton.name() + "\n";+      }+      LOG(DFATAL)+          << "Singletons of the following types had living references "+          << "after destroyInstances was finished:\n"+          << leakedTypes+          << "beware! It is very likely that those singleton instances "+          << "are leaked.";+    }+  }++  std::vector<detail::TypeDescriptor> leakedSingletons_;+};++#if defined(__APPLE__) || defined(_MSC_VER)+// OS X doesn't support constructor priorities.+FatalHelper fatalHelper;+#else+FatalHelper __attribute__((__init_priority__(101))) fatalHelper;+#endif++} // namespace++SingletonVault::SingletonVault(Type type) noexcept : type_(type) {+  AtFork::registerHandler(+      this,+      /*prepare*/+      [this]() {+        auto singletons = singletons_.rlock();+        auto creationOrder = creationOrder_.rlock();++        CHECK_GE(singletons->size(), creationOrder->size());++        for (const auto& singletonType : *creationOrder) {+          liveSingletonsPreFork_.insert(singletons->at(singletonType));+        }++        return true;+      },+      /*parent*/ [this]() { liveSingletonsPreFork_.clear(); },+      /*child*/+      [this]() {+        for (auto singleton : liveSingletonsPreFork_) {+          singleton->inChildAfterFork();+        }+        liveSingletonsPreFork_.clear();+      });+}++SingletonVault::~SingletonVault() {+  AtFork::unregisterHandler(this);+  destroyInstances();+}++void SingletonVault::registerSingleton(detail::SingletonHolderBase* entry) {+  auto state = state_.rlock();+  state->check(detail::SingletonVaultState::Type::Running);++  if (FOLLY_UNLIKELY(state->registrationComplete) &&+      type_.load(std::memory_order_relaxed) == Type::Strict) {+    LOG(ERROR) << "Registering singleton after registrationComplete().";+  }++  auto singletons = singletons_.wlock();+  CHECK_THROW(+      singletons->emplace(entry->type(), entry).second, std::logic_error);+}++void SingletonVault::addEagerInitSingleton(detail::SingletonHolderBase* entry) {+  auto state = state_.rlock();+  state->check(detail::SingletonVaultState::Type::Running);++  if (FOLLY_UNLIKELY(state->registrationComplete) &&+      type_.load(std::memory_order_relaxed) == Type::Strict) {+    LOG(ERROR) << "Registering for eager-load after registrationComplete().";+  }++  CHECK_THROW(singletons_.rlock()->count(entry->type()), std::logic_error);++  auto eagerInitSingletons = eagerInitSingletons_.wlock();+  eagerInitSingletons->insert(entry);+}++void SingletonVault::addEagerInitOnReenableSingleton(+    detail::SingletonHolderBase* entry) {+  auto state = state_.rlock();+  state->check(detail::SingletonVaultState::Type::Running);++  if (FOLLY_UNLIKELY(state->registrationComplete) &&+      type_.load(std::memory_order_relaxed) == Type::Strict) {+    LOG(ERROR)+        << "Registering for eager-load on re-enable after registrationComplete().";+  }++  CHECK_THROW(singletons_.rlock()->count(entry->type()), std::logic_error);++  auto eagerInitOnReenableSingletons = eagerInitOnReenableSingletons_.wlock();+  eagerInitOnReenableSingletons->insert(entry);+}++void SingletonVault::registrationComplete() {+  scheduleDestroyInstances();++  auto state = state_.wlock();+  state->check(detail::SingletonVaultState::Type::Running);++  if (state->registrationComplete) {+    return;+  }++  auto singletons = singletons_.rlock();+  if (type_.load(std::memory_order_relaxed) == Type::Strict) {+    for (const auto& p : *singletons) {+      if (p.second->hasLiveInstance()) {+        throw std::runtime_error(+            "Singleton " + p.first.name() ++            " created before registration was complete.");+      }+    }+  }++  state->registrationComplete = true;+}++void SingletonVault::doEagerInit() {+  {+    auto state = state_.rlock();+    state->check(detail::SingletonVaultState::Type::Running);+    if (FOLLY_UNLIKELY(!state->registrationComplete)) {+      throw std::logic_error("registrationComplete() not yet called");+    }+  }++  auto eagerInitSingletons = eagerInitSingletons_.rlock();+  for (auto* single : *eagerInitSingletons) {+    single->createInstance();+  }+}++void SingletonVault::doEagerInitVia(Executor& exe, folly::Baton<>* done) {+  {+    auto state = state_.rlock();+    state->check(detail::SingletonVaultState::Type::Running);+    if (FOLLY_UNLIKELY(!state->registrationComplete)) {+      throw std::logic_error("registrationComplete() not yet called");+    }+  }++  auto eagerInitSingletons = eagerInitSingletons_.rlock();+  auto countdown =+      std::make_shared<std::atomic<size_t>>(eagerInitSingletons->size());+  for (auto* single : *eagerInitSingletons) {+    // countdown is retained by shared_ptr, and will be alive until last lambda+    // is done.  notifyBaton is provided by the caller, and expected to remain+    // present (if it's non-nullptr).  singletonSet can go out of scope but+    // its values, which are SingletonHolderBase pointers, are alive as long as+    // SingletonVault is not being destroyed.+    exe.add([=] {+      // decrement counter and notify if requested, whether initialization+      // was successful, was skipped (already initialized), or exception thrown.+      SCOPE_EXIT {+        if (--(*countdown) == 0) {+          if (done != nullptr) {+            done->post();+          }+        }+      };+      // if initialization is in progress in another thread, don't try to init+      // here.  Otherwise the current thread will block on 'createInstance'.+      if (!single->creationStarted()) {+        single->createInstance();+      }+    });+  }+}++void SingletonVault::destroyInstances() {+  cancellationSource_.wlock()->requestCancellation();++  auto stateW = state_.wlock();+  if (stateW->state == detail::SingletonVaultState::Type::Quiescing) {+    return;+  }+  stateW->state = detail::SingletonVaultState::Type::Quiescing;++  auto stateR = stateW.moveFromWriteToRead();+  {+    auto singletons = singletons_.rlock();+    auto creationOrder = creationOrder_.rlock();++    CHECK_GE(singletons->size(), creationOrder->size());++    // Release all ReadMostlyMainPtrs at once+    {+      ReadMostlyMainPtrDeleter<> deleter;+      for (auto& singleton_type : *creationOrder) {+        singletons->at(singleton_type)->preDestroyInstance(deleter);+      }+    }++    for (auto type_iter = creationOrder->rbegin();+         type_iter != creationOrder->rend();+         ++type_iter) {+      singletons->at(*type_iter)->destroyInstance();+    }++    for (auto& singleton_type : *creationOrder) {+      auto instance = singletons->at(singleton_type);+      if (!instance->hasLiveInstance()) {+        continue;+      }++      fatalHelper.leakedSingletons_.push_back(instance->type());+    }+  }++  {+    auto creationOrder = creationOrder_.wlock();+    creationOrder->clear();+  }+}++void SingletonVault::reenableInstances() {+  CHECK(!shutdownTimerStarted_.load(std::memory_order_relaxed))+      << "reenableInstances() called after destroyInstancesFinal()";+  {+    auto state = state_.wlock();++    state->check(detail::SingletonVaultState::Type::Quiescing);++    state->state = detail::SingletonVaultState::Type::Running;+  }++  // reset the cancellation source+  cancellationSource_.withWLock([&](auto& cancellationSource) {+    cancellationSource = folly::CancellationSource{};+  });++  auto eagerInitOnReenableSingletons = eagerInitOnReenableSingletons_.copy();+  auto instantiatedAtLeastOnce = instantiatedAtLeastOnce_.copy();+  for (auto* single : eagerInitOnReenableSingletons) {+    if (!instantiatedAtLeastOnce.count(single->type())) {+      continue;+    }+    single->createInstance();+  }+}++void SingletonVault::scheduleDestroyInstances() {+  // Add a dependency on folly::ThreadLocal to make sure all its static+  // singletons are initalized first.+  threadlocal_detail::StaticMeta<void, void>::instance();+#if !defined(FOLLY_SINGLETON_SKIP_SCHEDULE_ATEXIT) || \+    !FOLLY_SINGLETON_SKIP_SCHEDULE_ATEXIT+  std::atexit([] { SingletonVault::singleton()->destroyInstancesFinal(); });+#endif+}++void SingletonVault::destroyInstancesFinal() {+  startShutdownTimer();+  destroyInstances();+}++void SingletonVault::addToShutdownLog(std::string message) {+  std::chrono::time_point<std::chrono::system_clock> now =+      std::chrono::system_clock::now();+  std::chrono::milliseconds millis =+      std::chrono::duration_cast<std::chrono::milliseconds>(+          now.time_since_epoch());+  shutdownLog_.wlock()->push_back(fmt::format("{:%T} {}", millis, message));+}++#if FOLLY_HAVE_LIBRT+namespace {+[[noreturn]] void fireShutdownSignalHelper(sigval_t sigval) {+  static_cast<SingletonVault*>(sigval.sival_ptr)->fireShutdownTimer();+}+} // namespace+#endif++void SingletonVault::startShutdownTimer() {+#if FOLLY_HAVE_LIBRT+  if (shutdownTimerStarted_.exchange(true)) {+    return;+  }++  if (!shutdownTimeout_.count()) {+    return;+  }++  struct sigevent sig;+  sig.sigev_notify = SIGEV_THREAD;+  sig.sigev_notify_function = fireShutdownSignalHelper;+  sig.sigev_value.sival_ptr = this;+  sig.sigev_notify_attributes = nullptr;+  timer_t timerId;+  PCHECK(timer_create(CLOCK_MONOTONIC, &sig, &timerId) == 0);++  struct itimerspec newValue, oldValue;+  newValue.it_value.tv_sec =+      std::chrono::milliseconds(shutdownTimeout_).count() / 1000;+  newValue.it_value.tv_nsec =+      std::chrono::milliseconds(shutdownTimeout_).count() % 1000 * 1000000;+  newValue.it_interval.tv_sec = 0;+  newValue.it_interval.tv_nsec = 0;+  PCHECK(timer_settime(timerId, 0, &newValue, &oldValue) == 0);+#endif+}++[[noreturn]] void SingletonVault::fireShutdownTimer() {+  std::string shutdownLog;+  for (auto& logMessage : shutdownLog_.copy()) {+    shutdownLog += logMessage + "\n";+  }++  auto msg = folly::to<std::string>(+      "Failed to complete shutdown within ",+      std::chrono::milliseconds(shutdownTimeout_).count(),+      "ms. Shutdown log:\n",+      shutdownLog);+  folly::terminate_with<std::runtime_error>(msg);+}++} // namespace folly
+ folly/folly/Singleton.h view
@@ -0,0 +1,906 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++//+// Docs: https://fburl.com/fbcref_singleton+//++/// Recommended usage of this class: suppose you have a class+/// called `MyExpensiveService`, and you only want to construct one (ie,+/// it's a singleton), but you only want to construct it if it is used.+///+/// In your .h file:+///+///     class MyExpensiveService {+///       // Caution - may return a null ptr during startup and shutdown.+///       static std::shared_ptr<MyExpensiveService> getInstance();+///       ....+///     };+///+/// In your .cpp file:+///+///     namespace { struct PrivateTag {}; }+///+///     static folly::Singleton<MyExpensiveService, PrivateTag> the_singleton;+///+///     std::shared_ptr<MyExpensiveService> MyExpensiveService::getInstance() {+///       return the_singleton.try_get();+///     }+///+/// Code in other modules can access it via:+///+///     auto instance = MyExpensiveService::getInstance();+///+/// ### Advanced usage and notes+///+/// You can also access a singleton instance with+/// `Singleton<ObjectType, TagType>::try_get()`. We recommend+/// that you prefer the form `the_singleton.try_get()` because it ensures that+/// `the_singleton` is used and cannot be garbage-collected during linking: this+/// is necessary because the constructor of `the_singleton` is what registers it+/// to the SingletonVault.+///+/// The singleton will be created on demand.  If the constructor for+/// MyExpensiveService actually makes use of *another* Singleton, then+/// the right thing will happen -- that other singleton will complete+/// construction before get() returns.  However, in the event of a+/// circular dependency, a runtime error will occur.+///+/// You can have multiple singletons of the same underlying type, but+/// each must be given a unique tag. If no tag is specified a default tag is+/// used. We recommend that you use a tag from an anonymous namespace private to+/// your implementation file, as this ensures that the singleton is only+/// available via your interface and not also through `Singleton<T>::try_get()`+///+///     namespace {+///     struct Tag1 {};+///     struct Tag2 {};+///     folly::Singleton<MyExpensiveService> s_default;+///     folly::Singleton<MyExpensiveService, Tag1> s1;+///     folly::Singleton<MyExpensiveService, Tag2> s2;+///     }+///     ...+///     MyExpensiveService* svc_default = s_default.get();+///     MyExpensiveService* svc1 = s1.get();+///     MyExpensiveService* svc2 = s2.get();+///+/// By default, the singleton instance is constructed via new and+/// deleted via delete, but this is configurable:+///+///     namespace {+///     folly::Singleton<MyExpensiveService> the_singleton(create, destroy);+///     }+///+/// Where create and destroy are functions, `Singleton<T>::CreateFunc`+/// `Singleton<T>::TeardownFunc`.+///+/// For example, if you need to pass arguments to your class's constructor:+///+///     class X {+///      public:+///        X(int a1, std::string a2);+///      // ...+///     }+///+/// Make your singleton like this:+///+///     folly::Singleton<X> singleton_x([]() { return new X(42, "foo"); });+///+/// The above examples detail a situation where an expensive singleton is loaded+/// on-demand (thus only if needed).  However if there is an expensive singleton+/// that will likely be needed, and initialization takes a potentially long+/// time, e.g. while initializing, parsing some files, talking to remote+/// services, making uses of other singletons, and so on, the initialization of+/// those can be scheduled up front, or "eagerly".+///+/// In that case the singleton can be declared this way:+///+///     namespace {+///     auto the_singleton =+///         folly::Singleton<MyExpensiveService>(+///                 /* optional args, destroy args */)+///             .shouldEagerInit();+///     }+///+/// This way the singleton's instance is built at program initialization,+/// if the program opted-in to that feature by calling "doEagerInit" or+/// "doEagerInitVia" during its startup.+///+/// What if you need to destroy all of your singletons?  Say, some of+/// your singletons manage threads, but you need to fork?  Or your unit+/// test wants to clean up all global state?  Then you can call+/// `SingletonVault::singleton()->destroyInstances()`, which invokes the+/// TeardownFunc for each singleton, in the reverse order they were+/// created.  It is your responsibility to ensure your singletons can+/// handle cases where the singletons they depend on go away, however.+/// Singletons won't be recreated after destroyInstances call. If you+/// want to re-enable singleton creation (say after fork was called) you+/// should call reenableInstances.+///+/// NOTE: Calling `try_get()` **before**+/// `SingletonVault::registrationComplete()` has completed (i.e. before+/// `folly::init()` completed) is a logic error. In this case, `try_get()`+///  will abort via `singletonWarnCreateBeforeRegistrationCompleteAndAbort()`.+/// @class folly::Singleton++#pragma once++#include <folly/CancellationToken.h>+#include <folly/Exception.h>+#include <folly/Executor.h>+#include <folly/Memory.h>+#include <folly/Synchronized.h>+#include <folly/concurrency/CoreCachedSharedPtr.h>+#include <folly/concurrency/memory/ReadMostlySharedPtr.h>+#include <folly/detail/Singleton.h>+#include <folly/detail/StaticSingletonManager.h>+#include <folly/hash/Hash.h>+#include <folly/lang/Exception.h>+#include <folly/memory/SanitizeLeak.h>+#include <folly/synchronization/Baton.h>++#include <algorithm>+#include <atomic>+#include <condition_variable>+#include <functional>+#include <list>+#include <memory>+#include <mutex>+#include <string>+#include <thread>+#include <typeindex>+#include <typeinfo>+#include <unordered_map>+#include <unordered_set>+#include <vector>++#include <glog/logging.h>++namespace folly {++// For actual usage, please see the Singleton<T> class at the bottom+// of this file; that is what you will actually interact with.++/// SingletonVault - a library to manage the creation and destruction+/// of interdependent singletons.+///+/// SingletonVault is the class that manages singleton instances.  It+/// is unaware of the underlying types of singletons, and simply+/// manages lifecycles and invokes CreateFunc and TeardownFunc when+/// appropriate.  In general, you won't need to interact with the+/// SingletonVault itself.+///+/// A vault goes through a few stages of life:+///+///   1. Registration phase; singletons can be registered:+///      a) Strict: no singleton can be created in this stage.+///      b) Relaxed: singleton can be created (the default vault is Relaxed).+///   2. registrationComplete() has been called; singletons can no+///      longer be registered, but they can be created.+///   3. A vault can return to stage 1 when destroyInstances is called.+///+/// In general, you don't need to worry about any of the above; just+/// ensure registrationComplete() is called near the top of your main()+/// function, otherwise no singletons can be instantiated.+/// @class folly::SingletonVault+class SingletonVault;++namespace detail {++// A TypeDescriptor is the unique handle for a given singleton.  It is+// a combination of the type and of the optional name, and is used as+// a key in unordered_maps.+class TypeDescriptor {+ public:+  TypeDescriptor(const std::type_info& ti, const std::type_info& tag_ti)+      : ti_(ti), tag_ti_(tag_ti) {}++  TypeDescriptor(const TypeDescriptor& other)+      : ti_(other.ti_), tag_ti_(other.tag_ti_) {}++  TypeDescriptor& operator=(const TypeDescriptor& other) {+    if (this != &other) {+      ti_ = other.ti_;+      tag_ti_ = other.tag_ti_;+    }++    return *this;+  }++  std::string name() const;++  friend class TypeDescriptorHasher;++  bool operator==(const TypeDescriptor& other) const {+    return ti_ == other.ti_ && tag_ti_ == other.tag_ti_;+  }++ private:+  std::type_index ti_;+  std::type_index tag_ti_;+};++class TypeDescriptorHasher {+ public:+  size_t operator()(const TypeDescriptor& ti) const {+    return folly::hash::hash_combine(ti.ti_, ti.tag_ti_);+  }+};++[[noreturn]] void singletonWarnLeakyDoubleRegistrationAndAbort(+    const TypeDescriptor& type);++[[noreturn]] void singletonWarnLeakyInstantiatingNotRegisteredAndAbort(+    const TypeDescriptor& type);++[[noreturn]] void singletonWarnRegisterMockEarlyAndAbort(+    const TypeDescriptor& type);++void singletonWarnDestroyInstanceLeak(+    const TypeDescriptor& type, const void* ptr);++[[noreturn]] void singletonWarnCreateCircularDependencyAndAbort(+    const TypeDescriptor& type);++[[noreturn]] void singletonWarnCreateUnregisteredAndAbort(+    const TypeDescriptor& type);++[[noreturn]] void singletonWarnCreateBeforeRegistrationCompleteAndAbort(+    const TypeDescriptor& type);++void singletonPrintDestructionStackTrace(const TypeDescriptor& type);++[[noreturn]] void singletonThrowNullCreator(const std::type_info& type);++[[noreturn]] void singletonThrowGetInvokedAfterDestruction(+    const TypeDescriptor& type);++struct SingletonVaultState {+  // The two stages of life for a vault, as mentioned in the class comment.+  enum class Type {+    Running,+    Quiescing,+  };++  Type state{Type::Running};+  bool registrationComplete{false};++  // Each singleton in the vault can be in two states: dead+  // (registered but never created), living (CreateFunc returned an instance).++  void check(+      Type expected,+      const char* msg = "Unexpected singleton state change") const {+    if (expected != state) {+      throw_exception<std::logic_error>(msg);+    }+  }++  bool isDisabled() const { return state == Type::Quiescing; }+};++// This interface is used by SingletonVault to interact with SingletonHolders.+// Having a non-template interface allows SingletonVault to keep a list of all+// SingletonHolders.+class SingletonHolderBase {+ public:+  explicit SingletonHolderBase(TypeDescriptor typeDesc) noexcept+      : type_(typeDesc) {}+  virtual ~SingletonHolderBase() = default;++  TypeDescriptor type() const { return type_; }+  virtual bool hasLiveInstance() = 0;+  virtual void createInstance() = 0;+  virtual bool creationStarted() = 0;+  virtual void preDestroyInstance(ReadMostlyMainPtrDeleter<>&) = 0;+  virtual void destroyInstance() = 0;+  virtual void inChildAfterFork() = 0;++ private:+  TypeDescriptor type_;+};++// An actual instance of a singleton, tracking the instance itself,+// its state as described above, and the create and teardown+// functions.+template <typename T>+struct SingletonHolder : public SingletonHolderBase {+ public:+  typedef std::function<void(T*)> TeardownFunc;+  typedef std::function<T*(void)> CreateFunc;++  template <typename Tag, typename VaultTag>+  inline static SingletonHolder<T>& singleton();++  inline T* get();+  inline std::weak_ptr<T> get_weak();+  inline std::shared_ptr<T> try_get();+  inline folly::ReadMostlySharedPtr<T> try_get_fast();+  template <typename Func>+  inline invoke_result_t<Func, T*> apply(Func f);+  inline void vivify();++  void registerSingleton(CreateFunc c, TeardownFunc t);+  void registerSingletonMock(CreateFunc c, TeardownFunc t);+  bool hasLiveInstance() override;+  void createInstance() override;+  bool creationStarted() override;+  void preDestroyInstance(ReadMostlyMainPtrDeleter<>&) override;+  void destroyInstance() override;+  void inChildAfterFork() override;++ private:+  template <typename Tag, typename VaultTag>+  struct Impl;++  SingletonHolder(TypeDescriptor type, SingletonVault& vault) noexcept;++  enum class SingletonHolderState {+    NotRegistered,+    Dead,+    Living,+    LivingInChildAfterFork,+  };++  SingletonVault& vault_;++  // mutex protects the entire entry during construction/destruction+  std::mutex mutex_;++  // State of the singleton entry. If state is Living, instance_ptr and+  // instance_weak can be safely accessed w/o synchronization.+  std::atomic<SingletonHolderState> state_{SingletonHolderState::NotRegistered};++  // the thread creating the singleton (only valid while creating an object)+  std::atomic<std::thread::id> creating_thread_{};++  // The singleton itself and related functions.++  // holds a ReadMostlyMainPtr to singleton instance, set when state is changed+  // from Dead to Living. Reset when state is changed from Living to Dead.+  folly::ReadMostlyMainPtr<T> instance_;+  // used to release all ReadMostlyMainPtrs at once+  folly::ReadMostlySharedPtr<T> instance_copy_;+  // per-core shared_ptrs that in turn hold the instance shared_ptr, to avoid+  // contention in acquiring them.+  folly::CoreCachedSharedPtr<T> instance_core_cached_;+  // weak references to the previous pointers. These are never written to after+  // initialization, so they're safe to read without synchronization once the+  // state has transitioned to Living.+  // instance_weak_ is a reference to the main instance, so it is authoritative+  // on whether the instance is expired.+  std::weak_ptr<T> instance_weak_;+  folly::ReadMostlyWeakPtr<T> instance_weak_fast_;+  folly::CoreCachedWeakPtr<T> instance_weak_core_cached_;++  // Time we wait on destroy_baton after releasing Singleton shared_ptr.+  std::shared_ptr<folly::Baton<>> destroy_baton_;+  T* instance_ptr_ = nullptr;+  CreateFunc create_ = nullptr;+  TeardownFunc teardown_ = nullptr;++  std::shared_ptr<std::atomic<bool>> print_destructor_stack_trace_;++  SingletonHolder(const SingletonHolder&) = delete;+  SingletonHolder& operator=(const SingletonHolder&) = delete;+  SingletonHolder& operator=(SingletonHolder&&) = delete;+  SingletonHolder(SingletonHolder&&) = delete;+};++} // namespace detail++class SingletonVault {+ public:+  enum class Type {+    Strict, // Singletons can't be created before registrationComplete()+    Relaxed, // Singletons can be created before registrationComplete()+  };++  /**+   * Clears all singletons in the given vault at ctor and dtor times.+   * Useful for unit-tests that need to clear the world.+   *+   * This need can arise when a unit-test needs to swap out an object used by a+   * singleton for a test-double, but the singleton needing its dependency to be+   * swapped has a type or a tag local to some other translation unit and+   * unavailable in the current translation unit.+   *+   * Other, better approaches to this need are "plz 2 refactor" ....+   */+  struct ScopedExpunger {+    SingletonVault* vault;+    explicit ScopedExpunger(SingletonVault* v) : vault(v) { expunge(); }+    ~ScopedExpunger() { expunge(); }+    void expunge() {+      vault->destroyInstances();+      vault->reenableInstances();+    }+  };++  static Type defaultVaultType();++  explicit SingletonVault(Type type = defaultVaultType()) noexcept;++  // Destructor is only called by unit tests to check destroyInstances.+  ~SingletonVault();++  typedef std::function<void(void*)> TeardownFunc;+  typedef std::function<void*(void)> CreateFunc;++  // Ensure that Singleton has not been registered previously and that+  // registration is not complete. If validations succeeds,+  // register a singleton of a given type with the create and teardown+  // functions.+  void registerSingleton(detail::SingletonHolderBase* entry);++  /**+   * Called by `Singleton<T>.shouldEagerInit()` to ensure the instance+   * is built when `doEagerInit[Via]` is called; see those methods+   * for more info.+   */+  void addEagerInitSingleton(detail::SingletonHolderBase* entry);++  void addEagerInitOnReenableSingleton(detail::SingletonHolderBase* entry);++  // Mark registration is complete; no more singletons can be+  // registered at this point.+  void registrationComplete();++  /**+   * Initialize all singletons which were marked as eager-initialized+   * (using `shouldEagerInit()`).  No return value.  Propagates exceptions+   * from constructors / create functions, as is the usual case when calling+   * for example `Singleton<Foo>::get_weak()`.+   */+  void doEagerInit();++  /**+   * Schedule eager singletons' initializations through the given executor.+   * If baton ptr is not null, its `post` method is called after all+   * early initialization has completed.+   *+   * If exceptions are thrown during initialization, this method will still+   * `post` the baton to indicate completion.  The exception will not propagate+   * and future attempts to `try_get` or `get_weak` the failed singleton will+   * retry initialization.+   *+   * Sample usage:+   *+   *   folly::IOThreadPoolExecutor executor(max_concurrency_level);+   *   folly::Baton<> done;+   *   doEagerInitVia(executor, &done);+   *   done.wait();  // or 'try_wait_for', etc.+   *+   */+  void doEagerInitVia(Executor& exe, folly::Baton<>* done = nullptr);++  // Destroy all singletons; when complete, the vault can't create+  // singletons once again until reenableInstances() is called.+  // If reenableInstances() will not be called, destroyInstancesFinal()+  // should be used instead.+  void destroyInstances();++  // Enable re-creating singletons after destroyInstances() was called.+  void reenableInstances();++  // Same as destroyInstances() but reenableInstances() should not be called+  // after it. Starts a shutdown timer.+  void destroyInstancesFinal();++  // For testing; how many registered and living singletons we have.+  size_t registeredSingletonCount() const {+    return singletons_.rlock()->size();+  }++  /**+   * Flips to true if eager initialization was used, and has completed.+   * Never set to true if "doEagerInit()" or "doEagerInitVia" never called.+   */+  bool eagerInitComplete() const;++  size_t livingSingletonCount() const {+    auto state = state_.rlock();+    auto singletons = singletons_.rlock();++    size_t ret = 0;+    for (const auto& p : *singletons) {+      if (p.second->hasLiveInstance()) {+        ++ret;+      }+    }++    return ret;+  }++  // A well-known vault; you can actually have others, but this is the+  // default.+  static SingletonVault* singleton() { return singleton<>(); }++  // Gets singleton vault for any Tag. Non-default tag should be used in unit+  // tests only.+  template <typename VaultTag = detail::DefaultTag>+  static SingletonVault* singleton() {+    return &detail::createGlobal<SingletonVault, VaultTag>();+  }++  // Get a cancellation token that gets triggered when singleton destruction+  // starts.+  //+  // The underlying cancellation source gets reset when reenableInstances() is+  // called.+  folly::CancellationToken getDestructionCancellationToken() {+    return cancellationSource_.wlock()->getToken();+  }++  void setType(Type type) { type_.store(type, std::memory_order_relaxed); }++  void setShutdownTimeout(std::chrono::milliseconds shutdownTimeout) {+    shutdownTimeout_ = shutdownTimeout;+  }++  void disableShutdownTimeout() {+    shutdownTimeout_ = std::chrono::milliseconds::zero();+  }++  void addToShutdownLog(std::string message);++  [[noreturn]] void fireShutdownTimer();++  void setFailOnUseAfterFork(bool failOnUseAfterFork) {+    failOnUseAfterFork_ = failOnUseAfterFork;+  }++  bool isDisabled() const { return state_.rlock()->isDisabled(); }++ private:+  template <typename T>+  friend struct detail::SingletonHolder;++  // This method only matters if registrationComplete() is never called.+  // Otherwise destroyInstances is scheduled to be executed atexit.+  //+  // Initializes static object, which calls destroyInstances on destruction.+  // Used to have better deletion ordering with singleton not managed by+  // folly::Singleton. The desruction will happen in the following order:+  // 1. Singletons, not managed by folly::Singleton, which were created after+  //    any of the singletons managed by folly::Singleton was requested.+  // 2. All singletons managed by folly::Singleton+  // 3. Singletons, not managed by folly::Singleton, which were created before+  //    any of the singletons managed by folly::Singleton was requested.+  static void scheduleDestroyInstances();++  void startShutdownTimer();++  typedef std::unordered_map<+      detail::TypeDescriptor,+      detail::SingletonHolderBase*,+      detail::TypeDescriptorHasher>+      SingletonMap;++  // Use SharedMutexSuppressTSAN to suppress noisy lock inversions when building+  // with TSAN. If TSAN is not enabled, SharedMutexSuppressTSAN is equivalent+  // to a normal SharedMutex.+  Synchronized<SingletonMap, SharedMutexSuppressTSAN> singletons_;+  Synchronized<+      std::unordered_set<detail::SingletonHolderBase*>,+      SharedMutexSuppressTSAN>+      eagerInitSingletons_;+  Synchronized<+      std::unordered_set<detail::SingletonHolderBase*>,+      SharedMutexSuppressTSAN>+      eagerInitOnReenableSingletons_;+  Synchronized<std::vector<detail::TypeDescriptor>, SharedMutexSuppressTSAN>+      creationOrder_;+  Synchronized<+      std::unordered_set<detail::TypeDescriptor, detail::TypeDescriptorHasher>,+      SharedMutexSuppressTSAN>+      instantiatedAtLeastOnce_;+  std::unordered_set<detail::SingletonHolderBase*> liveSingletonsPreFork_;++  // Using SharedMutexReadPriority is important here, because we want to make+  // sure we don't block nested singleton creation happening concurrently with+  // destroyInstances().+  Synchronized<detail::SingletonVaultState, SharedMutexReadPriority> state_;++  std::atomic<Type> type_;++  std::atomic<bool> shutdownTimerStarted_{false};+  std::chrono::milliseconds shutdownTimeout_{std::chrono::minutes{5}};+  Synchronized<std::vector<std::string>> shutdownLog_;+  // We use a lock around CancellationSource to get the guarantee that all+  // cancellation callbacks that got triggered on requestCancellation() are done+  // executing by the time we start destruction.  This prevents silent callbacks+  // that take long to block destruction.+  folly::Synchronized<CancellationSource> cancellationSource_;+  bool failOnUseAfterFork_{true};+};++/**+ * Singleton allows for simple access to registering and instantiating+ * singletons.  Create instances of this class in the global scope of+ * type Singleton<T> to register your singleton for later access via+ * Singleton<T>::try_get().+ *+ * There are many supporting libraries and classes for Singleton; this is the+ * one that users typically interact with.+ */+template <+    typename T,+    typename Tag = detail::DefaultTag,+    typename VaultTag = detail::DefaultTag /* for testing */>+class Singleton {+ public:+  typedef std::function<T*(void)> CreateFunc;+  typedef std::function<void(T*)> TeardownFunc;++  /**+   * Get a pointer to the singleton.+   *+   * It is preferable to call get() repeatedly than to store the returned+   * pointer. Accessing the returned pointer often fails during shutdown.+   *+   * Deprecated in favor of try_get().+   */+  [[deprecated("Replaced by try_get")]] static T* get() {+    return getEntry().get();+  }++  /**+   * Get a weak_ptr to the singleton.+   *+   * If you cannot lock the weak_ptr, this usually means the vault has been+   * destroyed.+   *+   * Deprecated in favor of try_get().+   */+  [[deprecated("Replaced by try_get")]] static std::weak_ptr<T> get_weak() {+    return getEntry().get_weak();+  }++  /**+   * Get a shared_ptr to the singleton.+   *+   * It is recommended to call try_get() repeatedly, rather than storing the+   * shared_ptr, because storing the shared_ptr prevents the singleton from+   * being destroyed during shutdown.+   */+  static std::shared_ptr<T> try_get() { return getEntry().try_get(); }++  /**+   * Get a ReadMostlySharedPtr to the singleton.+   *+   * It is recommended to call try_get_fast() repeatedly, rather than storing+   * the ReadMostlySharedPtr, because storing the ReadMostlySharedPtr prevents+   * the singleton from being destroyed during shutdown.+   */+  static folly::ReadMostlySharedPtr<T> try_get_fast() {+    return getEntry().try_get_fast();+  }++  /**+   * Applies a callback to the possibly-nullptr singleton instance, returning+   * the callback's result.+   *+   * That is, the following two are functionally equivalent:+   *    singleton.apply(std::ref(f));+   *    f(singleton.try_get().get());+   *+   * For example, the following returns the singleton+   * instance directly without any extra operations on the instance:+   * auto ret = Singleton<T>::apply([](auto* v) { return v; });+   */+  template <typename Func>+  static invoke_result_t<Func, T*> apply(Func f) {+    return getEntry().apply(std::ref(f));+  }++  /// Ensure the instance exists.+  static void vivify() { getEntry().vivify(); }++  /**+   * Create a singleton, which uses the default-constructor for its wrapped+   * type.+   *+   * @param t  The teardown function to use (in lieu of delete).+   */+  explicit Singleton(+      std::nullptr_t /* _ */ = nullptr,+      typename Singleton::TeardownFunc t = nullptr)+      : Singleton([]() { return new T; }, std::move(t)) {}++  /**+   * @param c  The create function to use (in lieu of new).+   * @param t  The teardown function to use (in lieu of delete).+   */+  explicit Singleton(+      typename Singleton::CreateFunc c,+      typename Singleton::TeardownFunc t = nullptr) {+    if (c == nullptr) {+      detail::singletonThrowNullCreator(typeid(T));+    }++    auto vault = SingletonVault::singleton<VaultTag>();+    getEntry().registerSingleton(std::move(c), getTeardownFunc(std::move(t)));+    vault->registerSingleton(&getEntry());+  }++  /**+   * Specify that the singleton should be eagerly initialized.+   *+   * Should be instantiated as soon as "doEagerInit[Via]" is called.+   * Singletons are usually lazy-loaded (built on-demand) but for those which+   * are known to be needed, to avoid the potential lag for objects that take+   * long to construct during runtime, there is an option to make sure these+   * are built up-front.+   *+   * Use like:+   *   auto gFooInstance = Singleton<Foo>(...).shouldEagerInit();+   *+   * Or alternately, define the singleton as usual, and say+   *   gFooInstance.shouldEagerInit();+   *+   * at some point prior to calling registrationComplete().+   * Then doEagerInit() or doEagerInitVia(Executor*) can be called.+   */+  Singleton& shouldEagerInit() {+    auto vault = SingletonVault::singleton<VaultTag>();+    vault->addEagerInitSingleton(&getEntry());+    return *this;+  }++  /**+   * Specify that the singleton should be eagerly initialized when singletons+   * reenable.+   *+   * Should be re-instantiated as soon as reenableInstances() is called.+   * Note that it will be re-instantiated only if it was instantiated before+   * destroyInstances() call.+   *+   * Use like:+   *   auto gFooInstance = Singleton<Foo>(...).shouldEagerInitOnReenable();+   */+  Singleton& shouldEagerInitOnReenable() {+    auto vault = SingletonVault::singleton<VaultTag>();+    vault->addEagerInitOnReenableSingleton(&getEntry());+    return *this;+  }++  /**+   * Inject a mock singleton, for testing.+   *+   * Construct and inject a mock singleton which should be used only from tests.+   * Unlike regular singletons which are initialized once per process lifetime,+   * mock singletons live for the duration of a test. This means that one+   * process running multiple tests can initialize and register the same+   * singleton multiple times. This functionality should be used only from tests+   * since it relaxes validation and performance in order to be able to perform+   * the injection. The returned mock singleton is functionality identical to+   * regular singletons.+   */+  static void make_mock(+      std::nullptr_t /* c */ = nullptr,+      typename Singleton<T>::TeardownFunc t = nullptr) {+    make_mock([]() { return new T; }, t);+  }++  static void make_mock(+      CreateFunc c, typename Singleton<T>::TeardownFunc t = nullptr) {+    if (c == nullptr) {+      detail::singletonThrowNullCreator(typeid(T));+    }++    auto& entry = getEntry();++    entry.registerSingletonMock(c, getTeardownFunc(t));+  }++ private:+  inline static detail::SingletonHolder<T>& getEntry() {+    return detail::SingletonHolder<T>::template singleton<Tag, VaultTag>();+  }++  // Construct TeardownFunc.+  static typename detail::SingletonHolder<T>::TeardownFunc getTeardownFunc(+      TeardownFunc t) {+    if (t == nullptr) {+      return [](T* v) { delete v; };+    } else {+      return t;+    }+  }+};++template <typename T, typename Tag = detail::DefaultTag>+class LeakySingleton {+ public:+  using CreateFunc = std::function<T*()>;++  LeakySingleton() : LeakySingleton([] { return new T(); }) {}++  explicit LeakySingleton(CreateFunc createFunc) {+    auto& entry = entryInstance();+    if (entry.state != State::NotRegistered) {+      detail::singletonWarnLeakyDoubleRegistrationAndAbort(entry.type_);+    }+    entry.createFunc = createFunc;+    entry.state = State::Dead;+  }++  static T& get() { return instance(); }++  static void make_mock(std::nullptr_t /* c */ = nullptr) {+    make_mock([]() { return new T; });+  }++  static void make_mock(CreateFunc createFunc) {+    if (createFunc == nullptr) {+      detail::singletonThrowNullCreator(typeid(T));+    }++    auto& entry = entryInstance();+    std::lock_guard lg(entry.mutex);+    if (entry.ptr) {+      lsan_ignore_object(std::atomic_exchange(&entry.ptr, (T*)nullptr));+    }+    entry.createFunc = createFunc;+    entry.state = State::Dead;+  }++ private:+  enum class State { NotRegistered, Dead, Living };++  struct Entry {+    Entry() noexcept {}+    Entry(const Entry&) = delete;+    Entry& operator=(const Entry&) = delete;++    std::atomic<State> state{State::NotRegistered};+    std::atomic<T*> ptr{nullptr};+    CreateFunc createFunc;+    std::mutex mutex;+    detail::TypeDescriptor type_{typeid(T), typeid(Tag)};+  };++  static Entry& entryInstance() { return detail::createGlobal<Entry, Tag>(); }++  static T& instance() {+    auto& entry = entryInstance();+    if (FOLLY_UNLIKELY(entry.state != State::Living)) {+      createInstance();+    }++    return *entry.ptr;+  }++  static void createInstance() {+    auto& entry = entryInstance();++    std::lock_guard lg(entry.mutex);+    if (entry.state == State::Living) {+      return;+    }++    if (entry.state == State::NotRegistered) {+      detail::singletonWarnLeakyInstantiatingNotRegisteredAndAbort(entry.type_);+    }++    entry.ptr = entry.createFunc();+    entry.state = State::Living;+  }+};+} // namespace folly++#include <folly/Singleton-inl.h>
+ folly/folly/SingletonThreadLocal.cpp view
@@ -0,0 +1,53 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#include <folly/SingletonThreadLocal.h>++namespace folly {++namespace detail {++FOLLY_NOINLINE SingletonThreadLocalState::Tracking::Tracking() noexcept {}++FOLLY_NOINLINE SingletonThreadLocalState::Tracking::~Tracking() {+  for (auto& kvp : caches) {+    kvp.first->object = nullptr;+  }+}++FOLLY_NOINLINE void SingletonThreadLocalState::LocalLifetime::destroy(+    Tracking& tracking) noexcept {+  auto& lifetimes = tracking.lifetimes[this];+  for (auto cache : lifetimes) {+    auto const it = tracking.caches.find(cache);+    if (!--it->second) {+      tracking.caches.erase(it);+      cache->object = nullptr;+    }+  }+  tracking.lifetimes.erase(this);+}++FOLLY_NOINLINE void SingletonThreadLocalState::LocalLifetime::track(+    LocalCache& cache, Tracking& tracking, void* object) noexcept {+  cache.object = object;+  auto const inserted = tracking.lifetimes[this].insert(&cache);+  tracking.caches[&cache] += inserted.second;+}++} // namespace detail++} // namespace folly
+ folly/folly/SingletonThreadLocal.h view
@@ -0,0 +1,263 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <thread>+#include <type_traits>+#include <unordered_map>+#include <unordered_set>++#include <folly/ScopeGuard.h>+#include <folly/ThreadLocal.h>+#include <folly/detail/Iterators.h>+#include <folly/detail/Singleton.h>+#include <folly/detail/UniqueInstance.h>+#include <folly/functional/Invoke.h>+#include <folly/lang/Hint.h>++namespace folly {++namespace detail {++struct SingletonThreadLocalState {+  struct LocalCache {+    void* object; // type-erased pointer to the object field of wrapper, below+  };+  static_assert( // pod avoids tls-init guard var and tls-fini ub use-after-dtor+      std::is_standard_layout<LocalCache>::value &&+          std::is_trivial<LocalCache>::value,+      "non-pod");++  struct LocalLifetime;++  struct Tracking {+    using LocalCacheSet = std::unordered_set<LocalCache*>;++    // per-cache refcounts, the number of lifetimes tracking that cache+    std::unordered_map<LocalCache*, size_t> caches;++    // per-lifetime cache tracking; 1-M lifetimes may track 1-N caches+    std::unordered_map<LocalLifetime*, LocalCacheSet> lifetimes;++    Tracking() noexcept;+    ~Tracking();+  };++  struct LocalLifetime {+    void destroy(Tracking& tracking) noexcept;+    void track(LocalCache& cache, Tracking& tracking, void* object) noexcept;+  };+};++} // namespace detail++/// SingletonThreadLocal+///+/// Useful for a per-thread leaky-singleton model in libraries and applications.+///+/// By "leaky" it is meant that the T instances held by the instantiation+/// SingletonThreadLocal<T> will survive until their owning thread exits.+/// Therefore, they can safely be used before main() begins and after main()+/// ends, and they can also safely be used in an application that spawns many+/// temporary threads throughout its life.+///+/// Example:+///+///   struct UsefulButHasExpensiveCtor {+///     UsefulButHasExpensiveCtor(); // this is expensive+///     Result operator()(Arg arg);+///   };+///+///   Result useful(Arg arg) {+///     using Useful = UsefulButHasExpensiveCtor;+///     auto& useful = folly::SingletonThreadLocal<Useful>::get();+///     return useful(arg);+///   }+///+/// As an example use-case, the random generators in <random> are expensive to+/// construct. And their constructors are deterministic, but many cases require+/// that they be randomly seeded. So folly::Random makes good canonical uses of+/// folly::SingletonThreadLocal so that a seed is computed from the secure+/// random device once per thread, and the random generator is constructed with+/// the seed once per thread.+///+/// Keywords to help people find this class in search:+/// Thread Local Singleton ThreadLocalSingleton+template <+    typename T,+    typename Tag = detail::DefaultTag,+    typename Make = detail::DefaultMake<T>,+    typename TLTag = std::+        conditional_t<std::is_same<Tag, detail::DefaultTag>::value, void, Tag>>+class SingletonThreadLocal {+ private:+  static detail::UniqueInstance unique;++  using State = detail::SingletonThreadLocalState;+  using LocalCache = State::LocalCache;++  using Object = invoke_result_t<Make>;+  static_assert(std::is_convertible<Object&, T&>::value, "inconvertible");++  struct ObjectWrapper {+    // keep as first field in first base, to save 1 instr in the fast path+    Object object{Make{}()};+  };+  struct Wrapper : ObjectWrapper, State::Tracking {+    /* implicit */ operator T&() { return ObjectWrapper::object; }+  };++  using WrapperTL = ThreadLocal<Wrapper, TLTag>;++  struct LocalLifetime : State::LocalLifetime {+    ~LocalLifetime() { destroy(getWrapper()); }+  };++  SingletonThreadLocal() = delete;++  FOLLY_ALWAYS_INLINE static WrapperTL& getWrapperTL() {+    (void)unique; // force the object not to be thrown out as unused+    return detail::createGlobal<WrapperTL, Tag>();+  }++  FOLLY_NOINLINE static Wrapper& getWrapper() { return *getWrapperTL(); }++  FOLLY_NOINLINE static Wrapper& getSlow(LocalCache& cache) {+    auto& wrapper = getWrapper();+    if (threadlocal_detail::StaticMetaBase::dying()) {+      return wrapper;+    }+    static thread_local LocalLifetime lifetime;+    lifetime.track(cache, wrapper, &wrapper.object); // idempotent+    return wrapper;+  }++ public:+  FOLLY_EXPORT FOLLY_ALWAYS_INLINE static T& get() {+    if (kIsMobile) {+      return getWrapper();+    }+    static thread_local LocalCache cache;+    auto* object = static_cast<Object*>(cache.object);+    return FOLLY_LIKELY(!!object) ? *object : getSlow(cache).object;+  }++  static T* try_get() {+    auto* wrapper = getWrapperTL().get_existing();+    return wrapper ? &static_cast<T&>(*wrapper) : nullptr;+  }++  class Accessor {+   private:+    using Inner = typename WrapperTL::Accessor;+    using IteratorBase = typename Inner::Iterator;+    using IteratorTag = typename IteratorBase::iterator_category;++    Inner inner_;++    explicit Accessor(Inner inner) noexcept : inner_(std::move(inner)) {}++   public:+    friend class SingletonThreadLocal<T, Tag, Make, TLTag>;++    class Iterator+        : public detail::+              IteratorAdaptor<Iterator, IteratorBase, T, IteratorTag> {+     private:+      using Super =+          detail::IteratorAdaptor<Iterator, IteratorBase, T, IteratorTag>;+      using Super::Super;++     public:+      friend class Accessor;++      T& dereference() const {+        return const_cast<Iterator*>(this)->base()->object;+      }++      std::thread::id getThreadId() const { return this->base().getThreadId(); }++      uint64_t getOSThreadId() const { return this->base().getOSThreadId(); }+    };++    Accessor(const Accessor&) = delete;+    Accessor& operator=(const Accessor&) = delete;+    Accessor(Accessor&&) = default;+    Accessor& operator=(Accessor&&) = default;++    Iterator begin() const { return Iterator(inner_.begin()); }++    Iterator end() const { return Iterator(inner_.end()); }+  };++  // Must use a unique Tag, takes a lock that is one per Tag+  static Accessor accessAllThreads() {+    return Accessor(getWrapperTL().accessAllThreads());+  }+};++FOLLY_PUSH_WARNING+FOLLY_CLANG_DISABLE_WARNING("-Wglobal-constructors")+template <typename T, typename Tag, typename Make, typename TLTag>+detail::UniqueInstance SingletonThreadLocal<T, Tag, Make, TLTag>::unique{+    tag<SingletonThreadLocal>, tag<T, Tag>, tag<Make, TLTag>};+FOLLY_POP_WARNING++} // namespace folly++/// FOLLY_DECLARE_REUSED+///+/// Useful for local variables of container types, where it is desired to avoid+/// the overhead associated with the local variable entering and leaving scope.+/// Rather, where it is desired that the memory be reused between invocations+/// of the same scope in the same thread rather than deallocated and reallocated+/// between invocations of the same scope in the same thread. Note that the+/// container will always be cleared between invocations; it is only the backing+/// memory allocation which is reused.+///+/// Example:+///+///   void traverse_perform(int root);+///   template <typename F>+///   void traverse_each_child_r(int root, F const&);+///   void traverse_depthwise(int root) {+///     // preserves some of the memory backing these per-thread data structures+///     FOLLY_DECLARE_REUSED(seen, std::unordered_set<int>);+///     FOLLY_DECLARE_REUSED(work, std::vector<int>);+///     // example algorithm that uses these per-thread data structures+///     work.push_back(root);+///     while (!work.empty()) {+///       root = work.back();+///       work.pop_back();+///       seen.insert(root);+///       traverse_perform(root);+///       traverse_each_child_r(root, [&](int item) {+///         if (!seen.count(item)) {+///           work.push_back(item);+///         }+///       });+///     }+///   }+#define FOLLY_DECLARE_REUSED(name, ...)                                        \+  struct __folly_reused_type_##name {                                          \+    __VA_ARGS__ object;                                                        \+  };                                                                           \+  [[maybe_unused]] ::folly::unsafe_for_async_usage                             \+      __folly_reused_g_prevent_async_##name;                                   \+  auto& name =                                                                 \+      ::folly::SingletonThreadLocal<__folly_reused_type_##name>::get().object; \+  auto __folly_reused_g_##name = ::folly::makeGuard([&] { name.clear(); })
+ folly/folly/SocketAddress.cpp view
@@ -0,0 +1,912 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#ifndef __STDC_FORMAT_MACROS+#define __STDC_FORMAT_MACROS+#endif++#include <folly/SocketAddress.h>++#include <cassert>+#include <cerrno>+#include <cstdio>+#include <cstring>+#include <sstream>+#include <string>+#include <system_error>+#include <type_traits>++#include <boost/functional/hash.hpp>++#include <fmt/core.h>++#include <folly/Exception.h>+#include <folly/hash/Hash.h>+#include <folly/net/NetOps.h>+#include <folly/net/NetworkSocket.h>++namespace {++/**+ * A structure to free a struct addrinfo when it goes out of scope.+ */+struct ScopedAddrInfo {+  explicit ScopedAddrInfo(struct addrinfo* addrinfo) : info(addrinfo) {}+  ~ScopedAddrInfo() { freeaddrinfo(info); }++  struct addrinfo* info;+};++/**+ * A simple data structure for parsing a host-and-port string.+ *+ * Accepts a string of the form "<host>:<port>" or just "<port>",+ * and contains two string pointers to the host and the port portion of the+ * string.+ *+ * The HostAndPort may contain pointers into the original string.  It is+ * responsible for the user to ensure that the input string is valid for the+ * lifetime of the HostAndPort structure.+ */+struct HostAndPort {+  HostAndPort(const char* str, bool hostRequired)+      : host(nullptr), port(nullptr), allocated(nullptr) {+    // Look for the last colon+    const char* colon = strrchr(str, ':');+    if (colon == nullptr) {+      // No colon, just a port number.+      if (hostRequired) {+        throw std::invalid_argument(+            "expected a host and port string of the "+            "form \"<host>:<port>\"");+      }+      port = str;+      return;+    }++    // We have to make a copy of the string so we can modify it+    // and change the colon to a NUL terminator.+    allocated = strdup(str);+    if (!allocated) {+      throw std::bad_alloc();+    }++    char* allocatedColon = allocated + (colon - str);+    *allocatedColon = '\0';+    host = allocated;+    port = allocatedColon + 1;+    // bracketed IPv6 address, remove the brackets+    // allocatedColon[-1] is fine, as allocatedColon >= host and+    // *allocatedColon != *host therefore allocatedColon > host+    if (*host == '[' && allocatedColon[-1] == ']') {+      allocatedColon[-1] = '\0';+      ++host;+    }+  }++  ~HostAndPort() { free(allocated); }++  const char* host;+  const char* port;+  char* allocated;+};++struct GetAddrInfoError {+#ifdef _WIN32+  std::string error;+  const char* str() const { return error.c_str(); }+  explicit GetAddrInfoError(int errorCode) {+    auto s = gai_strerror(errorCode);+    using Char = std::remove_reference_t<decltype(*s)>;+    error.assign(s, s + std::char_traits<Char>::length(s));+  }+#else+  const char* error;+  const char* str() const { return error ? error : "Unknown error"; }+  explicit GetAddrInfoError(int errorCode) : error(gai_strerror(errorCode)) {}+#endif+};++} // namespace++namespace folly {++bool SocketAddress::isPrivateAddress() const {+  if (holdsInet()) {+    return std::get<IPAddr>(storage_).ip.isPrivate();+  } else {+    // Unix and vsock addresses are always local to a host.  Return true,+    // since this conforms to the semantics of returning true for IP loopback+    // addresses.+    return true;+  }+}++bool SocketAddress::isLoopbackAddress() const {+  if (holdsInet()) {+    return std::get<IPAddr>(storage_).ip.isLoopback();+#if FOLLY_HAVE_VSOCK+  } else if (holdsVsock()) {+    // VSOCK addresses with CID_LOCAL are considered loopback+    const auto& vsockAddr = std::get<VsockAddr>(storage_);+    return vsockAddr.cid == VMADDR_CID_LOCAL;+#endif+  } else {+    // Return true for UNIX addresses, since they are always local to a host.+    return true;+  }+}++void SocketAddress::setFromHostPort(const char* host, uint16_t port) {+  ScopedAddrInfo results(getAddrInfo(host, port, 0));+  setFromAddrInfo(results.info);+}++void SocketAddress::setFromIpPort(const char* ip, uint16_t port) {+  ScopedAddrInfo results(getAddrInfo(ip, port, AI_NUMERICHOST));+  setFromAddrInfo(results.info);+}++void SocketAddress::setFromIpAddrPort(const IPAddress& ipAddr, uint16_t port) {+  storage_ = IPAddr(ipAddr, port);+}++void SocketAddress::setFromLocalPort(uint16_t port) {+  ScopedAddrInfo results(getAddrInfo(nullptr, port, AI_ADDRCONFIG));+  setFromLocalAddr(results.info);+}++void SocketAddress::setFromLocalPort(const char* port) {+  ScopedAddrInfo results(getAddrInfo(nullptr, port, AI_ADDRCONFIG));+  setFromLocalAddr(results.info);+}++void SocketAddress::setFromLocalIpPort(const char* addressAndPort) {+  HostAndPort hp(addressAndPort, false);+  ScopedAddrInfo results(+      getAddrInfo(hp.host, hp.port, AI_NUMERICHOST | AI_ADDRCONFIG));+  setFromLocalAddr(results.info);+}++void SocketAddress::setFromIpPort(const char* addressAndPort) {+  HostAndPort hp(addressAndPort, true);+  ScopedAddrInfo results(getAddrInfo(hp.host, hp.port, AI_NUMERICHOST));+  setFromAddrInfo(results.info);+}++void SocketAddress::setFromHostPort(const char* hostAndPort) {+  HostAndPort hp(hostAndPort, true);+  ScopedAddrInfo results(getAddrInfo(hp.host, hp.port, 0));+  setFromAddrInfo(results.info);+}++#if FOLLY_HAVE_VSOCK+void SocketAddress::setFromVsockCIDPort(uint32_t cid, uint32_t port) {+  storage_ = VsockAddr(cid, port);+}+#endif++int SocketAddress::getPortFrom(const struct sockaddr* address) {+  switch (address->sa_family) {+    case AF_INET:+      return ntohs(((sockaddr_in*)address)->sin_port);++    case AF_INET6:+      return ntohs(((sockaddr_in6*)address)->sin6_port);++#if FOLLY_HAVE_VSOCK+    case AF_VSOCK:+      return ((sockaddr_vm*)address)->svm_port;+#endif++    default:+      return -1;+  }+}++const char* SocketAddress::getFamilyNameFrom(+    const struct sockaddr* address, const char* defaultResult) {+#define GETFAMILYNAMEFROM_IMPL(Family) \+  case Family:                         \+    return #Family++  switch ((int)address->sa_family) {+    GETFAMILYNAMEFROM_IMPL(AF_INET);+    GETFAMILYNAMEFROM_IMPL(AF_INET6);+    GETFAMILYNAMEFROM_IMPL(AF_UNIX);+#if FOLLY_HAVE_VSOCK+    GETFAMILYNAMEFROM_IMPL(AF_VSOCK);+#endif+    GETFAMILYNAMEFROM_IMPL(AF_UNSPEC);++    default:+      return defaultResult;+  }++#undef GETFAMILYNAMEFROM_IMPL+}++void SocketAddress::setFromPath(StringPiece path) {+  // Before we touch storage_, check to see if the length is too big.+  if (path.size() > sizeof(ExternalUnixAddr().addr->sun_path)) {+    throw std::invalid_argument(+        "socket path too large to fit into sockaddr_un");+  }++  // Create a new ExternalUnixAddr if we don't already have one+  if (!holdsUnix()) {+    storage_ = ExternalUnixAddr();+  }++  auto& unixAddr = std::get<ExternalUnixAddr>(storage_);+  size_t len = path.size();+  unixAddr.len = socklen_t(offsetof(struct sockaddr_un, sun_path) + len);+  memcpy(unixAddr.addr->sun_path, path.data(), len);+  // If there is room, put a terminating NUL byte in sun_path.  In general the+  // path should be NUL terminated, although getsockname() and getpeername()+  // may return Unix socket addresses with paths that fit exactly in sun_path+  // with no terminating NUL.+  if (len < sizeof(unixAddr.addr->sun_path)) {+    unixAddr.addr->sun_path[len] = '\0';+  }+}++void SocketAddress::setFromPeerAddress(NetworkSocket socket) {+  setFromSocket(socket, netops::getpeername);+}++void SocketAddress::setFromLocalAddress(NetworkSocket socket) {+  setFromSocket(socket, netops::getsockname);+}++void SocketAddress::setFromSockaddr(const struct sockaddr* address) {+  uint16_t port;++  if (address->sa_family == AF_INET) {+    port = ntohs(((sockaddr_in*)address)->sin_port);+  } else if (address->sa_family == AF_INET6) {+    port = ntohs(((sockaddr_in6*)address)->sin6_port);+  } else if (address->sa_family == AF_UNIX) {+    // We need an explicitly specified length for AF_UNIX addresses,+    // to be able to distinguish anonymous addresses from addresses+    // in Linux's abstract namespace.+    throw std::invalid_argument(+        "SocketAddress::setFromSockaddr(): the address "+        "length must be explicitly specified when "+        "setting AF_UNIX addresses");+#if FOLLY_HAVE_VSOCK+  } else if (address->sa_family == AF_VSOCK) {+    // For VSOCK addresses, store the CID and port in the VsockAddr+    const auto* vsockAddr = reinterpret_cast<const sockaddr_vm*>(address);+    storage_ = VsockAddr(vsockAddr->svm_cid, vsockAddr->svm_port);+    return;+#endif+  } else {+    throw std::invalid_argument(fmt::format(+        "SocketAddress::setFromSockaddr() called "+        "with unsupported address type {}",+        address->sa_family));+  }++  // For IP addresses, use the IPAddress constructor+  storage_ = IPAddr(folly::IPAddress(address), port);+}++void SocketAddress::setFromSockaddr(+    const struct sockaddr* address, socklen_t addrlen) {+  // Check the length to make sure we can access address->sa_family+  if (addrlen <+      (offsetof(struct sockaddr, sa_family) + sizeof(address->sa_family))) {+    throw std::invalid_argument(+        "SocketAddress::setFromSockaddr() called "+        "with length too short for a sockaddr");+  }++  if (address->sa_family == AF_INET) {+    if (addrlen < sizeof(struct sockaddr_in)) {+      throw std::invalid_argument(+          "SocketAddress::setFromSockaddr() called "+          "with length too short for a sockaddr_in");+    }+    setFromSockaddr(reinterpret_cast<const struct sockaddr_in*>(address));+  } else if (address->sa_family == AF_INET6) {+    if (addrlen < sizeof(struct sockaddr_in6)) {+      throw std::invalid_argument(+          "SocketAddress::setFromSockaddr() called "+          "with length too short for a sockaddr_in6");+    }+    setFromSockaddr(reinterpret_cast<const struct sockaddr_in6*>(address));+  } else if (address->sa_family == AF_UNIX) {+    setFromSockaddr(+        reinterpret_cast<const struct sockaddr_un*>(address), addrlen);+#if FOLLY_HAVE_VSOCK+  } else if (address->sa_family == AF_VSOCK) {+    setFromSockaddr(reinterpret_cast<const struct sockaddr_vm*>(address));+#endif+  } else {+    throw std::invalid_argument(+        "SocketAddress::setFromSockaddr() called "+        "with unsupported address type");+  }+}++void SocketAddress::setFromSockaddr(const struct sockaddr_in* address) {+  assert(address->sin_family == AF_INET);+  setFromSockaddr((sockaddr*)address);+}++void SocketAddress::setFromSockaddr(const struct sockaddr_in6* address) {+  assert(address->sin6_family == AF_INET6);+  setFromSockaddr((sockaddr*)address);+}++void SocketAddress::setFromSockaddr(+    const struct sockaddr_un* address, socklen_t addrlen) {+  assert(address->sun_family == AF_UNIX);+  if (addrlen > sizeof(struct sockaddr_un)) {+    throw std::invalid_argument(+        "SocketAddress::setFromSockaddr() called "+        "with length too long for a sockaddr_un");+  }++  // Create a new ExternalUnixAddr if we don't already have one+  if (!holdsUnix()) {+    storage_ = ExternalUnixAddr();+  }++  auto& unixAddr = std::get<ExternalUnixAddr>(storage_);+  memcpy(unixAddr.addr, address, size_t(addrlen));+  updateUnixAddressLength(addrlen);++  // Fill the rest with 0s, just for safety+  if (addrlen < sizeof(struct sockaddr_un)) {+    auto p = reinterpret_cast<char*>(unixAddr.addr);+    memset(p + addrlen, 0, sizeof(struct sockaddr_un) - addrlen);+  }+}++#if FOLLY_HAVE_VSOCK+void SocketAddress::setFromSockaddr(const struct sockaddr_vm* address) {+  assert(address->svm_family == AF_VSOCK);+  storage_ = VsockAddr(address->svm_cid, address->svm_port);+}+#endif++const folly::IPAddress& SocketAddress::getIPAddress() const {+  auto family = getFamily();+  if (family != AF_INET && family != AF_INET6) {+    throw InvalidAddressFamilyException(family);+  }+  return std::get<IPAddr>(storage_).ip;+}++socklen_t SocketAddress::getActualSize() const {+  switch (getFamily()) {+    case AF_UNSPEC:+    case AF_INET:+      return sizeof(struct sockaddr_in);+    case AF_INET6:+      return sizeof(struct sockaddr_in6);+    case AF_UNIX:+      return std::get<ExternalUnixAddr>(storage_).len;+#if FOLLY_HAVE_VSOCK+    case AF_VSOCK:+      return sizeof(struct sockaddr_vm);+#endif+    default:+      throw std::invalid_argument(+          "SocketAddress::getActualSize() called "+          "with unrecognized address family");+  }+}++std::string SocketAddress::getFullyQualified() const {+  if (!isFamilyInet()) {+    throw std::invalid_argument("Can't get address str for non ip address");+  }+  return std::get<IPAddr>(storage_).ip.toFullyQualified();+}++std::string SocketAddress::getAddressStr() const {+  if (!isFamilyInet()) {+    throw std::invalid_argument("Can't get address str for non ip address");+  }+  return std::get<IPAddr>(storage_).ip.str();+}++bool SocketAddress::isFamilyInet() const {+  auto family = getFamily();+  return family == AF_INET || family == AF_INET6;+}++void SocketAddress::getAddressStr(char* buf, size_t buflen) const {+  auto ret = getAddressStr();+  size_t len = std::min(buflen - 1, ret.size());+  memcpy(buf, ret.data(), len);+  buf[len] = '\0';+}++uint16_t SocketAddress::getPort() const {+  switch (getFamily()) {+    case AF_INET:+    case AF_INET6:+      return std::get<IPAddr>(storage_).port;+    default:+      throw std::invalid_argument(+          "SocketAddress::getPort() called on non-IP "+          "address");+  }+}++#if FOLLY_HAVE_VSOCK+uint32_t SocketAddress::getVsockPort() const {+  switch (getFamily()) {+    case AF_VSOCK:+      return std::get<VsockAddr>(storage_).port;+    default:+      throw std::invalid_argument(+          "SocketAddress::getVsockPort() called on non-VSOCK address");+  }+}+#endif++void SocketAddress::setPort(uint16_t port) {+  switch (getFamily()) {+    case AF_INET:+    case AF_INET6:+      std::get<IPAddr>(storage_).port = port;+      return;+    default:+      throw std::invalid_argument(+          "SocketAddress::setPort() called on non-IP "+          "address");+  }+}++void SocketAddress::convertToIPv4() {+  if (!tryConvertToIPv4()) {+    throw std::invalid_argument(+        "convertToIPv4() called on an address that is "+        "not an IPv4-mapped address");+  }+}++bool SocketAddress::tryConvertToIPv4() {+  if (!isIPv4Mapped()) {+    return false;+  }++  auto& ipAddr = std::get<IPAddr>(storage_);+  ipAddr.ip = folly::IPAddress::createIPv4(ipAddr.ip);+  return true;+}++bool SocketAddress::mapToIPv6() {+  if (getFamily() != AF_INET) {+    return false;+  }++  auto& ipAddr = std::get<IPAddr>(storage_);+  ipAddr.ip = folly::IPAddress::createIPv6(ipAddr.ip);+  return true;+}++std::string SocketAddress::getHostStr() const {+  return getIpString(0);+}++std::string SocketAddress::getPath() const {+  if (!holdsUnix()) {+    throw std::invalid_argument(+        "SocketAddress: attempting to get path "+        "for a non-Unix address");+  }++  const auto& unixAddr = std::get<ExternalUnixAddr>(storage_);+  if (unixAddr.pathLength() == 0) {+    // anonymous address+    return std::string();+  }+  if (unixAddr.addr->sun_path[0] == '\0') {+    // abstract namespace+    return std::string(unixAddr.addr->sun_path, size_t(unixAddr.pathLength()));+  }++  return std::string(+      unixAddr.addr->sun_path,+      strnlen(unixAddr.addr->sun_path, size_t(unixAddr.pathLength())));+}++#if FOLLY_HAVE_VSOCK+uint32_t SocketAddress::getVsockCID() const {+  if (!holdsVsock()) {+    throw std::invalid_argument(+        "SocketAddress: attempting to get CID "+        "for a non-VSOCK address");+  }++  return std::get<VsockAddr>(storage_).cid;+}+#endif++std::string SocketAddress::describe() const {+  if (holdsUnix()) {+    const auto& unixAddr = std::get<ExternalUnixAddr>(storage_);+    if (unixAddr.pathLength() == 0) {+      return "<anonymous unix address>";+    }++    if (unixAddr.addr->sun_path[0] == '\0') {+      // Linux supports an abstract namespace for unix socket addresses+      return "<abstract unix address>";+    }++    return std::string(+        unixAddr.addr->sun_path,+        strnlen(unixAddr.addr->sun_path, size_t(unixAddr.pathLength())));+  }+  switch (getFamily()) {+    case AF_UNSPEC:+      return "<uninitialized address>";+    case AF_INET: {+      char buf[NI_MAXHOST + 16];+      getAddressStr(buf, sizeof(buf));+      size_t iplen = strlen(buf);+      snprintf(buf + iplen, sizeof(buf) - iplen, ":%" PRIu16, getPort());+      return buf;+    }+    case AF_INET6: {+      char buf[NI_MAXHOST + 18];+      buf[0] = '[';+      getAddressStr(buf + 1, sizeof(buf) - 1);+      size_t iplen = strlen(buf);+      snprintf(buf + iplen, sizeof(buf) - iplen, "]:%" PRIu16, getPort());+      return buf;+    }+#if FOLLY_HAVE_VSOCK+    case AF_VSOCK: {+      char buf[32];+      const auto& vsockAddr = std::get<VsockAddr>(storage_);+      auto* maybeName = vsockAddr.getMappedName();+      if (maybeName) {+        snprintf(+            buf, sizeof(buf), "[%s:%" PRIu32 "]", maybeName, vsockAddr.port);+      } else {+        snprintf(+            buf,+            sizeof(buf),+            "[%" PRIu32 ":%" PRIu32 "]",+            vsockAddr.cid,+            vsockAddr.port);+      }+      return buf;+    }+#endif+    default: {+      char buf[64];+      snprintf(buf, sizeof(buf), "<unknown address family %d>", getFamily());+      return buf;+    }+  }+}++bool SocketAddress::operator==(const SocketAddress& other) const {+  if (other.getFamily() != getFamily()) {+    return false;+  }++  if (holdsUnix()) {+    const auto& thisUnixAddr = std::get<ExternalUnixAddr>(storage_);+    const auto& otherUnixAddr = std::get<ExternalUnixAddr>(other.storage_);++    // anonymous addresses are never equal to any other addresses+    if (thisUnixAddr.pathLength() == 0 || otherUnixAddr.pathLength() == 0) {+      return false;+    }++    if (thisUnixAddr.len != otherUnixAddr.len) {+      return false;+    }+    int cmp = memcmp(+        thisUnixAddr.addr->sun_path,+        otherUnixAddr.addr->sun_path,+        size_t(thisUnixAddr.pathLength()));+    return cmp == 0;+  }++  switch (getFamily()) {+    case AF_INET:+    case AF_INET6:+      return (std::get<IPAddr>(other.storage_).ip ==+              std::get<IPAddr>(storage_).ip) &&+          (std::get<IPAddr>(other.storage_).port ==+           std::get<IPAddr>(storage_).port);+#if FOLLY_HAVE_VSOCK+    case AF_VSOCK:+      return (std::get<VsockAddr>(other.storage_).cid ==+              std::get<VsockAddr>(storage_).cid) &&+          (std::get<VsockAddr>(other.storage_).port ==+           std::get<VsockAddr>(storage_).port);+#endif+    case AF_UNSPEC:+      return std::get<IPAddr>(other.storage_).ip.empty();+    default:+      throw_exception<std::invalid_argument>(+          "SocketAddress: unsupported address family for comparison");+  }+}++bool SocketAddress::prefixMatch(+    const SocketAddress& other, unsigned prefixLength) const {+  if (other.getFamily() != getFamily()) {+    return false;+  }+  uint8_t mask_length = 128;+  switch (getFamily()) {+    case AF_INET:+      mask_length = 32;+      [[fallthrough]];+    case AF_INET6: {+      auto prefix = folly::IPAddress::longestCommonPrefix(+          {std::get<IPAddr>(storage_).ip, mask_length},+          {std::get<IPAddr>(other.storage_).ip, mask_length});+      return prefix.second >= prefixLength;+    }+    default:+      return false;+  }+}++size_t SocketAddress::hash() const {+  size_t seed = folly::hash::twang_mix64(getFamily());++  if (holdsUnix()) {+    const auto& unixAddr = std::get<ExternalUnixAddr>(storage_);+    enum { kUnixPathMax = sizeof(unixAddr.addr->sun_path) };+    const char* path = unixAddr.addr->sun_path;+    auto pathLength = unixAddr.pathLength();+    // TODO: this probably could be made more efficient+    for (off_t n = 0; n < pathLength; ++n) {+      boost::hash_combine(seed, folly::hash::twang_mix64(uint64_t(path[n])));+    }+  }++  switch ((int)getFamily()) {+    case AF_INET:+    case AF_INET6: {+      boost::hash_combine(seed, std::get<IPAddr>(storage_).port);+      boost::hash_combine(seed, std::get<IPAddr>(storage_).ip.hash());+      break;+    }+#if FOLLY_HAVE_VSOCK+    case AF_VSOCK: {+      boost::hash_combine(seed, std::get<VsockAddr>(storage_).port);+      boost::hash_combine(seed, std::get<VsockAddr>(storage_).cid);+      break;+    }+#endif+    case AF_UNIX:+      // Already handled above+      break;+    case AF_UNSPEC:+      boost::hash_combine(seed, std::get<IPAddr>(storage_).ip.hash());+      break;+    default:+      throw_exception<std::invalid_argument>(+          "SocketAddress: unsupported address family for comparison");+  }++  return seed;+}++struct addrinfo* SocketAddress::getAddrInfo(+    const char* host, uint16_t port, int flags) {+  // getaddrinfo() requires the port number as a string+  char portString[sizeof("65535")];+  snprintf(portString, sizeof(portString), "%" PRIu16, port);++  return getAddrInfo(host, portString, flags);+}++struct addrinfo* SocketAddress::getAddrInfo(+    const char* host, const char* port, int flags) {+  struct addrinfo hints;+  memset(&hints, 0, sizeof(hints));+  hints.ai_family = AF_UNSPEC;+  hints.ai_socktype = SOCK_STREAM;+  hints.ai_flags = AI_PASSIVE | AI_NUMERICSERV | flags;++  struct addrinfo* results;+  int error = getaddrinfo(host, port, &hints, &results);+  if (error != 0) {+    auto os = fmt::format(+        "Failed to resolve address for '{}': {} (error={})",+        (host ? host : "<null>"),+        GetAddrInfoError(error).str(),+        error);+    throw std::system_error(error, std::generic_category(), os);+  }++  return results;+}++void SocketAddress::setFromAddrInfo(const struct addrinfo* info) {+  setFromSockaddr(info->ai_addr, socklen_t(info->ai_addrlen));+}++void SocketAddress::setFromLocalAddr(const struct addrinfo* info) {+  // If an IPv6 address is present, prefer to use it, since IPv4 addresses+  // can be mapped into IPv6 space.+  for (const struct addrinfo* ai = info; ai != nullptr; ai = ai->ai_next) {+    if (ai->ai_family == AF_INET6) {+      setFromSockaddr(ai->ai_addr, socklen_t(ai->ai_addrlen));+      return;+    }+  }++  // Otherwise, just use the first address in the list.+  setFromSockaddr(info->ai_addr, socklen_t(info->ai_addrlen));+}++void SocketAddress::setFromSocket(+    NetworkSocket socket,+    int (*fn)(NetworkSocket, struct sockaddr*, socklen_t*)) {+  // Try to put the address into a local storage buffer.+  sockaddr_storage tmp_sock;+  socklen_t addrLen = sizeof(tmp_sock);+  if (fn(socket, (sockaddr*)&tmp_sock, &addrLen) != 0) {+    folly::throwSystemError("setFromSocket() failed");+  }++  setFromSockaddr((sockaddr*)&tmp_sock, addrLen);+}++std::string SocketAddress::getIpString(int flags) const {+  char addrString[NI_MAXHOST];+  getIpString(addrString, sizeof(addrString), flags);+  return std::string(addrString);+}++void SocketAddress::getIpString(char* buf, size_t buflen, int flags) const {+  auto family = getFamily();+  if (family != AF_INET && family != AF_INET6) {+    throw std::invalid_argument(+        "SocketAddress: attempting to get IP address "+        "for a non-IP address");+  }++  sockaddr_storage tmp_sock;+  std::get<IPAddr>(storage_).ip.toSockaddrStorage(+      &tmp_sock, std::get<IPAddr>(storage_).port);+  int rc = getnameinfo(+      (sockaddr*)&tmp_sock,+      sizeof(sockaddr_storage),+      buf,+      buflen,+      nullptr,+      0,+      flags);+  if (rc != 0) {+    auto os = fmt::format(+        "getnameinfo() failed in getIpString() error = {}",+        GetAddrInfoError(rc).str());+    throw std::system_error(rc, std::generic_category(), os);+  }+}++void SocketAddress::updateUnixAddressLength(socklen_t addrlen) {+  if (addrlen < offsetof(struct sockaddr_un, sun_path)) {+    throw std::invalid_argument(+        "SocketAddress: attempted to set a Unix socket "+        "with a length too short for a sockaddr_un");+  }++  auto& unixAddr = std::get<ExternalUnixAddr>(storage_);+  unixAddr.len = addrlen;+  if (unixAddr.pathLength() == 0) {+    // anonymous address+    return;+  }++  if (unixAddr.addr->sun_path[0] == '\0') {+    // abstract namespace.  honor the specified length+  } else {+    // Call strnlen(), just in case the length was overspecified.+    size_t maxLength = addrlen - offsetof(struct sockaddr_un, sun_path);+    size_t pathLength = strnlen(unixAddr.addr->sun_path, maxLength);+    unixAddr.len =+        socklen_t(offsetof(struct sockaddr_un, sun_path) + pathLength);+  }+}++bool SocketAddress::operator<(const SocketAddress& other) const {+  if (getFamily() != other.getFamily()) {+    return getFamily() < other.getFamily();+  }++  if (holdsUnix()) {+    // Anonymous addresses can't be compared to anything else.+    // Return that they are never less than anything.+    //+    // Note that this still meets the requirements for a strict weak+    // ordering, so we can use this operator<() with standard C+++    // containers.+    const auto& thisUnixAddr = std::get<ExternalUnixAddr>(storage_);+    auto thisPathLength = thisUnixAddr.pathLength();+    if (thisPathLength == 0) {+      return false;+    }+    const auto& otherUnixAddr = std::get<ExternalUnixAddr>(other.storage_);+    auto otherPathLength = otherUnixAddr.pathLength();+    if (otherPathLength == 0) {+      return true;+    }++    // Compare based on path length first, for efficiency+    if (thisPathLength != otherPathLength) {+      return thisPathLength < otherPathLength;+    }+    int cmp = memcmp(+        thisUnixAddr.addr->sun_path,+        otherUnixAddr.addr->sun_path,+        size_t(thisPathLength));+    return cmp < 0;+  }+  switch (getFamily()) {+    case AF_INET:+    case AF_INET6: {+      auto& thisAddr = std::get<IPAddr>(storage_);+      auto& otherAddr = std::get<IPAddr>(other.storage_);+      if (thisAddr.port != otherAddr.port) {+        return thisAddr.port < otherAddr.port;+      }++      return thisAddr.ip < otherAddr.ip;+    }+    case AF_UNSPEC:+    default:+      throw std::invalid_argument(+          "SocketAddress: unsupported address family for comparing");+  }+}++#if FOLLY_HAVE_VSOCK+const char* SocketAddress::VsockAddr::getMappedName() const {+  // Use special names for well-known CIDs+  if (cid == VMADDR_CID_ANY) {+    return "any";+  } else if (cid == VMADDR_CID_HYPERVISOR) {+    return "hypervisor";+  } else if (cid == VMADDR_CID_LOCAL) {+    return "local";+  } else if (cid == VMADDR_CID_HOST) {+    return "host";+  } else {+    return nullptr;+  }+}+#endif++size_t hash_value(const SocketAddress& address) {+  return address.hash();+}++std::ostream& operator<<(std::ostream& os, const SocketAddress& addr) {+  os << addr.describe();+  return os;+}++} // namespace folly
+ folly/folly/SocketAddress.h view
@@ -0,0 +1,828 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <sys/types.h>++#include <cassert>+#include <cstddef>+#include <iosfwd>+#include <string>++#include <variant>+#include <folly/IPAddress.h>+#include <folly/Portability.h>+#include <folly/Range.h>+#include <folly/net/NetworkSocket.h>+#include <folly/portability/Config.h>+#include <folly/portability/Sockets.h>++#if FOLLY_HAVE_VSOCK+#include <linux/vm_sockets.h>+#endif++namespace folly {++/**+ * Provides a unified interface for socket addresses.+ *+ * @class folly::SocketAddress+ *+ */++class SocketAddress {+ public:+  SocketAddress() = default;++  /**+   * Construct a SocketAddress from a hostname and port.+   *+   * Note: If the host parameter is not a numeric IP address, hostname+   * resolution will be performed, which can be quite slow.+   *+   * Raises std::system_error on error.+   *+   * @param host The IP address (or hostname, if allowNameLookup is true)+   * @param port The port (in host byte order)+   * @param allowNameLookup  If true, attempt to perform hostname lookup+   *        if the hostname does not appear to be a numeric IP address.+   *        This is potentially a very slow operation, so is disabled by+   *        default.+   */+  SocketAddress(const char* host, uint16_t port, bool allowNameLookup = false) {+    // Initialize the address family first,+    // since setFromHostPort() and setFromIpPort() will check it.++    if (allowNameLookup) {+      setFromHostPort(host, port);+    } else {+      setFromIpPort(host, port);+    }+  }+  /**+   * Similar to the constructor which accepts hostname and port.+   * This variant accepts host as std::string.+   */+  SocketAddress(+      const std::string& host, uint16_t port, bool allowNameLookup = false) {+    // Initialize the address family first,+    // since setFromHostPort() and setFromIpPort() will check it.++    if (allowNameLookup) {+      setFromHostPort(host.c_str(), port);+    } else {+      setFromIpPort(host.c_str(), port);+    }+  }+  /**+   * Construct a SocketAddress from a hostname and port.+   *+   * Raises std::system_error on error.+   *+   * @param ipAddr The IP address+   * @param port The port (in host byte order)+   */+  SocketAddress(const IPAddress& ipAddr, uint16_t port) {+    setFromIpAddrPort(ipAddr, port);+  }++  SocketAddress(const SocketAddress& addr) { storage_ = addr.storage_; }++  SocketAddress& operator=(const SocketAddress& addr) {+    storage_ = addr.storage_;+    return *this;+  }++  SocketAddress(SocketAddress&& addr) noexcept {+    storage_ = std::move(addr.storage_);+  }++  SocketAddress& operator=(SocketAddress&& addr) {+    storage_ = std::move(addr.storage_);+    return *this;+  }++  ~SocketAddress() = default;++  /**+   * Return whether this SocketAddress is initialized.+   */+  bool isInitialized() const { return (getFamily() != AF_UNSPEC); }++  /**+   * Return whether this address is within private network.+   *+   * According to RFC1918, the 10/8 prefix, 172.16/12 prefix, and 192.168/16+   * prefix are reserved for private networks.+   * fc00::/7 is the IPv6 version, defined in RFC4139.  IPv6 link-local+   * addresses (fe80::/10) are also considered private addresses.+   *+   * The loopback addresses 127/8 and ::1 are also regarded as private networks+   * for the purpose of this function.+   *+   * @return true if this is a private network address, false otherwise+   */+  bool isPrivateAddress() const;++  /**+   * Return whether this address is a loopback address.+   *+   * @return true if this is a loopback address, false otherwise+   */+  bool isLoopbackAddress() const;++  /**+   * Reset this SocketAddress by clearing the associated address and+   * freeing up any external storage being used.+   */+  void reset() { storage_ = IPAddr(); }++  /**+   * @overloadbrief Initialize this SocketAddress from a hostname and port.+   *+   * Note: If the host parameter is not a numeric IP address, hostname+   * resolution will be performed, which can be quite slow.+   *+   * If the hostname resolves to multiple addresses, only the first will be+   * returned.+   *+   * Raises std::system_error on error.+   *+   * @param host The hostname or IP address+   * @param port The port (in host byte order)+   */+  void setFromHostPort(const char* host, uint16_t port);+  /**+   * Similar to the function setFromHostPort above, but accepts the IP address+   * as a std::string.+   */+  void setFromHostPort(const std::string& host, uint16_t port) {+    setFromHostPort(host.c_str(), port);+  }++  /**+   * @overloadbrief Initialize this SocketAddress from an IP address and port.+   *+   * This is similar to setFromHostPort(), but only accepts numeric IP+   * addresses.  If the IP string does not look like an IP address, it throws a+   * std::invalid_argument rather than trying to perform a hostname resolution.+   *+   * Raises std::system_error on error.+   *+   * @param ip The IP address, as a human-readable string.+   * @param port The port (in host byte order)+   */+  void setFromIpPort(const char* ip, uint16_t port);+  /**+   * Similar to the function setFromIpPort above, but accepts the IP address as+   * a std::string.+   */+  void setFromIpPort(const std::string& ip, uint16_t port) {+    setFromIpPort(ip.c_str(), port);+  }++  /**+   * Initialize this SocketAddress from an IPAddress struct and port.+   *+   * @param ip The IP address in IPAddress format+   * @param port The port (in host byte order)+   */+  void setFromIpAddrPort(const IPAddress& ip, uint16_t port);++  /**+   * @overloadbrief Initialize this SocketAddress from a local port number.+   *+   * This is intended to be used by server code to determine the address to+   * listen on.+   *+   * If the current machine has any IPv6 addresses configured, an IPv6 address+   * will be returned (since connections from IPv4 clients can be mapped to the+   * IPv6 address).  If the machine does not have any IPv6 addresses, an IPv4+   * address will be returned.+   *+   * @param port The local port number (in host byte order)+   */+  void setFromLocalPort(uint16_t port);+  /**+   * Initialize this SocketAddress from a local port number.+   *+   * This version of setFromLocalPort() accepts the port as a string.  A+   * std::invalid_argument will be raised if the string does not refer to a port+   * number.  Non-numeric service port names are not accepted.+   *+   * @param port The local port number+   */+  void setFromLocalPort(const char* port);+  /**+   * Similar to the function setFromLocalPort above, but accepts the port as+   * a std::string.+   */+  void setFromLocalPort(const std::string& port) {+    return setFromLocalPort(port.c_str());+  }++  /**+   * @overloadbrief Initialize this SocketAddress from a local port number and+   * optional IP address.+   *+   * The addressAndPort string may be specified either as "<ip>:<port>", or+   * just as "<port>".  If the IP is not specified, the address will be+   * initialized to 0, so that a server socket bound to this address will+   * accept connections on all local IP addresses.+   *+   * Both the IP address and port number must be numeric.  DNS host names and+   * non-numeric service port names are not accepted.+   *+   * @param addressAndPort Address and the port separated by ':', or the port+   */+  void setFromLocalIpPort(const char* addressAndPort);+  /**+   * Similar to the function setFromLocalIpPort above, but accepts the address+   * and port as a std::string.+   */+  void setFromLocalIpPort(const std::string& addressAndPort) {+    return setFromLocalIpPort(addressAndPort.c_str());+  }++  /**+   * @overloadbrief Initialize this SocketAddress from an IP address and port+   * number.+   *+   * The addressAndPort string must be of the form "<ip>:<port>".  E.g.,+   * "10.0.0.1:1234".+   *+   * Both the IP address and port number must be numeric.  DNS host names and+   * non-numeric service port names are not accepted.+   *+   * @param addressAndPort Address and the port separated by ':'+   */+  void setFromIpPort(const char* addressAndPort);+  /**+   * Similar to the function setFromIpPort above, but accepts the address+   * and port as a std::string.+   */+  void setFromIpPort(const std::string& addressAndPort) {+    return setFromIpPort(addressAndPort.c_str());+  }++  /**+   * @overloadbrief Initialize this SocketAddress from a host name and port+   * number.+   *+   * The addressAndPort string must be of the form "<host>:<port>".  E.g.,+   * "www.facebook.com:443".+   *+   * If the host name is not a numeric IP address, a DNS lookup will be+   * performed.  Beware that the DNS lookup may be very slow.  The port number+   * must be numeric; non-numeric service port names are not accepted.+   *+   * @param hostAndPort Host name and the port separated by ':'+   */+  void setFromHostPort(const char* hostAndPort);+  /**+   * Similar to the function setFromHostPort above, but accepts the host name+   * and port as a std::string.+   */+  void setFromHostPort(const std::string& hostAndPort) {+    return setFromHostPort(hostAndPort.c_str());+  }++#if FOLLY_HAVE_VSOCK+  /**+   * Initialize this SocketAddress from a VSOCK CID and port.+   */+  void setFromVsockCIDPort(uint32_t cid, uint32_t port);+#endif++  /**+   * Returns the port number from the given socketaddr structure.+   *+   * Currently only IPv4 and IPv6 are supported.+   *+   * @param address The socketaddr structure to get port from+   *+   * @return The port number, or -1 for unsupported socket families.+   */+  static int getPortFrom(const struct sockaddr* address);++  /**+   * Returns the family name from the given socketaddr structure (e.g.: AF_INET6+   * for IPv6).+   *+   * @param address The socketaddr structure to get family name from+   * @param defaultResult The default family name to be returned in case of+   * unsupported socket. If no value is passed, `nullptr` is returned as default+   * family name.+   *+   * @return The family name, or `defaultResult` passed for unsupported socket+   * families.+   */+  static const char* getFamilyNameFrom(+      const struct sockaddr* address, const char* defaultResult = nullptr);++  /**+   * @overloadbrief Initialize this SocketAddress from a local unix path.+   *+   * Raises std::invalid_argument on error.+   *+   * @param path Local unix path+   */+  void setFromPath(StringPiece path);+  /**+   * Similar to setFromPath, but accepts local unix path as const char* and+   * its length.+   */+  void setFromPath(const char* path, size_t length) {+    setFromPath(StringPiece{path, length});+  }++  /**+   * Construct a SocketAddress from a local unix socket path.+   *+   * Raises std::invalid_argument on error.+   *+   * @param path The Unix domain socket path.+   */+  static SocketAddress makeFromPath(StringPiece path) {+    SocketAddress addr;+    addr.setFromPath(path);+    return addr;+  }++  /**+   * Initialize this SocketAddress from a socket's peer address.+   *+   * Raises std::system_error on error.+   *+   * @param socket Socket whose peer address is used to initialize+   */+  void setFromPeerAddress(NetworkSocket socket);++  /**+   * Initialize this SocketAddress from a socket's local address.+   *+   * Raises std::system_error on error.+   *+   * @param socket Socket whose local address is used to initialize+   */+  void setFromLocalAddress(NetworkSocket socket);++  /**+   * Initialize this folly::SocketAddress from a struct sockaddr.+   *+   * Raises std::system_error on error.+   *+   * This method is not supported for AF_UNIX addresses.  For unix addresses,+   * the address length must be explicitly specified.+   *+   * @param address  A struct sockaddr.  The size of the address is implied+   *                 from address->sa_family.+   */+  void setFromSockaddr(const struct sockaddr* address);++  /**+   * Initialize this SocketAddress from a struct sockaddr.+   *+   * Raises std::system_error on error.+   *+   * @param address  A struct sockaddr.+   * @param addrlen  The length of address data available.  This must be long+   *                 enough for the full address type required by+   *                 address->sa_family.+   */+  void setFromSockaddr(const struct sockaddr* address, socklen_t addrlen);++  /**+   * Initialize this SocketAddress from a struct sockaddr_in.+   *+   * @param address  A struct sockaddr_in to initialize from+   */+  void setFromSockaddr(const struct sockaddr_in* address);++  /**+   * Initialize this SocketAddress from a struct sockaddr_in6.+   *+   * @param address  A struct sockaddr_in6 to initialize from+   */+  void setFromSockaddr(const struct sockaddr_in6* address);++  /**+   * Initialize this SocketAddress from a struct sockaddr_un.+   *+   * Note that the addrlen parameter is necessary to properly detect anonymous+   * addresses, which have 0 valid path bytes, and may not even have a NUL+   * character at the start of the path.+   *+   * @param address  A struct sockaddr_un.+   * @param addrlen  The length of address data.  This should include all of+   *                 the valid bytes of sun_path, not including any NUL+   *                 terminator.+   */+  void setFromSockaddr(const struct sockaddr_un* address, socklen_t addrlen);++#if FOLLY_HAVE_VSOCK+  /**+   * Initialize this SocketAddress from a struct sockaddr_vm.+   *+   * @param address  A struct sockaddr_vm to initialize from+   */+  void setFromSockaddr(const struct sockaddr_vm* address);+#endif++  /**+   * Fill in a given sockaddr_storage with the ip or unix address.+   *+   * @param addr sockaddr_storage out parameter+   *+   * @return The actual size of the socket address+   */+  socklen_t getAddress(sockaddr_storage* addr) const {+    if (isFamilyInet()) {+      return std::get<IPAddr>(storage_).ip.toSockaddrStorage(+          addr, htons(std::get<IPAddr>(storage_).port));+#if FOLLY_HAVE_VSOCK+    } else if (holdsVsock()) {+      const auto& vsockAddr = std::get<VsockAddr>(storage_);+      auto* svm = reinterpret_cast<sockaddr_vm*>(addr);+      memset(svm, 0, sizeof(sockaddr_vm));+      svm->svm_family = AF_VSOCK;+      svm->svm_cid = vsockAddr.cid;+      svm->svm_port = vsockAddr.port;+      return sizeof(sockaddr_vm);+#endif+    } else {+      const auto& unixAddr = std::get<ExternalUnixAddr>(storage_);+      memcpy(addr, unixAddr.addr, sizeof(*unixAddr.addr));+      return unixAddr.len;+    }+  }++  /**+   * Return the IP address of this SocketAddress.+   *+   * @throws folly::InvalidAddressFamilyException if the family is not IPv4 or+   * IPv6+   *+   * @return IP address+   */+  const folly::IPAddress& getIPAddress() const;++  /**+   * DEPRECATED: SocketAddress::getAddress() above returns the same size as+   * getActualSize()+   *+   * Return the size of the underlying socket address+   *+   * @return The size of the socket address+   */+  socklen_t getActualSize() const;++  /**+   * Return the address family of this SocketAddress+   *+   * @return Socket address family+   */+  sa_family_t getFamily() const {+    if (holdsUnix()) {+      return sa_family_t(AF_UNIX);+#if FOLLY_HAVE_VSOCK+    } else if (holdsVsock()) {+      return sa_family_t(AF_VSOCK);+#endif+    } else {+      return std::get<IPAddr>(storage_).ip.family();+    }+  }++  /**+   * Return if the SocketAddress is `empty` i.e., the address family is+   * unspecified.+   *+   * @return true, if socket is `empty` i.e., address family is unspecified,+   * else false+   */+  bool empty() const { return getFamily() == AF_UNSPEC; }++  /**+   * Get a string representation of the IPv4 or IPv6 address.+   *+   * Raises std::invalid_argument if an error occurs (for example, if+   * the address is not an IPv4 or IPv6 address).+   *+   * @return String representation of the IP address+   */+  std::string getAddressStr() const;++  /**+   * Get a string representation of the IPv4 or IPv6 address.+   *+   * Raises std::invalid_argument if an error occurs (for example, if+   * the address is not an IPv4 or IPv6 address).+   *+   * @param buf Char buffer to write the string representation into+   * @param buflen Size of the buffer+   */+  void getAddressStr(char* buf, size_t buflen) const;++  /**+   * Return whether this address is a valid IPv4 or IPv6 address.+   *+   * @return true if address a valid IPv4 or IPv6 address, false otherwise+   */+  bool isFamilyInet() const;++  /**+   * For v4 & v6 addresses, return the fully qualified address string+   *+   * Raises std::invalid_argument if this is not an IPv4 or IPv6 address.+   *+   * @return Fully qualified IP address+   */+  std::string getFullyQualified() const;++  /**+   * Get the IPv4 or IPv6 port for this address.+   *+   * Raises std::invalid_argument if this is not an IPv4 or IPv6 address.+   *+   * @return The port, in host byte order+   */+  uint16_t getPort() const;++#if FOLLY_HAVE_VSOCK+  /**+   * Get the port for a VSOCK address.+   *+   * Raises std::invalid_argument if this is not a VSOCK address.+   *+   * @return The port, in host byte order+   */+  uint32_t getVsockPort() const;+#endif++  /**+   * Set the IPv4 or IPv6 port for this address.+   *+   * Raises std::invalid_argument if this is not an IPv4 or IPv6 address.+   *+   * @param port The port to set, in host byte order+   */+  void setPort(uint16_t port);++  /**+   * Return true if this is an IPv4-mapped IPv6 address.+   *+   * @return true if this address is a IPv6 address which is IPv4-mapped,+   * false otherwise+   */+  bool isIPv4Mapped() const {+    return (+        getFamily() == AF_INET6 &&+        std::get<IPAddr>(storage_).ip.isIPv4Mapped());+  }++  /**+   * Convert an IPv4-mapped IPv6 address to an IPv4 address.+   *+   * Raises std::invalid_argument if this is not an IPv4-mapped IPv6 address.+   *+   * @note SocketAddress::tryConvertToIPv4 is no-throw variant of this function+   */+  void convertToIPv4();++  /**+   * Try to convert an address to IPv4.+   *+   * This attempts to convert an address to an IPv4 address if possible.+   * If the address is an IPv4-mapped IPv6 address, it is converted to an IPv4+   * address and true is returned.  Otherwise nothing is done, and false is+   * returned.+   *+   * @return true if the address was converted to IPv4-mapped address, false+   * otherwise+   */+  bool tryConvertToIPv4();++  /**+   * Convert an IPv4 address to IPv6 [::ffff:a.b.c.d]+   *+   * @return true if the address conversion was done, false otherwise+   */+  bool mapToIPv6();++  /**+   * Get string representation of the host name (or IP address if the host name+   * cannot be resolved).+   *+   * Warning: Using this method is strongly discouraged.  It performs a+   * DNS lookup, which may block for many seconds.+   *+   * Raises std::invalid_argument if an error occurs.+   *+   * @return Host name (or IP address)+   */+  std::string getHostStr() const;++  /**+   * Get the path name for a Unix domain socket.+   *+   * Returns a std::string containing the path.  For anonymous sockets, an+   * empty string is returned.+   *+   * For addresses in the abstract namespace (Linux-specific), a std::string+   * containing binary data is returned.  In this case the first character will+   * always be a NUL character.+   *+   * Raises std::invalid_argument if called on a non-Unix domain socket.+   *+   * @return Path name for a Unix domain socket+   */+  std::string getPath() const;++#if FOLLY_HAVE_VSOCK+  /**+   * Get the CID (Context Identifier) for a VSOCK address.+   *+   * Raises std::invalid_argument if called on a non-VSOCK address.+   *+   * @return CID for a VSOCK address+   */+  uint32_t getVsockCID() const;+#endif++  /**+   * Get human-readable string representation of the address.+   *+   * This prints a string representation of the address, for human consumption.+   * For IP addresses, the string is of the form "<IP>:<port>".+   *+   * @return Human-readable representation of the address+   */+  std::string describe() const;++  bool operator==(const SocketAddress& other) const;+  bool operator!=(const SocketAddress& other) const {+    return !(*this == other);+  }++  /**+   * Check whether the first N bits of this address match the first N+   * bits of another address.+   *+   * @note returns false if the addresses are not from the same+   *       address family or if the family is neither IPv4 nor IPv6+   *+   * @param other The address to match against+   * @param prefixLength Length of the prefix to match+   * @return true if `prefixLength` this address matches with `other`,+   * false otherwise+   */+  bool prefixMatch(const SocketAddress& other, unsigned prefixLength) const;++  /**+   * Use this operator for storing maps based on SocketAddress.+   */+  bool operator<(const SocketAddress& other) const;++  /**+   * Compuate a hash of a SocketAddress.+   *+   * @return Hash for this SocketAddress+   */+  size_t hash() const;++ private:+  /**+   * Unix socket addresses require more storage than IPv4 and IPv6 addresses,+   * and are comparatively little-used.+   *+   * Therefore SocketAddress' internal storage_ member variable doesn't+   * contain room for a full unix address, to avoid wasting space in the common+   * case.  When we do need to store a Unix socket address, we use this+   * ExternalUnixAddr structure to allocate a struct sockaddr_un separately on+   * the heap.+   */+  struct ExternalUnixAddr {+    struct sockaddr_un* addr;+    socklen_t len;++    socklen_t pathLength() const {+      return socklen_t(len - offsetof(struct sockaddr_un, sun_path));+    }++    ExternalUnixAddr() {+      addr = new struct sockaddr_un;+      addr->sun_family = AF_UNIX;+      len = 0;+    }++    ExternalUnixAddr(const ExternalUnixAddr& other) : ExternalUnixAddr() {+      len = other.len;+      memcpy(addr, other.addr, size_t(len));+    }++    ExternalUnixAddr& operator=(const ExternalUnixAddr& other) {+      if (this != &other) {+        len = other.len;+        memcpy(addr, other.addr, size_t(len));+      }+      return *this;+    }++    ~ExternalUnixAddr() { delete addr; }+  };++  /**+   * This class stores an IP address and port.+   */+  struct IPAddr {+    folly::IPAddress ip;+    uint16_t port;++    IPAddr() : ip(), port(0) {}+    IPAddr(const folly::IPAddress& ip_, uint16_t port_)+        : ip(ip_), port(port_) {}+  };++  /**+   * This class stores the CID (Context Identifier) and port for VSOCK+   * addresses.+   */+  struct VsockAddr {+    uint32_t cid;+    uint32_t port;++    explicit VsockAddr(uint32_t cid_) : cid(cid_), port(0) {}+    VsockAddr(uint32_t cid_, uint32_t port_) : cid(cid_), port(port_) {}++#if FOLLY_HAVE_VSOCK+    const char* getMappedName() const;+#endif+  };++  bool holdsInet() const { return std::holds_alternative<IPAddr>(storage_); }++  bool holdsUnix() const {+    return std::holds_alternative<ExternalUnixAddr>(storage_);+  }++  bool holdsVsock() const {+    return std::holds_alternative<VsockAddr>(storage_);+  }++  struct addrinfo* getAddrInfo(const char* host, uint16_t port, int flags);+  struct addrinfo* getAddrInfo(const char* host, const char* port, int flags);+  void setFromAddrInfo(const struct addrinfo* info);+  void setFromLocalAddr(const struct addrinfo* info);+  void setFromSocket(+      NetworkSocket socket,+      int (*fn)(NetworkSocket, struct sockaddr*, socklen_t*));+  std::string getIpString(int flags) const;+  void getIpString(char* buf, size_t buflen, int flags) const;++  void updateUnixAddressLength(socklen_t addrlen);++  /*+   * storage_ contains either an IPAddr, an ExternalUnixAddr, or a VsockAddr.+   * IPAddr is used for IPv4 and IPv6 addresses.+   * ExternalUnixAddr is used for Unix domain sockets.+   * VsockAddr is used for VSOCK addresses.+   */+  std::variant<IPAddr, ExternalUnixAddr, VsockAddr> storage_{IPAddr()};+};++/**+ * Hash a SocketAddress object.+ *+ * boost::hash uses hash_value(), so this allows boost::hash to automatically+ * work for SocketAddress.+ */+size_t hash_value(const SocketAddress& address);++std::ostream& operator<<(std::ostream& os, const SocketAddress& addr);+} // namespace folly++namespace std {++// Provide an implementation for std::hash<SocketAddress>+template <>+struct hash<folly::SocketAddress> {+  size_t operator()(const folly::SocketAddress& addr) const {+    return addr.hash();+  }+};+} // namespace std
+ folly/folly/SpinLock.h view
@@ -0,0 +1,55 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++/*+ * N.B. You most likely do _not_ want to use SpinLock or any other+ * kind of spinlock.  Use std::mutex instead.+ *+ * In short, spinlocks in preemptive multi-tasking operating systems+ * have serious problems and fast mutexes like std::mutex are almost+ * certainly the better choice, because letting the OS scheduler put a+ * thread to sleep is better for system responsiveness and throughput+ * than wasting a timeslice repeatedly querying a lock held by a+ * thread that's blocked, and you can't prevent userspace+ * programs blocking.+ *+ * Spinlocks in an operating system kernel make much more sense than+ * they do in userspace.+ */++#pragma once++#include <type_traits>++#include <folly/Portability.h>+#include <folly/synchronization/SmallLocks.h>++namespace folly {++class SpinLock {+ public:+  FOLLY_ALWAYS_INLINE SpinLock() noexcept { lock_.init(); }+  FOLLY_ALWAYS_INLINE void lock() const noexcept { lock_.lock(); }+  FOLLY_ALWAYS_INLINE void unlock() const noexcept { lock_.unlock(); }+  FOLLY_ALWAYS_INLINE bool try_lock() const noexcept {+    return lock_.try_lock();+  }++ private:+  mutable folly::MicroSpinLock lock_;+};++} // namespace folly
+ folly/folly/String-inl.h view
@@ -0,0 +1,748 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <iterator>+#include <stdexcept>++#include <folly/CppAttributes.h>+#include <folly/container/Reserve.h>++#ifndef FOLLY_STRING_H_+#error This file may only be included from String.h+#endif++namespace folly {++namespace detail {+// Map from character code to value of one-character escape sequence+// ('\n' = 10 maps to 'n'), 'O' if the character should be printed as+// an octal escape sequence, or 'P' if the character is printable and+// should be printed as is.+extern const std::array<char, 256> cEscapeTable;+} // namespace detail++template <class String>+void cEscape(StringPiece str, String& out) {+  char esc[4];+  esc[0] = '\\';+  grow_capacity_by(out, str.size());+  auto p = str.begin();+  auto last = p; // last regular character+  // We advance over runs of regular characters (printable, not double-quote or+  // backslash) and copy them in one go; this is faster than calling push_back+  // repeatedly.+  while (p != str.end()) {+    char c = *p;+    unsigned char v = static_cast<unsigned char>(c);+    char e = detail::cEscapeTable[v];+    if (e == 'P') { // printable+      ++p;+    } else if (e == 'O') { // octal+      out.append(&*last, size_t(p - last));+      esc[1] = '0' + ((v >> 6) & 7);+      esc[2] = '0' + ((v >> 3) & 7);+      esc[3] = '0' + (v & 7);+      out.append(esc, 4);+      ++p;+      last = p;+    } else { // special 1-character escape+      out.append(&*last, size_t(p - last));+      esc[1] = e;+      out.append(esc, 2);+      ++p;+      last = p;+    }+  }+  out.append(&*last, size_t(p - last));+}++namespace detail {+// Map from the character code of the character following a backslash to+// the unescaped character if a valid one-character escape sequence+// ('n' maps to 10 = '\n'), 'O' if this is the first character of an+// octal escape sequence, 'X' if this is the first character of a+// hexadecimal escape sequence, or 'I' if this escape sequence is invalid.+extern const std::array<char, 256> cUnescapeTable;++// Map from the character code to the hex value, or 16 if invalid hex char.+extern const std::array<unsigned char, 256> hexTable;+} // namespace detail++template <class String>+void cUnescape(StringPiece str, String& out, bool strict) {+  grow_capacity_by(out, str.size());+  auto p = str.begin();+  auto last = p; // last regular character (not part of an escape sequence)+  // We advance over runs of regular characters (not backslash) and copy them+  // in one go; this is faster than calling push_back repeatedly.+  while (p != str.end()) {+    char c = *p;+    if (c != '\\') { // normal case+      ++p;+      continue;+    }+    out.append(&*last, p - last);+    ++p;+    if (p == str.end()) { // backslash at end of string+      if (strict) {+        throw_exception<std::invalid_argument>("incomplete escape sequence");+      }+      out.push_back('\\');+      last = p;+      continue;+    }+    char e = detail::cUnescapeTable[static_cast<unsigned char>(*p)];+    if (e == 'O') { // octal+      unsigned char val = 0;+      for (int i = 0; i < 3 && p != str.end() && *p >= '0' && *p <= '7';+           ++i, ++p) {+        val <<= 3;+        val |= (*p - '0');+      }+      out.push_back(val);+      last = p;+    } else if (e == 'X') { // hex+      ++p;+      if (p == str.end()) { // \x at end of string+        if (strict) {+          throw_exception<std::invalid_argument>(+              "incomplete hex escape sequence");+        }+        out.append("\\x");+        last = p;+        continue;+      }+      unsigned char val = 0;+      unsigned char h;+      for (; (p != str.end() &&+              (h = detail::hexTable[static_cast<unsigned char>(*p)]) < 16);+           ++p) {+        val <<= 4;+        val |= h;+      }+      out.push_back(val);+      last = p;+    } else if (e == 'I') { // invalid+      if (strict) {+        throw_exception<std::invalid_argument>("invalid escape sequence");+      }+      out.push_back('\\');+      out.push_back(*p);+      ++p;+      last = p;+    } else { // standard escape sequence, \' etc+      out.push_back(e);+      ++p;+      last = p;+    }+  }+  out.append(&*last, p - last);+}++namespace detail {+// Map from character code to escape mode:+// 0 = pass through+// 1 = unused+// 2 = pass through in PATH mode+// 3 = space, replace with '+' in QUERY mode+// 4 = percent-encode+extern const std::array<unsigned char, 256> uriEscapeTable;+} // namespace detail++template <class String>+void uriEscape(StringPiece str, String& out, UriEscapeMode mode) {+  static const char hexValues[] = "0123456789abcdef";+  char esc[3];+  esc[0] = '%';+  // Preallocate assuming that 25% of the input string will be escaped+  grow_capacity_by(out, str.size() + 3 * (str.size() / 4));+  auto p = str.begin();+  auto last = p; // last regular character+  // We advance over runs of passthrough characters and copy them in one go;+  // this is faster than calling push_back repeatedly.+  unsigned char minEncode = static_cast<unsigned char>(mode);+  while (p != str.end()) {+    char c = *p;+    unsigned char v = static_cast<unsigned char>(c);+    unsigned char discriminator = detail::uriEscapeTable[v];+    if (FOLLY_LIKELY(discriminator <= minEncode)) {+      ++p;+    } else if (mode == UriEscapeMode::QUERY && discriminator == 3) {+      out.append(&*last, size_t(p - last));+      out.push_back('+');+      ++p;+      last = p;+    } else {+      out.append(&*last, size_t(p - last));+      esc[1] = hexValues[v >> 4];+      esc[2] = hexValues[v & 0x0f];+      out.append(esc, 3);+      ++p;+      last = p;+    }+  }+  out.append(&*last, size_t(p - last));+}++template <class String>+bool tryUriUnescape(StringPiece str, String& out, UriEscapeMode mode) {+  grow_capacity_by(out, str.size());+  auto p = str.begin();+  auto last = p;+  // We advance over runs of passthrough characters and copy them in one go;+  // this is faster than calling push_back repeatedly.+  while (p != str.end()) {+    char c = *p;+    switch (c) {+      case '%': {+        if (FOLLY_UNLIKELY(std::distance(p, str.end()) < 3)) {+          return false;+        }+        auto h1 = detail::hexTable[static_cast<unsigned char>(p[1])];+        auto h2 = detail::hexTable[static_cast<unsigned char>(p[2])];+        if (FOLLY_UNLIKELY(h1 == 16 || h2 == 16)) {+          return false;+        }+        out.append(&*last, size_t(p - last));+        out.push_back(decltype(h1)(h1 << 4) | h2);+        p += 3;+        last = p;+        break;+      }+      case '+':+        if (mode == UriEscapeMode::QUERY) {+          out.append(&*last, size_t(p - last));+          out.push_back(' ');+          ++p;+          last = p;+          break;+        }+        // else fallthrough+        [[fallthrough]];+      default:+        ++p;+        break;+    }+  }+  out.append(&*last, size_t(p - last));++  return true;+}++template <class String>+void uriUnescape(StringPiece str, String& out, UriEscapeMode mode) {+  auto success = tryUriUnescape(str, out, mode);++  if (!success) {+    // tryUriEscape implementation only fails on invalid argument+    throw_exception<std::invalid_argument>(+        "incomplete percent encode sequence");+  }+}++namespace detail {++/*+ * The following functions are type-overloaded helpers for+ * internalSplit().+ */+inline size_t delimSize(char) {+  return 1;+}+inline size_t delimSize(StringPiece s) {+  return s.size();+}+inline bool atDelim(const char* s, char c) {+  return *s == c;+}+inline bool atDelim(const char* s, StringPiece sp) {+  return !std::memcmp(s, sp.start(), sp.size());+}++// These are used to short-circuit internalSplit() in the case of+// 1-character strings.+inline char delimFront(char) {+  // This one exists only for compile-time; it should never be called.+  std::abort();+}+inline char delimFront(StringPiece s) {+  assert(!s.empty() && s.start() != nullptr);+  return *s.start();+}++template <class OutStringT, class DelimT, class OutputIterator>+void internalSplit(+    DelimT delim, StringPiece sp, OutputIterator out, bool ignoreEmpty);++template <class OutStringT, class Container>+std::enable_if_t<+    IsSplitSupportedContainer<Container>::value &&+    HasSimdSplitCompatibleValueType<Container>::value>+internalSplitRecurseChar(+    char delim,+    folly::StringPiece sp,+    std::back_insert_iterator<Container> it,+    bool ignoreEmpty) {+  using base = std::back_insert_iterator<Container>;+  struct accessor : base {+    accessor(base b) : base(b) {}+    using base::container;+  };+  detail::simdSplitByChar(delim, sp, *accessor{it}.container, ignoreEmpty);+}++template <class OutStringT, class Iterator>+void internalSplitRecurseChar(+    char delim, folly::StringPiece sp, Iterator it, bool ignoreEmpty) {+  internalSplit<OutStringT>(delim, sp, it, ignoreEmpty);+}++/*+ * Shared implementation for all the split() overloads.+ *+ * This uses some external helpers that are overloaded to let this+ * algorithm be more performant if the deliminator is a single+ * character instead of a whole string.+ *+ * @param ignoreEmpty if true, don't copy empty segments to output+ */+template <class OutStringT, class DelimT, class OutputIterator>+void internalSplit(+    DelimT delim, StringPiece sp, OutputIterator out, bool ignoreEmpty) {+  assert(sp.empty() || sp.start() != nullptr);++  const char* s = sp.start();+  const size_t strSize = sp.size();+  const size_t dSize = delimSize(delim);++  if (dSize > strSize || dSize == 0) {+    if (!ignoreEmpty || strSize > 0) {+      *out++ = to<OutStringT>(sp);+    }+    return;+  }+  if (std::is_same<DelimT, StringPiece>::value && dSize == 1) {+    // Call the char version because it is significantly faster.+    return internalSplitRecurseChar<OutStringT>(+        delimFront(delim), sp, out, ignoreEmpty);+  }++  size_t tokenStartPos = 0;+  size_t tokenSize = 0;+  for (size_t i = 0; i <= strSize - dSize; ++i) {+    if (atDelim(&s[i], delim)) {+      if (!ignoreEmpty || tokenSize > 0) {+        *out++ = to<OutStringT>(sp.subpiece(tokenStartPos, tokenSize));+      }++      tokenStartPos = i + dSize;+      tokenSize = 0;+      i += dSize - 1;+    } else {+      ++tokenSize;+    }+  }+  tokenSize = strSize - tokenStartPos;+  if (!ignoreEmpty || tokenSize > 0) {+    *out++ = to<OutStringT>(sp.subpiece(tokenStartPos, tokenSize));+  }+}++template <class String>+StringPiece prepareDelim(const String& s) {+  return StringPiece(s);+}+inline char prepareDelim(char c) {+  return c;+}++template <class OutputType>+void toOrIgnore(StringPiece input, OutputType& output) {+  output = folly::to<OutputType>(input);+}++inline void toOrIgnore(StringPiece, decltype(std::ignore)&) {}++template <bool exact, class Delim, class OutputType>+bool splitFixed(const Delim& delimiter, StringPiece input, OutputType& output) {+  if (exact && FOLLY_UNLIKELY(std::string::npos != input.find(delimiter))) {+    return false;+  }+  toOrIgnore(input, output);+  return true;+}++template <bool exact, class Delim, class OutputType, class... OutputTypes>+bool splitFixed(+    const Delim& delimiter,+    StringPiece input,+    OutputType& outHead,+    OutputTypes&... outTail) {+  size_t cut = input.find(delimiter);+  if (FOLLY_UNLIKELY(cut == std::string::npos)) {+    return false;+  }+  StringPiece head(input.begin(), input.begin() + cut);+  StringPiece tail(+      input.begin() + cut + detail::delimSize(delimiter), input.end());+  if (FOLLY_LIKELY(splitFixed<exact>(delimiter, tail, outTail...))) {+    toOrIgnore(head, outHead);+    return true;+  }+  return false;+}++// Overload for no remaining output fields; requires empty input.+template <class Delim>+Expected<Unit, SubstringConversionCode> trySplitTo(+    StringPiece input, const Delim&) {+  if (input.empty()) {+    return unit;+  }+  return makeUnexpected(+      SubstringConversionCode{input, ConversionCode::SPLIT_ERROR});+}++// Replace custom conversion codes with folly::ConversionCode::CUSTOM_OTHER.+template <class CustomCode>+inline ConversionCode convertError(CustomCode&&) {+  return ConversionCode::CUSTOM;+}++inline ConversionCode convertError(ConversionCode code) {+  return code;+}++// tryFieldTo helpers, wrapping tryTo<>, but adding support for std::ignore and+// replacing custom error types with ConversionCode::CUSTOM.+template <class Output>+Expected<Output, ConversionCode> tryFieldTo(folly::StringPiece input) {+  if (auto result = tryTo<Output>(input)) {+    return std::move(result.value());+  } else {+    return makeUnexpected(convertError(result.error()));+  }+}++template <>+inline Expected<decltype(std::ignore), ConversionCode>+tryFieldTo<decltype(std::ignore)>(folly::StringPiece /*input*/) {+  return std::ignore;+}++template <class Delim, class Output, class... Outputs>+Expected<Unit, SubstringConversionCode> trySplitTo(+    StringPiece input,+    const Delim& delim,+    Output& output,+    Outputs&... outputs) {+  auto pos = input.find(delim);+  if ((pos == std::string::npos) != (sizeof...(outputs) == 0)) {+    return makeUnexpected(+        SubstringConversionCode{input, ConversionCode::SPLIT_ERROR});+  }+  StringPiece head, tail;+  if (pos == std::string::npos) {+    head = input;+  } else {+    head = input.subpiece(0, pos);+    tail = input.subpiece(pos + delimSize(delim));+  }+  // Eagerly attempt parsing the head value, but only assign on the way back+  // from the recursive calls to ensure all outputs are untouched on failure.+  if (auto headResult = tryFieldTo<Output>(head)) {+    if (auto tailResult = trySplitTo(tail, delim, outputs...)) {+      output = *headResult;+      return unit;++    } else {+      return makeUnexpected(tailResult.error());+    }+  } else {+    // First failure (left-to-right) is returned.+    return makeUnexpected(SubstringConversionCode{head, headResult.error()});+  }+}++} // namespace detail++//////////////////////////////////////////////////////////////////////++template <class Delim, class String, class OutputType>+std::enable_if_t<+    (!detail::IsSimdSupportedDelim<Delim>::value ||+     !detail::HasSimdSplitCompatibleValueType<OutputType>::value) &&+    detail::IsSplitSupportedContainer<OutputType>::value>+split(+    const Delim& delimiter,+    const String& input,+    OutputType& out,+    bool ignoreEmpty) {+  detail::internalSplit<typename OutputType::value_type>(+      detail::prepareDelim(delimiter),+      StringPiece(input),+      std::back_inserter(out),+      ignoreEmpty);+}++template <+    class OutputValueType,+    class Delim,+    class String,+    class OutputIterator>+void splitTo(+    const Delim& delimiter,+    const String& input,+    OutputIterator out,+    bool ignoreEmpty) {+  detail::internalSplit<OutputValueType>(+      detail::prepareDelim(delimiter), StringPiece(input), out, ignoreEmpty);+}++template <class Delim, class... OutputTypes>+typename std::enable_if<+    StrictConjunction<IsConvertible<OutputTypes>...>::value,+    Expected<Unit, SubstringConversionCode>>::type+trySplitTo(StringPiece input, const Delim& delim, OutputTypes&... outputs) {+  return detail::trySplitTo(input, detail::prepareDelim(delim), outputs...);+}++template <bool exact, class Delim, class... OutputTypes>+typename std::enable_if<+    StrictConjunction<IsConvertible<OutputTypes>...>::value &&+        sizeof...(OutputTypes) >= 1,+    bool>::type+split(const Delim& delimiter, StringPiece input, OutputTypes&... outputs) {+  return detail::splitFixed<exact>(+      detail::prepareDelim(delimiter), input, outputs...);+}++namespace detail {++/*+ * If a type can have its string size determined cheaply, we can more+ * efficiently append it in a loop (see internalJoinAppend). Note that the+ * struct need not conform to the std::string api completely (ex. does not need+ * to implement append()).+ */+template <class T>+struct IsSizableString {+  enum {+    value = IsSomeString<T>::value || std::is_same<T, StringPiece>::value+  };+};++template <class Iterator>+struct IsSizableStringContainerIterator+    : IsSizableString<typename std::iterator_traits<Iterator>::value_type> {};++template <class Delim, class Iterator, class String>+void internalJoinAppend(+    Delim delimiter, Iterator begin, Iterator end, String& output) {+  assert(begin != end);+  if (std::is_same<Delim, StringPiece>::value && delimSize(delimiter) == 1) {+    internalJoinAppend(delimFront(delimiter), begin, end, output);+    return;+  }+  toAppend(*begin, &output);+  while (++begin != end) {+    toAppend(delimiter, *begin, &output);+  }+}++template <class Delim, class Iterator, class String>+typename std::enable_if<IsSizableStringContainerIterator<Iterator>::value>::type+internalJoin(Delim delimiter, Iterator begin, Iterator end, String& output) {+  output.clear();+  if (begin == end) {+    return;+  }+  const size_t dsize = delimSize(delimiter);+  Iterator it = begin;+  size_t size = it->size();+  while (++it != end) {+    size += dsize + it->size();+  }+  output.reserve(size);+  internalJoinAppend(delimiter, begin, end, output);+}++template <class Delim, class Iterator, class String>+typename std::enable_if<+    !IsSizableStringContainerIterator<Iterator>::value>::type+internalJoin(Delim delimiter, Iterator begin, Iterator end, String& output) {+  output.clear();+  if (begin == end) {+    return;+  }+  internalJoinAppend(delimiter, begin, end, output);+}++} // namespace detail++template <class Delim, class Iterator, class String>+void join(+    const Delim& delimiter, Iterator begin, Iterator end, String& output) {+  detail::internalJoin(detail::prepareDelim(delimiter), begin, end, output);+}++template <class OutputString>+void backslashify(+    folly::StringPiece input, OutputString& output, bool hex_style) {+  static const char hexValues[] = "0123456789abcdef";+  output.clear();+  output.reserve(3 * input.size());+  for (unsigned char c : input) {+    // less than space or greater than '~' are considered unprintable+    if (c < 0x20 || c > 0x7e || c == '\\') {+      bool hex_append = false;+      output.push_back('\\');+      if (hex_style) {+        hex_append = true;+      } else {+        if (c == '\r') {+          output += 'r';+        } else if (c == '\n') {+          output += 'n';+        } else if (c == '\t') {+          output += 't';+        } else if (c == '\a') {+          output += 'a';+        } else if (c == '\b') {+          output += 'b';+        } else if (c == '\0') {+          output += '0';+        } else if (c == '\\') {+          output += '\\';+        } else {+          hex_append = true;+        }+      }+      if (hex_append) {+        output.push_back('x');+        output.push_back(hexValues[(c >> 4) & 0xf]);+        output.push_back(hexValues[c & 0xf]);+      }+    } else {+      output += c;+    }+  }+}++template <class String1, class String2>+void humanify(const String1& input, String2& output) {+  size_t numUnprintable = 0;+  size_t numPrintablePrefix = 0;+  for (unsigned char c : input) {+    if (c < 0x20 || c > 0x7e || c == '\\') {+      ++numUnprintable;+    }+    if (numUnprintable == 0) {+      ++numPrintablePrefix;+    }+  }++  // hexlify doubles a string's size; backslashify can potentially+  // explode it by 4x.  Now, the printable range of the ascii+  // "spectrum" is around 95 out of 256 values, so a "random" binary+  // string should be around 60% unprintable.  We use a 50% heuristic+  // here, so if a string is 60% unprintable, then we just use hex+  // output.  Otherwise we backslash.+  //+  // UTF8 is completely ignored; as a result, utf8 characters will+  // likely be \x escaped (since most common glyphs fit in two bytes).+  // This is a tradeoff of complexity/speed instead of a convenience+  // that likely would rarely matter.  Moreover, this function is more+  // about displaying underlying bytes, not about displaying glyphs+  // from languages.+  if (numUnprintable == 0) {+    output = input;+  } else if (5 * numUnprintable >= 3 * input.size()) {+    // However!  If we have a "meaningful" prefix of printable+    // characters, say 20% of the string, we backslashify under the+    // assumption viewing the prefix as ascii is worth blowing the+    // output size up a bit.+    if (5 * numPrintablePrefix >= input.size()) {+      backslashify(input, output);+    } else {+      output = "0x";+      hexlify(input, output, true /* append output */);+    }+  } else {+    backslashify(input, output);+  }+}++template <class InputString, class OutputString>+bool hexlify(+    const InputString& input, OutputString& output, bool append_output) {+  if (!append_output) {+    output.clear();+  }++  static char hexValues[] = "0123456789abcdef";+  auto j = output.size();+  output.resize(2 * input.size() + output.size());+  for (size_t i = 0; i < input.size(); ++i) {+    int ch = input[i];+    output[j++] = hexValues[(ch >> 4) & 0xf];+    output[j++] = hexValues[ch & 0xf];+  }+  return true;+}++template <class InputString, class OutputString>+bool unhexlify(const InputString& input, OutputString& output) {+  if (input.size() % 2 != 0) {+    return false;+  }+  output.resize(input.size() / 2);+  int j = 0;++  for (size_t i = 0; i < input.size(); i += 2) {+    int highBits = detail::hexTable[static_cast<uint8_t>(input[i])];+    int lowBits = detail::hexTable[static_cast<uint8_t>(input[i + 1])];+    if ((highBits | lowBits) & 0x10) {+      // One of the characters wasn't a hex digit+      return false;+    }+    output[j++] = (highBits << 4) + lowBits;+  }+  return true;+}++namespace detail {+/**+ * Hex-dump at most 16 bytes starting at offset from a memory area of size+ * bytes.  Return the number of bytes actually dumped.+ */+size_t hexDumpLine(+    const void* ptr, size_t offset, size_t size, std::string& line);+} // namespace detail++template <class OutIt>+void hexDump(const void* ptr, size_t size, OutIt out) {+  size_t offset = 0;+  std::string line;+  while (offset < size) {+    offset += detail::hexDumpLine(ptr, offset, size, line);+    *out++ = line;+  }+}++} // namespace folly
+ folly/folly/String.cpp view
@@ -0,0 +1,783 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#include <folly/String.h>++#include <cctype>+#include <cerrno>+#include <cstdarg>+#include <cstring>+#include <iterator>+#include <sstream>+#include <stdexcept>++#include <glog/logging.h>++#include <folly/Portability.h>+#include <folly/ScopeGuard.h>+#include <folly/container/Array.h>++namespace folly {++static_assert(IsConvertible<float>::value);+static_assert(IsConvertible<int>::value);+static_assert(IsConvertible<bool>::value);+static_assert(IsConvertible<int>::value);+static_assert(!IsConvertible<std::vector<int>>::value);++namespace detail {++struct string_table_c_escape_make_item {+  constexpr char operator()(std::size_t index) const {+    // clang-format off+    return+        index == '"' ? '"' :+        index == '\\' ? '\\' :+        index == '?' ? '?' :+        index == '\n' ? 'n' :+        index == '\r' ? 'r' :+        index == '\t' ? 't' :+        index < 32 || index > 126 ? 'O' : // octal+        'P'; // printable+    // clang-format on+  }+};++struct string_table_c_unescape_make_item {+  constexpr char operator()(std::size_t index) const {+    // clang-format off+    return+        index == '\'' ? '\'' :+        index == '?' ? '?' :+        index == '\\' ? '\\' :+        index == '"' ? '"' :+        index == 'a' ? '\a' :+        index == 'b' ? '\b' :+        index == 'f' ? '\f' :+        index == 'n' ? '\n' :+        index == 'r' ? '\r' :+        index == 't' ? '\t' :+        index == 'v' ? '\v' :+        index >= '0' && index <= '7' ? 'O' : // octal+        index == 'x' ? 'X' : // hex+        'I'; // invalid+    // clang-format on+  }+};++struct string_table_hex_make_item {+  constexpr unsigned char operator()(std::size_t index) const {+    // clang-format off+    return static_cast<unsigned char>(+        index >= '0' && index <= '9' ? index - '0' :+        index >= 'a' && index <= 'f' ? index - 'a' + 10 :+        index >= 'A' && index <= 'F' ? index - 'A' + 10 :+        16);+    // clang-format on+  }+};++struct string_table_uri_escape_make_item {+  //  0 = passthrough+  //  1 = unused+  //  2 = safe in path (/)+  //  3 = space (replace with '+' in query)+  //  4 = always percent-encode+  constexpr unsigned char operator()(std::size_t index) const {+    // clang-format off+    return+        index >= '0' && index <= '9' ? 0 :+        index >= 'A' && index <= 'Z' ? 0 :+        index >= 'a' && index <= 'z' ? 0 :+        index == '-' ? 0 :+        index == '_' ? 0 :+        index == '.' ? 0 :+        index == '~' ? 0 :+        index == '/' ? 2 :+        index == ' ' ? 3 :+        4;+    // clang-format on+  }+};++FOLLY_STORAGE_CONSTEXPR decltype(cEscapeTable) cEscapeTable =+    make_array_with<256>(string_table_c_escape_make_item{});+FOLLY_STORAGE_CONSTEXPR decltype(cUnescapeTable) cUnescapeTable =+    make_array_with<256>(string_table_c_unescape_make_item{});+FOLLY_STORAGE_CONSTEXPR decltype(hexTable) hexTable =+    make_array_with<256>(string_table_hex_make_item{});+FOLLY_STORAGE_CONSTEXPR decltype(uriEscapeTable) uriEscapeTable =+    make_array_with<256>(string_table_uri_escape_make_item{});++} // namespace detail++static inline bool is_oddspace(char c) {+  return c == '\n' || c == '\t' || c == '\r';+}++StringPiece ltrimWhitespace(StringPiece sp) {+  // Spaces other than ' ' characters are less common but should be+  // checked.  This configuration where we loop on the ' '+  // separately from oddspaces was empirically fastest.++  while (true) {+    while (!sp.empty() && sp.front() == ' ') {+      sp.pop_front();+    }+    if (!sp.empty() && is_oddspace(sp.front())) {+      sp.pop_front();+      continue;+    }++    return sp;+  }+}++StringPiece rtrimWhitespace(StringPiece sp) {+  // Spaces other than ' ' characters are less common but should be+  // checked.  This configuration where we loop on the ' '+  // separately from oddspaces was empirically fastest.++  while (true) {+    while (!sp.empty() && sp.back() == ' ') {+      sp.pop_back();+    }+    if (!sp.empty() && is_oddspace(sp.back())) {+      sp.pop_back();+      continue;+    }++    return sp;+  }+}++namespace {++int stringAppendfImplHelper(+    char* buf, size_t bufsize, const char* format, va_list args) {+  va_list args_copy;+  va_copy(args_copy, args);+  int bytes_used = vsnprintf(buf, bufsize, format, args_copy);+  va_end(args_copy);+  return bytes_used;+}++void stringAppendfImpl(std::string& output, const char* format, va_list args) {+  // Very simple; first, try to avoid an allocation by using an inline+  // buffer.  If that fails to hold the output string, allocate one on+  // the heap, use it instead.+  //+  // It is hard to guess the proper size of this buffer; some+  // heuristics could be based on the number of format characters, or+  // static analysis of a codebase.  Or, we can just pick a number+  // that seems big enough for simple cases (say, one line of text on+  // a terminal) without being large enough to be concerning as a+  // stack variable.+  std::array<char, 128> inline_buffer;++  int bytes_used = stringAppendfImplHelper(+      inline_buffer.data(), inline_buffer.size(), format, args);+  if (bytes_used < 0) {+    throw std::runtime_error(to<std::string>(+        "Invalid format string; snprintf returned negative "+        "with format string: ",+        format));+  }++  if (static_cast<size_t>(bytes_used) < inline_buffer.size()) {+    output.append(inline_buffer.data(), size_t(bytes_used));+    return;+  }++  // Couldn't fit.  Heap allocate a buffer, oh well.+  std::unique_ptr<char[]> heap_buffer(new char[size_t(bytes_used + 1)]);+  int final_bytes_used = stringAppendfImplHelper(+      heap_buffer.get(), size_t(bytes_used + 1), format, args);+  // The second call can take fewer bytes if, for example, we were printing a+  // string buffer with null-terminating char using a width specifier -+  // vsnprintf("%.*s", buf.size(), buf)+  CHECK(bytes_used >= final_bytes_used);++  // We don't keep the trailing '\0' in our output string+  output.append(heap_buffer.get(), size_t(final_bytes_used));+}++} // namespace++std::string stringPrintf(const char* format, ...) {+  va_list ap;+  va_start(ap, format);+  SCOPE_EXIT {+    va_end(ap);+  };+  return stringVPrintf(format, ap);+}++std::string stringVPrintf(const char* format, va_list ap) {+  std::string ret;+  stringAppendfImpl(ret, format, ap);+  return ret;+}++// Basic declarations; allow for parameters of strings and string+// pieces to be specified.+std::string& stringAppendf(std::string* output, const char* format, ...) {+  va_list ap;+  va_start(ap, format);+  SCOPE_EXIT {+    va_end(ap);+  };+  return stringVAppendf(output, format, ap);+}++std::string& stringVAppendf(+    std::string* output, const char* format, va_list ap) {+  stringAppendfImpl(*output, format, ap);+  return *output;+}++void stringPrintf(std::string* output, const char* format, ...) {+  va_list ap;+  va_start(ap, format);+  SCOPE_EXIT {+    va_end(ap);+  };+  return stringVPrintf(output, format, ap);+}++void stringVPrintf(std::string* output, const char* format, va_list ap) {+  output->clear();+  stringAppendfImpl(*output, format, ap);+}++namespace {++struct PrettySuffix {+  const char* suffix;+  double val;+};++const PrettySuffix kPrettyTimeSuffixes[] = {+    {"s ", 1e0L},+    {"ms", 1e-3L},+    {"us", 1e-6L},+    {"ns", 1e-9L},+    {"ps", 1e-12L},+    {"s ", 0},+    {nullptr, 0},+};++const PrettySuffix kPrettyTimeHmsSuffixes[] = {+    {"h ", 60L * 60L},+    {"m ", 60L},+    {"s ", 1e0L},+    {"ms", 1e-3L},+    {"us", 1e-6L},+    {"ns", 1e-9L},+    {"ps", 1e-12L},+    {"s ", 0},+    {nullptr, 0},+};++const PrettySuffix kPrettyBytesMetricSuffixes[] = {+    {"EB", 1e18L},+    {"PB", 1e15L},+    {"TB", 1e12L},+    {"GB", 1e9L},+    {"MB", 1e6L},+    {"kB", 1e3L},+    {"B ", 0},+    {nullptr, 0},+};++const PrettySuffix kPrettyBytesBinarySuffixes[] = {+    {"EB", int64_t(1) << 60},+    {"PB", int64_t(1) << 50},+    {"TB", int64_t(1) << 40},+    {"GB", int64_t(1) << 30},+    {"MB", int64_t(1) << 20},+    {"kB", int64_t(1) << 10},+    {"B ", 0},+    {nullptr, 0},+};++const PrettySuffix kPrettyBytesBinaryIECSuffixes[] = {+    {"EiB", int64_t(1) << 60},+    {"PiB", int64_t(1) << 50},+    {"TiB", int64_t(1) << 40},+    {"GiB", int64_t(1) << 30},+    {"MiB", int64_t(1) << 20},+    {"KiB", int64_t(1) << 10},+    {"B  ", 0},+    {nullptr, 0},+};++const PrettySuffix kPrettyUnitsMetricSuffixes[] = {+    {"qntl", 1e18L},+    {"qdrl", 1e15L},+    {"tril", 1e12L},+    {"bil", 1e9L},+    {"M", 1e6L},+    {"k", 1e3L},+    {" ", 0},+    {nullptr, 0},+};++const PrettySuffix kPrettyUnitsBinarySuffixes[] = {+    {"E", int64_t(1) << 60},+    {"P", int64_t(1) << 50},+    {"T", int64_t(1) << 40},+    {"G", int64_t(1) << 30},+    {"M", int64_t(1) << 20},+    {"k", int64_t(1) << 10},+    {" ", 0},+    {nullptr, 0},+};++const PrettySuffix kPrettyUnitsBinaryIECSuffixes[] = {+    {"Ei", int64_t(1) << 60},+    {"Pi", int64_t(1) << 50},+    {"Ti", int64_t(1) << 40},+    {"Gi", int64_t(1) << 30},+    {"Mi", int64_t(1) << 20},+    {"Ki", int64_t(1) << 10},+    {"  ", 0},+    {nullptr, 0},+};++const PrettySuffix kPrettySISuffixes[] = {+    {"Y", 1e24L},  {"Z", 1e21L},  {"E", 1e18L},  {"P", 1e15L},  {"T", 1e12L},+    {"G", 1e9L},   {"M", 1e6L},   {"k", 1e3L},   {"h", 1e2L},   {"da", 1e1L},+    {"d", 1e-1L},  {"c", 1e-2L},  {"m", 1e-3L},  {"u", 1e-6L},  {"n", 1e-9L},+    {"p", 1e-12L}, {"f", 1e-15L}, {"a", 1e-18L}, {"z", 1e-21L}, {"y", 1e-24L},+    {" ", 0},      {nullptr, 0},+};++const PrettySuffix* const kPrettySuffixes[PRETTY_NUM_TYPES] = {+    kPrettyTimeSuffixes,+    kPrettyTimeHmsSuffixes,+    kPrettyBytesMetricSuffixes,+    kPrettyBytesBinarySuffixes,+    kPrettyBytesBinaryIECSuffixes,+    kPrettyUnitsMetricSuffixes,+    kPrettyUnitsBinarySuffixes,+    kPrettyUnitsBinaryIECSuffixes,+    kPrettySISuffixes,+};++} // namespace++std::string prettyPrint(double val, PrettyType type, bool addSpace) {+  char buf[100];++  // pick the suffixes to use+  assert(type >= 0);+  assert(type < PRETTY_NUM_TYPES);+  const PrettySuffix* suffixes = kPrettySuffixes[type];++  // find the first suffix we're bigger than -- then use it+  double abs_val = fabs(val);+  for (int i = 0; suffixes[i].suffix; ++i) {+    if (abs_val >= suffixes[i].val) {+      snprintf(+          buf,+          sizeof buf,+          "%.4g%s%s",+          (suffixes[i].val != 0. ? (val / suffixes[i].val) : val),+          (addSpace ? " " : ""),+          suffixes[i].suffix);+      return std::string(buf);+    }+  }++  // no suffix, we've got a tiny value -- just print it in sci-notation+  snprintf(buf, sizeof buf, "%.4g", val);+  return std::string(buf);+}++// TODO:+// 1) Benchmark & optimize+double prettyToDouble(+    folly::StringPiece* const prettyString, const PrettyType type) {+  auto value = folly::to<double>(prettyString);+  while (!prettyString->empty() && std::isspace(prettyString->front())) {+    prettyString->advance(1); // Skipping spaces between number and suffix+  }+  const PrettySuffix* suffixes = kPrettySuffixes[type];+  int longestPrefixLen = -1;+  int bestPrefixId = -1;+  for (int j = 0; suffixes[j].suffix; ++j) {+    if (suffixes[j].suffix[0] == ' ') { // Checking for " " -> number rule.+      if (longestPrefixLen == -1) {+        longestPrefixLen = 0; // No characters to skip+        bestPrefixId = j;+      }+    } else if (prettyString->startsWith(suffixes[j].suffix)) {+      int suffixLen = int(strlen(suffixes[j].suffix));+      // We are looking for a longest suffix matching prefix of the string+      // after numeric value. We need this in case suffixes have common prefix.+      if (suffixLen > longestPrefixLen) {+        longestPrefixLen = suffixLen;+        bestPrefixId = j;+      }+    }+  }+  if (bestPrefixId == -1) { // No valid suffix rule found+    throw std::invalid_argument(folly::to<std::string>(+        "Unable to parse suffix \"", *prettyString, "\""));+  }+  prettyString->advance(size_t(longestPrefixLen));+  return suffixes[bestPrefixId].val != 0.+      ? value * suffixes[bestPrefixId].val+      : value;+}++double prettyToDouble(folly::StringPiece prettyString, const PrettyType type) {+  double result = prettyToDouble(&prettyString, type);+  detail::enforceWhitespace(prettyString);+  return result;+}++std::string hexDump(const void* ptr, size_t size) {+  std::ostringstream os;+  hexDump(ptr, size, std::ostream_iterator<StringPiece>(os, "\n"));+  return os.str();+}++// There are two variants of `strerror_r` function, one returns+// `int`, and another returns `char*`. Selecting proper version using+// preprocessor macros portably is extremely hard.+//+// For example, on Android function signature depends on `__USE_GNU` and+// `__ANDROID_API__` macros (https://git.io/fjBBE).+//+// So we are using C++ overloading trick: we pass a pointer of+// `strerror_r` to `invoke_strerror_r` function, and C++ compiler+// selects proper function.++[[maybe_unused]] static std::string invoke_strerror_r(+    int (*strerror_r)(int, char*, size_t), int err, char* buf, size_t buflen) {+  // Using XSI-compatible strerror_r+  int r = strerror_r(err, buf, buflen);++  // OSX/FreeBSD use EINVAL and Linux uses -1 so just check for non-zero+  if (r != 0) {+    return to<std::string>(+        "Unknown error ", err, " (strerror_r failed with error ", errno, ")");+  } else {+    return buf;+  }+}++[[maybe_unused]] static std::string invoke_strerror_r(+    char* (*strerror_r)(int, char*, size_t),+    int err,+    char* buf,+    size_t buflen) {+  // Using GNU strerror_r+  return strerror_r(err, buf, buflen);+}++std::string errnoStr(int err) {+  int savedErrno = errno;++  // Ensure that we reset errno upon exit.+  auto guard(makeGuard([&] { errno = savedErrno; }));++  char buf[1024];+  buf[0] = '\0';++  std::string result;++  // https://developer.apple.com/library/mac/documentation/Darwin/Reference/ManPages/man3/strerror_r.3.html+  // http://www.kernel.org/doc/man-pages/online/pages/man3/strerror.3.html+#if defined(_WIN32) && (defined(__MINGW32__) || defined(_MSC_VER))+  // mingw64 has no strerror_r, but Windows has strerror_s, which C11 added+  // as well. So maybe we should use this across all platforms (together+  // with strerrorlen_s). Note strerror_r and _s have swapped args.+  int r = strerror_s(buf, sizeof(buf), err);+  if (r != 0) {+    result = to<std::string>(+        "Unknown error ", err, " (strerror_r failed with error ", errno, ")");+  } else {+    result.assign(buf);+  }+#else+  // Using any strerror_r+  result.assign(invoke_strerror_r(strerror_r, err, buf, sizeof(buf)));+#endif++  return result;+}++namespace {++void toLowerAscii8(char& c) {+  // Branchless tolower, based on the input-rotating trick described+  // at http://www.azillionmonkeys.com/qed/asmexample.html+  //+  // This algorithm depends on an observation: each uppercase+  // ASCII character can be converted to its lowercase equivalent+  // by adding 0x20.++  // Step 1: Clear the high order bit. We'll deal with it in Step 5.+  auto rotated = uint8_t(c & 0x7f);+  // Currently, the value of rotated, as a function of the original c is:+  //   below 'A':   0- 64+  //   'A'-'Z':    65- 90+  //   above 'Z':  91-127++  // Step 2: Add 0x25 (37)+  rotated += 0x25;+  // Now the value of rotated, as a function of the original c is:+  //   below 'A':   37-101+  //   'A'-'Z':    102-127+  //   above 'Z':  128-164++  // Step 3: clear the high order bit+  rotated &= 0x7f;+  //   below 'A':   37-101+  //   'A'-'Z':    102-127+  //   above 'Z':    0- 36++  // Step 4: Add 0x1a (26)+  rotated += 0x1a;+  //   below 'A':   63-127+  //   'A'-'Z':    128-153+  //   above 'Z':   25- 62++  // At this point, note that only the uppercase letters have been+  // transformed into values with the high order bit set (128 and above).++  // Step 5: Shift the high order bit 2 spaces to the right: the spot+  // where the only 1 bit in 0x20 is.  But first, how we ignored the+  // high order bit of the original c in step 1?  If that bit was set,+  // we may have just gotten a false match on a value in the range+  // 128+'A' to 128+'Z'.  To correct this, need to clear the high order+  // bit of rotated if the high order bit of c is set.  Since we don't+  // care about the other bits in rotated, the easiest thing to do+  // is invert all the bits in c and bitwise-and them with rotated.+  rotated &= ~c;+  rotated >>= 2;++  // Step 6: Apply a mask to clear everything except the 0x20 bit+  // in rotated.+  rotated &= 0x20;++  // At this point, rotated is 0x20 if c is 'A'-'Z' and 0x00 otherwise++  // Step 7: Add rotated to c+  c += char(rotated);+}++void toLowerAscii32(uint32_t& c) {+  // Besides being branchless, the algorithm in toLowerAscii8() has another+  // interesting property: None of the addition operations will cause+  // an overflow in the 8-bit value.  So we can pack four 8-bit values+  // into a uint32_t and run each operation on all four values in parallel+  // without having to use any CPU-specific SIMD instructions.+  uint32_t rotated = c & uint32_t(0x7f7f7f7fUL);+  rotated += uint32_t(0x25252525UL);+  rotated &= uint32_t(0x7f7f7f7fUL);+  rotated += uint32_t(0x1a1a1a1aUL);++  // Step 5 involves a shift, so some bits will spill over from each+  // 8-bit value into the next.  But that's okay, because they're bits+  // that will be cleared by the mask in step 6 anyway.+  rotated &= ~c;+  rotated >>= 2;+  rotated &= uint32_t(0x20202020UL);+  c += rotated;+}++void toLowerAscii64(uint64_t& c) {+  // 64-bit version of toLower32+  uint64_t rotated = c & uint64_t(0x7f7f7f7f7f7f7f7fULL);+  rotated += uint64_t(0x2525252525252525ULL);+  rotated &= uint64_t(0x7f7f7f7f7f7f7f7fULL);+  rotated += uint64_t(0x1a1a1a1a1a1a1a1aULL);+  rotated &= ~c;+  rotated >>= 2;+  rotated &= uint64_t(0x2020202020202020ULL);+  c += rotated;+}++} // namespace++void toLowerAscii(char* str, size_t length) {+  static const size_t kAlignMask64 = 7;+  static const size_t kAlignMask32 = 3;++  // Convert a character at a time until we reach an address that+  // is at least 32-bit aligned+  auto n = (size_t)str;+  n &= kAlignMask32;+  n = std::min(n, length);+  size_t offset = 0;+  if (n != 0) {+    n = std::min(4 - n, length);+    do {+      toLowerAscii8(str[offset]);+      offset++;+    } while (offset < n);+  }++  n = (size_t)(str + offset);+  n &= kAlignMask64;+  if ((n != 0) && (offset + 4 <= length)) {+    // The next address is 32-bit aligned but not 64-bit aligned.+    // Convert the next 4 bytes in order to get to the 64-bit aligned+    // part of the input.+    toLowerAscii32(*(uint32_t*)(str + offset));+    offset += 4;+  }++  // Convert 8 characters at a time+  while (offset + 8 <= length) {+    toLowerAscii64(*(uint64_t*)(str + offset));+    offset += 8;+  }++  // Convert 4 characters at a time+  while (offset + 4 <= length) {+    toLowerAscii32(*(uint32_t*)(str + offset));+    offset += 4;+  }++  // Convert any characters remaining after the last 4-byte aligned group+  while (offset < length) {+    toLowerAscii8(str[offset]);+    offset++;+  }+}++namespace detail {++size_t hexDumpLine(+    const void* ptr, size_t offset, size_t size, std::string& line) {+  static char hexValues[] = "0123456789abcdef";+  // Line layout:+  // 8: address+  // 1: space+  // (1+2)*16: hex bytes, each preceded by a space+  // 1: space separating the two halves+  // 3: "  |"+  // 16: characters+  // 1: "|"+  // Total: 78+  line.clear();+  line.reserve(78);+  const uint8_t* p = reinterpret_cast<const uint8_t*>(ptr) + offset;+  size_t n = std::min(size - offset, size_t(16));+  line.push_back(hexValues[(offset >> 28) & 0xf]);+  line.push_back(hexValues[(offset >> 24) & 0xf]);+  line.push_back(hexValues[(offset >> 20) & 0xf]);+  line.push_back(hexValues[(offset >> 16) & 0xf]);+  line.push_back(hexValues[(offset >> 12) & 0xf]);+  line.push_back(hexValues[(offset >> 8) & 0xf]);+  line.push_back(hexValues[(offset >> 4) & 0xf]);+  line.push_back(hexValues[offset & 0xf]);+  line.push_back(' ');++  for (size_t i = 0; i < n; i++) {+    if (i == 8) {+      line.push_back(' ');+    }++    line.push_back(' ');+    line.push_back(hexValues[(p[i] >> 4) & 0xf]);+    line.push_back(hexValues[p[i] & 0xf]);+  }++  // 3 spaces for each byte we're not printing, one separating the halves+  // if necessary+  line.append(3 * (16 - n) + (n <= 8), ' ');+  line.append("  |");++  for (size_t i = 0; i < n; i++) {+    char c = (p[i] >= 32 && p[i] <= 126 ? static_cast<char>(p[i]) : '.');+    line.push_back(c);+  }+  line.append(16 - n, ' ');+  line.push_back('|');+  DCHECK_EQ(line.size(), 78u);++  return n;+}++} // namespace detail++std::string stripLeftMargin(std::string s) {+  std::vector<StringPiece> pieces;+  split("\n", s, pieces);+  auto piecer = range(pieces);++  auto piece = (piecer.end() - 1);+  auto needle = std::find_if(piece->begin(), piece->end(), [](char c) {+    return c != ' ' && c != '\t';+  });+  if (needle == piece->end()) {+    (piecer.end() - 1)->clear();+  }+  piece = piecer.begin();+  needle = std::find_if(piece->begin(), piece->end(), [](char c) {+    return c != ' ' && c != '\t';+  });+  if (needle == piece->end()) {+    piecer.erase(piecer.begin(), piecer.begin() + 1);+  }++  const auto sentinel = std::numeric_limits<size_t>::max();+  auto indent = sentinel;+  size_t max_length = 0;+  for (piece = piecer.begin(); piece != piecer.end(); piece++) {+    needle = std::find_if(piece->begin(), piece->end(), [](char c) {+      return c != ' ' && c != '\t';+    });+    if (needle != piece->end()) {+      indent = std::min<size_t>(indent, size_t(needle - piece->begin()));+    } else {+      max_length = std::max<size_t>(piece->size(), max_length);+    }+  }+  indent = indent == sentinel ? max_length : indent;+  for (piece = piecer.begin(); piece != piecer.end(); piece++) {+    if (piece->size() < indent) {+      piece->clear();+    } else {+      piece->erase(piece->begin(), piece->begin() + indent);+    }+  }+  return join("\n", piecer);+}++bool SubstringConversionCode::operator==(+    const SubstringConversionCode& other) const {+  return this->code == other.code && this->substring == other.substring;+}++} // namespace folly++#ifdef FOLLY_DEFINED_DMGL+#undef FOLLY_DEFINED_DMGL+#undef DMGL_NO_OPTS+#undef DMGL_PARAMS+#undef DMGL_ANSI+#undef DMGL_JAVA+#undef DMGL_VERBOSE+#undef DMGL_TYPES+#undef DMGL_RET_POSTFIX+#endif
+ folly/folly/String.h view
@@ -0,0 +1,855 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++//+// Docs: https://fburl.com/fbcref_string+//++/**+ * Convenience functions for working with strings.+ *+ * @file String.h+ */++#pragma once+#define FOLLY_STRING_H_++#include <cstdarg>+#include <exception>+#include <string>+#include <unordered_map>+#include <unordered_set>+#include <vector>++#include <folly/Conv.h>+#include <folly/ExceptionString.h>+#include <folly/Optional.h>+#include <folly/Portability.h>+#include <folly/Range.h>+#include <folly/ScopeGuard.h>+#include <folly/Traits.h>+#include <folly/Unit.h>+#include <folly/detail/SimpleSimdStringUtils.h>+#include <folly/detail/SplitStringSimd.h>++namespace folly {++/**+ * @overloadbrief C-escape a string.+ *+ * Make the string suitable for representation as a C string+ * literal.  Appends the result to the output string.+ *+ * Backslashes all occurrences of backslash, double-quote, and question mark:+ *   "  ->  \"+ *   \  ->  \\+ *   ?  ->  \?+ *+ * (Question marks are escaped in order to prevent creating trigraphs in+ * the output -- "??x" where x is one of "=/'()!<>-")+ *+ * Also backslashes certain whitespace characters: \n, \r, \t+ *+ * Replaces all non-printable ASCII characters with backslash-octal+ * representation:+ *   <ASCII 254> -> \376+ *+ * Note that we use backslash-octal instead of backslash-hex because the octal+ * representation is guaranteed to consume no more than 3 characters; "\3760"+ * represents two characters, one with value 254, and one with value 48 ('0'),+ * whereas "\xfe0" represents only one character (with value 4064, which leads+ * to implementation-defined behavior).+ */+template <class String>+void cEscape(StringPiece str, String& out);++/**+ * Similar to cEscape above, but returns the escaped string.+ */+template <class String>+String cEscape(StringPiece str) {+  String out;+  cEscape(str, out);+  return out;+}++/**+ * @overloadbrief C-Unescape a string.+ *+ * The opposite of cEscape above.  Appends the result+ * to the output string.+ *+ * Recognizes the standard C escape sequences:+ *+ * \code+ * \' \" \? \\ \a \b \f \n \r \t \v+ * \[0-7]++ * \x[0-9a-fA-F]++ * \endcode+ *+ * In strict mode (default), throws std::invalid_argument if it encounters+ * an unrecognized escape sequence.  In non-strict mode, it leaves+ * the escape sequence unchanged.+ */+template <class String>+void cUnescape(StringPiece str, String& out, bool strict = true);++/**+ * Similar to cUnescape above, but returns the escaped string.+ */+template <class String>+String cUnescape(StringPiece str, bool strict = true) {+  String out;+  cUnescape(str, out, strict);+  return out;+}++/**+ * @overloadbrief URI-escape a string.+ *+ * Appends the result to the output string.+ *+ * Alphanumeric characters and other characters marked as "unreserved" in RFC+ * 3986 ( -_.~ ) are left unchanged.  In PATH mode, the forward slash (/) is+ * also left unchanged.  In QUERY mode, spaces are replaced by '+'.  All other+ * characters are percent-encoded.+ */+enum class UriEscapeMode : unsigned char {+  // The values are meaningful, see generate_escape_tables.py+  ALL = 0,+  QUERY = 1,+  PATH = 2+};+template <class String>+void uriEscape(+    StringPiece str, String& out, UriEscapeMode mode = UriEscapeMode::ALL);++/**+ * Similar to uriEscape above, but returns the escaped string.+ */+template <class String>+String uriEscape(StringPiece str, UriEscapeMode mode = UriEscapeMode::ALL) {+  String out;+  uriEscape(str, out, mode);+  return out;+}++/**+ * @overloadbrief URI-unescape a string.+ *+ * Appends the result to the output string.+ *+ * In QUERY mode, '+' are replaced by space.  %XX sequences are decoded if+ * XX is a valid hex sequence, otherwise we return an unexpected+ * std::invalid_argument.+ */+template <class String>+bool tryUriUnescape(+    StringPiece str, String& out, UriEscapeMode mode = UriEscapeMode::ALL);++/**+ * Similar to tryUriUnescape above, but returning the unescaped string as a+ * folly::Expected.+ */+template <class String>+folly::Optional<String> tryUriUnescape(+    StringPiece str, UriEscapeMode mode = UriEscapeMode::ALL) {+  String out;+  auto success = tryUriUnescape(str, out, mode);++  if (!success) {+    return folly::none;+  }++  return out;+}++/**+ * Similar to tryUriUnescape above, but without folly::Expected wrapping, and+ * throwing std::invalid_argument on malformed input.+ */+template <class String>+void uriUnescape(+    StringPiece str, String& out, UriEscapeMode mode = UriEscapeMode::ALL);++/**+ * Similar to uriUnescape above, but returns the unescaped string.+ */+template <class String>+String uriUnescape(StringPiece str, UriEscapeMode mode = UriEscapeMode::ALL) {+  String out;+  uriUnescape(str, out, mode);+  return out;+}++/**+ * @overloadbrief printf into a string.+ *+ * stringPrintf is much like printf but deposits its result into a+ * string. Two signatures are supported: the first simply returns the+ * resulting string, and the second appends the produced characters to+ * the specified string and returns a reference to it.+ */+std::string stringPrintf(FOLLY_PRINTF_FORMAT const char* format, ...)+    FOLLY_PRINTF_FORMAT_ATTR(1, 2);++/* Similar to stringPrintf, with different signature. */+void stringPrintf(std::string* out, FOLLY_PRINTF_FORMAT const char* format, ...)+    FOLLY_PRINTF_FORMAT_ATTR(2, 3);++/**+ * Append printf-style output to string.+ */+std::string& stringAppendf(+    std::string* output, FOLLY_PRINTF_FORMAT const char* format, ...)+    FOLLY_PRINTF_FORMAT_ATTR(2, 3);++/**+ * @overloadbrief stringPrintf with va_list argument+ *+ * As with vsnprintf() itself, the value of ap is undefined after the call.+ * These functions do not call va_end() on ap.+ */+std::string stringVPrintf(const char* format, va_list ap);+void stringVPrintf(std::string* out, const char* format, va_list ap);++/**+ * Append va_list printf-style output to string.+ */+std::string& stringVAppendf(std::string* out, const char* format, va_list ap);++/**+ * Backslashify a string.+ *+ * That is, replace non-printable characters+ * with C-style (but NOT C compliant) "\xHH" encoding.  If hex_style+ * is false, then shorthand notations like "\0" will be used instead+ * of "\x00" for the most common backslash cases.+ *+ * There are two forms, one returning the input string, and one+ * creating output in the specified output string.+ *+ * This is mainly intended for printing to a terminal, so it is not+ * particularly optimized.+ *+ * Do *not* use this in situations where you expect to be able to feed+ * the string to a C or C++ compiler, as there are nuances with how C+ * parses such strings that lead to failures.  This is for display+ * purposed only.  If you want a string you can embed for use in C or+ * C++, use cEscape instead.  This function is for display purposes+ * only.+ */+template <class OutputString>+void backslashify(+    folly::StringPiece input, OutputString& output, bool hex_style = false);++template <class OutputString = std::string>+OutputString backslashify(StringPiece input, bool hex_style = false) {+  OutputString output;+  backslashify(input, output, hex_style);+  return output;+}++/**+ * Take a string and "humanify" it -- that is, make it look better.+ *+ * Since "better" is subjective, caveat emptor.  The basic approach is+ * to count the number of unprintable characters.  If there are none,+ * then the output is the input.  If there are relatively few, or if+ * there is a long "enough" prefix of printable characters, use+ * backslashify.  If it is mostly binary, then simply hex encode.+ *+ * This is an attempt to make a computer smart, and so likely is wrong+ * most of the time.+ */+template <class String1, class String2>+void humanify(const String1& input, String2& output);++template <class String>+String humanify(const String& input) {+  String output;+  humanify(input, output);+  return output;+}++/**+ * Convert input to hexadecimal representation.+ *+ * Same functionality as Python's binascii.hexlify.  Returns true+ * on successful conversion.+ *+ * If append_output is true, append data to the output rather than+ * replace it.+ */+template <class InputString, class OutputString>+bool hexlify(+    const InputString& input, OutputString& output, bool append = false);++template <class OutputString = std::string>+OutputString hexlify(ByteRange input) {+  OutputString output;+  if (!hexlify(input, output)) {+    // hexlify() currently always returns true, so this can't really happen+    throw_exception<std::runtime_error>("hexlify failed");+  }+  return output;+}++template <class OutputString = std::string>+OutputString hexlify(StringPiece input) {+  return hexlify<OutputString>(ByteRange{input});+}++/**+ * Get binary data from hexadecimal representation.+ *+ * Same functionality as Python's binascii.unhexlify.  Returns true+ * on successful conversion.+ */+template <class InputString, class OutputString>+bool unhexlify(const InputString& input, OutputString& output);++template <class OutputString = std::string>+OutputString unhexlify(StringPiece input) {+  OutputString output;+  if (!unhexlify(input, output)) {+    // unhexlify() fails if the input has non-hexidecimal characters,+    // or if it doesn't consist of a whole number of bytes+    throw_exception<std::domain_error>("unhexlify() called with non-hex input");+  }+  return output;+}++enum PrettyType {+  PRETTY_TIME,+  PRETTY_TIME_HMS,++  PRETTY_BYTES_METRIC,+  PRETTY_BYTES_BINARY,+  PRETTY_BYTES = PRETTY_BYTES_BINARY,+  PRETTY_BYTES_BINARY_IEC,+  PRETTY_BYTES_IEC = PRETTY_BYTES_BINARY_IEC,++  PRETTY_UNITS_METRIC,+  PRETTY_UNITS_BINARY,+  PRETTY_UNITS_BINARY_IEC,++  PRETTY_SI,+  PRETTY_NUM_TYPES,+};++/**+ * Pretty printer for numbers with units.+ *+ * A pretty-printer for numbers that appends suffixes of units of the+ * given type.  It prints 4 sig-figs of value with the most+ * appropriate unit.+ *+ * If `addSpace' is true, we put a space between the units suffix and+ * the value.+ *+ * Current types are:+ *     PRETTY_TIME         - s, ms, us, ns, etc.+ *     PRETTY_TIME_HMS     - h, m, s, ms, us, ns, etc.+ *     PRETTY_BYTES_METRIC - kB, MB, GB, etc (goes up by 10^3 = 1000 each time)+ *     PRETTY_BYTES        - kB, MB, GB, etc (goes up by 2^10 = 1024 each time)+ *     PRETTY_BYTES_IEC    - KiB, MiB, GiB, etc+ *     PRETTY_UNITS_METRIC - k, M, G, etc (goes up by 10^3 = 1000 each time)+ *     PRETTY_UNITS_BINARY - k, M, G, etc (goes up by 2^10 = 1024 each time)+ *     PRETTY_UNITS_BINARY_IEC - Ki, Mi, Gi, etc+ *     PRETTY_SI           - full SI metric prefixes from yocto to Yotta+ *                           http://en.wikipedia.org/wiki/Metric_prefix+ *+ */+std::string prettyPrint(double val, PrettyType, bool addSpace = true);++/**+ * @overloadbrief Reverse prettyPrint.+ *+ * This utility converts StringPiece in pretty format (look above) to double,+ * with progress information. Alters the  StringPiece parameter+ * to get rid of the already-parsed characters.+ * Expects string in form <floating point number> {space}* [<suffix>]+ * If string is not in correct format, utility finds longest valid prefix and+ * if there at least one, returns double value based on that prefix and+ * modifies string to what is left after parsing. Throws and std::range_error+ * exception if there is no correct parse.+ * Examples(for PRETTY_UNITS_METRIC):+ * '10M' => 10 000 000+ * '10 M' => 10 000 000+ * '10' => 10+ * '10 Mx' => 10 000 000, prettyString == "x"+ * 'abc' => throws std::range_error+ */+double prettyToDouble(+    folly::StringPiece* const prettyString, const PrettyType type);++/**+ * Same as prettyToDouble(folly::StringPiece*, PrettyType), but+ * expects whole string to be correctly parseable. Throws std::range_error+ * otherwise+ */+double prettyToDouble(folly::StringPiece prettyString, const PrettyType type);++/**+ * @overloadbrief Write a hex dump of size bytes starting at ptr to out.+ *+ * The hex dump is formatted as follows:+ *+ * for the string "abcdefghijklmnopqrstuvwxyz\x02"+00000000  61 62 63 64 65 66 67 68  69 6a 6b 6c 6d 6e 6f 70  |abcdefghijklmnop|+00000010  71 72 73 74 75 76 77 78  79 7a 02                 |qrstuvwxyz.     |+ *+ * that is, we write 16 bytes per line, both as hex bytes and as printable+ * characters.  Non-printable characters are replaced with '.'+ * Lines are written to out one by one (one StringPiece at a time) without+ * delimiters.+ */+template <class OutIt>+void hexDump(const void* ptr, size_t size, OutIt out);++/**+ * Return the hex dump of size bytes starting at ptr as a string.+ */+std::string hexDump(const void* ptr, size_t size);++/**+ * Pretty print an errno.+ *+ * Return a string containing the description of the given errno value.+ * Takes care not to overwrite the actual system errno, so calling+ * errnoStr(errno) is valid.+ */+std::string errnoStr(int err);++template <typename T, std::size_t M, typename P>+class small_vector;++template <typename T, typename Allocator>+class fbvector;++namespace detail {++// We don't use SimdSplitByCharIsDefinedFor because+// we would like the user to get an error where they could use SIMD+// implementation but didn't use quite correct parameters.+template <typename>+struct IsSplitSupportedContainer : std::false_type {};++template <typename T>+using HasSimdSplitCompatibleValueType =+    std::is_convertible<typename T::value_type, folly::StringPiece>;++template <typename T, typename A>+struct IsSplitSupportedContainer<std::vector<T, A>> : std::true_type {};++template <typename T, typename A>+struct IsSplitSupportedContainer<fbvector<T, A>> : std::true_type {};++template <typename T, std::size_t M, typename P>+struct IsSplitSupportedContainer<small_vector<T, M, P>> : std::true_type {};++template <typename>+struct IsSimdSupportedDelim : std::false_type {};++template <>+struct IsSimdSupportedDelim<char> : std::true_type {};++} // namespace detail++/**+ * Split a string into a list of tokens by delimiter.+ *+ * The split interface here supports different output types, selected+ * at compile time: StringPiece, fbstring, or std::string.  If you are+ * using a vector to hold the output, it detects the type based on+ * what your vector contains.  If the output vector is not empty, split+ * will append to the end of the vector.+ *+ * You can also use splitTo() to write the output to an arbitrary+ * OutputIterator (e.g. std::inserter() on a std::set<>), in which+ * case you have to tell the function the type.  (Rationale:+ * OutputIterators don't have a value_type, so we can't detect the+ * type in splitTo without being told.)+ *+ * Examples:+ *+ *   std::vector<folly::StringPiece> v;+ *   folly::split(':', "asd:bsd", v);+ *+ *   folly::small_vector<folly::StringPiece, 3> v;+ *   folly::split(':', "asd:bsd:csd", v)+ *+ *   std::set<StringPiece> s;+ *   folly::splitTo<StringPiece>("::", "asd::bsd::asd::csd",+ *    std::inserter(s, s.begin()));+ *+ * Split also takes a flag (ignoreEmpty) that indicates whether adjacent+ * delimiters should be treated as one single separator (ignoring empty tokens)+ * or not (generating empty tokens).+ */++template <class Delim, class String, class OutputType>+FOLLY_ALWAYS_INLINE std::enable_if_t<+    detail::IsSimdSupportedDelim<Delim>::value &&+    detail::HasSimdSplitCompatibleValueType<OutputType>::value &&+    detail::IsSplitSupportedContainer<OutputType>::value>+split(+    const Delim& delimiter,+    const String& input,+    OutputType& out,+    const bool ignoreEmpty = false) {+  return detail::simdSplitByChar(delimiter, input, out, ignoreEmpty);+}++template <class Delim, class String, class OutputType>+std::enable_if_t<+    (!detail::IsSimdSupportedDelim<Delim>::value ||+     !detail::HasSimdSplitCompatibleValueType<OutputType>::value) &&+    detail::IsSplitSupportedContainer<OutputType>::value>+split(+    const Delim& delimiter,+    const String& input,+    OutputType& out,+    const bool ignoreEmpty = false);++/**+ * split, to an output iterator+ */+template <+    class OutputValueType,+    class Delim,+    class String,+    class OutputIterator>+void splitTo(+    const Delim& delimiter,+    const String& input,+    OutputIterator out,+    const bool ignoreEmpty = false);++namespace detail {+template <typename Void, typename OutputType>+struct IsConvertible : std::false_type {};++template <>+struct IsConvertible<void, decltype(std::ignore)> : std::true_type {};++template <typename OutputType>+struct IsConvertible<+    void_t<decltype(parseTo(StringPiece{}, std::declval<OutputType&>()))>,+    OutputType> : std::true_type {};+} // namespace detail+template <typename OutputType>+struct IsConvertible : detail::IsConvertible<void, OutputType> {};++/**+ * Split a string into a fixed number of string pieces and/or numeric types+ * by delimiter. Conversions are supported for any type which folly:to<> can+ * target, including all overloads of parseTo(). Returns 'true' if the fields+ * were all successfully populated.  Returns 'false' if there were too few+ * fields in the input, or too many fields if exact=true.  Casting exceptions+ * will not be caught.+ *+ * Examples:+ *+ *  folly::StringPiece name, key, value;+ *  if (folly::split('\t', line, name, key, value))+ *    ...+ *+ *  folly::StringPiece name;+ *  double value;+ *  int id;+ *  if (folly::split('\t', line, name, value, id))+ *    ...+ *+ * The 'exact' template parameter specifies how the function behaves when too+ * many fields are present in the input string. When 'exact' is set to its+ * default value of 'true', a call to split will fail if the number of fields in+ * the input string does not exactly match the number of output parameters+ * passed. If 'exact' is overridden to 'false', all remaining fields will be+ * stored, unsplit, in the last field, as shown below:+ *+ *  folly::StringPiece x, y.+ *  if (folly::split<false>(':', "a:b:c", x, y))+ *    assert(x == "a" && y == "b:c");+ *+ * Note that this will likely not work if the last field's target is of numeric+ * type, in which case folly::to<> will throw an exception.+ */+template <bool exact = true, class Delim, class... OutputTypes>+typename std::enable_if<+    StrictConjunction<IsConvertible<OutputTypes>...>::value &&+        sizeof...(OutputTypes) >= 1,+    bool>::type+split(const Delim& delimiter, StringPiece input, OutputTypes&... outputs);++// Error type for trySplitTo(), below.+struct SubstringConversionCode {+  StringPiece substring;+  ConversionCode code;+  bool operator==(const SubstringConversionCode& other) const;+};++/**+ * Try to split a string into a fixed number of fields by delimiter, using+ * folly::tryTo<> for conversions. types by delimiter.+ * - On success, all output values will be initialized and the 'Unit{}' value is+ *   returned. Arguments are assigned in reverse order.+ * - On failure, the first failing 'ConversionCode' is returned with its+ *   associated substring in a 'SubstringConversionCode'.+ * - String splitting is performed prior to each conversion; field values will+ *   not contain the delimiter.+ * - All custom error codes are mapped to ConversionCode::CUSTOM.+ *+ * Examples:+ *+ *  folly::StringPiece name, key, value;+ *  if (folly::trySplitTo(line, '\t',  name, key, value))+ *    ...+ *+ *  folly::StringPiece name;+ *  double value;+ *  int id;+ *  if (folly::trySplitTo(line, '\t', name, value, id))+ *    ...+ *+ */+template <class Delim, class... OutputTypes>+typename std::enable_if<+    StrictConjunction<IsConvertible<OutputTypes>...>::value,+    Expected<Unit, SubstringConversionCode>>::type+trySplitTo(StringPiece input, const Delim& delimiter, OutputTypes&... outputs);++/**+ * Join list of tokens.+ *+ * Stores a string representation of tokens in the same order with+ * delimiter between each element.+ */+template <class Delim, class Iterator, class String>+void join(const Delim& delimiter, Iterator begin, Iterator end, String& output);++template <class Delim, class Container, class String>+void join(const Delim& delimiter, const Container& container, String& output) {+  join(delimiter, container.begin(), container.end(), output);+}++template <class Delim, class Value, class String>+void join(+    const Delim& delimiter,+    const std::initializer_list<Value>& values,+    String& output) {+  join(delimiter, values.begin(), values.end(), output);+}++template <class Delim, class Container>+std::string join(const Delim& delimiter, const Container& container) {+  std::string output;+  join(delimiter, container.begin(), container.end(), output);+  return output;+}++template <class Delim, class Value>+std::string join(+    const Delim& delimiter, const std::initializer_list<Value>& values) {+  std::string output;+  join(delimiter, values.begin(), values.end(), output);+  return output;+}++template <+    class Delim,+    class Iterator,+    typename std::enable_if<std::is_base_of<+        std::forward_iterator_tag,+        typename std::iterator_traits<Iterator>::iterator_category>::value>::+        type* = nullptr>+std::string join(const Delim& delimiter, Iterator begin, Iterator end) {+  std::string output;+  join(delimiter, begin, end, output);+  return output;+}++/**+ * Remove leading whitespace.+ *+ * Returns a subpiece with all whitespace removed from the front of @sp.+ * Whitespace means any of [' ', '\n', '\r', '\t'].+ */+StringPiece ltrimWhitespace(StringPiece sp);++/**+ * Remove trailing whitespace.+ *+ * Returns a subpiece with all whitespace removed from the back of @sp.+ * Whitespace means any of [' ', '\n', '\r', '\t'].+ */+StringPiece rtrimWhitespace(StringPiece sp);++/**+ * Remove leading and trailing whitespace.+ *+ * Returns a subpiece with all whitespace removed from the back and front of+ * @sp. Whitespace means any of [' ', '\n', '\r', '\t'].+ */+inline StringPiece trimWhitespace(StringPiece sp) {+  return ltrimWhitespace(rtrimWhitespace(sp));+}++/**+ * DEPRECATED: Use ltrimWhitespace instead+ *+ * Returns a subpiece with all whitespace removed from the front of @sp.+ * Whitespace means any of [' ', '\n', '\r', '\t'].+ */+inline StringPiece skipWhitespace(StringPiece sp) {+  return ltrimWhitespace(sp);+}++/**+ * Specify characters to ltrim.+ *+ * Returns a subpiece with all characters the provided @toTrim returns true+ * for removed from the front of @sp.+ */+template <typename ToTrim>+StringPiece ltrim(StringPiece sp, ToTrim toTrim) {+  while (!sp.empty() && toTrim(sp.front())) {+    sp.pop_front();+  }++  return sp;+}++/**+ * Specify characters to rtrim.+ *+ * Returns a subpiece with all characters the provided @toTrim returns true+ * for removed from the back of @sp.+ */+template <typename ToTrim>+StringPiece rtrim(StringPiece sp, ToTrim toTrim) {+  while (!sp.empty() && toTrim(sp.back())) {+    sp.pop_back();+  }++  return sp;+}++/**+ * Specify characters to trim.+ *+ * Returns a subpiece with all characters the provided @toTrim returns true+ * for removed from the back and front of @sp.+ */+template <typename ToTrim>+StringPiece trim(StringPiece sp, ToTrim toTrim) {+  return ltrim(rtrim(sp, std::ref(toTrim)), std::ref(toTrim));+}++/**+ * De-indent a string.+ *+ * Strips the leading and the trailing whitespace-only lines. Then looks for+ * the least indented non-whitespace-only line and removes its amount of+ * leading whitespace from every line. Assumes leading whitespace is either all+ * spaces or all tabs.+ *+ * Purpose: including a multiline string literal in source code, indented to+ * the level expected from context.+ */+std::string stripLeftMargin(std::string s);++/**+ * Convert ascii to lowercase, in-place.+ *+ * Leaves all other characters unchanged, including those with the 0x80+ * bit set.+ * @param str String to convert+ * @param length Length of str, in bytes+ */+void toLowerAscii(char* str, size_t length);++inline void toLowerAscii(MutableStringPiece str) {+  toLowerAscii(str.begin(), str.size());+}++inline void toLowerAscii(std::string& str) {+  // str[0] is legal also if the string is empty.+  toLowerAscii(&str[0], str.size());+}++/**+ * Returns if string contains std::isspace or std::iscntrl characters.+ **/+inline bool hasSpaceOrCntrlSymbols(folly::StringPiece s) {+  return detail::simdHasSpaceOrCntrlSymbols(s);+}++struct format_string_for_each_named_arg_fn {+  struct options {+    bool numeric_args_as_named = false;++    options& set_numeric_args_as_named(bool value) noexcept {+      numeric_args_as_named = value;+      return *this;+    }+  };++  template <typename C, typename CT, typename Fn>+  constexpr void operator()(std::basic_string_view<C, CT> str, Fn fn) const+      noexcept(noexcept(fn(str))) {+    return operator()(options{}, str, std::ref(fn));+  }++  template <typename C, typename CT, typename Fn>+  constexpr void operator()(+      options const& opts, std::basic_string_view<C, CT> str, Fn fn) const+      noexcept(noexcept(fn(str))) {+    using view = std::basic_string_view<C, CT>;+    while (true) {+      auto const pos = str.find('{');+      auto const beg = pos == view::npos ? str.size() : pos + 1;+      if (beg == str.size()) {+        return; // completed+      }+      if (str[beg] == '{') {+        str = str.substr(beg + 1);+        continue; // escaped+      }+      auto const end = std::min(str.find('}', pos), str.find(':', pos));+      if (end == view::npos) {+        return; // malformed+      }+      auto const arg = str.substr(beg, end - beg);+      auto const c = arg.empty() ? 0 : arg[0];+      if (c && (opts.numeric_args_as_named || !(c >= '0' && c <= '9'))) {+        fn(arg);+      }+      str = str.substr(end);+    }+  }+};++inline constexpr format_string_for_each_named_arg_fn+    format_string_for_each_named_arg{};++using format_string_for_each_named_arg_options =+    format_string_for_each_named_arg_fn::options;++} // namespace folly++#include <folly/String-inl.h>
+ folly/folly/Subprocess.cpp view
@@ -0,0 +1,1449 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#ifndef _GNU_SOURCE+#define _GNU_SOURCE+#endif++#include <folly/Subprocess.h>++#if defined(__linux__)+#include <sys/prctl.h>+#endif+#include <dlfcn.h>+#include <fcntl.h>++#include <algorithm>+#include <array>+#include <system_error>+#include <thread>++#include <boost/container/flat_set.hpp>+#include <boost/range/adaptors.hpp>++#include <folly/Conv.h>+#include <folly/Exception.h>+#include <folly/ScopeGuard.h>+#include <folly/String.h>+#include <folly/io/Cursor.h>+#include <folly/lang/Assume.h>+#include <folly/logging/xlog.h>+#include <folly/portability/Dirent.h>+#include <folly/portability/Fcntl.h>+#include <folly/portability/Sockets.h>+#include <folly/portability/Stdlib.h>+#include <folly/portability/SysSyscall.h>+#include <folly/portability/Unistd.h>+#include <folly/system/AtFork.h>+#include <folly/system/Shell.h>++/// interceptors to work around:+///+/// https://github.com/llvm/llvm-project/blob/llvmorg-19.1.7/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp+/// https://github.com/llvm/llvm-project/blob/llvmorg-19.1.7/compiler-rt/lib/sanitizer_common/sanitizer_signal_interceptors.inc++/// In sanitized builds, explicitly disable sanitization in the child process.+/// * Disable sanitizer function transformation, including __tsan_func_entry and+///   __tsan_func_exit hooks (under clang).+/// * Bypass sanitizer interceptors of libc/posix functions, including of vfork+///   and of all other libc/posix callees in the child process. Interceptors+///   look like __interceptor_trampoline_{name} for libc/posix function {name}.++#if __has_attribute(disable_sanitizer_instrumentation)+#define FOLLY_DETAIL_SUBPROCESS_RAW                  \+  __attribute__((                                    \+      noinline,                                      \+      no_sanitize("address", "undefined", "thread"), \+      disable_sanitizer_instrumentation))+#else+#define FOLLY_DETAIL_SUBPROCESS_RAW \+  __attribute__((noinline, no_sanitize("address", "undefined", "thread")))+#endif++constexpr int kExecFailure = 127;+constexpr int kChildFailure = 126;++namespace folly {++namespace detail {++SubprocessFdActionsList::SubprocessFdActionsList(+    span<value_type const> rep) noexcept+    : begin_{rep.data()}, end_{rep.data() + rep.size()} {+  [[maybe_unused]] auto lt = [](auto a, auto b) { return a.first < b.first; };+  assert(std::is_sorted(begin_, end_, lt));+  [[maybe_unused]] auto eq = [](auto a, auto b) { return a.first == b.first; };+  assert(std::adjacent_find(begin_, end_, eq) == end_);+}++FOLLY_DETAIL_SUBPROCESS_RAW+auto SubprocessFdActionsList::begin() const noexcept -> value_type const* {+  return begin_;+}+FOLLY_DETAIL_SUBPROCESS_RAW+auto SubprocessFdActionsList::end() const noexcept -> value_type const* {+  return end_;+}+FOLLY_DETAIL_SUBPROCESS_RAW+auto SubprocessFdActionsList::find(int fd) const noexcept -> int const* {+  auto lo = begin_;+  auto hi = end_;+  while (lo < hi) {+    auto mid = lo + (hi - lo) / 2;+    if (mid->first == fd) {+      return &mid->second;+    }+    if (mid->first < fd) {+      lo = mid + 1;+    } else {+      hi = mid;+    }+  }+  return nullptr;+}++// clang-format off+static inline constexpr auto subprocess_libc_soname =+    kIsLinux ? "libc.so.6" :+    kIsFreeBSD ? "libc.so.7" :+    kIsApple ? "/usr/lib/libSystem.B.dylib" :+    nullptr;+// clang-format on++template <typename Ret>+static Ret subprocess_libc_load(+    void* const handle, Ret const ptr, char const* const name) {+  return !kIsSanitize ? ptr : reinterpret_cast<Ret>(::dlsym(handle, name));+}++#define FOLLY_DETAIL_SUBPROCESS_LIBC_X_BASE(X) \+  X(_exit, _exit)                              \+  X(close, close)                              \+  X(dup2, dup2)                                \+  X(fcntl, fcntl)                              \+  X(pthread_sigmask, pthread_sigmask)          \+  X(signal, signal)                            \+  X(sprintf, sprintf)                          \+  X(strtol, strtol)                            \+  X(vfork, vfork)                              \+  X(write, write)++#if defined(__BIONIC_INCLUDE_FORTIFY_HEADERS)+#define FOLLY_DETAIL_SUBPROCESS_LIBC_X_OPEN(X) \+  X(open, __open_real)                         \+  X(openat, __openat_real)+#else+#define FOLLY_DETAIL_SUBPROCESS_LIBC_X_OPEN(X) \+  X(open, open)                                \+  X(openat, openat)+#endif++#if defined(__linux__)+#define FOLLY_DETAIL_SUBPROCESS_LIBC_X_PRCTL(X) X(prctl, prctl)+#else+#define FOLLY_DETAIL_SUBPROCESS_LIBC_X_PRCTL(X)+#endif++#define FOLLY_DETAIL_SUBPROCESS_LIBC_X(X) \+  FOLLY_DETAIL_SUBPROCESS_LIBC_X_BASE(X)  \+  FOLLY_DETAIL_SUBPROCESS_LIBC_X_OPEN(X)  \+  FOLLY_DETAIL_SUBPROCESS_LIBC_X_PRCTL(X)++#define FOLLY_DETAIL_SUBPROCESS_LIBC_FIELD_DECL(name, func) \+  static decltype(&::func) name;+#define FOLLY_DETAIL_SUBPROCESS_LIBC_FIELD_DEFN(name, func) \+  decltype(&::func) subprocess_libc::name;+#define FOLLY_DETAIL_SUBPROCESS_LIBC_INIT(name, func) \+  subprocess_libc::name = subprocess_libc_load(handle, &::func, #name);++struct subprocess_libc {+  FOLLY_DETAIL_SUBPROCESS_LIBC_X(FOLLY_DETAIL_SUBPROCESS_LIBC_FIELD_DECL)+};++FOLLY_DETAIL_SUBPROCESS_LIBC_X(FOLLY_DETAIL_SUBPROCESS_LIBC_FIELD_DEFN)++__attribute__((constructor(101))) static void subprocess_libc_init() {+  auto handle = !kIsSanitize+      ? nullptr+      : ::dlopen(subprocess_libc_soname, RTLD_LAZY | RTLD_LOCAL | RTLD_NOLOAD);+  assert(!kIsSanitize || !!handle);+  FOLLY_DETAIL_SUBPROCESS_LIBC_X(FOLLY_DETAIL_SUBPROCESS_LIBC_INIT)+}++#undef FOLLY_DETAIL_SUBPROCESS_LIBC_FIELD_DECL+#undef FOLLY_DETAIL_SUBPROCESS_LIBC_FIELD_DEFN+#undef FOLLY_DETAIL_SUBPROCESS_LIBC_INIT+#undef FOLLY_DETAIL_SUBPROCESS_LIBC_X+#undef FOLLY_DETAIL_SUBPROCESS_LIBC_X_PRCTL+#undef FOLLY_DETAIL_SUBPROCESS_LIBC_X_OPEN+#undef FOLLY_DETAIL_SUBPROCESS_LIBC_X_BASE++} // namespace detail++struct Subprocess::SpawnRawArgs {+  struct Scratch {+    std::vector<std::pair<int, int>> fdActions;+    std::vector<char*> setPrintPidToBuffer;+    std::vector<std::pair<int, Options::AttrWithMeta<rlimit>>> rlimits;++    explicit Scratch(Options const& options)+        : fdActions{options.fdActions_.begin(), options.fdActions_.end()},+          setPrintPidToBuffer{+              options.setPrintPidToBuffer_.begin(),+              options.setPrintPidToBuffer_.end()},+          rlimits{options.rlimits_.begin(), options.rlimits_.end()} {+      std::sort(fdActions.begin(), fdActions.end());+    }+  };++  template <typename T>+  struct AttrWithMeta {+    T value{};+    int* errout{};+  };++  static char const* getCStrForNonEmpty(std::string const& str) {+    return str.empty() ? nullptr : str.c_str();+  }++  // from options+  char const* childDir{};+  AttrWithMeta<int> linuxCGroupFd{-1, nullptr};+  AttrWithMeta<char const*> linuxCGroupPath{nullptr, nullptr};+  bool closeOtherFds{};+#if defined(__linux__)+  Options::AttrWithMeta<cpu_set_t> const* cpuSet{};+#endif+  bool detach{};+  detail::SubprocessFdActionsList fdActions;+  int parentDeathSignal{};+  bool processGroupLeader{};+  bool usePath{};+  Options::AttrWithMeta<uid_t> const* uid{};+  Options::AttrWithMeta<gid_t> const* gid{};+  Options::AttrWithMeta<uid_t> const* euid{};+  Options::AttrWithMeta<gid_t> const* egid{};+  char* const* setPrintPidToBufferData{};+  size_t setPrintPidToBufferSize{};+  std::pair<int, Options::AttrWithMeta<rlimit>> const* rlimitsData{};+  size_t rlimitsSize{};++  // assigned explicitly+  char const* const* argv{};+  char const* const* envv{};+  char const* executable{};+  ChildErrorInfo* err{};+  sigset_t oldSignals{};++  explicit SpawnRawArgs(Scratch const& scratch, Options const& options)+      : childDir{getCStrForNonEmpty(options.childDir_)},+        linuxCGroupFd{+            options.linuxCGroupFd_.value, options.linuxCGroupFd_.errout},+        linuxCGroupPath{+            getCStrForNonEmpty(options.linuxCGroupPath_.value),+            options.linuxCGroupPath_.errout},+        closeOtherFds{options.closeOtherFds_},+#if defined(__linux__)+        cpuSet{get_pointer(options.cpuSet_)},+#endif+        detach{options.detach_},+        fdActions{scratch.fdActions},+#if defined(__linux__)+        parentDeathSignal{options.parentDeathSignal_},+#endif+        processGroupLeader{options.processGroupLeader_},+        usePath{options.usePath_},+        uid{options.uid_.get_pointer()},+        gid{options.gid_.get_pointer()},+        euid{options.euid_.get_pointer()},+        egid{options.egid_.get_pointer()},+        setPrintPidToBufferData{scratch.setPrintPidToBuffer.data()},+        setPrintPidToBufferSize{scratch.setPrintPidToBuffer.size()},+        rlimitsData{scratch.rlimits.data()},+        rlimitsSize{scratch.rlimits.size()} {+    static_assert(std::is_standard_layout_v<Subprocess::SpawnRawArgs>);+    static_assert(std::is_trivially_destructible_v<Subprocess::SpawnRawArgs>);+  }+};++ProcessReturnCode ProcessReturnCode::make(int status) {+  if (!WIFEXITED(status) && !WIFSIGNALED(status)) {+    throw std::runtime_error(+        to<std::string>("Invalid ProcessReturnCode: ", status));+  }+  return ProcessReturnCode(status);+}++ProcessReturnCode::ProcessReturnCode(ProcessReturnCode&& p) noexcept+    : rawStatus_(p.rawStatus_) {+  p.rawStatus_ = ProcessReturnCode::RV_NOT_STARTED;+}++ProcessReturnCode& ProcessReturnCode::operator=(+    ProcessReturnCode&& p) noexcept {+  rawStatus_ = p.rawStatus_;+  p.rawStatus_ = ProcessReturnCode::RV_NOT_STARTED;+  return *this;+}++ProcessReturnCode::State ProcessReturnCode::state() const {+  if (rawStatus_ == RV_NOT_STARTED) {+    return NOT_STARTED;+  }+  if (rawStatus_ == RV_RUNNING) {+    return RUNNING;+  }+  if (WIFEXITED(rawStatus_)) {+    return EXITED;+  }+  if (WIFSIGNALED(rawStatus_)) {+    return KILLED;+  }+  assume_unreachable();+}++void ProcessReturnCode::enforce(State expected) const {+  State s = state();+  if (s != expected) {+    throw std::logic_error(to<std::string>(+        "Bad use of ProcessReturnCode; state is ", s, " expected ", expected));+  }+}++int ProcessReturnCode::exitStatus() const {+  enforce(EXITED);+  return WEXITSTATUS(rawStatus_);+}++int ProcessReturnCode::killSignal() const {+  enforce(KILLED);+  return WTERMSIG(rawStatus_);+}++bool ProcessReturnCode::coreDumped() const {+  enforce(KILLED);+  return WCOREDUMP(rawStatus_);+}++bool ProcessReturnCode::succeeded() const {+  return exited() && exitStatus() == 0;+}++std::string ProcessReturnCode::str() const {+  switch (state()) {+    case NOT_STARTED:+      return "not started";+    case RUNNING:+      return "running";+    case EXITED:+      return to<std::string>("exited with status ", exitStatus());+    case KILLED:+      return to<std::string>(+          "killed by signal ",+          killSignal(),+          (coreDumped() ? " (core dumped)" : ""));+  }+  assume_unreachable();+}++CalledProcessError::CalledProcessError(ProcessReturnCode rc)+    : SubprocessError(rc.str()), returnCode_(rc) {}++static inline std::string toSubprocessSpawnErrorMessage(+    char const* executable, int errCode, int errnoValue) {+  auto prefix = errCode == kExecFailure+      ? "failed to execute "+      : "error preparing to execute ";+  return to<std::string>(prefix, executable, ": ", errnoStr(errnoValue));+}++SubprocessSpawnError::SubprocessSpawnError(+    const char* executable, int errCode, int errnoValue)+    : SubprocessError(+          toSubprocessSpawnErrorMessage(executable, errCode, errnoValue)),+      errnoValue_(errnoValue) {}++namespace {++// Copy pointers to the given strings in a format suitable for posix_spawn+std::unique_ptr<const char*[]> cloneStrings(const std::vector<std::string>& s) {+  std::unique_ptr<const char*[]> d(new const char*[s.size() + 1]);+  for (size_t i = 0; i < s.size(); i++) {+    d[i] = s[i].c_str();+  }+  d[s.size()] = nullptr;+  return d;+}++// Check a wait() status, throw on non-successful+void checkStatus(ProcessReturnCode returnCode) {+  if (returnCode.state() != ProcessReturnCode::EXITED ||+      returnCode.exitStatus() != 0) {+    throw CalledProcessError(returnCode);+  }+}++} // namespace++Subprocess::Options& Subprocess::Options::fd(int fd, int action) {+  if (fdActions_.contains(fd)) {+    throw std::invalid_argument("fd already added");+  }+  if (action == Subprocess::PIPE) {+    if (fd == 0) {+      action = Subprocess::PIPE_IN;+    } else if (fd == 1 || fd == 2) {+      action = Subprocess::PIPE_OUT;+    } else {+      throw std::invalid_argument(+          to<std::string>("Only fds 0, 1, 2 are valid for action=PIPE: ", fd));+    }+  }+  fdActions_[fd] = action;+  return *this;+}++#if defined(__linux__)++Subprocess::Options& Subprocess::Options::setLinuxCGroupFd(+    int cgroupFd, std::shared_ptr<int> errout) {+  if (linuxCGroupFd_.value >= 0 || !linuxCGroupPath_.value.empty()) {+    throw std::runtime_error("setLinuxCGroup* called more than once");+  }+  linuxCGroupFd_ = {cgroupFd, std::move(errout)};+  return *this;+}++Subprocess::Options& Subprocess::Options::setLinuxCGroupPath(+    const std::string& cgroupPath, std::shared_ptr<int> errout) {+  if (linuxCGroupFd_.value >= 0 || !linuxCGroupPath_.value.empty()) {+    throw std::runtime_error("setLinuxCGroup* called more than once");+  }+  linuxCGroupPath_ = {cgroupPath, std::move(errout)};+  return *this;+}++#endif++Subprocess::Options& Subprocess::Options::addPrintPidToBuffer(span<char> buf) {+  if (buf.size() < kPidBufferMinSize) {+    throw std::invalid_argument("buf size too small");+  }+  setPrintPidToBuffer_.insert(buf.data());+  return *this;+}++Subprocess::Options& Subprocess::Options::addRLimit(+    int resource, rlimit limit, std::shared_ptr<int> errout) {+  if (rlimits_.count(resource)) {+    throw std::runtime_error("addRLimit called with same limit more than once");+  }+  rlimits_[resource] = AttrWithMeta<rlimit>{limit, std::move(errout)};+  return *this;+}++Subprocess::Subprocess() = default;++Subprocess::Subprocess(+    const std::vector<std::string>& argv,+    const Options& options,+    const char* executable,+    const std::vector<std::string>* env)+    : destroyBehavior_(options.destroyBehavior_) {+  if (argv.empty()) {+    throw std::invalid_argument("argv must not be empty");+  }+  if (!executable) {+    executable = argv[0].c_str();+  }+  spawn(cloneStrings(argv), executable, options, env);+}++Subprocess::Subprocess(+    const std::string& cmd,+    const Options& options,+    const std::vector<std::string>* env)+    : destroyBehavior_(options.destroyBehavior_) {+  if (options.usePath_) {+    throw std::invalid_argument("usePath() not allowed when running in shell");+  }++  std::vector<std::string> argv = {"/bin/sh", "-c", cmd};+  spawn(cloneStrings(argv), argv[0].c_str(), options, env);+}++Subprocess Subprocess::fromExistingProcess(pid_t pid) {+  Subprocess sp;+  sp.pid_ = pid;+  sp.destroyBehavior_ = DestroyBehaviorLeak;+  sp.returnCode_ = ProcessReturnCode::makeRunning();+  return sp;+}++Subprocess::~Subprocess() {+  if (returnCode_.state() == ProcessReturnCode::RUNNING) {+    if (destroyBehavior_ == DestroyBehaviorFatal) {+      // Explicitly crash if we are destroyed without reaping the child process.+      //+      // If you are running into this crash, you are destroying a Subprocess+      // without cleaning up the child process first, which can leave behind a+      // zombie process on the system until the current process exits.  You may+      // want to use one of the following options instead when creating the+      // Subprocess:+      // - Options::detach()+      //   If you do not want to wait on the child process to complete, and do+      //   not care about its exit status, use detach().+      // - Options::killChildOnDestruction()+      //   If you want the child process to be automatically killed when the+      //   Subprocess is destroyed, use killChildOnDestruction() or+      //   terminateChildOnDestruction()+      XLOG(FATAL) << "Subprocess destroyed without reaping child";+    } else if (destroyBehavior_ == DestroyBehaviorLeak) {+      // Do nothing if we are destroyed without reaping the child process.+      XLOG(DBG) << "Subprocess destroyed without reaping child process";+    } else {+      // If we are killed without reaping the child process, explicitly+      // terminate/kill it and wait for it to exit.+      try {+        TimeoutDuration timeout(destroyBehavior_);+        terminateOrKill(timeout);+      } catch (const std::exception& ex) {+        XLOG(WARN) << "error terminating process in Subprocess destructor: "+                   << ex.what();+      }+    }+  }+}++struct Subprocess::ChildErrorInfo {+  int errCode;+  int errnoValue;+};++[[noreturn]]+FOLLY_DETAIL_SUBPROCESS_RAW void Subprocess::childError(+    SpawnRawArgs const& args, int errCode, int errnoValue) {+  *args.err = {errCode, errnoValue};+  detail::subprocess_libc::_exit(errCode);+  __builtin_unreachable();+}++void Subprocess::setAllNonBlocking() {+  for (auto& p : pipes_) {+    int fd = p.pipe.fd();+    int flags = ::fcntl(fd, F_GETFL);+    checkUnixError(flags, "fcntl");+    int r = ::fcntl(fd, F_SETFL, flags | O_NONBLOCK);+    checkUnixError(r, "fcntl");+  }+}++void Subprocess::spawn(+    std::unique_ptr<const char*[]> argv,+    const char* executable,+    const Options& optionsIn,+    const std::vector<std::string>* env) {+  if (optionsIn.usePath_ && env) {+    throw std::invalid_argument(+        "usePath() not allowed when overriding environment");+  }++  // Make a copy, we'll mutate options+  Options options(optionsIn);++  // On error, close all pipes_ (ignoring errors, but that seems fine here).+  auto pipesGuard = makeGuard([this] { pipes_.clear(); });++  ChildErrorInfo err{};++  // Perform the actual work of setting up pipes then forking and+  // executing the child.+  spawnInternal(std::move(argv), executable, options, env, &err);++  // After spawnInternal() returns the child is alive.  We have to be very+  // careful about throwing after this point.  We are inside the constructor,+  // so if we throw the Subprocess object will have never existed, and the+  // destructor will never be called.+  //+  // We should only throw if we got an error via the ChildErrorInfo, and we know+  // the child has exited and can be immediately waited for.  In all other+  // cases, we have no way of cleaning up the child.+  readChildErrorNum(err, executable);++  // If we spawned a detached child, wait on the intermediate child process.+  // It always exits immediately.+  if (options.detach_) {+    wait();+  }++  // We have fully succeeded now, so release the guard on pipes_+  pipesGuard.dismiss();+}++void Subprocess::spawnInternal(+    std::unique_ptr<const char*[]> argv,+    const char* executable,+    Options& options,+    const std::vector<std::string>* env,+    ChildErrorInfo* err) {+  // Parent work, pre-fork: create pipes+  std::vector<int> childFds;+  // Close all of the childFds as we leave this scope+  SCOPE_EXIT {+    // These are only pipes, closing them shouldn't fail+    for (int cfd : childFds) {+      CHECK_ERR(fileops::close(cfd));+    }+  };++  int r;+  for (auto& p : options.fdActions_) {+    if (p.second == PIPE_IN || p.second == PIPE_OUT) {+      int fds[2];+      // We're setting both ends of the pipe as close-on-exec. The child+      // doesn't need to reset the flag on its end, as we always dup2() the fd,+      // and dup2() fds don't share the close-on-exec flag.+#if FOLLY_HAVE_PIPE2+      // If possible, set close-on-exec atomically. Otherwise, a concurrent+      // Subprocess invocation can fork() between "pipe" and "fnctl",+      // causing FDs to leak.+      r = ::pipe2(fds, O_CLOEXEC);+      checkUnixError(r, "pipe2");+#else+      r = fileops::pipe(fds);+      checkUnixError(r, "pipe");+      r = fcntl(fds[0], F_SETFD, FD_CLOEXEC);+      checkUnixError(r, "set FD_CLOEXEC");+      r = fcntl(fds[1], F_SETFD, FD_CLOEXEC);+      checkUnixError(r, "set FD_CLOEXEC");+#endif+      pipes_.emplace_back();+      Pipe& pipe = pipes_.back();+      pipe.direction = p.second;+      int cfd;+      if (p.second == PIPE_IN) {+        // Child gets reading end+        pipe.pipe = folly::File(fds[1], /*ownsFd=*/true);+        cfd = fds[0];+      } else {+        pipe.pipe = folly::File(fds[0], /*ownsFd=*/true);+        cfd = fds[1];+      }+      p.second = cfd; // ensure it gets dup2()ed+      pipe.childFd = p.first;+      childFds.push_back(cfd);+    }+  }++  // This should already be sorted, as options.fdActions_ is+  DCHECK(std::is_sorted(pipes_.begin(), pipes_.end()));++  // Note that the const casts below are legit, per+  // http://pubs.opengroup.org/onlinepubs/009695399/functions/exec.html++  // Set up environment+  std::unique_ptr<const char*[]> envHolder;+  if (env) {+    envHolder = cloneStrings(*env);+  }++  // Block all signals around vfork; see http://ewontfix.com/7/.+  //+  // As the child may run in the same address space as the parent until+  // the actual execve() system call, any (custom) signal handlers that+  // the parent has might alter parent's memory if invoked in the child,+  // with undefined results.  So we block all signals in the parent before+  // vfork(), which will cause them to be blocked in the child as well (we+  // rely on the fact that Linux, just like all sane implementations, only+  // clones the calling thread).  Then, in the child, we reset all signals+  // to their default dispositions (while still blocked), and unblock them+  // (so the exec()ed process inherits the parent's signal mask)+  //+  // The parent also unblocks all signals as soon as vfork() returns.+  sigset_t allBlocked;+  r = sigfillset(&allBlocked);+  checkUnixError(r, "sigfillset");+  sigset_t oldSignals;++  r = pthread_sigmask(SIG_SETMASK, &allBlocked, &oldSignals);+  checkPosixError(r, "pthread_sigmask");+  SCOPE_EXIT {+    // Restore signal mask+    r = pthread_sigmask(SIG_SETMASK, &oldSignals, nullptr);+    CHECK_EQ(r, 0) << "pthread_sigmask: " << errnoStr(r); // shouldn't fail+  };++  SpawnRawArgs::Scratch scratch{options};+  SpawnRawArgs args{scratch, options};+  args.argv = argv.get();+  args.envv = env ? envHolder.get() : environ;+  args.executable = executable;+  args.err = err;+  args.oldSignals = options.sigmask_.value_or(oldSignals);++  // Child is alive.  We have to be very careful about throwing after this+  // point.  We are inside the constructor, so if we throw the Subprocess+  // object will have never existed, and the destructor will never be called.+  //+  // We should only throw if we got an error via the errFd, and we know the+  // child has exited and can be immediately waited for.  In all other cases,+  // we have no way of cleaning up the child.+  pid_ = spawnInternalDoFork(args);+  returnCode_ = ProcessReturnCode::makeRunning();+}++// With -Wclobbered, gcc complains about vfork potentially cloberring the+// childDir variable, even though we only use it on the child side of the+// vfork.++FOLLY_PUSH_WARNING+FOLLY_GCC_DISABLE_WARNING("-Wclobbered")+FOLLY_DETAIL_SUBPROCESS_RAW+pid_t Subprocess::spawnInternalDoFork(SpawnRawArgs const& args) {+  pid_t pid = detail::subprocess_libc::vfork();+  checkUnixError(pid, errno, "failed to fork");+  if (pid != 0) {+    return pid;+  }++  // From this point onward, we are in the child.++  // Fork a second time if detach_ was requested.+  // This must be done before signals are restored in prepareChild()+  if (args.detach) {+    pid = detail::subprocess_libc::vfork();+    if (pid == -1) {+      // Inform our parent process of the error so it can throw in the parent.+      childError(args, kChildFailure, errno);+    } else if (pid != 0) {+      // We are the intermediate process.  Exit immediately.+      // Our child will still inform the original parent of success/failure+      // through errFd.  The pid of the grandchild process never gets+      // propagated back up to the original parent.  In the future we could+      // potentially send it back using errFd if we needed to.+      detail::subprocess_libc::_exit(0);+    }+  }++  int errnoValue = prepareChild(args);+  if (errnoValue != 0) {+    childError(args, kChildFailure, errnoValue);+  }++  errnoValue = runChild(args);+  // If we get here, exec() failed.+  childError(args, kExecFailure, errnoValue);++  return 0; // unreachable+}+FOLLY_POP_WARNING++FOLLY_DETAIL_SUBPROCESS_RAW+int Subprocess::prepareChildDoOptionalError(int* errout) {+  if (errout) {+    *errout = errno;+    return 0;+  } else {+    return errno;+  }+}++FOLLY_DETAIL_SUBPROCESS_RAW+int Subprocess::prepareChildDoLinuxCGroup(SpawnRawArgs const& args) {+  auto cgroupPath = args.linuxCGroupPath;+  auto cgroupFd = args.linuxCGroupFd;+  if (nullptr != cgroupPath.value) {+    int fd = detail::subprocess_libc::open(+        cgroupPath.value, O_RDONLY | O_DIRECTORY | O_CLOEXEC);+    if (-1 == fd) {+      return prepareChildDoOptionalError(cgroupPath.errout);+    }+    cgroupFd = {fd, cgroupPath.errout};+  }+  if (-1 != cgroupFd.value) {+    int fd = detail::subprocess_libc::openat(+        cgroupFd.value, "cgroup.procs", O_WRONLY | O_CLOEXEC);+    if (fd == -1) {+      return prepareChildDoOptionalError(cgroupFd.errout);+    }+    int rc = 0;+    do {+      constexpr char const buf = '0';+      rc = detail::subprocess_libc::write(fd, &buf, 1);+    } while (rc == -1 && errno == EINTR);+    if (rc == -1) {+      return prepareChildDoOptionalError(cgroupFd.errout);+    }+  }+  return 0;+}++// If requested, close all other file descriptors.  Don't close+// any fds in options.fdActions_, and don't touch stdin, stdout, stderr.+// Ignore errors.+//+//+// This function is called in the child after fork but before exec so+// there is very little it can do. It cannot allocate memory and+// it cannot lock a mutex, just as if it were running in a signal+// handler.+FOLLY_DETAIL_SUBPROCESS_RAW+void Subprocess::closeInheritedFds(const SpawnRawArgs& args) {+#if defined(__linux__)+  int dirfd = detail::subprocess_libc::open("/proc/self/fd", O_RDONLY);+  if (dirfd != -1) {+    char buffer[32768];+    int res;+    while ((res = syscall(SYS_getdents64, dirfd, buffer, sizeof(buffer))) > 0) {+      // linux_dirent64 is part of the kernel ABI for the getdents64 system+      // call. It is currently the same as struct dirent64 in both glibc and+      // musl, but those are library specific and could change. linux_dirent64+      // is not defined in the standard set of Linux userspace headers+      // (/usr/include/linux)+      //+      // We do not use the POSIX interfaces (opendir, readdir, etc..) for+      // reading a directory since they may allocate memory / grab a lock, which+      // is unsafe in this context.+      FOLLY_PUSH_WARNING+      FOLLY_CLANG_DISABLE_WARNING("-Wzero-length-array")+      struct linux_dirent64 {+        uint64_t d_ino;+        int64_t d_off;+        uint16_t d_reclen;+        unsigned char d_type;+        char d_name[0];+      } const* entry;+      FOLLY_POP_WARNING+      for (int offset = 0; offset < res; offset += entry->d_reclen) {+        entry = reinterpret_cast<struct linux_dirent64*>(buffer + offset);+        if (entry->d_type != DT_LNK) {+          continue;+        }+        char* end_p = nullptr;+        errno = 0;+        int fd = static_cast<int>(+            detail::subprocess_libc::strtol(entry->d_name, &end_p, 10));+        if (errno == ERANGE || fd < 3 || end_p == entry->d_name) {+          continue;+        }+        if ((fd != dirfd) && (args.fdActions.find(fd) == nullptr)) {+          detail::subprocess_libc::close(fd);+        }+      }+    }+    detail::subprocess_libc::close(dirfd);+    return;+  }+#endif+  // If not running on Linux or if we failed to open /proc/self/fd, try to close+  // all possible open file descriptors.+  for (auto fd = sysconf(_SC_OPEN_MAX) - 1; fd >= 3; --fd) {+    if (args.fdActions.find(fd) == nullptr) {+      detail::subprocess_libc::close(fd);+    }+  }+}++FOLLY_DETAIL_SUBPROCESS_RAW+int Subprocess::prepareChild(SpawnRawArgs const& args) {+  // While all signals are blocked, we must reset their+  // dispositions to default.+  for (int sig = 1; sig < NSIG; ++sig) {+    detail::subprocess_libc::signal(sig, SIG_DFL);+  }++  {+    // Unblock signals; restore signal mask.+    int r = detail::subprocess_libc::pthread_sigmask(+        SIG_SETMASK, &args.oldSignals, nullptr);+    if (r != 0) {+      return r; // pthread_sigmask() returns an errno value+    }+  }++  // Move the child process into a linux cgroup, if one is given+  if (auto rc = prepareChildDoLinuxCGroup(args)) {+    return rc;+  }++  for (size_t i = 0; i < args.rlimitsSize; ++i) {+    auto const& limit = args.rlimitsData[i];+    if (setrlimit(limit.first, &limit.second.value) == -1) {+      if (limit.second.errout) {+        *limit.second.errout = errno;+      } else {+        return errno;+      }+    }+  }++  // Change the working directory, if one is given+  if (args.childDir) {+    if (::chdir(args.childDir) == -1) {+      return errno;+    }+  }++#ifdef __linux__+  // Best effort+  if (args.cpuSet) {+    const auto& cpuSet = *args.cpuSet;+    if (::sched_setaffinity(0, sizeof(cpuSet.value), &cpuSet.value) == -1) {+      if (cpuSet.errout) {+        *cpuSet.errout = errno;+      } else {+        return errno;+      }+    }+  }+#endif++  // Change effective/real group/user, if requested+  if (auto& ptr = args.egid; ptr && 0 != ::setegid(ptr->value)) {+    if (auto out = ptr->errout) {+      *out = errno;+    } else {+      return errno;+    }+  }+  if (auto& ptr = args.gid; ptr && 0 != ::setgid(ptr->value)) {+    if (auto out = ptr->errout) {+      *out = errno;+    } else {+      return errno;+    }+  }+  if (auto& ptr = args.euid; ptr && 0 != ::seteuid(ptr->value)) {+    if (auto out = ptr->errout) {+      *out = errno;+    } else {+      return errno;+    }+  }+  if (auto& ptr = args.uid; ptr && 0 != ::setuid(ptr->value)) {+    if (auto out = ptr->errout) {+      *out = errno;+    } else {+      return errno;+    }+  }++  // We don't have to explicitly close the parent's end of all pipes,+  // as they all have the FD_CLOEXEC flag set and will be closed at+  // exec time.++  // Redirect requested FDs to /dev/null or NUL+  // dup2 any explicitly specified FDs+  for (auto p : args.fdActions) {+    if (p.second == DEV_NULL) {+      // folly/portability/Fcntl provides an impl of open that will+      // map this to NUL on Windows.+      auto devNull =+          detail::subprocess_libc::open("/dev/null", O_RDWR | O_CLOEXEC);+      if (devNull == -1) {+        return errno;+      }+      // note: dup2 will not set CLOEXEC on the destination+      if (detail::subprocess_libc::dup2(devNull, p.first) == -1) {+        // explicit close on error to avoid leaking fds+        detail::subprocess_libc::close(devNull);+        return errno;+      }+      detail::subprocess_libc::close(devNull);+    } else if (p.second != p.first && p.second != NO_CLOEXEC) {+      if (detail::subprocess_libc::dup2(p.second, p.first) == -1) {+        return errno;+      }+    } else if (p.second == p.first || p.second == NO_CLOEXEC) {+      int flags = detail::subprocess_libc::fcntl(p.first, F_GETFD);+      if (flags == -1) {+        return errno;+      }+      if (int newflags = flags & ~FD_CLOEXEC; newflags != flags) {+        if (detail::subprocess_libc::fcntl(p.first, F_SETFD, newflags) == -1) {+          return errno;+        }+      }+    }+  }++  if (args.closeOtherFds) {+    closeInheritedFds(args);+  }++#if defined(__linux__)+  // Opt to receive signal on parent death, if requested+  if (args.parentDeathSignal != 0) {+    const auto parentDeathSignal =+        static_cast<unsigned long>(args.parentDeathSignal);+    if (detail::subprocess_libc::prctl(+            PR_SET_PDEATHSIG, parentDeathSignal, 0, 0, 0) == -1) {+      return errno;+    }+  }+#endif++  if (args.processGroupLeader) {+#if !defined(__FreeBSD__)+    if (setpgrp() == -1) {+#else+    if (setpgrp(getpid(), getpgrp()) == -1) {+#endif+      return errno;+    }+  }++  for (size_t i = 0; i < args.setPrintPidToBufferSize; ++i) {+    auto buf = args.setPrintPidToBufferData[i];+    detail::subprocess_libc::sprintf(buf, "%d", getpid());+  }++  return 0;+}++FOLLY_DETAIL_SUBPROCESS_RAW+int Subprocess::runChild(SpawnRawArgs const& args) {+  auto argv = const_cast<char* const*>(args.argv);+  auto envv = const_cast<char* const*>(args.envv);+  // Now, finally, exec.+  if (args.usePath) {+    ::execvp(args.executable, argv);+  } else {+    ::execve(args.executable, argv, envv);+  }+  return errno;+}++void Subprocess::readChildErrorNum(ChildErrorInfo err, const char* executable) {+  if (err.errCode == 0) {+    return;+  }++  // We got error data from the child.  The child should exit immediately in+  // this case, so wait on it to clean up.+  wait();++  // Throw to signal the error+  throw SubprocessSpawnError(executable, err.errCode, err.errnoValue);+}++ProcessReturnCode Subprocess::poll(struct rusage* ru) {+  returnCode_.enforce(ProcessReturnCode::RUNNING);+  DCHECK_GT(pid_, 0);+  int status;+  pid_t found = ::wait4(pid_, &status, WNOHANG, ru);+  // The spec guarantees that EINTR does not occur with WNOHANG, so the only+  // two remaining errors are ECHILD (other code reaped the child?), or+  // EINVAL (cosmic rays?), both of which merit an abort:+  PCHECK(found != -1) << "waitpid(" << pid_ << ", &status, WNOHANG)";+  if (found != 0) {+    // Though the child process had quit, this call does not close the pipes+    // since its descendants may still be using them.+    returnCode_ = ProcessReturnCode::make(status);+    pid_ = -1;+  }+  return returnCode_;+}++bool Subprocess::pollChecked() {+  if (poll().state() == ProcessReturnCode::RUNNING) {+    return false;+  }+  checkStatus(returnCode_);+  return true;+}++ProcessReturnCode Subprocess::wait() {+  returnCode_.enforce(ProcessReturnCode::RUNNING);+  DCHECK_GT(pid_, 0);+  int status;+  pid_t found;+  do {+    found = ::waitpid(pid_, &status, 0);+  } while (found == -1 && errno == EINTR);+  // The only two remaining errors are ECHILD (other code reaped the+  // child?), or EINVAL (cosmic rays?), and both merit an abort:+  PCHECK(found != -1) << "waitpid(" << pid_ << ", &status, 0)";+  // Though the child process had quit, this call does not close the pipes+  // since its descendants may still be using them.+  DCHECK_EQ(found, pid_);+  returnCode_ = ProcessReturnCode::make(status);+  pid_ = -1;+  return returnCode_;+}++ProcessReturnCode Subprocess::waitAndGetRusage(struct rusage* ru) {+  returnCode_.enforce(ProcessReturnCode::RUNNING);+  DCHECK_GT(pid_, 0);+  int status;+  pid_t found;+  do {+    found = ::wait4(pid_, &status, 0, ru);+  } while (found == -1 && errno == EINTR);+  // The only two remaining errors are ECHILD (other code reaped the+  // child?), or EINVAL (cosmic rays?), and both merit an abort:+  PCHECK(found != -1) << "wait4(" << pid_ << ", &status, 0, resourceUsage)";+  // Though the child process had quit, this call does not close the pipes+  // since its descendants may still be using them.+  DCHECK_EQ(found, pid_);+  returnCode_ = ProcessReturnCode::make(status);+  pid_ = -1;+  return returnCode_;+}++void Subprocess::waitChecked() {+  wait();+  checkStatus(returnCode_);+}++ProcessReturnCode Subprocess::waitTimeout(TimeoutDuration timeout) {+  returnCode_.enforce(ProcessReturnCode::RUNNING);+  DCHECK_GT(pid_, 0) << "The subprocess has been waited already";++  auto pollUntil = std::chrono::steady_clock::now() + timeout;+  auto sleepDuration = std::chrono::milliseconds{2};+  constexpr auto maximumSleepDuration = std::chrono::milliseconds{100};++  for (;;) {+    // Always call waitpid once after the full timeout has elapsed.+    auto now = std::chrono::steady_clock::now();++    int status;+    pid_t found;+    do {+      found = ::waitpid(pid_, &status, WNOHANG);+    } while (found == -1 && errno == EINTR);+    PCHECK(found != -1) << "waitpid(" << pid_ << ", &status, WNOHANG)";+    if (found) {+      // Just on the safe side, make sure it's the actual pid we are waiting.+      DCHECK_EQ(found, pid_);+      returnCode_ = ProcessReturnCode::make(status);+      // Change pid_ to -1 to detect programming error like calling+      // this method multiple times.+      pid_ = -1;+      return returnCode_;+    }+    if (now > pollUntil) {+      // Timed out: still running().+      return returnCode_;+    }+    // The subprocess is still running, sleep for increasing periods of time.+    std::this_thread::sleep_for(sleepDuration);+    sleepDuration =+        std::min(maximumSleepDuration, sleepDuration + sleepDuration);+  }+}++void Subprocess::sendSignal(int signal) {+  returnCode_.enforce(ProcessReturnCode::RUNNING);+  int r = ::kill(pid_, signal);+  checkUnixError(r, "kill");+}++ProcessReturnCode Subprocess::waitOrTerminateOrKill(+    TimeoutDuration waitTimeout, TimeoutDuration sigtermTimeout) {+  returnCode_.enforce(ProcessReturnCode::RUNNING);+  DCHECK_GT(pid_, 0) << "The subprocess has been waited already";++  this->waitTimeout(waitTimeout);++  if (returnCode_.running()) {+    return terminateOrKill(sigtermTimeout);+  }+  return returnCode_;+}++ProcessReturnCode Subprocess::terminateOrKill(TimeoutDuration sigtermTimeout) {+  returnCode_.enforce(ProcessReturnCode::RUNNING);+  DCHECK_GT(pid_, 0) << "The subprocess has been waited already";++  if (sigtermTimeout > TimeoutDuration(0)) {+    // 1. Send SIGTERM to kill the process+    terminate();+    // 2. check whether subprocess has terminated using non-blocking waitpid+    waitTimeout(sigtermTimeout);+    if (!returnCode_.running()) {+      return returnCode_;+    }+  }++  // 3. If we are at this point, we have waited enough time after+  // sending SIGTERM, we have to use nuclear option SIGKILL to kill+  // the subprocess.+  XLOGF(INFO, "Send SIGKILL to {}", pid_);+  kill();+  // 4. SIGKILL should kill the process otherwise there must be+  // something seriously wrong, just use blocking wait to wait for the+  // subprocess to finish.+  return wait();+}++pid_t Subprocess::pid() const {+  return pid_;+}++namespace {++ByteRange queueFront(const IOBufQueue& queue) {+  auto* p = queue.front();+  if (!p) {+    return ByteRange{};+  }+  return io::Cursor(p).peekBytes();+}++// fd write+bool handleWrite(int fd, IOBufQueue& queue) {+  for (;;) {+    auto b = queueFront(queue);+    if (b.empty()) {+      return true; // EOF+    }++    ssize_t n = writeNoInt(fd, b.data(), b.size());+    if (n == -1 && errno == EAGAIN) {+      return false;+    }+    checkUnixError(n, "write");+    queue.trimStart(n);+  }+}++// fd read+bool handleRead(int fd, IOBufQueue& queue) {+  for (;;) {+    auto p = queue.preallocate(100, 65000);+    ssize_t n = readNoInt(fd, p.first, p.second);+    if (n == -1 && errno == EAGAIN) {+      return false;+    }+    checkUnixError(n, "read");+    if (n == 0) {+      return true;+    }+    queue.postallocate(n);+  }+}++bool discardRead(int fd) {+  static const size_t bufSize = 65000;+  // Thread unsafe, but it doesn't matter.+  static std::unique_ptr<char[]> buf(new char[bufSize]);++  for (;;) {+    ssize_t n = readNoInt(fd, buf.get(), bufSize);+    if (n == -1 && errno == EAGAIN) {+      return false;+    }+    checkUnixError(n, "read");+    if (n == 0) {+      return true;+    }+  }+}++} // namespace++std::pair<std::string, std::string> Subprocess::communicate(StringPiece input) {+  IOBufQueue inputQueue;+  inputQueue.wrapBuffer(input.data(), input.size());++  auto outQueues = communicateIOBuf(std::move(inputQueue));+  auto outBufs =+      std::make_pair(outQueues.first.move(), outQueues.second.move());+  std::pair<std::string, std::string> out;+  if (outBufs.first) {+    outBufs.first->coalesce();+    out.first.assign(+        reinterpret_cast<const char*>(outBufs.first->data()),+        outBufs.first->length());+  }+  if (outBufs.second) {+    outBufs.second->coalesce();+    out.second.assign(+        reinterpret_cast<const char*>(outBufs.second->data()),+        outBufs.second->length());+  }+  return out;+}++std::pair<IOBufQueue, IOBufQueue> Subprocess::communicateIOBuf(+    IOBufQueue input) {+  // If the user supplied a non-empty input buffer, make sure+  // that stdin is a pipe so we can write the data.+  if (!input.empty()) {+    // findByChildFd() will throw std::invalid_argument if no pipe for+    // STDIN_FILENO exists+    findByChildFd(STDIN_FILENO);+  }++  std::pair<IOBufQueue, IOBufQueue> out;++  auto readCallback = [&](int pfd, int cfd) -> bool {+    if (cfd == STDOUT_FILENO) {+      return handleRead(pfd, out.first);+    } else if (cfd == STDERR_FILENO) {+      return handleRead(pfd, out.second);+    } else {+      // Don't close the file descriptor, the child might not like SIGPIPE,+      // just read and throw the data away.+      return discardRead(pfd);+    }+  };++  auto writeCallback = [&](int pfd, int cfd) -> bool {+    if (cfd == STDIN_FILENO) {+      return handleWrite(pfd, input);+    } else {+      // If we don't want to write to this fd, just close it.+      return true;+    }+  };++  communicate(std::move(readCallback), std::move(writeCallback));++  return out;+}++void Subprocess::communicate(+    FdCallback readCallback, FdCallback writeCallback) {+  // This serves to prevent wait() followed by communicate(), but if you+  // legitimately need that, send a patch to delete this line.+  returnCode_.enforce(ProcessReturnCode::RUNNING);+  setAllNonBlocking();++  std::vector<pollfd> fds;+  fds.reserve(pipes_.size());+  std::vector<size_t> toClose; // indexes into pipes_+  toClose.reserve(pipes_.size());++  while (!pipes_.empty()) {+    fds.clear();+    toClose.clear();++    for (auto& p : pipes_) {+      pollfd pfd;+      pfd.fd = p.pipe.fd();+      // Yes, backwards, PIPE_IN / PIPE_OUT are defined from the+      // child's point of view.+      if (!p.enabled) {+        // Still keeping fd in watched set so we get notified of POLLHUP /+        // POLLERR+        pfd.events = 0;+      } else if (p.direction == PIPE_IN) {+        pfd.events = POLLOUT;+      } else {+        pfd.events = POLLIN;+      }+      fds.push_back(pfd);+    }++    int r;+    do {+      r = ::poll(fds.data(), fds.size(), -1);+    } while (r == -1 && errno == EINTR);+    checkUnixError(r, "poll");++    for (size_t i = 0; i < pipes_.size(); ++i) {+      auto& p = pipes_[i];+      auto parentFd = p.pipe.fd();+      DCHECK_EQ(fds[i].fd, parentFd);+      short events = fds[i].revents;++      bool closed = false;+      if (events & POLLOUT) {+        DCHECK(!(events & POLLIN));+        if (writeCallback(parentFd, p.childFd)) {+          toClose.push_back(i);+          closed = true;+        }+      }++      // Call read callback on POLLHUP, to give it a chance to read (and act+      // on) end of file+      if (events & (POLLIN | POLLHUP)) {+        DCHECK(!(events & POLLOUT));+        if (readCallback(parentFd, p.childFd)) {+          toClose.push_back(i);+          closed = true;+        }+      }++      if ((events & (POLLHUP | POLLERR)) && !closed) {+        toClose.push_back(i);+      }+    }++    // Close the fds in reverse order so the indexes hold after erase()+    for (int idx : boost::adaptors::reverse(toClose)) {+      auto pos = pipes_.begin() + idx;+      pos->pipe.close(); // Throws on error+      pipes_.erase(pos);+    }+  }+}++void Subprocess::enableNotifications(int childFd, bool enabled) {+  pipes_[findByChildFd(childFd)].enabled = enabled;+}++bool Subprocess::notificationsEnabled(int childFd) const {+  return pipes_[findByChildFd(childFd)].enabled;+}++size_t Subprocess::findByChildFd(int childFd) const {+  auto pos = std::lower_bound(+      pipes_.begin(), pipes_.end(), childFd, [](const Pipe& pipe, int fd) {+        return pipe.childFd < fd;+      });+  if (pos == pipes_.end() || pos->childFd != childFd) {+    throw std::invalid_argument(+        folly::to<std::string>("child fd not found ", childFd));+  }+  return pos - pipes_.begin();+}++void Subprocess::closeParentFd(int childFd) {+  int idx = findByChildFd(childFd);+  pipes_[idx].pipe.close(); // May throw+  pipes_.erase(pipes_.begin() + idx);+}++std::vector<Subprocess::ChildPipe> Subprocess::takeOwnershipOfPipes() {+  std::vector<Subprocess::ChildPipe> pipes;+  for (auto& p : pipes_) {+    pipes.emplace_back(p.childFd, std::move(p.pipe));+  }+  // release memory+  std::vector<Pipe>().swap(pipes_);+  return pipes;+}++namespace {++class Initializer {+ public:+  Initializer() {+    // We like EPIPE, thanks.+    ::signal(SIGPIPE, SIG_IGN);+  }+};++Initializer initializer;++} // namespace++} // namespace folly
+ folly/folly/Subprocess.h view
@@ -0,0 +1,1079 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++/**+ * Subprocess library, modeled after Python's subprocess module+ * (http://docs.python.org/2/library/subprocess.html)+ *+ * This library defines one class (Subprocess) which represents a child+ * process.  Subprocess has two constructors: one that takes a vector<string>+ * and executes the given executable without using the shell, and one+ * that takes a string and executes the given command using the shell.+ * Subprocess allows you to redirect the child's standard input, standard+ * output, and standard error to/from child descriptors in the parent,+ * or to create communication pipes between the child and the parent.+ *+ * The simplest example is a thread-safe [1] version of the system() library+ * function:+ *    Subprocess(cmd).wait();+ * which executes the command using the default shell and waits for it+ * to complete, returning the exit status.+ *+ * A thread-safe [1] version of popen() (type="r", to read from the child):+ *    Subprocess proc(cmd, Subprocess::Options().pipeStdout());+ *    // read from proc.stdoutFd()+ *    proc.wait();+ *+ * A thread-safe [1] version of popen() (type="w", to write to the child):+ *    Subprocess proc(cmd, Subprocess::Options().pipeStdin());+ *    // write to proc.stdinFd()+ *    proc.wait();+ *+ * If you want to redirect both stdin and stdout to pipes, you can, but note+ * that you're subject to a variety of deadlocks.  You'll want to use+ * nonblocking I/O, like the callback version of communicate().+ *+ * The string or IOBuf-based variants of communicate() are the simplest way+ * to communicate with a child via its standard input, standard output, and+ * standard error.  They buffer everything in memory, so they are not great+ * for large amounts of data (or long-running processes), but they are much+ * simpler than the callback version.+ *+ * == A note on thread-safety ==+ *+ * [1] "thread-safe" refers ONLY to the fact that Subprocess is very careful+ * to fork in a way that does not cause grief in multithreaded programs.+ *+ * Caveat: If your system does not have the atomic pipe2 system call, it is+ * not safe to concurrently call Subprocess from different threads.+ * Therefore, it is best to have a single thread be responsible for spawning+ * subprocesses.+ *+ * A particular instances of Subprocess is emphatically **not** thread-safe.+ * If you need to simultaneously communicate via the pipes, and interact+ * with the Subprocess state, your best bet is to:+ *  - takeOwnershipOfPipes() to separate the pipe I/O from the subprocess.+ *  - Only interact with the Subprocess from one thread at a time.+ *+ * The current implementation of communicate() cannot be safely interrupted.+ * To do so correctly, one would need to use EventFD, or open a dedicated+ * pipe to be messaged from a different thread -- in particular, kill() will+ * not do, since a descendant may keep the pipes open indefinitely.+ *+ * So, once you call communicate(), you must wait for it to return, and not+ * touch the pipes from other threads.  closeParentFd() is emphatically+ * unsafe to call concurrently, and even sendSignal() is not a good idea.+ * You can perhaps give the Subprocess's PID to a different thread before+ * starting communicate(), and use that PID to send a signal without+ * accessing the Subprocess object.  In that case, you will need a mutex+ * that ensures you don't wait() before you sent said signal.  In a+ * nutshell, don't do this.+ *+ * In fact, signals are inherently concurrency-unsafe on Unix: if you signal+ * a PID, while another thread is in waitpid(), the signal may fire either+ * before or after the process is reaped.  This means that your signal can,+ * in pathological circumstances, be delivered to the wrong process (ouch!).+ * To avoid this, you should only use non-blocking waits (i.e. poll()), and+ * make sure to serialize your signals (i.e. kill()) with the waits --+ * either wait & signal from the same thread, or use a mutex.+ */++#pragma once++#ifdef _WIN32+#error Subprocess is not supported on Windows.+#endif++#include <signal.h>+#include <sys/resource.h>+#include <sys/types.h>+#include <sys/wait.h>++#include <chrono>+#include <exception>+#include <string>+#include <tuple>+#include <vector>++#include <boost/container/flat_map.hpp>+#include <boost/operators.hpp>++#include <folly/Exception.h>+#include <folly/File.h>+#include <folly/FileUtil.h>+#include <folly/Function.h>+#include <folly/MapUtil.h>+#include <folly/Optional.h>+#include <folly/Portability.h>+#include <folly/Range.h>+#include <folly/container/span.h>+#include <folly/gen/String.h>+#include <folly/io/IOBufQueue.h>+#include <folly/portability/SysResource.h>++namespace folly {++namespace detail {++/// SubprocessFdActionsList+///+/// A sorted vector-map with a binary search. Declared in the header so that the+/// binary search can be unit-tested.+///+/// Not using a library container type since this binary search is done in the+/// child process after a vfork(), including in sanitized builds. Relevant+/// member functions are explicitly marked non-sanitized (under clang).+class SubprocessFdActionsList {+ private:+  using value_type = std::pair<int, int>;++  value_type const* begin_;+  value_type const* end_;++ public:+  explicit SubprocessFdActionsList(span<value_type const> rep) noexcept;++  value_type const* begin() const noexcept;+  value_type const* end() const noexcept;++  int const* find(int fd) const noexcept;+};++} // namespace detail++/**+ * Class to wrap a process return code.+ */+class Subprocess;+class ProcessReturnCode {+ public:+  enum State {+    // Subprocess starts in the constructor, so this state designates only+    // default-initialized or moved-out ProcessReturnCodes.+    NOT_STARTED,+    RUNNING,+    EXITED,+    KILLED,+  };++  static ProcessReturnCode makeNotStarted() {+    return ProcessReturnCode(RV_NOT_STARTED);+  }++  static ProcessReturnCode makeRunning() {+    return ProcessReturnCode(RV_RUNNING);+  }++  static ProcessReturnCode make(int status);++  // Default-initialized for convenience. Subprocess::returnCode() will+  // never produce this value.+  ProcessReturnCode() : rawStatus_(RV_NOT_STARTED) {}++  // Trivially copyable+  ProcessReturnCode(const ProcessReturnCode& p) = default;+  ProcessReturnCode& operator=(const ProcessReturnCode& p) = default;+  // Non-default move: In order for Subprocess to be movable, the "moved+  // out" state must not be "running", or ~Subprocess() will abort.+  ProcessReturnCode(ProcessReturnCode&& p) noexcept;+  ProcessReturnCode& operator=(ProcessReturnCode&& p) noexcept;++  /**+   * Process state.  One of:+   * NOT_STARTED: process hasn't been started successfully+   * RUNNING: process is currently running+   * EXITED: process exited (successfully or not)+   * KILLED: process was killed by a signal.+   */+  State state() const;++  /**+   * Helper wrappers around state().+   */+  bool notStarted() const { return state() == NOT_STARTED; }+  bool running() const { return state() == RUNNING; }+  bool exited() const { return state() == EXITED; }+  bool killed() const { return state() == KILLED; }++  /**+   * Exit status.  Only valid if state() == EXITED; throws otherwise.+   */+  int exitStatus() const;++  /**+   * Signal that caused the process's termination.  Only valid if+   * state() == KILLED; throws otherwise.+   */+  int killSignal() const;++  /**+   * Was a core file generated?  Only valid if state() == KILLED; throws+   * otherwise.+   */+  bool coreDumped() const;++  /**+   * Process exited normally with a zero exit status+   */+  bool succeeded() const;++  /**+   * String representation; one of+   * "not started"+   * "running"+   * "exited with status <status>"+   * "killed by signal <signal>"+   * "killed by signal <signal> (core dumped)"+   */+  std::string str() const;++  /**+   * Helper function to enforce a precondition based on this.+   * Throws std::logic_error if in an unexpected state.+   */+  void enforce(State expected) const;++ private:+  explicit ProcessReturnCode(int rv) : rawStatus_(rv) {}+  static constexpr int RV_NOT_STARTED = -2;+  static constexpr int RV_RUNNING = -1;++  int rawStatus_;+};++/**+ * Base exception thrown by the Subprocess methods.+ */+class FOLLY_EXPORT SubprocessError : public std::runtime_error {+ public:+  using std::runtime_error::runtime_error;+};++/**+ * Exception thrown by *Checked methods of Subprocess.+ */+class FOLLY_EXPORT CalledProcessError : public SubprocessError {+ public:+  explicit CalledProcessError(ProcessReturnCode rc);+  ~CalledProcessError() noexcept override = default;+  ProcessReturnCode returnCode() const { return returnCode_; }++ private:+  ProcessReturnCode returnCode_;+};++/**+ * Exception thrown if the subprocess cannot be started.+ */+class FOLLY_EXPORT SubprocessSpawnError : public SubprocessError {+ public:+  SubprocessSpawnError(const char* executable, int errCode, int errnoValue);+  ~SubprocessSpawnError() noexcept override = default;+  int errnoValue() const { return errnoValue_; }++ private:+  int errnoValue_;+};++/**+ * Subprocess.+ */+class Subprocess {+ public:+  using TimeoutDuration = std::chrono::milliseconds;++  // removed CLOSE = -1+  static const int PIPE = -2;+  static const int PIPE_IN = -3;+  static const int PIPE_OUT = -4;+  static const int DEV_NULL = -5;+  static const int NO_CLOEXEC = -6;++  /**+   * Class representing various options: file descriptor behavior, and+   * whether to use $PATH for searching for the executable,+   *+   * By default, we don't use $PATH, file descriptors are closed if+   * the close-on-exec flag is set (fcntl FD_CLOEXEC) and inherited+   * otherwise.+   */+  class Options {+    friend class Subprocess;++   public:+    // digits10 is the maximum number of decimal digits such that any number+    // up to this many decimal digits can always be represented in the given+    // integer type+    // but we need to have storage for the decimal representation of any+    // integer, so +1, and we need to have storage for the terminal null, so+    // again +1.+    static inline constexpr size_t kPidBufferMinSize =+        std::numeric_limits<pid_t>::digits10 + 2;++    Options() {} // E.g. https://gcc.gnu.org/bugzilla/show_bug.cgi?id=58328++    /**+     * Change action for file descriptor fd.+     *+     * "action" may be another file descriptor number (dup2()ed before the+     * child execs), or one of CLOSE, PIPE_IN, and PIPE_OUT.+     *+     * CLOSE: close the file descriptor in the child+     * PIPE_IN: open a pipe *from* the child+     * PIPE_OUT: open a pipe *to* the child+     *+     * PIPE is a shortcut; same as PIPE_IN for stdin (fd 0), same as+     * PIPE_OUT for stdout (fd 1) or stderr (fd 2), and an error for+     * other file descriptors.+     */+    Options& fd(int fd, int action);++    /**+     * Shortcut to change the action for standard input.+     */+    Options& stdinFd(int action) { return fd(STDIN_FILENO, action); }++    /**+     * Shortcut to change the action for standard output.+     */+    Options& stdoutFd(int action) { return fd(STDOUT_FILENO, action); }++    /**+     * Shortcut to change the action for standard error.+     * Note that stderr(1) will redirect the standard error to the same+     * file descriptor as standard output; the equivalent of bash's "2>&1"+     */+    Options& stderrFd(int action) { return fd(STDERR_FILENO, action); }++    Options& pipeStdin() { return fd(STDIN_FILENO, PIPE_IN); }+    Options& pipeStdout() { return fd(STDOUT_FILENO, PIPE_OUT); }+    Options& pipeStderr() { return fd(STDERR_FILENO, PIPE_OUT); }++    /**+     * Close all other fds (other than standard input, output, error,+     * and file descriptors explicitly specified with fd()).+     *+     * This is potentially slow; it's generally a better idea to+     * set the close-on-exec flag on all file descriptors that shouldn't+     * be inherited by the child.+     *+     * Even with this option set, standard input, output, and error are+     * not closed; use stdin(CLOSE), stdout(CLOSE), stderr(CLOSE) if you+     * desire this.+     */+    Options& closeOtherFds() {+      closeOtherFds_ = true;+      return *this;+    }++    /**+     * Use the search path ($PATH) when searching for the executable.+     */+    Options& usePath() {+      usePath_ = true;+      return *this;+    }++    /**+     * Change the child's working directory, after the vfork.+     */+    Options& chdir(const std::string& dir) {+      childDir_ = dir;+      return *this;+    }++#if defined(__linux__)+    /**+     * Child will receive a signal when the parent *thread* exits.+     *+     * This is especially important when this option is used but the calling+     * thread does not block for the duration of the subprocess. If the original+     * thread that created the subprocess ends then the subprocess will+     * terminate. For example, thread pool executors which can reap unused+     * threads may trigger this behavior.+     */+    Options& parentDeathSignal(int sig) {+      parentDeathSignal_ = sig;+      return *this;+    }+#endif++    /**+     * Child will be made a process group leader when it starts. Upside: one+     * can reliably kill all its non-daemonizing descendants.  Downside: the+     * child will not receive Ctrl-C etc during interactive use.+     */+    Options& processGroupLeader() {+      processGroupLeader_ = true;+      return *this;+    }++    /**+     * Detach the spawned process, to allow destroying the Subprocess object+     * without waiting for the child process to finish.+     *+     * This causes the code to vfork twice before executing the command. The+     * intermediate child process will exit immediately after execve, causing+     * the process running the executable to be reparented to init (pid 1).+     *+     * Subprocess objects created with detach() enabled will already be in an+     * "EXITED" state when the constructor returns.  The caller should not call+     * wait() or poll() on the Subprocess, and pid() will return -1.+     */+    Options& detach() {+      detach_ = true;+      return *this;+    }++    /**+     * If the Subprocess object is destroyed while the process is still running,+     * automatically kill the child with SIGKILL and wait on the pid.+     */+    Options& killChildOnDestruction() {+      destroyBehavior_ = 0;+      return *this;+    }++    /**+     * If the Subprocess object is destroyed while the process is still running,+     * use terminateOrKill() to stop it and wait for it to exit.+     *+     * Beware that this may cause the Subprocess destructor to block while+     * waiting on the child process to exit.+     */+    Options& terminateChildOnDestruction(TimeoutDuration timeout) {+      destroyBehavior_ = std::max(TimeoutDuration::rep(0), timeout.count());+      return *this;+    }++    /**+     * By default, if Subprocess is destroyed while the child process is+     * still RUNNING, the destructor will log a fatal.  You can skip this+     * behavior by setting it to true here.+     *+     * Note that detach()ed processes are never in RUNNING state, so this+     * setting does not impact such processes.+     *+     * BEWARE: setting this flag can leave zombie processes behind on the system+     * after the folly::Subprocess is destroyed.  In general you should avoid+     * using this setting.  In general, prefer using one of the following+     * options instead:+     * - If you do not care about monitoring the child process or waiting for it+     *   to complete, use detach().+     * - If you want to automatically clean up the child process when the+     *   Subprocess is destroyed, use killChildOnDestruction() or+     *   terminateChildOnDestruction()+     * - If you want to allow the parent process to exit without waiting on thie+     *   child, prefer simply leaking the folly::Subprocess object when the+     *   parent process exits.  You could exit with _exit(), or you could+     *   explicitly leak the Subprocess using std::unique_ptr::release() or+     *   similar mechanisms.+     */+    Options& allowDestructionWhileProcessRunning(bool val) {+      destroyBehavior_ = val ? DestroyBehaviorLeak : DestroyBehaviorFatal;+      return *this;+    }++#if defined(__linux__)+    Options& setCpuSet(+        const cpu_set_t& cpuSet, std::shared_ptr<int> errout = nullptr) {+      cpuSet_ = AttrWithMeta<cpu_set_t>{cpuSet, std::move(errout)};+      return *this;+    }++    /*+     * setLinuxCGroup*+     * Takes a fd or a path to the cgroup dir. Only one may be provided.+     * Note that the cgroup filesystem may be mounted at any arbitrary point in+     * the filesystem hierarchy, and that different distributions may have their+     * own standard points. So just taking a cgroup name would be non-portable.+     */+    Options& setLinuxCGroupFd(+        int cgroupFd, std::shared_ptr<int> errout = nullptr);+    Options& setLinuxCGroupPath(+        const std::string& cgroupPath, std::shared_ptr<int> errout = nullptr);+#endif++    Options& setUid(uid_t uid, std::shared_ptr<int> errout = nullptr) {+      uid_ = AttrWithMeta<uid_t>{uid, std::move(errout)};+      return *this;+    }+    Options& setGid(gid_t gid, std::shared_ptr<int> errout = nullptr) {+      gid_ = AttrWithMeta<gid_t>{gid, std::move(errout)};+      return *this;+    }+    Options& setEUid(uid_t uid, std::shared_ptr<int> errout = nullptr) {+      euid_ = AttrWithMeta<uid_t>{uid, std::move(errout)};+      return *this;+    }+    Options& setEGid(gid_t gid, std::shared_ptr<int> errout = nullptr) {+      egid_ = AttrWithMeta<gid_t>{gid, std::move(errout)};+      return *this;+    }++    Options& setSignalMask(sigset_t sigmask) {+      sigmask_ = sigmask;+      return *this;+    }++    Options& addPrintPidToBuffer(span<char> buf);++    Options& addRLimit(+        int resource, rlimit limit, std::shared_ptr<int> errout = nullptr);++   private:+    template <typename T>+    struct AttrWithMeta {+      T value{};++      /// nullptr if required, ptr if optional to report failure+      std::shared_ptr<int> erroutLifetime_{}; // do not access in child+      int* errout{erroutLifetime_.get()};+    };++    typedef boost::container::flat_map<int, int> FdMap;+    FdMap fdActions_;+    bool closeOtherFds_{false};+    bool usePath_{false};+    bool processGroupLeader_{false};+    bool detach_{false};+    // The behavior to take if the Subprocess destructor is invoked while the+    // child process is still running.  This is either+    // DestroyBehaviorFatal, DestroyBehaviorLeak, or a timeout value to pass to+    // terminateOrKill() to kill the child process.+    TimeoutDuration::rep destroyBehavior_{DestroyBehaviorFatal};+    std::string childDir_; // "" keeps the parent's working directory+    AttrWithMeta<int> linuxCGroupFd_{-1, nullptr}; // -1 means no cgroup+    AttrWithMeta<std::string> linuxCGroupPath_{}; // empty means no cgroup+#if defined(__linux__)+    int parentDeathSignal_{0};+    Optional<AttrWithMeta<cpu_set_t>> cpuSet_;+#endif+    Optional<AttrWithMeta<uid_t>> uid_;+    Optional<AttrWithMeta<gid_t>> gid_;+    Optional<AttrWithMeta<uid_t>> euid_;+    Optional<AttrWithMeta<gid_t>> egid_;+    Optional<sigset_t> sigmask_;+    std::unordered_set<char*> setPrintPidToBuffer_;+    std::unordered_map<int, AttrWithMeta<rlimit>> rlimits_;+  };++  // Non-copyable, but movable+  Subprocess(const Subprocess&) = delete;+  Subprocess& operator=(const Subprocess&) = delete;+  Subprocess(Subprocess&&) = default;+  Subprocess& operator=(Subprocess&&) = default;++  /**+   * Create an uninitialized subprocess.+   *+   * In this state it can only be destroyed, or assigned to using the move+   * assignment operator.+   */+  Subprocess();++  /**+   * Create a subprocess from the given arguments.  argv[0] must be listed.+   * If not-null, executable must be the actual executable+   * being used (otherwise it's the same as argv[0]).+   *+   * If env is not-null, it must contain name=value strings to be used+   * as the child's environment; otherwise, we inherit the environment+   * from the parent.  env must be null if options.usePath is set.+   */+  explicit Subprocess(+      const std::vector<std::string>& argv,+      const Options& options = Options(),+      const char* executable = nullptr,+      const std::vector<std::string>* env = nullptr);+  ~Subprocess();++  /**+   * Create a Subprocess object for an existing child process ID.+   *+   * The process ID must refer to an immediate child process of the current+   * process.  This allows using the poll() and wait() APIs on a process ID+   * that was not originally spawned by Subprocess.+   */+  static Subprocess fromExistingProcess(pid_t pid);++  /**+   * Create a subprocess run as a shell command (as shell -c 'command')+   *+   * The shell to use is taken from the environment variable $SHELL,+   * or /bin/sh if $SHELL is unset.+   */+  // clang-format off+  [[deprecated(+      "Prefer not running in a shell or use `shellify`.")]]+  explicit Subprocess(+      const std::string& cmd,+      const Options& options = Options(),+      const std::vector<std::string>* env = nullptr);+  // clang-format on++  ////+  //// The methods below only manipulate the process state, and do not+  //// affect its communication pipes.+  ////++  /**+   * Return the child's pid, or -1 if the child wasn't successfully spawned+   * or has already been wait()ed upon.+   */+  pid_t pid() const;++  /**+   * Return the child's status (as per wait()) if the process has already+   * been waited on, -1 if the process is still running, or -2 if the+   * process hasn't been successfully started.  NOTE that this does not call+   * waitpid() or Subprocess::poll(), but simply returns the status stored+   * in the Subprocess object.+   */+  ProcessReturnCode returnCode() const { return returnCode_; }++  /**+   * Poll the child's status and return it. Return the exit status if the+   * subprocess had quit, or RUNNING otherwise.  Throws an std::logic_error+   * if called on a Subprocess whose status is no longer RUNNING.  No other+   * exceptions are possible.  Aborts on egregious violations of contract,+   * e.g. if you wait for the underlying process without going through this+   * Subprocess instance.+   */+  ProcessReturnCode poll(struct rusage* ru = nullptr);++  /**+   * Poll the child's status.  If the process is still running, return false.+   * Otherwise, return true if the process exited with status 0 (success),+   * or throw CalledProcessError if the process exited with a non-zero status.+   */+  bool pollChecked();++  /**+   * Wait for the process to terminate and return its status.  Like poll(),+   * the only exception this can throw is std::logic_error if you call this+   * on a Subprocess whose status is not RUNNING.  Aborts on egregious+   * violations of contract, like an out-of-band waitpid(p.pid(), 0, 0).+   */+  ProcessReturnCode wait();++  /**+   * Wait for the process to terminate and return its status and rusage.  Like+   * poll(), the only exception this can throw is std::logic_error if you call+   * this on a Subprocess whose status is not RUNNING.  Aborts on egregious+   * violations of contract, like an out-of-band wait4(p.pid(), 0, 0, nullptr).+   */+  ProcessReturnCode waitAndGetRusage(struct rusage* ru);++  /**+   * Wait for the process to terminate, throw if unsuccessful.+   */+  void waitChecked();++  /**+   * Call `waitpid` non-blockingly up to `timeout`. Throws std::logic_error if+   * called on a Subprocess whose status is not RUNNING.+   *+   * The return code will be running() if waiting timed out.+   */+  ProcessReturnCode waitTimeout(TimeoutDuration timeout);++  /**+   * Send a signal to the child.  Shortcuts for the commonly used Unix+   * signals are below.+   */+  void sendSignal(int signal);+  void terminate() { sendSignal(SIGTERM); }+  void kill() { sendSignal(SIGKILL); }++  /**+   * Call `waitpid` non-blockingly up to `waitTimeout`. If the process hasn't+   * terminated after that, fall back on `terminateOrKill` with+   * `sigtermTimeoutSeconds`.+   */+  ProcessReturnCode waitOrTerminateOrKill(+      TimeoutDuration waitTimeout, TimeoutDuration sigtermTimeout);++  /**+   * Send the SIGTERM to terminate the process, poll `waitpid` non-blockingly+   * several times up to `sigtermTimeout`. If the process hasn't terminated+   * after that, send SIGKILL to kill the process and call `waitpid` blockingly.+   * Return the exit code of process.+   *+   * If sigtermTimeout is 0 or negative, this will immediately send SIGKILL+   * without first sending SIGTERM.+   */+  ProcessReturnCode terminateOrKill(TimeoutDuration sigtermTimeout);++  ////+  //// The methods below only affect the process's communication pipes, but+  //// not its return code or state (they do not poll() or wait()).+  ////++  /**+   * Communicate with the child until all pipes to/from the child are closed.+   *+   * The input buffer is written to the process' stdin pipe, and data is read+   * from the stdout and stderr pipes.  Non-blocking I/O is performed on all+   * pipes simultaneously to avoid deadlocks.+   *+   * The stdin pipe will be closed after the full input buffer has been written.+   * An error will be thrown if a non-empty input buffer is supplied but stdin+   * was not configured as a pipe.+   *+   * Returns a pair of buffers containing the data read from stdout and stderr.+   * If stdout or stderr is not a pipe, an empty IOBuf queue will be returned+   * for the respective buffer.+   *+   * Note that communicate() and communicateIOBuf() both return when all+   * pipes to/from the child are closed; the child might stay alive after+   * that, so you must still wait().+   *+   * communicateIOBuf() uses IOBufQueue for buffering (which has the+   * advantage that it won't try to allocate all data at once), but it does+   * store the subprocess's entire output in memory before returning.+   *+   * communicate() uses strings for simplicity.+   */+  std::pair<IOBufQueue, IOBufQueue> communicateIOBuf(+      IOBufQueue input = IOBufQueue());++  std::pair<std::string, std::string> communicate(+      StringPiece input = StringPiece());++  /**+   * Communicate with the child until all pipes to/from the child are closed.+   *+   * == Semantics ==+   *+   * readCallback(pfd, cfd) will be called whenever there's data available+   * on any pipe *from* the child (PIPE_OUT).  pfd is the file descriptor+   * in the parent (that you use to read from); cfd is the file descriptor+   * in the child (used for identifying the stream; 1 = child's standard+   * output, 2 = child's standard error, etc)+   *+   * writeCallback(pfd, cfd) will be called whenever a pipe *to* the child is+   * writable (PIPE_IN).  pfd is the file descriptor in the parent (that you+   * use to write to); cfd is the file descriptor in the child (used for+   * identifying the stream; 0 = child's standard input, etc)+   *+   * The read and write callbacks must read from / write to pfd and return+   * false during normal operation.  Return true to tell communicate() to+   * close the pipe.  For readCallback, this might send SIGPIPE to the+   * child, or make its writes fail with EPIPE, so you should generally+   * avoid returning true unless you've reached end-of-file.+   *+   * communicate() returns when all pipes to/from the child are closed; the+   * child might stay alive after that, so you must still wait().+   * Conversely, the child may quit long before its pipes are closed, since+   * its descendants can keep them alive forever.+   *+   * Most users won't need to use this callback version; the simpler version+   * of communicate (which buffers data in memory) will probably work fine.+   *+   * == Things you must get correct ==+   *+   * 1) You MUST consume all data passed to readCallback (or return true to+   * close the pipe).  Similarly, you MUST write to a writable pipe (or+   * return true to close the pipe).  To do otherwise is an error that can+   * result in a deadlock.  You must do this even for pipes you are not+   * interested in.+   *+   * 2) pfd is nonblocking, so be prepared for read() / write() to return -1+   * and set errno to EAGAIN (in which case you should return false).  Use+   * readNoInt() from FileUtil.h to handle interrupted reads for you.+   *+   * 3) Your callbacks MUST NOT call any of the Subprocess methods that+   * manipulate the pipe FDs.  Check the docblocks, but, for example,+   * neither closeParentFd (return true instead) nor takeOwnershipOfPipes+   * are safe.  Stick to reading/writing from pfd, as appropriate.+   *+   * == Good to know ==+   *+   * 1) See ReadLinesCallback for an easy way to consume the child's output+   * streams line-by-line (or tokenized by another delimiter).+   *+   * 2) "Wait until the descendants close the pipes" is usually the behavior+   * you want, since the descendants may have something to say even if the+   * immediate child is dead.  If you need to be able to force-close all+   * parent FDs, communicate() will NOT work for you.  Do it your own way by+   * using takeOwnershipOfPipes().+   *+   * Why not? You can return "true" from your callbacks to sever active+   * pipes, but inactive ones can remain open indefinitely.  It is+   * impossible to safely close inactive pipes while another thread is+   * blocked in communicate().  This is BY DESIGN.  Racing communicate()'s+   * read/write callbacks can result in wrong I/O and data corruption.  This+   * class would need internal synchronization and timeouts, a poor and+   * expensive implementation choice, in order to make closeParentFd()+   * thread-safe.+   */+  using FdCallback = folly::Function<bool(int, int)>;+  void communicate(FdCallback readCallback, FdCallback writeCallback);++  /**+   * A readCallback for Subprocess::communicate() that helps you consume+   * lines (or other delimited pieces) from your subprocess's file+   * descriptors.  Use the readLinesCallback() helper to get template+   * deduction.  For example:+   *+   *   subprocess.communicate(+   *     Subprocess::readLinesCallback(+   *       [](int fd, folly::StringPiece s) {+   *         std::cout << fd << " said: " << s;+   *         return false;  // Keep reading from the child+   *       }+   *     ),+   *     [](int pdf, int cfd){ return true; }  // Don't write to the child+   *   );+   *+   * If a file line exceeds maxLineLength, your callback will get some+   * initial chunks of maxLineLength with no trailing delimiters.  The final+   * chunk of a line is delimiter-terminated iff the delimiter was present+   * in the input.  In particular, the last line in a file always lacks a+   * delimiter -- so if a file ends on a delimiter, the final line is empty.+   *+   * Like a regular communicate() callback, your fdLineCb() normally returns+   * false.  It may return true to tell Subprocess to close the underlying+   * file descriptor.  The child process may then receive SIGPIPE or get+   * EPIPE errors on writes.+   */+  template <class Callback>+  class ReadLinesCallback {+   private:+    // Binds an FD to the client-provided FD+line callback+    struct StreamSplitterCallback {+      StreamSplitterCallback(Callback& cb, int fd) : cb_(cb), fd_(fd) {}+      // The return value semantics are inverted vs StreamSplitter+      bool operator()(StringPiece s) { return !cb_(fd_, s); }+      Callback& cb_;+      int fd_;+    };+    typedef gen::StreamSplitter<StreamSplitterCallback> LineSplitter;++   public:+    explicit ReadLinesCallback(+        Callback&& fdLineCb,+        uint64_t maxLineLength = 0, // No line length limit by default+        char delimiter = '\n',+        uint64_t bufSize = 1024)+        : fdLineCb_(std::forward<Callback>(fdLineCb)),+          maxLineLength_(maxLineLength),+          delimiter_(delimiter),+          bufSize_(bufSize) {}++    bool operator()(int pfd, int cfd) {+      // Make a splitter for this cfd if it doesn't already exist+      auto it = fdToSplitter_.find(cfd);+      auto& splitter = (it != fdToSplitter_.end())+          ? it->second+          : fdToSplitter_+                .emplace(+                    cfd,+                    LineSplitter(+                        delimiter_,+                        StreamSplitterCallback(fdLineCb_, cfd),+                        maxLineLength_))+                .first->second;+      // Read as much as we can from this FD+      char buf[bufSize_];+      while (true) {+        ssize_t ret = readNoInt(pfd, buf, bufSize_);+        if (ret == -1 && errno == EAGAIN) { // No more data for now+          return false;+        }+        checkUnixError(ret, "read");+        if (ret == 0) { // Reached end-of-file+          splitter.flush(); // Ignore return since the file is over anyway+          return true;+        }+        if (!splitter(StringPiece(buf, ret))) {+          return true; // The callback told us to stop+        }+      }+    }++   private:+    Callback fdLineCb_;+    const uint64_t maxLineLength_;+    const char delimiter_;+    const uint64_t bufSize_;+    // We lazily make splitters for all cfds that get used.+    std::unordered_map<int, LineSplitter> fdToSplitter_;+  };++  // Helper to enable template deduction+  template <class Callback>+  static auto readLinesCallback(+      Callback&& fdLineCb,+      uint64_t maxLineLength = 0, // No line length limit by default+      char delimiter = '\n',+      uint64_t bufSize = 1024)+      -> ReadLinesCallback<typename std::decay<Callback>::type> {+    return ReadLinesCallback<typename std::decay<Callback>::type>(+        std::forward<Callback>(fdLineCb), maxLineLength, delimiter, bufSize);+  }++  /**+   * communicate() callbacks can use this to temporarily enable/disable+   * notifications (callbacks) for a pipe to/from the child.  By default,+   * all are enabled.  Useful for "chatty" communication -- you want to+   * disable write callbacks until you receive the expected message.+   *+   * Disabling a pipe does not free you from the requirement to consume all+   * incoming data.  Failing to do so will easily create deadlock bugs.+   *+   * Throws if the childFd is not known.+   */+  void enableNotifications(int childFd, bool enabled);++  /**+   * Are notifications for one pipe to/from child enabled?  Throws if the+   * childFd is not known.+   */+  bool notificationsEnabled(int childFd) const;++  ////+  //// The following methods are meant for the cases when communicate() is+  //// not suitable.  You should not need them when you call communicate(),+  //// and, in fact, it is INHERENTLY UNSAFE to use closeParentFd() or+  //// takeOwnershipOfPipes() from a communicate() callback.+  ////++  /**+   * Close the parent file descriptor given a file descriptor in the child.+   * DO NOT USE from communicate() callbacks; make them return true instead.+   */+  void closeParentFd(int childFd);++  /**+   * Set all pipes from / to child to be non-blocking.  communicate() does+   * this for you.+   */+  void setAllNonBlocking();++  /**+   * Get parent file descriptor corresponding to the given file descriptor+   * in the child.  Throws if childFd isn't a pipe (PIPE_IN / PIPE_OUT).+   * Do not close() the returned file descriptor; use closeParentFd, above.+   */+  int parentFd(int childFd) const {+    return pipes_[findByChildFd(childFd)].pipe.fd();+  }+  int stdinFd() const { return parentFd(0); }+  int stdoutFd() const { return parentFd(1); }+  int stderrFd() const { return parentFd(2); }++  /**+   * The child's pipes are logically separate from the process metadata+   * (they may even be kept alive by the child's descendants).  This call+   * lets you manage the pipes' lifetime separately from the lifetime of the+   * child process.+   *+   * After this call, the Subprocess instance will have no knowledge of+   * these pipes, and the caller assumes responsibility for managing their+   * lifetimes.  Pro-tip: prefer to explicitly close() the pipes, since+   * folly::File would otherwise silently suppress I/O errors.+   *+   * No, you may NOT call this from a communicate() callback.+   */+  struct ChildPipe {+    ChildPipe(int fd, folly::File&& ppe) : childFd(fd), pipe(std::move(ppe)) {}+    int childFd;+    folly::File pipe; // Owns the parent FD+  };+  std::vector<ChildPipe> takeOwnershipOfPipes();++ private:+  struct LibcReal;+  struct SpawnRawArgs;+  struct ChildErrorInfo;++  // spawn() sets up a pipe to read errors from the child,+  // then calls spawnInternal() to do the bulk of the work.  Once+  // spawnInternal() returns it reads the error pipe to see if the child+  // encountered any errors.+  void spawn(+      std::unique_ptr<const char*[]> argv,+      const char* executable,+      const Options& options,+      const std::vector<std::string>* env);+  void spawnInternal(+      std::unique_ptr<const char*[]> argv,+      const char* executable,+      Options& options,+      const std::vector<std::string>* env,+      ChildErrorInfo* err);++  static pid_t spawnInternalDoFork(SpawnRawArgs const& args);+  [[noreturn]] static void childError(+      SpawnRawArgs const& args, int errCode, int errnoValue);++  // Actions to run in child.+  // Note that this runs after vfork(), so tread lightly.+  // Returns 0 on success, or an errno value on failure.+  static int prepareChild(SpawnRawArgs const& args);+  static int prepareChildDoOptionalError(int* errout);+  static int prepareChildDoLinuxCGroup(SpawnRawArgs const& args);+  static int runChild(SpawnRawArgs const& args);++  // Closes fds inherited from parent in child process+  static void closeInheritedFds(const SpawnRawArgs& args);++  /**+   * Read from the error pipe, and throw SubprocessSpawnError if the child+   * failed before calling exec().+   */+  void readChildErrorNum(ChildErrorInfo err, const char* executable);++  // Returns an index into pipes_. Throws std::invalid_argument if not found.+  size_t findByChildFd(const int childFd) const;++  static constexpr TimeoutDuration::rep DestroyBehaviorFatal = -1;+  static constexpr TimeoutDuration::rep DestroyBehaviorLeak = -2;++  pid_t pid_{-1};+  ProcessReturnCode returnCode_;+  TimeoutDuration::rep destroyBehavior_ = DestroyBehaviorFatal;++  /**+   * Represents a pipe between this process, and the child process (or its+   * descendant).  To interact with these pipes, you can use communicate(),+   * or use parentFd() and related methods, or separate them from the+   * Subprocess instance entirely via takeOwnershipOfPipes().+   */+  struct Pipe : private boost::totally_ordered<Pipe> {+    folly::File pipe; // Our end of the pipe, wrapped in a File to auto-close.+    int childFd = -1; // Identifies the pipe: what FD is this in the child?+    int direction = PIPE_IN; // one of PIPE_IN / PIPE_OUT+    bool enabled = true; // Are notifications enabled in communicate()?++    bool operator<(const Pipe& other) const { return childFd < other.childFd; }+    bool operator==(const Pipe& other) const {+      return childFd == other.childFd;+    }+  };++  // Populated at process start according to fdActions, empty after+  // takeOwnershipOfPipes().  Sorted by childFd.  Can only have elements+  // erased, but not inserted, after being populated.+  //+  // The number of pipes between parent and child is assumed to be small,+  // so we're happy with a vector here, even if it means linear erase.+  std::vector<Pipe> pipes_;+};++} // namespace folly
+ folly/folly/Synchronized.h view
@@ -0,0 +1,1841 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++//+// Docs: https://fburl.com/fbcref_synchronized+//++/**+ * This module implements a Synchronized abstraction useful in+ * mutex-based concurrency.+ *+ * The Synchronized<T, Mutex> class is the primary public API exposed by this+ * module.  See folly/docs/Synchronized.md for a more complete explanation of+ * this class and its benefits.+ */++#pragma once++#include <folly/Function.h>+#include <folly/Likely.h>+#include <folly/Preprocessor.h>+#include <folly/SharedMutex.h>+#include <folly/Traits.h>+#include <folly/Utility.h>+#include <folly/container/Foreach.h>+#include <folly/functional/ApplyTuple.h>+#include <folly/synchronization/Lock.h>++#include <glog/logging.h>++#include <array>+#include <mutex>+#include <tuple>+#include <type_traits>+#include <utility>++namespace folly {++namespace detail {++template <typename, typename Mutex>+inline constexpr bool kSynchronizedMutexIsUnique = false;+template <typename Mutex>+inline constexpr bool kSynchronizedMutexIsUnique<+    decltype(void(std::declval<Mutex&>().lock())),+    Mutex> = true;++template <typename, typename Mutex>+inline constexpr bool kSynchronizedMutexIsShared = false;+template <typename Mutex>+inline constexpr bool kSynchronizedMutexIsShared<+    decltype(void(std::declval<Mutex&>().lock_shared())),+    Mutex> = true;++template <typename, typename Mutex>+inline constexpr bool kSynchronizedMutexIsUpgrade = false;+template <typename Mutex>+inline constexpr bool kSynchronizedMutexIsUpgrade<+    decltype(void(std::declval<Mutex&>().lock_upgrade())),+    Mutex> = true;++/**+ * An enum to describe the "level" of a mutex.  The supported levels are+ *  Unique - a normal mutex that supports only exclusive locking+ *  Shared - a shared mutex which has shared locking and unlocking functions;+ *  Upgrade - a mutex that has all the methods of the two above along with+ *            support for upgradable locking+ */+enum class SynchronizedMutexLevel { Unknown, Unique, Shared, Upgrade };++template <typename Mutex>+inline constexpr SynchronizedMutexLevel kSynchronizedMutexLevel =+    kSynchronizedMutexIsUpgrade<void, Mutex>  ? SynchronizedMutexLevel::Upgrade+    : kSynchronizedMutexIsShared<void, Mutex> ? SynchronizedMutexLevel::Shared+    : kSynchronizedMutexIsUnique<void, Mutex>+    ? SynchronizedMutexLevel::Unique+    : SynchronizedMutexLevel::Unknown;++enum class SynchronizedMutexMethod { Lock, TryLock };++template <SynchronizedMutexLevel Level, SynchronizedMutexMethod Method>+struct SynchronizedLockPolicy {+  static constexpr SynchronizedMutexLevel level = Level;+  static constexpr SynchronizedMutexMethod method = Method;+};+using SynchronizedLockPolicyExclusive = SynchronizedLockPolicy<+    SynchronizedMutexLevel::Unique,+    SynchronizedMutexMethod::Lock>;+using SynchronizedLockPolicyTryExclusive = SynchronizedLockPolicy<+    SynchronizedMutexLevel::Unique,+    SynchronizedMutexMethod::TryLock>;+using SynchronizedLockPolicyShared = SynchronizedLockPolicy<+    SynchronizedMutexLevel::Shared,+    SynchronizedMutexMethod::Lock>;+using SynchronizedLockPolicyTryShared = SynchronizedLockPolicy<+    SynchronizedMutexLevel::Shared,+    SynchronizedMutexMethod::TryLock>;+using SynchronizedLockPolicyUpgrade = SynchronizedLockPolicy<+    SynchronizedMutexLevel::Upgrade,+    SynchronizedMutexMethod::Lock>;+using SynchronizedLockPolicyTryUpgrade = SynchronizedLockPolicy<+    SynchronizedMutexLevel::Upgrade,+    SynchronizedMutexMethod::TryLock>;++template <SynchronizedMutexLevel>+struct SynchronizedLockType_ {};+template <>+struct SynchronizedLockType_<SynchronizedMutexLevel::Unique> {+  template <typename Mutex>+  using apply = std::unique_lock<Mutex>;+};+template <>+struct SynchronizedLockType_<SynchronizedMutexLevel::Shared> {+  template <typename Mutex>+  using apply = std::shared_lock<Mutex>;+};+template <>+struct SynchronizedLockType_<SynchronizedMutexLevel::Upgrade> {+  template <typename Mutex>+  using apply = upgrade_lock<Mutex>;+};+template <SynchronizedMutexLevel Level, typename MutexType>+using SynchronizedLockType =+    typename SynchronizedLockType_<Level>::template apply<MutexType>;++} // namespace detail++/**+ * SynchronizedBase is a helper parent class for Synchronized<T>.+ *+ * It provides wlock() and rlock() methods for shared mutex types,+ * or lock() methods for purely exclusive mutex types.+ */+template <class Subclass, detail::SynchronizedMutexLevel level>+class SynchronizedBase;++template <class LockedType, class Mutex, class LockPolicy>+class LockedPtrBase;+template <class LockedType, class LockPolicy>+class LockedPtr;++/**+ * SynchronizedBase specialization for shared mutex types.+ *+ * This class provides wlock() and rlock() methods for acquiring the lock and+ * accessing the data.+ */+template <class Subclass>+class SynchronizedBase<Subclass, detail::SynchronizedMutexLevel::Shared> {+ private:+  template <typename T, typename P>+  using LockedPtr_ = ::folly::LockedPtr<T, P>;++ public:+  using LockPolicyExclusive = detail::SynchronizedLockPolicyExclusive;+  using LockPolicyShared = detail::SynchronizedLockPolicyShared;+  using LockPolicyTryExclusive = detail::SynchronizedLockPolicyTryExclusive;+  using LockPolicyTryShared = detail::SynchronizedLockPolicyTryShared;++  using WLockedPtr = LockedPtr_<Subclass, LockPolicyExclusive>;+  using ConstWLockedPtr = LockedPtr_<const Subclass, LockPolicyExclusive>;++  using RLockedPtr = LockedPtr_<Subclass, LockPolicyShared>;+  using ConstRLockedPtr = LockedPtr_<const Subclass, LockPolicyShared>;++  using TryWLockedPtr = LockedPtr_<Subclass, LockPolicyTryExclusive>;+  using ConstTryWLockedPtr = LockedPtr_<const Subclass, LockPolicyTryExclusive>;++  using TryRLockedPtr = LockedPtr_<Subclass, LockPolicyTryShared>;+  using ConstTryRLockedPtr = LockedPtr_<const Subclass, LockPolicyTryShared>;++  // These aliases are deprecated.+  // TODO: Codemod them away.+  using LockedPtr = WLockedPtr;+  using ConstLockedPtr = ConstRLockedPtr;++  /**+   * @brief Acquire an exclusive lock.+   *+   * Acquire an exclusive lock, and return a LockedPtr that can be used to+   * safely access the datum.+   *+   * LockedPtr offers operator -> and * to provide access to the datum.+   * The lock will be released when the LockedPtr is destroyed.+   *+   * @methodset Exclusive lock+   */+  LockedPtr wlock() { return LockedPtr(static_cast<Subclass*>(this)); }+  ConstWLockedPtr wlock() const {+    return ConstWLockedPtr(static_cast<const Subclass*>(this));+  }++  /**+   * @brief Acquire an exclusive lock, or null.+   *+   * Attempts to acquire the lock in exclusive mode.  If acquisition is+   * unsuccessful, the returned LockedPtr will be null.+   *+   * (Use LockedPtr::operator bool() or LockedPtr::isNull() to check for+   * validity.)+   *+   * @methodset Exclusive lock+   */+  TryWLockedPtr tryWLock() {+    return TryWLockedPtr{static_cast<Subclass*>(this)};+  }+  ConstTryWLockedPtr tryWLock() const {+    return ConstTryWLockedPtr{static_cast<const Subclass*>(this)};+  }++  /**+   * @brief Acquire a read lock.+   *+   * The returned LockedPtr will force const access to the data unless the lock+   * is acquired in non-const context and asNonConstUnsafe() is used.+   *+   * @methodset Shared lock+   */+  RLockedPtr rlock() { return RLockedPtr(static_cast<Subclass*>(this)); }+  ConstLockedPtr rlock() const {+    return ConstLockedPtr(static_cast<const Subclass*>(this));+  }++  /**+   * @brief Acquire a read lock, or null.+   *+   * Attempts to acquire the lock in shared mode.  If acquisition is+   * unsuccessful, the returned LockedPtr will be null.+   *+   * (Use LockedPtr::operator bool() or LockedPtr::isNull() to check for+   * validity.)+   *+   * @methodset Shared lock+   */+  TryRLockedPtr tryRLock() {+    return TryRLockedPtr{static_cast<Subclass*>(this)};+  }+  ConstTryRLockedPtr tryRLock() const {+    return ConstTryRLockedPtr{static_cast<const Subclass*>(this)};+  }++  /**+   * Attempts to acquire the lock, or fails if the timeout elapses first.+   * If acquisition is unsuccessful, the returned LockedPtr will be null.+   *+   * (Use LockedPtr::operator bool() or LockedPtr::isNull() to check for+   * validity.)+   *+   * @methodset Exclusive lock+   */+  template <class Rep, class Period>+  LockedPtr wlock(const std::chrono::duration<Rep, Period>& timeout) {+    return LockedPtr(static_cast<Subclass*>(this), timeout);+  }+  template <class Rep, class Period>+  LockedPtr wlock(const std::chrono::duration<Rep, Period>& timeout) const {+    return LockedPtr(static_cast<const Subclass*>(this), timeout);+  }++  /**+   * Attempts to acquire the lock, or fails if the timeout elapses first.+   * If acquisition is unsuccessful, the returned LockedPtr will be null.+   *+   * (Use LockedPtr::operator bool() or LockedPtr::isNull() to check for+   * validity.)+   *+   * @methodset Shared lock+   */+  template <class Rep, class Period>+  RLockedPtr rlock(const std::chrono::duration<Rep, Period>& timeout) {+    return RLockedPtr(static_cast<Subclass*>(this), timeout);+  }+  template <class Rep, class Period>+  ConstRLockedPtr rlock(+      const std::chrono::duration<Rep, Period>& timeout) const {+    return ConstRLockedPtr(static_cast<const Subclass*>(this), timeout);+  }++  /**+   * Invoke a function while holding the lock exclusively.+   *+   * A reference to the datum will be passed into the function as its only+   * argument.+   *+   * This can be used with a lambda argument for easily defining small critical+   * sections in the code.  For example:+   *+   *   auto value = obj.withWLock([](auto& data) {+   *     data.doStuff();+   *     return data.getValue();+   *   });+   *+   * @methodset Exclusive lock+   */+  template <class Function>+  auto withWLock(Function&& function) {+    return function(*wlock());+  }+  template <class Function>+  auto withWLock(Function&& function) const {+    return function(*wlock());+  }++  /**+   * Invoke a function while holding the lock exclusively.+   *+   * This is similar to withWLock(), but the function will be passed a+   * LockedPtr rather than a reference to the data itself.+   *+   * This allows scopedUnlock() to be called on the LockedPtr argument if+   * desired.+   *+   * @methodset Exclusive lock+   */+  template <class Function>+  auto withWLockPtr(Function&& function) {+    return function(wlock());+  }+  template <class Function>+  auto withWLockPtr(Function&& function) const {+    return function(wlock());+  }++  /**+   * Invoke a function while holding an the lock in shared mode.+   *+   * A const reference to the datum will be passed into the function as its+   * only argument.+   *+   * @methodset Shared lock+   */+  template <class Function>+  auto withRLock(Function&& function) const {+    return function(*rlock());+  }++  /**+   * Invoke a function while holding the lock in shared mode.+   *+   * This is similar to withRLock(), but the function will be passed a+   * LockedPtr rather than a reference to the data itself.+   *+   * This allows scopedUnlock() to be called on the LockedPtr argument if+   * desired.+   *+   * @methodset Shared lock+   */+  template <class Function>+  auto withRLockPtr(Function&& function) {+    return function(rlock());+  }++  template <class Function>+  auto withRLockPtr(Function&& function) const {+    return function(rlock());+  }+};++/**+ * SynchronizedBase specialization for upgrade mutex types.+ *+ * This class provides all the functionality provided by the SynchronizedBase+ * specialization for shared mutexes and a ulock() method that returns an+ * upgrade lock RAII proxy+ */+template <class Subclass>+class SynchronizedBase<Subclass, detail::SynchronizedMutexLevel::Upgrade>+    : public SynchronizedBase<+          Subclass,+          detail::SynchronizedMutexLevel::Shared> {+ private:+  template <typename T, typename P>+  using LockedPtr_ = ::folly::LockedPtr<T, P>;++ public:+  using LockPolicyUpgrade = detail::SynchronizedLockPolicyUpgrade;+  using LockPolicyTryUpgrade = detail::SynchronizedLockPolicyTryUpgrade;++  using UpgradeLockedPtr = LockedPtr_<Subclass, LockPolicyUpgrade>;+  using ConstUpgradeLockedPtr = LockedPtr_<const Subclass, LockPolicyUpgrade>;++  using TryUpgradeLockedPtr = LockedPtr_<Subclass, LockPolicyTryUpgrade>;+  using ConstTryUpgradeLockedPtr =+      LockedPtr_<const Subclass, LockPolicyTryUpgrade>;++  /**+   * @brief Acquire an upgrade lock.+   *+   * The returned LockedPtr will have force const access to the data unless the+   * lock is acquired in non-const context and asNonConstUnsafe() is used.+   *+   * @methodset Upgrade lock+   */+  UpgradeLockedPtr ulock() {+    return UpgradeLockedPtr(static_cast<Subclass*>(this));+  }+  ConstUpgradeLockedPtr ulock() const {+    return ConstUpgradeLockedPtr(static_cast<const Subclass*>(this));+  }++  /**+   * @brief Acquire an upgrade lock, or null.+   *+   * Attempts to acquire the lock in upgrade mode.  If acquisition is+   * unsuccessful, the returned LockedPtr will be null.+   *+   * (Use LockedPtr::operator bool() or LockedPtr::isNull() to check for+   * validity.)+   *+   * @methodset Upgrade lock+   */+  TryUpgradeLockedPtr tryULock() {+    return TryUpgradeLockedPtr{static_cast<Subclass*>(this)};+  }++  /**+   * Acquire an upgrade lock and return a LockedPtr that can be used to safely+   * access the datum+   *+   * And the const version+   *+   * @methodset Upgrade lock+   */+  template <class Rep, class Period>+  UpgradeLockedPtr ulock(const std::chrono::duration<Rep, Period>& timeout) {+    return UpgradeLockedPtr(static_cast<Subclass*>(this), timeout);+  }++  /**+   * Invoke a function while holding the lock.+   *+   * A reference to the datum will be passed into the function as its only+   * argument.+   *+   * This can be used with a lambda argument for easily defining small critical+   * sections in the code.  For example:+   *+   *   auto value = obj.withULock([](auto& data) {+   *     data.doStuff();+   *     return data.getValue();+   *   });+   *+   * This is probably not the function you want.  If the intent is to read the+   * data object and determine whether you should upgrade to a write lock then+   * the withULockPtr() method should be called instead, since it gives access+   * to the LockedPtr proxy (which can be upgraded via the+   * moveFromUpgradeToWrite() method)+   *+   * @methodset Upgrade lock+   */+  template <class Function>+  auto withULock(Function&& function) {+    return function(*ulock());+  }+  template <class Function>+  auto withULock(Function&& function) const {+    return function(*ulock());+  }++  /**+   * Invoke a function while holding the lock exclusively.+   *+   * This is similar to withULock(), but the function will be passed a+   * LockedPtr rather than a reference to the data itself.+   *+   * This allows scopedUnlock() and as_lock() to be called on the+   * LockedPtr argument.+   *+   * This also allows you to upgrade the LockedPtr proxy to a write state so+   * that changes can be made to the underlying data+   *+   * @methodset Upgrade lock+   */+  template <class Function>+  auto withULockPtr(Function&& function) {+    return function(ulock());+  }+  template <class Function>+  auto withULockPtr(Function&& function) const {+    return function(ulock());+  }+};++/**+ * SynchronizedBase specialization for non-shared mutex types.+ *+ * This class provides lock() methods for acquiring the lock and accessing the+ * data.+ */+template <class Subclass>+class SynchronizedBase<Subclass, detail::SynchronizedMutexLevel::Unique> {+ private:+  template <typename T, typename P>+  using LockedPtr_ = ::folly::LockedPtr<T, P>;++ public:+  using LockPolicyExclusive = detail::SynchronizedLockPolicyExclusive;+  using LockPolicyTryExclusive = detail::SynchronizedLockPolicyTryExclusive;++  using LockedPtr = LockedPtr_<Subclass, LockPolicyExclusive>;+  using ConstLockedPtr = LockedPtr_<const Subclass, LockPolicyExclusive>;++  using TryLockedPtr = LockedPtr_<Subclass, LockPolicyTryExclusive>;+  using ConstTryLockedPtr = LockedPtr_<const Subclass, LockPolicyTryExclusive>;++  /**+   * @brief Acquire the lock.+   *+   * Return a LockedPtr that can be used to safely access the datum.+   *+   * @methodset Non-shareable lock+   */+  LockedPtr lock() { return LockedPtr(static_cast<Subclass*>(this)); }++  /**+   * Acquire a lock, and return a ConstLockedPtr that can be used to safely+   * access the datum.+   *+   * @methodset Non-shareable lock+   */+  ConstLockedPtr lock() const {+    return ConstLockedPtr(static_cast<const Subclass*>(this));+  }++  /**+   * @brief Acquire the lock, or null.+   *+   * Attempts to acquire the lock in exclusive mode.  If acquisition is+   * unsuccessful, the returned LockedPtr will be null.+   *+   * (Use LockedPtr::operator bool() or LockedPtr::isNull() to check for+   * validity.)+   *+   * @methodset Non-shareable lock+   */+  TryLockedPtr tryLock() { return TryLockedPtr{static_cast<Subclass*>(this)}; }+  ConstTryLockedPtr tryLock() const {+    return ConstTryLockedPtr{static_cast<const Subclass*>(this)};+  }++  /**+   * Attempts to acquire the lock, or fails if the timeout elapses first.+   * If acquisition is unsuccessful, the returned LockedPtr will be null.+   *+   * @methodset Non-shareable lock+   */+  template <class Rep, class Period>+  LockedPtr lock(const std::chrono::duration<Rep, Period>& timeout) {+    return LockedPtr(static_cast<Subclass*>(this), timeout);+  }++  /**+   * Attempts to acquire the lock, or fails if the timeout elapses first.+   * If acquisition is unsuccessful, the returned LockedPtr will be null.+   *+   * @methodset Non-shareable lock+   */+  template <class Rep, class Period>+  ConstLockedPtr lock(const std::chrono::duration<Rep, Period>& timeout) const {+    return ConstLockedPtr(static_cast<const Subclass*>(this), timeout);+  }++  /**+   * Invoke a function while holding the lock.+   *+   * A reference to the datum will be passed into the function as its only+   * argument.+   *+   * This can be used with a lambda argument for easily defining small critical+   * sections in the code.  For example:+   *+   *   auto value = obj.withLock([](auto& data) {+   *     data.doStuff();+   *     return data.getValue();+   *   });+   *+   * @methodset Non-shareable lock+   */+  template <class Function>+  auto withLock(Function&& function) {+    return function(*lock());+  }+  template <class Function>+  auto withLock(Function&& function) const {+    return function(*lock());+  }++  /**+   * Invoke a function while holding the lock exclusively.+   *+   * This is similar to withWLock(), but the function will be passed a+   * LockedPtr rather than a reference to the data itself.+   *+   * This allows scopedUnlock() and as_lock() to be called on the+   * LockedPtr argument.+   *+   * @methodset Non-shareable lock+   */+  template <class Function>+  auto withLockPtr(Function&& function) {+    return function(lock());+  }+  template <class Function>+  auto withLockPtr(Function&& function) const {+    return function(lock());+  }+};++/**+ * `folly::Synchronized` pairs a datum with a mutex. The datum can only be+ * reached through a `LockedPtr`, typically acquired via `.rlock()` or+ * `.wlock()`; the mutex is held for the lifetime of the `LockedPtr`.+ *+ * It is recommended to explicitly open a new nested scope when aquiring+ * a `LockedPtr` object, to help visibly delineate the critical section and to+ * ensure that the `LockedPtr` is destroyed as soon as it is no longer needed.+ *+ * @tparam T  The type of datum to be stored.+ * @tparam Mutex  The mutex type that guards the datum. Must be Lockable.+ *+ * @refcode folly/docs/examples/folly/Synchronized.cpp+ */+template <class T, class Mutex = SharedMutex>+struct Synchronized+    : public SynchronizedBase<+          Synchronized<T, Mutex>,+          detail::kSynchronizedMutexLevel<Mutex>> {+ private:+  using Base = SynchronizedBase<+      Synchronized<T, Mutex>,+      detail::kSynchronizedMutexLevel<Mutex>>;+  static constexpr bool nxCopyCtor{+      std::is_nothrow_copy_constructible<T>::value};+  static constexpr bool nxMoveCtor{+      std::is_nothrow_move_constructible<T>::value};++  // used to disable copy construction and assignment+  class NonImplementedType;++ public:+  using LockedPtr = typename Base::LockedPtr;+  using ConstLockedPtr = typename Base::ConstLockedPtr;+  using DataType = T;+  using MutexType = Mutex;++  /**+   * Default constructor leaves both members call their own default constructor.+   */+  constexpr Synchronized() = default;++ public:+  /**+   * Copy constructor. Enabled only when the data type is copy-constructible.+   *+   * Takes a shared-or-exclusive lock on the source mutex while performing the+   * copy-construction of the destination data from the source data. No lock is+   * taken on the destination mutex.+   *+   * May throw even when the data type is is nothrow-copy-constructible because+   * acquiring a lock may throw.+   *+   * deprecated+   */+  /* implicit */ Synchronized(+      typename std::conditional<+          std::is_copy_constructible<T>::value,+          const Synchronized&,+          NonImplementedType>::type rhs) /* may throw */+      : Synchronized(rhs.copy()) {}++  /**+   * Move-constructs from the source data without locking either the source or+   * the destination mutex.+   *+   * Semantically, assumes that the source object is a true rvalue and therefore+   * that no synchronization is required for accessing it.+   *+   * deprecated+   */+  Synchronized(Synchronized&& rhs) noexcept(nxMoveCtor)+      : Synchronized(std::move(rhs.datum_)) {}++  /**+   * Constructor taking a datum as argument copies it. There is no+   * need to lock the constructing object.+   */+  explicit Synchronized(const T& rhs) noexcept(nxCopyCtor) : datum_(rhs) {}++  /**+   * Constructor taking a datum rvalue as argument moves it. There is no need+   * to lock the constructing object.+   */+  explicit Synchronized(T&& rhs) noexcept(nxMoveCtor)+      : datum_(std::move(rhs)) {}++  /**+   * Lets you construct non-movable types in-place. Use the constexpr+   * instance `in_place` as the first argument.+   */+  template <typename... Args>+  explicit constexpr Synchronized(std::in_place_t, Args&&... args)+      : datum_(std::forward<Args>(args)...) {}++  /**+   * Lets you construct the synchronized object and also pass construction+   * parameters to the underlying mutex if desired+   */+  template <typename... DatumArgs, typename... MutexArgs>+  Synchronized(+      std::piecewise_construct_t,+      std::tuple<DatumArgs...> datumArgs,+      std::tuple<MutexArgs...> mutexArgs)+      : Synchronized{+            std::piecewise_construct,+            std::move(datumArgs),+            std::move(mutexArgs),+            std::make_index_sequence<sizeof...(DatumArgs)>{},+            std::make_index_sequence<sizeof...(MutexArgs)>{}} {}++  /**+   * Copy assignment operator.+   *+   * Enabled only when the data type is copy-constructible and move-assignable.+   *+   * Move-assigns from a copy of the source data.+   *+   * Takes a shared-or-exclusive lock on the source mutex while copying the+   * source data to a temporary. Takes an exclusive lock on the destination+   * mutex while move-assigning from the temporary.+   *+   * This technique consts an extra temporary but avoids the need to take locks+   * on both mutexes together.+   *+   * deprecated+   */+  Synchronized& operator=(+      typename std::conditional<+          std::is_copy_constructible<T>::value &&+              std::is_move_assignable<T>::value,+          const Synchronized&,+          NonImplementedType>::type rhs) {+    return *this = rhs.copy();+  }++  /**+   * Move assignment operator.+   *+   * Takes an exclusive lock on the destination mutex while move-assigning the+   * destination data from the source data. The source mutex is not locked or+   * otherwise accessed.+   *+   * Semantically, assumes that the source object is a true rvalue and therefore+   * that no synchronization is required for accessing it.+   *+   * deprecated+   */+  Synchronized& operator=(Synchronized&& rhs) {+    return *this = std::move(rhs.datum_);+  }++  /**+   * Lock object, assign datum.+   */+  Synchronized& operator=(const T& rhs) {+    if (&datum_ != &rhs) {+      auto guard = LockedPtr{this};+      datum_ = rhs;+    }+    return *this;+  }++  /**+   * Lock object, move-assign datum.+   */+  Synchronized& operator=(T&& rhs) {+    if (&datum_ != &rhs) {+      auto guard = LockedPtr{this};+      datum_ = std::move(rhs);+    }+    return *this;+  }++  /**+   * @brief Acquire some lock.+   *+   * If the mutex is a shared mutex, and the Synchronized instance is const,+   * this acquires a shared lock.  Otherwise this acquires an exclusive lock.+   *+   * In general, prefer using the explicit rlock() and wlock() methods+   * for read-write locks, and lock() for purely exclusive locks.+   *+   * contextualLock() is primarily intended for use in other template functions+   * that do not necessarily know the lock type.+   */+  LockedPtr contextualLock() { return LockedPtr(this); }+  ConstLockedPtr contextualLock() const { return ConstLockedPtr(this); }+  template <class Rep, class Period>+  LockedPtr contextualLock(const std::chrono::duration<Rep, Period>& timeout) {+    return LockedPtr(this, timeout);+  }+  template <class Rep, class Period>+  ConstLockedPtr contextualLock(+      const std::chrono::duration<Rep, Period>& timeout) const {+    return ConstLockedPtr(this, timeout);+  }+  /**+   * @brief Acquire a lock for reading.+   *+   * contextualRLock() acquires a read lock if the mutex type is shared,+   * or a regular exclusive lock for non-shared mutex types.+   *+   * contextualRLock() when you know that you prefer a read lock (if+   * available), even if the Synchronized<T> object itself is non-const.+   */+  ConstLockedPtr contextualRLock() const { return ConstLockedPtr(this); }+  template <class Rep, class Period>+  ConstLockedPtr contextualRLock(+      const std::chrono::duration<Rep, Period>& timeout) const {+    return ConstLockedPtr(this, timeout);+  }++  /**+   * @brief Acquire a LockedPtr with timeout.+   *+   * Attempts to acquire for a given number of milliseconds. If+   * acquisition is unsuccessful, the returned LockedPtr is nullptr.+   *+   * NOTE: This API is deprecated.  Use lock(), wlock(), or rlock() instead.+   * In the future it will be marked with a deprecation attribute to emit+   * build-time warnings, and then it will be removed entirely.+   */+  LockedPtr timedAcquire(unsigned int milliseconds) {+    return LockedPtr(this, std::chrono::milliseconds(milliseconds));+  }++  /**+   * Attempts to acquire for a given number of milliseconds. If+   * acquisition is unsuccessful, the returned ConstLockedPtr is nullptr.+   *+   * NOTE: This API is deprecated.  Use lock(), wlock(), or rlock() instead.+   * In the future it will be marked with a deprecation attribute to emit+   * build-time warnings, and then it will be removed entirely.+   */+  ConstLockedPtr timedAcquire(unsigned int milliseconds) const {+    return ConstLockedPtr(this, std::chrono::milliseconds(milliseconds));+  }++  /**+   * @brief Swap datum.+   *+   * Swaps with another Synchronized. Protected against+   * self-swap. Only data is swapped. Locks are acquired in increasing+   * address order.+   */+  void swap(Synchronized& rhs) {+    if (this == &rhs) {+      return;+    }+    if (this > &rhs) {+      return rhs.swap(*this);+    }+    auto guard1 = LockedPtr{this};+    auto guard2 = LockedPtr{&rhs};++    using std::swap;+    swap(datum_, rhs.datum_);+  }++  /**+   * Swap with another datum. Recommended because it keeps the mutex+   * held only briefly.+   */+  void swap(T& rhs) {+    LockedPtr guard(this);++    using std::swap;+    swap(datum_, rhs);+  }++  /**+   * @brief Exchange datum.+   *+   * Assign another datum and return the original value. Recommended+   * because it keeps the mutex held only briefly.+   */+  T exchange(T&& rhs) {+    swap(rhs);+    return std::move(rhs);+  }++  /**+   * Copies datum to a given target.+   */+  void copyInto(T& target) const {+    ConstLockedPtr guard(this);+    target = datum_;+  }++  /**+   * Returns a fresh copy of the datum.+   */+  T copy() const {+    ConstLockedPtr guard(this);+    return datum_;+  }++  /**+   * @brief Access datum without locking.+   *+   * Returns a reference to the datum without acquiring a lock.+   *+   * Provided as a backdoor for call-sites where it is known safe to be used.+   * For example, when it is known that only one thread has access to the+   * Synchronized instance.+   *+   * To be used with care - this method explicitly overrides the normal safety+   * guarantees provided by the rest of the Synchronized API.+   */+  T& unsafeGetUnlocked() { return datum_; }+  const T& unsafeGetUnlocked() const { return datum_; }++  /**+   * @brief Access underlying mutex_ directly.+   *+   * Provided as a backdoor for call-sites where the lock and unlock are paired+   * in different calls. For example, in fork handlers. Use carefully as the+   * caller is responsible to ensure it is paired with an unlock and there is+   * nothing else in between that tries to implicitly or explicitly acquire the+   * lock again.+   */+  Mutex& unsafeGetMutex() { return mutex_; }++ private:+  template <class LockedType, class MutexType, class LockPolicy>+  friend class folly::LockedPtrBase;+  template <class LockedType, class LockPolicy>+  friend class folly::LockedPtr;++  /**+   * Helper constructors to enable Synchronized for+   * non-default constructible types T.+   * Guards are created in actual public constructors and are alive+   * for the time required to construct the object+   */+  Synchronized(+      const Synchronized& rhs,+      const ConstLockedPtr& /*guard*/) noexcept(nxCopyCtor)+      : datum_(rhs.datum_) {}++  Synchronized(Synchronized&& rhs, const LockedPtr& /*guard*/) noexcept(+      nxMoveCtor)+      : datum_(std::move(rhs.datum_)) {}++  template <+      typename... DatumArgs,+      typename... MutexArgs,+      std::size_t... IndicesOne,+      std::size_t... IndicesTwo>+  Synchronized(+      std::piecewise_construct_t,+      std::tuple<DatumArgs...> datumArgs,+      std::tuple<MutexArgs...> mutexArgs,+      std::index_sequence<IndicesOne...>,+      std::index_sequence<IndicesTwo...>)+      : datum_{std::get<IndicesOne>(std::move(datumArgs))...},+        mutex_{std::get<IndicesTwo>(std::move(mutexArgs))...} {}++  // simulacrum of data members - keep data members in sync!+  // LockedPtr needs offsetof() which is specified only for standard-layout+  // types which Synchronized is not so we define a simulacrum for offsetof+  struct Simulacrum {+    aligned_storage_for_t<DataType> datum_;+    aligned_storage_for_t<MutexType> mutex_;+  };++  // data members - keep simulacrum of data members in sync!+  T datum_;+  mutable Mutex mutex_;+};++/**+ * Deprecated subclass of Synchronized that provides implicit locking+ * via operator->. This is intended to ease migration while preventing+ * accidental use of operator-> in new code.+ */+template <class T, class Mutex = SharedMutex>+struct [[deprecated(+    "use Synchronized and explicit lock(), wlock(), or rlock() instead")]] ImplicitSynchronized+    : Synchronized<T, Mutex> {+ private:+  using Base = Synchronized<T, Mutex>;++ public:+  using LockedPtr = typename Base::LockedPtr;+  using ConstLockedPtr = typename Base::ConstLockedPtr;+  using DataType = typename Base::DataType;+  using MutexType = typename Base::MutexType;++  using Base::Base;+  using Base::operator=;++  /**+   * @brief Access the datum under lock.+   *+   * deprecated+   *+   * This accessor offers a LockedPtr. In turn, LockedPtr offers+   * operator-> returning a pointer to T. The operator-> keeps+   * expanding until it reaches a pointer, so syncobj->foo() will lock+   * the object and call foo() against it.+   *+   * NOTE: This API is planned to be deprecated in an upcoming diff.+   * Prefer using lock(), wlock(), or rlock() instead.+   */+  [[deprecated("use explicit lock(), wlock(), or rlock() instead")]] LockedPtr+  operator->() {+    return LockedPtr(this);+  }++  /**+   * deprecated+   *+   * Obtain a ConstLockedPtr.+   *+   * NOTE: This API is planned to be deprecated in an upcoming diff.+   * Prefer using lock(), wlock(), or rlock() instead.+   */+  [[deprecated(+      "use explicit lock(), wlock(), or rlock() instead")]] ConstLockedPtr+  operator->() const {+    return ConstLockedPtr(this);+  }+};++template <class SynchronizedType, class LockPolicy>+class ScopedUnlocker;++namespace detail {+/*+ * A helper alias that resolves to "const T" if the template parameter+ * is a const Synchronized<T>, or "T" if the parameter is not const.+ */+template <class SynchronizedType, bool AllowsConcurrentAccess>+using SynchronizedDataType = typename std::conditional<+    AllowsConcurrentAccess || std::is_const<SynchronizedType>::value,+    typename SynchronizedType::DataType const,+    typename SynchronizedType::DataType>::type;+/*+ * A helper alias that resolves to a ConstLockedPtr if the template parameter+ * is a const Synchronized<T>, or a LockedPtr if the parameter is not const.+ */+template <class SynchronizedType>+using LockedPtrType = typename std::conditional<+    std::is_const<SynchronizedType>::value,+    typename SynchronizedType::ConstLockedPtr,+    typename SynchronizedType::LockedPtr>::type;++template <+    typename Synchronized,+    typename LockFunc,+    typename TryLockFunc,+    typename... Args>+class SynchronizedLocker {+ public:+  using LockedPtr = invoke_result_t<LockFunc&, Synchronized&, const Args&...>;++  template <typename LockFuncType, typename TryLockFuncType, typename... As>+  SynchronizedLocker(+      Synchronized& sync,+      LockFuncType&& lockFunc,+      TryLockFuncType tryLockFunc,+      As&&... as)+      : synchronized{sync},+        lockFunc_{std::forward<LockFuncType>(lockFunc)},+        tryLockFunc_{std::forward<TryLockFuncType>(tryLockFunc)},+        args_{std::forward<As>(as)...} {}++  auto lock() const {+    auto args = std::tuple<const Args&...>{args_};+    return apply(lockFunc_, std::tuple_cat(std::tie(synchronized), args));+  }+  auto tryLock() const { return tryLockFunc_(synchronized); }++ private:+  Synchronized& synchronized;+  LockFunc lockFunc_;+  TryLockFunc tryLockFunc_;+  std::tuple<Args...> args_;+};++template <+    typename Synchronized,+    typename LockFunc,+    typename TryLockFunc,+    typename... Args>+auto makeSynchronizedLocker(+    Synchronized& synchronized,+    LockFunc&& lockFunc,+    TryLockFunc&& tryLockFunc,+    Args&&... args) {+  using LockFuncType = std::decay_t<LockFunc>;+  using TryLockFuncType = std::decay_t<TryLockFunc>;+  return SynchronizedLocker<+      Synchronized,+      LockFuncType,+      TryLockFuncType,+      std::decay_t<Args>...>{+      synchronized,+      std::forward<LockFunc>(lockFunc),+      std::forward<TryLockFunc>(tryLockFunc),+      std::forward<Args>(args)...};+}++/**+ * Acquire locks for multiple Synchronized<T> objects, in a deadlock-safe+ * manner.+ *+ * The function uses the "smart and polite" algorithm from this link+ * http://howardhinnant.github.io/dining_philosophers.html#Polite+ *+ * The gist of the algorithm is that it locks a mutex, then tries to lock the+ * other mutexes in a non-blocking manner.  If all the locks succeed, we are+ * done, if not, we release the locks we have held, yield to allow other+ * threads to continue and then block on the mutex that we failed to acquire.+ *+ * This allows dynamically yielding ownership of all the mutexes but one, so+ * that other threads can continue doing work and locking the other mutexes.+ * See the benchmarks in folly/test/SynchronizedBenchmark.cpp for more.+ */+template <typename... SynchronizedLocker>+auto lock(SynchronizedLocker... lockersIn)+    -> std::tuple<typename SynchronizedLocker::LockedPtr...> {+  // capture the list of lockers as a tuple+  auto lockers = std::forward_as_tuple(lockersIn...);++  // make a list of null LockedPtr instances that we will return to the caller+  auto lockedPtrs = std::tuple<typename SynchronizedLocker::LockedPtr...>{};++  // start by locking the first thing in the list+  std::get<0>(lockedPtrs) = std::get<0>(lockers).lock();+  auto indexLocked = 0;++  while (true) {+    auto couldLockAll = true;++    for_each(lockers, [&](auto& locker, auto index) {+      // if we should try_lock on the current locker then do so+      if (index != indexLocked) {+        auto lockedPtr = locker.tryLock();++        // if we were unable to lock this mutex,+        //+        // 1. release all the locks,+        // 2. yield control to another thread to be nice+        // 3. block on the mutex we failed to lock, acquire the lock+        // 4. break out and set the index of the current mutex to indicate+        //    which mutex we have locked+        if (!lockedPtr) {+          // writing lockedPtrs = decltype(lockedPtrs){} does not compile on+          // gcc, I believe this is a bug D7676798+          lockedPtrs = std::tuple<typename SynchronizedLocker::LockedPtr...>{};++          std::this_thread::yield();+          fetch(lockedPtrs, index) = locker.lock();+          indexLocked = index;+          couldLockAll = false;++          return loop_break;+        }++        // else store the locked mutex in the list we return+        fetch(lockedPtrs, index) = std::move(lockedPtr);+      }++      return loop_continue;+    });++    if (couldLockAll) {+      return lockedPtrs;+    }+  }+}++template <typename Synchronized, typename... Args>+auto wlock(Synchronized& synchronized, Args&&... args) {+  return detail::makeSynchronizedLocker(+      synchronized,+      [](auto& s, auto&&... a) {+        return s.wlock(std::forward<decltype(a)>(a)...);+      },+      [](auto& s) { return s.tryWLock(); },+      std::forward<Args>(args)...);+}+template <typename Synchronized, typename... Args>+auto rlock(Synchronized& synchronized, Args&&... args) {+  return detail::makeSynchronizedLocker(+      synchronized,+      [](auto& s, auto&&... a) {+        return s.rlock(std::forward<decltype(a)>(a)...);+      },+      [](auto& s) { return s.tryRLock(); },+      std::forward<Args>(args)...);+}+template <typename Synchronized, typename... Args>+auto ulock(Synchronized& synchronized, Args&&... args) {+  return detail::makeSynchronizedLocker(+      synchronized,+      [](auto& s, auto&&... a) {+        return s.ulock(std::forward<decltype(a)>(a)...);+      },+      [](auto& s) { return s.tryULock(); },+      std::forward<Args>(args)...);+}+template <typename Synchronized, typename... Args>+auto lock(Synchronized& synchronized, Args&&... args) {+  return detail::makeSynchronizedLocker(+      synchronized,+      [](auto& s, auto&&... a) {+        return s.lock(std::forward<decltype(a)>(a)...);+      },+      [](auto& s) { return s.tryLock(); },+      std::forward<Args>(args)...);+}++} // namespace detail++/**+ * This class temporarily unlocks a LockedPtr in a scoped manner.+ */+template <class SynchronizedType, class LockPolicy>+class ScopedUnlocker {+ public:+  explicit ScopedUnlocker(LockedPtr<SynchronizedType, LockPolicy>* p) noexcept+      : ptr_(p), parent_(p->parent()) {+    ptr_->releaseLock();+  }+  ScopedUnlocker(const ScopedUnlocker&) = delete;+  ScopedUnlocker& operator=(const ScopedUnlocker&) = delete;+  ScopedUnlocker(ScopedUnlocker&& other) noexcept+      : ptr_(std::exchange(other.ptr_, nullptr)),+        parent_(std::exchange(other.parent_, nullptr)) {}+  ScopedUnlocker& operator=(ScopedUnlocker&& other) = delete;++  ~ScopedUnlocker() noexcept(false) {+    if (ptr_) {+      ptr_->reacquireLock(parent_);+    }+  }++ private:+  LockedPtr<SynchronizedType, LockPolicy>* ptr_{nullptr};+  SynchronizedType* parent_{nullptr};+};++/**+ * A LockedPtr keeps a Synchronized<T> object locked for the duration of+ * LockedPtr's existence.+ *+ * It provides access the datum's members directly by using operator->() and+ * operator*().+ *+ * The LockPolicy parameter controls whether or not the lock is acquired in+ * exclusive or shared mode.+ */+template <class SynchronizedType, class LockPolicy>+class LockedPtr {+ private:+  constexpr static bool AllowsConcurrentAccess =+      LockPolicy::level != detail::SynchronizedMutexLevel::Unique;++  using CDataType = // the DataType with the appropriate const-qualification+      detail::SynchronizedDataType<SynchronizedType, AllowsConcurrentAccess>;++  template <typename LockPolicyOther>+  using EnableIfSameLevel =+      std::enable_if_t<LockPolicy::level == LockPolicyOther::level>;++  template <typename, typename>+  friend class LockedPtr;++  friend class ScopedUnlocker<SynchronizedType, LockPolicy>;++ public:+  using DataType = typename SynchronizedType::DataType;+  using MutexType = typename SynchronizedType::MutexType;+  using Synchronized = typename std::remove_const<SynchronizedType>::type;+  using LockType = detail::SynchronizedLockType<LockPolicy::level, MutexType>;++  /**+   * Creates an uninitialized LockedPtr.+   *+   * Dereferencing an uninitialized LockedPtr is not allowed.+   */+  LockedPtr() = default;++  /**+   * Takes a Synchronized<T> and locks it.+   */+  explicit LockedPtr(SynchronizedType* parent)+      : lock_{!parent ? LockType{} : doLock(parent->mutex_)} {}++  /**+   * Takes a Synchronized<T> and attempts to lock it, within the specified+   * timeout.+   *+   * Blocks until the lock is acquired or until the specified timeout expires.+   * If the timeout expired without acquiring the lock, the LockedPtr will be+   * null, and LockedPtr::isNull() will return true.+   */+  template <class Rep, class Period>+  LockedPtr(+      SynchronizedType* parent,+      const std::chrono::duration<Rep, Period>& timeout)+      : lock_{parent ? LockType{parent->mutex_, timeout} : LockType{}} {+    if (isNull()) {+      lock_ = {};+    }+  }++  /**+   * Move constructor.+   */+  LockedPtr(LockedPtr&& rhs) noexcept = default;+  template <+      typename Type = SynchronizedType,+      std::enable_if_t<std::is_const<Type>::value, int> = 0>+  /* implicit */ LockedPtr(LockedPtr<Synchronized, LockPolicy>&& rhs) noexcept+      : lock_{std::move(rhs.lock_)} {}+  template <+      typename LockPolicyType,+      EnableIfSameLevel<LockPolicyType>* = nullptr>+  explicit LockedPtr(+      LockedPtr<SynchronizedType, LockPolicyType>&& other) noexcept+      : lock_{std::move(other.lock_)} {}+  template <+      typename Type = SynchronizedType,+      typename LockPolicyType,+      std::enable_if_t<std::is_const<Type>::value, int> = 0,+      EnableIfSameLevel<LockPolicyType>* = nullptr>+  explicit LockedPtr(LockedPtr<Synchronized, LockPolicyType>&& rhs) noexcept+      : lock_{std::move(rhs.lock_)} {}++  /**+   * Move assignment operator.+   */+  LockedPtr& operator=(LockedPtr&& rhs) noexcept = default;+  template <+      typename LockPolicyType,+      EnableIfSameLevel<LockPolicyType>* = nullptr>+  LockedPtr& operator=(+      LockedPtr<SynchronizedType, LockPolicyType>&& other) noexcept {+    lock_ = std::move(other.lock_);+    return *this;+  }+  template <+      typename Type = SynchronizedType,+      typename LockPolicyType,+      std::enable_if_t<std::is_const<Type>::value, int> = 0,+      EnableIfSameLevel<LockPolicyType>* = nullptr>+  LockedPtr& operator=(+      LockedPtr<Synchronized, LockPolicyType>&& other) noexcept {+    lock_ = std::move(other.lock_);+    return *this;+  }++  /*+   * Copy constructor and assignment operator are deleted.+   */+  LockedPtr(const LockedPtr& rhs) = delete;+  LockedPtr& operator=(const LockedPtr& rhs) = delete;++  /**+   * Destructor releases.+   */+  ~LockedPtr() = default;++  /**+   * Access the underlying lock object.+   */+  LockType& as_lock() noexcept { return lock_; }+  LockType const& as_lock() const noexcept { return lock_; }++  /**+   * Check if this LockedPtr is uninitialized, or points to valid locked data.+   *+   * This method can be used to check if a timed-acquire operation succeeded.+   * If an acquire operation times out it will result in a null LockedPtr.+   *+   * A LockedPtr is always either null, or holds a lock to valid data.+   * Methods such as scopedUnlock() reset the LockedPtr to null for the+   * duration of the unlock.+   */+  bool isNull() const { return !lock_.owns_lock(); }++  /**+   * Explicit boolean conversion.+   *+   * Returns !isNull()+   */+  explicit operator bool() const { return lock_.owns_lock(); }++  /**+   * Access the locked data.+   *+   * This method should only be used if the LockedPtr is valid.+   */+  CDataType* operator->() const { return std::addressof(parent()->datum_); }++  /**+   * Access the locked data.+   *+   * This method should only be used if the LockedPtr is valid.+   */+  CDataType& operator*() const { return parent()->datum_; }++  void unlock() noexcept { lock_ = {}; }++  /**+   * Locks that allow concurrent access (shared, upgrade) force const+   * access with the standard accessors even if the Synchronized+   * object is non-const.+   *+   * In some cases non-const access can be needed, for example:+   *+   *   - Under an upgrade lock, to get references that will be mutated+   *     after upgrading to a write lock.+   *+   *   - Under an read lock, if some mutating operations on the data+   *     are thread safe (e.g. mutating the value in an associative+   *     container with reference stability).+   *+   * asNonConstUnsafe() returns a non-const reference to the data if+   * the parent Synchronized object was non-const at the point of lock+   * acquisition.+   */+  template <typename = void>+  DataType& asNonConstUnsafe() const {+    static_assert(+        AllowsConcurrentAccess && !std::is_const<SynchronizedType>::value,+        "asNonConstUnsafe() is only available on non-exclusive locks"+        " acquired in a non-const context");++    return parent()->datum_;+  }++  /**+   * Temporarily unlock the LockedPtr, and reset it to null.+   *+   * Returns an helper object that will re-lock and restore the LockedPtr when+   * the helper is destroyed.  The LockedPtr may not be dereferenced for as+   * long as this helper object exists.+   */+  ScopedUnlocker<SynchronizedType, LockPolicy> scopedUnlock() {+    return ScopedUnlocker<SynchronizedType, LockPolicy>(this);+  }++  /***************************************************************************+   * Upgrade lock methods.+   * These are disabled via SFINAE when the mutex is not an upgrade mutex.+   **************************************************************************/+  /**+   * Move the locked ptr from an upgrade state to an exclusive state.  The+   * current lock is left in a null state.+   */+  template <+      typename SyncType = SynchronizedType,+      decltype(void(std::declval<typename SyncType::MutexType&>()+                        .lock_upgrade()))* = nullptr>+  LockedPtr<SynchronizedType, detail::SynchronizedLockPolicyExclusive>+  moveFromUpgradeToWrite() {+    static_assert(std::is_same<SyncType, SynchronizedType>::value, "mismatch");+    return transition_to_unique_lock(lock_);+  }++  /**+   * Move the locked ptr from an exclusive state to an upgrade state.  The+   * current lock is left in a null state.+   */+  template <+      typename SyncType = SynchronizedType,+      decltype(void(std::declval<typename SyncType::MutexType&>()+                        .lock_upgrade()))* = nullptr>+  LockedPtr<SynchronizedType, detail::SynchronizedLockPolicyUpgrade>+  moveFromWriteToUpgrade() {+    static_assert(std::is_same<SyncType, SynchronizedType>::value, "mismatch");+    return transition_to_upgrade_lock(lock_);+  }++  /**+   * Move the locked ptr from an upgrade state to a shared state.  The+   * current lock is left in a null state.+   */+  template <+      typename SyncType = SynchronizedType,+      decltype(void(std::declval<typename SyncType::MutexType&>()+                        .lock_upgrade()))* = nullptr>+  LockedPtr<SynchronizedType, detail::SynchronizedLockPolicyShared>+  moveFromUpgradeToRead() {+    static_assert(std::is_same<SyncType, SynchronizedType>::value, "mismatch");+    return transition_to_shared_lock(lock_);+  }++  /**+   * Move the locked ptr from an exclusive state to a shared state.  The+   * current lock is left in a null state.+   */+  template <+      typename SyncType = SynchronizedType,+      decltype(void(std::declval<typename SyncType::MutexType&>()+                        .lock_shared()))* = nullptr>+  LockedPtr<SynchronizedType, detail::SynchronizedLockPolicyShared>+  moveFromWriteToRead() {+    static_assert(std::is_same<SyncType, SynchronizedType>::value, "mismatch");+    return transition_to_shared_lock(lock_);+  }++  SynchronizedType* parent() const {+    using simulacrum = typename SynchronizedType::Simulacrum;+    static_assert(sizeof(simulacrum) == sizeof(SynchronizedType), "mismatch");+    static_assert(alignof(simulacrum) == alignof(SynchronizedType), "mismatch");+    auto off = offsetof(simulacrum, mutex_);+    const auto raw = reinterpret_cast<char*>(lock_.mutex());+    return reinterpret_cast<SynchronizedType*>(raw - (raw ? off : 0));+  }++ private:+  /* implicit */ LockedPtr(LockType lock) noexcept : lock_{std::move(lock)} {}++  template <typename LP>+  static constexpr bool is_try =+      LP::method == detail::SynchronizedMutexMethod::TryLock;++  template <+      typename MT,+      typename LT = LockType,+      typename LP = LockPolicy,+      std::enable_if_t<is_try<LP>, int> = 0>+  FOLLY_ERASE static LT doLock(MT& mutex) {+    return LT{mutex, std::try_to_lock};+  }+  template <+      typename MT,+      typename LT = LockType,+      typename LP = LockPolicy,+      std::enable_if_t<!is_try<LP>, int> = 0>+  FOLLY_ERASE static LT doLock(MT& mutex) {+    return LT{mutex};+  }++  void releaseLock() noexcept {+    DCHECK(lock_.owns_lock());+    lock_ = {};+  }+  void reacquireLock(SynchronizedType* parent) {+    DCHECK(parent);+    DCHECK(!lock_.owns_lock());+    lock_ = doLock(parent->mutex_);+  }++  LockType lock_;+};++/**+ * Helper functions that should be passed to either a lock() or synchronized()+ * invocation, these return implementation defined structs that will be used+ * to lock the synchronized instance appropriately.+ *+ *    lock(wlock(one), rlock(two), wlock(three));+ *    synchronized([](auto one, two) { ... }, wlock(one), rlock(two));+ *+ * For example in the above rlock() produces an implementation defined read+ * locking helper instance and wlock() a write locking helper+ *+ * Subsequent arguments passed to these locking helpers, after the first, will+ * be passed by const-ref to the corresponding function on the synchronized+ * instance.  This means that if the function accepts these parameters by+ * value, they will be copied.  Note that it is not necessary that the primary+ * locking function will be invoked at all (for eg.  the implementation might+ * just invoke the try*Lock() method)+ *+ *    // Try to acquire the lock for one second+ *    synchronized([](auto) { ... }, wlock(one, 1s));+ *+ *    // The timed lock acquire might never actually be called, if it is not+ *    // needed by the underlying deadlock avoiding algorithm+ *    synchronized([](auto, auto) { ... }, rlock(one), wlock(two, 1s));+ *+ * Note that the arguments passed to to *lock() calls will be passed by+ * const-ref to the function invocation, as the implementation might use them+ * many times+ */+template <typename D, typename M, typename... Args>+auto wlock(Synchronized<D, M>& synchronized, Args&&... args) {+  return detail::wlock(synchronized, std::forward<Args>(args)...);+}+template <typename D, typename M, typename... Args>+auto wlock(const Synchronized<D, M>& synchronized, Args&&... args) {+  return detail::wlock(synchronized, std::forward<Args>(args)...);+}+template <typename Data, typename Mutex, typename... Args>+auto rlock(const Synchronized<Data, Mutex>& synchronized, Args&&... args) {+  return detail::rlock(synchronized, std::forward<Args>(args)...);+}+template <typename D, typename M, typename... Args>+auto ulock(Synchronized<D, M>& synchronized, Args&&... args) {+  return detail::ulock(synchronized, std::forward<Args>(args)...);+}+template <typename D, typename M, typename... Args>+auto lock(Synchronized<D, M>& synchronized, Args&&... args) {+  return detail::lock(synchronized, std::forward<Args>(args)...);+}+template <typename D, typename M, typename... Args>+auto lock(const Synchronized<D, M>& synchronized, Args&&... args) {+  return detail::lock(synchronized, std::forward<Args>(args)...);+}++/**+ * Acquire locks for multiple Synchronized<> objects, in a deadlock-safe+ * manner.+ *+ * Wrap the synchronized instances with the appropriate locking strategy by+ * using one of the four strategies - folly::lock (exclusive acquire for+ * exclusive only mutexes), folly::rlock (shared acquire for shareable+ * mutexes), folly::wlock (exclusive acquire for shareable mutexes) or+ * folly::ulock (upgrade acquire for upgrade mutexes) (see above)+ *+ * The locks will be acquired and the passed callable will be invoked with the+ * LockedPtr instances in the order that they were passed to the function+ */+template <typename Func, typename... SynchronizedLockers>+decltype(auto) synchronized(Func&& func, SynchronizedLockers&&... lockers) {+  return apply(+      std::forward<Func>(func),+      lock(std::forward<SynchronizedLockers>(lockers)...));+}++/**+ * Acquire locks on many lockables or synchronized instances in such a way+ * that the sequence of calls within the function does not cause deadlocks.+ *+ * This can often result in a performance boost as compared to simply+ * acquiring your locks in an ordered manner.  Even for very simple cases.+ * The algorithm tried to adjust to contention by blocking on the mutex it+ * thinks is the best fit, leaving all other mutexes open to be locked by+ * other threads.  See the benchmarks in folly/test/SynchronizedBenchmark.cpp+ * for more+ *+ * This works differently as compared to the locking algorithm in libstdc+++ * and is the recommended way to acquire mutexes in a generic order safe+ * manner.  Performance benchmarks show that this does better than the one in+ * libstdc++ even for the simple cases+ *+ * Usage is the same as std::lock() for arbitrary lockables+ *+ *    folly::lock(one, two, three);+ *+ * To make it work with folly::Synchronized you have to specify how you want+ * the locks to be acquired, use the folly::wlock(), folly::rlock(),+ * folly::ulock() and folly::lock() helpers defined below+ *+ *    auto [one, two] = lock(folly::wlock(a), folly::rlock(b));+ *+ * Note that you can/must avoid the folly:: namespace prefix on the lock()+ * function if you use the helpers, ADL lookup is done to find the lock function+ *+ * This will execute the deadlock avoidance algorithm and acquire a write lock+ * for a and a read lock for b+ */+template <typename LockableOne, typename LockableTwo, typename... Lockables>+void lock(LockableOne& one, LockableTwo& two, Lockables&... lockables) {+  auto locker = [](auto& lockable) {+    using Lockable = std::remove_reference_t<decltype(lockable)>;+    return detail::makeSynchronizedLocker(+        lockable,+        [](auto& l) { return std::unique_lock<Lockable>{l}; },+        [](auto& l) {+          return std::unique_lock<Lockable>{l, std::try_to_lock};+        });+  };+  auto locks = lock(locker(one), locker(two), locker(lockables)...);++  // release ownership of the locks from the RAII lock wrapper returned by the+  // function above+  for_each(locks, [&](auto& lock) { lock.release(); });+}++/**+ * Acquire locks for multiple Synchronized<T> objects, in a deadlock-safe+ * manner.+ *+ * The locks are acquired in order from lowest address to highest address.+ * (Note that this is not necessarily the same algorithm used by std::lock().)+ * For parameters that are const and support shared locks, a read lock is+ * acquired.  Otherwise an exclusive lock is acquired.+ *+ * use lock() with folly::wlock(), folly::rlock() and folly::ulock() for+ * arbitrary locking without causing a deadlock (as much as possible), with the+ * same effects as std::lock()+ */+template <class Sync1, class Sync2>+std::tuple<detail::LockedPtrType<Sync1>, detail::LockedPtrType<Sync2>>+acquireLocked(Sync1& l1, Sync2& l2) {+  if (static_cast<const void*>(&l1) < static_cast<const void*>(&l2)) {+    auto p1 = l1.contextualLock();+    auto p2 = l2.contextualLock();+    return std::make_tuple(std::move(p1), std::move(p2));+  } else {+    auto p2 = l2.contextualLock();+    auto p1 = l1.contextualLock();+    return std::make_tuple(std::move(p1), std::move(p2));+  }+}++/**+ * A version of acquireLocked() that returns a std::pair rather than a+ * std::tuple, which is easier to use in many places.+ */+template <class Sync1, class Sync2>+std::pair<detail::LockedPtrType<Sync1>, detail::LockedPtrType<Sync2>>+acquireLockedPair(Sync1& l1, Sync2& l2) {+  auto lockedPtrs = acquireLocked(l1, l2);+  return {+      std::move(std::get<0>(lockedPtrs)), std::move(std::get<1>(lockedPtrs))};+}++/************************************************************************+ * NOTE: All APIs below this line will be deprecated in upcoming diffs.+ ************************************************************************/++// Non-member swap primitive+template <class T, class M>+void swap(Synchronized<T, M>& lhs, Synchronized<T, M>& rhs) {+  lhs.swap(rhs);+}++/**+ * Disambiguate the name var by concatenating the line number of the original+ * point of expansion. This avoids shadowing warnings for nested+ * SYNCHRONIZEDs. The name is consistent if used multiple times within+ * another macro.+ * Only for internal use.+ */+#define SYNCHRONIZED_VAR(var) FB_CONCATENATE(SYNCHRONIZED_##var##_, __LINE__)++namespace detail {+struct [[deprecated(+    "use explicit lock(), wlock(), or rlock() instead")]] SYNCHRONIZED_macro_is_deprecated {+};+} // namespace detail++/**+ * NOTE: This API is deprecated.  Use lock(), wlock(), rlock() or the withLock+ * functions instead.  In the future it will be marked with a deprecation+ * attribute to emit build-time warnings, and then it will be removed entirely.+ *+ * SYNCHRONIZED is the main facility that makes Synchronized<T>+ * helpful. It is a pseudo-statement that introduces a scope where the+ * object is locked. Inside that scope you get to access the unadorned+ * datum.+ *+ * Example:+ *+ * Synchronized<vector<int>> svector;+ * ...+ * SYNCHRONIZED (svector) { ... use svector as a vector<int> ... }+ * or+ * SYNCHRONIZED (v, svector) { ... use v as a vector<int> ... }+ *+ * Refer to folly/docs/Synchronized.md for a detailed explanation and more+ * examples.+ */+#define SYNCHRONIZED(...)                                                 \+  FOLLY_PUSH_WARNING                                                      \+  FOLLY_GNU_DISABLE_WARNING("-Wshadow")                                   \+  FOLLY_MSVC_DISABLE_WARNING(4189) /* initialized but unreferenced */     \+  FOLLY_MSVC_DISABLE_WARNING(4456) /* declaration hides local */          \+  FOLLY_MSVC_DISABLE_WARNING(4457) /* declaration hides parameter */      \+  FOLLY_MSVC_DISABLE_WARNING(4458) /* declaration hides member */         \+  FOLLY_MSVC_DISABLE_WARNING(4459) /* declaration hides global */         \+  FOLLY_GCC_DISABLE_NEW_SHADOW_WARNINGS                                   \+  if (bool SYNCHRONIZED_VAR(state) = false) {                             \+    (void)::folly::detail::SYNCHRONIZED_macro_is_deprecated{};            \+  } else                                                                  \+    for (auto SYNCHRONIZED_VAR(lockedPtr) =                               \+             (FB_VA_GLUE(FB_ARG_2_OR_1, (__VA_ARGS__))).contextualLock(); \+         !SYNCHRONIZED_VAR(state);                                        \+         SYNCHRONIZED_VAR(state) = true)                                  \+      for ([[maybe_unused]] auto& FB_VA_GLUE(FB_ARG_1, (__VA_ARGS__)) =   \+               *SYNCHRONIZED_VAR(lockedPtr).operator->();                 \+           !SYNCHRONIZED_VAR(state);                                      \+           SYNCHRONIZED_VAR(state) = true)                                \+    FOLLY_POP_WARNING++/**+ * NOTE: This API is deprecated.  Use lock(), wlock(), rlock() or the withLock+ * functions instead.  In the future it will be marked with a deprecation+ * attribute to emit build-time warnings, and then it will be removed entirely.+ *+ * Similar to SYNCHRONIZED, but only uses a read lock.+ */+#define SYNCHRONIZED_CONST(...)            \+  SYNCHRONIZED(                            \+      FB_VA_GLUE(FB_ARG_1, (__VA_ARGS__)), \+      std::as_const(FB_VA_GLUE(FB_ARG_2_OR_1, (__VA_ARGS__))))++/**+ * NOTE: This API is deprecated.  Use lock(), wlock(), rlock() or the withLock+ * functions instead.  In the future it will be marked with a deprecation+ * attribute to emit build-time warnings, and then it will be removed entirely.+ *+ * Synchronizes two Synchronized objects (they may encapsulate+ * different data). Synchronization is done in increasing address of+ * object order, so there is no deadlock risk.+ */+#define SYNCHRONIZED_DUAL(n1, e1, n2, e2)                                      \+  if (bool SYNCHRONIZED_VAR(state) = false) {                                  \+    (void)::folly::detail::SYNCHRONIZED_macro_is_deprecated{};                 \+  } else                                                                       \+    for (auto SYNCHRONIZED_VAR(ptrs) = acquireLockedPair(e1, e2);              \+         !SYNCHRONIZED_VAR(state);                                             \+         SYNCHRONIZED_VAR(state) = true)                                       \+      for (auto& n1 = *SYNCHRONIZED_VAR(ptrs).first; !SYNCHRONIZED_VAR(state); \+           SYNCHRONIZED_VAR(state) = true)                                     \+        for (auto& n2 = *SYNCHRONIZED_VAR(ptrs).second;                        \+             !SYNCHRONIZED_VAR(state);                                         \+             SYNCHRONIZED_VAR(state) = true)++} /* namespace folly */
+ folly/folly/SynchronizedPtr.h view
@@ -0,0 +1,106 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <folly/Synchronized.h>++/* `SynchronizedPtr` is a variation on the `Synchronized` idea that's useful for+ * some cases where you want to protect a pointed-to object (or an object within+ * some pointer-like wrapper). If you would otherwise need to use+ * `Synchronized<smart_ptr<Synchronized<T>>>` consider using+ * `SynchronizedPtr<smart_ptr<T>>`as it is a bit easier to use and it works when+ * you want the `T` object at runtime to actually a subclass of `T`.+ *+ * You can access the contained `T` with `.rlock()`, and `.wlock()`, and the+ * pointer or pointer-like wrapper with `.wlockPointer()`. The corresponding+ * `with...` methods take a callback, invoke it with a `T const&`, `T&` or+ * `smart_ptr<T>&` respectively, and return the callback's result.+ */+namespace folly {+template <typename LockHolder, typename Element>+struct SynchronizedPtrLockedElement {+  explicit SynchronizedPtrLockedElement(LockHolder&& holder)+      : holder_(std::move(holder)) {}++  Element& operator*() const { return **holder_; }++  Element* operator->() const { return &**holder_; }++  explicit operator bool() const { return static_cast<bool>(*holder_); }++ private:+  LockHolder holder_;+};++template <typename PointerType, typename MutexType = SharedMutex>+class SynchronizedPtr {+  using inner_type = Synchronized<PointerType, MutexType>;+  inner_type inner_;++ public:+  using pointer_type = PointerType;+  using element_type = typename std::pointer_traits<pointer_type>::element_type;+  using const_element_type = typename std::add_const<element_type>::type;+  using read_locked_element = SynchronizedPtrLockedElement<+      typename inner_type::ConstLockedPtr,+      const_element_type>;+  using write_locked_element = SynchronizedPtrLockedElement<+      typename inner_type::LockedPtr,+      element_type>;+  using write_locked_pointer = typename inner_type::LockedPtr;++  template <typename... Args>+  explicit SynchronizedPtr(Args... args)+      : inner_(std::forward<Args>(args)...) {}++  SynchronizedPtr() = default;+  SynchronizedPtr(SynchronizedPtr const&) = default;+  SynchronizedPtr(SynchronizedPtr&&) = default;+  SynchronizedPtr& operator=(SynchronizedPtr const&) = default;+  SynchronizedPtr& operator=(SynchronizedPtr&&) = default;++  // Methods to provide appropriately locked and const-qualified access to the+  // element.++  read_locked_element rlock() const {+    return read_locked_element(inner_.rlock());+  }++  template <class Function>+  auto withRLock(Function&& function) const {+    return function(*rlock());+  }++  write_locked_element wlock() { return write_locked_element(inner_.wlock()); }++  template <class Function>+  auto withWLock(Function&& function) {+    return function(*wlock());+  }++  // Methods to provide write-locked access to the pointer. We deliberately make+  // it difficult to get a read-locked pointer because that provides read-locked+  // non-const access to the element, and the purpose of this class is to+  // discourage that.+  write_locked_pointer wlockPointer() { return inner_.wlock(); }++  template <class Function>+  auto withWLockPointer(Function&& function) {+    return function(*wlockPointer());+  }+};+} // namespace folly
+ folly/folly/ThreadCachedInt.h view
@@ -0,0 +1,175 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++/**+ * Higher performance (up to 10x) atomic increment using thread caching.+ */++#pragma once++#include <atomic>++#include <folly/Likely.h>+#include <folly/ThreadLocal.h>++namespace folly {++// Note that readFull requires holding a lock and iterating through all of the+// thread local objects with the same Tag, so if you have a lot of+// ThreadCachedInt's you should considering breaking up the Tag space even+// further.+template <class IntT, class Tag = IntT>+class ThreadCachedInt {+  struct IntCache;++ public:+  explicit ThreadCachedInt(IntT initialVal = 0, uint32_t cacheSize = 1000)+      : target_(initialVal), cacheSize_(cacheSize) {}++  ThreadCachedInt(const ThreadCachedInt&) = delete;+  ThreadCachedInt& operator=(const ThreadCachedInt&) = delete;++  void increment(IntT inc) {+    auto cache = cache_.get();+    if (FOLLY_UNLIKELY(cache == nullptr)) {+      cache = new IntCache(*this);+      cache_.reset(cache);+    }+    cache->increment(inc);+  }++  // Quickly grabs the current value which may not include some cached+  // increments.+  IntT readFast() const { return target_.load(std::memory_order_relaxed); }++  // Reads the current value plus all the cached increments.  Requires grabbing+  // a lock, so this is significantly slower than readFast().+  IntT readFull() const {+    // This could race with thread destruction and so the access lock should be+    // acquired before reading the current value+    const auto accessor = cache_.accessAllThreads();+    IntT ret = readFast();+    for (const auto& cache : accessor) {+      if (!cache.reset_.load(std::memory_order_acquire)) {+        ret += cache.val_.load(std::memory_order_relaxed);+      }+    }+    return ret;+  }++  // Quickly reads and resets current value (doesn't reset cached increments).+  IntT readFastAndReset() {+    return target_.exchange(0, std::memory_order_release);+  }++  // This function is designed for accumulating into another counter, where you+  // only want to count each increment once.  It can still get the count a+  // little off, however, but it should be much better than calling readFull()+  // and set(0) sequentially.+  IntT readFullAndReset() {+    // This could race with thread destruction and so the access lock should be+    // acquired before reading the current value+    auto accessor = cache_.accessAllThreads();+    IntT ret = readFastAndReset();+    for (auto& cache : accessor) {+      if (!cache.reset_.load(std::memory_order_acquire)) {+        ret += cache.val_.load(std::memory_order_relaxed);+        cache.reset_.store(true, std::memory_order_release);+      }+    }+    return ret;+  }++  void setCacheSize(uint32_t newSize) {+    cacheSize_.store(newSize, std::memory_order_release);+  }++  uint32_t getCacheSize() const { return cacheSize_.load(); }++  ThreadCachedInt& operator+=(IntT inc) {+    increment(inc);+    return *this;+  }+  ThreadCachedInt& operator-=(IntT inc) {+    increment(-inc);+    return *this;+  }+  // pre-increment (we don't support post-increment)+  ThreadCachedInt& operator++() {+    increment(1);+    return *this;+  }+  ThreadCachedInt& operator--() {+    increment(IntT(-1));+    return *this;+  }++  // Thread-safe set function.+  // This is a best effort implementation. In some edge cases, there could be+  // data loss (missing counts)+  void set(IntT newVal) {+    for (auto& cache : cache_.accessAllThreads()) {+      cache.reset_.store(true, std::memory_order_release);+    }+    target_.store(newVal, std::memory_order_release);+  }++ private:+  std::atomic<IntT> target_;+  std::atomic<uint32_t> cacheSize_;+  ThreadLocalPtr<IntCache, Tag, AccessModeStrict>+      cache_; // Must be last for dtor ordering++  // This should only ever be modified by one thread+  struct IntCache {+    ThreadCachedInt* parent_;+    mutable std::atomic<IntT> val_;+    mutable uint32_t numUpdates_;+    std::atomic<bool> reset_;++    explicit IntCache(ThreadCachedInt& parent)+        : parent_(&parent), val_(0), numUpdates_(0), reset_(false) {}++    void increment(IntT inc) {+      if (FOLLY_LIKELY(!reset_.load(std::memory_order_acquire))) {+        // This thread is the only writer to val_, so it's fine do do+        // a relaxed load and do the addition non-atomically.+        val_.store(+            val_.load(std::memory_order_relaxed) + inc,+            std::memory_order_release);+      } else {+        val_.store(inc, std::memory_order_relaxed);+        reset_.store(false, std::memory_order_release);+      }+      ++numUpdates_;+      if (FOLLY_UNLIKELY(+              numUpdates_ >+              parent_->cacheSize_.load(std::memory_order_acquire))) {+        flush();+      }+    }++    void flush() const {+      parent_->target_.fetch_add(val_, std::memory_order_release);+      val_.store(0, std::memory_order_release);+      numUpdates_ = 0;+    }++    ~IntCache() { flush(); }+  };+};++} // namespace folly
+ folly/folly/ThreadLocal.h view
@@ -0,0 +1,475 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++/**+ * Improved thread local storage for non-trivial types (similar speed as+ * pthread_getspecific but only consumes a single pthread_key_t, and 4x faster+ * than boost::thread_specific_ptr).+ *+ * ThreadLocal objects can be grouped together logically under a tag. Within+ * a tag, each object has a unique id. The combination of tag and id is used to+ * locate the managed object corresponding to the current thread.+ *+ * Also includes an accessor interface to iterate all of the managed+ * objects owned by a ThreadLocal object, each corresponding to a+ * separate thread.  accessAllThreads() initializes an accessor+ * which holds+ * a lock *that blocks all creation and destruction of managed+ * objects managed by the ThreadLocal. The accessor can be used+ * as an iterable container. Note: for now, the accessor also happens to hold+ * other per tag global locks and hence calls to accessAllThreads() are+ * serialized at tag level.+ *+ * accessAllThreads() can race with destruction of thread-local elements. We+ * provide a strict mode which is dangerous because it requires the access lock+ * to be held while destroying thread-local elements which could cause+ * deadlocks. We gate this mode behind the AccessModeStrict template parameter.+ *+ * Intended use is for frequent write, infrequent read data access patterns such+ * as counters.+ *+ * There are two classes here - ThreadLocal and ThreadLocalPtr.  ThreadLocalPtr+ * has semantics similar to boost::thread_specific_ptr. ThreadLocal is a thin+ * wrapper around ThreadLocalPtr that manages allocation automatically.+ */++#pragma once++#include <iterator>+#include <thread>+#include <type_traits>+#include <utility>++#include <folly/Likely.h>+#include <folly/Portability.h>+#include <folly/ScopeGuard.h>+#include <folly/SharedMutex.h>+#include <folly/detail/ThreadLocalDetail.h>++namespace folly {++template <class T, class Tag, class AccessMode>+class ThreadLocalPtr;++template <class T, class Tag = void, class AccessMode = void>+class ThreadLocal {+ public:+  constexpr ThreadLocal() noexcept : constructor_([]() { return T(); }) {}++  template <typename F, std::enable_if_t<is_invocable_r_v<T, F>, int> = 0>+  explicit ThreadLocal(F&& constructor)+      : constructor_(std::forward<F>(constructor)) {}++  ThreadLocal(ThreadLocal&& that) noexcept+      : tlp_{std::move(that.tlp_)},+        constructor_{std::exchange(that.constructor_, {})} {}++  ThreadLocal& operator=(ThreadLocal&& that) noexcept {+    assert(this != &that);+    tlp_ = std::exchange(that.tlp_, {});+    constructor_ = std::exchange(that.constructor_, {});+    return *this;+  }++  FOLLY_ERASE T* get() const {+    auto const ptr = tlp_.get();+    return FOLLY_LIKELY(!!ptr) ? ptr : makeTlp();+  }++  // may return null+  FOLLY_ERASE T* get_existing() const { return tlp_.get(); }++  T* operator->() const { return get(); }++  T& operator*() const { return *get(); }++  void reset(T* newPtr = nullptr) { tlp_.reset(newPtr); }++  typedef typename ThreadLocalPtr<T, Tag, AccessMode>::Accessor Accessor;+  Accessor accessAllThreads() const { return tlp_.accessAllThreads(); }++ private:+  // non-copyable+  ThreadLocal(const ThreadLocal&) = delete;+  ThreadLocal& operator=(const ThreadLocal&) = delete;++  FOLLY_NOINLINE T* makeTlp() const {+    auto const ptr = new T(constructor_());+    tlp_.reset(ptr);+    return ptr;+  }++  mutable ThreadLocalPtr<T, Tag, AccessMode> tlp_;+  std::function<T()> constructor_;+};++/*+ * The idea here is that __thread is faster than pthread_getspecific, so we+ * keep a __thread array of pointers to objects (ThreadEntry::elements) where+ * each array has an index for each unique instance of the ThreadLocalPtr+ * object.  Each ThreadLocalPtr object has a unique id that is an index into+ * these arrays so we can fetch the correct object from thread local storage+ * very efficiently.+ *+ * In order to prevent unbounded growth of the id space and thus huge+ * ThreadEntry::elements, arrays, for example due to continuous creation and+ * destruction of ThreadLocalPtr objects, we keep a set of all active+ * instances.  When an instance is destroyed we remove it from the active+ * set and insert the id into freeIds_ for reuse.  These operations require a+ * global mutex, but only happen at construction and destruction time.+ *+ * We use a single global pthread_key_t per Tag to manage object destruction and+ * memory cleanup upon thread exit because there is a finite number of+ * pthread_key_t's available per machine.+ *+ * NOTE: Apple platforms don't support the same semantics for __thread that+ *       Linux does (and it's only supported at all on i386). For these, use+ *       pthread_setspecific()/pthread_getspecific() for the per-thread+ *       storage.  Windows (MSVC and GCC) does support the same semantics+ *       with __declspec(thread)+ */++template <class T, class Tag = void, class AccessMode = void>+class ThreadLocalPtr {+ private:+  typedef threadlocal_detail::StaticMeta<Tag, AccessMode> StaticMeta;++  using AccessAllThreadsEnabled = Negation<std::is_same<Tag, void>>;++ public:+  constexpr ThreadLocalPtr() noexcept : id_() {}++  ThreadLocalPtr(ThreadLocalPtr&& other) noexcept : id_(std::move(other.id_)) {}++  ThreadLocalPtr& operator=(ThreadLocalPtr&& other) noexcept {+    assert(this != &other);+    destroy(); // user-provided dtors invoked within here must not throw+    id_ = std::move(other.id_);+    return *this;+  }++  ~ThreadLocalPtr() { destroy(); }++  T* get() const {+    threadlocal_detail::ElementWrapper& w = StaticMeta::get(&id_);+    return static_cast<T*>(w.ptr);+  }++  T* operator->() const { return get(); }++  T& operator*() const { return *get(); }++  T* release() {+    auto rlocked = getForkGuard();+    threadlocal_detail::ThreadEntry* te = StaticMeta::getThreadEntry(&id_);+    auto id = id_.getOrInvalid();+    // Only valid index into the elements array+    DCHECK_NE(id, threadlocal_detail::kEntryIDInvalid);+    return static_cast<T*>(te->releaseElement(id));+  }++  void reset(T* newPtr = nullptr) {+    auto rlocked = getForkGuard();+    auto guard = makeGuard([&] { delete newPtr; });+    threadlocal_detail::ThreadEntry* te = StaticMeta::getThreadEntry(&id_);+    uint32_t id = id_.getOrInvalid();+    // Only valid index into the elements array+    DCHECK_NE(id, threadlocal_detail::kEntryIDInvalid);+    te->resetElement(newPtr, id);+    guard.dismiss();+  }++  explicit operator bool() const { return get() != nullptr; }++  /**+   * reset() that transfers ownership from a smart pointer+   */+  template <+      typename SourceT,+      typename Deleter,+      typename = typename std::enable_if<+          std::is_convertible<SourceT*, T*>::value>::type>+  void reset(std::unique_ptr<SourceT, Deleter> source) {+    auto deleter =+        [delegate = source.get_deleter()](T* ptr, TLPDestructionMode) {+          delegate(ptr);+        };+    reset(source.release(), deleter);+  }++  /**+   * reset() that transfers ownership from a smart pointer with the default+   * deleter+   */+  template <+      typename SourceT,+      typename = typename std::enable_if<+          std::is_convertible<SourceT*, T*>::value>::type>+  void reset(std::unique_ptr<SourceT> source) {+    reset(source.release());+  }++  /**+   * reset() with a custom deleter:+   * deleter(T* ptr, TLPDestructionMode mode)+   * "mode" is ALL_THREADS if we're destructing this ThreadLocalPtr (and thus+   * deleting pointers for all threads), and THIS_THREAD if we're only deleting+   * the member for one thread (because of thread exit or reset()).+   * Invoking the deleter must not throw.+   */+  template <class Deleter>+  void reset(T* newPtr, const Deleter& deleter) {+    auto guard = makeGuard([&] {+      if (newPtr) {+        deleter(newPtr, TLPDestructionMode::THIS_THREAD);+      }+    });++    auto rlocked = getForkGuard();+    threadlocal_detail::ThreadEntry* te = StaticMeta::getThreadEntry(&id_);+    uint32_t id = id_.getOrInvalid();+    // Only valid index into the elements array+    DCHECK_NE(id, threadlocal_detail::kEntryIDInvalid);+    te->resetElement(newPtr, deleter, id);+    guard.dismiss();+  }++  void reset(const std::shared_ptr<T>& newPtr) {+    reset(newPtr.get(), threadlocal_detail::SharedPtrDeleter{newPtr});+  }++  // Holds a global lock for iteration through all thread local child objects.+  // Can be used as an iterable container.+  // Use accessAllThreads() to obtain one.+  class Accessor {+    friend class ThreadLocalPtr<T, Tag, AccessMode>;++    threadlocal_detail::StaticMetaBase& meta_ =+        threadlocal_detail::StaticMeta<Tag, AccessMode>::instance();+    std::unique_lock<SharedMutex> accessAllThreadsLock_;+    std::shared_lock<SharedMutex> forkHandlerLock_;+    uint32_t id_ = 0;++    // Prevent the entry set from changing while we are iterating over it.+    // reset() calls to populate will acquire shared lock on the id's set.+    threadlocal_detail::StaticMetaBase::SynchronizedThreadEntrySet::WLockedPtr+        wlockedThreadEntrySet_;++   public:+    class Iterator;+    friend class Iterator;++    // The iterators obtained from Accessor are bidirectional iterators.+    class Iterator {+      friend class Accessor;+      const Accessor* accessor_{nullptr};+      using InnerVector = threadlocal_detail::ThreadEntrySet::ElementVector;+      using InnerIterator = InnerVector::iterator;++      InnerVector& vec_;+      InnerIterator iter_;++      void increment() {+        if (iter_ != vec_.end()) {+          ++iter_;+          incrementToValid();+        }+      }++      void decrement() {+        if (iter_ != vec_.begin()) {+          --iter_;+          decrementToValid();+        }+      }++      const T& dereference() const {+        return *static_cast<T*>(iter_->wrapper.ptr);+      }++      T& dereference() { return *static_cast<T*>(iter_->wrapper.ptr); }++      bool equal(const Iterator& other) const {+        return (accessor_->id_ == other.accessor_->id_ && iter_ == other.iter_);+      }++      void setToEnd() { iter_ = vec_.end(); }++      explicit Iterator(const Accessor* accessor, bool toEnd = false)+          : accessor_(accessor),+            vec_(accessor_->wlockedThreadEntrySet_->threadElements),+            iter_(vec_.begin()) {+        if (toEnd) {+          setToEnd();+        } else {+          incrementToValid();+        }+      }++      // we just need to check the ptr since it can be set to nullptr+      // even if the entry is part of the list+      bool valid() const { return (iter_ != vec_.end() && iter_->wrapper.ptr); }++      void incrementToValid() {+        for (; iter_ != vec_.end() && !valid(); ++iter_) {+        }+      }++      void decrementToValid() {+        for (; iter_ != vec_.begin() && !valid(); --iter_) {+        }+      }++     public:+      using difference_type = ssize_t;+      using value_type = T;+      using reference = T const&;+      using pointer = T const*;+      using iterator_category = std::bidirectional_iterator_tag;++      Iterator() = default;++      Iterator& operator++() {+        increment();+        return *this;+      }++      Iterator& operator++(int) {+        Iterator copy(*this);+        increment();+        return copy;+      }++      Iterator& operator--() {+        decrement();+        return *this;+      }++      Iterator& operator--(int) {+        Iterator copy(*this);+        decrement();+        return copy;+      }++      T& operator*() { return dereference(); }++      T const& operator*() const { return dereference(); }++      T* operator->() { return &dereference(); }++      T const* operator->() const { return &dereference(); }++      bool operator==(Iterator const& rhs) const { return equal(rhs); }++      bool operator!=(Iterator const& rhs) const { return !equal(rhs); }++      std::thread::id getThreadId() const { return iter_->threadEntry->tid(); }++      uint64_t getOSThreadId() const { return iter_->threadEntry->tid_os; }+    };++    ~Accessor() { release(); }++    Iterator begin() const { return Iterator(this); }++    Iterator end() const { return Iterator(this, true); }++    Accessor(const Accessor&) = delete;+    Accessor& operator=(const Accessor&) = delete;++    Accessor(Accessor&& other) noexcept+        : meta_(other.meta_),+          accessAllThreadsLock_(std::move(other.accessAllThreadsLock_)),+          forkHandlerLock_(std::move(other.forkHandlerLock_)),+          id_(std::exchange(other.id_, 0)) {+      wlockedThreadEntrySet_ = std::move(other.wlockedThreadEntrySet_);+    }++    Accessor& operator=(Accessor&& other) noexcept {+      // Each Tag has its own unique meta, and accessors with different Tags+      // have different types.  So either *this is empty, or this and other+      // have the same tag.  But if they have the same tag, they have the same+      // meta (and lock), so they'd both hold the lock at the same time,+      // which is impossible, which leaves only one possible scenario --+      // *this is empty.  Assert it.+      assert(&meta_ == &other.meta_);+      using std::swap;+      swap(accessAllThreadsLock_, other.accessAllThreadsLock_);+      swap(forkHandlerLock_, other.forkHandlerLock_);+      swap(id_, other.id_);+      wlockedThreadEntrySet_.unlock();+      swap(wlockedThreadEntrySet_, other.wlockedThreadEntrySet_);+    }++    Accessor() = default;++   private:+    explicit Accessor(uint32_t id)+        : accessAllThreadsLock_(meta_.accessAllThreadsLock_, std::defer_lock),+          forkHandlerLock_(meta_.forkHandlerLock_, std::defer_lock),+          id_(id) {+      forkHandlerLock_.lock();+      accessAllThreadsLock_.lock();+      wlockedThreadEntrySet_ = meta_.allId2ThreadEntrySets_[id_].wlock();+    }++    void release() {+      if (accessAllThreadsLock_) {+        wlockedThreadEntrySet_.unlock();+        accessAllThreadsLock_.unlock();+        DCHECK(forkHandlerLock_);+        forkHandlerLock_.unlock();+        id_ = 0;+      }+    }+  };++  // accessor allows a client to iterate through all thread local child+  // elements of this ThreadLocal instance.  Holds a global lock for each <Tag>+  Accessor accessAllThreads() const {+    static_assert(+        AccessAllThreadsEnabled::value,+        "Must use a unique Tag to use the accessAllThreads feature");+    return Accessor(id_.getOrAllocate(StaticMeta::instance()));+  }++ private:+  void destroy() noexcept {+    auto const val = id_.value.load(std::memory_order_relaxed);+    if (val == threadlocal_detail::kEntryIDInvalid) {+      return;+    }+    StaticMeta::instance().destroy(&id_);+    // User provided destructors should not cause the TL to have its id+    // reallocated.+    DCHECK(+        id_.value.load(std::memory_order_relaxed) ==+        threadlocal_detail::kEntryIDInvalid);+  }++  // non-copyable+  ThreadLocalPtr(const ThreadLocalPtr&) = delete;+  ThreadLocalPtr& operator=(const ThreadLocalPtr&) = delete;++  static auto getForkGuard() {+    auto& mutex = StaticMeta::instance().forkHandlerLock_;+    return std::shared_lock{mutex};+  }++  mutable typename StaticMeta::EntryID id_;+};++} // namespace folly
+ folly/folly/TimeoutQueue.cpp view
@@ -0,0 +1,79 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#include <folly/TimeoutQueue.h>++#include <algorithm>+#include <limits>+#include <vector>++namespace folly {++TimeoutQueue::Id TimeoutQueue::add(+    int64_t now, int64_t delay, Callback callback) {+  Id id = nextId_++;+  timeouts_.insert({id, now + delay, -1, std::move(callback)});+  return id;+}++TimeoutQueue::Id TimeoutQueue::addRepeating(+    int64_t now, int64_t interval, Callback callback) {+  Id id = nextId_++;+  timeouts_.insert({id, now + interval, interval, std::move(callback)});+  return id;+}++int64_t TimeoutQueue::nextExpiration() const {+  return (+      timeouts_.empty()+          ? std::numeric_limits<int64_t>::max()+          : timeouts_.get<BY_EXPIRATION>().begin()->expiration);+}++bool TimeoutQueue::erase(Id id) {+  return timeouts_.get<BY_ID>().erase(id);+}++int64_t TimeoutQueue::runInternal(int64_t now, bool onceOnly) {+  auto& byExpiration = timeouts_.get<BY_EXPIRATION>();+  int64_t nextExp;+  do {+    const auto end = byExpiration.upper_bound(now);+    std::vector<Event> expired;+    std::move(byExpiration.begin(), end, std::back_inserter(expired));+    byExpiration.erase(byExpiration.begin(), end);+    for (const auto& event : expired) {+      // Reinsert if repeating, do this before executing callbacks+      // so the callbacks have a chance to call erase+      if (event.repeatInterval >= 0) {+        timeouts_.insert(+            {event.id,+             now + event.repeatInterval,+             event.repeatInterval,+             event.callback});+      }+    }++    // Call callbacks+    for (const auto& event : expired) {+      event.callback(event.id, now);+    }+    nextExp = nextExpiration();+  } while (!onceOnly && nextExp <= now);+  return nextExp;+}++} // namespace folly
+ folly/folly/TimeoutQueue.h view
@@ -0,0 +1,123 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++/**+ * Simple timeout queue.  Call user-specified callbacks when their timeouts+ * expire.+ *+ * This class assumes that "time" is an int64_t and doesn't care about time+ * units (seconds, milliseconds, etc).  You call runOnce() / runLoop() using+ * the same time units that you use to specify callbacks.+ */++#pragma once++#include <cstdint>+#include <functional>++#include <boost/multi_index/indexed_by.hpp>+#include <boost/multi_index/member.hpp>+#include <boost/multi_index/ordered_index.hpp>+#include <boost/multi_index_container.hpp>++namespace folly {++class TimeoutQueue {+ public:+  typedef int64_t Id;+  typedef std::function<void(Id, int64_t)> Callback;++  TimeoutQueue() : nextId_(1) {}++  /**+   * Add a one-time timeout event that will fire "delay" time units from "now"+   * (that is, the first time that run*() is called with a time value >= now+   * + delay).+   */+  Id add(int64_t now, int64_t delay, Callback callback);++  /**+   * Add a repeating timeout event that will fire every "interval" time units+   * (it will first fire when run*() is called with a time value >=+   * now + interval).+   *+   * run*() will always invoke each repeating event at most once, even if+   * more than one "interval" period has passed.+   */+  Id addRepeating(int64_t now, int64_t interval, Callback callback);++  /**+   * Erase a given timeout event, returns true if the event was actually+   * erased and false if it didn't exist in our queue.+   */+  bool erase(Id id);++  /**+   * Process all events that are due at times <= "now" by calling their+   * callbacks.+   *+   * Callbacks are allowed to call back into the queue and add / erase events;+   * they might create more events that are already due.  In this case,+   * runOnce() will only go through the queue once, and return a "next+   * expiration" time in the past or present (<= now); runLoop()+   * will process the queue again, until there are no events already due.+   *+   * Note that it is then possible for runLoop to never return if+   * callbacks re-add themselves to the queue (or if you have repeating+   * callbacks with an interval of 0).+   *+   * Return the time that the next event will be due (same as+   * nextExpiration(), below)+   */+  int64_t runOnce(int64_t now) { return runInternal(now, true); }+  int64_t runLoop(int64_t now) { return runInternal(now, false); }++  /**+   * Return the time that the next event will be due.+   */+  int64_t nextExpiration() const;++ private:+  int64_t runInternal(int64_t now, bool onceOnly);+  TimeoutQueue(const TimeoutQueue&) = delete;+  TimeoutQueue& operator=(const TimeoutQueue&) = delete;++  struct Event {+    Id id;+    int64_t expiration;+    int64_t repeatInterval;+    Callback callback;+  };++  typedef boost::multi_index_container<+      Event,+      boost::multi_index::indexed_by<+          boost::multi_index::ordered_unique<+              boost::multi_index::member<Event, Id, &Event::id>>,+          boost::multi_index::ordered_non_unique<+              boost::multi_index::member<Event, int64_t, &Event::expiration>>>>+      Set;++  enum {+    BY_ID = 0,+    BY_EXPIRATION = 1,+  };++  Set timeouts_;+  Id nextId_;+};++} // namespace folly
+ folly/folly/TokenBucket.h view
@@ -0,0 +1,673 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <algorithm>+#include <atomic>+#include <chrono>+#include <thread>++#include <folly/ConstexprMath.h>+#include <folly/Likely.h>+#include <folly/Optional.h>+#include <folly/concurrency/CacheLocality.h>++namespace folly {++struct TokenBucketPolicyDefault {+  using align =+      std::integral_constant<size_t, hardware_destructive_interference_size>;++  template <typename T>+  using atom = std::atomic<T>;++  using clock = std::chrono::steady_clock;++  using concurrent = std::true_type;+};++/**+ * Thread-safe (atomic) token bucket primitive.+ *+ * This primitive can be used to implement a token bucket+ * (http://en.wikipedia.org/wiki/Token_bucket).  It handles+ * the storage of the state in an atomic way, and presents+ * an interface dealing with tokens, rate, burstSize and time.+ *+ * This primitive records the last time it was updated. This allows the+ * token bucket to add tokens "just in time" when tokens are requested.+ *+ * @tparam Policy A policy.+ */+template <typename Policy = TokenBucketPolicyDefault>+class TokenBucketStorage {+  template <typename T>+  using Atom = typename Policy::template atom<T>;+  using Align = typename Policy::align;+  using Clock = typename Policy::clock; // do we need clock here?+  using Concurrent = typename Policy::concurrent;++  static_assert(Clock::is_steady, "clock must be steady"); // do we need clock?++ public:+  /**+   * Constructor.+   *+   * @param zeroTime Initial time at which to consider the token bucket+   *                 starting to fill. Defaults to 0, so by default token+   *                 buckets are "empty" after construction.+   */+  explicit TokenBucketStorage(double zeroTime = 0) noexcept+      : zeroTime_(zeroTime) {}++  /**+   * Copy constructor.+   *+   * Thread-safe. (Copy constructors of derived classes may not be thread-safe+   * however.)+   */+  TokenBucketStorage(const TokenBucketStorage& other) noexcept+      : zeroTime_(other.zeroTime_.load(std::memory_order_relaxed)) {}++  /**+   * Copy-assignment operator.+   *+   * Warning: not thread safe for the object being assigned to (including+   * self-assignment). Thread-safe for the other object.+   */+  TokenBucketStorage& operator=(const TokenBucketStorage& other) noexcept {+    zeroTime_.store(other.zeroTime(), std::memory_order_relaxed);+    return *this;+  }++  /**+   * Re-initialize token bucket.+   *+   * Thread-safe.+   *+   * @param zeroTime Initial time at which to consider the token bucket+   *                 starting to fill. Defaults to 0, so by default token+   *                 bucket is reset to "empty".+   */+  void reset(double zeroTime = 0) noexcept {+    zeroTime_.store(zeroTime, std::memory_order_relaxed);+  }++  /**+   * Returns the token balance at specified time (negative if bucket in debt).+   *+   * Thread-safe (but returned value may immediately be outdated).+   */+  double balance(+      double rate, double burstSize, double nowInSeconds) const noexcept {+    assert(rate > 0);+    assert(burstSize > 0);+    double zt = this->zeroTime_.load(std::memory_order_relaxed);+    return std::min((nowInSeconds - zt) * rate, burstSize);+  }++  /**+   * Consume tokens at the given rate/burst/time.+   *+   * Consumption is actually done by the callback function: it's given a+   * reference with the number of available tokens and returns the number+   * consumed.  Typically the return value would be between 0.0 and available,+   * but there are no restrictions.+   *+   * Note: the callback may be called multiple times, so please no side-effects+   */+  template <typename Callback>+  double consume(+      double rate,+      double burstSize,+      double nowInSeconds,+      const Callback& callback) {+    assert(rate > 0);+    assert(burstSize > 0);++    double zeroTimeOld;+    double zeroTimeNew;+    double consumed;+    do {+      zeroTimeOld = zeroTime();+      double tokens = std::min((nowInSeconds - zeroTimeOld) * rate, burstSize);+      consumed = callback(tokens);+      double tokensNew = tokens - consumed;+      if (consumed == 0.0) {+        return consumed;+      }++      zeroTimeNew = nowInSeconds - tokensNew / rate;+    } while (FOLLY_UNLIKELY(+        !compare_exchange_weak_relaxed(zeroTime_, zeroTimeOld, zeroTimeNew)));++    return consumed;+  }++  /**+   * returns the time at which the bucket will have `target` tokens available.+   *+   * Caution: it doesn't make sense to ask about target > burstSize+   *+   * Eg.+   *     // time debt repaid+   *     bucket.timeWhenBucket(rate, 0);+   *+   *     // time bucket is full+   *     bucket.timeWhenBucket(rate, burstSize);+   */++  double timeWhenBucket(double rate, double target) {+    return zeroTime() + target / rate;+  }++  /**+   * Return extra tokens back to the bucket.+   *+   * Thread-safe.+   */+  void returnTokens(double tokensToReturn, double rate) {+    assert(rate > 0);++    returnTokensImpl(tokensToReturn, rate);+  }++ private:+  /**+   * Adjust zeroTime based on rate and tokenCount and return the new value of+   * zeroTime_. Note: Token count can be negative to move the zeroTime_+   * into the future.+   */+  double returnTokensImpl(double tokenCount, double rate) {+    auto zeroTimeOld = zeroTime_.load(std::memory_order_relaxed);++    double zeroTimeNew;+    do {+      zeroTimeNew = zeroTimeOld - tokenCount / rate;++    } while (FOLLY_UNLIKELY(+        !compare_exchange_weak_relaxed(zeroTime_, zeroTimeOld, zeroTimeNew)));+    return zeroTimeNew;+  }++  static bool compare_exchange_weak_relaxed(++      Atom<double>& atom, double& expected, double zeroTime) {+    if (Concurrent::value) {+      return atom.compare_exchange_weak(+          expected, zeroTime, std::memory_order_relaxed);+    } else {+      return atom.store(zeroTime, std::memory_order_relaxed), true;+    }+  }++  double zeroTime() const {+    return this->zeroTime_.load(std::memory_order_relaxed);+  }++  static constexpr size_t AlignZeroTime =+      constexpr_max(Align::value, alignof(Atom<double>));+  alignas(AlignZeroTime) Atom<double> zeroTime_;+};++/**+ * Thread-safe (atomic) token bucket implementation.+ *+ * A token bucket (http://en.wikipedia.org/wiki/Token_bucket) models a stream+ * of events with an average rate and some amount of burstiness. The canonical+ * example is a packet switched network: the network can accept some number of+ * bytes per second and the bytes come in finite packets (bursts). A token+ * bucket stores up to a fixed number of tokens (the burst size). Some number+ * of tokens are removed when an event occurs. The tokens are replenished at a+ * fixed rate. Failure to allocate tokens implies resource is unavailable and+ * caller needs to implement its own retry mechanism. For simple cases where+ * caller is okay with a FIFO starvation-free scheduling behavior, there are+ * also APIs to 'borrow' from the future effectively assigning a start time to+ * the caller when it should proceed with using the resource. It is also+ * possible to 'return' previously allocated tokens to make them available to+ * other users. Returns in excess of burstSize are considered expired and+ * will not be available to later callers.+ *+ * This implementation records the last time it was updated. This allows the+ * token bucket to add tokens "just in time" when tokens are requested.+ *+ * The "dynamic" base variant allows the token generation rate and maximum+ * burst size to change with every token consumption.+ *+ * @tparam Policy A policy.+ */+template <typename Policy = TokenBucketPolicyDefault>+class BasicDynamicTokenBucket {+  template <typename T>+  using Atom = typename Policy::template atom<T>;+  using Align = typename Policy::align;+  using Clock = typename Policy::clock;+  using Concurrent = typename Policy::concurrent;++  static_assert(Clock::is_steady, "clock must be steady");++ public:+  /**+   * Constructor.+   *+   * @param zeroTime Initial time at which to consider the token bucket+   *                 starting to fill. Defaults to 0, so by default token+   *                 buckets are "empty" after construction.+   */+  explicit BasicDynamicTokenBucket(double zeroTime = 0) noexcept+      : bucket_(zeroTime) {}++  /**+   * Copy constructor and copy assignment operator.+   *+   * Thread-safe. (Copy constructors of derived classes may not be thread-safe+   * however.)+   */+  BasicDynamicTokenBucket(const BasicDynamicTokenBucket& other) noexcept =+      default;+  BasicDynamicTokenBucket& operator=(+      const BasicDynamicTokenBucket& other) noexcept = default;++  /**+   * Re-initialize token bucket.+   *+   * Thread-safe.+   *+   * @param zeroTime Initial time at which to consider the token bucket+   *                 starting to fill. Defaults to 0, so by default token+   *                 bucket is reset to "empty".+   */+  void reset(double zeroTime = 0) noexcept { bucket_.reset(zeroTime); }++  /**+   * Returns the current time in seconds since Epoch.+   */+  static double defaultClockNow() noexcept {+    auto const now = Clock::now().time_since_epoch();+    return std::chrono::duration<double>(now).count();+  }++  /**+   * Attempts to consume some number of tokens. Tokens are first added to the+   * bucket based on the time elapsed since the last attempt to consume tokens.+   * Note: Attempts to consume more tokens than the burst size will always+   * fail.+   *+   * Thread-safe.+   *+   * @param toConsume The number of tokens to consume.+   * @param rate Number of tokens to generate per second.+   * @param burstSize Maximum burst size. Must be greater than 0.+   * @param nowInSeconds Current time in seconds. Should be monotonically+   *                     increasing from the nowInSeconds specified in+   *                     this token bucket's constructor.+   * @return True if the rate limit check passed, false otherwise.+   */+  bool consume(+      double toConsume,+      double rate,+      double burstSize,+      double nowInSeconds = defaultClockNow()) {+    assert(rate > 0);+    assert(burstSize > 0);++    if (bucket_.balance(rate, burstSize, nowInSeconds) < 0.0) {+      return 0;+    }++    double consumed = bucket_.consume(+        rate, burstSize, nowInSeconds, [toConsume](double available) {+          return available < toConsume ? 0.0 : toConsume;+        });++    assert(consumed == toConsume || consumed == 0.0);+    return consumed == toConsume;+  }++  /**+   * Similar to consume, but always consumes some number of tokens.  If the+   * bucket contains enough tokens - consumes toConsume tokens.  Otherwise the+   * bucket is drained.+   *+   * Thread-safe.+   *+   * @param toConsume The number of tokens to consume.+   * @param rate Number of tokens to generate per second.+   * @param burstSize Maximum burst size. Must be greater than 0.+   * @param nowInSeconds Current time in seconds. Should be monotonically+   *                     increasing from the nowInSeconds specified in+   *                     this token bucket's constructor.+   * @return number of tokens that were consumed.+   */+  double consumeOrDrain(+      double toConsume,+      double rate,+      double burstSize,+      double nowInSeconds = defaultClockNow()) {+    assert(rate > 0);+    assert(burstSize > 0);++    if (bucket_.balance(rate, burstSize, nowInSeconds) <= 0.0) {+      return 0;+    }++    double consumed = bucket_.consume(+        rate, burstSize, nowInSeconds, [toConsume](double available) {+          return constexpr_min(available, toConsume);+        });+    return consumed;+  }++  /**+   * Return extra tokens back to the bucket.+   *+   * Thread-safe.+   */+  void returnTokens(double tokensToReturn, double rate) {+    assert(rate > 0);+    assert(tokensToReturn > 0);++    bucket_.returnTokens(tokensToReturn, rate);+  }++  /**+   * Like consumeOrDrain but the call will always satisfy the asked for count.+   * It does so by borrowing tokens from the future if the currently available+   * count isn't sufficient.+   *+   * Returns a folly::Optional<double>. The optional wont be set if the request+   * cannot be satisfied: only case is when it is larger than burstSize. The+   * value of the optional is a double indicating the time in seconds that the+   * caller needs to wait at which the reservation becomes valid. The caller+   * could simply sleep for the returned duration to smooth out the allocation+   * to match the rate limiter or do some other computation in the meantime. In+   * any case, any regular consume or consumeOrDrain calls will fail to allocate+   * any tokens until the future time is reached.+   *+   * Note: It is assumed the caller will not ask for a very large count nor use+   * it immediately (if not waiting inline) as that would break the burst+   * prevention the limiter is meant to be used for.+   *+   * Thread-safe.+   */+  Optional<double> consumeWithBorrowNonBlocking(+      double toConsume,+      double rate,+      double burstSize,+      double nowInSeconds = defaultClockNow()) {+    assert(rate > 0);+    assert(burstSize > 0);++    if (burstSize < toConsume) {+      return folly::none;+    }++    while (toConsume > 0) {+      double consumed =+          consumeOrDrain(toConsume, rate, burstSize, nowInSeconds);+      if (consumed > 0) {+        toConsume -= consumed;+      } else {+        bucket_.returnTokens(-toConsume, rate);+        double debtPaid = bucket_.timeWhenBucket(rate, 0);+        double napTime = std::max(0.0, debtPaid - nowInSeconds);+        return napTime;+      }+    }+    return 0;+  }++  /**+   * Convenience wrapper around non-blocking borrow to sleep inline until+   * reservation is valid.+   */+  bool consumeWithBorrowAndWait(+      double toConsume,+      double rate,+      double burstSize,+      double nowInSeconds = defaultClockNow()) {+    auto res =+        consumeWithBorrowNonBlocking(toConsume, rate, burstSize, nowInSeconds);+    if (res.value_or(0) > 0) {+      const auto napUSec = static_cast<int64_t>(res.value() * 1000000);+      std::this_thread::sleep_for(std::chrono::microseconds(napUSec));+    }+    return res.has_value();+  }++  /**+   * Returns the tokens available at specified time (zero if in debt).+   *+   * Use balance() to get the balance of tokens.+   *+   * Thread-safe (but returned value may immediately be outdated).+   */+  double available(+      double rate,+      double burstSize,+      double nowInSeconds = defaultClockNow()) const noexcept {+    return std::max(0.0, balance(rate, burstSize, nowInSeconds));+  }++  /**+   * Returns the token balance at specified time (negative if bucket in debt).+   *+   * Thread-safe (but returned value may immediately be outdated).+   */+  double balance(+      double rate,+      double burstSize,+      double nowInSeconds = defaultClockNow()) const noexcept {+    return bucket_.balance(rate, burstSize, nowInSeconds);+  }++ private:+  TokenBucketStorage<Policy> bucket_;+};++/**+ * Specialization of BasicDynamicTokenBucket with a fixed token+ * generation rate and a fixed maximum burst size.+ */+template <typename Policy = TokenBucketPolicyDefault>+class BasicTokenBucket {+ private:+  using Impl = BasicDynamicTokenBucket<Policy>;++ public:+  /**+   * Construct a token bucket with a specific maximum rate and burst size.+   *+   * @param genRate Number of tokens to generate per second.+   * @param burstSize Maximum burst size. Must be greater than 0.+   * @param zeroTime Initial time at which to consider the token bucket+   *                 starting to fill. Defaults to 0, so by default token+   *                 bucket is "empty" after construction.+   */+  BasicTokenBucket(+      double genRate, double burstSize, double zeroTime = 0) noexcept+      : tokenBucket_(zeroTime), rate_(genRate), burstSize_(burstSize) {+    assert(rate_ > 0);+    assert(burstSize_ > 0);+  }++  /**+   * Copy constructor.+   *+   * Warning: not thread safe!+   */+  BasicTokenBucket(const BasicTokenBucket& other) noexcept = default;++  /**+   * Copy-assignment operator.+   *+   * Warning: not thread safe!+   */+  BasicTokenBucket& operator=(const BasicTokenBucket& other) noexcept = default;++  /**+   * Returns the current time in seconds since Epoch.+   */+  static double defaultClockNow() noexcept(noexcept(Impl::defaultClockNow())) {+    return Impl::defaultClockNow();+  }++  /**+   * Change rate and burst size.+   *+   * Warning: not thread safe!+   *+   * @param genRate Number of tokens to generate per second.+   * @param burstSize Maximum burst size. Must be greater than 0.+   * @param nowInSeconds Current time in seconds. Should be monotonically+   *                     increasing from the nowInSeconds specified in+   *                     this token bucket's constructor.+   */+  void reset(+      double genRate,+      double burstSize,+      double nowInSeconds = defaultClockNow()) noexcept {+    assert(genRate > 0);+    assert(burstSize > 0);+    const double availTokens = available(nowInSeconds);+    rate_ = genRate;+    burstSize_ = burstSize;+    setCapacity(availTokens, nowInSeconds);+  }++  /**+   * Change number of tokens in bucket.+   *+   * Warning: not thread safe!+   *+   * @param tokens Desired number of tokens in bucket after the call.+   * @param nowInSeconds Current time in seconds. Should be monotonically+   *                     increasing from the nowInSeconds specified in+   *                     this token bucket's constructor.+   */+  void setCapacity(double tokens, double nowInSeconds) noexcept {+    tokenBucket_.reset(nowInSeconds - tokens / rate_);+  }++  /**+   * Attempts to consume some number of tokens. Tokens are first added to the+   * bucket based on the time elapsed since the last attempt to consume tokens.+   * Note: Attempts to consume more tokens than the burst size will always+   * fail.+   *+   * Thread-safe.+   *+   * @param toConsume The number of tokens to consume.+   * @param nowInSeconds Current time in seconds. Should be monotonically+   *                     increasing from the nowInSeconds specified in+   *                     this token bucket's constructor.+   * @return True if the rate limit check passed, false otherwise.+   */+  bool consume(double toConsume, double nowInSeconds = defaultClockNow()) {+    return tokenBucket_.consume(toConsume, rate_, burstSize_, nowInSeconds);+  }++  /**+   * Similar to consume, but always consumes some number of tokens.  If the+   * bucket contains enough tokens - consumes toConsume tokens.  Otherwise the+   * bucket is drained.+   *+   * Thread-safe.+   *+   * @param toConsume The number of tokens to consume.+   * @param nowInSeconds Current time in seconds. Should be monotonically+   *                     increasing from the nowInSeconds specified in+   *                     this token bucket's constructor.+   * @return number of tokens that were consumed.+   */+  double consumeOrDrain(+      double toConsume, double nowInSeconds = defaultClockNow()) {+    return tokenBucket_.consumeOrDrain(+        toConsume, rate_, burstSize_, nowInSeconds);+  }++  /**+   * Returns extra token back to the bucket.  Cannot be negative.+   * For negative tokens, setCapacity() can be used+   */+  void returnTokens(double tokensToReturn) {+    return tokenBucket_.returnTokens(tokensToReturn, rate_);+  }++  /**+   * Reserve tokens and return time to wait for in order for the reservation to+   * be compatible with the bucket configuration.+   */+  Optional<double> consumeWithBorrowNonBlocking(+      double toConsume, double nowInSeconds = defaultClockNow()) {+    return tokenBucket_.consumeWithBorrowNonBlocking(+        toConsume, rate_, burstSize_, nowInSeconds);+  }++  /**+   * Reserve tokens. Blocks if need be until reservation is satisfied.+   */+  bool consumeWithBorrowAndWait(+      double toConsume, double nowInSeconds = defaultClockNow()) {+    return tokenBucket_.consumeWithBorrowAndWait(+        toConsume, rate_, burstSize_, nowInSeconds);+  }++  /**+   * Returns the tokens available at specified time (zero if in debt).+   *+   * Use balance() to get the balance of tokens.+   *+   * Thread-safe (but returned value may immediately be outdated).+   */+  double available(double nowInSeconds = defaultClockNow()) const noexcept {+    return std::max(0.0, balance(nowInSeconds));+  }++  /**+   * Returns the token balance at specified time (negative if bucket in debt).+   *+   * Thread-safe (but returned value may immediately be outdated).+   */+  double balance(double nowInSeconds = defaultClockNow()) const noexcept {+    return tokenBucket_.balance(rate_, burstSize_, nowInSeconds);+  }++  /**+   * Returns the number of tokens generated per second.+   *+   * Thread-safe (but returned value may immediately be outdated).+   */+  double rate() const noexcept { return rate_; }++  /**+   * Returns the maximum burst size.+   *+   * Thread-safe (but returned value may immediately be outdated).+   */+  double burst() const noexcept { return burstSize_; }++ private:+  Impl tokenBucket_;+  double rate_;+  double burstSize_;+};++using TokenBucket = BasicTokenBucket<>;+using DynamicTokenBucket = BasicDynamicTokenBucket<>;++} // namespace folly
+ folly/folly/Traits.h view
@@ -0,0 +1,1497 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <cstdint>+#include <functional>+#include <limits>+#include <memory>+#include <tuple>+#include <type_traits>++#include <folly/Portability.h>++namespace folly {++#if defined(__cpp_lib_type_identity) && __cpp_lib_type_identity >= 201806L++using std::type_identity;+using std::type_identity_t;++#else++/// type_identity_t+/// type_identity+///+/// mimic: std::type_identity_t, std::type_identity, c++20+template <typename T>+struct type_identity {+  using type = T;+};+template <typename T>+using type_identity_t = typename type_identity<T>::type;++#endif++/// tag_t+/// tag+///+/// A generic type-list value type and value.+///+/// A type-list is a class template parameterized by a pack of types.+template <typename...>+struct tag_t {};+template <typename... T>+inline constexpr tag_t<T...> tag{};++/// vtag_t+/// vtag+///+/// A generic value-list value type and value.+///+/// A value-list is a class template parameterized by a pack of values.+template <auto...>+struct vtag_t {};+template <auto... V>+inline constexpr vtag_t<V...> vtag{};++template <std::size_t I>+using index_constant = std::integral_constant<std::size_t, I>;++/// always_false+///+/// A variable template that is always false but requires template arguments to+/// be provided (which are then ignored). This is useful in very specific cases+/// where we want type-dependent expressions to defer static_assert's.+///+/// A common use-case is for exhaustive constexpr if branches:+///+///   template <typename T>+///   void foo(T value) {+///     if constexpr (std::is_integral_v<T>) foo_integral(value);+///     else if constexpr (std::is_same_v<T, std::string>) foo_string(value);+///     else static_assert(always_false<T>, "Unsupported type");+///   }+///+/// If we had used static_assert(false), then this would always fail to compile,+/// even if foo is never instantiated!+///+/// Another use case is if a template that is expected to always be specialized+/// is erroneously instantiated with the base template.+///+///   template <typename T>+///   struct Foo {+///     static_assert(always_false<T>, "Unsupported type");+///   };+///   template <>+///   struct Foo<int> {};+///+///   Foo<int> a;         // fine+///   Foo<std::string> b; // fails! And you get a nice (custom) error message+///+/// This is similar to leaving the base template undefined but we get a nicer+/// compiler error message with static_assert.+template <typename...>+inline constexpr bool always_false = false;++namespace detail {++template <typename Void, typename T>+struct require_sizeof_ {+  static_assert(always_false<T>, "application of sizeof fails substitution");+};+template <typename T>+struct require_sizeof_<decltype(void(sizeof(T))), T> {+  template <typename V>+  using apply_t = V;++  static constexpr std::size_t size = sizeof(T);+};++} // namespace detail++/// require_sizeof+///+/// Equivalent to sizeof, but with a static_assert enforcing that application of+/// sizeof would not fail substitution.+template <typename T>+constexpr std::size_t require_sizeof = detail::require_sizeof_<void, T>::size;++/// is_complete+/// is_complete_v+///+/// It is tempting to define is_complete and is_complete_v, but ultimately these+/// would be a bad idea. These traits are defined here to witness that these are+/// intentionally excluded and not merely a missing feature.+template <typename T>+struct is_complete {+  static_assert(always_false<T>, "is_complete would break ODR");+};+template <typename T>+constexpr auto is_complete_v = is_complete<T>::value;++/// is_unbounded_array_v+/// is_unbounded_array+///+/// A trait variable and type to check if a given type is an unbounded array.+///+/// mimic: std::is_unbounded_array_d, std::is_unbounded_array (C++20)+template <typename T>+inline constexpr bool is_unbounded_array_v = false;+template <typename T>+inline constexpr bool is_unbounded_array_v<T[]> = true;+template <typename T>+struct is_unbounded_array : std::bool_constant<is_unbounded_array_v<T>> {};++/// is_bounded_array_v+/// is_bounded_array+///+/// A trait variable and type to check if a given type is a bounded array.+///+/// mimic: std::is_bounded_array_d, std::is_bounded_array (C++20)+template <typename T>+inline constexpr bool is_bounded_array_v = false;+template <typename T, std::size_t S>+inline constexpr bool is_bounded_array_v<T[S]> = true;+template <typename T>+struct is_bounded_array : std::bool_constant<is_bounded_array_v<T>> {};++/// is_instantiation_of_v+/// is_instantiation_of+/// instantiated_from+/// uncvref_instantiated_from+///+/// A trait variable and type to check if a given type is an instantiation of a+/// class template. And corresponding concepts.+///+/// Note that this only works with type template parameters. It does not work+/// with non-type template parameters, template template parameters, or alias+/// templates.+template <template <typename...> class, typename>+inline constexpr bool is_instantiation_of_v = false;+template <template <typename...> class C, typename... T>+inline constexpr bool is_instantiation_of_v<C, C<T...>> = true;+template <template <typename...> class C, typename... T>+struct is_instantiation_of+    : std::bool_constant<is_instantiation_of_v<C, T...>> {};++#if defined(__cpp_concepts)++template <typename T, template <typename...> class Templ>+concept instantiated_from = is_instantiation_of_v<Templ, T>;++template <typename T, template <typename...> class Templ>+concept uncvref_instantiated_from =+    is_instantiation_of_v<Templ, std::remove_cvref_t<T>>;++#endif++/// member_pointer_traits+///+/// For a member-pointer, reveals its constituent member-type and object-type.+///+/// Works for both member-object-pointer and member-function-pointer.+template <typename>+struct member_pointer_traits;+template <typename M, typename O>+struct member_pointer_traits<M O::*> {+  using member_type = M;+  using object_type = O;+};++/// member_pointer_member_t+///+/// The member-type of a pointer-to-member type.+template <typename P>+using member_pointer_member_t = typename member_pointer_traits<P>::member_type;++/// member_pointer_object_t+///+/// The object-type of a pointer-to-member type.+template <typename P>+using member_pointer_object_t = typename member_pointer_traits<P>::object_type;++namespace detail {++struct is_constexpr_default_constructible_ {+  template <typename T>+  static constexpr auto make(tag_t<T>) -> decltype(void(T()), 0) {+    return (void(T()), 0);+  }+  //  second param should just be: int = (void(T()), 0)+  //  but under clang 10, crash: https://bugs.llvm.org/show_bug.cgi?id=47620+  //  and, with assertions disabled, expectation failures showing compiler+  //  deviation from the language spec+  //  xcode renumbers clang versions so detection is tricky, but, if detection+  //  were desired, a combination of __apple_build_version__ and __clang_major__+  //  may be used to reduce frontend overhead under correct compilers: clang 12+  //  under xcode and clang 10 otherwise+  template <typename T, int = make(tag<T>)>+  static std::true_type sfinae(T*);+  static std::false_type sfinae(void*);+  template <typename T>+  static constexpr bool apply =+      !require_sizeof<T> || decltype(sfinae(static_cast<T*>(nullptr)))::value;+};++} // namespace detail++/// is_constexpr_default_constructible_v+/// is_constexpr_default_constructible+///+/// A trait variable and type which determines whether the type parameter is+/// constexpr default-constructible, that is, default-constructible in a+/// constexpr context.+template <typename T>+inline constexpr bool is_constexpr_default_constructible_v =+    detail::is_constexpr_default_constructible_::apply<T>;+template <typename T>+struct is_constexpr_default_constructible+    : std::bool_constant<is_constexpr_default_constructible_v<T>> {};++/***+ *  _t+ *+ *  Instead of:+ *+ *    using decayed = typename std::decay<T>::type;+ *+ *  With the C++14 standard trait aliases, we could use:+ *+ *    using decayed = std::decay_t<T>;+ *+ *  Without them, we could use:+ *+ *    using decayed = _t<std::decay<T>>;+ *+ *  Also useful for any other library with template types having dependent+ *  member types named `type`, like the standard trait types.+ */+template <typename T>+using _t = typename T::type;++/**+ * A type trait to remove all const volatile and reference qualifiers on a+ * type T+ */+template <typename T>+struct remove_cvref {+  using type =+      typename std::remove_cv<typename std::remove_reference<T>::type>::type;+};+template <typename T>+using remove_cvref_t = typename remove_cvref<T>::type;++namespace detail {+template <typename Src>+struct copy_cvref_ {+  template <typename Dst>+  using apply = Dst;+};+template <typename Src>+struct copy_cvref_<Src const> {+  template <typename Dst>+  using apply = Dst const;+};+template <typename Src>+struct copy_cvref_<Src volatile> {+  template <typename Dst>+  using apply = Dst volatile;+};+template <typename Src>+struct copy_cvref_<Src const volatile> {+  template <typename Dst>+  using apply = Dst const volatile;+};+template <typename Src>+struct copy_cvref_<Src&> {+  template <typename Dst>+  using apply = typename copy_cvref_<Src>::template apply<Dst>&;+};+template <typename Src>+struct copy_cvref_<Src&&> {+  template <typename Dst>+  using apply = typename copy_cvref_<Src>::template apply<Dst>&&;+};+} // namespace detail++/// copy_cvref_t+///+/// A trait alias to replace the cvref category of `Dst` with that of `Src`.+///+/// CAUTION: This is not what is typically wanted in a forwarding or+/// deducing-`this` context, or in most cases of casting one reference+/// to another. In such cases, the most appropriate tool would be `like_t`,+/// or `std::forward_like` in C++23.+///+/// Some of the problems with forwarding via `copy_cvref` are:+/// - Removing `const` from a `Dst` that is backed by a value is quite+///   problematic. The case of `static_cast<copy_cvref_t<...>>(dst)`+///   would yield a compile error. The case of a C-style cast, `const_cast`,+///   `reinterpret_cast`, or functional case may compile but may have+///   undefined behavior by treating non-writable memory as writable.+/// - The `Dst` value would typically have an address, and the `volatile`+///   qualifier is a function of that address, so it would be incorrect+///   to derive that from `Src`.+///+/// `like_t` and `forward_like` avoid these problems.+template <typename Src, typename Dst>+using copy_cvref_t =+    typename detail::copy_cvref_<Src>::template apply<remove_cvref_t<Dst>>;++namespace detail {+// These `copy_ref_` functions assume `Dst` is not a reference.+template <typename Src>+struct copy_ref_ {+  template <typename Dst>+  using apply = Dst;+};+template <typename Src>+struct copy_ref_<Src&> {+  template <typename Dst>+  using apply = Dst&;+};+template <typename Src>+struct copy_ref_<Src&&> {+  template <typename Dst>+  using apply = Dst&&;+};+template <typename Src, typename Dst>+using copy_const_t = std::conditional_t<+    std::is_const_v<std::remove_reference_t<Src>>,+    Dst const,+    Dst>;+} // namespace detail++/// like+/// like_t+///+/// Similar to `like` and `like_t` from p0847r0, but with semantics made+/// compatible with the C++23 `std::forward_like` from p2445r0.+///+/// Differences:+/// - Never removes `const` qualifiers from `Dst`.+/// - Leaves any `volatile` as it was on `Dst`, `Src` volatility is ignored.+/// - Unlike `__forward_like_t` from p2445r0, distinguishes between value+///   `Src` and rvalue-reference `Src` types.+///+/// The above are the right semantics for most cases.+///+/// The rare cases which need to replace all cvref qualifiers from `Src` onto+/// `Dst` may use `copy_cvref_t`. But heed the cautions in its documentation.+///+/// mimic: `like`, p0847r0+template <typename Src, typename Dst>+using like_t = typename detail::copy_ref_<Src>::template apply<+    detail::copy_const_t<Src, std::remove_reference_t<Dst>>>;+template <typename Src, typename Dst>+struct like {+  using type = like_t<Src, Dst>;+};++#if defined(__cpp_concepts)++/**+ *  Concept to check that a type is same as a given type,+ *  when stripping qualifiers and refernces.+ *  Especially useful for perfect forwarding of a specific type.+ *+ *  Example:+ *+ *    void foo(folly::uncvref_same_as<std::vector<int>> auto&& vec);+ *+ */+template <typename Ref, typename To>+concept uncvref_same_as = std::is_same_v<std::remove_cvref_t<Ref>, To>;++#endif++/**+ *  type_t+ *+ *  A type alias for the first template type argument. `type_t` is useful for+ *  controlling class-template and function-template partial specialization.+ *+ *  Example:+ *+ *    template <typename Value>+ *    class Container {+ *     public:+ *      template <typename... Args>+ *      Container(+ *          type_t<in_place_t, decltype(Value(std::declval<Args>()...))>,+ *          Args&&...);+ *    };+ *+ *  void_t+ *+ *  A type alias for `void`. `void_t` is useful for controlling class-template+ *  and function-template partial specialization.+ *+ *  Example:+ *+ *    // has_value_type<T>::value is true if T has a nested type `value_type`+ *    template <class T, class = void>+ *    struct has_value_type+ *        : std::false_type {};+ *+ *    template <class T>+ *    struct has_value_type<T, folly::void_t<typename T::value_type>>+ *        : std::true_type {};+ */++/**+ * There is a bug in libstdc++, libc++, and MSVC's STL that causes it to+ * ignore unused template parameter arguments in template aliases and does not+ * cause substitution failures. This defect has been recorded here:+ * http://open-std.org/JTC1/SC22/WG21/docs/cwg_defects.html#1558.+ *+ * This causes the implementation of std::void_t to be buggy, as it is likely+ * defined as something like the following:+ *+ *  template <typename...>+ *  using void_t = void;+ *+ * This causes the compiler to ignore all the template arguments and does not+ * help when one wants to cause substitution failures.  Rather declarations+ * which have void_t in orthogonal specializations are treated as the same.+ * For example, assuming the possible `T` types are only allowed to have+ * either the alias `one` or `two` and never both or none:+ *+ *  template <typename T,+ *            typename std::void_t<std::decay_t<T>::one>* = nullptr>+ *  void foo(T&&) {}+ *  template <typename T,+ *            typename std::void_t<std::decay_t<T>::two>* = nullptr>+ *  void foo(T&&) {}+ *+ * The second foo() will be a redefinition because it conflicts with the first+ * one; void_t does not cause substitution failures - the template types are+ * just ignored.+ */++namespace traits_detail {+template <class T, class...>+struct type_t_ {+  using type = T;+};+} // namespace traits_detail++template <class T, class... Ts>+using type_t = typename traits_detail::type_t_<T, Ts...>::type;+template <class... Ts>+using void_t = type_t<void, Ts...>;++/// nonesuch+///+/// A tag type which traits may use to indicate lack of a result type.+///+/// Similar to void in that no values of this type may be constructed. Different+/// from void in that no functions may be defined with this return type and no+/// complete expressions may evaluate with this expression type.+///+/// mimic: std::experimental::nonesuch, Library Fundamentals TS v2+struct nonesuch {+  ~nonesuch() = delete;+  nonesuch(nonesuch const&) = delete;+  void operator=(nonesuch const&) = delete;+};++namespace detail {++template <typename Void, typename D, template <typename...> class, typename...>+struct detected_ {+  using value_t = std::false_type;+  using type = D;+};+template <typename D, template <typename...> class T, typename... A>+struct detected_<void_t<T<A...>>, D, T, A...> {+  using value_t = std::true_type;+  using type = T<A...>;+};++} // namespace detail++/// detected_or+///+/// If T<A...> substitutes, has member type alias value_t as std::true_type+/// and has member type alias type as T<A...>. Otherwise, has member type+/// alias value_t as std::false_type and has member type alias type as D.+///+/// mimic: std::experimental::detected_or, Library Fundamentals TS v2+///+/// Note: not resilient against incomplete types; may violate ODR.+template <typename D, template <typename...> class T, typename... A>+using detected_or = detail::detected_<void, D, T, A...>;++/// detected_or_t+///+/// A trait type alias which results in T<A...> if substitution would succeed+/// and in D otherwise.+///+/// Equivalent to detected_or<D, T, A...>::type.+///+/// mimic: std::experimental::detected_or_t, Library Fundamentals TS v2+///+/// Note: not resilient against incomplete types; may violate ODR.+template <typename D, template <typename...> class T, typename... A>+using detected_or_t = typename detected_or<D, T, A...>::type;++/// detected_t+///+/// A trait type alias which results in T<A...> if substitution would succeed+/// and in nonesuch otherwise.+///+/// Equivalent to detected_or_t<nonesuch, T, A...>.+///+/// mimic: std::experimental::detected_t, Library Fundamentals TS v2+///+/// Note: not resilient against incomplete types; may violate ODR.+template <template <typename...> class T, typename... A>+using detected_t = detected_or_t<nonesuch, T, A...>;++//  is_detected_v+//  is_detected+//+//  A trait variable and type to test whether some metafunction from types to+//  types would succeed or fail in substitution over a given set of arguments.+//+//  The trait variable is_detected_v<T, A...> is equivalent to+//  detected_or<nonesuch, T, A...>::value_t::value.+//  The trait type is_detected<T, A...> unambiguously inherits+//  std::bool_constant<V> where V is is_detected_v<T, A...>.+//+//  mimic: std::experimental::is_detected, std::experimental::is_detected_v,+//    Library Fundamentals TS v2+//+//  Note: not resilient against incomplete types; may violate ODR.+//+//  Note: the trait type is_detected differs here by being deferred.+template <template <typename...> class T, typename... A>+inline constexpr bool is_detected_v =+    detected_or<nonesuch, T, A...>::value_t::value;+template <template <typename...> class T, typename... A>+struct is_detected : detected_or<nonesuch, T, A...>::value_t {};++template <typename T>+using aligned_storage_for_t =+    typename std::aligned_storage<sizeof(T), alignof(T)>::type;++//  ----++namespace fallback {+template <typename From, typename To>+inline constexpr bool is_nothrow_convertible_v =+    (std::is_void<From>::value && std::is_void<To>::value) ||+    ( //+        std::is_convertible<From, To>::value &&+        std::is_nothrow_constructible<To, From>::value);+template <typename From, typename To>+struct is_nothrow_convertible+    : std::bool_constant<is_nothrow_convertible_v<From, To>> {};+} // namespace fallback++//  is_nothrow_convertible+//  is_nothrow_convertible_v+//+//  Import or backport:+//  * std::is_nothrow_convertible+//  * std::is_nothrow_convertible_v+//+//  mimic: is_nothrow_convertible, C++20+#if defined(__cpp_lib_is_nothrow_convertible) && \+    __cpp_lib_is_nothrow_convertible >= 201806L+using std::is_nothrow_convertible;+using std::is_nothrow_convertible_v;+#else+using fallback::is_nothrow_convertible;+using fallback::is_nothrow_convertible_v;+#endif++/**+ * IsRelocatable<T>::value describes the ability of moving around+ * memory a value of type T by using memcpy (as opposed to the+ * conservative approach of calling the copy constructor and then+ * destroying the old temporary. Essentially for a relocatable type,+ * the following two sequences of code should be semantically+ * equivalent:+ *+ * void move1(T * from, T * to) {+ *   new(to) T(from);+ *   (*from).~T();+ * }+ *+ * void move2(T * from, T * to) {+ *   memcpy(to, from, sizeof(T));+ * }+ *+ * Most C++ types are relocatable; the ones that aren't would include+ * internal pointers or (very rarely) would need to update remote+ * pointers to pointers tracking them. All C++ primitive types and+ * type constructors are relocatable.+ *+ * This property can be used in a variety of optimizations. Currently+ * fbvector uses this property intensively.+ *+ * The default conservatively assumes the type is not+ * relocatable. Several specializations are defined for known+ * types. You may want to add your own specializations. Do so in+ * namespace folly and make sure you keep the specialization of+ * IsRelocatable<SomeStruct> in the same header as SomeStruct.+ *+ * You may also declare a type to be relocatable by including+ *    `typedef std::true_type IsRelocatable;`+ * in the class header.+ *+ * It may be unset in a base class by overriding the typedef to false_type.+ */+/*+ * IsZeroInitializable describes the property that value-initialization+ * is the same as memset(dst, 0, sizeof(T)).+ */++namespace traits_detail {++#define FOLLY_HAS_TRUE_XXX(name)                                             \+  template <typename T>                                                      \+  using detect_##name = typename T::name;                                    \+  template <class T>                                                         \+  struct name##_is_true : std::is_same<typename T::name, std::true_type> {}; \+  template <class T>                                                         \+  struct has_true_##name                                                     \+      : std::conditional<                                                    \+            is_detected_v<detect_##name, T>,                                 \+            name##_is_true<T>,                                               \+            std::false_type>::type {}++FOLLY_HAS_TRUE_XXX(IsRelocatable);+FOLLY_HAS_TRUE_XXX(IsZeroInitializable);++#undef FOLLY_HAS_TRUE_XXX++} // namespace traits_detail++struct Ignore {+  Ignore() = default;+  template <class T>+  constexpr /* implicit */ Ignore(const T&) {}+  template <class T>+  const Ignore& operator=(T const&) const {+    return *this;+  }+};++template <class...>+using Ignored = Ignore;++namespace traits_detail_IsEqualityComparable {+Ignore operator==(Ignore, Ignore);++template <class T, class U = T>+struct IsEqualityComparable+    : std::is_convertible<+          decltype(std::declval<T>() == std::declval<U>()),+          bool> {};+} // namespace traits_detail_IsEqualityComparable++/* using override */ using traits_detail_IsEqualityComparable::+    IsEqualityComparable;++namespace traits_detail_IsLessThanComparable {+Ignore operator<(Ignore, Ignore);++template <class T, class U = T>+struct IsLessThanComparable+    : std::is_convertible<+          decltype(std::declval<T>() < std::declval<U>()),+          bool> {};+} // namespace traits_detail_IsLessThanComparable++/* using override */ using traits_detail_IsLessThanComparable::+    IsLessThanComparable;++template <class T>+struct IsRelocatable+    : std::conditional<+          !require_sizeof<T> ||+              is_detected_v<traits_detail::detect_IsRelocatable, T>,+          traits_detail::has_true_IsRelocatable<T>,+#if defined(__cpp_lib_is_trivially_relocatable) // P1144+          std::is_trivially_relocatable<T>+#else+          std::is_trivially_copyable<T>+#endif+          >::type {+};++template <class T>+struct IsZeroInitializable+    : std::conditional<+          !require_sizeof<T> ||+              is_detected_v<traits_detail::detect_IsZeroInitializable, T>,+          traits_detail::has_true_IsZeroInitializable<T>,+          std::bool_constant< //+              !std::is_class<T>::value && //+              !std::is_union<T>::value && //+              !std::is_member_object_pointer<T>::value && // itanium+              true>>::type {};++namespace detail {+template <bool>+struct conditional_;+template <>+struct conditional_<false> {+  template <typename, typename T>+  using apply = T;+};+template <>+struct conditional_<true> {+  template <typename T, typename>+  using apply = T;+};+} // namespace detail++/// conditional_t+///+/// Like std::conditional_t but with only two total class template instances,+/// rather than as many class template instances as there are uses.+///+/// As one effect, the result can be used in deducible contexts, allowing+/// deduction of conditional_t<V, T, F> to work when T or F is a template param.+template <bool V, typename T, typename F>+using conditional_t = typename detail::conditional_<V>::template apply<T, F>;++template <typename...>+struct Conjunction : std::true_type {};+template <typename T>+struct Conjunction<T> : T {};+template <typename T, typename... TList>+struct Conjunction<T, TList...>+    : std::conditional<T::value, Conjunction<TList...>, T>::type {};++template <typename...>+struct Disjunction : std::false_type {};+template <typename T>+struct Disjunction<T> : T {};+template <typename T, typename... TList>+struct Disjunction<T, TList...>+    : std::conditional<T::value, T, Disjunction<TList...>>::type {};++template <typename T>+struct Negation : std::bool_constant<!T::value> {};++template <bool... Bs>+struct Bools {+  using valid_type = bool;+  static constexpr std::size_t size() { return sizeof...(Bs); }+};++//  Lighter-weight than Conjunction, but evaluates all sub-conditions eagerly.+template <class... Ts>+struct StrictConjunction+    : std::is_same<Bools<Ts::value...>, Bools<(Ts::value || true)...>> {};++template <class... Ts>+struct StrictDisjunction+    : Negation<+          std::is_same<Bools<Ts::value...>, Bools<(Ts::value && false)...>>> {};++namespace detail {+template <typename T>+using is_transparent_ = typename T::is_transparent;+} // namespace detail++/// is_transparent_v+/// is_transparent+///+/// A trait variable and type to test whether a less, equal-to, or hash type+/// follows the is-transparent protocol used by containers with optional+/// heterogeneous access.+template <typename T>+inline constexpr bool is_transparent_v =+    is_detected_v<detail::is_transparent_, T>;+template <typename T>+struct is_transparent : std::bool_constant<is_transparent_v<T>> {};++namespace detail {++template <typename T, typename = void>+inline constexpr bool is_allocator_ = !require_sizeof<T>;+template <typename T>+inline constexpr bool is_allocator_<+    T,+    void_t<+        typename T::value_type,+        decltype(std::declval<T&>().allocate(std::size_t{})),+        decltype(std::declval<T&>().deallocate(+            static_cast<typename T::value_type*>(nullptr), std::size_t{}))>> =+    true;++} // namespace detail++/// is_allocator_v+/// is_allocator+///+/// A trait variable and type to test whether a type is an allocator according+/// to the minimum protocol required by std::allocator_traits.+template <typename T>+inline constexpr bool is_allocator_v = detail::is_allocator_<T>;+template <typename T>+struct is_allocator : std::bool_constant<is_allocator_v<T>> {};++} // namespace folly++/**+ * Use this macro ONLY inside namespace folly. When using it with a+ * regular type, use it like this:+ *+ * // Make sure you're at namespace ::folly scope+ * template <> FOLLY_ASSUME_RELOCATABLE(MyType)+ *+ * When using it with a template type, use it like this:+ *+ * // Make sure you're at namespace ::folly scope+ * template <class T1, class T2>+ * FOLLY_ASSUME_RELOCATABLE(MyType<T1, T2>)+ */+#define FOLLY_ASSUME_RELOCATABLE(...) \+  struct IsRelocatable<__VA_ARGS__> : std::true_type {}++/**+ * The FOLLY_ASSUME_FBVECTOR_COMPATIBLE* macros below encode the+ * assumption that the type is relocatable per IsRelocatable+ * above. Many types can be assumed to satisfy this condition, but+ * it is the responsibility of the user to state that assumption.+ * User-defined classes will not be optimized for use with+ * fbvector (see FBVector.h) unless they state that assumption.+ *+ * Use FOLLY_ASSUME_FBVECTOR_COMPATIBLE with regular types like this:+ *+ * FOLLY_ASSUME_FBVECTOR_COMPATIBLE(MyType)+ *+ * The versions FOLLY_ASSUME_FBVECTOR_COMPATIBLE_1, _2, _3, and _4+ * allow using the macro for describing templatized classes with 1, 2,+ * 3, and 4 template parameters respectively. For template classes+ * just use the macro with the appropriate number and pass the name of+ * the template to it. Example:+ *+ * template <class T1, class T2> class MyType { ... };+ * ...+ * // Make sure you're at global scope+ * FOLLY_ASSUME_FBVECTOR_COMPATIBLE_2(MyType)+ */++// Use this macro ONLY at global level (no namespace)+#define FOLLY_ASSUME_FBVECTOR_COMPATIBLE(...) \+  namespace folly {                           \+  template <>                                 \+  FOLLY_ASSUME_RELOCATABLE(__VA_ARGS__);      \+  }+// Use this macro ONLY at global level (no namespace)+#define FOLLY_ASSUME_FBVECTOR_COMPATIBLE_1(...) \+  namespace folly {                             \+  template <class T1>                           \+  FOLLY_ASSUME_RELOCATABLE(__VA_ARGS__<T1>);    \+  }+// Use this macro ONLY at global level (no namespace)+#define FOLLY_ASSUME_FBVECTOR_COMPATIBLE_2(...)  \+  namespace folly {                              \+  template <class T1, class T2>                  \+  FOLLY_ASSUME_RELOCATABLE(__VA_ARGS__<T1, T2>); \+  }+// Use this macro ONLY at global level (no namespace)+#define FOLLY_ASSUME_FBVECTOR_COMPATIBLE_3(...)      \+  namespace folly {                                  \+  template <class T1, class T2, class T3>            \+  FOLLY_ASSUME_RELOCATABLE(__VA_ARGS__<T1, T2, T3>); \+  }+// Use this macro ONLY at global level (no namespace)+#define FOLLY_ASSUME_FBVECTOR_COMPATIBLE_4(...)          \+  namespace folly {                                      \+  template <class T1, class T2, class T3, class T4>      \+  FOLLY_ASSUME_RELOCATABLE(__VA_ARGS__<T1, T2, T3, T4>); \+  }++namespace folly {++// STL commonly-used types+template <class T, class U>+struct IsRelocatable<std::pair<T, U>>+    : std::bool_constant<IsRelocatable<T>::value && IsRelocatable<U>::value> {};++// Is T one of T1, T2, ..., Tn?+template <typename T, typename... Ts>+using IsOneOf = StrictDisjunction<std::is_same<T, Ts>...>;++/*+ * Complementary type traits for integral comparisons.+ *+ * For instance, `if(x < 0)` yields an error in clang for unsigned types+ * when -Werror is used due to -Wtautological-compare+ */++// same as `x < 0`+template <typename T>+constexpr bool is_negative(T x) {+  return std::is_signed<T>::value && x < T(0);+}++// same as `x <= 0`+template <typename T>+constexpr bool is_non_positive(T x) {+  return !x || folly::is_negative(x);+}++// same as `x > 0`+template <typename T>+constexpr bool is_positive(T x) {+  return !is_non_positive(x);+}++// same as `x >= 0`+template <typename T>+constexpr bool is_non_negative(T x) {+  return !x || is_positive(x);+}++namespace detail {++//  folly::to integral specializations can end up generating code+//  inside what are really static ifs (not executed because of the templated+//  types) that violate -Wsign-compare and/or -Wbool-compare so suppress them+//  in order to not prevent all calling code from using it.+FOLLY_PUSH_WARNING+FOLLY_GNU_DISABLE_WARNING("-Wsign-compare")+FOLLY_GCC_DISABLE_WARNING("-Wbool-compare")+FOLLY_MSVC_DISABLE_WARNING(4287) // unsigned/negative constant mismatch+FOLLY_MSVC_DISABLE_WARNING(4388) // sign-compare+FOLLY_MSVC_DISABLE_WARNING(4804) // bool-compare++template <typename RHS, RHS rhs, typename LHS>+bool less_than_impl(LHS const lhs) {+  // clang-format off+  return+      // Ensure signed and unsigned values won't be compared directly.+      (!std::is_signed<RHS>::value && is_negative(lhs)) ? true :+      (!std::is_signed<LHS>::value && is_negative(rhs)) ? false :+      rhs > std::numeric_limits<LHS>::max() ? true :+      rhs <= std::numeric_limits<LHS>::lowest() ? false :+      lhs < rhs;+  // clang-format on+}++template <typename RHS, RHS rhs, typename LHS>+bool greater_than_impl(LHS const lhs) {+  // clang-format off+  return+      // Ensure signed and unsigned values won't be compared directly.+      (!std::is_signed<RHS>::value && is_negative(lhs)) ? false :+      (!std::is_signed<LHS>::value && is_negative(rhs)) ? true :+      rhs > std::numeric_limits<LHS>::max() ? false :+      rhs < std::numeric_limits<LHS>::lowest() ? true :+      lhs > rhs;+  // clang-format on+}++FOLLY_POP_WARNING++} // namespace detail++template <typename RHS, RHS rhs, typename LHS>+bool less_than(LHS const lhs) {+  return detail::+      less_than_impl<RHS, rhs, typename std::remove_reference<LHS>::type>(lhs);+}++template <typename RHS, RHS rhs, typename LHS>+bool greater_than(LHS const lhs) {+  return detail::+      greater_than_impl<RHS, rhs, typename std::remove_reference<LHS>::type>(+          lhs);+}+} // namespace folly++// Assume nothing when compiling with MSVC.+#ifndef _MSC_VER+FOLLY_ASSUME_FBVECTOR_COMPATIBLE_2(std::unique_ptr)+FOLLY_ASSUME_FBVECTOR_COMPATIBLE_1(std::shared_ptr)+#endif++namespace folly {++//  Some compilers have signed __int128 and unsigned __int128 types, and some+//  libraries with some compilers have traits for those types. It's a mess.+//  Import things into folly and then fill in whatever is missing.+//+//  The aliases:+//    int128_t+//    uint128_t+//+//  The traits:+//    is_arithmetic+//    is_arithmetic_v+//    is_integral+//    is_integral_v+//    is_signed+//    is_signed_v+//    is_unsigned+//    is_unsigned_v+//    make_signed+//    make_signed_t+//    make_unsigned+//    make_unsigned_t++template <typename T>+struct is_arithmetic : std::is_arithmetic<T> {};+template <typename T>+inline constexpr bool is_arithmetic_v = is_arithmetic<T>::value;++template <typename T>+struct is_integral : std::is_integral<T> {};+template <typename T>+inline constexpr bool is_integral_v = is_integral<T>::value;++template <typename T>+struct is_signed : std::is_signed<T> {};+template <typename T>+inline constexpr bool is_signed_v = is_signed<T>::value;++template <typename T>+struct is_unsigned : std::is_unsigned<T> {};+template <typename T>+inline constexpr bool is_unsigned_v = is_unsigned<T>::value;++template <typename T>+struct make_signed : std::make_signed<T> {};+template <typename T>+using make_signed_t = typename make_signed<T>::type;++template <typename T>+struct make_unsigned : std::make_unsigned<T> {};+template <typename T>+using make_unsigned_t = typename make_unsigned<T>::type;++#if FOLLY_HAVE_INT128_T++using int128_t = signed __int128;+using uint128_t = unsigned __int128;++template <>+struct is_arithmetic<int128_t> : std::true_type {};+template <>+struct is_arithmetic<uint128_t> : std::true_type {};++template <>+struct is_integral<int128_t> : std::true_type {};+template <>+struct is_integral<uint128_t> : std::true_type {};++template <>+struct is_signed<int128_t> : std::true_type {};+template <>+struct is_signed<uint128_t> : std::false_type {};+template <>+struct is_unsigned<int128_t> : std::false_type {};+template <>+struct is_unsigned<uint128_t> : std::true_type {};++template <>+struct make_signed<int128_t> {+  using type = int128_t;+};+template <>+struct make_signed<uint128_t> {+  using type = int128_t;+};++template <>+struct make_unsigned<int128_t> {+  using type = uint128_t;+};+template <>+struct make_unsigned<uint128_t> {+  using type = uint128_t;+};+#endif // FOLLY_HAVE_INT128_T++namespace traits_detail {+template <std::size_t>+struct uint_bits_t_ {};+template <>+struct uint_bits_t_<8> : type_t_<std::uint8_t> {};+template <>+struct uint_bits_t_<16> : type_t_<std::uint16_t> {};+template <>+struct uint_bits_t_<32> : type_t_<std::uint32_t> {};+template <>+struct uint_bits_t_<64> : type_t_<std::uint64_t> {};+#if FOLLY_HAVE_INT128_T+template <>+struct uint_bits_t_<128> : type_t_<uint128_t> {};+#endif // FOLLY_HAVE_INT128_T+} // namespace traits_detail++template <std::size_t bits>+using uint_bits_t = _t<traits_detail::uint_bits_t_<bits>>;++template <std::size_t lg_bits>+using uint_bits_lg_t = uint_bits_t<(1u << lg_bits)>;++template <std::size_t bits>+using int_bits_t = make_signed_t<uint_bits_t<bits>>;++template <std::size_t lg_bits>+using int_bits_lg_t = make_signed_t<uint_bits_lg_t<lg_bits>>;++namespace traits_detail {++template <std::size_t I, typename T>+struct type_pack_element_indexed_type {+  using type = T;+};++template <typename, typename...>+struct type_pack_element_set;+template <std::size_t... I, typename... T>+struct type_pack_element_set<std::index_sequence<I...>, T...>+    : type_pack_element_indexed_type<I, T>... {};+template <typename... T>+using type_pack_element_set_t =+    type_pack_element_set<std::index_sequence_for<T...>, T...>;++template <std::size_t I>+struct type_pack_element_test {+  template <typename T>+  static type_pack_element_indexed_type<I, T> impl(+      type_pack_element_indexed_type<I, T>*);+};++template <std::size_t I, typename... Ts>+using type_pack_element_fallback = _t<decltype(type_pack_element_test<I>::impl(+    static_cast<type_pack_element_set_t<Ts...>*>(nullptr)))>;++} // namespace traits_detail++/// type_pack_element_t+///+/// In the type pack Ts..., the Ith element.+///+/// Wraps the builtin __type_pack_element where the builtin is available; where+/// not, implemented directly.+///+/// Under gcc, the builtin is available but does not mangle. Therefore, this+/// trait must not be used anywhere it might be subject to mangling, such as in+/// a return-type expression.++#if FOLLY_HAS_BUILTIN(__type_pack_element)++template <std::size_t I, typename... Ts>+using type_pack_element_t = __type_pack_element<I, Ts...>;++#else++template <std::size_t I, typename... Ts>+using type_pack_element_t = traits_detail::type_pack_element_fallback<I, Ts...>;++#endif++/// type_pack_size_v+///+/// The size of a type pack.+///+/// A metafunction around sizeof...(Ts).+template <typename... Ts>+inline constexpr std::size_t type_pack_size_v = sizeof...(Ts);++/// type_pack_size_t+///+/// The size of a type pack.+///+/// A metafunction around index_constant<sizeof...(Ts)>.+template <typename... Ts>+using type_pack_size_t = index_constant<sizeof...(Ts)>;++namespace traits_detail {++template <std::size_t I, template <typename...> class List, typename... T>+type_identity<type_pack_element_t<I, T...>> type_list_element_(+    List<T...> const*);++template <template <typename...> class List, typename... T>+index_constant<sizeof...(T)> type_list_size_(List<T...> const*);++} // namespace traits_detail++/// type_list_element_t+///+/// In the type list List<T...>, where List has kind template <typename...> and+/// T... is a type-pack, equivalent to type_pack_element_t<I, T...>.+template <std::size_t I, typename List>+using type_list_element_t = _t<decltype(traits_detail::type_list_element_<I>(+    static_cast<List const*>(nullptr)))>;++/// type_list_size_v+///+/// The size of a type list.+///+/// For List<T...>, equivalent to type_pack_size_v<T...>.+template <typename List>+inline constexpr std::size_t type_list_size_v =+    decltype(traits_detail::type_list_size_(+        static_cast<List const*>(nullptr)))::value;++/// type_list_size_t+///+/// The size of a type list.+///+/// For List<T...>, equivalent to type_pack_size_t<T...>.+template <typename List>+using type_list_size_t =+    decltype(traits_detail::type_list_size_(static_cast<List const*>(nullptr)));++namespace detail {++// The arguments to this "error" type help the user debug bad invocations.+// It is purposely undefined to cause a compile error.+template <typename...>+struct error_list_concat_params_should_be_non_cvref;++// The primary template is only invoked for invalid parameters.+template <template <typename...> class Out, typename... T>+inline constexpr auto type_list_concat_ =+    error_list_concat_params_should_be_non_cvref<T...>{};++template <template <typename...> class Out>+inline constexpr type_identity<Out<>> type_list_concat_<Out>;++template <+    template <typename...>+    class Out,+    template <typename...>+    class In,+    typename... T>+inline constexpr auto type_list_concat_<Out, In<T...>> =+    type_identity<Out<T...>>{};++template <+    template <typename...>+    class Out,+    // Allow input lists to come from heterogeneous templates.+    template <typename...>+    class InA,+    typename... A,+    template <typename...>+    class InB,+    typename... B,+    typename... Tail>+inline constexpr auto type_list_concat_<Out, InA<A...>, InB<B...>, Tail...> =+    // Avoid instantiating the `In*` or `Out` types for the intermediate+    // lists, since those types may be invalid, or expensive.  Per my tests+    // on clang using `tag_t` for the intermediate list is no more expensive+    // than using a dedicated incomplete list type.+    type_list_concat_<Out, tag_t<A..., B...>, Tail...>;++} // namespace detail++/// type_list_concat_t+///+/// Each `List` is a type list of the form `InK<TypeK...>`, where the+/// templates `InK` are potentially heterogeneous.  Concatenates these+/// `List`s into a single type list `Out<Type1..., Type2..., ...>`.+template <template <typename...> class Out, typename... List>+using type_list_concat_t =+    typename decltype(detail::type_list_concat_<Out, List...>)::type;++namespace traits_detail {++template <decltype(auto) V>+struct value_pack_constant {+  inline static constexpr decltype(V) value = V;+};++} // namespace traits_detail++/// value_pack_size_v+///+/// The size of a value pack.+///+/// A metafunction around sizeof...(V).+template <auto... V>+inline constexpr std::size_t value_pack_size_v = sizeof...(V);++/// value_pack_size_t+///+/// The size of a value pack.+///+/// A metafunction around index_constant<sizeof...(V)>.+template <auto... V>+using value_pack_size_t = index_constant<sizeof...(V)>;++/// value_pack_element_type_t+///+/// In the value pack V..., the type of the Ith element.+template <std::size_t I, auto... V>+using value_pack_element_type_t = type_pack_element_t<I, decltype(V)...>;++/// value_pack_element_type_t+///+/// In the value pack V..., the Ith element.+template <std::size_t I, auto... V>+inline constexpr value_pack_element_type_t<I, V...> value_pack_element_v =+    type_pack_element_t<I, traits_detail::value_pack_constant<V>...>::value;++namespace traits_detail {++template <typename List>+struct value_list_traits_;+template <template <auto...> class List, auto... V>+struct value_list_traits_<List<V...>> {+  static constexpr std::size_t size = sizeof...(V);+  template <std::size_t I>+  using element_type = value_pack_element_type_t<I, V...>;+  template <std::size_t I>+  static constexpr value_pack_element_type_t<I, V...> element =+      value_pack_element_v<I, V...>;+};++} // namespace traits_detail++/// value_list_size_v+///+/// The size of a value list.+///+/// For List<V...>, equivalent to value_pack_size_v<V...>.+template <typename List>+inline constexpr std::size_t value_list_size_v =+    traits_detail::value_list_traits_<List>::size;++/// value_list_size_t+///+/// The size of a value list.+///+/// For List<V...>, equivalent to value_pack_size_t<V...>.+template <typename List>+using value_list_size_t = index_constant<value_list_size_v<List>>;++/// value_list_element_type_t+///+/// For List<V...>, the type of the Ith element.+template <std::size_t I, typename List>+using value_list_element_type_t =+    typename traits_detail::value_list_traits_<List>::template element_type<I>;++/// value_list_element_v+///+/// For List<V...>, the Ith element.+template <std::size_t I, typename List>+inline constexpr value_list_element_type_t<I, List> value_list_element_v =+    traits_detail::value_list_traits_<List>::template element<I>;++namespace detail {++// The primary template is only invoked for invalid parameters.+template <template <auto...> class Out, typename... T>+inline constexpr auto value_list_concat_ =+    error_list_concat_params_should_be_non_cvref<T...>{};++template <template <auto...> class Out>+inline constexpr type_identity<Out<>> value_list_concat_<Out>;++template <template <auto...> class Out, template <auto...> class In, auto... V>+inline constexpr auto value_list_concat_<Out, In<V...>> =+    type_identity<Out<V...>>{};++template <+    template <auto...>+    class Out,+    // Allow input lists to come from heterogeneous templates.+    template <auto...>+    class InA,+    auto... A,+    template <auto...>+    class InB,+    auto... B,+    typename... Tail>+inline constexpr auto value_list_concat_<Out, InA<A...>, InB<B...>, Tail...> =+    // The use of `vtag_t` is explained in the analogous `type_list_concat_.+    value_list_concat_<Out, vtag_t<A..., B...>, Tail...>;++} // namespace detail++/// value_list_concat_t+///+/// Each `List` is a value list of the form `InK<ValK...>`, where the+/// templates `InK` are potentially heterogeneous.  Concatenates these+/// `List`s into a single value list `Out<Val1..., Val2..., ...>`.+template <template <auto...> class Out, typename... List>+using value_list_concat_t =+    typename decltype(detail::value_list_concat_<Out, List...>)::type;++namespace detail {++template <typename V, typename... T>+constexpr std::size_t type_pack_find_() {+  bool eq[] = {std::is_same_v<V, T>..., true};+  for (size_t i = 0; i < sizeof...(T); ++i) {+    if (eq[i]) {+      return i;+    }+  }+  return sizeof...(T);+}++template <typename>+struct type_list_find_;+template <template <typename...> class List, typename... T>+struct type_list_find_<List<T...>> {+  template <typename V>+  static inline constexpr std::size_t apply = type_pack_find_<V, T...>();+};++} // namespace detail++/// type_pack_find_v+///+/// The index of the element of the type pack which is identical to the given+/// type, or the size of the pack if there is no such element.+template <typename V, typename... T>+inline constexpr std::size_t type_pack_find_v =+    detail::type_pack_find_<V, T...>();++/// type_pack_find_t+///+/// The index of the element of the type pack which is identical to the given+/// type, or the size of the pack if there is no such element.+template <typename V, typename... T>+using type_pack_find_t = index_constant<type_pack_find_v<V, T...>>;++/// type_list_find_v+///+/// The index of the element of the type list which is identical to the given+/// type, or the size of the list if there is no such element.+template <typename V, typename List>+inline constexpr std::size_t type_list_find_v =+    detail::type_list_find_<List>::template apply<V>;++/// type_list_find_t+///+/// The index of the element of the type list which is identical to the given+/// type, or the size of the list if there is no such element.+template <typename V, typename List>+using type_list_find_t = index_constant<type_list_find_v<V, List>>;++} // namespace folly
+ folly/folly/Try-inl.h view
@@ -0,0 +1,373 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <folly/Utility.h>+#include <folly/functional/Invoke.h>++#include <stdexcept>+#include <tuple>+#include <utility>++namespace folly {++namespace detail {+template <class T>+TryBase<T>::TryBase(TryBase<T>&& t) noexcept(+    std::is_nothrow_move_constructible<T>::value)+    : contains_(t.contains_) {+  if (contains_ == Contains::VALUE) {+    ::new (static_cast<void*>(std::addressof(value_))) T(std::move(t.value_));+  } else if (contains_ == Contains::EXCEPTION) {+    new (&e_) exception_wrapper(std::move(t.e_));+  }+}++template <class T>+TryBase<T>& TryBase<T>::operator=(TryBase<T>&& t) noexcept(+    std::is_nothrow_move_constructible<T>::value) {+  if (this == &t) {+    return *this;+  }++  destroy();++  if (t.contains_ == Contains::VALUE) {+    ::new (static_cast<void*>(std::addressof(value_))) T(std::move(t.value_));+  } else if (t.contains_ == Contains::EXCEPTION) {+    new (&e_) exception_wrapper(std::move(t.e_));+  }++  contains_ = t.contains_;++  return *this;+}++template <class T>+TryBase<T>::TryBase(const TryBase<T>& t) noexcept(+    std::is_nothrow_copy_constructible<T>::value) {+  contains_ = t.contains_;+  if (contains_ == Contains::VALUE) {+    ::new (static_cast<void*>(std::addressof(value_))) T(t.value_);+  } else if (contains_ == Contains::EXCEPTION) {+    new (&e_) exception_wrapper(t.e_);+  }+}++template <class T>+TryBase<T>& TryBase<T>::operator=(const TryBase<T>& t) noexcept(+    std::is_nothrow_copy_constructible<T>::value) {+  if (this == &t) {+    return *this;+  }++  destroy();++  if (t.contains_ == Contains::VALUE) {+    ::new (static_cast<void*>(std::addressof(value_))) T(t.value_);+  } else if (t.contains_ == Contains::EXCEPTION) {+    new (&e_) exception_wrapper(t.e_);+  }++  contains_ = t.contains_;++  return *this;+}++template <class T>+void TryBase<T>::destroy() noexcept {+  auto oldContains = std::exchange(contains_, Contains::NOTHING);+  if (FOLLY_LIKELY(oldContains == Contains::VALUE)) {+    value_.~T();+  } else if (FOLLY_UNLIKELY(oldContains == Contains::EXCEPTION)) {+    e_.~exception_wrapper();+  }+}++template <class T>+template <class T2>+TryBase<T>::TryBase(+    typename std::enable_if<std::is_same<Unit, T2>::value, Try<void> const&>::+        type t) noexcept+    : contains_(Contains::NOTHING) {+  if (t.hasValue()) {+    contains_ = Contains::VALUE;+    ::new (static_cast<void*>(std::addressof(value_))) T();+  } else if (t.hasException()) {+    contains_ = Contains::EXCEPTION;+    new (&e_) exception_wrapper(t.exception());+  }+}++template <class T>+TryBase<T>::~TryBase() {+  if (FOLLY_LIKELY(contains_ == Contains::VALUE)) {+    value_.~T();+  } else if (FOLLY_UNLIKELY(contains_ == Contains::EXCEPTION)) {+    e_.~exception_wrapper();+  }+}++} // namespace detail++Try<void>::Try(const Try<Unit>& t) noexcept : hasValue_(!t.hasException()) {+  if (t.hasException()) {+    new (&this->e_) exception_wrapper(t.exception());+  }+}++template <typename T>+template <typename... Args>+T& Try<T>::emplace(Args&&... args) noexcept(+    std::is_nothrow_constructible<T, Args&&...>::value) {+  this->destroy();+  ::new (static_cast<void*>(std::addressof(this->value_)))+      T(static_cast<Args&&>(args)...);+  this->contains_ = Contains::VALUE;+  return this->value_;+}++template <typename T>+template <typename... Args>+exception_wrapper& Try<T>::emplaceException(Args&&... args) noexcept(+    std::is_nothrow_constructible<exception_wrapper, Args&&...>::value) {+  this->destroy();+  new (&this->e_) exception_wrapper(static_cast<Args&&>(args)...);+  this->contains_ = Contains::EXCEPTION;+  return this->e_;+}++template <class T>+T& Try<T>::value() & {+  throwUnlessValue();+  return this->value_;+}++template <class T>+T&& Try<T>::value() && {+  throwUnlessValue();+  return std::move(this->value_);+}++template <class T>+const T& Try<T>::value() const& {+  throwUnlessValue();+  return this->value_;+}++template <class T>+const T&& Try<T>::value() const&& {+  throwUnlessValue();+  return std::move(this->value_);+}++template <class T>+template <class U>+T Try<T>::value_or(U&& defaultValue) const& {+  return hasValue() ? **this : static_cast<T>(static_cast<U&&>(defaultValue));+}++template <class T>+template <class U>+T Try<T>::value_or(U&& defaultValue) && {+  return hasValue()+      ? std::move(**this)+      : static_cast<T>(static_cast<U&&>(defaultValue));+}++template <class T>+void Try<T>::throwUnlessValue() const {+  switch (this->contains_) {+    case Contains::VALUE:+      return;+    case Contains::EXCEPTION:+      this->e_.throw_exception();+    case Contains::NOTHING:+    default:+      throw_exception<UsingUninitializedTry>();+  }+}++template <class T>+void Try<T>::throwIfFailed() const {+  throwUnlessValue();+}++Try<void>& Try<void>::operator=(const Try<void>& t) noexcept {+  if (t.hasException()) {+    if (hasException()) {+      this->e_ = t.e_;+    } else {+      new (&this->e_) exception_wrapper(t.e_);+      hasValue_ = false;+    }+  } else {+    if (hasException()) {+      this->e_.~exception_wrapper();+      hasValue_ = true;+    }+  }+  return *this;+}++template <typename... Args>+exception_wrapper& Try<void>::emplaceException(Args&&... args) noexcept(+    std::is_nothrow_constructible<exception_wrapper, Args&&...>::value) {+  if (hasException()) {+    this->e_.~exception_wrapper();+  }+  new (&this->e_) exception_wrapper(static_cast<Args&&>(args)...);+  hasValue_ = false;+  return this->e_;+}++void Try<void>::throwIfFailed() const {+  if (hasException()) {+    this->e_.throw_exception();+  }+}++void Try<void>::throwUnlessValue() const {+  throwIfFailed();+}++template <typename F>+typename std::enable_if<+    !std::is_same<invoke_result_t<F>, void>::value,+    Try<invoke_result_t<F>>>::type+makeTryWithNoUnwrap(F&& f) noexcept {+  using ResultType = invoke_result_t<F>;+  try {+    return Try<ResultType>(f());+  } catch (...) {+    return Try<ResultType>(exception_wrapper(current_exception()));+  }+}++template <typename F>+typename std::+    enable_if<std::is_same<invoke_result_t<F>, void>::value, Try<void>>::type+    makeTryWithNoUnwrap(F&& f) noexcept {+  try {+    f();+    return Try<void>();+  } catch (...) {+    return Try<void>(exception_wrapper(current_exception()));+  }+}++template <typename F>+typename std::+    enable_if<!isTry<invoke_result_t<F>>::value, Try<invoke_result_t<F>>>::type+    makeTryWith(F&& f) noexcept {+  return makeTryWithNoUnwrap(std::forward<F>(f));+}++template <typename F>+typename std::enable_if<isTry<invoke_result_t<F>>::value, invoke_result_t<F>>::+    type+    makeTryWith(F&& f) noexcept {+  using ResultType = invoke_result_t<F>;+  try {+    return f();+  } catch (...) {+    return ResultType(exception_wrapper(current_exception()));+  }+}++template <typename T, typename... Args>+T* tryEmplace(Try<T>& t, Args&&... args) noexcept {+  try {+    return std::addressof(t.emplace(static_cast<Args&&>(args)...));+  } catch (...) {+    t.emplaceException(current_exception());+    return nullptr;+  }+}++void tryEmplace(Try<void>& t) noexcept {+  t.emplace();+}++template <typename T, typename Func>+T* tryEmplaceWith(Try<T>& t, Func&& func) noexcept {+  static_assert(+      std::is_constructible<T, folly::invoke_result_t<Func>>::value,+      "Unable to initialise a value of type T with the result of 'func'");+  try {+    return std::addressof(t.emplace(static_cast<Func&&>(func)()));+  } catch (...) {+    t.emplaceException(current_exception());+    return nullptr;+  }+}++template <typename Func>+bool tryEmplaceWith(Try<void>& t, Func&& func) noexcept {+  static_assert(+      std::is_void<folly::invoke_result_t<Func>>::value,+      "Func returns non-void. Cannot be used to emplace Try<void>");+  try {+    static_cast<Func&&>(func)();+    t.emplace();+    return true;+  } catch (...) {+    t.emplaceException(current_exception());+    return false;+  }+}++namespace try_detail {++/**+ * Trait that removes the layer of Try abstractions from the passed in type+ */+template <typename Type>+struct RemoveTry;+template <template <typename...> class TupleType, typename... Types>+struct RemoveTry<TupleType<folly::Try<Types>...>> {+  using type = TupleType<Types...>;+};++template <std::size_t... Indices, typename Tuple>+auto unwrapTryTupleImpl(std::index_sequence<Indices...>, Tuple&& instance) {+  using std::get;+  using ReturnType = typename RemoveTry<typename std::decay<Tuple>::type>::type;+  return ReturnType{(get<Indices>(std::forward<Tuple>(instance)).value())...};+}+} // namespace try_detail++template <typename Tuple>+auto unwrapTryTuple(Tuple&& instance) {+  using TupleDecayed = typename std::decay<Tuple>::type;+  using Seq = std::make_index_sequence<std::tuple_size<TupleDecayed>::value>;+  return try_detail::unwrapTryTupleImpl(Seq{}, std::forward<Tuple>(instance));+}++template <typename T>+void tryAssign(Try<T>& t, Try<T>&& other) noexcept {+  try {+    t = std::move(other);+  } catch (...) {+    t.emplaceException(current_exception());+  }+}++// limited to the instances unconditionally forced by the futures library+extern template class Try<Unit>;++} // namespace folly
+ folly/folly/Try.cpp view
@@ -0,0 +1,23 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#include <folly/Try.h>++namespace folly {++template class Try<Unit>;++} // namespace folly
+ folly/folly/Try.h view
@@ -0,0 +1,729 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <exception>+#include <stdexcept>+#include <type_traits>+#include <utility>++#include <folly/ExceptionWrapper.h>+#include <folly/Likely.h>+#include <folly/Memory.h>+#include <folly/Portability.h>+#include <folly/Unit.h>+#include <folly/Utility.h>+#include <folly/functional/Invoke.h>+#include <folly/lang/Exception.h>++namespace folly {++class FOLLY_EXPORT TryException : public std::logic_error {+ public:+  using std::logic_error::logic_error;+  TryException() : std::logic_error{""} {}+};++class FOLLY_EXPORT UsingUninitializedTry : public TryException {+ public:+  UsingUninitializedTry() = default;+  char const* what() const noexcept override {+    return "Using uninitialized try";+  }+};++template <class T>+class Try;++namespace detail {+template <class T>+class TryBase {+ protected:+  enum class Contains {+    VALUE,+    EXCEPTION,+    NOTHING,+  };++ public:+  /*+   * Construct an empty Try+   */+  TryBase() noexcept : contains_(Contains::NOTHING) {}++  /*+   * Construct a Try with a value by copy+   *+   * @param v The value to copy in+   */+  explicit TryBase(const T& v) noexcept(+      std::is_nothrow_copy_constructible<T>::value)+      : contains_(Contains::VALUE), value_(v) {}++  /*+   * Construct a Try with a value by move+   *+   * @param v The value to move in+   */+  explicit TryBase(T&& v) noexcept(std::is_nothrow_move_constructible<T>::value)+      : contains_(Contains::VALUE), value_(std::move(v)) {}++  template <typename... Args>+  explicit TryBase(std::in_place_t, Args&&... args) noexcept(+      std::is_nothrow_constructible<T, Args&&...>::value)+      : contains_(Contains::VALUE), value_(static_cast<Args&&>(args)...) {}++  /// Implicit conversion from Try<void> to Try<Unit>+  template <class T2 = T>+  /* implicit */ TryBase(+      typename std::enable_if<std::is_same<Unit, T2>::value, Try<void> const&>::+          type t) noexcept;++  /*+   * Construct a Try with an exception_wrapper+   *+   * @param e The exception_wrapper+   */+  explicit TryBase(exception_wrapper e) noexcept+      : contains_(Contains::EXCEPTION), e_(std::move(e)) {}++  ~TryBase();++  // Move constructor+  TryBase(TryBase&& t) noexcept(std::is_nothrow_move_constructible<T>::value);+  // Move assigner+  TryBase& operator=(TryBase&& t) noexcept(+      std::is_nothrow_move_constructible<T>::value);++  // Copy constructor+  TryBase(const TryBase& t) noexcept(+      std::is_nothrow_copy_constructible<T>::value);+  // Copy assigner+  TryBase& operator=(const TryBase& t) noexcept(+      std::is_nothrow_copy_constructible<T>::value);++ protected:+  void destroy() noexcept;++  Contains contains_;+  union {+    T value_;+    exception_wrapper e_;+  };+};+} // namespace detail++/*+ * Try<T> is a wrapper that contains either an instance of T, an exception, or+ * nothing. Exceptions are stored as exception_wrappers so that the user can+ * minimize rethrows if so desired.+ *+ * To represent success or a captured exception, use Try<Unit>.+ */+template <class T>+class Try+    : detail::TryBase<T>,+      moveonly_::EnableCopyMove<+          std::is_copy_constructible<T>::value,+          std::is_move_constructible<T>::value> {+  static_assert(+      !std::is_reference<T>::value, "Try may not be used with reference types");+  using typename detail::TryBase<T>::Contains;++ public:+  using detail::TryBase<T>::TryBase;++  /*+   * The value type for the Try+   */+  using element_type = T;++  /*+   * In-place construct the value in the Try object.+   *+   * Destroys any previous value prior to constructing the new value.+   * Leaves *this in an empty state if the construction of T throws.+   *+   * @returns reference to the newly constructed value.+   */+  template <typename... Args>+  T& emplace(Args&&... args) noexcept(+      std::is_nothrow_constructible<T, Args&&...>::value);++  /*+   * In-place construct an exception in the Try object.+   *+   * Destroys any previous value prior to constructing the new value.+   * Leaves *this in an empty state if the construction of the exception_wrapper+   * throws.+   *+   * Any arguments passed to emplaceException() are forwarded on to the+   * exception_wrapper constructor.+   *+   * @returns reference to the newly constructed exception_wrapper.+   */+  template <typename... Args>+  exception_wrapper& emplaceException(Args&&... args) noexcept(+      std::is_nothrow_constructible<exception_wrapper, Args&&...>::value);++  /*+   * Get a mutable reference to the contained value.+   * [Re]throws if the Try contains an exception or is empty.+   *+   * @returns mutable reference to the contained value+   */+  T& value() &;+  /*+   * Get a rvalue reference to the contained value.+   * [Re]throws if the Try contains an exception or is empty.+   *+   * @returns rvalue reference to the contained value+   */+  T&& value() &&;+  /*+   * Get a const reference to the contained value.+   * [Re]throws if the Try contains an exception or is empty.+   *+   * @returns const reference to the contained value+   */+  const T& value() const&;+  /*+   * Get a const rvalue reference to the contained value.+   * [Re]throws if the Try contains an exception or is empty.+   *+   * @returns const rvalue reference to the contained value+   */+  const T&& value() const&&;++  /*+   * Returns a copy of the contained value if *this has a value,+   * otherwise returns a value constructed from defaultValue.+   *+   * The selected constructor of the return value may throw exceptions.+   */+  template <class U>+  T value_or(U&& defaultValue) const&;+  template <class U>+  T value_or(U&& defaultValue) &&;++  /*+   * [Re]throw if the Try contains an exception or is empty. Otherwise do+   * nothing.+   */+  void throwUnlessValue() const;+  [[deprecated("Replaced by throwUnlessValue")]] void throwIfFailed() const;++  /*+   * Const dereference operator.+   * [Re]throws if the Try contains an exception or is empty.+   *+   * @returns const reference to the contained value+   */+  const T& operator*() const& { return value(); }+  /*+   * Dereference operator. If the Try contains an exception it will be rethrown.+   *+   * @returns mutable reference to the contained value+   */+  T& operator*() & { return value(); }+  /*+   * Mutable rvalue dereference operator.  If the Try contains an exception it+   * will be rethrown.+   *+   * @returns rvalue reference to the contained value+   */+  T&& operator*() && { return std::move(value()); }+  /*+   * Const rvalue dereference operator.  If the Try contains an exception it+   * will be rethrown.+   *+   * @returns rvalue reference to the contained value+   */+  const T&& operator*() const&& { return std::move(value()); }++  /*+   * Const arrow operator.+   * [Re]throws if the Try contains an exception or is empty.+   *+   * @returns const reference to the contained value+   */+  const T* operator->() const { return &value(); }+  /*+   * Arrow operator. If the Try contains an exception it will be rethrown.+   *+   * @returns mutable reference to the contained value+   */+  T* operator->() { return &value(); }++  /*+   * @returns True if the Try contains a value, false otherwise+   */+  bool hasValue() const noexcept { return this->contains_ == Contains::VALUE; }+  /*+   * @returns True if the Try contains an exception, false otherwise+   */+  bool hasException() const noexcept {+    return this->contains_ == Contains::EXCEPTION;+  }++  /*+   * @returns True if the Try contains an exception of type Ex, false otherwise+   */+  template <class Ex>+  bool hasException() const noexcept {+    return hasException() && this->e_.template is_compatible_with<Ex>();+  }++  exception_wrapper& exception() & {+    if (!hasException()) {+      throw_exception<TryException>("Try does not contain an exception");+    }+    return this->e_;+  }++  exception_wrapper&& exception() && {+    if (!hasException()) {+      throw_exception<TryException>("Try does not contain an exception");+    }+    return std::move(this->e_);+  }++  const exception_wrapper& exception() const& {+    if (!hasException()) {+      throw_exception<TryException>("Try does not contain an exception");+    }+    return this->e_;+  }++  const exception_wrapper&& exception() const&& {+    if (!hasException()) {+      throw_exception<TryException>("Try does not contain an exception");+    }+    return std::move(this->e_);+  }++  /*+   * @returns a pointer to the `std::exception` held by `*this`, if one is held;+   *          otherwise, returns `nullptr`.+   */+  std::exception* tryGetExceptionObject() noexcept {+    return hasException() ? this->e_.get_exception() : nullptr;+  }+  std::exception const* tryGetExceptionObject() const noexcept {+    return hasException() ? this->e_.get_exception() : nullptr;+  }++  /*+   * @returns a pointer to the `Ex` held by `*this`, if it holds an object whose+   *          type `From` permits `std::is_convertible<From*, Ex*>`; otherwise,+   *          returns `nullptr`.+   */+  template <class Ex>+  Ex* tryGetExceptionObject() noexcept {+    return hasException() ? this->e_.template get_exception<Ex>() : nullptr;+  }+  template <class Ex>+  Ex const* tryGetExceptionObject() const noexcept {+    return hasException() ? this->e_.template get_exception<Ex>() : nullptr;+  }++  /*+   * If the Try contains an exception and it is of type Ex, execute func(Ex)+   *+   * @param func a function that takes a single parameter of type const Ex&+   *+   * @returns True if the Try held an Ex and func was executed, false otherwise+   */+  template <class Ex, class F>+  bool withException(F func) {+    if (!hasException()) {+      return false;+    }+    return this->e_.template with_exception<Ex>(std::move(func));+  }+  template <class Ex, class F>+  bool withException(F func) const {+    if (!hasException()) {+      return false;+    }+    return this->e_.template with_exception<Ex>(std::move(func));+  }++  /*+   * If the Try contains an exception and it is of type compatible with Ex as+   * deduced from the first parameter of func, execute func(Ex)+   *+   * @param func a function that takes a single parameter of type const Ex&+   *+   * @returns True if the Try held an Ex and func was executed, false otherwise+   */+  template <class F>+  bool withException(F func) {+    if (!hasException()) {+      return false;+    }+    return this->e_.with_exception(std::move(func));+  }+  template <class F>+  bool withException(F func) const {+    if (!hasException()) {+      return false;+    }+    return this->e_.with_exception(std::move(func));+  }++  template <bool isTry, typename R>+  typename std::enable_if<isTry, R>::type get() {+    return std::forward<R>(*this);+  }++  template <bool isTry, typename R>+  typename std::enable_if<!isTry, R>::type get() {+    return std::forward<R>(value());+  }+};++/*+ * Specialization of Try for void value type. Encapsulates either success or an+ * exception.+ */+template <>+class Try<void> {+ public:+  /*+   * The value type for the Try+   */+  typedef void element_type;++  // Construct a Try holding a successful and void result+  Try() noexcept : hasValue_(true) {}++  /*+   * Construct a Try with an exception_wrapper+   *+   * @param e The exception_wrapper+   */+  explicit Try(exception_wrapper e) noexcept+      : hasValue_(false), e_(std::move(e)) {}++  /// Implicit conversion from Try<Unit> to Try<void>+  /* implicit */ inline Try(const Try<Unit>& t) noexcept;++  // Copy assigner+  inline Try& operator=(const Try<void>& t) noexcept;++  // Copy constructor+  Try(const Try<void>& t) noexcept : hasValue_(t.hasValue_) {+    if (t.hasException()) {+      new (&e_) exception_wrapper(t.e_);+    }+  }++  ~Try() {+    if (hasException()) {+      e_.~exception_wrapper();+    }+  }++  /*+   * In-place construct a 'void' value into this Try object.+   *+   * This has the effect of clearing any existing exception stored in the+   * Try object.+   */+  void emplace() noexcept {+    if (hasException()) {+      e_.~exception_wrapper();+      hasValue_ = true;+    }+  }++  /*+   * In-place construct an exception in the Try object.+   *+   * Destroys any previous value prior to constructing the new value.+   * Leaves *this in an empty state if the construction of the exception_wrapper+   * throws.+   *+   * Any arguments passed to emplaceException() are forwarded on to the+   * exception_wrapper constructor.+   *+   * @returns reference to the newly constructed exception_wrapper.+   */+  template <typename... Args>+  exception_wrapper& emplaceException(Args&&... args) noexcept(+      std::is_nothrow_constructible<exception_wrapper, Args&&...>::value);++  // If the Try contains an exception, throws it+  void value() const { throwIfFailed(); }+  // Dereference operator. If the Try contains an exception, throws it+  void operator*() const { return value(); }++  // If the Try contains an exception, throws it+  inline void throwIfFailed() const;+  inline void throwUnlessValue() const;++  // @returns False if the Try contains an exception, true otherwise+  bool hasValue() const noexcept { return hasValue_; }+  // @returns True if the Try contains an exception, false otherwise+  bool hasException() const noexcept { return !hasValue_; }++  // @returns True if the Try contains an exception of type Ex, false otherwise+  template <class Ex>+  bool hasException() const noexcept {+    return hasException() && e_.is_compatible_with<Ex>();+  }++  /*+   * @throws TryException if the Try doesn't contain an exception+   *+   * @returns mutable reference to the exception contained by this Try+   */+  exception_wrapper& exception() & {+    if (!hasException()) {+      throw_exception<TryException>("Try does not contain an exception");+    }+    return e_;+  }++  exception_wrapper&& exception() && {+    if (!hasException()) {+      throw_exception<TryException>("Try does not contain an exception");+    }+    return std::move(e_);+  }++  const exception_wrapper& exception() const& {+    if (!hasException()) {+      throw_exception<TryException>("Try does not contain an exception");+    }+    return e_;+  }++  const exception_wrapper&& exception() const&& {+    if (!hasException()) {+      throw_exception<TryException>("Try does not contain an exception");+    }+    return std::move(e_);+  }++  /*+   * @returns a pointer to the `std::exception` held by `*this`, if one is held;+   *          otherwise, returns `nullptr`.+   */+  std::exception* tryGetExceptionObject() noexcept {+    return hasException() ? e_.get_exception() : nullptr;+  }+  std::exception const* tryGetExceptionObject() const noexcept {+    return hasException() ? e_.get_exception() : nullptr;+  }++  /*+   * @returns a pointer to the `Ex` held by `*this`, if it holds an object whose+   *          type `From` permits `std::is_convertible<From*, Ex*>`; otherwise,+   *          returns `nullptr`.+   */+  template <class E>+  E* tryGetExceptionObject() noexcept {+    return hasException() ? e_.get_exception<E>() : nullptr;+  }+  template <class E>+  E const* tryGetExceptionObject() const noexcept {+    return hasException() ? e_.get_exception<E>() : nullptr;+  }++  /*+   * If the Try contains an exception and it is of type Ex, execute func(Ex)+   *+   * @param func a function that takes a single parameter of type const Ex&+   *+   * @returns True if the Try held an Ex and func was executed, false otherwise+   */+  template <class Ex, class F>+  bool withException(F func) {+    if (!hasException()) {+      return false;+    }+    return e_.with_exception<Ex>(std::move(func));+  }+  template <class Ex, class F>+  bool withException(F func) const {+    if (!hasException()) {+      return false;+    }+    return e_.with_exception<Ex>(std::move(func));+  }++  /*+   * If the Try contains an exception and it is of type compatible with Ex as+   * deduced from the first parameter of func, execute func(Ex)+   *+   * @param func a function that takes a single parameter of type const Ex&+   *+   * @returns True if the Try held an Ex and func was executed, false otherwise+   */+  template <class F>+  bool withException(F func) {+    if (!hasException()) {+      return false;+    }+    return e_.with_exception(std::move(func));+  }+  template <class F>+  bool withException(F func) const {+    if (!hasException()) {+      return false;+    }+    return e_.with_exception(std::move(func));+  }++  template <bool, typename R>+  R get() {+    return std::forward<R>(*this);+  }++ private:+  bool hasValue_;+  union {+    exception_wrapper e_;+  };+};++template <typename T>+struct isTry : std::false_type {};++template <typename T>+struct isTry<Try<T>> : std::true_type {};++/*+ * @param f a function to execute and capture the result of (value or exception)+ *+ * @returns Try holding the result of f+ */+template <typename F>+typename std::enable_if<+    !std::is_same<invoke_result_t<F>, void>::value,+    Try<invoke_result_t<F>>>::type+makeTryWithNoUnwrap(F&& f) noexcept;++/*+ * Specialization of makeTryWith for void return+ *+ * @param f a function to execute and capture the result of+ *+ * @returns Try<void> holding the result of f+ */+template <typename F>+typename std::+    enable_if<std::is_same<invoke_result_t<F>, void>::value, Try<void>>::type+    makeTryWithNoUnwrap(F&& f) noexcept;++/*+ * @param f a function to execute and capture the result of (value or exception)+ *+ * @returns Try holding the result of f+ */+template <typename F>+typename std::+    enable_if<!isTry<invoke_result_t<F>>::value, Try<invoke_result_t<F>>>::type+    makeTryWith(F&& f) noexcept;++/*+ * Specialization of makeTryWith for functions that return Try<T>+ * Causes makeTryWith to not double-wrap the try.+ *+ * @param f a function to execute and capture the result of+ *+ * @returns result of f if f did not throw. Otherwise Try<T> containing+ * exception+ */+template <typename F>+typename std::enable_if<isTry<invoke_result_t<F>>::value, invoke_result_t<F>>::+    type+    makeTryWith(F&& f) noexcept;++/*+ * Try to in-place construct a new value from the specified arguments.+ *+ * If T's constructor throws an exception then this is caught and the Try<T>+ * object is initialised to hold that exception.+ *+ * @param args Are passed to T's constructor.+ */+template <typename T, typename... Args>+T* tryEmplace(Try<T>& t, Args&&... args) noexcept;++/*+ * Overload of tryEmplace() for Try<void>.+ */+inline void tryEmplace(Try<void>& t) noexcept;++/*+ * Try to in-place construct a new value from the result of a function.+ *+ * If the function completes successfully then attempts to in-place construct+ * a value of type, T, passing the result of the function as the only parameter.+ *+ * If either the call to the function completes with an exception or the+ * constructor completes with an exception then the exception is caught and+ * stored in the Try object.+ *+ * @returns A pointer to the newly constructed object if it completed+ * successfully, otherwise returns nullptr if the operation completed with+ * an exception.+ */+template <typename T, typename Func>+T* tryEmplaceWith(Try<T>& t, Func&& func) noexcept;++/*+ * Specialization of tryEmplaceWith() for Try<void>.+ *+ * Calls func() and if it doesn't throw an exception then calls t.emplace().+ * If func() throws then captures the exception in t using t.emplaceException().+ *+ * Func must be callable with zero parameters and must return void.+ *+ * @returns true if func() completed without an exception, false if func()+ * threw an exception.+ */+template <typename Func>+bool tryEmplaceWith(Try<void>& t, Func&& func) noexcept;++/**+ * Tuple<Try<Type>...> -> std::tuple<Type...>+ *+ * Unwraps a tuple-like type containing a sequence of Try<Type> instances to+ * std::tuple<Type>+ */+template <typename Tuple>+auto unwrapTryTuple(Tuple&&);++/*+ * Try to move the value/exception from another Try object.+ *+ * If T's constructor throws an exception then this is caught and the Try<T>+ * object is initialised to hold that exception.+ */+template <typename T>+void tryAssign(Try<T>& t, Try<T>&& other) noexcept;++template <typename T>+Try(T&&) -> Try<std::decay_t<T>>;++} // namespace folly++#include <folly/Try-inl.h>
+ folly/folly/UTF8String.h view
@@ -0,0 +1,56 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <string>++#include <boost/regex/pending/unicode_iterator.hpp>++#include <folly/Range.h>++namespace folly {++class UTF8StringPiece {+ public:+  using iterator = boost::u8_to_u32_iterator<const char*>;+  using size_type = std::size_t;++  /* implicit */ UTF8StringPiece(const folly::StringPiece piece)+      : begin_{piece.begin(), piece.begin(), piece.end()},+        end_{piece.end(), piece.begin(), piece.end()} {}+  template <+      typename T,+      std::enable_if_t<std::is_convertible_v<T, folly::StringPiece>, int> = 0>+  /* implicit */ UTF8StringPiece(const T& t)+      : UTF8StringPiece(folly::StringPiece(t)) {}++  iterator begin() const noexcept { return begin_; }+  iterator cbegin() const noexcept { return begin_; }+  iterator end() const noexcept { return end_; }+  iterator cend() const noexcept { return end_; }++  bool empty() const noexcept { return begin_ == end_; }+  size_type walk_size() const {+    return static_cast<size_type>(std::distance(begin_, end_));+  }++ private:+  iterator begin_;+  iterator end_;+};++} // namespace folly
+ folly/folly/Unicode.cpp view
@@ -0,0 +1,201 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#include <folly/Unicode.h>++#include <initializer_list>++#include <folly/Conv.h>++namespace folly {++namespace {++template <class F>+void codePointToUtf8Impl(char32_t cp, F&& f) {+  // Based on description from http://en.wikipedia.org/wiki/UTF-8.++  if (cp <= 0x7f) {+    f({+        static_cast<char>(cp),+    });+  } else if (cp <= 0x7FF) {+    f({+        static_cast<char>(0xC0 | (cp >> 6)),+        static_cast<char>(0x80 | (0x3f & cp)),+    });+  } else if (cp <= 0xFFFF) {+    f({+        static_cast<char>(0xE0 | (cp >> 12)),+        static_cast<char>(0x80 | (0x3f & (cp >> 6))),+        static_cast<char>(0x80 | (0x3f & cp)),+    });+  } else if (cp <= 0x10FFFF) {+    f({+        static_cast<char>(0xF0 | (cp >> 18)),+        static_cast<char>(0x80 | (0x3f & (cp >> 12))),+        static_cast<char>(0x80 | (0x3f & (cp >> 6))),+        static_cast<char>(0x80 | (0x3f & cp)),+    });+  }+}++} // namespace++std::string codePointToUtf8(char32_t cp) {+  std::string result;+  codePointToUtf8Impl(cp, [&](std::initializer_list<char> data) {+    result.assign(data.begin(), data.end());+  });+  return result;+}++void appendCodePointToUtf8(char32_t cp, std::string& out) {+  codePointToUtf8Impl(cp, [&](std::initializer_list<char> data) {+    out.append(data.begin(), data.end());+  });+}++char32_t utf8ToCodePoint(+    const unsigned char*& p, const unsigned char* const e, bool skipOnError) {+  // clang-format off+  /** UTF encodings+  *  | # of B | First CP |  Last CP  | Bit Pattern+  *  |   1    |   0x0000 |   0x007F  | 0xxxxxxx+  *  |   2    |   0x0080 |   0x07FF  | 110xxxxx 10xxxxxx+  *  |   3    |   0x0800 |   0xFFFF  | 1110xxxx 10xxxxxx 10xxxxxx+  *  |   4    |  0x10000 | 0x10FFFF  | 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx+  *  |   5    |       -  |        -  | 111110xx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx+  *  |   6    |       -  |        -  | 1111110x 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx+  *+  *+  * NOTE:+  * - the 4B encoding can encode values up to 0x1FFFFF,+  *   but Unicode defines 0x10FFFF to be the largest code point+  * - the 5B & 6B encodings will all encode values larger than 0x1FFFFF+  *   (so larger than the largest code point value 0x10FFFF) so they form invalid+  *   Unicode code points+  *+  * On invalid input (invalid encoding or code points larger than 0x10FFFF):+  * - When skipOnError is true, this function will skip the first byte and return+  *   U'\ufffd'. Potential optimization: skip the whole invalid range.+  * - When skipOnError is false, throws.+  */+  // clang-format on++  const auto skip = [&] {+    ++p;+    return U'\ufffd';+  };++  if (p >= e) {+    if (skipOnError) {+      return skip();+    }+    throw std::runtime_error("folly::utf8ToCodePoint empty/invalid string");+  }++  unsigned char fst = *p;+  if (!(fst & 0x80)) {+    // trivial case, 1 byte encoding+    return *p++;+  }++  static const uint32_t bitMask[] = {+      (1 << 7) - 1,+      (1 << 11) - 1,+      (1 << 16) - 1,+      (1 << 21) - 1,+  };++  // upper control bits are masked out later+  uint32_t d = fst;++  // multi-byte encoded values must start with bits 0b11. 0xC0 is 0b11000000+  if ((fst & 0xC0) != 0xC0) {+    if (skipOnError) {+      return skip();+    }+    throw std::runtime_error(+        to<std::string>("folly::utf8ToCodePoint i=0 d=", d));+  }++  fst <<= 1;++  for (unsigned int i = 1; i != 4 && p + i < e; ++i) {+    const unsigned char tmp = p[i];++    // from the second byte on, format should be 10xxxxxx+    if ((tmp & 0xC0) != 0x80) {+      if (skipOnError) {+        return skip();+      }+      throw std::runtime_error(to<std::string>(+          "folly::utf8ToCodePoint i=", i, " tmp=", (uint32_t)tmp));+    }++    // gradually fill a 32 bit integer d with non control bits in tmp+    // 0x3F is 0b00111111 which clears out the first 2 control bits+    d = (d << 6) | (tmp & 0x3F);+    fst <<= 1;++    if (!(fst & 0x80)) {+      // We know the length of encoding now, since we encounter the first "0" in+      // fst (the first byte). This branch processes the last byte of encoding.+      d &= bitMask[i]; // d is now the code point++      // overlong, could have been encoded with i bytes+      if ((d & ~bitMask[i - 1]) == 0) {+        if (skipOnError) {+          return skip();+        }+        throw std::runtime_error(+            to<std::string>("folly::utf8ToCodePoint i=", i, " d=", d));+      }++      // check for surrogates only needed for 3 bytes+      if (i == 2) {+        if (d >= 0xD800 && d <= 0xDFFF) {+          if (skipOnError) {+            return skip();+          }+          throw std::runtime_error(+              to<std::string>("folly::utf8ToCodePoint i=", i, " d=", d));+        }+      }++      // While UTF-8 encoding can encode arbitrary numbers, 0x10FFFF is the+      // largest defined Unicode code point.+      // Only >=4 bytes can UTF-8 encode such values, so i=3 here.+      if (d > 0x10FFFF) {+        if (skipOnError) {+          return skip();+        }+        throw std::runtime_error(+            "folly::utf8ToCodePoint encoding exceeds max unicode code point");+      }+      p += i + 1;+      return d;+    }+  }++  if (skipOnError) {+    return skip();+  }+  throw std::runtime_error("folly::utf8ToCodePoint encoding length maxed out");+}++} // namespace folly
+ folly/folly/Unicode.h view
@@ -0,0 +1,94 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++// Some utility routines relating to Unicode.++#pragma once++#include <cstdint>+#include <stdexcept>+#include <string>++#include <folly/lang/Exception.h>++namespace folly {++class FOLLY_EXPORT unicode_error : public std::runtime_error {+ public:+  using std::runtime_error::runtime_error;+};++//  Unicode code points are split into 17 planes.+//+//  The Basic Multilingual Plane covers code points in [0-0xFFFF] but reserves+//  two invalid ranges:+//  - High surrogates: [0xD800-0xDBFF].+//  - Low surrogates: [0xDC00-0xDFFF].+//+//  UTF-16 code units are 2 bytes wide and are represented here with char16_t.+//  Unicode code points are represented in UTF-16 across either 1-2 code units:+//  - Valid BMP code points [0x0000-0xD7FF] + [0xE000-0xFFFF] are encoded+//    directly as 1 code unit.+//  - Code points larger than BMP (>0xFFFF) are encoded as 2 code units, with+//    values respectively in the high surrogates and low surrogates ranges.+//+//  JSON text permits the inclusion of Unicode escape sequences within quoted+//  strings:+//  - Valid BMP code points are encoded as \xXXXX, where XXXX are the base-16+//    digits of the code point.+//  - Code points larger than BMP are encoded as \uHHHH\uLLLL, where HHHH and+//    LLLL are respectively the base-16 digits of the high and low surrogates of+//    the UTF-16 encoding of the code point.++inline bool utf16_code_unit_is_bmp(char16_t const c) {+  return c < 0xd800 || c >= 0xe000;+}+inline bool utf16_code_unit_is_high_surrogate(char16_t const c) {+  return c >= 0xd800 && c < 0xdc00;+}+inline bool utf16_code_unit_is_low_surrogate(char16_t const c) {+  return c >= 0xdc00 && c < 0xe000;+}+inline char32_t unicode_code_point_from_utf16_surrogate_pair(+    char16_t const high, char16_t const low) {+  if (!utf16_code_unit_is_high_surrogate(high)) {+    throw_exception<unicode_error>("invalid high surrogate");+  }+  if (!utf16_code_unit_is_low_surrogate(low)) {+    throw_exception<unicode_error>("invalid low surrogate");+  }+  return 0x10000 + ((char32_t(high) & 0x3ff) << 10) + (char32_t(low) & 0x3ff);+}++//////////////////////////////////////////////////////////////////////++/*+ * Encode a single Unicode code point into a UTF-8 byte sequence.+ *+ * Result is undefined if `cp' is an invalid code point.+ */+std::string codePointToUtf8(char32_t cp);+void appendCodePointToUtf8(char32_t cp, std::string& out);++/*+ * Decode a single Unicode code point from UTF-8 byte sequence.+ */+char32_t utf8ToCodePoint(+    const unsigned char*& p, const unsigned char* const e, bool skipOnError);++//////////////////////////////////////////////////////////////////////++} // namespace folly
+ folly/folly/Unit.h view
@@ -0,0 +1,65 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <type_traits>++namespace folly {++/// In functional programming, the degenerate case is often called "unit". In+/// C++, "void" is often the best analogue. However, because of the syntactic+/// special-casing required for void, it is frequently a liability for template+/// metaprogramming. So, instead of writing specializations to handle cases like+/// SomeContainer<void>, a library author may instead rule that out and simply+/// have library users use SomeContainer<Unit>. Contained values may be ignored.+/// Much easier.+///+/// "void" is the type that admits of no values at all. It is not possible to+/// construct a value of this type.+/// "unit" is the type that admits of precisely one unique value. It is+/// possible to construct a value of this type, but it is always the same value+/// every time, so it is uninteresting.+struct Unit {+  constexpr bool operator==(const Unit& /*other*/) const { return true; }+  constexpr bool operator!=(const Unit& /*other*/) const { return false; }+};++constexpr Unit unit{};++template <typename T>+struct lift_unit {+  using type = T;+};+template <>+struct lift_unit<void> {+  using type = Unit;+};+template <typename T>+using lift_unit_t = typename lift_unit<T>::type;++template <typename T>+struct drop_unit {+  using type = T;+};+template <>+struct drop_unit<Unit> {+  using type = void;+};+template <typename T>+using drop_unit_t = typename drop_unit<T>::type;++} // namespace folly
+ folly/folly/Uri-inl.h view
@@ -0,0 +1,101 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#ifndef FOLLY_URI_H_+#error This file may only be included from folly/Uri.h+#endif++#include <functional>+#include <tuple>++#include <folly/Conv.h>+#include <folly/hash/Hash.h>++namespace folly {++namespace uri_detail {++using UriTuple = std::tuple<+    const std::string&,+    const std::string&,+    const std::string&,+    const std::string&,+    uint16_t,+    const std::string&,+    const std::string&,+    const std::string&>;++inline UriTuple as_tuple(const folly::Uri& k) {+  return UriTuple(+      k.scheme(),+      k.username(),+      k.password(),+      k.host(),+      k.port(),+      k.path(),+      k.query(),+      k.fragment());+}++} // namespace uri_detail++template <class String>+String Uri::toString() const {+  String str;+  if (hasAuthority_) {+    toAppend(scheme_, "://", &str);+    if (!password_.empty()) {+      toAppend(username_, ":", password_, "@", &str);+    } else if (!username_.empty()) {+      toAppend(username_, "@", &str);+    }+    toAppend(host_, &str);+    if (port_ != 0) {+      toAppend(":", port_, &str);+    }+  } else {+    toAppend(scheme_, ":", &str);+  }+  toAppend(path_, &str);+  if (!query_.empty()) {+    toAppend("?", query_, &str);+  }+  if (!fragment_.empty()) {+    toAppend("#", fragment_, &str);+  }+  return str;+}++} // namespace folly++namespace std {++template <>+struct hash<folly::Uri> {+  std::size_t operator()(const folly::Uri& k) const {+    return std::hash<folly::uri_detail::UriTuple>{}(+        folly::uri_detail::as_tuple(k));+  }+};++template <>+struct equal_to<folly::Uri> {+  bool operator()(const folly::Uri& a, const folly::Uri& b) const {+    return folly::uri_detail::as_tuple(a) == folly::uri_detail::as_tuple(b);+  }+};++} // namespace std
+ folly/folly/Uri.cpp view
@@ -0,0 +1,190 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#include <folly/Uri.h>++#include <algorithm>+#include <cctype>++#include <boost/regex.hpp>++namespace folly {++namespace {++std::string submatch(const boost::cmatch& m, int idx) {+  const auto& sub = m[idx];+  return std::string(sub.first, sub.second);+}++} // namespace++// private default contructor+Uri::Uri() : hasAuthority_(false), port_(0) {}++// public string constructor+Uri::Uri(StringPiece str) : hasAuthority_(false), port_(0) {+  auto maybeUri = tryFromString(str);+  if (maybeUri.hasError()) {+    switch (maybeUri.error()) {+      case UriFormatError::INVALID_URI_AUTHORITY:+        throw std::invalid_argument(+            to<std::string>("invalid URI authority: ", str));+      case UriFormatError::INVALID_URI_PORT:+        throw std::invalid_argument(to<std::string>("invalid URI port: ", str));+      case UriFormatError::INVALID_URI:+      default:+        throw std::invalid_argument(to<std::string>("invalid URI: ", str));+    }+  }+  *this = maybeUri.value();+}++Expected<Uri, UriFormatError> Uri::tryFromString(StringPiece str) noexcept {+  Uri result;++  static const boost::regex uriRegex(+      "([a-zA-Z][a-zA-Z0-9+.-]*):" // scheme:+      "([^?#]*)" // authority and path+      "(?:\\?([^#]*))?" // ?query+      "(?:#(.*))?"); // #fragment+  static const boost::regex authorityAndPathRegex("//([^/]*)(/.*)?");++  boost::cmatch match;+  if (FOLLY_UNLIKELY(+          !boost::regex_match(str.begin(), str.end(), match, uriRegex))) {+    return makeUnexpected(UriFormatError::INVALID_URI);+  }++  result.scheme_ = submatch(match, 1);+  std::transform(+      result.scheme_.begin(),+      result.scheme_.end(),+      result.scheme_.begin(),+      ::tolower);++  StringPiece authorityAndPath(match[2].first, match[2].second);+  boost::cmatch authorityAndPathMatch;+  if (!boost::regex_match(+          authorityAndPath.begin(),+          authorityAndPath.end(),+          authorityAndPathMatch,+          authorityAndPathRegex)) {+    // Does not start with //, doesn't have authority+    result.hasAuthority_ = false;+    result.path_ = authorityAndPath.str();+  } else {+    static const boost::regex authorityRegex(+        "(?:([^@:]*)(?::([^@]*))?@)?" // username, password+        "(\\[[^\\]]*\\]|[^\\[:]*)" // host (IP-literal (e.g. '['+IPv6+']',+                                   // dotted-IPv4, or named host)+        "(?::(\\d*))?"); // port++    const auto authority = authorityAndPathMatch[1];+    boost::cmatch authorityMatch;+    if (!boost::regex_match(+            authority.first,+            authority.second,+            authorityMatch,+            authorityRegex)) {+      return makeUnexpected(UriFormatError::INVALID_URI_AUTHORITY);+    }++    StringPiece port(authorityMatch[4].first, authorityMatch[4].second);+    if (!port.empty()) {+      try {+        result.port_ = to<uint16_t>(port);+      } catch (ConversionError const&) {+        return makeUnexpected(UriFormatError::INVALID_URI_PORT);+      }+    }++    result.hasAuthority_ = true;+    result.username_ = submatch(authorityMatch, 1);+    result.password_ = submatch(authorityMatch, 2);+    result.host_ = submatch(authorityMatch, 3);+    result.path_ = submatch(authorityAndPathMatch, 2);+  }++  result.query_ = submatch(match, 3);+  result.fragment_ = submatch(match, 4);++  return result;+}++std::string Uri::authority() const {+  std::string result;++  // Port is 5 characters max and we have up to 3 delimiters.+  result.reserve(host().size() + username().size() + password().size() + 8);++  if (!username().empty() || !password().empty()) {+    result.append(username());++    if (!password().empty()) {+      result.push_back(':');+      result.append(password());+    }++    result.push_back('@');+  }++  result.append(host());++  if (port() != 0) {+    result.push_back(':');+    toAppend(port(), &result);+  }++  return result;+}++std::string Uri::hostname() const {+  if (!host_.empty() && host_[0] == '[') {+    // If it starts with '[', then it should end with ']', this is ensured by+    // regex+    return host_.substr(1, host_.size() - 2);+  }+  return host_;+}++const std::vector<std::pair<std::string, std::string>>& Uri::getQueryParams() {+  if (!query_.empty() && queryParams_.empty()) {+    // Parse query string+    static const boost::regex queryParamRegex(+        "(^|&)" /*start of query or start of parameter "&"*/+        "([^=&]*)=?" /*parameter name and "=" if value is expected*/+        "([^=&]*)" /*parameter value*/+        "(?=(&|$))" /*forward reference, next should be end of query or+                      start of next parameter*/);+    const boost::cregex_iterator paramBeginItr(+        query_.data(), query_.data() + query_.size(), queryParamRegex);+    boost::cregex_iterator paramEndItr;+    for (auto itr = paramBeginItr; itr != paramEndItr; ++itr) {+      if (itr->length(2) == 0) {+        // key is empty, ignore it+        continue;+      }+      queryParams_.emplace_back(+          std::string((*itr)[2].first, (*itr)[2].second), // parameter name+          std::string((*itr)[3].first, (*itr)[3].second) // parameter value+      );+    }+  }+  return queryParams_;+}++} // namespace folly
+ folly/folly/Uri.h view
@@ -0,0 +1,142 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once+#define FOLLY_URI_H_++#include <string>+#include <vector>++#include <folly/Expected.h>+#include <folly/String.h>++namespace folly {+/**+ * Error codes for parsing issues. Used by tryFromString()+ */+enum class UriFormatError {+  INVALID_URI,+  INVALID_URI_AUTHORITY,+  INVALID_URI_PORT,+};++/**+ * Class representing a URI.+ *+ * Consider http://www.facebook.com/foo/bar?key=foo#anchor+ *+ * The URI is broken down into its parts: scheme ("http"), authority+ * (ie. host and port, in most cases: "www.facebook.com"), path+ * ("/foo/bar"), query ("key=foo") and fragment ("anchor").  The scheme is+ * lower-cased.+ *+ * If this Uri represents a URL, note that, to prevent ambiguity, the component+ * parts are NOT percent-decoded; you should do this yourself with+ * uriUnescape() (for the authority and path) and uriUnescape(...,+ * UriEscapeMode::QUERY) (for the query, but probably only after splitting at+ * '&' to identify the individual parameters).+ */+class Uri {+ public:+  /**+   * Parse a Uri from a string.  Same as tryFromString except it throws+   * a std::invalid_argument if there's an error.+   */+  explicit Uri(StringPiece str);++  /**+   * Parse a Uri from a string.+   *+   * On failure, returns UriFormatError.+   */+  static Expected<Uri, UriFormatError> tryFromString(StringPiece str) noexcept;++  const std::string& scheme() const { return scheme_; }+  const std::string& username() const { return username_; }+  const std::string& password() const { return password_; }+  /**+   * Get host part of URI. If host is an IPv6 address, square brackets will be+   * returned, for example: "[::1]".+   */+  const std::string& host() const { return host_; }+  /**+   * Get host part of URI. If host is an IPv6 address, square brackets will not+   * be returned, for example "::1"; otherwise it returns the same thing as+   * host().+   *+   * hostname() is what one needs to call if passing the host to any other tool+   * or API that connects to that host/port; e.g. getaddrinfo() only understands+   * IPv6 host without square brackets+   */+  std::string hostname() const;+  uint16_t port() const { return port_; }+  const std::string& path() const { return path_; }+  const std::string& query() const { return query_; }+  const std::string& fragment() const { return fragment_; }++  std::string authority() const;++  template <class String>+  String toString() const;++  std::string str() const { return toString<std::string>(); }+  fbstring fbstr() const { return toString<fbstring>(); }++  void setPort(uint16_t port) {+    hasAuthority_ = true;+    port_ = port;+  }++  /**+   * Get query parameters as key-value pairs.+   * e.g. for URI containing query string:  key1=foo&key2=&key3&=bar&=bar=+   * In returned list, there are 3 entries:+   *     "key1" => "foo"+   *     "key2" => ""+   *     "key3" => ""+   * Parts "=bar" and "=bar=" are ignored, as they are not valid query+   * parameters. "=bar" is missing parameter name, while "=bar=" has more than+   * one equal signs, we don't know which one is the delimiter for key and+   * value.+   *+   * Note, this method is not thread safe, it might update internal state, but+   * only the first call to this method update the state. After the first call+   * is finished, subsequent calls to this method are thread safe.+   *+   * @return  query parameter key-value pairs in a vector, each element is a+   *          pair of which the first element is parameter name and the second+   *          one is parameter value+   */+  const std::vector<std::pair<std::string, std::string>>& getQueryParams();++ private:+  explicit Uri();++  std::string scheme_;+  std::string username_;+  std::string password_;+  std::string host_;+  bool hasAuthority_;+  uint16_t port_;+  std::string path_;+  std::string query_;+  std::string fragment_;+  std::vector<std::pair<std::string, std::string>> queryParams_;+};++} // namespace folly++#include <folly/Uri-inl.h>
+ folly/folly/Utility.h view
@@ -0,0 +1,945 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <cstdint>+#include <limits>+#include <type_traits>+#include <utility>++#include <folly/CPortability.h>+#include <folly/Portability.h>+#include <folly/Traits.h>++namespace folly {++/*+ * FOLLY_DECLVAL(T)+ *+ * This macro works like std::declval<T>() but does the same thing in a way+ * that does not require instantiating a function template.+ *+ * Use this macro instead of std::declval<T>() in places that are widely+ * instantiated to reduce compile-time overhead of instantiating function+ * templates.+ *+ * Note that, like std::declval<T>(), this macro can only be used in+ * unevaluated contexts.+ *+ * There are some small differences between this macro and std::declval<T>().+ * - This macro results in a value of type 'T' instead of 'T&&'.+ * - This macro requires the type T to be a complete type at the+ *   point of use.+ *   If this is a problem then use FOLLY_DECLVAL(T&&) instead, or if T might+ *   be 'void', then use FOLLY_DECLVAL(std::add_rvalue_reference_t<T>).+ */+#define FOLLY_DECLVAL(...) static_cast<__VA_ARGS__ (*)() noexcept>(nullptr)()++namespace detail {+template <typename T>+T decay_1_(T const volatile&&);+template <typename T>+T decay_1_(T const&);+template <typename T>+T* decay_1_(T*);++template <typename T>+auto decay_0_(int) -> decltype(detail::decay_1_(FOLLY_DECLVAL(T&&)));+template <typename T>+auto decay_0_(short) -> void;++template <typename T>+using decay_t = decltype(detail::decay_0_<T>(0));+} // namespace detail++//  decay_t+//+//  Like std::decay_t but possibly faster to compile.+//+//  mimic: std::decay_t, C++14+using detail::decay_t;++/**+ *  copy+ *+ *  Usable when you have a function with two overloads:+ *+ *      class MyData;+ *      void something(MyData&&);+ *      void something(const MyData&);+ *+ *  Where the purpose is to make copies and moves explicit without having to+ *  spell out the full type names - in this case, for copies, to invoke copy+ *  constructors.+ *+ *  When the caller wants to pass a copy of an lvalue, the caller may:+ *+ *      void foo() {+ *        MyData data;+ *        something(folly::copy(data)); // explicit copy+ *        something(std::move(data)); // explicit move+ *        something(data); // const& - neither move nor copy+ *      }+ *+ *  Note: If passed an rvalue, invokes the move-ctor, not the copy-ctor. This+ *  can be used to to force a move, where just using std::move would not:+ *+ *      folly::copy(std::move(data)); // force-move, not just a cast to &&+ *+ *  Note: The following text appears in the standard:+ *+ *      In several places in this Clause the operation //DECAY_COPY(x)// is+ *      used. All such uses mean call the function `decay_copy(x)` and use the+ *      result, where `decay_copy` is defined as follows:+ *+ *        template <class T> decay_t<T> decay_copy(T&& v)+ *          { return std::forward<T>(v); }+ *+ *      http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2014/n4296.pdf+ *        30.2.6 `decay_copy` [thread.decaycopy].+ *+ *  We mimic it, with a `noexcept` specifier for good measure.+ */++template <typename T>+constexpr detail::decay_t<T> copy(T&& value) noexcept(+    noexcept(detail::decay_t<T>(static_cast<T&&>(value)))) {+  return static_cast<T&&>(value);+}++//  mimic: `std::forward_like`, C++23 / p2445r0.+template <typename Src, typename Dst>+constexpr like_t<Src, Dst>&& forward_like(Dst&& dst) noexcept {+  return std::forward<like_t<Src, Dst>>(static_cast<Dst&&>(dst));+}++/**+ * Initializer lists are a powerful compile time syntax introduced in C++11+ * but due to their often conflicting syntax they are not used by APIs for+ * construction.+ *+ * Further standard conforming compilers *strongly* favor an+ * std::initializer_list overload for construction if one exists.  The+ * following is a simple tag used to disambiguate construction with+ * initializer lists and regular uniform initialization.+ *+ * For example consider the following case+ *+ *  class Something {+ *  public:+ *    explicit Something(int);+ *    Something(std::initializer_list<int>);+ *+ *    operator int();+ *  };+ *+ *  ...+ *  Something something{1}; // SURPRISE!!+ *+ * The last call to instantiate the Something object will go to the+ * initializer_list overload.  Which may be surprising to users.+ *+ * If however this tag was used to disambiguate such construction it would be+ * easy for users to see which construction overload their code was referring+ * to.  For example+ *+ *  class Something {+ *  public:+ *    explicit Something(int);+ *    Something(folly::initlist_construct_t, std::initializer_list<int>);+ *+ *    operator int();+ *  };+ *+ *  ...+ *  Something something_one{1}; // not the initializer_list overload+ *  Something something_two{folly::initlist_construct, {1}}; // correct+ */+struct initlist_construct_t {};+constexpr initlist_construct_t initlist_construct{};++//  sorted_unique_t, sorted_unique+//+//  A generic tag type and value to indicate that some constructor or method+//  accepts a container in which the values are sorted and unique.+//+//  Example:+//+//    void takes_numbers(folly::sorted_unique_t, std::vector<int> alist) {+//      assert(std::is_sorted(alist.begin(), alist.end()));+//      assert(std::unique(alist.begin(), alist.end()) == alist.end());+//      for (i : alist) {+//        // some behavior which safe only when alist is sorted and unique+//      }+//    }+//    void takes_numbers(std::vector<int> alist) {+//      std::sort(alist.begin(), alist.end());+//      alist.erase(std::unique(alist.begin(), alist.end()), alist.end());+//      takes_numbers(folly::sorted_unique, alist);+//    }+//+//  mimic: std::sorted_unique_t, std::sorted_unique, p0429r6+struct sorted_unique_t {};+constexpr sorted_unique_t sorted_unique{};++//  sorted_equivalent_t, sorted_equivalent+//+//  A generic tag type and value to indicate that some constructor or method+//  accepts a container in which the values are sorted but not necessarily+//  unique.+//+//  Example:+//+//    void takes_numbers(folly::sorted_equivalent_t, std::vector<int> alist) {+//      assert(std::is_sorted(alist.begin(), alist.end()));+//      for (i : alist) {+//        // some behavior which safe only when alist is sorted+//      }+//    }+//    void takes_numbers(std::vector<int> alist) {+//      std::sort(alist.begin(), alist.end());+//      takes_numbers(folly::sorted_equivalent, alist);+//    }+//+//  mimic: std::sorted_equivalent_t, std::sorted_equivalent, p0429r6+struct sorted_equivalent_t {};+constexpr sorted_equivalent_t sorted_equivalent{};++template <typename T>+struct transparent : T {+  using is_transparent = void;+  using T::T;+};++/**+ * A simple function object that passes its argument through unchanged.+ *+ * Example:+ *+ *   int i = 42;+ *   int &j = identity(i);+ *   assert(&i == &j);+ *+ * Warning: passing a prvalue through identity turns it into an xvalue,+ * which can effect whether lifetime extension occurs or not. For instance:+ *+ *   auto&& x = std::make_unique<int>(42);+ *   cout << *x ; // OK, x refers to a valid unique_ptr.+ *+ *   auto&& y = identity(std::make_unique<int>(42));+ *   cout << *y ; // ERROR: y did not lifetime-extend the unique_ptr. It+ *                // is no longer valid+ */+struct identity_fn {+  template <class T>+  constexpr T&& operator()(T&& x) const noexcept {+    return static_cast<T&&>(x);+  }+};+using Identity = identity_fn;+inline constexpr identity_fn identity{};++#if FOLLY_CPLUSPLUS >= 202002 && !defined(__NVCC__)++/// literal_c_str+///+/// This can only wrap literal strings, since the constructor is marked+/// consteval.  Like with fmt::format_string.+struct literal_c_str {+  const char* const ptr;+  /* implicit */ consteval literal_c_str(const char* p) : ptr(p) {}+};++#endif++/// literal_string+///+/// A structural type representing a literal string. A structural type may be+/// a non-type template argument.+///+/// May at times be useful since language-level literal strings are not allowed+/// as non-type template arguments.+///+/// This may typically be used with vtag for passing the literal string as a+/// constant-expression via a non-type template argument.+///+/// Example:+///+///   template <size_t N, literal_string<char, N> Str>+///   void do_something_with_literal_string(vtag_t<Str>);+///+///   void do_something() {+///     do_something_with_literal_string(vtag<literal_string{"foobar"}>);+///   }+template <typename C, std::size_t N>+struct literal_string {+  C buffer[N] = {};++  FOLLY_CONSTEVAL /* implicit */ literal_string(C const (&buf)[N]) noexcept {+    for (std::size_t i = 0; i < N; ++i) {+      buffer[i] = buf[i];+    }+  }++  constexpr std::size_t size() const noexcept { return N - 1; }+  constexpr C const* data() const noexcept { return buffer; }+  constexpr C const* c_str() const noexcept { return buffer; }++  template <+      typename String,+      decltype((void(String(FOLLY_DECLVAL(C const*), N - 1)), 0)) = 0>+  constexpr explicit operator String() const //+      noexcept(noexcept(String(FOLLY_DECLVAL(C const*), N - 1))) {+    return String(data(), N - 1);+  }+};++inline namespace literals {+inline namespace string_literals {++#if FOLLY_CPLUSPLUS >= 202002 && !defined(__NVCC__)+template <literal_string Str>+FOLLY_CONSTEVAL decltype(Str) operator""_lit() noexcept {+  return Str;+}+template <literal_string Str>+FOLLY_CONSTEVAL vtag_t<Str> operator""_litv() noexcept {+  return vtag<Str>;+}+#endif++} // namespace string_literals+} // namespace literals++namespace detail {++template <typename T>+struct inheritable_inherit_ : T {+  using T::T;+  template <+      typename... A,+      std::enable_if_t<std::is_constructible<T, A...>::value, int> = 0>+  /* implicit */ FOLLY_ERASE inheritable_inherit_(A&&... a) noexcept(+      noexcept(T(static_cast<A&&>(a)...)))+      : T(static_cast<A&&>(a)...) {}+};++template <typename T>+struct inheritable_contain_ {+  T v;+  template <+      typename... A,+      std::enable_if_t<std::is_constructible<T, A...>::value, int> = 0>+  /* implicit */ FOLLY_ERASE inheritable_contain_(A&&... a) noexcept(+      noexcept(T(static_cast<A&&>(a)...)))+      : v(static_cast<A&&>(a)...) {}+  FOLLY_ERASE operator T&() & noexcept { return v; }+  FOLLY_ERASE operator T&&() && noexcept { return static_cast<T&&>(v); }+  FOLLY_ERASE operator T const&() const& noexcept { return v; }+  FOLLY_ERASE operator T const&&() const&& noexcept {+    return static_cast<T const&&>(v);+  }+};++template <bool>+struct inheritable_;+template <>+struct inheritable_<false> {+  template <typename T>+  using apply = inheritable_inherit_<T>;+};+template <>+struct inheritable_<true> {+  template <typename T>+  using apply = inheritable_contain_<T>;+};++//  inheritable+//+//  A class wrapping an arbitrary type T which is always inheritable, and which+//  enables empty-base-optimization when possible.+template <typename T>+using inheritable =+    typename inheritable_<std::is_final<T>::value>::template apply<T>;++} // namespace detail++// Prevent child classes from finding anything in folly:: by ADL.+namespace moveonly_ {++/**+ * Disallow copy but not move in derived types. This is essentially+ * boost::noncopyable (the implementation is almost identical), except:+ * 1) It doesn't delete move constructor and move assignment.+ * 2) It has public methods, enabling aggregate initialization.+ */+struct MoveOnly {+  constexpr MoveOnly() noexcept = default;+  ~MoveOnly() noexcept = default;++  MoveOnly(MoveOnly&&) noexcept = default;+  MoveOnly& operator=(MoveOnly&&) noexcept = default;+  MoveOnly(const MoveOnly&) = delete;+  MoveOnly& operator=(const MoveOnly&) = delete;+};++/**+ * Disallow copy and move for derived types. This is essentially+ * boost::noncopyable (the implementation is almost identical), except it has+ * public methods, enabling aggregate initialization.+ */+struct NonCopyableNonMovable {+  constexpr NonCopyableNonMovable() noexcept = default;+  ~NonCopyableNonMovable() noexcept = default;++  NonCopyableNonMovable(NonCopyableNonMovable&&) = delete;+  NonCopyableNonMovable& operator=(NonCopyableNonMovable&&) = delete;+  NonCopyableNonMovable(const NonCopyableNonMovable&) = delete;+  NonCopyableNonMovable& operator=(const NonCopyableNonMovable&) = delete;+};++struct Default {};++template <bool Copy, bool Move>+using EnableCopyMove = std::conditional_t<+    Copy,+    Default,+    std::conditional_t<Move, MoveOnly, NonCopyableNonMovable>>;++} // namespace moveonly_++using moveonly_::MoveOnly;+using moveonly_::NonCopyableNonMovable;++/// variadic_noop+/// variadic_noop_fn+///+/// An invocable object and type that has no side-effects - that does nothing+/// when invoked regardless of the arguments with which it is invoked - and that+/// returns void.+///+/// May be invoked with any arguments. Returns void.+struct variadic_noop_fn {+  template <typename... A>+  constexpr void operator()(A&&...) const noexcept {}+};+inline constexpr variadic_noop_fn variadic_noop;++/// variadic_constant_of+/// variadic_constant_of_fn+///+/// An invocable object and type that has no side-effects - that does nothing+/// when invoked regardless of the arguments with which it is invoked - and that+/// returns a constant value.+template <auto Value>+struct variadic_constant_of_fn {+  using value_type = decltype(Value);+  static inline constexpr value_type value = Value;+  template <typename... A>+  constexpr value_type operator()(A&&...) const noexcept {+    return value;+  }+};+template <auto Value>+inline constexpr variadic_constant_of_fn<Value> variadic_constant_of;++//  unsafe_default_initialized+//  unsafe_default_initialized_cv+//+//  An object which is explicitly convertible to any default-constructible type+//  and which, upon conversion, yields a default-initialized value of that type.+//+//  https://en.cppreference.com/w/cpp/language/default_initialization+//+//  For fundamental types, a default-initialized instance may have indeterminate+//  value. Reading an indeterminate value is undefined behavior but may offer a+//  performance optimization. When using an indeterminate value as a performance+//  optimization, it is best to be explicit.+//+//  Useful as an escape hatch when enabling warnings or errors:+//  * gcc:+//    * uninitialized+//    * maybe-uninitialized+//  * clang:+//    * uninitialized+//    * conditional-uninitialized+//    * sometimes-uninitialized+//    * uninitialized-const-reference+//  * msvc:+//    * C4701: potentially uninitialized local variable used+//    * C4703: potentially uninitialized local pointer variable used+//+//  Example:+//+//      int local = folly::unsafe_default_initialized;+//      store_value_into_int_ptr(&value); // suppresses possible warning+//      use_value(value); // suppresses possible warning+struct unsafe_default_initialized_cv {+  FOLLY_PUSH_WARNING+  // MSVC requires warning disables to be outside of function definition+  // Uninitialized local variable 'uninit' used+  FOLLY_MSVC_DISABLE_WARNING(4700)+  // Potentially uninitialized local variable 'uninit' used+  FOLLY_MSVC_DISABLE_WARNING(4701)+  // Potentially uninitialized local pointer variable 'uninit' used+  FOLLY_MSVC_DISABLE_WARNING(4703)+  FOLLY_GNU_DISABLE_WARNING("-Wuninitialized")+  // Clang doesn't implement -Wmaybe-uninitialized and warns about it+  FOLLY_GCC_DISABLE_WARNING("-Wmaybe-uninitialized")+  template <typename T>+  FOLLY_ERASE constexpr /* implicit */ operator T() const noexcept {+#if defined(__cpp_lib_is_constant_evaluated)+#if __cpp_lib_is_constant_evaluated >= 201811L+#if (defined(_MSC_VER) && !defined(__MSVC_RUNTIME_CHECKS)) || \+    (defined(__clang__) && !defined(__GNUC__))+    if (!std::is_constant_evaluated()) {+      T uninit;+      return uninit;+    }+#endif+#endif+#endif+    return T();+  }+  FOLLY_POP_WARNING+};+inline constexpr unsafe_default_initialized_cv unsafe_default_initialized{};++/// to_bool+/// to_bool_fn+///+/// Constructs a boolean from the argument.+///+/// Particularly useful for testing sometimes-weak function declarations. They+/// may be declared weak on some platforms but not on others. GCC likes to warn+/// about them but the warning is unhelpful.+struct to_bool_fn {+  template <typename..., typename T>+  FOLLY_ERASE constexpr auto operator()(T const& t) const noexcept+      -> decltype(static_cast<bool>(t)) {+    FOLLY_PUSH_WARNING+    FOLLY_GCC_DISABLE_WARNING("-Waddress")+    FOLLY_GCC_DISABLE_WARNING("-Wnonnull-compare")+    return static_cast<bool>(t);+    FOLLY_POP_WARNING+  }+};+inline constexpr to_bool_fn to_bool{};++struct to_signed_fn {+  template <typename..., typename T>+  constexpr auto operator()(T const& t) const noexcept ->+      typename std::make_signed<T>::type {+    using S = typename std::make_signed<T>::type;+    // note: static_cast<S>(t) would be more straightforward, but it would also+    // be implementation-defined behavior and that is typically to be avoided;+    // the following code optimized into the same thing, though+    constexpr auto m = static_cast<T>(std::numeric_limits<S>::max());+    return m < t ? -static_cast<S>(~t) + S{-1} : static_cast<S>(t);+  }+};+inline constexpr to_signed_fn to_signed{};++struct to_unsigned_fn {+  template <typename..., typename T>+  constexpr auto operator()(T const& t) const noexcept ->+      typename std::make_unsigned<T>::type {+    using U = typename std::make_unsigned<T>::type;+    return static_cast<U>(t);+  }+};+inline constexpr to_unsigned_fn to_unsigned{};++namespace detail {+template <typename Src, typename Dst>+inline constexpr bool is_to_narrow_convertible_v =+    (std::is_integral<Dst>::value) &&+    (std::is_signed<Dst>::value == std::is_signed<Src>::value);+}++template <typename Src>+class to_narrow_convertible {+  static_assert(std::is_integral<Src>::value, "not an integer");++  template <typename Dst>+  struct to_+      : std::bool_constant<detail::is_to_narrow_convertible_v<Src, Dst>> {};++ public:+  explicit constexpr to_narrow_convertible(Src const& value) noexcept+      : value_(value) {}+  explicit to_narrow_convertible(to_narrow_convertible const&) = default;+  explicit to_narrow_convertible(to_narrow_convertible&&) = default;+  to_narrow_convertible& operator=(to_narrow_convertible const&) = default;+  to_narrow_convertible& operator=(to_narrow_convertible&&) = default;++  template <typename Dst, std::enable_if_t<to_<Dst>::value, int> = 0>+  /* implicit */ constexpr operator Dst() const noexcept {+    FOLLY_PUSH_WARNING+    FOLLY_MSVC_DISABLE_WARNING(4244) // lossy conversion: arguments+    FOLLY_MSVC_DISABLE_WARNING(4267) // lossy conversion: variables+    FOLLY_GNU_DISABLE_WARNING("-Wconversion")+    return value_;+    FOLLY_POP_WARNING+  }++ private:+  Src value_;+};++//  to_narrow+//+//  A utility for performing explicit possibly-narrowing integral conversion+//  without specifying the destination type. Does not permit changing signs.+//  Sometimes preferable to static_cast<Dst>(src) to document the intended+//  semantics of the cast.+//+//  Models explicit conversion with an elided destination type. Sits in between+//  a stricter explicit conversion with a named destination type and a more+//  lenient implicit conversion. Implemented with implicit conversion in order+//  to take advantage of the undefined-behavior sanitizer's inspection of all+//  implicit conversions - it checks for truncation, with suppressions in place+//  for warnings which guard against narrowing implicit conversions.+struct to_narrow_fn {+  template <typename..., typename Src>+  constexpr auto operator()(Src const& src) const noexcept+      -> to_narrow_convertible<Src> {+    return to_narrow_convertible<Src>{src};+  }+};+inline constexpr to_narrow_fn to_narrow{};++template <typename Src>+class to_integral_convertible {+  static_assert(std::is_floating_point<Src>::value, "not a floating-point");++  template <typename Dst>+  static constexpr bool to_ = std::is_integral<Dst>::value;++ public:+  explicit constexpr to_integral_convertible(Src const& value) noexcept+      : value_(value) {}++  explicit to_integral_convertible(to_integral_convertible const&) = default;+  explicit to_integral_convertible(to_integral_convertible&&) = default;+  to_integral_convertible& operator=(to_integral_convertible const&) = default;+  to_integral_convertible& operator=(to_integral_convertible&&) = default;++  template <typename Dst, std::enable_if_t<to_<Dst>, int> = 0>+  /* implicit */ constexpr operator Dst() const noexcept {+    FOLLY_PUSH_WARNING+    FOLLY_MSVC_DISABLE_WARNING(4244) // lossy conversion: arguments+    FOLLY_MSVC_DISABLE_WARNING(4267) // lossy conversion: variables+    FOLLY_GNU_DISABLE_WARNING("-Wconversion")+    return value_;+    FOLLY_POP_WARNING+  }++ private:+  Src value_;+};++//  to_integral+//+//  A utility for performing explicit floating-point-to-integral conversion+//  without specifying the destination type. Sometimes preferable to+//  static_cast<Dst>(src) to document the intended semantics of the cast.+//+//  Models explicit conversion with an elided destination type. Sits in between+//  a stricter explicit conversion with a named destination type and a more+//  lenient implicit conversion. Implemented with implicit conversion in order+//  to take advantage of the undefined-behavior sanitizer's inspection of all+//  implicit conversions.+struct to_integral_fn {+  template <typename..., typename Src>+  constexpr auto operator()(Src const& src) const noexcept+      -> to_integral_convertible<Src> {+    return to_integral_convertible<Src>{src};+  }+};+inline constexpr to_integral_fn to_integral{};++template <typename Src>+class to_floating_point_convertible {+  static_assert(std::is_integral<Src>::value, "not a floating-point");++  template <typename Dst>+  static constexpr bool to_ = std::is_floating_point<Dst>::value;++ public:+  explicit constexpr to_floating_point_convertible(Src const& value) noexcept+      : value_(value) {}++  explicit to_floating_point_convertible(to_floating_point_convertible const&) =+      default;+  explicit to_floating_point_convertible(to_floating_point_convertible&&) =+      default;+  to_floating_point_convertible& operator=(+      to_floating_point_convertible const&) = default;+  to_floating_point_convertible& operator=(to_floating_point_convertible&&) =+      default;++  template <typename Dst, std::enable_if_t<to_<Dst>, int> = 0>+  /* implicit */ constexpr operator Dst() const noexcept {+    FOLLY_PUSH_WARNING+    FOLLY_GNU_DISABLE_WARNING("-Wconversion")+    return value_;+    FOLLY_POP_WARNING+  }++ private:+  Src value_;+};++//  to_floating_point+//+//  A utility for performing explicit integral-to-floating-point conversion+//  without specifying the destination type. Sometimes preferable to+//  static_cast<Dst>(src) to document the intended semantics of the cast.+//+//  Models explicit conversion with an elided destination type. Sits in between+//  a stricter explicit conversion with a named destination type and a more+//  lenient implicit conversion. Implemented with implicit conversion in order+//  to take advantage of the undefined-behavior sanitizer's inspection of all+//  implicit conversions.+struct to_floating_point_fn {+  template <typename..., typename Src>+  constexpr auto operator()(Src const& src) const noexcept+      -> to_floating_point_convertible<Src> {+    return to_floating_point_convertible<Src>{src};+  }+};+inline constexpr to_floating_point_fn to_floating_point{};++struct to_underlying_fn {+  template <typename..., class E>+  constexpr std::underlying_type_t<E> operator()(E e) const noexcept {+    static_assert(std::is_enum<E>::value, "not an enum type");+    return static_cast<std::underlying_type_t<E>>(e);+  }+};+inline constexpr to_underlying_fn to_underlying{};++namespace detail {+template <typename R>+using invocable_to_detect = decltype(FOLLY_DECLVAL(R)());++template <+    typename F,+    //  MSVC 14.16.27023 does not permit these to be in the class body:+    //    error C2833: 'operator decltype' is not a recognized operator or type+    //  TODO: return these to the class body and remove the static assertions+    typename TML = detected_t<invocable_to_detect, F&>,+    typename TCL = detected_t<invocable_to_detect, F const&>,+    typename TMR = detected_t<invocable_to_detect, F&&>,+    typename TCR = detected_t<invocable_to_detect, F const&&>>+class invocable_to_convertible : private inheritable<F> {+ private:+  static_assert(std::is_same<F, decay_t<F>>::value, "mismatch");++  template <typename R>+  using result_t = detected_t<invocable_to_detect, R>;+  template <typename R>+  static constexpr bool detected_v = is_detected_v<invocable_to_detect, R>;+  template <typename R>+  using if_invocable_as_v = std::enable_if_t<detected_v<R>, int>;+  template <typename R>+  static constexpr bool nx_v = noexcept(FOLLY_DECLVAL(R)());+  template <typename G>+  static constexpr bool constructible_v = std::is_constructible<F, G&&>::value;++  using FML = F&;+  using FCL = F const&;+  using FMR = F&&;+  using FCR = F const&&;+  static_assert(std::is_same<TML, result_t<FML>>::value, "mismatch");+  static_assert(std::is_same<TCL, result_t<FCL>>::value, "mismatch");+  static_assert(std::is_same<TMR, result_t<FMR>>::value, "mismatch");+  static_assert(std::is_same<TCR, result_t<FCR>>::value, "mismatch");++ public:+  template <typename G, std::enable_if_t<constructible_v<G&&>, int> = 0>+  FOLLY_ERASE explicit constexpr invocable_to_convertible(G&& g) noexcept(+      noexcept(F(static_cast<G&&>(g))))+      : inheritable<F>(static_cast<G&&>(g)) {}++  template <typename..., typename R = FML, if_invocable_as_v<R> = 0>+  FOLLY_ERASE constexpr operator TML() & noexcept(nx_v<R>) {+    return static_cast<FML>(*this)();+  }+  template <typename..., typename R = FCL, if_invocable_as_v<R> = 0>+  FOLLY_ERASE constexpr operator TCL() const& noexcept(nx_v<R>) {+    return static_cast<FCL>(*this)();+  }+  template <typename..., typename R = FMR, if_invocable_as_v<R> = 0>+  FOLLY_ERASE constexpr operator TMR() && noexcept(nx_v<R>) {+    return static_cast<FMR>(*this)();+  }+  template <typename..., typename R = FCR, if_invocable_as_v<R> = 0>+  FOLLY_ERASE constexpr operator TCR() const&& noexcept(nx_v<R>) {+    return static_cast<FCR>(*this)();+  }+};+} // namespace detail++//  invocable_to+//  invocable_to_fn+//+//  Given an invocable, returns an object which is implicitly convertible to the+//  type which the invocable returns when invoked with no arguments. Conversion+//  invokes the invocables and returns the value.+//+//  The return object has unspecified type with the following semantics:+//  * It stores a decay-copy of the passed invocable.+//  * It defines four-way conversion operators. Each conversion operator purely+//    forwards to the invocable as forwarded-like the convertible, and has the+//    same exception specification and the same participation in overload+//    resolution as invocation of the invocable.+//+//  Example:+//+//    Given a setup:+//+//      struct stable {+//        int value = 0;+//        stable() = default;+//        stable(stable const&); // expensive!+//      };+//      std::list<stable const> list;+//+//    The goal is to insert a stable with a value of 7 to the back of the list.+//+//    The obvious ways are expensive:+//+//      stable obj;+//      obj.value = 7;+//      list.push_back(obj); // or variations with emplace_back or std::move+//+//    With a lambda and copy elision optimization (NRVO), the expense remains:+//+//      list.push_back(std::invoke([] {+//        stable obj;+//        obj.value = 7;+//        return obj;+//      }));+//+//    But conversion, as done with this utility, makes this goal achievable.+//+//      list.emplace_back(folly::invocable_to([] {+//        stable obj;+//        obj.value = 7;+//        return obj;+//      }));+struct invocable_to_fn {+  template <+      typename F,+      typename...,+      typename D = detail::decay_t<F>,+      typename R = detail::invocable_to_convertible<D>,+      std::enable_if_t<std::is_constructible<D, F&&>::value, int> = 0>+  FOLLY_ERASE constexpr R operator()(F&& f) const+      noexcept(noexcept(R(static_cast<F&&>(f)))) {+    return R(static_cast<F&&>(f));+  }+};+inline constexpr invocable_to_fn invocable_to{};++#define FOLLY_DETAIL_FORWARD_BODY(...)                     \+  noexcept(noexcept(__VA_ARGS__))->decltype(__VA_ARGS__) { \+    return __VA_ARGS__;                                    \+  }++/// FOLLY_FOR_EACH_THIS_OVERLOAD_IN_CLASS_BODY_DELEGATE+///+/// Helper macro to add 4 delegated, qualifier-overloaded methods to a class+///+/// Example:+///+///     template <typename T>+///     class optional {+///      public:+///       bool has_value() const;+///+///       T& value() & {+///         if (!has_value()) { throw std::bad_optional_access(); }+///         return m_value;+///       }+///+///       const T& value() const& {+///         if (!has_value()) { throw std::bad_optional_access(); }+///         return m_value;+///       }+///+///       T&& value() && {+///         if (!has_value()) { throw std::bad_optional_access(); }+///         return std::move(m_value);+///       }+///+///       const T&& value() const&& {+///         if (!has_value()) { throw std::bad_optional_access(); }+///         return std::move(m_value);+///       }+///     };+///+/// This is equivalent to+///+///     template <typename T>+///     class optional {+///       template <typename Self>+///       decltype(auto) value_impl(Self&& self) {+///         if (!self.has_value()) {+///           throw std::bad_optional_access();+///         }+///         return std::forward<Self>(self).m_value;+///       }+///       // ...+///+///      public:+///       bool has_value() const;+///+///       FOLLY_FOR_EACH_THIS_OVERLOAD_IN_CLASS_BODY_DELEGATE(value,+///       value_impl);+///     };+///+/// Note: This can be migrated to C++23's deducing this:+/// https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2021/p0847r7.html+///+// clang-format off+#define FOLLY_FOR_EACH_THIS_OVERLOAD_IN_CLASS_BODY_DELEGATE(MEMBER, DELEGATE) \+  template <class... Args>                                                    \+  [[maybe_unused]] FOLLY_ERASE_HACK_GCC                                       \+  constexpr auto MEMBER(Args&&... args) & FOLLY_DETAIL_FORWARD_BODY(          \+      ::folly::remove_cvref_t<decltype(*this)>::DELEGATE(                     \+          *this, static_cast<Args&&>(args)...))                               \+  template <class... Args>                                                    \+  [[maybe_unused]] FOLLY_ERASE_HACK_GCC                                       \+  constexpr auto MEMBER(Args&&... args) const& FOLLY_DETAIL_FORWARD_BODY(     \+      ::folly::remove_cvref_t<decltype(*this)>::DELEGATE(                     \+          *this, static_cast<Args&&>(args)...))                               \+  template <class... Args>                                                    \+  [[maybe_unused]] FOLLY_ERASE_HACK_GCC                                       \+  constexpr auto MEMBER(Args&&... args) && FOLLY_DETAIL_FORWARD_BODY(         \+      ::folly::remove_cvref_t<decltype(*this)>::DELEGATE(                     \+          std::move(*this), static_cast<Args&&>(args)...))                    \+  template <class... Args>                                                    \+  [[maybe_unused]] FOLLY_ERASE_HACK_GCC                                       \+  constexpr auto MEMBER(Args&&... args) const&& FOLLY_DETAIL_FORWARD_BODY(    \+      ::folly::remove_cvref_t<decltype(*this)>::DELEGATE(                     \+          std::move(*this), static_cast<Args&&>(args)...))                    \+  /* enforce semicolon after macro */ static_assert(true)+// clang-format on+} // namespace folly
+ folly/folly/Varint.h view
@@ -0,0 +1,230 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <type_traits>++#include <folly/Conv.h>+#include <folly/Expected.h>+#include <folly/Likely.h>+#include <folly/Portability.h>+#include <folly/Range.h>++namespace folly {++/**+ * Variable-length integer encoding, using a little-endian, base-128+ * representation.+ *+ * The MSb is set on all bytes except the last.+ *+ * Details:+ * https://developers.google.com/protocol-buffers/docs/encoding#varints+ *+ * If you want to encode multiple values, GroupVarint (in GroupVarint.h)+ * is faster and likely smaller.+ */++/**+ * Maximum length (in bytes) of the varint encoding of a 32-bit value.+ */+constexpr size_t kMaxVarintLength32 = 5;++/**+ * Maximum length (in bytes) of the varint encoding of a 64-bit value.+ */+constexpr size_t kMaxVarintLength64 = 10;++/**+ * Encode a value in the given buffer, returning the number of bytes used+ * for encoding.+ * buf must have enough space to represent the value (at least+ * kMaxVarintLength64 bytes to encode arbitrary 64-bit values)+ */+size_t encodeVarint(uint64_t val, uint8_t* buf);++/**+ * Determine the number of bytes needed to represent "val".+ * 32-bit values need at most 5 bytes.+ * 64-bit values need at most 10 bytes.+ */+int encodeVarintSize(uint64_t val);++/**+ * Decode a value from a given buffer, advances data past the returned value.+ * Throws on error.+ */+template <class T>+uint64_t decodeVarint(Range<T*>& data);++enum class DecodeVarintError {+  TooManyBytes = 0,+  TooFewBytes = 1,+};++/**+ * A variant of decodeVarint() that does not throw on error. Useful in contexts+ * where only part of a serialized varint may be attempted to be decoded, e.g.,+ * when a serialized varint arrives on the boundary of a network packet.+ */+template <class T>+Expected<uint64_t, DecodeVarintError> tryDecodeVarint(Range<T*>& data);++/**+ * ZigZag encoding that maps signed integers with a small absolute value+ * to unsigned integers with a small (positive) values. Without this,+ * encoding negative values using Varint would use up 9 or 10 bytes.+ *+ * if x >= 0, encodeZigZag(x) == 2*x+ * if x <  0, encodeZigZag(x) == -2*x - 1+ */++inline uint64_t encodeZigZag(int64_t val) {+  // Bit-twiddling magic stolen from the Google protocol buffer document;+  // val >> 63 is an arithmetic shift because val is signed+  auto uval = static_cast<uint64_t>(val);+  return static_cast<uint64_t>((uval << 1) ^ (val >> 63));+}++inline int64_t decodeZigZag(uint64_t val) {+  return static_cast<int64_t>((val >> 1) ^ -(val & 1));+}++// Implementation below++inline size_t encodeVarint(uint64_t val, uint8_t* buf) {+  uint8_t* p = buf;+  while (val >= 128) {+    *p++ = 0x80 | (val & 0x7f);+    val >>= 7;+  }+  *p++ = uint8_t(val);+  return size_t(p - buf);+}++inline int encodeVarintSize(uint64_t val) {+  if (folly::kIsArchAmd64) {+    // __builtin_clzll is undefined for 0+    int highBit = 64 - __builtin_clzll(val | 1);+    return (highBit + 6) / 7;+  } else {+    int s = 1;+    while (val >= 128) {+      ++s;+      val >>= 7;+    }+    return s;+  }+}++template <class T>+inline uint64_t decodeVarint(Range<T*>& data) {+  auto expected = tryDecodeVarint(data);+  if (!expected) {+    throw std::invalid_argument(+        expected.error() == DecodeVarintError::TooManyBytes+            ? "Invalid varint value: too many bytes."+            : "Invalid varint value: too few bytes.");+  }+  return *expected;+}++template <class T>+inline Expected<uint64_t, DecodeVarintError> tryDecodeVarint(Range<T*>& data) {+  static_assert(+      std::is_same<typename std::remove_cv<T>::type, char>::value ||+          std::is_same<typename std::remove_cv<T>::type, unsigned char>::value,+      "Only character ranges are supported");++  const int8_t* begin = reinterpret_cast<const int8_t*>(data.begin());+  const int8_t* end = reinterpret_cast<const int8_t*>(data.end());+  const int8_t* p = begin;+  uint64_t val = 0;++  // end is always greater than or equal to begin, so this subtraction is safe+  if (FOLLY_LIKELY(size_t(end - begin) >= kMaxVarintLength64)) { // fast path+    int64_t b;+    do {+      b = *p++;+      val = (b & 0x7f);+      if (b >= 0) {+        break;+      }+      b = *p++;+      val |= (b & 0x7f) << 7;+      if (b >= 0) {+        break;+      }+      b = *p++;+      val |= (b & 0x7f) << 14;+      if (b >= 0) {+        break;+      }+      b = *p++;+      val |= (b & 0x7f) << 21;+      if (b >= 0) {+        break;+      }+      b = *p++;+      val |= (b & 0x7f) << 28;+      if (b >= 0) {+        break;+      }+      b = *p++;+      val |= (b & 0x7f) << 35;+      if (b >= 0) {+        break;+      }+      b = *p++;+      val |= (b & 0x7f) << 42;+      if (b >= 0) {+        break;+      }+      b = *p++;+      val |= (b & 0x7f) << 49;+      if (b >= 0) {+        break;+      }+      b = *p++;+      val |= (b & 0x7f) << 56;+      if (b >= 0) {+        break;+      }+      b = *p++;+      val |= (b & 0x01) << 63;+      if (b >= 0) {+        break;+      }+      return makeUnexpected(DecodeVarintError::TooManyBytes);+    } while (false);+  } else {+    int shift = 0;+    while (p != end && *p < 0) {+      val |= static_cast<uint64_t>(*p++ & 0x7f) << shift;+      shift += 7;+    }+    if (p == end) {+      return makeUnexpected(DecodeVarintError::TooFewBytes);+    }+    val |= static_cast<uint64_t>(*p++) << shift;+  }++  data.uncheckedAdvance(p - begin);+  return val;+}++} // namespace folly
+ folly/folly/VirtualExecutor.h view
@@ -0,0 +1,17 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#include <folly/executors/VirtualExecutor.h>
+ folly/folly/algorithm/BinaryHeap.h view
@@ -0,0 +1,60 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <algorithm>+#include <iterator>++#include <folly/lang/Builtin.h>++namespace folly {++/**+ * Companion to std::push/pop_heap(). Restores the heap property if the heap's+ * top is modified.+ */+template <class RandomIt, class Compare>+void down_heap(RandomIt first, RandomIt last, Compare comp) {+  size_t size = last - first;+  size_t parent = 0;+  size_t child;+  // Iterate while both left and right children exist.+  while ((child = 2 * parent + 2) < size) {+    // Find the max among the two children.+    child = FOLLY_BUILTIN_UNPREDICTABLE(comp(first[child], first[child - 1]))+        ? child - 1+        : child;+    if (comp(first[parent], first[child])) {+      std::iter_swap(first + parent, first + child);+      parent = child;+    } else {+      return;+    }+  }++  // Now parent can have either no children or only a left child.+  if (--child < size && comp(first[parent], first[child])) {+    std::iter_swap(first + parent, first + child);+  }+}++template <class RandomIt>+void down_heap(RandomIt first, RandomIt last) {+  down_heap(first, last, std::less<>{});+}++} // namespace folly
+ folly/folly/algorithm/simd/Contains.cpp view
@@ -0,0 +1,43 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#include <folly/algorithm/simd/Contains.h>++#include <algorithm>+#include <cstring>+#include <folly/algorithm/simd/detail/ContainsImpl.h>++namespace folly::simd::detail {++bool containsU8(+    folly::span<const std::uint8_t> haystack, std::uint8_t needle) noexcept {+  return containsImpl(haystack, needle);+}+bool containsU16(+    folly::span<const std::uint16_t> haystack, std::uint16_t needle) noexcept {+  return containsImpl(haystack, needle);+}+bool containsU32(+    folly::span<const std::uint32_t> haystack, std::uint32_t needle) noexcept {+  return containsImpl(haystack, needle);+}++bool containsU64(+    folly::span<const std::uint64_t> haystack, std::uint64_t needle) noexcept {+  return containsImpl(haystack, needle);+}++} // namespace folly::simd::detail
+ folly/folly/algorithm/simd/Contains.h view
@@ -0,0 +1,117 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <folly/CPortability.h>+#include <folly/algorithm/simd/detail/Traits.h>++#include <iterator>++namespace folly::simd {+namespace detail {++// no overloading for easier profiling.++bool containsU8(+    folly::span<const std::uint8_t> haystack, std::uint8_t needle) noexcept;+bool containsU16(+    folly::span<const std::uint16_t> haystack, std::uint16_t needle) noexcept;+bool containsU32(+    folly::span<const std::uint32_t> haystack, std::uint32_t needle) noexcept;+bool containsU64(+    folly::span<const std::uint64_t> haystack, std::uint64_t needle) noexcept;++template <typename R>+using std_range_value_t = typename std::iterator_traits<decltype(std::begin(+    std::declval<R&>()))>::value_type;++// Constexpr check that we can always safely cast from From to To.+// If we don't require this, we might silently get different semantics from+// standard algorithms.+template <typename From, typename To>+constexpr bool convertible_with_no_loss() {+  if (sizeof(From) > sizeof(To)) {+    return false;+  }+  if (std::is_signed_v<From>) {+    return std::is_signed_v<To>;+  }++  return std::is_unsigned_v<To> || sizeof(From) < sizeof(To);+}++// All the requirements to call contains(haystack, needle);+//  * both are simd friendly (contigious range, primitive types)+//  * integrals only+//  * needle can be converted to the value_type of haystack and+//    the result of equality comparison will be the same.+template <typename R, typename T>+constexpr bool contains_haystack_needle_test() {+  if constexpr (!std::is_invocable_v<AsSimdFriendlyUintFn, R>) {+    return false;+  } else if constexpr (!has_integral_simd_friendly_equivalent_scalar_v<T>) {+    return false;+  } else {+    using simd_haystack_value =+        simd_friendly_equivalent_scalar_t<std_range_value_t<R>>;+    using simd_needle = simd_friendly_equivalent_scalar_t<T>;+    return convertible_with_no_loss<simd_needle, simd_haystack_value>();+  }+}++} // namespace detail++/**+ * folly::simd::contains+ * folly::simd::contains_fn+ *+ * A vectorized version of `std::ranges::find(r, x) != r.end()`.+ * Only works for "simd friendly cases" -+ * specifically the ones where the type can be reasonably cast+ * to `uint8/16/32/64_t`.+ *+ **/+struct contains_fn {+  template <+      typename R,+      typename T,+      typename =+          std::enable_if_t<detail::contains_haystack_needle_test<R, T>()>>+  FOLLY_ERASE bool operator()(R&& r, T x) const {+    auto castR = detail::asSimdFriendlyUint(folly::span(r));+    using value_type = detail::std_range_value_t<decltype(castR)>;++    auto castX = static_cast<value_type>(detail::asSimdFriendlyUint(x));++    if constexpr (std::is_same_v<value_type, std::uint8_t>) {+      return detail::containsU8(castR, castX);+    } else if constexpr (std::is_same_v<value_type, std::uint16_t>) {+      return detail::containsU16(castR, castX);+    } else if constexpr (std::is_same_v<value_type, std::uint32_t>) {+      return detail::containsU32(castR, castX);+    } else {+      static_assert(+          std::is_same_v<value_type, std::uint64_t>,+          "internal error, unknown type");+      return detail::containsU64(castR, castX);+    }+  }+};++inline constexpr contains_fn contains;++} // namespace folly::simd
+ folly/folly/algorithm/simd/FindFixed.h view
@@ -0,0 +1,301 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <algorithm>+#include <array>+#include <bit>+#include <concepts>+#include <cstdint>+#include <cstring>+#include <optional>+#include <span>+#include <type_traits>++#include <folly/Portability.h>+#include <folly/algorithm/simd/Movemask.h>+#include <folly/algorithm/simd/detail/Traits.h>++#if FOLLY_X64+#include <immintrin.h>+#endif++#if FOLLY_AARCH64+#include <arm_neon.h>+#endif++namespace folly {++namespace detail {++// Note: using std::same_as will just be slower to compile than is_same_v+template <typename T>+concept SimdFriendlyType =+    (std::is_same_v<std::int8_t, T> || std::is_same_v<std::uint8_t, T> ||+     std::is_same_v<std::int16_t, T> || std::is_same_v<std::uint16_t, T> ||+     std::is_same_v<std::int32_t, T> || std::is_same_v<std::uint32_t, T> ||+     std::is_same_v<std::int64_t, T> || std::is_same_v<std::uint64_t, T>);++} // namespace detail++template <typename T>+concept FollyFindFixedSupportedType = detail::SimdFriendlyType<T> ||+    (std::is_enum_v<T> && detail::SimdFriendlyType<std::underlying_type_t<T>>);++/*+ * # folly::findFixed+ *+ * A function to linear search in number of elements, known at compiled time.+ *+ * Example:+ *   std::vector<int> v {1, 3, 1, 2};+ *   std::span<const int, 4> vspan(v.data(), 4);+ *   auto m0 = folly::findFixed(vspan, 3); // m0 == 1;+ *   auto m1 = folly::findFixed(vspan, 5); // m0 == std::nullopt;+ *+ * Supported types:+ *  any 8,16,32,64 bit integers+ *  enums+ *+ * Max supported size of the range is 64 bytes.+ */+template <+    FollyFindFixedSupportedType T,+    std::convertible_to<T> U,+    std::size_t N>+constexpr std::optional<std::size_t> findFixed(std::span<const T, N> where, U x)+  requires(sizeof(T) * N <= 64);++// implementation ---------------------------------------------------------++namespace find_fixed_detail {++template <typename T>+constexpr std::optional<std::size_t> findFixedConstexpr(+    std::span<const T> where, T x) {+  std::size_t res = 0;+  for (T e : where) {+    if (e == x) {+      return res;+    }+    ++res;+  }+  return std::nullopt;+}++// clang just checks all elements one by one, without any vectorization.+// even for not very friendly to SIMD cases we could do better but for+// now only special powers of 2 were interesting.+template <typename T, std::size_t N>+std::optional<std::size_t> findFixedLetTheCompilerDoIt(+    std::span<const T, N> where, T x) {+  // this get's unrolled by both clang and gcc.+  // Experimenting with more complex ways of writing this code+  // didn't yield any results.+  return findFixedConstexpr(std::span<const T>(where), x);+}++#if FOLLY_X64+#if defined(__AVX2__)+constexpr std::size_t kMaxSimdRegister = 32;+#else+constexpr std::size_t kMaxSimdRegister = 16;+#endif+#elif FOLLY_AARCH64+constexpr std::size_t kMaxSimdRegister = 16;+#else+constexpr std::size_t kMaxSimdRegister = 1;+#endif++template <typename T>+std::optional<std::size_t> find8bytes(const T* from, T x);+template <typename T>+std::optional<std::size_t> find16bytes(const T* from, T x);+template <typename T>+std::optional<std::size_t> find32bytes(const T* from, T x);++template <typename T, std::size_t N>+std::optional<std::size_t> find2Overlaping(std::span<const T, N> where, T x);++template <typename T, std::size_t N>+std::optional<std::size_t> findSplitFirstRegister(+    std::span<const T, N> where, T x);++template <typename T, std::size_t N>+std::optional<std::size_t> findFixedDispatch(std::span<const T, N> where, T x) {+  constexpr std::size_t kNumBytes = N * sizeof(T);++  if constexpr (N == 0) {+    return std::nullopt;+  } else if constexpr (N <= 2 || kNumBytes < 8 || kMaxSimdRegister == 1) {+    return findFixedLetTheCompilerDoIt(where, x);+  } else if constexpr (kNumBytes == 8) {+    return find8bytes(where.data(), x);+  } else if constexpr (kNumBytes == 16) {+    return find16bytes(where.data(), x);+  } else if constexpr (kMaxSimdRegister >= 32 && kNumBytes == 32) {+    return find32bytes(where.data(), x);+  } else if constexpr (kMaxSimdRegister * 2 <= kNumBytes) {+    return findSplitFirstRegister(where, x);+  } else {+    // we can maybe do one better here probably with either out of bounds+    // loads or combined two register search but it's ok for now.+    return find2Overlaping(where, x);+  }+}++template <typename T, std::size_t N>+std::optional<std::size_t> find2Overlaping(std::span<const T, N> where, T x) {+  constexpr std::size_t kRegSize = std::bit_floor(N);++  std::span<const T, kRegSize> firstOverlap(where.data(), kRegSize);+  if (auto res = findFixed(firstOverlap, x)) {+    return res;+  }++  std::span<const T, kRegSize> secondOverlap(+      where.data() + (N - kRegSize), kRegSize);+  if (auto res = findFixed(secondOverlap, x)) {+    return *res + (N - kRegSize);+  }+  return std::nullopt;+}++template <typename T, std::size_t N>+std::optional<std::size_t> findSplitFirstRegister(+    std::span<const T, N> where, T x) {+  constexpr std::size_t kRegSize = kMaxSimdRegister / sizeof(T);++  std::span<const T, kRegSize> head(where.data(), kRegSize);+  if (auto res = findFixed(head, x)) {+    return res;+  }++  std::span<const T, N - kRegSize> tail(where.data() + kRegSize, N - kRegSize);+  if (auto res = findFixed(tail, x)) {+    return *res + kRegSize;+  }+  return std::nullopt;+}++template <typename Scalar, typename Reg>+std::optional<std::size_t> firstTrue(Reg reg) {+  auto [bits, bitsPerElement] = folly::simd::movemask<Scalar>(reg);+  if (bits) {+    return std::countr_zero(bits) / bitsPerElement();+  }+  return std::nullopt;+}++#if FOLLY_X64++template <typename T>+std::optional<std::size_t> find16ByteReg(__m128i reg, T x) {+  if constexpr (sizeof(T) == 1) {+    return firstTrue<T>(_mm_cmpeq_epi8(reg, _mm_set1_epi8(x)));+  } else if constexpr (sizeof(T) == 2) {+    return firstTrue<T>(_mm_cmpeq_epi16(reg, _mm_set1_epi16(x)));+  } else if constexpr (sizeof(T) == 4) {+    return firstTrue<T>(_mm_cmpeq_epi32(reg, _mm_set1_epi32(x)));+  }+}++template <typename T>+std::optional<std::size_t> find8bytes(const T* from, T x) {+  std::uint64_t reg;+  std::memcpy(&reg, from, 8);+  return find16ByteReg(_mm_set1_epi64x(reg), x);+}++template <typename T>+std::optional<std::size_t> find16bytes(const T* from, T x) {+  __m128i reg = _mm_loadu_si128(reinterpret_cast<const __m128i*>(from));+  return find16ByteReg(reg, x);+}++#if defined(__AVX2__)+template <typename T>+std::optional<std::size_t> find32ByteReg(__m256i reg, T x) {+  if constexpr (sizeof(T) == 1) {+    return firstTrue<T>(_mm256_cmpeq_epi8(reg, _mm256_set1_epi8(x)));+  } else if constexpr (sizeof(T) == 2) {+    return firstTrue<T>(_mm256_cmpeq_epi16(reg, _mm256_set1_epi16(x)));+  } else if constexpr (sizeof(T) == 4) {+    return firstTrue<T>(_mm256_cmpeq_epi32(reg, _mm256_set1_epi32(x)));+  } else if constexpr (sizeof(T) == 8) {+    return firstTrue<T>(_mm256_cmpeq_epi64(reg, _mm256_set1_epi64x(x)));+  }+}++template <typename T>+std::optional<std::size_t> find32bytes(const T* from, T x) {+  __m256i reg = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(from));+  return find32ByteReg(reg, x);+}++#endif+#endif++#if FOLLY_AARCH64++template <typename T>+std::optional<std::size_t> find8bytes(const T* from, T x) {+  if constexpr (std::same_as<T, std::uint8_t>) {+    return firstTrue<T>(vceq_u8(vld1_u8(from), vdup_n_u8(x)));+  } else if constexpr (std::same_as<T, std::uint16_t>) {+    return firstTrue<T>(vceq_u16(vld1_u16(from), vdup_n_u16(x)));+  } else {+    return firstTrue<T>(vceq_u32(vld1_u32(from), vdup_n_u32(x)));+  }+}++template <typename T>+std::optional<std::size_t> find16bytes(const T* from, T x) {+  if constexpr (std::same_as<T, std::uint8_t>) {+    return firstTrue<T>(vceqq_u8(vld1q_u8(from), vdupq_n_u8(x)));+  } else if constexpr (std::same_as<T, std::uint16_t>) {+    return firstTrue<T>(vceqq_u16(vld1q_u16(from), vdupq_n_u16(x)));+  } else if constexpr (std::same_as<T, std::uint32_t>) {+    return firstTrue<T>(vceqq_u32(vld1q_u32(from), vdupq_n_u32(x)));+  } else {+    return firstTrue<T>(vceqq_u64(vld1q_u64(from), vdupq_n_u64(x)));+  }+}++#endif++} // namespace find_fixed_detail++template <+    FollyFindFixedSupportedType T,+    std::convertible_to<T> U,+    std::size_t N>+constexpr std::optional<std::size_t> findFixed(std::span<const T, N> where, U x)+  requires(sizeof(T) * N <= 64)+{+  if constexpr (!std::is_same_v<T, U>) {+    return findFixed(where, static_cast<T>(x));+  } else if (std::is_constant_evaluated()) {+    return find_fixed_detail::findFixedConstexpr(std::span<const T>(where), x);+  } else {+    return find_fixed_detail::findFixedDispatch(+        simd::detail::asSimdFriendlyUint(where),+        simd::detail::asSimdFriendlyUint(x));+  }+}++} // namespace folly
+ folly/folly/algorithm/simd/Ignore.h view
@@ -0,0 +1,52 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <folly/lang/Bits.h>++#include <type_traits>++namespace folly::simd {++/**+ * ignore(_none/_extrema)+ *+ * tag types to be used in some simd operations.+ *+ * They are used to indicate to the function that+ * some of the elements in are garbage.+ *+ * ignore_none indicates that the whole register is used.+ * ignore_extrema.first, .last show how many elements are out of the data.+ *+ * Example:+ * register: [true, true, false, false, false, false, false, true]+ * indexes   [0,    1,    2,     3,     4,     5,     6,     7   ]+ *+ * ignore_extema{.first = 1, .last = 2}+ * means that elements with indexes 0, 6, and 7 will be ignored+ * (w/e that means for an operation)+ */++struct ignore_extrema {+  int first = 0;+  int last = 0;+};++struct ignore_none {};++} // namespace folly::simd
+ folly/folly/algorithm/simd/Movemask.h view
@@ -0,0 +1,208 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <folly/Portability.h>+#include <folly/algorithm/simd/Ignore.h>+#include <folly/lang/Bits.h>++#include <cstdint>+#include <type_traits>+#include <utility>++#if FOLLY_SSE >= 2+#include <immintrin.h>+#endif++#if FOLLY_NEON+#include <arm_neon.h>+#endif++FOLLY_PUSH_WARNING+FOLLY_GCC_DISABLE_WARNING("-Wignored-attributes")++namespace folly::simd {++/**+ * movemask+ * movemask_fn+ *+ * This is a low level utility used for simd search algorithms.+ *+ * It is a logical extension of _mm_movemask_epi8 for different types+ * for both x86 and arm.+ *+ * Main interface looks like this:+ * folly::simd::movemask<scalar_type>(simdRegister)+ *   -> std::pair<Bits, BitsPerElement>;+ *+ * scalar type - type of element in the simdRegister+ *+ *  Bits - unsigned integral, containing the bitmask (first is lowest bit).+ *  BitsPerElement - std::integral_constant with number of bits per element.+ *+ * There are also overloads taking `ignore`+ *+ *   folly::simd::movemask<T>(nativeRegister, ignore_extrema)+ *   folly::simd::movemask<T>(nativeRegister, ignore_none)+ *+ * These are there if not all the native register contains valid results,+ * and some need to be ignored (zeroed out)+ *+ * Example: find in 8 shorts on arm.+ *+ *  std::optional<std::uint32_t> findUint16(+ *       std::span<const std::uint16_t> haystack,+ *       std::uint16_t needle) {+ *    uint16x8_t loaded = vld1q_u16(arr.data());+ *    uint16x8_t simdNeedle = vdupq_n_u16(needle);+ *    uint16x8_t test = vceqq_u16(loaded, simdNeedle);+ *+ *    auto [bits, bitsPerElement] = folly::simd::movemask<std::uint16_t>(test);+ *    if (!bits) {+ *      return std::nullopt;+ *    }+ *    return std::countl_zero(bits) / bitsPerElement();+ *  }+ *+ * Arm implementation is based on:+ * https://github.com/jfalcou/eve/blob/a2e2cf539e36e9a3326800194ad5206a8ef3f5b7/include/eve/detail/function/simd/arm/neon/movemask.hpp#L48+ *+ **/++template <typename Scalar>+struct movemask_fn {+  template <typename Reg>+  auto operator()(Reg reg) const;++  template <typename Reg, typename Ignore>+  auto operator()(Reg reg, Ignore ignore) const;+};++template <typename Scalar>+inline constexpr movemask_fn<Scalar> movemask;++#if FOLLY_SSE >= 2++template <typename Scalar>+template <typename Reg>+FOLLY_ERASE auto movemask_fn<Scalar>::operator()(Reg reg) const {+  std::integral_constant<std::uint32_t, sizeof(Scalar) == 2 ? 2 : 1>+      bitsPerElement;++  using uint_t = std::+      conditional_t<std::is_same_v<Reg, __m128i>, std::uint16_t, std::uint32_t>;++  auto mmask = static_cast<uint_t>([&] {+    if constexpr (std::is_same_v<Reg, __m128i>) {+      if constexpr (sizeof(Scalar) <= 2) {+        return _mm_movemask_epi8(reg);+      } else if constexpr (sizeof(Scalar) == 4) {+        return _mm_movemask_ps(_mm_castsi128_ps(reg));+      } else if constexpr (sizeof(Scalar) == 8) {+        return _mm_movemask_pd(_mm_castsi128_pd(reg));+      }+    }+#if defined(__AVX2__)+    else if constexpr (std::is_same_v<Reg, __m256i>) {+      if constexpr (sizeof(Scalar) <= 2) {+        return _mm256_movemask_epi8(reg);+      } else if constexpr (sizeof(Scalar) == 4) {+        return _mm256_movemask_ps(_mm256_castsi256_ps(reg));+      } else if constexpr (sizeof(Scalar) == 8) {+        return _mm256_movemask_pd(_mm256_castsi256_pd(reg));+      }+    }+#endif+  }());+  return std::pair{mmask, bitsPerElement};+}++#endif++#if FOLLY_NEON++namespace detail {++FOLLY_ERASE auto movemaskChars16Aarch64(uint8x16_t reg) {+  uint16x8_t u16s = vreinterpretq_u16_u8(reg);+  u16s = vshrq_n_u16(u16s, 4);+  uint8x8_t packed = vmovn_u16(u16s);+  std::uint64_t bits = vget_lane_u64(vreinterpret_u64_u8(packed), 0);+  return std::pair{bits, std::integral_constant<std::uint32_t, 4>{}};+}++template <typename Reg>+FOLLY_ERASE uint64x1_t asUint64x1Aarch64(Reg reg) {+  if constexpr (std::is_same_v<Reg, uint64x1_t>) {+    return reg;+  } else if constexpr (std::is_same_v<Reg, uint32x2_t>) {+    return vreinterpret_u64_u32(reg);+  } else if constexpr (std::is_same_v<Reg, uint16x4_t>) {+    return vreinterpret_u64_u16(reg);+  } else {+    return vreinterpret_u64_u8(reg);+  }+}++} // namespace detail++template <typename Scalar>+template <typename Reg>+FOLLY_ERASE auto movemask_fn<Scalar>::operator()(Reg reg) const {+  if constexpr (std::is_same_v<Reg, uint64x2_t>) {+    return movemask<std::uint32_t>(vmovn_u64(reg));+  } else if constexpr (std::is_same_v<Reg, uint32x4_t>) {+    return movemask<std::uint16_t>(vmovn_u32(reg));+  } else if constexpr (std::is_same_v<Reg, uint16x8_t>) {+    return movemask<std::uint8_t>(vmovn_u16(reg));+  } else if constexpr (std::is_same_v<Reg, uint8x16_t>) {+    return detail::movemaskChars16Aarch64(reg);+  } else {+    std::uint64_t mmask = vget_lane_u64(detail::asUint64x1Aarch64(reg), 0);+    return std::pair{+        mmask, std::integral_constant<std::uint32_t, sizeof(Scalar) * 8>{}};+  }+}++#endif++#if FOLLY_SSE >= 2 || FOLLY_NEON++template <typename Scalar>+template <typename Reg, typename Ignore>+FOLLY_ERASE auto movemask_fn<Scalar>::operator()(Reg reg, Ignore ignore) const {+  auto [bits, bitsPerElement] = operator()(reg);++  if constexpr (std::is_same_v<Ignore, ignore_none>) {+    return std::pair{bits, bitsPerElement};+  } else {+    static constexpr int kCardinal = sizeof(Reg) / sizeof(Scalar);++    int bitsToKeep = (kCardinal - ignore.last) * bitsPerElement;++    bits = clear_n_least_significant_bits(bits, ignore.first * bitsPerElement);+    bits = clear_n_most_significant_bits(bits, sizeof(bits) * 8 - bitsToKeep);+    return std::pair{bits, bitsPerElement};+  }+}++#endif++} // namespace folly::simd++FOLLY_POP_WARNING
+ folly/folly/algorithm/simd/detail/ContainsImpl.h view
@@ -0,0 +1,92 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <algorithm>+#include <cstring>+#include <cwchar>+#include <type_traits>++#include <folly/CPortability.h>+#include <folly/algorithm/simd/detail/SimdAnyOf.h>+#include <folly/algorithm/simd/detail/SimdPlatform.h>+#include <folly/container/span.h>++namespace folly::simd::detail {++/*+ * The functions in this file are FOLLY_ERASE to make sure+ * that the only place behind a call boundary is the explicit one.+ */++template <typename T>+FOLLY_ALWAYS_INLINE bool containsImplStd(+    folly::span<const T> haystack, T needle) {+  static_assert(+      std::is_unsigned_v<T>, "we should only get here for uint8/16/32/64");+  if constexpr (sizeof(T) == 1) {+    auto* ptr = reinterpret_cast<const char*>(haystack.data());+    auto castNeedle = static_cast<char>(needle);+    if (haystack.empty()) { // memchr requires not null+      return false;+    }+    return std::memchr(ptr, castNeedle, haystack.size()) != nullptr;+  } else if constexpr (sizeof(T) == sizeof(wchar_t)) {+    auto* ptr = reinterpret_cast<const wchar_t*>(haystack.data());+    auto castNeedle = static_cast<wchar_t>(needle);+    if (haystack.empty()) { // wmemchr requires not null+      return false;+    }+    return std::wmemchr(ptr, castNeedle, haystack.size()) != nullptr;+  } else {+    // Using find instead of any_of on an off chance that the standard library+    // will add some custom vectorization.+    // That wouldn't be possible for any_of because of the predicates.+    return std::find(haystack.begin(), haystack.end(), needle) !=+        haystack.end();+  }+}++template <typename T>+constexpr bool hasHandwrittenContains() {+  return !std::is_same_v<SimdPlatform<T>, void> &&+      (std::is_same_v<std::uint8_t, T> || std::is_same_v<std::uint16_t, T> ||+       std::is_same_v<std::uint32_t, T> || std::is_same_v<std::uint64_t, T>);+}++template <typename T, typename Platform = SimdPlatform<T>>+FOLLY_ALWAYS_INLINE bool containsImplHandwritten(+    folly::span<const T> haystack, T needle) {+  static_assert(!std::is_same_v<Platform, void>);+  return simdAnyOf<Platform, 4>(+      haystack.data(),+      haystack.data() + haystack.size(),+      [&](typename Platform::reg_t x) {+        return Platform::equal(x, static_cast<T>(needle));+      });+}++template <typename T>+FOLLY_ALWAYS_INLINE bool containsImpl(folly::span<const T> haystack, T needle) {+  if constexpr (hasHandwrittenContains<T>()) {+    return containsImplHandwritten(haystack, needle);+  } else {+    return containsImplStd(haystack, needle);+  }+}++} // namespace folly::simd::detail
+ folly/folly/algorithm/simd/detail/SimdAnyOf.h view
@@ -0,0 +1,84 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <folly/CPortability.h>+#include <folly/algorithm/simd/detail/SimdForEach.h>+#include <folly/algorithm/simd/detail/UnrollUtils.h>++namespace folly {+namespace simd::detail {++/**+ * AnyOfDelegate+ *+ * Implementation detail of simdAnyOf+ * This is a delegate to simdForEach+ *+ * Based on+ * https://github.com/jfalcou/eve/blob/9309d6d17a35004adb371099d79082c8cc75d3a6/include/eve/module/algo/algo/any_of.hpp#L23+ */+template <typename Platform, typename I, typename P>+struct AnyOfDelegate {+  // _p to deal with a shadow warning on an old gcc+  explicit AnyOfDelegate(P _p) : p(_p) {}++  template <typename Ignore, typename UnrollStep>+  FOLLY_ALWAYS_INLINE bool step(I it, Ignore ignore, UnrollStep) {+    auto test = p(Platform::loada(it, ignore));+    res = Platform::any(test, ignore);+    return res;+  }++  template <std::size_t N>+  FOLLY_ALWAYS_INLINE bool unrolledStep(std::array<I, N> arr) {+    // Don't have to forceinline - no user code dependency+    auto loaded = detail::UnrollUtils::arrayMap(arr, [](I it) {+      return Platform::loada(it, ignore_none{});+    });+    auto tests = detail::UnrollUtils::arrayMap(loaded, p);+    auto test = detail::UnrollUtils::arrayReduce(tests, Platform::logical_or);+    res = Platform::any(test, ignore_none{});+    return res;+  }++  P p;+  bool res = false;+};++/**+ * simdAnyOf<Platform, unrolling = 4>(f, l, p);+ *+ * Like std::any_of but with vectorized predicates.+ * Predicate shoud accept Platform::reg_t and return Platform::logical_t.+ *+ * By default is unrolled 4 ways but for expensive predicates you might want to+ * use an unroll factor of 1.+ *+ * Function is marked as FOLLY_ALWAYS_INLINE because we don't want the end users+ * to include this directly, we want to implement a specific function and hide+ * that code behind a compile time boundary.+ */+template <typename Platform, int unrolling = 4, typename T, typename P>+FOLLY_ALWAYS_INLINE bool simdAnyOf(T* f, T* l, P p) {+  AnyOfDelegate<Platform, T*, P> delegate{p};+  simdForEachAligning<unrolling>(Platform::kCardinal, f, l, delegate);+  return delegate.res;+}++} // namespace simd::detail+} // namespace folly
+ folly/folly/algorithm/simd/detail/SimdForEach.h view
@@ -0,0 +1,209 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <folly/CPortability.h>+#include <folly/Traits.h>+#include <folly/algorithm/simd/Ignore.h>+#include <folly/algorithm/simd/detail/UnrollUtils.h>+#include <folly/lang/Align.h>++#include <array>+#include <cstdint>+#include <type_traits>++namespace folly {+namespace simd::detail {++// Based on+// https://github.com/jfalcou/eve/blob/5264e20c51aeca17675e67abf236ce1ead781c52/include/eve/module/algo/algo/for_each_iteration.hpp#L148+//+// Everything is ALWAYS_INLINE because we want to have one top level noinline+// function that does everything. Otherwise sometimes the compiler tends+// to mess that up.+//++/**+ * simdForEachAligning<unrolling>(cardinal, f, l, delegate);+ *+ * The main idea is that you can read from the memory within the page.+ * The beginning and end of the page are aligned to 4KB.+ * That means, the previous aligned address is always safe to read from+ * (requires asan disablement). This is how strlen works+ * https://stackoverflow.com/questions/25566302/vectorized-strlen-getting-away-with-reading-unallocated-memory+ *+ * The interface parameters are:+ * - unrolling: by how much do you want to unroll the main loop. 4 is a good+ * default for simple operations.+ * - cardinal: how big is your register in elements.+ * - f, l: [first, last) range+ * - delegate:+ *     conceptually a callback but has 2 different operations.+ *     - bool step(T*, ignore): to process one register.+ *       Is called for tails and gets ignore_none/ignore_extrema.+ *     - bool unrolledStep(std::array<T*, unrolling>) -+ *       to process the unrolled part. unrolledStep is not called+ *       if unrolling == 1.+ *   Both step and unrolledStep should return true if they would like to break.+ *   Delegate is passed by reference so that the caller can store state in it.+ */+template <int unrolling, typename T, typename Delegate>+FOLLY_ALWAYS_INLINE void simdForEachAligning(+    int cardinal, T* f, T* l, Delegate& delegate);++/**+ * previousAlignedAddress+ *+ * Given a pointer returns a closest pointer aligned to a given size+ * (in elements).+ */+template <typename T>+FOLLY_ALWAYS_INLINE T* previousAlignedAddress(T* ptr, int to) {+  return align_floor(ptr, sizeof(T) * to);+}++/**+ * SimdForEachMainLoop+ *+ * Implementaiton detail of simdForEach+ *+ * Regardless of how you chose to handle tails, the middle will be the same.+ * The operator() returns true if the delegate returned to break.+ *+ * There are two variations:+ * - no unrolling (unroll<1>)+ * - unrolling > 1+ *+ * For explanation of parameters see simdForEachAligning+ */+struct SimdForEachMainLoop {+  template <typename T, typename Delegate>+  FOLLY_ALWAYS_INLINE bool operator()(+      int cardinal, T*& f, T* l, Delegate& delegate, index_constant<1>) const {+    while (f != l) {+      if (delegate.step(f, ignore_none{}, index_constant<0>{}))+        return true;+      f += cardinal;+    }++    return false;+  }++  template <typename T, typename Delegate>+  struct SmallStepsLambda {+    bool& shouldBreak;+    int cardinal;+    T*& f;+    T* l;+    Delegate& delegate;++    template <std::size_t i>+    FOLLY_ALWAYS_INLINE bool operator()(index_constant<i> unrollI) {+      if (f == l)+        return true;++      shouldBreak = delegate.step(f, ignore_none{}, unrollI);+      f += cardinal;+      return shouldBreak;+    }+  };++  template <typename T, typename Delegate, std::size_t unrolling>+  FOLLY_ALWAYS_INLINE bool operator()(+      int cardinal, T*& f, T* l, Delegate& delegate, index_constant<unrolling>)+      const {+    // Not enough to fully unroll explanation.+    //+    // There are a few approaches to handle this:+    // 1. Duff's device. Is not good for simd algorithms, we are not that+    // pressed for space and we'd like the code to work differently when+    // we have many registers to process.+    // 2. Traditional: do unrolled steps first, then single steps.+    // 3. What we do here: put single steps before the unrolled ones to also do+    //    them in the beginning.+    //+    // The reason to prefer 3 over 2 is that unrolled part often would need more+    // set up, and this allows us to avoid it for small arrays.++    // Jump to this while true will be performed at most once, to do final+    // single steps.+    while (true) {+      // Delegate said we should break. We might also stop because we reached+      // the end.+      bool shouldBreak = false;++      // single steps+      if (UnrollUtils::unrollUntil<unrolling>(SmallStepsLambda<T, Delegate>{+              shouldBreak, cardinal, f, l, delegate})) {+        return shouldBreak;+      }++      for (std::ptrdiff_t bigStepsCount = (l - f) / (cardinal * unrolling);+           bigStepsCount != 0;+           --bigStepsCount) {+        std::array<T*, unrolling> arr;+        // Since there is no callback, we can rely on the inlining+        UnrollUtils::unrollUntil<unrolling>([&](auto idx) {+          arr[idx()] = f;+          f += cardinal;+          return false;+        });+        if (delegate.unrolledStep(arr)) {+          return true;+        }+      }+    }+  }+};++// Comment is at the top of the file.+template <int unrolling, typename T, typename Delegate>+FOLLY_ALWAYS_INLINE void simdForEachAligning(+    int cardinal, T* f, T* l, Delegate& delegate) {+  if (f == l) {+    return;+  }++  T* af = previousAlignedAddress(f, cardinal);+  T* al = previousAlignedAddress(l, cardinal);++  ignore_extrema ignore{static_cast<int>(f - af), 0};+  if (af != al) {+    // first chunk+    if (delegate.step(af, ignore, index_constant<0>{})) {+      return;+    }+    ignore.first = 0;+    af += cardinal;++    if (SimdForEachMainLoop{}(+            cardinal, af, al, delegate, index_constant<unrolling>{})) {+      return;+    }++    // Here af might be exactly at the end of page.+    if (af == l) {+      return;+    }+  }++  ignore.last = static_cast<int>(af + cardinal - l);+  delegate.step(af, ignore, index_constant<0>{});+}++} // namespace simd::detail+} // namespace folly
+ folly/folly/algorithm/simd/detail/SimdPlatform.h view
@@ -0,0 +1,518 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <folly/Portability.h>+#include <folly/algorithm/simd/Ignore.h>+#include <folly/algorithm/simd/Movemask.h>+#include <folly/algorithm/simd/detail/SimdPlatform.h>+#include <folly/lang/SafeAssert.h>++#include <array>++#if FOLLY_X64 && FOLLY_SSE_PREREQ(4, 2)+#include <immintrin.h>+#endif++#if FOLLY_AARCH64+#include <arm_neon.h>+#endif++namespace folly {+namespace simd::detail {++/**+ * SimdPlatform<T>+ *+ * Common interface for some SIMD operations between: sse4.2, avx2,+ * arm-neon.+ *+ * Supported types for T at the moment are uint8_16/uint16_t/uint32_t/uint64_t+ *+ * If it's not one of the supported platforms:+ *  std::same_as<SimdPlatform<T>, void>+ *  There is also a macro: FOLLY_DETAIL_HAS_SIMD_PLATFORM set to 1 or 0+ *+ **/++#if FOLLY_X64 && FOLLY_SSE_PREREQ(4, 2) || FOLLY_AARCH64++template <typename Platform>+struct SimdPlatformCommon {+  /**+   * sclar_t - type of scalar we operate on (uint8_t, uint16_t etc)+   * reg_t - type of a simd register (__m128i)+   * logical_t - type of a simd logical register (matches reg_t so far)+   **/+  using scalar_t = typename Platform::scalar_t;+  using reg_t = typename Platform::reg_t;+  using logical_t = typename Platform::logical_t;++  static constexpr int kCardinal = sizeof(reg_t) / sizeof(scalar_t);++  /**+   * loads:+   * precondition: at least one element should be not ignored.+   *+   * loada - load from an aligned (to sizeof(reg_t)) address+   * loadu - load from an unaligned address+   * unsafeLoadu - load from an unaligned address that disables sanitizers.+   *               This is for reading a register within a page+   *               but maybe outside of the array's boundary.+   *+   * Ignored values can be garbage.+   **/+  template <typename Ignore>+  static reg_t loada(const scalar_t* ptr, Ignore);+  static reg_t loadu(const scalar_t* ptr, ignore_none);+  static reg_t unsafeLoadu(const scalar_t* ptr, ignore_none);++  /**+   * Comparing reg_t against the scalar.+   *+   * NOTE: less_equal only implemented for uint8_t+   *       for now.+   **/+  static logical_t equal(reg_t reg, scalar_t x);+  static logical_t less_equal(reg_t reg, scalar_t x);++  /**+   * logical reduction+   **/+  template <typename Ignore>+  static bool any(logical_t logical, Ignore ignore);++  template <typename Ignore>+  static bool all(logical_t logical, Ignore ignore);++  /**+   * logical operations+   **/+  static logical_t logical_or(logical_t x, logical_t y);++  /**+   * Converting register to an array for debugging+   **/+  static auto toArray(reg_t x);+};++template <typename Platform>+template <typename Ignore>+FOLLY_ERASE auto SimdPlatformCommon<Platform>::loada(+    const scalar_t* ptr, [[maybe_unused]] Ignore ignore) -> reg_t {+  if constexpr (std::is_same_v<ignore_none, Ignore>) {+    // There is not point to aligned load instructions+    // on modern cpus. Arm doesn't even have any.+    return loadu(ptr, ignore_none{});+  } else {+    // We have a precondition: at least one element is loaded.+    // From this we can prove that we can unsafely load from+    // and aligned address.+    //+    // Here is an explanation from Stephen Canon:+    // https://stackoverflow.com/questions/25566302/vectorized-strlen-getting-away-with-reading-unallocated-memory+    if constexpr (!kIsSanitizeAddress) {+      return unsafeLoadu(ptr, ignore_none{});+    } else {+      // If the sanitizers are enabled, we want to trigger the issues.+      // We also want to match the garbage values with/without asan,+      // so that testing works on the same values as prod.+      scalar_t buf[kCardinal];+      std::memcpy(+          buf + ignore.first,+          ptr + ignore.first,+          (kCardinal - ignore.first - ignore.last) * sizeof(scalar_t));++      auto testAgainst = loadu(buf, ignore_none{});+      auto res = unsafeLoadu(ptr, ignore_none{});++      // Extra sanity check.+      FOLLY_SAFE_CHECK(all(Platform::equal(res, testAgainst), ignore));+      return res;+    }+  }+}++template <typename Platform>+FOLLY_ERASE auto SimdPlatformCommon<Platform>::loadu(+    const scalar_t* ptr, ignore_none) -> reg_t {+  return Platform::loadu(ptr);+}++template <typename Platform>+FOLLY_ERASE auto SimdPlatformCommon<Platform>::unsafeLoadu(+    const scalar_t* ptr, ignore_none) -> reg_t {+  return Platform::unsafeLoadu(ptr);+}++template <typename Platform>+FOLLY_ERASE auto SimdPlatformCommon<Platform>::equal(reg_t reg, scalar_t x)+    -> logical_t {+  return Platform::equal(reg, Platform::broadcast(x));+}++template <typename Platform>+FOLLY_ERASE auto SimdPlatformCommon<Platform>::less_equal(reg_t reg, scalar_t x)+    -> logical_t {+  static_assert(std::is_same_v<scalar_t, std::uint8_t>, "not implemented");+  return Platform::less_equal(reg, Platform::broadcast(x));+}++template <typename Platform>+template <typename Ignore>+FOLLY_ERASE bool SimdPlatformCommon<Platform>::any(+    logical_t logical, Ignore ignore) {+  if constexpr (std::is_same_v<Ignore, ignore_none>) {+    return Platform::any(logical);+  } else {+    return movemask<scalar_t>(logical, ignore).first;+  }+}++template <typename Platform>+template <typename Ignore>+FOLLY_ERASE bool SimdPlatformCommon<Platform>::all(+    logical_t logical, Ignore ignore) {+  if constexpr (std::is_same_v<Ignore, ignore_none>) {+    return Platform::all(logical);+  } else {+    auto [bits, bitsPerElement] = movemask<scalar_t>(logical, ignore_none{});++    auto expected = n_least_significant_bits<decltype(bits)>(+        bitsPerElement * (kCardinal - ignore.last));+    expected =+        clear_n_least_significant_bits(expected, ignore.first * bitsPerElement);++    return (bits & expected) == expected;+  }+}++template <typename Platform>+FOLLY_ERASE auto SimdPlatformCommon<Platform>::logical_or(+    logical_t x, logical_t y) -> logical_t {+  return Platform::logical_or(x, y);+}++template <typename Platform>+FOLLY_ERASE auto SimdPlatformCommon<Platform>::toArray(reg_t x) {+  std::array<scalar_t, kCardinal> res;+  std::memcpy(&res, &x, sizeof(x));+  return res;+}++#endif++#if FOLLY_X64 && FOLLY_SSE_PREREQ(4, 2)++template <typename T>+struct SimdSse42PlatformSpecific {+  using scalar_t = T;+  using reg_t = __m128i;+  using logical_t = reg_t;++  static constexpr std::size_t kCardinal = sizeof(reg_t) / sizeof(scalar_t);++  FOLLY_ERASE+  static reg_t loadu(const scalar_t* p) {+    return _mm_loadu_si128(reinterpret_cast<const reg_t*>(p));+  }++  FOLLY_DISABLE_SANITIZERS+  FOLLY_ERASE+  static reg_t unsafeLoadu(const scalar_t* p) {+    return _mm_loadu_si128(reinterpret_cast<const reg_t*>(p));+  }++  FOLLY_ERASE+  static reg_t broadcast(scalar_t x) {+    if constexpr (std::is_same_v<scalar_t, std::uint8_t>) {+      return _mm_set1_epi8(x);+    } else if constexpr (std::is_same_v<scalar_t, std::uint16_t>) {+      return _mm_set1_epi16(x);+    } else if constexpr (std::is_same_v<scalar_t, std::uint32_t>) {+      return _mm_set1_epi32(x);+    } else if constexpr (std::is_same_v<scalar_t, std::uint64_t>) {+      return _mm_set1_epi64x(x);+    }+  }++  FOLLY_ERASE+  static logical_t equal(reg_t x, reg_t y) {+    if constexpr (std::is_same_v<scalar_t, std::uint8_t>) {+      return _mm_cmpeq_epi8(x, y);+    } else if constexpr (std::is_same_v<scalar_t, std::uint16_t>) {+      return _mm_cmpeq_epi16(x, y);+    } else if constexpr (std::is_same_v<scalar_t, std::uint32_t>) {+      return _mm_cmpeq_epi32(x, y);+    } else if constexpr (std::is_same_v<scalar_t, std::uint64_t>) {+      return _mm_cmpeq_epi64(x, y);+    }+  }++  FOLLY_ERASE+  static logical_t less_equal(reg_t x, reg_t y) {+    static_assert(+        std::is_same_v<std::uint8_t, scalar_t>, "other types not implemented");+    // No unsigned comparisons on x86+    // less equal <=> equal (min)+    reg_t min = _mm_min_epu8(x, y);+    return equal(x, min);+  }++  FOLLY_ERASE+  static logical_t logical_or(logical_t x, logical_t y) {+    return _mm_or_si128(x, y);+  }++  FOLLY_ERASE+  static bool any(logical_t log) { return movemask<scalar_t>(log).first; }++#if 0 // disabled untill we have a test where this is relevant+  FOLLY_ERASE+  static bool all(logical_t log) {+    auto [bits, bitsPerElement] = movemask<scalar_t>(log);+    return movemask<scalar_t>(log) ==+        n_least_significant_bits<decltype(bits)>(kCardinal * bitsPerElement);+  }+#endif+};++#define FOLLY_DETAIL_HAS_SIMD_PLATFORM 1++template <typename T>+struct SimdSse42Platform : SimdPlatformCommon<SimdSse42PlatformSpecific<T>> {};++#if defined(__AVX2__)++template <typename T>+struct SimdAvx2PlatformSpecific {+  using scalar_t = T;+  using reg_t = __m256i;+  using logical_t = reg_t;++  static constexpr std::size_t kCardinal = sizeof(reg_t) / sizeof(scalar_t);++  FOLLY_ERASE+  static reg_t loadu(const scalar_t* p) {+    return _mm256_loadu_si256(reinterpret_cast<const reg_t*>(p));+  }++  FOLLY_DISABLE_SANITIZERS+  FOLLY_ERASE+  static reg_t unsafeLoadu(const scalar_t* p) {+    return _mm256_loadu_si256(reinterpret_cast<const reg_t*>(p));+  }++  FOLLY_ERASE+  static reg_t broadcast(scalar_t x) {+    if constexpr (std::is_same_v<scalar_t, std::uint8_t>) {+      return _mm256_set1_epi8(x);+    } else if constexpr (std::is_same_v<scalar_t, std::uint16_t>) {+      return _mm256_set1_epi16(x);+    } else if constexpr (std::is_same_v<scalar_t, std::uint32_t>) {+      return _mm256_set1_epi32(x);+    } else if constexpr (std::is_same_v<scalar_t, std::uint64_t>) {+      return _mm256_set1_epi64x(x);+    }+  }++  FOLLY_ERASE+  static logical_t equal(reg_t x, reg_t y) {+    if constexpr (std::is_same_v<scalar_t, std::uint8_t>) {+      return _mm256_cmpeq_epi8(x, y);+    } else if constexpr (std::is_same_v<scalar_t, std::uint16_t>) {+      return _mm256_cmpeq_epi16(x, y);+    } else if constexpr (std::is_same_v<scalar_t, std::uint32_t>) {+      return _mm256_cmpeq_epi32(x, y);+    } else if constexpr (std::is_same_v<scalar_t, std::uint64_t>) {+      return _mm256_cmpeq_epi64(x, y);+    }+  }++  FOLLY_ERASE+  static logical_t less_equal(reg_t x, reg_t y) {+    static_assert(+        std::is_same_v<std::uint8_t, scalar_t>, "other types not implemented");+    // See SSE comment+    reg_t min = _mm256_min_epu8(x, y);+    return _mm256_cmpeq_epi8(x, min);+  }++  FOLLY_ERASE+  static logical_t logical_or(logical_t x, logical_t y) {+    return _mm256_or_si256(x, y);+  }++  FOLLY_ERASE+  static bool any(logical_t log) { return simd::movemask<scalar_t>(log).first; }++#if 0 // disabled untill we have a test where this is relevant+  FOLLY_ERASE+  static bool all(logical_t log) {+    auto [bits, bitsPerElement] = movemask<scalar_t>(log);+    return movemask<scalar_t>(log) ==+        n_least_significant_bits<decltype(bits)>(kCardinal * bitsPerElement);+  }+#endif+};++template <typename T>+struct SimdAvx2Platform : SimdPlatformCommon<SimdAvx2PlatformSpecific<T>> {};++template <typename T>+using SimdPlatform = SimdAvx2Platform<T>;++#else++template <typename T>+using SimdPlatform = SimdSse42Platform<T>;++#endif++#elif FOLLY_AARCH64++template <typename T>+struct SimdAarch64PlatformSpecific {+  using scalar_t = T;++  FOLLY_ERASE+  static auto loadu(const scalar_t* p) {+    if constexpr (std::is_same_v<scalar_t, std::uint8_t>) {+      return vld1q_u8(p);+    } else if constexpr (std::is_same_v<scalar_t, std::uint16_t>) {+      return vld1q_u16(p);+    } else if constexpr (std::is_same_v<scalar_t, std::uint32_t>) {+      return vld1q_u32(p);+    } else if constexpr (std::is_same_v<scalar_t, std::uint64_t>) {+      return vld1q_u64(p);+    }+  }++  using reg_t = decltype(loadu(nullptr));+  using logical_t = reg_t;++  FOLLY_DISABLE_SANITIZERS+  FOLLY_ERASE+  static reg_t unsafeLoadu(const scalar_t* p) {+    if constexpr (std::is_same_v<scalar_t, std::uint8_t>) {+      return vld1q_u8(p);+    } else if constexpr (std::is_same_v<scalar_t, std::uint16_t>) {+      return vld1q_u16(p);+    } else if constexpr (std::is_same_v<scalar_t, std::uint32_t>) {+      return vld1q_u32(p);+    } else if constexpr (std::is_same_v<scalar_t, std::uint64_t>) {+      return vld1q_u64(p);+    }+  }++  FOLLY_ERASE+  static reg_t broadcast(scalar_t x) {+    if constexpr (std::is_same_v<scalar_t, std::uint8_t>) {+      return vdupq_n_u8(x);+    } else if constexpr (std::is_same_v<scalar_t, std::uint16_t>) {+      return vdupq_n_u16(x);+    } else if constexpr (std::is_same_v<scalar_t, std::uint32_t>) {+      return vdupq_n_u32(x);+    } else if constexpr (std::is_same_v<scalar_t, std::uint64_t>) {+      return vdupq_n_u64(x);+    }+  }++  FOLLY_ERASE+  static logical_t equal(reg_t x, reg_t y) {+    if constexpr (std::is_same_v<scalar_t, std::uint8_t>) {+      return vceqq_u8(x, y);+    } else if constexpr (std::is_same_v<scalar_t, std::uint16_t>) {+      return vceqq_u16(x, y);+    } else if constexpr (std::is_same_v<scalar_t, std::uint32_t>) {+      return vceqq_u32(x, y);+    } else if constexpr (std::is_same_v<scalar_t, std::uint64_t>) {+      return vceqq_u64(x, y);+    }+  }++  FOLLY_ERASE+  static logical_t less_equal(reg_t x, reg_t y) {+    if constexpr (std::is_same_v<scalar_t, std::uint8_t>) {+      return vcleq_u8(x, y);+    } else if constexpr (std::is_same_v<scalar_t, std::uint16_t>) {+      return vcleq_u16(x, y);+    } else if constexpr (std::is_same_v<scalar_t, std::uint32_t>) {+      return vcleq_u32(x, y);+    } else if constexpr (std::is_same_v<scalar_t, std::uint64_t>) {+      return vcleq_u64(x, y);+    }+  }++  FOLLY_ALWAYS_INLINE+  static logical_t logical_or(logical_t x, logical_t y) {+    if constexpr (std::is_same_v<scalar_t, std::uint8_t>) {+      return vorrq_u8(x, y);+    } else if constexpr (std::is_same_v<scalar_t, std::uint16_t>) {+      return vorrq_u16(x, y);+    } else if constexpr (std::is_same_v<scalar_t, std::uint32_t>) {+      return vorrq_u32(x, y);+    } else if constexpr (std::is_same_v<scalar_t, std::uint64_t>) {+      return vorrq_u64(x, y);+    }+  }++  FOLLY_ALWAYS_INLINE+  static bool any(logical_t log) {+    // https://github.com/dotnet/runtime/pull/75864+    auto u32 = bit_cast<uint32x4_t>(log);+    u32 = vpmaxq_u32(u32, u32);+    auto u64 = bit_cast<uint64x2_t>(u32);+    return vgetq_lane_u64(u64, 0);+  }++#if 0 // disabled untill we have a test where this is relevant+  FOLLY_ERASE+  static bool all(logical_t log) {+    // Not quite what they did in .Net runtime, but+    // should be close.+    // https://github.com/dotnet/runtime/pull/75864+    auto u32 = bit_cast<uint32x4_t>(log);+    u32 = vpminq_u32(u32, u32);+    auto u64 = bit_cast<uint64x2_t>(u32);+    return u64 == n_least_significant_bits<std::uint64_t>(64);+  }+#endif+};++#define FOLLY_DETAIL_HAS_SIMD_PLATFORM 1++template <typename T>+struct SimdAarch64Platform+    : SimdPlatformCommon<SimdAarch64PlatformSpecific<T>> {};++template <typename T>+using SimdPlatform = SimdAarch64Platform<T>;++#define FOLLY_DETAIL_HAS_SIMD_PLATFORM 1++#else++#define FOLLY_DETAIL_HAS_SIMD_PLATFORM 0++template <typename T>+using SimdPlatform = void;++#endif++} // namespace simd::detail+} // namespace folly
+ folly/folly/algorithm/simd/detail/Traits.h view
@@ -0,0 +1,126 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <folly/CPortability.h>+#include <folly/Memory.h>+#include <folly/Traits.h>+#include <folly/container/span.h>++#include <concepts>+#include <type_traits>++namespace folly::simd::detail {++template <typename T>+auto findSimdFriendlyEquivalent() {+  static_assert(std::is_same_v<T, remove_cvref_t<T>>);+  if constexpr (std::is_enum_v<T>) {+    return findSimdFriendlyEquivalent<std::underlying_type_t<T>>();+  } else if constexpr (std::is_pointer_v<T>) {+    // We use signed numbers for pointers because x86 support for signed+    // numbers is better and we can get away with it, in terms of correctness.+    return int_bits_t<sizeof(T) * 8>{};+  } else if constexpr (std::is_floating_point_v<T>) {+    if constexpr (sizeof(T) == 4) {+      return float{};+    } else {+      return double{};+    }+  } else if constexpr (std::is_signed_v<T>) {+    return int_bits_t<sizeof(T) * 8>{};+  } else if constexpr (std::is_unsigned_v<T>) {+    return uint_bits_t<sizeof(T) * 8>{};+  }+}++template <typename T>+constexpr bool has_simd_friendly_equivalent_scalar = !std::is_void_v<+    decltype(findSimdFriendlyEquivalent<std::remove_const_t<T>>())>;++template <typename T>+using simd_friendly_equivalent_scalar_t = std::enable_if_t<+    has_simd_friendly_equivalent_scalar<T>,+    like_t<T, decltype(findSimdFriendlyEquivalent<std::remove_const_t<T>>())>>;++template <typename T>+constexpr bool has_integral_simd_friendly_equivalent_scalar_v =+    std::is_integral_v< // void will return false+        decltype(findSimdFriendlyEquivalent<std::remove_const_t<T>>())>;++template <typename T>+using unsigned_simd_friendly_equivalent_scalar_t = std::enable_if_t<+    has_integral_simd_friendly_equivalent_scalar_v<T>,+    like_t<T, uint_bits_t<sizeof(T) * 8>>>;++template <typename R>+using span_for = decltype(folly::span(std::declval<const R&>()));++struct AsSimdFriendlyFn {+  template <typename T, std::size_t extent>+  FOLLY_ERASE auto operator()(folly::span<T, extent> s) const+      -> folly::span<simd_friendly_equivalent_scalar_t<T>, extent> {+    return reinterpret_span_cast<simd_friendly_equivalent_scalar_t<T>>(s);+  }++  template <typename R>+  FOLLY_ERASE auto operator()(R&& r) const+      -> decltype(operator()(span_for<R>(r))) {+    return operator()(folly::span(r));+  }++  template <typename T>+  FOLLY_ERASE constexpr auto operator()(T x) const+      -> simd_friendly_equivalent_scalar_t<T> {+    using res_t = simd_friendly_equivalent_scalar_t<T>;+    if constexpr (!std::is_pointer_v<T>) {+      return static_cast<res_t>(x);+    } else {+      return reinterpret_cast<res_t>(x);+    }+  }+};+inline constexpr AsSimdFriendlyFn asSimdFriendly;++struct AsSimdFriendlyUintFn {+  template <typename T, std::size_t extent>+  FOLLY_ERASE auto operator()(folly::span<T, extent> s) const+      -> folly::span<unsigned_simd_friendly_equivalent_scalar_t<T>, extent> {+    return reinterpret_span_cast<unsigned_simd_friendly_equivalent_scalar_t<T>>(+        s);+  }++  template <typename R>+  FOLLY_ERASE auto operator()(R&& r) const+      -> decltype(operator()(span_for<R>(r))) {+    return operator()(folly::span(r));+  }++  template <typename T>+  FOLLY_ERASE constexpr auto operator()(T x) const+      -> unsigned_simd_friendly_equivalent_scalar_t<T> {+    using res_t = unsigned_simd_friendly_equivalent_scalar_t<T>;+    if constexpr (!std::is_pointer_v<T>) {+      return static_cast<res_t>(x);+    } else {+      return reinterpret_cast<res_t>(x);+    }+  }+};+inline constexpr AsSimdFriendlyUintFn asSimdFriendlyUint;++} // namespace folly::simd::detail
+ folly/folly/algorithm/simd/detail/UnrollUtils.h view
@@ -0,0 +1,126 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <folly/Portability.h>+#include <folly/Traits.h>++#include <array>+#include <type_traits>++namespace folly::simd::detail {++/**+ * UnrollUtils+ *+ * Unfortunately compilers often don't unroll the loops with small+ * fixed number of iterations and/or not unroll them properly.+ *+ * This is a collection of helpers that use templates to do some+ * common unrolled loops.+ */+struct UnrollUtils {+ public:+  /**+   * arrayMap(x, op)+   *+   * Typical "map" from functional languages: apply op for each element,+   * return an array of results.+   */+  template <typename T, std::size_t N, typename Op>+  FOLLY_NODISCARD FOLLY_ALWAYS_INLINE static constexpr auto arrayMap(+      const std::array<T, N>& x, Op op) {+    return arrayMapImpl(x, op, std::make_index_sequence<N>());+  }++  /**+   * arrayReduce(x, op)+   *+   * std::reduce(x.begin(), x.end(), op) but unrolled and orders operations+   * to minimize dependencies.+   *+   * (a + b) + (c + d)+   */+  template <typename T, std::size_t N, typename Op>+  FOLLY_NODISCARD FOLLY_ALWAYS_INLINE static constexpr T arrayReduce(+      const std::array<T, N>& x, Op op) {+    return arrayReduceImpl<0, N>(x, op);+  }++  /**+   * unrollUntil<N>(op)+   *+   *  Do operation N times or until it returns true to break.+   *  Op accepts integral_constant<i> so it can keep track of a step begin+   * executed.+   *+   *  Returns wether true if it was interrupted (you can know if the op breaked)+   */+  template <std::size_t N, typename Op>+  FOLLY_ALWAYS_INLINE static constexpr bool unrollUntil(Op op) {+    return unrollUntilImpl(op, std::make_index_sequence<N>{});+  }++ private:+  template <typename T, std::size_t N, typename Op, std::size_t... i>+  FOLLY_ALWAYS_INLINE static constexpr auto arrayMapImpl(+      const std::array<T, N>& x, Op op, std::index_sequence<i...>) {+    using U = decltype(op(std::declval<const T&>()));++    FOLLY_PUSH_WARNING+    // This is a very common gcc issue,+    // https://gcc.gnu.org/bugzilla/show_bug.cgi?id=97222 apparently discarding+    // it here is fine and done through out.+    FOLLY_GCC_DISABLE_WARNING("-Wignored-attributes")+    std::array<U, N> res{{op(x[i])...}};+    FOLLY_POP_WARNING+    return res;+  }++  template <+      std::size_t f,+      std::size_t l,+      typename T,+      std::size_t N,+      typename Op>+  FOLLY_ALWAYS_INLINE static constexpr std::enable_if_t<l - f == 1, T>+  arrayReduceImpl(std::array<T, N> const& x, Op) {+    return x[f];+  }++  template <+      std::size_t f,+      std::size_t l,+      typename T,+      std::size_t N,+      typename Op>+  FOLLY_ALWAYS_INLINE static constexpr std::enable_if_t<l - f != 1, T>+  arrayReduceImpl(std::array<T, N> const& x, Op op) {+    constexpr std::size_t n = l - f;+    T leftSum = arrayReduceImpl<f, f + n / 2>(x, op);+    T rightSum = arrayReduceImpl<f + n / 2, l>(x, op);+    return op(leftSum, rightSum);+  }++  template <typename Op, std::size_t... i>+  FOLLY_ALWAYS_INLINE static constexpr bool unrollUntilImpl(+      Op op, std::index_sequence<i...>) {+    return (... || op(index_constant<i>{}));+  }+};++} // namespace folly::simd::detail
+ folly/folly/algorithm/simd/find_first_of.h view
@@ -0,0 +1,671 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <algorithm>+#include <array>+#include <cstddef>+#include <cstdint>++#include <folly/Portability.h>+#include <folly/Utility.h>+#include <folly/algorithm/simd/Movemask.h>+#include <folly/container/SparseByteSet.h>+#include <folly/container/span.h>+#include <folly/lang/Align.h>+#include <folly/lang/Bits.h>+#include <folly/lang/Hint.h>++#if FOLLY_SSE+#include <immintrin.h>+#endif++#if FOLLY_NEON+#include <arm_neon.h>+#endif++#if FOLLY_ARM_FEATURE_SVE+#include <arm_sve.h>+#if __has_include(<arm_neon_sve_bridge.h>)+#include <arm_neon_sve_bridge.h> // @manual+#endif+#endif++namespace folly::simd {++namespace detail {++/// stdfind_scalar_finder_first_of+///+/// A find-first-of finder which simply wraps std::find.+template <typename CharT>+class stdfind_scalar_finder_first_of {+ private:+  using value_type = CharT;+  using view = span<CharT const>;++  alignas(sizeof(view)) view const alphabet_;++ public:+  constexpr explicit stdfind_scalar_finder_first_of(+      view const alphabet) noexcept+      : alphabet_{alphabet} {}++  size_t operator()(view const input, size_t const pos = 0) const noexcept {+    auto const r = std::find_first_of(+        input.subspan(pos).begin(),+        input.end(),+        alphabet_.begin(),+        alphabet_.end());+    return r - input.begin();+  }+};++/// default_scalar_finder_first_of+///+/// A find-first-of finder which, for each element of the input, iterates the+/// search alphabet. Has complexity O(MN), with M the length of the alphabet and+/// with N the length of the input.+///+/// Requires no precomputation or storage.+template <typename CharT, bool Eq>+class default_scalar_finder_first_op_of {+ private:+  using value_type = CharT;+  using view = span<CharT const>;++  alignas(sizeof(view)) view alphabet_;++  bool match(value_type const c) const noexcept {+    bool ret = !Eq;+    for (auto const a : alphabet_) {+      auto const v = a == c;+      ret = Eq ? ret || v : ret && !v;+    }+    return ret;+  }++ public:+  constexpr explicit default_scalar_finder_first_op_of(+      view const alphabet) noexcept+      : alphabet_{alphabet} {}++  size_t operator()(view const input, size_t const pos = 0) const noexcept {+    for (size_t i = pos; i < input.size(); ++i) {+      if (match(input[i])) {+        return i;+      }+    }+    return input.size();+  }+};++/// ltindex_scalar_finder_first_of+///+/// A find-first-of finder which, for each element of the input, looks up that+/// element in a lookup table. Has complexity O(N), with N the length of the+/// input.+///+/// Precomputes and stores a 256-byte lookup table. Precomputation has+/// complexity O(M), with M the length of the alphabet.+///+/// Restricted to elements which are 1 byte wide.+template <typename CharT, bool Eq>+class ltindex_scalar_finder_first_op_of {+ private:+  using value_type = CharT;+  using view = span<CharT const>;+  using index = std::array<bool, 256>;++  static_assert(sizeof(value_type) == 1);++  alignas(hardware_destructive_interference_size) index const ltindex_;++  static constexpr index make_index(view const alphabet) noexcept {+    index ltindex{};+    for (auto const a : alphabet) {+      ltindex[static_cast<uint8_t>(a)] = true;+    }+    return ltindex;+  }++  bool match(value_type const c) const noexcept {+    return Eq == ltindex_[static_cast<uint8_t>(c)];+  }++ public:+  constexpr explicit ltindex_scalar_finder_first_op_of(+      view const alphabet) noexcept+      : ltindex_{make_index(alphabet)} {}++  size_t operator()(view const input, size_t const pos = 0) const noexcept {+    for (size_t i = pos; i < input.size(); ++i) {+      if (match(input[i])) {+        return i;+      }+    }+    return input.size();+  }+};++/// ltsparse_scalar_finder_first_of+///+/// A find-first-of finder which, for each element of the input, looks up that+/// element in a lookup table. Has complexity O(M+N), with M the length of the+/// alphabet and with N the length of the input.+///+/// Similar to ltindex_scalar_finder_first_of, but where the precomputation is+/// instead done at the beginning of each search using an alternative set type.+/// This alternative set type has lower setup cost but higher lookup cost as+/// compared with the set type in ltindex_scalar_finder_first_of, making this+/// implementation more suitable for unpredictable alphabets.+///+/// Requires no precomputation or storage.+///+/// Restricted to elements which are 1 byte wide.+template <typename CharT, bool Eq>+class ltsparse_scalar_finder_first_op_of {+ private:+  using value_type = CharT;+  using view = span<CharT const>;++  static_assert(sizeof(value_type) == 1);++  alignas(sizeof(view)) view alphabet_;++  void prep(SparseByteSet& set) const noexcept {+    for (auto const a : alphabet_) {+      set.add(static_cast<uint8_t>(a));+    }+  }++  bool match(value_type const c, SparseByteSet const& set) const noexcept {+    return Eq == set.contains(static_cast<uint8_t>(c));+  }++ public:+  constexpr explicit ltsparse_scalar_finder_first_op_of(+      view const alphabet) noexcept+      : alphabet_{alphabet} {}++  size_t operator()(view const input, size_t const pos = 0) const noexcept {+    [[FOLLY_ATTR_CLANG_UNINITIALIZED]] SparseByteSet set;+    prep(set);+    for (size_t i = pos; i < input.size(); ++i) {+      if (match(input[i], set)) {+        return i;+      }+    }+    return input.size();+  }+};++/// default_vector_finder_first_of+///+/// A find-first-of finder which, for each element of the input, iterates the+/// search alphabet. Has complexity O(MN), with M the length of the alphabet and+/// with N the length of the input.+///+/// Like default_scalar_finder_first_of, but accelerated with simd instructions+/// to search up to 16 elements of the input at a time.+///+/// Requires no precomputation or storage.+///+/// Restricted to elements which are 1 byte wide.+///+/// Implemented for x86-64 and aarch64 architectures.+///+/// Requires a fallback scalar finder for not-implemented-architecture and for+/// near-end-of-input.+template <typename CharT, bool Eq>+class default_vector_finder_first_op_of {+ private:+  using value_type = CharT;+  using view = span<CharT const>;++  static_assert(sizeof(value_type) == 1);++  alignas(sizeof(view)) view const alphabet_;++ public:+  constexpr explicit default_vector_finder_first_op_of(+      view const alphabet) noexcept+      : alphabet_{alphabet} {}++  template <typename Scalar>+  size_t operator()(+      Scalar const& scalar,+      view const input,+      size_t const pos = 0) const noexcept {+    return operator()(scalar, true, input, pos);+  }++  template <typename Scalar>+  size_t operator()(+      Scalar const& scalar,+      bool const vector,+      view const input,+      size_t const pos = 0) const noexcept {+    size_t size = pos;+    if (vector) {+#if (FOLLY_SSE >= 2 || (FOLLY_NEON && FOLLY_AARCH64))+      while (input.size() >= size + 16) {+#if FOLLY_SSE+        auto const vhaystack = _mm_loadu_si128(+            reinterpret_cast<__m128i const*>(input.data() + size));+        auto vmask = _mm_set1_epi8(Eq ? 0 : -1);+        for (auto const a : alphabet_) {+          auto const veq = _mm_cmpeq_epi8(vhaystack, _mm_set1_epi8(a));+          vmask = Eq ? _mm_or_si128(veq, vmask) : _mm_andnot_si128(veq, vmask);+        }+#elif FOLLY_NEON+        auto const vhaystack =+            vld1q_u8(reinterpret_cast<uint8_t const*>(input.data() + size));+        auto vmask = vdupq_n_u8(Eq ? 0 : -1);+        for (auto const a : alphabet_) {+          auto const veq = vhaystack == vdupq_n_u8(a);+          vmask = Eq ? veq | vmask : ~veq & vmask;+        }+#endif+        if (auto const [word, bits] = movemask<CharT>(vmask); word) {+          return size + to_signed((findFirstSet(word) - 1) / bits);+        }+        size += 16;+      }+      if (input.size() < size) {+        compiler_may_unsafely_assume_unreachable();+      }+#endif+    }+    return scalar(input, size);+  }+};++/// shuffle_vector_finder_first_of+///+/// A find-first-of finder which, for each element of the input, looks up that+/// element in a lookup table. Has complexity O(MN), with M the length of the+/// alphabet after deduplication and with N the length of the input.+///+/// Precomputes and stores a 256-byte lookup table. Precomputation has+/// complexity O(M), with M the length of the alphabet.+///+/// Like ltindex_scalar_finder_first_of, but accelerated with simd instructions+/// to search up to 16 elements of the input at a time and decelerated by+/// splitting the lookup table into a sequence of lookup tables of length O(M).+///+/// Restricted to elements which are 1 byte wide.+///+/// Implemented for x86-64 and aarch64 architectures.+///+/// Requires a fallback scalar finder for not-implemented-architecture and for+/// near-end-of-input.+template <typename CharT, bool Eq>+class shuffle_vector_finder_first_op_of {+ private:+  using value_type = CharT;+  using view = span<CharT const>;+  using shufvec = std::array<value_type, 256>;++  struct shuffle {+    shufvec table;+    size_t rounds;+  };++  static_assert(sizeof(value_type) == 1);++  //  invariant: (a in alphabet) <=> (exists k : shufvec[k * 16 + a % 16] = a)+  alignas(hardware_destructive_interference_size) shuffle const shuffle_;++  //  mimic: std::exchange (constexpr), C++20+  template <typename T, typename U = T>+  static constexpr T exchange(T& obj, U&& val) noexcept {+    auto ret = std::move(obj);+    obj = std::forward<U>(val);+    return ret;+  }++  static constexpr shuffle make_shuffle(view const alphabet) noexcept {+    //  init requires: forall k, a : result[k * 16 + a % 16] != a+    shufvec table{1}; // 1, 0, 0, ...+    size_t maxk{};++    std::array<bool, 256> seen{};+    std::array<size_t, 16> lo_seen{};++    for (auto const a : alphabet) {+      auto const v = static_cast<uint8_t>(a);+      if (!exchange(seen[v], true)) {+        auto const k = lo_seen[v % 16]++;+        maxk = maxk < k ? k : maxk;+        table[k * 16 + v % 16] = v;+      }+    }++    return {table, maxk + 1};+  }++ public:+  constexpr explicit shuffle_vector_finder_first_op_of(+      view const alphabet) noexcept+      : shuffle_{make_shuffle(alphabet)} {}++  template <typename Scalar>+  size_t operator()(+      Scalar const& scalar,+      view const input,+      size_t const pos = 0) const noexcept {+    return operator()(scalar, true, input, pos);+  }++  template <typename Scalar>+  size_t operator()(+      Scalar const& scalar,+      bool const vector,+      view const input,+      size_t const pos = 0) const noexcept {+    size_t size = pos;+    if (vector) {+#if ((FOLLY_SSE >= 2 && FOLLY_SSSE >= 3) || (FOLLY_NEON && FOLLY_AARCH64))+      auto const table = shuffle_.table.data();+      while (input.size() >= size + 16) {+#if FOLLY_SSE+        auto const vtable = reinterpret_cast<__m128i const*>(table);+        auto const vhaystack = _mm_loadu_si128(+            reinterpret_cast<__m128i const*>(input.data() + size));+        auto const vhaystackm = _mm_and_si128(vhaystack, _mm_set1_epi8(15));+        auto vmask = _mm_set1_epi8(Eq ? 0 : -1);+        for (size_t i = 0; i < shuffle_.rounds; ++i) {+          auto const vshuffle = _mm_shuffle_epi8(vtable[i], vhaystackm);+          auto const veq = _mm_cmpeq_epi8(vshuffle, vhaystack);+          vmask = Eq ? _mm_or_si128(veq, vmask) : _mm_andnot_si128(veq, vmask);+        }+#elif FOLLY_NEON+        auto const vtable = reinterpret_cast<uint8x16_t const*>(table);+        auto const vhaystack =+            vld1q_u8(reinterpret_cast<uint8_t const*>(input.data() + size));+        auto vmask = vdupq_n_u8(Eq ? 0 : -1);+        for (size_t i = 0; i < shuffle_.rounds; ++i) {+          auto const veq = vqtbl1q_u8(vtable[i], vhaystack & 15) == vhaystack;+          vmask = Eq ? veq | vmask : ~veq & vmask;+        }+#endif+        if (auto const [word, bits] = movemask<CharT>(vmask); word) {+          return size + to_signed((findFirstSet(word) - 1) / bits);+        }+        size += 16;+      }+      if (input.size() < size) {+        compiler_may_unsafely_assume_unreachable();+      }+#endif+    }+    return scalar(input, size);+  }+};++/// azmatch_vector_finder_first_of+///+/// A find-first-of finder which, for each element of the input, looks up that+/// element in a lookup table. Has complexity O(MN), with M the length of the+/// alphabet after deduplication and with N the length of the input.+///+/// Precomputes and stores a 256-byte lookup table. Precomputation has+/// complexity O(M), with M the length of the alphabet.+///+/// Like ltindex_scalar_finder_first_of, but accelerated with simd instructions+/// to search up to 16 elements of the input at a time and decelerated by+/// splitting the lookup table into a sequence of lookup tables of length O(M).+///+/// Like ltindex_vector_finder_first_of, but with a different technique.+///+/// Restricted to elements which are 1 byte wide.+///+/// Implemented for aarch64 architectures with sve.+///+/// Requires a fallback scalar finder for not-implemented-architecture and for+/// near-end-of-input.+template <typename CharT, bool Eq>+class azmatch_vector_finder_first_op_of {+ private:+  using value_type = CharT;+  using view = span<CharT const>;+  using matchvec = std::array<value_type, 256>;++  struct meta {+    matchvec table;+    size_t rounds;+  };++  static_assert(sizeof(value_type) == 1);++  alignas(hardware_destructive_interference_size) meta const meta_;++  //  mimic: std::exchange (constexpr), C++20+  template <typename T, typename U = T>+  static constexpr T exchange(T& obj, U&& val) noexcept {+    auto ret = std::move(obj);+    obj = std::forward<U>(val);+    return ret;+  }++  static constexpr size_t next_segment(+      view& alphabet, span<value_type, 16> out, span<bool, 256> seen) noexcept {+    if (!alphabet.size()) {+      return 0;+    }+    for (size_t i = 0; i < 16; ++i) {+      out[i] = alphabet[0];+    }+    size_t items = 0;+    while (items < 16 && alphabet.size()) {+      auto const v = static_cast<uint8_t>(alphabet[0]);+      alphabet = alphabet.subspan(1);+      if (!exchange(seen[v], true)) {+        out[items++] = v;+      }+    }+    return items;+  }++  static constexpr meta make_meta(view alphabet) noexcept {+    std::array<bool, 256> seen{};+    size_t rounds = 0;+    matchvec vec{};+    while (true) {+      auto segment = span<value_type, 16>{vec.data() + 16 * rounds, 16};+      auto segsize = next_segment(alphabet, segment, seen);+      if (!segsize) {+        break;+      }+      ++rounds;+    }+    return meta{vec, rounds};+  }++#if FOLLY_ARM_FEATURE_SVE+  static auto svld1_u8_nopred_16(uint8_t const* p) noexcept {+#if __has_include(<arm_neon_sve_bridge.h>)+    return svset_neonq_u8(svundef_u8(), vld1q_u8(p));+#else+    return svld1_u8(svptrue_pat_b8(SV_VL16), p);+#endif+  }+#endif++ public:+  constexpr explicit azmatch_vector_finder_first_op_of(+      view const alphabet) noexcept+      : meta_{make_meta(alphabet)} {}++  template <typename Scalar>+  size_t operator()(+      Scalar const& scalar,+      view const input,+      size_t const pos = 0) const noexcept {+    return operator()(scalar, true, input, pos);+  }++  template <typename Scalar>+  size_t operator()(+      Scalar const& scalar,+      bool const vector,+      view const input,+      size_t const pos = 0) const noexcept {+    size_t size = pos;+    if (vector) {+#if FOLLY_ARM_FEATURE_SVE+      auto const table = reinterpret_cast<uint8_t const*>(meta_.table.data());+      while (input.size() >= size + 16) {+        auto const pred = svptrue_b8();+        auto const vhaystack = svld1_u8_nopred_16(+            reinterpret_cast<uint8_t const*>(input.data() + size));+        auto vmask = Eq ? svpfalse_b() : pred;+        for (size_t i = 0; i < meta_.rounds; ++i) {+          auto const vsegment = svld1_u8_nopred_16(table + 16 * i);+          vmask = Eq+              ? svorr_b_z(pred, vmask, svmatch_u8(pred, vhaystack, vsegment))+              : svand_b_z(pred, vmask, svnmatch_u8(pred, vhaystack, vsegment));+        }+        // an important optimization that llvm-17 *could*, but doesn't, do for+        // sve+        if (meta_.rounds == 1) {+          auto const vsegment = svld1_u8_nopred_16(table);+          vmask = Eq+              ? svmatch_u8(pred, vhaystack, vsegment)+              : svnmatch_u8(pred, vhaystack, vsegment);+        }+        auto const count = svcntp_b8(pred, svbrkb_b_z(pred, vmask));+        if (count < 16) {+          return size + count;+        }+        size += 16;+      }+      if (input.size() < size) {+        compiler_may_unsafely_assume_unreachable();+      }+#endif+    }+    return scalar(input, size);+  }+};++} // namespace detail++template <typename CharT>+using basic_stdfind_scalar_finder_first_of =+    detail::stdfind_scalar_finder_first_of<CharT>;++template <typename CharT>+using basic_default_scalar_finder_first_of =+    detail::default_scalar_finder_first_op_of<CharT, true>;+template <typename CharT>+using basic_default_scalar_finder_first_not_of =+    detail::default_scalar_finder_first_op_of<CharT, false>;++template <typename CharT>+using basic_ltindex_scalar_finder_first_of =+    detail::ltindex_scalar_finder_first_op_of<CharT, true>;+template <typename CharT>+using basic_ltindex_scalar_finder_first_not_of =+    detail::ltindex_scalar_finder_first_op_of<CharT, false>;++template <typename CharT>+using basic_ltsparse_scalar_finder_first_of =+    detail::ltsparse_scalar_finder_first_op_of<CharT, true>;+template <typename CharT>+using basic_ltsparse_scalar_finder_first_not_of =+    detail::ltsparse_scalar_finder_first_op_of<CharT, false>;++template <typename CharT>+using basic_default_vector_finder_first_of =+    detail::default_vector_finder_first_op_of<CharT, true>;+template <typename CharT>+using basic_default_vector_finder_first_not_of =+    detail::default_vector_finder_first_op_of<CharT, false>;++template <typename CharT>+using basic_shuffle_vector_finder_first_of =+    detail::shuffle_vector_finder_first_op_of<CharT, true>;+template <typename CharT>+using basic_shuffle_vector_finder_first_not_of =+    detail::shuffle_vector_finder_first_op_of<CharT, false>;++template <typename CharT>+using basic_azmatch_vector_finder_first_of =+    detail::azmatch_vector_finder_first_op_of<CharT, true>;+template <typename CharT>+using basic_azmatch_vector_finder_first_not_of =+    detail::azmatch_vector_finder_first_op_of<CharT, false>;++using stdfind_scalar_finder_first_of =+    basic_stdfind_scalar_finder_first_of<char>;++using default_scalar_finder_first_of =+    basic_default_scalar_finder_first_of<char>;+using default_scalar_finder_first_not_of =+    basic_default_scalar_finder_first_not_of<char>;++using ltindex_scalar_finder_first_of =+    basic_ltindex_scalar_finder_first_of<char>;+using ltindex_scalar_finder_first_not_of =+    basic_ltindex_scalar_finder_first_not_of<char>;++using ltsparse_scalar_finder_first_of =+    basic_ltsparse_scalar_finder_first_of<char>;+using ltsparse_scalar_finder_first_not_of =+    basic_ltsparse_scalar_finder_first_not_of<char>;++using default_vector_finder_first_of =+    basic_default_vector_finder_first_of<char>;+using default_vector_finder_first_not_of =+    basic_default_vector_finder_first_not_of<char>;++using shuffle_vector_finder_first_of =+    basic_shuffle_vector_finder_first_of<char>;+using shuffle_vector_finder_first_not_of =+    basic_shuffle_vector_finder_first_not_of<char>;++using azmatch_vector_finder_first_of =+    basic_azmatch_vector_finder_first_of<char>;+using azmatch_vector_finder_first_not_of =+    basic_azmatch_vector_finder_first_not_of<char>;++/// composite_finder+///+/// A find-first-of finder which composes a vector finder with a scalar finder.+///+/// A vector finder requires a scalar finder for not-implemented-architecture+/// and for near-end-of-input. This combinator producers a finder which uses the+/// vector finder where possible and otherwise falls back to the scalar finder.+template <typename Vector, typename Scalar>+class composite_finder_first_of : private Vector, Scalar {+ private:+  using view = span<char const>;++ public:+  constexpr explicit composite_finder_first_of(view const alphabet) noexcept+      : Vector{alphabet}, Scalar{alphabet} {}++  size_t operator()(view const input, size_t const pos = 0) const noexcept {+    auto const& vector = static_cast<Vector const&>(*this);+    auto const& scalar = static_cast<Scalar const&>(*this);+    return vector(scalar, input, pos);+  }+};++} // namespace folly::simd
+ folly/folly/algorithm/simd/find_first_of_extra.h view
@@ -0,0 +1,126 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <algorithm>+#include <cstddef>+#include <cstdint>+#include <execution>++#include <folly/Range.h>+#include <folly/container/span.h>++namespace folly::simd {++namespace detail {++#if __cpp_lib_execution >= 201902L++/// stdfind_vector_finder_first_of+///+/// A find-first-of finder which simply wraps std::find_first_of with an+/// execution policy.+///+/// Like stdfind_scalar_finder_first_of, but potentially accelerated. Depends+/// on the implementation.+///+/// Requires no precomputation or storage.+///+/// Extracted to a separate facility since this has high startup cost and to+/// isolate the dependency on tbb which libstdc++ brings.+///+/// Only supports positive-match (find-first-of) and not negative-match+/// (find-first-not-of) since std::find+template <typename CharT>+class stdfind_vector_finder_first_of {+ private:+  using value_type = CharT;+  using view = span<CharT const>;++  alignas(sizeof(view)) view const alphabet_;++ public:+  constexpr explicit stdfind_vector_finder_first_of(+      view const alphabet) noexcept+      : alphabet_{alphabet} {}++  size_t operator()(view const input, size_t const pos = 0) const noexcept {+    auto const r = std::find_first_of(+        std::execution::unseq,+        input.subspan(pos).begin(),+        input.end(),+        alphabet_.begin(),+        alphabet_.end());+    return r - input.begin();+  }+};++#endif++/// rngfind_vector_finder_first_of+///+/// A find-first-of finder which simply wraps folly::Range::find_first_of. This+/// algorithm has its own internal acceleration.+///+/// Requires no precomputation or storage.+///+/// Implemented for x86-64 architecture.+///+/// Extracted to a separate facility since this has high startup cost.+///+/// Only supports positive-match (find-first-of) and not negative-match+/// (find-first-not-of) since std::find+template <typename CharT>+class rngfind_vector_finder_first_of {+ private:+  using value_type = CharT;+  using view = span<CharT const>;++  alignas(sizeof(view)) view const alphabet_;++ public:+  constexpr explicit rngfind_vector_finder_first_of(+      view const alphabet) noexcept+      : alphabet_{alphabet} {}++  size_t operator()(view const input, size_t const pos = 0) const noexcept {+    auto const r = crange(input).find_first_of(crange(alphabet_), pos);+    return r == size_t(-1) ? input.size() : r;+  }+};++} // namespace detail++#if __cpp_lib_execution >= 201902L++template <typename CharT>+using basic_stdfind_vector_finder_first_of =+    detail::stdfind_vector_finder_first_of<CharT>;++using stdfind_vector_finder_first_of =+    basic_stdfind_vector_finder_first_of<char>;++#endif++template <typename CharT>+using basic_rngfind_vector_finder_first_of =+    detail::rngfind_vector_finder_first_of<CharT>;++using rngfind_vector_finder_first_of =+    basic_rngfind_vector_finder_first_of<char>;++} // namespace folly::simd
+ folly/folly/base64.h view
@@ -0,0 +1,275 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <cstdint>+#include <stdexcept>+#include <string>+#include <string_view>+#include <vector>+#include <folly/CPortability.h>+#include <folly/Portability.h>+#include <folly/detail/base64_detail/Base64Api.h>+#include <folly/detail/base64_detail/Base64Common.h>+#include <folly/lang/Exception.h>+#include <folly/memory/UninitializedMemoryHacks.h>++namespace folly {++//+// base64 encoding/decoding+//+// There are a few variations of base64 encoding.+//+// We have 2: base64 and base64URL.+//+// base64 uses '+' '/' for encoding 62 and 63 and uses '=' padding symbol.+// (padding symbols are required on decoding)+//+// base64URL uses '-' '_' for encoding 62 and 63 and has no padding.+// Decoding with base64URL will accept both base64 and base64URL encoded data ++// padding is always optional.+//+// SIMD implementation is based on 0x80 blog.+// See details explained in folly/detail/base64_detail/README.md+//++//+// High level API.+// Encoding never fails, except for allocation.+// Decoding will throw base64_decode_error if it fails.+//+// NOTE: the expection does not contain detailed information+//       about the error because keeping track of that is overhead.+//       We can potentially improve error reporting by doing a second+//       pass if we decide that it's benefitial.++struct base64_decode_error;++inline auto base64Encode(std::string_view s) -> std::string;+inline auto base64Decode(std::string_view s) -> std::string;+inline auto base64URLEncode(std::string_view s) -> std::string;+inline auto base64URLDecode(std::string_view s) -> std::string;++// Low level API.+//+// This API does not throw and is constexpr enabled.+//+// Encode returns a pointer past the last the byte written+// Decode returns a struct with `is_success` flag and the pointer `o`+// past the last char written.+//+// NOTE: decode will not stop writing when encountering a failure+//       and can always write up to size.+//+// NOTE: since on C++17 we cannot always adequately determine if+//       the function is running in compile time or not,+//       we provide explicit runime versions too.++constexpr std::size_t base64EncodedSize(std::size_t inSize) noexcept;+constexpr std::size_t base64URLEncodedSize(std::size_t inSize) noexcept;++inline constexpr char* base64Encode(+    const char* f, const char* l, char* o) noexcept;+inline constexpr char* base64URLEncode(+    const char* f, const char* l, char* o) noexcept;++inline char* base64EncodeRuntime(+    const char* f, const char* l, char* o) noexcept;+inline char* base64URLEncodeRuntime(+    const char* f, const char* l, char* o) noexcept;++constexpr std::size_t base64DecodedSize(const char* f, const char* l) noexcept;+constexpr std::size_t base64DecodedSize(std::string_view s) noexcept;++constexpr std::size_t base64URLDecodedSize(+    const char* f, const char* l) noexcept;+constexpr std::size_t base64URLDecodedSize(std::string_view s) noexcept;++struct base64_decode_result {+  bool is_success;+  char* o;+};++inline constexpr base64_decode_result base64Decode(+    const char* f, const char* l, char* o) noexcept;+inline constexpr base64_decode_result base64Decode(+    std::string_view s, char* o) noexcept;++inline constexpr base64_decode_result base64URLDecode(+    const char* f, const char* l, char* o) noexcept;+inline constexpr base64_decode_result base64URLDecode(+    std::string_view s, char* o) noexcept;++inline base64_decode_result base64DecodeRuntime(+    const char* f, const char* l, char* o) noexcept;+inline base64_decode_result base64DecodeRuntime(+    std::string_view s, char* o) noexcept;++inline base64_decode_result base64URLDecodeRuntime(+    const char* f, const char* l, char* o) noexcept;+inline base64_decode_result base64URLDecodeRuntime(+    std::string_view s, char* o) noexcept;++// -----------------------------------------------------------------+// implementation++struct base64_decode_error : std::runtime_error {+  using std::runtime_error::runtime_error;+};++constexpr std::size_t base64EncodedSize(std::size_t inSize) noexcept {+  return detail::base64_detail::base64EncodedSize(inSize);+}++constexpr std::size_t base64URLEncodedSize(std::size_t inSize) noexcept {+  return detail::base64_detail::base64URLEncodedSize(inSize);+}++inline constexpr char* base64Encode(+    const char* f, const char* l, char* o) noexcept {+  return detail::base64_detail::base64Encode(f, l, o);+}++inline constexpr char* base64URLEncode(+    const char* f, const char* l, char* o) noexcept {+  return detail::base64_detail::base64URLEncode(f, l, o);+}++inline char* base64EncodeRuntime(+    const char* f, const char* l, char* o) noexcept {+  return detail::base64_detail::base64EncodeRuntime(f, l, o);+}++inline char* base64URLEncodeRuntime(+    const char* f, const char* l, char* o) noexcept {+  return detail::base64_detail::base64URLEncodeRuntime(f, l, o);+}++inline std::string base64Encode(std::string_view s) {+  std::string res;+  std::size_t resSize = folly::base64EncodedSize(s.size());+  folly::resizeWithoutInitialization(res, resSize);+  folly::base64EncodeRuntime(s.data(), s.data() + s.size(), res.data());+  return res;+}++inline std::string base64URLEncode(std::string_view s) {+  std::string res;+  std::size_t resSize = folly::base64URLEncodedSize(s.size());+  folly::resizeWithoutInitialization(res, resSize);+  folly::base64URLEncodeRuntime(s.data(), s.data() + s.size(), res.data());+  return res;+}++constexpr std::size_t base64DecodedSize(const char* f, const char* l) noexcept {+  return detail::base64_detail::base64DecodedSize(f, l);+}++constexpr std::size_t base64DecodedSize(std::string_view s) noexcept {+  return folly::base64DecodedSize(s.data(), s.data() + s.size());+}++constexpr std::size_t base64URLDecodedSize(+    const char* f, const char* l) noexcept {+  return detail::base64_detail::base64URLDecodedSize(f, l);+}++constexpr std::size_t base64URLDecodedSize(std::string_view s) noexcept {+  return folly::base64URLDecodedSize(s.data(), s.data() + s.size());+}++inline constexpr base64_decode_result base64Decode(+    const char* f, const char* l, char* o) noexcept {+  auto detailResult = detail::base64_detail::base64Decode(f, l, o);+  return {detailResult.isSuccess, detailResult.o};+}++inline constexpr base64_decode_result base64Decode(+    std::string_view s, char* o) noexcept {+  return folly::base64Decode(s.data(), s.data() + s.size(), o);+}++inline constexpr base64_decode_result base64URLDecode(+    const char* f, const char* l, char* o) noexcept {+  auto detailResult = detail::base64_detail::base64URLDecode(f, l, o);+  return {detailResult.isSuccess, detailResult.o};+}++inline constexpr base64_decode_result base64URLDecode(+    std::string_view s, char* o) noexcept {+  return folly::base64URLDecode(s.data(), s.data() + s.size(), o);+}++inline base64_decode_result base64DecodeRuntime(+    const char* f, const char* l, char* o) noexcept {+  auto detailResult = detail::base64_detail::base64DecodeRuntime(f, l, o);+  return {detailResult.isSuccess, detailResult.o};+}++inline base64_decode_result base64DecodeRuntime(+    std::string_view s, char* o) noexcept {+  return folly::base64DecodeRuntime(s.data(), s.data() + s.size(), o);+}++inline base64_decode_result base64URLDecodeRuntime(+    const char* f, const char* l, char* o) noexcept {+  auto detailResult = detail::base64_detail::base64URLDecodeRuntime(f, l, o);+  return {detailResult.isSuccess, detailResult.o};+}++inline base64_decode_result base64URLDecodeRuntime(+    std::string_view s, char* o) noexcept {+  return folly::base64URLDecodeRuntime(s.data(), s.data() + s.size(), o);+}++// NOTE: for resizeWithoutInitialization we don't need to declare the macros,+//       since we are using char which is already included by default.+inline std::string base64Decode(std::string_view s) {+  std::string res;+  std::size_t resSize = folly::base64DecodedSize(s);+  folly::resizeWithoutInitialization(res, resSize);++  if (!folly::base64DecodeRuntime(s, res.data()).is_success) {+    folly::throw_exception<base64_decode_error>("Base64 Decoding failed");+  }+  return res;+}++inline std::string base64URLDecode(std::string_view s) {+  std::string res;+  std::size_t resSize = folly::base64URLDecodedSize(s);+  folly::resizeWithoutInitialization(res, resSize);++  if (!folly::base64URLDecodeRuntime(s, res.data()).is_success) {+    folly::throw_exception<base64_decode_error>("Base64URL Decoding failed");+  }+  return res;+}++inline bool isBase64URL(std::string_view s) {+  std::string res;+  std::size_t resSize = folly::base64URLDecodedSize(s);+  folly::resizeWithoutInitialization(res, resSize);++  if (!folly::base64URLDecodeRuntime(s, res.data()).is_success) {+    return false;+  }+  return true;+}++} // namespace folly
+ folly/folly/channels/Channel-fwd.h view
@@ -0,0 +1,51 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <folly/experimental/channels/detail/ChannelBridge.h>++#include <optional>++namespace folly {+namespace channels {++template <typename TValue>+class Receiver;++template <typename TValue>+class Sender;++namespace detail {+template <typename TValue>+ChannelBridgePtr<TValue>& senderGetBridge(Sender<TValue>& sender);++template <typename TValue>+bool receiverWait(+    Receiver<TValue>& receiver, detail::IChannelCallback* receiverCallback);++template <typename TValue>+detail::IChannelCallback* cancelReceiverWait(Receiver<TValue>& receiver);++template <typename TValue>+std::optional<Try<TValue>> receiverGetValue(Receiver<TValue>& receiver);++template <typename TValue>+std::pair<detail::ChannelBridgePtr<TValue>, detail::ReceiverQueue<TValue>>+receiverUnbuffer(Receiver<TValue>&& receiver);+} // namespace detail+} // namespace channels+} // namespace folly
+ folly/folly/channels/Channel-inl.h view
@@ -0,0 +1,223 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <folly/CancellationToken.h>+#include <folly/Synchronized.h>+#include <folly/coro/Coroutine.h>+#include <folly/experimental/channels/detail/ChannelBridge.h>++namespace folly {+namespace channels {++namespace detail {+template <typename TValue>+ChannelBridgePtr<TValue>& senderGetBridge(Sender<TValue>& sender) {+  return sender.bridge_;+}++template <typename TValue>+bool receiverWait(+    Receiver<TValue>& receiver, detail::IChannelCallback* callback) {+  if (!receiver.buffer_.empty()) {+    return false;+  }+  return receiver.bridge_->receiverWait(callback);+}++template <typename TValue>+detail::IChannelCallback* cancelReceiverWait(Receiver<TValue>& receiver) {+  return receiver.bridge_->cancelReceiverWait();+}++template <typename TValue>+std::optional<Try<TValue>> receiverGetValue(Receiver<TValue>& receiver) {+  if (receiver.buffer_.empty()) {+    receiver.buffer_ = receiver.bridge_->receiverGetValues();+    if (receiver.buffer_.empty()) {+      return std::nullopt;+    }+  }+  auto result = std::move(receiver.buffer_.front());+  receiver.buffer_.pop();+  return result;+}++template <typename TValue>+std::pair<detail::ChannelBridgePtr<TValue>, detail::ReceiverQueue<TValue>>+receiverUnbuffer(Receiver<TValue>&& receiver) {+  return std::make_pair(+      std::move(receiver.bridge_), std::move(receiver.buffer_));+}++} // namespace detail++template <typename TValue>+class Receiver<TValue>::Waiter : public detail::IChannelCallback {+ public:+  Waiter(+      Receiver<TValue>* receiver,+      folly::CancellationToken cancelToken,+      bool closeOnCancel)+      : state_(State{.receiver = receiver}),+        cancelCallback_(+            makeCancellationCallback(std::move(cancelToken), closeOnCancel)) {}++  bool await_ready() const noexcept {+    // We are ready immediately if the receiver is either cancelled or closed.+    return state_.withRLock([&](const State& state) {+      return state.cancelled || !state.receiver;+    });+  }++  bool await_suspend(folly::coro::coroutine_handle<> awaitingCoroutine) {+    return state_.withWLock([&](State& state) {+      if (state.cancelled || !state.receiver ||+          !receiverWait(*state.receiver, this)) {+        // We will not suspend at all if the receiver is either cancelled or+        // closed.+        return false;+      }+      state.awaitingCoroutine = awaitingCoroutine;+      return true;+    });+  }++  std::optional<TValue> await_resume() {+    auto result = getResult();+    if (!result.hasValue() && !result.hasException()) {+      return std::nullopt;+    }+    return std::move(result.value());+  }++  // FIXME: The default implementation of `co_await_result` will convert the+  // `getResult()` branch below that returns an empty `Try` into an error+  // state.  That's because the normal `folly::coro` contract does not allow+  // producing an empty `Try`.  Therefore, any future `await_resume_result()`+  // implementation should investigate the intended semantics of this+  // `getResult()` logic, and provide better handling:+  //+  //   if (!state.receiver) {+  //     return Try<TValue>();+  //   }+  //+  // Also, after D73731681 (rich_error), supporting `await_resume_result()`+  // will be also motivated by better perf for signaling cancellation.+  Try<TValue> await_resume_try() { return getResult(); }++ protected:+  struct State {+    Receiver<TValue>* receiver;+    folly::coro::coroutine_handle<> awaitingCoroutine{};+    bool cancelled{false};+  };++  std::unique_ptr<folly::CancellationCallback> makeCancellationCallback(+      folly::CancellationToken cancelToken, bool closeOnCancel) {+    if (!cancelToken.canBeCancelled()) {+      return nullptr;+    }+    return std::make_unique<folly::CancellationCallback>(+        std::move(cancelToken), [this, closeOnCancel] {+          auto receiver = state_.withWLock([&](State& state) {+            state.cancelled = true;+            return std::exchange(state.receiver, nullptr);+          });+          if (!receiver) {+            return;+          }+          if (closeOnCancel) {+            std::move(*receiver).cancel();+          } else {+            auto* callback = detail::cancelReceiverWait(*receiver);+            if (callback) {+              callback->canceled(nullptr);+            }+          }+        });+  }++  void consume(detail::ChannelBridgeBase*) override { resume(); }++  void canceled(detail::ChannelBridgeBase*) override { resume(); }++  void resume() {+    auto awaitingCoroutine = state_.withWLock([&](State& state) {+      return std::exchange(state.awaitingCoroutine, nullptr);+    });+    awaitingCoroutine.resume();+  }++  Try<TValue> getResult() {+    cancelCallback_.reset();+    return state_.withWLock([&](State& state) {+      if (state.cancelled) {+        return Try<TValue>(+            folly::make_exception_wrapper<folly::OperationCancelled>());+      }+      if (!state.receiver) {+        return Try<TValue>();+      }+      auto result =+          std::move(detail::receiverGetValue(*state.receiver).value());+      if (!result.hasValue()) {+        std::move(*state.receiver).cancel();+        state.receiver = nullptr;+      }+      return result;+    });+  }++  folly::Synchronized<State> state_;+  std::unique_ptr<folly::CancellationCallback> cancelCallback_;+};++template <typename TValue>+struct Receiver<TValue>::NextSemiAwaitable {+ public:+  explicit NextSemiAwaitable(+      Receiver<TValue>* receiver,+      bool closeOnCancel,+      std::optional<folly::CancellationToken> cancelToken = std::nullopt)+      : receiver_(receiver),+        closeOnCancel_(closeOnCancel),+        cancelToken_(std::move(cancelToken)) {}++  [[nodiscard]] Waiter operator co_await() {+    return Waiter(+        receiver_,+        cancelToken_.value_or(folly::CancellationToken()),+        closeOnCancel_);+  }++  friend NextSemiAwaitable co_withCancellation(+      folly::CancellationToken cancelToken, NextSemiAwaitable&& awaitable) {+    if (awaitable.cancelToken_.has_value()) {+      return std::move(awaitable);+    }+    return NextSemiAwaitable(+        awaitable.receiver_, awaitable.closeOnCancel_, std::move(cancelToken));+  }++ private:+  Receiver<TValue>* receiver_;+  bool closeOnCancel_;+  std::optional<folly::CancellationToken> cancelToken_;+};+} // namespace channels+} // namespace folly
+ folly/folly/channels/Channel.h view
@@ -0,0 +1,289 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <folly/channels/Channel-fwd.h>+#include <folly/experimental/channels/detail/ChannelBridge.h>++namespace folly {+namespace channels {++/*+ * A channel is a sender and receiver pair that allows one component to send+ * values to another. A sender and receiver pair is similar to an AsyncPipe and+ * AsyncGenerator pair. However, unlike AsyncPipe/AsyncGenerator, senders and+ * receivers can be used by memory-efficient higher level transformation+ * abstractions.+ *+ * Typical usage:+ *   auto [receiver, sender] = Channel<T>::create();+ *   sender.write(val1);+ *   auto val2 = co_await receiver.next();+ */+template <typename TValue>+class Channel {+ public:+  /**+   * Creates a new channel with a sender/receiver pair. The channel will be+   * closed if the sender is destroyed, and will be cancelled if the receiver is+   * destroyed.+   */+  static std::pair<Receiver<TValue>, Sender<TValue>> create() {+    auto senderBridge = detail::ChannelBridge<TValue>::create();+    auto receiverBridge = senderBridge->copy();+    return std::make_pair(+        Receiver<TValue>(std::move(receiverBridge)),+        Sender<TValue>(std::move(senderBridge)));+  }+};++/**+ * A sender sends values to be consumed by a receiver.+ */+template <typename TValue>+class Sender {+ public:+  friend Channel<TValue>;+  using ValueType = TValue;++  Sender(Sender&& other) noexcept : bridge_(std::move(other.bridge_)) {}++  Sender& operator=(Sender&& other) noexcept {+    if (this == &other) {+      return *this;+    }++    if (bridge_) {+      std::move(*this).close();+    }+    bridge_ = std::move(other.bridge_);+    return *this;+  }++  ~Sender() {+    if (bridge_) {+      std::move(*this).close();+    }+  }++  /**+   * Returns whether or not this sender instance is valid. This will return+   * false if the sender was closed or moved away.+   */+  explicit operator bool() const { return bridge_ != nullptr; }++  /**+   * Writes a value into the pipe.+   */+  template <typename U = TValue>+  void write(U&& element) {+    if (!bridge_->isSenderClosed()) {+      bridge_->senderPush(std::forward<U>(element));+    }+  }++  /**+   * Closes the pipe without an exception.+   */+  void close() && {+    if (!bridge_->isSenderClosed()) {+      bridge_->senderClose();+    }+    bridge_ = nullptr;+  }++  /**+   * Closes the pipe with an exception.+   */+  void close(exception_wrapper exception) && {+    if (!bridge_->isSenderClosed()) {+      bridge_->senderClose(std::move(exception));+    }+    bridge_ = nullptr;+  }++  /**+   * Returns whether or not the corresponding receiver has been cancelled or+   * destroyed.+   */+  bool isReceiverCancelled() {+    if (bridge_->isSenderClosed()) {+      return true;+    }+    auto values = bridge_->senderGetValues();+    if (!values.empty()) {+      bridge_->senderClose();+      return true;+    }+    return false;+  }++ private:+  friend detail::ChannelBridgePtr<TValue>& detail::senderGetBridge<>(+      Sender<TValue>&);++  explicit Sender(detail::ChannelBridgePtr<TValue> bridge)+      : bridge_(std::move(bridge)) {}++  detail::ChannelBridgePtr<TValue> bridge_;+};++/**+ * A receiver that receives values sent by a sender. There are several ways that+ * a receiver can be consumed:+ *+ * 1. Call co_await receiver.next() to get the next value. See the docstring of+ *    next() for more details. This is the easiest way to consume the values+ *    from a receiver, but it is also the most expensive memory-wise (as it+ *    creates a long-lived coroutine frame). This is typically used in scenarios+ *    where O(1) channels are being consumed (and therefore coroutine memory+ *    overhead is negligible).+ *+ * 2. Call consumeChannelWithCallback to get a callback when each value comes+ *    in. See ConsumeChannel.h for more details. This uses less memory than+ *    #1, as it only needs to allocate coroutine frames when processing values+ *    (rather than always having such frames allocated when waiting for values).+ *+ * 3. Use MergeChannel in folly/experimental/channels/MergeChannel.h.+ *    This construct allows you to consume the merged output of a dynamically+ *    changing set of receivers. This is the cheapest way to consume the output+ *    of a large number of receivers. It is useful when the consumer wants to+ *    process all values from all receivers sequentially.+ *+ * 4. Use ChannelProcessor in folly/experimental/channels/ChannelProcessor.h.+ *    This construct allows you to consume a dynamically changing set of+ *    receivers in parallel.+ *+ * 5. A receiver may also be passed to other framework primitives that consume+ *    the receiver (such as transform). As with options 2-4, these primitives+ *    do not require coroutine frames to be allocated when waiting for values.+ */+template <typename TValue>+class Receiver {+  class Waiter;+  struct NextSemiAwaitable;++ public:+  friend Channel<TValue>;+  using ValueType = TValue;++  Receiver() {}++  Receiver(Receiver&& other) noexcept+      : bridge_(std::move(other.bridge_)), buffer_(std::move(other.buffer_)) {}++  Receiver& operator=(Receiver&& other) noexcept {+    if (this == &other) {+      return *this;+    }+    if (bridge_ != nullptr) {+      std::move(*this).cancel();+    }+    bridge_ = std::move(other.bridge_);+    buffer_ = std::move(other.buffer_);+    return *this;+  }++  ~Receiver() {+    if (bridge_ != nullptr) {+      std::move(*this).cancel();+    }+  }++  /**+   * Returns whether or not this receiver instance is valid. This will return+   * false if the receiver was cancelled or moved away.+   */+  explicit operator bool() const { return bridge_ != nullptr; }++  /**+   * Returns the next value sent by a sender. The behavior similar to the+   * behavior of next() on folly::coro::AsyncGenerator<TValue>.+   *+   * When closeOnCancel is true, if the returned semi-awaitable is cancelled,+   * the underlying channel will be closed. No more values will be received,+   * even if they were sent by the sender. This matches the behavior of+   * folly::coro::AsyncGenerator.+   *+   * When closeOnCancel is false, cancelling the returned semi-awaitable will+   * not close the underlying channel. Instead, it will just cancel the next()+   * operation. This means that the caller can call next() again and continue+   * to receive values sent by the sender.+   *+   * If consumed directly with co_await, next() will return an std::optional:+   *+   *    std::optional<TValue> value = co_await receiver.next();+   *+   *    - If a value is sent, the std::optional will contain the value.+   *    - If the channel is closed by the sender with no exception, the optional+   *        will be empty.+   *    - If the channel is closed by the sender with an exception, next() will+   *        throw the exception.+   *    - If the next() call was cancelled, next() will throw an exception of+   *        type folly::OperationCancelled.+   *+   * If consumed with folly::coro::co_awaitTry, this will return a Try:+   *+   *    Try<TValue> value = co_await folly::coro::co_awaitTry(+   *        receiver.next());+   *+   *    - If a value is sent, the Try will contain the value.+   *    - If the channel is closed by the sender with no exception, the try will+   *        be empty (with no value or exception).+   *    - If the channel is closed by the sender with an exception, the try will+   *        contain the exception.+   *    - If the next() call was cancelled, the try will contain an exception of+   *        type folly::OperationCancelled.+   */+  NextSemiAwaitable next(bool closeOnCancel = true) {+    return NextSemiAwaitable(*this ? this : nullptr, closeOnCancel);+  }++  /**+   * Cancels this receiver. If the receiver is currently being consumed, the+   * consumer will receive a folly::OperationCancelled exception.+   */+  void cancel() && {+    bridge_->receiverCancel();+    bridge_ = nullptr;+    buffer_.clear();+  }++ private:+  explicit Receiver(detail::ChannelBridgePtr<TValue> bridge)+      : bridge_(std::move(bridge)) {}++  friend bool detail::receiverWait<>(+      Receiver<TValue>&, detail::IChannelCallback*);++  friend detail::IChannelCallback* detail::cancelReceiverWait<>(+      Receiver<TValue>&);++  friend std::optional<Try<TValue>> detail::receiverGetValue<>(+      Receiver<TValue>&);++  friend std::+      pair<detail::ChannelBridgePtr<TValue>, detail::ReceiverQueue<TValue>>+      detail::receiverUnbuffer<>(Receiver<TValue>&& receiver);++  detail::ChannelBridgePtr<TValue> bridge_;+  detail::ReceiverQueue<TValue> buffer_;+};+} // namespace channels+} // namespace folly++#include <folly/channels/Channel-inl.h>
+ folly/folly/channels/ChannelCallbackHandle.h view
@@ -0,0 +1,162 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <folly/IntrusiveList.h>+#include <folly/ScopeGuard.h>+#include <folly/channels/Channel.h>++namespace folly {+namespace channels {++namespace detail {+class ChannelCallbackProcessor : public IChannelCallback {+ public:+  virtual void onHandleDestroyed() = 0;+};+} // namespace detail++/**+ * A callback handle for a consumption operation on a channel. The consumption+ * operation will be cancelled when this handle is destroyed.+ */+class ChannelCallbackHandle {+ public:+  ChannelCallbackHandle() : processor_(nullptr) {}++  explicit ChannelCallbackHandle(detail::ChannelCallbackProcessor* processor)+      : processor_(processor) {}++  ~ChannelCallbackHandle() {+    if (processor_) {+      processor_->onHandleDestroyed();+    }+  }++  ChannelCallbackHandle(ChannelCallbackHandle&& other) noexcept+      : processor_(std::exchange(other.processor_, nullptr)) {}++  ChannelCallbackHandle& operator=(ChannelCallbackHandle&& other) {+    if (&other == this) {+      return *this;+    }+    reset();+    processor_ = std::exchange(other.processor_, nullptr);+    return *this;+  }++  void reset() {+    if (processor_) {+      processor_->onHandleDestroyed();+      processor_ = nullptr;+    }+  }++ private:+  detail::ChannelCallbackProcessor* processor_;+};++namespace detail {++/**+ * A wrapper around a ChannelCallbackHandle that belongs to an intrusive linked+ * list. When the holder is destroyed, the object will automatically be unlinked+ * from the linked list that it is in (if any).+ */+struct ChannelCallbackHandleHolder {+  explicit ChannelCallbackHandleHolder(ChannelCallbackHandle _handle)+      : handle(std::move(_handle)) {}++  ChannelCallbackHandleHolder(ChannelCallbackHandleHolder&& other) noexcept+      : handle(std::move(other.handle)) {+    hook.swap_nodes(other.hook);+  }++  ChannelCallbackHandleHolder& operator=(+      ChannelCallbackHandleHolder&& other) noexcept {+    if (&other == this) {+      return *this;+    }+    handle = std::move(other.handle);+    hook.unlink();+    hook.swap_nodes(other.hook);+    return *this;+  }++  void requestCancellation() { handle.reset(); }++  ChannelCallbackHandle handle;+  folly::IntrusiveListHook hook;+};++template <typename TValue, typename OnNextFunc>+class ChannelCallbackProcessorImplWithList;+} // namespace detail++/**+ * A list of channel callback handles. When consumeChannelWithCallback is+ * invoked with a list, a cancellation handle is automatically added to the list+ * for the consumption operation. Similarly, when a consumption operation is+ * completed, the handle is automatically removed from the lists.+ *+ * If the list still has any cancellation handles remaining when the list is+ * destroyed, cancellation is triggered for each handle in the list.+ *+ * This list is not thread safe.+ */+class ChannelCallbackHandleList {+ public:+  ChannelCallbackHandleList() {}++  ChannelCallbackHandleList(ChannelCallbackHandleList&& other) noexcept {+    holders_.swap(other.holders_);+  }++  ChannelCallbackHandleList& operator=(+      ChannelCallbackHandleList&& other) noexcept {+    if (&other == this) {+      return *this;+    }+    holders_.swap(other.holders_);+    return *this;+  }++  ~ChannelCallbackHandleList() { clear(); }++  void clear() {+    for (auto& holder : holders_) {+      holder.requestCancellation();+    }+    holders_.clear();+  }++ private:+  template <typename TValue, typename OnNextFunc>+  friend class detail::ChannelCallbackProcessorImplWithList;++  void add(detail::ChannelCallbackHandleHolder& holder) {+    holders_.push_back(holder);+  }++  using ChannelCallbackHandleListImpl = folly::IntrusiveList<+      detail::ChannelCallbackHandleHolder,+      &detail::ChannelCallbackHandleHolder::hook>;++  ChannelCallbackHandleListImpl holders_;+};+} // namespace channels+} // namespace folly
+ folly/folly/channels/ChannelProcessor-inl.h view
@@ -0,0 +1,378 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <fmt/format.h>+#include <folly/channels/ChannelProcessor.h>+#include <folly/channels/ConsumeChannel.h>+#include <folly/channels/MergeChannel.h>+#include <folly/channels/Transform.h>+#include <folly/executors/SerialExecutor.h>+#include <folly/experimental/channels/detail/IntrusivePtr.h>++namespace folly {+namespace channels {+namespace detail {++template <typename KeyType>+class ChannelProcessorImpl {+ public:+  ChannelProcessorImpl(+      std::vector<folly::Executor::KeepAlive<folly::SequencedExecutor>>+          executors,+      std::shared_ptr<folly::channels::RateLimiter> rateLimiter,+      MergeChannel<KeyType, Unit> mergeChannel,+      Receiver<MergeChannelEvent<KeyType, Unit>> mergeChannelReceiver)+      : implState_(make_intrusive<ImplState>(+            std::move(executors), std::move(rateLimiter))),+        channels_(std::move(mergeChannel)),+        handle_(consumeChannelWithCallback(+            std::move(mergeChannelReceiver),+            implState_->executors[0],+            [](Try<MergeChannelEvent<KeyType, Unit>>)+                -> folly::coro::Task<bool> {+              // Do nothing+              co_return true;+            })) {}++  template <typename ReceiverType, typename OnUpdateFunc>+  void addChannel(KeyType key, ReceiverType receiver, OnUpdateFunc onUpdate) {+    using InputValueType = typename ReceiverType::ValueType;+    channels_.removeReceiver(key);+    channels_.addNewReceiver(+        std::move(key),+        transform(+            std::move(receiver),+            Transformer<InputValueType, OnUpdateFunc>(+                implState_, std::move(onUpdate))));+  }++  template <+      typename InitializeArg,+      typename InitializeFunc,+      typename OnUpdateFunc>+  void addResumableChannelWithState(+      KeyType key,+      InitializeArg initializeArg,+      InitializeFunc initialize,+      OnUpdateFunc onUpdate) {+    addResumableChannelWithState(+        std::move(key),+        std::move(initializeArg),+        std::move(initialize),+        std::move(onUpdate),+        NoChannelState());+  }++  template <+      typename InitializeArg,+      typename InitializeFunc,+      typename OnUpdateFunc,+      typename ChannelState>+  void addResumableChannelWithState(+      KeyType key,+      InitializeArg initializeArg,+      InitializeFunc initialize,+      OnUpdateFunc onUpdate,+      ChannelState channelState) {+    using ReceiverType = typename decltype(initialize(+        std::move(initializeArg), channelState))::StorageType;+    using InputValueType = typename ReceiverType::ValueType;+    channels_.removeReceiver(key);+    channels_.addNewReceiver(+        std::move(key),+        resumableTransform(+            std::move(initializeArg),+            ResumableTransformer<+                InitializeArg,+                InputValueType,+                InitializeFunc,+                OnUpdateFunc,+                ChannelState>(+                implState_,+                std::move(initialize),+                std::move(onUpdate),+                std::move(channelState))));+  }++  void removeChannel(const KeyType& keyType) {+    channels_.removeReceiver(keyType);+  }++ private:+  struct NoChannelState {};++  template <+      typename Function,+      typename ReturnType =+          typename std::invoke_result_t<Function>::StorageType>+  static folly::coro::Task<ReturnType> catchNonCoroException(Function func) {+    auto result = folly::makeTryWith(std::move(func));+    if (result.hasException()) {+      return folly::coro::makeErrorTask<ReturnType>(+          std::move(result.exception()));+    } else {+      return std::move(result.value());+    }+  }++  struct ImplState : public IntrusivePtrBase<ImplState> {+    ImplState(+        std::vector<folly::Executor::KeepAlive<folly::SequencedExecutor>>+            _executors,+        std::shared_ptr<folly::channels::RateLimiter> _rateLimiter)+        : executors(std::move(_executors)),+          rateLimiter(std::move(_rateLimiter)) {}++    std::vector<folly::Executor::KeepAlive<folly::SequencedExecutor>> executors;+    std::shared_ptr<folly::channels::RateLimiter> rateLimiter;+  };++  template <typename InputValueType, typename OnUpdateFunc>+  class Transformer : public std::tuple<OnUpdateFunc> {+   public:+    Transformer(intrusive_ptr<ImplState> implState, OnUpdateFunc onUpdate)+        : std::tuple<OnUpdateFunc>(std::move(onUpdate)),+          implState_(std::move(implState)) {}++    folly::Executor::KeepAlive<folly::SequencedExecutor> getExecutor() {+      return implState_->executors+          [std::hash<decltype(this)>()(this) % implState_->executors.size()];+    }++    std::shared_ptr<folly::channels::RateLimiter> getRateLimiter() {+      return implState_->rateLimiter;+    }++    folly::coro::AsyncGenerator<Unit&&> transformValue(+        Try<InputValueType> value) {+      auto result = co_await folly::coro::co_awaitTry(catchNonCoroException(+          [&] { return std::get<OnUpdateFunc>(*this)(std::move(value)); }));+      if (result.template hasException<folly::OperationCancelled>() ||+          result.template hasException<OnClosedException>()) {+        co_yield folly::coro::co_error(OnClosedException());+      } else if (result.hasException()) {+        LOG(FATAL) << fmt::format(+            "Encountered exception from callback when consuming channel of "+            "type {}: {}",+            typeid(InputValueType).name(),+            result.exception().what());+      }+    }++   private:+    intrusive_ptr<ImplState> implState_;+  };++  template <+      typename InitializeArg,+      typename InputValueType,+      typename InitializeFunc,+      typename OnUpdateFunc,+      typename ChannelState>+  class ResumableTransformer+      : public std::tuple<InitializeFunc, OnUpdateFunc, ChannelState> {+   public:+    ResumableTransformer(+        intrusive_ptr<ImplState> implState,+        InitializeFunc initialize,+        OnUpdateFunc onUpdate,+        ChannelState channelState)+        : std::tuple<InitializeFunc, OnUpdateFunc, ChannelState>(+              std::move(initialize),+              std::move(onUpdate),+              std::move(channelState)),+          implState_(std::move(implState)) {}++    folly::Executor::KeepAlive<folly::SequencedExecutor> getExecutor() {+      return implState_->executors+          [std::hash<decltype(this)>()(this) % implState_->executors.size()];+    }++    std::shared_ptr<folly::channels::RateLimiter> getRateLimiter() {+      return implState_->rateLimiter;+    }++    folly::coro::Task<std::pair<std::vector<Unit>, Receiver<InputValueType>>>+    initializeTransform(InitializeArg initializeArg) {+      auto result = co_await folly::coro::co_awaitTry(+          initialize(std::move(initializeArg)));+      if (result.template hasException<folly::OperationCancelled>() ||+          result.template hasException<OnClosedException>()) {+        co_yield folly::coro::co_error(OnClosedException());+      } else if (result.hasException()) {+        LOG(FATAL) << fmt::format(+            "Encountered exception from callback when consuming channel of "+            "type {}: {}",+            typeid(InputValueType).name(),+            result.exception().what());+      }+      co_return std::make_pair(std::vector<Unit>(), std::move(result.value()));+    }++    folly::coro::AsyncGenerator<Unit&&> transformValue(+        Try<InputValueType> value) {+      auto result =+          co_await folly::coro::co_awaitTry(onUpdate(std::move(value)));+      if (result+              .template hasException<ReinitializeException<InitializeArg>>()) {+        co_yield folly::coro::co_error(std::move(result.exception()));+      } else if (+          result.template hasException<folly::OperationCancelled>() ||+          result.template hasException<OnClosedException>()) {+        co_yield folly::coro::co_error(OnClosedException());+      } else if (result.hasException()) {+        LOG(FATAL) << fmt::format(+            "Encountered exception from callback when consuming channel of "+            "type {}: {}",+            typeid(InputValueType).name(),+            result.exception().what());+      }+    }++   private:+    folly::coro::Task<Receiver<InputValueType>> initialize(+        InitializeArg initializeArg) {+      if constexpr (std::is_same_v<ChannelState, NoChannelState>) {+        co_return co_await catchNonCoroException([&] {+          return std::get<InitializeFunc>(*this)(std::move(initializeArg));+        });+      } else {+        co_return co_await catchNonCoroException([&] {+          return std::get<InitializeFunc>(*this)(+              std::move(initializeArg), std::get<ChannelState>(*this));+        });+      }+    }++    folly::coro::Task<void> onUpdate(Try<InputValueType> value) {+      if constexpr (std::is_same_v<ChannelState, NoChannelState>) {+        co_await catchNonCoroException([&] {+          return std::get<OnUpdateFunc>(*this)(std::move(value));+        });+      } else {+        co_await catchNonCoroException([&] {+          return std::get<OnUpdateFunc>(*this)(+              std::move(value), std::get<ChannelState>(*this));+        });+      }+    }++    intrusive_ptr<ImplState> implState_;+  };++  intrusive_ptr<ImplState> implState_;+  MergeChannel<KeyType, Unit> channels_;+  ChannelCallbackHandle handle_;+};+} // namespace detail++template <typename KeyType>+ChannelProcessor<KeyType>::ChannelProcessor(+    std::unique_ptr<detail::ChannelProcessorImpl<KeyType>> impl)+    : impl_(std::move(impl)) {}++template <typename KeyType>+ChannelProcessor<KeyType>::operator bool() const {+  return impl_ != nullptr;+}++template <typename KeyType>+template <typename ReceiverType, typename OnUpdateFunc>+void ChannelProcessor<KeyType>::addChannel(+    KeyType key, ReceiverType receiver, OnUpdateFunc onUpdate) {+  impl_->addChannel(std::move(key), std::move(receiver), std::move(onUpdate));+}++template <typename KeyType>+template <+    typename InitializeArg,+    typename InitializeFunc,+    typename OnUpdateFunc>+void ChannelProcessor<KeyType>::addResumableChannel(+    KeyType key,+    InitializeArg initializeArg,+    InitializeFunc initialize,+    OnUpdateFunc onUpdate) {+  impl_->addResumableChannel(+      std::move(key),+      std::move(initializeArg),+      std::move(initialize),+      std::move(onUpdate));+}++template <typename KeyType>+template <+    typename InitializeArg,+    typename InitializeFunc,+    typename OnUpdateFunc,+    typename ChannelState>+void ChannelProcessor<KeyType>::addResumableChannelWithState(+    KeyType key,+    InitializeArg initializeArg,+    InitializeFunc initialize,+    OnUpdateFunc onUpdate,+    ChannelState channelState) {+  impl_->addResumableChannelWithState(+      std::move(key),+      std::move(initializeArg),+      std::move(initialize),+      std::move(onUpdate),+      std::move(channelState));+}++template <typename KeyType>+void ChannelProcessor<KeyType>::removeChannel(const KeyType& keyType) {+  impl_->removeChannel(keyType);+}++template <typename KeyType>+void ChannelProcessor<KeyType>::close() && {+  impl_.reset();+}++template <typename KeyType>+ChannelProcessor<KeyType> createChannelProcessor(+    std::vector<folly::Executor::KeepAlive<folly::SequencedExecutor>> executors,+    std::shared_ptr<RateLimiter> rateLimiter) {+  CHECK_GT(executors.size(), 0);+  auto [mergeChannelReceiver, mergeChannel] =+      createMergeChannel<KeyType, Unit>(executors[0]);+  return ChannelProcessor<KeyType>(+      std::make_unique<detail::ChannelProcessorImpl<KeyType>>(+          std::move(executors),+          std::move(rateLimiter),+          std::move(mergeChannel),+          std::move(mergeChannelReceiver)));+}++template <typename KeyType>+ChannelProcessor<KeyType> createChannelProcessor(+    folly::Executor::KeepAlive<> executor,+    std::shared_ptr<RateLimiter> rateLimiter,+    size_t numSequencedExecutors) {+  CHECK_GT(numSequencedExecutors, 0);+  auto executors =+      std::vector<folly::Executor::KeepAlive<folly::SequencedExecutor>>();+  for (size_t i = 0; i < numSequencedExecutors; i++) {+    executors.push_back(folly::SerialExecutor::create(executor));+  }+  return createChannelProcessor<KeyType>(+      std::move(executors), std::move(rateLimiter));+}+} // namespace channels+} // namespace folly
+ folly/folly/channels/ChannelProcessor.h view
@@ -0,0 +1,252 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <folly/Executor.h>+#include <folly/channels/RateLimiter.h>++namespace folly {+namespace channels {++namespace detail {+template <typename KeyType>+class ChannelProcessorImpl;+}++/**+ * This object allows for memory-efficient processing of values many channels.+ *+ * A channel is added with a unique key and a callback. The callback will be+ * called for every value pushed to the receiver.+ *+ * A resumable channel can also be added. A resumable channel involves two+ * callbacks. An initialization callback is called to get the receiver, and the+ * update callback is called on every update (as for a normal channel). The+ * update callback can throw a ReinitializeException at any time, which will+ * trigger the initialize callback to re-run.+ *+ * Values for a given channel are processed until one of the following occurs:+ *     1. The channel is closed+ *     2. The channel callback throws an OnClosedException+ *     3. The channel callback throws a folly::OperationCancelled exception.+ *     4. The channel is removed with a call to removeChannel.+ *+ * If a channel is removed with removeChannel, processing will eventually stop+ * for that channel. This will not necessarily happen immediately.+ *+ * If a channel is added for an already existing key, the previous channel for+ * that key will be removed and processing will eventually stop.+ *+ * Processing for all channels will run on the user-provided executor. For any+ * particular channel, all processing will happen sequentially. For any two+ * distinct channels, processing may happen in parallel (subject to any+ * constraints of the provided executor).+ */+template <typename KeyType>+class ChannelProcessor {+ public:+  explicit ChannelProcessor(+      std::unique_ptr<detail::ChannelProcessorImpl<KeyType>> impl);++  /**+   * Returns whether this ChannelProcessor is a valid object.+   */+  explicit operator bool() const;++  /**+   * Processes a channel with a given key and callback. For a receiver of type+   * Receiver<InputValueType>, the callback must accept a single parameter of+   * type Try<InputValueType>, and return a void task. If the callback+   * throws an exception of type OperationCancelled or OnClosedException, the+   * channel will be removed. Any other exception thrown by the callback will+   * terminate the process.+   *+   * If there is an existing channel with the same key, it will be removed+   * before the new channel is added. The old channel's callback can check the+   * current cancellation token to see if it was removed while processing+   * values. See removeChannel for more details.+   *+   * Example:+   *+   *   // Example function that returns a receiver for a given entity+   *   Receiver<int> subscribe(const std::string& entity);+   *+   *   // Example function that returns an executor+   *   folly::Executor::KeepAlive<> getExecutor();+   *+   *   auto channelProcessor = createChannelProcessor<std::string>(+   *       getExecutor());+   *+   *   channelProcessor.addChannel(+   *       "abc",+   *       subscribe("abc"),+   *       [](Try<int> value) -> folly::coro::Task<void> {+   *         LOG(INFO) << fmt::format("Received value {}", *value);+   *         co_return;+   *       });+   */+  template <typename ReceiverType, typename OnUpdateFunc>+  void addChannel(KeyType key, ReceiverType receiver, OnUpdateFunc onUpdate);++  /**+   * Processing a resumable channel involves two callbacks. The initialization+   * callback accepts an initialization argument of a user-defined type, and+   * must return a folly::coro::Task<Receiver<InputValueType>>. The onUpdate+   * callback accepts a Try<InputValueType>, and returns a void task. The+   * onUpdate callback can throw a ReinitializeException<InitializeArg> at any+   * time, which will trigger the initialize function to be run again. In+   * addition, if either callback throws an exception of type OperationCancelled+   * or OnClosedException, the channel will be removed. Any other exception+   * thrown by either callback will terminate the process.+   *+   * If there is an existing channel with the same key, it will be removed+   * before the new channel is added. The old channel's callbacks can check the+   * current cancellation token to see if it was removed while processing+   * values. See removeChannel for more details.+   *+   * Example:+   *+   *   struct InitializeArg {+   *     std::string param;+   *   }+   *+   *   // Example function that returns a receiver for a given entity+   *   Receiver<int> subscribe(const InitializeArg& initializeArg);+   *+   *   // Example function that returns an executor+   *   folly::Executor::KeepAlive<> getExecutor();+   *+   *   auto channelProcessor = createChannelProcessor<std::string>(+   *       getExecutor());+   *+   *   channelProcessor.addResumableChannel(+   *       "abc",+   *       InitializeArg({"param"}),+   *       [](InitializeArg initializeArg) -> folly::coro::Task<Receiver<int>> {+   *         co_return subscribe(initializeArg);+   *       },+   *       [](Try<int> value) -> folly::coro::Task<void> {+   *         if (*value == -1) {+   *           throw ReinitializeException(InitializeArg({"param"}));+   *         }+   *         LOG(INFO) << fmt::format("Received value {}", *value);+   *         co_return;+   *       });+   */+  template <+      typename InitializeArg,+      typename InitializeFunc,+      typename OnUpdateFunc>+  void addResumableChannel(+      KeyType key,+      InitializeArg initializeArg,+      InitializeFunc initialize,+      OnUpdateFunc onUpdate);++  /*+   * This is similar to addResumableChannel. However, it allows a user-provided+   * state object to be stored with the channel. That state object will be+   * passed to both callbacks, and will be destructed when the channel is+   * removed or closed.+   *+   * * Example:+   *+   *   struct InitializeArg {+   *     std::string param;+   *   }+   *+   *   struct State {+   *     int prevValue{-1};+   *   }+   *+   *   // Example function that returns a receiver for a given entity+   *   Receiver<int> subscribe(const InitializeArg& initializeArg);+   *+   *   // Example function that returns an executor+   *   folly::Executor::KeepAlive<> getExecutor();+   *+   *   auto channelProcessor = createChannelProcessor<std::string>(+   *       getExecutor());+   *+   *   channelProcessor.addResumableChannelWithState(+   *       "abc",+   *       InitializeArg({"param"}),+   *       [](InitializeArg initializeArg, State& state)+   *                        -> folly::coro::Task<Receiver<int>> {+   *         co_return subscribe(initializeArg);+   *       },+   *       [](Try<int> value, State& state) -> folly::coro::Task<void> {+   *         if (*value == -1) {+   *           throw ReinitializeException(InitializeArg({"param"}));+   *         }+   *         LOG(INFO) << fmt::format(+   *             "Received value {}. Previous: {}.", *value, state.prevValue);+   *         state.prevValue = *value;+   *         co_return;+   *       },+   *       State());+   */+  template <+      typename InitializeArg,+      typename InitializeFunc,+      typename OnUpdateFunc,+      typename ChannelState>+  void addResumableChannelWithState(+      KeyType key,+      InitializeArg initializeArg,+      InitializeFunc initialize,+      OnUpdateFunc onUpdate,+      ChannelState channelState);++  /**+   * Removes the channel with the given key, if such a channel exists. The+   * channel will be asynchronously removed, so the channels' callback may+   * still receive some values after this call. The callback can detect whether+   * or not the channel was removed by examining its current cancellation token.+   */+  void removeChannel(const KeyType& keyType);++  /**+   * Closes all channels being processed, causing all processing to eventually+   * stop. Calling this function will make the object invalid.+   */+  void close() &&;++ private:+  std::unique_ptr<detail::ChannelProcessorImpl<KeyType>> impl_;+};++/**+ * Creates a new channel processor.+ */+template <typename KeyType>+ChannelProcessor<KeyType> createChannelProcessor(+    folly::Executor::KeepAlive<> executor,+    std::shared_ptr<RateLimiter> rateLimiter = nullptr,+    size_t numSequencedExecutors = 1);++/**+ * Creates a new channel processor.+ */+template <typename KeyType>+ChannelProcessor<KeyType> createChannelProcessor(+    std::vector<folly::Executor::KeepAlive<folly::SequencedExecutor>> executors,+    std::shared_ptr<RateLimiter> rateLimiter = nullptr);+} // namespace channels+} // namespace folly++#include <folly/channels/ChannelProcessor-inl.h>
+ folly/folly/channels/ConsumeChannel-inl.h view
@@ -0,0 +1,253 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <fmt/format.h>+#include <folly/Executor.h>+#include <folly/Format.h>+#include <folly/IntrusiveList.h>+#include <folly/ScopeGuard.h>+#include <folly/channels/Channel.h>+#include <folly/channels/ChannelCallbackHandle.h>+#include <folly/coro/Task.h>+#include <folly/experimental/channels/detail/Utility.h>++namespace folly {+namespace channels {++namespace detail {++template <typename TValue, typename OnNextFunc>+class ChannelCallbackProcessorImpl : public ChannelCallbackProcessor {+ public:+  ChannelCallbackProcessorImpl(+      ChannelBridgePtr<TValue> receiver,+      folly::Executor::KeepAlive<folly::SequencedExecutor> executor,+      OnNextFunc onNext)+      : receiver_(std::move(receiver)),+        executor_(std::move(executor)),+        onNext_(std::move(onNext)),+        cancelSource_(folly::CancellationSource::invalid()) {}++  void start(std::optional<detail::ReceiverQueue<TValue>> buffer) {+    co_withExecutor(+        executor_,+        runCoroutineWithCancellation(+            processAllAvailableValues(std::move(buffer))))+        .start();+  }++ private:+  /**+   * Called when the handle is destroyed.+   */+  void onHandleDestroyed() override {+    executor_->add([=, this]() { processHandleDestroyed(); });+  }++  /**+   * Called when the channel we are listening to has an update.+   */+  void consume(ChannelBridgeBase*) override {+    co_withExecutor(+        executor_, runCoroutineWithCancellation(processAllAvailableValues()))+        .start();+  }++  /**+   * Called after we cancelled the input channel (which happens after the handle+   * is destroyed).+   */+  void canceled(ChannelBridgeBase*) override {+    co_withExecutor(+        executor_,+        runCoroutineWithCancellation(+            processReceiverCancelled(true /* fromHandleDestruction */)))+        .start();+  }++  /**+   * Processes all available values from the input receiver (starting from the+   * provided buffer, if present).+   *+   * If a value was received indicating that the input channel has been closed,+   * we will process cancellation for the input receiver.+   */+  folly::coro::Task<void> processAllAvailableValues(+      std::optional<ReceiverQueue<TValue>> buffer = std::nullopt) {+    bool closed = buffer.has_value()+        ? !co_await processValues(std::move(buffer.value()))+        : false;+    while (!closed) {+      if (receiver_->receiverWait(this)) {+        // There are no more values available right now, but more values may+        // come in the future. We will stop processing for now, until we+        // re-start processing when the consume() callback is fired.+        break;+      }+      auto values = receiver_->receiverGetValues();+      CHECK(!values.empty());+      closed = !co_await processValues(std::move(values));+    }+    if (closed) {+      // The input receiver was closed.+      receiver_->receiverCancel();+      co_await processReceiverCancelled(false /* fromHandleDestruction */);+    }+  }++  /**+   * Processes values from the channel. Returns false if the channel has been+   * closed, so the caller can stop processing values from it.+   */+  folly::coro::Task<bool> processValues(ReceiverQueue<TValue> values) {+    auto cancelToken = co_await folly::coro::co_current_cancellation_token;+    while (!values.empty()) {+      if (cancelToken.isCancellationRequested()) {+        co_return true;+      }+      auto result = std::move(values.front());+      values.pop();+      bool closed = !result.hasValue();+      if (!co_await callCallback(std::move(result))) {+        closed = true;+      }+      if (closed) {+        co_return false;+      }+      co_await folly::coro::co_reschedule_on_current_executor;+    }+    co_return true;+  }++  /**+   * Process cancellation of the input receiver.+   *+   * @param fromHandleDestruction: Whether the cancellation was prompted by the+   *    handle being destroyed. If true, we will call the user's callback with+   *    a folly::OperationCancelled exception. This will be false if the+   *    cancellation was prompted by the closure of the channel.+   */+  folly::coro::Task<void> processReceiverCancelled(bool fromHandleDestruction) {+    CHECK_EQ(getReceiverState(), ChannelState::CancellationTriggered);+    receiver_ = nullptr;+    if (fromHandleDestruction) {+      co_await callCallback(Try<TValue>(+          folly::make_exception_wrapper<folly::OperationCancelled>()));+    }+    maybeDelete();+  }++  /**+   * Processes the destruction of the handle.+   */+  void processHandleDestroyed() {+    CHECK(!handleDestroyed_);+    handleDestroyed_ = true;+    cancelSource_.requestCancellation();+    if (getReceiverState() == ChannelState::Active) {+      receiver_->receiverCancel();+    }+    maybeDelete();+  }++  /**+   * Deletes this object if we have already processed cancellation for the+   * receiver and the handle.+   */+  void maybeDelete() {+    if (getReceiverState() == ChannelState::CancellationProcessed &&+        handleDestroyed_) {+      delete this;+    }+  }++  /**+   * Calls the user's callback with the given result.+   */+  folly::coro::Task<bool> callCallback(Try<TValue> result) {+    auto retVal = co_await folly::coro::co_awaitTry(onNext_(std::move(result)));+    if (retVal.template hasException<folly::OperationCancelled>()) {+      co_return false;+    } else if (retVal.hasException()) {+      LOG(FATAL) << fmt::format(+          "Encountered exception from callback when consuming channel of "+          "type {}: {}",+          typeid(TValue).name(),+          retVal.exception().what());+    }+    co_return retVal.value();+  }++  /**+   * Runs the given coroutine while listening for cancellation triggered by the+   * handle's destruction.+   */+  folly::coro::Task<void> runCoroutineWithCancellation(+      folly::coro::Task<void> task) {+    cancelSource_ = folly::CancellationSource();+    if (handleDestroyed_) {+      // The handle was already destroyed before we even started the coroutine.+      // Request cancellation so that the user's callback knows to stop quickly.+      cancelSource_.requestCancellation();+    }+    auto token = cancelSource_.getToken();+    auto retVal = co_await folly::coro::co_awaitTry(+        folly::coro::co_withCancellation(token, std::move(task)));+    CHECK(!retVal.hasException()) << fmt::format(+        "Unexpected exception when running coroutine: {}",+        retVal.exception().what());+    if (!token.isCancellationRequested()) {+      cancelSource_ = folly::CancellationSource::invalid();+    }+  }++  ChannelState getReceiverState() {+    return detail::getReceiverState(receiver_.get());+  }++  ChannelBridgePtr<TValue> receiver_;+  folly::Executor::KeepAlive<folly::SequencedExecutor> executor_;+  OnNextFunc onNext_;+  folly::CancellationSource cancelSource_;+  bool handleDestroyed_{false};+};+} // namespace detail++template <+    typename TReceiver,+    typename OnNextFunc,+    typename TValue,+    std::enable_if_t<+        std::is_constructible_v<+            folly::Function<folly::coro::Task<bool>(Try<TValue>)>,+            OnNextFunc>,+        int>>+ChannelCallbackHandle consumeChannelWithCallback(+    TReceiver receiver,+    folly::Executor::KeepAlive<folly::SequencedExecutor> executor,+    OnNextFunc onNext) {+  detail::ChannelCallbackProcessorImpl<TValue, OnNextFunc>* processor = nullptr;+  auto [unbufferedReceiver, buffer] =+      detail::receiverUnbuffer(std::move(receiver));+  processor = new detail::ChannelCallbackProcessorImpl<TValue, OnNextFunc>(+      std::move(unbufferedReceiver), std::move(executor), std::move(onNext));+  processor->start(std::move(buffer));+  return ChannelCallbackHandle(processor);+}+} // namespace channels+} // namespace folly
+ folly/folly/channels/ConsumeChannel.h view
@@ -0,0 +1,71 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <folly/Executor.h>+#include <folly/IntrusiveList.h>+#include <folly/channels/Channel.h>+#include <folly/channels/ChannelCallbackHandle.h>+#include <folly/coro/Task.h>+#include <folly/executors/SequencedExecutor.h>++namespace folly {+namespace channels {++/**+ * This function takes a Receiver, and consumes updates from that receiver with+ * a callback.+ *+ * This function returns a ChannelCallbackHandle. On destruction of this handle,+ * the callback will receive a try containing an exception of type+ * folly::OperationCancelled. If an active callback is running at the time the+ * cancellation request is received, cancellation will be requested on the+ * ambient cancellation token of the callback.+ *+ * The callback is run for each received value on the given executor. A try+ * is passed to the callback with the result:+ *+ *    - If a value is sent, the Try will contain the value.+ *    - If the channel is closed by the sender with no exception, the try will+ *          be empty (with no value or exception).+ *    - If the channel is closed by the sender with an exception, the try will+ *          contain the exception.+ *    - If the channel was cancelled (by the destruction of the returned+ *          handle), the try will contain an exception of type+ *          folly::OperationCancelled.+ *+ * If the callback returns false or throws a folly::OperationCancelled+ * exception, the channel will be cancelled and no further values will be+ * received.+ */+template <+    typename TReceiver,+    typename OnNextFunc,+    typename TValue = typename TReceiver::ValueType,+    std::enable_if_t<+        std::is_constructible_v<+            folly::Function<folly::coro::Task<bool>(Try<TValue>)>,+            OnNextFunc>,+        int> = 0>+ChannelCallbackHandle consumeChannelWithCallback(+    TReceiver receiver,+    folly::Executor::KeepAlive<folly::SequencedExecutor> executor,+    OnNextFunc onNext);+} // namespace channels+} // namespace folly++#include <folly/channels/ConsumeChannel-inl.h>
+ folly/folly/channels/FanoutChannel-inl.h view
@@ -0,0 +1,369 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <folly/channels/Channel.h>+#include <folly/channels/FanoutSender.h>+#include <folly/container/F14Set.h>+#include <folly/executors/SequencedExecutor.h>+#include <folly/experimental/channels/detail/Utility.h>++namespace folly {+namespace channels {++template <typename ValueType, typename ContextType>+FanoutChannel<ValueType, ContextType>::FanoutChannel(TProcessor* processor)+    : processor_(processor) {}++template <typename ValueType, typename ContextType>+FanoutChannel<ValueType, ContextType>::FanoutChannel(+    FanoutChannel&& other) noexcept+    : processor_(std::exchange(other.processor_, nullptr)) {}++template <typename ValueType, typename ContextType>+FanoutChannel<ValueType, ContextType>&+FanoutChannel<ValueType, ContextType>::operator=(+    FanoutChannel&& other) noexcept {+  if (&other == this) {+    return *this;+  }+  if (processor_) {+    std::move(*this).close();+  }+  processor_ = std::exchange(other.processor_, nullptr);+  return *this;+}++template <typename ValueType, typename ContextType>+FanoutChannel<ValueType, ContextType>::~FanoutChannel() {+  if (processor_ != nullptr) {+    std::move(*this).close(exception_wrapper());+  }+}++template <typename ValueType, typename ContextType>+FanoutChannel<ValueType, ContextType>::operator bool() const {+  return processor_ != nullptr;+}++template <typename ValueType, typename ContextType>+Receiver<ValueType> FanoutChannel<ValueType, ContextType>::subscribe(+    folly::Function<std::vector<ValueType>(const ContextType&)>+        getInitialValues) {+  return processor_->subscribe(std::move(getInitialValues));+}++template <typename ValueType, typename ContextType>+bool FanoutChannel<ValueType, ContextType>::anySubscribers() const {+  return processor_->anySubscribers();+}++template <typename ValueType, typename ContextType>+void FanoutChannel<ValueType, ContextType>::closeSubscribers(+    exception_wrapper ex) {+  processor_->closeSubscribers(+      ex ? detail::CloseResult(std::move(ex)) : detail::CloseResult());+}++template <typename ValueType, typename ContextType>+void FanoutChannel<ValueType, ContextType>::close(exception_wrapper ex) && {+  processor_->destroyHandle(+      ex ? detail::CloseResult(std::move(ex)) : detail::CloseResult());+  processor_ = nullptr;+}++template <typename ValueType, typename ContextType>+ContextType FanoutChannel<ValueType, ContextType>::getContext() const {+  return processor_->getContext();+}++namespace detail {++template <typename ValueType, typename ContextType>+class IFanoutChannelProcessor : public IChannelCallback {+ public:+  virtual Receiver<ValueType> subscribe(+      folly::Function<std::vector<ValueType>(const ContextType&)>+          getInitialValues) = 0;++  virtual bool anySubscribers() = 0;++  virtual void closeSubscribers(CloseResult closeResult) = 0;++  virtual void destroyHandle(CloseResult closeResult) = 0;++  virtual ContextType getContext() = 0;+};++/**+ * This object fans out values from the input receiver to all output receivers.+ * The lifetime of this object is described by the following state machine.+ *+ * The input receiver can be in one of three conceptual states: Active,+ * CancellationTriggered, or CancellationProcessed (removed). When the input+ * receiver reaches the CancellationProcessed state AND the user's FanoutChannel+ * object is deleted, this object is deleted.+ *+ * When an input receiver receives a value indicating that the channel has+ * been closed, the state of the input receiver transitions from Active directly+ * to CancellationProcessed (and this object will be deleted once the user+ * destroys their FanoutChannel object).+ *+ * When the user destroys their FanoutChannel object, the state of the input+ * receiver transitions from Active to CancellationTriggered. This object will+ * then be deleted once the input receiver transitions to the+ * CancellationProcessed state.+ */+template <typename ValueType, typename ContextType>+class FanoutChannelProcessor+    : public IFanoutChannelProcessor<ValueType, ContextType> {+ private:+  struct State {+    State(ContextType _context) : context(std::move(_context)) {}++    ChannelState getReceiverState() {+      return detail::getReceiverState(receiver.get());+    }++    ChannelBridgePtr<ValueType> receiver;+    FanoutSender<ValueType> fanoutSender;+    ContextType context;+    bool handleDeleted{false};+  };++  using WLockedStatePtr = typename folly::Synchronized<State>::WLockedPtr;++ public:+  explicit FanoutChannelProcessor(+      folly::Executor::KeepAlive<folly::SequencedExecutor> executor,+      ContextType context)+      : executor_(std::move(executor)), state_(std::move(context)) {}++  /**+   * Starts fanning out values from the input receiver to all output receivers.+   *+   * @param inputReceiver: The input receiver to fan out values from.+   */+  void start(Receiver<ValueType> inputReceiver) {+    auto state = state_.wlock();+    auto [unbufferedInputReceiver, buffer] =+        detail::receiverUnbuffer(std::move(inputReceiver));+    state->receiver = std::move(unbufferedInputReceiver);++    // Start processing new values that come in from the input receiver.+    processAllAvailableValues(state, std::move(buffer));+  }++  /**+   * Returns a new output receiver that will receive all values from the input+   * receiver. If a getInitialValues parameter is provided, it will be executed+   * to determine the set of initial values that will (only) go to the new input+   * receiver.+   */+  Receiver<ValueType> subscribe(+      folly::Function<std::vector<ValueType>(const ContextType&)>+          getInitialValues) override {+    auto state = state_.wlock();+    auto initialValues = getInitialValues+        ? getInitialValues(state->context)+        : std::vector<ValueType>();+    if (!state->receiver) {+      auto [receiver, sender] = Channel<ValueType>::create();+      for (auto&& value : initialValues) {+        sender.write(std::move(value));+      }+      std::move(sender).close();+      return std::move(receiver);+    }+    return state->fanoutSender.subscribe(std::move(initialValues));+  }++  /**+   * Closes all subscribers without closing the fanout channel.+   */+  void closeSubscribers(CloseResult closeResult) override {+    auto state = state_.wlock();+    std::move(state->fanoutSender)+        .close(+            closeResult.exception.has_value()+                ? closeResult.exception.value()+                : exception_wrapper());+  }++  /**+   * This is called when the user's FanoutChannel object has been destroyed.+   */+  void destroyHandle(CloseResult closeResult) override {+    auto state = state_.wlock();+    processHandleDestroyed(state, std::move(closeResult));+  }++  /**+   * Returns whether this fanout channel has any output receivers.+   */+  bool anySubscribers() override {+    return state_.wlock()->fanoutSender.anySubscribers();+  }++  ContextType getContext() override { return state_.rlock()->context; }++ private:+  /**+   * Called when one of the channels we are listening to has an update (either+   * a value from the input receiver or a cancellation from an output receiver).+   */+  void consume(ChannelBridgeBase*) override {+    executor_->add([=, this]() {+      // One or more values are now available from the input receiver.+      auto state = state_.wlock();+      CHECK_NE(state->getReceiverState(), ChannelState::CancellationProcessed);+      processAllAvailableValues(state);+    });+  }++  void canceled(ChannelBridgeBase*) override {+    executor_->add([=, this]() {+      // We previously cancelled this input receiver, due to the destruction of+      // the handle. Process the cancellation for this input receiver.+      auto state = state_.wlock();+      processReceiverCancelled(state, CloseResult());+    });+  }++  /**+   * Processes all available values from the input receiver (starting from the+   * provided buffer, if present).+   *+   * If an value was received indicating that the input channel has been closed+   * (or if the transform function indicated that channel should be closed), we+   * will process cancellation for the input receiver.+   */+  void processAllAvailableValues(+      WLockedStatePtr& state,+      std::optional<ReceiverQueue<ValueType>> buffer = std::nullopt) {+    auto closeResult = state->receiver->isReceiverCancelled()+        ? CloseResult()+        : (buffer.has_value() ? processValues(state, std::move(buffer.value()))+                              : std::nullopt);+    while (!closeResult.has_value()) {+      if (state->receiver->receiverWait(this)) {+        // There are no more values available right now. We will stop processing+        // until the channel fires the consume() callback (indicating that more+        // values are available).+        break;+      }+      auto values = state->receiver->receiverGetValues();+      CHECK(!values.empty());+      closeResult = processValues(state, std::move(values));+    }+    if (closeResult.has_value()) {+      // The receiver received a value indicating channel closure.+      state->receiver->receiverCancel();+      processReceiverCancelled(state, std::move(closeResult.value()));+    }+  }++  /**+   * Processes the given set of values for the input receiver. Returns a+   * CloseResult if channel was closed, so the caller can stop attempting to+   * process values from it.+   */+  std::optional<CloseResult> processValues(+      WLockedStatePtr& state, ReceiverQueue<ValueType> values) {+    while (!values.empty()) {+      auto inputResult = std::move(values.front());+      values.pop();+      if (inputResult.hasValue()) {+        // We have received a normal value from the input receiver. Write it to+        // all output senders.+        state->context.update(+            inputResult.value(), state->fanoutSender.numSubscribers());+        state->fanoutSender.write(std::move(inputResult.value()));+      } else {+        // The input receiver was closed.+        return inputResult.hasException()+            ? CloseResult(std::move(inputResult.exception()))+            : CloseResult();+      }+    }+    return std::nullopt;+  }++  /**+   * Processes the cancellation of the input receiver. We will close all senders+   * with the exception received from the input receiver (if any).+   */+  void processReceiverCancelled(+      WLockedStatePtr& state, CloseResult closeResult) {+    CHECK_EQ(state->getReceiverState(), ChannelState::CancellationTriggered);+    state->receiver = nullptr;+    std::move(state->fanoutSender)+        .close(+            closeResult.exception.has_value()+                ? closeResult.exception.value()+                : exception_wrapper());+    maybeDelete(state);+  }++  /**+   * Processes the destruction of the user's FanoutChannel object.  We will+   * cancel the receiver and trigger cancellation for all senders not already+   * cancelled.+   */+  void processHandleDestroyed(WLockedStatePtr& state, CloseResult closeResult) {+    state->handleDeleted = true;+    if (state->getReceiverState() == ChannelState::Active) {+      state->receiver->receiverCancel();+    }+    std::move(state->fanoutSender)+        .close(+            closeResult.exception.has_value()+                ? closeResult.exception.value()+                : exception_wrapper());+    maybeDelete(state);+  }++  /**+   * Deletes this object if we have already processed cancellation for the+   * receiver and all senders, and if the user's FanoutChannel object was+   * destroyed.+   */+  void maybeDelete(WLockedStatePtr& state) {+    if (state->getReceiverState() == ChannelState::CancellationProcessed &&+        state->handleDeleted) {+      state.unlock();+      delete this;+    }+  }++  folly::Executor::KeepAlive<folly::SequencedExecutor> executor_;+  folly::Synchronized<State> state_;+};+} // namespace detail++template <typename TReceiver, typename ValueType, typename ContextType>+FanoutChannel<ValueType, ContextType> createFanoutChannel(+    TReceiver inputReceiver,+    folly::Executor::KeepAlive<folly::SequencedExecutor> executor,+    ContextType context) {+  auto* processor = new detail::FanoutChannelProcessor<ValueType, ContextType>(+      std::move(executor), std::move(context));+  processor->start(std::move(inputReceiver));+  return FanoutChannel<ValueType, ContextType>(processor);+}+} // namespace channels+} // namespace folly
+ folly/folly/channels/FanoutChannel.h view
@@ -0,0 +1,151 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <folly/channels/Channel.h>+#include <folly/executors/SequencedExecutor.h>++namespace folly {+namespace channels {++namespace detail {+template <typename ValueType, typename ContextType>+class IFanoutChannelProcessor;+}++template <typename TValue>+struct NoContext {+  void update(const TValue&, size_t) {}+};++/**+ * A fanout channel allows fanning out updates from a single input receiver+ * to multiple output receivers.+ *+ * When a new output receiver is added, an optional function will be run that+ * computes a set of initial values. These initial values will only be sent to+ * the new receiver.+ *+ * FanoutChannel allows specifying an optional context object. If specified, the+ * context object must have a void update function:+ *+ *   void update(const ValueType&);+ *+ * This update function will be called on every value from the input receiver.+ * The context will be passed to the getInitialUpdates argument to subscribe,+ * allowing for initial updates to depend on the context. This facilitates the+ * common pattern of letting new subscribers know where they are starting from.+ *+ * Example without context:+ *+ *   // Function that returns a receiver:+ *   Receiver<int> getInputReceiver();+ *+ *   // Function that returns an executor+ *   folly::Executor::KeepAlive<folly::SequencedExecutor> getExecutor();+ *+ *   auto fanoutChannel = createFanoutChannel(getReceiver(), getExecutor());+ *   auto receiver1 = fanoutChannel.subscribe();+ *   auto receiver2 = fanoutChannel.subscribe();+ *   auto receiver3 = fanoutChannel.subscribe([]{ return {1, 2, 3}; });+ *+ * Example with context:+ *+ *   struct Context {+ *     int lastValue{-1};+ *+ *     void update(const int& value) {+ *       lastValue = value;+ *     }+ *   };+ *+ *   auto fanoutChannel =+ *       createFanoutChannel(getReceiver(), getExecutor(), Context());+ *   auto receiver1 = fanoutChannel.subscribe(+ *       [](const Context& context) { return {context.latestValue}; });+ *   auto receiver2 = fanoutChannel.subscribe(+ *       [](const Context& context) { return {context.latestValue}; });+ *   std::move(fanoutChannel).close();+ */+template <typename ValueType, typename ContextType = NoContext<ValueType>>+class FanoutChannel {+  using TProcessor = detail::IFanoutChannelProcessor<ValueType, ContextType>;++ public:+  explicit FanoutChannel(TProcessor* processor);+  FanoutChannel(FanoutChannel&& other) noexcept;+  FanoutChannel& operator=(FanoutChannel&& other) noexcept;+  ~FanoutChannel();++  /**+   * Returns whether this FanoutChannel is a valid object.+   */+  explicit operator bool() const;++  /**+   * Returns a new output receiver that will receive all values from the input+   * receiver.+   *+   * If a getInitialValues parameter is provided, it will be executed+   * to determine the set of initial values that will (only) go to the new input+   * receiver. Other functions on this class should not be called from within+   * getInitialValues, or a deadlock will occur.+   */+  Receiver<ValueType> subscribe(+      folly::Function<std::vector<ValueType>(const ContextType&)>+          getInitialValues = {});++  /**+   * Returns whether this fanout channel has any subscribers.+   */+  bool anySubscribers() const;++  /**+   * Closes all subscribers, without closing the fanout channel. New subscribers+   * can be added after this call.+   */+  void closeSubscribers(exception_wrapper ex = exception_wrapper());++  /**+   * Closes the fanout channel.+   */+  void close(exception_wrapper ex = exception_wrapper()) &&;++  /**+   * Get the context+   */+  ContextType getContext() const;++ private:+  TProcessor* processor_;+};++/**+ * Creates a new fanout channel that fans out updates from an input receiver.+ */+template <+    typename ReceiverType,+    typename ValueType = typename ReceiverType::ValueType,+    typename ContextType = NoContext<typename ReceiverType::ValueType>>+FanoutChannel<ValueType, ContextType> createFanoutChannel(+    ReceiverType inputReceiver,+    folly::Executor::KeepAlive<folly::SequencedExecutor> executor,+    ContextType context = ContextType());+} // namespace channels+} // namespace folly++#include <folly/channels/FanoutChannel-inl.h>
+ folly/folly/channels/FanoutSender-inl.h view
@@ -0,0 +1,358 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <folly/channels/Channel.h>+#include <folly/container/F14Set.h>+#include <folly/executors/SequencedExecutor.h>+#include <folly/experimental/channels/detail/Utility.h>++namespace folly {+namespace channels {++namespace detail {+template <typename ValueType>+class FanoutSenderProcessor : public IChannelCallback {+ private:+  struct State {+    folly::F14FastSet<ChannelBridge<ValueType>*> senders_;+    bool handleDestroyed_{false};+  };++  using WLockedStatePtr = typename folly::Synchronized<State>::WLockedPtr;++ public:+  /**+   * Subscribes with an already-created sender.+   */+  void addSender(detail::ChannelBridgePtr<ValueType> sender) {+    auto state = state_.wlock();+    sender->senderWait(this);+    state->senders_.insert(sender.release());+  }++  /**+   * Sends the given value to all corresponding receivers.+   */+  template <typename U = ValueType>+  void write(U&& element) {+    auto state = state_.wlock();+    for (auto* sender : state->senders_) {+      sender->senderPush(element);+    }+  }++  /**+   * This is called when the user's FanoutSender object has been destroyed.+   */+  void destroyHandle(CloseResult closeResult) {+    processHandleDestroyed(state_.wlock(), std::move(closeResult));+  }++  /**+   * Returns whether this fanout channel has any output receivers.+   */+  size_t numSubscribers() const { return state_.rlock()->senders_.size(); }++  std::pair<bool, ChannelBridgePtr<ValueType>>+  stealSenderAndDestorySelfIfSingle() {+    auto state = state_.wlock();+    if (state->senders_.empty()) {+      // There are no remaining senders. We will destroy ourselves.+      state->handleDestroyed_ = true;+      maybeDelete(std::move(state));+      return std::make_pair(true, ChannelBridgePtr<ValueType>());+    } else if (state->senders_.size() == 1) {+      // There is one remaining sender.+      auto* sender = *state->senders_.begin();+      auto* callback = sender->cancelSenderWait();+      if (callback) {+        // We successfully cancelled the callback, so we can now destroy+        // ourselves and return the sender.+        state->senders_.clear();+        state->handleDestroyed_ = true;+        maybeDelete(std::move(state));+        return std::make_pair(true, ChannelBridgePtr<ValueType>(sender));+      } else {+        // We failed to cancel the callback. This means that another thread is+        // invoking the callback by calling consume(), letting us know that the+        // corresponding receiver was deleted. The callback will start running+        // once we release the lock, so we will let the callback delete the+        // sender (and then destroy ourselves).+        return std::make_pair(true, ChannelBridgePtr<ValueType>());+      }+    } else {+      // There is more than one sender. Do not destroy ourselves.+      return std::make_pair(false, ChannelBridgePtr<ValueType>());+    }+  }++  static ChannelState getSenderState(ChannelBridge<ValueType>* sender) {+    return detail::getSenderState(sender);+  }++ private:+  /**+   * Called when receiving a cancellation from an output receiver.+   */+  void consume(ChannelBridgeBase* bridge) override {+    // The consumer of an output receiver has stopped consuming.+    auto state = state_.wlock();+    auto* sender = static_cast<ChannelBridge<ValueType>*>(bridge);+    CHECK_NE(getSenderState(sender), ChannelState::CancellationProcessed);+    sender->senderClose();+    processSenderCancelled(std::move(state), sender);+  }++  void canceled(ChannelBridgeBase*) override {+    // We cancel the callback before we close the sender explicitly, so this+    // should never be hit.+    CHECK(false);+  }++  /**+   * Processes the cancellation of a sender (indicating that the consumer of+   * the corresponding output receiver has stopped consuming).+   */+  void processSenderCancelled(+      WLockedStatePtr state, ChannelBridge<ValueType>* sender) {+    CHECK_EQ(getSenderState(sender), ChannelState::CancellationTriggered);+    state->senders_.erase(sender);+    deleteSender(sender);+    maybeDelete(std::move(state));+  }++  /**+   * Processes the destruction of the user's FanoutChannel object.  We will+   * cancel the receiver and trigger cancellation for all senders not already+   * cancelled.+   */+  void processHandleDestroyed(WLockedStatePtr state, CloseResult closeResult) {+    CHECK(!state->handleDestroyed_);+    state->handleDestroyed_ = true;+    auto senders = state->senders_;+    for (auto* sender : senders) {+      auto* callback = sender->cancelSenderWait();+      if (closeResult.exception.has_value()) {+        sender->senderClose(closeResult.exception.value());+      } else {+        sender->senderClose();+      }+      if (callback) {+        // We successfully cancelled the callback, so we can now delete the+        // sender.+        CHECK_EQ(callback, this);+        state->senders_.erase(sender);+        deleteSender(sender);+      } else {+        // We failed to cancel the callback. This means that another thread is+        // invoking the callback by calling consume(), letting us know that the+        // corresponding receiver was deleted. The callback will start running+        // once we release the lock, so we will let the callback delete the+        // sender.+      }+    }+    maybeDelete(std::move(state));+  }++  /*+   * Deletes the given sender.+   */+  void deleteSender(ChannelBridge<ValueType>* sender) {+    (ChannelBridgePtr<ValueType>(sender));+  }++  /**+   * Deletes this object if we have already processed cancellation for the+   * receiver and all senders, and if the user's FanoutChannel object was+   * destroyed.+   */+  void maybeDelete(WLockedStatePtr state) {+    if (state->senders_.empty() && state->handleDestroyed_) {+      state.unlock();+      delete this;+    }+  }++  folly::Synchronized<State> state_;+};+} // namespace detail++template <typename ValueType>+FanoutSender<ValueType>::FanoutSender()+    : senders_(static_cast<detail::ChannelBridge<ValueType>*>(nullptr)) {}++template <typename ValueType>+FanoutSender<ValueType>::FanoutSender(FanoutSender&& other) noexcept+    : senders_(std::move(other.senders_)) {}++template <typename ValueType>+FanoutSender<ValueType>& FanoutSender<ValueType>::operator=(+    FanoutSender&& other) noexcept {+  if (&other == this) {+    return *this;+  }+  std::move(*this).close();+  senders_ = std::move(senders_);+  return *this;+}++template <typename ValueType>+FanoutSender<ValueType>::~FanoutSender() {+  std::move(*this).close();+}++template <typename ValueType>+Receiver<ValueType> FanoutSender<ValueType>::subscribe(+    std::vector<ValueType> initialValues) {+  auto [newReceiver, newSender] = Channel<ValueType>::create();+  for (auto&& initialValue : initialValues) {+    newSender.write(std::move(initialValue));+  }+  subscribe(std::move(newSender));+  return std::move(newReceiver);+}++template <typename ValueType>+void FanoutSender<ValueType>::subscribe(Sender<ValueType> newSender) {+  clearSendersWithClosedReceivers();+  if (!anySubscribersImpl()) {+    // There are currently no output receivers. Store the new output receiver.+    senders_.set(detail::senderGetBridge(newSender).release());+  } else if (!hasProcessor()) {+    // There is currently exactly one output receiver. Convert to a processor.+    auto* processor = new detail::FanoutSenderProcessor<ValueType>();+    processor->addSender(+        detail::ChannelBridgePtr<ValueType>(getSingleSender()));+    processor->addSender(std::move(detail::senderGetBridge(newSender)));+    senders_.set(processor);+  } else {+    // There are currently more than one output receivers. Add the new receiver+    // to the existing processor.+    auto* processor = getProcessor();+    processor->addSender(std::move(detail::senderGetBridge(newSender)));+  }+}++template <typename ValueType>+bool FanoutSender<ValueType>::anySubscribers() const {+  clearSendersWithClosedReceivers();+  return anySubscribersImpl();+}++template <typename ValueType>+std::uint64_t FanoutSender<ValueType>::numSubscribers() const {+  clearSendersWithClosedReceivers();+  if (!anySubscribersImpl()) {+    return 0;+  } else if (!hasProcessor()) {+    return 1;+  } else {+    return getProcessor()->numSubscribers();+  }+}++template <typename ValueType>+template <typename U>+void FanoutSender<ValueType>::write(U&& element) {+  clearSendersWithClosedReceivers();+  if (!anySubscribersImpl()) {+    // There are currently no output receivers to write to.+    return;+  } else if (!hasProcessor()) {+    // There is exactly one output receiver. Write the value to that receiver.+    getSingleSender()->senderPush(std::forward<U>(element));+  } else {+    getProcessor()->write(std::forward<U>(element));+  }+}++template <typename ValueType>+void FanoutSender<ValueType>::close(exception_wrapper ex) && {+  clearSendersWithClosedReceivers();+  if (!anySubscribersImpl()) {+    // There are no output receivers to close.+    return;+  } else if (!hasProcessor()) {+    // There is exactly one output receiver to close.+    if (ex) {+      getSingleSender()->senderClose(ex);+    } else {+      getSingleSender()->senderClose();+    }+    // Delete the output receiver.+    (detail::ChannelBridgePtr<ValueType>(getSingleSender()));+    senders_.set(static_cast<detail::ChannelBridge<ValueType>*>(nullptr));+  } else {+    // There is more than one output receiver to close.+    getProcessor()->destroyHandle(+        ex ? detail::CloseResult(std::move(ex)) : detail::CloseResult());+    senders_.set(static_cast<detail::ChannelBridge<ValueType>*>(nullptr));+  }+}++template <typename ValueType>+bool FanoutSender<ValueType>::anySubscribersImpl() const {+  return hasProcessor() || getSingleSender() != nullptr;+}++template <typename ValueType>+bool FanoutSender<ValueType>::hasProcessor() const {+  return senders_.index() == 1;+}++template <typename ValueType>+detail::ChannelBridge<ValueType>* FanoutSender<ValueType>::getSingleSender()+    const {+  return senders_.get(folly::tag_t<detail::ChannelBridge<ValueType>>{});+}++template <typename ValueType>+detail::FanoutSenderProcessor<ValueType>*+FanoutSender<ValueType>::getProcessor() const {+  return senders_.get(folly::tag_t<detail::FanoutSenderProcessor<ValueType>>{});+}++template <typename ValueType>+void FanoutSender<ValueType>::clearSendersWithClosedReceivers() const {+  if (hasProcessor()) {+    auto [processorDestroyed, remainingSender] =+        getProcessor()->stealSenderAndDestorySelfIfSingle();+    if (processorDestroyed) {+      if (remainingSender) {+        senders_.set(remainingSender.release());+      } else {+        senders_.set(static_cast<detail::ChannelBridge<ValueType>*>(nullptr));+      }+    }+  } else {+    auto* bridge = getSingleSender();+    if (bridge) {+      // There is currently exactly one output receiver. Check to see if it has+      // been cancelled.+      auto values = bridge->senderGetValues();+      if (!values.empty()) {+        bridge->senderClose();+        senders_.set(static_cast<detail::ChannelBridge<ValueType>*>(nullptr));+        // Delete the output receiver.+        (detail::ChannelBridgePtr<ValueType>(bridge));+      }+    }+  }+}+} // namespace channels+} // namespace folly
+ folly/folly/channels/FanoutSender.h view
@@ -0,0 +1,111 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <folly/channels/Channel.h>+#include <folly/container/F14Set.h>+#include <folly/experimental/channels/detail/PointerVariant.h>++namespace folly {+namespace channels {++namespace detail {+template <typename ValueType>+class FanoutSenderProcessor;+}++/**+ * A FanoutSender allows fanning out updates to multiple output receivers.+ * Values can be written as with a normal Sender. When there is only one output+ * receiver, the memory used by a FanoutSender (and the corresponding output+ * receiver) is the same as the memory used by a normal channel.+ *+ * When a new output receiver is added, an optional vector of initial values+ * can be provided. These initial values will only be sent to the new receiver.+ *+ * Memory used by closed receivers is reclaimed lazily (when iterating over+ * receivers).+ *+ * Example:+ *+ *  FanoutSender<int> fanoutSender;+ *  auto receiver1 = fanoutSender.subscribe();+ *  auto receiver2 = fanoutSender.subscribe();+ *  auto receiver3 = fanoutSender.subscribe({1, 2, 3});+ *  std::move(fanoutSender).close();+ */+template <typename ValueType>+class FanoutSender {+ public:+  FanoutSender();+  FanoutSender(FanoutSender&& other) noexcept;+  FanoutSender& operator=(FanoutSender&& other) noexcept;+  ~FanoutSender();++  /**+   * Returns a new output receiver that will receive all values written to the+   * FanoutSender. If the initialValues parameter is provided, the given values+   * will (only) go to the new output receiver.+   */+  Receiver<ValueType> subscribe(std::vector<ValueType> initialValues = {});++  /**+   * Subscribes with an already-created sender.+   */+  void subscribe(Sender<ValueType> sender);++  /**+   * Returns whether this fanout sender has any active output receivers.+   */+  bool anySubscribers() const;++  /**+   * Returns the number of output receivers for this fanout sender.+   */+  std::uint64_t numSubscribers() const;++  /**+   * Sends the given value to all corresponding receivers.+   */+  template <typename U = ValueType>+  void write(U&& element);++  /**+   * Closes the fanout sender.+   */+  void close(exception_wrapper ex = exception_wrapper()) &&;++ private:+  bool anySubscribersImpl() const;++  bool hasProcessor() const;++  detail::ChannelBridge<ValueType>* getSingleSender() const;++  detail::FanoutSenderProcessor<ValueType>* getProcessor() const;++  void clearSendersWithClosedReceivers() const;++  mutable detail::PointerVariant<+      detail::ChannelBridge<ValueType>,+      detail::FanoutSenderProcessor<ValueType>>+      senders_;+};+} // namespace channels+} // namespace folly++#include <folly/channels/FanoutSender-inl.h>
+ folly/folly/channels/MaxConcurrentRateLimiter.cpp view
@@ -0,0 +1,84 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#include <folly/channels/MaxConcurrentRateLimiter.h>++namespace folly {+namespace channels {++class MaxConcurrentRateLimiter::Token : public RateLimiter::Token {+ public:+  explicit Token(+      std::shared_ptr<MaxConcurrentRateLimiter> maxConcurrentRateLimiter)+      : maxConcurrentRateLimiter_{std::move(maxConcurrentRateLimiter)} {}++  Token(Token&&) = default;+  Token& operator=(Token&&) = default;+  Token(const Token&) = delete;+  Token& operator=(const Token&) = delete;++  ~Token() override {+    if (maxConcurrentRateLimiter_) {+      maxConcurrentRateLimiter_->release();+    }+  }++ private:+  std::shared_ptr<MaxConcurrentRateLimiter> maxConcurrentRateLimiter_;+};++std::shared_ptr<MaxConcurrentRateLimiter> MaxConcurrentRateLimiter::create(+    size_t maxConcurrent) {+  return std::shared_ptr<MaxConcurrentRateLimiter>(+      new MaxConcurrentRateLimiter(maxConcurrent));+}++MaxConcurrentRateLimiter::MaxConcurrentRateLimiter(size_t maxConcurrent)+    : maxConcurrent_(maxConcurrent) {}++void MaxConcurrentRateLimiter::executeWhenReady(+    folly::Function<void(std::unique_ptr<RateLimiter::Token>)> func,+    Executor::KeepAlive<SequencedExecutor> executor) {+  auto state = state_.wlock();+  if (state->running < maxConcurrent_) {+    CHECK(state->queue.empty());+    state->running++;+    executor->add(+        [func = std::move(func),+         token = std::make_unique<MaxConcurrentRateLimiter::Token>(+             std::static_pointer_cast<MaxConcurrentRateLimiter>(+                 shared_from_this()))]() mutable { func(std::move(token)); });+  } else {+    state->queue.enqueue(QueueItem{std::move(func), std::move(executor)});+  }+}++void MaxConcurrentRateLimiter::release() {+  auto state = state_.wlock();+  if (!state->queue.empty()) {+    auto queueItem = state->queue.dequeue();+    queueItem.executor->add(+        [func = std::move(queueItem.func),+         token = std::make_unique<MaxConcurrentRateLimiter::Token>(+             std::static_pointer_cast<MaxConcurrentRateLimiter>(+                 shared_from_this()))]() mutable { func(std::move(token)); });+  } else {+    state->running--;+  }+}++} // namespace channels+} // namespace folly
+ folly/folly/channels/MaxConcurrentRateLimiter.h view
@@ -0,0 +1,56 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <folly/Synchronized.h>+#include <folly/channels/RateLimiter.h>+#include <folly/concurrency/UnboundedQueue.h>+#include <folly/executors/SequencedExecutor.h>++namespace folly {+namespace channels {++class MaxConcurrentRateLimiter : public RateLimiter {+ public:+  static std::shared_ptr<MaxConcurrentRateLimiter> create(size_t maxConcurrent);++  void executeWhenReady(+      folly::Function<void(std::unique_ptr<Token>)> func,+      Executor::KeepAlive<SequencedExecutor> executor) override;++ private:+  class Token;+  friend class Token;++  explicit MaxConcurrentRateLimiter(size_t maxConcurrent);+  void release();++  struct QueueItem {+    folly::Function<void(std::unique_ptr<Token>)> func;+    Executor::KeepAlive<SequencedExecutor> executor;+  };++  struct State {+    USPSCQueue<QueueItem, false /* MayBlock */, 6 /* LgSegmentSize */> queue;+    size_t running{0};+  };++  const size_t maxConcurrent_;+  folly::Synchronized<State> state_;+};+} // namespace channels+} // namespace folly
+ folly/folly/channels/Merge-inl.h view
@@ -0,0 +1,302 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <folly/channels/Channel.h>+#include <folly/container/F14Set.h>+#include <folly/executors/SequencedExecutor.h>+#include <folly/experimental/channels/detail/Utility.h>++namespace folly {+namespace channels {++namespace detail {++/**+ * This object does the merging of values from the input receiver to the output+ * receiver. It is not an object that the user is aware of or holds a pointer+ * to. The lifetime of this object is described by the following state machine.+ *+ * The sender and all receivers can be in one of three conceptual states:+ * Active, CancellationTriggered, or CancellationProcessed. When the sender and+ * all receivers reach the CancellationProcessed state, this object is deleted.+ *+ * When an input receiver receives a value indicating that the channel has been+ * closed, the state of that receiver transitions from Active directly to+ * CancellationProcessed.+ *+ * If this is the last receiver to be closed, or if the receiver closed with an+ * exception, the state of the sender and all other receivers transitions from+ * Active to CancellationTriggered. In that case, once we receive callbacks+ * indicating the cancellation signal has been received for all other receivers+ * and the sender, the state of the sender and all other receivers transitions+ * to CancellationProcessed (and this object is deleted).+ *+ * When the sender receives notification that the consumer of the output+ * receiver has stopped consuming, the state of the sender transitions from+ * Active directly to CancellationProcessed, and the state of all remaining+ * input receivers transitions from Active to CancellationTriggered. This+ * object will then be deleted once each remaining input receiver transitions to+ * the CancellationProcessed state (after we receive each cancelled callback).+ */+template <typename TValue, bool WaitForAllInputsToClose>+class MergeProcessor : public IChannelCallback {+ public:+  MergeProcessor(+      Sender<TValue> sender,+      folly::Executor::KeepAlive<folly::SequencedExecutor> executor)+      : sender_(std::move(detail::senderGetBridge(sender))),+        executor_(std::move(executor)) {}++  /**+   * Starts merging inputs from all input receivers into the output receiver.+   *+   * @param inputReceivers: The collection of input receivers to merge.+   */+  void start(std::vector<Receiver<TValue>> inputReceivers) {+    executor_->add([=,+                    this,+                    inputReceivers = std::move(inputReceivers)]() mutable {+      if (!sender_->senderWait(this)) {+        sender_->senderClose();+        processSenderCancelled();+        return;+      }+      auto buffers =+          folly::F14FastMap<ChannelBridge<TValue>*, ReceiverQueue<TValue>>();+      receivers_.reserve(inputReceivers.size());+      buffers.reserve(inputReceivers.size());+      for (auto& inputReceiver : inputReceivers) {+        auto [unbufferedInputReceiver, buffer] =+            detail::receiverUnbuffer(std::move(inputReceiver));+        CHECK(unbufferedInputReceiver != nullptr)+            << "The bridge in the input receiver is null.";+        CHECK(buffers+                  .insert(std::make_pair(+                      unbufferedInputReceiver.get(), std::move(buffer)))+                  .second);+        receivers_.insert(unbufferedInputReceiver.release());+      }+      for (auto* receiver : receivers_) {+        processAllAvailableValues(+            receiver,+            !buffers.empty()+                ? std::make_optional(std::move(buffers.at(receiver)))+                : std::nullopt);+      }+    });+  }++  /**+   * Called when one of the channels we are listening to has an update (either+   * a value from an input receiver or a cancellation from the output receiver).+   */+  void consume(ChannelBridgeBase* bridge) override {+    executor_->add([=, this]() {+      if (bridge == sender_.get()) {+        // The consumer of the output receiver has stopped consuming.+        CHECK_NE(getSenderState(), ChannelState::CancellationProcessed);+        sender_->senderClose();+        processSenderCancelled();+      } else {+        // One or more values are now available from an input receiver.+        auto* receiver = static_cast<ChannelBridge<TValue>*>(bridge);+        CHECK_NE(+            getReceiverState(receiver), ChannelState::CancellationProcessed);+        processAllAvailableValues(receiver);+      }+    });+  }++  /**+   * Called after we cancelled one of the channels we were listening to (either+   * the sender or an input receiver).+   */+  void canceled(ChannelBridgeBase* bridge) override {+    executor_->add([=, this]() {+      if (bridge == sender_.get()) {+        // We previously cancelled the sender due to an input receiver closure+        // with an exception (or the closure of all input receivers without an+        // exception). Process the cancellation for the sender.+        CHECK_EQ(getSenderState(), ChannelState::CancellationTriggered);+        processSenderCancelled();+      } else {+        // We previously cancelled this input receiver, either because the+        // consumer of the output receiver stopped consuming or because another+        // input receiver received an exception. Process the cancellation for+        // this input receiver.+        auto* receiver = static_cast<ChannelBridge<TValue>*>(bridge);+        CHECK_EQ(+            getReceiverState(receiver), ChannelState::CancellationTriggered);+        processReceiverCancelled(receiver, CloseResult());+      }+    });+  }++ private:+  /**+   * Processes all available values from the input receiver (starting from the+   * provided buffer, if present).+   *+   * If an value was received indicating that the input channel has been closed+   * (or if the transform function indicated that channel should be closed), we+   * will process cancellation for the input receiver.+   */+  void processAllAvailableValues(+      ChannelBridge<TValue>* receiver,+      std::optional<ReceiverQueue<TValue>> buffer = std::nullopt) {+    auto closeResult =+        getReceiverState(receiver) == ChannelState::CancellationTriggered+        ? CloseResult()+        : buffer.has_value()+        ? processValues(std::move(buffer.value()))+        : std::nullopt;+    while (!closeResult.has_value()) {+      if (receiver->receiverWait(this)) {+        // There are no more values available right now. We will stop processing+        // until the channel fires the consume() callback (indicating that more+        // values are available).+        break;+      }+      auto values = receiver->receiverGetValues();+      CHECK(!values.empty());+      closeResult = processValues(std::move(values));+    }+    if (closeResult.has_value()) {+      // The receiver received a value indicating channel closure.+      receiver->receiverCancel();+      processReceiverCancelled(receiver, std::move(closeResult.value()));+    }+  }++  /**+   * Processes the given set of values for an input receiver. Returns a+   * CloseResult if the given channel was closed, so the caller can stop+   * attempting to process values from it.+   */+  std::optional<CloseResult> processValues(ReceiverQueue<TValue> values) {+    while (!values.empty()) {+      auto inputResult = std::move(values.front());+      values.pop();+      if (inputResult.hasValue()) {+        // We have received a normal value from an input receiver. Write it to+        // the output receiver.+        sender_->senderPush(std::move(inputResult.value()));+      } else {+        // The input receiver was closed.+        return inputResult.hasException()+            ? CloseResult(std::move(inputResult.exception()))+            : CloseResult();+      }+    }+    return std::nullopt;+  }++  /**+   * Processes the cancellation of an input receiver. If the cancellation was+   * due to receipt of an exception (or the cancellation was the last input+   * receiver to be closed), we will also trigger cancellation for the sender+   * (and all other input receivers).+   */+  void processReceiverCancelled(+      ChannelBridge<TValue>* receiver, CloseResult closeResult) {+    CHECK_EQ(getReceiverState(receiver), ChannelState::CancellationTriggered);+    receivers_.erase(receiver);+    (ChannelBridgePtr<TValue>(receiver));+    if (closeResult.exception.has_value() || !WaitForAllInputsToClose) {+      // We need to close the sender and all other receivers.+      if (getSenderState() == ChannelState::Active) {+        if (closeResult.exception.has_value()) {+          sender_->senderClose(std::move(closeResult.exception.value()));+        } else {+          sender_->senderClose();+        }+      }+      for (auto* otherReceiver : receivers_) {+        if (getReceiverState(otherReceiver) == ChannelState::Active) {+          otherReceiver->receiverCancel();+        }+      }+    } else if (receivers_.empty()) {+      // We just closed the last receiver. Close the sender.+      if (getSenderState() == ChannelState::Active) {+        sender_->senderClose();+      }+    }+    maybeDelete();+  }++  /**+   * Processes the cancellation of the sender (indicating that the consumer of+   * the output receiver has stopped consuming). We will trigger cancellation+   * for all input receivers not already cancelled.+   */+  void processSenderCancelled() {+    CHECK_EQ(getSenderState(), ChannelState::CancellationTriggered);+    sender_ = nullptr;+    for (auto* receiver : receivers_) {+      if (getReceiverState(receiver) == ChannelState::Active) {+        receiver->receiverCancel();+      }+    }+    maybeDelete();+  }++  /**+   * Deletes this object if we have already processed cancellation for the+   * sender and all input receivers.+   */+  void maybeDelete() {+    if (getSenderState() == ChannelState::CancellationProcessed &&+        receivers_.empty()) {+      delete this;+    }+  }++  ChannelState getReceiverState(ChannelBridge<TValue>* receiver) {+    return detail::getReceiverState(receiver);+  }++  ChannelState getSenderState() {+    return detail::getSenderState(sender_.get());+  }++  folly::F14FastSet<ChannelBridge<TValue>*> receivers_;+  ChannelBridgePtr<TValue> sender_;+  folly::Executor::KeepAlive<folly::SequencedExecutor> executor_;+};+} // namespace detail++template <typename TReceiver, typename TValue>+Receiver<TValue> merge(+    std::vector<TReceiver> inputReceivers,+    folly::Executor::KeepAlive<folly::SequencedExecutor> executor,+    bool waitForAllInputsToClose) {+  auto [outputReceiver, outputSender] = Channel<TValue>::create();+  if (waitForAllInputsToClose) {+    auto* processor = new detail::MergeProcessor<TValue, true>(+        std::move(outputSender), std::move(executor));+    processor->start(std::move(inputReceivers));+  } else {+    auto* processor = new detail::MergeProcessor<TValue, false>(+        std::move(outputSender), std::move(executor));+    processor->start(std::move(inputReceivers));+  }+  return std::move(outputReceiver);+}+} // namespace channels+} // namespace folly
+ folly/folly/channels/Merge.h view
@@ -0,0 +1,59 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <folly/channels/Channel.h>+#include <folly/executors/SequencedExecutor.h>++namespace folly {+namespace channels {++/**+ * Merge takes a list of receivers, and returns a new receiver that receives+ * all updates from all input receivers. If any input receiver closes with+ * an exception, the exception is forwarded and the channel is closed. If any+ * input receiver closes without an exception, the channel continues to merge+ * values from the other input receivers until all input receivers are closed.+ *+ * @param inputReceivers: The collection of input receivers to merge.+ *+ * @param executor: A SequencedExecutor used to merge input values.+ *+ * @param waitForAllInputsToClose: When true, if any input receiver closes+ * without an exception, the channel continues to merge values from the other+ * input receivers until all input receivers are closed. If false, the channel+ * closes as soon as any input receiver has closed.+ *+ * Example:+ *+ *  // Example function that returns a list of receivers+ *  std::vector<Receiver<int>> getReceivers();+ *+ *  // Example function that returns an executor+ *  folly::Executor::KeepAlive<folly::SequencedExecutor> getExecutor();+ *+ *  Receiver<int> mergedReceiver = merge(getReceivers(), getExecutor());+ */+template <typename TReceiver, typename TValue = typename TReceiver::ValueType>+Receiver<TValue> merge(+    std::vector<TReceiver> inputReceivers,+    folly::Executor::KeepAlive<folly::SequencedExecutor> executor,+    bool waitForAllInputsToClose = true);+} // namespace channels+} // namespace folly++#include <folly/channels/Merge-inl.h>
+ folly/folly/channels/MergeChannel-inl.h view
@@ -0,0 +1,470 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <folly/channels/Channel.h>+#include <folly/container/F14Map.h>+#include <folly/container/F14Set.h>+#include <folly/executors/SequencedExecutor.h>+#include <folly/experimental/channels/detail/Utility.h>++namespace folly {+namespace channels {++template <typename KeyType, typename ValueType>+MergeChannel<KeyType, ValueType>::MergeChannel(TProcessor* processor)+    : processor_(processor) {}++template <typename KeyType, typename ValueType>+MergeChannel<KeyType, ValueType>::MergeChannel(MergeChannel&& other) noexcept+    : processor_(std::exchange(other.processor_, nullptr)) {}++template <typename KeyType, typename ValueType>+MergeChannel<KeyType, ValueType>& MergeChannel<KeyType, ValueType>::operator=(+    MergeChannel&& other) noexcept {+  if (&other == this) {+    return *this;+  }+  if (processor_) {+    std::move(*this).close();+  }+  processor_ = std::exchange(other.processor_, nullptr);+  return *this;+}++template <typename KeyType, typename ValueType>+MergeChannel<KeyType, ValueType>::~MergeChannel() {+  if (processor_) {+    std::move(*this).close(std::nullopt /* ex */);+  }+}++template <typename KeyType, typename ValueType>+MergeChannel<KeyType, ValueType>::operator bool() const {+  return processor_;+}++template <typename KeyType, typename ValueType>+template <typename TReceiver>+void MergeChannel<KeyType, ValueType>::addNewReceiver(+    KeyType key, TReceiver receiver) {+  processor_->addNewReceiver(key, std::move(receiver));+}++template <typename KeyType, typename ValueType>+void MergeChannel<KeyType, ValueType>::removeReceiver(KeyType key) {+  processor_->removeReceiver(key);+}++template <typename KeyType, typename ValueType>+folly::F14FastSet<KeyType> MergeChannel<KeyType, ValueType>::getReceiverKeys() {+  return processor_->getReceiverKeys();+}++template <typename KeyType, typename ValueType>+void MergeChannel<KeyType, ValueType>::close(+    std::optional<exception_wrapper> ex) && {+  processor_->destroyHandle(+      ex.has_value() ? detail::CloseResult(std::move(ex.value()))+                     : detail::CloseResult());+  processor_ = nullptr;+}++namespace detail {++template <typename KeyType, typename ValueType>+class IMergeChannelProcessor : public IChannelCallback {+ public:+  virtual void addNewReceiver(KeyType key, Receiver<ValueType> receiver) = 0;++  virtual void removeReceiver(KeyType key) = 0;++  virtual folly::F14FastSet<KeyType> getReceiverKeys() = 0;++  virtual void destroyHandle(CloseResult closeResult) = 0;+};++/**+ * This object does the merging of values from the input receivers to the output+ * receiver. The lifetime of this object is described by the following state+ * machine.+ *+ * The sender and all active receivers can be in one of three conceptual states:+ * Active, CancellationTriggered, or CancellationProcessed (removed). When the+ * sender and all receivers reach the CancellationProcessed state AND the user's+ * MergeChannel object is deleted, this object is deleted.+ *+ * When an input receiver receives a value indicating that the channel has+ * been closed, the state of that receiver transitions from Active directly to+ * CancellationProcessed and the receiver is removed.+ *+ * If the receiver closed with an exception, the state of the sender and all+ * other receivers transitions from Active to CancellationTriggered. In that+ * case, once we receive callbacks indicating the cancellation signal has been+ * received for all other receivers and the sender, the state of the sender and+ * all other receivers transitions to CancellationProcessed (and this object+ * will be deleted once the user destroys their MergeChannel object).+ *+ * When the sender receives notification that the consumer of the output+ * receiver has stopped consuming, the state of the sender transitions from+ * Active directly to CancellationProcessed, and the state of all remaining+ * input receivers transitions from Active to CancellationTriggered. Once we+ * receive callbacks for all input receivers indicating that the cancellation+ * signal has been received, each such receiver is transitioned to the+ * CancellationProcessed state (and this object will be deleted once the user+ * destroys their MergeChannel object).+ *+ * When the user destroys their MergeChannel object, the state of the sender and+ * all remaining receivers transition from Active to CancellationTriggered. This+ * object will then be deleted once the sender and each remaining input receiver+ * transitions to the CancellationProcessed state (after we receive each+ * cancelled callback).+ */+template <typename KeyType, typename ValueType>+class MergeChannelProcessor+    : public IMergeChannelProcessor<KeyType, ValueType> {+ private:+  struct State {+    explicit State(+        ChannelBridgePtr<MergeChannelEvent<KeyType, ValueType>> _sender)+        : sender(std::move(_sender)) {}++    ChannelState getSenderState() {+      return detail::getSenderState(sender.get());+    }++    // The output sender for the merge channel.+    ChannelBridgePtr<MergeChannelEvent<KeyType, ValueType>> sender;++    // A non-owning map from key to receiver.+    folly::F14NodeMap<KeyType, ChannelBridge<ValueType>*> receiversByKey;++    // The set of receivers that feed into this MergeChannel. This map "owns"+    // its receivers. MergeChannelProcessor must free any receiver removed from+    // this map.+    folly::F14NodeMap<ChannelBridge<ValueType>*, const KeyType*> receivers;++    // Whether or not the handle to the MergeChannel has been destroyed.+    bool handleDestroyed{false};+  };++  using WLockedStatePtr = typename folly::Synchronized<State>::WLockedPtr;++ public:+  MergeChannelProcessor(+      Sender<MergeChannelEvent<KeyType, ValueType>> sender,+      folly::Executor::KeepAlive<folly::SequencedExecutor> executor)+      : executor_(std::move(executor)),+        state_(State(std::move(detail::senderGetBridge(sender)))) {+    auto state = state_.wlock();+    CHECK(state->sender->senderWait(this));+  }++  /**+   * Adds a new receiver to be merged, along with a key to allow for later+   * removal.+   */+  void addNewReceiver(KeyType key, Receiver<ValueType> receiver) {+    auto state = state_.wlock();+    if (state->getSenderState() != ChannelState::Active) {+      return;+    }+    auto [unbufferedReceiver, buffer] =+        detail::receiverUnbuffer(std::move(receiver));+    auto existingReceiverIt = state->receiversByKey.find(key);+    if (existingReceiverIt != state->receiversByKey.end()) {+      CHECK(state->receivers.contains(existingReceiverIt->second));+      if (!existingReceiverIt->second->isReceiverCancelled()) {+        // We already have a receiver with the given key. Trigger cancellation+        // on that previous receiver.+        existingReceiverIt->second->receiverCancel();+      }+      auto keyToRemove = existingReceiverIt->first;+      state->receivers[existingReceiverIt->second] = nullptr;+      state->receiversByKey.erase(existingReceiverIt);+      state->sender->senderPush(MergeChannelEvent<KeyType, ValueType>{+          keyToRemove, MergeChannelReceiverRemoved{}});+    }+    auto [it, _] = state->receiversByKey.insert(+        std::make_pair(key, unbufferedReceiver.get()));+    auto* receiverPtr = unbufferedReceiver.get();+    state->receivers.insert(+        std::make_pair(unbufferedReceiver.release(), &it->first));+    state->sender->senderPush(MergeChannelEvent<KeyType, ValueType>{+        key, MergeChannelReceiverAdded{}});+    processAllAvailableValues(state, receiverPtr, std::move(buffer));+  }++  /**+   * Removes the receiver with the given key.+   */+  void removeReceiver(KeyType key) {+    auto state = state_.wlock();+    if (state->getSenderState() != ChannelState::Active) {+      return;+    }+    auto receiverIt = state->receiversByKey.find(key);+    if (receiverIt == state->receiversByKey.end()) {+      return;+    }+    CHECK(state->receivers.contains(receiverIt->second));+    if (!receiverIt->second->isReceiverCancelled()) {+      receiverIt->second->receiverCancel();+    }+    auto keyToRemove = receiverIt->first;+    state->receivers[receiverIt->second] = nullptr;+    state->receiversByKey.erase(receiverIt);+    state->sender->senderPush(MergeChannelEvent<KeyType, ValueType>{+        keyToRemove, MergeChannelReceiverRemoved{}});+  }++  folly::F14FastSet<KeyType> getReceiverKeys() {+    auto state = state_.rlock();+    auto receiverKeys = folly::F14FastSet<KeyType>();+    receiverKeys.reserve(state->receiversByKey.size());+    for (const auto& [key, _] : state->receiversByKey) {+      receiverKeys.insert(key);+    }+    return receiverKeys;+  }++  /**+   * Called when the user's MergeChannel object is destroyed.+   */+  void destroyHandle(CloseResult closeResult) {+    auto state = state_.wlock();+    processHandleDestroyed(state, std::move(closeResult));+  }++  /**+   * Called when one of the channels we are listening to has an update (either+   * a value from an input receiver or a cancellation from the output receiver).+   */+  void consume(ChannelBridgeBase* bridge) override {+    executor_->add([=, this]() {+      auto state = state_.wlock();+      if (bridge == state->sender.get()) {+        // The consumer of the output receiver has stopped consuming.+        CHECK(state->getSenderState() != ChannelState::CancellationProcessed);+        state->sender->senderClose();+        processSenderCancelled(state);+      } else {+        // One or more values are now available from an input receiver.+        auto* receiver = static_cast<ChannelBridge<ValueType>*>(bridge);+        CHECK(+            getReceiverState(receiver) != ChannelState::CancellationProcessed);+        processAllAvailableValues(state, receiver);+      }+    });+  }++  /**+   * Called after we cancelled one of the channels we were listening to (either+   * the sender or an input receiver).+   */+  void canceled(ChannelBridgeBase* bridge) override {+    executor_->add([=, this]() {+      auto state = state_.wlock();+      if (bridge == state->sender.get()) {+        // We previously cancelled the sender due to an input receiver closure+        // with an exception (or the closure of all input receivers without an+        // exception). Process the cancellation for the sender.+        CHECK(state->getSenderState() == ChannelState::CancellationTriggered);+        processSenderCancelled(state);+      } else {+        // We previously cancelled this input receiver, either because the+        // consumer of the output receiver stopped consuming or because another+        // input receiver received an exception. Process the cancellation for+        // this input receiver.+        auto* receiver = static_cast<ChannelBridge<ValueType>*>(bridge);+        processReceiverCancelled(state, receiver, CloseResult());+      }+    });+  }++ protected:+  /**+   * Processes all available values from the given input receiver channel+   * (starting from the provided buffer, if present).+   *+   * If an value was received indicating that the input channel has been closed+   * (or if the transform function indicated that channel should be closed), we+   * will process cancellation for the input receiver.+   */+  void processAllAvailableValues(+      WLockedStatePtr& state,+      ChannelBridge<ValueType>* receiver,+      std::optional<ReceiverQueue<ValueType>> buffer = std::nullopt) {+    CHECK(state->receivers.contains(receiver));+    const auto* key = state->receivers.at(receiver);+    auto closeResult = receiver->isReceiverCancelled()+        ? CloseResult()+        : (buffer.has_value()+               ? processValues(state, std::move(buffer.value()), key)+               : std::nullopt);+    while (!closeResult.has_value()) {+      if (receiver->receiverWait(this)) {+        // There are no more values available right now. We will stop processing+        // until the channel fires the consume() callback (indicating that more+        // values are available).+        break;+      }+      auto values = receiver->receiverGetValues();+      CHECK(!values.empty());+      closeResult = processValues(state, std::move(values), key);+    }+    if (closeResult.has_value()) {+      // The receiver received a value indicating channel closure.+      receiver->receiverCancel();+      processReceiverCancelled(state, receiver, std::move(closeResult.value()));+    }+  }++  /**+   * Processes the given set of values for an input receiver. Returns a+   * CloseResult if the given channel was closed, so the caller can stop+   * attempting to process values from it.+   */+  std::optional<CloseResult> processValues(+      WLockedStatePtr& state,+      ReceiverQueue<ValueType> values,+      const KeyType* key) {+    while (!values.empty()) {+      auto inputResult = std::move(values.front());+      values.pop();+      if (inputResult.hasValue()) {+        // We have received a normal value from an input receiver. Write it to+        // the output receiver.+        state->sender->senderPush(MergeChannelEvent<KeyType, ValueType>{+            *key, std::move(inputResult.value())});+      } else {+        // The input receiver was closed.+        return inputResult.hasException()+            ? CloseResult(std::move(inputResult.exception()))+            : CloseResult();+      }+    }+    return std::nullopt;+  }++  /**+   * Processes the cancellation of an input receiver.+   */+  void processReceiverCancelled(+      WLockedStatePtr& state,+      ChannelBridge<ValueType>* receiver,+      CloseResult closeResult) {+    CHECK(getReceiverState(receiver) == ChannelState::CancellationTriggered);+    auto* key = state->receivers.at(receiver);+    if (key != nullptr) {+      auto keyToRemove = *key;+      CHECK_EQ(state->receiversByKey.erase(keyToRemove), 1);+      if (state->getSenderState() == ChannelState::Active) {+        state->sender->senderPush(MergeChannelEvent<KeyType, ValueType>{+            keyToRemove,+            MergeChannelReceiverClosed{+                closeResult.exception.has_value()+                    ? std::move(closeResult.exception.value())+                    : exception_wrapper()}});+      }+    }+    state->receivers.erase(receiver);+    (ChannelBridgePtr<ValueType>(receiver));+    maybeDelete(state);+  }++  /**+   * Processes the cancellation of the sender (indicating that the consumer of+   * the output receiver has stopped consuming). We will trigger cancellation+   * for all input receivers not already cancelled.+   */+  void processSenderCancelled(WLockedStatePtr& state) {+    CHECK(state->getSenderState() == ChannelState::CancellationTriggered);+    state->sender = nullptr;+    for (auto [receiver, _] : state->receivers) {+      if (getReceiverState(receiver) == ChannelState::Active) {+        receiver->receiverCancel();+      }+    }+    maybeDelete(state);+  }++  /**+   * Processes the destruction of the user's MergeChannel object.  We will+   * close the sender and trigger cancellation for all input receivers not+   * already cancelled.+   */+  void processHandleDestroyed(WLockedStatePtr& state, CloseResult closeResult) {+    CHECK(!state->handleDestroyed);+    state->handleDestroyed = true;+    if (state->getSenderState() == ChannelState::Active) {+      for (auto [key, receiver] : state->receiversByKey) {+        state->receivers[receiver] = nullptr;+        state->sender->senderPush(MergeChannelEvent<KeyType, ValueType>{+            key, MergeChannelReceiverRemoved{}});+      }+      if (closeResult.exception.has_value()) {+        state->sender->senderClose(std::move(closeResult.exception.value()));+      } else {+        state->sender->senderClose();+      }+    }+    for (auto [receiver, _] : state->receivers) {+      if (getReceiverState(receiver) == ChannelState::Active) {+        receiver->receiverCancel();+      }+    }+    maybeDelete(state);+  }++  /**+   * Deletes this object if we have already processed cancellation for the+   * sender and all input receivers, and if the user's MergeChannel object was+   * destroyed.+   */+  void maybeDelete(WLockedStatePtr& state) {+    if (state->getSenderState() == ChannelState::CancellationProcessed &&+        state->receivers.empty() && state->handleDestroyed) {+      state.unlock();+      delete this;+    }+  }++  ChannelState getReceiverState(ChannelBridge<ValueType>* receiver) {+    return detail::getReceiverState(receiver);+  }++  folly::Executor::KeepAlive<folly::SequencedExecutor> executor_;+  folly::Synchronized<State> state_;+};+} // namespace detail++template <typename KeyType, typename ValueType>+std::pair<+    Receiver<MergeChannelEvent<KeyType, ValueType>>,+    MergeChannel<KeyType, ValueType>>+createMergeChannel(+    folly::Executor::KeepAlive<folly::SequencedExecutor> executor) {+  auto [receiver, sender] =+      Channel<MergeChannelEvent<KeyType, ValueType>>::create();+  auto* processor = new detail::MergeChannelProcessor<KeyType, ValueType>(+      std::move(sender), std::move(executor));+  return std::make_pair(+      std::move(receiver), MergeChannel<KeyType, ValueType>(processor));+}+} // namespace channels+} // namespace folly
+ folly/folly/channels/MergeChannel.h view
@@ -0,0 +1,131 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <folly/channels/Channel.h>+#include <folly/container/F14Set.h>+#include <folly/executors/SequencedExecutor.h>++namespace folly {+namespace channels {++namespace detail {+template <typename KeyType, typename ValueType>+class IMergeChannelProcessor;+}++struct MergeChannelReceiverAdded {};+struct MergeChannelReceiverRemoved {};+struct MergeChannelReceiverClosed {+  exception_wrapper exception;+};++template <typename KeyType, typename ValueType>+struct MergeChannelEvent {+  using EventType = std::variant<+      ValueType,+      MergeChannelReceiverAdded,+      MergeChannelReceiverRemoved,+      MergeChannelReceiverClosed>;++  KeyType key;+  EventType event;+};++/**+ * A merge channel allows one to merge multiple receivers into a single+ * output receiver. The set of receivers being merged can be changed at+ * runtime. Each receiver is added with a key that can be used to remove+ * the receiver at a later point.+ *+ * Example:+ *+ *  // Example function that returns a receiver for a given entity:+ *  Receiver<int> subscribe(std::string entity);+ *+ *  // Example function that returns an executor+ *  folly::Executor::KeepAlive<folly::SequencedExecutor> getExecutor();+ *+ *  auto [outputReceiver, mergeChannel]+ *      = createMergeChannel<std::string, int>(getExecutor());+ *  mergeChannel.addNewReceiver("abc", subscribe("abc"));+ *  mergeChannel.addNewReceiver("def", subscribe("def"));+ *  mergeChannel.removeReceiver("abc");+ *  std::move(mergeChannel).close();+ */+template <typename KeyType, typename ValueType>+class MergeChannel {+  using TProcessor = detail::IMergeChannelProcessor<KeyType, ValueType>;++ public:+  explicit MergeChannel(+      detail::IMergeChannelProcessor<KeyType, ValueType>* processor);+  MergeChannel(MergeChannel&& other) noexcept;+  MergeChannel& operator=(MergeChannel&& other) noexcept;+  ~MergeChannel();++  /**+   * Returns whether this MergeChannel is a valid object.+   */+  explicit operator bool() const;++  /**+   * Adds a new receiver to be merged, along with a given key. If the key+   * matches the key of an existing receiver, that existing receiver is replaced+   * with the new one (and updates from the old receiver will no longer be+   * merged). An added receiver can later be removed by passing the same key to+   * removeReceiver.+   */+  template <typename TReceiver>+  void addNewReceiver(KeyType key, TReceiver receiver);++  /**+   * Removes the receiver added with the given key. The receiver will be+   * asynchronously removed, so the consumer may still receive some values from+   * this receiver after this call.+   */+  void removeReceiver(KeyType key);++  /**+   * Returns a set of keys for receivers that are merged into this MergeChannel.+   */+  folly::F14FastSet<KeyType> getReceiverKeys();++  /**+   * Closes the merge channel.+   */+  void close(std::optional<exception_wrapper> ex = std::nullopt) &&;++ private:+  TProcessor* processor_;+};++/**+ * Creates a new merge channel.+ *+ * @param executor: The SequencedExecutor to use for merging values.+ */+template <typename KeyType, typename ValueType>+std::pair<+    Receiver<MergeChannelEvent<KeyType, ValueType>>,+    MergeChannel<KeyType, ValueType>>+createMergeChannel(+    folly::Executor::KeepAlive<folly::SequencedExecutor> executor);+} // namespace channels+} // namespace folly++#include <folly/channels/MergeChannel-inl.h>
+ folly/folly/channels/MultiplexChannel-inl.h view
@@ -0,0 +1,513 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <folly/channels/MultiplexChannel.h>+#include <folly/channels/RateLimiter.h>+#include <folly/coro/FutureUtil.h>+#include <folly/coro/Mutex.h>+#include <folly/coro/Promise.h>+#include <folly/experimental/channels/detail/Utility.h>++namespace folly {+namespace channels {++template <typename MultiplexerType>+MultiplexedSubscriptions<MultiplexerType>::MultiplexedSubscriptions(+    SubscriptionMap& subscriptions)+    : subscriptions_(subscriptions) {}++template <typename MultiplexerType>+bool MultiplexedSubscriptions<MultiplexerType>::hasSubscription(+    const MultiplexedSubscriptions::KeyType& key) {+  return subscriptions_.contains(key) && !closedSubscriptionKeys_.contains(key);+}++template <typename MultiplexerType>+typename MultiplexedSubscriptions<MultiplexerType>::KeyContextType&+MultiplexedSubscriptions<MultiplexerType>::getKeyContext(+    const MultiplexedSubscriptions::KeyType& key) {+  ensureKeyExists(key);+  return std::get<KeyContextType>(subscriptions_.at(key));+}++template <typename MultiplexerType>+template <typename U>+void MultiplexedSubscriptions<MultiplexerType>::write(+    const MultiplexedSubscriptions::KeyType& key, U&& value) {+  ensureKeyExists(key);+  auto& sender =+      std::get<FanoutSender<OutputValueType>>(subscriptions_.at(key));+  sender.write(std::forward<U>(value));+}++template <typename MultiplexerType>+void MultiplexedSubscriptions<MultiplexerType>::close(+    const MultiplexedSubscriptions::KeyType& key, exception_wrapper ex) {+  ensureKeyExists(key);+  auto& sender =+      std::get<FanoutSender<OutputValueType>>(subscriptions_.at(key));+  if (ex) {+    std::move(sender).close(std::move(ex));+  } else {+    std::move(sender).close();+  }+  // We do not erase from the subscriptions_ map yet, because we do not want+  // to invalidate the view returned by getSubscriptionKeys.+  closedSubscriptionKeys_.insert(key);+}++template <typename MultiplexerType>+void MultiplexedSubscriptions<MultiplexerType>::ensureKeyExists(+    const KeyType& key) {+  if (!subscriptions_.contains(key) || closedSubscriptionKeys_.contains(key)) {+    throw std::runtime_error("Subscription with the given key does not exist.");+  }+}++template <typename MultiplexerType>+MultiplexChannel<MultiplexerType>::MultiplexChannel(TProcessor* processor)+    : processor_(processor) {}++template <typename MultiplexerType>+MultiplexChannel<MultiplexerType>::MultiplexChannel(+    MultiplexChannel&& other) noexcept+    : processor_(std::exchange(other.processor_, nullptr)) {}++template <typename MultiplexerType>+MultiplexChannel<MultiplexerType>& MultiplexChannel<MultiplexerType>::operator=(+    MultiplexChannel&& other) noexcept {+  if (&other == this) {+    return *this;+  }+  if (processor_) {+    std::move(*this).close();+  }+  processor_ = std::exchange(other.processor_, nullptr);+  return *this;+}++template <typename MultiplexerType>+MultiplexChannel<MultiplexerType>::~MultiplexChannel() {+  if (processor_ != nullptr) {+    std::move(*this).close(exception_wrapper());+  }+}++template <typename MultiplexerType>+MultiplexChannel<MultiplexerType>::operator bool() const {+  return processor_;+}++template <typename MultiplexerType>+Receiver<typename MultiplexChannel<MultiplexerType>::OutputValueType>+MultiplexChannel<MultiplexerType>::subscribe(+    KeyType key, SubscriptionArgType subscriptionArg) {+  return processor_->subscribe(std::move(key), std::move(subscriptionArg));+}++template <typename MultiplexerType>+folly::coro::Task<std::vector<std::pair<+    typename MultiplexChannel<MultiplexerType>::KeyType,+    typename MultiplexChannel<MultiplexerType>::KeyContextType>>>+MultiplexChannel<MultiplexerType>::clearUnusedSubscriptions() {+  co_return co_await processor_->clearUnusedSubscriptions();+}++template <typename MultiplexerType>+bool MultiplexChannel<MultiplexerType>::anySubscribers() const {+  return processor_->anySubscribers();+}++template <typename MultiplexerType>+void MultiplexChannel<MultiplexerType>::close(exception_wrapper ex) && {+  processor_->destroyHandle(+      ex ? detail::CloseResult(std::move(ex)) : detail::CloseResult());+  processor_ = nullptr;+}++namespace detail {++/**+ * This object fans out values from the input receiver to all output receivers.+ * The lifetime of this object is described by the following state machine.+ *+ * The input receiver can be in one of three conceptual states: Active,+ * CancellationTriggered, or CancellationProcessed (removed). When the input+ * receiver reaches the CancellationProcessed state AND the user's+ * MultiplexChannel object is deleted, this object is deleted.+ *+ * When an input receiver receives a value indicating that the channel has+ * been closed, the state of the input receiver transitions from Active directly+ * to CancellationProcessed (and this object will be deleted once the user+ * destroys their MultiplexChannel object).+ *+ * When the user destroys their MultiplexChannel object, the state of the input+ * receiver transitions from Active to CancellationTriggered. This object will+ * then be deleted once the input receiver transitions to the+ * CancellationProcessed state.+ */+template <typename MultiplexerType>+class MultiplexChannelProcessor : public IChannelCallback {+ private:+  using MultiplexerTypeTraits = detail::MultiplexerTraits<MultiplexerType>;+  using KeyType = typename MultiplexerTypeTraits::KeyType;+  using KeyContextType = typename MultiplexerTypeTraits::KeyContextType;+  using SubscriptionArgType =+      typename MultiplexerTypeTraits::SubscriptionArgType;+  using InputValueType = typename MultiplexerTypeTraits::InputValueType;+  using OutputValueType = typename MultiplexerTypeTraits::OutputValueType;++ public:+  explicit MultiplexChannelProcessor(MultiplexerType multiplexer)+      : multiplexer_(std::move(multiplexer)),+        totalSubscriptions_(0),+        pendingAsyncCalls_(0) {}++  /**+   * Starts multiplexing values from the input receiver to to one or more keyed+   * subscriptions.+   */+  void start(Receiver<InputValueType> inputReceiver) {+    executeWithMutexWhenReady(+        [this, inputReceiver = std::move(inputReceiver)]() mutable+        -> folly::coro::Task<void> {+          co_await processStart(std::move(inputReceiver));+        });+  }++  Receiver<OutputValueType> subscribe(+      KeyType key, SubscriptionArgType subscriptionArg) {+    auto [receiver, sender] = Channel<OutputValueType>::create();+    totalSubscriptions_.fetch_add(1);+    executeWithMutexWhenReady(+        [this,+         key = std::move(key),+         subscriptionArg = std::move(subscriptionArg),+         sender_2 = std::move(sender)]() mutable -> folly::coro::Task<void> {+          co_await processNewSubscription(+              std::move(key), std::move(subscriptionArg), std::move(sender_2));+        });+    return std::move(receiver);+  }++  folly::coro::Task<std::vector<std::pair<KeyType, KeyContextType>>>+  clearUnusedSubscriptions() {+    auto [promise, future] = folly::coro::makePromiseContract<+        std::vector<std::pair<KeyType, KeyContextType>>>();+    executeWithMutexWhenReady(+        [this,+         promise_2 = std::move(promise)]() mutable -> folly::coro::Task<void> {+          co_await processClearUnusedSubscriptions(std::move(promise_2));+        });+    return folly::coro::toTask(std::move(future));+  }++  bool anySubscribers() { return totalSubscriptions_.load() > 0; }++  /**+   * This is called when the user's MultiplexChannel object has been destroyed.+   */+  void destroyHandle(CloseResult closeResult) {+    executeWithMutexWhenReady(+        [this, closeResult = std::move(closeResult)]() mutable+        -> folly::coro::Task<void> {+          co_await processHandleDestroyed(std::move(closeResult));+        });+  }++ private:+  /**+   * Called when the input receiver has an update.+   */+  void consume(ChannelBridgeBase*) override {+    executeWithMutexWhenReady([this]() -> folly::coro::Task<void> {+      co_await processAllAvailableValues();+    });+  }++  /**+   * Called after we cancelled this input receiver, due to the destruction of+   * the handle.+   */+  void canceled(ChannelBridgeBase*) override {+    executeWithMutexWhenReady([this]() -> folly::coro::Task<void> {+      auto closeResult = CloseResult(); // Declaring first due to GCC bug+      co_await processReceiverCancelled(std::move(closeResult));+    });+  }++  folly::coro::Task<void> processStart(Receiver<InputValueType> inputReceiver) {+    auto [unbufferedInputReceiver, buffer] =+        detail::receiverUnbuffer(std::move(inputReceiver));+    receiver_ = std::move(unbufferedInputReceiver);++    // Start processing new values that come in from the input receiver.+    co_await processAllAvailableValues(std::move(buffer));+  }++  /**+   * Processes all available values from the input receiver (starting from the+   * provided buffer, if present).+   *+   * If an value was received indicating that the input channel has been closed+   * (or if the transform function indicated that channel should be closed), we+   * will process cancellation for the input receiver.+   */+  folly::coro::Task<void> processAllAvailableValues(+      std::optional<ReceiverQueue<InputValueType>> buffer = std::nullopt) {+    CHECK_NE(getReceiverState(), ChannelState::CancellationProcessed);+    auto closeResult = receiver_->isReceiverCancelled()+        ? CloseResult()+        : (buffer.has_value()+               ? co_await processValues(std::move(buffer.value()))+               : std::nullopt);+    while (!closeResult.has_value()) {+      if (receiver_->receiverWait(this)) {+        // There are no more values available right now. We will stop processing+        // until the channel fires the consume() callback (indicating that more+        // values are available).+        break;+      }+      auto values = receiver_->receiverGetValues();+      CHECK(!values.empty());+      closeResult = co_await processValues(std::move(values));+    }+    if (closeResult.has_value()) {+      // The receiver received a value indicating channel closure.+      receiver_->receiverCancel();+      co_await processReceiverCancelled(std::move(closeResult.value()));+    }+  }++  /**+   * Processes the given set of values for the input receiver. Returns a+   * CloseResult if channel was closed, so the caller can stop attempting to+   * process values from it.+   */+  folly::coro::Task<std::optional<CloseResult>> processValues(+      ReceiverQueue<InputValueType> values) {+    while (!values.empty()) {+      auto inputResult = std::move(values.front());+      values.pop();+      bool inputClosed = !inputResult.hasValue();+      auto subscriptions =+          MultiplexedSubscriptions<MultiplexerType>(subscriptions_);+      if (inputClosed && !inputResult.hasException()) {+        // The input channel was closed. We will send an OnClosedException to+        // onInputValue.+        inputResult = Try<InputValueType>(+            folly::make_exception_wrapper<OnClosedException>());+      }++      // Process the input value by calling onInputValue on the user's+      // multiplexer.+      auto onInputValueResult = co_await folly::coro::co_awaitTry(+          multiplexer_.onInputValue(std::move(inputResult), subscriptions));++      // If the user closed any subscriptions, erase them from the subscriptions+      // map.+      for (const auto& key : subscriptions.closedSubscriptionKeys_) {+        subscriptions_.erase(key);+      }+      if (!subscriptions.closedSubscriptionKeys_.empty()) {+        totalSubscriptions_.fetch_sub(+            subscriptions.closedSubscriptionKeys_.size());+        subscriptions.closedSubscriptionKeys_.clear();+      }++      if (inputClosed && onInputValueResult.hasValue()) {+        // The input channel was closed, but the onInputValue function did not+        // throw. We need to close all output receivers.+        onInputValueResult =+            Try<void>(folly::make_exception_wrapper<OnClosedException>());+      }+      if (!onInputValueResult.hasValue()) {+        co_return onInputValueResult.template hasException<OnClosedException>()+            ? CloseResult()+            : CloseResult(std::move(onInputValueResult.exception()));+      }+    }+    co_return std::nullopt;+  }++  /**+   * Processes the cancellation of the input receiver. We will close all+   * senders with the exception received from the input receiver (if any).+   */+  folly::coro::Task<void> processReceiverCancelled(CloseResult closeResult) {+    CHECK_EQ(getReceiverState(), ChannelState::CancellationTriggered);+    receiver_ = nullptr;+    closeAllSubscriptions(std::move(closeResult));+    co_return;+  }++  folly::coro::Task<void> processNewSubscription(+      KeyType key,+      SubscriptionArgType subscriptionArg,+      Sender<OutputValueType> newSender) {+    if (subscriptions_.contains(key)) {+      // We already had a subscription for this key.+      totalSubscriptions_.fetch_sub(1);+    }+    auto& [sender, context] = subscriptions_[key];+    auto initialValues =+        co_await folly::coro::co_awaitTry(multiplexer_.onNewSubscription(+            key, context, std::move(subscriptionArg)));+    if (initialValues.hasException()) {+      std::move(newSender).close(initialValues.exception());+      co_return;+    }+    for (auto& initialValue : initialValues.value()) {+      newSender.write(std::move(initialValue));+    }+    sender.subscribe(std::move(newSender));+  }++  folly::coro::Task<void> processClearUnusedSubscriptions(+      folly::coro::Promise<std::vector<std::pair<KeyType, KeyContextType>>>+          promise) {+    auto clearedSubscriptions =+        std::vector<std::pair<KeyType, KeyContextType>>();+    size_t subscriptionsToRemove = 0;+    for (auto it = subscriptions_.begin(); it != subscriptions_.end();) {+      auto& sender = std::get<FanoutSender<OutputValueType>>(it->second);+      if (!sender.anySubscribers()) {+        clearedSubscriptions.push_back(std::make_pair(+            it->first, std::move(std::get<KeyContextType>(it->second))));+        it = subscriptions_.erase(it);+        subscriptionsToRemove++;+      } else {+        ++it;+      }+    }++    totalSubscriptions_.fetch_sub(subscriptionsToRemove);+    promise.setValue(std::move(clearedSubscriptions));+    co_return;+  }++  /**+   * Processes the destruction of the user's MultiplexChannel object.  We will+   * cancel the receiver and trigger cancellation for all senders not already+   * cancelled.+   */+  folly::coro::Task<void> processHandleDestroyed(CloseResult closeResult) {+    handleDeleted_ = true;+    if (getReceiverState() == ChannelState::Active) {+      receiver_->receiverCancel();+    }+    closeAllSubscriptions(std::move(closeResult));+    co_return;+  }++  /**+   * Deletes this object if we have already processed cancellation for the+   * receiver and all senders, and if the user's MultiplexChannel object was+   * destroyed.+   */+  void maybeDelete(std::unique_lock<folly::coro::Mutex>& lock) {+    if (getReceiverState() == ChannelState::CancellationProcessed &&+        handleDeleted_ && pendingAsyncCalls_ == 0) {+      lock.unlock();+      delete this;+    }+  }++  void executeWithMutexWhenReady(+      folly::Function<folly::coro::Task<void>()> func) {+    pendingAsyncCalls_++;+    auto rateLimiter = multiplexer_.getRateLimiter();+    if (rateLimiter != nullptr) {+      rateLimiter->executeWhenReady(+          [this, func = std::move(func), executor = multiplexer_.getExecutor()](+              std::unique_ptr<RateLimiter::Token> token) mutable {+            co_withExecutor(+                executor,+                folly::coro::co_invoke(+                    [this,+                     token = std::move(token),+                     func =+                         std::move(func)]() mutable -> folly::coro::Task<void> {+                      auto lock = co_await mutex_.co_scoped_lock();+                      co_await func();+                      pendingAsyncCalls_--;+                      maybeDelete(lock);+                    }))+                .start();+          },+          multiplexer_.getExecutor());+    } else {+      co_withExecutor(+          multiplexer_.getExecutor(),+          folly::coro::co_invoke(+              [this,+               func = std::move(func)]() mutable -> folly::coro::Task<void> {+                auto lock = co_await mutex_.co_scoped_lock();+                co_await func();+                pendingAsyncCalls_--;+                maybeDelete(lock);+              }))+          .start();+    }+  }++  ChannelState getReceiverState() {+    return detail::getReceiverState(receiver_.get());+  }++  void closeAllSubscriptions(CloseResult closeResult) {+    for (auto& [key, subscription] : subscriptions_) {+      auto& sender = std::get<FanoutSender<OutputValueType>>(subscription);+      std::move(sender).close(+          closeResult.exception.has_value()+              ? closeResult.exception.value()+              : exception_wrapper());+    }+    totalSubscriptions_.fetch_sub(subscriptions_.size());+    subscriptions_.clear();+  }++  using SubscriptionMap = folly::F14FastMap<+      KeyType,+      std::tuple<FanoutSender<OutputValueType>, KeyContextType>>;++  coro::Mutex mutex_;++  // The above coro mutex must be acquired before accessing this state.+  ChannelBridgePtr<InputValueType> receiver_;+  SubscriptionMap subscriptions_;+  bool handleDeleted_{false};++  // The above coro mutex does not need to be acquired before accessing this+  // state.+  MultiplexerType multiplexer_;+  std::atomic<uint64_t> totalSubscriptions_; // Includes pending subscriptions+  std::atomic<uint64_t> pendingAsyncCalls_;+};+} // namespace detail++template <typename MultiplexerType, typename InputReceiverType>+MultiplexChannel<MultiplexerType> createMultiplexChannel(+    MultiplexerType multiplexer, InputReceiverType inputReceiver) {+  auto* processor = new detail::MultiplexChannelProcessor<MultiplexerType>(+      std::move(multiplexer));+  processor->start(std::move(inputReceiver));+  return MultiplexChannel<MultiplexerType>(processor);+}+} // namespace channels+} // namespace folly
+ folly/folly/channels/MultiplexChannel.h view
@@ -0,0 +1,244 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <range/v3/view/map.hpp>+#include <folly/channels/Channel.h>+#include <folly/channels/FanoutSender.h>+#include <folly/channels/OnClosedException.h>+#include <folly/container/F14Map.h>+#include <folly/coro/Task.h>+#include <folly/executors/SequencedExecutor.h>+#include <folly/experimental/channels/detail/MultiplexerTraits.h>++namespace folly {+namespace channels {++template <typename MultiplexerType>+class MultiplexChannel;++/**+ * Creates a new multiplex channel that multiplexes updates from a single input+ * receiver to one or more keyed subscriptions.+ *+ * The creator of a multiplex channel must pass a Multiplexer class that+ * implements the following functions:+ *+ *   folly::Executor::KeepAlive<folly::SequencedExecutor> getExecutor();+ *+ *   std::shared_ptr<RateLimiter> getRateLimiter(); // Can return nullptr+ *+ *   // This function is called for each call to subscribe on the multiplex+ *   // channel object. It returns a vector of output values that should be sent+ *   // directly to the new output receiver for that subscription.+ *   folly::coro::Task<std::vector<OutputValueType>> onNewSubscription(+ *       KeyType key,+ *       KeyContextType& keyContext,+ *       SubscriptionArgType subscriptionArg);+ *+ *   // This function is called with each value fromm the given input receiver.+ *   // This function sends any corresponding values to the relevant output+ *   // receivers, using the subscriptions parameter.+ *   folly::coro::Task<void> onInputValue(+ *       Try<InputValueType> inputValue,+ *       MultiplexedSubscriptions<MultiplexerType>& subscriptions);+ *+ * Example:+ *+ *   struct InputValue {+ *     std::string key;+ *     int64_t value;+ *   };+ *+ *   struct NoContext {}+ *   struct NoSubscriptionArg {};+ *+ *   class Multiplexer {+ *    public:+ *     explicit Multiplexer(+ *         folly::Executor::KeepAlive<folly::SequencedExecutor> executor)+ *         : executor_(std::move(executor)) {}+ *+ *    folly::Executor::KeepAlive<folly::SequencedExecutor> getExecutor() {+ *      return executor_;+ *    }+ *+ *    std::shared_ptr<RateLimiter> getRateLimiter() {+ *      return nullptr; // No rate limiting+ *    }+ *+ *    folly::coro::Task<std::vector<OutputValueType>> onNewSubscription(+ *        std::string key,+ *        NoContext&,+ *        NoSubscriptionArg&) {+ *      co_return std::vector<int64_t>(); // No initial values+ *    }+ *+ *    folly::coro::Task<void> onInputValue(+ *        Try<InputValue> inputValue,+ *        MultiplexedSubscriptions<Multiplexer>& subscriptions) {+ *      if (subscriptions.hasSubscription(inputValue->key)) {+ *        subscriptions.write(inputValue->key, inputValue->value);+ *      }+ *      co_return;+ *    }+ *+ *    private:+ *     folly::Executor::KeepAlive<folly::SequencedExecutor> executor_;+ *   }+ *+ *   // Function that returns a receiver:+ *   Receiver<InputValue> getInputReceiver();+ *+ *   // Function that returns an executor+ *   folly::Executor::KeepAlive<folly::SequencedExecutor> getExecutor();+ *+ *   auto multiplexChannel = createMultiplexChannel(+ *       Multiplexer(getExecutor()),+ *       getInputReceiver());+ *+ *   auto receiver1a = multiplexChannel.subscribe("one");+ *   auto receiver1b = multiplexChannel.subscribe("one");+ *   auto receiver2a = multiplexChannel.subscribe("two");+ */+template <typename MultiplexerType, typename InputReceiverType>+MultiplexChannel<MultiplexerType> createMultiplexChannel(+    MultiplexerType multiplexer, InputReceiverType inputReceiver);++namespace detail {+template <typename MultiplexerType>+class MultiplexChannelProcessor;+} // namespace detail++/**+ * A multiplex channel allows multiplexing updates from a single input receiver+ * to one or more keyed subscriptions.+ */+template <typename MultiplexerType>+class MultiplexChannel {+  using TProcessor = detail::MultiplexChannelProcessor<MultiplexerType>;+  using MultiplexerTypeTraits = detail::MultiplexerTraits<MultiplexerType>;++  using KeyType = typename MultiplexerTypeTraits::KeyType;+  using KeyContextType = typename MultiplexerTypeTraits::KeyContextType;+  using SubscriptionArgType =+      typename MultiplexerTypeTraits::SubscriptionArgType;+  using OutputValueType = typename MultiplexerTypeTraits::OutputValueType;++ public:+  MultiplexChannel(MultiplexChannel&& other) noexcept;+  MultiplexChannel& operator=(MultiplexChannel&& other) noexcept;+  ~MultiplexChannel();++  /**+   * Returns whether this MultiplexChannel is a valid object. This will return+   * false if the object was moved from.+   */+  explicit operator bool() const;++  /**+   * Returns a new output receiver for the given key.+   */+  Receiver<OutputValueType> subscribe(+      KeyType key, SubscriptionArgType subscriptionArg);++  /**+   * Removes keys with no subscribers, and returns their contexts.+   */+  folly::coro::Task<std::vector<std::pair<KeyType, KeyContextType>>>+  clearUnusedSubscriptions();++  /**+   * Returns whether this multiplex channel has any subscribers for any keys.+   * Note that if any output receivers returned from subscribe have been+   * destroyed, such subscriptions will still be considered to exist until the+   * clearUnusedSubscriptions function is called.+   */+  bool anySubscribers() const;++  /**+   * Closes the multiplex channel.+   */+  void close(exception_wrapper ex = exception_wrapper()) &&;++ private:+  template <typename Multiplexer, typename InputValueType>+  friend MultiplexChannel<Multiplexer> createMultiplexChannel(+      Multiplexer, InputValueType);++  explicit MultiplexChannel(TProcessor* processor);++  TProcessor* processor_;+};++/**+ * A class that allows one to see which keys are subscribed, and to write+ * values for particular subscriptions. This is passed to the onInputValue+ * function of the user-provided Multiplexer class.+ */+template <typename MultiplexerType>+class MultiplexedSubscriptions {+ public:+  using MultiplexerTypeTraits = detail::MultiplexerTraits<MultiplexerType>;+  using KeyType = typename MultiplexerTypeTraits::KeyType;+  using KeyContextType = typename MultiplexerTypeTraits::KeyContextType;+  using OutputValueType = typename MultiplexerTypeTraits::OutputValueType;++  friend class detail::MultiplexChannelProcessor<MultiplexerType>;++  /**+   * Returns whether or not a subscription exists for the given key.+   */+  bool hasSubscription(const KeyType& key);++  /**+   * Returns a reference to the context object for the given key.+   */+  KeyContextType& getKeyContext(const KeyType& key);++  /**+   * Sends a value to all subscribers of a given key.+   */+  template <typename U = OutputValueType>+  void write(const KeyType& key, U&& value);++  /**+   * Closes all subscribers for the given key.+   */+  void close(const KeyType& key, exception_wrapper ex);++  /**+   * Returns a view containing a list of subscribed keys.+   */+  auto getAllSubscriptionKeys() { return subscriptions_ | ranges::views::keys; }++ private:+  using SubscriptionMap = folly::F14FastMap<+      KeyType,+      std::tuple<FanoutSender<OutputValueType>, KeyContextType>>&;++  explicit MultiplexedSubscriptions(SubscriptionMap& subscriptions);++  void ensureKeyExists(const KeyType& key);++  SubscriptionMap& subscriptions_;+  folly::F14FastSet<KeyType> closedSubscriptionKeys_;+};+} // namespace channels+} // namespace folly++#include <folly/channels/MultiplexChannel-inl.h>
+ folly/folly/channels/OnClosedException.h view
@@ -0,0 +1,35 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <exception>++namespace folly {+namespace channels {++/**+ * An OnClosedException passed to a transform or multiplex callback indicates+ * that the input channel was closed. An OnClosedException can also be thrown by+ * a transform or multiplex callback, which will close the output channel.+ */+struct OnClosedException : public std::exception {+  const char* what() const noexcept override {+    return "The channel has been closed.";+  }+};+} // namespace channels+} // namespace folly
+ folly/folly/channels/Producer-inl.h view
@@ -0,0 +1,143 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <fmt/format.h>+#include <folly/CancellationToken.h>+#include <folly/channels/Channel.h>+#include <folly/channels/ConsumeChannel.h>+#include <folly/channels/Producer.h>+#include <folly/coro/Task.h>++namespace folly {+namespace channels {++template <typename TValue>+Producer<TValue>::KeepAlive::KeepAlive(Producer<TValue>* ptr) : ptr_(ptr) {}++template <typename TValue>+Producer<TValue>::KeepAlive::~KeepAlive() {+  if (ptr_ && --ptr_->refCount_ == 0) {+    auto deleteTask =+        folly::coro::co_invoke([ptr = ptr_]() -> folly::coro::Task<void> {+          delete ptr;+          co_return;+        });+    co_withExecutor(ptr_->getExecutor(), std::move(deleteTask)).start();+  }+}++template <typename TValue>+Producer<TValue>::KeepAlive::KeepAlive(+    Producer<TValue>::KeepAlive&& other) noexcept+    : ptr_(std::exchange(other.ptr_, nullptr)) {}++template <typename TValue>+typename Producer<TValue>::KeepAlive& Producer<TValue>::KeepAlive::operator=(+    Producer<TValue>::KeepAlive&& other) noexcept {+  if (&other == this) {+    return *this;+  }+  ptr_ = std::exchange(other.ptr_, nullptr);+  return *this;+}++template <typename TValue>+Producer<TValue>::Producer(+    Sender<TValue> sender,+    folly::Executor::KeepAlive<folly::SequencedExecutor> executor)+    : sender_(std::move(detail::senderGetBridge(sender))),+      executor_(std::move(executor)) {+  CHECK(sender_->senderWait(this));+}++template <typename TValue>+void Producer<TValue>::write(TValue value) {+  executor_->add([this, value = std::move(value)]() mutable {+    sender_->senderPush(std::move(value));+  });+}++template <typename TValue>+void Producer<TValue>::close(std::optional<exception_wrapper> ex) {+  executor_->add([this, ex = std::move(ex)]() mutable {+    if (ex.has_value()) {+      sender_->senderClose(std::move(ex.value()));+    } else {+      sender_->senderClose();+    }+  });+}++template <typename TValue>+bool Producer<TValue>::isClosed() {+  return sender_->isSenderClosed();+}++template <typename TValue>+folly::Executor::KeepAlive<folly::SequencedExecutor>+Producer<TValue>::getExecutor() {+  return executor_;+}++template <typename TValue>+typename Producer<TValue>::KeepAlive Producer<TValue>::getKeepAlive() {+  refCount_.fetch_add(1, std::memory_order_relaxed);+  return KeepAlive(this);+}++template <typename TValue>+void Producer<TValue>::consume(detail::ChannelBridgeBase*) {+  co_withExecutor(getExecutor(), onClosed()).start([=, this](auto) {+    // Decrement ref count+    KeepAlive(this);+  });+}++template <typename TValue>+void Producer<TValue>::canceled(detail::ChannelBridgeBase* bridge) {+  consume(bridge);+}++namespace detail {+template <typename TProducer>+class ProducerImpl : public TProducer {+  template <typename ProducerType, typename... Args>+  friend Receiver<typename ProducerType::ValueType> makeProducer(+      folly::Executor::KeepAlive<folly::SequencedExecutor> executor,+      Args&&... args);++ public:+  using TProducer::TProducer;++ private:+  void ensureMakeProducerUsedForCreation() override {}+};+} // namespace detail++template <typename TProducer, typename... Args>+Receiver<typename TProducer::ValueType> makeProducer(+    folly::Executor::KeepAlive<folly::SequencedExecutor> executor,+    Args&&... args) {+  using TValue = typename TProducer::ValueType;+  auto [receiver, sender] = Channel<TValue>::create();+  new detail::ProducerImpl<TProducer>(+      std::move(sender), std::move(executor), std::forward<Args>(args)...);+  return std::move(receiver);+}+} // namespace channels+} // namespace folly
+ folly/folly/channels/Producer.h view
@@ -0,0 +1,181 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <folly/channels/ChannelCallbackHandle.h>+#include <folly/coro/Task.h>+#include <folly/executors/SequencedExecutor.h>++namespace folly {+namespace channels {++/**+ * A Producer is a base class for an object that produces a channel. The+ * subclass can call write to write a new value to the channel, and close to+ * close the channel. It is a useful way to generate output values for a+ * receiver, without having to keep alive an extraneous object that produces+ * those values.+ *+ * When the consumer of the channel stops consuming, the onClosed function will+ * be called. The subclass should cancel any ongoing work in this function.+ * After onCancelled is called, the object will be deleted once the last+ * outstanding KeepAlive is destroyed.+ *+ * Example:+ *   // Function that returns an executor+ *   folly::Executor::KeepAlive<folly::SequencedExecutor> getExecutor();+ *+ *   // Function that returns output values+ *   std::vector<int> getLatestOutputValues();+ *+ *   // Example producer implementation+ *   class PollingProducer : public Producer<int> {+ *    public:+ *     PollingProducer(+ *         Sender<int> sender,+ *         folly::Executor::KeepAlive<folly::SequencedExecutor> executor)+ *         : Producer<int>(std::move(sender), std::move(executor)) {+ *       // Start polling for values.+ *       co_withExecutor(getExecutor(), folly::coro::co_withCancellation(+ *             cancelSource_.getToken(),+ *             [=, keepAlive = getKeepAlive()]() {+ *               return pollForOutputValues();+ *             })+ *         )+ *         .start();+ *     }+ *+ *     folly::coro::Task<void> onClosed() override {+ *       // The consumer has stopped consuming our values. Stop polling.+ *       cancelSource_.requestCancellation();+ *     }+ *+ *    private:+ *     folly::coro::Task<void> pollForOutputValues() {+ *       auto cancelToken = co_await folly::coro::co_current_cancellation_token;+ *       while (!cancelToken.isCancellationRequested()) {+ *         auto outputValues = getLatestOutputValues();+ *         for (auto& outputValue : outputValues) {+ *           write(std::move(outputValue));+ *         }+ *       }+ *       co_await folly::coro::sleep(std::chrono::seconds(1));+ *     }+ *+ *     folly::CancellationSource cancelSource_;+ *   };+ *+ *   // Producer usage+ *   Receiver<int> receiver = makeProducer<PollingProducer>(getExecutor());+ */+template <typename TValue>+class Producer : public detail::IChannelCallback {+ public:+  using ValueType = TValue;++ protected:+  /**+   * This object will ensure that the corresponding Producer that created it+   * will not be destroyed.+   */+  class KeepAlive {+   public:+    ~KeepAlive();+    KeepAlive(KeepAlive&&) noexcept;+    KeepAlive& operator=(KeepAlive&&) noexcept;++   private:+    friend class Producer<TValue>;++    explicit KeepAlive(Producer<TValue>* ptr);++    Producer<TValue>* ptr_;+  };++  Producer(+      Sender<TValue> sender,+      folly::Executor::KeepAlive<folly::SequencedExecutor> executor);+  virtual ~Producer() override = default;++  /**+   * Writes a value into the channel.+   */+  void write(TValue value);++  /**+   * Closes the channel.+   */+  void close(std::optional<exception_wrapper> ex = std::nullopt);++  /**+   * Returns whether or not this producer is closed or cancelled.+   */+  bool isClosed();++  /**+   * Returns the executor used for this producer.+   */+  folly::Executor::KeepAlive<folly::SequencedExecutor> getExecutor();++  /**+   * Returns a KeepAlive object. This object will not be destroyed before all+   * KeepAlive objects are destroyed.+   */+  KeepAlive getKeepAlive();++  /**+   * Called when the corresponding receiver is cancelled, or the sender is+   * closed.+   */+  virtual folly::coro::Task<void> onClosed() { co_return; }++  /**+   * If you get an error that this function is not implemented, do not+   * implement it. Instead, create your object with makeProducer+   * below.+   */+  virtual void ensureMakeProducerUsedForCreation() = 0;++ private:+  template <typename TProducer, typename... Args>+  friend Receiver<typename TProducer::ValueType> makeProducer(+      folly::Executor::KeepAlive<folly::SequencedExecutor> executor,+      Args&&... args);++  void consume(detail::ChannelBridgeBase* bridge) override;++  void canceled(detail::ChannelBridgeBase* bridge) override;++  detail::ChannelBridgePtr<TValue> sender_;+  folly::Executor::KeepAlive<folly::SequencedExecutor> executor_;+  std::atomic<int> refCount_{1};+};++/**+ * Creates a new object that extends the Producer class, and returns a receiver.+ * The receiver will receive any values produced by the producer. See the+ * description of the Producer class for information on how to implement a+ * producer.+ */+template <typename TProducer, typename... Args>+Receiver<typename TProducer::ValueType> makeProducer(+    folly::Executor::KeepAlive<folly::SequencedExecutor> executor,+    Args&&... args);+} // namespace channels+} // namespace folly++#include <folly/channels/Producer-inl.h>
+ folly/folly/channels/ProxyChannel-inl.h view
@@ -0,0 +1,350 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <folly/channels/ProxyChannel.h>+#include <folly/experimental/channels/detail/Utility.h>++namespace folly {+namespace channels {++template <typename ValueType>+ProxyChannel<ValueType>::ProxyChannel(TProcessor* processor)+    : processor_(processor) {}++template <typename ValueType>+ProxyChannel<ValueType>::ProxyChannel(ProxyChannel&& other) noexcept+    : processor_(std::exchange(other.processor_, nullptr)) {}++template <typename ValueType>+ProxyChannel<ValueType>& ProxyChannel<ValueType>::operator=(+    ProxyChannel&& other) noexcept {+  if (&other == this) {+    return *this;+  }+  if (processor_) {+    std::move(*this).close();+  }+  processor_ = std::exchange(other.processor_, nullptr);+  return *this;+}++template <typename ValueType>+ProxyChannel<ValueType>::~ProxyChannel() {+  if (processor_) {+    std::move(*this).close();+  }+}++template <typename ValueType>+ProxyChannel<ValueType>::operator bool() const {+  return processor_;+}++template <typename ValueType>+void ProxyChannel<ValueType>::setInputReceiver(Receiver<ValueType> receiver) {+  processor_->setInputReceiver(std::move(receiver));+}++template <typename ValueType>+void ProxyChannel<ValueType>::removeInputReceiver() {+  processor_->removeInputReceiver();+}++template <typename ValueType>+void ProxyChannel<ValueType>::close(folly::exception_wrapper&& ex) && {+  processor_->destroyHandle(+      ex ? detail::CloseResult(std::move(ex)) : detail::CloseResult());+  processor_ = nullptr;+}++namespace detail {++/**+ * This object does the proxying of values from the input receiver to the output+ * receiver.+ */+template <typename ValueType>+class ProxyChannelProcessor : public IChannelCallback {+ private:+  struct State {+    explicit State(ChannelBridgePtr<ValueType> _sender)+        : sender(std::move(_sender)) {}++    ChannelState getSenderState() {+      return detail::getSenderState(sender.get());+    }++    // The output sender for the proxy channel.+    ChannelBridgePtr<ValueType> sender;++    // The current input receiver for the proxy channel.+    ChannelBridge<ValueType>* receiver{nullptr};++    // The refcount for this proxy channel. The handle (if not yet destroyed),+    // the sender (if not yet cancelled), the current input receiver (if any),+    // and any previous input receivers not yet joined (if any) will contribute+    // to this refcount. It starts at 2, since a new ProxyChannel always has+    // one handle, one output receiver, and no input receivers.+    size_t refCount{2};+  };++  using WLockedStatePtr = typename folly::Synchronized<State>::WLockedPtr;++ public:+  ProxyChannelProcessor(+      Sender<ValueType> sender,+      folly::Executor::KeepAlive<folly::SequencedExecutor> executor)+      : executor_(std::move(executor)),+        state_(State(std::move(detail::senderGetBridge(sender)))) {+    auto state = state_.wlock();+    CHECK(state->sender->senderWait(this));+  }++  /**+   * Sets a new input receiver (removing the old input receiver, if any).+   */+  void setInputReceiver(Receiver<ValueType> receiver) {+    auto state = state_.wlock();+    if (state->getSenderState() != ChannelState::Active) {+      return;+    }+    auto [unbufferedReceiver, buffer] =+        detail::receiverUnbuffer(std::move(receiver));+    cancelInputReceiverIfExists(state);+    auto receiverPtr = unbufferedReceiver.release();+    state->receiver = receiverPtr;+    state->refCount++;+    processAllAvailableValues(std::move(state), receiverPtr, std::move(buffer));+  }++  /**+   * Removes the current input receiver.+   */+  void removeInputReceiver() {+    auto state = state_.wlock();+    if (state->getSenderState() != ChannelState::Active) {+      return;+    }+    cancelInputReceiverIfExists(state);+  }++  /**+   * Called when the user's ProxyChannel object is destroyed.+   */+  void destroyHandle(CloseResult closeResult) {+    processHandleDestroyed(state_.wlock(), std::move(closeResult));+  }++  /**+   * Called when one of the channels we are listening to has an update (either+   * a value from an input receiver or a cancellation from the output receiver).+   */+  void consume(ChannelBridgeBase* bridge) override {+    executor_->add([=, this]() {+      auto state = state_.wlock();+      if (bridge == state->sender.get()) {+        // The consumer of the output receiver has stopped consuming.+        state->sender->senderClose();+        processSenderCancelled(std::move(state));+      } else {+        // One or more values are now available from an input receiver.+        auto* receiver = static_cast<ChannelBridge<ValueType>*>(bridge);+        processAllAvailableValues(std::move(state), receiver);+      }+    });+  }++  /**+   * Called after we cancelled one of the channels we were listening to (either+   * the sender or an input receiver).+   */+  void canceled(ChannelBridgeBase* bridge) override {+    executor_->add([=, this]() {+      auto state = state_.wlock();+      if (bridge == state->sender.get()) {+        // We previously cancelled the sender due to an input receiver closure.+        // Process the cancellation for the sender.+        CHECK(state->getSenderState() == ChannelState::CancellationTriggered);+        processSenderCancelled(std::move(state));+      } else {+        // We previously cancelled this input receiver. Process the cancellation+        // for this input receiver.+        auto* receiver = static_cast<ChannelBridge<ValueType>*>(bridge);+        processReceiverCancelled(std::move(state), receiver, CloseResult());+      }+    });+  }++ protected:+  /**+   * Processes all available values from the current input receiver channel+   * (starting from the provided buffer, if present).+   *+   * If an value was received indicating that the input channel has been closed+   * we will process cancellation for the input receiver.+   */+  void processAllAvailableValues(+      WLockedStatePtr state,+      ChannelBridge<ValueType>* receiver,+      std::optional<ReceiverQueue<ValueType>> buffer = std::nullopt) {+    CHECK_NOTNULL(receiver);+    if (!receiver->isReceiverCancelled()) {+      CHECK_EQ(receiver, state->receiver);+    }+    auto closeResult = receiver->isReceiverCancelled()+        ? CloseResult()+        : (buffer.has_value() ? processValues(state, std::move(buffer.value()))+                              : std::nullopt);+    while (!closeResult.has_value()) {+      if (receiver->receiverWait(this)) {+        // There are no more values available right now. We will stop processing+        // until the channel fires the consume() callback (indicating that more+        // values are available).+        break;+      }+      auto values = receiver->receiverGetValues();+      CHECK(!values.empty());+      closeResult = processValues(state, std::move(values));+    }+    if (closeResult.has_value()) {+      // The receiver received a value indicating channel closure.+      receiver->receiverCancel();+      processReceiverCancelled(+          std::move(state), receiver, std::move(closeResult.value()));+    }+  }++  /**+   * Processes the given set of values for an input receiver. Returns a+   * CloseResult if the given channel was closed, so the caller can stop+   * attempting to process values from it.+   */+  std::optional<CloseResult> processValues(+      WLockedStatePtr& state, ReceiverQueue<ValueType> values) {+    while (!values.empty()) {+      auto inputResult = std::move(values.front());+      values.pop();+      if (inputResult.hasValue()) {+        // We have received a normal value from an input receiver. Write it to+        // the output receiver.+        state->sender->senderPush(std::move(inputResult.value()));+      } else {+        // The input receiver was closed.+        return inputResult.hasException()+            ? CloseResult(std::move(inputResult.exception()))+            : CloseResult();+      }+    }+    return std::nullopt;+  }++  /**+   * Processes the cancellation of an input receiver.+   */+  void processReceiverCancelled(+      WLockedStatePtr state,+      ChannelBridge<ValueType>* receiver,+      CloseResult closeResult) {+    CHECK(receiver->isReceiverCancelled());+    if (receiver == state->receiver &&+        state->getSenderState() == ChannelState::Active) {+      if (closeResult.exception.has_value()) {+        state->sender->senderClose(std::move(closeResult.exception.value()));+      } else {+        state->sender->senderClose();+      }+    }+    if (state->receiver == receiver) {+      state->receiver = nullptr;+    }+    (ChannelBridgePtr<ValueType>(receiver)); // Delete the receiver+    state->refCount--;+    maybeDelete(std::move(state));+  }++  /**+   * Processes the cancellation of the sender (indicating that the consumer of+   * the output receiver has stopped consuming). We will trigger cancellation+   * for the input receiver if it is not already cancelled.+   */+  void processSenderCancelled(WLockedStatePtr state) {+    CHECK(state->getSenderState() == ChannelState::CancellationTriggered);+    state->sender.reset();+    state->refCount--;+    cancelInputReceiverIfExists(state);+    maybeDelete(std::move(state));+  }++  /**+   * Processes the destruction of the user's ProxyChannel object.  We will+   * close the sender and trigger cancellation for the input receiver (if any).+   */+  void processHandleDestroyed(WLockedStatePtr state, CloseResult closeResult) {+    if (state->getSenderState() == ChannelState::Active) {+      if (closeResult.exception.has_value()) {+        state->sender->senderClose(std::move(closeResult.exception.value()));+      } else {+        state->sender->senderClose();+      }+    }+    cancelInputReceiverIfExists(state);+    state->refCount--;+    maybeDelete(std::move(state));+  }++  /**+   * Cancels the current input receiver if it exists.+   */+  void cancelInputReceiverIfExists(WLockedStatePtr& state) {+    if (state->receiver != nullptr) {+      CHECK(!state->receiver->isReceiverCancelled());+      state->receiver->receiverCancel();+      state->receiver = nullptr;+    }+  }++  /**+   * Deletes this object if we have already processed cancellation for the+   * sender, the current input receiver, and all previous input receivers, and+   * if the user's ProxyChannel object was destroyed.+   */+  void maybeDelete(WLockedStatePtr state) {+    if (state->refCount == 0) {+      CHECK_EQ(state->sender.get(), static_cast<void*>(NULL));+      CHECK_EQ(state->receiver, static_cast<void*>(NULL));+      state.unlock();+      delete this;+    }+  }++  folly::Executor::KeepAlive<folly::SequencedExecutor> executor_;+  folly::Synchronized<State> state_;+};+} // namespace detail++template <typename ValueType>+std::pair<Receiver<ValueType>, ProxyChannel<ValueType>> createProxyChannel(+    folly::Executor::KeepAlive<folly::SequencedExecutor> executor) {+  auto [receiver, sender] = Channel<ValueType>::create();+  auto* processor = new detail::ProxyChannelProcessor<ValueType>(+      std::move(sender), std::move(executor));+  return std::make_pair(+      std::move(receiver), ProxyChannel<ValueType>(processor));+}+} // namespace channels+} // namespace folly
+ folly/folly/channels/ProxyChannel.h view
@@ -0,0 +1,101 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <folly/channels/Channel.h>+#include <folly/executors/SequencedExecutor.h>++namespace folly {+namespace channels {++namespace detail {+template <typename ValueType>+class ProxyChannelProcessor;+}++/**+ * A proxy allows one to create a channel whose input is proxied from the output+ * of another channel, which can change over time. This is more memory-efficient+ * than using a MergeChannel with one input receiver.+ *+ * Example:+ *+ *  // Example function that returns a receiver for a given entity:+ *  Receiver<int> subscribe(std::string entity);+ *+ *  // Example function that returns an executor+ *  folly::Executor::KeepAlive<folly::SequencedExecutor> getExecutor();+ *+ *  auto [outputReceiver, proxyChannel]+ *      = createProxyChannel<int>(getExecutor());+ *  proxyChannel.setInputReceiver(subscribe("abc"));+ *  proxyChannel.setInputReceiver(subscribe("def"));+ *  proxyChannel.removeInputReceiver();+ *  proxyChannel.setInputReceiver(subscribe("ghi"));+ *  std::move(proxyChannel).close();+ */+template <typename ValueType>+class ProxyChannel {+  using TProcessor = detail::ProxyChannelProcessor<ValueType>;++ public:+  explicit ProxyChannel(detail::ProxyChannelProcessor<ValueType>* processor);+  ProxyChannel(ProxyChannel&& other) noexcept;+  ProxyChannel& operator=(ProxyChannel&& other) noexcept;+  ~ProxyChannel();++  /**+   * Returns whether this ProxyChannel is a valid object.+   */+  explicit operator bool() const;++  /**+   * Sets a new input receiver. As soon as this function returns, values from+   * the old input receiver (if any) will no longer be sent to the output+   * receiver. Values from the new input receiver will start being sent to the+   * output receiver, unless a previous input receiver was closed.+   */+  void setInputReceiver(Receiver<ValueType> receiver);++  /**+   * Removes the current input receiver (if any). As soon as this function+   * returns, values from the old input receiver (if any) will no longer be sent+   * to the output receiver.+   */+  void removeInputReceiver();++  /**+   * Closes the proxy channel.+   */+  void close(folly::exception_wrapper&& ex = {}) &&;++ private:+  TProcessor* processor_;+};++/**+ * Creates a new proxy channel.+ *+ * @param executor: The SequencedExecutor to use for proxying values.+ */+template <typename ValueType>+std::pair<Receiver<ValueType>, ProxyChannel<ValueType>> createProxyChannel(+    folly::Executor::KeepAlive<folly::SequencedExecutor> executor);+} // namespace channels+} // namespace folly++#include <folly/channels/ProxyChannel-inl.h>
+ folly/folly/channels/RateLimiter.h view
@@ -0,0 +1,60 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <folly/Function.h>+#include <folly/executors/SequencedExecutor.h>++namespace folly {+namespace channels {++/**+ * A rate-limiter used by the channels framework to limit the number of+ * in-flight requests.+ *+ * A default implementation is provided in MaxConcurrentRateLimiter.h but users+ * can provide custom rate-limiters.+ */+class RateLimiter : public std::enable_shared_from_this<RateLimiter> {+ public:+  class Token;+  virtual ~RateLimiter() = default;++  /**+   * Executes the given function when there is capacity available in the+   * rate-limiter.+   *+   * The function is considered finished when the token is destroyed.+   */+  virtual void executeWhenReady(+      folly::Function<void(std::unique_ptr<Token>)> function,+      Executor::KeepAlive<SequencedExecutor> executor) = 0;+};++/**+ * A token on destruction signals termination of the user provided function. So+ * it's expected that a derived class override the destructor to provide the+ * desired functionality.  Or piggyback on destruction of the compiler generated+ * overridden destructor.+ */+class RateLimiter::Token {+ public:+  virtual ~Token() = default;+};++} // namespace channels+} // namespace folly
+ folly/folly/channels/Transform-inl.h view
@@ -0,0 +1,656 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <folly/channels/Channel.h>+#include <folly/coro/AsyncGenerator.h>+#include <folly/coro/Task.h>+#include <folly/executors/SequencedExecutor.h>+#include <folly/experimental/channels/detail/Utility.h>++namespace folly {+namespace channels {++namespace detail {++/**+ * This object transforms values from the input receiver to the output receiver.+ * It is not an object that the user is aware of or holds a pointer to. The+ * lifetime of this object is described by the following state machine.+ *+ * Both the sender and receiver can be in one of three states: Active,+ * CancellationTriggered, or CancellationProcessed. When both the sender and+ * receiver reach the CancellationProcessed state, this object is deleted.+ *+ * When the input receiver receives a value indicating that the channel has been+ * closed, the state of the receiver transitions from Active directly to+ * CancellationProcessed and the state of the sender transitions from Active to+ * CancellationTriggered. Once we receive a callback indicating the sender's+ * cancellation signal has been received, the sender's state is transitioned+ * from CancellationTriggered to CancellationProcessed (and the object is+ * deleted).+ *+ * When the sender receives notification that the consumer of the output+ * receiver has stopped consuming, the state of the sender transitions from+ * Active directly to CancellationProcessed, and the state of the input receiver+ * transitions from Active to CancellationTriggered. Once we receive a callback+ * indicating that the input receiver's cancellation signal has been received,+ * the input receiver's state is transitioned from CancellationTriggered to+ * to CancellationProcessed (and the object is deleted).+ */+template <+    typename InputValueType,+    typename OutputValueType,+    typename TransformerType>+class TransformProcessorBase : public IChannelCallback {+ public:+  TransformProcessorBase(+      Sender<OutputValueType> sender, TransformerType transformer)+      : sender_(std::move(senderGetBridge(sender))),+        transformer_(std::move(transformer)) {}++  template <typename ReceiverType>+  void startTransform(ReceiverType receiver) {+    executeWhenReady([=, this, receiver = std::move(receiver)](+                         std::unique_ptr<RateLimiter::Token> token) mutable {+      runOperationWithSenderCancellation(+          transformer_.getExecutor(),+          this->sender_,+          false /* alreadyStartedWaiting */,+          this /* channelCallbackToRestore */,+          startTransformImpl(std::move(receiver)),+          std::move(token));+    });+  }++ protected:+  /**+   * Starts transforming values from the input receiver and sending the+   * resulting transformed values to the output receiver.+   *+   * @param inputReceiver: The input receiver to transform values from.+   */+  folly::coro::Task<void> startTransformImpl(+      Receiver<InputValueType> receiver) {+    auto [unbufferedInputReceiver, buffer] =+        detail::receiverUnbuffer(std::move(receiver));+    receiver_ = std::move(unbufferedInputReceiver);+    co_await processAllAvailableValues(std::move(buffer));+  }++  /**+   * This is called when one of the channels we are listening to has an update+   * (either a value from the input receiver or a cancellation signal from the+   * sender).+   */+  void consume(ChannelBridgeBase* bridge) override {+    executeWhenReady([=, this](std::unique_ptr<RateLimiter::Token> token) {+      if (bridge == receiver_.get()) {+        // We have received new values from the input receiver.+        CHECK_NE(getReceiverState(), ChannelState::CancellationProcessed);+        runOperationWithSenderCancellation(+            transformer_.getExecutor(),+            this->sender_,+            true /* alreadyStartedWaiting */,+            this /* channelCallbackToRestore */,+            processAllAvailableValues(),+            std::move(token));+      } else {+        CHECK_NE(getSenderState(), ChannelState::CancellationProcessed);+        // The consumer of the output receiver has stopped consuming.+        if (getSenderState() == ChannelState::Active) {+          sender_->senderClose();+        }+        processSenderCancelled();+      }+    });+  }++  /**+   * This is called after we explicitly cancel one of the channels we are+   * listening to.+   */+  void canceled(ChannelBridgeBase* bridge) override {+    executeWhenReady([=, this](std::unique_ptr<RateLimiter::Token> token) {+      if (bridge == receiver_.get()) {+        // We previously cancelled the input receiver (because the consumer of+        // the output receiver stopped consuming). Process the cancellation for+        // the input receiver.+        CHECK_EQ(getReceiverState(), ChannelState::CancellationTriggered);+        runOperationWithSenderCancellation(+            transformer_.getExecutor(),+            this->sender_,+            true /* alreadyStartedWaiting */,+            this /* channelCallbackToRestore */,+            processReceiverCancelled(CloseResult()),+            std::move(token));+      } else {+        // We previously cancelled the sender due to the closure of the input+        // receiver. Process the cancellation for the sender.+        CHECK_EQ(getSenderState(), ChannelState::CancellationTriggered);+        processSenderCancelled();+      }+    });+  }++  /**+   * Processes all available values from the input receiver (starting from the+   * provided buffer, if present).+   *+   * If a value was received indicating that the input channel has been closed+   * (or if the transform function indicated that channel should be closed), we+   * will process cancellation for the input receiver.+   */+  folly::coro::Task<void> processAllAvailableValues(+      std::optional<ReceiverQueue<InputValueType>> buffer = std::nullopt) {+    auto closeResult = buffer.has_value()+        ? co_await processValues(std::move(buffer.value()))+        : std::nullopt;+    while (!closeResult.has_value()) {+      if (receiver_->receiverWait(this)) {+        // There are no more values available right now, but more values may+        // come in the future. We will stop processing for now, until we+        // re-start processing when the consume() callback is fired.+        break;+      }+      auto values = receiver_->receiverGetValues();+      CHECK(!values.empty());+      closeResult = co_await processValues(std::move(values));+    }+    if (closeResult.has_value()) {+      // The output receiver should be closed (either because the input receiver+      // was closed or the transform function desired the closure of the output+      // receiver).+      receiver_->receiverCancel();+      co_await processReceiverCancelled(std::move(closeResult.value()));+    }+  }++  /**+   * Processes the given set of values for the input receiver. If the output+   * receiver should be closed (either because the input receiver was closed or+   * the transform function desired the closure of the output receiver), a+   * CloseResult is returned containing the exception (if any) that should be+   * used to close the output receiver.+   */+  folly::coro::Task<std::optional<CloseResult>> processValues(+      ReceiverQueue<InputValueType> values) {+    auto cancelToken = co_await folly::coro::co_current_cancellation_token;+    while (!values.empty()) {+      auto inputResult = std::move(values.front());+      values.pop();+      bool inputClosed = !inputResult.hasValue();+      if (!inputResult.hasValue() && !inputResult.hasException()) {+        inputResult = Try<InputValueType>(OnClosedException());+      }+      auto outputGen = folly::makeTryWith([&]() {+        return transformer_.transformValue(std::move(inputResult));+      });+      if (!outputGen.hasValue()) {+        // The transform function threw an exception and was not a coroutine.+        // We will close the output receiver.+        co_return outputGen.template hasException<OnClosedException>()+            ? CloseResult()+            : CloseResult(std::move(outputGen.exception()));+      }+      while (true) {+        auto outputResult =+            co_await folly::coro::co_awaitTry(outputGen->next());+        if (!outputResult.hasException() && !outputResult->has_value()) {+          break;+        }+        if (cancelToken.isCancellationRequested()) {+          co_return CloseResult();+        }+        if (!outputResult.hasException()) {+          sender_->senderPush(std::move(outputResult->value()));+        } else {+          // The transform coroutine threw an exception. We will close the+          // output receiver.+          co_return outputResult.template hasException<OnClosedException>()+              ? CloseResult()+              : CloseResult(std::move(outputResult.exception()));+        }+      }+      if (inputClosed) {+        // The input receiver was closed, and the transform function did not+        // explicitly close the output receiver. We will therefore close it+        // anyway, as it does not make sense to keep it open when no future+        // values will arrive.+        co_return CloseResult();+      }+    }+    co_return std::nullopt;+  }++  /**+   * Process cancellation for the input receiver.+   */+  virtual folly::coro::Task<void> processReceiverCancelled(+      CloseResult closeResult, bool noRetriesAllowed = false) = 0;++  /**+   * Process cancellation for the sender.+   */+  void processSenderCancelled() {+    CHECK_EQ(getSenderState(), ChannelState::CancellationTriggered);+    sender_ = nullptr;+    if (getReceiverState() == ChannelState::Active) {+      receiver_->receiverCancel();+    }+    maybeDelete();+  }++  /**+   * Deletes this object if we have already processed cancellation for the+   * receiver and the sender.+   */+  void maybeDelete() {+    if (getReceiverState() == ChannelState::CancellationProcessed &&+        getSenderState() == ChannelState::CancellationProcessed) {+      delete this;+    }+  }++  ChannelState getReceiverState() {+    return detail::getReceiverState(receiver_.get());+  }++  ChannelState getSenderState() {+    return detail::getSenderState(sender_.get());+  }++  void executeWhenReady(+      folly::Function<void(std::unique_ptr<RateLimiter::Token>)> func) {+    auto rateLimiter = transformer_.getRateLimiter();+    if (rateLimiter != nullptr) {+      rateLimiter->executeWhenReady(+          std::move(func), transformer_.getExecutor());+    } else {+      transformer_.getExecutor()->add([func = std::move(func)]() mutable {+        func(std::unique_ptr<RateLimiter::Token>(nullptr));+      });+    }+  }++  ChannelBridgePtr<InputValueType> receiver_;+  ChannelBridgePtr<OutputValueType> sender_;+  TransformerType transformer_;+};++/**+ * This subclass is used for simple transformations triggered by a call to the+ * transform function (i.e. with a single input receiver and no initialization+ * function).+ */+template <+    typename InputValueType,+    typename OutputValueType,+    typename TransformerType>+class TransformProcessor+    : public TransformProcessorBase<+          InputValueType,+          OutputValueType,+          TransformerType> {+ public:+  using Base =+      TransformProcessorBase<InputValueType, OutputValueType, TransformerType>;+  using Base::Base;++ private:+  /**+   * Process cancellation for the input receiver.+   */+  folly::coro::Task<void> processReceiverCancelled(+      CloseResult closeResult, bool /* noRetriesAllowed */) override {+    CHECK_EQ(this->getReceiverState(), ChannelState::CancellationTriggered);+    this->receiver_ = nullptr;+    if (this->getSenderState() == ChannelState::Active) {+      if (closeResult.exception.has_value()) {+        this->sender_->senderClose(std::move(closeResult.exception.value()));+      } else {+        this->sender_->senderClose();+      }+    }+    this->maybeDelete();+    co_return;+  }+};++/**+ * This subclass is used for resumable transformations triggered by a call to+ * the resumableTransform function.+ */+template <+    typename InitializeArg,+    typename InputValueType,+    typename OutputValueType,+    typename TransformerType>+class ResumableTransformProcessor+    : public TransformProcessorBase<+          InputValueType,+          OutputValueType,+          TransformerType> {+ public:+  using Base =+      TransformProcessorBase<InputValueType, OutputValueType, TransformerType>;+  using Base::Base;++  void initialize(InitializeArg initializeArg) {+    this->executeWhenReady(+        [=, this, initializeArg = std::move(initializeArg)](+            std::unique_ptr<RateLimiter::Token> token) mutable {+          runOperationWithSenderCancellation(+              this->transformer_.getExecutor(),+              this->sender_,+              false /* currentlyWaiting */,+              this /* channelCallbackToRestore */,+              initializeImpl(std::move(initializeArg)),+              std::move(token));+        });+  }++ private:+  /**+   * Runs the user-provided initialization function to get a set of initial+   * values and a receiver to continue transforming. This is called when the+   * resumableTransform is created, and again whenever the previous input+   * receiver closed without an exception.+   */+  folly::coro::Task<void> initializeImpl(InitializeArg initializeArg) {+    auto cancelToken = co_await folly::coro::co_current_cancellation_token;+    auto initializeResult = co_await folly::coro::co_awaitTry(+        this->transformer_.initializeTransform(std::move(initializeArg)));+    if (initializeResult.hasException()) {+      auto closeResult =+          initializeResult.template hasException<OnClosedException>()+          ? CloseResult()+          : CloseResult(std::move(initializeResult.exception()));+      co_await processReceiverCancelled(+          std::move(closeResult), true /* noRetriesAllowed */);+      co_return;+    }+    auto [initialValues, inputReceiver] = std::move(initializeResult.value());+    CHECK(inputReceiver)+        << "The initialize function of a resumableTransform returned an "+           "invalid receiver.";+    if (cancelToken.isCancellationRequested()) {+      // The sender was closed before we finished running the initialization+      // function. We will ignore the results from that function and proceed+      // to process cancellation for the receiver.+      co_await processReceiverCancelled(+          CloseResult(), true /* noRetriesAllowed */);+      co_return;+    }+    for (auto& initialValue : initialValues) {+      this->sender_->senderPush(std::move(initialValue));+    }+    co_await this->startTransformImpl(std::move(inputReceiver));+  }++  /**+   * Process cancellation for the input receiver.+   */+  folly::coro::Task<void> processReceiverCancelled(+      CloseResult closeResult, bool noRetriesAllowed) override {+    if (this->receiver_) {+      CHECK_EQ(this->getReceiverState(), ChannelState::CancellationTriggered);+      this->receiver_ = nullptr;+    }+    auto cancelToken = co_await folly::coro::co_current_cancellation_token;+    if (this->getSenderState() == ChannelState::Active &&+        !cancelToken.isCancellationRequested()) {+      if (!closeResult.exception.has_value()) {+        // We were closed without an exception. We will close the sender without+        // an exception.+        this->sender_->senderClose();+      } else if (+          noRetriesAllowed ||+          !closeResult.exception+               ->is_compatible_with<ReinitializeException<InitializeArg>>()) {+        // We were closed with an exception. We will close the sender with that+        // exception.+        this->sender_->senderClose(std::move(closeResult.exception.value()));+      } else {+        // We were closed with a ReinitializeException. We will re-run the+        // user's initialization function and resume the resumableTransform.+        auto* reinitializeEx =+            closeResult.exception+                ->get_exception<ReinitializeException<InitializeArg>>();+        co_await initializeImpl(std::move(reinitializeEx->initializeArg));+        co_return;+      }+    }+    this->maybeDelete();+  }+};++template <bool Enabled>+class RateLimiterHolder;++template <>+class RateLimiterHolder<true> {+ public:+  explicit RateLimiterHolder(std::shared_ptr<RateLimiter> rateLimiter)+      : rateLimiter_(std::move(rateLimiter)) {}++  std::shared_ptr<RateLimiter> getRateLimiter() { return rateLimiter_; }++ private:+  std::shared_ptr<RateLimiter> rateLimiter_;+};++template <>+class RateLimiterHolder<false> {+ public:+  explicit RateLimiterHolder(std::shared_ptr<RateLimiter> rateLimiter) {+    CHECK_EQ(rateLimiter.get(), static_cast<void*>(NULL));+  }++  std::shared_ptr<RateLimiter> getRateLimiter() { return nullptr; }+};++template <+    typename InputValueType,+    typename OutputValueType,+    typename TransformValueFunc,+    bool RateLimiterEnabled>+class DefaultTransformer : public RateLimiterHolder<RateLimiterEnabled> {+ public:+  DefaultTransformer(+      folly::Executor::KeepAlive<folly::SequencedExecutor> executor,+      TransformValueFunc transformValue,+      std::shared_ptr<RateLimiter> rateLimiter)+      : RateLimiterHolder<RateLimiterEnabled>(std::move(rateLimiter)),+        executor_(std::move(executor)),+        transformValue_(std::move(transformValue)) {}++  folly::Executor::KeepAlive<folly::SequencedExecutor> getExecutor() {+    return executor_;+  }++  auto transformValue(Try<InputValueType> inputValue) {+    return transformValue_(std::move(inputValue));+  }++ private:+  folly::Executor::KeepAlive<folly::SequencedExecutor> executor_;+  TransformValueFunc transformValue_;+};++template <+    typename InitializeArg,+    typename InputValueType,+    typename OutputValueType,+    typename InitializeTransformFunc,+    typename TransformValueFunc,+    bool RateLimiterEnabled>+class DefaultResumableTransformer+    : public DefaultTransformer<+          InputValueType,+          OutputValueType,+          TransformValueFunc,+          RateLimiterEnabled> {+ public:+  using Base = DefaultTransformer<+      InputValueType,+      OutputValueType,+      TransformValueFunc,+      RateLimiterEnabled>;++  DefaultResumableTransformer(+      folly::Executor::KeepAlive<folly::SequencedExecutor> executor,+      InitializeTransformFunc initializeTransform,+      TransformValueFunc transformValue,+      std::shared_ptr<RateLimiter> rateLimiter)+      : Base(+            std::move(executor),+            std::move(transformValue),+            std::move(rateLimiter)),+        initializeTransform_(std::move(initializeTransform)) {}++  auto initializeTransform(InitializeArg initializeArg) {+    return initializeTransform_(std::move(initializeArg));+  }++ private:+  InitializeTransformFunc initializeTransform_;+};+} // namespace detail++template <+    typename ReceiverType,+    typename TransformValueFunc,+    typename InputValueType,+    typename OutputValueType>+Receiver<OutputValueType> transform(+    ReceiverType inputReceiver,+    folly::Executor::KeepAlive<folly::SequencedExecutor> executor,+    TransformValueFunc transformValue,+    std::shared_ptr<RateLimiter> rateLimiter) {+  if (rateLimiter != nullptr) {+    using TransformerType = detail::DefaultTransformer<+        InputValueType,+        OutputValueType,+        TransformValueFunc,+        true /* RateLimiterEnabled */>;+    return transform(+        std::move(inputReceiver),+        TransformerType(+            std::move(executor),+            std::move(transformValue),+            std::move(rateLimiter)));++  } else {+    using TransformerType = detail::DefaultTransformer<+        InputValueType,+        OutputValueType,+        TransformValueFunc,+        false /* RateLimiterEnabled */>;+    return transform(+        std::move(inputReceiver),+        TransformerType(+            std::move(executor),+            std::move(transformValue),+            nullptr /* rateLimiter */));+  }+}++template <+    typename ReceiverType,+    typename TransformerType,+    typename InputValueType,+    typename OutputValueType>+Receiver<OutputValueType> transform(+    ReceiverType inputReceiver, TransformerType transformer) {+  auto [outputReceiver, outputSender] = Channel<OutputValueType>::create();+  using TProcessor = detail::+      TransformProcessor<InputValueType, OutputValueType, TransformerType>;+  auto* processor =+      new TProcessor(std::move(outputSender), std::move(transformer));+  processor->startTransform(std::move(inputReceiver));+  return std::move(outputReceiver);+}++template <+    typename InitializeArg,+    typename InitializeTransformFunc,+    typename TransformValueFunc,+    typename ReceiverType,+    typename InputValueType,+    typename OutputValueType>+Receiver<OutputValueType> resumableTransform(+    folly::Executor::KeepAlive<folly::SequencedExecutor> executor,+    InitializeArg initializeArg,+    InitializeTransformFunc initializeTransform,+    TransformValueFunc transformValue,+    std::shared_ptr<RateLimiter> rateLimiter) {+  if (rateLimiter != nullptr) {+    using TransformerType = detail::DefaultResumableTransformer<+        InitializeArg,+        InputValueType,+        OutputValueType,+        InitializeTransformFunc,+        TransformValueFunc,+        true /* RateLimiterEnabled */>;+    return resumableTransform(+        std::move(initializeArg),+        TransformerType(+            std::move(executor),+            std::move(initializeTransform),+            std::move(transformValue),+            std::move(rateLimiter)));+  } else {+    using TransformerType = detail::DefaultResumableTransformer<+        InitializeArg,+        InputValueType,+        OutputValueType,+        InitializeTransformFunc,+        TransformValueFunc,+        false /* RateLimiterEnabled */>;+    return resumableTransform(+        std::move(initializeArg),+        TransformerType(+            std::move(executor),+            std::move(initializeTransform),+            std::move(transformValue),+            nullptr /* rateLimiter */));+  }+}++template <+    typename InitializeArg,+    typename TransformerType,+    typename ReceiverType,+    typename InputValueType,+    typename OutputValueType>+Receiver<OutputValueType> resumableTransform(+    InitializeArg initializeArg, TransformerType transformer) {+  auto [outputReceiver, outputSender] = Channel<OutputValueType>::create();+  using TProcessor = detail::ResumableTransformProcessor<+      InitializeArg,+      InputValueType,+      OutputValueType,+      TransformerType>;+  auto* processor =+      new TProcessor(std::move(outputSender), std::move(transformer));+  processor->initialize(std::move(initializeArg));+  return std::move(outputReceiver);+}++} // namespace channels+} // namespace folly
+ folly/folly/channels/Transform.h view
@@ -0,0 +1,237 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <folly/channels/Channel.h>+#include <folly/channels/OnClosedException.h>+#include <folly/channels/RateLimiter.h>+#include <folly/executors/SequencedExecutor.h>++namespace folly {+namespace channels {++/**+ * Returns an output receiver that applies a given transformation function to+ * each value from an input receiver.+ *+ * The TransformValue function takes a Try<InputValueType>, and returns a+ * folly::coro::AsyncGenerator<OutputValueType>.+ *+ *   - If the TransformValue function yields one or more output values, those+ *      output values are sent to the output receiver.+ *+ *   - If the TransformValue function throws an OnClosedException, the output+ *      receiver is closed (without an exception).+ *+ *   - If the TransformValue function throws any other type of exception, the+ *      output receiver is closed with that exception.+ *+ * If the input receiver was closed, the TransformValue function is called with+ * a Try containing an exception (either OnClosedException if the input receiver+ * was closed without an exception, or the closure exception if the input+ * receiver was closed with an exception). In this case, regardless of what the+ * TransformValue function returns, the output receiver will be closed+ * (potentially after receiving the last output values the TransformValue+ * function returned, if any).+ *+ * @param inputReceiver: The input receiver.+ *+ * @param executor: A folly::SequencedExecutor used to transform the values.+ *+ * @param transformValue: A function as described above.+ *+ * @param rateLimiter: An optional rate limiter. If specified, the given rate+ *     limiter will limit the number of transformation functions that are+ *     simultaneously running.+ *+ * Example:+ *+ *  // Function that returns a receiver+ *  Receiver<int> getInputReceiver();+ *+ *  // Function that returns an executor+ *  folly::Executor::KeepAlive<folly::SequencedExecutor> getExecutor();+ *+ *  Receiver<std::string> outputReceiver = transform(+ *      getInputReceiver(),+ *      getExecutor(),+ *      [](Try<int> try) -> folly::coro::AsyncGenerator<std::string&&> {+ *          co_yield folly::to<std::string>(try.value());+ *      });+ */+template <+    typename ReceiverType,+    typename TransformValueFunc,+    typename InputValueType = typename ReceiverType::ValueType,+    typename OutputValueType = typename folly::invoke_result_t< //+        TransformValueFunc,+        Try<InputValueType>>::value_type>+Receiver<OutputValueType> transform(+    ReceiverType inputReceiver,+    folly::Executor::KeepAlive<folly::SequencedExecutor> executor,+    TransformValueFunc transformValue,+    std::shared_ptr<RateLimiter> rateLimiter = nullptr);++/**+ * This overload accepts arguments in the form of a transformer object. The+ * transformer object must have the following functions:+ *+ * folly::Executor::KeepAlive<folly::SequencedExecutor> getExecutor();+ *+ * folly::coro::AsyncGenerator<OutputValueType&&> transformValue(+ *     Try<InputValueType> inputValue);+ *+ * std::shared_ptr<RateLimiter> getRateLimiter(); // Can return nullptr+ */+template <+    typename ReceiverType,+    typename TransformerType,+    typename InputValueType = typename ReceiverType::ValueType,+    typename OutputValueType =+        typename decltype(std::declval<TransformerType>().transformValue(+            std::declval<Try<InputValueType>>()))::value_type>+Receiver<OutputValueType> transform(+    ReceiverType inputReceiver, TransformerType transformer);++/**+ * This function is similar to the above transform function. However, instead of+ * taking a single input receiver, it takes an initialization function that+ * accepts a value of type InitializeArg, and returns a+ * std::pair<std::vector<OutputValueType>, Receiver<InputValueType>>.+ *+ *  - If the InitializeTransform function returns successfully, the vector's+ *      output values will be immediately sent to the output receiver. The input+ *      receiver is then processed as described in the transform function's+ *      documentation, unless and until it throws a ReinitializeException. At+ *      that point, the InitializationTransform is re-run with the InitializeArg+ *      specified in the ReinitializeException, and the transform begins anew.+ *+ *  - If the InitializeTransform function or the TransformValue function throws+ *      an OnClosedException, the output receiver is closed (with no exception).+ *+ *  - If the InitializeTransform function or the TransformValue function throws+ *      any other type of exception, the output receiver is closed with that+ *      exception.+ *+ * @param executor: A folly::SequencedExecutor used to transform the values.+ *+ * @param initializeArg: The initial argument passed to the InitializeTransform+ *  function.+ *+ * @param initializeTransform: The InitializeTransform function as described+ *  above.+ *+ * @param transformValue: The TransformValue function as described above.+ *+ * @param rateLimiter: An optional rate limiter. If specified, the given rate+ *     limiter will limit the number of transformation functions that are+ *     simultaneously running.+ *+ * Example:+ *+ *  struct InitializeArg {+ *    std::string param;+ *  }+ *+ *  // Function that returns a receiver+ *  Receiver<int> getInputReceiver(InitializeArg initializeArg);+ *+ *  // Function that returns an executor+ *  folly::Executor::KeepAlive<folly::SequencedExecutor> getExecutor();+ *+ *  Receiver<std::string> outputReceiver = resumableTransform(+ *      getExecutor(),+ *      InitializeArg{"param"},+ *      [](InitializeArg initializeArg) -> folly::coro::Task<+ *                  std::pair<std::vector<std::string>, Receiver<int>> {+ *          co_return std::make_pair(+ *              std::vector<std::string>({"Initialized"}),+ *              getInputReceiver(initializeArg));+ *      },+ *      [](Try<int> try) -> folly::coro::AsyncGenerator<std::string&&> {+ *          try {+ *            co_yield folly::to<std::string>(try.value());+ *          } catch (const SomeApplicationException& ex) {+ *            throw ReinitializeException(InitializeArg{ex.getParam()});+ *          }+ *      });+ *+ */+template <+    typename InitializeArg,+    typename InitializeTransformFunc,+    typename TransformValueFunc,+    typename ReceiverType = typename folly::invoke_result_t<+        InitializeTransformFunc,+        InitializeArg>::StorageType::second_type,+    typename InputValueType = typename ReceiverType::ValueType,+    typename OutputValueType = typename folly::invoke_result_t< //+        TransformValueFunc,+        Try<InputValueType>>::value_type>+Receiver<OutputValueType> resumableTransform(+    folly::Executor::KeepAlive<folly::SequencedExecutor> executor,+    InitializeArg initializeArg,+    InitializeTransformFunc initializeTransform,+    TransformValueFunc transformValue,+    std::shared_ptr<RateLimiter> rateLimiter = nullptr);++/**+ * This overload accepts arguments in the form of a transformer object. The+ * transformer object must have the following functions:+ *+ * folly::Executor::KeepAlive<folly::SequencedExecutor> getExecutor();+ *+ * std::pair<std::vector<OutputValueType>, Receiver<InputValueType>>+ * initializeTransform(InitializeArg initializeArg);+ *+ * folly::coro::AsyncGenerator<OutputValueType&&> transformValue(+ *     Try<InputValueType> inputValue);+ *+ * std::shared_ptr<RateLimiter> getRateLimiter(); // Can return nullptr+ */+template <+    typename InitializeArg,+    typename TransformerType,+    typename ReceiverType =+        typename decltype(std::declval<TransformerType>().initializeTransform(+            std::declval<InitializeArg>()))::StorageType::second_type,+    typename InputValueType = typename ReceiverType::ValueType,+    typename OutputValueType =+        typename decltype(std::declval<TransformerType>().transformValue(+            std::declval<Try<InputValueType>>()))::value_type>+Receiver<OutputValueType> resumableTransform(+    InitializeArg initializeArg, TransformerType transformer);++/**+ * A ReinitializeException thrown by a transform callback indicates that the+ * resumable transform needs to be re-initialized.+ */+template <typename InitializeArg>+struct ReinitializeException : public std::exception {+  explicit ReinitializeException(InitializeArg _initializeArg)+      : initializeArg(std::move(_initializeArg)) {}++  const char* what() const noexcept override {+    return "This resumable transform should be re-initialized.";+  }++  InitializeArg initializeArg;+};+} // namespace channels+} // namespace folly++#include <folly/channels/Transform-inl.h>
+ folly/folly/channels/detail/AtomicQueue.h view
@@ -0,0 +1,267 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <atomic>+#include <cassert>+#include <memory>+#include <utility>+#include <glog/logging.h>++#include <folly/lang/Assume.h>++namespace folly {+namespace channels {+namespace detail {++template <typename T>+class Queue {+ public:+  constexpr Queue() noexcept {}+  constexpr Queue(Queue&& other) noexcept+      : head_(std::exchange(other.head_, nullptr)) {}+  Queue& operator=(Queue&& other) noexcept {+    clear();+    std::swap(head_, other.head_);+    return *this;+  }+  ~Queue() { clear(); }++  bool empty() const noexcept { return !head_; }++  T& front() noexcept { return head_->value; }++  void pop() noexcept {+    std::unique_ptr<Node>(std::exchange(head_, head_->next));+  }++  void clear() {+    while (!empty()) {+      pop();+    }+  }++  explicit operator bool() const { return !empty(); }++  struct Node {+    explicit Node(T&& t) : value(std::move(t)) {}++    T value;+    Node* next{nullptr};+  };++  constexpr explicit Queue(Node* head) noexcept : head_(head) {}+  static Queue fromReversed(Node* tail) noexcept {+    // Reverse a linked list.+    Node* head{nullptr};+    while (tail) {+      head = std::exchange(tail, std::exchange(tail->next, head));+    }+    return Queue(head);+  }++  Node* head_{nullptr};+};++template <typename Consumer, typename Message>+class AtomicQueue {+ public:+  using MessageQueue = Queue<Message>;++  AtomicQueue() { static_assert(alignof(Consumer) > kTypeMask); }+  ~AtomicQueue() {+    auto storage = storage_.load(std::memory_order_acquire);+    auto type = static_cast<Type>(storage & kTypeMask);+    auto ptr = storage & kPointerMask;+    switch (type) {+      case Type::EMPTY:+      case Type::CLOSED:+        return;+      case Type::TAIL:+        MessageQueue::fromReversed(+            reinterpret_cast<typename MessageQueue::Node*>(ptr));+        return;+      case Type::CONSUMER:+      default:+        folly::assume_unreachable();+    }+  }+  AtomicQueue(const AtomicQueue&) = delete;+  AtomicQueue& operator=(const AtomicQueue&) = delete;++  template <typename... ConsumerArgs>+  void push(Message&& value, ConsumerArgs&&... consumerArgs) {+    std::unique_ptr<typename MessageQueue::Node> node(+        new typename MessageQueue::Node(std::move(value)));+    assert(!(reinterpret_cast<intptr_t>(node.get()) & kTypeMask));++    auto storage = storage_.load(std::memory_order_relaxed);+    while (true) {+      auto type = static_cast<Type>(storage & kTypeMask);+      auto ptr = storage & kPointerMask;+      switch (type) {+        case Type::EMPTY:+        case Type::TAIL:+          node->next = reinterpret_cast<typename MessageQueue::Node*>(ptr);+          if (storage_.compare_exchange_weak(+                  storage,+                  reinterpret_cast<intptr_t>(node.get()) |+                      static_cast<intptr_t>(Type::TAIL),+                  std::memory_order_release,+                  std::memory_order_relaxed)) {+            node.release();+            return;+          }+          break;+        case Type::CLOSED:+          return;+        case Type::CONSUMER:+          node->next = nullptr;+          if (storage_.compare_exchange_weak(+                  storage,+                  reinterpret_cast<intptr_t>(node.get()) |+                      static_cast<intptr_t>(Type::TAIL),+                  std::memory_order_acq_rel,+                  std::memory_order_relaxed)) {+            node.release();+            auto consumer = reinterpret_cast<Consumer*>(ptr);+            consumer->consume(std::forward<ConsumerArgs>(consumerArgs)...);+            return;+          }+          break;+        default:+          folly::assume_unreachable();+      }+    }+  }++  template <typename... ConsumerArgs>+  bool wait(Consumer* consumer, ConsumerArgs&&... consumerArgs) {+    assert(!(reinterpret_cast<intptr_t>(consumer) & kTypeMask));+    auto storage = storage_.load(std::memory_order_relaxed);+    while (true) {+      auto type = static_cast<Type>(storage & kTypeMask);+      switch (type) {+        case Type::EMPTY:+          if (storage_.compare_exchange_weak(+                  storage,+                  reinterpret_cast<intptr_t>(consumer) |+                      static_cast<intptr_t>(Type::CONSUMER),+                  std::memory_order_release,+                  std::memory_order_relaxed)) {+            return true;+          }+          break;+        case Type::CLOSED:+          consumer->canceled(std::forward<ConsumerArgs>(consumerArgs)...);+          return true;+        case Type::TAIL:+          return false;+        case Type::CONSUMER:+        default:+          folly::assume_unreachable();+      }+    }+  }++  template <typename... ConsumerArgs>+  void close(ConsumerArgs&&... consumerArgs) {+    auto storage = storage_.exchange(+        static_cast<intptr_t>(Type::CLOSED), std::memory_order_acquire);+    auto type = static_cast<Type>(storage & kTypeMask);+    auto ptr = storage & kPointerMask;+    switch (type) {+      case Type::EMPTY:+        return;+      case Type::TAIL:+        MessageQueue::fromReversed(+            reinterpret_cast<typename MessageQueue::Node*>(ptr));+        return;+      case Type::CONSUMER:+        reinterpret_cast<Consumer*>(ptr)->canceled(+            std::forward<ConsumerArgs>(consumerArgs)...);+        return;+      case Type::CLOSED:+      default:+        folly::assume_unreachable();+    }+  }++  bool isClosed() {+    auto type = static_cast<Type>(storage_ & kTypeMask);+    return type == Type::CLOSED;+  }++  template <typename... ConsumerArgs>+  MessageQueue getMessages(ConsumerArgs&&... consumerArgs) {+    auto storage = storage_.exchange(+        static_cast<intptr_t>(Type::EMPTY), std::memory_order_acquire);+    auto type = static_cast<Type>(storage & kTypeMask);+    auto ptr = storage & kPointerMask;+    switch (type) {+      case Type::TAIL:+        return MessageQueue::fromReversed(+            reinterpret_cast<typename MessageQueue::Node*>(ptr));+      case Type::EMPTY:+        return MessageQueue();+      case Type::CLOSED:+        // We accidentally re-opened the queue, so close it again.+        // This is only safe to do because isClosed() can't be called+        // concurrently with getMessages().+        close(std::forward<ConsumerArgs>(consumerArgs)...);+        return MessageQueue();+      case Type::CONSUMER:+      default:+        folly::assume_unreachable();+    }+  }++  Consumer* cancelCallback() {+    auto storage = storage_.load(std::memory_order_acquire);+    while (true) {+      auto type = static_cast<Type>(storage & kTypeMask);+      auto ptr = storage & kPointerMask;+      switch (type) {+        case Type::CONSUMER:+          if (storage_.compare_exchange_weak(+                  storage,+                  static_cast<intptr_t>(Type::EMPTY),+                  std::memory_order_relaxed,+                  std::memory_order_relaxed)) {+            return reinterpret_cast<Consumer*>(ptr);+          }+          break;+        case Type::TAIL:+        case Type::EMPTY:+        case Type::CLOSED:+        default:+          return nullptr;+      }+    }+  }++ private:+  enum class Type : intptr_t { EMPTY = 0, CONSUMER = 1, TAIL = 2, CLOSED = 3 };++  static constexpr intptr_t kTypeMask = 3;+  static constexpr intptr_t kPointerMask = ~kTypeMask;++  std::atomic<intptr_t> storage_{0};+};+} // namespace detail+} // namespace channels+} // namespace folly
+ folly/folly/channels/detail/ChannelBridge.h view
@@ -0,0 +1,139 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <folly/Try.h>+#include <folly/experimental/channels/detail/AtomicQueue.h>++namespace folly {+namespace channels {+namespace detail {++class ChannelBridgeBase {};++class IChannelCallback {+ public:+  virtual ~IChannelCallback() = default;++  virtual void consume(ChannelBridgeBase* bridge) = 0;++  virtual void canceled(ChannelBridgeBase* bridge) = 0;+};++using SenderQueue = typename folly::channels::detail::Queue<Unit>;++template <typename TValue>+using ReceiverQueue = typename folly::channels::detail::Queue<Try<TValue>>;++template <typename TValue>+class ChannelBridge : public ChannelBridgeBase {+ public:+  struct Deleter {+    void operator()(ChannelBridge<TValue>* ptr) { ptr->decref(); }+  };+  using Ptr = std::unique_ptr<ChannelBridge<TValue>, Deleter>;++  static Ptr create() { return Ptr(new ChannelBridge<TValue>()); }++  Ptr copy() {+    auto refCount = refCount_.fetch_add(1, std::memory_order_relaxed);+    DCHECK(refCount > 0);+    return Ptr(this);+  }++  // These should only be called from the sender thread++  template <typename U = TValue>+  void senderPush(U&& value) {+    receiverQueue_.push(+        Try<TValue>(std::forward<U>(value)),+        static_cast<ChannelBridgeBase*>(this));+  }++  bool senderWait(IChannelCallback* callback) {+    return senderQueue_.wait(callback, static_cast<ChannelBridgeBase*>(this));+  }++  IChannelCallback* cancelSenderWait() { return senderQueue_.cancelCallback(); }++  void senderClose() {+    if (!isSenderClosed()) {+      receiverQueue_.push(Try<TValue>(), static_cast<ChannelBridgeBase*>(this));+      senderQueue_.close(static_cast<ChannelBridgeBase*>(this));+    }+  }++  void senderClose(exception_wrapper ex) {+    if (!isSenderClosed()) {+      receiverQueue_.push(+          Try<TValue>(std::move(ex)), static_cast<ChannelBridgeBase*>(this));+      senderQueue_.close(static_cast<ChannelBridgeBase*>(this));+    }+  }++  bool isSenderClosed() { return senderQueue_.isClosed(); }++  SenderQueue senderGetValues() {+    return senderQueue_.getMessages(static_cast<ChannelBridgeBase*>(this));+  }++  // These should only be called from the receiver thread++  void receiverCancel() {+    if (!isReceiverCancelled()) {+      senderQueue_.push(Unit(), static_cast<ChannelBridgeBase*>(this));+      receiverQueue_.close(static_cast<ChannelBridgeBase*>(this));+    }+  }++  bool isReceiverCancelled() { return receiverQueue_.isClosed(); }++  bool receiverWait(IChannelCallback* callback) {+    return receiverQueue_.wait(callback, static_cast<ChannelBridgeBase*>(this));+  }++  IChannelCallback* cancelReceiverWait() {+    return receiverQueue_.cancelCallback();+  }++  ReceiverQueue<TValue> receiverGetValues() {+    return receiverQueue_.getMessages(static_cast<ChannelBridgeBase*>(this));+  }++ private:+  using ReceiverAtomicQueue = typename folly::channels::detail::+      AtomicQueue<IChannelCallback, Try<TValue>>;++  using SenderAtomicQueue =+      typename folly::channels::detail::AtomicQueue<IChannelCallback, Unit>;++  void decref() {+    if (refCount_.fetch_sub(1, std::memory_order_acq_rel) == 1) {+      delete this;+    }+  }++  ReceiverAtomicQueue receiverQueue_;+  SenderAtomicQueue senderQueue_;+  std::atomic<int8_t> refCount_{1};+};++template <typename TValue>+using ChannelBridgePtr = typename ChannelBridge<TValue>::Ptr;+} // namespace detail+} // namespace channels+} // namespace folly
+ folly/folly/channels/detail/IntrusivePtr.h view
@@ -0,0 +1,52 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <boost/intrusive_ptr.hpp>+#include <boost/smart_ptr/intrusive_ref_counter.hpp>++namespace folly {+namespace channels {+namespace detail {++/**+ * An intrusive_ptr is like an std::shared_ptr. However, unlike a shared_ptr,+ * the reference count for an intrusive_ptr lives on the object itself. This has+ * two advantages:+ *+ * 1. Each intrusive_ptr is 8 bytes instead of 16 bytes.+ *+ * 2. An intrusive_ptr can be created from a raw pointer/reference, unlike a+ *    shared_ptr.+ *+ * To use intrusive_ptr<T>, ensure that T inherits from IntrusivePtrBase<T>.+ */++template <typename T>+using intrusive_ptr = boost::intrusive_ptr<T>;++template <typename T>+using IntrusivePtrBase =+    boost::intrusive_ref_counter<T, boost::thread_safe_counter>;++template <typename T, typename... Args>+intrusive_ptr<T> make_intrusive(Args&&... args) {+  return intrusive_ptr<T>(new T(std::forward<Args>(args)...));+}+} // namespace detail+} // namespace channels+} // namespace folly
+ folly/folly/channels/detail/MultiplexerTraits.h view
@@ -0,0 +1,55 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <folly/Traits.h>+#include <folly/functional/traits.h>++namespace folly {+namespace channels {+namespace detail {++template <typename MultiplexerType>+struct MultiplexerTraits {+  using OnNewSubscriptionSig =+      member_pointer_member_t<decltype(&MultiplexerType::onNewSubscription)>;++  using OnInputValueSig =+      member_pointer_member_t<decltype(&MultiplexerType::onInputValue)>;++  // First parameter type of MultiplexerType::onNewSubscription+  using KeyType = function_arguments_element_t<0, OnNewSubscriptionSig>;++  // Second parameter type for MultiplexerType::onNewSubscription+  using KeyContextType =+      std::decay_t<function_arguments_element_t<1, OnNewSubscriptionSig>>;++  // Third parameter type for MultiplexerType::onNewSubscription+  using SubscriptionArgType =+      function_arguments_element_t<2, OnNewSubscriptionSig>;++  // First parameter value type of MultiplexerType::onInputValue+  using InputValueType =+      typename function_arguments_element_t<0, OnInputValueSig>::element_type;++  // Element type of the returned vector from MultiplexerType::onNewSubscription+  using OutputValueType =+      typename function_result_t<OnNewSubscriptionSig>::StorageType::value_type;+};+} // namespace detail+} // namespace channels+} // namespace folly
+ folly/folly/channels/detail/PointerVariant.h view
@@ -0,0 +1,104 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <fmt/format.h>+#include <folly/Demangle.h>++namespace folly {+namespace channels {+namespace detail {++/**+ * A PointerVariant stores a pointer of one of two possible types.+ */+template <typename FirstType, typename SecondType>+class PointerVariant {+ public:+  template <typename T>+  explicit PointerVariant(T* pointer) {+    set(pointer);+  }++  PointerVariant(PointerVariant&& other) noexcept+      : storage_(std::exchange(other.storage_, 0)) {}++  PointerVariant& operator=(PointerVariant&& other) noexcept {+    storage_ = std::exchange(other.storage_, 0);+    return *this;+  }++  /**+   * Returns the zero-based index of the type that is currently held.+   */+  size_t index() const { return static_cast<size_t>(storage_ & kTypeMask); }++  /**+   * Returns the pointer stored in the PointerVariant, if the type matches the+   * first type. If the stored type does not match the first type, an exception+   * will be thrown.+   */+  inline FirstType* get(folly::tag_t<FirstType>) const {+    ensureCorrectType(false /* secondType */);+    return reinterpret_cast<FirstType*>(storage_ & kPointerMask);+  }++  /**+   * Returns the pointer stored in the PointerVariant, if the type matches the+   * second type. If the stored type does not match the second type, an+   * exception will be thrown.+   */+  inline SecondType* get(folly::tag_t<SecondType>) const {+    ensureCorrectType(true /* secondType */);+    return reinterpret_cast<SecondType*>(storage_ & kPointerMask);+  }++  /**+   * Store a new pointer of type FirstType in the PointerVariant.+   */+  void set(FirstType* pointer) {+    storage_ = reinterpret_cast<intptr_t>(pointer);+  }++  /**+   * Store a new pointer of type SecondType in the PointerVariant.+   */+  void set(SecondType* pointer) {+    storage_ = reinterpret_cast<intptr_t>(pointer) | kTypeMask;+  }++ private:+  void ensureCorrectType(bool secondType) const {+    if (secondType != !!(storage_ & kTypeMask)) {+      throw std::runtime_error(fmt::format(+          "Incorrect type specified. Given: {}, Stored: {}",+          secondType ? folly::demangle(typeid(SecondType).name())+                     : folly::demangle(typeid(FirstType).name()),+          storage_ & kTypeMask+              ? folly::demangle(typeid(SecondType).name())+              : folly::demangle(typeid(FirstType).name())));+    }+  }++  static constexpr intptr_t kTypeMask = 1;+  static constexpr intptr_t kPointerMask = ~kTypeMask;++  intptr_t storage_;+};+} // namespace detail+} // namespace channels+} // namespace folly
+ folly/folly/channels/detail/Utility.h view
@@ -0,0 +1,286 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <optional>+#include <folly/ExceptionWrapper.h>+#include <folly/Function.h>+#include <folly/ScopeGuard.h>+#include <folly/channels/Channel.h>+#include <folly/channels/RateLimiter.h>+#include <folly/coro/Promise.h>+#include <folly/coro/Task.h>+#include <folly/executors/SequencedExecutor.h>++namespace folly {+namespace channels {+namespace detail {++struct CloseResult {+  CloseResult() {}++  explicit CloseResult(exception_wrapper _exception)+      : exception(std::move(_exception)) {}++  std::optional<exception_wrapper> exception;+};++enum class ChannelState {+  Active,+  CancellationTriggered,+  CancellationProcessed+};++template <typename TSender>+ChannelState getSenderState(TSender* sender) {+  if (sender == nullptr) {+    return ChannelState::CancellationProcessed;+  } else if (sender->isSenderClosed()) {+    return ChannelState::CancellationTriggered;+  } else {+    return ChannelState::Active;+  }+}++template <typename TReceiver>+ChannelState getReceiverState(TReceiver* receiver) {+  if (receiver == nullptr) {+    return ChannelState::CancellationProcessed;+  } else if (receiver->isReceiverCancelled()) {+    return ChannelState::CancellationTriggered;+  } else {+    return ChannelState::Active;+  }+}++inline std::ostream& operator<<(std::ostream& os, ChannelState state) {+  switch (state) {+    case ChannelState::Active:+      return os << "Active";+    case ChannelState::CancellationTriggered:+      return os << "CancellationTriggered";+    case ChannelState::CancellationProcessed:+      return os << "CancellationProcessed";+    default:+      return os << "Should never be hit";+  }+}++/**+ * A cancellation callback that wraps an existing channel callback. When the+ * callback is fired, this object will trigger cancellation on its cancellation+ * source (in addition to firing the wrapped callback).+ */+template <typename TSender>+class SenderCancellationCallback : public IChannelCallback {+ public:+  explicit SenderCancellationCallback(+      TSender& sender,+      folly::Executor::KeepAlive<folly::SequencedExecutor> executor,+      IChannelCallback* channelCallback)+      : sender_(sender),+        executor_(std::move(executor)),+        channelCallback_(channelCallback),+        callbackToFire_(folly::coro::makePromiseContract<CallbackToFire>()) {+    if (channelCallback_ == nullptr) {+      // The sender was already canceled runOperationWithSenderCancellation was+      // even called. This means the cancelled callback already was fired, so+      // we will not set the callback to fire here.+      cancelSource_.requestCancellation();+      return;+    }+    CHECK(sender_);+    if (!sender_->senderWait(this)) {+      // The sender was cancelled after runOperationWithSenderCancellation was+      // called, but before we had a chance to start the operation. This means+      // that the cancelled callback was never called. We will therefore set it+      // to fire here, when the operation is complete.+      cancelSource_.requestCancellation();+      callbackToFire_.first.setValue(CallbackToFire::Consume);+    }+  }++  folly::coro::Task<void> onTaskCompleted() {+    if (!channelCallback_) {+      co_return;+    }+    auto callbackToFire = std::optional<CallbackToFire>();+    bool promiseSet = false;+    if (callbackToFire_.second.isReady()) {+      // The callback was fired.+      promiseSet = true;+      callbackToFire = co_await std::move(callbackToFire_.second);+    } else {+      // The callback has not yet been fired.+      if (!sender_->cancelSenderWait()) {+        // The sender has been cancelled, but the callback has not been called+        // yet. Wait for the callback to be called.+        promiseSet = true;+        callbackToFire = co_await std::move(callbackToFire_.second);+      } else if (!sender_->senderWait(channelCallback_)) {+        // The sender was cancelled between the call to cancelSenderWait and+        // the call to senderWait. This means that the cancelled callback was+        // never called. We will therefore set it to fire here.+        callbackToFire = CallbackToFire::Consume;+      }+    }+    if (!promiseSet) {+      // Set a default value here, so we don't need to waste time constructing a+      // broken promise exception when the promise is destructed. This value+      // will not be read.+      callbackToFire_.first.setValue(CallbackToFire::Consume);+    }+    if (callbackToFire.has_value()) {+      switch (callbackToFire.value()) {+        case CallbackToFire::Consume:+          channelCallback_->consume(sender_.get());+          co_return;+        case CallbackToFire::Canceled:+          channelCallback_->canceled(sender_.get());+          co_return;+      }+    }+    // The sender has not yet been cancelled, and we are now back in the state+    // where the sender is waiting on the user-provided callback. We are done.+  }++  /**+   * Returns a cancellation token that will trigger when the sender+   */+  folly::CancellationToken getCancellationToken() {+    return cancelSource_.getToken();+  }++  /**+   * Requests cancellation, and triggers the consume function on the callback+   * if the callback was not previously triggered.+   */+  void consume(ChannelBridgeBase*) override {+    cancelSource_.requestCancellation();+    executor_->add([this]() {+      CHECK(!callbackToFire_.second.isReady());+      callbackToFire_.first.setValue(CallbackToFire::Consume);+    });+  }++  /**+   * Requests cancellation, and triggers the canceled function on the callback+   * if the callback was not previously triggered.+   */+  void canceled(ChannelBridgeBase*) override {+    cancelSource_.requestCancellation();+    executor_->add([this]() {+      CHECK(!callbackToFire_.second.isReady());+      callbackToFire_.first.setValue(CallbackToFire::Canceled);+    });+  }++ private:+  enum class CallbackToFire { Consume, Canceled };++  TSender& sender_;+  folly::Executor::KeepAlive<folly::SequencedExecutor> executor_;+  IChannelCallback* channelCallback_;+  folly::CancellationSource cancelSource_;+  std::pair<+      folly::coro::Promise<CallbackToFire>,+      folly::coro::Future<CallbackToFire>>+      callbackToFire_;+};++/**+ * Any object that produces an output receiver (transform, merge,+ * MergeChannel, etc) will listen for a cancellation signal from that output+ * receiver. Once the consumer of the output receiver stops consuming, a+ * callback will be called that triggers these objects to start cleaning+ * themselves up (and eventually destroy themselves).+ *+ * However, when one of these objects decides to run a user coroutine, they+ * would like that user coroutine to be able to get notified when that+ * cancellation signal is received. That allows the coroutine to stop any+ * long-running operations quickly, rather than running a long time when the+ * consumer of the output receiver no longer cares about the result.+ *+ * This function enables that behavior. It will run the provided operation+ * coroutine. While that coroutine is running, it will listen to cancellation+ * events from the output receiver (through its sender). If it receives a+ * cancellation signal from the sender, it will trigger cancellation of the+ * operation coroutine.+ *+ * Once the coroutine finishes, it will then call the given channel callback+ * to notify it of the cancellation event (the same way that callback would+ * have been notified if no coroutine had been started). It will also resume+ * waiting on the channel callback.+ *+ * @param executor: The executor to run the coroutine on.+ *+ * @param sender: The sender to use to listen for cancellation. If this is+ * null, we will assume that cancellation already occurred.+ *+ * @param alreadyStartedWaiting: Whether or not the caller already started+ * listening for a cancellation signal from the output receiver. If so, this+ * function will temporarily stop waiting with that callback (so it can listen+ * for the cancellation signal to stop the coroutine).+ *+ * @param channelCallbackToRestore: The channel callback to restore once the+ *  coroutine operation is complete.+ *+ * @param operation: The operation to run.+ *+ * @param token: The rate limiter token for this operation.+ */+template <typename TSender>+void runOperationWithSenderCancellation(+    folly::Executor::KeepAlive<folly::SequencedExecutor> executor,+    TSender& sender,+    bool alreadyStartedWaiting,+    IChannelCallback* channelCallbackToRestore,+    folly::coro::Task<void> operation,+    std::unique_ptr<RateLimiter::Token> token) noexcept {+  if (alreadyStartedWaiting && (!sender || !sender->cancelSenderWait())) {+    // The output receiver was cancelled before starting this operation+    // (indicating that the channel callback already ran).+    channelCallbackToRestore = nullptr;+  }+  co_withExecutor(+      executor,+      folly::coro::co_invoke(+          [&sender,+           executor,+           channelCallbackToRestore,+           token = std::move(token),+           operation =+               std::move(operation)]() mutable -> folly::coro::Task<void> {+            auto senderCancellationCallback = SenderCancellationCallback(+                sender, executor, channelCallbackToRestore);+            auto result = co_await folly::coro::co_awaitTry(+                folly::coro::co_withCancellation(+                    senderCancellationCallback.getCancellationToken(),+                    std::move(operation)));+            if (result.hasException()) {+              LOG(FATAL) << fmt::format(+                  "Unexpected exception when running coroutine operation with "+                  "sender cancellation: {}",+                  result.exception().what());+            }+            co_await senderCancellationCallback.onTaskCompleted();+          }))+      .start();+}+} // namespace detail+} // namespace channels+} // namespace folly
+ folly/folly/chrono/Clock.h view
@@ -0,0 +1,65 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <chrono>++/**+ * Namespace for folly chrono types.+ *+ * Using a separate namespace for clock types to minimize type conflicts in+ * tests and other code which may be using `using namespace folly` while also+ * having aliased chrono types.+ */+namespace folly::chrono {++/**+ * Clock interface.+ *+ * Abstraction enables tests to control the current time.+ */+template <typename ClockType>+class Clock {+ public:+  using TimePoint = typename ClockType::time_point;+  Clock() = default;+  virtual ~Clock() = default;++  /**+   * Returns current time.+   */+  [[nodiscard]] virtual TimePoint now() const = 0;+};++/**+ * Implementation of ClockInterface for given std::chrono ClockType.+ */+template <typename ClockType>+class ClockImpl : public Clock<ClockType> {+ public:+  using TimePoint = typename ClockType::time_point;+  ClockImpl() = default;+  ~ClockImpl() override = default;+  [[nodiscard]] TimePoint now() const override { return ClockType::now(); }+};++using SteadyClock = Clock<std::chrono::steady_clock>;+using SteadyClockImpl = ClockImpl<std::chrono::steady_clock>;+using SystemClock = Clock<std::chrono::system_clock>;+using SystemClockImpl = ClockImpl<std::chrono::system_clock>;++} // namespace folly::chrono
+ folly/folly/chrono/Conv.h view
@@ -0,0 +1,712 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++/**+ * Conversions between std::chrono types and POSIX time types.+ *+ * These conversions will fail with a ConversionError if an overflow would+ * occur performing the conversion.  (e.g., if the input value cannot fit in+ * the destination type).  However they allow loss of precision (e.g.,+ * converting nanoseconds to a struct timeval which only has microsecond+ * granularity, or a struct timespec to std::chrono::minutes).+ */++#pragma once++#include <chrono>+#include <limits>+#include <type_traits>++#include <folly/ConstexprMath.h>+#include <folly/Conv.h>+#include <folly/Expected.h>+#include <folly/Utility.h>+#include <folly/portability/SysTime.h>+#include <folly/portability/SysTypes.h>++namespace folly {+namespace detail {++template <typename T>+struct is_duration : std::false_type {};+template <typename Rep, typename Period>+struct is_duration<std::chrono::duration<Rep, Period>> : std::true_type {};+template <typename T>+struct is_time_point : std::false_type {};+template <typename Clock, typename Duration>+struct is_time_point<std::chrono::time_point<Clock, Duration>>+    : std::true_type {};+template <typename T>+struct is_std_chrono_type {+  static constexpr bool value =+      is_duration<T>::value || is_time_point<T>::value;+};+template <typename T>+struct is_posix_time_type {+  static constexpr bool value = std::is_same<T, struct timespec>::value ||+      std::is_same<T, struct timeval>::value;+};+template <typename Tgt, typename Src>+struct is_chrono_conversion {+  static constexpr bool value =+      ((is_std_chrono_type<Tgt>::value && is_posix_time_type<Src>::value) ||+       (is_posix_time_type<Tgt>::value && is_std_chrono_type<Src>::value));+};++/**+ * This converts a number in some input type to time_t while ensuring that it+ * fits in the range of numbers representable by time_t.+ *+ * This is similar to the normal folly::tryTo() behavior when converting+ * arthmetic types to an integer type, except that it does not complain about+ * floating point conversions losing precision.+ */+template <typename Src>+Expected<time_t, ConversionCode> chronoRangeCheck(Src value) {+  static_assert(+      std::is_integral_v<time_t> && std::is_signed_v<time_t>,+      "This function is only implemented for time_t that are signed integrals. Please update it if you need to support a different time_t type.");+  if constexpr (std::is_floating_point_v<Src>) {+    // time_t max converted to a floating point does not have+    // an exact representation.+    // 18446742974197923840 <- Largest float before time_t max+    // 18446744073709549568 <- Largest double before time_t max+    // 18446744073709551615 <- time_t max (when time_t is int64_t)+    // 18446744073709551616 <- next representable float or double.+    // The floating point value that gets chosen depends on the floating point+    // implementation. IEEE arthimetic rounds to nearest.+    static_assert(+        std::numeric_limits<Src>::round_style == std::round_to_nearest,+        "This function is only implemented for IEEE round to nearest. Please update it if you need other round styles.");+    if (value >= static_cast<Src>(std::numeric_limits<time_t>::max())) {+      return makeUnexpected(ConversionCode::POSITIVE_OVERFLOW);+    }+  } else {+    constexpr bool isIntegralWithLargerRange = sizeof(Src) > sizeof(time_t) ||+        (sizeof(Src) == sizeof(time_t) && std::is_unsigned_v<Src>);+    if constexpr (isIntegralWithLargerRange) {+      if (value > static_cast<Src>(std::numeric_limits<time_t>::max())) {+        return makeUnexpected(ConversionCode::POSITIVE_OVERFLOW);+      }+    }+  }+  if (std::is_signed<Src>::value) {+    // int64_t lowest converted to a floating point has+    // has an exact representation because it is a power of 2.+    if (value < std::numeric_limits<time_t>::lowest()) {+      return makeUnexpected(ConversionCode::NEGATIVE_OVERFLOW);+    }+  }++  return static_cast<time_t>(value);+}++/**+ * Convert a std::chrono::duration with second granularity to a pair of+ * (seconds, subseconds)+ *+ * The SubsecondRatio template parameter specifies what type of subseconds to+ * return.  This must have a numerator of 1.+ */+template <typename SubsecondRatio, typename Rep>+static Expected<std::pair<time_t, long>, ConversionCode> durationToPosixTime(+    const std::chrono::duration<Rep, std::ratio<1, 1>>& duration) {+  static_assert(+      SubsecondRatio::num == 1, "subsecond numerator should always be 1");++  auto sec = chronoRangeCheck(duration.count());+  if (sec.hasError()) {+    return makeUnexpected(sec.error());+  }++  time_t secValue = sec.value();+  long subsec = 0L;+  if (std::is_floating_point<Rep>::value) {+    auto fraction = (duration.count() - secValue);+    subsec = static_cast<long>(fraction * SubsecondRatio::den);+    if (duration.count() < 0 && fraction < 0) {+      if (secValue == std::numeric_limits<time_t>::lowest()) {+        return makeUnexpected(ConversionCode::NEGATIVE_OVERFLOW);+      }+      secValue -= 1;+      subsec += SubsecondRatio::den;+    }+  }+  return std::pair<time_t, long>{secValue, subsec};+}++/**+ * Convert a std::chrono::duration with subsecond granularity to a pair of+ * (seconds, subseconds)+ */+template <typename SubsecondRatio, typename Rep, std::intmax_t Denominator>+static Expected<std::pair<time_t, long>, ConversionCode> durationToPosixTime(+    const std::chrono::duration<Rep, std::ratio<1, Denominator>>& duration) {+  static_assert(Denominator != 1, "special case expecting den != 1");+  static_assert(+      SubsecondRatio::num == 1, "subsecond numerator should always be 1");++  auto sec = chronoRangeCheck(duration.count() / Denominator);+  if (sec.hasError()) {+    return makeUnexpected(sec.error());+  }+  auto secTimeT = static_cast<time_t>(sec.value());++  auto remainder = duration.count() - (secTimeT * Denominator);+  long subsec =+      static_cast<long>((remainder * SubsecondRatio::den) / Denominator);+  if (FOLLY_UNLIKELY(duration.count() < 0) && remainder != 0) {+    if (secTimeT == std::numeric_limits<time_t>::lowest()) {+      return makeUnexpected(ConversionCode::NEGATIVE_OVERFLOW);+    }+    secTimeT -= 1;+    subsec += SubsecondRatio::den;+  }++  return std::pair<time_t, long>{secTimeT, subsec};+}++/**+ * Convert a std::chrono::duration with coarser-than-second granularity to a+ * pair of (seconds, subseconds)+ */+template <typename SubsecondRatio, typename Rep, std::intmax_t Numerator>+static Expected<std::pair<time_t, long>, ConversionCode> durationToPosixTime(+    const std::chrono::duration<Rep, std::ratio<Numerator, 1>>& duration) {+  static_assert(Numerator != 1, "special case expecting num!=1");+  static_assert(+      SubsecondRatio::num == 1, "subsecond numerator should always be 1");++  constexpr auto maxValue = std::numeric_limits<time_t>::max() / Numerator;+  constexpr auto minValue = std::numeric_limits<time_t>::lowest() / Numerator;+  if (duration.count() > maxValue) {+    return makeUnexpected(ConversionCode::POSITIVE_OVERFLOW);+  }+  if (duration.count() < minValue) {+    return makeUnexpected(ConversionCode::NEGATIVE_OVERFLOW);+  }++  // Note that we can't use chronoRangeCheck() here since we have to check+  // if (duration.count() * Numerator) would overflow (which we do above).+  auto secOriginalRep = (duration.count() * Numerator);+  auto sec = static_cast<time_t>(secOriginalRep);++  long subsec = 0L;+  if (std::is_floating_point<Rep>::value) {+    auto fraction = secOriginalRep - sec;+    subsec = static_cast<long>(fraction * SubsecondRatio::den);+    if (duration.count() < 0 && fraction < 0) {+      if (sec == std::numeric_limits<time_t>::lowest()) {+        return makeUnexpected(ConversionCode::NEGATIVE_OVERFLOW);+      }+      sec -= 1;+      subsec += SubsecondRatio::den;+    }+  }+  return std::pair<time_t, long>{sec, subsec};+}++/*+ * Helper classes for picking an intermediate duration type to use+ * when doing conversions to/from durations where neither the numerator nor+ * denominator are 1.+ */+template <typename T, bool IsFloatingPoint, bool IsSigned>+struct IntermediateTimeRep {};+template <typename T, bool IsSigned>+struct IntermediateTimeRep<T, true, IsSigned> {+  using type = T;+};+template <typename T>+struct IntermediateTimeRep<T, false, true> {+  using type = intmax_t;+};+template <typename T>+struct IntermediateTimeRep<T, false, false> {+  using type = uintmax_t;+};+// For IntermediateDuration we always use 1 as the numerator, and the original+// Period denominator.  This ensures that we do not lose precision when+// performing the conversion.+template <typename Rep, typename Period>+using IntermediateDuration = std::chrono::duration<+    typename IntermediateTimeRep<+        Rep,+        std::is_floating_point<Rep>::value,+        std::is_signed<Rep>::value>::type,+    std::ratio<1, Period::den>>;++/**+ * Convert a std::chrono::duration to a pair of (seconds, subseconds)+ *+ * This overload is only used for unusual durations where neither the numerator+ * nor denominator are 1.+ */+template <typename SubsecondRatio, typename Rep, typename Period>+Expected<std::pair<time_t, long>, ConversionCode> durationToPosixTime(+    const std::chrono::duration<Rep, Period>& duration) {+  static_assert(Period::num != 1, "should use special-case code when num==1");+  static_assert(Period::den != 1, "should use special-case code when den==1");+  static_assert(+      SubsecondRatio::num == 1, "subsecond numerator should always be 1");++  // Perform this conversion by first converting to a duration where the+  // numerator is 1, then convert to the output type.+  using IntermediateType = IntermediateDuration<Rep, Period>;+  using IntermediateRep = typename IntermediateType::rep;++  // Check to see if we would have overflow converting to the intermediate+  // type.+  constexpr auto maxInput =+      std::numeric_limits<IntermediateRep>::max() / Period::num;+  if (duration.count() > maxInput) {+    return makeUnexpected(ConversionCode::POSITIVE_OVERFLOW);+  }+  constexpr auto minInput =+      std::numeric_limits<IntermediateRep>::min() / Period::num;+  if (duration.count() < minInput) {+    return makeUnexpected(ConversionCode::NEGATIVE_OVERFLOW);+  }+  auto intermediate = IntermediateType{+      static_cast<IntermediateRep>(duration.count()) *+      static_cast<IntermediateRep>(Period::num)};++  return durationToPosixTime<SubsecondRatio>(intermediate);+}++/**+ * Check for overflow when converting to a duration type that is second+ * granularity or finer (e.g., nanoseconds, milliseconds, seconds)+ *+ * This assumes the input is normalized, with subseconds >= 0 and subseconds+ * less than 1 second.+ */+template <bool IsFloatingPoint>+struct CheckOverflowToDuration {+  template <+      typename Tgt,+      typename SubsecondRatio,+      typename Seconds,+      typename Subseconds>+  static ConversionCode check(Seconds seconds, Subseconds subseconds) {+    static_assert(+        Tgt::period::num == 1,+        "this implementation should only be used for subsecond granularity "+        "duration types");+    static_assert(+        !std::is_floating_point<typename Tgt::rep>::value, "incorrect usage");+    static_assert(+        SubsecondRatio::num == 1, "subsecond numerator should always be 1");++    if (FOLLY_LIKELY(seconds >= 0)) {+      constexpr auto maxCount = std::numeric_limits<typename Tgt::rep>::max();+      constexpr auto maxSeconds = maxCount / Tgt::period::den;++      auto unsignedSeconds = to_unsigned(seconds);+      if (FOLLY_LIKELY(unsignedSeconds < maxSeconds)) {+        return ConversionCode::SUCCESS;+      }++      if (FOLLY_UNLIKELY(unsignedSeconds == maxSeconds)) {+        constexpr auto maxRemainder =+            maxCount - (maxSeconds * Tgt::period::den);+        constexpr auto maxSubseconds =+            (maxRemainder * SubsecondRatio::den) / Tgt::period::den;+        if (subseconds <= 0) {+          return ConversionCode::SUCCESS;+        }+        if (to_unsigned(subseconds) <= maxSubseconds) {+          return ConversionCode::SUCCESS;+        }+      }+      return ConversionCode::POSITIVE_OVERFLOW;+    } else if (std::is_unsigned<typename Tgt::rep>::value) {+      return ConversionCode::NEGATIVE_OVERFLOW;+    } else {+      constexpr auto minCount =+          to_signed(std::numeric_limits<typename Tgt::rep>::lowest());+      constexpr auto minSeconds = (minCount / Tgt::period::den);+      if (FOLLY_LIKELY(seconds >= minSeconds)) {+        return ConversionCode::SUCCESS;+      }++      if (FOLLY_UNLIKELY(seconds == minSeconds - 1)) {+        constexpr auto maxRemainder =+            minCount - (minSeconds * Tgt::period::den) + Tgt::period::den;+        constexpr auto maxSubseconds =+            (maxRemainder * SubsecondRatio::den) / Tgt::period::den;+        if (subseconds <= 0) {+          return ConversionCode::NEGATIVE_OVERFLOW;+        }+        if (subseconds >= maxSubseconds) {+          return ConversionCode::SUCCESS;+        }+      }+      return ConversionCode::NEGATIVE_OVERFLOW;+    }+  }+};++template <>+struct CheckOverflowToDuration<true> {+  template <+      typename Tgt,+      typename SubsecondRatio,+      typename Seconds,+      typename Subseconds>+  static ConversionCode check(+      Seconds /* seconds */, Subseconds /* subseconds */) {+    static_assert(+        std::is_floating_point<typename Tgt::rep>::value, "incorrect usage");+    static_assert(+        SubsecondRatio::num == 1, "subsecond numerator should always be 1");++    // We expect floating point types to have much a wider representable range+    // than integer types, so we don't bother actually checking the input+    // integer value here.+    static_assert(+        std::numeric_limits<typename Tgt::rep>::max() >=+            std::numeric_limits<Seconds>::max(),+        "unusually limited floating point type");+    static_assert(+        std::numeric_limits<typename Tgt::rep>::lowest() <=+            std::numeric_limits<Seconds>::lowest(),+        "unusually limited floating point type");++    return ConversionCode::SUCCESS;+  }+};++/**+ * Convert a timeval or a timespec to a std::chrono::duration with second+ * granularity.+ *+ * The SubsecondRatio template parameter specifies what type of subseconds to+ * return.  This must have a numerator of 1.+ *+ * The input must be in normalized form: the subseconds field must be greater+ * than or equal to 0, and less than SubsecondRatio::den (i.e., less than 1+ * second).+ */+template <+    typename SubsecondRatio,+    typename Seconds,+    typename Subseconds,+    typename Rep>+auto posixTimeToDuration(+    Seconds seconds,+    Subseconds subseconds,+    std::chrono::duration<Rep, std::ratio<1, 1>> dummy)+    -> Expected<decltype(dummy), ConversionCode> {+  using Tgt = decltype(dummy);+  static_assert(Tgt::period::num == 1, "special case expecting num==1");+  static_assert(Tgt::period::den == 1, "special case expecting den==1");+  static_assert(+      SubsecondRatio::num == 1, "subsecond numerator should always be 1");++  auto outputSeconds = tryTo<typename Tgt::rep>(seconds);+  if (outputSeconds.hasError()) {+    return makeUnexpected(outputSeconds.error());+  }++  if (std::is_floating_point<typename Tgt::rep>::value) {+    return Tgt{+        typename Tgt::rep(seconds) ++        (typename Tgt::rep(subseconds) / SubsecondRatio::den)};+  }++  // If the value is negative, we have to round up a non-zero subseconds value+  if (FOLLY_UNLIKELY(outputSeconds.value() < 0) && subseconds > 0) {+    if (FOLLY_UNLIKELY(+            outputSeconds.value() ==+            std::numeric_limits<typename Tgt::rep>::lowest())) {+      return makeUnexpected(ConversionCode::NEGATIVE_OVERFLOW);+    }+    return Tgt{outputSeconds.value() + 1};+  }++  return Tgt{outputSeconds.value()};+}++/**+ * Convert a timeval or a timespec to a std::chrono::duration with subsecond+ * granularity+ */+template <+    typename SubsecondRatio,+    typename Seconds,+    typename Subseconds,+    typename Rep,+    std::intmax_t Denominator>+auto posixTimeToDuration(+    Seconds seconds,+    Subseconds subseconds,+    std::chrono::duration<Rep, std::ratio<1, Denominator>> dummy)+    -> Expected<decltype(dummy), ConversionCode> {+  using Tgt = decltype(dummy);+  static_assert(Tgt::period::num == 1, "special case expecting num==1");+  static_assert(Tgt::period::den != 1, "special case expecting den!=1");+  static_assert(+      SubsecondRatio::num == 1, "subsecond numerator should always be 1");++  auto errorCode = detail::CheckOverflowToDuration<+      std::is_floating_point<typename Tgt::rep>::value>::+      template check<Tgt, SubsecondRatio>(seconds, subseconds);+  if (errorCode != ConversionCode::SUCCESS) {+    return makeUnexpected(errorCode);+  }++  if (FOLLY_LIKELY(seconds >= 0)) {+    return std::chrono::duration_cast<Tgt>(+               std::chrono::duration<typename Tgt::rep>{seconds}) ++        std::chrono::duration_cast<Tgt>(+               std::chrono::duration<typename Tgt::rep, SubsecondRatio>{+                   subseconds});+  } else {+    // For negative numbers we have to round subseconds up towards zero, even+    // though it is a positive value, since the overall value is negative.+    return std::chrono::duration_cast<Tgt>(+               std::chrono::duration<typename Tgt::rep>{seconds + 1}) -+        std::chrono::duration_cast<Tgt>(+               std::chrono::duration<typename Tgt::rep, SubsecondRatio>{+                   SubsecondRatio::den - subseconds});+  }+}++/**+ * Convert a timeval or a timespec to a std::chrono::duration with+ * granularity coarser than 1 second.+ */+template <+    typename SubsecondRatio,+    typename Seconds,+    typename Subseconds,+    typename Rep,+    std::intmax_t Numerator>+auto posixTimeToDuration(+    Seconds seconds,+    Subseconds subseconds,+    std::chrono::duration<Rep, std::ratio<Numerator, 1>> dummy)+    -> Expected<decltype(dummy), ConversionCode> {+  using Tgt = decltype(dummy);+  static_assert(Tgt::period::num != 1, "special case expecting num!=1");+  static_assert(Tgt::period::den == 1, "special case expecting den==1");+  static_assert(+      SubsecondRatio::num == 1, "subsecond numerator should always be 1");++  if (FOLLY_UNLIKELY(seconds < 0) && subseconds > 0) {+    // Increment seconds by one to handle truncation of negative numbers+    // properly.+    if (FOLLY_UNLIKELY(seconds == std::numeric_limits<Seconds>::lowest())) {+      return makeUnexpected(ConversionCode::NEGATIVE_OVERFLOW);+    }+    seconds += 1;+  }++  if (std::is_floating_point<typename Tgt::rep>::value) {+    // Convert to the floating point type before performing the division+    return Tgt{static_cast<typename Tgt::rep>(seconds) / Tgt::period::num};+  } else {+    // Perform the division as an integer, and check that the result fits in+    // the output integer type+    auto outputValue = (seconds / Tgt::period::num);+    auto expectedOuput = tryTo<typename Tgt::rep>(outputValue);+    if (expectedOuput.hasError()) {+      return makeUnexpected(expectedOuput.error());+    }++    return Tgt{expectedOuput.value()};+  }+}++/**+ * Convert a timeval or timespec to a std::chrono::duration+ *+ * This overload is only used for unusual durations where neither the numerator+ * nor denominator are 1.+ */+template <+    typename SubsecondRatio,+    typename Seconds,+    typename Subseconds,+    typename Rep,+    std::intmax_t Denominator,+    std::intmax_t Numerator>+auto posixTimeToDuration(+    Seconds seconds,+    Subseconds subseconds,+    std::chrono::duration<Rep, std::ratio<Numerator, Denominator>> dummy)+    -> Expected<decltype(dummy), ConversionCode> {+  using Tgt = decltype(dummy);+  static_assert(+      Tgt::period::num != 1, "should use special-case code when num==1");+  static_assert(+      Tgt::period::den != 1, "should use special-case code when den==1");+  static_assert(+      SubsecondRatio::num == 1, "subsecond numerator should always be 1");++  // Cast through an intermediate type with subsecond granularity.+  // Note that this could fail due to overflow during the initial conversion+  // even if the result is representable in the output POSIX-style types.+  //+  // Note that for integer type conversions going through this intermediate+  // type can result in slight imprecision due to truncating the intermediate+  // calculation to an integer.+  using IntermediateType =+      IntermediateDuration<typename Tgt::rep, typename Tgt::period>;+  auto intermediate = posixTimeToDuration<SubsecondRatio>(+      seconds, subseconds, IntermediateType{});+  if (intermediate.hasError()) {+    return makeUnexpected(intermediate.error());+  }+  // Now convert back to the target duration.  Use tryTo() to confirm that the+  // result fits in the target representation type.+  return tryTo<typename Tgt::rep>(+             intermediate.value().count() / Tgt::period::num)+      .then([](typename Tgt::rep tgt) { return Tgt{tgt}; });+}++template <+    typename Tgt,+    typename SubsecondRatio,+    typename Seconds,+    typename Subseconds>+Expected<Tgt, ConversionCode> tryPosixTimeToDuration(+    Seconds seconds, Subseconds subseconds) {+  static_assert(+      SubsecondRatio::num == 1, "subsecond numerator should always be 1");++  // Normalize the input if required+  if (FOLLY_UNLIKELY(subseconds < 0)) {+    const auto overflowSeconds = (subseconds / SubsecondRatio::den);+    const auto remainder = (subseconds % SubsecondRatio::den);+    if (std::numeric_limits<Seconds>::lowest() + 1 - overflowSeconds >+        seconds) {+      return makeUnexpected(ConversionCode::NEGATIVE_OVERFLOW);+    }+    seconds = seconds - 1 + overflowSeconds;+    subseconds = to_narrow(remainder + SubsecondRatio::den);+  } else if (FOLLY_UNLIKELY(subseconds >= SubsecondRatio::den)) {+    const auto overflowSeconds = (subseconds / SubsecondRatio::den);+    const auto remainder = (subseconds % SubsecondRatio::den);+    if (std::numeric_limits<Seconds>::max() - overflowSeconds < seconds) {+      return makeUnexpected(ConversionCode::POSITIVE_OVERFLOW);+    }+    seconds += overflowSeconds;+    subseconds = to_narrow(remainder);+  }++  return posixTimeToDuration<SubsecondRatio>(seconds, subseconds, Tgt{});+}++} // namespace detail++/**+ * struct timespec to std::chrono::duration+ */+template <typename Tgt>+typename std::enable_if<+    detail::is_duration<Tgt>::value,+    Expected<Tgt, ConversionCode>>::type+tryTo(const struct timespec& ts) {+  return detail::tryPosixTimeToDuration<Tgt, std::nano>(ts.tv_sec, ts.tv_nsec);+}++/**+ * struct timeval to std::chrono::duration+ */+template <typename Tgt>+typename std::enable_if<+    detail::is_duration<Tgt>::value,+    Expected<Tgt, ConversionCode>>::type+tryTo(const struct timeval& tv) {+  return detail::tryPosixTimeToDuration<Tgt, std::micro>(tv.tv_sec, tv.tv_usec);+}++/**+ * timespec or timeval to std::chrono::time_point+ */+template <typename Tgt, typename Src>+typename std::enable_if<+    detail::is_time_point<Tgt>::value && detail::is_posix_time_type<Src>::value,+    Expected<Tgt, ConversionCode>>::type+tryTo(const Src& value) {+  return tryTo<typename Tgt::duration>(value).then(+      [](typename Tgt::duration result) { return Tgt(result); });+}++/**+ * std::chrono::duration to struct timespec+ */+template <typename Tgt, typename Rep, typename Period>+typename std::enable_if<+    std::is_same<Tgt, struct timespec>::value,+    Expected<Tgt, ConversionCode>>::type+tryTo(const std::chrono::duration<Rep, Period>& duration) {+  auto result = detail::durationToPosixTime<std::nano>(duration);+  if (result.hasError()) {+    return makeUnexpected(result.error());+  }++  struct timespec ts;+  ts.tv_sec = result.value().first;+  ts.tv_nsec = result.value().second;+  return ts;+}++/**+ * std::chrono::duration to struct timeval+ */+template <typename Tgt, typename Rep, typename Period>+typename std::enable_if<+    std::is_same<Tgt, struct timeval>::value,+    Expected<Tgt, ConversionCode>>::type+tryTo(const std::chrono::duration<Rep, Period>& duration) {+  auto result = detail::durationToPosixTime<std::micro>(duration);+  if (result.hasError()) {+    return makeUnexpected(result.error());+  }++  struct timeval tv;+  tv.tv_sec = result.value().first;+  tv.tv_usec = result.value().second;+  return tv;+}++/**+ * std::chrono::time_point to timespec or timeval+ */+template <typename Tgt, typename Clock, typename Duration>+typename std::enable_if<+    detail::is_posix_time_type<Tgt>::value,+    Expected<Tgt, ConversionCode>>::type+tryTo(const std::chrono::time_point<Clock, Duration>& timePoint) {+  return tryTo<Tgt>(timePoint.time_since_epoch());+}++/**+ * For all chrono conversions, to() wraps tryTo()+ */+template <typename Tgt, typename Src>+typename std::enable_if<detail::is_chrono_conversion<Tgt, Src>::value, Tgt>::+    type+    to(const Src& value) {+  return tryTo<Tgt>(value).thenOrThrow(+      [](Tgt res) { return res; },+      [&](ConversionCode e) { return makeConversionError(e, StringPiece{}); });+}++} // namespace folly
+ folly/folly/chrono/Hardware.h view
@@ -0,0 +1,155 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <folly/Portability.h>++#include <chrono>+#include <cstdint>++//  Description of and implementation of sequences for precise measurement:+//    https://github.com/abseil/abseil-cpp/blob/20240116.2/absl/random/internal/nanobenchmark.cc#L164-L277++#if defined(_MSC_VER) && !defined(__clang__) && \+    (defined(_M_IX86) || defined(_M_X64))+extern "C" std::uint64_t __rdtsc();+extern "C" std::uint64_t __rdtscp(unsigned int*);+extern "C" void _ReadWriteBarrier();+extern "C" void _mm_lfence();+#pragma intrinsic(__rdtsc)+#pragma intrinsic(__rdtscp)+#pragma intrinsic(_ReadWriteBarrier)+#pragma intrinsic(_mm_lfence)+#endif++namespace folly {++inline std::uint64_t hardware_timestamp() {+#if defined(_MSC_VER) && (defined(_M_IX86) || defined(_M_X64))+  return __rdtsc();+#elif defined(__GNUC__) && (defined(__i386__) || FOLLY_X64)+  return __builtin_ia32_rdtsc();+#elif FOLLY_AARCH64 && !FOLLY_MOBILE+  uint64_t cval;+  asm volatile("mrs %0, cntvct_el0" : "=r"(cval));+  return cval;+#else+  // use steady_clock::now() as an approximation for the timestamp counter on+  // non-x86 systems+  return std::chrono::steady_clock::now().time_since_epoch().count();+#endif+}++/// hardware_timestamp_measurement_start+/// hardware_timestamp_measurement_stop+///+/// Suitable for beginning precise measurement of a region of code.+///+/// Prevents the compiler from reordering instructions across the call. This is+/// in contrast with hardware_timestamp(), which may not prevent the compiler+/// from reordering instructions across the call.+///+/// Prevents the processor from pipelining the call with loads. This is in+/// contrast with hardware_timestamp(), which allows the processor to pipeline+/// the call with surrounding loads.+///+/// Does not prevent instruction pipelines from continuing execution across the+/// call. For example, does not prevent a store issued before the call from+/// continuing execution (becoming globally visible) across the call. This means+/// that this form may not be suitable for certain measurement use-cases which+/// would require such prevention. However, this would be suitable for typical+/// measurement use-cases which do not specifically need to measure background+/// work such as store execution.+std::uint64_t hardware_timestamp_measurement_start() noexcept;+std::uint64_t hardware_timestamp_measurement_stop() noexcept;++inline std::uint64_t hardware_timestamp_measurement_start() noexcept {+#if defined(_MSC_VER) && !defined(__clang__) && \+    (defined(_M_IX86) || defined(_M_X64))+  // msvc does not have embedded assembly+  _ReadWriteBarrier();+  _mm_lfence();+  _ReadWriteBarrier();+  auto const ret = __rdtsc();+  _ReadWriteBarrier();+  _mm_lfence();+  _ReadWriteBarrier();+  return ret;+#elif defined(__GNUC__) && FOLLY_X64+  uint64_t ret = 0;+#if !defined(__clang_major__) || __clang_major__ >= 11+  asm volatile inline(+#else+  asm volatile(+#endif+      "lfence\n"+      "rdtsc\n" // loads 64-bit tsc into edx:eax+      "shl $32, %%rdx\n" // prep rdx for combine into rax+      "or %%rdx, %[ret]\n" // combine rdx into rax+      "lfence\n"+      : [ret] "=a"(ret) // bind ret to rax for output+      : // no inputs+      : "rdx", // rdtsc loads into edx:eax+        "cc", // shl clobbers condition-code+        "memory" // memory clobber asks gcc/clang not to reorder+  );+  return ret;+#else+  // use steady_clock::now() as an approximation for the timestamp counter on+  // non-x64 systems+  return std::chrono::steady_clock::now().time_since_epoch().count();+#endif+}++inline std::uint64_t hardware_timestamp_measurement_stop() noexcept {+#if defined(_MSC_VER) && !defined(__clang__) && \+    (defined(_M_IX86) || defined(_M_X64))+  // msvc does not have embedded assembly+  _ReadWriteBarrier();+  unsigned int aux;+  auto const ret = __rdtscp(&aux);+  _ReadWriteBarrier();+  _mm_lfence();+  _ReadWriteBarrier();+  return ret;+#elif defined(__GNUC__) && FOLLY_X64+  uint64_t ret = 0;+#if !defined(__clang_major__) || __clang_major__ >= 11+  asm volatile inline(+#else+  asm volatile(+#endif+      "rdtscp\n" // loads 64-bit tsc into edx:eax, clobbers ecx+      "shl $32, %%rdx\n" // prep rdx for combine into rax+      "or %%rdx, %[ret]\n" // combine rdx into rax+      "lfence\n"+      : [ret] "=a"(ret) // bind ret to rax for output+      : // no inputs+      : "rdx", // rdtscp loads into edx:eax+        "rcx", // rdtscp clobbers rcx+        "cc", // shl clobbers condition-code+        "memory" // memory clobber asks gcc/clang not to reorder+  );+  return ret;+#else+  // use steady_clock::now() as an approximation for the timestamp counter on+  // non-x64 systems+  return std::chrono::steady_clock::now().time_since_epoch().count();+#endif+}++} // namespace folly
+ folly/folly/cli/NestedCommandLineApp.cpp view
@@ -0,0 +1,359 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#include <folly/cli/NestedCommandLineApp.h>++#include <iostream>++#include <fmt/format.h>+#include <glog/logging.h>++#include <folly/FileUtil.h>+#include <folly/Format.h>+#include <folly/experimental/io/FsUtil.h>++namespace po = ::boost::program_options;++namespace folly {++namespace {++// Guess the program name as basename(executable)+std::string guessProgramName() {+  try {+    return fs::executable_path().filename().string();+  } catch (const std::exception&) {+    return "UNKNOWN";+  }+}++} // namespace++ProgramExit::ProgramExit(int status, const std::string& msg)+    : std::runtime_error(msg), status_(status) {+  // Message is only allowed for non-zero exit status+  CHECK(status_ != 0 || msg.empty());+}++NestedCommandLineApp::NestedCommandLineApp(+    std::string programName,+    std::string version,+    std::string programHeading,+    std::string programHelpFooter,+    InitFunction initFunction)+    : programName_(std::move(programName)),+      programHeading_(std::move(programHeading)),+      programHelpFooter_(std::move(programHelpFooter)),+      version_(std::move(version)),+      globalOptions_("Global options"),+      optionStyle_(po::command_line_style::default_style) {+  addCommand(+      kHelpCommand.str(),+      "[command]",+      "Display help (globally or for a given command)",+      "Displays help (globally or for a given command).",+      [this](+          const po::variables_map& vm, const std::vector<std::string>& args) {+        displayHelp(vm, args);+      });+  builtinCommands_.insert(kHelpCommand);+  addAlias(kShortHelpCommand.str(), kHelpCommand.str());++  addCommand(+      kVersionCommand.str(),+      "[command]",+      "Display version information",+      "Displays version information.",+      [this](const po::variables_map&, const std::vector<std::string>&) {+        displayVersion();+      });+  builtinCommands_.insert(kVersionCommand);++  globalOptions_.add_options()(+      (kHelpCommand.str() + "," + kShortHelpCommand.str()).c_str(),+      "Display help (globally or for a given command)")(+      kVersionCommand.str().c_str(), "Display version information");++  if (initFunction) {+    callbackFunctions_.emplace_back(std::move(initFunction));+  }+}++po::options_description& NestedCommandLineApp::addCommand(+    std::string name,+    std::string argStr,+    std::string shortHelp,+    std::string fullHelp,+    Command command,+    folly::Optional<po::positional_options_description> positionalOptions) {+  CommandInfo info{+      std::move(argStr),+      std::move(shortHelp),+      std::move(fullHelp),+      std::move(command),+      po::options_description(fmt::format("Options for `{}'", name)),+      std::move(positionalOptions)};++  auto p = commands_.emplace(std::move(name), std::move(info));+  CHECK(p.second) << "Command already exists";++  return p.first->second.options;+}++void NestedCommandLineApp::addAlias(std::string newName, std::string oldName) {+  CHECK(aliases_.count(oldName) || commands_.count(oldName))+      << "Alias old name does not exist";+  CHECK(!aliases_.count(newName) && !commands_.count(newName))+      << "Alias new name already exists";+  aliases_.emplace(std::move(newName), std::move(oldName));+}++void NestedCommandLineApp::setOptionStyle(+    boost::program_options::command_line_style::style_t style) {+  optionStyle_ = style;+}++void NestedCommandLineApp::displayHelp(+    const po::variables_map& /* globalOptions */,+    const std::vector<std::string>& args) const {+  if (args.empty()) {+    // General help+    printf(+        "%s\nUsage: %s [global_options...] <command> [command_options...] "+        "[command_args...]\n\n",+        programHeading_.c_str(),+        programName_.c_str());+    std::cout << globalOptions_;+    printf("\nAvailable commands:\n");++    size_t maxLen = 0;+    for (auto& p : commands_) {+      maxLen = std::max(maxLen, p.first.size());+    }+    for (auto& p : aliases_) {+      maxLen = std::max(maxLen, p.first.size());+    }++    for (auto& p : commands_) {+      printf(+          "  %-*s    %s\n",+          int(maxLen),+          p.first.c_str(),+          p.second.shortHelp.c_str());+    }++    if (!aliases_.empty()) {+      printf("\nAvailable aliases:\n");+      for (auto& p : aliases_) {+        printf(+            "  %-*s => %s\n",+            int(maxLen),+            p.first.c_str(),+            resolveAlias(p.second).c_str());+      }+    }+    std::cout << "\n" << programHelpFooter_ << "\n";+  } else {+    // Help for a given command+    auto& p = findCommand(args.front());+    if (p.first != args.front()) {+      printf(+          "`%s' is an alias for `%s'; showing help for `%s'\n",+          args.front().c_str(),+          p.first.c_str(),+          p.first.c_str());+    }+    auto& info = p.second;++    printf(+        "Usage: %s [global_options...] %s%s%s%s\n\n",+        programName_.c_str(),+        p.first.c_str(),+        info.options.options().empty() ? "" : " [command_options...]",+        info.argStr.empty() ? "" : " ",+        info.argStr.c_str());++    printf("%s\n", info.fullHelp.c_str());++    std::cout << globalOptions_;++    if (!info.options.options().empty()) {+      printf("\n");+      std::cout << info.options;+    }+  }+}++void NestedCommandLineApp::displayVersion() const {+  printf("%s %s\n", programName_.c_str(), version_.c_str());+}++const std::string& NestedCommandLineApp::resolveAlias(+    const std::string& name) const {+  auto dest = &name;+  for (;;) {+    auto pos = aliases_.find(*dest);+    if (pos == aliases_.end()) {+      break;+    }+    dest = &pos->second;+  }+  return *dest;+}++auto NestedCommandLineApp::findCommand(const std::string& name) const+    -> const std::pair<const std::string, CommandInfo>& {+  auto pos = commands_.find(resolveAlias(name));+  if (pos == commands_.end()) {+    throw ProgramExit(+        1,+        fmt::format(+            "Command '{}' not found. Run '{} {}' for help.",+            name,+            programName_,+            kHelpCommand));+  }+  return *pos;+}++int NestedCommandLineApp::run(int argc, const char* const argv[]) {+  if (programName_.empty()) {+    programName_ = fs::path(argv[0]).filename().string();+  }+  return run(std::vector<std::string>(argv + 1, argv + argc));+}++int NestedCommandLineApp::run(const std::vector<std::string>& args) {+  int status;+  try {+    doRun(args);+    status = 0;+  } catch (const ProgramExit& ex) {+    if (ex.what()[0]) { // if not empty+      fprintf(stderr, "%s\n", ex.what());+    }+    status = ex.status();+  } catch (const po::error& ex) {+    fprintf(+        stderr,+        "%s",+        fmt::format(+            "{}. Run '{} help' for {}.\n",+            ex.what(),+            programName_,+            kHelpCommand)+            .c_str());+    status = 1;+  }++  if (status == 0) {+    if (ferror(stdout)) {+      fprintf(stderr, "error on standard output\n");+      status = 1;+    } else if (fflush(stdout)) {+      fprintf(+          stderr,+          "standard output flush failed: %s\n",+          errnoStr(errno).c_str());+      status = 1;+    }+  }++  return status;+}++void NestedCommandLineApp::doRun(const std::vector<std::string>& args) {+  if (programName_.empty()) {+    programName_ = guessProgramName();+  }++  bool not_clean = false;+  std::vector<std::string> cleanArgs;+  std::vector<std::string> endArgs;++  for (auto& na : args) {+    if (not_clean) {+      endArgs.push_back(na);+    } else if (na == "--") {+      not_clean = true;+    } else {+      cleanArgs.push_back(na);+    }+  }++  auto parsed = parseNestedCommandLine(cleanArgs, globalOptions_, optionStyle_);+  po::variables_map vm;+  po::store(parsed.options, vm);+  if (vm.count(kHelpCommand.str())) {+    std::vector<std::string> helpArgs;+    if (parsed.command) {+      helpArgs.push_back(*parsed.command);+    }+    displayHelp(vm, helpArgs);+    return;+  }++  if (vm.count(kVersionCommand.str())) {+    displayVersion();+    return;+  }++  if (!parsed.command) {+    throw ProgramExit(+        1,+        fmt::format(+            "Command not specified. Run '{} {}' for help.",+            programName_,+            kHelpCommand));+  }++  auto& p = findCommand(*parsed.command);+  auto& cmd = p.first;+  auto& info = p.second;++  auto parser =+      po::command_line_parser(parsed.rest)+          .options(info.options)+          .style(optionStyle_);+  if (info.positionalOptions) {+    parser = parser.positional(*info.positionalOptions);+  }++  auto cmdOptions = parser.run();++  po::store(cmdOptions, vm);+  po::notify(vm);++  // If positional arguments are specified they should get mapped to a named arg+  // and don't need to be double collected+  auto cmdArgs = po::collect_unrecognized(+      cmdOptions.options,+      info.positionalOptions ? po::exclude_positional : po::include_positional);++  cmdArgs.insert(cmdArgs.end(), endArgs.begin(), endArgs.end());++  for (const auto& callback : callbackFunctions_) {+    callback(cmd, vm, cmdArgs);+  }++  info.command(vm, cmdArgs);+}++bool NestedCommandLineApp::isBuiltinCommand(const std::string& name) const {+  return builtinCommands_.count(name);+}++} // namespace folly
+ folly/folly/cli/NestedCommandLineApp.h view
@@ -0,0 +1,211 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <functional>+#include <set>+#include <stdexcept>++#include <folly/CPortability.h>+#include <folly/String.h>+#include <folly/cli/ProgramOptions.h>++namespace folly {++/**+ * Exception that commands may throw to force the program to exit cleanly+ * with a given exit code. NestedCommandLineApp::run() catches this and+ * makes run() print the given message on stderr (followed by a newline, unless+ * empty; the message is only allowed when exiting with a non-zero status), and+ * return the exit code. (Other exceptions will propagate out of run())+ */+class FOLLY_EXPORT ProgramExit : public std::runtime_error {+ public:+  explicit ProgramExit(int status, const std::string& msg = std::string());+  int status() const { return status_; }++ private:+  int status_;+};++/**+ * App that uses a nested command line, of the form:+ *+ * program [--global_options...] command [--command_options...] command_args...+ *+ * Note: Global options (including GFlags, if added using addGFlags()) are+ * recognized anywhere in the command line, and are prefix matched with higher+ * priority than command options. For example, a global option named "--foobar"+ * would be matched over a command option named "--foo", even if you specify+ * "--foo" on the command line. You can disable prefix matching with:+ *+ * int style = boost::program_options::command_line_style::default_style;+ * style &= ~boost::program_options::command_line_style::allow_guessing;+ * app.setOptionStyle(+ *   static_cast<boost::program_options::command_line_style::style_t>(style));+ */+class NestedCommandLineApp {+ public:+  typedef std::function<void(+      const std::string& command,+      const boost::program_options::variables_map& options,+      const std::vector<std::string>& args)>+      InitFunction;++  typedef std::function<void(+      const boost::program_options::variables_map& options,+      const std::vector<std::string>&)>+      Command;++  struct CommandInfo {+    std::string argStr;+    std::string shortHelp;+    std::string fullHelp;+    Command command;+    boost::program_options::options_description options;+    folly::Optional<boost::program_options::positional_options_description>+        positionalOptions;+  };++  static constexpr StringPiece const kHelpCommand = "help";+  static constexpr StringPiece const kShortHelpCommand = "h";+  static constexpr StringPiece const kVersionCommand = "version";+  /**+   * Initialize the app.+   *+   * If programName is not set, we try to guess (readlink("/proc/self/exe")).+   *+   * version is the version string printed when given the --version flag.+   *+   * initFunction, if specified, is called after parsing the command line,+   * right before executing the command.+   */+  explicit NestedCommandLineApp(+      std::string programName = std::string(),+      std::string version = std::string(),+      std::string programHeading = std::string(),+      std::string programHelpFooter = std::string(),+      InitFunction initFunction = InitFunction());++#if FOLLY_HAVE_LIBGFLAGS && __has_include(<gflags/gflags.h>)+  /**+   * Add GFlags to the list of supported options with the given style.+   */+  void addGFlags(ProgramOptionsStyle style = ProgramOptionsStyle::GNU) {+    globalOptions_.add(getGFlags(style));+  }+#endif++  /**+   * Return the global options object, so you can add options.+   */+  boost::program_options::options_description& globalOptions() {+    return globalOptions_;+  }++  /*+   * Return the commands map, so you can see the registered commands and get+   * access to their respective options descriptions.+   */+  const std::map<std::string, CommandInfo>& commands() { return commands_; }++  /**+   * Add a command.+   *+   * name:  command name+   * argStr: description of arguments in help strings+   *   (<filename> <N>)+   * shortHelp: one-line summary help string+   * fullHelp: full help string+   * command: function to run+   *+   * Returns a reference to the options_description object that you can+   * use to add options for this command.+   */+  boost::program_options::options_description& addCommand(+      std::string name,+      std::string argStr,+      std::string shortHelp,+      std::string fullHelp,+      Command command,+      folly::Optional<boost::program_options::positional_options_description>+          positionalOptions = folly::none);++  /**+   * Add an alias; running the command newName will have the same effect+   * as running oldName.+   */+  void addAlias(std::string newName, std::string oldName);++  /**+   * Sets the style in which options will be accepted by the parser.+   */+  void setOptionStyle(+      boost::program_options::command_line_style::style_t style);++  /**+   * Run the command and return; the return code is 0 on success or+   * non-zero on error, so it is idiomatic to call this at the end of main():+   * return app.run(argc, argv);+   *+   * On successful exit, run() will check for errors on stdout (and flush+   * it) to help command-line applications that need to write to stdout+   * (failing to write to stdout is an error). If there is an error on stdout,+   * we'll print a helpful message on stderr and return an error status (1).+   */+  int run(int argc, const char* const argv[]);+  int run(const std::vector<std::string>& args);++  /**+   * Return true if name represent known built-in command (help, version)+   */+  bool isBuiltinCommand(const std::string& name) const;++  /**+   * Add a callback to be invoked after command-line arguments are parsed+   */+  void addCallback(InitFunction callback) {+    callbackFunctions_.emplace_back(std::move(callback));+  }++ private:+  void doRun(const std::vector<std::string>& args);++  const std::string& resolveAlias(const std::string& name) const;++  const std::pair<const std::string, CommandInfo>& findCommand(+      const std::string& name) const;++  void displayHelp(+      const boost::program_options::variables_map& options,+      const std::vector<std::string>& args) const;++  void displayVersion() const;++  std::string programName_;+  std::string programHeading_;+  std::string programHelpFooter_;+  std::string version_;+  std::vector<InitFunction> callbackFunctions_;+  boost::program_options::options_description globalOptions_;+  boost::program_options::command_line_style::style_t optionStyle_;+  std::map<std::string, CommandInfo> commands_;+  std::map<std::string, std::string> aliases_;+  std::set<folly::StringPiece> builtinCommands_;+};++} // namespace folly
+ folly/folly/cli/ProgramOptions.cpp view
@@ -0,0 +1,346 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#include <folly/cli/ProgramOptions.h>++#include <unordered_map>+#include <unordered_set>++#include <boost/version.hpp>+#include <glog/logging.h>++#ifdef __ANDROID__+#include <gflags/gflags.h>+#endif++#include <folly/Conv.h>+#include <folly/Portability.h>+#include <folly/portability/GFlags.h>++namespace po = ::boost::program_options;++namespace folly {++#if FOLLY_HAVE_LIBGFLAGS && __has_include(<gflags/gflags.h>)+namespace {++// Information about one GFlag. Handled via shared_ptr, as, in the case+// of boolean flags, two boost::program_options options (--foo and --nofoo)+// may share the same GFlag underneath.+//+// We're slightly abusing the boost::program_options interface; the first+// time we (successfully) parse a value that matches this GFlag, we'll set+// it and remember not to set it again; this prevents, for example, the+// default value of --foo from overwriting the GFlag if --nofoo is set.+template <class T>+class GFlagInfo {+ public:+  explicit GFlagInfo(folly::gflags::CommandLineFlagInfo info)+      : info_(std::move(info)), isSet_(false) {}++  void set(const T& value) {+    if (isSet_) {+      return;+    }++    auto strValue = folly::to<std::string>(value);+    auto msg = folly::gflags::SetCommandLineOption(+        info_.name.c_str(), strValue.c_str());+    if (msg.empty()) {+      throw po::invalid_option_value(strValue);+    }+    isSet_ = true;+  }++  T get() const {+    std::string str;+    CHECK(folly::gflags::GetCommandLineOption(info_.name.c_str(), &str));+    return folly::to<T>(str);+  }++  const folly::gflags::CommandLineFlagInfo& info() const { return info_; }++ private:+  folly::gflags::CommandLineFlagInfo info_;+  bool isSet_;+};++template <class T>+class GFlagValueSemanticBase : public po::value_semantic {+ public:+  explicit GFlagValueSemanticBase(std::shared_ptr<GFlagInfo<T>> info)+      : info_(std::move(info)) {}++  std::string name() const override { return "arg"; }+#if BOOST_VERSION >= 105900 && BOOST_VERSION <= 106400+  bool adjacent_tokens_only() const override { return false; }+#endif+  bool is_composing() const override { return false; }+  bool is_required() const override { return false; }+  // We handle setting the GFlags from parse(), so notify() does nothing.+  void notify(const boost::any& /* valueStore */) const override {}+  bool apply_default(boost::any& valueStore) const override {+    // We're using the *current* rather than *default* value here, and+    // this is intentional; GFlags-using programs assign to FLAGS_foo+    // before ParseCommandLineFlags() in order to change the default value,+    // and we obey that.+    auto val = info_->get();+    this->transform(val);+    valueStore = val;+    return true;+  }++  void parse(+      boost::any& valueStore,+      const std::vector<std::string>& tokens,+      bool /* utf8 */) const override;++ private:+  virtual T parseValue(const std::vector<std::string>& tokens) const = 0;+  virtual void transform(T& /* val */) const {}++  mutable std::shared_ptr<GFlagInfo<T>> info_;+};++template <class T>+void GFlagValueSemanticBase<T>::parse(+    boost::any& valueStore,+    const std::vector<std::string>& tokens,+    bool /* utf8 */) const {+  T val;+  try {+    val = this->parseValue(tokens);+    this->transform(val);+  } catch (const std::exception&) {+    throw po::invalid_option_value(+        tokens.empty() ? std::string() : tokens.front());+  }+  this->info_->set(val);+  valueStore = val;+}++template <class T>+class GFlagValueSemantic : public GFlagValueSemanticBase<T> {+ public:+  explicit GFlagValueSemantic(std::shared_ptr<GFlagInfo<T>> info)+      : GFlagValueSemanticBase<T>(std::move(info)) {}++  unsigned min_tokens() const override { return 1; }+  unsigned max_tokens() const override { return 1; }++  T parseValue(const std::vector<std::string>& tokens) const override {+    DCHECK(tokens.size() == 1);+    return folly::to<T>(tokens.front());+  }+};++class BoolGFlagValueSemantic : public GFlagValueSemanticBase<bool> {+ public:+  explicit BoolGFlagValueSemantic(std::shared_ptr<GFlagInfo<bool>> info)+      : GFlagValueSemanticBase<bool>(std::move(info)) {}++  unsigned min_tokens() const override { return 0; }+  unsigned max_tokens() const override { return 0; }++  bool parseValue(const std::vector<std::string>& tokens) const override {+    DCHECK(tokens.empty());+    return true;+  }+};++class NegativeBoolGFlagValueSemantic : public BoolGFlagValueSemantic {+ public:+  explicit NegativeBoolGFlagValueSemantic(std::shared_ptr<GFlagInfo<bool>> info)+      : BoolGFlagValueSemantic(std::move(info)) {}++ private:+  void transform(bool& val) const override { val = !val; }+};++const std::string& getName(const std::string& name) {+  static const std::unordered_map<std::string, std::string> gFlagOverrides{+      // Allow -v in addition to --v+      {"v", "v,v"},+  };+  auto pos = gFlagOverrides.find(name);+  return pos != gFlagOverrides.end() ? pos->second : name;+}++template <class T>+void addGFlag(+    folly::gflags::CommandLineFlagInfo&& flag,+    po::options_description& desc,+    ProgramOptionsStyle style) {+  auto gflagInfo = std::make_shared<GFlagInfo<T>>(std::move(flag));+  auto& info = gflagInfo->info();+  auto name = getName(info.name);++  switch (style) {+    case ProgramOptionsStyle::GFLAGS:+      break;+    case ProgramOptionsStyle::GNU:+      std::replace(name.begin(), name.end(), '_', '-');+      break;+  }+  desc.add_options()(+      name.c_str(),+      new GFlagValueSemantic<T>(gflagInfo),+      info.description.c_str());+}++template <>+void addGFlag<bool>(+    folly::gflags::CommandLineFlagInfo&& flag,+    po::options_description& desc,+    ProgramOptionsStyle style) {+  auto gflagInfo = std::make_shared<GFlagInfo<bool>>(std::move(flag));+  auto& info = gflagInfo->info();+  auto name = getName(info.name);+  std::string negationPrefix;++  switch (style) {+    case ProgramOptionsStyle::GFLAGS:+      negationPrefix = "no";+      break;+    case ProgramOptionsStyle::GNU:+      std::replace(name.begin(), name.end(), '_', '-');+      negationPrefix = "no-";+      break;+  }++  // clang-format off+  desc.add_options()+    (name.c_str(),+     new BoolGFlagValueSemantic(gflagInfo),+     info.description.c_str())+    ((negationPrefix + name).c_str(),+     new NegativeBoolGFlagValueSemantic(gflagInfo),+     folly::to<std::string>("(no) ", info.description).c_str());+  // clang-format on+}++typedef void (*FlagAdder)(+    folly::gflags::CommandLineFlagInfo&&,+    po::options_description&,+    ProgramOptionsStyle);++const std::unordered_map<std::string, FlagAdder> gFlagAdders = {+#define X(NAME, TYPE) \+  { NAME, addGFlag<TYPE> }+    X("bool", bool),+    X("int32", int32_t),+    X("int64", int64_t),+    X("uint32", uint32_t),+    X("uint64", uint64_t),+    X("double", double),+    X("string", std::string),+#undef X+};++} // namespace++po::options_description getGFlags(ProgramOptionsStyle style) {+  static const std::unordered_set<std::string> gSkipFlags{+      "flagfile",+      "fromenv",+      "tryfromenv",+      "undefok",+      "help",+      "helpful",+      "helpshort",+      "helpon",+      "helpmatch",+      "helppackage",+      "helpxml",+      "version",+      "tab_completion_columns",+      "tab_completion_word",+  };++  po::options_description desc("GFlags");++  std::vector<folly::gflags::CommandLineFlagInfo> allFlags;+  folly::gflags::GetAllFlags(&allFlags);++  for (auto& f : allFlags) {+    if (gSkipFlags.count(f.name)) {+      continue;+    }+    auto pos = gFlagAdders.find(f.type);+    CHECK(pos != gFlagAdders.end()) << "Invalid flag type: " << f.type;+    (*pos->second)(std::move(f), desc, style);+  }++  return desc;+}+#endif++namespace {++NestedCommandLineParseResult doParseNestedCommandLine(+    po::command_line_parser&& parser,+    const po::options_description& desc,+    boost::program_options::command_line_style::style_t style) {+  NestedCommandLineParseResult result;++  result.options = parser.options(desc).style(style).allow_unregistered().run();++  bool setCommand = true;+  for (auto& opt : result.options.options) {+    auto& tokens = opt.original_tokens;+    auto tokensStart = tokens.begin();++    if (setCommand && opt.position_key != -1) {+      DCHECK(tokensStart != tokens.end());+      result.command = *(tokensStart++);+    }++    if (opt.position_key != -1 || opt.unregistered) {+      // If we see an unrecognized option before the first positional+      // argument, assume we don't have a valid command name, because+      // we don't know how to parse it otherwise.+      //+      // program --wtf foo bar+      //+      // Is "foo" an argument to "--wtf", or the command name?+      setCommand = false;+      result.rest.insert(result.rest.end(), tokensStart, tokens.end());+    }+  }++  return result;+}++} // namespace++NestedCommandLineParseResult parseNestedCommandLine(+    int argc,+    const char* const argv[],+    const po::options_description& desc,+    boost::program_options::command_line_style::style_t style) {+  return doParseNestedCommandLine(+      po::command_line_parser(argc, argv), desc, style);+}++NestedCommandLineParseResult parseNestedCommandLine(+    const std::vector<std::string>& cmdline,+    const po::options_description& desc,+    boost::program_options::command_line_style::style_t style) {+  return doParseNestedCommandLine(+      po::command_line_parser(cmdline), desc, style);+}++} // namespace folly
+ folly/folly/cli/ProgramOptions.h view
@@ -0,0 +1,92 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <boost/program_options.hpp>++#include <folly/Optional.h>+#include <folly/portability/GFlags.h>++namespace folly {++#if FOLLY_HAVE_LIBGFLAGS && __has_include(<gflags/gflags.h>)+enum class ProgramOptionsStyle {+  GFLAGS,+  GNU,+};++// Add all GFlags to the given options_description.+// Use this *instead of* gflags::ParseCommandLineFlags().+//+// in GFLAGS style, the flags are named as per gflags conventions:+//   names_with_underscores+//   boolean flags have a "no" prefix+//+// in GNU style, the flags are named as per GNU conventions:+//   names-with-dashes+//   boolean flags have a "no-" prefix+//+// Consider (for example) a boolean flag:+//   DEFINE_bool(flying_pigs, false, "...");+//+// In GFLAGS style, the corresponding flags are named+//   flying_pigs+//   noflying_pigs+//+// In GNU style, the corresponding flags are named+//   flying-pigs+//   no-flying-pigs+//+// You may not pass arguments to boolean flags, so you must use the+// "no" / "no-" prefix to set them to false; "--flying_pigs false"+// and "--flying_pigs=false" are not allowed, to prevent ambiguity.+boost::program_options::options_description getGFlags(+    ProgramOptionsStyle style = ProgramOptionsStyle::GNU);+#endif++// Helper when parsing nested command lines:+//+// program [--common_options...] command [--command_options...] args+//+// The result has "command" set to the first positional argument, if any,+// and "rest" set to the remaining options and arguments. Note that any+// unrecognized flags must appear after the command name.+//+// You may pass "rest" to parseNestedCommandLine again, etc.+struct NestedCommandLineParseResult {+  NestedCommandLineParseResult() {}++  boost::program_options::parsed_options options{nullptr};++  Optional<std::string> command;+  std::vector<std::string> rest;+};++NestedCommandLineParseResult parseNestedCommandLine(+    int argc,+    const char* const argv[],+    const boost::program_options::options_description& desc,+    boost::program_options::command_line_style::style_t style =+        boost::program_options::command_line_style::default_style);++NestedCommandLineParseResult parseNestedCommandLine(+    const std::vector<std::string>& cmdline,+    const boost::program_options::options_description& desc,+    boost::program_options::command_line_style::style_t style =+        boost::program_options::command_line_style::default_style);++} // namespace folly
+ folly/folly/codec/Uuid.h view
@@ -0,0 +1,358 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <array>+#include <cstring>+#include <string>+#include <string_view>++#include <folly/Portability.h>+#include <folly/memory/UninitializedMemoryHacks.h>++#if FOLLY_X64+#include <immintrin.h>+#endif++namespace folly {++enum class [[nodiscard]] UuidParseCode : unsigned char {+  SUCCESS,+  WRONG_LENGTH,+  INVALID_CHAR,+};++namespace detail {+template <auto buffer_to_buffer_func>+FOLLY_ALWAYS_INLINE UuidParseCode+uuid_parse_generic(std::string& out, std::string_view s) {+  if (s.size() != 36) {+    return UuidParseCode::WRONG_LENGTH;+  }+  folly::resizeWithoutInitialization(out, 16);+  return buffer_to_buffer_func(out.data(), s.data());+}++#if FOLLY_X64 && defined(__AVX2__)++// given a register full of hexadecimal digits (0-9, a-f, A-F),+// compute a register where within each 16-byte lane, the first 8 bytes+// have the values of the parsed 2-digit spans in the input.+//+// clang-format off+// example:+// in =  [790455cb98134298][87ec9ede1dc38e10] (ascii string starting with "7904")+// out = [0x79, 0x04, 0x55, 0xcb, 0x98, 0x13, 0x42, 0x98, _, _, _, _, _, _, _, _]+//       [0x87, 0xec, 0x9e, 0xde, 0x1d, 0xc3, 0x8e, 0x10, _, _, _, _, _, _, _, _]+// where each '_' represents some arbitrary value+// clang-format on+FOLLY_ALWAYS_INLINE __m256i+uuid_parse_parse_hex_without_validation_avx2(const __m256i in) {+  const __m256i mask = _mm256_set1_epi8(0xf0);+  const __m256i hi_nibble = _mm256_srli_epi16(_mm256_and_si256(in, mask), 4);+  // 0-9 are 0x30-0x39, so we should subtract '0'+  // A-F are 0x41-0x46, so we should subtract ('A' - 10)+  // a-f are 0x61-0x66, so we should subtract ('a' - 10)+  // clang-format off+  const __m256i hi_nibble_to_offset = _mm256_setr_epi8(+    0, 0, 0, '0', ('A' - 10), 0, ('a' - 10), 0, 0, 0, 0, 0, 0, 0, 0, 0,+    0, 0, 0, '0', ('A' - 10), 0, ('a' - 10), 0, 0, 0, 0, 0, 0, 0, 0, 0);+  // clang-format on+  const __m256i per_byte_offset =+      _mm256_shuffle_epi8(hi_nibble_to_offset, hi_nibble);+  const __m256i digits = _mm256_sub_epi8(in, per_byte_offset);++  // shift left 12 bits+  // that is, 0x01 0x02 0x03 0x04 -> 0x00 0x10 0x00 0x30+  const __m256i out_hi_nibbles = _mm256_slli_epi16(digits, 12);+  // 0x01 0x12 0x03 0x34 etc.+  const __m256i out_combined = _mm256_or_si256(out_hi_nibbles, digits);+  // clang-format off+  const __m256i pack_odd = _mm256_setr_epi8(+    1, 3, 5, 7, 9, 11, 13, 15, -1, -1, -1, -1, -1, -1, -1, -1,+    1, 3, 5, 7, 9, 11, 13, 15, -1, -1, -1, -1, -1, -1, -1, -1);+  // clang-format on+  // 0x12 0x34 0x56 0x78 etc.+  return _mm256_shuffle_epi8(out_combined, pack_odd);+}++// returns a bitmask of which input lanes were actually hex digits+FOLLY_ALWAYS_INLINE unsigned int uuid_parse_validate_hex_avx2(+    const __m256i in) {+  const __m256i mask = _mm256_set1_epi8(0xf0);+  const __m256i hi_nibble = _mm256_srli_epi16(_mm256_and_si256(in, mask), 4);+  // We're using a technique based on "Special case 1 — small sets" from+  // http://0x80.pl/notesen/2018-10-18-simd-byte-lookup.html#special-case-1-small-sets+  // The technique is: actually this special case isn't for set of up to 8+  // values. It's for a union U of up to 8 sets S_i where each set+  // S_i = {all values with lo nibble in some set A and hi nibble in some set B}+  // In our case U={0-9a-fA-F}, S_1={0x3}x{0x0-0x9}, S_2={0x4,0x6}x{0x1-0x6}+  //+  // Also, it's not necessary to extract the lower nibbles because none of the+  // values we are looking for have the high bit set.+  //+  // clang-format off+  const __m256i lo_nibble_lookup = _mm256_setr_epi8(+    1, 3, 3, 3, 3, 3, 3, 1, 1, 1, 0, 0, 0, 0, 0, 0,+    1, 3, 3, 3, 3, 3, 3, 1, 1, 1, 0, 0, 0, 0, 0, 0);+  const __m256i hi_nibble_lookup = _mm256_setr_epi8(+    0, 0, 0, 1, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0,+    0, 0, 0, 1, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0);+  // clang-format on++  const __m256i hi_mask = _mm256_shuffle_epi8(hi_nibble_lookup, hi_nibble);+  const __m256i lo_mask = _mm256_shuffle_epi8(lo_nibble_lookup, in);+  const __m256i valid_input = _mm256_cmpgt_epi8(+      _mm256_and_si256(lo_mask, hi_mask), _mm256_set1_epi8(0));+  return _mm256_movemask_epi8(valid_input);+}++FOLLY_ALWAYS_INLINE UuidParseCode+uuid_parse_buffer_to_buffer_avx2(char* out, const char* s) {+  // clang-format off+  // read the 36-byte input into two 32-byte values+  // a = [790455cb-9813-42][98-87ec-9ede1dc3]+  // b =     [55cb-9813-4298-8][7ec-9ede1dc38e10]+  // clang-format on+  const __m256i a = _mm256_loadu_si256((const __m256i_u*)(s + 0));+  const __m256i b = _mm256_loadu_si256((const __m256i_u*)(s + 4));++  // clang-format off+  // merge the values into one, discarding the positions where we expect dashes+  const __m256i shuffle_a = _mm256_setr_epi8(+    0,  1,  2,  3,  4,  5,  6,  7,  9, 10, 11, 12, 14, 15, -1, -1,+    3,  4,  5,  6,  8,  9, 10, 11, 12, 13, 14, 15, -1, -1, -1, -1);+  const __m256i shuffle_b = _mm256_setr_epi8(+   -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 12, 13,+   -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 12, 13, 14, 15);+  // a  = [11111111-2222-33][44-5555-66666666]+  // b  =     [1111-2222-3344-5][555-666666667777]+  // sa = [11111111222233__][555566666666____]+  // sb = [______________44][____________7777]+  // c  = [1111111122223344][5555666666667777]+  // clang-format on+  const __m256i sa = _mm256_shuffle_epi8(a, shuffle_a);+  const __m256i sb = _mm256_shuffle_epi8(b, shuffle_b);+  const __m256i c = _mm256_or_si256(sa, sb);++  // check that the dashes are in the right places in the input+  const unsigned int my_dashes_mask =+      _mm256_movemask_epi8(_mm256_cmpeq_epi8(a, _mm256_set1_epi8('-')));+  const unsigned int desired_non_dashes_mask =+      0b11111111011110111101111011111111u;+  // dashes_mask should be all ones!+  const unsigned int dashes_mask = my_dashes_mask ^ desired_non_dashes_mask;++  const __m256i ret = uuid_parse_parse_hex_without_validation_avx2(c);+  const unsigned int valid_hex_mask = uuid_parse_validate_hex_avx2(c);+  if (~(dashes_mask & valid_hex_mask)) {+    return UuidParseCode::INVALID_CHAR;+  }++  const unsigned long long retA = _mm256_extract_epi64(ret, 0);+  const unsigned long long retB = _mm256_extract_epi64(ret, 2);+  std::memcpy(out + 0, &retA, 8);+  std::memcpy(out + 8, &retB, 8);++  return UuidParseCode::SUCCESS;+}++inline UuidParseCode uuid_parse_avx2(std::string& out, std::string_view s) {+  return uuid_parse_generic<uuid_parse_buffer_to_buffer_avx2>(out, s);+}++#endif // FOLLY_X64 && defined(__AVX2__)++#if FOLLY_X64 && defined(__SSSE3__)++FOLLY_ALWAYS_INLINE __m128i+uuid_parse_parse_hex_without_validation_ssse3(const __m128i in) {+  const __m128i mask = _mm_set1_epi8(0xf0);+  const __m128i hi_nibble = _mm_srli_epi16(_mm_and_si128(in, mask), 4);+  // clang-format off+  const __m128i hi_nibble_to_offset = _mm_setr_epi8(+    0, 0, 0, '0', ('A' - 10), 0, ('a' - 10), 0, 0, 0, 0, 0, 0, 0, 0, 0);+  // clang-format on+  const __m128i per_byte_offset =+      _mm_shuffle_epi8(hi_nibble_to_offset, hi_nibble);+  const __m128i digits = _mm_sub_epi8(in, per_byte_offset);++  const __m128i out_hi_nibbles = _mm_slli_epi16(digits, 12);+  const __m128i out_combined = _mm_or_si128(out_hi_nibbles, digits);+  // clang-format off+  const __m128i pack_odd = _mm_setr_epi8(+    1, 3, 5, 7, 9, 11, 13, 15, -1, -1, -1, -1, -1, -1, -1, -1);+  // clang-format on+  return _mm_shuffle_epi8(out_combined, pack_odd);+}++FOLLY_ALWAYS_INLINE unsigned int uuid_parse_validate_hex_ssse3(+    const __m128i in) {+  const __m128i mask = _mm_set1_epi8(0xf0);+  const __m128i hi_nibble = _mm_srli_epi16(_mm_and_si128(in, mask), 4);+  // clang-format off+  const __m128i lo_nibble_lookup = _mm_setr_epi8(+    1, 3, 3, 3, 3, 3, 3, 1, 1, 1, 0, 0, 0, 0, 0, 0);+  const __m128i hi_nibble_lookup = _mm_setr_epi8(+    0, 0, 0, 1, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0);+  // clang-format on++  const __m128i hi_mask = _mm_shuffle_epi8(hi_nibble_lookup, hi_nibble);+  const __m128i lo_mask = _mm_shuffle_epi8(lo_nibble_lookup, in);+  const __m128i valid_input =+      _mm_cmpgt_epi8(_mm_and_si128(lo_mask, hi_mask), _mm_set1_epi8(0));+  return _mm_movemask_epi8(valid_input);+}++FOLLY_ALWAYS_INLINE UuidParseCode+uuid_parse_buffer_to_buffer_ssse3(char* out, const char* s) {+  // clang-format off+  // read the 36-byte input into three 16-byte values+  // a = [01234567-89ab-cd]+  // b =         [-89ab-cdef-0123-]+  // c =                     [123-456789abcdef]+  // clang-format on+  const __m128i a = _mm_loadu_si128((const __m128i_u*)(s + 0));+  const __m128i b = _mm_loadu_si128((const __m128i_u*)(s + 8));+  const __m128i c = _mm_loadu_si128((const __m128i_u*)(s + 20));++  // clang-format off+  // merge the values into one, discarding the positions where we expect dashes+  const __m128i shuffle_b = _mm_setr_epi8(+   11, 12, 13, 14, -1, -1, -1, -1, 1, 2, 3, 4, 6, 7, 8, 9);+  // a  = [11111111-2222-33]+  // b  =         [-2222-3344-5666-]+  // c  =                     [666-777777777777]+  // sb =             [5666____22223344]+  // ascii_digits_a = [1111111122223344]+  // ascii_digits_b = [5666777777777777]+  // clang-format on+  const __m128i sb = _mm_shuffle_epi8(b, shuffle_b);+  const __m128i ascii_digits_a = _mm_castpd_si128( // no _mm_blend_epi64+      _mm_blend_pd(_mm_castsi128_pd(a), _mm_castsi128_pd(sb), 0b10));+  const __m128i ascii_digits_b = _mm_castps_si128( // _mm_blend_epi32 is AVX2+      _mm_blend_ps(_mm_castsi128_ps(c), _mm_castsi128_ps(sb), 0b0001));++  // check that the dashes are in the right places in the input+  const unsigned int my_dashes_mask =+      _mm_movemask_epi8(_mm_cmpeq_epi8(b, _mm_set1_epi8('-')));+  const unsigned int desired_non_dashes_mask =+      0b11111111111111110111101111011110u;+  // dashes_mask should be all ones!+  const unsigned int dashes_mask = my_dashes_mask ^ desired_non_dashes_mask;++  const __m128i ret_a =+      uuid_parse_parse_hex_without_validation_ssse3(ascii_digits_a);+  const __m128i ret_b =+      uuid_parse_parse_hex_without_validation_ssse3(ascii_digits_b);+  const unsigned int valid_hex_mask_a =+      uuid_parse_validate_hex_ssse3(ascii_digits_a);+  const unsigned int valid_hex_mask_b =+      uuid_parse_validate_hex_ssse3(ascii_digits_b);+  const unsigned int valid_hex_mask =+      (valid_hex_mask_a << 16) | valid_hex_mask_b;++  if (~(dashes_mask & valid_hex_mask)) {+    return UuidParseCode::INVALID_CHAR;+  }++  const unsigned long long ret_a_scalar = _mm_extract_epi64(ret_a, 0);+  const unsigned long long ret_b_scalar = _mm_extract_epi64(ret_b, 0);+  std::memcpy(out + 0, &ret_a_scalar, 8);+  std::memcpy(out + 8, &ret_b_scalar, 8);+  return UuidParseCode::SUCCESS;+}++inline UuidParseCode uuid_parse_ssse3(std::string& out, std::string_view s) {+  return uuid_parse_generic<uuid_parse_buffer_to_buffer_ssse3>(out, s);+}++#endif // FOLLY_X64 && defined(__SSSE3__)++// NOTE: add NEON or SVE?++// scalar fallback+constexpr std::array<std::uint8_t, 256> generateValueTable() {+  std::array<std::uint8_t, 256> table = {};+  for (size_t i = 0; i < 256; ++i) {+    if (i >= '0' && i <= '9') {+      table[i] = static_cast<std::uint8_t>(i - '0');+    } else if (i >= 'A' && i <= 'F') {+      table[i] = static_cast<std::uint8_t>(i - 'A' + 10);+    } else if (i >= 'a' && i <= 'f') {+      table[i] = static_cast<std::uint8_t>(i - 'a' + 10);+    } else {+      table[i] = 16;+    }+  }+  return table;+}+inline constexpr std::array<std::uint8_t, 256> value_table =+    generateValueTable();+static_assert(+    value_table[0] == 16, "value_table must be initialized at compile time");++FOLLY_ALWAYS_INLINE UuidParseCode+uuid_parse_buffer_to_buffer_scalar(char* out, const char* s) {+  int i = 0;+  std::uint8_t tmp[32];+  std::memcpy(tmp + 0, s + 0, 8);+  std::memcpy(tmp + 8, s + 9, 4);+  std::memcpy(tmp + 12, s + 14, 4);+  std::memcpy(tmp + 16, s + 19, 4);+  std::memcpy(tmp + 20, s + 24, 12);+  if (s[8] != '-' || s[13] != '-' || s[18] != '-' || s[23] != '-') {+    return UuidParseCode::INVALID_CHAR;+  }+  while (i < 32) {+    const char hi = value_table[tmp[i++]];+    const char lo = value_table[tmp[i++]];+    if (hi == 16 || lo == 16) {+      return UuidParseCode::INVALID_CHAR;+    }+    *(out++) = (hi << 4) | lo;+  }+  return UuidParseCode::SUCCESS;+}++inline UuidParseCode uuid_parse_scalar(std::string& out, std::string_view s) {+  return uuid_parse_generic<uuid_parse_buffer_to_buffer_scalar>(out, s);+}+} // namespace detail++FOLLY_ALWAYS_INLINE UuidParseCode+uuid_parse_buffer_to_buffer(char* out, const char* s) {+#if FOLLY_X64 && defined(__AVX2__)+  return detail::uuid_parse_buffer_to_buffer_avx2(out, s);+#elif FOLLY_X64 && defined(__SSSE3__)+  return detail::uuid_parse_buffer_to_buffer_ssse3(out, s);+#else+  return detail::uuid_parse_buffer_to_buffer_scalar(out, s);+#endif+}++// reads 36 bytes from s and writes 16 bytes to out+inline UuidParseCode uuid_parse(std::uint8_t* out, const char* s) {+  return uuid_parse_buffer_to_buffer((char*)out, s);+}++// checks that s is 36 bytes long and overwrites s with 16 bytes+inline UuidParseCode uuid_parse(std::string& out, std::string_view s) {+  return detail::uuid_parse_generic<uuid_parse_buffer_to_buffer>(out, s);+}++} // namespace folly
+ folly/folly/codec/hex.h view
@@ -0,0 +1,160 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <array>+#include <cstdint>+#include <string_view>++#include <folly/lang/Align.h>+#include <folly/lang/Assume.h>++namespace folly {++/// hex_alphabet_lower+constexpr auto hex_alphabet_lower = std::string_view("0123456789abcdef");++/// hex_alphabet_upper+constexpr auto hex_alphabet_upper = std::string_view("0123456789ABCDEF");++namespace detail {++constexpr std::array<uint8_t, 256> make_hex_alphabet_table() {+  std::array<uint8_t, 256> array{};+  for (auto& v : array) {+    v = ~0;+  }+  for (uint8_t i = 0; i < hex_alphabet_lower.size(); ++i) {+    array[hex_alphabet_lower[i]] = i;+  }+  for (uint8_t i = 0; i < hex_alphabet_upper.size(); ++i) {+    array[hex_alphabet_upper[i]] = i;+  }+  return array;+}++} // namespace detail++/// hex_alphabet_table+alignas(hardware_constructive_interference_size) //+    constexpr auto hex_alphabet_table = detail::make_hex_alphabet_table();++/// hex_decoded_digit_is_valid+constexpr bool hex_decoded_digit_is_valid(uint8_t const v) {+  return !(v & 0x80);+}++/// hex_is_digit_table+constexpr bool hex_is_digit_table(char const h) {+  return int8_t(hex_alphabet_table[uint8_t(h)]) >= 0;+}++/// hex_decode_digit_table+///+/// For characters which are in the lower or upper hex alphabets, returns the+/// decoded value. For other characters, returns a value with the high bit set.+constexpr uint8_t hex_decode_digit_table(char const h) {+  return hex_alphabet_table[uint8_t(h)];+}++/// hex_is_digit_flavor_x86_64+///+/// from: https://github.com/stedonet/chex+constexpr bool hex_is_digit_flavor_x86_64(char const h) {+  uint8_t const uh = uint8_t(h);+  uint8_t const n09 = uh - '0';+  uint8_t const nAF = (uh | 0x20) - 'a';+  return (n09 < 10) || (nAF < 6);+}++/// hex_decode_digit_raw_flavor_x86_64+///+/// from: https://github.com/stedonet/chex+constexpr uint8_t hex_decode_digit_raw_flavor_x86_64(char const h) {+  return (h + (h >> 6) * 9) & 0xf;+}++/// hex_decode_digit_flavor_x86_64+///+/// from: https://github.com/stedonet/chex+constexpr uint8_t hex_decode_digit_flavor_x86_64(char const h) {+  if (!hex_is_digit_flavor_x86_64(h)) {+    return ~0;+  }+  auto const raw = hex_decode_digit_raw_flavor_x86_64(h);+  assume(!(raw & 0xf0));+  return raw;+}++/// hex_is_digit_flavor_aarch64+constexpr bool hex_is_digit_flavor_aarch64(char const h) {+  auto const uh = uint8_t(h);+  auto const cond = uh <= '9';+  auto const base = uint8_t(cond ? uh - '0' : (uh | 0x20) - 'a');+  auto const bound = uint8_t(cond ? 10 : 6);+  return base < bound;+}++/// hex_decode_digit_raw_flavor_aarch64+constexpr uint8_t hex_decode_digit_raw_flavor_aarch64(char const h) {+  auto const uh = uint8_t(h);+  auto const cond = uh <= '9';+  auto const base = uint8_t(cond ? uh - '0' : (uh | 0x20) - 'a');+  auto const offset = uint8_t(cond ? 0 : 10);+  return base + offset;+}++/// hex_decode_digit_flavor_aarch64+constexpr uint8_t hex_decode_digit_flavor_aarch64(char const h) {+  if (!hex_is_digit_flavor_aarch64(h)) {+    return ~0;+  }+  auto const raw = hex_decode_digit_raw_flavor_aarch64(h);+  assume(!(raw & 0xf0));+  return raw;+}++/// hex_is_digit+constexpr bool hex_is_digit(char const h) {+  return kIsArchAArch64 //+      ? hex_is_digit_flavor_aarch64(h)+      : hex_is_digit_flavor_x86_64(h);+}++/// hex_decode_digit_raw+///+/// For characters which are in the lower or upper hex alphabets, returns the+/// decoded value. For other characters, the behavior is unspecified and any+/// value may be returned. This function cannot be used to distinguish between+/// characters in and outside of the alphabets.+constexpr uint8_t hex_decode_digit_raw(char const h) {+  return kIsArchAArch64 //+      ? hex_decode_digit_raw_flavor_aarch64(h)+      : hex_decode_digit_raw_flavor_x86_64(h);+}++/// hex_decode_digit+///+/// For characters which are in the lower or upper hex alphabets, returns the+/// decoded value. For other characters, returns a value with the high bit set.+constexpr uint8_t hex_decode_digit(char const h) {+  return kIsArchAArch64 //+      ? hex_decode_digit_flavor_aarch64(h)+      : hex_decode_digit_flavor_x86_64(h);+}++} // namespace folly
+ folly/folly/compression/Compression.cpp view
@@ -0,0 +1,2047 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#include <folly/compression/Compression.h>++#if FOLLY_HAVE_LIBLZ4+#include <lz4.h>+#include <lz4hc.h>+#if LZ4_VERSION_NUMBER >= 10301+#include <lz4frame.h>+#endif+#endif++#include <glog/logging.h>++#if FOLLY_HAVE_LIBSNAPPY+#include <snappy-sinksource.h>+#include <snappy.h>+#endif++#if FOLLY_HAVE_LIBZ+#include <folly/compression/Zlib.h>+#endif++#if FOLLY_HAVE_LIBLZMA+#include <lzma.h>+#endif++#if FOLLY_HAVE_LIBZSTD+#include <folly/compression/Zstd.h>+#endif++#if FOLLY_HAVE_LIBBZ2+#include <folly/portability/Windows.h>++#include <bzlib.h>+#endif++#include <algorithm>+#include <unordered_set>++#include <folly/Conv.h>+#include <folly/Memory.h>+#include <folly/Portability.h>+#include <folly/Random.h>+#include <folly/ScopeGuard.h>+#include <folly/Utility.h>+#include <folly/Varint.h>+#include <folly/compression/Utils.h>+#include <folly/io/Cursor.h>+#include <folly/lang/Bits.h>+#include <folly/stop_watch.h>++using folly::compression::detail::dataStartsWithLE;+using folly::compression::detail::prefixToStringLE;++namespace folly {+namespace compression {++Codec::Codec(CodecType type, Optional<int> /* level */, StringPiece /* name */)+    : type_(type) {}++// Ensure consistent behavior in the nullptr case+std::unique_ptr<IOBuf> Codec::compress(const IOBuf* data) {+  if (data == nullptr) {+    throw std::invalid_argument("Codec: data must not be nullptr");+  }+  const uint64_t len = data->computeChainDataLength();+  if (len > maxUncompressedLength()) {+    throw std::runtime_error("Codec: uncompressed length too large");+  }+  return doCompress(data);+}++std::string Codec::compress(const StringPiece data) {+  const uint64_t len = data.size();+  if (len > maxUncompressedLength()) {+    throw std::runtime_error("Codec: uncompressed length too large");+  }+  return doCompressString(data);+}++std::unique_ptr<IOBuf> Codec::uncompress(+    const IOBuf* data, Optional<uint64_t> uncompressedLength) {+  if (data == nullptr) {+    throw std::invalid_argument("Codec: data must not be nullptr");+  }+  if (!uncompressedLength) {+    if (needsUncompressedLength()) {+      throw std::invalid_argument("Codec: uncompressed length required");+    }+  } else if (*uncompressedLength > maxUncompressedLength()) {+    throw std::runtime_error("Codec: uncompressed length too large");+  }++  if (data->empty()) {+    if (uncompressedLength.value_or(0) != 0) {+      throw std::runtime_error("Codec: invalid uncompressed length");+    }+    return IOBuf::create(0);+  }++  return doUncompress(data, uncompressedLength);+}++std::string Codec::uncompress(+    const StringPiece data, Optional<uint64_t> uncompressedLength) {+  if (!uncompressedLength) {+    if (needsUncompressedLength()) {+      throw std::invalid_argument("Codec: uncompressed length required");+    }+  } else if (*uncompressedLength > maxUncompressedLength()) {+    throw std::runtime_error("Codec: uncompressed length too large");+  }++  if (data.empty()) {+    if (uncompressedLength.value_or(0) != 0) {+      throw std::runtime_error("Codec: invalid uncompressed length");+    }+    return "";+  }++  return doUncompressString(data, uncompressedLength);+}++bool Codec::needsUncompressedLength() const {+  return doNeedsUncompressedLength();+}++uint64_t Codec::maxUncompressedLength() const {+  return doMaxUncompressedLength();+}++bool Codec::doNeedsUncompressedLength() const {+  return false;+}++uint64_t Codec::doMaxUncompressedLength() const {+  return UNLIMITED_UNCOMPRESSED_LENGTH;+}++std::vector<std::string> Codec::validPrefixes() const {+  return {};+}++bool Codec::canUncompress(const IOBuf*, Optional<uint64_t>) const {+  return false;+}++bool Codec::canUncompress(+    StringPiece data, Optional<uint64_t> uncompressedLength) const {+  auto buf = IOBuf::wrapBufferAsValue(data.data(), data.size());+  return canUncompress(&buf, uncompressedLength);+}++std::string Codec::doCompressString(const StringPiece data) {+  const IOBuf inputBuffer{IOBuf::WRAP_BUFFER, data};+  auto outputBuffer = doCompress(&inputBuffer);+  return outputBuffer->to<std::string>();+}++std::string Codec::doUncompressString(+    const StringPiece data, Optional<uint64_t> uncompressedLength) {+  const IOBuf inputBuffer{IOBuf::WRAP_BUFFER, data};+  auto outputBuffer = doUncompress(&inputBuffer, uncompressedLength);+  std::string output;+  output.reserve(outputBuffer->computeChainDataLength());+  for (auto range : *outputBuffer) {+    output.append(reinterpret_cast<const char*>(range.data()), range.size());+  }+  return output;+}++uint64_t Codec::maxCompressedLength(uint64_t uncompressedLength) const {+  return doMaxCompressedLength(uncompressedLength);+}++Optional<uint64_t> Codec::getUncompressedLength(+    const folly::IOBuf* data, Optional<uint64_t> uncompressedLength) const {+  auto const compressedLength = data->computeChainDataLength();+  if (compressedLength == 0) {+    if (uncompressedLength.value_or(0) != 0) {+      throw std::runtime_error("Invalid uncompressed length");+    }+    return 0;+  }+  return doGetUncompressedLength(data, uncompressedLength);+}++Optional<uint64_t> Codec::getUncompressedLength(+    StringPiece data, Optional<uint64_t> uncompressedLength) const {+  auto buf = IOBuf::wrapBufferAsValue(data.data(), data.size());+  return getUncompressedLength(&buf, uncompressedLength);+}++Optional<uint64_t> Codec::doGetUncompressedLength(+    const folly::IOBuf*, Optional<uint64_t> uncompressedLength) const {+  return uncompressedLength;+}++bool StreamCodec::needsDataLength() const {+  return doNeedsDataLength();+}++bool StreamCodec::doNeedsDataLength() const {+  return false;+}++void StreamCodec::assertStateIs(State expected) const {+  if (state_ != expected) {+    throw std::logic_error(folly::to<std::string>(+        "Codec: state is ", state_, "; expected state ", expected));+  }+}++void StreamCodec::resetStream(Optional<uint64_t> uncompressedLength) {+  state_ = State::RESET;+  uncompressedLength_ = uncompressedLength;+  progressMade_ = true;+  doResetStream();+}++bool StreamCodec::compressStream(+    ByteRange& input, MutableByteRange& output, StreamCodec::FlushOp flushOp) {+  if (state_ == State::RESET && input.empty() &&+      flushOp == StreamCodec::FlushOp::END &&+      uncompressedLength().value_or(0) != 0) {+    throw std::runtime_error("Codec: invalid uncompressed length");+  }++  if (!uncompressedLength() && needsDataLength()) {+    throw std::runtime_error("Codec: uncompressed length required");+  }+  if (state_ == State::RESET && !input.empty() &&+      uncompressedLength() == uint64_t(0)) {+    throw std::runtime_error("Codec: invalid uncompressed length");+  }+  // Handle input state transitions+  switch (flushOp) {+    case StreamCodec::FlushOp::NONE:+      if (state_ == State::RESET) {+        state_ = State::COMPRESS;+      }+      assertStateIs(State::COMPRESS);+      break;+    case StreamCodec::FlushOp::FLUSH:+      if (state_ == State::RESET || state_ == State::COMPRESS) {+        state_ = State::COMPRESS_FLUSH;+      }+      assertStateIs(State::COMPRESS_FLUSH);+      break;+    case StreamCodec::FlushOp::END:+      if (state_ == State::RESET || state_ == State::COMPRESS) {+        state_ = State::COMPRESS_END;+      }+      assertStateIs(State::COMPRESS_END);+      break;+  }+  size_t const inputSize = input.size();+  size_t const outputSize = output.size();+  bool const done = doCompressStream(input, output, flushOp);+  if (!done && inputSize == input.size() && outputSize == output.size()) {+    if (!progressMade_) {+      throw std::runtime_error("Codec: No forward progress made");+    }+    // Throw an exception if there is no progress again next time+    progressMade_ = false;+  } else {+    progressMade_ = true;+  }+  // Handle output state transitions+  if (done) {+    if (state_ == State::COMPRESS_FLUSH) {+      state_ = State::COMPRESS;+    } else if (state_ == State::COMPRESS_END) {+      state_ = State::END;+    }+    // Check internal invariants+    DCHECK(input.empty());+    DCHECK(flushOp != StreamCodec::FlushOp::NONE);+  }+  return done;+}++bool StreamCodec::uncompressStream(+    ByteRange& input, MutableByteRange& output, StreamCodec::FlushOp flushOp) {+  if (state_ == State::RESET && input.empty()) {+    return uncompressedLength().value_or(0) == 0;+  }+  // Handle input state transitions+  if (state_ == State::RESET) {+    state_ = State::UNCOMPRESS;+  }+  assertStateIs(State::UNCOMPRESS);+  size_t const inputSize = input.size();+  size_t const outputSize = output.size();+  bool const done = doUncompressStream(input, output, flushOp);+  if (!done && inputSize == input.size() && outputSize == output.size()) {+    if (!progressMade_) {+      throw std::runtime_error("Codec: no forward progress made");+    }+    // Throw an exception if there is no progress again next time+    progressMade_ = false;+  } else {+    progressMade_ = true;+  }+  // Handle output state transitions+  if (done) {+    state_ = State::END;+  }+  return done;+}++static std::unique_ptr<IOBuf> addOutputBuffer(+    MutableByteRange& output, uint64_t size) {+  DCHECK(output.empty());+  auto buffer = IOBuf::create(size);+  buffer->append(buffer->capacity());+  output = {buffer->writableData(), buffer->length()};+  return buffer;+}++std::unique_ptr<IOBuf> StreamCodec::doCompress(IOBuf const* data) {+  uint64_t const uncompressedLength = data->computeChainDataLength();+  resetStream(uncompressedLength);+  uint64_t const maxCompressedLen = maxCompressedLength(uncompressedLength);++  auto constexpr kMaxSingleStepLength = uint64_t(64) << 20; // 64 MB+  auto constexpr kDefaultBufferLength = uint64_t(4) << 20; // 4 MB++  MutableByteRange output;+  auto buffer = addOutputBuffer(+      output,+      maxCompressedLen <= kMaxSingleStepLength+          ? maxCompressedLen+          : kDefaultBufferLength);++  // Compress the entire IOBuf chain into the IOBuf chain pointed to by buffer+  IOBuf const* current = data;+  ByteRange input{current->data(), current->length()};+  StreamCodec::FlushOp flushOp = StreamCodec::FlushOp::NONE;+  bool done = false;+  while (!done) {+    while (input.empty() && current->next() != data) {+      current = current->next();+      input = {current->data(), current->length()};+    }+    if (current->next() == data) {+      // This is the last input buffer so end the stream+      flushOp = StreamCodec::FlushOp::END;+    }+    if (output.empty()) {+      buffer->prependChain(addOutputBuffer(output, kDefaultBufferLength));+    }+    done = compressStream(input, output, flushOp);+    if (done) {+      DCHECK(input.empty());+      DCHECK(flushOp == StreamCodec::FlushOp::END);+      DCHECK_EQ(current->next(), data);+    }+  }+  buffer->prev()->trimEnd(output.size());+  return buffer;+}++static uint64_t computeBufferLength(+    uint64_t const compressedLength, uint64_t const blockSize) {+  uint64_t constexpr kMaxBufferLength = uint64_t(4) << 20; // 4 MiB+  uint64_t const goodBufferSize = 4 * std::max(blockSize, compressedLength);+  return std::min(goodBufferSize, kMaxBufferLength);+}++std::unique_ptr<IOBuf> StreamCodec::doUncompress(+    IOBuf const* data, Optional<uint64_t> uncompressedLength) {+  auto constexpr kMaxSingleStepLength = uint64_t(64) << 20; // 64 MB+  auto constexpr kBlockSize = uint64_t(128) << 10;+  auto const defaultBufferLength =+      computeBufferLength(data->computeChainDataLength(), kBlockSize);++  uncompressedLength = getUncompressedLength(data, uncompressedLength);+  resetStream(uncompressedLength);++  MutableByteRange output;+  auto buffer = addOutputBuffer(+      output,+      (uncompressedLength && *uncompressedLength <= kMaxSingleStepLength+           ? *uncompressedLength+           : defaultBufferLength));++  // Uncompress the entire IOBuf chain into the IOBuf chain pointed to by buffer+  IOBuf const* current = data;+  ByteRange input{current->data(), current->length()};+  StreamCodec::FlushOp flushOp = StreamCodec::FlushOp::NONE;+  bool done = false;+  while (!done) {+    while (input.empty() && current->next() != data) {+      current = current->next();+      input = {current->data(), current->length()};+    }+    if (current->next() == data) {+      // Tell the uncompressor there is no more input (it may optimize)+      flushOp = StreamCodec::FlushOp::END;+    }+    if (output.empty()) {+      buffer->prependChain(addOutputBuffer(output, defaultBufferLength));+    }+    done = uncompressStream(input, output, flushOp);+  }+  if (!input.empty()) {+    throw std::runtime_error("Codec: Junk after end of data");+  }++  buffer->prev()->trimEnd(output.size());+  if (uncompressedLength &&+      *uncompressedLength != buffer->computeChainDataLength()) {+    throw std::runtime_error("Codec: invalid uncompressed length");+  }++  return buffer;+}++namespace {++/**+ * No compression+ */+class NoCompressionCodec final : public Codec {+ public:+  static std::unique_ptr<Codec> create(int level, CodecType type);+  explicit NoCompressionCodec(int level, CodecType type);++ private:+  uint64_t doMaxCompressedLength(uint64_t uncompressedLength) const override;+  std::unique_ptr<IOBuf> doCompress(const IOBuf* data) override;+  std::unique_ptr<IOBuf> doUncompress(+      const IOBuf* data, Optional<uint64_t> uncompressedLength) override;+};++std::unique_ptr<Codec> NoCompressionCodec::create(int level, CodecType type) {+  return std::make_unique<NoCompressionCodec>(level, type);+}++NoCompressionCodec::NoCompressionCodec(int level, CodecType type)+    : Codec(type) {+  DCHECK(type == CodecType::NO_COMPRESSION);+  switch (level) {+    case COMPRESSION_LEVEL_DEFAULT:+    case COMPRESSION_LEVEL_FASTEST:+    case COMPRESSION_LEVEL_BEST:+      level = 0;+  }+  if (level != 0) {+    throw std::invalid_argument(+        to<std::string>("NoCompressionCodec: invalid level ", level));+  }+}++uint64_t NoCompressionCodec::doMaxCompressedLength(+    uint64_t uncompressedLength) const {+  return uncompressedLength;+}++std::unique_ptr<IOBuf> NoCompressionCodec::doCompress(const IOBuf* data) {+  return data->clone();+}++std::unique_ptr<IOBuf> NoCompressionCodec::doUncompress(+    const IOBuf* data, Optional<uint64_t> uncompressedLength) {+  if (uncompressedLength &&+      data->computeChainDataLength() != *uncompressedLength) {+    throw std::runtime_error(+        to<std::string>("NoCompressionCodec: invalid uncompressed length"));+  }+  return data->clone();+}++#if (FOLLY_HAVE_LIBLZ4 || FOLLY_HAVE_LIBLZMA)++void encodeVarintToIOBuf(uint64_t val, folly::IOBuf* out) {+  DCHECK_GE(out->tailroom(), kMaxVarintLength64);+  out->append(encodeVarint(val, out->writableTail()));+}++inline uint64_t decodeVarintFromCursor(folly::io::Cursor& cursor) {+  uint64_t val = 0;+  int8_t b = 0;+  for (int shift = 0; shift <= 63; shift += 7) {+    b = cursor.read<int8_t>();+    val |= static_cast<uint64_t>(b & 0x7f) << shift;+    if (b >= 0) {+      break;+    }+  }+  if (b < 0) {+    throw std::invalid_argument("Invalid varint value. Too big.");+  }+  return val;+}++#endif // FOLLY_HAVE_LIBLZ4 || FOLLY_HAVE_LIBLZMA++#if FOLLY_HAVE_LIBLZ4++#if LZ4_VERSION_NUMBER >= 10802 && defined(LZ4_STATIC_LINKING_ONLY) && \+    defined(LZ4_HC_STATIC_LINKING_ONLY) && !defined(FOLLY_USE_LZ4_FAST_RESET)+#define FOLLY_USE_LZ4_FAST_RESET 1+#endif++#if FOLLY_USE_LZ4_FAST_RESET+void lz4_stream_t_deleter(LZ4_stream_t* ctx) {+  LZ4_freeStream(ctx);+}++void lz4_streamhc_t_deleter(LZ4_streamHC_t* ctx) {+  LZ4_freeStreamHC(ctx);+}+#endif++/**+ * LZ4 compression+ */+class LZ4Codec final : public Codec {+ public:+  static std::unique_ptr<Codec> create(int level, CodecType type);+  explicit LZ4Codec(int level, CodecType type);++ private:+  bool doNeedsUncompressedLength() const override;+  uint64_t doMaxUncompressedLength() const override;+  uint64_t doMaxCompressedLength(uint64_t uncompressedLength) const override;++  bool encodeSize() const { return type() == CodecType::LZ4_VARINT_SIZE; }++  std::unique_ptr<IOBuf> doCompress(const IOBuf* data) override;+  std::unique_ptr<IOBuf> doUncompress(+      const IOBuf* data, Optional<uint64_t> uncompressedLength) override;++#if FOLLY_USE_LZ4_FAST_RESET+  std::unique_ptr<+      LZ4_stream_t,+      folly::static_function_deleter<LZ4_stream_t, lz4_stream_t_deleter>>+      ctx;+  std::unique_ptr<+      LZ4_streamHC_t,+      folly::static_function_deleter<LZ4_streamHC_t, lz4_streamhc_t_deleter>>+      hcctx;+#endif++  bool highCompression_;+};++std::unique_ptr<Codec> LZ4Codec::create(int level, CodecType type) {+  return std::make_unique<LZ4Codec>(level, type);+}++int lz4ConvertLevel(int level) {+  switch (level) {+    case 1:+    case COMPRESSION_LEVEL_FASTEST:+    case COMPRESSION_LEVEL_DEFAULT:+      return 1;+    case 2:+    case COMPRESSION_LEVEL_BEST:+      return 2;+  }+  throw std::invalid_argument(+      to<std::string>("LZ4Codec: invalid level: ", level));+}++LZ4Codec::LZ4Codec(int level, CodecType type)+    : Codec(type, lz4ConvertLevel(level)),+      highCompression_(lz4ConvertLevel(level) > 1) {+  DCHECK(type == CodecType::LZ4 || type == CodecType::LZ4_VARINT_SIZE);+}++bool LZ4Codec::doNeedsUncompressedLength() const {+  return !encodeSize();+}++// The value comes from lz4.h in lz4-r117, but older versions of lz4 don't+// define LZ4_MAX_INPUT_SIZE (even though the max size is the same), so do it+// here.+#ifndef LZ4_MAX_INPUT_SIZE+#define LZ4_MAX_INPUT_SIZE 0x7E000000+#endif++uint64_t LZ4Codec::doMaxUncompressedLength() const {+  return LZ4_MAX_INPUT_SIZE;+}++uint64_t LZ4Codec::doMaxCompressedLength(uint64_t uncompressedLength) const {+  return LZ4_compressBound(uncompressedLength) ++      (encodeSize() ? kMaxVarintLength64 : 0);+}++std::unique_ptr<IOBuf> LZ4Codec::doCompress(const IOBuf* data) {+  IOBuf clone;+  if (data->isChained()) {+    // LZ4 doesn't support streaming, so we have to coalesce+    clone = data->cloneCoalescedAsValue();+    data = &clone;+  }++  auto out = IOBuf::create(maxCompressedLength(data->length()));+  if (encodeSize()) {+    encodeVarintToIOBuf(data->length(), out.get());+  }++  int n;+  auto input = reinterpret_cast<const char*>(data->data());+  auto output = reinterpret_cast<char*>(out->writableTail());+  const auto inputLength = data->length();++#if FOLLY_USE_LZ4_FAST_RESET+  if (!highCompression_ && !ctx) {+    ctx.reset(LZ4_createStream());+  }+  if (highCompression_ && !hcctx) {+    hcctx.reset(LZ4_createStreamHC());+  }++  if (highCompression_) {+    n = LZ4_compress_HC_extStateHC_fastReset(+        hcctx.get(), input, output, inputLength, out->tailroom(), 0);+  } else {+    n = LZ4_compress_fast_extState_fastReset(+        ctx.get(), input, output, inputLength, out->tailroom(), 1);+  }+#elif LZ4_VERSION_NUMBER >= 10700+  if (highCompression_) {+    n = LZ4_compress_HC(input, output, inputLength, out->tailroom(), 0);+  } else {+    n = LZ4_compress_default(input, output, inputLength, out->tailroom());+  }+#else+  if (highCompression_) {+    n = LZ4_compressHC(input, output, inputLength);+  } else {+    n = LZ4_compress(input, output, inputLength);+  }+#endif++  CHECK_GE(n, 0);+  CHECK_LE(n, out->capacity());++  out->append(n);+  return out;+}++std::unique_ptr<IOBuf> LZ4Codec::doUncompress(+    const IOBuf* data, Optional<uint64_t> uncompressedLength) {+  IOBuf clone;+  if (data->isChained()) {+    // LZ4 doesn't support streaming, so we have to coalesce+    clone = data->cloneCoalescedAsValue();+    data = &clone;+  }++  folly::io::Cursor cursor(data);+  uint64_t actualUncompressedLength;+  if (encodeSize()) {+    actualUncompressedLength = decodeVarintFromCursor(cursor);+    if (uncompressedLength && *uncompressedLength != actualUncompressedLength) {+      throw std::runtime_error("LZ4Codec: invalid uncompressed length");+    }+  } else {+    // Invariants+    DCHECK(uncompressedLength.has_value());+    DCHECK(*uncompressedLength <= maxUncompressedLength());+    actualUncompressedLength = *uncompressedLength;+  }++  auto sp = StringPiece{cursor.peekBytes()};+  auto out = IOBuf::create(actualUncompressedLength);+  int n = LZ4_decompress_safe(+      sp.data(),+      reinterpret_cast<char*>(out->writableTail()),+      sp.size(),+      actualUncompressedLength);++  if (n < 0 || uint64_t(n) != actualUncompressedLength) {+    throw std::runtime_error(+        to<std::string>("LZ4 decompression returned invalid value ", n));+  }+  out->append(actualUncompressedLength);+  return out;+}++#if LZ4_VERSION_NUMBER >= 10301++class LZ4FrameCodec final : public Codec {+ public:+  static std::unique_ptr<Codec> create(int level, CodecType type);+  explicit LZ4FrameCodec(int level, CodecType type);+  ~LZ4FrameCodec() override;++  std::vector<std::string> validPrefixes() const override;+  bool canUncompress(+      const IOBuf* data, Optional<uint64_t> uncompressedLength) const override;++ private:+  uint64_t doMaxCompressedLength(uint64_t uncompressedLength) const override;++  std::unique_ptr<IOBuf> doCompress(const IOBuf* data) override;+  std::unique_ptr<IOBuf> doUncompress(+      const IOBuf* data, Optional<uint64_t> uncompressedLength) override;++  // Reset the dctx_ if it is dirty or null.+  void resetDCtx();++  int level_;+#if FOLLY_USE_LZ4_FAST_RESET+  LZ4F_compressionContext_t cctx_{nullptr};+#endif+  LZ4F_decompressionContext_t dctx_{nullptr};+  bool dirty_{false};+};++/* static */ std::unique_ptr<Codec> LZ4FrameCodec::create(+    int level, CodecType type) {+  return std::make_unique<LZ4FrameCodec>(level, type);+}++constexpr uint32_t kLZ4FrameMagicLE = 0x184D2204;++std::vector<std::string> LZ4FrameCodec::validPrefixes() const {+  return {prefixToStringLE(kLZ4FrameMagicLE)};+}++bool LZ4FrameCodec::canUncompress(const IOBuf* data, Optional<uint64_t>) const {+  return dataStartsWithLE(data, kLZ4FrameMagicLE);+}++uint64_t LZ4FrameCodec::doMaxCompressedLength(+    uint64_t uncompressedLength) const {+  LZ4F_preferences_t prefs{};+  prefs.compressionLevel = level_;+  prefs.frameInfo.contentSize = uncompressedLength;+  return LZ4F_compressFrameBound(uncompressedLength, &prefs);+}++size_t lz4FrameThrowOnError(size_t code) {+  if (LZ4F_isError(code)) {+    throw std::runtime_error(+        to<std::string>("LZ4Frame error: ", LZ4F_getErrorName(code)));+  }+  return code;+}++void LZ4FrameCodec::resetDCtx() {+  if (dctx_ && !dirty_) {+    return;+  }+  if (dctx_) {+    LZ4F_freeDecompressionContext(dctx_);+  }+  lz4FrameThrowOnError(LZ4F_createDecompressionContext(&dctx_, 100));+  dirty_ = false;+}++int lz4fConvertLevel(int level) {+  switch (level) {+    case COMPRESSION_LEVEL_FASTEST:+    case COMPRESSION_LEVEL_DEFAULT:+      return 0;+    case COMPRESSION_LEVEL_BEST:+      return 16;+  }+  return level;+}++LZ4FrameCodec::LZ4FrameCodec(int level, CodecType type)+    : Codec(type, lz4fConvertLevel(level)), level_(lz4fConvertLevel(level)) {+  DCHECK(type == CodecType::LZ4_FRAME);+}++LZ4FrameCodec::~LZ4FrameCodec() {+  if (dctx_) {+    LZ4F_freeDecompressionContext(dctx_);+  }+#if FOLLY_USE_LZ4_FAST_RESET+  if (cctx_) {+    LZ4F_freeCompressionContext(cctx_);+  }+#endif+}++std::unique_ptr<IOBuf> LZ4FrameCodec::doCompress(const IOBuf* data) {+  // LZ4 Frame compression doesn't support streaming so we have to coalesce+  IOBuf clone;+  if (data->isChained()) {+    clone = data->cloneCoalescedAsValue();+    data = &clone;+  }++#if FOLLY_USE_LZ4_FAST_RESET+  if (!cctx_) {+    lz4FrameThrowOnError(LZ4F_createCompressionContext(&cctx_, LZ4F_VERSION));+  }+#endif++  // Set preferences+  const auto uncompressedLength = data->length();+  LZ4F_preferences_t prefs{};+  prefs.compressionLevel = level_;+  prefs.frameInfo.contentSize = uncompressedLength;+  // Compress+  auto buf = IOBuf::create(maxCompressedLength(uncompressedLength));+  const size_t written = lz4FrameThrowOnError(+#if FOLLY_USE_LZ4_FAST_RESET+      LZ4F_compressFrame_usingCDict(+          cctx_,+          buf->writableTail(),+          buf->tailroom(),+          data->data(),+          data->length(),+          nullptr,+          &prefs)+#else+      LZ4F_compressFrame(+          buf->writableTail(),+          buf->tailroom(),+          data->data(),+          data->length(),+          &prefs)+#endif+  );+  buf->append(written);+  return buf;+}++std::unique_ptr<IOBuf> LZ4FrameCodec::doUncompress(+    const IOBuf* data, Optional<uint64_t> uncompressedLength) {+  // Reset the dctx if any errors have occurred+  resetDCtx();+  // Coalesce the data+  ByteRange in = *data->begin();+  IOBuf clone;+  if (data->isChained()) {+    clone = data->cloneCoalescedAsValue();+    in = clone.coalesce();+  }+  data = nullptr;+  // Select decompression options+  LZ4F_decompressOptions_t options{};+  options.stableDst = 1;+  // Select blockSize and growthSize for the IOBufQueue+  IOBufQueue queue(IOBufQueue::cacheChainLength());+  auto blockSize = uint64_t{64} << 10;+  auto growthSize = uint64_t{4} << 20;+  if (uncompressedLength) {+    // Allocate uncompressedLength in one chunk (up to 64 MB)+    const auto allocateSize = std::min(*uncompressedLength, uint64_t{64} << 20);+    queue.preallocate(allocateSize, allocateSize);+    blockSize = std::min(*uncompressedLength, blockSize);+    growthSize = std::min(*uncompressedLength, growthSize);+  } else {+    // Reduce growthSize for small data+    const auto guessUncompressedLen =+        4 * std::max<uint64_t>(blockSize, in.size());+    growthSize = std::min(guessUncompressedLen, growthSize);+  }+  // Once LZ4_decompress() is called, the dctx_ cannot be reused until it+  // returns 0+  dirty_ = true;+  // Decompress until the frame is over+  size_t code = 0;+  do {+    // Allocate enough space to decompress at least a block+    void* out;+    size_t outSize;+    std::tie(out, outSize) = queue.preallocate(blockSize, growthSize);+    // Decompress+    size_t inSize = in.size();+    code = lz4FrameThrowOnError(+        LZ4F_decompress(dctx_, out, &outSize, in.data(), &inSize, &options));+    if (in.empty() && outSize == 0 && code != 0) {+      // We passed no input, no output was produced, and the frame isn't over+      // No more forward progress is possible+      throw std::runtime_error("LZ4Frame error: Incomplete frame");+    }+    in.uncheckedAdvance(inSize);+    queue.postallocate(outSize);+  } while (code != 0);+  // At this point the decompression context can be reused+  dirty_ = false;+  if (uncompressedLength && queue.chainLength() != *uncompressedLength) {+    throw std::runtime_error("LZ4Frame error: Invalid uncompressedLength");+  }+  return queue.move();+}++#endif // LZ4_VERSION_NUMBER >= 10301+#endif // FOLLY_HAVE_LIBLZ4++#if FOLLY_HAVE_LIBSNAPPY++/**+ * Snappy compression+ */++/**+ * Implementation of snappy::Source that reads from a IOBuf chain.+ */+class IOBufSnappySource final : public snappy::Source {+ public:+  explicit IOBufSnappySource(const IOBuf* data);+  size_t Available() const override;+  const char* Peek(size_t* len) override;+  void Skip(size_t n) override;++ private:+  size_t available_;+  io::Cursor cursor_;+};++IOBufSnappySource::IOBufSnappySource(const IOBuf* data)+    : available_(data->computeChainDataLength()), cursor_(data) {}++size_t IOBufSnappySource::Available() const {+  return available_;+}++const char* IOBufSnappySource::Peek(size_t* len) {+  auto sp = StringPiece{cursor_.peekBytes()};+  *len = sp.size();+  return sp.data();+}++void IOBufSnappySource::Skip(size_t n) {+  CHECK_LE(n, available_);+  cursor_.skip(n);+  available_ -= n;+}++class SnappyCodec final : public Codec {+ public:+  static std::unique_ptr<Codec> create(int level, CodecType type);+  explicit SnappyCodec(int level, CodecType type);++ private:+  uint64_t doMaxUncompressedLength() const override;+  uint64_t doMaxCompressedLength(uint64_t uncompressedLength) const override;+  std::unique_ptr<IOBuf> doCompress(const IOBuf* data) override;+  std::unique_ptr<IOBuf> doUncompress(+      const IOBuf* data, Optional<uint64_t> uncompressedLength) override;+  folly::Optional<uint64_t> doGetUncompressedLength(+      const folly::IOBuf* data,+      folly::Optional<uint64_t> uncompressedLength) const override;+};++std::unique_ptr<Codec> SnappyCodec::create(int level, CodecType type) {+  return std::make_unique<SnappyCodec>(level, type);+}++SnappyCodec::SnappyCodec(int level, CodecType type) : Codec(type) {+  DCHECK(type == CodecType::SNAPPY);+  switch (level) {+    case COMPRESSION_LEVEL_FASTEST:+    case COMPRESSION_LEVEL_DEFAULT:+    case COMPRESSION_LEVEL_BEST:+      level = 1;+  }+  if (level != 1) {+    throw std::invalid_argument(+        to<std::string>("SnappyCodec: invalid level: ", level));+  }+}++uint64_t SnappyCodec::doMaxUncompressedLength() const {+  // snappy.h uses uint32_t for lengths, so there's that.+  return std::numeric_limits<uint32_t>::max();+}++uint64_t SnappyCodec::doMaxCompressedLength(uint64_t uncompressedLength) const {+  return snappy::MaxCompressedLength(uncompressedLength);+}++std::unique_ptr<IOBuf> SnappyCodec::doCompress(const IOBuf* data) {+  IOBufSnappySource source(data);+  auto out = IOBuf::create(maxCompressedLength(source.Available()));++  snappy::UncheckedByteArraySink sink(+      reinterpret_cast<char*>(out->writableTail()));++  size_t n = snappy::Compress(&source, &sink);++  CHECK_LE(n, out->capacity());+  out->append(n);+  return out;+}++std::unique_ptr<IOBuf> SnappyCodec::doUncompress(+    const IOBuf* data, Optional<uint64_t> uncompressedLength) {+  uint32_t actualUncompressedLength = 0;++  {+    IOBufSnappySource source(data);+    if (!snappy::GetUncompressedLength(&source, &actualUncompressedLength)) {+      throw std::runtime_error("snappy::GetUncompressedLength failed");+    }+    if (uncompressedLength && *uncompressedLength != actualUncompressedLength) {+      throw std::runtime_error("snappy: invalid uncompressed length");+    }+  }++  auto out = IOBuf::create(actualUncompressedLength);++  {+    IOBufSnappySource source(data);+    if (!snappy::RawUncompress(+            &source, reinterpret_cast<char*>(out->writableTail()))) {+      throw std::runtime_error("snappy::RawUncompress failed");+    }+  }++  out->append(actualUncompressedLength);+  return out;+}++folly::Optional<uint64_t> SnappyCodec::doGetUncompressedLength(+    const folly::IOBuf* data,+    folly::Optional<uint64_t> uncompressedLength) const {+  uint32_t actualUncompressedLength = 0;+  IOBufSnappySource source(data);+  if (!snappy::GetUncompressedLength(&source, &actualUncompressedLength)) {+    throw std::runtime_error("snappy::GetUncompressedLength failed");+  }+  if (uncompressedLength && *uncompressedLength != actualUncompressedLength) {+    throw std::runtime_error("snappy: invalid uncompressed length");+  }++  return actualUncompressedLength;+}++#endif // FOLLY_HAVE_LIBSNAPPY++#if FOLLY_HAVE_LIBLZMA++/**+ * LZMA2 compression+ */+class LZMA2StreamCodec final : public StreamCodec {+ public:+  static std::unique_ptr<Codec> createCodec(int level, CodecType type);+  static std::unique_ptr<StreamCodec> createStream(int level, CodecType type);+  explicit LZMA2StreamCodec(int level, CodecType type);+  ~LZMA2StreamCodec() override;++  std::vector<std::string> validPrefixes() const override;+  bool canUncompress(+      const IOBuf* data, Optional<uint64_t> uncompressedLength) const override;++ private:+  bool doNeedsDataLength() const override;+  uint64_t doMaxUncompressedLength() const override;+  uint64_t doMaxCompressedLength(uint64_t uncompressedLength) const override;++  bool encodeSize() const { return type() == CodecType::LZMA2_VARINT_SIZE; }++  void doResetStream() override;+  bool doCompressStream(+      ByteRange& input,+      MutableByteRange& output,+      StreamCodec::FlushOp flushOp) override;+  bool doUncompressStream(+      ByteRange& input,+      MutableByteRange& output,+      StreamCodec::FlushOp flushOp) override;++  void resetCStream();+  void resetDStream();++  bool decodeAndCheckVarint(ByteRange& input);+  bool flushVarintBuffer(MutableByteRange& output);+  void resetVarintBuffer();++  Optional<lzma_stream> cstream_{};+  Optional<lzma_stream> dstream_{};++  std::array<uint8_t, kMaxVarintLength64> varintBuffer_;+  ByteRange varintToEncode_;+  size_t varintBufferPos_{0};++  int level_;+  bool needReset_{true};+  bool needDecodeSize_{false};+};++constexpr uint64_t kLZMA2MagicLE = 0x005A587A37FD;+constexpr unsigned kLZMA2MagicBytes = 6;++std::vector<std::string> LZMA2StreamCodec::validPrefixes() const {+  if (type() == CodecType::LZMA2_VARINT_SIZE) {+    return {};+  }+  return {prefixToStringLE(kLZMA2MagicLE, kLZMA2MagicBytes)};+}++bool LZMA2StreamCodec::doNeedsDataLength() const {+  return encodeSize();+}++bool LZMA2StreamCodec::canUncompress(+    const IOBuf* data, Optional<uint64_t>) const {+  if (type() == CodecType::LZMA2_VARINT_SIZE) {+    return false;+  }+  // Returns false for all inputs less than 8 bytes.+  // This is okay, because no valid LZMA2 streams are less than 8 bytes.+  return dataStartsWithLE(data, kLZMA2MagicLE, kLZMA2MagicBytes);+}++std::unique_ptr<Codec> LZMA2StreamCodec::createCodec(+    int level, CodecType type) {+  return std::make_unique<LZMA2StreamCodec>(level, type);+}++std::unique_ptr<StreamCodec> LZMA2StreamCodec::createStream(+    int level, CodecType type) {+  return std::make_unique<LZMA2StreamCodec>(level, type);+}++LZMA2StreamCodec::LZMA2StreamCodec(int level, CodecType type)+    : StreamCodec(type) {+  DCHECK(type == CodecType::LZMA2 || type == CodecType::LZMA2_VARINT_SIZE);+  switch (level) {+    case COMPRESSION_LEVEL_FASTEST:+      level = 0;+      break;+    case COMPRESSION_LEVEL_DEFAULT:+      level = LZMA_PRESET_DEFAULT;+      break;+    case COMPRESSION_LEVEL_BEST:+      level = 9;+      break;+  }+  if (level < 0 || level > 9) {+    throw std::invalid_argument(+        to<std::string>("LZMA2Codec: invalid level: ", level));+  }+  level_ = level;+}++LZMA2StreamCodec::~LZMA2StreamCodec() {+  if (cstream_) {+    lzma_end(cstream_.get_pointer());+    cstream_.reset();+  }+  if (dstream_) {+    lzma_end(dstream_.get_pointer());+    dstream_.reset();+  }+}++uint64_t LZMA2StreamCodec::doMaxUncompressedLength() const {+  // From lzma/base.h: "Stream is roughly 8 EiB (2^63 bytes)"+  return uint64_t(1) << 63;+}++uint64_t LZMA2StreamCodec::doMaxCompressedLength(+    uint64_t uncompressedLength) const {+  return lzma_stream_buffer_bound(uncompressedLength) ++      (encodeSize() ? kMaxVarintLength64 : 0);+}++void LZMA2StreamCodec::doResetStream() {+  needReset_ = true;+}++void LZMA2StreamCodec::resetCStream() {+  if (!cstream_) {+    cstream_.assign(LZMA_STREAM_INIT);+  }+  lzma_ret const rc =+      lzma_easy_encoder(cstream_.get_pointer(), level_, LZMA_CHECK_NONE);+  if (rc != LZMA_OK) {+    throw std::runtime_error(folly::to<std::string>(+        "LZMA2StreamCodec: lzma_easy_encoder error: ", rc));+  }+}++void LZMA2StreamCodec::resetDStream() {+  if (!dstream_) {+    dstream_.assign(LZMA_STREAM_INIT);+  }+  lzma_ret const rc = lzma_auto_decoder(+      dstream_.get_pointer(), std::numeric_limits<uint64_t>::max(), 0);+  if (rc != LZMA_OK) {+    throw std::runtime_error(folly::to<std::string>(+        "LZMA2StreamCodec: lzma_auto_decoder error: ", rc));+  }+}++FOLLY_PUSH_WARNING+FOLLY_GNU_DISABLE_WARNING("-Wswitch-enum")+lzma_ret lzmaThrowOnError(lzma_ret const rc) {+  switch (rc) {+    case LZMA_OK:+    case LZMA_STREAM_END:+    case LZMA_BUF_ERROR: // not fatal: returned if no progress was made twice+      return rc;+    case LZMA_NO_CHECK:+    case LZMA_UNSUPPORTED_CHECK:+    case LZMA_GET_CHECK:+    case LZMA_MEM_ERROR:+    case LZMA_MEMLIMIT_ERROR:+    case LZMA_FORMAT_ERROR:+    case LZMA_OPTIONS_ERROR:+    case LZMA_DATA_ERROR:+    case LZMA_PROG_ERROR:+    default:+      throw std::runtime_error(+          to<std::string>("LZMA2StreamCodec: error: ", rc));+  }+}+FOLLY_POP_WARNING++lzma_action lzmaTranslateFlush(StreamCodec::FlushOp flush) {+  switch (flush) {+    case StreamCodec::FlushOp::NONE:+      return LZMA_RUN;+    case StreamCodec::FlushOp::FLUSH:+      return LZMA_SYNC_FLUSH;+    case StreamCodec::FlushOp::END:+      return LZMA_FINISH;+    default:+      throw std::invalid_argument("LZMA2StreamCodec: Invalid flush");+  }+}++/**+ * Flushes the varint buffer.+ * Advances output by the number of bytes written.+ * Returns true when flushing is complete.+ */+bool LZMA2StreamCodec::flushVarintBuffer(MutableByteRange& output) {+  if (varintToEncode_.empty()) {+    return true;+  }+  const size_t numBytesToCopy = std::min(varintToEncode_.size(), output.size());+  if (numBytesToCopy > 0) {+    memcpy(output.data(), varintToEncode_.data(), numBytesToCopy);+  }+  varintToEncode_.advance(numBytesToCopy);+  output.advance(numBytesToCopy);+  return varintToEncode_.empty();+}++bool LZMA2StreamCodec::doCompressStream(+    ByteRange& input, MutableByteRange& output, StreamCodec::FlushOp flushOp) {+  if (needReset_) {+    resetCStream();+    if (encodeSize()) {+      varintBufferPos_ = 0;+      size_t const varintSize =+          encodeVarint(*uncompressedLength(), varintBuffer_.data());+      varintToEncode_ = {varintBuffer_.data(), varintSize};+    }+    needReset_ = false;+  }++  if (!flushVarintBuffer(output)) {+    return false;+  }++  cstream_->next_in = const_cast<uint8_t*>(input.data());+  cstream_->avail_in = input.size();+  cstream_->next_out = output.data();+  cstream_->avail_out = output.size();+  SCOPE_EXIT {+    input.uncheckedAdvance(input.size() - cstream_->avail_in);+    output.uncheckedAdvance(output.size() - cstream_->avail_out);+  };+  lzma_ret const rc = lzmaThrowOnError(+      lzma_code(cstream_.get_pointer(), lzmaTranslateFlush(flushOp)));+  switch (flushOp) {+    case StreamCodec::FlushOp::NONE:+      return false;+    case StreamCodec::FlushOp::FLUSH:+      return cstream_->avail_in == 0 && cstream_->avail_out != 0;+    case StreamCodec::FlushOp::END:+      return rc == LZMA_STREAM_END;+    default:+      throw std::invalid_argument("LZMA2StreamCodec: invalid FlushOp");+  }+}++/**+ * Attempts to decode a varint from input.+ * The function advances input by the number of bytes read.+ *+ * If there are too many bytes and the varint is not valid, throw a+ * runtime_error.+ *+ * If the uncompressed length was provided and a decoded varint does not match+ * the provided length, throw a runtime_error.+ *+ * Returns true if the varint was successfully decoded and matches the+ * uncompressed length if provided, and false if more bytes are needed.+ */+bool LZMA2StreamCodec::decodeAndCheckVarint(ByteRange& input) {+  if (input.empty()) {+    return false;+  }+  size_t const numBytesToCopy =+      std::min(kMaxVarintLength64 - varintBufferPos_, input.size());+  memcpy(varintBuffer_.data() + varintBufferPos_, input.data(), numBytesToCopy);++  size_t const rangeSize = varintBufferPos_ + numBytesToCopy;+  ByteRange range{varintBuffer_.data(), rangeSize};+  auto const ret = tryDecodeVarint(range);++  if (ret.hasValue()) {+    size_t const varintSize = rangeSize - range.size();+    input.advance(varintSize - varintBufferPos_);+    if (uncompressedLength() && *uncompressedLength() != ret.value()) {+      throw std::runtime_error("LZMA2StreamCodec: invalid uncompressed length");+    }+    return true;+  } else if (ret.error() == DecodeVarintError::TooManyBytes) {+    throw std::runtime_error("LZMA2StreamCodec: invalid uncompressed length");+  } else {+    // Too few bytes+    input.advance(numBytesToCopy);+    varintBufferPos_ += numBytesToCopy;+    return false;+  }+}++bool LZMA2StreamCodec::doUncompressStream(+    ByteRange& input, MutableByteRange& output, StreamCodec::FlushOp flushOp) {+  if (needReset_) {+    resetDStream();+    needReset_ = false;+    needDecodeSize_ = encodeSize();+    if (encodeSize()) {+      // Reset buffer+      varintBufferPos_ = 0;+    }+  }++  if (needDecodeSize_) {+    // Try decoding the varint. If the input does not contain the entire varint,+    // buffer the input. If the varint can not be decoded, fail.+    if (!decodeAndCheckVarint(input)) {+      return false;+    }+    needDecodeSize_ = false;+  }++  dstream_->next_in = const_cast<uint8_t*>(input.data());+  dstream_->avail_in = input.size();+  dstream_->next_out = output.data();+  dstream_->avail_out = output.size();+  SCOPE_EXIT {+    input.advance(input.size() - dstream_->avail_in);+    output.advance(output.size() - dstream_->avail_out);+  };++  lzma_ret rc;+  switch (flushOp) {+    case StreamCodec::FlushOp::NONE:+    case StreamCodec::FlushOp::FLUSH:+      rc = lzmaThrowOnError(lzma_code(dstream_.get_pointer(), LZMA_RUN));+      break;+    case StreamCodec::FlushOp::END:+      rc = lzmaThrowOnError(lzma_code(dstream_.get_pointer(), LZMA_FINISH));+      break;+    default:+      throw std::invalid_argument("LZMA2StreamCodec: invalid flush");+  }+  return rc == LZMA_STREAM_END;+}+#endif // FOLLY_HAVE_LIBLZMA++#if FOLLY_HAVE_LIBZSTD++int zstdConvertLevel(int level) {+  switch (level) {+    case COMPRESSION_LEVEL_FASTEST:+      return 1;+    case COMPRESSION_LEVEL_DEFAULT:+      return 1;+    case COMPRESSION_LEVEL_BEST:+      return 19;+  }+  if (level < 1 || level > ZSTD_maxCLevel()) {+    throw std::invalid_argument(+        to<std::string>("ZSTD: invalid level: ", level));+  }+  return level;+}++int zstdFastConvertLevel(int level) {+  switch (level) {+    case COMPRESSION_LEVEL_FASTEST:+      return -5;+    case COMPRESSION_LEVEL_DEFAULT:+      return -1;+    case COMPRESSION_LEVEL_BEST:+      return -1;+  }+  if (level < 1) {+    throw std::invalid_argument(+        to<std::string>("ZSTD: invalid level: ", level));+  }+  return -level;+}++std::unique_ptr<Codec> getZstdCodec(int level, CodecType type) {+  DCHECK(type == CodecType::ZSTD);+  return zstd::getCodec(zstd::Options(zstdConvertLevel(level)));+}++std::unique_ptr<StreamCodec> getZstdStreamCodec(int level, CodecType type) {+  DCHECK(type == CodecType::ZSTD);+  return zstd::getStreamCodec(zstd::Options(zstdConvertLevel(level)));+}++std::unique_ptr<Codec> getZstdFastCodec(int level, CodecType type) {+  DCHECK(type == CodecType::ZSTD_FAST);+  return zstd::getCodec(zstd::Options(zstdFastConvertLevel(level)));+}++std::unique_ptr<StreamCodec> getZstdFastStreamCodec(int level, CodecType type) {+  DCHECK(type == CodecType::ZSTD_FAST);+  return zstd::getStreamCodec(zstd::Options(zstdFastConvertLevel(level)));+}++#endif // FOLLY_HAVE_LIBZSTD++#if FOLLY_HAVE_LIBBZ2++class Bzip2StreamCodec final : public StreamCodec {+ public:+  static std::unique_ptr<Codec> createCodec(int level, CodecType type);+  static std::unique_ptr<StreamCodec> createStream(int level, CodecType type);+  explicit Bzip2StreamCodec(int level, CodecType type);++  ~Bzip2StreamCodec() override;++  std::vector<std::string> validPrefixes() const override;+  bool canUncompress(+      IOBuf const* data, Optional<uint64_t> uncompressedLength) const override;++ private:+  uint64_t doMaxCompressedLength(uint64_t uncompressedLength) const override;++  void doResetStream() override;+  bool doCompressStream(+      ByteRange& input,+      MutableByteRange& output,+      StreamCodec::FlushOp flushOp) override;+  bool doUncompressStream(+      ByteRange& input,+      MutableByteRange& output,+      StreamCodec::FlushOp flushOp) override;++  void resetCStream();+  void resetDStream();++  Optional<bz_stream> cstream_{};+  Optional<bz_stream> dstream_{};++  int level_;+  bool needReset_{true};+};++/* static */ std::unique_ptr<Codec> Bzip2StreamCodec::createCodec(+    int level, CodecType type) {+  return createStream(level, type);+}++/* static */ std::unique_ptr<StreamCodec> Bzip2StreamCodec::createStream(+    int level, CodecType type) {+  return std::make_unique<Bzip2StreamCodec>(level, type);+}++Bzip2StreamCodec::Bzip2StreamCodec(int level, CodecType type)+    : StreamCodec(type) {+  DCHECK(type == CodecType::BZIP2);+  switch (level) {+    case COMPRESSION_LEVEL_FASTEST:+      level = 1;+      break;+    case COMPRESSION_LEVEL_DEFAULT:+      level = 9;+      break;+    case COMPRESSION_LEVEL_BEST:+      level = 9;+      break;+  }+  if (level < 1 || level > 9) {+    throw std::invalid_argument(+        to<std::string>("Bzip2: invalid level: ", level));+  }+  level_ = level;+}++uint32_t constexpr kBzip2MagicLE = 0x685a42;+uint64_t constexpr kBzip2MagicBytes = 3;++std::vector<std::string> Bzip2StreamCodec::validPrefixes() const {+  return {prefixToStringLE(kBzip2MagicLE, kBzip2MagicBytes)};+}++bool Bzip2StreamCodec::canUncompress(+    IOBuf const* data, Optional<uint64_t>) const {+  return dataStartsWithLE(data, kBzip2MagicLE, kBzip2MagicBytes);+}++uint64_t Bzip2StreamCodec::doMaxCompressedLength(+    uint64_t uncompressedLength) const {+  // http://www.bzip.org/1.0.5/bzip2-manual-1.0.5.html#bzbufftobuffcompress+  //   To guarantee that the compressed data will fit in its buffer, allocate an+  //   output buffer of size 1% larger than the uncompressed data, plus six+  //   hundred extra bytes.+  return uncompressedLength + uncompressedLength / 100 + 600;+}++bz_stream createBzStream() {+  bz_stream stream;+  stream.bzalloc = nullptr;+  stream.bzfree = nullptr;+  stream.opaque = nullptr;+  stream.next_in = stream.next_out = nullptr;+  stream.avail_in = stream.avail_out = 0;+  return stream;+}++// Throws on error condition, otherwise returns the code.+int bzCheck(int const rc) {+  switch (rc) {+    case BZ_OK:+    case BZ_RUN_OK:+    case BZ_FLUSH_OK:+    case BZ_FINISH_OK:+    case BZ_STREAM_END:+    // Allow BZ_PARAM_ERROR.+    // It can get returned if no progress is made, but we handle that.+    case BZ_PARAM_ERROR:+      return rc;+    default:+      throw std::runtime_error(to<std::string>("Bzip2 error: ", rc));+  }+}++Bzip2StreamCodec::~Bzip2StreamCodec() {+  if (cstream_) {+    BZ2_bzCompressEnd(cstream_.get_pointer());+    cstream_.reset();+  }+  if (dstream_) {+    BZ2_bzDecompressEnd(dstream_.get_pointer());+    dstream_.reset();+  }+}++void Bzip2StreamCodec::doResetStream() {+  needReset_ = true;+}++void Bzip2StreamCodec::resetCStream() {+  if (cstream_) {+    BZ2_bzCompressEnd(cstream_.get_pointer());+  }+  cstream_ = createBzStream();+  bzCheck(BZ2_bzCompressInit(cstream_.get_pointer(), level_, 0, 0));+}++int bzip2TranslateFlush(StreamCodec::FlushOp flushOp) {+  switch (flushOp) {+    case StreamCodec::FlushOp::NONE:+      return BZ_RUN;+    case StreamCodec::FlushOp::END:+      return BZ_FINISH;+    case StreamCodec::FlushOp::FLUSH:+      throw std::invalid_argument(+          "Bzip2StreamCodec: FlushOp::FLUSH not supported");+    default:+      throw std::invalid_argument("Bzip2StreamCodec: Invalid flush");+  }+}++bool Bzip2StreamCodec::doCompressStream(+    ByteRange& input, MutableByteRange& output, StreamCodec::FlushOp flushOp) {+  // Bzip2 uses uint32_t for sizes, so we can't compress more than 4GB at a time+  return detail::chunkedStream(+      detail::kDefaultChunkSizeFor32BitSizes,+      input,+      output,+      flushOp,+      [this](auto& input, auto& output, auto flushOp) {+        if (needReset_) {+          resetCStream();+          needReset_ = false;+        }+        if (input.empty() && output.empty()) {+          return false;+        }++        cstream_->next_in =+            const_cast<char*>(reinterpret_cast<const char*>(input.data()));+        cstream_->avail_in = to_narrow(input.size());+        cstream_->next_out = reinterpret_cast<char*>(output.data());+        cstream_->avail_out = to_narrow(output.size());+        DCHECK_EQ(cstream_->avail_in, input.size());+        DCHECK_EQ(cstream_->avail_out, output.size());+        SCOPE_EXIT {+          input.uncheckedAdvance(input.size() - cstream_->avail_in);+          output.uncheckedAdvance(output.size() - cstream_->avail_out);+        };+        int const rc = bzCheck(BZ2_bzCompress(+            cstream_.get_pointer(), bzip2TranslateFlush(flushOp)));+        switch (flushOp) {+          case StreamCodec::FlushOp::NONE:+            return false;+          case StreamCodec::FlushOp::FLUSH:+            if (rc == BZ_RUN_OK) {+              DCHECK_EQ(cstream_->avail_in, 0);+              DCHECK(input.empty() || cstream_->avail_out != output.size());+              return true;+            }+            return false;+          case StreamCodec::FlushOp::END:+            return rc == BZ_STREAM_END;+          default:+            throw std::invalid_argument("Bzip2StreamCodec: invalid FlushOp");+        }+        return false;+      });+}++void Bzip2StreamCodec::resetDStream() {+  if (dstream_) {+    BZ2_bzDecompressEnd(dstream_.get_pointer());+  }+  dstream_ = createBzStream();+  bzCheck(BZ2_bzDecompressInit(dstream_.get_pointer(), 0, 0));+}++bool Bzip2StreamCodec::doUncompressStream(+    ByteRange& input, MutableByteRange& output, StreamCodec::FlushOp flushOp) {+  // Bzip2 uses uint32_t for sizes, so we can't uncompress more than 4GB at a+  // time+  return detail::chunkedStream(+      detail::kDefaultChunkSizeFor32BitSizes,+      input,+      output,+      flushOp,+      [this](auto& input, auto& output, auto flushOp) {+        if (flushOp == StreamCodec::FlushOp::FLUSH) {+          throw std::invalid_argument(+              "Bzip2StreamCodec: FlushOp::FLUSH not supported");+        }+        if (needReset_) {+          resetDStream();+          needReset_ = false;+        }++        dstream_->next_in =+            const_cast<char*>(reinterpret_cast<const char*>(input.data()));+        dstream_->avail_in = to_narrow(input.size());+        dstream_->next_out = reinterpret_cast<char*>(output.data());+        dstream_->avail_out = to_narrow(output.size());+        DCHECK_EQ(dstream_->avail_in, input.size());+        DCHECK_EQ(dstream_->avail_out, output.size());+        SCOPE_EXIT {+          input.uncheckedAdvance(input.size() - dstream_->avail_in);+          output.uncheckedAdvance(output.size() - dstream_->avail_out);+        };+        int const rc = bzCheck(BZ2_bzDecompress(dstream_.get_pointer()));+        return rc == BZ_STREAM_END;+      });+}++#endif // FOLLY_HAVE_LIBBZ2++#if FOLLY_HAVE_LIBZ++zlib::Options getZlibOptions(CodecType type) {+  DCHECK(type == CodecType::GZIP || type == CodecType::ZLIB);+  return type == CodecType::GZIP+      ? zlib::defaultGzipOptions()+      : zlib::defaultZlibOptions();+}++std::unique_ptr<Codec> getZlibCodec(int level, CodecType type) {+  return zlib::getCodec(getZlibOptions(type), level);+}++std::unique_ptr<StreamCodec> getZlibStreamCodec(int level, CodecType type) {+  return zlib::getStreamCodec(getZlibOptions(type), level);+}++#endif // FOLLY_HAVE_LIBZ++/**+ * Automatic decompression+ */+class AutomaticCodec final : public Codec {+ public:+  static std::unique_ptr<Codec> create(+      std::vector<std::unique_ptr<Codec>> customCodecs,+      std::unique_ptr<Codec> terminalCodec);+  explicit AutomaticCodec(+      std::vector<std::unique_ptr<Codec>> customCodecs,+      std::unique_ptr<Codec> terminalCodec);++  std::vector<std::string> validPrefixes() const override;+  bool canUncompress(+      const IOBuf* data, Optional<uint64_t> uncompressedLength) const override;++ private:+  bool doNeedsUncompressedLength() const override;+  uint64_t doMaxUncompressedLength() const override;++  uint64_t doMaxCompressedLength(uint64_t) const override {+    throw std::runtime_error(+        "AutomaticCodec error: maxCompressedLength() not supported.");+  }+  std::unique_ptr<IOBuf> doCompress(const IOBuf*) override {+    throw std::runtime_error("AutomaticCodec error: compress() not supported.");+  }+  std::unique_ptr<IOBuf> doUncompress(+      const IOBuf* data, Optional<uint64_t> uncompressedLength) override;++  void addCodecIfSupported(CodecType type);++  // Throws iff the codecs aren't compatible (very slow)+  void checkCompatibleCodecs() const;++  std::vector<std::unique_ptr<Codec>> codecs_;+  std::unique_ptr<Codec> terminalCodec_;+  bool needsUncompressedLength_;+  uint64_t maxUncompressedLength_;+};++std::vector<std::string> AutomaticCodec::validPrefixes() const {+  std::unordered_set<std::string> prefixes;+  for (const auto& codec : codecs_) {+    const auto codecPrefixes = codec->validPrefixes();+    prefixes.insert(codecPrefixes.begin(), codecPrefixes.end());+  }+  return std::vector<std::string>{prefixes.begin(), prefixes.end()};+}++bool AutomaticCodec::canUncompress(+    const IOBuf* data, Optional<uint64_t> uncompressedLength) const {+  return std::any_of(+      codecs_.begin(),+      codecs_.end(),+      [data, uncompressedLength](std::unique_ptr<Codec> const& codec) {+        return codec->canUncompress(data, uncompressedLength);+      });+}++void AutomaticCodec::addCodecIfSupported(CodecType type) {+  const bool present = std::any_of(+      codecs_.begin(),+      codecs_.end(),+      [&type](std::unique_ptr<Codec> const& codec) {+        return codec->type() == type;+      });+  bool const isTerminalType = terminalCodec_ && terminalCodec_->type() == type;+  if (hasCodec(type) && !present && !isTerminalType) {+    codecs_.push_back(getCodec(type));+  }+}++/* static */ std::unique_ptr<Codec> AutomaticCodec::create(+    std::vector<std::unique_ptr<Codec>> customCodecs,+    std::unique_ptr<Codec> terminalCodec) {+  return std::make_unique<AutomaticCodec>(+      std::move(customCodecs), std::move(terminalCodec));+}++AutomaticCodec::AutomaticCodec(+    std::vector<std::unique_ptr<Codec>> customCodecs,+    std::unique_ptr<Codec> terminalCodec)+    : Codec(CodecType::USER_DEFINED, folly::none, "auto"),+      codecs_(std::move(customCodecs)),+      terminalCodec_(std::move(terminalCodec)) {+  // Fastest -> slowest+  std::array<CodecType, 6> defaultTypes{{+      CodecType::LZ4_FRAME,+      CodecType::ZSTD,+      CodecType::ZLIB,+      CodecType::GZIP,+      CodecType::LZMA2,+      CodecType::BZIP2,+  }};++  for (auto type : defaultTypes) {+    addCodecIfSupported(type);+  }++  if (kIsDebug) {+    checkCompatibleCodecs();+  }++  // Check that none of the codecs are null+  DCHECK(std::none_of(+      codecs_.begin(), codecs_.end(), [](std::unique_ptr<Codec> const& codec) {+        return codec == nullptr;+      }));++  // Check that the terminal codec's type is not duplicated (with the exception+  // of USER_DEFINED).+  if (terminalCodec_) {+    DCHECK(std::none_of(+        codecs_.begin(),+        codecs_.end(),+        [&](std::unique_ptr<Codec> const& codec) {+          return codec->type() != CodecType::USER_DEFINED &&+              codec->type() == terminalCodec_->type();+        }));+  }++  bool const terminalNeedsUncompressedLength =+      terminalCodec_ && terminalCodec_->needsUncompressedLength();+  needsUncompressedLength_ =+      std::any_of(+          codecs_.begin(),+          codecs_.end(),+          [](std::unique_ptr<Codec> const& codec) {+            return codec->needsUncompressedLength();+          }) ||+      terminalNeedsUncompressedLength;++  const auto it = std::max_element(+      codecs_.begin(),+      codecs_.end(),+      [](std::unique_ptr<Codec> const& lhs, std::unique_ptr<Codec> const& rhs) {+        return lhs->maxUncompressedLength() < rhs->maxUncompressedLength();+      });+  DCHECK(it != codecs_.end());+  auto const terminalMaxUncompressedLength =+      terminalCodec_ ? terminalCodec_->maxUncompressedLength() : 0;+  maxUncompressedLength_ =+      std::max((*it)->maxUncompressedLength(), terminalMaxUncompressedLength);+}++void AutomaticCodec::checkCompatibleCodecs() const {+  // Keep track of all the possible headers.+  std::unordered_set<std::string> headers;+  // The empty header is not allowed.+  headers.insert("");+  // Step 1:+  // Construct a set of headers and check that none of the headers occur twice.+  // Eliminate edge cases.+  for (auto&& codec : codecs_) {+    const auto codecHeaders = codec->validPrefixes();+    // Codecs without any valid headers are not allowed.+    if (codecHeaders.empty()) {+      throw std::invalid_argument{+          "AutomaticCodec: validPrefixes() must not be empty."};+    }+    // Insert all the headers for the current codec.+    const size_t beforeSize = headers.size();+    headers.insert(codecHeaders.begin(), codecHeaders.end());+    // Codecs are not compatible if any header occurred twice.+    if (beforeSize + codecHeaders.size() != headers.size()) {+      throw std::invalid_argument{+          "AutomaticCodec: Two valid prefixes collide."};+    }+  }+  // Step 2:+  // Check if any strict non-empty prefix of any header is a header.+  for (const auto& header : headers) {+    for (size_t i = 1; i < header.size(); ++i) {+      if (headers.count(header.substr(0, i))) {+        throw std::invalid_argument{+            "AutomaticCodec: One valid prefix is a prefix of another valid "+            "prefix."};+      }+    }+  }+}++bool AutomaticCodec::doNeedsUncompressedLength() const {+  return needsUncompressedLength_;+}++uint64_t AutomaticCodec::doMaxUncompressedLength() const {+  return maxUncompressedLength_;+}++std::unique_ptr<IOBuf> AutomaticCodec::doUncompress(+    const IOBuf* data, Optional<uint64_t> uncompressedLength) {+  try {+    for (auto&& codec : codecs_) {+      if (codec->canUncompress(data, uncompressedLength)) {+        return codec->uncompress(data, uncompressedLength);+      }+    }+  } catch (std::exception const& e) {+    if (!terminalCodec_) {+      throw e;+    }+  }++  // Try terminal codec+  if (terminalCodec_) {+    return terminalCodec_->uncompress(data, uncompressedLength);+  }++  throw std::runtime_error("AutomaticCodec error: Unknown compressed data");+}++using CodecFactory = std::unique_ptr<Codec> (*)(int, CodecType);+using StreamCodecFactory = std::unique_ptr<StreamCodec> (*)(int, CodecType);+struct Factory {+  CodecFactory codec;+  StreamCodecFactory stream;+};++constexpr Factory codecFactories[static_cast<size_t>(+    CodecType::NUM_CODEC_TYPES)] = {+    {}, // USER_DEFINED+    {NoCompressionCodec::create, nullptr},++#if FOLLY_HAVE_LIBLZ4+    {LZ4Codec::create, nullptr},+#else+    {},+#endif++#if FOLLY_HAVE_LIBSNAPPY+    {SnappyCodec::create, nullptr},+#else+    {},+#endif++#if FOLLY_HAVE_LIBZ+    {getZlibCodec, getZlibStreamCodec},+#else+    {},+#endif++#if FOLLY_HAVE_LIBLZ4+    {LZ4Codec::create, nullptr},+#else+    {},+#endif++#if FOLLY_HAVE_LIBLZMA+    {LZMA2StreamCodec::createCodec, LZMA2StreamCodec::createStream},+    {LZMA2StreamCodec::createCodec, LZMA2StreamCodec::createStream},+#else+    {},+    {},+#endif++#if FOLLY_HAVE_LIBZSTD+    {getZstdCodec, getZstdStreamCodec},+#else+    {},+#endif++#if FOLLY_HAVE_LIBZ+    {getZlibCodec, getZlibStreamCodec},+#else+    {},+#endif++#if (FOLLY_HAVE_LIBLZ4 && LZ4_VERSION_NUMBER >= 10301)+    {LZ4FrameCodec::create, nullptr},+#else+    {},+#endif++#if FOLLY_HAVE_LIBBZ2+    {Bzip2StreamCodec::createCodec, Bzip2StreamCodec::createStream},+#else+    {},+#endif++#if FOLLY_HAVE_LIBZSTD+    {getZstdFastCodec, getZstdFastStreamCodec},+#else+    {},+#endif+};++Factory const& getFactory(CodecType type) {+  auto const idx = static_cast<size_t>(type);+  if (idx >= static_cast<size_t>(CodecType::NUM_CODEC_TYPES)) {+    throw std::invalid_argument(+        to<std::string>("Compression type ", idx, " invalid"));+  }+  return codecFactories[idx];+}+} // namespace++bool hasCodec(CodecType type) {+  return getFactory(type).codec != nullptr;+}++std::unique_ptr<Codec> getCodec(CodecType type, int level) {+  auto const factory = getFactory(type).codec;+  if (!factory) {+    throw std::invalid_argument(+        to<std::string>("Compression type ", type, " not supported"));+  }+  auto codec = (*factory)(level, type);+  DCHECK(codec->type() == type);+  return codec;+}++bool hasStreamCodec(CodecType type) {+  return getFactory(type).stream != nullptr;+}++std::unique_ptr<StreamCodec> getStreamCodec(CodecType type, int level) {+  auto const factory = getFactory(type).stream;+  if (!factory) {+    throw std::invalid_argument(+        to<std::string>("Compression type ", type, " not supported"));+  }+  auto codec = (*factory)(level, type);+  DCHECK(codec->type() == type);+  return codec;+}++std::unique_ptr<Codec> getAutoUncompressionCodec(+    std::vector<std::unique_ptr<Codec>> customCodecs,+    std::unique_ptr<Codec> terminalCodec) {+  return AutomaticCodec::create(+      std::move(customCodecs), std::move(terminalCodec));+}+} // namespace compression+} // namespace folly
+ folly/folly/compression/Compression.h view
@@ -0,0 +1,547 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <cstdint>+#include <limits>+#include <memory>+#include <string>+#include <vector>++#include <folly/Optional.h>+#include <folly/Range.h>+#include <folly/io/IOBuf.h>++/**+ * Compression / decompression over IOBufs+ */++namespace folly {+namespace compression {++enum class CodecType {+  /**+   * This codec type is not defined; getCodec() will throw an exception+   * if used. Useful if deriving your own classes from Codec without+   * going through the getCodec() interface.+   */+  USER_DEFINED = 0,++  /**+   * Use no compression.+   * Levels supported: 0+   */+  NO_COMPRESSION = 1,++  /**+   * Use LZ4 compression.+   * Levels supported: 1 = fast, 2 = best; default = 1+   */+  LZ4 = 2,++  /**+   * Use Snappy compression.+   * Levels supported: 1+   */+  SNAPPY = 3,++  /**+   * Use zlib compression.+   * Levels supported: 0 = no compression, 1 = fast, ..., 9 = best; default = 6+   * Streaming compression is supported.+   */+  ZLIB = 4,++  /**+   * Use LZ4 compression, prefixed with size (as Varint).+   */+  LZ4_VARINT_SIZE = 5,++  /**+   * Use LZMA2 compression.+   * Levels supported: 0 = no compression, 1 = fast, ..., 9 = best; default = 6+   * Streaming compression is supported.+   */+  LZMA2 = 6,+  LZMA2_VARINT_SIZE = 7,++  /**+   * Use ZSTD compression.+   * Levels supported: 1 = fast, ..., 19 = best; default = 1+   * Use ZSTD_FAST for the fastest zstd compression (negative levels).+   * Streaming compression is supported.+   */+  ZSTD = 8,++  /**+   * Use gzip compression.  This is the same compression algorithm as ZLIB but+   * gzip-compressed files tend to be easier to work with from the command line.+   * Levels supported: 0 = no compression, 1 = fast, ..., 9 = best; default = 6+   * Streaming compression is supported.+   */+  GZIP = 9,++  /**+   * Use LZ4 frame compression.+   * Levels supported: 0 = fast, 16 = best; default = 0+   */+  LZ4_FRAME = 10,++  /**+   * Use bzip2 compression.+   * Levels supported: 1 = fast, 9 = best; default = 9+   * Streaming compression is supported BUT FlushOp::FLUSH does NOT ensure that+   * the decompressor can read all the data up to that point, due to a bug in+   * the bzip2 library.+   */+  BZIP2 = 11,++  /**+   * Use ZSTD compression with a negative compression level (1=-1, 2=-2, ...).+   * Higher compression levels mean faster.+   * Level 1 is around the same speed as Snappy with better compression.+   * Level 5 is around the same speed as LZ4 with slightly worse compression.+   * Each level gains about 6-15% speed and loses 3-7% compression.+   * Decompression speed improves for each level, and level 1 decompression+   * speed is around 25% faster than ZSTD.+   * This codec is fully compatible with ZSTD.+   * Levels supported: 1 = best, ..., 5 = fast; default = 1+   * Streaming compression is supported.+   */+  ZSTD_FAST = 12,++  NUM_CODEC_TYPES = 13,+};++class Codec {+ public:+  virtual ~Codec() {}++  static constexpr uint64_t UNLIMITED_UNCOMPRESSED_LENGTH = uint64_t(-1);+  /**+   * Return the maximum length of data that may be compressed with this codec.+   * NO_COMPRESSION and ZLIB support arbitrary lengths;+   * LZ4 supports up to 1.9GiB; SNAPPY supports up to 4GiB.+   * May return UNLIMITED_UNCOMPRESSED_LENGTH if unlimited.+   */+  uint64_t maxUncompressedLength() const;++  /**+   * Return the codec's type.+   */+  CodecType type() const { return type_; }++  /**+   * Does this codec need the exact uncompressed length on decompression?+   */+  bool needsUncompressedLength() const;++  /**+   * Compress data, returning an IOBuf (which may share storage with data).+   * Throws std::invalid_argument if data is larger than+   * maxUncompressedLength().+   */+  std::unique_ptr<IOBuf> compress(const folly::IOBuf* data);++  /**+   * Compresses data. May involve additional copies compared to the overload+   * that takes and returns IOBufs. Has the same error semantics as the IOBuf+   * version.+   */+  std::string compress(StringPiece data);++  /**+   * Uncompress data. Throws std::runtime_error on decompression error.+   *+   * Some codecs (LZ4) require the exact uncompressed length; this is indicated+   * by needsUncompressedLength().+   *+   * For other codes (zlib), knowing the exact uncompressed length ahead of+   * time might be faster.+   *+   * Regardless of the behavior of the underlying compressor, uncompressing+   * an empty IOBuf chain will return an empty IOBuf chain.+   */+  std::unique_ptr<IOBuf> uncompress(+      const IOBuf* data,+      folly::Optional<uint64_t> uncompressedLength = folly::none);++  /**+   * Uncompresses data. May involve additional copies compared to the overload+   * that takes and returns IOBufs. Has the same error semantics as the IOBuf+   * version.+   */+  std::string uncompress(+      StringPiece data,+      folly::Optional<uint64_t> uncompressedLength = folly::none);++  /**+   * Returns a bound on the maximum compressed length when compressing data with+   * the given uncompressed length.+   */+  uint64_t maxCompressedLength(uint64_t uncompressedLength) const;++  /**+   * Extracts the uncompressed length from the compressed data if possible.+   * If the codec doesn't store the uncompressed length, or the data is+   * corrupted it returns the given uncompressedLength.+   * If the uncompressed length is stored in the compressed data and+   * uncompressedLength is not none and they do not match a std::runtime_error+   * is thrown.+   */+  folly::Optional<uint64_t> getUncompressedLength(+      const folly::IOBuf* data,+      folly::Optional<uint64_t> uncompressedLength = folly::none) const;++  /**+   * Helper wrapper around getUncompressedLength(IOBuf)+   */+  folly::Optional<uint64_t> getUncompressedLength(+      folly::StringPiece data,+      folly::Optional<uint64_t> uncompressedLength = folly::none) const;++ protected:+  Codec(+      CodecType type,+      folly::Optional<int> level = folly::none,+      folly::StringPiece name = {});++ public:+  /**+   * Returns a superset of the set of prefixes for which canUncompress() will+   * return true. A superset is allowed for optimizations in canUncompress()+   * based on other knowledge such as length. None of the prefixes may be empty.+   * default: No prefixes.+   */+  virtual std::vector<std::string> validPrefixes() const;++  /**+   * Returns true if the codec thinks it can uncompress the data.+   * If a codec doesn't have magic bytes at the beginning, like LZ4 and Snappy,+   * it can always return false.+   * default: Returns false.+   */+  virtual bool canUncompress(+      const folly::IOBuf* data,+      folly::Optional<uint64_t> uncompressedLength = folly::none) const;++  /**+   * Helper wrapper around canUncompress(IOBuf)+   */+  bool canUncompress(+      folly::StringPiece data,+      folly::Optional<uint64_t> uncompressedLength = folly::none) const;++ private:+  // default: no limits (save for special value UNKNOWN_UNCOMPRESSED_LENGTH)+  virtual uint64_t doMaxUncompressedLength() const;+  // default: doesn't need uncompressed length+  virtual bool doNeedsUncompressedLength() const;+  virtual std::unique_ptr<IOBuf> doCompress(const folly::IOBuf* data) = 0;+  virtual std::unique_ptr<IOBuf> doUncompress(+      const folly::IOBuf* data,+      folly::Optional<uint64_t> uncompressedLength) = 0;+  // default: an implementation is provided by default to wrap the strings into+  // IOBufs and delegate to the IOBuf methods. This incurs a copy of the output+  // from IOBuf to string. Implementers, at their discretion, can override+  // these methods to avoid the copy.+  virtual std::string doCompressString(StringPiece data);+  virtual std::string doUncompressString(+      StringPiece data, folly::Optional<uint64_t> uncompressedLength);++  virtual uint64_t doMaxCompressedLength(uint64_t uncompressedLength) const = 0;+  // default: returns the passed uncompressedLength.+  virtual folly::Optional<uint64_t> doGetUncompressedLength(+      const folly::IOBuf* data,+      folly::Optional<uint64_t> uncompressedLength) const;++  CodecType type_;+};++class StreamCodec : public Codec {+ public:+  ~StreamCodec() override {}++  /**+   * Does the codec need the data length before compression streaming?+   */+  bool needsDataLength() const;++  /*****************************************************************************+   * Streaming API+   *****************************************************************************+   * A low-level stateful streaming API.+   * Streaming operations can be started in two ways:+   *   1. From a clean Codec on which no non-const methods have been called.+   *   2. A call to resetStream(), which will reset any codec to a clean state.+   * After a streaming operation has begun, either compressStream() or+   * uncompressStream() must be called until the streaming operation ends.+   * compressStream() ends when it returns true with flushOp END.+   * uncompressStream() ends when it returns true. At this point the codec+   * may be reused by calling resetStream().+   *+   * compress() and uncompress() can be called at any time, but they interrupt+   * any ongoing streaming operations (state is lost and resetStream() must be+   * called before another streaming operation).+   */++  /**+   * Reset the state of the codec, and set the uncompressed length for the next+   * streaming operation. If uncompressedLength is not none it must be exactly+   * the uncompressed length. compressStream() must be passed exactly+   * uncompressedLength input bytes before the stream is ended.+   * uncompressStream() must be passed a compressed frame that uncompresses to+   * uncompressedLength.+   */+  void resetStream(folly::Optional<uint64_t> uncompressedLength = folly::none);++  enum class FlushOp { NONE, FLUSH, END };++  /**+   * Compresses some data from the input buffer and writes the compressed data+   * into the output buffer. It may read input without producing any output,+   * except when forced to flush.+   *+   * The input buffer is advanced to point to the range of data that hasn't yet+   * been read. Compression will resume at this point for the next call to+   * compressStream(). The output buffer is advanced one byte past the last byte+   * written.+   *+   * The default flushOp is NONE, which allows compressStream() complete+   * discretion in how much data to gather before writing any output.+   * Always returns false in with FlushOp::NONE.+   *+   * If flushOp is END, all pending and input data is flushed to the output+   * buffer, and the frame is ended. compressStream() must be called with the+   * same input and flushOp END until it returns true. At this point the caller+   * must call resetStream() to use the codec again.+   *+   * If flushOp is FLUSH, all pending and input data is flushed to the output+   * buffer, but the frame is not ended. compressStream() must be called with+   * the same input and flushOp FLUSH until it returns true. At this point the+   * caller can continue to compressStream() with any input data and flushOp.+   * The uncompressor, if passed all the produced output data, will be able to+   * uncompress all the input data passed to compressStream() so far. Excessive+   * use of flushOp FLUSH will deteriorate compression ratio. This is useful for+   * stateful streaming across a network. Most users don't need to use this+   * flushOp.+   *+   * A std::logic_error is thrown on incorrect usage of the API.+   * A std::runtime_error is thrown upon error conditions or if no forward+   * progress could be made twice in a row.+   *+   * @returns true iff @p flushOp is FLUSH or END and the entire @p input+   * has been consumed and flushed into @p output. If @p flushOp is END, then+   * the compressed frame is complete. Always returns false if @p flushOp is+   * NONE.+   */+  bool compressStream(+      folly::ByteRange& input,+      folly::MutableByteRange& output,+      FlushOp flushOp = StreamCodec::FlushOp::NONE);++  /**+   * Uncompresses some data from the input buffer and writes the uncompressed+   * data into the output buffer. It may read input without producing any+   * output.+   *+   * The input buffer is advanced to point to the range of data that hasn't yet+   * been read. Uncompression will resume at this point for the next call to+   * uncompressStream(). The output buffer is advanced one byte past the last+   * byte written.+   *+   * The default flushOp is NONE, which allows uncompressStream() complete+   * discretion in how much output data to flush. The uncompressor may not make+   * maximum forward progress, but will make some forward progress when+   * possible.+   *+   * If flushOp is END, the caller guarantees that no more input will be+   * presented to uncompressStream(). uncompressStream() must be called with the+   * same input and flushOp END until it returns true. This is not mandatory,+   * but if the input is all available in one buffer, and there is enough output+   * space to write the entire frame, codecs can uncompress faster.+   *+   * If flushOp is FLUSH, uncompressStream() is guaranteed to make the maximum+   * amount of forward progress possible. When using this flushOp and+   * uncompressStream() returns with `!output.empty()` the caller knows that all+   * pending output has been flushed. This is useful for stateful streaming+   * across a network, and it should be used in conjunction with+   * compressStream() with flushOp FLUSH. Most users don't need to use this+   * flushOp.+   *+   * A std::runtime_error is thrown upon error conditions or if no forward+   * progress could be made upon two consecutive calls to the function (only the+   * second call will throw an exception).+   *+   * @returns true at the end of a frame. At this point resetStream() must be+   * called to reuse the codec.+   */+  bool uncompressStream(+      folly::ByteRange& input,+      folly::MutableByteRange& output,+      FlushOp flushOp = StreamCodec::FlushOp::NONE);++ protected:+  StreamCodec(+      CodecType type,+      folly::Optional<int> level = folly::none,+      folly::StringPiece name = {})+      : Codec(type, std::move(level), name) {}++  // Returns the uncompressed length last passed to resetStream() or none if it+  // hasn't been called yet.+  folly::Optional<uint64_t> uncompressedLength() const {+    return uncompressedLength_;+  }++ private:+  // default: Implemented using the streaming API.+  std::unique_ptr<IOBuf> doCompress(const folly::IOBuf* data) override;+  std::unique_ptr<IOBuf> doUncompress(+      const folly::IOBuf* data,+      folly::Optional<uint64_t> uncompressedLength) override;++  // default: Returns false+  virtual bool doNeedsDataLength() const;+  virtual void doResetStream() = 0;+  virtual bool doCompressStream(+      folly::ByteRange& input,+      folly::MutableByteRange& output,+      FlushOp flushOp) = 0;+  virtual bool doUncompressStream(+      folly::ByteRange& input,+      folly::MutableByteRange& output,+      FlushOp flushOp) = 0;++  enum class State {+    RESET,+    COMPRESS,+    COMPRESS_FLUSH,+    COMPRESS_END,+    UNCOMPRESS,+    END,+  };+  void assertStateIs(State expected) const;++  State state_{State::RESET};+  ByteRange previousInput_{};+  folly::Optional<uint64_t> uncompressedLength_{};+  bool progressMade_{true};+};++constexpr int COMPRESSION_LEVEL_FASTEST = -1;+constexpr int COMPRESSION_LEVEL_DEFAULT = -2;+constexpr int COMPRESSION_LEVEL_BEST = -3;++/**+ * Return a codec for the given type. Throws on error.  The level+ * is a non-negative codec-dependent integer indicating the level of+ * compression desired, or one of the following constants:+ *+ * COMPRESSION_LEVEL_FASTEST is fastest (uses least CPU / memory,+ *   worst compression)+ * COMPRESSION_LEVEL_DEFAULT is the default (likely a tradeoff between+ *   FASTEST and BEST)+ * COMPRESSION_LEVEL_BEST is the best compression (uses most CPU / memory,+ *   best compression)+ *+ * When decompressing, the compression level is ignored. All codecs will+ * decompress all data compressed with the a codec of the same type, regardless+ * of compression level.+ */+std::unique_ptr<Codec> getCodec(+    CodecType type, int level = COMPRESSION_LEVEL_DEFAULT);++/**+ * Return a codec for the given type. Throws on error.  The level+ * is a non-negative codec-dependent integer indicating the level of+ * compression desired, or one of the following constants:+ *+ * COMPRESSION_LEVEL_FASTEST is fastest (uses least CPU / memory,+ *   worst compression)+ * COMPRESSION_LEVEL_DEFAULT is the default (likely a tradeoff between+ *   FASTEST and BEST)+ * COMPRESSION_LEVEL_BEST is the best compression (uses most CPU / memory,+ *   best compression)+ *+ * When decompressing, the compression level is ignored. All codecs will+ * decompress all data compressed with the a codec of the same type, regardless+ * of compression level.+ */+std::unique_ptr<StreamCodec> getStreamCodec(+    CodecType type, int level = COMPRESSION_LEVEL_DEFAULT);++/**+ * Returns a codec that can uncompress any of the given codec types as well as+ * {LZ4_FRAME, ZSTD, ZLIB, GZIP, LZMA2, BZIP2}. Appends each default codec to+ * customCodecs in order, so long as a codec with the same type() isn't already+ * present in customCodecs or as the terminalCodec. When uncompress() is called,+ * each codec's canUncompress() is called in the order that they are given.+ * Appended default codecs are checked last.  uncompress() is called on the+ * first codec whose canUncompress() returns true.+ *+ * In addition, an optional `terminalCodec` can be provided. This codec's+ * uncompress() will be called either when no other codec canUncompress() the+ * data or the chosen codec throws an exception on the data. The terminalCodec+ * is intended for ambiguous headers, when canUncompress() is false for some+ * data it can actually uncompress. The terminalCodec does not need to override+ * validPrefixes() or canUncompress() and overriding these functions will have+ * no effect on the returned codec's validPrefixes() or canUncompress()+ * functions. The terminalCodec's needsUncompressedLength() and+ * maxUncompressedLength() will affect the returned codec's respective+ * functions. The terminalCodec must not be duplicated in customCodecs.+ *+ * An exception is thrown if no codec canUncompress() the data and either no+ * terminal codec was provided or a terminal codec was provided and it throws on+ * the data.+ * An exception is thrown if the chosen codec's uncompress() throws on the data+ * and either no terminal codec was provided or a terminal codec was provided+ * and it also throws on the data.+ * An exception is thrown if compress() is called on the returned codec.+ *+ * Requirements are checked in debug mode and are as follows:+ * Let headers be the concatenation of every codec's validPrefixes().+ *  1. Each codec must override validPrefixes() and canUncompress().+ *  2. No codec's validPrefixes() may be empty.+ *  3. No header in headers may be empty.+ *  4. headers must not contain any duplicate elements.+ *  5. No strict non-empty prefix of any header in headers may be in headers.+ *  6. The terminalCodec's type must not be the same as any other codec's type+ *     (with USER_DEFINED being the exception).+ */+std::unique_ptr<Codec> getAutoUncompressionCodec(+    std::vector<std::unique_ptr<Codec>> customCodecs = {},+    std::unique_ptr<Codec> terminalCodec = {});++/**+ * Check if a specified codec is supported.+ */+bool hasCodec(CodecType type);++/**+ * Check if a specified codec is supported and supports streaming.+ */+bool hasStreamCodec(CodecType type);++/**+ * Added here so users of folly can figure out whether the header+ * folly/compression/CompressionContextPoolSingletons.h is present, and+ * therefore whether it can be included.+ */+#define FOLLY_COMPRESSION_HAS_CONTEXT_POOL_SINGLETONS+} // namespace compression+} // namespace folly
+ folly/folly/compression/CompressionContextPool.h view
@@ -0,0 +1,114 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <memory>++#include <folly/Memory.h>+#include <folly/Synchronized.h>++namespace folly {+namespace compression {++template <typename T, typename Creator, typename Deleter, typename Resetter>+class CompressionContextPool {+ private:+  using InternalRef = std::unique_ptr<T, Deleter>;++  class ReturnToPoolDeleter {+   public:+    using Pool = CompressionContextPool<T, Creator, Deleter, Resetter>;++    explicit ReturnToPoolDeleter(Pool* pool) : pool_(pool) { DCHECK(pool); }++    void operator()(T* t) {+      InternalRef ptr(t, pool_->deleter_);+      pool_->add(std::move(ptr));+    }++   private:+    Pool* pool_;+  };++ public:+  using Object = T;+  using Ref = std::unique_ptr<T, ReturnToPoolDeleter>;++  explicit CompressionContextPool(+      Creator creator = Creator(),+      Deleter deleter = Deleter(),+      Resetter resetter = Resetter())+      : creator_(std::move(creator)),+        deleter_(std::move(deleter)),+        resetter_(std::move(resetter)),+        stack_(),+        created_(0) {}++  Ref get() {+    auto stack = stack_.wlock();+    if (stack->empty()) {+      T* t = creator_();+      if (t == nullptr) {+        throw_exception<std::bad_alloc>();+      }+      created_++;+      return Ref(t, get_deleter());+    }+    auto ptr = std::move(stack->back());+    stack->pop_back();+    if (!ptr) {+      throw_exception<std::logic_error>(+          "A nullptr snuck into our context pool!?!?");+    }+    return Ref(ptr.release(), get_deleter());+  }++  size_t created_count() const { return created_.load(); }++  size_t size() { return stack_.rlock()->size(); }++  ReturnToPoolDeleter get_deleter() { return ReturnToPoolDeleter(this); }++  Resetter& get_resetter() { return resetter_; }++  void flush_deep() {+    flush_shallow();+    // no backing stack, so deep == shallow+  }++  void flush_shallow() {+    auto stack = stack_.wlock();+    stack->resize(0);+  }++ private:+  void add(InternalRef ptr) {+    DCHECK(ptr);+    resetter_(ptr.get());+    stack_.wlock()->push_back(std::move(ptr));+  }++  Creator creator_;+  Deleter deleter_;+  Resetter resetter_;++  folly::Synchronized<std::vector<InternalRef>> stack_;++  std::atomic<size_t> created_;+};+} // namespace compression+} // namespace folly
+ folly/folly/compression/CompressionContextPoolSingletons.cpp view
@@ -0,0 +1,150 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#include <folly/compression/CompressionContextPoolSingletons.h>++#include <stdlib.h>++#include <folly/Portability.h>+#include <folly/memory/Malloc.h>++#ifndef FOLLY_COMPRESSION_USE_HUGEPAGES+#if defined(__linux__) && !defined(__ANDROID__)+#define FOLLY_COMPRESSION_USE_HUGEPAGES 1+#else+#define FOLLY_COMPRESSION_USE_HUGEPAGES 0+#endif+#endif++#if FOLLY_COMPRESSION_USE_HUGEPAGES+#include <folly/memory/JemallocHugePageAllocator.h>+#endif++#if FOLLY_HAVE_LIBZSTD+#ifndef ZSTD_STATIC_LINKING_ONLY+#define ZSTD_STATIC_LINKING_ONLY+#endif+#include <zstd.h>+#endif++namespace folly {+namespace compression {+namespace contexts {++#if FOLLY_HAVE_LIBZSTD+namespace {+// These objects have no static dependencies and therefore no SIOF issues.+ZSTD_CCtx_Pool zstd_cctx_pool_singleton;+ZSTD_DCtx_Pool zstd_dctx_pool_singleton;++#if FOLLY_COMPRESSION_USE_HUGEPAGES+constexpr bool use_huge_pages = kIsArchAmd64;++void* huge_page_alloc(void*, size_t size) {+  if (size < 16 * 4096) {+    // Arbritrary cutoff: ZSTD_CCtx'es only ever make two kinds of allocations:+    // 1. one small one for the CCtx itself.+    // 2. "big" ones for the workspace (ZSTD_cwksp)+    // The CCtx allocation doesn't need to be in a huge page.+    return malloc(size);+  }+  return JemallocHugePageAllocator::allocate(size);+}++void huge_page_free(void*, void* address) {+  if (address != nullptr) {+    if (JemallocHugePageAllocator::addressInArena(address)) {+      JemallocHugePageAllocator::deallocate(address);+    } else {+      free(address);+    }+  }+}++ZSTD_customMem huge_page_custom_mem = (use_huge_pages && usingJEMalloc())+    ? (ZSTD_customMem){huge_page_alloc, huge_page_free, nullptr}+    : ZSTD_defaultCMem;+#else+ZSTD_customMem huge_page_custom_mem = ZSTD_defaultCMem;+#endif++} // anonymous namespace++ZSTD_CCtx* ZSTD_CCtx_Creator::operator()() const noexcept {+  return ZSTD_createCCtx_advanced(huge_page_custom_mem);+}++ZSTD_DCtx* ZSTD_DCtx_Creator::operator()() const noexcept {+  return ZSTD_createDCtx_advanced(huge_page_custom_mem);+}++void ZSTD_CCtx_Deleter::operator()(ZSTD_CCtx* ctx) const noexcept {+  ZSTD_freeCCtx(ctx);+}++void ZSTD_DCtx_Deleter::operator()(ZSTD_DCtx* ctx) const noexcept {+  ZSTD_freeDCtx(ctx);+}++void ZSTD_CCtx_Resetter::operator()(ZSTD_CCtx* ctx) const noexcept {+  size_t const err = ZSTD_CCtx_reset(ctx, ZSTD_reset_session_and_parameters);+  assert(!ZSTD_isError(err)); // This function doesn't actually fail+  (void)err;+}++void ZSTD_DCtx_Resetter::operator()(ZSTD_DCtx* ctx) const noexcept {+  size_t const err = ZSTD_DCtx_reset(ctx, ZSTD_reset_session_and_parameters);+  assert(!ZSTD_isError(err)); // This function doesn't actually fail+  (void)err;+}++ZSTD_CCtx_Pool::Ref getZSTD_CCtx() {+  return zstd_cctx_pool_singleton.get();+}++ZSTD_DCtx_Pool::Ref getZSTD_DCtx() {+  return zstd_dctx_pool_singleton.get();+}++ZSTD_CCtx_Pool::Ref getNULL_ZSTD_CCtx() {+  return zstd_cctx_pool_singleton.getNull();+}++ZSTD_DCtx_Pool::Ref getNULL_ZSTD_DCtx() {+  return zstd_dctx_pool_singleton.getNull();+}++ZSTD_CCtx_Pool& zstd_cctx_pool() {+  return zstd_cctx_pool_singleton;+}++ZSTD_DCtx_Pool& zstd_dctx_pool() {+  return zstd_dctx_pool_singleton;+}++size_t get_zstd_cctx_created_count() {+  return zstd_cctx_pool_singleton.created_count();+}++size_t get_zstd_dctx_created_count() {+  return zstd_dctx_pool_singleton.created_count();+}++#endif // FOLLY_HAVE_LIBZSTD++} // namespace contexts+} // namespace compression+} // namespace folly
+ folly/folly/compression/CompressionContextPoolSingletons.h view
@@ -0,0 +1,102 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <folly/portability/Config.h>++#if FOLLY_HAVE_LIBZSTD+#include <zstd.h>+#endif++#include <folly/compression/CompressionCoreLocalContextPool.h>++// When this header is present, folly/compression/Compression.h defines+// FOLLY_COMPRESSION_HAS_CONTEXT_POOL_SINGLETONS.++namespace folly {+namespace compression {+namespace contexts {++#if FOLLY_HAVE_LIBZSTD++// Additional feature test macro for zstd singletons.+#define FOLLY_COMPRESSION_HAS_ZSTD_CONTEXT_POOL_SINGLETONS++struct ZSTD_CCtx_Creator {+  ZSTD_CCtx* operator()() const noexcept;+};++struct ZSTD_DCtx_Creator {+  ZSTD_DCtx* operator()() const noexcept;+};++struct ZSTD_CCtx_Deleter {+  void operator()(ZSTD_CCtx* ctx) const noexcept;+};++struct ZSTD_DCtx_Deleter {+  void operator()(ZSTD_DCtx* ctx) const noexcept;+};++struct ZSTD_CCtx_Resetter {+  void operator()(ZSTD_CCtx* ctx) const noexcept;+};++struct ZSTD_DCtx_Resetter {+  void operator()(ZSTD_DCtx* ctx) const noexcept;+};++using ZSTD_CCtx_Pool = CompressionCoreLocalContextPool<+    ZSTD_CCtx,+    ZSTD_CCtx_Creator,+    ZSTD_CCtx_Deleter,+    ZSTD_CCtx_Resetter,+    4>;+using ZSTD_DCtx_Pool = CompressionCoreLocalContextPool<+    ZSTD_DCtx,+    ZSTD_DCtx_Creator,+    ZSTD_DCtx_Deleter,+    ZSTD_DCtx_Resetter,+    16>;++/**+ * Returns a clean ZSTD_CCtx.+ */+ZSTD_CCtx_Pool::Ref getZSTD_CCtx();++/**+ * Returns a clean ZSTD_DCtx.+ */+ZSTD_DCtx_Pool::Ref getZSTD_DCtx();++ZSTD_CCtx_Pool::Ref getNULL_ZSTD_CCtx();++ZSTD_DCtx_Pool::Ref getNULL_ZSTD_DCtx();++ZSTD_CCtx_Pool& zstd_cctx_pool();++ZSTD_DCtx_Pool& zstd_dctx_pool();++size_t get_zstd_cctx_created_count();++size_t get_zstd_dctx_created_count();++#endif // FOLLY_HAVE_LIBZSTD++} // namespace contexts+} // namespace compression+} // namespace folly
+ folly/folly/compression/CompressionCoreLocalContextPool.h view
@@ -0,0 +1,142 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <folly/compression/CompressionContextPool.h>+#include <folly/concurrency/CacheLocality.h>++namespace folly {+namespace compression {++/**+ * This class is intended to reduce contention on reserving a compression+ * context and improve cache locality (but maybe not hotness) of the contexts+ * it manages.+ *+ * This class uses folly::AccessSpreader to spread the managed object across+ * NumStripes domains (which should correspond to a topologically close set of+ * hardware threads). This cache is still backed by the basic locked stack in+ * the folly::compression::CompressionContextPool.+ *+ * Note that there is a tradeoff in choosing the number of stripes. More stripes+ * make for less contention, but mean that a context is less likely to be hot+ * in cache.+ */+template <+    typename T,+    typename Creator,+    typename Deleter,+    typename Resetter,+    size_t NumStripes = 8>+class CompressionCoreLocalContextPool {+ private:+  /**+   * Force each pointer to be on a different cache line.+   */+  class alignas(folly::hardware_destructive_interference_size) Storage {+   public:+    Storage() : ptr(nullptr) {}++    std::atomic<T*> ptr;+  };++  class ReturnToPoolDeleter {+   public:+    using Pool = CompressionCoreLocalContextPool<+        T,+        Creator,+        Deleter,+        Resetter,+        NumStripes>;++    explicit ReturnToPoolDeleter(Pool* pool) : pool_(pool) { DCHECK(pool_); }++    void operator()(T* ptr) { pool_->store(ptr); }++   private:+    Pool* pool_;+  };++  using BackingPool = CompressionContextPool<T, Creator, Deleter, Resetter>;+  using BackingPoolRef = typename BackingPool::Ref;++ public:+  using Object = T;+  using Ref = std::unique_ptr<T, ReturnToPoolDeleter>;++  explicit CompressionCoreLocalContextPool(+      Creator creator = Creator(),+      Deleter deleter = Deleter(),+      Resetter resetter = Resetter())+      : pool_(std::move(creator), std::move(deleter), std::move(resetter)),+        caches_() {}++  ~CompressionCoreLocalContextPool() { flush_shallow(); }++  Ref get() {+    auto ptr = local().ptr.exchange(nullptr);+    if (ptr == nullptr) {+      // no local ctx, get from backing pool+      ptr = pool_.get().release();+      DCHECK(ptr);+    }+    return Ref(ptr, get_deleter());+  }++  Ref getNull() { return Ref(nullptr, get_deleter()); }++  size_t created_count() const { return pool_.created_count(); }++  void flush_deep() {+    flush_shallow();+    pool_.flush_deep();+  }++  void flush_shallow() {+    for (auto& cache : caches_) {+      // Return all cached contexts back to the backing pool.+      auto ptr = cache.ptr.exchange(nullptr);+      return_to_backing_pool(ptr);+    }+  }++ private:+  ReturnToPoolDeleter get_deleter() { return ReturnToPoolDeleter(this); }++  void store(T* ptr) {+    DCHECK(ptr);+    pool_.get_resetter()(ptr);+    auto other = local().ptr.exchange(ptr);+    if (other != nullptr) {+      return_to_backing_pool(other);+    }+  }++  void return_to_backing_pool(T* ptr) {+    BackingPoolRef(ptr, pool_.get_deleter());+  }++  Storage& local() {+    const auto idx = folly::AccessSpreader<>::cachedCurrent(NumStripes);+    return caches_[idx];+  }++  BackingPool pool_;+  std::array<Storage, NumStripes> caches_{};+};+} // namespace compression+} // namespace folly
+ folly/folly/compression/Instructions.h view
@@ -0,0 +1,211 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <glog/logging.h>++#ifdef _MSC_VER+#include <immintrin.h>+#endif++#include <string_view>++#include <folly/CpuId.h>+#include <folly/Portability.h>+#include <folly/lang/Assume.h>+#include <folly/portability/Builtins.h>++namespace folly {+namespace compression {+namespace instructions {++// NOTE: It's recommended to compile EF coding with -msse4.2, starting+// with Nehalem, Intel CPUs support POPCNT instruction and gcc will emit+// it for __builtin_popcountll intrinsic.+// But we provide an alternative way for the client code: it can switch to+// the appropriate version of EliasFanoReader<> at runtime (client should+// implement this switching logic itself) by specifying instruction set to+// use explicitly.++struct Default {+  static std::string_view name() noexcept { return "Default"; }+  static bool supported(const folly::CpuId& /* cpuId */ = {}) { return true; }+  static FOLLY_ALWAYS_INLINE uint64_t popcount(uint64_t value) {+    return uint64_t(__builtin_popcountll(value));+  }+  static FOLLY_ALWAYS_INLINE int ctz(uint64_t value) {+    DCHECK_GT(value, 0u);+    return __builtin_ctzll(value);+  }+  static FOLLY_ALWAYS_INLINE int clz(uint64_t value) {+    DCHECK_GT(value, 0u);+    return __builtin_clzll(value);+  }+  static FOLLY_ALWAYS_INLINE uint64_t blsr(uint64_t value) {+    return value & (value - 1);+  }++  // Extract `length` bits starting from `start` from value. Only bits [0:63]+  // will be extracted. All higher order bits in the+  // result will be zeroed. If no bits are extracted, return 0.+  static FOLLY_ALWAYS_INLINE uint64_t+  bextr(uint64_t value, uint32_t start, uint32_t length) {+    if (start > 63) {+      return 0ULL;+    }+    if (start + length > 64) {+      length = 64 - start;+    }++    return (value >> start) &+        ((length == 64) ? (~0ULL) : ((1ULL << length) - 1ULL));+  }++  // Clear high bits starting at position index.+  static FOLLY_ALWAYS_INLINE uint64_t bzhi(uint64_t value, uint32_t index) {+    if (index > 63) {+      return 0;+    }+    return value & ((uint64_t(1) << index) - 1);+  }+};++#if FOLLY_X64 || defined(__i386__)+struct Nehalem : public Default {+  static std::string_view name() noexcept { return "Nehalem"; }++  static bool supported(const folly::CpuId& cpuId = {}) {+    return cpuId.popcnt();+  }++  static FOLLY_ALWAYS_INLINE uint64_t popcount(uint64_t value) {+// POPCNT is supported starting with Intel Nehalem, AMD K10.+#if defined(__GNUC__)+    // GCC and Clang won't inline the intrinsics.+    uint64_t result;+    asm("popcntq %1, %0" : "=r"(result) : "r"(value));+    return result;+#else+    return uint64_t(_mm_popcnt_u64(value));+#endif+  }+};++struct Haswell : public Nehalem {+  static std::string_view name() noexcept { return "Haswell"; }++  static bool supported(const folly::CpuId& cpuId = {}) {+    return Nehalem::supported(cpuId) && cpuId.bmi1() && cpuId.bmi2();+  }++  static FOLLY_ALWAYS_INLINE uint64_t blsr(uint64_t value) {+// BMI1 is supported starting with Intel Haswell, AMD Piledriver.+// BLSR combines two instructions into one and reduces register pressure.+#if defined(__GNUC__)+    // GCC and Clang won't inline the intrinsics.+    uint64_t result;+    asm("blsrq %1, %0" : "=r"(result) : "r"(value));+    return result;+#else+    return _blsr_u64(value);+#endif+  }++  static FOLLY_ALWAYS_INLINE uint64_t+  bextr(uint64_t value, uint32_t start, uint32_t length) {+#if defined(__GNUC__)+    // GCC and Clang won't inline the intrinsics.+    // Encode parameters in `pattern` where `pattern[0:7]` is `start` and+    // `pattern[8:15]` is `length`.+    // Ref: Intel Advanced Vector Extensions Programming Reference+    uint64_t pattern = start & 0xFF;+    pattern = pattern | ((length & 0xFF) << 8);+    uint64_t result;+    asm("bextrq %2, %1, %0" : "=r"(result) : "r"(value), "r"(pattern));+    return result;+#else+    return _bextr_u64(value, start, length);+#endif+  }++  static FOLLY_ALWAYS_INLINE uint64_t bzhi(uint64_t value, uint32_t index) {+#if defined(__GNUC__)+    // GCC and Clang won't inline the intrinsics.+    const uint64_t index64 = index;+    uint64_t result;+    asm("bzhiq %2, %1, %0" : "=r"(result) : "r"(value), "r"(index64));+    return result;+#else+    return _bzhi_u64(value, index);+#endif+  }+};+#endif++enum class Type {+  DEFAULT,+  NEHALEM,+  HASWELL,+};++inline Type detect() {+  const static Type type = [] {+#if FOLLY_X64 || defined(__i386)+    if (instructions::Haswell::supported()) {+      VLOG(2) << "Will use folly::compression::instructions::Haswell";+      return Type::HASWELL;+    } else if (instructions::Nehalem::supported()) {+      VLOG(2) << "Will use folly::compression::instructions::Nehalem";+      return Type::NEHALEM;+    } else {+      VLOG(2) << "Will use folly::compression::instructions::Default";+      return Type::DEFAULT;+    }+#else+    return Type::DEFAULT;+#endif+  }();+  return type;+}++template <class F>+auto dispatch(Type type, F&& f) -> decltype(f(std::declval<Default>())) {+#if FOLLY_X64 || defined(__i386)+  switch (type) {+    case Type::HASWELL:+      return f(Haswell());+    case Type::NEHALEM:+      return f(Nehalem());+    case Type::DEFAULT:+      return f(Default());+  }+#else+  (void)type;+  return f(Default());+#endif++  assume_unreachable();+}++template <class F>+auto dispatch(F&& f) -> decltype(f(std::declval<Default>())) {+  return dispatch(detect(), std::forward<F>(f));+}++} // namespace instructions+} // namespace compression+} // namespace folly
+ folly/folly/compression/QuotientMultiSet-inl.h view
@@ -0,0 +1,412 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#include <folly/compression/QuotientMultiSet.h>++#include <folly/Format.h>+#include <folly/Portability.h>+#include <folly/compression/Select64.h>+#include <folly/lang/Bits.h>+#include <folly/lang/BitsClass.h>+#include <folly/lang/SafeAssert.h>++#include <glog/logging.h>++#if FOLLY_QUOTIENT_MULTI_SET_SUPPORTED++namespace folly {++namespace qms_detail {++/**+ * Reference: Faster Remainder by Direct Computation: Applications to Compilers+ * and Software Libraries, Software: Practice and Experience 49 (6), 2019.+ */+FOLLY_ALWAYS_INLINE UInt64InverseType getInverse(uint64_t divisor) {+  UInt64InverseType fraction = UInt64InverseType(-1);+  fraction /= divisor;+  fraction += 1;+  return fraction;+}++FOLLY_ALWAYS_INLINE uint64_t+mul128(UInt64InverseType lowbits, uint64_t divisor) {+  UInt64InverseType bottomHalf =+      (lowbits & UINT64_C(0xFFFFFFFFFFFFFFFF)) * divisor;+  bottomHalf >>= 64;+  UInt64InverseType topHalf = (lowbits >> 64) * divisor;+  UInt64InverseType bothHalves = bottomHalf + topHalf;+  bothHalves >>= 64;+  return static_cast<uint64_t>(bothHalves);+}++FOLLY_ALWAYS_INLINE std::pair<uint64_t, uint64_t> getQuotientAndRemainder(+    uint64_t dividend, uint64_t divisor, UInt64InverseType inverse) {+  if (FOLLY_UNLIKELY(divisor == 1)) {+    return {dividend, 0};+  }+  auto quotient = mul128(inverse, dividend);+  auto remainder = dividend - quotient * divisor;+  DCHECK_LT(remainder, divisor);+  return {quotient, remainder};+}++// Max value for given bits.+FOLLY_ALWAYS_INLINE uint64_t maxValue(uint32_t nbits) {+  return nbits == 64+      ? std::numeric_limits<uint64_t>::max()+      : (uint64_t(1) << nbits) - 1;+}++} // namespace qms_detail++template <class Instructions>+struct QuotientMultiSet<Instructions>::Metadata {+  // Total number of blocks.+  uint64_t numBlocks;+  uint64_t numKeys;+  uint64_t divisor;+  uint8_t keyBits;+  uint8_t remainderBits;++  std::string debugString() const {+    return fmt::format(+        "Number of blocks: {}\n"+        "Number of elements: {}\n"+        "Divisor: {}\n"+        "Key bits: {}\n"+        "Remainder bits: {}",+        numBlocks,+        numKeys,+        divisor,+        keyBits,+        remainderBits);+  }+} FOLLY_PACK_ATTR;++template <class Instructions>+struct QuotientMultiSet<Instructions>::Block {+  static const Block* get(const char* data) {+    return reinterpret_cast<const Block*>(data);+  }++  static BlockPtr make(size_t remainderBits) {+    auto ptr = reinterpret_cast<Block*>(calloc(blockSize(remainderBits), 1));+    return {ptr, free};+  }++  uint64_t payload;+  uint64_t occupieds;+  uint64_t offset;+  uint64_t runends;+  char remainders[0];++  static uint64_t blockSize(size_t remainderBits) {+    return sizeof(Block) + remainderBits * 8;+  }++  FOLLY_ALWAYS_INLINE bool isOccupied(size_t offsetInBlock) const {+    return ((occupieds >> offsetInBlock) & uint64_t(1)) != 0;+  }++  FOLLY_ALWAYS_INLINE bool isRunend(size_t offsetInBlock) const {+    return ((runends >> offsetInBlock) & uint64_t(1)) != 0;+  }++  FOLLY_ALWAYS_INLINE uint64_t getRemainder(+      size_t offsetInBlock, size_t remainderBits, size_t remainderMask) const {+    DCHECK_LE(remainderBits, 56);+    const size_t bitPos = offsetInBlock * remainderBits;+    const uint64_t remainderWord =+        loadUnaligned<uint64_t>(remainders + (bitPos / 8));+    return (remainderWord >> (bitPos % 8)) & remainderMask;+  }++  void setOccupied(size_t offsetInBlock) {+    occupieds |= uint64_t(1) << offsetInBlock;+  }++  void setRunend(size_t offsetInBlock) {+    runends |= uint64_t(1) << offsetInBlock;+  }++  void setRemainder(+      size_t offsetInBlock, size_t remainderBits, uint64_t remainder) {+    DCHECK_LT(offsetInBlock, kBlockSize);+    if (FOLLY_UNLIKELY(remainderBits == 0)) {+      return;+    }+    Bits<uint64_t>::set(+        reinterpret_cast<uint64_t*>(remainders),+        offsetInBlock * remainderBits,+        remainderBits,+        remainder);+  }+} FOLLY_PACK_ATTR;++template <class Instructions>+QuotientMultiSet<Instructions>::QuotientMultiSet(StringPiece data) {+  static_assert(+      kIsLittleEndian, "QuotientMultiSet requires little endianness.");+  StringPiece sp = data;+  CHECK_GE(sp.size(), sizeof(Metadata));+  sp.advance(sp.size() - sizeof(Metadata));+  metadata_ = reinterpret_cast<const Metadata*>(sp.data());+  VLOG(2) << "Metadata: " << metadata_->debugString();++  numBlocks_ = metadata_->numBlocks;+  numSlots_ = numBlocks_ * kBlockSize;+  divisor_ = metadata_->divisor;+  fraction_ = qms_detail::getInverse(divisor_);+  keyBits_ = metadata_->keyBits;+  maxKey_ = qms_detail::maxValue(keyBits_);+  remainderBits_ = metadata_->remainderBits;+  remainderMask_ = qms_detail::maxValue(remainderBits_);+  blockSize_ = Block::blockSize(remainderBits_);++  CHECK_EQ(data.size(), numBlocks_ * blockSize_ + sizeof(Metadata));+  data_ = data.data();+}++template <class Instructions>+auto QuotientMultiSet<Instructions>::equalRange(uint64_t key) const+    -> SlotRange {+  if (key > maxKey_) {+    return {0, 0};+  }+  const auto qr = qms_detail::getQuotientAndRemainder(key, divisor_, fraction_);+  const auto& quotient = qr.first;+  const auto& remainder = qr.second;+  const size_t blockIndex = quotient / kBlockSize;+  const size_t offsetInBlock = quotient % kBlockSize;++  if (FOLLY_UNLIKELY(blockIndex >= numBlocks_)) {+    return {0, 0};+  }++  const auto* occBlock = getBlock(blockIndex);+  __builtin_prefetch(reinterpret_cast<const char*>(&occBlock->occupieds) + 64);+  const auto firstRunend = occBlock->offset;+  if (!occBlock->isOccupied(offsetInBlock)) {+    // Return a position that depends on the contents of the block so+    // we can create a dependency in benchmarks.+    return {firstRunend, firstRunend};+  }++  // Look for the right runend for the given key.+  const uint64_t occupiedRank = Instructions::popcount(+      Instructions::bzhi(occBlock->occupieds, offsetInBlock));+  auto runend = findRunend(occupiedRank, firstRunend);+  auto& slot = runend.first;+  auto& block = runend.second;++  // Iterates over the run backwards to find the slots whose remainder+  // matches the key.+  SlotRange range = {slot + 1, slot + 1};+  while (true) {+    uint64_t slotRemainder =+        block->getRemainder(slot % kBlockSize, remainderBits_, remainderMask_);++    if (slotRemainder > remainder) {+      range.begin = slot;+      range.end = slot;+    } else if (slotRemainder == remainder) {+      range.begin = slot;+    } else {+      break;+    }++    if (FOLLY_UNLIKELY(slot % kBlockSize == 0)) {+      // Reached block start and the run starts from a prev block.+      size_t slotBlockIndex = slot / kBlockSize;+      if (slotBlockIndex > blockIndex) {+        block = getBlock(slotBlockIndex - 1);+      } else {+        break;+      }+    }++    --slot;+    // Encounters the previous run.+    if (block->isRunend(slot % kBlockSize)) {+      break;+    }+  }++  return range;+}++template <class Instructions>+auto QuotientMultiSet<Instructions>::findRunend(+    uint64_t occupiedRank,+    uint64_t firstRunend) const -> std::pair<uint64_t, const Block*> {+  // Look for the right runend.+  size_t slotBlockIndex = firstRunend / kBlockSize;+  auto block = getBlock(slotBlockIndex);+  uint64_t runendWord =+      block->runends & (uint64_t(-1) << (firstRunend % kBlockSize));++  while (true) {+    DCHECK_LE(slotBlockIndex, numBlocks_);++    const size_t numRuns = Instructions::popcount(runendWord);+    if (FOLLY_LIKELY(numRuns > occupiedRank)) {+      break;+    }+    occupiedRank -= numRuns;+    ++slotBlockIndex;+    block = getBlock(slotBlockIndex);+    runendWord = block->runends;+  }++  return {+      slotBlockIndex * kBlockSize ++          select64<Instructions>(runendWord, occupiedRank),+      block};+}++template <class Instructions>+uint64_t QuotientMultiSet<Instructions>::getBlockPayload(+    uint64_t blockIndex) const {+  DCHECK_LT(blockIndex, numBlocks_);+  return getBlock(blockIndex)->payload;+}++template <class Instructions>+QuotientMultiSet<Instructions>::Iterator::Iterator(+    const QuotientMultiSet<Instructions>* qms)+    : qms_(qms) {}++template <class Instructions>+bool QuotientMultiSet<Instructions>::Iterator::next() {+  if (pos_ == size_t(-1) ||+      qms_->getBlock(pos_ / kBlockSize)->isRunend(pos_ % kBlockSize)) {+    // Move to start of next run.+    if (!nextOccupied()) {+      return setEnd();+    }++    // Next run either starts at pos + 1 or the start of block+    // specified by occupied slot.+    pos_ = std::max<uint64_t>(pos_ + 1, occBlockIndex_ * kBlockSize);+  } else {+    // Move to next slot since a run must be contiguous.+    pos_++;+  }++  const Block* block = qms_->getBlock(pos_ / kBlockSize);+  uint64_t quotient =+      (occBlockIndex_ * kBlockSize + occOffsetInBlock_) * qms_->divisor_;+  key_ = quotient ++      block->getRemainder(+          pos_ % kBlockSize, qms_->remainderBits_, qms_->remainderMask_);++  DCHECK_LT(pos_, qms_->numBlocks_ * kBlockSize);+  return true;+}++template <class Instructions>+bool QuotientMultiSet<Instructions>::Iterator::skipTo(uint64_t key) {+  if (key > qms_->maxKey_) {+    return setEnd();+  }+  const auto qr =+      qms_detail::getQuotientAndRemainder(key, qms_->divisor_, qms_->fraction_);+  const auto& quotient = qr.first;+  occBlockIndex_ = quotient / kBlockSize;+  occOffsetInBlock_ = quotient % kBlockSize;++  if (FOLLY_UNLIKELY(occBlockIndex_ >= qms_->numBlocks_)) {+    return setEnd();+  }++  occBlock_ = qms_->getBlock(occBlockIndex_);+  occWord_ = occBlock_->occupieds & (uint64_t(-1) << occOffsetInBlock_);+  if (!nextOccupied()) {+    return setEnd();+  }++  // Search for the next runend.+  uint64_t occupiedRank = Instructions::popcount(+      Instructions::bzhi(occBlock_->occupieds, occOffsetInBlock_));+  auto runend = qms_->findRunend(occupiedRank, occBlock_->offset);+  auto& slot = runend.first;+  auto& block = runend.second;+  uint64_t slotBlockIndex = slot / kBlockSize;+  uint64_t slotOffsetInBlock = slot % kBlockSize;+  pos_ = slot;++  uint64_t nextQuotient =+      (occBlockIndex_ * kBlockSize + occOffsetInBlock_) * qms_->divisor_;+  uint64_t nextRemainder = block->getRemainder(+      slotOffsetInBlock, qms_->remainderBits_, qms_->remainderMask_);++  if (nextQuotient + nextRemainder < key) {+    // Lower bound element is at the start of next run.+    return next();+  }++  // Iterate over the run backwards to find the first key that is larger than+  // or equal to the given key.+  while (true) {+    uint64_t slotRemainder = block->getRemainder(+        slotOffsetInBlock, qms_->remainderBits_, qms_->remainderMask_);+    if (nextQuotient + slotRemainder < key) {+      break;+    }+    pos_ = slot;+    nextRemainder = slotRemainder;+    if (FOLLY_UNLIKELY(slotOffsetInBlock == 0)) {+      // Reached block start and the run starts from a prev block.+      if (slotBlockIndex > occBlockIndex_) {+        --slotBlockIndex;+        block = qms_->getBlock(slotBlockIndex);+        slotOffsetInBlock = kBlockSize;+      } else {+        break;+      }+    }+    --slot;+    --slotOffsetInBlock;++    // Encounters the previous run.+    if (block->isRunend(slotOffsetInBlock)) {+      break;+    }+  }++  key_ = nextQuotient + nextRemainder;+  return true;+}++template <class Instructions>+bool QuotientMultiSet<Instructions>::Iterator::nextOccupied() {+  while (FOLLY_UNLIKELY(occWord_ == 0)) {+    if (FOLLY_UNLIKELY(++occBlockIndex_ >= qms_->numBlocks_)) {+      return false;+    }+    occBlock_ = qms_->getBlock(occBlockIndex_);+    occWord_ = occBlock_->occupieds;+  }++  occOffsetInBlock_ = Instructions::ctz(occWord_);+  occWord_ = Instructions::blsr(occWord_);+  return true;+}++} // namespace folly++#endif // FOLLY_QUOTIENT_MULTI_SET_SUPPORTED
+ folly/folly/compression/QuotientMultiSet.cpp view
@@ -0,0 +1,184 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#include <folly/compression/QuotientMultiSet.h>++#include <cmath>++#include <folly/Math.h>++#if FOLLY_QUOTIENT_MULTI_SET_SUPPORTED++namespace folly {++QuotientMultiSetBuilder::QuotientMultiSetBuilder(+    size_t keyBits, size_t expectedElements, double loadFactor)+    : keyBits_(keyBits), maxKey_(qms_detail::maxValue(keyBits_)) {+  expectedElements = std::max<size_t>(expectedElements, 1);+  uint64_t numSlots = to_integral(ceil(expectedElements / loadFactor));++  // Make sure 1:1 mapping between key space and <divisor, remainder> pairs.+  divisor_ = divCeil(maxKey_, numSlots);+  remainderBits_ = findLastSet(divisor_ - 1);++  // We only support remainders as long as 56 bits. If the set is very+  // sparse, force the maximum allowed remainder size. This will waste+  // up to 3 extra blocks (because of 8-bit quotients) but be correct.+  if (remainderBits_ > 56) {+    remainderBits_ = 56;+    divisor_ = uint64_t(1) << remainderBits_;+  }++  blockSize_ = Block::blockSize(remainderBits_);+  fraction_ = qms_detail::getInverse(divisor_);+}++QuotientMultiSetBuilder::~QuotientMultiSetBuilder() = default;++bool QuotientMultiSetBuilder::maybeAllocateBlocks(size_t limitIndex) {+  bool blockAllocated = false;+  for (; numBlocks_ <= limitIndex; numBlocks_++) {+    auto block = Block::make(remainderBits_);+    blocks_.emplace_back(std::move(block), numBlocks_);+    blockAllocated = true;+  }+  return blockAllocated;+}++bool QuotientMultiSetBuilder::insert(uint64_t key) {+  FOLLY_SAFE_CHECK(key <= maxKey_, "Invalid key");+  FOLLY_SAFE_CHECK(+      key >= prevKey_, "Keys need to be inserted in nondecreasing order");+  const auto qr = qms_detail::getQuotientAndRemainder(key, divisor_, fraction_);+  const auto& quotient = qr.first;+  const auto& remainder = qr.second;+  const size_t blockIndex = quotient / kBlockSize;+  const size_t offsetInBlock = quotient % kBlockSize;++  bool newBlockAllocated = false;+  // Allocate block for the given key if necessary.+  newBlockAllocated |= maybeAllocateBlocks(+      std::max<uint64_t>(blockIndex, nextSlot_ / kBlockSize));+  auto block = getBlock(nextSlot_ / kBlockSize).block.get();++  // Start a new run.+  if (prevOccupiedQuotient_ != quotient) {+    closePreviousRun();++    if (blockIndex > nextSlot_ / kBlockSize) {+      nextSlot_ = (blockIndex * kBlockSize);+      newBlockAllocated |= maybeAllocateBlocks(blockIndex);+      block = getBlock(blockIndex).block.get();+    }++    // Update previous run info.+    prevRunStart_ = nextSlot_;+    prevOccupiedQuotient_ = quotient;+  }++  block->setRemainder(nextSlot_ % kBlockSize, remainderBits_, remainder);++  // Set occupied bit for the given key.+  block = getBlock(blockIndex).block.get();+  block->setOccupied(offsetInBlock);++  nextSlot_++;+  prevKey_ = key;+  numKeys_++;+  return newBlockAllocated;+}++void QuotientMultiSetBuilder::setBlockPayload(uint64_t payload) {+  DCHECK(!blocks_.empty());+  blocks_.back().block->payload = payload;+}++void QuotientMultiSetBuilder::closePreviousRun() {+  if (FOLLY_UNLIKELY(nextSlot_ == 0)) {+    return;+  }++  // Mark runend for previous run.+  const auto runEnd = nextSlot_ - 1;+  auto block = getBlock(runEnd / kBlockSize).block.get();+  block->setRunend(runEnd % kBlockSize);+  numRuns_++;++  // Set the offset of previous block if this run is the first one in that+  // block.+  auto prevRunOccupiedBlock =+      getBlock(prevOccupiedQuotient_ / kBlockSize).block.get();+  if (isPowTwo(prevRunOccupiedBlock->occupieds)) {+    prevRunOccupiedBlock->offset = runEnd;+  }++  // Update mark all blocks before prevOccupiedQuotient_ + 1 to be ready.+  size_t limitIndex = (prevOccupiedQuotient_ + 1) / kBlockSize;+  for (size_t idx = readyBlocks_; idx < blocks_.size(); idx++) {+    if (blocks_[idx].index < limitIndex) {+      blocks_[idx].ready = true;+      readyBlocks_++;+    } else {+      break;+    }+  }+}++void QuotientMultiSetBuilder::moveReadyBlocks(IOBufQueue& buff) {+  while (!blocks_.empty()) {+    if (!blocks_.front().ready) {+      break;+    }+    buff.append(+        IOBuf::takeOwnership(blocks_.front().block.release(), blockSize_));+    blocks_.pop_front();+  }+}++void QuotientMultiSetBuilder::flush(IOBufQueue& buff) {+  moveReadyBlocks(buff);+  readyBlocks_ = 0;+}++void QuotientMultiSetBuilder::close(IOBufQueue& buff) {+  closePreviousRun();++  // Mark all blocks as ready.+  for (auto iter = blocks_.rbegin(); iter != blocks_.rend(); iter++) {+    if (iter->ready) {+      break;+    }+    iter->ready = true;+  }++  moveReadyBlocks(buff);++  // Add metadata trailer. This will also allows getRemainder() to access whole+  // 64-bits at any position without bounds-checking.+  static_assert(sizeof(Metadata) > 7, "getRemainder() is not safe");+  auto metadata = reinterpret_cast<Metadata*>(calloc(1, sizeof(Metadata)));+  metadata->numBlocks = numBlocks_;+  metadata->numKeys = numKeys_;+  metadata->divisor = divisor_;+  metadata->keyBits = keyBits_;+  metadata->remainderBits = remainderBits_;+  VLOG(2) << "Metadata: " << metadata->debugString();+  buff.append(IOBuf::takeOwnership(metadata, sizeof(Metadata)));+}++} // namespace folly++#endif // FOLLY_QUOTIENT_MULTI_SET_SUPPORTED
+ folly/folly/compression/QuotientMultiSet.h view
@@ -0,0 +1,341 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <deque>+#include <utility>++#include <folly/Portability.h>+#include <folly/Range.h>+#include <folly/compression/Instructions.h>+#include <folly/io/IOBuf.h>+#include <folly/io/IOBufQueue.h>++// A 128-bit integer type is needed for fast division.+#define FOLLY_QUOTIENT_MULTI_SET_SUPPORTED FOLLY_HAVE_INT128_T++#if FOLLY_QUOTIENT_MULTI_SET_SUPPORTED++namespace folly {++namespace qms_detail {++using UInt64InverseType = __uint128_t;++} // namespace qms_detail++/**+ * A space-efficient static data structure to store a non-decreasing sequence of+ * b-bit integers. If the integers are uniformly distributed lookup is O(1)-time+ * and performs a single random memory lookup with high probability.+ *+ * Space for n keys is bounded by (5 + b - log(n / loadFactor)) / loadFactor+ * bits per element, which makes it particularly efficient for very dense+ * sets. Note that 1 bit is taken up by the user-provided block payloads, and 1+ * depends on how close the table size is to a power of 2. Experimentally,+ * performance is good up to load factor 95%.+ *+ * Lookup returns a range of positions in the table. The intended use case is to+ * store hashes, as the first layer of a multi-layer hash table. If b is sized+ * to floor(log(n)) + k, the probability of a false positive (a non-empty range+ * is returned for a non-existent key) is approximately 2^-k, which makes it+ * competitive with a Bloom filter for low FP probabilities, with the additional+ * benefit that it also returns a range of positions to restrict the search in+ * subsequent layers.+ *+ * The data structure is inspired by the Rank-Select Quotient Filter+ * introduced in+ *+ *   Prashant Pandey, Michael A. Bender, Rob Johnson and Robert Patro,+ *   A General-Purpose Counting Filter: Making Every Bit Count, SIGMOD, 2017+ *+ * Besides being static, QuotientMultiSet differs from the data structure from+ * the paper in the following ways:+ *+ * - The table size can be arbitrary, rather than just powers-of-2. This can+ *   waste up to a bit for each residual, but it prevents 2x overhead when the+ *   desired table size is slightly larger than a power of 2.+ *+ * - Within each block all the holes are moved at the end. This enables+ *   efficient iteration, and makes the returned positions a contiguous range+ *   for each block, which allows to use them to index into a secondary data+ *   structure. An arbitrary 64-bit payload can be attached to each block; for+ *   example, this can be used to store the number of elements up to that block,+ *   so that positions can be translated to the element rank. Alternatively, the+ *   payload can be used to address blocks in the secondary data structure.+ *+ * - Correctness does not depend on the keys being uniformly distributed.+ *   However, performance does, as for arbitrary keys the worst-case lookup time+ *   can be linear.+ *+ * Implemented by Matt Ma based on a prototype by Giuseppe Ottaviano and+ * Sebastiano Vigna.+ *+ * Data layout:+ * ------------------------------------------------------------------------+ * | Block | Block | Block | Block |  ...                         | Block |+ * ------------------------------------------------------------------------+ *                /        |+ * ------------------------------------------------------------------------+ * | Payload | Occupieds | Offset | Runends |       Remainders * 64       |+ * ------------------------------------------------------------------------+ *+ * Each block contains 64 slots. Keys mapping to the same slot are stored+ * contiguously in a run. The occupieds and runends bitvectors are the+ * concatenation of the corresponding words in each block.+ *+ * - Occupieds bit indicates whether there is a key mapping to this quotient.+ *+ * - Offset stores the position of the runend of the first run in this block.+ *+ * - Runends bit indicates whether the slot is the end of some run. 1s in+ *   occupieds and runends bits are in 1-1 correspondence: the i-th 1 in the+ *   runends vector marks the run end of the i-th 1 in the occupieds.+ */++template <class Instructions = compression::instructions::Default>+class QuotientMultiSet final {+ public:+  explicit QuotientMultiSet(StringPiece data);++  // Each block contains 64 elements.+  static constexpr size_t kBlockSize = 64;++  // Position range of given key. End is not included. Range can be empty if the+  // key is not found, in which case the values of begin and end are+  // unspecified.+  struct SlotRange {+    size_t begin = 0;+    size_t end = 0;++    explicit operator bool() const {+      DCHECK_LE(begin, end);+      return begin < end;+    }+  };++  class Iterator;++  // Get the position range for the given key.+  SlotRange equalRange(uint64_t key) const;++  // Get payload of given block.+  uint64_t getBlockPayload(uint64_t blockIndex) const;++  friend class QuotientMultiSetBuilder;++ private:+  // Metadata to describe a quotient table.+  struct Metadata;++  // Block contains payload, occupieds, runends, offsets and 64 remainders.+  struct Block;+  using BlockPtr = std::unique_ptr<Block, decltype(free)*>;++  const Block* getBlock(size_t blockIndex) const {+    return Block::get(data_ + blockIndex * blockSize_);+  }++  FOLLY_ALWAYS_INLINE std::pair<uint64_t, const Block*> findRunend(+      uint64_t occupiedRank, uint64_t startPos) const;++  const Metadata* metadata_;+  const char* data_;+  // Total number of blocks.+  size_t numBlocks_;+  size_t numSlots_;+  // Number of bytes per block.+  size_t blockSize_;+  // Divisor for mapping from keys to slots.+  uint64_t divisor_;+  // fraction_ = 1 / divisor_.+  qms_detail::UInt64InverseType fraction_;+  // Number of key bits.+  size_t keyBits_;+  uint64_t maxKey_;+  // Number of remainder bits.+  size_t remainderBits_;+  uint64_t remainderMask_;+};++template <class Instructions>+class QuotientMultiSet<Instructions>::Iterator {+ public:+  explicit Iterator(const QuotientMultiSet<Instructions>* qms);++  // Advance to the next key.+  bool next();++  // Skip forward to the first key >= the given key.+  bool skipTo(uint64_t key);++  bool done() const { return pos_ == qms_->numSlots_; }++  // Return current key.+  uint64_t key() const { return key_; }++  // Return current position in quotient multiset.+  size_t pos() const { return pos_; }++ private:+  // Position the iterator at the end and return false.+  // Shortcut for use when implementing doNext, etc: return setEnd();+  bool setEnd() {+    pos_ = qms_->numSlots_;+    return false;+  }++  // Move to next occupied.+  bool nextOccupied();++  const QuotientMultiSet<Instructions>* qms_;+  uint64_t key_ = 0;++  // State members for the quotient occupied position.+  // Block index of key_'s occupied slot.+  size_t occBlockIndex_ = -1;+  // Block offset of key_'s occupied slot.+  uint64_t occOffsetInBlock_ = 0;+  // Occupied words of the occupiedBlock_ after quotientBlockOffset_.+  uint64_t occWord_ = 0;+  // Block of the current occupied slot.+  const Block* occBlock_ = nullptr;++  // State member for the actual key position.+  // Position of the current key_.+  size_t pos_ = -1;+};++/**+ * Class to build a QuotientMultiSet.+ *+ * The builder requires inserting elements in non-decreasing order.+ * Example usage:+ *   QuotientMultiSetBuilder builder(...);+ *   while (...) {+ *     if (builder.insert(key)) {+ *       builder.setBlockPayload(payload);+ *     }+ *     if (builder.numReadyBlocks() > N) {+ *       buff = builder.flush();+ *       write(buff);+ *     }+ *   }+ *   buff = builder.close();+ *   write(buff)+ */+class QuotientMultiSetBuilder final {+ public:+  QuotientMultiSetBuilder(+      size_t keyBits,+      size_t expectedElements,+      double loadFactor = kDefaultMaxLoadFactor);+  ~QuotientMultiSetBuilder();++  using Metadata = QuotientMultiSet<>::Metadata;+  using Block = QuotientMultiSet<>::Block;++  // Keeps load factor <= 0.95.+  constexpr static double kDefaultMaxLoadFactor = 0.95;++  constexpr static size_t kBlockSize = QuotientMultiSet<>::kBlockSize;++  // Returns whether the key's slot is in a newly created block.+  // Only allows insert keys in nondecreasing order.+  bool insert(uint64_t key);++  // Set payload of the latest created block.+  // Can only be called immediately after an add() that returns true.+  void setBlockPayload(uint64_t payload);++  // Return all ready blocks till now. The ownership of these blocks will be+  // transferred to the caller.+  void flush(IOBufQueue& buff);++  // Return all remaining blocks since last flush call and the final quotient+  // table metadata. The ownership of these blocks will be transferred to the+  // caller.+  void close(folly::IOBufQueue& buff);++  size_t numReadyBlocks() { return readyBlocks_; }++ private:+  using BlockPtr = QuotientMultiSet<>::BlockPtr;++  struct BlockWithState {+    BlockWithState(BlockPtr ptr, size_t idx)+        : block(std::move(ptr)), index(idx), ready(false) {}++    BlockPtr block;+    size_t index;+    bool ready;+  };++  // Allocate space for blocks until limitIndex (included).+  bool maybeAllocateBlocks(size_t limitIndex);++  // Close the previous run.+  void closePreviousRun();++  // Move ready blocks to given IOBufQueue.+  void moveReadyBlocks(IOBufQueue& buff);++  // Get block for given block index.+  BlockWithState& getBlock(uint64_t blockIndex) {+    CHECK_GE(blockIndex, blocks_.front().index);+    return blocks_[blockIndex - blocks_.front().index];+  }++  // Number of key bits.+  const size_t keyBits_;+  const uint64_t maxKey_;++  // Total number of blocks.+  size_t numBlocks_ = 0;+  // Number of bytes per block.+  size_t blockSize_ = 0;+  // Divisor for mapping from keys to slots.+  uint64_t divisor_;+  // fraction_ = 1 / divisor_.+  qms_detail::UInt64InverseType fraction_;+  // Number of remainder bits.+  uint64_t remainderBits_;++  size_t numKeys_ = 0;+  size_t numRuns_ = 0;++  uint64_t prevKey_ = 0;+  // Next slot to be used.+  size_t nextSlot_ = 0;+  // The actual start of previous run.+  size_t prevRunStart_ = 0;+  // The quotient of previous run.+  size_t prevOccupiedQuotient_ = 0;+  // Number of ready blocks in deque.+  size_t readyBlocks_ = 0;++  // Contains blocks since last flush call.+  std::deque<BlockWithState> blocks_;++  IOBufQueue buff_;+};++} // namespace folly++#include <folly/compression/QuotientMultiSet-inl.h>++#endif // FOLLY_QUOTIENT_MULTI_SET_SUPPORTED
+ folly/folly/compression/Select64.cpp view
@@ -0,0 +1,61 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#include <folly/compression/Select64.h>++#include <cstdint>+#include <utility>++#include <folly/ConstexprMath.h>+#include <folly/Portability.h>+#include <folly/Utility.h>++namespace folly {+namespace detail {++namespace {++constexpr std::uint8_t selectInByte(std::size_t i, std::size_t j) {+  auto r = std::uint8_t(0);+  while (j--) {+    auto const s = folly::constexpr_find_first_set(i);+    r += s;+    i >>= s;+  }+  return i == 0 ? 8 : r + folly::constexpr_find_first_set(i) - 1;+}++template <std::size_t... I, std::size_t J>+constexpr auto makeSelectInByteNestedArray(+    std::index_sequence<I...>, index_constant<J>) {+  return std::array<std::uint8_t, sizeof...(I)>{{selectInByte(I, J)...}};+}++template <typename Is, std::size_t... J>+constexpr auto makeSelectInByteArray(Is is, std::index_sequence<J...>) {+  using inner = std::array<std::uint8_t, Is::size()>;+  using outer = std::array<inner, sizeof...(J)>;+  return outer{{makeSelectInByteNestedArray(is, index_constant<J>{})...}};+}++} // namespace++FOLLY_STORAGE_CONSTEXPR std::array<std::array<std::uint8_t, 256>, 8> const+    kSelectInByte = makeSelectInByteArray(+        std::make_index_sequence<256>{}, std::make_index_sequence<8>{});++} // namespace detail+} // namespace folly
+ folly/folly/compression/Select64.h view
@@ -0,0 +1,109 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <array>++#include <glog/logging.h>++#include <folly/Portability.h>+#include <folly/compression/Instructions.h>++namespace folly {++namespace detail {++//  kSelectInByte+//+//  Described in:+//    http://dsiutils.di.unimi.it/docs/it/unimi/dsi/bits/Fast.html#selectInByte+//+//  A precomputed tabled containing the positions of the set bits in the binary+//  representations of all 8-bit unsigned integers.+//+//  For i: [0, 256) ranging over all 8-bit unsigned integers and for j: [0, 8)+//  ranging over all 0-based bit positions in an 8-bit unsigned integer, the+//  table entry kSelectInByte[i][j] is the 0-based bit position of the j-th set+//  bit in the binary representation of i, or 8 if it has fewer than j set bits.+//+//  Example: i: 17 (b00010001), j: [0, 8)+//    kSelectInByte[b00010001][0] = 0+//    kSelectInByte[b00010001][1] = 4+//    kSelectInByte[b00010001][2] = 8+//    ...+//    kSelectInByte[b00010001][7] = 8+extern std::array<std::array<std::uint8_t, 256>, 8> const kSelectInByte;++} // namespace detail++/**+ * Returns the position of the k-th 1 in the 64-bit word x.+ * k is 0-based, so k=0 returns the position of the first 1.+ *+ * Uses the broadword selection algorithm by Vigna [1], improved by Gog+ * and Petri [2] and Vigna [3].+ *+ * [1] Sebastiano Vigna. Broadword Implementation of Rank/Select+ *     Queries. WEA, 2008+ *+ * [2] Simon Gog, Matthias Petri. Optimized succinct data structures+ *     for massive data. Softw. Pract. Exper., 2014+ *+ * [3] Sebastiano Vigna. MG4J 5.2.1. http://mg4j.di.unimi.it/+ */+template <class Instructions>+inline uint64_t select64(uint64_t x, uint64_t k) {+  DCHECK_LT(k, Instructions::popcount(x));++  constexpr uint64_t kOnesStep4 = 0x1111111111111111ULL;+  constexpr uint64_t kOnesStep8 = 0x0101010101010101ULL;+  constexpr uint64_t kMSBsStep8 = 0x80ULL * kOnesStep8;++  auto s = x;+  s = s - ((s & 0xA * kOnesStep4) >> 1);+  s = (s & 0x3 * kOnesStep4) + ((s >> 2) & 0x3 * kOnesStep4);+  s = (s + (s >> 4)) & 0xF * kOnesStep8;+  uint64_t byteSums = s * kOnesStep8;++  uint64_t kStep8 = k * kOnesStep8;+  uint64_t geqKStep8 = (((kStep8 | kMSBsStep8) - byteSums) & kMSBsStep8);+  uint64_t place = Instructions::popcount(geqKStep8) * 8;+  uint64_t byteRank = k - (((byteSums << 8) >> place) & uint64_t(0xFF));+  return place + detail::kSelectInByte[byteRank][((x >> place) & 0xFF)];+}++#if FOLLY_X64 || defined(__i386)+template <>+FOLLY_ALWAYS_INLINE uint64_t+select64<compression::instructions::Haswell>(uint64_t x, uint64_t k) {+#if defined(__GNUC__)+  // GCC and Clang won't inline the intrinsics.+  uint64_t result = uint64_t(1) << k;++  asm("pdep %1, %0, %0\n\t"+      "tzcnt %0, %0"+      : "+r"(result)+      : "r"(x));++  return result;+#else+  return _tzcnt_u64(_pdep_u64(1ULL << k, x));+#endif+}+#endif++} // namespace folly
+ folly/folly/compression/Utils.h view
@@ -0,0 +1,149 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <type_traits>++#include <folly/compression/Compression.h>+#include <folly/io/Cursor.h>+#include <folly/io/IOBuf.h>+#include <folly/lang/Bits.h>++/**+ * Helper functions for compression codecs.+ */+namespace folly {+namespace compression {+namespace detail {++/**+ * Reads sizeof(T) bytes, and returns false if not enough bytes are available.+ * Returns true if the first n bytes are equal to prefix when interpreted as+ * a little endian T.+ */+template <typename T>+typename std::enable_if<std::is_unsigned<T>::value, bool>::type+dataStartsWithLE(const IOBuf* data, T prefix, uint64_t n = sizeof(T)) {+  DCHECK_GT(n, 0);+  DCHECK_LE(n, sizeof(T));+  T value;+  io::Cursor cursor{data};+  if (!cursor.tryReadLE(value)) {+    return false;+  }+  const T mask = n == sizeof(T) ? T(-1) : (T(1) << (8 * n)) - 1;+  return prefix == (value & mask);+}++template <typename T>+typename std::enable_if<std::is_arithmetic<T>::value, std::string>::type+prefixToStringLE(T prefix, uint64_t n = sizeof(T)) {+  DCHECK_GT(n, 0);+  DCHECK_LE(n, sizeof(T));+  prefix = Endian::little(prefix);+  std::string result;+  result.resize(n);+  memcpy(&result[0], &prefix, n);+  return result;+}++/**+ * Calls @p streamFn in a loop, and guarantees that @p streamFn is never called+ * with `input.size() > chunkSize` or `output.size() > chunkSize`. It behaves+ * as-if streamFn(input, output, flushOp) was called.+ *+ * This function relies only on the rules defined by+ * StreamCodec::compressStream() and StreamCodec::uncompressStream() for+ * correctness.+ */+template <typename StreamFn>+bool chunkedStream(+    size_t chunkSize,+    ByteRange& input,+    MutableByteRange& output,+    StreamCodec::FlushOp flushOp,+    StreamFn&& streamFn) {+  for (;;) {+    const auto chunkInputSize = std::min<size_t>(input.size(), chunkSize);+    const auto chunkOutputSize = std::min<size_t>(output.size(), chunkSize);+    auto chunkInput = input.subpiece(0, chunkInputSize);+    auto chunkOutput = output.subpiece(0, chunkOutputSize);++    // If we're presenting the entire suffix of the input in this call, use the+    // users flush op, otherwise use NONE.+    const StreamCodec::FlushOp chunkFlushOp =+        (chunkInput.size() == input.size())+        ? flushOp+        : StreamCodec::FlushOp::NONE;++    const bool finished = streamFn(chunkInput, chunkOutput, chunkFlushOp);++    // Update input / output buffers+    const size_t inputConsumed = chunkInputSize - chunkInput.size();+    const size_t outputProduced = chunkOutputSize - chunkOutput.size();+    input.advance(inputConsumed);+    output.advance(outputProduced);++    // If the underlying streaming function returns true, we want to forward+    // that to the caller.+    // Compression: If flushOp == NONE, this is guaranteed not to happen.+    // Decompression: This signals the end of a frame, and we must forward that+    //                signal to the caller without consuming any more input.+    if (finished) {+      return true;+    }++    // We've consumed the entire input, which means we fulfilled our as-if+    // guarantee.+    if (input.empty()) {+      DCHECK(!finished);+      return false;+    }++    // Compression: Presenting more input bytes guarantees that there must be+    //              more output bytes to produce. Therefore we've made maximal+    //              forward progress.+    // Decompression: The only flushOp that guarantees maximal forward progress+    //                is FLUSH. Its signal that the flush is complete is+    //                !output.empty(). So we can safely return if+    //                output.empty().+    if (output.empty()) {+      DCHECK(!input.empty() && !finished);+      return false;+    }++    // If we've failed to make forward progress, return to the caller.+    // The underlying streaming function is always required to make some forward+    // progress if any forward progress is possible. So forward progress must be+    // impossible.+    // This preserves StreamCodec's strong guarantee of forward progress, and+    // protects us from infinite loops.+    if (inputConsumed == 0 && outputProduced == 0) {+      DCHECK(!output.empty() && !input.empty() && !finished);+      return false;+    }+  }+}++// Some codecs use uint32_t for sizes so we need to limit the chunk size to 4GB.+// Rather than allow 4GB inputs / outputs, which nearly never happens, limit to+// 4MB, so we don't have complex logic that almost never runs.+constexpr size_t kDefaultChunkSizeFor32BitSizes = size_t(4) << 20;++} // namespace detail+} // namespace compression+} // namespace folly
+ folly/folly/compression/Zlib.cpp view
@@ -0,0 +1,437 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#include <folly/compression/Zlib.h>++#if FOLLY_HAVE_LIBZ++#include <folly/Conv.h>+#include <folly/Optional.h>+#include <folly/Range.h>+#include <folly/ScopeGuard.h>+#include <folly/compression/Compression.h>+#include <folly/compression/Utils.h>+#include <folly/io/Cursor.h>++using folly::compression::detail::dataStartsWithLE;+using folly::compression::detail::prefixToStringLE;++namespace folly {+namespace compression {+namespace zlib {++namespace {++bool isValidStrategy(int strategy) {+  std::array<int, 5> strategies{{+      Z_DEFAULT_STRATEGY,+      Z_FILTERED,+      Z_HUFFMAN_ONLY,+      Z_RLE,+      Z_FIXED,+  }};+  return std::any_of(strategies.begin(), strategies.end(), [&](int i) {+    return i == strategy;+  });+}++int getWindowBits(Options::Format format, int windowSize) {+  switch (format) {+    case Options::Format::ZLIB:+      return windowSize;+    case Options::Format::GZIP:+      return windowSize + 16;+    case Options::Format::RAW:+      return -windowSize;+    case Options::Format::AUTO:+      return windowSize + 32;+    default:+      return windowSize;+  }+}++CodecType getCodecType(Options options) {+  if (options.windowSize == 15 && options.format == Options::Format::ZLIB) {+    return CodecType::ZLIB;+  } else if (+      options.windowSize == 15 && options.format == Options::Format::GZIP) {+    return CodecType::GZIP;+  } else {+    return CodecType::USER_DEFINED;+  }+}++class ZlibStreamCodec final : public StreamCodec {+ public:+  static std::unique_ptr<Codec> createCodec(Options options, int level);+  static std::unique_ptr<StreamCodec> createStream(Options options, int level);++  explicit ZlibStreamCodec(Options options, int level);+  ~ZlibStreamCodec() override;++  std::vector<std::string> validPrefixes() const override;+  bool canUncompress(+      const IOBuf* data, Optional<uint64_t> uncompressedLength) const override;++ private:+  uint64_t doMaxCompressedLength(uint64_t uncompressedLength) const override;++  void doResetStream() override;+  bool doCompressStream(+      ByteRange& input,+      MutableByteRange& output,+      StreamCodec::FlushOp flush) override;+  bool doUncompressStream(+      ByteRange& input,+      MutableByteRange& output,+      StreamCodec::FlushOp flush) override;++  void resetDeflateStream();+  void resetInflateStream();++  Options options_;++  Optional<z_stream> deflateStream_{};+  Optional<z_stream> inflateStream_{};+  int level_;+  bool needReset_{true};+};+constexpr uint16_t kGZIPMagicLE = 0x8B1F;++std::vector<std::string> ZlibStreamCodec::validPrefixes() const {+  if (type() == CodecType::ZLIB) {+    // Zlib streams start with a 2 byte header.+    //+    //   0   1+    // +---+---++    // |CMF|FLG|+    // +---+---++    //+    // We won't restrict the values of any sub-fields except as described below.+    //+    // The lowest 4 bits of CMF is the compression method (CM).+    // CM == 0x8 is the deflate compression method, which is currently the only+    // supported compression method, so any valid prefix must have CM == 0x8.+    //+    // The lowest 5 bits of FLG is FCHECK.+    // FCHECK must be such that the two header bytes are a multiple of 31 when+    // interpreted as a big endian 16-bit number.+    std::vector<std::string> result;+    // 16 values for the first byte, 8 values for the second byte.+    // There are also 4 combinations where both 0x00 and 0x1F work as FCHECK.+    result.reserve(132);+    // Select all values for the CMF byte that use the deflate algorithm 0x8.+    for (uint32_t first = 0x0800; first <= 0xF800; first += 0x1000) {+      // Select all values for the FLG, but leave FCHECK as 0 since it's fixed.+      for (uint32_t second = 0x00; second <= 0xE0; second += 0x20) {+        uint16_t prefix = first | second;+        // Compute FCHECK.+        prefix += 31 - (prefix % 31);+        result.push_back(prefixToStringLE(Endian::big(prefix)));+        // zlib won't produce this, but it is a valid prefix.+        if ((prefix & 0x1F) == 31) {+          prefix -= 31;+          result.push_back(prefixToStringLE(Endian::big(prefix)));+        }+      }+    }+    return result;+  } else if (type() == CodecType::GZIP) {+    // The gzip frame starts with 2 magic bytes.+    return {prefixToStringLE(kGZIPMagicLE)};+  } else {+    return {};+  }+}++bool ZlibStreamCodec::canUncompress(+    const IOBuf* data, Optional<uint64_t>) const {+  if (type() == CodecType::ZLIB) {+    uint16_t value;+    io::Cursor cursor{data};+    if (!cursor.tryReadBE(value)) {+      return false;+    }+    // zlib compressed if using deflate and is a multiple of 31.+    return (value & 0x0F00) == 0x0800 && value % 31 == 0;+  } else if (type() == CodecType::GZIP) {+    return dataStartsWithLE(data, kGZIPMagicLE);+  } else {+    return false;+  }+}++uint64_t ZlibStreamCodec::doMaxCompressedLength(+    uint64_t uncompressedLength) const {+  // When passed a nullptr, deflateBound() adds 6 bytes for a zlib wrapper. A+  // gzip wrapper is 18 bytes, so we add the 12 byte difference.+  return deflateBound(nullptr, uncompressedLength) ++      (options_.format == Options::Format::GZIP ? 12 : 0);+}++std::unique_ptr<Codec> ZlibStreamCodec::createCodec(+    Options options, int level) {+  return std::make_unique<ZlibStreamCodec>(options, level);+}++std::unique_ptr<StreamCodec> ZlibStreamCodec::createStream(+    Options options, int level) {+  return std::make_unique<ZlibStreamCodec>(options, level);+}++bool inBounds(int value, int low, int high) {+  return (value >= low) && (value <= high);+}++int zlibConvertLevel(int level) {+  switch (level) {+    case COMPRESSION_LEVEL_FASTEST:+      return 1;+    case COMPRESSION_LEVEL_DEFAULT:+      return 6;+    case COMPRESSION_LEVEL_BEST:+      return 9;+  }+  if (!inBounds(level, 0, 9)) {+    throw std::invalid_argument(+        to<std::string>("ZlibStreamCodec: invalid level: ", level));+  }+  return level;+}++ZlibStreamCodec::ZlibStreamCodec(Options options, int level)+    : StreamCodec(+          getCodecType(options),+          zlibConvertLevel(level),+          getCodecType(options) == CodecType::GZIP ? "gzip" : "zlib"),+      level_(zlibConvertLevel(level)) {+  options_ = options;++  // Although zlib allows a windowSize of 8..15, a value of 8 is not+  // properly supported and is treated as a value of 9. This means data deflated+  // with windowSize==8 can not be re-inflated with windowSize==8. windowSize==8+  // is also not supported for gzip and raw deflation.+  // Hence, the codec supports only 9..15.+  if (!inBounds(options_.windowSize, 9, 15)) {+    throw std::invalid_argument(to<std::string>(+        "ZlibStreamCodec: invalid windowSize option: ", options.windowSize));+  }+  if (!inBounds(options_.memLevel, 1, 9)) {+    throw std::invalid_argument(to<std::string>(+        "ZlibStreamCodec: invalid memLevel option: ", options.memLevel));+  }+  if (!isValidStrategy(options_.strategy)) {+    throw std::invalid_argument(to<std::string>(+        "ZlibStreamCodec: invalid strategy: ", options.strategy));+  }+}++ZlibStreamCodec::~ZlibStreamCodec() {+  if (deflateStream_) {+    deflateEnd(deflateStream_.get_pointer());+    deflateStream_.reset();+  }+  if (inflateStream_) {+    inflateEnd(inflateStream_.get_pointer());+    inflateStream_.reset();+  }+}++void ZlibStreamCodec::doResetStream() {+  needReset_ = true;+}++void ZlibStreamCodec::resetDeflateStream() {+  if (deflateStream_) {+    int const rc = deflateReset(deflateStream_.get_pointer());+    if (rc != Z_OK) {+      deflateStream_.reset();+      throw std::runtime_error(+          to<std::string>("ZlibStreamCodec: deflateReset error: ", rc));+    }+    return;+  }+  deflateStream_ = z_stream{};++  // The automatic header detection format is only for inflation.+  // Use zlib for deflation if the format is auto.+  int const windowBits = getWindowBits(+      options_.format == Options::Format::AUTO+          ? Options::Format::ZLIB+          : options_.format,+      options_.windowSize);++  int const rc = deflateInit2(+      deflateStream_.get_pointer(),+      level_,+      Z_DEFLATED,+      windowBits,+      options_.memLevel,+      options_.strategy);+  if (rc != Z_OK) {+    deflateStream_.reset();+    throw std::runtime_error(+        to<std::string>("ZlibStreamCodec: deflateInit error: ", rc));+  }+}++void ZlibStreamCodec::resetInflateStream() {+  if (inflateStream_) {+    int const rc = inflateReset(inflateStream_.get_pointer());+    if (rc != Z_OK) {+      inflateStream_.reset();+      throw std::runtime_error(+          to<std::string>("ZlibStreamCodec: inflateReset error: ", rc));+    }+    return;+  }+  inflateStream_ = z_stream{};+  int const rc = inflateInit2(+      inflateStream_.get_pointer(),+      getWindowBits(options_.format, options_.windowSize));+  if (rc != Z_OK) {+    inflateStream_.reset();+    throw std::runtime_error(+        to<std::string>("ZlibStreamCodec: inflateInit error: ", rc));+  }+}++int zlibTranslateFlush(StreamCodec::FlushOp flush) {+  switch (flush) {+    case StreamCodec::FlushOp::NONE:+      return Z_NO_FLUSH;+    case StreamCodec::FlushOp::FLUSH:+      return Z_SYNC_FLUSH;+    case StreamCodec::FlushOp::END:+      return Z_FINISH;+    default:+      throw std::invalid_argument("ZlibStreamCodec: Invalid flush");+  }+}++int zlibThrowOnError(int rc) {+  switch (rc) {+    case Z_OK:+    case Z_BUF_ERROR:+    case Z_STREAM_END:+      return rc;+    default:+      throw std::runtime_error(to<std::string>("ZlibStreamCodec: error: ", rc));+  }+}++bool ZlibStreamCodec::doCompressStream(+    ByteRange& input, MutableByteRange& output, StreamCodec::FlushOp flush) {+  // Zlib uses uint32_t for sizes, so we can't compress more than 4GB at a time+  return detail::chunkedStream(+      detail::kDefaultChunkSizeFor32BitSizes,+      input,+      output,+      flush,+      [this](auto& input, auto& output, auto flush) {+        if (needReset_) {+          resetDeflateStream();+          needReset_ = false;+        }+        DCHECK(deflateStream_.has_value());+        // zlib will return Z_STREAM_ERROR if output.data() is null.+        if (output.data() == nullptr) {+          return false;+        }+        deflateStream_->next_in = const_cast<uint8_t*>(input.data());+        deflateStream_->avail_in = to_narrow(input.size());+        deflateStream_->next_out = output.data();+        deflateStream_->avail_out = to_narrow(output.size());+        DCHECK_EQ(deflateStream_->avail_in, input.size());+        DCHECK_EQ(deflateStream_->avail_out, output.size());+        SCOPE_EXIT {+          input.uncheckedAdvance(input.size() - deflateStream_->avail_in);+          output.uncheckedAdvance(output.size() - deflateStream_->avail_out);+        };+        int const rc = zlibThrowOnError(+            deflate(deflateStream_.get_pointer(), zlibTranslateFlush(flush)));+        switch (flush) {+          case StreamCodec::FlushOp::NONE:+            return false;+          case StreamCodec::FlushOp::FLUSH:+            return deflateStream_->avail_in == 0 &&+                deflateStream_->avail_out != 0;+          case StreamCodec::FlushOp::END:+            return rc == Z_STREAM_END;+          default:+            throw std::invalid_argument("ZlibStreamCodec: Invalid flush");+        }+      });+}++bool ZlibStreamCodec::doUncompressStream(+    ByteRange& input, MutableByteRange& output, StreamCodec::FlushOp flush) {+  // Zlib uses uint32_t for sizes, so we can't uncompress more than 4GB at a+  // time.+  return detail::chunkedStream(+      detail::kDefaultChunkSizeFor32BitSizes,+      input,+      output,+      flush,+      [this](auto& input, auto& output, auto flush) {+        if (needReset_) {+          resetInflateStream();+          needReset_ = false;+        }+        DCHECK(inflateStream_.has_value());+        // zlib will return Z_STREAM_ERROR if output.data() is null.+        if (output.data() == nullptr) {+          return false;+        }+        inflateStream_->next_in = const_cast<uint8_t*>(input.data());+        inflateStream_->avail_in = to_narrow(input.size());+        inflateStream_->next_out = output.data();+        inflateStream_->avail_out = to_narrow(output.size());+        DCHECK_EQ(inflateStream_->avail_in, input.size());+        DCHECK_EQ(inflateStream_->avail_out, output.size());+        SCOPE_EXIT {+          input.advance(input.size() - inflateStream_->avail_in);+          output.advance(output.size() - inflateStream_->avail_out);+        };+        int const rc = zlibThrowOnError(+            inflate(inflateStream_.get_pointer(), zlibTranslateFlush(flush)));+        return rc == Z_STREAM_END;+      });+}++} // namespace++Options defaultGzipOptions() {+  return Options(Options::Format::GZIP);+}++Options defaultZlibOptions() {+  return Options(Options::Format::ZLIB);+}++std::unique_ptr<Codec> getCodec(Options options, int level) {+  return ZlibStreamCodec::createCodec(options, level);+}++std::unique_ptr<StreamCodec> getStreamCodec(Options options, int level) {+  return ZlibStreamCodec::createStream(options, level);+}++} // namespace zlib+} // namespace compression+} // namespace folly++#endif // FOLLY_HAVE_LIBZ
+ folly/folly/compression/Zlib.h view
@@ -0,0 +1,127 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <folly/Portability.h>+#include <folly/compression/Compression.h>++#if FOLLY_HAVE_LIBZ++#include <zlib.h>++/**+ * Interface for Zlib-specific codec initialization.+ */+namespace folly {+namespace compression {+namespace zlib {++struct Options {+  /**+   * ZLIB: default option -- write a zlib wrapper as documented in RFC 1950.+   *+   * GZIP: write a simple gzip header and trailer around the compressed data+   * instead of a zlib wrapper.+   *+   * RAW: deflate will generate raw deflate data with no zlib header or+   * trailer, and will not compute a check value.+   *+   * AUTO: enable automatic header detection for decoding gzip or zlib data.+   * For deflation, ZLIB will be used.+   */+  enum class Format { ZLIB, GZIP, RAW, AUTO };++  explicit Options(+      Format format_ = Format::ZLIB,+      int windowSize_ = 15,+      int memLevel_ = 8,+      int strategy_ = Z_DEFAULT_STRATEGY)+      : format(format_),+        windowSize(windowSize_),+        memLevel(memLevel_),+        strategy(strategy_) {}++  Format format;++  /**+   * windowSize is the base two logarithm of the window size (the size of the+   * history buffer). It should be in the range 9..15. Larger values of this+   * parameter result in better compression at the expense of memory usage.+   *+   * The default value is 15.+   *+   * NB: when inflating/uncompressing data, the windowSize must be greater than+   * or equal to the size used when deflating/compressing.+   */+  int windowSize;++  /**+   * "The memLevel parameter specifies how much memory should be allocated for+   * the internal compression state. memLevel=1 uses minimum memory but is slow+   * and reduces compression ratio; memLevel=9 uses maximum memory for optimal+   * speed. The default value is 8."+   */+  int memLevel;++  /**+   * The strategy parameter is used to tune the compression algorithm.+   * Supported values:+   * - Z_DEFAULT_STRATEGY: normal data+   * - Z_FILTERED: data produced by a filter (or predictor)+   * - Z_HUFFMAN_ONLY: force Huffman encoding only (no string match)+   * - Z_RLE: limit match distances to one+   * - Z_FIXED: prevents the use of dynamic Huffman codes+   *+   * The strategy parameter only affects the compression ratio but not the+   * correctness of the compressed output.+   */+  int strategy;+};++/**+ * Get the default options for gzip compression.+ * A codec created with these options will have type CodecType::GZIP.+ */+Options defaultGzipOptions();++/**+ * Get the default options for zlib compression.+ * A codec created with these options will have type CodecType::ZLIB.+ */+Options defaultZlibOptions();++/**+ * Get a codec with the given options and compression level.+ *+ * If the windowSize is 15 and the format is Format::ZLIB or Format::GZIP, then+ * the type of the codec will be CodecType::ZLIB or CodecType::GZIP+ * respectively. Otherwise, the type will be CodecType::USER_DEFINED.+ *+ * Automatic uncompression is not supported with USER_DEFINED codecs.+ *+ * Levels supported: 0 = no compression, 1 = fast, ..., 9 = best; default = 6+ */+std::unique_ptr<Codec> getCodec(+    Options options = Options(), int level = COMPRESSION_LEVEL_DEFAULT);+std::unique_ptr<StreamCodec> getStreamCodec(+    Options options = Options(), int level = COMPRESSION_LEVEL_DEFAULT);++} // namespace zlib+} // namespace compression+} // namespace folly++#endif // FOLLY_HAVE_LIBZ
+ folly/folly/compression/Zstd.cpp view
@@ -0,0 +1,250 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#include <folly/compression/Zstd.h>++#if FOLLY_HAVE_LIBZSTD++#include <stdexcept>+#include <string>++#include <zstd.h>++#include <folly/Conv.h>+#include <folly/Range.h>+#include <folly/ScopeGuard.h>+#include <folly/compression/CompressionContextPoolSingletons.h>+#include <folly/compression/Utils.h>++static_assert(+    ZSTD_VERSION_NUMBER >= 10400,+    "zstd-1.4.0 is the minimum supported zstd version.");++using folly::compression::detail::dataStartsWithLE;+using folly::compression::detail::prefixToStringLE;++using namespace folly::compression::contexts;++namespace folly {+namespace compression {+namespace zstd {+namespace {++size_t zstdThrowIfError(size_t rc) {+  if (!ZSTD_isError(rc)) {+    return rc;+  }+  throw std::runtime_error(+      to<std::string>("ZSTD returned an error: ", ZSTD_getErrorName(rc)));+}++ZSTD_EndDirective zstdTranslateFlush(StreamCodec::FlushOp flush) {+  switch (flush) {+    case StreamCodec::FlushOp::NONE:+      return ZSTD_e_continue;+    case StreamCodec::FlushOp::FLUSH:+      return ZSTD_e_flush;+    case StreamCodec::FlushOp::END:+      return ZSTD_e_end;+    default:+      throw std::invalid_argument("ZSTDStreamCodec: Invalid flush");+  }+}++class ZSTDStreamCodec final : public StreamCodec {+ public:+  explicit ZSTDStreamCodec(Options options);++  std::vector<std::string> validPrefixes() const override;+  bool canUncompress(+      const IOBuf* data, Optional<uint64_t> uncompressedLength) const override;++ private:+  bool doNeedsUncompressedLength() const override;+  uint64_t doMaxCompressedLength(uint64_t uncompressedLength) const override;+  Optional<uint64_t> doGetUncompressedLength(+      IOBuf const* data, Optional<uint64_t> uncompressedLength) const override;++  void doResetStream() override;+  bool doCompressStream(+      ByteRange& input,+      MutableByteRange& output,+      StreamCodec::FlushOp flushOp) override;+  bool doUncompressStream(+      ByteRange& input,+      MutableByteRange& output,+      StreamCodec::FlushOp flushOp) override;++  void resetCCtx();+  void resetDCtx();++  Options options_;+  ZSTD_CCtx_Pool::Ref cctx_{getNULL_ZSTD_CCtx()};+  ZSTD_DCtx_Pool::Ref dctx_{getNULL_ZSTD_DCtx()};+};++constexpr uint32_t kZSTDMagicLE = 0xFD2FB528;++std::vector<std::string> ZSTDStreamCodec::validPrefixes() const {+  return {prefixToStringLE(kZSTDMagicLE)};+}++bool ZSTDStreamCodec::canUncompress(+    const IOBuf* data, Optional<uint64_t>) const {+  return dataStartsWithLE(data, kZSTDMagicLE);+}++CodecType codecType(Options const& options) {+  int const level = options.level();+  DCHECK_NE(level, 0);+  return level > 0 ? CodecType::ZSTD : CodecType::ZSTD_FAST;+}++ZSTDStreamCodec::ZSTDStreamCodec(Options options)+    : StreamCodec(codecType(options), options.level()),+      options_(std::move(options)) {}++bool ZSTDStreamCodec::doNeedsUncompressedLength() const {+  return false;+}++uint64_t ZSTDStreamCodec::doMaxCompressedLength(+    uint64_t uncompressedLength) const {+  return ZSTD_compressBound(uncompressedLength);+}++Optional<uint64_t> ZSTDStreamCodec::doGetUncompressedLength(+    IOBuf const* data, Optional<uint64_t> uncompressedLength) const {+  // Read decompressed size from frame if available in first IOBuf.+  auto const decompressedSize =+      ZSTD_getFrameContentSize(data->data(), data->length());+  if (decompressedSize == ZSTD_CONTENTSIZE_UNKNOWN ||+      decompressedSize == ZSTD_CONTENTSIZE_ERROR) {+    return uncompressedLength;+  }+  if (uncompressedLength && *uncompressedLength != decompressedSize) {+    throw std::runtime_error("ZSTD: invalid uncompressed length");+  }+  return decompressedSize;+}++void ZSTDStreamCodec::doResetStream() {+  cctx_.reset(nullptr);+  dctx_.reset(nullptr);+}++void ZSTDStreamCodec::resetCCtx() {+  DCHECK(cctx_ == nullptr);+  cctx_ = getZSTD_CCtx(); // Gives us a clean context+  DCHECK(cctx_ != nullptr);+  zstdThrowIfError(+      ZSTD_CCtx_setParametersUsingCCtxParams(cctx_.get(), options_.params()));+  zstdThrowIfError(ZSTD_CCtx_setPledgedSrcSize(+      cctx_.get(), uncompressedLength().value_or(ZSTD_CONTENTSIZE_UNKNOWN)));+}++bool ZSTDStreamCodec::doCompressStream(+    ByteRange& input, MutableByteRange& output, StreamCodec::FlushOp flushOp) {+  if (cctx_ == nullptr) {+    resetCCtx();+  }+  ZSTD_inBuffer in = {input.data(), input.size(), 0};+  ZSTD_outBuffer out = {output.data(), output.size(), 0};+  SCOPE_EXIT {+    input.uncheckedAdvance(in.pos);+    output.uncheckedAdvance(out.pos);+  };+  size_t const rc = zstdThrowIfError(ZSTD_compressStream2(+      cctx_.get(), &out, &in, zstdTranslateFlush(flushOp)));+  switch (flushOp) {+    case StreamCodec::FlushOp::NONE:+      return false;+    case StreamCodec::FlushOp::FLUSH:+      return rc == 0;+    case StreamCodec::FlushOp::END:+      if (rc == 0) {+        // Surrender our cctx_+        doResetStream();+      }+      return rc == 0;+    default:+      throw std::invalid_argument("ZSTD: invalid FlushOp");+  }+}++void ZSTDStreamCodec::resetDCtx() {+  DCHECK(dctx_ == nullptr);+  dctx_ = getZSTD_DCtx(); // Gives us a clean context+  DCHECK(dctx_ != nullptr);+  if (options_.maxWindowSize() != 0) {+    zstdThrowIfError(+        ZSTD_DCtx_setMaxWindowSize(dctx_.get(), options_.maxWindowSize()));+  }+}++bool ZSTDStreamCodec::doUncompressStream(+    ByteRange& input, MutableByteRange& output, StreamCodec::FlushOp) {+  if (dctx_ == nullptr) {+    resetDCtx();+  }+  ZSTD_inBuffer in = {input.data(), input.size(), 0};+  ZSTD_outBuffer out = {output.data(), output.size(), 0};+  SCOPE_EXIT {+    input.uncheckedAdvance(in.pos);+    output.uncheckedAdvance(out.pos);+  };+  size_t const rc =+      zstdThrowIfError(ZSTD_decompressStream(dctx_.get(), &out, &in));+  if (rc == 0) {+    // Surrender our dctx_+    doResetStream();+  }+  return rc == 0;+}++} // namespace++Options::Options(int level) : params_(ZSTD_createCCtxParams()), level_(level) {+  if (params_ == nullptr) {+    throw std::bad_alloc{};+  }+  zstdThrowIfError(ZSTD_CCtxParams_init(params_.get(), level));+}++void Options::set(ZSTD_cParameter param, unsigned value) {+  zstdThrowIfError(ZSTD_CCtxParams_setParameter(params_.get(), param, value));+  if (param == ZSTD_c_compressionLevel) {+    level_ = static_cast<int>(value);+  }+}++/* static */ void Options::freeCCtxParams(ZSTD_CCtx_params* params) {+  ZSTD_freeCCtxParams(params);+}++std::unique_ptr<Codec> getCodec(Options options) {+  return std::make_unique<ZSTDStreamCodec>(std::move(options));+}++std::unique_ptr<StreamCodec> getStreamCodec(Options options) {+  return std::make_unique<ZSTDStreamCodec>(std::move(options));+}++} // namespace zstd+} // namespace compression+} // namespace folly++#endif
+ folly/folly/compression/Zstd.h view
@@ -0,0 +1,92 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <memory.h>++#include <folly/Memory.h>+#include <folly/Portability.h>+#include <folly/compression/Compression.h>++#if FOLLY_HAVE_LIBZSTD++#ifndef ZSTD_STATIC_LINKING_ONLY+#define ZSTD_STATIC_LINKING_ONLY+#endif+#include <zstd.h>++namespace folly {+namespace compression {+namespace zstd {++/**+ * Interface for zstd-specific codec initialization.+ */+class Options {+ public:+  /* Create an Options struct with the default options for the given `level`.+   * NOTE: This is the zstd level, COMPRESSION_LEVEL_DEFAULT and such aren't+   *       supported, since zstd supports negative compression levels.+   */+  explicit Options(int level);++  /**+   * Set the compression `param` to `value`.+   * See the zstd documentation for ZSTD_CCtx_setParameter() for details, this+   * is just a thin wrapper.+   */+  void set(ZSTD_cParameter param, unsigned value);++  /**+   * Set the maximum allowed window size during decompression.+   * `maxWindowSize == 0` means don't set the maximum window size.+   * zstd's current default limit is 2^27.+   * See the zstd documentation for ZSTD_DCtx_setMaxWindowSize() for details.+   */+  void setMaxWindowSize(size_t maxWindowSize) {+    maxWindowSize_ = maxWindowSize;+  }++  /// Get a reference to the ZSTD_CCtx_params.+  ZSTD_CCtx_params const* params() const { return params_.get(); }++  /// Get the compression level.+  int level() const { return level_; }++  /// Get the maximum window size.+  size_t maxWindowSize() const { return maxWindowSize_; }++ private:+  static void freeCCtxParams(ZSTD_CCtx_params* params);+  std::unique_ptr<+      ZSTD_CCtx_params,+      folly::static_function_deleter<ZSTD_CCtx_params, &freeCCtxParams>>+      params_;+  size_t maxWindowSize_{0};+  int level_;+};++/// Get a zstd Codec with the given options.+std::unique_ptr<Codec> getCodec(Options options);+/// Get a zstd StreamCodec with the given options.+std::unique_ptr<StreamCodec> getStreamCodec(Options options);++} // namespace zstd+} // namespace compression+} // namespace folly++#endif
+ folly/folly/compression/elias_fano/BitVectorCoding.h view
@@ -0,0 +1,447 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <cstdlib>+#include <limits>+#include <type_traits>++#include <glog/logging.h>++#include <folly/Likely.h>+#include <folly/Portability.h>+#include <folly/Range.h>+#include <folly/compression/Instructions.h>+#include <folly/compression/Select64.h>+#include <folly/compression/elias_fano/CodingDetail.h>+#include <folly/lang/Bits.h>+#include <folly/lang/BitsClass.h>++namespace folly {+namespace compression {++static_assert(kIsLittleEndian, "BitVectorCoding.h requires little endianness");++template <class Pointer>+struct BitVectorCompressedListBase {+  BitVectorCompressedListBase() = default;++  template <class OtherPointer>+  BitVectorCompressedListBase(+      const BitVectorCompressedListBase<OtherPointer>& other)+      : size(other.size),+        upperBound(other.upperBound),+        data(other.data),+        bits(reinterpret_cast<Pointer>(other.bits)),+        skipPointers(reinterpret_cast<Pointer>(other.skipPointers)),+        forwardPointers(reinterpret_cast<Pointer>(other.forwardPointers)) {}++  template <class T = Pointer>+  auto free() -> decltype(::free(T(nullptr))) {+    return ::free(data.data());+  }++  size_t size = 0;+  size_t upperBound = 0;++  folly::Range<Pointer> data;++  Pointer bits = nullptr;+  Pointer skipPointers = nullptr;+  Pointer forwardPointers = nullptr;+};++typedef BitVectorCompressedListBase<const uint8_t*> BitVectorCompressedList;+typedef BitVectorCompressedListBase<uint8_t*> MutableBitVectorCompressedList;++template <+    class Value,+    class SkipValue,+    size_t kSkipQuantum = 0,+    size_t kForwardQuantum = 0>+struct BitVectorEncoder {+  static_assert(+      std::is_integral<Value>::value && std::is_unsigned<Value>::value,+      "Value should be unsigned integral");++  typedef BitVectorCompressedList CompressedList;+  typedef MutableBitVectorCompressedList MutableCompressedList;++  typedef Value ValueType;+  typedef SkipValue SkipValueType;+  struct Layout;++  static constexpr size_t skipQuantum = kSkipQuantum;+  static constexpr size_t forwardQuantum = kForwardQuantum;++  template <class RandomAccessIterator>+  static MutableCompressedList encode(+      RandomAccessIterator begin, RandomAccessIterator end) {+    if (begin == end) {+      return MutableCompressedList();+    }+    BitVectorEncoder encoder(size_t(end - begin), *(end - 1));+    for (; begin != end; ++begin) {+      encoder.add(*begin);+    }+    return encoder.finish();+  }++  explicit BitVectorEncoder(const MutableCompressedList& result)+      : bits_(result.bits),+        skipPointers_(result.skipPointers),+        forwardPointers_(result.forwardPointers),+        result_(result) {+    memset(result.data.data(), 0, result.data.size());+  }++  BitVectorEncoder(size_t size, ValueType upperBound)+      : BitVectorEncoder(+            Layout::fromUpperBoundAndSize(upperBound, size).allocList()) {}++  void add(ValueType value) {+    CHECK_LT(value, std::numeric_limits<ValueType>::max());+    // Also works when lastValue_ == -1.+    CHECK_GT(value + 1, lastValue_ + 1)+        << "BitVectorCoding only supports stricly monotone lists";++    auto block = bits_ + (value / 64) * sizeof(uint64_t);+    size_t inner = value % 64;+    folly::Bits<folly::Unaligned<uint64_t>>::set(+        reinterpret_cast<folly::Unaligned<uint64_t>*>(block), inner);++    if constexpr (skipQuantum != 0) {+      size_t nextSkipPointerSize = value / skipQuantum;+      while (skipPointersSize_ < nextSkipPointerSize) {+        auto pos = skipPointersSize_++;+        folly::storeUnaligned<SkipValueType>(+            skipPointers_ + pos * sizeof(SkipValueType), size_);+      }+    }++    if constexpr (forwardQuantum != 0) {+      if (size_ != 0 && (size_ % forwardQuantum == 0)) {+        const auto pos = size_ / forwardQuantum - 1;+        folly::storeUnaligned<SkipValueType>(+            forwardPointers_ + pos * sizeof(SkipValueType), value);+      }+    }++    lastValue_ = value;+    ++size_;+  }++  const MutableCompressedList& finish() const {+    CHECK_EQ(size_, result_.size);+    // TODO(ott): Relax this assumption.+    CHECK_EQ(result_.upperBound, lastValue_);+    return result_;+  }++ private:+  uint8_t* const bits_ = nullptr;+  uint8_t* const skipPointers_ = nullptr;+  uint8_t* const forwardPointers_ = nullptr;++  ValueType lastValue_ = -1;+  size_t size_ = 0;+  size_t skipPointersSize_ = 0;++  MutableCompressedList result_;+};++template <+    class Value,+    class SkipValue,+    size_t kSkipQuantum,+    size_t kForwardQuantum>+struct BitVectorEncoder<Value, SkipValue, kSkipQuantum, kForwardQuantum>::+    Layout {+  static Layout fromUpperBoundAndSize(size_t upperBound, size_t size) {+    Layout layout;+    layout.size = size;+    layout.upperBound = upperBound;++    size_t bitVectorSizeInBytes = (upperBound / 8) + 1;+    layout.bits = bitVectorSizeInBytes;++    if constexpr (skipQuantum != 0) {+      size_t numSkipPointers = upperBound / skipQuantum;+      layout.skipPointers = numSkipPointers * sizeof(SkipValueType);+    }+    if constexpr (forwardQuantum != 0) {+      size_t numForwardPointers = size / forwardQuantum;+      layout.forwardPointers = numForwardPointers * sizeof(SkipValueType);+    }++    CHECK_LT(size, std::numeric_limits<SkipValueType>::max());++    return layout;+  }++  size_t bytes() const { return bits + skipPointers + forwardPointers; }++  template <class Range>+  BitVectorCompressedListBase<typename Range::iterator> openList(+      Range& buf) const {+    BitVectorCompressedListBase<typename Range::iterator> result;+    result.size = size;+    result.upperBound = upperBound;+    result.data = buf.subpiece(0, bytes());+    auto advance = [&](size_t n) {+      auto begin = buf.data();+      buf.advance(n);+      return begin;+    };++    result.bits = advance(bits);+    result.skipPointers = advance(skipPointers);+    result.forwardPointers = advance(forwardPointers);+    CHECK_EQ(buf.data() - result.data.data(), bytes());++    return result;+  }++  MutableCompressedList allocList() const {+    uint8_t* buf = nullptr;+    if (size > 0) {+      buf = static_cast<uint8_t*>(malloc(bytes() + 7));+    }+    folly::MutableByteRange bufRange(buf, bytes());+    return openList(bufRange);+  }++  size_t size = 0;+  size_t upperBound = 0;++  // Sizes in bytes.+  size_t bits = 0;+  size_t skipPointers = 0;+  size_t forwardPointers = 0;+};++template <+    class Encoder,+    class Instructions = instructions::Default,+    bool kUnchecked = false>+class BitVectorReader+    : detail::ForwardPointers<Encoder::forwardQuantum>,+      detail::SkipPointers<Encoder::skipQuantum> {+ public:+  typedef Encoder EncoderType;+  typedef typename Encoder::ValueType ValueType;+  // A bitvector can only be as large as its largest value.+  typedef typename Encoder::ValueType SizeType;+  typedef typename Encoder::SkipValueType SkipValueType;++  explicit BitVectorReader(const typename Encoder::CompressedList& list)+      : detail::ForwardPointers<Encoder::forwardQuantum>(list.forwardPointers),+        detail::SkipPointers<Encoder::skipQuantum>(list.skipPointers),+        bits_(list.bits),+        size_(list.size),+        upperBound_(kUnchecked || list.size == 0 ? 0 : list.upperBound) {+    reset();+  }++  void reset() {+    // Pretend the bitvector is prefixed by a block of zeroes.+    block_ = 0;+    position_ = static_cast<SizeType>(-1);+    outer_ = static_cast<SizeType>(-sizeof(uint64_t));+    value_ = kInvalidValue;+  }++  bool next() {+    if (!kUnchecked && FOLLY_UNLIKELY(position() + 1 >= size_)) {+      return setDone();+    }++    while (block_ == 0) {+      outer_ += sizeof(uint64_t);+      block_ = folly::loadUnaligned<uint64_t>(bits_ + outer_);+    }++    ++position_;+    auto inner = Instructions::ctz(block_);+    block_ = Instructions::blsr(block_);++    return setValue(inner);+  }++  bool skip(SizeType n) {+    if (n == 0) {+      return valid();+    }++    if (!kUnchecked && position() + n >= size_) {+      return setDone();+    }+    // Small skip optimization.+    if (FOLLY_LIKELY(n < kLinearScanThreshold)) {+      for (size_t i = 0; i < n; ++i) {+        next();+      }+      return true;+    }++    position_ += n;++    // Use forward pointer.+    if constexpr (Encoder::forwardQuantum > 0) {+      if (n > Encoder::forwardQuantum) {+        const size_t steps = position_ / Encoder::forwardQuantum;+        const size_t dest = folly::loadUnaligned<SkipValueType>(+            this->forwardPointers_ + (steps - 1) * sizeof(SkipValueType));++        reposition(dest);+        n = position_ + 1 - steps * Encoder::forwardQuantum;+      }+    }++    size_t cnt;+    // Find necessary block.+    while ((cnt = Instructions::popcount(block_)) < n) {+      n -= cnt;+      outer_ += sizeof(uint64_t);+      block_ = folly::loadUnaligned<uint64_t>(bits_ + outer_);+    }++    // Skip to the n-th one in the block.+    DCHECK_GT(n, 0);+    auto inner = select64<Instructions>(block_, n - 1);+    block_ &= (uint64_t(-1) << inner) << 1;++    return setValue(inner);+  }++  template <bool kCanBeAtValue = true>+  bool skipTo(ValueType v) {+    // Also works when value_ == kInvalidValue.+    if (v != kInvalidValue) {+      DCHECK_GE(v + 1, value_ + 1);+    }++    if (!kUnchecked && v > upperBound_) {+      return setDone();+    } else if (kCanBeAtValue && v == value_) {+      return true;+    }++    // Small skip optimization.+    if (v - value_ < kLinearScanThreshold) {+      do {+        next();+      } while (value() < v);++      return true;+    }++    if constexpr (Encoder::skipQuantum > 0) {+      if (v - value_ > Encoder::skipQuantum) {+        size_t q = v / Encoder::skipQuantum;+        auto skipPointer = folly::loadUnaligned<SkipValueType>(+            this->skipPointers_ + (q - 1) * sizeof(SkipValueType));+        position_ = static_cast<SizeType>(skipPointer) - 1;++        reposition(q * Encoder::skipQuantum);+      }+    }++    // Find the value.+    size_t outer = v / 64 * sizeof(uint64_t);++    while (outer_ != outer) {+      position_ += Instructions::popcount(block_);+      outer_ += sizeof(uint64_t);+      block_ = folly::loadUnaligned<uint64_t>(bits_ + outer_);+      DCHECK_LE(outer_, outer);+    }++    uint64_t mask = ~((uint64_t(1) << (v % 64)) - 1);+    position_ += Instructions::popcount(block_ & ~mask) + 1;+    block_ &= mask;++    while (block_ == 0) {+      outer_ += sizeof(uint64_t);+      block_ = folly::loadUnaligned<uint64_t>(bits_ + outer_);+    }++    auto inner = Instructions::ctz(block_);+    block_ = Instructions::blsr(block_);++    setValue(inner);+    return true;+  }++  SizeType size() const { return size_; }++  bool valid() const {+    return position() < size(); // Also checks that position() != -1.+  }++  SizeType position() const { return position_; }+  ValueType value() const {+    DCHECK(valid());+    return value_;+  }++  bool jump(SizeType n) {+    reset();+    return skip(n + 1);+  }++  bool jumpTo(ValueType v) {+    reset();+    return skipTo(v);+  }++  bool setDone() {+    value_ = kInvalidValue;+    position_ = size_;+    return false;+  }++ private:+  // Must hold kInvalidValue + 1 == 0.+  constexpr static ValueType kInvalidValue = -1;++  bool setValue(size_t inner) {+    value_ = static_cast<ValueType>(8 * outer_ + inner);+    return true;+  }++  void reposition(size_t dest) {+    outer_ = dest / 64 * 8;+    // We maintain the invariant that outer_ is divisible by 8.+    block_ = folly::loadUnaligned<uint64_t>(bits_ + outer_);+    block_ &= ~((uint64_t(1) << (dest % 64)) - 1);+  }++  constexpr static size_t kLinearScanThreshold = 4;++  const uint8_t* const bits_;+  uint64_t block_;+  SizeType outer_;+  SizeType position_;+  ValueType value_;++  const SizeType size_;+  const ValueType upperBound_;+};++} // namespace compression+} // namespace folly
+ folly/folly/compression/elias_fano/CodingDetail.h view
@@ -0,0 +1,63 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++/**+ * Shared utils for BitVectorCoding.h and EliasFanoCoding.h.+ */++#pragma once++#include <stddef.h>++namespace folly {+namespace compression {+namespace detail {++/**+ * Helpers to store pointers to forward and skip pointer arrays only+ * if they are used, that is, the quantum is nonzero. If it is 0, the+ * class is empty, and the member is static to keep the syntax valid,+ * thus it will take no space in a derived class thanks to empty base+ * class optimization.+ */+template <size_t>+class ForwardPointers {+ protected:+  explicit ForwardPointers(const unsigned char* ptr) : forwardPointers_(ptr) {}+  const unsigned char* const forwardPointers_;+};+template <>+class ForwardPointers<0> {+ protected:+  explicit ForwardPointers(const unsigned char*) {}+  constexpr static const unsigned char* const forwardPointers_{};+};++template <size_t>+class SkipPointers {+ protected:+  explicit SkipPointers(const unsigned char* ptr) : skipPointers_(ptr) {}+  const unsigned char* const skipPointers_;+};+template <>+class SkipPointers<0> {+ protected:+  explicit SkipPointers(const unsigned char*) {}+  constexpr static const unsigned char* const skipPointers_{};+};+} // namespace detail+} // namespace compression+} // namespace folly
+ folly/folly/compression/elias_fano/EliasFanoCoding.h view
@@ -0,0 +1,890 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++/**+ * Based on the paper by Sebastiano Vigna,+ * "Quasi-succinct indices" (arxiv:1206.4300).+ */++#pragma once++#include <algorithm>+#include <cstdlib>+#include <limits>+#include <type_traits>++#include <glog/logging.h>++#include <folly/Likely.h>+#include <folly/Portability.h>+#include <folly/Range.h>+#include <folly/compression/Instructions.h>+#include <folly/compression/Select64.h>+#include <folly/compression/elias_fano/CodingDetail.h>+#include <folly/lang/Assume.h>+#include <folly/lang/Bits.h>++namespace folly {+namespace compression {++static_assert(kIsLittleEndian, "EliasFanoCoding.h requires little endianness");++constexpr size_t kCacheLineSize = 64;++template <class Pointer>+struct EliasFanoCompressedListBase {+  EliasFanoCompressedListBase() = default;++  template <class OtherPointer>+  EliasFanoCompressedListBase(+      const EliasFanoCompressedListBase<OtherPointer>& other)+      : size(other.size),+        numLowerBits(other.numLowerBits),+        upperSizeBytes(other.upperSizeBytes),+        data(other.data),+        skipPointers(reinterpret_cast<Pointer>(other.skipPointers)),+        forwardPointers(reinterpret_cast<Pointer>(other.forwardPointers)),+        lower(reinterpret_cast<Pointer>(other.lower)),+        upper(reinterpret_cast<Pointer>(other.upper)) {}++  template <class T = Pointer>+  auto free() -> decltype(::free(T(nullptr))) {+    return ::free(data.data());+  }++  size_t size = 0;+  uint8_t numLowerBits = 0;+  size_t upperSizeBytes = 0;++  // WARNING: EliasFanoCompressedList has no ownership of data. The 7 bytes+  // following the last byte should be readable if kUpperFirst = false, 8 bytes+  // otherwise.+  Range<Pointer> data;++  Pointer skipPointers = nullptr;+  Pointer forwardPointers = nullptr;+  Pointer lower = nullptr;+  Pointer upper = nullptr;+};++using EliasFanoCompressedList = EliasFanoCompressedListBase<const uint8_t*>;+using MutableEliasFanoCompressedList = EliasFanoCompressedListBase<uint8_t*>;++template <+    class Value,+    // SkipValue must be wide enough to be able to represent the list length.+    class SkipValue = uint64_t,+    size_t kSkipQuantum = 0, // 0 = disabled+    size_t kForwardQuantum = 0, // 0 = disabled+    bool kUpperFirst = false>+struct EliasFanoEncoder {+  static_assert(+      std::is_integral_v<Value> && std::is_unsigned_v<Value>,+      "Value should be unsigned integral");++  using CompressedList = EliasFanoCompressedList;+  using MutableCompressedList = MutableEliasFanoCompressedList;++  using ValueType = Value;+  using SkipValueType = SkipValue;+  struct Layout;++  static constexpr size_t skipQuantum = kSkipQuantum;+  static constexpr size_t forwardQuantum = kForwardQuantum;++  static uint8_t defaultNumLowerBits(size_t upperBound, size_t size) {+    if (FOLLY_UNLIKELY(size == 0 || upperBound < size)) {+      return 0;+    }+    // Result that should be returned is "floor(log(upperBound / size))".+    // In order to avoid expensive division, we rely on+    // "floor(a) - floor(b) - 1 <= floor(a - b) <= floor(a) - floor(b)".+    // Assuming "candidate = floor(log(upperBound)) - floor(log(upperBound))",+    // then result is either "candidate - 1" or "candidate".+    auto candidate = findLastSet(upperBound) - findLastSet(size);+    // NOTE: As size != 0, "candidate" is always < 64.+    return (size > (upperBound >> candidate)) ? candidate - 1 : candidate;+  }++  // Requires: input range (begin, end) is sorted (encoding+  // crashes if it's not).+  // WARNING: encode() mallocates EliasFanoCompressedList::data. As+  // EliasFanoCompressedList has no ownership of it, you need to call+  // free() explicitly.+  template <class RandomAccessIterator>+  static MutableCompressedList encode(+      RandomAccessIterator begin, RandomAccessIterator end) {+    if (begin == end) {+      return MutableCompressedList();+    }+    EliasFanoEncoder encoder(size_t(end - begin), *(end - 1));+    for (; begin != end; ++begin) {+      encoder.add(*begin);+    }+    return encoder.finish();+  }++  explicit EliasFanoEncoder(const MutableCompressedList& result)+      : lower_(result.lower),+        upper_(result.upper),+        skipPointers_(reinterpret_cast<SkipValueType*>(result.skipPointers)),+        forwardPointers_(+            reinterpret_cast<SkipValueType*>(result.forwardPointers)),+        result_(result) {+    std::fill(result.data.begin(), result.data.end(), '\0');+  }++  EliasFanoEncoder(size_t size, ValueType upperBound)+      : EliasFanoEncoder(+            Layout::fromUpperBoundAndSize(upperBound, size).allocList()) {}++  void add(ValueType value) {+    CHECK_GE(value, lastValue_);+    CHECK_LT(size_, result_.size)+        << "add() called more times than the size specified in construction";++    const auto numLowerBits = result_.numLowerBits;+    const ValueType upperBits = value >> numLowerBits;++    // Upper sequence consists of upperBits 0-bits and (size_ + 1) 1-bits.+    const size_t pos = upperBits + size_;+    upper_[pos / 8] |= 1U << (pos % 8);+    // Append numLowerBits bits to lower sequence.+    if (numLowerBits != 0) {+      const ValueType lowerBits = value & ((ValueType(1) << numLowerBits) - 1);+      writeBits56(lower_, size_ * numLowerBits, numLowerBits, lowerBits);+    }++    fillSkipPointersUpTo(upperBits);++    if constexpr (forwardQuantum != 0) {+      if ((size_ + 1) % forwardQuantum == 0) {+        DCHECK_LE(upperBits, std::numeric_limits<SkipValueType>::max());+        const auto k = size_ / forwardQuantum;+        // Store the number of preceding 0-bits.+        forwardPointers_[k] = upperBits;+      }+    }++    lastValue_ = value;+    ++size_;+  }++  const MutableCompressedList& finish() {+    CHECK_EQ(size_, result_.size)+        << "Number of add()s must be equal to the size specified in construction";+    const ValueType upperBitsUniverse =+        (8 * result_.upperSizeBytes - result_.size);+    // Populate skip pointers up to the universe upper bound (inclusive).+    fillSkipPointersUpTo(upperBitsUniverse);+    return result_;+  }++ private:+  void fillSkipPointersUpTo(ValueType fillBoundary) {+    if constexpr (skipQuantum != 0) {+      DCHECK_LE(size_, std::numeric_limits<SkipValueType>::max());+      // The first skip pointer is omitted (it would always be 0), so the+      // calculation is shifted by 1.+      while ((skipPointersSize_ + 1) * skipQuantum <= fillBoundary) {+        // Store the number of preceding 1-bits.+        skipPointers_[skipPointersSize_++] = static_cast<SkipValueType>(size_);+      }+    }+  }+  // Writes value (with len up to 56 bits) to data starting at pos-th bit.+  static void writeBits56(+      unsigned char* data, size_t pos, uint8_t len, uint64_t value) {+    DCHECK_LE(uint32_t(len), 56);+    DCHECK_EQ(0, value & ~((uint64_t(1) << len) - 1));+    unsigned char* const ptr = data + (pos / 8);+    uint64_t ptrv = loadUnaligned<uint64_t>(ptr);+    ptrv |= value << (pos % 8);+    storeUnaligned<uint64_t>(ptr, ptrv);+  }++  unsigned char* lower_ = nullptr;+  unsigned char* upper_ = nullptr;+  SkipValueType* skipPointers_ = nullptr;+  SkipValueType* forwardPointers_ = nullptr;++  ValueType lastValue_ = 0;+  size_t size_ = 0;+  size_t skipPointersSize_ = 0;++  MutableCompressedList result_;+};++template <+    class Value,+    class SkipValue,+    size_t kSkipQuantum,+    size_t kForwardQuantum,+    bool kUpperFirst>+struct EliasFanoEncoder<+    Value,+    SkipValue,+    kSkipQuantum,+    kForwardQuantum,+    kUpperFirst>::Layout {+  static Layout fromUpperBoundAndSize(size_t upperBound, size_t size) {+    // numLowerBits can be at most 56 because of detail::writeBits56.+    const uint8_t numLowerBits =+        std::min(defaultNumLowerBits(upperBound, size), uint8_t(56));+    // *** Upper bits.+    // Upper bits are stored using unary delta encoding.+    // For example, (3 5 5 9) will be encoded as 1000011001000_2.+    const size_t upperSizeBits =+        (upperBound >> numLowerBits) + // Number of 0-bits to be stored.+        size; // 1-bits.+    const size_t upper = (upperSizeBits + 7) / 8;++    // *** Validity checks.+    // Shift by numLowerBits must be valid.+    CHECK_LT(static_cast<int>(numLowerBits), 8 * sizeof(Value));+    CHECK_LE(+        upperBound >> numLowerBits, std::numeric_limits<SkipValueType>::max());++    return fromInternalSizes(numLowerBits, upper, size);+  }++  static Layout fromInternalSizes(+      uint8_t numLowerBits, size_t upper, size_t size) {+    Layout layout;+    layout.size = size;+    layout.numLowerBits = numLowerBits;++    layout.lower = (numLowerBits * size + 7) / 8;+    layout.upper = upper;++    // *** Skip pointers.+    // Store (1-indexed) position of every skipQuantum-th+    // 0-bit in upper bits sequence.+    if constexpr (skipQuantum != 0) {+      // 8 * upper is used here instead of upperSizeBits, as that is+      // more serialization-friendly way (upperSizeBits doesn't need+      // to be known by this function, unlike upper).++      size_t numSkipPointers = (8 * upper - size) / skipQuantum;+      layout.skipPointers = numSkipPointers * sizeof(SkipValueType);+    }++    // *** Forward pointers.+    // Store (1-indexed) position of every forwardQuantum-th+    // 1-bit in upper bits sequence.+    if constexpr (forwardQuantum != 0) {+      size_t numForwardPointers = size / forwardQuantum;+      layout.forwardPointers = numForwardPointers * sizeof(SkipValueType);+    }++    return layout;+  }++  size_t bytes() const {+    return lower + upper + skipPointers + forwardPointers;+  }++  template <class Range>+  EliasFanoCompressedListBase<typename Range::iterator> openList(+      Range& buf) const {+    EliasFanoCompressedListBase<typename Range::iterator> result;+    result.size = size;+    result.numLowerBits = numLowerBits;+    result.upperSizeBytes = upper;+    result.data = buf.subpiece(0, bytes());++    auto advance = [&](size_t n) {+      auto begin = buf.data();+      buf.advance(n);+      return begin;+    };++    result.skipPointers = advance(skipPointers);+    result.forwardPointers = advance(forwardPointers);+    if constexpr (kUpperFirst) {+      result.upper = advance(upper);+      result.lower = advance(lower);+    } else {+      result.lower = advance(lower);+      result.upper = advance(upper);+    }++    return result;+  }++  MutableCompressedList allocList() const {+    uint8_t* buf = nullptr;+    // WARNING: Current read/write logic assumes that the 7 bytes+    // following the upper bytes and the 8 bytes following the lower bytes+    // sequences are readable (stored value doesn't matter and won't be+    // changed), so we allocate additional 8 bytes, but do not include them in+    // size of returned value.+    if (size > 0) {+      buf = static_cast<uint8_t*>(malloc(bytes() + 8));+    }+    MutableByteRange bufRange(buf, bytes());+    return openList(bufRange);+  }++  size_t size = 0;+  uint8_t numLowerBits = 0;++  // Sizes in bytes.+  size_t lower = 0;+  size_t upper = 0;+  size_t skipPointers = 0;+  size_t forwardPointers = 0;+};++namespace detail {++// Add a and b in the domain of T. This guarantees that if T is a sub-int type,+// we cast away the promotion to int, so that unsigned overflow and underflow+// work as expected.+template <class T, class U>+FOLLY_ALWAYS_INLINE T addT(T a, U b) {+  static_assert(std::is_unsigned_v<T>);+  return static_cast<T>(a + static_cast<T>(b));+}++template <+    class Encoder,+    class Instructions,+    class SizeType,+    bool kUnchecked = false>+class UpperBitsReader+    : ForwardPointers<Encoder::forwardQuantum>,+      SkipPointers<Encoder::skipQuantum> {+  using SkipValueType = typename Encoder::SkipValueType;++ public:+  using ValueType = typename Encoder::ValueType;++  static_assert(+      std::is_integral_v<SizeType> && std::is_unsigned_v<SizeType>,+      "SizeType should be unsigned integral");+  // Functions like `jump()` and `next()` rely on this being the predecessor+  // of 0.  `valid()` also needs it to be the largest possible `SizeType`.+  static constexpr SizeType kBeforeFirstPos = -1;++  explicit UpperBitsReader(const typename Encoder::CompressedList& list)+      : ForwardPointers<Encoder::forwardQuantum>(list.forwardPointers),+        SkipPointers<Encoder::skipQuantum>(list.skipPointers),+        start_(list.upper),+        size_(list.size),+        upperBound_(estimateUpperBound(list)) {+    reset();+  }++  void reset() {+    // Pretend the bitvector is prefixed by a block of zeroes.+    block_ = 0;+    position_ = kBeforeFirstPos;+    outer_ = static_cast<OuterType>(-sizeof(block_t));+    value_ = 0;+  }++  FOLLY_ALWAYS_INLINE SizeType position() const { return position_; }++  FOLLY_ALWAYS_INLINE ValueType value() const { return value_; }++  FOLLY_ALWAYS_INLINE bool valid() const {+    // SizeType is unsigned, so this also ensures position() != kBeforeFirstPos+    return position() < size();+  }++  FOLLY_ALWAYS_INLINE SizeType size() const { return size_; }++  FOLLY_ALWAYS_INLINE bool previous() {+    if (!kUnchecked && FOLLY_UNLIKELY(position() == 0)) {+      return false;+    }++    size_t inner;+    block_t block;+    DCHECK_GE(outer_, 0);+    getPreviousInfo(block, inner, outer_); // Updates outer_.+    block_ = loadUnaligned<block_t>(start_ + outer_);+    block_ ^= block;+    --position_;+    return setValue(inner);+  }++  FOLLY_ALWAYS_INLINE bool next() {+    if (!kUnchecked && FOLLY_UNLIKELY(addT(position(), 1) >= size())) {+      return setDone();+    }++    // Skip to the first non-zero block.+    while (FOLLY_UNLIKELY(block_ == 0)) {+      outer_ += sizeof(block_t);+      block_ = loadUnaligned<block_t>(start_ + outer_);+    }++    ++position_;+    size_t inner = Instructions::ctz(block_);+    block_ = Instructions::blsr(block_);++    return setValue(inner);+  }++  FOLLY_ALWAYS_INLINE bool skip(SizeType n) {+    DCHECK_GT(n, 0);+    if (!kUnchecked && FOLLY_UNLIKELY(addT(position_, n) >= size())) {+      return setDone();+    }++    position_ += n; // n 1-bits will be read.++    // Use forward pointer.+    if constexpr (Encoder::forwardQuantum > 0) {+      if (FOLLY_UNLIKELY(n > Encoder::forwardQuantum)) {+        const size_t steps = position_ / Encoder::forwardQuantum;+        const size_t dest = loadUnaligned<SkipValueType>(+            this->forwardPointers_ + (steps - 1) * sizeof(SkipValueType));++        reposition(dest + steps * Encoder::forwardQuantum);+        n = position_ + 1 - steps * Encoder::forwardQuantum; // n is > 0.+      }+    }++    size_t cnt;+    // Find necessary block.+    while ((cnt = Instructions::popcount(block_)) < n) {+      n -= cnt;+      outer_ += sizeof(block_t);+      block_ = loadUnaligned<block_t>(start_ + outer_);+    }++    // Skip to the n-th one in the block.+    DCHECK_GT(n, 0);+    size_t inner = select64<Instructions>(block_, n - 1);+    block_ &= (block_t(-1) << inner) << 1;++    return setValue(inner);+  }++  // Skip to the first element that is >= v and located *after* the current+  // one (so even if current value equals v, position will be increased by 1).+  FOLLY_ALWAYS_INLINE bool skipToNext(ValueType v) {+    DCHECK_GE(v, value_);+    if (!kUnchecked && FOLLY_UNLIKELY(v > upperBound_)) {+      return setDone();+    }++    // Use skip pointer.+    if constexpr (Encoder::skipQuantum > 0) {+      // NOTE: The addition can overflow here, but that means value_ is within+      // skipQuantum_ distance from the maximum representable value, and thus+      // the last value, so the comparison is still correct.+      if (FOLLY_UNLIKELY(v >= addT(value_, Encoder::skipQuantum))) {+        const size_t steps = v / Encoder::skipQuantum;+        const size_t dest = loadUnaligned<SkipValueType>(+            this->skipPointers_ + (steps - 1) * sizeof(SkipValueType));++        DCHECK_LE(dest, size());+        if (!kUnchecked && FOLLY_UNLIKELY(dest == size())) {+          return setDone();+        }++        reposition(dest + Encoder::skipQuantum * steps);+        position_ = dest - 1;++        // Correct value_ will be set during the next() call at the end.++        // NOTE: Corresponding block of lower bits sequence may be+        // prefetched here (via __builtin_prefetch), but experiments+        // didn't show any significant improvements.+      }+    }++    // Skip by blocks.+    size_t cnt;+    // outer_ and position_ rely on negative sentinel values. We enforce the+    // overflown bits are dropped by explicitly casting the final value to+    // SizeType first, followed by a potential implicit cast to size_t.+    size_t skip = static_cast<SizeType>(v - (8 * outer_ - position_ - 1));++    constexpr size_t kBitsPerBlock = 8 * sizeof(block_t);+    while ((cnt = Instructions::popcount(~block_)) < skip) {+      skip -= cnt;+      position_ += kBitsPerBlock - cnt;+      outer_ += sizeof(block_t);+      DCHECK_LT(outer_, (static_cast<size_t>(upperBound_) + size() + 7) / 8);+      block_ = loadUnaligned<block_t>(start_ + outer_);+    }++    if (FOLLY_LIKELY(skip)) {+      auto inner = select64<Instructions>(~block_, skip - 1);+      position_ += inner - skip + 1;+      block_ &= block_t(-1) << inner;+    }++    DCHECK_LT(addT(position(), 1), addT(size(), 1));+    return next();+  }++  /**+   * Try to prepare to skip to value. This is a constant-time operation that+   * will attempt to prefetch memory required for a subsequent skipTo(value)+   * call if the value to skip to is within this list.+   *+   * Returns:+   *   {true, position of the reader} if the skip is valid,+   *   {false, size()} otherwise.+   */+  FOLLY_ALWAYS_INLINE std::pair<bool, SizeType> prepareSkipTo(+      ValueType v) const {+    if (!kUnchecked && FOLLY_UNLIKELY(v > upperBound_)) {+      return std::make_pair(false, size());+    }+    auto position = position_;++    if constexpr (Encoder::skipQuantum > 0) {+      if (v >= addT(value_, Encoder::skipQuantum)) {+        auto outer = outer_;+        const size_t steps = v / Encoder::skipQuantum;+        const size_t dest = loadUnaligned<SkipValueType>(+            this->skipPointers_ + (steps - 1) * sizeof(SkipValueType));++        DCHECK_LE(dest, size());+        if (!kUnchecked && FOLLY_UNLIKELY(dest == size())) {+          return std::make_pair(false, size());+        }++        position = dest - 1;+        outer = (dest + Encoder::skipQuantum * steps) / 8;++        // Prefetch up to the beginning of where we linear search. After that,+        // hardware prefetching will outperform our own. In addition, this+        // simplifies calculating what to prefetch as we don't have to calculate+        // the entire destination address. Two cache lines are prefetched+        // because this results in fewer cycles used (based on practical+        // results) than one. However, three cache lines does not have any+        // additional effect.+        const auto addr = start_ + outer;+        __builtin_prefetch(addr);+        __builtin_prefetch(addr + kCacheLineSize);+      }+    }++    return std::make_pair(true, position);+  }++  FOLLY_ALWAYS_INLINE ValueType previousValue() const {+    block_t block;+    size_t inner;+    OuterType outer;+    getPreviousInfo(block, inner, outer);+    return static_cast<ValueType>(8 * outer + inner - (position_ - 1));+  }++  // Returns true if we're at the beginning of the list, or previousValue() !=+  // value().+  FOLLY_ALWAYS_INLINE bool isAtBeginningOfRun() const {+    DCHECK_NE(position(), kBeforeFirstPos);+    if (position_ == 0) {+      return true;+    }+    size_t bitPos = size_t(value_) + position_ - 1;+    return (start_[bitPos / 8] & (1 << (bitPos % 8))) == 0;+  }++ private:+  using block_t = uint64_t;+  // The size in bytes of the upper bits is limited by n + universe / 8,+  // so a type that can hold either sizes or values is sufficient.+  using OuterType = typename std::common_type_t<ValueType, SizeType>;++  static ValueType estimateUpperBound(+      const typename Encoder::CompressedList& list) {+    size_t upperBound = 8 * list.upperSizeBytes - list.size;+    // The bitvector is byte-aligned, so we may be overestimating the universe+    // size. Make sure it fits in ValueType.+    return static_cast<ValueType>(std::min<size_t>(+        upperBound,+        std::numeric_limits<ValueType>::max() >> list.numLowerBits));+  }++  FOLLY_ALWAYS_INLINE bool setValue(size_t inner) {+    value_ = static_cast<ValueType>(8 * outer_ + inner - position_);+    return true;+  }++  FOLLY_ALWAYS_INLINE bool setDone() {+    position_ = size_;+    return false;+  }++  // NOTE: dest is a position in the bit vector, use size_t as SizeType may+  // not be sufficient here.+  FOLLY_ALWAYS_INLINE void reposition(size_t dest) {+    outer_ = dest / 8;+    DCHECK_LT(outer_, (static_cast<size_t>(upperBound_) + size() + 7) / 8);+    block_ = loadUnaligned<block_t>(start_ + outer_);+    block_ &= ~((block_t(1) << (dest % 8)) - 1);+  }++  FOLLY_ALWAYS_INLINE void getPreviousInfo(+      block_t& block, size_t& inner, OuterType& outer) const {+    DCHECK_GT(position(), 0);+    DCHECK_LT(position(), size());++    outer = outer_;+    block = loadUnaligned<block_t>(start_ + outer);+    inner = size_t(value_) - 8 * outer_ + position_;+    block &= (block_t(1) << inner) - 1;+    while (FOLLY_UNLIKELY(block == 0)) {+      DCHECK_GT(outer, 0);+      outer -= std::min<OuterType>(sizeof(block_t), outer);+      block = loadUnaligned<block_t>(start_ + outer);+    }+    inner = 8 * sizeof(block_t) - 1 - Instructions::clz(block);+  }++  const unsigned char* const start_;+  const SizeType size_; // Size of the list.+  const ValueType upperBound_; // Upper bound of values in this list.+  block_t block_;+  SizeType position_; // Index of current value (= #reads - 1).+  OuterType outer_; // Outer offset: number of consumed bytes in upper.+  ValueType value_;+};++} // namespace detail++// If kUnchecked = true the caller must guarantee that all the operations return+// valid elements, i.e., they would never return false if checked.+//+// If the list length is known to be representable with a type narrower than the+// SkipValueType used in the format, the reader footprint can be reduced by+// passing the type as SizeType.+template <+    class Encoder,+    class Instructions = instructions::Default,+    bool kUnchecked = false,+    class SizeT = typename Encoder::SkipValueType>+class EliasFanoReader {+  using UpperBitsReader =+      detail::UpperBitsReader<Encoder, Instructions, SizeT, kUnchecked>;++ public:+  using EncoderType = Encoder;+  using ValueType = typename Encoder::ValueType;+  using SizeType = SizeT;++  explicit EliasFanoReader(const typename Encoder::CompressedList& list)+      : upper_(list),+        lower_(list.lower),+        value_(),+        numLowerBits_(list.numLowerBits) {+    DCHECK_LE(list.size, std::numeric_limits<SizeType>::max());+    DCHECK(Instructions::supported());+  }++  void reset() { upper_.reset(); }++  bool previous() {+    if (FOLLY_LIKELY(upper_.previous())) {+      return setValue(readCurrentValue());+    }+    reset();+    return false;+  }++  bool next() {+    if (FOLLY_LIKELY(upper_.next())) {+      return setValue(readCurrentValue());+    }+    return false;+  }++  /**+   * Advances by n elements. n = 0 is allowed and has no effect. Returns false+   * if the end of the list is reached. position() + n must be representable by+   * SizeType.+   */+  bool skip(SizeType n) {+    if (n == 0) {+      return valid();+    }+    if (!upper_.skip(n)) {+      return false;+    }+    return setValue(readCurrentValue());+  }++  /**+   * Skips to the first element >= value whose position is greater or equal to+   * the current position.+   * Requires that value >= value() (or that the reader is positioned before the+   * first element). Returns false if no such element exists.+   * If kCanBeAtValue is false, the requirement above becomes value > value().+   */+  template <bool kCanBeAtValue = true>+  bool skipTo(ValueType value) {+    if (valid()) {+      if constexpr (kCanBeAtValue) {+        DCHECK_GE(value, value_);+        if (FOLLY_UNLIKELY(value == value_)) {+          return true;+        }+      } else {+        DCHECK_GT(value, value_);+      }+    }++    ValueType upperValue = value >> numLowerBits_;++    if (FOLLY_UNLIKELY(!upper_.skipToNext(upperValue))) {+      return false;+    }++    do {+      if (auto cur = readCurrentValue(); FOLLY_LIKELY(cur >= value)) {+        return setValue(cur);+      }+    } while (FOLLY_LIKELY(upper_.next()));++    return false;+  }++  /**+   * Prepare to skip to `value` by prefetching appropriate memory in both the+   * upper and lower bits.+   */+  template <bool kCanBeAtValue = true>+  void prepareSkipTo(ValueType value) const {+    if (valid()) {+      if constexpr (kCanBeAtValue) {+        DCHECK_GE(value, value_);+        if (FOLLY_UNLIKELY(value == value_)) {+          return;+        }+      } else {+        DCHECK_GT(value, value_);+      }+    }++    // Do minimal computation required to prefetch address used in+    // `readLowerPart()`.+    ValueType upperValue = value >> numLowerBits_;+    const auto [valid, upperPosition] = upper_.prepareSkipTo(upperValue);+    if (!valid) {+      return;+    }+    const auto addr = lower_ + (upperPosition * numLowerBits_ / 8);+    __builtin_prefetch(addr);+    __builtin_prefetch(addr + kCacheLineSize);+  }++  /**+   * Jumps to the element at position n. The reader can be in any state. Returns+   * false if n >= size().+   */+  bool jump(SizeType n) {+    // Also works if position() == -1, since `kBeforeFirstPos + 1 == 0`.+    if (detail::addT(n, 1) < detail::addT(position(), 1)) {+      reset();+      n += 1; // Initial position is -1.+    } else {+      n -= position();+    }+    return skip(n);+  }++  /**+   * Jumps to the first element >= value. The reader can be in any+   * state. Returns false if no such element exists.+   *+   * If all the values in the list can be assumed distinct, setting+   * assumeDistinct = true can enable some optimizations.+   */+  bool jumpTo(ValueType value, bool assumeDistinct = false) {+    if (valid() && value == value_) {+      if (assumeDistinct == true) {+        return true;+      }++      // We might be in the middle of a run of equal values, reposition by+      // iterating backwards to its first element.+      auto valueLower = Instructions::bzhi(value_, numLowerBits_);+      while (!upper_.isAtBeginningOfRun() &&+             readLowerPart(position() - 1) == valueLower) {+        upper_.previous();+      }+      return true;+    }++    // We need to reset if we're not in the initial state and the jump is+    // backwards.+    if (position() != UpperBitsReader::kBeforeFirstPos &&+        (position() == size() || value < value_)) {+      reset();+    }+    return skipTo(value);+  }++  ValueType previousValue() const {+    DCHECK_GT(position(), 0);+    DCHECK_LT(position(), size());+    return readLowerPart(position() - 1) |+        (upper_.previousValue() << numLowerBits_);+  }++  SizeType size() const { return upper_.size(); }++  bool valid() const { return upper_.valid(); }++  SizeType position() const { return upper_.position(); }++  ValueType value() const {+    DCHECK(valid());+    return value_;+  }++ private:+  FOLLY_ALWAYS_INLINE bool setValue(ValueType value) {+    DCHECK(valid());+    value_ = value;+    return true;+  }++  FOLLY_ALWAYS_INLINE ValueType readLowerPart(SizeType i) const {+    DCHECK_LT(i, size());+    const size_t pos = i * numLowerBits_;+    const unsigned char* ptr = lower_ + (pos / 8);+    const uint64_t ptrv = loadUnaligned<uint64_t>(ptr);+    // This removes the branch in the fallback implementation of+    // bextr. The condition is verified at encoding time.+    assume(numLowerBits_ < sizeof(ValueType) * 8);+    assume((pos % 8) + numLowerBits_ < 64);+    return Instructions::bextr(ptrv, pos % 8, numLowerBits_);+  }++  FOLLY_ALWAYS_INLINE ValueType readCurrentValue() {+    return readLowerPart(position()) | (upper_.value() << numLowerBits_);+  }++  // Ordering of fields is counter-intutive but it optimizes the layout.+  UpperBitsReader upper_;+  const uint8_t* const lower_;+  ValueType value_;+  const uint8_t numLowerBits_;+};++} // namespace compression+} // namespace folly
+ folly/folly/concurrency/AtomicSharedPtr.h view
@@ -0,0 +1,484 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <atomic>+#include <cstdint>+#include <thread>++#include <folly/PackedSyncPtr.h>+#include <folly/concurrency/detail/AtomicSharedPtr-detail.h>+#include <folly/memory/SanitizeLeak.h>+#include <folly/synchronization/AtomicStruct.h>+#include <folly/synchronization/AtomicUtil.h>+#include <folly/synchronization/detail/AtomicUtils.h>++#if defined(__GLIBCXX__) && FOLLY_HAS_PACKED_SYNC_PTR+#define FOLLY_HAS_ATOMIC_SHARED_PTR_HOOKED 1+#else+#define FOLLY_HAS_ATOMIC_SHARED_PTR_HOOKED 0+#endif++#if FOLLY_HAS_ATOMIC_SHARED_PTR_HOOKED++/*+ * This is an implementation of the std::atomic_shared_ptr TS+ * http://en.cppreference.com/w/cpp/experimental/atomic_shared_ptr+ * https://isocpp.org/files/papers/N4162.pdf+ *+ * AFAIK, the only other implementation is Anthony Williams from+ * Just::thread library:+ *+ * https://github.com/anthonywilliams/atomic_shared_ptr+ *+ * implementation details:+ *+ * Basically, three things need to be atomically exchanged to make this work:+ * * the local count+ * * the pointer to the control block+ * * the aliased pointer, if any.+ *+ * The Williams version does it with DWcas: 32 bits for local count, 64+ * bits for control block ptr, and he changes the shared_ptr+ * implementation to also store the aliased pointers using a linked list+ * like structure, and provides 32-bit index accessors to them (like+ * IndexedMemPool trick).+ *+ * This version instead stores the 48 bits of address, plus 16 bits of+ * local count in a single 8byte pointer.  This avoids 'lock cmpxchg16b',+ * which is much slower than 'lock xchg' in the normal 'store' case.  In+ * the less-common aliased pointer scenario, we just allocate it in a new+ * block, and store a pointer to that instead.+ *+ * Note that even if we only want to use the 3-bits of pointer alignment,+ * this trick should still work - Any more than 4 concurrent accesses+ * will have to go to an external map count instead (slower, but lots of+ * concurrent access will be slow anyway due to bouncing cachelines).+ *+ * As a perf optimization, we currently batch up local count and only+ * move it global every once in a while.  This means load() is usually+ * only a single atomic operation, instead of 3.  For this trick to work,+ * we probably need at least 8 bits to make batching worth it.+ */++// A note on noexcept: If the pointer is an aliased pointer,+// store() will allocate.  Otherwise is noexcept.+namespace folly {++template <+    typename T,+    template <typename> class Atom = std::atomic,+    typename CountedDetail = detail::shared_ptr_internals>+class atomic_shared_ptr {+  using SharedPtr = typename CountedDetail::template CountedPtr<T>;+  using BasePtr = typename CountedDetail::counted_base;+  using PackedPtr = folly::PackedSyncPtr<BasePtr>;++ public:+  atomic_shared_ptr() noexcept { init(); }+  explicit atomic_shared_ptr(SharedPtr foo) /* noexcept */+      : atomic_shared_ptr() {+    store(std::move(foo));+  }+  atomic_shared_ptr(const atomic_shared_ptr<T>&) = delete;++  ~atomic_shared_ptr() { store(SharedPtr(nullptr)); }+  void operator=(SharedPtr desired) /* noexcept */ {+    store(std::move(desired));+  }+  void operator=(const atomic_shared_ptr<T>&) = delete;++  bool is_lock_free() const noexcept {+    // lock free unless more than EXTERNAL_OFFSET threads are+    // contending and they all get unlucky and scheduled out during+    // load().+    //+    // TODO: Could use a lock-free external map to fix this+    // corner case.+    return true;+  }++  SharedPtr load(+      std::memory_order order = std::memory_order_seq_cst) const noexcept {+    auto local = takeOwnedBase(order);+    return get_shared_ptr(local, false);+  }++  /* implicit */ operator SharedPtr() const { return load(); }++  void store(+      SharedPtr n,+      std::memory_order order = std::memory_order_seq_cst) /* noexcept */ {+    auto newptr = get_newptr(std::move(n));+    auto old = ptr_.exchange(newptr, order);+    release_external(old);+  }++  SharedPtr exchange(+      SharedPtr n,+      std::memory_order order = std::memory_order_seq_cst) /* noexcept */ {+    auto newptr = get_newptr(std::move(n));+    auto old = ptr_.exchange(newptr, order);++    SharedPtr old_ptr;++    if (old.get()) {+      old_ptr = get_shared_ptr(old);+      release_external(old);+    }++    return old_ptr;+  }++  bool compare_exchange_weak(+      SharedPtr& expected,+      const SharedPtr& n,+      std::memory_order mo = std::memory_order_seq_cst) noexcept {+    return compare_exchange_weak(+        expected, n, mo, detail::default_failure_memory_order(mo));+  }+  bool compare_exchange_weak(+      SharedPtr& expected,+      const SharedPtr& n,+      std::memory_order success,+      std::memory_order failure) /* noexcept */ {+    auto newptr = get_newptr(n);+    PackedPtr oldptr, expectedptr;++    oldptr = takeOwnedBase(success);+    if (!owners_eq(oldptr, CountedDetail::get_counted_base(expected))) {+      expected = get_shared_ptr(oldptr, false);+      release_external(newptr);+      return false;+    }+    expectedptr = oldptr; // Need oldptr to release if failed+    if (ptr_.compare_exchange_weak(expectedptr, newptr, success, failure)) {+      if (oldptr.get()) {+        release_external(oldptr, -1);+      }+      return true;+    } else {+      if (oldptr.get()) {+        expected = get_shared_ptr(oldptr, false);+      } else {+        expected = SharedPtr(nullptr);+      }+      release_external(newptr);+      return false;+    }+  }+  bool compare_exchange_weak(+      SharedPtr& expected,+      SharedPtr&& desired,+      std::memory_order mo = std::memory_order_seq_cst) noexcept {+    return compare_exchange_weak(+        expected, desired, mo, detail::default_failure_memory_order(mo));+  }+  bool compare_exchange_weak(+      SharedPtr& expected,+      SharedPtr&& desired,+      std::memory_order success,+      std::memory_order failure) /* noexcept */ {+    return compare_exchange_weak(expected, desired, success, failure);+  }+  bool compare_exchange_strong(+      SharedPtr& expected,+      const SharedPtr& n,+      std::memory_order mo = std::memory_order_seq_cst) noexcept {+    return compare_exchange_strong(+        expected, n, mo, detail::default_failure_memory_order(mo));+  }+  bool compare_exchange_strong(+      SharedPtr& expected,+      const SharedPtr& n,+      std::memory_order success,+      std::memory_order failure) /* noexcept */ {+    auto local_expected = expected;+    do {+      if (compare_exchange_weak(expected, n, success, failure)) {+        return true;+      }+    } while (local_expected == expected);++    return false;+  }+  bool compare_exchange_strong(+      SharedPtr& expected,+      SharedPtr&& desired,+      std::memory_order mo = std::memory_order_seq_cst) noexcept {+    return compare_exchange_strong(+        expected, desired, mo, detail::default_failure_memory_order(mo));+  }+  bool compare_exchange_strong(+      SharedPtr& expected,+      SharedPtr&& desired,+      std::memory_order success,+      std::memory_order failure) /* noexcept */ {+    return compare_exchange_strong(expected, desired, success, failure);+  }++ private:+  // Matches packed_sync_pointer.  Must be > max number of local+  // counts.  This is the max number of threads that can access this+  // atomic_shared_ptr at once before we start blocking.+  static constexpr std::uint16_t EXTERNAL_OFFSET{0x2000};+  // Bit signifying aliased constructor+  static constexpr std::uint16_t ALIASED_PTR{0x4000};++  static std::uint16_t get_local_count(const PackedPtr& p) {+    return p.extra() & ~ALIASED_PTR;+  }++  static void add_external(BasePtr* res, int64_t c = 0) {+    assert(res);+    CountedDetail::inc_shared_count(res, EXTERNAL_OFFSET + c);+    annotate_object_leaked(res);+  }+  static void release_external(PackedPtr& res, int64_t c = 0) {+    if (!res.get()) {+      return;+    }+    annotate_object_collected(res.get());+    int64_t count = get_local_count(res) + c;+    int64_t diff = EXTERNAL_OFFSET - count;+    assert(diff >= 0);+    CountedDetail::template release_shared<T>(res.get(), diff);+  }++  static PackedPtr get_newptr(const SharedPtr& n) {+    return get_newptr_impl<false>(n);+  }+  static PackedPtr get_newptr(SharedPtr&& n) {+    return get_newptr_impl<true>(std::move(n));+  }+  template <bool kOwn, class S>+  static PackedPtr get_newptr_impl(S&& n) {+    std::uint16_t count = 0;+    BasePtr* newval = CountedDetail::get_counted_base(n);+    if (!n && newval == nullptr) {+      // n is default-constructed, nothing to do.+    } else if (+        newval == nullptr ||+        n.get() != CountedDetail::template get_shared_ptr<T>(newval)) {+      // This is an aliased sharedptr.  Make an un-aliased one by wrapping in+      // *another* shared_ptr.+      auto data =+          CountedDetail::template make_ptr<SharedPtr>(std::forward<S>(n));+      newval = CountedDetail::get_counted_base(data);+      count = ALIASED_PTR;+      CountedDetail::release_ptr(data);+      add_external(newval, -1);+    } else {+      if constexpr (kOwn) {+        CountedDetail::release_ptr(n);+      }+      add_external(newval, kOwn ? -1 : 0);+    }++    PackedPtr newptr;+    newptr.init(newval, count);++    return newptr;+  }++  void init() {+    PackedPtr data;+    data.init();+    ptr_.store(data);+  }++  // Check pointer equality considering wrapped aliased pointers.+  bool owners_eq(PackedPtr& p1, BasePtr* p2) {+    bool aliased1 = p1.extra() & ALIASED_PTR;+    if (aliased1) {+      auto p1a = CountedDetail::template get_shared_ptr_from_counted_base<T>(+          p1.get(), false);+      return CountedDetail::get_counted_base(p1a) == p2;+    }+    return p1.get() == p2;+  }++  SharedPtr get_shared_ptr(const PackedPtr& p, bool inc = true) const {+    bool aliased = p.extra() & ALIASED_PTR;++    auto res = CountedDetail::template get_shared_ptr_from_counted_base<T>(+        p.get(), inc);+    if (aliased) {+      auto aliasedp =+          CountedDetail::template get_shared_ptr_from_counted_base<SharedPtr>(+              p.get());+      res = *aliasedp;+    }+    return res;+  }++  /* Get a reference to the pointer, either from the local batch or+   * from the global count.+   *+   * return is the base ptr, and the previous local count, if it is+   * needed for compare_and_swap later.+   */+  PackedPtr takeOwnedBase(std::memory_order order) const noexcept {+    PackedPtr local, newlocal;+    local = ptr_.load(std::memory_order_acquire);+    while (true) {+      if (!local.get()) {+        return local;+      }+      newlocal = local;+      if (get_local_count(newlocal) + 1 > EXTERNAL_OFFSET) {+        // spinlock in the rare case we have more than+        // EXTERNAL_OFFSET threads trying to access at once.+        std::this_thread::yield();+        // Force DeterministicSchedule to choose a different thread+        local = ptr_.load(std::memory_order_acquire);+      } else {+        newlocal.setExtra(newlocal.extra() + 1);+        assert(get_local_count(newlocal) > 0);+        if (ptr_.compare_exchange_weak(local, newlocal, order)) {+          break;+        }+      }+    }++    // Check if we need to push a batch from local -> global+    std::uint16_t batchcount = EXTERNAL_OFFSET / 2;+    if (get_local_count(newlocal) > batchcount) {+      CountedDetail::inc_shared_count(newlocal.get(), batchcount);+      putOwnedBase(newlocal.get(), batchcount, order);+    }++    return newlocal;+  }++  void putOwnedBase(+      BasePtr* p, std::uint16_t count, std::memory_order mo) const noexcept {+    PackedPtr local = ptr_.load(std::memory_order_acquire);+    while (true) {+      if (local.get() != p) {+        break;+      }+      auto newlocal = local;+      if (get_local_count(local) > count) {+        newlocal.setExtra(local.extra() - count);+      } else {+        // Otherwise it may be the same pointer, but someone else won+        // the compare_exchange below, local count was already made+        // global.  We decrement the global count directly instead of+        // the local one.+        break;+      }+      if (ptr_.compare_exchange_weak(local, newlocal, mo)) {+        return;+      }+    }++    CountedDetail::template release_shared<T>(p, count);+  }++  mutable AtomicStruct<PackedPtr, Atom> ptr_;+};++} // namespace folly++#else++namespace folly {++template <typename T>+class atomic_shared_ptr {+ private:+  std::shared_ptr<T> rep_;++ public:+  using value_type = std::shared_ptr<T>;++  atomic_shared_ptr() = default;+  atomic_shared_ptr(std::nullptr_t) noexcept {}+  atomic_shared_ptr(std::shared_ptr<T> desired) noexcept+      : rep_{std::move(desired)} {}++  atomic_shared_ptr(atomic_shared_ptr const&) = delete;+  atomic_shared_ptr(atomic_shared_ptr&&) = delete;++  void operator=(std::nullptr_t) noexcept { store(nullptr); }+  void operator=(std::shared_ptr<T> desired) noexcept {+    store(std::move(desired));+  }++  void operator=(atomic_shared_ptr const&) = delete;+  void operator=(atomic_shared_ptr&&) = delete;++  /* implicit */ operator std::shared_ptr<T>() const noexcept { return load(); }++  bool is_lock_free() const noexcept { return atomic_is_lock_free(&rep_); }++  std::shared_ptr<T> load(+      std::memory_order order = std::memory_order_seq_cst) const noexcept {+    return atomic_load_explicit(&rep_, order);+  }++  void store(+      std::shared_ptr<T> desired,+      std::memory_order order = std::memory_order_seq_cst) noexcept {+    atomic_store_explicit(&rep_, std::move(desired), order);+  }++  std::shared_ptr<T> exchange(+      std::shared_ptr<T> desired,+      std::memory_order order = std::memory_order_seq_cst) noexcept {+    return atomic_exchange_explicit(&rep_, std::move(desired), order);+  }++  bool compare_exchange_weak(+      std::shared_ptr<T>& expected,+      std::shared_ptr<T> desired,+      std::memory_order success,+      std::memory_order failure) noexcept {+    return atomic_compare_exchange_weak_explicit(+        &rep_, &expected, std::move(desired), success, failure);+  }++  bool compare_exchange_weak(+      std::shared_ptr<T>& expected,+      std::shared_ptr<T> desired,+      std::memory_order order = std::memory_order_seq_cst) noexcept {+    return atomic_compare_exchange_weak_explicit(+        &rep_, &expected, std::move(desired), order, memory_order_load(order));+  }++  bool compare_exchange_strong(+      std::shared_ptr<T>& expected,+      std::shared_ptr<T> desired,+      std::memory_order success,+      std::memory_order failure) noexcept {+    return atomic_compare_exchange_strong_explicit(+        &rep_, &expected, std::move(desired), success, failure);+  }++  bool compare_exchange_strong(+      std::shared_ptr<T>& expected,+      std::shared_ptr<T> desired,+      std::memory_order order = std::memory_order_seq_cst) noexcept {+    return atomic_compare_exchange_strong_explicit(+        &rep_, &expected, std::move(desired), order, memory_order_load(order));+  }+};++} // namespace folly++#endif // FOLLY_HAS_ATOMIC_SHARED_PTR_HOOKED
+ folly/folly/concurrency/CacheLocality.cpp view
@@ -0,0 +1,703 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#include <folly/concurrency/CacheLocality.h>++#ifndef _MSC_VER+#define _GNU_SOURCE 1 // for RTLD_NOLOAD+#include <dlfcn.h>+#endif+#include <fstream>+#include <mutex>+#include <numeric>+#include <optional>+#include <unordered_map>++#include <fmt/core.h>+#include <glog/logging.h>+#include <folly/Indestructible.h>+#include <folly/Memory.h>+#include <folly/ScopeGuard.h>+#include <folly/detail/StaticSingletonManager.h>+#include <folly/hash/Hash.h>+#include <folly/lang/Exception.h>+#include <folly/portability/Unistd.h>+#include <folly/system/ThreadId.h>++namespace folly {++///////////// CacheLocality++/// Returns the CacheLocality information best for this machine+static CacheLocality getSystemLocalityInfo() {+  if (kIsLinux) {+    // First try to parse /proc/cpuinfo.+    // If that fails, then try to parse /sys/devices/.+    // The latter is slower but more accurate.+    try {+      return CacheLocality::readFromProcCpuinfo();+    } catch (...) {+      // /proc/cpuinfo might be non-standard+      // lets try with sysfs /sys/devices/cpu+    }++    try {+      return CacheLocality::readFromSysfs();+    } catch (...) {+      // keep trying+    }+  }++  long numCpus = sysconf(_SC_NPROCESSORS_CONF);+  if (numCpus <= 0) {+    // This shouldn't happen, but if it does we should try to keep+    // going.  We are probably not going to be able to parse /sys on+    // this box either (although we will try), which means we are going+    // to fall back to the SequentialThreadId splitter.  On my 16 core+    // (x hyperthreading) dev box 16 stripes is enough to get pretty good+    // contention avoidance with SequentialThreadId, and there is little+    // improvement from going from 32 to 64.  This default gives us some+    // wiggle room+    numCpus = 32;+  }+  return CacheLocality::uniform(size_t(numCpus));+}++template <>+const CacheLocality& CacheLocality::system<std::atomic>() {+  static std::atomic<const CacheLocality*> cache;+  auto value = cache.load(std::memory_order_acquire);+  if (value != nullptr) {+    return *value;+  }+  auto next = new CacheLocality(getSystemLocalityInfo());+  if (cache.compare_exchange_strong(value, next, std::memory_order_acq_rel)) {+    return *next;+  }+  delete next;+  return *value;+}++CacheLocality::CacheLocality(std::vector<std::vector<size_t>> equivClasses) {+  numCpus = equivClasses.size();++  for (size_t cpu = 0; cpu < numCpus; ++cpu) {+    for (size_t level = 0; level < equivClasses[cpu].size(); ++level) {+      if (equivClasses[cpu][level] == cpu) {+        // we only want to count the equiv classes once, so we do it when we+        // are processing their representative.+        while (numCachesByLevel.size() <= level) {+          numCachesByLevel.push_back(0);+        }+        numCachesByLevel[level]++;+      }+    }+  }++  std::vector<size_t> cpus(numCpus);+  std::iota(cpus.begin(), cpus.end(), 0);++  std::sort(cpus.begin(), cpus.end(), [&](size_t lhs, size_t rhs) -> bool {+    auto& lhsEquiv = equivClasses[lhs];+    auto& rhsEquiv = equivClasses[rhs];++    // If different cpus have different numbers of caches group first by number+    // of caches to guarantee strict weak ordering, even though the resulting+    // order may be sub-optimal.+    if (lhsEquiv.size() != rhsEquiv.size()) {+      return lhsEquiv.size() < rhsEquiv.size();+    }++    // Order by equiv class of cache with highest index, direction doesn't+    // matter.+    for (size_t i = lhsEquiv.size(); i > 0; --i) {+      auto idx = i - 1;+      if (lhsEquiv[idx] != rhsEquiv[idx]) {+        return lhsEquiv[idx] < rhsEquiv[idx];+      }+    }++    // Break ties deterministically by cpu.+    return lhs < rhs;+  });++  // The cpus are now sorted by locality, with neighboring entries closer+  // to each other than entries that are far away.  For striping we want+  // the inverse map, since we are starting with the cpu.+  localityIndexByCpu.resize(numCpus);+  for (size_t i = 0; i < cpus.size(); ++i) {+    localityIndexByCpu[cpus[i]] = i;+  }++  equivClassesByCpu = std::move(equivClasses);+}++// Each level of cache has sharing sets, which are the set of cpus that share a+// common cache at that level.  These are available in a hex bitset form+// (/sys/devices/system/cpu/cpu0/cache/index0/shared_cpu_map, for example).+// They are also available in human-readable form in the shared_cpu_list file in+// the same directory.  The list is a comma-separated list of numbers and+// ranges, where the ranges are pairs of decimal numbers separated by a '-'.+//+// To sort the cpus for optimum locality we don't really need to parse the+// sharing sets, we just need a unique representative from the equivalence+// class.  The smallest value works fine, and happens to be the first decimal+// number in the file.  We load all of the equivalence class information from+// all of the cpu*/index* directories, order the cpus first by increasing+// last-level cache equivalence class, then by the smaller caches.  Finally, we+// break ties with the cpu number itself.++/// Returns the first decimal number in the line, or throws an exception if the+/// line does not start with a number terminated by ',', '-', '\n', or EOS.+static size_t parseLeadingNumber(const std::string& line) {+  auto raw = line.c_str();+  char* end;+  unsigned long val = strtoul(raw, &end, 10);+  if (end == raw || (*end != ',' && *end != '-' && *end != '\n' && *end != 0)) {+    throw std::runtime_error(fmt::format("error parsing list '{}'", line));+  }+  return val;+}++CacheLocality CacheLocality::readFromSysfsTree(+    const std::function<std::string(std::string const&)>& mapping) {+  // the list of cache equivalence classes, where equivalence classes+  // are named by the smallest cpu in the class+  std::vector<std::vector<size_t>> equivClassesByCpu;++  for (size_t cpu = 0;; ++cpu) {+    std::vector<size_t> levels;+    for (size_t index = 0;; ++index) {+      auto dir = fmt::format(+          "/sys/devices/system/cpu/cpu{}/cache/index{}/", cpu, index);+      auto cacheType = mapping(dir + "type");+      auto equivStr = mapping(dir + "shared_cpu_list");+      if (cacheType.empty() || equivStr.empty()) {+        // no more caches+        break;+      }+      if (cacheType[0] == 'I') {+        // cacheType in { "Data", "Instruction", "Unified" }. skip icache+        continue;+      }+      auto equiv = parseLeadingNumber(equivStr);+      levels.push_back(equiv);+    }++    if (levels.empty()) {+      // no levels at all for this cpu, we must be done+      break;+    }+    equivClassesByCpu.emplace_back(std::move(levels));+  }++  if (equivClassesByCpu.empty()) {+    throw std::runtime_error("unable to load cache sharing info");+  }++  return CacheLocality{std::move(equivClassesByCpu)};+}++CacheLocality CacheLocality::readFromSysfs() {+  return readFromSysfsTree([](std::string const& name) {+    std::ifstream xi(name.c_str());+    std::string rv;+    std::getline(xi, rv);+    return rv;+  });+}++namespace {++static bool procCpuinfoLineRelevant(std::string const& line) {+  return line.size() > 4 && (line[0] == 'p' || line[0] == 'c');+}++std::vector<std::tuple<size_t, size_t, size_t>> parseProcCpuinfoLines(+    std::vector<std::string> const& lines) {+  std::vector<std::tuple<size_t, size_t, size_t>> cpus;+  size_t physicalId = 0;+  size_t coreId = 0;+  size_t maxCpu = 0;+  size_t numberOfPhysicalIds = 0;+  size_t numberOfCoreIds = 0;+  for (auto iter = lines.rbegin(); iter != lines.rend(); ++iter) {+    auto& line = *iter;+    if (!procCpuinfoLineRelevant(line)) {+      continue;+    }++    auto sepIndex = line.find(':');+    if (sepIndex == std::string::npos || sepIndex + 2 > line.size()) {+      continue;+    }+    auto arg = line.substr(sepIndex + 2);++    // "physical id" is socket, which is the most important locality+    // context.  "core id" is a real core, so two "processor" entries with+    // the same physical id and core id are hyperthreads of each other.+    // "processor" is the top line of each record, so when we hit it in+    // the reverse order then we can emit a record.+    if (line.find("physical id") == 0) {+      physicalId = parseLeadingNumber(arg);+      ++numberOfPhysicalIds;+    } else if (line.find("core id") == 0) {+      coreId = parseLeadingNumber(arg);+      ++numberOfCoreIds;+    } else if (line.find("processor") == 0) {+      auto cpu = parseLeadingNumber(arg);+      maxCpu = std::max(cpu, maxCpu);+      cpus.emplace_back(physicalId, coreId, cpu);+    }+  }++  if (cpus.empty()) {+    throw std::runtime_error("no CPUs parsed from /proc/cpuinfo");+  }+  if (maxCpu != cpus.size() - 1) {+    throw std::runtime_error(+        "offline CPUs not supported for /proc/cpuinfo cache locality source");+  }+  if (numberOfPhysicalIds == 0) {+    throw std::runtime_error("no physical ids found");+  }+  if (numberOfCoreIds == 0) {+    throw std::runtime_error("no core ids found");+  }++  return cpus;+}++} // namespace++CacheLocality CacheLocality::readFromProcCpuinfoLines(+    std::vector<std::string> const& lines) {+  // (physicalId, coreId, cpu)+  std::vector<std::tuple<size_t, size_t, size_t>> cpus =+      parseProcCpuinfoLines(lines);+  // Sort to make equivalence classes contiguous.+  std::sort(cpus.begin(), cpus.end());++  // We can't tell the real cache hierarchy from /proc/cpuinfo, but it works+  // well enough to assume there are 3 levels, L1 and L2 per-core and L3 per+  // socket.+  std::vector<std::vector<size_t>> equivClassesByCpu(cpus.size());+  size_t l1Equiv = 0;+  size_t l3Equiv = 0;+  for (size_t i = 0; i < cpus.size(); ++i) {+    auto [physicalId, coreId, cpu] = cpus[i];+    // The representative for each L1 and L3 equivalence class is the first cpu+    // in the class.+    if (i == 0 || physicalId != std::get<0>(cpus[i - 1]) ||+        coreId != std::get<1>(cpus[i - 1])) {+      l1Equiv = cpu;+    }+    if (i == 0 || physicalId != std::get<0>(cpus[i - 1])) {+      l3Equiv = cpu;+    }+    equivClassesByCpu[cpu] = {l1Equiv, l1Equiv, l3Equiv};+  }++  return CacheLocality{std::move(equivClassesByCpu)};+}++CacheLocality CacheLocality::readFromProcCpuinfo() {+  std::vector<std::string> lines;+  {+    std::ifstream xi("/proc/cpuinfo");+    if (xi.fail()) {+      throw std::runtime_error("unable to open /proc/cpuinfo");+    }+    char buf[8192];+    while (xi.good() && lines.size() < 20000) {+      xi.getline(buf, sizeof(buf));+      std::string str(buf);+      if (procCpuinfoLineRelevant(str)) {+        lines.emplace_back(std::move(str));+      }+    }+  }+  return readFromProcCpuinfoLines(lines);+}++CacheLocality CacheLocality::uniform(size_t numCpus) {+  // One cache shared by all cpus.+  std::vector<std::vector<size_t>> equivClassesByCpu(numCpus, {0});+  return CacheLocality{std::move(equivClassesByCpu)};+}++////////////// Getcpu++Getcpu::Func Getcpu::resolveVdsoFunc() {+#if !defined(FOLLY_HAVE_LINUX_VDSO) || defined(FOLLY_SANITIZE_MEMORY)+  return nullptr;+#else+  void* h = dlopen("linux-vdso.so.1", RTLD_LAZY | RTLD_LOCAL | RTLD_NOLOAD);+  if (h == nullptr) {+    return nullptr;+  }++  auto func = Getcpu::Func(dlsym(h, "__vdso_getcpu"));+  if (func == nullptr) {+    // technically a null result could either be a failure or a successful+    // lookup of a symbol with the null value, but the second can't actually+    // happen for this symbol.  No point holding the handle forever if+    // we don't need the code+    dlclose(h);+  }++  return func;+#endif+}++/////////////// SequentialThreadId+unsigned SequentialThreadId::get() {+  static std::atomic<unsigned> global{0};+  static thread_local unsigned local{0};+  return FOLLY_LIKELY(local) ? local : (local = ++global);+}++/////////////// HashingThreadId+unsigned HashingThreadId::get() {+  return hash::twang_32from64(getCurrentThreadID());+}++namespace detail {++int AccessSpreaderBase::degenerateGetcpu(unsigned* cpu, unsigned* node, void*) {+  if (cpu != nullptr) {+    *cpu = 0;+  }+  if (node != nullptr) {+    *node = 0;+  }+  return 0;+}++struct AccessSpreaderStaticInit {+  static AccessSpreaderStaticInit instance;+  AccessSpreaderStaticInit() { (void)AccessSpreader<>::current(~size_t(0)); }+};+AccessSpreaderStaticInit AccessSpreaderStaticInit::instance;++bool AccessSpreaderBase::initialize(+    GlobalState& state,+    Getcpu::Func (&pickGetcpuFunc)(),+    const CacheLocality& (&system)()) {+  (void)AccessSpreaderStaticInit::instance; // ODR-use it so it is not dropped+  constexpr auto relaxed = std::memory_order_relaxed;+  auto& cacheLocality = system();+  auto n = cacheLocality.numCpus;+  for (size_t width = 0; width <= kMaxCpus; ++width) {+    auto& row = state.table[width];+    auto numStripes = std::max(size_t{1}, width);+    for (size_t cpu = 0; cpu < kMaxCpus && cpu < n; ++cpu) {+      auto index = cacheLocality.localityIndexByCpu[cpu];+      assert(index < n);+      // as index goes from 0..n, post-transform value goes from+      // 0..numStripes+      make_atomic_ref(row[cpu]).store(+          static_cast<CompactStripe>((index * numStripes) / n), relaxed);+      assert(make_atomic_ref(row[cpu]).load(relaxed) < numStripes);+    }+    size_t filled = n;+    while (filled < kMaxCpus) {+      size_t len = std::min(filled, kMaxCpus - filled);+      for (size_t i = 0; i < len; ++i) {+        make_atomic_ref(row[filled + i])+            .store(make_atomic_ref(row[i]).load(relaxed), relaxed);+      }+      filled += len;+    }+    for (size_t cpu = n; cpu < kMaxCpus; ++cpu) {+      assert(+          make_atomic_ref(row[cpu]).load(relaxed) ==+          make_atomic_ref(row[cpu - n]).load(relaxed));+    }+  }+  state.getcpu.exchange(pickGetcpuFunc(), std::memory_order_acq_rel);+  return true;+}++} // namespace detail++/* static */ LLCAccessSpreader& LLCAccessSpreader::get() {+  static folly::Indestructible<LLCAccessSpreader> instance(PrivateTag{});+  return *instance;+}++LLCAccessSpreader::LLCAccessSpreader(PrivateTag) {+#ifdef __linux__+  auto getcpu = Getcpu::resolveVdsoFunc();+  getcpu_ = getcpu ? getcpu : &FallbackGetcpuType::getcpu;++  // /proc/cpuinfo does not have accurate LLC info, we need to force a read+  // from sysfs.+  auto cl = CacheLocality::readFromSysfs();+  stripeByCpu_.resize(cl.numCpus);++  // Only have stripes for LLCs that are accessible from the current process,+  // and number them sequentially as we discover them.+  std::unordered_map<size_t, size_t> cacheIdx;+  cpu_set_t cpuset;+  PCHECK(sched_getaffinity(0, sizeof(cpuset), &cpuset) == 0);+  for (size_t cpu = 0; cpu < cl.numCpus; ++cpu) {+    if (!CPU_ISSET(cpu, &cpuset)) {+      continue;+    }++    auto llcEquiv = cl.equivClassesByCpu[cpu].back();+    auto [it, _] = cacheIdx.try_emplace(llcEquiv, cacheIdx.size());+    stripeByCpu_[cpu] = it->second;+  }++  numStripes_ = cacheIdx.size();+#else+  numStripes_ = 1;+  (void)getcpu_;+  (void)stripeByCpu_;+#endif // __linux__+}++size_t LLCAccessSpreader::current() const {+#ifdef __linux__+  struct ThreadCache {+    size_t usesLeft = 0;+    size_t value;+  };++  // We expect that a thread doesn't migrate LLC often, so reuse the value a+  // few times before refreshing it.+  // TODO(ott): Re-evaluate once we can use rseq to get the current CPU.+  thread_local ThreadCache tc;+  if (tc.usesLeft-- == 0) {+    tc.usesLeft = 16; // A small number is enough to amortize.+    unsigned cpu;+    getcpu_(&cpu, nullptr, nullptr);+    // If the set of active CPUs can change at runtime just return the 0+    // stripe. This should not happen in normal operations.+    tc.value = cpu < stripeByCpu_.size() ? stripeByCpu_[cpu] : 0;+  }++  return tc.value;+#else+  return 0;+#endif // __linux__+}++size_t LLCAccessSpreader::numStripes() const {+  return numStripes_;+}++namespace {++/**+ * A simple freelist allocator.  Allocates things of size sz, from slabs of size+ * kAllocSize.  Takes a lock on each allocation/deallocation.+ */+class SimpleAllocator {+ public:+  // To support array aggregate initialization without an implicit constructor.+  struct Ctor {};++  SimpleAllocator(Ctor, size_t sz) : sz_(sz) {}+  ~SimpleAllocator() {+    std::lock_guard g(m_);+    for (auto& block : blocks_) {+      folly::aligned_free(block);+    }+  }++  void* allocate() {+    std::lock_guard g(m_);+    // Freelist allocation.+    if (freelist_) {+      auto mem = freelist_;+      freelist_ = *static_cast<void**>(freelist_);+      return mem;+    }++    if (mem_) {+      // Bump-ptr allocation.+      if (intptr_t(mem_) % kMallocAlign == 0) {+        // Avoid allocating pointers that may look like malloc+        // pointers.+        mem_ += std::min(sz_, max_align_v);+      }+      if (mem_ + sz_ <= end_) {+        auto mem = mem_;+        mem_ += sz_;++        assert(intptr_t(mem) % kMallocAlign != 0);+        return mem;+      }+    }++    return allocateHard();+  }++  static void deallocate(void* ptr) {+    assert(intptr_t(ptr) % kMallocAlign != 0);+    // Find the allocator instance.+    auto addr =+        reinterpret_cast<void*>(intptr_t(ptr) & ~intptr_t(kAllocSize - 1));+    auto allocator = *static_cast<SimpleAllocator**>(addr);++    std::lock_guard g(allocator->m_);+    *static_cast<void**>(ptr) = allocator->freelist_;+    if constexpr (kIsSanitizeAddress) {+      // If running under ASAN, scrub the memory on deallocation, so we don't+      // leave pointers that could hide leaks at shutdown, since the backing+      // slabs may not be deallocated if the instance is a leaky singleton.+      auto* base = static_cast<char*>(ptr);+      std::fill(+          base + sizeof(void*), base + allocator->sz_, static_cast<char>(0));+    }+    allocator->freelist_ = ptr;+  }++  constexpr static size_t kMallocAlign =+      std::max(size_t(128), hardware_destructive_interference_size);+  static_assert(+      kMallocAlign % hardware_destructive_interference_size == 0,+      "Large allocations should be cacheline-aligned");++ private:+  constexpr static size_t kAllocSize = 4096;++  void* allocateHard() {+    // Allocate a new slab.+    mem_ = static_cast<uint8_t*>(folly::aligned_malloc(kAllocSize, kAllocSize));+    if (!mem_) {+      throw_exception<std::bad_alloc>();+    }+    end_ = mem_ + kAllocSize;+    blocks_.push_back(mem_);++    // Install a pointer to ourselves as the allocator.+    *reinterpret_cast<SimpleAllocator**>(mem_) = this;+    static_assert(+        max_align_v >= sizeof(SimpleAllocator*), "alignment too small");+    mem_ += std::min(sz_, max_align_v);++    // New allocation.+    auto mem = mem_;+    mem_ += sz_;+    assert(intptr_t(mem) % kMallocAlign != 0);+    return mem;+  }++  std::mutex m_;+  uint8_t* mem_{nullptr};+  uint8_t* end_{nullptr};+  void* freelist_{nullptr};+  size_t sz_;+  std::vector<void*> blocks_;+};++class Allocator {+ public:+  void* allocate(size_t size) {+    if (auto cl = sizeClass(size)) {+      return allocators_[*cl].allocate();+    }++    // Fall back to malloc, returning a kMallocAlign-aligned allocation so it+    // can be distinguished from SimpleAllocator allocations.+    size = size + (SimpleAllocator::kMallocAlign - 1);+    size &= ~size_t(SimpleAllocator::kMallocAlign - 1);+    void* mem = aligned_malloc(size, SimpleAllocator::kMallocAlign);+    if (!mem) {+      throw_exception<std::bad_alloc>();+    }+    return mem;+  }++  static void deallocate(void* ptr) {+    if (!ptr) {+      return;+    }++    // See if it came from SimpleAllocator or malloc.+    if (intptr_t(ptr) % SimpleAllocator::kMallocAlign != 0) {+      SimpleAllocator::deallocate(ptr);+    } else {+      aligned_free(ptr);+    }+  }++ private:+  std::optional<uint8_t> sizeClass(size_t size) {+    if (size <= 8) {+      return 0;+    } else if (size <= 16) {+      return 1;+    } else if (size <= 32) {+      return 2;+    } else if (size <= 64) {+      return 3;+    } else {+      return std::nullopt;+    }+  }++  std::array<SimpleAllocator, 4> allocators_{+      {{SimpleAllocator::Ctor{}, 8},+       {SimpleAllocator::Ctor{}, 16},+       {SimpleAllocator::Ctor{}, 32},+       {SimpleAllocator::Ctor{}, 64}}};+};++} // namespace++void* coreMalloc(size_t size, size_t numStripes, size_t stripe) {+  static folly::Indestructible<Allocator>+      allocators[AccessSpreader<>::maxLocalityIndexValue()];+  auto index = AccessSpreader<>::localityIndexForStripe(numStripes, stripe);+  return allocators[index]->allocate(size);+}++void coreFree(void* ptr) {+  Allocator::deallocate(ptr);+}++namespace {+thread_local CoreAllocatorGuard* gCoreAllocatorGuard = nullptr;+}++CoreAllocatorGuard::CoreAllocatorGuard(size_t numStripes, size_t stripe)+    : numStripes_(numStripes), stripe_(stripe) {+  CHECK(gCoreAllocatorGuard == nullptr)+      << "CoreAllocator::Guard cannot be used recursively";+  gCoreAllocatorGuard = this;+}++CoreAllocatorGuard::~CoreAllocatorGuard() {+  gCoreAllocatorGuard = nullptr;+}++namespace detail {++void* coreMallocFromGuard(size_t size) {+  CHECK(gCoreAllocatorGuard != nullptr)+      << "CoreAllocator::allocator called without an active Guard";+  return coreMalloc(+      size, gCoreAllocatorGuard->numStripes_, gCoreAllocatorGuard->stripe_);+}++} // namespace detail++} // namespace folly
+ folly/folly/concurrency/CacheLocality.h view
@@ -0,0 +1,482 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <algorithm>+#include <array>+#include <atomic>+#include <cassert>+#include <functional>+#include <limits>+#include <string>+#include <type_traits>+#include <vector>++#include <folly/Likely.h>+#include <folly/Portability.h>+#include <folly/lang/Align.h>+#include <folly/synchronization/AtomicRef.h>++namespace folly {++// This file contains several classes that might be useful if you are+// trying to dynamically optimize cache locality: CacheLocality reads+// cache sharing information from sysfs to determine how CPUs should be+// grouped to minimize contention, Getcpu provides fast access to the+// current CPU via __vdso_getcpu, and AccessSpreader uses these two to+// optimally spread accesses among a predetermined number of stripes.+//+// AccessSpreader<>::current(n) microbenchmarks at 22 nanos, which is+// substantially less than the cost of a cache miss.  This means that we+// can effectively use it to reduce cache line ping-pong on striped data+// structures such as IndexedMemPool or statistics counters.+//+// Because CacheLocality looks at all of the cache levels, it can be+// used for different levels of optimization.  AccessSpreader(2) does+// per-chip spreading on a dual socket system.  AccessSpreader(numCpus)+// does perfect per-cpu spreading.  AccessSpreader(numCpus / 2) does+// perfect L1 spreading in a system with hyperthreading enabled.++struct CacheLocality {+  /// 1 more than the maximum value that can be returned from sched_getcpu+  /// or getcpu.  This is the number of hardware thread contexts provided+  /// by the processors.+  size_t numCpus;++  /// NOTE: The information below may be a heuristic approximation based on the+  /// available mechanisms to parse cpu topology.++  /// Holds the number of caches present at each cache level (0 is+  /// the closest to the cpu).  This is the number of AccessSpreader+  /// stripes needed to avoid cross-cache communication at the specified+  /// layer.  numCachesByLevel.front() is the number of L1 caches and+  /// numCachesByLevel.back() is the number of last-level caches.+  std::vector<size_t> numCachesByLevel;++  /// A map from cpu (from sched_getcpu or getcpu) to an index in the+  /// range 0..numCpus-1, where neighboring locality indices are more+  /// likely to share caches then indices far away.  All of the members+  /// of a particular cache level be contiguous in their locality index.+  /// For example, if numCpus is 32 and numCachesByLevel.back() is 2,+  /// then cpus with a locality index < 16 will share one last-level+  /// cache and cpus with a locality index >= 16 will share the other.+  std::vector<size_t> localityIndexByCpu;++  /// For each cpu, a list of cache identifiers following the same layout as+  /// numCachesByLevel. The identifier itself is an arbitrary number: it only+  /// signifies that cpus with the same identifier share a cache at that level.+  std::vector<std::vector<size_t>> equivClassesByCpu;++  /// Returns the best CacheLocality information available for the current+  /// system, cached for fast access.  This will be loaded from sysfs if+  /// possible, otherwise it will be correct in the number of CPUs but+  /// not in their sharing structure.+  ///+  /// If you are into yo dawgs, this is a shared cache of the local+  /// locality of the shared caches.+  ///+  /// The template parameter here is used to allow injection of a+  /// repeatable CacheLocality structure during testing.  Rather than+  /// inject the type of the CacheLocality provider into every data type+  /// that transitively uses it, all components select between the default+  /// sysfs implementation and a deterministic implementation by keying+  /// off the type of the underlying atomic.  See DeterministicScheduler.+  template <template <typename> class Atom = std::atomic>+  static const CacheLocality& system();++  /// Reads CacheLocality information from a tree structured like+  /// the sysfs filesystem.  The provided function will be evaluated+  /// for each sysfs file that needs to be queried.  The function+  /// should return a string containing the first line of the file+  /// (not including the newline), or an empty string if the file does+  /// not exist.  The function will be called with paths of the form+  /// /sys/devices/system/cpu/cpu*/cache/index*/{type,shared_cpu_list} .+  /// Throws an exception if no caches can be parsed at all.+  static CacheLocality readFromSysfsTree(+      const std::function<std::string(std::string const&)>& mapping);++  /// Reads CacheLocality information from the real sysfs filesystem.+  /// Throws an exception if no cache information can be loaded.+  static CacheLocality readFromSysfs();++  /// readFromProcCpuinfo(), except input is taken from memory rather+  /// than the file system.+  static CacheLocality readFromProcCpuinfoLines(+      std::vector<std::string> const& lines);++  /// Returns an estimate of the CacheLocality information by reading+  /// /proc/cpuinfo.  This isn't as accurate as readFromSysfs(), but+  /// is a lot faster because the info isn't scattered across+  /// hundreds of files.  Throws an exception if no cache information+  /// can be loaded.+  static CacheLocality readFromProcCpuinfo();++  /// Returns a usable (but probably not reflective of reality)+  /// CacheLocality structure with the specified number of cpus and a+  /// single cache level that associates one cpu per cache.+  static CacheLocality uniform(size_t numCpus);++ private:+  explicit CacheLocality(std::vector<std::vector<size_t>> equivClasses);+};++/// Knows how to derive a function pointer to the VDSO implementation of+/// getcpu(2), if available+struct Getcpu {+  /// Function pointer to a function with the same signature as getcpu(2).+  typedef int (*Func)(unsigned* cpu, unsigned* node, void* unused);++  /// Returns a pointer to the VDSO implementation of getcpu(2), if+  /// available, or nullptr otherwise.  This function may be quite+  /// expensive, be sure to cache the result.+  static Func resolveVdsoFunc();+};++struct SequentialThreadId {+  static unsigned get();+};++struct HashingThreadId {+  static unsigned get();+};++/// A class that lazily binds a unique (for each implementation of Atom)+/// identifier to a thread.  This is a fallback mechanism for the access+/// spreader if __vdso_getcpu can't be loaded+template <typename ThreadId>+struct FallbackGetcpu {+  /// Fills the thread id into the cpu and node out params (if they+  /// are non-null).  This method is intended to act like getcpu when a+  /// fast-enough form of getcpu isn't available or isn't desired+  static int getcpu(unsigned* cpu, unsigned* node, void* /* unused */) {+    auto id = ThreadId::get();+    if (cpu) {+      *cpu = id;+    }+    if (node) {+      *node = id;+    }+    return 0;+  }+};++using FallbackGetcpuType = FallbackGetcpu<+    conditional_t<kIsMobile, HashingThreadId, SequentialThreadId>>;++namespace detail {++class AccessSpreaderBase {+ protected:+  /// If there are more cpus than this nothing will crash, but there+  /// might be unnecessary sharing+  enum {+    // Android phones with 8 cores exist today; 16 for future-proofing.+    kMaxCpus = kIsMobile ? 16 : 256,+  };++  using CompactStripe = uint8_t;++  static_assert(+      (kMaxCpus & (kMaxCpus - 1)) == 0,+      "kMaxCpus should be a power of two so modulo is fast");+  static_assert(+      kMaxCpus - 1 <= std::numeric_limits<CompactStripe>::max(),+      "stripeByCpu element type isn't wide enough");++  using CompactStripeTable = CompactStripe[kMaxCpus + 1][kMaxCpus];++  struct GlobalState {+    /// For each level of splitting up to kMaxCpus, maps the cpu (mod+    /// kMaxCpus) to the stripe.  Rather than performing any inequalities+    /// or modulo on the actual number of cpus, we just fill in the entire+    /// array.+    /// Keep as the first field to avoid extra + in the fastest path.+    mutable CompactStripeTable table;++    /// Points to the getcpu-like function we are using to obtain the+    /// current cpu. It should not be assumed that the returned cpu value+    /// is in range.+    std::atomic<Getcpu::Func> getcpu; // nullptr -> not initialized+  };++  /// Always claims to be on CPU zero, node zero+  static int degenerateGetcpu(unsigned* cpu, unsigned* node, void*);++  static bool initialize(+      GlobalState& out, Getcpu::Func (&)(), const CacheLocality& (&)());+};++} // namespace detail++/// AccessSpreader arranges access to a striped data structure in such a+/// way that concurrently executing threads are likely to be accessing+/// different stripes.  It does NOT guarantee uncontended access.+/// Your underlying algorithm must be thread-safe without spreading, this+/// is merely an optimization.  AccessSpreader::current(n) is typically+/// much faster than a cache miss (12 nanos on my dev box, tested fast+/// in both 2.6 and 3.2 kernels).+///+/// If available (and not using the deterministic testing implementation)+/// AccessSpreader uses the getcpu system call via VDSO and the+/// precise locality information retrieved from sysfs by CacheLocality.+/// This provides optimal anti-sharing at a fraction of the cost of a+/// cache miss.+///+/// When there are not as many stripes as processors, we try to optimally+/// place the cache sharing boundaries.  This means that if you have 2+/// stripes and run on a dual-socket system, your 2 stripes will each get+/// all of the cores from a single socket.  If you have 16 stripes on a+/// 16 core system plus hyperthreading (32 cpus), each core will get its+/// own stripe and there will be no cache sharing at all.+///+/// AccessSpreader has a fallback mechanism for when __vdso_getcpu can't be+/// loaded, or for use during deterministic testing.  Using sched_getcpu+/// or the getcpu syscall would negate the performance advantages of+/// access spreading, so we use a thread-local value and a shared atomic+/// counter to spread access out.  On systems lacking both a fast getcpu()+/// and TLS, we hash the thread id to spread accesses.+///+/// AccessSpreader is templated on the template type that is used+/// to implement atomics, as a way to instantiate the underlying+/// heuristics differently for production use and deterministic unit+/// testing.  See DeterministicScheduler for more.  If you aren't using+/// DeterministicScheduler, you can just use the default template parameter+/// all of the time.+template <template <typename> class Atom = std::atomic>+struct AccessSpreader : private detail::AccessSpreaderBase {+ private:+  struct GlobalState : detail::AccessSpreaderBase::GlobalState {};+  static_assert(+      std::is_trivially_destructible<GlobalState>::value,+      "unsuitable for global state");++ public:+  FOLLY_EXPORT static GlobalState& state() {+    static FOLLY_CONSTINIT GlobalState state{};+    if (FOLLY_UNLIKELY(!state.getcpu.load(std::memory_order_acquire))) {+      initialize(state);+    }+    return state;+  }++  /// Returns the stripe associated with the current CPU.  The returned+  /// value will be < numStripes.+  static size_t current(size_t numStripes, const GlobalState& s = state()) {+    // s.table[0] will actually work okay (all zeros), but+    // something's wrong with the caller+    assert(numStripes > 0);++    unsigned cpu;+    s.getcpu.load(std::memory_order_relaxed)(&cpu, nullptr, nullptr);+    cpu = cpu % kMaxCpus;+    auto& ref = s.table[std::min(size_t(kMaxCpus), numStripes)][cpu];+    return make_atomic_ref(ref).load(std::memory_order_relaxed);+  }++  /// Returns the stripe associated with the current CPU.  The returned+  /// value will be < numStripes.+  /// This function caches the current cpu in a thread-local variable for a+  /// certain small number of calls, which can make the result imprecise, but+  /// it is more efficient (amortized 2 ns on my dev box, compared to 12 ns for+  /// current()).+  static size_t cachedCurrent(+      size_t numStripes, const GlobalState& s = state()) {+    if (kIsMobile) {+      return current(numStripes, s);+    }+    unsigned cpu = cpuCache().cpu(s);+    auto& ref = s.table[std::min(size_t(kMaxCpus), numStripes)][cpu];+    return make_atomic_ref(ref).load(std::memory_order_relaxed);+  }++  /// Forces the next cachedCurrent() call in this thread to re-probe the+  /// current CPU.+  static void invalidateCachedCurrent() {+    if (kIsMobile) {+      return;+    }+    cpuCache().invalidate();+  }++  /// Returns a canonical index in [0, maxLocalityIndexValue()) for each+  /// stripe. This can be used to share global data structures accessed with+  /// different stripings. For optimal spread, it is best for numStripes to be a+  /// divisor of the number of L1 caches.+  static size_t localityIndexForStripe(size_t numStripes, size_t stripe) {+    assert(stripe < numStripes);+    return stripe *+        std::min(size_t(kMaxCpus), CacheLocality::system<Atom>().numCpus) /+        numStripes;+  }++  /// Returns the maximum stripe value that can be returned under any+  /// dynamic configuration, based on the current compile-time platform+  static constexpr size_t maxStripeValue() { return kMaxCpus; }++  /// Returns the maximum locality index value that can be returned under any+  /// dynamic configuration, based on the current compile-time platform+  static constexpr size_t maxLocalityIndexValue() { return kMaxCpus; }++ private:+  /// Caches the current CPU and refreshes the cache every so often.+  class CpuCache {+   public:+    unsigned cpu(GlobalState const& s) {+      if (FOLLY_UNLIKELY(cachedCpuUses_-- == 0)) {+        unsigned cpu;+        s.getcpu.load(std::memory_order_relaxed)(&cpu, nullptr, nullptr);+        cachedCpu_ = cpu % kMaxCpus;+        cachedCpuUses_ = kMaxCachedCpuUses - 1;+      }+      return cachedCpu_;+    }++    void invalidate() { cachedCpuUses_ = 0; }++   private:+    static constexpr unsigned kMaxCachedCpuUses = 32;++    unsigned cachedCpu_ = 0;+    unsigned cachedCpuUses_ = 0;+  };++  FOLLY_EXPORT FOLLY_ALWAYS_INLINE static CpuCache& cpuCache() {+    static thread_local CpuCache cpuCache;+    return cpuCache;+  }++  /// Returns the best getcpu implementation for Atom+  static Getcpu::Func pickGetcpuFunc() {+    auto best = Getcpu::resolveVdsoFunc();+    return best ? best : &FallbackGetcpuType::getcpu;+  }++  // The function to call for fast lookup of getcpu is a singleton, as+  // is the precomputed table of locality information.  AccessSpreader+  // is used in very tight loops, however (we're trying to race an L1+  // cache miss!), so the normal singleton mechanisms are noticeably+  // expensive.  Even a not-taken branch guarding access to getcpuFunc+  // slows AccessSpreader::current from 12 nanos to 14.  As a result, we+  // populate the static members with simple (but valid) values that can+  // be filled in by the linker, and then follow up with a normal static+  // initializer call that puts in the proper version.  This means that+  // when there are initialization order issues we will just observe a+  // zero stripe.  Once a sanitizer gets smart enough to detect this as+  // a race or undefined behavior, we can annotate it.++  static bool initialize(GlobalState& state) {+    return detail::AccessSpreaderBase::initialize(+        state, pickGetcpuFunc, CacheLocality::system<Atom>);+  }+};++/// Similar to AccessSpreader, but it has exactly one stripe for each last-level+/// cache that is accessible by the current process.+///+/// Only supported on Linux; on other systems, numStripes() always returns 1+/// and current() always returns 0.+class LLCAccessSpreader {+  struct PrivateTag {};++ public:+  static LLCAccessSpreader& get();++  explicit LLCAccessSpreader(PrivateTag);++  size_t current() const;+  size_t numStripes() const;++ private:+  Getcpu::Func getcpu_;+  size_t numStripes_;+  std::vector<size_t> stripeByCpu_;+};++/**+ * An allocator that can be used with AccessSpreader to allocate core-local+ * memory.+ *+ * There is actually nothing special about the memory itself (it is not bound to+ * NUMA nodes or anything), but the allocator guarantees that memory allocatd+ * from the same stripe will only come from cache lines also allocated to the+ * same stripe, for the given numStripes.  This means multiple things using+ * AccessSpreader can allocate memory in smaller-than cacheline increments, and+ * be assured that it won't cause more false sharing than it otherwise would.+ *+ * Note that allocation and deallocation takes a per-size-class lock.+ *+ * Memory allocated with coreMalloc() must be freed with coreFree().+ */+void* coreMalloc(size_t size, size_t numStripes, size_t stripe);+void coreFree(void* ptr);++namespace detail {+void* coreMallocFromGuard(size_t size);+}++/**+ * An C++ allocator adapter for coreMalloc/Free. The allocator is stateless, to+ * avoid increasing the footprint of the container that uses it, so the stripe+ * needs to be passed out of band: allocate() can only be called while there is+ * an active CoreAllocatorGuard. deallocate() can instead be called at any+ * point.+ *+ * This makes CoreAllocator unsuitable for containers that can grow, and it is+ * meant for container where all allocations happen at construction time.+ */+template <typename T>+class CoreAllocator : private std::allocator<T> {+ public:+  using value_type = T;++  CoreAllocator() = default;++  template <class U>+  /* implicit */ CoreAllocator(const CoreAllocator<U>&) {}++  T* allocate(std::size_t n) {+    return reinterpret_cast<T*>(detail::coreMallocFromGuard(n * sizeof(T)));+  }++  void deallocate(T* p, std::size_t) { coreFree(p); }++  friend bool operator==(const CoreAllocator&, const CoreAllocator&) noexcept {+    return true;+  }+  friend bool operator!=(const CoreAllocator&, const CoreAllocator&) noexcept {+    return false;+  }++  template <typename U>+  struct rebind {+    using other = CoreAllocator<U>;+  };+};++class FOLLY_NODISCARD CoreAllocatorGuard {+ public:+  CoreAllocatorGuard(size_t numStripes, size_t stripe);+  ~CoreAllocatorGuard();++ private:+  friend void* detail::coreMallocFromGuard(size_t size);++  size_t numStripes_;+  size_t stripe_;+};++} // namespace folly
+ folly/folly/concurrency/ConcurrentHashMap.h view
@@ -0,0 +1,849 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <atomic>+#include <mutex>++#include <folly/Optional.h>+#include <folly/concurrency/detail/ConcurrentHashMap-detail.h>+#include <folly/synchronization/Hazptr.h>++namespace folly {++/**+ * Implementations of high-performance Concurrent Hashmaps that+ * support erase and update.+ *+ * Readers are always wait-free.+ * Writers are sharded, but take a lock that only locks part of the map.+ *+ * Multithreaded performance beats anything except the lock-free+ *      atomic maps (AtomicUnorderedMap, AtomicHashMap), BUT only+ *      if you can perfectly size the atomic maps, and you don't+ *      need erase().  If you don't know the size in advance or+ *      your workload needs erase(), this is the better choice.+ *+ * The interface is as close to std::unordered_map as possible, but there+ * are a handful of changes:+ *+ * * Iterators hold hazard pointers to the returned elements.  Elements can only+ *   be accessed while Iterators are still valid!+ *+ * * Therefore operator[] and at() return copies, since they do not+ *   return an iterator.  The returned value is const, to remind you+ *   that changes do not affect the value in the map.+ *+ * * erase() calls the hash function, and may fail if the hash+ *   function throws an exception.+ *+ * * clear() initializes new segments, and is not noexcept.+ *+ * * The interface adds assign_if_equal, since find() doesn't take a lock.+ *+ * * Only const version of find() is supported, and const iterators.+ *   Mutation must use functions provided, like assign().+ *+ * * iteration iterates over all the buckets in the table, unlike+ *   std::unordered_map which iterates over a linked list of elements.+ *   If the table is sparse, this may be more expensive.+ *+ * * Allocator must be stateless.+ *+ * 1: ConcurrentHashMap, based on Java's ConcurrentHashMap.+ *    Very similar to std::unordered_map in performance.+ *+ * 2: ConcurrentHashMapSIMD, based on F14ValueMap.  If the map is+ *    larger than the cache size, it has superior performance due to+ *    vectorized key lookup.+ *+ *+ *+ * USAGE FAQs+ *+ * Q: Is simultaneous iteration and erase() threadsafe?+ *       Example:+ *+ *       ConcurrentHashMap<int, int> map;+ *+ *       Thread 1: auto it = map.begin();+ *                   while (it != map.end()) {+ *                      // Do something with it+ *                      it++;+ *                   }+ *+ *       Thread 2:    map.insert(2, 2);  map.erase(2);+ *+ * A: Yes, this is safe.  However, the iterating thread is not+ * guaranteed to see (or not see) concurrent insertions and erasures.+ * Inserts may cause a rehash, but the old table is still valid as+ * long as any iterator pointing to it exists.+ *+ * Q: How do I update an existing object atomically?+ *+ * A: assign_if_equal is the recommended way - readers will see the+ * old value until the new value is completely constructed and+ * inserted.+ *+ * Q: Why do map.erase() and clear() not actually destroy elements?+ *+ * A: Hazard Pointers are used to improve the performance of+ * concurrent access.  They can be thought of as a simple Garbage+ * Collector.  To reduce the GC overhead, a GC pass is only run after+ * reaching a certain memory bound.  erase() will remove the element+ * from being accessed via the map, but actual destruction may happen+ * later, after iterators that may point to it have been deleted.+ *+ * The only guarantee is that a GC pass will be run on map destruction+ * - no elements will remain after map destruction.+ *+ * Q: Are pointers to values safe to access *without* holding an+ * iterator?+ *+ * A: The SIMD version guarantees that references to elements are+ * stable across rehashes, the non-SIMD version does *not*.  Note that+ * unless you hold an iterator, you need to ensure there are no+ * concurrent deletes/updates to that key if you are accessing it via+ * reference.+ */++template <+    typename KeyType,+    typename ValueType,+    typename HashFn = std::hash<KeyType>,+    typename KeyEqual = std::equal_to<KeyType>,+    typename Allocator = std::allocator<uint8_t>,+    uint8_t ShardBits = 8,+    template <typename> class Atom = std::atomic,+    class Mutex = std::mutex,+    template <+        typename,+        typename,+        uint8_t,+        typename,+        typename,+        typename,+        template <typename>+        class,+        class>+    class Impl = detail::concurrenthashmap::bucket::BucketTable>+class ConcurrentHashMap {+  using SegmentT = detail::ConcurrentHashMapSegment<+      KeyType,+      ValueType,+      ShardBits,+      HashFn,+      KeyEqual,+      Allocator,+      Atom,+      Mutex,+      Impl>;+  using SegmentTAllocator = typename std::allocator_traits<+      Allocator>::template rebind_alloc<SegmentT>;+  template <typename K, typename T>+  using EnableHeterogeneousFind = std::enable_if_t<+      detail::EligibleForHeterogeneousFind<KeyType, HashFn, KeyEqual, K>::value,+      T>;++  float load_factor_ = SegmentT::kDefaultLoadFactor;++  static constexpr uint64_t NumShards = (1 << ShardBits);++ public:+  class ConstIterator;++  typedef KeyType key_type;+  typedef ValueType mapped_type;+  typedef std::pair<const KeyType, ValueType> value_type;+  typedef std::size_t size_type;+  typedef HashFn hasher;+  typedef KeyEqual key_equal;+  typedef ConstIterator const_iterator;++ private:+  template <typename K, typename T>+  using EnableHeterogeneousInsert = std::enable_if_t<+      ::folly::detail::+          EligibleForHeterogeneousInsert<KeyType, HashFn, KeyEqual, K>::value,+      T>;++  template <typename K>+  using IsIter = std::is_same<ConstIterator, remove_cvref_t<K>>;++  template <typename K, typename T>+  using EnableHeterogeneousErase = std::enable_if_t<+      ::folly::detail::EligibleForHeterogeneousFind<+          KeyType,+          HashFn,+          KeyEqual,+          std::conditional_t<IsIter<K>::value, KeyType, K>>::value &&+          !IsIter<K>::value,+      T>;++ public:+  /*+   * Construct a ConcurrentHashMap with 1 << ShardBits shards, size+   * and max_size given.  Both size and max_size will be rounded up to+   * the next power of two, if they are not already a power of two, so+   * that we can index in to Shards efficiently.+   *+   * Insertion functions will throw bad_alloc if max_size is exceeded.+   */+  explicit ConcurrentHashMap(size_t size = 8, size_t max_size = 0) {+    size_ = folly::nextPowTwo(size);+    if (max_size != 0) {+      max_size_ = folly::nextPowTwo(max_size);+    }+    CHECK(max_size_ == 0 || max_size_ >= size_);+    for (uint64_t i = 0; i < NumShards; i++) {+      segments_[i].store(nullptr, std::memory_order_relaxed);+    }+  }++  ConcurrentHashMap(ConcurrentHashMap&& o) noexcept+      : size_(o.size_), max_size_(o.max_size_) {+    for (uint64_t i = 0; i < NumShards; i++) {+      segments_[i].store(+          o.segments_[i].load(std::memory_order_relaxed),+          std::memory_order_relaxed);+      o.segments_[i].store(nullptr, std::memory_order_relaxed);+    }+    cohort_.store(o.cohort(), std::memory_order_relaxed);+    o.cohort_.store(nullptr, std::memory_order_relaxed);+    beginSeg_.store(+        o.beginSeg_.load(std::memory_order_relaxed), std::memory_order_relaxed);+    o.beginSeg_.store(NumShards, std::memory_order_relaxed);+    endSeg_.store(+        o.endSeg_.load(std::memory_order_relaxed), std::memory_order_relaxed);+    o.endSeg_.store(0, std::memory_order_relaxed);+  }++  ConcurrentHashMap& operator=(ConcurrentHashMap&& o) {+    for (uint64_t i = 0; i < NumShards; i++) {+      auto seg = segments_[i].load(std::memory_order_relaxed);+      if (seg) {+        seg->~SegmentT();+        SegmentTAllocator().deallocate(seg, 1);+      }+      segments_[i].store(+          o.segments_[i].load(std::memory_order_relaxed),+          std::memory_order_relaxed);+      o.segments_[i].store(nullptr, std::memory_order_relaxed);+    }+    size_ = o.size_;+    max_size_ = o.max_size_;+    cohort_shutdown_cleanup();+    cohort_.store(o.cohort(), std::memory_order_relaxed);+    o.cohort_.store(nullptr, std::memory_order_relaxed);+    beginSeg_.store(+        o.beginSeg_.load(std::memory_order_relaxed), std::memory_order_relaxed);+    o.beginSeg_.store(NumShards, std::memory_order_relaxed);+    endSeg_.store(+        o.endSeg_.load(std::memory_order_relaxed), std::memory_order_relaxed);+    o.endSeg_.store(0, std::memory_order_relaxed);+    return *this;+  }++  ~ConcurrentHashMap() {+    uint64_t begin = beginSeg_.load(std::memory_order_acquire);+    uint64_t end = endSeg_.load(std::memory_order_acquire);+    for (uint64_t i = begin; i < end; ++i) {+      auto seg = segments_[i].load(std::memory_order_relaxed);+      if (seg) {+        seg->~SegmentT();+        SegmentTAllocator().deallocate(seg, 1);+      }+    }+    cohort_shutdown_cleanup();+  }++  bool empty() const noexcept {+    uint64_t begin = beginSeg_.load(std::memory_order_acquire);+    uint64_t end = endSeg_.load(std::memory_order_acquire);+    // Note: beginSeg_ and endSeg_ are only conservative hints of the+    // range of non-empty segments. This function cannot conclude that+    // a map is nonempty merely because beginSeg_ < endSeg_.+    for (uint64_t i = begin; i < end; ++i) {+      auto seg = segments_[i].load(std::memory_order_acquire);+      if (seg) {+        if (!seg->empty()) {+          return false;+        }+      }+    }+    return true;+  }++  ConstIterator find(const KeyType& k) const { return findImpl(k); }++  template <typename K, EnableHeterogeneousFind<K, int> = 0>+  ConstIterator find(const K& k) const {+    return findImpl(k);+  }++  ConstIterator cend() const noexcept { return ConstIterator(NumShards); }++  ConstIterator cbegin() const noexcept { return ConstIterator(this); }++  ConstIterator end() const noexcept { return cend(); }++  ConstIterator begin() const noexcept { return cbegin(); }++  std::pair<ConstIterator, bool> insert(+      std::pair<key_type, mapped_type>&& foo) {+    return insertImpl(std::move(foo));+  }++  template <typename Key, EnableHeterogeneousInsert<Key, int> = 0>+  std::pair<ConstIterator, bool> insert(std::pair<Key, mapped_type>&& foo) {+    return insertImpl(std::move(foo));+  }++  template <typename Key, typename Value>+  std::pair<ConstIterator, bool> insert(Key&& k, Value&& v) {+    auto h = HashFn{}(k);+    auto segment = pickSegment(h);+    std::pair<ConstIterator, bool> res(+        std::piecewise_construct,+        std::forward_as_tuple(this, segment),+        std::forward_as_tuple(false));+    res.second = ensureSegment(segment)->insert(+        res.first.it_, h, std::forward<Key>(k), std::forward<Value>(v));+    return res;+  }++  template <typename Key, typename... Args>+  std::pair<ConstIterator, bool> try_emplace(Key&& k, Args&&... args) {+    auto h = HashFn{}(k);+    auto segment = pickSegment(h);+    std::pair<ConstIterator, bool> res(+        std::piecewise_construct,+        std::forward_as_tuple(this, segment),+        std::forward_as_tuple(false));+    res.second = ensureSegment(segment)->try_emplace(+        res.first.it_, h, std::forward<Key>(k), std::forward<Args>(args)...);+    return res;+  }++  template <typename... Args>+  std::pair<ConstIterator, bool> emplace(Args&&... args) {+    using Node = typename SegmentT::Node;+    detail::concurrenthashmap::AllocNodeGuard<Node, Allocator> g(+        Allocator(), ensureCohort(), std::forward<Args>(args)...);+    auto h = HashFn{}(g.node->getItem().first);+    auto segment = pickSegment(h);+    std::pair<ConstIterator, bool> res(+        std::piecewise_construct,+        std::forward_as_tuple(this, segment),+        std::forward_as_tuple(false));+    res.second = ensureSegment(segment)->emplace(+        res.first.it_, h, g.node->getItem().first, g.node);+    if (res.second) {+      g.dismiss();+    }+    return res;+  }++  /*+   * The bool component will always be true if the map has been updated via+   * either insertion or assignment. Note that this is different from the+   * std::map::insert_or_assign interface.+   */+  template <typename Key, typename Value>+  std::pair<ConstIterator, bool> insert_or_assign(Key&& k, Value&& v) {+    auto h = HashFn{}(k);+    auto segment = pickSegment(h);+    std::pair<ConstIterator, bool> res(+        std::piecewise_construct,+        std::forward_as_tuple(this, segment),+        std::forward_as_tuple(false));+    res.second = ensureSegment(segment)->insert_or_assign(+        res.first.it_, h, std::forward<Key>(k), std::forward<Value>(v));+    return res;+  }++  /*+   * Insert desired if the key doesn't exist, or assign to desired if the+   * predicate returns true for the current value. The bool component will+   * always be true if the map has been updated via either insertion or+   * assignment. Note that this is different from the std::map::insert_or_assign+   * interface.+   */+  template <typename Key, typename Value, typename Predicate>+  std::pair<ConstIterator, bool> insert_or_assign_if(+      Key&& k, Value&& desired, Predicate&& predicate) {+    auto h = HashFn{}(k);+    auto segment = pickSegment(h);+    std::pair<ConstIterator, bool> res(+        std::piecewise_construct,+        std::forward_as_tuple(this, segment),+        std::forward_as_tuple(false));+    res.second = ensureSegment(segment)->insert_or_assign_if(+        res.first.it_,+        h,+        std::forward<Key>(k),+        std::forward<Value>(desired),+        std::forward<Predicate>(predicate));+    return res;+  }++  template <typename Key, typename Value>+  folly::Optional<ConstIterator> assign(Key&& k, Value&& v) {+    auto h = HashFn{}(k);+    auto segment = pickSegment(h);+    ConstIterator res(this, segment);+    auto seg = segments_[segment].load(std::memory_order_acquire);+    if (!seg) {+      return none;+    } else {+      auto r =+          seg->assign(res.it_, h, std::forward<Key>(k), std::forward<Value>(v));+      if (!r) {+        return none;+      }+    }+    return std::move(res);+  }++  // Assign to desired if and only if the predicate returns true+  // for the current value.+  template <typename Key, typename Value, typename Predicate>+  folly::Optional<ConstIterator> assign_if(+      Key&& k, Value&& desired, Predicate&& predicate) {+    auto h = HashFn{}(k);+    auto segment = pickSegment(h);+    ConstIterator res(this, segment);+    auto seg = segments_[segment].load(std::memory_order_acquire);+    if (!seg) {+      return none;+    } else {+      auto r = seg->assign_if(+          res.it_,+          h,+          std::forward<Key>(k),+          std::forward<Value>(desired),+          std::forward<Predicate>(predicate));+      if (!r) {+        return none;+      }+    }+    return std::move(res);+  }++  // Assign to desired if and only if current value is equal to expected+  template <typename Key, typename Value>+  folly::Optional<ConstIterator> assign_if_equal(+      Key&& k, const ValueType& expected, Value&& desired) {+    auto h = HashFn{}(k);+    auto segment = pickSegment(h);+    ConstIterator res(this, segment);+    auto seg = segments_[segment].load(std::memory_order_acquire);+    if (!seg) {+      return none;+    } else {+      auto r = seg->assign_if_equal(+          res.it_,+          h,+          std::forward<Key>(k),+          expected,+          std::forward<Value>(desired));+      if (!r) {+        return none;+      }+    }+    return std::move(res);+  }++  // Copying wrappers around insert and find.+  // Only available for copyable types.+  const ValueType operator[](const KeyType& key) {+    auto item = insert(key, ValueType());+    return item.first->second;+  }++  template <typename Key, EnableHeterogeneousInsert<Key, int> = 0>+  const ValueType operator[](const Key& key) {+    auto item = insert(key, ValueType());+    return item.first->second;+  }++  const ValueType at(const KeyType& key) const { return atImpl(key); }++  template <typename K, EnableHeterogeneousFind<K, int> = 0>+  const ValueType at(const K& key) const {+    return atImpl(key);+  }++  // TODO update assign interface, operator[], at++  size_type erase(const key_type& k) { return eraseImpl(k); }++  template <typename K, EnableHeterogeneousErase<K, int> = 0>+  size_type erase(const K& k) {+    return eraseImpl(k);+  }++  // Calls the hash function, and therefore may throw.+  // This function doesn't necessarily delete the item that pos points to.+  // It simply tries erasing the item associated with the same key.+  // While this behavior can be confusing, erase(iterator) is often found in+  // std data structures so we follow a similar pattern here.+  ConstIterator erase(ConstIterator& pos) {+    auto h = HashFn{}(pos->first);+    auto segment = pickSegment(h);+    ConstIterator res(this, segment);+    ensureSegment(segment)->erase(res.it_, pos.it_, h);+    res.advanceIfAtSegmentEnd();+    return res;+  }++  // Erase if and only if key k is equal to expected+  size_type erase_if_equal(const key_type& k, const ValueType& expected) {+    return erase_key_if(k, [&expected](const ValueType& v) {+      return v == expected;+    });+  }++  template <typename K, EnableHeterogeneousErase<K, int> = 0>+  size_type erase_if_equal(const K& k, const ValueType& expected) {+    return erase_key_if(k, [&expected](const ValueType& v) {+      return v == expected;+    });+  }++  // Erase if predicate evaluates to true on the existing value+  template <typename Predicate>+  size_type erase_key_if(const key_type& k, Predicate&& predicate) {+    return eraseKeyIfImpl(k, std::forward<Predicate>(predicate));+  }++  template <+      typename K,+      typename Predicate,+      EnableHeterogeneousErase<K, int> = 0>+  size_type erase_key_if(const K& k, Predicate&& predicate) {+    return eraseKeyIfImpl(k, std::forward<Predicate>(predicate));+  }++  // NOT noexcept, initializes new shard segments vs.+  void clear() {+    uint64_t begin = beginSeg_.load(std::memory_order_acquire);+    uint64_t end = endSeg_.load(std::memory_order_acquire);+    for (uint64_t i = begin; i < end; ++i) {+      auto seg = segments_[i].load(std::memory_order_acquire);+      if (seg) {+        seg->clear();+      }+    }+  }++  void reserve(size_t count) {+    count = count >> ShardBits;+    if (!count)+      return;+    uint64_t begin = beginSeg_.load(std::memory_order_acquire);+    uint64_t end = endSeg_.load(std::memory_order_acquire);+    for (uint64_t i = begin; i < end; ++i) {+      auto seg = segments_[i].load(std::memory_order_acquire);+      if (seg) {+        seg->rehash(count);+      }+    }+  }++  // This is a rolling size, and is not exact at any moment in time.+  size_t size() const noexcept {+    size_t res = 0;+    uint64_t begin = beginSeg_.load(std::memory_order_acquire);+    uint64_t end = endSeg_.load(std::memory_order_acquire);+    for (uint64_t i = begin; i < end; ++i) {+      auto seg = segments_[i].load(std::memory_order_acquire);+      if (seg) {+        res += seg->size();+      }+    }+    return res;+  }++  float max_load_factor() const { return load_factor_; }++  void max_load_factor(float factor) {+    uint64_t begin = beginSeg_.load(std::memory_order_acquire);+    uint64_t end = endSeg_.load(std::memory_order_acquire);+    for (uint64_t i = begin; i < end; ++i) {+      auto seg = segments_[i].load(std::memory_order_acquire);+      if (seg) {+        seg->max_load_factor(factor);+      }+    }+  }++  class ConstIterator {+   public:+    friend class ConcurrentHashMap;++    const value_type& operator*() const { return *it_; }++    const value_type* operator->() const { return &*it_; }++    ConstIterator& operator++() {+      ++it_;+      advanceIfAtSegmentEnd();+      return *this;+    }++    bool operator==(const ConstIterator& o) const {+      return it_ == o.it_ && segment_ == o.segment_;+    }++    bool operator!=(const ConstIterator& o) const { return !(*this == o); }++    ConstIterator& operator=(const ConstIterator& o) = delete;++    ConstIterator& operator=(ConstIterator&& o) noexcept {+      if (this != &o) {+        it_ = std::move(o.it_);+        segment_ = std::exchange(o.segment_, uint64_t(NumShards));+        parent_ = std::exchange(o.parent_, nullptr);+      }+      return *this;+    }++    ConstIterator(const ConstIterator& o) = delete;++    ConstIterator(ConstIterator&& o) noexcept+        : it_(std::move(o.it_)),+          segment_(std::exchange(o.segment_, uint64_t(NumShards))),+          parent_(std::exchange(o.parent_, nullptr)) {}++    ConstIterator(const ConcurrentHashMap* parent, uint64_t segment)+        : segment_(segment), parent_(parent) {}++   private:+    // cbegin iterator+    explicit ConstIterator(const ConcurrentHashMap* parent)+        : it_(nullptr),+          segment_(parent->beginSeg_.load(std::memory_order_acquire)),+          parent_(parent) {+      advanceToSegmentBegin();+    }++    // cend iterator+    explicit ConstIterator(uint64_t shards) : it_(nullptr), segment_(shards) {}++    void advanceIfAtSegmentEnd() {+      DCHECK_LT(segment_, parent_->NumShards);+      SegmentT* seg =+          parent_->segments_[segment_].load(std::memory_order_acquire);+      DCHECK(seg);+      if (it_ == seg->cend()) {+        ++segment_;+        advanceToSegmentBegin();+      }+    }++    FOLLY_ALWAYS_INLINE void advanceToSegmentBegin() {+      // Advance to the beginning of the next nonempty segment+      // starting from segment_.+      uint64_t end = parent_->endSeg_.load(std::memory_order_acquire);+      while (segment_ < end) {+        SegmentT* seg =+            parent_->segments_[segment_].load(std::memory_order_acquire);+        if (seg) {+          it_ = seg->cbegin();+          if (it_ != seg->cend()) {+            return;+          }+        }+        ++segment_;+      }+      // All segments are empty. Advance to end.+      segment_ = parent_->NumShards;+    }++    typename SegmentT::Iterator it_;+    uint64_t segment_;+    const ConcurrentHashMap* parent_;+  };++ private:+  template <typename K>+  ConstIterator findImpl(const K& k) const {+    auto h = HashFn{}(k);+    auto segment = pickSegment(h);+    ConstIterator res(this, segment);+    auto seg = segments_[segment].load(std::memory_order_acquire);+    if (!seg || !seg->find(res.it_, h, k)) {+      res.segment_ = NumShards;+    }+    return res;+  }++  template <typename K>+  const ValueType atImpl(const K& k) const {+    auto item = find(k);+    if (item == cend()) {+      throw_exception<std::out_of_range>("at(): key not in map");+    }+    return item->second;+  }++  template <typename Key>+  std::pair<ConstIterator, bool> insertImpl(std::pair<Key, mapped_type>&& foo) {+    auto h = HashFn{}(foo.first);+    auto segment = pickSegment(h);+    std::pair<ConstIterator, bool> res(+        std::piecewise_construct,+        std::forward_as_tuple(this, segment),+        std::forward_as_tuple(false));+    res.second =+        ensureSegment(segment)->insert(res.first.it_, h, std::move(foo));+    return res;+  }++  template <typename K>+  size_type eraseImpl(const K& k) {+    auto h = HashFn{}(k);+    auto segment = pickSegment(h);+    auto seg = segments_[segment].load(std::memory_order_acquire);+    if (!seg) {+      return 0;+    } else {+      return seg->erase(h, k);+    }+  }++  template <typename K, typename Predicate>+  size_type eraseKeyIfImpl(const K& k, Predicate&& predicate) {+    auto h = HashFn{}(k);+    auto segment = pickSegment(h);+    auto seg = segments_[segment].load(std::memory_order_acquire);+    if (!seg) {+      return 0;+    }+    return seg->erase_key_if(h, k, std::forward<Predicate>(predicate));+  }++  uint64_t pickSegment(size_t h) const {+    // Use the lowest bits for our shard bits.+    //+    // This works well even if the hash function is biased towards the+    // low bits: The sharding will happen in the segments_ instead of+    // in the segment buckets, so we'll still get write sharding as+    // well.+    //+    // Low-bit bias happens often for std::hash using small numbers,+    // since the integer hash function is the identity function.+    return h & (NumShards - 1);+  }++  SegmentT* ensureSegment(uint64_t i) const {+    SegmentT* seg = segments_[i].load(std::memory_order_acquire);+    if (!seg) {+      auto b = ensureCohort();+      SegmentT* newseg = SegmentTAllocator().allocate(1);+      newseg = new (newseg)+          SegmentT(size_ >> ShardBits, load_factor_, max_size_ >> ShardBits, b);+      if (!segments_[i].compare_exchange_strong(seg, newseg)) {+        // seg is updated with new value, delete ours.+        newseg->~SegmentT();+        SegmentTAllocator().deallocate(newseg, 1);+      } else {+        seg = newseg;+        updateBeginAndEndSegments(i);+      }+    }+    return seg;+  }++  void updateBeginAndEndSegments(uint64_t i) const {+    uint64_t val = beginSeg_.load(std::memory_order_acquire);+    while (i < val && !casSeg(beginSeg_, val, i)) {+    }+    val = endSeg_.load(std::memory_order_acquire);+    while (i + 1 > val && !casSeg(endSeg_, val, i + 1)) {+    }+  }++  bool casSeg(Atom<uint64_t>& seg, uint64_t& expval, uint64_t newval) const {+    return seg.compare_exchange_weak(+        expval, newval, std::memory_order_acq_rel, std::memory_order_acquire);+  }++  hazptr_obj_cohort<Atom>* cohort() const noexcept {+    return cohort_.load(std::memory_order_acquire);+  }++  hazptr_obj_cohort<Atom>* ensureCohort() const {+    auto b = cohort();+    if (!b) {+      auto storage = Allocator().allocate(sizeof(hazptr_obj_cohort<Atom>));+      auto newcohort = new (storage) hazptr_obj_cohort<Atom>();+      if (cohort_.compare_exchange_strong(b, newcohort)) {+        b = newcohort;+      } else {+        newcohort->~hazptr_obj_cohort<Atom>();+        Allocator().deallocate(storage, sizeof(hazptr_obj_cohort<Atom>));+      }+    }+    return b;+  }++  void cohort_shutdown_cleanup() {+    auto b = cohort();+    if (b) {+      b->~hazptr_obj_cohort<Atom>();+      Allocator().deallocate((uint8_t*)b, sizeof(hazptr_obj_cohort<Atom>));+    }+  }++  mutable Atom<SegmentT*> segments_[NumShards];+  size_t size_{0};+  size_t max_size_{0};+  mutable Atom<hazptr_obj_cohort<Atom>*> cohort_{nullptr};+  mutable Atom<uint64_t> beginSeg_{NumShards};+  mutable Atom<uint64_t> endSeg_{0};+};++template <+    typename KeyType,+    typename ValueType,+    typename HashFn = std::hash<KeyType>,+    typename KeyEqual = std::equal_to<KeyType>,+    typename Allocator = std::allocator<uint8_t>,+    uint8_t ShardBits = 8,+    template <typename> class Atom = std::atomic,+    class Mutex = std::mutex>+using ConcurrentHashMapSIMD = ConcurrentHashMap<+    KeyType,+    ValueType,+    HashFn,+    KeyEqual,+    Allocator,+    ShardBits,+    Atom,+    Mutex,+#if (FOLLY_SSE_PREREQ(4, 2) || FOLLY_AARCH64) && \+    FOLLY_F14_VECTOR_INTRINSICS_AVAILABLE+    detail::concurrenthashmap::simd::SIMDTable+#else+    // fallback to regular impl+    detail::concurrenthashmap::bucket::BucketTable+#endif+    >;++} // namespace folly
+ folly/folly/concurrency/CoreCachedSharedPtr.h view
@@ -0,0 +1,232 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <array>+#include <atomic>+#include <memory>++#include <folly/CppAttributes.h>+#include <folly/Portability.h>+#include <folly/Unit.h>+#include <folly/concurrency/CacheLocality.h>+#include <folly/synchronization/Hazptr.h>++namespace folly {++// On mobile we do not expect high concurrency, and memory is more important, so+// use more conservative caching.+constexpr size_t kCoreCachedSharedPtrDefaultMaxSlots = kIsMobile ? 4 : 64;++namespace core_cached_shared_ptr_detail {++template <size_t kMaxSlots>+class SlotsConfig {+ public:+  FOLLY_EXPORT static void initialize() {+    [[maybe_unused]] static const Unit _ = [] {+      // We need at most as many slots as the number of L1 caches, so we can+      // avoid wasting memory if more slots are requested.+      const auto l1Caches = CacheLocality::system().numCachesByLevel.front();+      num_ = std::min(std::max<size_t>(1, l1Caches), kMaxSlots);+      return unit;+    }();+  }++  static size_t num() { return num_.load(std::memory_order_relaxed); }++ private:+  static std::atomic<size_t> num_;+};++// Initialize with a valid num so that get() always returns a valid stripe, even+// if initialize() has not been called yet.+template <size_t kMaxSlots>+std::atomic<size_t> SlotsConfig<kMaxSlots>::num_{1};++template <size_t kMaxSlots, class T>+void makeSlots(std::shared_ptr<T> p, folly::Range<std::shared_ptr<T>*> slots) {+  // Allocate each holder and its control block in a different CoreAllocator+  // stripe to prevent false sharing.+  for (size_t i = 0; i < slots.size(); ++i) {+    CoreAllocatorGuard guard(slots.size(), i);+    auto holder = std::allocate_shared<std::shared_ptr<T>>(+        CoreAllocator<std::shared_ptr<T>>{});+    auto ptr = p.get();+    if (i != slots.size() - 1) {+      *holder = p;+    } else {+      *holder = std::move(p);+    }+    slots[i] = std::shared_ptr<T>(std::move(holder), ptr);+  }+}++// Check whether a shared_ptr is equivalent to default-constructed. Because of+// aliasing constructors, there can be both nullptr with a managed object, and+// non-nullptr with no managed object, so we need to check both.+template <class T>+bool isDefault(const std::shared_ptr<T>& p) {+  return p == nullptr && p.use_count() == 0;+}++} // namespace core_cached_shared_ptr_detail++/**+ * This class creates core-local caches for a given shared_ptr, to+ * mitigate contention when acquiring/releasing it.+ *+ * It has the same thread-safety guarantees as shared_ptr: it is safe+ * to concurrently call get(), but reset()s must be synchronized with+ * reads and other reset()s.+ */+template <class T, size_t kMaxSlots = kCoreCachedSharedPtrDefaultMaxSlots>+class CoreCachedSharedPtr {+  using SlotsConfig = core_cached_shared_ptr_detail::SlotsConfig<kMaxSlots>;++ public:+  CoreCachedSharedPtr() = default;+  explicit CoreCachedSharedPtr(std::shared_ptr<T> p) { reset(std::move(p)); }++  void reset(std::shared_ptr<T> p = nullptr) {+    SlotsConfig::initialize();++    folly::Range<std::shared_ptr<T>*> slots{slots_.data(), SlotsConfig::num()};+    for (auto& slot : slots) {+      slot = {};+    }+    if (!core_cached_shared_ptr_detail::isDefault(p)) {+      core_cached_shared_ptr_detail::makeSlots<kMaxSlots>(std::move(p), slots);+    }+  }++  std::shared_ptr<T> get() const {+    return slots_[AccessSpreader<>::cachedCurrent(SlotsConfig::num())];+  }++ private:+  template <class, size_t>+  friend class CoreCachedWeakPtr;++  std::array<std::shared_ptr<T>, kMaxSlots> slots_;+};++template <class T, size_t kMaxSlots = kCoreCachedSharedPtrDefaultMaxSlots>+class CoreCachedWeakPtr {+  using SlotsConfig = core_cached_shared_ptr_detail::SlotsConfig<kMaxSlots>;++ public:+  CoreCachedWeakPtr() = default;+  explicit CoreCachedWeakPtr(const CoreCachedSharedPtr<T, kMaxSlots>& p) {+    reset(p);+  }++  void reset() { *this = {}; }+  void reset(const CoreCachedSharedPtr<T, kMaxSlots>& p) {+    SlotsConfig::initialize();+    for (size_t i = 0; i < SlotsConfig::num(); ++i) {+      slots_[i] = p.slots_[i];+    }+  }++  std::weak_ptr<T> get() const {+    return slots_[AccessSpreader<>::cachedCurrent(SlotsConfig::num())];+  }++  // Faster than get().lock(), as it avoid one weak count cycle.+  std::shared_ptr<T> lock() const {+    return slots_[AccessSpreader<>::cachedCurrent(SlotsConfig::num())].lock();+  }++ private:+  std::array<std::weak_ptr<T>, kMaxSlots> slots_;+};++/**+ * This class creates core-local caches for a given shared_ptr, to+ * mitigate contention when acquiring/releasing it.+ *+ * All methods are threadsafe.  Hazard pointers are used to avoid+ * use-after-free for concurrent reset() and get() operations.+ *+ * Concurrent reset()s are sequenced with respect to each other: the+ * sharded shared_ptrs will always all be set to the same value.+ * get()s will never see a newer pointer on one core, and an older+ * pointer on another after a subsequent thread migration.+ */+template <class T, size_t kMaxSlots = kCoreCachedSharedPtrDefaultMaxSlots>+class AtomicCoreCachedSharedPtr {+  using SlotsConfig = core_cached_shared_ptr_detail::SlotsConfig<kMaxSlots>;++ public:+  AtomicCoreCachedSharedPtr() = default;+  explicit AtomicCoreCachedSharedPtr(std::shared_ptr<T> p) {+    reset(std::move(p));+  }++  AtomicCoreCachedSharedPtr(AtomicCoreCachedSharedPtr&& other) noexcept+      : slots_(other.slots_.load(std::memory_order_relaxed)) {+    other.slots_.store(nullptr, std::memory_order_relaxed);+  }+  AtomicCoreCachedSharedPtr& operator=(AtomicCoreCachedSharedPtr&& other) =+      delete;++  ~AtomicCoreCachedSharedPtr() {+    // Delete of AtomicCoreCachedSharedPtr must be synchronized, no+    // need for slots->retire().+    delete slots_.load(std::memory_order_acquire);+  }++  void reset(std::shared_ptr<T> p = nullptr) {+    SlotsConfig::initialize();+    std::unique_ptr<Slots> newslots;+    if (!core_cached_shared_ptr_detail::isDefault(p)) {+      newslots = std::make_unique<Slots>();+      core_cached_shared_ptr_detail::makeSlots<kMaxSlots>(+          std::move(p), {newslots->slots.data(), SlotsConfig::num()});+    }++    if (auto oldslots = slots_.exchange(newslots.release())) {+      oldslots->retire();+    }+  }++  std::shared_ptr<T> get() const {+    // Avoid the hazptr cost if empty.+    auto slots = slots_.load(std::memory_order_relaxed);+    if (slots == nullptr) {+      return nullptr;+    }++    folly::hazptr_local<1> hazptr;+    while (!hazptr[0].try_protect(slots, slots_)) {+      // Lost the update race, retry.+    }+    if (slots == nullptr) { // Need to check again, try_protect reloads slots.+      return nullptr;+    }+    return slots->slots[AccessSpreader<>::cachedCurrent(SlotsConfig::num())];+  }++ private:+  struct Slots : folly::hazptr_obj_base<Slots> {+    std::array<std::shared_ptr<T>, kMaxSlots> slots;+  };+  std::atomic<Slots*> slots_{nullptr};+};++} // namespace folly
+ folly/folly/concurrency/DeadlockDetector.cpp view
@@ -0,0 +1,27 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#include <folly/concurrency/DeadlockDetector.h>++namespace folly {+/* static */ DeadlockDetectorFactory* DeadlockDetectorFactory::instance() {+  if (get_deadlock_detector_factory_instance) {+    return get_deadlock_detector_factory_instance();+  }++  return nullptr;+}+} // namespace folly
+ folly/folly/concurrency/DeadlockDetector.h view
@@ -0,0 +1,44 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <folly/Executor.h>+#include <folly/executors/QueueObserver.h>++namespace folly {+class DeadlockDetector {+ public:+  virtual ~DeadlockDetector() {}+};++class DeadlockDetectorFactory {+ public:+  virtual ~DeadlockDetectorFactory() {}+  virtual std::unique_ptr<DeadlockDetector> create(+      Executor* executor, const std::string& name) = 0;+  static DeadlockDetectorFactory* instance();+};++using GetDeadlockDetectorFactoryInstance = DeadlockDetectorFactory*();+#if FOLLY_HAVE_WEAK_SYMBOLS+FOLLY_ATTR_WEAK GetDeadlockDetectorFactoryInstance+    get_deadlock_detector_factory_instance;+#else+constexpr GetDeadlockDetectorFactoryInstance*+    get_deadlock_detector_factory_instance = nullptr;+#endif+} // namespace folly
+ folly/folly/concurrency/DynamicBoundedQueue.h view
@@ -0,0 +1,751 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <folly/concurrency/CacheLocality.h>+#include <folly/concurrency/UnboundedQueue.h>++#include <glog/logging.h>++#include <atomic>+#include <chrono>++namespace folly {++/// DynamicBoundedQueue supports:+/// - Dynamic memory usage that grows and shrink in proportion to the+///   number of elements in the queue.+/// - Adjustable capacity that helps throttle pathological cases of+///   producer-consumer imbalance that may lead to excessive memory+///   usage.+/// - The adjustable capacity can also help prevent deadlock by+///   allowing users to temporarily increase capacity substantially to+///   guarantee accommodating producer requests that cannot wait.+/// - SPSC, SPMC, MPSC, MPMC variants.+/// - Blocking and spinning-only variants.+/// - Inter-operable non-waiting, timed until, timed for, and waiting+///   variants of producer and consumer operations.+/// - Optional variable element weights.+///+/// Element Weights+/// - Queue elements may have variable weights (calculated using a+///   template parameter) that are by default 1.+/// - Element weights count towards the queue's capacity.+/// - Elements weights are not priorities and do not affect element+///   order. Queues with variable element weights follow FIFO order,+///   the same as default queues.+///+/// When to use DynamicBoundedQueue:+/// - If a small maximum capacity may lead to deadlock or performance+///   degradation under bursty patterns and a larger capacity is+///   sufficient.+/// - If the typical queue size is expected to be much lower than the+///   maximum capacity+/// - If an unbounded queue is susceptible to growing too much.+/// - If support for variable element weights is needed.+///+/// When not to use DynamicBoundedQueue?+/// - If dynamic memory allocation is unacceptable or if the maximum+///   capacity needs to be small, then use fixed-size MPMCQueue or (if+///   non-blocking SPSC) ProducerConsumerQueue.+/// - If there is no risk of the queue growing too much, then use+///   UnboundedQueue.+///+/// Setting capacity+/// - The general rule is to set the capacity as high as acceptable.+///   The queue performs best when it is not near full capacity.+/// - The implementation may allow extra slack in capacity (~10%) for+///   amortizing some costly steps. Therefore, precise capacity is not+///   guaranteed and cannot be relied on for synchronization; i.e.,+///   this queue cannot be used as a semaphore.+///+/// Performance expectations:+/// - As long as the queue size is below capacity in the common case,+///   performance is comparable to MPMCQueue and better in cases of+///   higher producer demand.+/// - Performance degrades gracefully at full capacity.+/// - It is recommended to measure performance with different variants+///   when applicable, e.g., DMPMC vs DMPSC. Depending on the use+///   case, sometimes the variant with the higher sequential overhead+///   may yield better results due to, for example, more favorable+///   producer-consumer balance or favorable timing for avoiding+///   costly blocking.+/// - See DynamicBoundedQueueTest.cpp for some benchmark results.+///+/// Template parameters:+/// - T: element type+/// - SingleProducer: true if there can be only one producer at a+///   time.+/// - SingleConsumer: true if there can be only one consumer at a+///   time.+/// - MayBlock: true if producers or consumers may block.+/// - LgSegmentSize (default 8): Log base 2 of number of elements per+///   UnboundedQueue segment.+/// - LgAlign (default 7): Log base 2 of alignment directive; can be+///   used to balance scalability (avoidance of false sharing) with+///   memory efficiency.+/// - WeightFn (DefaultWeightFn<T>): A customizable weight computing type+///   for computing the weights of elements. The default weight is 1.+///+/// Template Aliases:+///   DSPSCQueue<T, MayBlock, LgSegmentSize, LgAlign>+///   DMPSCQueue<T, MayBlock, LgSegmentSize, LgAlign>+///   DSPMCQueue<T, MayBlock, LgSegmentSize, LgAlign>+///   DMPMCQueue<T, MayBlock, LgSegmentSize, LgAlign>+///+/// Functions:+///   Constructor+///     Takes a capacity value as an argument.+///+///   Producer functions:+///     void enqueue(const T&);+///     void enqueue(T&&);+///         Adds an element to the end of the queue. Waits until+///         capacity is available if necessary.+///     bool try_enqueue(const T&);+///     bool try_enqueue(T&&);+///         Tries to add an element to the end of the queue if+///         capacity allows it. Returns true if successful. Otherwise+///         Returns false.+///     bool try_enqueue_until(const T&, time_point& deadline);+///     bool try_enqueue_until(T&&, time_point& deadline);+///         Tries to add an element to the end of the queue if+///         capacity allows it until the specified deadline. Returns+///         true if successful, otherwise false.+///     bool try_enqueue_for(const T&, duration&);+///     bool try_enqueue_for(T&&, duration&);+///         Tries to add an element to the end of the queue if+///         capacity allows until the expiration of the specified+///         duration. Returns true if successful, otherwise false.+///+///   Consumer functions:+///     void dequeue(T&);+///         Extracts an element from the front of the queue. Waits+///         until an element is available if necessary.+///     bool try_dequeue(T&);+///         Tries to extracts an element from the front of the queue+///         if available. Returns true if successful, otherwise false.+///     bool try_dequeue_until(T&, time_point& deadline);+///         Tries to extracts an element from the front of the queue+///         if available until the specified daedline. Returns true+///         if successful. Otherwise Returns false.+///     bool try_dequeue_for(T&, duration&);+///         Tries to extracts an element from the front of the queue+///         if available until the expiration of the specified+///         duration.  Returns true if successful. Otherwise Returns+///         false.+///+///   Secondary functions:+///     void reset_capacity(size_t capacity);+///        Changes the capacity of the queue. Does not affect the+///        current contents of the queue. Guaranteed only to affect+///        subsequent enqueue operations. May or may not affect+///        concurrent operations. Capacity must be at least 1000.+///     Weight weight();+///        Returns an estimate of the total weight of the elements in+///        the queue.+///     size_t size();+///         Returns an estimate of the total number of elements.+///     bool empty();+///         Returns true only if the queue was empty during the call.+///     Note: weight(), size(), and empty() are guaranteed to be+///     accurate only if there are no concurrent changes to the queue.+///+/// Usage example with default weight:+/// @code+///   /* DMPSC, doesn't block, 1024 int elements per segment */+///   DMPSCQueue<int, false, 10> q(100000);+///   ASSERT_TRUE(q.empty());+///   ASSERT_EQ(q.size(), 0);+///   q.enqueue(1));+///   ASSERT_TRUE(q.try_enqueue(2));+///   ASSERT_TRUE(q.try_enqueue_until(3, deadline));+///   ASSERT_TRUE(q.try_enqueue(4, duration));+///   // ... enqueue more elements until capacity is full+///   // See above comments about imprecise capacity guarantees+///   ASSERT_FALSE(q.try_enqueue(100001)); // can't enqueue but can't wait+///   size_t sz = q.size();+///   ASSERT_GE(sz, 100000);+///   q.reset_capacity(1000000000); // set huge capacity+///   ASSERT_TRUE(q.try_enqueue(100001)); // now enqueue succeeds+///   q.reset_capacity(100000); // set capacity back to 100,000+///   ASSERT_FALSE(q.try_enqueue(100002));+///   ASSERT_EQ(q.size(), sz + 1);+///   int v;+///   q.dequeue(v);+///   ASSERT_EQ(v, 1);+///   ASSERT_TRUE(q.try_dequeue(v));+///   ASSERT_EQ(v, 2);+///   ASSERT_TRUE(q.try_dequeue_until(v, deadline));+///   ASSERT_EQ(v, 3);+///   ASSERT_TRUE(q.try_dequeue_for(v, duration));+///   ASSERT_EQ(v, 4);+///   ASSERT_EQ(q.size(), sz - 3);+/// @endcode+///+/// Usage example with custom weights:+/// @code+///   struct CustomWeightFn {+///     uint64_t operator()(int val) { return val / 100; }+///   };+///   DMPMCQueue<int, false, 10, CustomWeightFn> q(20);+///   ASSERT_TRUE(q.empty());+///   q.enqueue(100);+///   ASSERT_TRUE(q.try_enqueue(200));+///   ASSERT_TRUE(q.try_enqueue_until(500, now() + seconds(1)));+///   ASSERT_EQ(q.size(), 3);+///   ASSERT_EQ(q.weight(), 8);+///   ASSERT_FALSE(q.try_enqueue_for(1700, microseconds(1)));+///   q.reset_capacity(1000000); // set capacity to 1000000 instead of 20+///   ASSERT_TRUE(q.try_enqueue_for(1700, microseconds(1)));+///   q.reset_capacity(20); // set capacity to 20 again+///   ASSERT_FALSE(q.try_enqueue(100));+///   ASSERT_EQ(q.size(), 4);+///   ASSERT_EQ(q.weight(), 25);+///   int v;+///   q.dequeue(v);+///   ASSERT_EQ(v, 100);+///   ASSERT_TRUE(q.try_dequeue(v));+///   ASSERT_EQ(v, 200);+///   ASSERT_TRUE(q.try_dequeue_until(v, now() + seconds(1)));+///   ASSERT_EQ(v, 500);+///   ASSERT_EQ(q.size(), 1);+///   ASSERT_EQ(q.weight(), 17);+/// @endcode+///+/// Design:+/// - The implementation is on top of UnboundedQueue.+/// - The main FIFO functionality is in UnboundedQueue.+///   DynamicBoundedQueue manages keeping the total queue weight+///   within the specified capacity.+/// - For the sake of scalability, the data structures are designed to+///   minimize interference between producers on one side and+///   consumers on the other.+/// - Producers add to a debit variable the weight of the added+///   element and check capacity.+/// - Consumers add to a credit variable the weight of the removed+///   element.+/// - Producers, for the sake of scalability, use fetch_add to add to+///   the debit variable and subtract if it exceeded capacity,+///   rather than using compare_exchange to avoid overshooting.+/// - Consumers, infrequently, transfer credit to a transfer variable+///   and unblock any blocked producers. The transfer variable can be+///   used by producers to decrease their debit when needed.+/// - Note that a low capacity will trigger frequent credit transfer+///   by consumers that may degrade performance. Capacity should not+///   be set too low.+/// - Transfer of credit by consumers is triggered when the amount of+///   credit reaches a threshold (1/10 of capacity).+/// - The waiting of consumers is handled in UnboundedQueue.+///   The waiting of producers is handled in this template.+/// - For a producer operation, if the difference between debit and+///   capacity (plus some slack to account for the transfer threshold)+///   does not accommodate the weight of the new element, it first+///   tries to transfer credit that may have already been made+///   available by consumers. If this is insufficient and MayBlock is+///   true, then the producer uses a futex to block until new credit+///   is transferred by a consumer.+///+/// Memory Usage:+/// - Aside from three cache lines for managing capacity, the memory+///   for queue elements is managed using UnboundedQueue and grows and+///   shrinks dynamically with the number of elements.+/// - The template parameter LgAlign can be used to reduce memory usage+///   at the cost of increased chance of false sharing.++template <typename T>+struct DefaultWeightFn {+  template <typename Arg>+  uint64_t operator()(Arg&&) const noexcept {+    return 1;+  }+};++template <+    typename T,+    bool SingleProducer,+    bool SingleConsumer,+    bool MayBlock,+    size_t LgSegmentSize = 8,+    size_t LgAlign = 7,+    typename WeightFn = DefaultWeightFn<T>,+    template <typename> class Atom = std::atomic>+class DynamicBoundedQueue {+  using Weight = uint64_t;++  enum WaitingState : uint32_t {+    NOTWAITING = 0,+    WAITING = 1,+  };++  static constexpr bool SPSC = SingleProducer && SingleConsumer;+  static constexpr size_t Align = 1u << LgAlign;++  static_assert(LgAlign < 16, "LgAlign must be < 16");++  /// Data members++  // Read mostly by producers+  alignas(Align) Atom<Weight> debit_; // written frequently only by producers+  Atom<Weight> capacity_; // written rarely by capacity resets++  // Read mostly by consumers+  alignas(Align) Atom<Weight> credit_; // written frequently only by consumers+  Atom<Weight> threshold_; // written rarely only by capacity resets++  // Normally written and read rarely by producers and consumers+  // May be read frequently by producers when capacity is full+  alignas(Align) Atom<Weight> transfer_;+  detail::Futex<Atom> waiting_;++  // Underlying unbounded queue+  UnboundedQueue<+      T,+      SingleProducer,+      SingleConsumer,+      MayBlock,+      LgSegmentSize,+      LgAlign,+      Atom>+      q_;++ public:+  using value_type = T;+  using size_type = size_t;++  /** constructor */+  explicit DynamicBoundedQueue(Weight capacity)+      : debit_(0),+        capacity_(capacity + threshold(capacity)), // capacity slack+        credit_(0),+        threshold_(threshold(capacity)),+        transfer_(0),+        waiting_(0) {}++  /** destructor */+  ~DynamicBoundedQueue() {}++  /// Enqueue functions++  /** enqueue */+  FOLLY_ALWAYS_INLINE void enqueue(const T& v) { enqueueImpl(v); }++  FOLLY_ALWAYS_INLINE void enqueue(T&& v) { enqueueImpl(std::move(v)); }++  /** try_enqueue */+  FOLLY_ALWAYS_INLINE bool try_enqueue(const T& v) { return tryEnqueueImpl(v); }++  FOLLY_ALWAYS_INLINE bool try_enqueue(T&& v) {+    return tryEnqueueImpl(std::move(v));+  }++  /** try_enqueue_until */+  template <typename Clock, typename Duration>+  FOLLY_ALWAYS_INLINE bool try_enqueue_until(+      const T& v, const std::chrono::time_point<Clock, Duration>& deadline) {+    return tryEnqueueUntilImpl(v, deadline);+  }++  template <typename Clock, typename Duration>+  FOLLY_ALWAYS_INLINE bool try_enqueue_until(+      T&& v, const std::chrono::time_point<Clock, Duration>& deadline) {+    return tryEnqueueUntilImpl(std::move(v), deadline);+  }++  /** try_enqueue_for */+  template <typename Rep, typename Period>+  FOLLY_ALWAYS_INLINE bool try_enqueue_for(+      const T& v, const std::chrono::duration<Rep, Period>& duration) {+    return tryEnqueueForImpl(v, duration);+  }++  template <typename Rep, typename Period>+  FOLLY_ALWAYS_INLINE bool try_enqueue_for(+      T&& v, const std::chrono::duration<Rep, Period>& duration) {+    return tryEnqueueForImpl(std::move(v), duration);+  }++  /// Dequeue functions++  /** dequeue */+  FOLLY_ALWAYS_INLINE void dequeue(T& elem) {+    q_.dequeue(elem);+    addCredit(WeightFn()(elem));+  }++  /** try_dequeue */+  FOLLY_ALWAYS_INLINE bool try_dequeue(T& elem) {+    if (q_.try_dequeue(elem)) {+      addCredit(WeightFn()(elem));+      return true;+    }+    return false;+  }++  FOLLY_ALWAYS_INLINE folly::Optional<T> try_dequeue() {+    auto elem = q_.try_dequeue();+    if (elem.hasValue()) {+      addCredit(WeightFn()(*elem));+    }+    return elem;+  }++  /** try_dequeue_until */+  template <typename Clock, typename Duration>+  FOLLY_ALWAYS_INLINE bool try_dequeue_until(+      T& elem, const std::chrono::time_point<Clock, Duration>& deadline) {+    if (q_.try_dequeue_until(elem, deadline)) {+      addCredit(WeightFn()(elem));+      return true;+    }+    return false;+  }++  template <typename Clock, typename Duration>+  FOLLY_ALWAYS_INLINE folly::Optional<T> try_dequeue_until(+      const std::chrono::time_point<Clock, Duration>& deadline) {+    auto elem = q_.try_dequeue_until(deadline);+    if (elem.hasValue()) {+      addCredit(WeightFn()(*elem));+    }+    return elem;+  }++  /** try_dequeue_for */+  template <typename Rep, typename Period>+  FOLLY_ALWAYS_INLINE bool try_dequeue_for(+      T& elem, const std::chrono::duration<Rep, Period>& duration) {+    if (q_.try_dequeue_for(elem, duration)) {+      addCredit(WeightFn()(elem));+      return true;+    }+    return false;+  }++  template <typename Rep, typename Period>+  FOLLY_ALWAYS_INLINE folly::Optional<T> try_dequeue_for(+      const std::chrono::duration<Rep, Period>& duration) {+    auto elem = q_.try_dequeue_for(duration);+    if (elem.hasValue()) {+      addCredit(WeightFn()(*elem));+    }+    return elem;+  }++  /// Secondary functions++  /** reset_capacity */+  void reset_capacity(Weight capacity) noexcept {+    Weight thresh = threshold(capacity);+    capacity_.store(capacity + thresh, std::memory_order_release);+    threshold_.store(thresh, std::memory_order_release);+  }++  /** weight */+  Weight weight() const noexcept {+    auto d = getDebit();+    auto c = getCredit();+    auto t = getTransfer();+    return d > (c + t) ? d - (c + t) : 0;+  }++  /** size */+  size_t size() const noexcept { return q_.size(); }++  /** empty */+  bool empty() const noexcept { return q_.empty(); }++ private:+  /// Private functions ///++  // Calculation of threshold to move credits in bulk from consumers+  // to producers+  constexpr Weight threshold(Weight capacity) const noexcept {+    return (capacity + 9) / 10;+  }++  // Functions called frequently by producers++  template <typename Arg>+  FOLLY_ALWAYS_INLINE void enqueueImpl(Arg&& v) {+    tryEnqueueUntilImpl(+        std::forward<Arg>(v), std::chrono::steady_clock::time_point::max());+  }++  template <typename Arg>+  FOLLY_ALWAYS_INLINE bool tryEnqueueImpl(Arg&& v) {+    return tryEnqueueUntilImpl(+        std::forward<Arg>(v), std::chrono::steady_clock::time_point::min());+  }++  template <typename Clock, typename Duration, typename Arg>+  FOLLY_ALWAYS_INLINE bool tryEnqueueUntilImpl(+      Arg&& v, const std::chrono::time_point<Clock, Duration>& deadline) {+    Weight weight = WeightFn()(std::forward<Arg>(v));+    if (FOLLY_LIKELY(tryAddDebit(weight))) {+      q_.enqueue(std::forward<Arg>(v));+      return true;+    }+    return tryEnqueueUntilSlow(std::forward<Arg>(v), deadline);+  }++  template <typename Rep, typename Period, typename Arg>+  FOLLY_ALWAYS_INLINE bool tryEnqueueForImpl(+      Arg&& v, const std::chrono::duration<Rep, Period>& duration) {+    if (FOLLY_LIKELY(tryEnqueueImpl(std::forward<Arg>(v)))) {+      return true;+    }+    auto deadline = std::chrono::steady_clock::now() + duration;+    return tryEnqueueUntilSlow(std::forward<Arg>(v), deadline);+  }++  FOLLY_ALWAYS_INLINE bool tryAddDebit(Weight weight) noexcept {+    Weight capacity = getCapacity();+    Weight before = fetchAddDebit(weight);+    if (FOLLY_LIKELY(before + weight <= capacity)) {+      return true;+    } else {+      subDebit(weight);+      return false;+    }+  }++  FOLLY_ALWAYS_INLINE Weight getCapacity() const noexcept {+    return capacity_.load(std::memory_order_acquire);+  }++  FOLLY_ALWAYS_INLINE Weight fetchAddDebit(Weight weight) noexcept {+    Weight before;+    if (SingleProducer) {+      before = getDebit();+      debit_.store(before + weight, std::memory_order_relaxed);+    } else {+      before = debit_.fetch_add(weight, std::memory_order_acq_rel);+    }+    return before;+  }++  FOLLY_ALWAYS_INLINE Weight getDebit() const noexcept {+    return debit_.load(std::memory_order_acquire);+  }++  // Functions called frequently by consumers++  FOLLY_ALWAYS_INLINE void addCredit(Weight weight) noexcept {+    Weight before = fetchAddCredit(weight);+    Weight thresh = getThreshold();+    if (before + weight >= thresh && before < thresh) {+      transferCredit();+    }+  }++  FOLLY_ALWAYS_INLINE Weight fetchAddCredit(Weight weight) noexcept {+    Weight before;+    if (SingleConsumer) {+      before = getCredit();+      credit_.store(before + weight, std::memory_order_relaxed);+    } else {+      before = credit_.fetch_add(weight, std::memory_order_acq_rel);+    }+    return before;+  }++  FOLLY_ALWAYS_INLINE Weight getCredit() const noexcept {+    return credit_.load(std::memory_order_acquire);+  }++  FOLLY_ALWAYS_INLINE Weight getThreshold() const noexcept {+    return threshold_.load(std::memory_order_acquire);+  }++  /** Functions called infrequently by producers */++  void subDebit(Weight weight) noexcept {+    Weight before;+    if (SingleProducer) {+      before = getDebit();+      debit_.store(before - weight, std::memory_order_relaxed);+    } else {+      before = debit_.fetch_sub(weight, std::memory_order_acq_rel);+    }+    DCHECK_GE(before, weight);+  }++  template <typename Clock, typename Duration, typename Arg>+  bool tryEnqueueUntilSlow(+      Arg&& v, const std::chrono::time_point<Clock, Duration>& deadline) {+    Weight weight = WeightFn()(std::forward<Arg>(v));+    if (canEnqueue(deadline, weight)) {+      q_.enqueue(std::forward<Arg>(v));+      return true;+    } else {+      return false;+    }+  }++  template <typename Clock, typename Duration>+  bool canEnqueue(+      const std::chrono::time_point<Clock, Duration>& deadline,+      Weight weight) noexcept {+    Weight capacity = getCapacity();+    while (true) {+      tryReduceDebit();+      Weight debit = getDebit();+      if ((debit + weight <= capacity) && tryAddDebit(weight)) {+        return true;+      }+      if (deadline < Clock::time_point::max() && Clock::now() >= deadline) {+        return false;+      }+      if (MayBlock) {+        if (canBlock(weight, capacity)) {+          detail::futexWaitUntil(&waiting_, WAITING, deadline);+        }+      } else {+        asm_volatile_pause();+      }+    }+  }++  bool canBlock(Weight weight, Weight capacity) noexcept {+    waiting_.store(WAITING, std::memory_order_relaxed);+    std::atomic_thread_fence(std::memory_order_seq_cst);+    tryReduceDebit();+    Weight debit = getDebit();+    return debit + weight > capacity;+  }++  bool tryReduceDebit() noexcept {+    Weight w = takeTransfer();+    if (w > 0) {+      subDebit(w);+    }+    return w > 0;+  }++  Weight takeTransfer() noexcept {+    Weight w = getTransfer();+    if (w > 0) {+      w = transfer_.exchange(0, std::memory_order_acq_rel);+    }+    return w;+  }++  Weight getTransfer() const noexcept {+    return transfer_.load(std::memory_order_acquire);+  }++  /** Functions called infrequently by consumers */++  void transferCredit() noexcept {+    Weight credit = takeCredit();+    transfer_.fetch_add(credit, std::memory_order_acq_rel);+    if (MayBlock) {+      std::atomic_thread_fence(std::memory_order_seq_cst);+      waiting_.store(NOTWAITING, std::memory_order_relaxed);+      detail::futexWake(&waiting_);+    }+  }++  Weight takeCredit() noexcept {+    Weight credit;+    if (SingleConsumer) {+      credit = credit_.load(std::memory_order_relaxed);+      credit_.store(0, std::memory_order_relaxed);+    } else {+      credit = credit_.exchange(0, std::memory_order_acq_rel);+    }+    return credit;+  }++}; // DynamicBoundedQueue++/// Aliases++/** DSPSCQueue */+template <+    typename T,+    bool MayBlock,+    size_t LgSegmentSize = 8,+    size_t LgAlign = 7,+    typename WeightFn = DefaultWeightFn<T>,+    template <typename> class Atom = std::atomic>+using DSPSCQueue = DynamicBoundedQueue<+    T,+    true,+    true,+    MayBlock,+    LgSegmentSize,+    LgAlign,+    WeightFn,+    Atom>;++/** DMPSCQueue */+template <+    typename T,+    bool MayBlock,+    size_t LgSegmentSize = 8,+    size_t LgAlign = 7,+    typename WeightFn = DefaultWeightFn<T>,+    template <typename> class Atom = std::atomic>+using DMPSCQueue = DynamicBoundedQueue<+    T,+    false,+    true,+    MayBlock,+    LgSegmentSize,+    LgAlign,+    WeightFn,+    Atom>;++/** DSPMCQueue */+template <+    typename T,+    bool MayBlock,+    size_t LgSegmentSize = 8,+    size_t LgAlign = 7,+    typename WeightFn = DefaultWeightFn<T>,+    template <typename> class Atom = std::atomic>+using DSPMCQueue = DynamicBoundedQueue<+    T,+    true,+    false,+    MayBlock,+    LgSegmentSize,+    LgAlign,+    WeightFn,+    Atom>;++/** DMPMCQueue */+template <+    typename T,+    bool MayBlock,+    size_t LgSegmentSize = 8,+    size_t LgAlign = 7,+    typename WeightFn = DefaultWeightFn<T>,+    template <typename> class Atom = std::atomic>+using DMPMCQueue = DynamicBoundedQueue<+    T,+    false,+    false,+    MayBlock,+    LgSegmentSize,+    LgAlign,+    WeightFn,+    Atom>;++} // namespace folly
+ folly/folly/concurrency/PriorityUnboundedQueueSet.h view
@@ -0,0 +1,207 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <atomic>+#include <vector>++#include <folly/Memory.h>+#include <folly/concurrency/UnboundedQueue.h>+#include <folly/lang/Align.h>++namespace folly {++/// PriorityUnboundedQueueSet+///+/// A set of per-priority queues, and an interface for accessing them.+///+/// Functions:+///   Consumer operations:+///     bool try_dequeue(T&);+///     Optional<T> try_dequeue();+///       Tries to extract an element from the front of the least-priority+///       backing queue which has an element, if any.+///     T const* try_peek();+///       Returns a pointer to the element at the front of the least-priority+///       backing queue which has an element, if any. Only allowed when+///       SingleConsumer is true.+///     Note:+///       Queues at lower priority are tried before queues at higher priority.+///+///   Secondary functions:+///     queue& at_priority(size_t);+///     queue const& at_priority(size_t) const;+///       Returns a reference to the owned queue at the given priority.+///     size_t size() const;+///       Returns an estimate of the total size of the owned queues.+///     bool empty() const;+///       Returns true only if all of the owned queues were empty during the+///       call.+///     Note: size() and empty() are guaranteed to be accurate only if the+///       owned queues are not changed concurrently.+template <+    typename T,+    bool SingleProducer,+    bool SingleConsumer,+    bool MayBlock,+    size_t LgSegmentSize = 8,+    size_t LgAlign = constexpr_log2(hardware_destructive_interference_size),+    template <typename> class Atom = std::atomic>+class PriorityUnboundedQueueSet {+ public:+  using queue = UnboundedQueue<+      T,+      SingleProducer,+      SingleConsumer,+      MayBlock,+      LgSegmentSize,+      LgAlign,+      Atom>;++  explicit PriorityUnboundedQueueSet(size_t priorities) : queues_(priorities) {}++  PriorityUnboundedQueueSet(PriorityUnboundedQueueSet const&) = delete;+  PriorityUnboundedQueueSet(PriorityUnboundedQueueSet&&) = delete;+  PriorityUnboundedQueueSet& operator=(PriorityUnboundedQueueSet const&) =+      delete;+  PriorityUnboundedQueueSet& operator=(PriorityUnboundedQueueSet&&) = delete;++  queue& at_priority(size_t priority) { return queues_.at(priority); }++  queue const& at_priority(size_t priority) const {+    return queues_.at(priority);+  }++  bool try_dequeue(T& item) noexcept {+    for (auto& q : queues_) {+      if (q.try_dequeue(item)) {+        return true;+      }+    }+    return false;+  }++  Optional<T> try_dequeue() noexcept {+    for (auto& q : queues_) {+      if (auto item = q.try_dequeue()) {+        return item;+      }+    }+    return none;+  }++  T const* try_peek() noexcept {+    DCHECK(SingleConsumer);+    for (auto& q : queues_) {+      if (auto ptr = q.try_peek()) {+        return ptr;+      }+    }+    return nullptr;+  }++  size_t size() const noexcept {+    size_t size = 0;+    for (auto& q : queues_) {+      size += q.size();+    }+    return size;+  }++  bool empty() const noexcept {+    for (auto& q : queues_) {+      if (!q.empty()) {+        return false;+      }+    }+    return true;+  }++  size_t priorities() const noexcept { return queues_.size(); }++ private:+  //  queue_alloc custom allocator is necessary until C++17+  //    http://open-std.org/JTC1/SC22/WG21/docs/papers/2012/n3396.htm+  //    https://gcc.gnu.org/bugzilla/show_bug.cgi?id=65122+  //    https://bugs.llvm.org/show_bug.cgi?id=22634+  using queue_alloc = AlignedSysAllocator<queue, FixedAlign<alignof(queue)>>;+  std::vector<queue, queue_alloc> queues_;+}; // PriorityUnboundedQueueSet++/* Aliases */++template <+    typename T,+    bool MayBlock,+    size_t LgSegmentSize = 8,+    size_t LgAlign = constexpr_log2(hardware_destructive_interference_size),+    template <typename> class Atom = std::atomic>+using PriorityUSPSCQueueSet = PriorityUnboundedQueueSet<+    T,+    true,+    true,+    MayBlock,+    LgSegmentSize,+    LgAlign,+    Atom>;++template <+    typename T,+    bool MayBlock,+    size_t LgSegmentSize = 8,+    size_t LgAlign = constexpr_log2(hardware_destructive_interference_size),+    template <typename> class Atom = std::atomic>+using PriorityUMPSCQueueSet = PriorityUnboundedQueueSet<+    T,+    false,+    true,+    MayBlock,+    LgSegmentSize,+    LgAlign,+    Atom>;++template <+    typename T,+    bool MayBlock,+    size_t LgSegmentSize = 8,+    size_t LgAlign = constexpr_log2(hardware_destructive_interference_size),+    template <typename> class Atom = std::atomic>+using PriorityUSPMCQueueSet = PriorityUnboundedQueueSet<+    T,+    true,+    false,+    MayBlock,+    LgSegmentSize,+    LgAlign,+    Atom>;++template <+    typename T,+    bool MayBlock,+    size_t LgSegmentSize = 8,+    size_t LgAlign = constexpr_log2(hardware_destructive_interference_size),+    template <typename> class Atom = std::atomic>+using PriorityUMPMCQueueSet = PriorityUnboundedQueueSet<+    T,+    false,+    false,+    MayBlock,+    LgSegmentSize,+    LgAlign,+    Atom>;++} // namespace folly
+ folly/folly/concurrency/ProcessLocalUniqueId.cpp view
@@ -0,0 +1,46 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#include <folly/Likely.h>+#include <folly/concurrency/ProcessLocalUniqueId.h>+#include <folly/synchronization/RelaxedAtomic.h>++#include <atomic>++namespace folly {++uint64_t processLocalUniqueId() {+  FOLLY_CONSTINIT static relaxed_atomic<uint64_t> nextEpoch{0};+  // Id format is <epoch: 48 bits> <counter: 16 bits>.+  // Ephemeral threads, if any, can waste a whole epoch, so we keep epochs+  // relatively small, but large enough to amortize the atomic epoch increment.+  constexpr int kCounterBits = 16;+  constexpr uint64_t kCounterMask = (uint64_t(1) << kCounterBits) - 1;+  thread_local uint64_t next{0};++  // If first call in thread, or counter wrapped around, start new epoch.+  if (FOLLY_UNLIKELY((next & kCounterMask) == 0)) {+    next = nextEpoch++ << kCounterBits;+    // Skip 0 as per contract.+    if (FOLLY_UNLIKELY(next == 0)) {+      ++next;+    }+  }++  return next++;+}++} // namespace folly
+ folly/folly/concurrency/ProcessLocalUniqueId.h view
@@ -0,0 +1,38 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <cstdint>++namespace folly {++/**+ * Generates a 64-bit id that is unique within the process. The returned ids+ * should not be persisted or passed to other processes, and there are no+ * ordering guarantees.+ *+ * It is guaranteed that 0 is never returned, hence 0 can be used as a sentinel+ * value, similarly to nullptr.+ *+ * The function is thread-safe.+ *+ * The uniqueness guarantee can be broken if enough ids are generated, but even+ * in the most pessimistic scenario it would take a few hundred years.+ */+uint64_t processLocalUniqueId();++} // namespace folly
+ folly/folly/concurrency/SingletonRelaxedCounter.h view
@@ -0,0 +1,327 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <array>+#include <atomic>+#include <type_traits>+#include <unordered_map>++#include <folly/Likely.h>+#include <folly/Portability.h>+#include <folly/Synchronized.h>+#include <folly/Utility.h>+#include <folly/detail/StaticSingletonManager.h>+#include <folly/detail/thread_local_globals.h>+#include <folly/lang/SafeAssert.h>+#include <folly/synchronization/AtomicRef.h>++namespace folly {++namespace detail {++//  SingletonRelaxedCounterBase+//+//  Extracts tag-independent functionality from SingletonRelaxedCounter below+//  to avoid the monomorphization the compiler would otherwise perform.+//+//  Tricks:+//  * Use constexpr vtables as the polymorphization mechanism.+//  * Use double-noinline outer-only-noexcept definitions to shrink code size in+//    inline or monomorphized slow paths.+template <typename Int>+class SingletonRelaxedCounterBase {+ protected:+  using Signed = std::make_signed_t<Int>;+  using Counter = Signed; // should be atomic but clang generates worse code++  struct CounterAndCache {+    Counter counter; // valid during LocalLifetime object lifetime+    Counter* cache; // points to counter when counter is valid+  };+  static_assert(std::is_trivial_v<CounterAndCache>);++  struct CounterRefAndLocal {+    Counter* counter; // refers either to local counter or to global counter+    bool local; // if true, definitely local; if false, could be global+  };++  struct LocalLifetime;++  //  Global+  //+  //  Tracks all of the per-thread/per-dso counters and lifetimes and maintains+  //  a global fallback counter.+  struct Global {+    struct Tracking {+      std::unordered_map<LocalLifetime*, CounterAndCache*> lifetimes;+    };++    Counter fallback; // used instead of local during thread destruction+    folly::Synchronized<Tracking> tracking;+  };++  using GetGlobal = Global&();+  using GetLocal = CounterAndCache&();+  using GetLifetime = LocalLifetime&();++  struct Arg {+    GetGlobal& global;+    GetLocal& local;+    GetLifetime& lifetime;+  };++  //  LocalLifetime+  //+  //  Manages local().cache, global().tracking, and moving outstanding counts+  //  from local().counter to global().counter during thread destruction and dso+  //  unload.+  //+  //  The index map is within Global to reduce per-thread overhead for threads+  //  which do not participate in counter mutations, rather than being a member+  //  field of LocalLifetime. This comes at the cost of the slow path always+  //  acquiring a unique lock on the global mutex.+  struct LocalLifetime {+    FOLLY_NOINLINE void destroy(GetGlobal& get_global) noexcept {+      destroy_(get_global);+    }+    FOLLY_NOINLINE void destroy_(GetGlobal& get_global) {+      auto& global = get_global();+      auto const tracking = global.tracking.wlock();+      auto& entry = tracking->lifetimes[this];+      FOLLY_SAFE_CHECK(entry);+      auto entry_counter = atomic_ref(entry->counter);+      auto const current = entry_counter.load(std::memory_order_relaxed);+      atomic_ref(global.fallback).fetch_add(current, std::memory_order_relaxed);+      entry_counter.store(Signed(0), std::memory_order_relaxed);+      entry->cache = nullptr;+      tracking->lifetimes.erase(this);+    }++    FOLLY_NOINLINE void track(Global& global, CounterAndCache& state) noexcept {+      track_(global, state);+    }+    FOLLY_NOINLINE void track_(Global& global, CounterAndCache& state) {+      state.cache = &state.counter;+      auto const tracking = global.tracking.wlock();+      auto& entry = tracking->lifetimes[this];+      FOLLY_SAFE_CHECK(!entry || &state == entry);+      entry = &state;+    }+  };++  FOLLY_NOINLINE static Int aggregate(GetGlobal& get_global) noexcept {+    return aggregate_(get_global);+  }+  FOLLY_NOINLINE static Int aggregate_(GetGlobal& get_global) {+    auto& global = get_global();+    auto count = atomic_ref(global.fallback).load(std::memory_order_relaxed);+    auto const tracking = global.tracking.rlock();+    for (auto const& [_, entry] : tracking->lifetimes) {+      FOLLY_SAFE_CHECK(entry);+      count += atomic_ref(entry->counter).load(std::memory_order_relaxed);+    }+    return std::is_unsigned<Int>::value+        ? to_unsigned(std::max(Signed(0), count))+        : count;+  }++  FOLLY_ERASE static void mutate(Signed v, CounterRefAndLocal cl) {+    auto c = atomic_ref(*cl.counter);+    if (cl.local) {+      //  splitting load/store on the local counter is faster than fetch-and-add+      c.store(c.load(std::memory_order_relaxed) + v, std::memory_order_relaxed);+    } else {+      //  but is not allowed on the global counter because mutations may be lost+      c.fetch_add(v, std::memory_order_relaxed);+    }+  }++  FOLLY_NOINLINE static void mutate_slow(Signed v, Arg const& arg) noexcept {+    mutate(v, counter(arg));+  }++  FOLLY_NOINLINE static Counter& counter_slow(Arg const& arg) noexcept {+    auto& global = arg.global();+    if (thread_is_dying()) {+      return global.fallback;+    }+    auto& state = arg.local();+    arg.lifetime().track(global, state); // idempotent+    auto const cache = state.cache;+    return FOLLY_LIKELY(!!cache) ? *cache : global.fallback;+  }++  FOLLY_ERASE static CounterRefAndLocal counter(Arg const& arg) {+    auto& state = arg.local();+    auto const cache = state.cache; // a copy! null before/after LocalLifetime+    auto const counter = FOLLY_LIKELY(!!cache) ? cache : &counter_slow(arg);+    //  cache is a stale nullptr after the first call to counter_slow; this is+    //  intentional for the side-effect of shrinking the inline fast path+    return CounterRefAndLocal{counter, !!cache};+  }+};++} // namespace detail++//  SingletonRelaxedCounter+//+//  A singleton-per-tag relaxed counter. Optimized for increment/decrement+//  runtime performance under contention and inlined fast path code size.+//+//  The cost of computing the value of the counter is linear in the number of+//  threads which perform increments/decrements, and computing the value of the+//  counter is exclusive with thread exit and dlclose. The result of this+//  computation is not a point-in-time snapshot of increments and decrements+//  summed, but is an approximation which may exclude any subset of increments+//  and decrements that do not happen before the start of the computation.+//+//  Templated over the integral types. When templated over an unsigned integral+//  type, it is assumed that decrements do not exceed increments, and if within+//  computation of the value of the counter more decrements are observed to+//  exceed increments then the excess decrements are ignored. This avoids the+//  scenario of incrementing and decrementing once each in different threads,+//  and concurrently observing a computed value of the counter of 2^64 - 1.+//+//  Templated over the tag types. Each unique pair of integral type and tag type+//  is a different counter.+//+//  Implementation:+//  Uses a thread-local counter when possible to avoid contention, and a global+//  counter as a fallback. The total count at any given time is computed by+//  summing over the global counter plus all of the thread-local counters; since+//  the total sum is not a snapshot of the value at any given point in time, it+//  is a relaxed sum; when the system quiesces (i.e., when no concurrent+//  increments or decrements are happening and no threads are going through+//  thread exit phase), the sum is exact.+//+//  Most of the implementation is in SingletonRelaxedCounterBase to avoid excess+//  monomorphization.+template <typename Int, typename Tag>+class SingletonRelaxedCounter+    : private detail::SingletonRelaxedCounterBase<Int> {+ public:+  static void add(Int value) { mutate(+to_signed(value)); }+  static void sub(Int value) { mutate(-to_signed(value)); }+  static Int count() { return aggregate(global); }++ private:+  using Base = detail::SingletonRelaxedCounterBase<Int>;+  using Base::aggregate;+  using Base::mutate;+  using Base::mutate_slow;+  using typename Base::Arg;+  using typename Base::CounterAndCache;+  using typename Base::GetGlobal;+  using typename Base::Global;+  using typename Base::LocalLifetime;+  using typename Base::Signed;++  struct MonoLocalLifetime : Base::LocalLifetime {+    ~MonoLocalLifetime() noexcept(false) {+      Base::LocalLifetime::destroy(global);+    }+  };++  //  It is an invariant that in a single call to mutate which calls mutate_slow+  //  the thread_local local and the thread_local lifetime that are used are in+  //  the same DSO as each other.+  //+  //  The following functions are all [[gnu::visibility("hidden")]] in order to+  //  ensure this invariant.++  FOLLY_ERASE_NOINLINE static void mutate_slow(Signed v) noexcept {+    static constexpr Arg arg{global, local, lifetime};+    mutate_slow(v, arg);+  }++  FOLLY_ERASE static void mutate(Signed v, void (&slow)(Signed) = mutate_slow) {+    auto const cache = local().cache; // a copy! null before/after LocalLifetime+    //  fun-ref to trick compiler into emitting a tail call+    FOLLY_LIKELY(!!cache) ? mutate(v, {cache, true}) : slow(v);+  }++  static constexpr GetGlobal& global = folly::detail::createGlobal<Global, Tag>;++  FOLLY_ERASE static CounterAndCache& local() {+    //  this is a member function local instead of a class member because of+    //  https://gcc.gnu.org/bugzilla/show_bug.cgi?id=66944+    static thread_local CounterAndCache instance;+    return instance;+  }++  FOLLY_ERASE static LocalLifetime& lifetime() {+    static thread_local MonoLocalLifetime lifetime;+    return lifetime;+  }+};++template <typename Counted>+class SingletonRelaxedCountableAccess;++//  SingletonRelaxedCountable+//+//  A CRTP base class for making the instances of a type within a process be+//  globally counted. The running counter is a relaxed counter.+//+//  To avoid adding any new names from the base class to the counted type, the+//  count is exposed via a separate type SingletonRelaxedCountableAccess.+//+//  This type is a convenience interface around SingletonRelaxedCounter.+template <typename Counted>+class SingletonRelaxedCountable {+ public:+  SingletonRelaxedCountable() noexcept {+    static_assert(+        std::is_base_of<SingletonRelaxedCountable, Counted>::value, "non-crtp");+    Counter::add(1);+  }+  ~SingletonRelaxedCountable() noexcept {+    static_assert(+        std::is_base_of<SingletonRelaxedCountable, Counted>::value, "non-crtp");+    Counter::sub(1);+  }++  SingletonRelaxedCountable(const SingletonRelaxedCountable&) noexcept+      : SingletonRelaxedCountable() {}+  SingletonRelaxedCountable(SingletonRelaxedCountable&&) noexcept+      : SingletonRelaxedCountable() {}++  SingletonRelaxedCountable& operator=(const SingletonRelaxedCountable&) =+      default;+  SingletonRelaxedCountable& operator=(SingletonRelaxedCountable&&) = default;++ private:+  friend class SingletonRelaxedCountableAccess<Counted>;++  struct Tag;+  using Counter = SingletonRelaxedCounter<size_t, Tag>;+};++//  SingletonRelaxedCountableAccess+//+//  Provides access to the running count of instances of a type using the CRTP+//  base class SingletonRelaxedCountable.+template <typename Counted>+class SingletonRelaxedCountableAccess {+ public:+  static size_t count() {+    return SingletonRelaxedCountable<Counted>::Counter::count();+  }+};++} // namespace folly
+ folly/folly/concurrency/ThreadCachedSynchronized.h view
@@ -0,0 +1,227 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <cstdint>+#include <memory>+#include <type_traits>+#include <utility>++#include <folly/SharedMutex.h>+#include <folly/ThreadLocal.h>+#include <folly/Utility.h>+#include <folly/lang/Access.h>+#include <folly/synchronization/Lock.h>+#include <folly/synchronization/RelaxedAtomic.h>++namespace folly {++//  thread_cached_synchronized+//+//  Roughly equivalent to Synchronized, but with a per-thread cache for+//  acceleration.+//+//  Use in hot code when Synchronized alone, with its shared lock and unlock,+//  would be too costly.+//+//  Avoid when acceleration is marginal since per-thread caches are expensive.+//+//  Example:+//+//    struct writer_and_readers {+//      folly::thread_cached_synchronized<std::shared_ptr<data>> obj_;+//      std::jthread background_writer_{std::bind(loop_update, this)};+//+//      std::shared_ptr<data> get_recent_data_fast() { return obj; }+//+//      data fetch_recent_data();+//      bool needs_data_and_not_signaled_done();+//      void loop_update() {+//        while (needs_data_and_not_signaled_done()) {+//          obj.exchange(folly::copy_to_shared_ptr(fetch_recent_data()));+//        }+//      }+//    };+//+//  Note:+//    A singleton variation of this with SingletonThreadLocal would remove one+//    of the branches when looking up the per-thread cache but would introduce+//    a new branch when looking up the global version.+template <typename T, typename Mutex = SharedMutex>+class thread_cached_synchronized {+  static_assert(std::is_same<std::decay_t<T>, T>::value, "not decayed");+  static_assert(std::is_copy_constructible<T>::value, "not copy-constructible");++ public:+  using value_type = T;++ private:+  using version_type = std::uint64_t; // 64 bits will not overflow++  struct truth_state {+    relaxed_atomic<version_type> version{0}; // tiny optimization if first field+    Mutex mutex{}; // protects value and sometimes version+    value_type value;++    template <typename... A>+    truth_state(A&&... a) noexcept(+        std::is_nothrow_constructible<Mutex>{} &&+        std::is_nothrow_constructible<value_type, A...>{})+        : value{static_cast<A&&>(a)...} {}+  };++  struct cache_state {+    version_type version{0};+    value_type value;++    template <typename... A>+    cache_state(A&&... a) //+        noexcept(std::is_nothrow_constructible<value_type, A...>{})+        : value{static_cast<A&&>(a)...} {}+  };+  using tlp_cache_state = ThreadLocalPtr<cache_state>;++  template <typename... A>+  static constexpr bool nx =+      noexcept(truth_state{std::in_place, FOLLY_DECLVAL(A)...});++  template <bool C>+  using if_ = std::enable_if_t<C, int>;++  using swap_fn = access::swap_fn;++ public:+  template <typename A = value_type, if_<std::is_constructible<A>{}> = 0>+  thread_cached_synchronized() noexcept(nx<>) : truth_{std::in_place} {}+  explicit thread_cached_synchronized(value_type const& a) //+      noexcept(nx<value_type const&>)+      : truth_{a} {}+  explicit thread_cached_synchronized(value_type&& a) noexcept(nx<value_type&&>)+      : truth_{static_cast<value_type&&>(a)} {}+  template <typename A, if_<std::is_constructible<value_type, A>{}> = 0>+  explicit thread_cached_synchronized(A&& a) noexcept(nx<A&&>)+      : truth_{static_cast<A&&>(a)} {}+  template <typename... A, if_<std::is_constructible<value_type, A...>{}> = 0>+  explicit thread_cached_synchronized(std::in_place_t, A&&... a) noexcept(+      nx<A&&...>)+      : truth_{static_cast<A&&>(a)...} {}++  template <typename A, if_<std::is_assignable<value_type&, A>{}> = 0>+  thread_cached_synchronized& operator=(A&& a) noexcept(false) {+    store(static_cast<A&&>(a));+    return *this;+  }++  template <typename A = value_type>+  void store(A&& a = A{}) {+    mutate([&](auto& value) { value = static_cast<A&&>(a); });+  }++  template <typename A = value_type>+  value_type exchange(A&& a = A{}) {+    return mutate([&](auto& value) { //+      return std::exchange(value, static_cast<A&&>(a));+    });+  }++  template <typename A>+  bool compare_exchange(value_type& expected, A&& desired) {+    return mutate_cx(expected, static_cast<A&&>(desired));+  }++  template <typename A>+  void swap(A& that) noexcept(false) {+    mutate([&](auto& value) { access::swap(value, that); });+  }++  template <typename A, if_<is_invocable_v<swap_fn, value_type&, A&>> = 0>+  friend void swap(thread_cached_synchronized& self, A& that) noexcept(false) {+    self.swap(that);+  }++  value_type const& operator*() const { return ref(); }+  value_type const* operator->() const { return std::addressof(ref()); }+  value_type load() const { return std::as_const(ref()); }+  /* implicit */ operator value_type() const { return load(); }++ private:+  void invalidate_caches() {+    truth_.version = truth_.version + 1; // intentionally not +=+  }++  // TODO: past C++17, just use if-constexpr in mutate()+  template <+      typename F,+      typename R = invoke_result_t<F, value_type&>,+      if_<std::is_void<R>{}> = 0>+  R mutate_locked(F f) {+    f(truth_.value); // value first: mutation may throw+    invalidate_caches();+  }+  template <+      typename F,+      typename R = invoke_result_t<F, value_type&>,+      if_<!std::is_void<R>{}> = 0>+  R mutate_locked(F f) {+    decltype(auto) ret = f(truth_.value); // value first: mutation may throw+    invalidate_caches();+    return ret;+  }+  template <typename F, typename R = invoke_result_t<F, value_type&>>+  R mutate(F f) {+    unique_lock<Mutex> lock{truth_.mutex};+    return mutate_locked(f);+  }++  template <typename A>+  bool mutate_cx(value_type& expected, A&& desired) {+    unique_lock<Mutex> lock{truth_.mutex};+    auto const eq = std::as_const(truth_.value) == std::as_const(expected);+    if (eq) {+      truth_.value = // value first: mutation may throw+          static_cast<A&&>(desired);+      invalidate_caches();+    } else {+      expected = std::as_const(truth_.value);+    }+    return eq;+  }++  FOLLY_ERASE value_type& ref() const {+    auto const cache = cache_.get();+    auto const unexpired = cache && cache->version == truth_.version;+    return FOLLY_LIKELY(unexpired) ? cache->value : get_slow();+  }++  FOLLY_NOINLINE value_type& get_slow() const {+    hybrid_lock<Mutex> lock{truth_.mutex};+    auto cache = cache_.get();+    if (cache == nullptr) {+      cache = new cache_state{truth_.value}; // value first: copy may throw+      cache_.reset(cache);+    } else {+      cache->value = truth_.value; // value first: copy may throw+    }+    cache->version = truth_.version;+    return cache->value;+  }++  mutable truth_state truth_;+  mutable tlp_cache_state cache_;+};++} // namespace folly
+ folly/folly/concurrency/UnboundedQueue.h view
@@ -0,0 +1,892 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <atomic>+#include <chrono>+#include <memory>++#include <glog/logging.h>++#include <folly/ConstexprMath.h>+#include <folly/Optional.h>+#include <folly/Traits.h>+#include <folly/concurrency/CacheLocality.h>+#include <folly/lang/Align.h>+#include <folly/synchronization/Hazptr.h>+#include <folly/synchronization/SaturatingSemaphore.h>+#include <folly/synchronization/WaitOptions.h>+#include <folly/synchronization/detail/Spin.h>++namespace folly {++/// UnboundedQueue supports a variety of options for unbounded+/// dynamically expanding an shrinking queues, including variations of:+/// - Single vs. multiple producers+/// - Single vs. multiple consumers+/// - Blocking vs. spin-waiting+/// - Non-waiting, timed, and waiting consumer operations.+/// Producer operations never wait or fail (unless out-of-memory).+///+/// Template parameters:+/// - T: element type+/// - SingleProducer: true if there can be only one producer at a+///   time.+/// - SingleConsumer: true if there can be only one consumer at a+///   time.+/// - MayBlock: true if consumers may block, false if they only+///   spin. A performance tuning parameter.+/// - LgSegmentSize (default 8): Log base 2 of number of elements per+///   segment. A performance tuning parameter. See below.+/// - LgAlign (default 7): Log base 2 of alignment directive; can be+///   used to balance scalability (avoidance of false sharing) with+///   memory efficiency.+///+/// When to use UnboundedQueue:+/// - If a small bound may lead to deadlock or performance degradation+///   under bursty patterns.+/// - If there is no risk of the queue growing too much.+///+/// When not to use UnboundedQueue:+/// - If there is risk of the queue growing too much and a large bound+///   is acceptable, then use DynamicBoundedQueue.+/// - If the queue must not allocate on enqueue or it must have a+///   small bound, then use fixed-size MPMCQueue or (if non-blocking+///   SPSC) ProducerConsumerQueue.+///+/// Template Aliases:+///   USPSCQueue<T, MayBlock, LgSegmentSize, LgAlign>+///   UMPSCQueue<T, MayBlock, LgSegmentSize, LgAlign>+///   USPMCQueue<T, MayBlock, LgSegmentSize, LgAlign>+///   UMPMCQueue<T, MayBlock, LgSegmentSize, LgAlign>+///+/// Functions:+///   Producer operations never wait or fail (unless OOM)+///     void enqueue(const T&);+///     void enqueue(T&&);+///         Adds an element to the end of the queue.+///+///   Consumer operations:+///     void dequeue(T&);+///     T dequeue();+///         Extracts an element from the front of the queue. Waits+///         until an element is available if needed.+///     bool try_dequeue(T&);+///     folly::Optional<T> try_dequeue();+///         Tries to extract an element from the front of the queue+///         if available.+///     bool try_dequeue_until(T&, time_point& deadline);+///     folly::Optional<T> try_dequeue_until(time_point& deadline);+///         Tries to extract an element from the front of the queue+///         if available until the specified deadline.+///     bool try_dequeue_for(T&, duration&);+///     folly::Optional<T> try_dequeue_for(duration&);+///         Tries to extract an element from the front of the queue if+///         available until the expiration of the specified duration.+///     const T* try_peek();+///         Returns pointer to the element at the front of the queue+///         if available, or nullptr if the queue is empty. Only for+///         SPSC and MPSC.+///+///   Secondary functions:+///     size_t size();+///         Returns an estimate of the size of the queue.+///     bool empty();+///         Returns true only if the queue was empty during the call.+///     Note: size() and empty() are guaranteed to be accurate only if+///     the queue is not changed concurrently.+///+/// Usage examples:+/// @code+///   /* UMPSC, doesn't block, 1024 int elements per segment */+///   UMPSCQueue<int, false, 10> q;+///   q.enqueue(1);+///   q.enqueue(2);+///   q.enqueue(3);+///   ASSERT_FALSE(q.empty());+///   ASSERT_EQ(q.size(), 3);+///   int v;+///   q.dequeue(v);+///   ASSERT_EQ(v, 1);+///   ASSERT_TRUE(try_dequeue(v));+///   ASSERT_EQ(v, 2);+///   ASSERT_TRUE(try_dequeue_until(v, now() + seconds(1)));+///   ASSERT_EQ(v, 3);+///   ASSERT_TRUE(q.empty());+///   ASSERT_EQ(q.size(), 0);+///   ASSERT_FALSE(try_dequeue(v));+///   ASSERT_FALSE(try_dequeue_for(v, microseconds(100)));+/// @endcode+///+/// Design:+/// - The queue is composed of one or more segments. Each segment has+///   a fixed size of 2^LgSegmentSize entries. Each segment is used+///   exactly once.+/// - Each entry is composed of a futex and a single element.+/// - Each segment's array of entries is strided to avoid false sharing.+///   I.e., to reduce any cacheline contention that might be induced by+///   concurrent mutations to the queue that might happen to affect+///   otherwise-adjacent locations that might happen to share cacheline.+/// - The queue contains two 64-bit ticket variables. The producer+///   ticket counts the number of producer tickets issued so far, and+///   the same for the consumer ticket. Each ticket number corresponds+///   to a specific entry in a specific segment.+/// - The queue maintains two pointers, head and tail. Head points to+///   the segment that corresponds to the current consumer+///   ticket. Similarly, tail pointer points to the segment that+///   corresponds to the producer ticket.+/// - Segments are organized as a singly linked list.+/// - The producer with the first ticket in the current producer+///   segment has primary responsibility for allocating and linking+///   the next segment. Other producers and connsumers may help do so+///   when needed if that thread is delayed.+/// - The producer with the last ticket in the current producer+///   segment is primarily responsible for advancing the tail pointer+///   to the next segment. Other producers and consumers may help do+///   so when needed if that thread is delayed.+/// - Similarly, the consumer with the last ticket in the current+///   consumer segment is primarily responsible for advancing the head+///   pointer to the next segment. Other consumers may help do so when+///   needed if that thread is delayed.+/// - The tail pointer must not lag behind the head pointer.+///   Otherwise, the algorithm cannot be certain about the removal of+///   segment and would have to incur higher costs to ensure safe+///   reclamation.  Consumers must ensure that head never overtakes+///   tail.+///+/// Memory Usage:+/// - An empty queue contains one segment. A nonempty queue contains+///   one or two more segment than fits its contents.+/// - Removed segments are not reclaimed until there are no threads,+///   producers or consumers, with references to them or their+///   predecessors. That is, a lagging thread may delay the reclamation+///   of a chain of removed segments.+/// - The template parameter LgAlign can be used to reduce memory usage+///   at the cost of increased chance of false sharing.+///+/// Performance considerations:+/// - All operations take constant time, excluding the costs of+///   allocation, reclamation, interference from other threads, and+///   waiting for actions by other threads.+/// - In general, using the single producer and or single consumer+///   variants yield better performance than the MP and MC+///   alternatives.+/// - SPSC without blocking is the fastest configuration. It doesn't+///   include any read-modify-write atomic operations, full fences, or+///   system calls in the critical path.+/// - MP adds a fetch_add to the critical path of each producer operation.+/// - MC adds a fetch_add or compare_exchange to the critical path of+///   each consumer operation.+/// - The possibility of consumers blocking, even if they never do,+///   adds a compare_exchange to the critical path of each producer+///   operation.+/// - MPMC, SPMC, MPSC require the use of a deferred reclamation+///   mechanism to guarantee that segments removed from the linked+///   list, i.e., unreachable from the head pointer, are reclaimed+///   only after they are no longer needed by any lagging producers or+///   consumers.+/// - The overheads of segment allocation and reclamation are intended+///   to be mostly out of the critical path of the queue's throughput.+/// - If the template parameter LgSegmentSize is changed, it should be+///   set adequately high to keep the amortized cost of allocation and+///   reclamation low.+/// - It is recommended to measure performance with different variants+///   when applicable, e.g., UMPMC vs UMPSC. Depending on the use+///   case, sometimes the variant with the higher sequential overhead+///   may yield better results due to, for example, more favorable+///   producer-consumer balance or favorable timing for avoiding+///   costly blocking.+///+/// Guarantees:+/// - The queues are linearizable:+///   - For two enqueue operations q(A) and q(B), if q(A) < q(B) in+///     the happens-before relation, then A precedes B in the queue.+///   - For two dequeue operations d(A) and d(B), if d(A) < d(B) in+///     the happens-before relation, then A preceded B in the queue.++template <+    typename T,+    bool SingleProducer,+    bool SingleConsumer,+    bool MayBlock,+    size_t LgSegmentSize = 8,+    size_t LgAlign = constexpr_log2(hardware_destructive_interference_size),+    template <typename> class Atom = std::atomic>+class UnboundedQueue {+  using Ticket = uint64_t;+  class Entry;+  class Segment;++  static constexpr bool SPSC = SingleProducer && SingleConsumer;+  static constexpr size_t Stride = SPSC || (LgSegmentSize <= 1) ? 1 : 27;+  static constexpr size_t SegmentSize = 1u << LgSegmentSize;+  static constexpr size_t Align = 1u << LgAlign;++  static_assert(+      std::is_nothrow_destructible<T>::value, "T must be nothrow_destructible");+  static_assert((Stride & 1) == 1, "Stride must be odd");+  static_assert(LgSegmentSize < 32, "LgSegmentSize must be < 32");+  static_assert(LgAlign < 16, "LgAlign must be < 16");++  using Sem = folly::SaturatingSemaphore<MayBlock, Atom>;++  struct Consumer {+    Atom<Segment*> head;+    Atom<Ticket> ticket;+    hazptr_obj_cohort<Atom> cohort;+    explicit Consumer(Segment* s) : head(s), ticket(0) {+      s->set_cohort_no_tag(&cohort); // defined in hazptr_obj+    }+  };+  struct Producer {+    Atom<Segment*> tail;+    Atom<Ticket> ticket;+    explicit Producer(Segment* s) : tail(s), ticket(0) {}+  };++  alignas(Align) Consumer c_;+  alignas(Align) Producer p_;++ public:+  using value_type = T;+  using size_type = size_t;++  /** constructor */+  UnboundedQueue()+      : c_(new Segment(0)), p_(c_.head.load(std::memory_order_relaxed)) {}++  /** destructor */+  ~UnboundedQueue() {+    cleanUpRemainingItems();+    reclaimRemainingSegments();+  }++  /** enqueue */+  FOLLY_ALWAYS_INLINE void enqueue(const T& arg) { enqueueImpl(arg); }++  FOLLY_ALWAYS_INLINE void enqueue(T&& arg) { enqueueImpl(std::move(arg)); }++  /** dequeue */+  FOLLY_ALWAYS_INLINE void dequeue(T& item) noexcept { item = dequeueImpl(); }++  FOLLY_ALWAYS_INLINE T dequeue() noexcept { return dequeueImpl(); }++  /** try_dequeue */+  FOLLY_ALWAYS_INLINE bool try_dequeue(T& item) noexcept {+    auto o = try_dequeue();+    if (FOLLY_LIKELY(o.has_value())) {+      item = std::move(*o);+      return true;+    }+    return false;+  }++  FOLLY_ALWAYS_INLINE folly::Optional<T> try_dequeue() noexcept {+    return tryDequeueUntil(std::chrono::steady_clock::time_point::min());+  }++  /** try_dequeue_until */+  template <typename Clock, typename Duration>+  FOLLY_ALWAYS_INLINE bool try_dequeue_until(+      T& item,+      const std::chrono::time_point<Clock, Duration>& deadline) noexcept {+    folly::Optional<T> o = try_dequeue_until(deadline);++    if (FOLLY_LIKELY(o.has_value())) {+      item = std::move(*o);+      return true;+    }++    return false;+  }++  template <typename Clock, typename Duration>+  FOLLY_ALWAYS_INLINE folly::Optional<T> try_dequeue_until(+      const std::chrono::time_point<Clock, Duration>& deadline) noexcept {+    return tryDequeueUntil(deadline);+  }++  /** try_dequeue_for */+  template <typename Rep, typename Period>+  FOLLY_ALWAYS_INLINE bool try_dequeue_for(+      T& item, const std::chrono::duration<Rep, Period>& duration) noexcept {+    folly::Optional<T> o = try_dequeue_for(duration);++    if (FOLLY_LIKELY(o.has_value())) {+      item = std::move(*o);+      return true;+    }++    return false;+  }++  template <typename Rep, typename Period>+  FOLLY_ALWAYS_INLINE folly::Optional<T> try_dequeue_for(+      const std::chrono::duration<Rep, Period>& duration) noexcept {+    folly::Optional<T> o = try_dequeue();+    if (FOLLY_LIKELY(o.has_value())) {+      return o;+    }+    return tryDequeueUntil(std::chrono::steady_clock::now() + duration);+  }++  /** try_peek */+  FOLLY_ALWAYS_INLINE const T* try_peek() noexcept {+    static_assert(SingleConsumer, "not single-consumer");+    return tryPeekUntil(std::chrono::steady_clock::time_point::min());+  }++  /** size */+  size_t size() const noexcept {+    auto p = producerTicket();+    auto c = consumerTicket();+    return p > c ? p - c : 0;+  }++  /** empty */+  bool empty() const noexcept {+    auto c = consumerTicket();+    auto p = producerTicket();+    return p <= c;+  }++ private:+  /** enqueueImpl */+  template <typename Arg>+  FOLLY_ALWAYS_INLINE void enqueueImpl(Arg&& arg) {+    if (SPSC) {+      Segment* s = tail();+      enqueueCommon(s, std::forward<Arg>(arg));+    } else {+      // Using hazptr_holder instead of hazptr_local because it is+      // possible that the T ctor happens to use hazard pointers.+      hazptr_holder<Atom> hptr = make_hazard_pointer<Atom>();+      Segment* s = hptr.protect(p_.tail);+      enqueueCommon(s, std::forward<Arg>(arg));+    }+  }++  /** enqueueCommon */+  template <typename Arg>+  FOLLY_ALWAYS_INLINE void enqueueCommon(Segment* s, Arg&& arg) {+    Ticket t = fetchIncrementProducerTicket();+    if (!SingleProducer) {+      s = findSegment(s, t);+    }+    DCHECK_GE(t, s->minTicket());+    DCHECK_LT(t, s->minTicket() + SegmentSize);+    size_t idx = index(t);+    Entry& e = s->entry(idx);+    e.putItem(std::forward<Arg>(arg));+    if (responsibleForAlloc(t)) {+      allocNextSegment(s);+    }+    if (responsibleForAdvance(t)) {+      advanceTail(s);+    }+  }++  /** dequeueImpl */+  FOLLY_ALWAYS_INLINE T dequeueImpl() noexcept {+    if (SPSC) {+      Segment* s = head();+      return dequeueCommon(s);+    } else {+      // Using hazptr_holder instead of hazptr_local because it is+      // possible to call the T dtor and it may happen to use hazard+      // pointers.+      hazptr_holder<Atom> hptr = make_hazard_pointer<Atom>();+      Segment* s = hptr.protect(c_.head);+      return dequeueCommon(s);+    }+  }++  /** dequeueCommon */+  FOLLY_ALWAYS_INLINE T dequeueCommon(Segment* s) noexcept {+    Ticket t = fetchIncrementConsumerTicket();+    if (!SingleConsumer) {+      s = findSegment(s, t);+    }+    size_t idx = index(t);+    Entry& e = s->entry(idx);+    auto res = e.takeItem();+    if (responsibleForAdvance(t)) {+      advanceHead(s);+    }+    return res;+  }++  /** tryDequeueUntil */+  template <typename Clock, typename Duration>+  FOLLY_ALWAYS_INLINE folly::Optional<T> tryDequeueUntil(+      const std::chrono::time_point<Clock, Duration>& deadline) noexcept {+    if (SingleConsumer) {+      Segment* s = head();+      return tryDequeueUntilSC(s, deadline);+    } else {+      // Using hazptr_holder instead of hazptr_local because it is+      //  possible to call ~T() and it may happen to use hazard pointers.+      hazptr_holder<Atom> hptr = make_hazard_pointer<Atom>();+      Segment* s = hptr.protect(c_.head);+      return tryDequeueUntilMC(s, deadline);+    }+  }++  /** tryDequeueUntilSC */+  template <typename Clock, typename Duration>+  FOLLY_ALWAYS_INLINE folly::Optional<T> tryDequeueUntilSC(+      Segment* s,+      const std::chrono::time_point<Clock, Duration>& deadline) noexcept {+    Ticket t = consumerTicket();+    DCHECK_GE(t, s->minTicket());+    DCHECK_LT(t, (s->minTicket() + SegmentSize));+    size_t idx = index(t);+    Entry& e = s->entry(idx);+    if (FOLLY_UNLIKELY(!tryDequeueWaitElem(e, t, deadline))) {+      return folly::Optional<T>();+    }+    setConsumerTicket(t + 1);+    folly::Optional<T> ret = e.takeItem();+    if (responsibleForAdvance(t)) {+      advanceHead(s);+    }+    return ret;+  }++  /** tryDequeueUntilMC */+  template <typename Clock, typename Duration>+  FOLLY_ALWAYS_INLINE folly::Optional<T> tryDequeueUntilMC(+      Segment* s,+      const std::chrono::time_point<Clock, Duration>& deadline) noexcept {+    while (true) {+      Ticket t = consumerTicket();+      if (FOLLY_UNLIKELY(t >= (s->minTicket() + SegmentSize))) {+        s = getAllocNextSegment(s, t);+        DCHECK(s);+        continue;+      }+      size_t idx = index(t);+      Entry& e = s->entry(idx);+      if (FOLLY_UNLIKELY(!tryDequeueWaitElem(e, t, deadline))) {+        return folly::Optional<T>();+      }+      if (!c_.ticket.compare_exchange_weak(+              t, t + 1, std::memory_order_acq_rel, std::memory_order_acquire)) {+        continue;+      }+      folly::Optional<T> ret = e.takeItem();+      if (responsibleForAdvance(t)) {+        advanceHead(s);+      }+      return ret;+    }+  }++  /** tryDequeueWaitElem */+  template <typename Clock, typename Duration>+  FOLLY_ALWAYS_INLINE bool tryDequeueWaitElem(+      Entry& e,+      Ticket t,+      const std::chrono::time_point<Clock, Duration>& deadline) noexcept {+    if (FOLLY_LIKELY(e.tryWaitUntil(deadline))) {+      return true;+    }+    return t < producerTicket();+  }++  /** tryPeekUntil */+  template <typename Clock, typename Duration>+  FOLLY_ALWAYS_INLINE const T* tryPeekUntil(+      const std::chrono::time_point<Clock, Duration>& deadline) noexcept {+    // This function is supported only for USPSC and UMPSC queues.+    DCHECK(SingleConsumer);+    Segment* s = head();+    Ticket t = consumerTicket();+    DCHECK_GE(t, s->minTicket());+    DCHECK_LT(t, (s->minTicket() + SegmentSize));+    size_t idx = index(t);+    Entry& e = s->entry(idx);+    if (FOLLY_UNLIKELY(!tryDequeueWaitElem(e, t, deadline))) {+      return nullptr;+    }+    return e.peekItem();+  }++  /** findSegment */+  FOLLY_ALWAYS_INLINE+  Segment* findSegment(Segment* s, const Ticket t) noexcept {+    while (FOLLY_UNLIKELY(t >= (s->minTicket() + SegmentSize))) {+      s = getAllocNextSegment(s, t);+      DCHECK(s);+    }+    return s;+  }++  /** getAllocNextSegment */+  Segment* getAllocNextSegment(Segment* s, Ticket t) noexcept {+    Segment* next = s->nextSegment();+    if (!next) {+      DCHECK_GE(t, s->minTicket() + SegmentSize);+      auto diff = t - (s->minTicket() + SegmentSize);+      if (diff > 0) {+        auto dur = std::chrono::microseconds(diff);+        auto deadline = std::chrono::steady_clock::now() + dur;+        WaitOptions opt;+        opt.spin_max(dur);+        detail::spin_pause_until(deadline, opt, [s] {+          return s->nextSegment();+        });+        next = s->nextSegment();+        if (next) {+          return next;+        }+      }+      next = allocNextSegment(s);+    }+    DCHECK(next);+    return next;+  }++  /** allocNextSegment */+  Segment* allocNextSegment(Segment* s) {+    auto t = s->minTicket() + SegmentSize;+    Segment* next = new Segment(t);+    next->set_cohort_no_tag(&c_.cohort); // defined in hazptr_obj+    next->acquire_ref_safe(); // defined in hazptr_obj_base_linked+    if (!s->casNextSegment(next)) {+      delete next;+      next = s->nextSegment();+    }+    DCHECK(next);+    return next;+  }++  /** advanceTail */+  void advanceTail(Segment* s) noexcept {+    if (SPSC) {+      Segment* next = s->nextSegment();+      DCHECK(next);+      setTail(next);+    } else {+      Ticket t = s->minTicket() + SegmentSize;+      advanceTailToTicket(t);+    }+  }++  /** advanceTailToTicket */+  void advanceTailToTicket(Ticket t) noexcept {+    Segment* s = tail();+    while (s->minTicket() < t) {+      Segment* next = s->nextSegment();+      if (!next) {+        next = allocNextSegment(s);+      }+      DCHECK(next);+      casTail(s, next);+      s = tail();+    }+  }++  /** advanceHead */+  void advanceHead(Segment* s) noexcept {+    if (SPSC) {+      while (tail() == s) {+        /* Wait for producer to advance tail. */+        asm_volatile_pause();+      }+      Segment* next = s->nextSegment();+      DCHECK(next);+      setHead(next);+      reclaimSegment(s);+    } else {+      Ticket t = s->minTicket() + SegmentSize;+      advanceHeadToTicket(t);+    }+  }++  /** advanceHeadToTicket */+  void advanceHeadToTicket(Ticket t) noexcept {+    /* Tail must not lag behind head. Otherwise, the algorithm cannot+       be certain about removal of segments. */+    advanceTailToTicket(t);+    Segment* s = head();+    if (SingleConsumer) {+      DCHECK_EQ(s->minTicket() + SegmentSize, t);+      Segment* next = s->nextSegment();+      DCHECK(next);+      setHead(next);+      reclaimSegment(s);+    } else {+      while (s->minTicket() < t) {+        Segment* next = s->nextSegment();+        DCHECK(next);+        if (casHead(s, next)) {+          reclaimSegment(s);+          s = next;+        }+      }+    }+  }++  /** reclaimSegment */+  void reclaimSegment(Segment* s) noexcept {+    if (SPSC) {+      delete s;+    } else {+      s->retire(); // defined in hazptr_obj_base_linked+    }+  }++  /** cleanUpRemainingItems */+  void cleanUpRemainingItems() {+    auto end = producerTicket();+    auto s = head();+    for (auto t = consumerTicket(); t < end; ++t) {+      if (t >= s->minTicket() + SegmentSize) {+        s = s->nextSegment();+      }+      DCHECK_LT(t, (s->minTicket() + SegmentSize));+      auto idx = index(t);+      auto& e = s->entry(idx);+      e.destroyItem();+    }+  }++  /** reclaimRemainingSegments */+  void reclaimRemainingSegments() {+    auto h = head();+    auto s = h->nextSegment();+    h->setNextSegment(nullptr);+    reclaimSegment(h);+    while (s) {+      auto next = s->nextSegment();+      delete s;+      s = next;+    }+  }++  FOLLY_ALWAYS_INLINE size_t index(Ticket t) const noexcept {+    return (t * Stride) & (SegmentSize - 1);+  }++  FOLLY_ALWAYS_INLINE bool responsibleForAlloc(Ticket t) const noexcept {+    return (t & (SegmentSize - 1)) == 0;+  }++  FOLLY_ALWAYS_INLINE bool responsibleForAdvance(Ticket t) const noexcept {+    return (t & (SegmentSize - 1)) == (SegmentSize - 1);+  }++  FOLLY_ALWAYS_INLINE Segment* head() const noexcept {+    return c_.head.load(std::memory_order_acquire);+  }++  FOLLY_ALWAYS_INLINE Segment* tail() const noexcept {+    return p_.tail.load(std::memory_order_acquire);+  }++  FOLLY_ALWAYS_INLINE Ticket producerTicket() const noexcept {+    return p_.ticket.load(std::memory_order_acquire);+  }++  FOLLY_ALWAYS_INLINE Ticket consumerTicket() const noexcept {+    return c_.ticket.load(std::memory_order_acquire);+  }++  void setHead(Segment* s) noexcept {+    DCHECK(SingleConsumer);+    c_.head.store(s, std::memory_order_relaxed);+  }++  void setTail(Segment* s) noexcept {+    DCHECK(SPSC);+    p_.tail.store(s, std::memory_order_release);+  }++  bool casHead(Segment*& s, Segment* next) noexcept {+    DCHECK(!SingleConsumer);+    return c_.head.compare_exchange_strong(+        s, next, std::memory_order_release, std::memory_order_acquire);+  }++  void casTail(Segment*& s, Segment* next) noexcept {+    DCHECK(!SPSC);+    p_.tail.compare_exchange_strong(+        s, next, std::memory_order_release, std::memory_order_relaxed);+  }++  FOLLY_ALWAYS_INLINE void setProducerTicket(Ticket t) noexcept {+    p_.ticket.store(t, std::memory_order_release);+  }++  FOLLY_ALWAYS_INLINE void setConsumerTicket(Ticket t) noexcept {+    c_.ticket.store(t, std::memory_order_release);+  }++  FOLLY_ALWAYS_INLINE Ticket fetchIncrementConsumerTicket() noexcept {+    if (SingleConsumer) {+      Ticket oldval = consumerTicket();+      setConsumerTicket(oldval + 1);+      return oldval;+    } else { // MC+      return c_.ticket.fetch_add(1, std::memory_order_acq_rel);+    }+  }++  FOLLY_ALWAYS_INLINE Ticket fetchIncrementProducerTicket() noexcept {+    if (SingleProducer) {+      Ticket oldval = producerTicket();+      setProducerTicket(oldval + 1);+      return oldval;+    } else { // MP+      return p_.ticket.fetch_add(1, std::memory_order_acq_rel);+    }+  }++  /**+   *  Entry+   */+  class Entry {+    Sem flag_;+    aligned_storage_for_t<T> item_;++   public:+    template <typename Arg>+    FOLLY_ALWAYS_INLINE void putItem(Arg&& arg) {+      new (&item_) T(std::forward<Arg>(arg));+      flag_.post();+    }++    FOLLY_ALWAYS_INLINE T takeItem() noexcept {+      flag_.wait();+      return getItem();+    }++    FOLLY_ALWAYS_INLINE const T* peekItem() noexcept {+      flag_.wait();+      return itemPtr();+    }++    template <typename Clock, typename Duration>+    FOLLY_EXPORT FOLLY_ALWAYS_INLINE bool tryWaitUntil(+        const std::chrono::time_point<Clock, Duration>& deadline) noexcept {+      // wait-options from benchmarks on contended queues:+      static constexpr auto const opt =+          Sem::wait_options().spin_max(std::chrono::microseconds(10));+      return flag_.try_wait_until(deadline, opt);+    }++    FOLLY_ALWAYS_INLINE void destroyItem() noexcept { itemPtr()->~T(); }++   private:+    FOLLY_ALWAYS_INLINE T getItem() noexcept {+      T ret = std::move(*(itemPtr()));+      destroyItem();+      return ret;+    }++    FOLLY_ALWAYS_INLINE T* itemPtr() noexcept {+      return static_cast<T*>(static_cast<void*>(&item_));+    }+  }; // Entry++  /**+   *  Segment+   */+  class Segment : public hazptr_obj_base_linked<Segment, Atom> {+    Atom<Segment*> next_{nullptr};+    const Ticket min_;+    alignas(Align) Entry b_[SegmentSize];++   public:+    explicit Segment(const Ticket t) noexcept : min_(t) {}++    Segment* nextSegment() const noexcept {+      return next_.load(std::memory_order_acquire);+    }++    void setNextSegment(Segment* next) {+      next_.store(next, std::memory_order_relaxed);+    }++    bool casNextSegment(Segment* next) noexcept {+      Segment* expected = nullptr;+      return next_.compare_exchange_strong(+          expected, next, std::memory_order_release, std::memory_order_relaxed);+    }++    FOLLY_ALWAYS_INLINE Ticket minTicket() const noexcept {+      DCHECK_EQ((min_ & (SegmentSize - 1)), Ticket(0));+      return min_;+    }++    FOLLY_ALWAYS_INLINE Entry& entry(size_t index) noexcept {+      return b_[index];+    }++    template <typename S>+    void push_links(bool m, S& s) {+      if (m == false) { // next_ is immutable+        auto p = nextSegment();+        if (p) {+          s.push(p);+        }+      }+    }+  }; // Segment++}; // UnboundedQueue++/* Aliases */++template <+    typename T,+    bool MayBlock,+    size_t LgSegmentSize = 8,+    size_t LgAlign = constexpr_log2(hardware_destructive_interference_size),+    template <typename> class Atom = std::atomic>+using USPSCQueue =+    UnboundedQueue<T, true, true, MayBlock, LgSegmentSize, LgAlign, Atom>;++template <+    typename T,+    bool MayBlock,+    size_t LgSegmentSize = 8,+    size_t LgAlign = constexpr_log2(hardware_destructive_interference_size),+    template <typename> class Atom = std::atomic>+using UMPSCQueue =+    UnboundedQueue<T, false, true, MayBlock, LgSegmentSize, LgAlign, Atom>;++template <+    typename T,+    bool MayBlock,+    size_t LgSegmentSize = 8,+    size_t LgAlign = constexpr_log2(hardware_destructive_interference_size),+    template <typename> class Atom = std::atomic>+using USPMCQueue =+    UnboundedQueue<T, true, false, MayBlock, LgSegmentSize, LgAlign, Atom>;++template <+    typename T,+    bool MayBlock,+    size_t LgSegmentSize = 8,+    size_t LgAlign = constexpr_log2(hardware_destructive_interference_size),+    template <typename> class Atom = std::atomic>+using UMPMCQueue =+    UnboundedQueue<T, false, false, MayBlock, LgSegmentSize, LgAlign, Atom>;++} // namespace folly
+ folly/folly/concurrency/container/FlatCombiningPriorityQueue.h view
@@ -0,0 +1,426 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <atomic>+#include <chrono>+#include <memory>+#include <mutex>+#include <queue>++#include <glog/logging.h>++#include <folly/Optional.h>+#include <folly/detail/Futex.h>+#include <folly/synchronization/FlatCombining.h>++namespace folly {++/// Thread-safe priority queue based on flat combining. If the+/// constructor parameter maxSize is greater than 0 (default = 0),+/// then the queue is bounded. This template provides blocking,+/// non-blocking, and timed variants of each of push(), pop(), and+/// peek() operations. The empty() and size() functions are inherently+/// non-blocking.+///+/// PriorityQueue must support the interface of std::priority_queue,+/// specifically empty(), size(), push(), top(), and pop().  Mutex+/// must meet the standard Lockable requirements.+///+/// By default FlatCombining uses a dedicated combiner thread, which+/// yields better latency and throughput under high contention but+/// higher overheads under low contention. If the constructor+/// parameter dedicated is false, then there will be no dedicated+/// combiner thread and any requester may do combining of operations+/// requested by other threads. For more details see the comments for+/// FlatCombining.+///+/// Usage examples:+/// @code+///   FlatCombiningPriorityQueue<int> pq(1);+///   CHECK(pq.empty());+///   CHECK(pq.size() == 0);+///   int v;+///   CHECK(!try_pop(v));+///   CHECK(!try_pop_until(v, now() + seconds(1)));+///   CHECK(!try_peek(v));+///   CHECK(!try_peek_until(v, now() + seconds(1)));+///   pq.push(10);+///   CHECK(!pq.empty());+///   CHECK(pq.size() == 1);+///   CHECK(!pq.try_push(20));+///   CHECK(!pq.try_push_until(20), now() + seconds(1)));+///   peek(v);+///   CHECK_EQ(v, 10);+///   CHECK(pq.size() == 1);+///   pop(v);+///   CHECK_EQ(v, 10);+///   CHECK(pq.empty());+/// @encode++template <+    typename T,+    typename PriorityQueue = std::priority_queue<T>,+    typename Mutex = std::mutex,+    template <typename> class Atom = std::atomic>+class FlatCombiningPriorityQueue+    : public folly::FlatCombining<+          FlatCombiningPriorityQueue<T, PriorityQueue, Mutex, Atom>,+          Mutex,+          Atom> {+  using FCPQ = FlatCombiningPriorityQueue<T, PriorityQueue, Mutex, Atom>;+  using FC = folly::FlatCombining<FCPQ, Mutex, Atom>;++ public:+  template <+      typename... PQArgs,+      typename = decltype(PriorityQueue(std::declval<PQArgs>()...))>+  explicit FlatCombiningPriorityQueue(+      // Concurrent priority queue parameter+      const size_t maxSize = 0,+      // Flat combining parameters+      const bool dedicated = true,+      const uint32_t numRecs = 0,+      const uint32_t maxOps = 0,+      // (Sequential) PriorityQueue Parameters+      PQArgs... args)+      : FC(dedicated, numRecs, maxOps),+        maxSize_(maxSize),+        pq_(std::forward<PQArgs>(args)...) {}++  /// Returns true iff the priority queue is empty+  bool empty() const {+    bool res;+    auto fn = [&] { res = pq_.empty(); };+    const_cast<FCPQ*>(this)->requestFC(fn);+    return res;+  }++  /// Returns the number of items in the priority queue+  size_t size() const {+    size_t res;+    auto fn = [&] { res = pq_.size(); };+    const_cast<FCPQ*>(this)->requestFC(fn);+    return res;+  }++  /// Non-blocking push. Succeeds if there is space in the priority+  /// queue to insert the new item. Tries once if no time point is+  /// provided or until the provided time_point is reached. If+  /// successful, inserts the provided item in the priority queue+  /// according to its priority.+  bool try_push(const T& val) {+    return try_push_impl(+        val, std::chrono::time_point<std::chrono::steady_clock>::min());+  }++  /// Non-blocking pop. Succeeds if the priority queue is+  /// nonempty. Tries once if no time point is provided or until the+  /// provided time_point is reached.  If successful, copies the+  /// highest priority item and removes it from the priority queue.+  bool try_pop(T& val) {+    return try_pop_impl(+        val, std::chrono::time_point<std::chrono::steady_clock>::min());+  }++  /// Non-blocking peek. Succeeds if the priority queue is+  /// nonempty. Tries once if no time point is provided or until the+  /// provided time_point is reached.  If successful, copies the+  /// highest priority item without removing it.+  bool try_peek(T& val) {+    return try_peek_impl(+        val, std::chrono::time_point<std::chrono::steady_clock>::min());+  }++  /// Blocking push. Inserts the provided item in the priority+  /// queue. If it is full, this function blocks until there is space+  /// for the new item.+  void push(const T& val) {+    try_push_impl(+        val, std::chrono::time_point<std::chrono::steady_clock>::max());+  }++  /// Blocking pop. Copies the highest priority item and removes+  /// it. If the priority queue is empty, this function blocks until+  /// it is nonempty.+  void pop(T& val) {+    try_pop_impl(+        val, std::chrono::time_point<std::chrono::steady_clock>::max());+  }++  /// Blocking peek. Copies the highest priority item without+  /// removing it. If the priority queue is empty, this function+  /// blocks until it is nonempty.+  void peek(T& val) {+    try_peek_impl(+        val, std::chrono::time_point<std::chrono::steady_clock>::max());+  }++  folly::Optional<T> try_pop() {+    T val;+    if (try_pop(val)) {+      return std::move(val);+    }+    return folly::none;+  }++  folly::Optional<T> try_peek() {+    T val;+    if (try_peek(val)) {+      return std::move(val);+    }+    return folly::none;+  }++  template <typename Rep, typename Period>+  folly::Optional<T> try_pop_for(+      const std::chrono::duration<Rep, Period>& timeout) {+    T val;+    if (try_pop(val) ||+        try_pop_impl(val, std::chrono::steady_clock::now() + timeout)) {+      return std::move(val);+    }+    return folly::none;+  }++  template <typename Rep, typename Period>+  bool try_push_for(+      const T& val, const std::chrono::duration<Rep, Period>& timeout) {+    return (+        try_push(val) ||+        try_push_impl(val, std::chrono::steady_clock::now() + timeout));+  }++  template <typename Rep, typename Period>+  folly::Optional<T> try_peek_for(+      const std::chrono::duration<Rep, Period>& timeout) {+    T val;+    if (try_peek(val) ||+        try_peek_impl(val, std::chrono::steady_clock::now() + timeout)) {+      return std::move(val);+    }+    return folly::none;+  }++  template <typename Clock, typename Duration>+  folly::Optional<T> try_pop_until(+      const std::chrono::time_point<Clock, Duration>& deadline) {+    T val;+    if (try_pop_impl(val, deadline)) {+      return std::move(val);+    }+    return folly::none;+  }++  template <typename Clock, typename Duration>+  bool try_push_until(+      const T& val, const std::chrono::time_point<Clock, Duration>& deadline) {+    return try_push_impl(val, deadline);+  }++  template <typename Clock, typename Duration>+  folly::Optional<T> try_peek_until(+      const std::chrono::time_point<Clock, Duration>& deadline) {+    T val;+    if (try_peek_impl(val, deadline)) {+      return std::move(val);+    }+    return folly::none;+  }++ private:+  size_t maxSize_;+  PriorityQueue pq_;+  detail::Futex<Atom> empty_{};+  detail::Futex<Atom> full_{};++  bool isTrue(detail::Futex<Atom>& futex) {+    return futex.load(std::memory_order_relaxed) != 0;+  }++  void setFutex(detail::Futex<Atom>& futex, uint32_t val) {+    futex.store(val, std::memory_order_relaxed);+  }++  bool futexSignal(detail::Futex<Atom>& futex) {+    if (isTrue(futex)) {+      setFutex(futex, 0);+      return true;+    } else {+      return false;+    }+  }++  template <typename Clock, typename Duration>+  bool try_push_impl(+      const T& val, const std::chrono::time_point<Clock, Duration>& when);++  template <typename Clock, typename Duration>+  bool try_pop_impl(+      T& val, const std::chrono::time_point<Clock, Duration>& when);++  template <typename Clock, typename Duration>+  bool try_peek_impl(+      T& val, const std::chrono::time_point<Clock, Duration>& when);+};++/// Implementation++template <+    typename T,+    typename PriorityQueue,+    typename Mutex,+    template <typename>+    class Atom>+template <typename Clock, typename Duration>+inline bool+FlatCombiningPriorityQueue<T, PriorityQueue, Mutex, Atom>::try_push_impl(+    const T& val, const std::chrono::time_point<Clock, Duration>& when) {+  while (true) {+    bool res;+    bool wake;++    auto fn = [&] {+      if (maxSize_ > 0 && pq_.size() == maxSize_) {+        setFutex(full_, 1);+        res = false;+        return;+      }+      DCHECK(maxSize_ == 0 || pq_.size() < maxSize_);+      try {+        pq_.push(val);+        wake = futexSignal(empty_);+        res = true;+        return;+      } catch (const std::bad_alloc&) {+        setFutex(full_, 1);+        res = false;+        return;+      }+    };+    this->requestFC(fn);++    if (res) {+      if (wake) {+        detail::futexWake(&empty_);+      }+      return true;+    }+    if (when == std::chrono::time_point<Clock>::min()) {+      return false;+    }+    while (isTrue(full_)) {+      if (when == std::chrono::time_point<Clock>::max()) {+        detail::futexWait(&full_, 1);+      } else {+        if (Clock::now() > when) {+          return false;+        } else {+          detail::futexWaitUntil(&full_, 1, when);+        }+      }+    } // inner while loop+  } // outer while loop+}++template <+    typename T,+    typename PriorityQueue,+    typename Mutex,+    template <typename>+    class Atom>+template <typename Clock, typename Duration>+inline bool+FlatCombiningPriorityQueue<T, PriorityQueue, Mutex, Atom>::try_pop_impl(+    T& val, const std::chrono::time_point<Clock, Duration>& when) {+  while (true) {+    bool res;+    bool wake;++    auto fn = [&] {+      res = !pq_.empty();+      if (res) {+        val = pq_.top();+        pq_.pop();+        wake = futexSignal(full_);+      } else {+        setFutex(empty_, 1);+      }+    };+    this->requestFC(fn);++    if (res) {+      if (wake) {+        detail::futexWake(&full_);+      }+      return true;+    }+    while (isTrue(empty_)) {+      if (when == std::chrono::time_point<Clock>::max()) {+        detail::futexWait(&empty_, 1);+      } else {+        if (Clock::now() > when) {+          return false;+        } else {+          detail::futexWaitUntil(&empty_, 1, when);+        }+      }+    } // inner while loop+  } // outer while loop+}++template <+    typename T,+    typename PriorityQueue,+    typename Mutex,+    template <typename>+    class Atom>+template <typename Clock, typename Duration>+inline bool+FlatCombiningPriorityQueue<T, PriorityQueue, Mutex, Atom>::try_peek_impl(+    T& val, const std::chrono::time_point<Clock, Duration>& when) {+  while (true) {+    bool res;++    auto fn = [&] {+      res = !pq_.empty();+      if (res) {+        val = pq_.top();+      } else {+        setFutex(empty_, 1);+      }+    };+    this->requestFC(fn);++    if (res) {+      return true;+    }+    while (isTrue(empty_)) {+      if (when == std::chrono::time_point<Clock>::max()) {+        detail::futexWait(&empty_, 1);+      } else {+        if (Clock::now() > when) {+          return false;+        } else {+          detail::futexWaitUntil(&empty_, 1, when);+        }+      }+    } // inner while loop+  } // outer while loop+}++} // namespace folly
+ folly/folly/concurrency/container/LockFreeRingBuffer.h view
@@ -0,0 +1,306 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <atomic>+#include <cstring>+#include <memory>+#include <type_traits>+#include <utility>++#include <boost/operators.hpp>+#include <folly/Portability.h>+#include <folly/Traits.h>+#include <folly/detail/TurnSequencer.h>+#include <folly/portability/Unistd.h>+#include <folly/synchronization/SanitizeThread.h>++namespace folly {+namespace detail {++template <+    typename T,+    template <typename>+    class Atom,+    template <typename>+    class Storage>+class RingBufferSlot;+template <typename T>+class RingBufferTrivialStorage;+template <typename T>+class RingBufferBrokenStorage;++} // namespace detail++/// LockFreeRingBuffer<T> is a fixed-size, concurrent ring buffer with the+/// following semantics:+///+///  1. Writers cannot block on other writers UNLESS they are <capacity> writes+///     apart from each other (writing to the same slot after a wrap-around)+///  2. Writers cannot block on readers+///  3. Readers can wait for writes that haven't occurred yet+///  4. Readers can detect if they are lagging behind+///+/// In this sense, reads from this buffer are best-effort but writes+/// are guaranteed.+///+/// Another way to think about this is as an unbounded stream of writes. The+/// buffer contains the last <capacity> writes but readers can attempt to read+/// any part of the stream, even outside this window. The read API takes a+/// Cursor that can point anywhere in this stream of writes. Reads from the+/// "future" can optionally block but reads from the "past" will always fail.+///++template <+    typename T,+    template <typename> class Atom = std::atomic,+    template <typename> class Storage = detail::RingBufferTrivialStorage>+class LockFreeRingBuffer {+  static_assert(+      std::is_nothrow_default_constructible<T>::value,+      "Element type must be nothrow default constructible");++ public:+  /// Opaque pointer to a past or future write.+  /// Can be moved relative to its current location but not in absolute terms.+  struct Cursor : boost::totally_ordered<Cursor> {+    explicit Cursor(uint64_t initialTicket) noexcept : ticket(initialTicket) {}++    /// Returns true if this cursor now points to a different+    /// write, false otherwise.+    bool moveForward(uint64_t steps = 1) noexcept {+      uint64_t prevTicket = ticket;+      ticket += steps;+      return prevTicket != ticket;+    }++    /// Returns true if this cursor now points to a previous+    /// write, false otherwise.+    bool moveBackward(uint64_t steps = 1) noexcept {+      uint64_t prevTicket = ticket;+      if (steps > ticket) {+        ticket = 0;+      } else {+        ticket -= steps;+      }+      return prevTicket != ticket;+    }++    bool operator==(const Cursor& that) const noexcept {+      return ticket == that.ticket;+    }++    bool operator<(const Cursor& that) const noexcept {+      return ticket < that.ticket;+    }++   protected: // for test visibility reasons+    uint64_t ticket;+    friend class LockFreeRingBuffer;+  };++  explicit LockFreeRingBuffer(uint32_t capacity) noexcept+      : capacity_(capacity), slots_(new Slot[capacity]), ticket_(0) {}++  LockFreeRingBuffer(const LockFreeRingBuffer&) = delete;+  LockFreeRingBuffer& operator=(const LockFreeRingBuffer&) = delete;++  uint32_t capacity() const noexcept { return capacity_; }++  /// Perform a single write of an object of type T.+  /// Writes can block iff a previous writer has not yet completed a write+  /// for the same slot (before the most recent wrap-around).+  template <typename V>+  void write(const V& value) noexcept {+    uint64_t ticket = ticket_.fetch_add(1);+    slots_[idx(ticket)].write(turn(ticket), value);+  }++  /// Perform a single write of an object of type T.+  /// Writes can block iff a previous writer has not yet completed a write+  /// for the same slot (before the most recent wrap-around).+  /// Returns a Cursor pointing to the just-written T.+  template <typename V>+  Cursor writeAndGetCursor(const V& value) noexcept {+    uint64_t ticket = ticket_.fetch_add(1);+    slots_[idx(ticket)].write(turn(ticket), value);+    return Cursor(ticket);+  }++  /// Read the value at the cursor.+  /// Returns true if the read succeeded, false otherwise. If the return+  /// value is false, dest is to be considered partially read and in an+  /// inconsistent state. Readers are advised to discard it.+  template <typename V>+  bool tryRead(V& dest, const Cursor& cursor) const noexcept {+    return slots_[idx(cursor.ticket)].tryRead(dest, turn(cursor.ticket));+  }++  /// Read the value at the cursor or block if the write has not occurred yet.+  /// Returns true if the read succeeded, false otherwise. If the return+  /// value is false, dest is to be considered partially read and in an+  /// inconsistent state. Readers are advised to discard it.+  template <typename V>+  bool waitAndTryRead(V& dest, const Cursor& cursor) noexcept {+    return slots_[idx(cursor.ticket)].waitAndTryRead(dest, turn(cursor.ticket));+  }++  /// Returns a Cursor pointing to the first write that has not occurred yet.+  Cursor currentHead() const noexcept { return Cursor(ticket_.load()); }++  /// Returns a Cursor pointing to the earliest readable write.+  Cursor currentTail() const noexcept {+    uint64_t ticket = ticket_.load();++    // can't go back more steps than we've taken+    uint64_t backStep = std::min<uint64_t>(ticket, capacity_);++    return Cursor(ticket - backStep);+  }++  /// Returns the address and length of the internal buffer.+  /// Unsafe to inspect this region at runtime. And not useful.+  /// Useful when using LockFreeRingBuffer to store data which must be retrieved+  /// from a core dump after a crash if the given region is added to the list of+  /// dumped memory regions.+  std::pair<void const*, size_t> internalBufferLocation() const {+    return std::make_pair(+        static_cast<void const*>(slots_.get()), capacity_ * sizeof(Slot));+  }++ private:+  using Slot = detail::RingBufferSlot<T, Atom, Storage>;++  const uint32_t capacity_;++  const std::unique_ptr<Slot[]> slots_;++  Atom<uint64_t> ticket_;++  uint32_t idx(uint64_t ticket) const noexcept { return ticket % capacity_; }++  uint32_t turn(uint64_t ticket) const noexcept {+    return (uint32_t)(ticket / capacity_);+  }+}; // LockFreeRingBuffer++namespace detail {+template <+    typename T,+    template <typename>+    class Atom,+    template <typename>+    class Storage>+class RingBufferSlot {+ public:+  explicit RingBufferSlot() noexcept {}++  template <typename V>+  void write(const uint32_t turn, const V& value) noexcept {+    Atom<uint32_t> cutoff(0);+    sequencer_.waitForTurn(turn * 2, cutoff, false);++    // Change to an odd-numbered turn to indicate write in process+    sequencer_.completeTurn(turn * 2);++    storage_.store(value);+    sequencer_.completeTurn(turn * 2 + 1);+    // At (turn + 1) * 2+  }++  template <typename V>+  bool waitAndTryRead(V& dest, uint32_t turn) noexcept {+    uint32_t desired_turn = (turn + 1) * 2;+    Atom<uint32_t> cutoff(0);+    if (sequencer_.tryWaitForTurn(desired_turn, cutoff, false) !=+        TurnSequencer<Atom>::TryWaitResult::SUCCESS) {+      return false;+    }+    storage_.load(dest);++    // if it's still the same turn, we read the value successfully+    return sequencer_.isTurn(desired_turn);+  }++  template <typename V>+  bool tryRead(V& dest, uint32_t turn) const noexcept {+    // The write that started at turn 0 ended at turn 2+    if (!sequencer_.isTurn((turn + 1) * 2)) {+      return false;+    }+    storage_.load(dest);++    // if it's still the same turn, we read the value successfully+    return sequencer_.isTurn((turn + 1) * 2);+  }++ private:+  TurnSequencer<Atom> sequencer_;+  Storage<T> storage_;+};++template <typename T>+class RingBufferTrivialStorage {+  static_assert(std::is_trivially_copyable_v<T>, "T must trivially copyable");++  // Note: If T fits in 8 bytes, folly::AtomicStruct could be used instead.++ public:+  RingBufferTrivialStorage() noexcept {+    annotate_benign_race_sized(+        &data_,+        sizeof(T),+        "T is trivial and sequencer is checked to determine validity",+        __FILE__,+        __LINE__);+  }++  void store(const T& src) {+    // technically undefined behavior: once p1478 is accepted in a future c++,+    // this memcpy may be replaced with atomic_store_per_byte_memcpy+    std::memcpy(&data_, &src, sizeof(T));+    // The sequencer protects this store with its own state_ store-release+  }++  void load(T& dest) const {+    // the sequencer protects this load with its own state_ load-acquire+    // technically undefined behavior: once p1478 is accepted in a future c++,+    // this memcpy may be replaced with atomic_store_per_byte_memcpy+    std::memcpy(&dest, &data_, sizeof(T));+  }++ private:+  // No initialization is necessary because the sequencer is checked before data+  // is returned.+  T data_;+};++template <typename T>+class [[deprecated(+    "It is UB to race loads and stores across multiple threads. "+    "Use RingBufferTrivialStorage.")]] RingBufferBrokenStorage {+ public:+  void store(const T& src) { data_ = src; }++  void load(T& dest) const { dest = data_; }++ private:+  T data_{};+};++} // namespace detail+} // namespace folly
+ folly/folly/concurrency/container/RelaxedConcurrentPriorityQueue.h view
@@ -0,0 +1,1213 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <algorithm>+#include <atomic>+#include <climits>+#include <cmath>+#include <iomanip>+#include <iostream>+#include <mutex>++#include <folly/Random.h>+#include <folly/SpinLock.h>+#include <folly/ThreadLocal.h>+#include <folly/detail/Futex.h>+#include <folly/lang/Align.h>+#include <folly/synchronization/Hazptr.h>+#include <folly/synchronization/WaitOptions.h>+#include <folly/synchronization/detail/Spin.h>++/// ------ Concurrent Priority Queue Implementation ------+// The concurrent priority queue implementation is based on the+// Mound data structure (Mounds: Array-Based Concurrent Priority Queues+// by Yujie Liu and Michael Spear, ICPP 2012)+//+/// --- Overview ---+// This relaxed implementation extends the Mound algorithm, and provides+// following features:+// - Arbitrary priorities.+// - Unbounded size.+// - Push, pop, empty, size functions. [TODO: Non-waiting and timed wait pop]+// - Supports blocking.+// - Fast and Scalable.+//+/// --- Mound ---+// A Mound is a heap where each element is a sorted linked list.+// First nodes in the lists maintain the heap property. Push randomly+// selects a leaf at the bottom level, then uses binary search to find+// a place to insert the new node to the head of the list. Pop gets+// the node from the head of the list at the root, then swap the+// list down until the heap feature holds. To use Mound in our+// implementation, we need to solve the following problems:+// - 1. Lack of general relaxed implementations. Mound is appealing+// for relaxed priority queue implementation because pop the whole+// list from the root is straightforward. One thread pops the list+// and following threads can pop from the list until its empty.+// Those pops only trigger one swap done operation. Thus reduce+// the latency for pop and reduce the contention for Mound.+// The difficulty is to provide a scalable and fast mechanism+// to let threads concurrently get elements from the list.+// - 2. Lack of control of list length. The length for every+// lists is critical for the performance. Mound suffers from not+// only the extreme cases(Push with increasing priorities, Mound+// becomes a sorted linked list; Push with decreasing priorities,+// Mound becomes to a regular heap), but also the common case(for+// random generated priorities, Mound degrades to the regular heap+// after millions of push/pop operations). The difficulty is to+// stabilize the list length without losing the accuracy and performance.+// - 3. Does not support blocking. Blocking is an important feature.+// Mound paper does not mention it. Designing the new algorithm for+// efficient blocking is challenging.+// - 4. Memory management. Mound allows optimistic reads. We need to+// protect the node from been reclaimed.+//+/// --- Design ---+// Our implementation extends Mound algorithm to support+// efficient relaxed pop. We employ a shared buffer algorithm to+// share the popped list. Our algorithm makes popping from shared+// buffer as fast as fetch_and_add. We improve the performance+// and compact the heap structure by stabilizing the size of each list.+// The implementation exposes the template parameter to set the+// preferred list length. Under the hood, we provide algorithms for+// fast inserting, pruning, and merging. The blocking algorithm is+// tricky. It allows one producer only wakes one consumer at a time.+// It also does not block the producer. For optimistic read, we use+// hazard pointer to protect the node from been reclaimed. We optimize the+// check-lock-check pattern by using test-test-and-set spin lock.++/// --- Template Parameters: ---+// 1. PopBatch could be 0 or a positive integer.+// If it is 0, only pop one node at a time.+// This is the strict implementation. It guarantees the return+// priority is alway the highest.  If it is > 0, we keep+// up to that number of nodes in a shared buffer to be consumed by+// subsequent pop operations.+//+// 2. ListTargetSize represents the minimal length for the list. It+// solves the problem when inserting to Mound with+// decreasing priority order (degrade to a heap).  Moreover,+// it maintains the Mound structure stable after trillions of+// operations, which causes unbalanced problem in the original+// Mound algorithm. We set the prunning length and merging lengtyh+// based on this parameter.+//+/// --- Interface ---+//  void push(const T& val)+//  void pop(T& val)+//  size_t size()+//  bool empty()++namespace folly {++template <+    typename T,+    bool MayBlock = false,+    bool SupportsSize = false,+    size_t PopBatch = 16,+    size_t ListTargetSize = 25,+    typename Mutex = folly::SpinLock,+    template <typename> class Atom = std::atomic>+class RelaxedConcurrentPriorityQueue {+  // Max height of the tree+  static constexpr uint32_t MAX_LEVELS = 32;+  // The default minimum value+  static constexpr T MIN_VALUE = std::numeric_limits<T>::min();++  // Align size for the shared buffer node+  static constexpr size_t Align = 1u << 7;+  static constexpr int LevelForForceInsert = 3;+  static constexpr int LevelForTraverseParent = 7;++  static_assert(PopBatch <= 256, "PopBatch must be <= 256");+  static_assert(+      ListTargetSize >= 1 && ListTargetSize <= 256,+      "TargetSize must be in the range [1, 256]");++  // The maximal length for the list+  static constexpr size_t PruningSize = ListTargetSize * 2;+  // When pop from Mound, tree elements near the leaf+  // level are likely be very small (the length of the list). When+  // swapping down after pop a list, we check the size of the+  // children to decide whether to merge them to their parent.+  static constexpr size_t MergingSize = ListTargetSize;++  /// List Node structure+  struct Node : public folly::hazptr_obj_base<Node, Atom> {+    Node* next;+    T val;+  };++  /// Mound Element (Tree node), head points to a linked list+  struct MoundElement {+    // Reading (head, size) without acquiring the lock+    Atom<Node*> head;+    Atom<size_t> size;+    alignas(Align) Mutex lock;+    MoundElement() { // initializer+      head.store(nullptr, std::memory_order_relaxed);+      size.store(0, std::memory_order_relaxed);+    }+  };++  /// The pos strcture simplify the implementation+  struct Position {+    uint32_t level;+    uint32_t index;+  };++  /// Node for shared buffer should be aligned+  struct BufferNode {+    alignas(Align) Atom<Node*> pnode;+  };++  /// Data members++  // Mound structure -> 2D array to represent a tree+  MoundElement* levels_[MAX_LEVELS];+  // Record the current leaf level (root is 0)+  Atom<uint32_t> bottom_;+  // It is used when expanding the tree+  Atom<uint32_t> guard_;++  // Mound with shared buffer+  // Following two members are accessed by consumers+  std::unique_ptr<BufferNode[]> shared_buffer_;+  alignas(Align) Atom<int> top_loc_;++  /// Blocking algorithm+  // Numbers of futexs in the array+  static constexpr size_t NumFutex = 128;+  // The index gap for accessing futex in the array+  static constexpr size_t Stride = 33;+  std::unique_ptr<folly::detail::Futex<Atom>[]> futex_array_;+  alignas(Align) Atom<uint32_t> cticket_;+  alignas(Align) Atom<uint32_t> pticket_;++  // Two counters to calculate size of the queue+  alignas(Align) Atom<size_t> counter_p_;+  alignas(Align) Atom<size_t> counter_c_;++ public:+  /// Constructor+  RelaxedConcurrentPriorityQueue()+      : cticket_(1), pticket_(1), counter_p_(0), counter_c_(0) {+    if (MayBlock) {+      futex_array_.reset(new folly::detail::Futex<Atom>[NumFutex]);+    }++    if (PopBatch > 0) {+      top_loc_ = -1;+      shared_buffer_.reset(new BufferNode[PopBatch]);+      for (size_t i = 0; i < PopBatch; i++) {+        shared_buffer_[i].pnode = nullptr;+      }+    }+    bottom_.store(0, std::memory_order_relaxed);+    guard_.store(0, std::memory_order_relaxed);+    // allocate the root MoundElement and initialize Mound+    levels_[0] = new MoundElement[1]; // default MM for MoundElement+    for (uint32_t i = 1; i < MAX_LEVELS; i++) {+      levels_[i] = nullptr;+    }+  }++  ~RelaxedConcurrentPriorityQueue() {+    if (PopBatch > 0) {+      deleteSharedBuffer();+    }+    if (MayBlock) {+      futex_array_.reset();+    }+    Position pos;+    pos.level = pos.index = 0;+    deleteAllNodes(pos);+    // default MM for MoundElement+    for (int i = getBottomLevel(); i >= 0; i--) {+      delete[] levels_[i];+    }+  }++  void push(const T& val) {+    moundPush(val);+    if (SupportsSize) {+      counter_p_.fetch_add(1, std::memory_order_relaxed);+    }+  }++  void pop(T& val) {+    moundPop(val);+    if (SupportsSize) {+      counter_c_.fetch_add(1, std::memory_order_relaxed);+    }+  }++  /// Note: size() and empty() are guaranteed to be accurate only if+  ///       the queue is not changed concurrently.+  /// Returns an estimate of the size of the queue+  size_t size() {+    DCHECK(SupportsSize);+    size_t p = counter_p_.load(std::memory_order_acquire);+    size_t c = counter_c_.load(std::memory_order_acquire);+    return (p > c) ? p - c : 0;+  }++  /// Returns true only if the queue was empty during the call.+  bool empty() { return isEmpty(); }++ private:+  uint32_t getBottomLevel() { return bottom_.load(std::memory_order_acquire); }++  /// This function is only called by the destructor+  void deleteSharedBuffer() {+    DCHECK(PopBatch > 0);+    // delete nodes in the buffer+    int loc = top_loc_.load(std::memory_order_relaxed);+    while (loc >= 0) {+      Node* n = shared_buffer_[loc--].pnode.load(std::memory_order_relaxed);+      delete n;+    }+    // delete buffer+    shared_buffer_.reset();+  }++  /// This function is only called by the destructor+  void deleteAllNodes(const Position& pos) {+    if (getElementSize(pos) == 0) {+      // current list is empty, do not need to check+      // its children again.+      return;+    }++    Node* curList = getList(pos);+    setTreeNode(pos, nullptr);+    while (curList != nullptr) { // reclaim nodes+      Node* n = curList;+      curList = curList->next;+      delete n;+    }++    if (!isLeaf(pos)) {+      deleteAllNodes(leftOf(pos));+      deleteAllNodes(rightOf(pos));+    }+  }++  /// Check the first node in TreeElement keeps the heap structure.+  bool isHeap(const Position& pos) {+    if (isLeaf(pos)) {+      return true;+    }+    Position lchild = leftOf(pos);+    Position rchild = rightOf(pos);+    return isHeap(lchild) && isHeap(rchild) &&+        readValue(pos) >= readValue(lchild) &&+        readValue(pos) >= readValue(rchild);+  }++  /// Current position is leaf?+  FOLLY_ALWAYS_INLINE bool isLeaf(const Position& pos) {+    return pos.level == getBottomLevel();+  }++  /// Current element is the root?+  FOLLY_ALWAYS_INLINE bool isRoot(const Position& pos) {+    return pos.level == 0;+  }++  /// Locate the parent node+  FOLLY_ALWAYS_INLINE Position parentOf(const Position& pos) {+    Position res;+    res.level = pos.level - 1;+    res.index = pos.index / 2;+    return res;+  }++  /// Locate the left child+  FOLLY_ALWAYS_INLINE Position leftOf(const Position& pos) {+    Position res;+    res.level = pos.level + 1;+    res.index = pos.index * 2;+    return res;+  }++  /// Locate the right child+  FOLLY_ALWAYS_INLINE Position rightOf(const Position& pos) {+    Position res;+    res.level = pos.level + 1;+    res.index = pos.index * 2 + 1;+    return res;+  }++  /// get the list size in current MoundElement+  FOLLY_ALWAYS_INLINE size_t getElementSize(const Position& pos) {+    return levels_[pos.level][pos.index].size.load(std::memory_order_relaxed);+  }++  /// Set the size of current MoundElement+  FOLLY_ALWAYS_INLINE void setElementSize(+      const Position& pos, const uint32_t& v) {+    levels_[pos.level][pos.index].size.store(v, std::memory_order_relaxed);+  }++  /// Extend the tree level+  void grow(uint32_t btm) {+    while (true) {+      if (guard_.fetch_add(1, std::memory_order_acq_rel) == 0) {+        break;+      }+      // someone already expanded the tree+      if (btm != getBottomLevel()) {+        return;+      }+      std::this_thread::yield();+    }+    // double check the bottom has not changed yet+    if (btm != getBottomLevel()) {+      guard_.store(0, std::memory_order_release);+      return;+    }+    // create and initialize the new level+    uint32_t tmp_btm = getBottomLevel();+    uint32_t size = 1 << (tmp_btm + 1);+    MoundElement* new_level = new MoundElement[size]; // MM+    levels_[tmp_btm + 1] = new_level;+    bottom_.store(tmp_btm + 1, std::memory_order_release);+    guard_.store(0, std::memory_order_release);+  }++  /// TODO: optimization+  // This function is important, it selects a position to insert the+  // node, there are two execution paths when this function returns.+  // 1. It returns a position with head node has lower priority than the target.+  // Thus it could be potentially used as the starting element to do the binary+  // search to find the fit position.  (slow path)+  // 2. It returns a position, which is not the best fit.+  // But it prevents aggressively grow the Mound. (fast path)+  Position selectPosition(+      const T& val,+      bool& path,+      uint32_t& seed,+      folly::hazptr_holder<Atom>& hptr) {+    while (true) {+      uint32_t b = getBottomLevel();+      int bound = 1 << b; // number of elements in this level+      int steps = 1 + b * b; // probe the length+      ++seed;+      uint32_t index = seed % bound;++      for (int i = 0; i < steps; i++) {+        int loc = (index + i) % bound;+        Position pos;+        pos.level = b;+        pos.index = loc;+        // the first round, we do the quick check+        if (optimisticReadValue(pos, hptr) <= val) {+          path = false;+          seed = ++loc;+          return pos;+        } else if (+            b > LevelForForceInsert && getElementSize(pos) < ListTargetSize) {+          // [fast path] conservative implementation+          // it makes sure every tree element should+          // have more than the given number of nodes.+          seed = ++loc;+          path = true;+          return pos;+        }+        if (b != getBottomLevel()) {+          break;+        }+      }+      // failed too many times grow+      if (b == getBottomLevel()) {+        grow(b);+      }+    }+  }++  /// Swap two Tree Elements (head, size)+  void swapList(const Position& a, const Position& b) {+    Node* tmp = getList(a);+    setTreeNode(a, getList(b));+    setTreeNode(b, tmp);++    // need to swap the tree node meta-data+    uint32_t sa = getElementSize(a);+    uint32_t sb = getElementSize(b);+    setElementSize(a, sb);+    setElementSize(b, sa);+  }++  FOLLY_ALWAYS_INLINE void lockNode(const Position& pos) {+    levels_[pos.level][pos.index].lock.lock();+  }++  FOLLY_ALWAYS_INLINE void unlockNode(const Position& pos) {+    levels_[pos.level][pos.index].lock.unlock();+  }++  FOLLY_ALWAYS_INLINE bool trylockNode(const Position& pos) {+    return levels_[pos.level][pos.index].lock.try_lock();+  }++  FOLLY_ALWAYS_INLINE T+  optimisticReadValue(const Position& pos, folly::hazptr_holder<Atom>& hptr) {+    Node* tmp = hptr.protect(levels_[pos.level][pos.index].head);+    return (tmp == nullptr) ? MIN_VALUE : tmp->val;+  }++  // Get the value from the head of the list as the elementvalue+  FOLLY_ALWAYS_INLINE T readValue(const Position& pos) {+    Node* tmp = getList(pos);+    return (tmp == nullptr) ? MIN_VALUE : tmp->val;+  }++  FOLLY_ALWAYS_INLINE Node* getList(const Position& pos) {+    return levels_[pos.level][pos.index].head.load(std::memory_order_acquire);+  }++  FOLLY_ALWAYS_INLINE void setTreeNode(const Position& pos, Node* t) {+    levels_[pos.level][pos.index].head.store(t, std::memory_order_release);+  }++  // Merge two sorted lists+  Node* mergeList(Node* base, Node* source) {+    if (base == nullptr) {+      return source;+    } else if (source == nullptr) {+      return base;+    }++    Node *res, *p;+    // choose the head node+    if (base->val >= source->val) {+      res = base;+      base = base->next;+      p = res;+    } else {+      res = source;+      source = source->next;+      p = res;+    }++    while (base != nullptr && source != nullptr) {+      if (base->val >= source->val) {+        p->next = base;+        base = base->next;+      } else {+        p->next = source;+        source = source->next;+      }+      p = p->next;+    }+    if (base == nullptr) {+      p->next = source;+    } else {+      p->next = base;+    }+    return res;+  }++  /// Merge list t to the Element Position+  void mergeListTo(const Position& pos, Node* t, const size_t& list_length) {+    Node* head = getList(pos);+    setTreeNode(pos, mergeList(head, t));+    uint32_t ns = getElementSize(pos) + list_length;+    setElementSize(pos, ns);+  }++  bool pruningLeaf(const Position& pos) {+    if (getElementSize(pos) <= PruningSize) {+      unlockNode(pos);+      return true;+    }++    int b = getBottomLevel();+    int leaves = 1 << b;+    int cnodes = 0;+    for (int i = 0; i < leaves; i++) {+      Position tmp;+      tmp.level = b;+      tmp.index = i;+      if (getElementSize(tmp) != 0) {+        cnodes++;+      }+      if (cnodes > leaves * 2 / 3) {+        break;+      }+    }++    if (cnodes <= leaves * 2 / 3) {+      unlockNode(pos);+      return true;+    }+    return false;+  }++  /// Split the current list into two lists,+  /// then split the tail list and merge to two children.+  void startPruning(const Position& pos) {+    if (isLeaf(pos) && pruningLeaf(pos)) {+      return;+    }++    // split the list, record the tail+    Node* pruning_head = getList(pos);+    int steps = ListTargetSize; // keep in the original list+    for (int i = 0; i < steps - 1; i++) {+      pruning_head = pruning_head->next;+    }+    Node* t = pruning_head;+    pruning_head = pruning_head->next;+    t->next = nullptr;+    int tail_length = getElementSize(pos) - steps;+    setElementSize(pos, steps);++    // split the tail list into two lists+    // evenly merge to two children+    if (pos.level != getBottomLevel()) {+      // split the rest into two lists+      int left_length = (tail_length + 1) / 2;+      int right_length = tail_length - left_length;+      Node *to_right, *to_left = pruning_head;+      for (int i = 0; i < left_length - 1; i++) {+        pruning_head = pruning_head->next;+      }+      to_right = pruning_head->next;+      pruning_head->next = nullptr;++      Position lchild = leftOf(pos);+      Position rchild = rightOf(pos);+      if (left_length != 0) {+        lockNode(lchild);+        mergeListTo(lchild, to_left, left_length);+      }+      if (right_length != 0) {+        lockNode(rchild);+        mergeListTo(rchild, to_right, right_length);+      }+      unlockNode(pos);+      if (left_length != 0 && getElementSize(lchild) > PruningSize) {+        startPruning(lchild);+      } else if (left_length != 0) {+        unlockNode(lchild);+      }+      if (right_length != 0 && getElementSize(rchild) > PruningSize) {+        startPruning(rchild);+      } else if (right_length != 0) {+        unlockNode(rchild);+      }+    } else { // time to grow the Mound+      grow(pos.level);+      // randomly choose a child to insert+      if (steps % 2 == 1) {+        Position rchild = rightOf(pos);+        lockNode(rchild);+        mergeListTo(rchild, pruning_head, tail_length);+        unlockNode(pos);+        unlockNode(rchild);+      } else {+        Position lchild = leftOf(pos);+        lockNode(lchild);+        mergeListTo(lchild, pruning_head, tail_length);+        unlockNode(pos);+        unlockNode(lchild);+      }+    }+  }++  // This function insert the new node (always) at the head of the+  // current list. It needs to lock the parent & current+  // This function may cause the list becoming tooooo long, so we+  // provide pruning algorithm.+  bool regularInsert(const Position& pos, const T& val, Node* newNode) {+    // insert to the root node+    if (isRoot(pos)) {+      lockNode(pos);+      T nv = readValue(pos);+      if (FOLLY_LIKELY(nv <= val)) {+        newNode->next = getList(pos);+        setTreeNode(pos, newNode);+        uint32_t sz = getElementSize(pos);+        setElementSize(pos, sz + 1);+        if (FOLLY_UNLIKELY(sz > PruningSize)) {+          startPruning(pos);+        } else {+          unlockNode(pos);+        }+        return true;+      }+      unlockNode(pos);+      return false;+    }++    // insert to an inner node+    Position parent = parentOf(pos);+    if (!trylockNode(parent)) {+      return false;+    }+    if (!trylockNode(pos)) {+      unlockNode(parent);+      return false;+    }+    T pv = readValue(parent);+    T nv = readValue(pos);+    if (FOLLY_LIKELY(pv > val && nv <= val)) {+      // improve the accuracy by getting the node(R) with less priority than the+      // new value from parent level, insert the new node to the parent list+      // and insert R to the current list.+      // It only happens at >= LevelForTraverseParent for reducing contention+      uint32_t sz = getElementSize(pos);+      if (pos.level >= LevelForTraverseParent) {+        Node* start = getList(parent);+        while (start->next != nullptr && start->next->val >= val) {+          start = start->next;+        }+        if (start->next != nullptr) {+          newNode->next = start->next;+          start->next = newNode;+          while (start->next->next != nullptr) {+            start = start->next;+          }+          newNode = start->next;+          start->next = nullptr;+        }+        unlockNode(parent);++        Node* curList = getList(pos);+        if (curList == nullptr) {+          newNode->next = nullptr;+          setTreeNode(pos, newNode);+        } else {+          Node* p = curList;+          if (p->val <= newNode->val) {+            newNode->next = curList;+            setTreeNode(pos, newNode);+          } else {+            while (p->next != nullptr && p->next->val >= newNode->val) {+              p = p->next;+            }+            newNode->next = p->next;+            p->next = newNode;+          }+        }+        setElementSize(pos, sz + 1);+      } else {+        unlockNode(parent);+        newNode->next = getList(pos);+        setTreeNode(pos, newNode);+        setElementSize(pos, sz + 1);+      }+      if (FOLLY_UNLIKELY(sz > PruningSize)) {+        startPruning(pos);+      } else {+        unlockNode(pos);+      }+      return true;+    }+    unlockNode(parent);+    unlockNode(pos);+    return false;+  }++  bool forceInsertToRoot(Node* newNode) {+    Position pos;+    pos.level = pos.index = 0;+    std::unique_lock lck(levels_[pos.level][pos.index].lock, std::try_to_lock);+    if (!lck.owns_lock()) {+      return false;+    }+    uint32_t sz = getElementSize(pos);+    if (sz >= ListTargetSize) {+      return false;+    }++    Node* curList = getList(pos);+    if (curList == nullptr) {+      newNode->next = nullptr;+      setTreeNode(pos, newNode);+    } else {+      Node* p = curList;+      if (p->val <= newNode->val) {+        newNode->next = curList;+        setTreeNode(pos, newNode);+      } else {+        while (p->next != nullptr && p->next->val >= newNode->val) {+          p = p->next;+        }+        newNode->next = p->next;+        p->next = newNode;+      }+    }+    setElementSize(pos, sz + 1);+    return true;+  }++  // This function forces the new node inserting to the current position+  // if the element does not hold the enough nodes. It is safe to+  // lock just one position to insert, because it won't be the first+  // node to sustain the heap structure.+  bool forceInsert(const Position& pos, const T& val, Node* newNode) {+    if (isRoot(pos)) {+      return forceInsertToRoot(newNode);+    }++    while (true) {+      std::unique_lock lck(+          levels_[pos.level][pos.index].lock, std::try_to_lock);+      if (!lck.owns_lock()) {+        if (getElementSize(pos) < ListTargetSize && readValue(pos) >= val) {+          continue;+        } else {+          return false;+        }+      }+      T nv = readValue(pos);+      uint32_t sz = getElementSize(pos);+      // do not allow the new node to be the first one+      // do not allow the list size tooooo big+      if (FOLLY_UNLIKELY(nv < val || sz >= ListTargetSize)) {+        return false;+      }++      Node* p = getList(pos);+      // find a place to insert the node+      while (p->next != nullptr && p->next->val > val) {+        p = p->next;+      }+      newNode->next = p->next;+      p->next = newNode;+      // do not forget to change the metadata+      setElementSize(pos, sz + 1);+      return true;+    }+  }++  void binarySearchPosition(+      Position& cur, const T& val, folly::hazptr_holder<Atom>& hptr) {+    Position parent, mid;+    if (cur.level == 0) {+      return;+    }+    // start from the root+    parent.level = parent.index = 0;++    while (true) { // binary search+      mid.level = (cur.level + parent.level) / 2;+      mid.index = cur.index >> (cur.level - mid.level);++      T mv = optimisticReadValue(mid, hptr);+      if (val < mv) {+        parent = mid;+      } else {+        cur = mid;+      }++      if (mid.level == 0 || // the root+          ((parent.level + 1 == cur.level) && parent.level != 0)) {+        return;+      }+    }+  }++  // The push keeps the length of each element stable+  void moundPush(const T& val) {+    Position cur;+    folly::hazptr_holder<Atom> hptr = folly::make_hazard_pointer<Atom>();+    Node* newNode = new Node;+    newNode->val = val;+    uint32_t seed = folly::Random::rand32() % (1 << 21);++    while (true) {+      // shell we go the fast path?+      bool go_fast_path = false;+      // chooice the right node to start+      cur = selectPosition(val, go_fast_path, seed, hptr);+      if (go_fast_path) {+        if (FOLLY_LIKELY(forceInsert(cur, val, newNode))) {+          if (MayBlock) {+            blockingPushImpl();+          }+          return;+        } else {+          continue;+        }+      }++      binarySearchPosition(cur, val, hptr);+      if (FOLLY_LIKELY(regularInsert(cur, val, newNode))) {+        if (MayBlock) {+          blockingPushImpl();+        }+        return;+      }+    }+  }++  int popToSharedBuffer(const uint32_t rsize, Node* head) {+    Position pos;+    pos.level = pos.index = 0;++    int num = std::min(rsize, (uint32_t)PopBatch);+    for (int i = num - 1; i >= 0; i--) {+      // wait until this block is empty+      while (shared_buffer_[i].pnode.load(std::memory_order_relaxed) != nullptr)+        ;+      shared_buffer_[i].pnode.store(head, std::memory_order_relaxed);+      head = head->next;+    }+    if (num > 0) {+      top_loc_.store(num - 1, std::memory_order_release);+    }+    setTreeNode(pos, head);+    return rsize - num;+  }++  void mergeDown(const Position& pos) {+    if (isLeaf(pos)) {+      unlockNode(pos);+      return;+    }++    // acquire locks for L and R and compare+    Position lchild = leftOf(pos);+    Position rchild = rightOf(pos);+    lockNode(lchild);+    lockNode(rchild);+    // read values+    T nv = readValue(pos);+    T lv = readValue(lchild);+    T rv = readValue(rchild);+    if (nv >= lv && nv >= rv) {+      unlockNode(pos);+      unlockNode(lchild);+      unlockNode(rchild);+      return;+    }++    // If two children contains nodes less than the+    // threshold, we merge two children to the parent+    // and do merge down on both of them.+    size_t sum =+        getElementSize(rchild) + getElementSize(lchild) + getElementSize(pos);+    if (sum <= MergingSize) {+      Node* l1 = mergeList(getList(rchild), getList(lchild));+      setTreeNode(pos, mergeList(l1, getList(pos)));+      setElementSize(pos, sum);+      setTreeNode(lchild, nullptr);+      setElementSize(lchild, 0);+      setTreeNode(rchild, nullptr);+      setElementSize(rchild, 0);+      unlockNode(pos);+      mergeDown(lchild);+      mergeDown(rchild);+      return;+    }+    // pull from right+    if (rv >= lv && rv > nv) {+      swapList(rchild, pos);+      unlockNode(pos);+      unlockNode(lchild);+      mergeDown(rchild);+    } else if (lv >= rv && lv > nv) {+      // pull from left+      swapList(lchild, pos);+      unlockNode(pos);+      unlockNode(rchild);+      mergeDown(lchild);+    }+  }++  bool deferSettingRootSize(Position& pos) {+    if (isLeaf(pos)) {+      setElementSize(pos, 0);+      unlockNode(pos);+      return true;+    }++    // acquire locks for L and R and compare+    Position lchild = leftOf(pos);+    Position rchild = rightOf(pos);+    lockNode(lchild);+    lockNode(rchild);+    if (getElementSize(lchild) == 0 && getElementSize(rchild) == 0) {+      setElementSize(pos, 0);+      unlockNode(pos);+      unlockNode(lchild);+      unlockNode(rchild);+      return true;+    } else {+      // read values+      T lv = readValue(lchild);+      T rv = readValue(rchild);+      if (lv >= rv) {+        swapList(lchild, pos);+        setElementSize(lchild, 0);+        unlockNode(pos);+        unlockNode(rchild);+        pos = lchild;+      } else {+        swapList(rchild, pos);+        setElementSize(rchild, 0);+        unlockNode(pos);+        unlockNode(lchild);+        pos = rchild;+      }+      return false;+    }+  }++  bool moundPopMany(T& val) {+    // pop from the root+    Position pos;+    pos.level = pos.index = 0;+    // the root is nullptr, return false+    Node* head = getList(pos);+    if (head == nullptr) {+      unlockNode(pos);+      return false;+    }++    // shared buffer already filled by other threads+    if (PopBatch > 0 && top_loc_.load(std::memory_order_acquire) >= 0) {+      unlockNode(pos);+      return false;+    }++    uint32_t sz = getElementSize(pos);+    // get the one node first+    val = head->val;+    Node* p = head;+    head = head->next;+    sz--;++    if (PopBatch > 0) {+      sz = popToSharedBuffer(sz, head);+    } else {+      setTreeNode(pos, head);+    }++    bool done = false;+    if (FOLLY_LIKELY(sz == 0)) {+      done = deferSettingRootSize(pos);+    } else {+      setElementSize(pos, sz);+    }++    if (FOLLY_LIKELY(!done)) {+      mergeDown(pos);+    }++    p->retire();+    return true;+  }++  void blockingPushImpl() {+    auto p = pticket_.fetch_add(1, std::memory_order_acq_rel);+    auto loc = getFutexArrayLoc(p);+    uint32_t curfutex = futex_array_[loc].load(std::memory_order_acquire);++    while (true) {+      uint32_t ready = p << 1; // get the lower 31 bits+      // avoid the situation that push has larger ticket already set the value+      if (FOLLY_UNLIKELY(+              ready + 1 < curfutex ||+              ((curfutex > ready) && (curfutex - ready > 0x40000000)))) {+        return;+      }++      if (futex_array_[loc].compare_exchange_strong(curfutex, ready)) {+        if (curfutex &+            1) { // One or more consumers may be blocked on this futex+          detail::futexWake(&futex_array_[loc]);+        }+        return;+      } else {+        curfutex = futex_array_[loc].load(std::memory_order_acquire);+      }+    }+  }++  // This could guarentee the Mound is empty+  FOLLY_ALWAYS_INLINE bool isMoundEmpty() {+    Position pos;+    pos.level = pos.index = 0;+    return getElementSize(pos) == 0;+  }++  // Return true if the shared buffer is empty+  FOLLY_ALWAYS_INLINE bool isSharedBufferEmpty() {+    return top_loc_.load(std::memory_order_acquire) < 0;+  }++  FOLLY_ALWAYS_INLINE bool isEmpty() {+    if (PopBatch > 0) {+      return isMoundEmpty() && isSharedBufferEmpty();+    }+    return isMoundEmpty();+  }++  FOLLY_ALWAYS_INLINE bool futexIsReady(const size_t& curticket) {+    auto loc = getFutexArrayLoc(curticket);+    auto curfutex = futex_array_[loc].load(std::memory_order_acquire);+    uint32_t short_cticket = curticket & 0x7FFFFFFF;+    uint32_t futex_ready = curfutex >> 1;+    // handle unsigned 31 bits overflow+    return futex_ready >= short_cticket ||+        short_cticket - futex_ready > 0x40000000;+  }++  template <typename Clock, typename Duration>+  FOLLY_NOINLINE bool trySpinBeforeBlock(+      const size_t& curticket,+      const std::chrono::time_point<Clock, Duration>& deadline,+      const folly::WaitOptions& opt = wait_options()) {+    return folly::detail::spin_pause_until(deadline, opt, [=] {+             return futexIsReady(curticket);+           }) == folly::detail::spin_result::success;+  }++  void tryBlockingPop(const size_t& curticket) {+    auto loc = getFutexArrayLoc(curticket);+    auto curfutex = futex_array_[loc].load(std::memory_order_acquire);+    if (curfutex &+        1) { /// The last round consumers are still waiting, go to sleep+      detail::futexWait(&futex_array_[loc], curfutex);+    }+    if (trySpinBeforeBlock(+            curticket,+            std::chrono::time_point<std::chrono::steady_clock>::max())) {+      return; /// Spin until the push ticket is ready+    }+    while (true) {+      curfutex = futex_array_[loc].load(std::memory_order_acquire);+      if (curfutex &+          1) { /// The last round consumers are still waiting, go to sleep+        detail::futexWait(&futex_array_[loc], curfutex);+      } else if (!futexIsReady(curticket)) { // current ticket < pop ticket+        uint32_t blocking_futex = curfutex + 1;+        if (futex_array_[loc].compare_exchange_strong(+                curfutex, blocking_futex)) {+          detail::futexWait(&futex_array_[loc], blocking_futex);+        }+      } else {+        return;+      }+    }+  }++  void blockingPopImpl() {+    auto ct = cticket_.fetch_add(1, std::memory_order_acq_rel);+    // fast path check+    if (futexIsReady(ct)) {+      return;+    }+    // Blocking+    tryBlockingPop(ct);+  }++  bool tryPopFromMound(T& val) {+    if (isMoundEmpty()) {+      return false;+    }+    Position pos;+    pos.level = pos.index = 0;++    // lock the root+    if (trylockNode(pos)) {+      return moundPopMany(val);+    }+    return false;+  }++  FOLLY_ALWAYS_INLINE static folly::WaitOptions wait_options() { return {}; }++  template <typename Clock, typename Duration>+  FOLLY_NOINLINE bool tryWait(+      const std::chrono::time_point<Clock, Duration>& deadline,+      const folly::WaitOptions& opt = wait_options()) {+    // Fast path, by quick check the status+    switch (folly::detail::spin_pause_until(deadline, opt, [=] {+      return !isEmpty();+    })) {+      case folly::detail::spin_result::success:+        return true;+      case folly::detail::spin_result::timeout:+        return false;+      case folly::detail::spin_result::advance:+        break;+    }++    // Spinning strategy+    while (true) {+      auto res = folly::detail::spin_yield_until(deadline, [=] {+        return !isEmpty();+      });+      if (res == folly::detail::spin_result::success) {+        return true;+      } else if (res == folly::detail::spin_result::timeout) {+        return false;+      }+    }+    return true;+  }++  bool tryPopFromSharedBuffer(T& val) {+    int get_or = -1;+    if (!isSharedBufferEmpty()) {+      get_or = top_loc_.fetch_sub(1, std::memory_order_acq_rel);+      if (get_or >= 0) {+        Node* c = shared_buffer_[get_or].pnode.load(std::memory_order_relaxed);+        shared_buffer_[get_or].pnode.store(nullptr, std::memory_order_release);+        val = c->val;+        c->retire();+        return true;+      }+    }+    return false;+  }++  size_t getFutexArrayLoc(size_t s) {+    return ((s - 1) * Stride) & (NumFutex - 1);+  }++  void moundPop(T& val) {+    if (MayBlock) {+      blockingPopImpl();+    }++    if (PopBatch > 0) {+      if (tryPopFromSharedBuffer(val)) {+        return;+      }+    }++    while (true) {+      if (FOLLY_LIKELY(tryPopFromMound(val))) {+        return;+      }+      tryWait(std::chrono::time_point<std::chrono::steady_clock>::max());+      if (PopBatch > 0 && tryPopFromSharedBuffer(val)) {+        return;+      }+    }+  }+};++} // namespace folly
+ folly/folly/concurrency/container/SingleWriterFixedHashMap.h view
@@ -0,0 +1,322 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <atomic>++#include <folly/lang/Bits.h>++#include <glog/logging.h>++namespace folly {++/// SingleWriterFixedHashMap:+///+/// Minimal single-writer fixed hash map implementation that supports:+/// - Copy construction with optional capacity expansion.+/// - Concurrent read-only lookup.+/// - Concurrent read-only iteration.+///+/// Assumes that higher level code:+/// - Checks availability of empty slots before calling insert+/// - Manages expansion and/or cleanup of tombstones+/// - Ensures no concurrent mutations to the copy constructor argument+///+/// Notes on algorithm:+/// - Tombstones are used to mark previously occupied slots.+/// - A slot with a tombstone can only be reused for the same key. The+///   reason for that is to enforce that once a key occupies a slot,+///   that key cannot use any other slot for the lifetime of the+///   map. This is to guarantee that when readers iterate over the map+///   they do not encounter any key more than once.+///+/// Writer-only operations:+/// - insert()+/// - erase()+/// - used()+/// - available()+///+/// This implementation guarantees that a copy from a map with+/// tombstones will have at least one available empty element.+///+template <typename Key, typename Value>+class SingleWriterFixedHashMap {+  static_assert(+      std::atomic<Value>::is_always_lock_free,+      "This implementation depends on having fast atomic "+      "data-race-free loads and stores of Value type.");+  static_assert(+      std::is_trivial<Key>::value,+      "This implementation depends on using a single key instance "+      "for all insert and erase operations. The reason is to allow "+      "readers to read keys data-race-free concurrently with possible "+      "concurrent insert and erase operations on the keys.");++  class Elem;++  enum class State : uint8_t { EMPTY, VALID, TOMBSTONE };++  size_t capacity_;+  size_t used_{0};+  std::atomic<size_t> size_{0};+  std::unique_ptr<Elem[]> elem_;++ public:+  class Iterator;++  explicit SingleWriterFixedHashMap(size_t capacity)+      : capacity_(folly::nextPowTwo(capacity)) {}++  explicit SingleWriterFixedHashMap(+      size_t capacity, const SingleWriterFixedHashMap& o)+      : capacity_(folly::nextPowTwo(capacity)) {+    if (o.empty()) {+      return;+    }+    elem_ = std::make_unique<Elem[]>(capacity_);+    if (capacity_ == o.capacity_ &&+        (o.used_ < o.capacity_ || o.size() == o.capacity_)) {+      std::memcpy(+          static_cast<void*>(elem_.get()),+          static_cast<const void*>(o.elem_.get()),+          capacity_ * sizeof(Elem));+      used_ = o.used_;+      setSize(o.size());+      return;+    }+    for (size_t i = 0; i < o.capacity_; ++i) {+      Elem& e = o.elem_[i];+      if (e.valid()) {+        insert(e.key(), e.value());+      }+    }+  }++  FOLLY_ALWAYS_INLINE Iterator begin() const {+    return empty() ? end() : Iterator(*this);+  }++  FOLLY_ALWAYS_INLINE Iterator end() const {+    return Iterator(*this, capacity_);+  }++  size_t capacity() const { return capacity_; }++  /* not data-race-free, to be called only by the single writer */+  size_t used() const { return used_; }++  /* not-data race-free, to be called only by the single writer */+  size_t available() const { return capacity_ - used_; }++  /* data-race-free, can be called by readers */+  FOLLY_ALWAYS_INLINE size_t size() const {+    return size_.load(std::memory_order_acquire);+  }++  FOLLY_ALWAYS_INLINE bool empty() const { return size() == 0; }++  bool insert(Key key, Value value) {+    if (!elem_) {+      elem_ = std::make_unique<Elem[]>(capacity_);+    }+    DCHECK_LT(used_, capacity_);+    if (writer_find(key) < capacity_) {+      return false;+    }+    size_t index = hash(key);+    auto attempts = capacity_;+    size_t mask = capacity_ - 1;+    while (attempts--) {+      Elem& e = elem_[index];+      auto state = e.state();+      if (state == State::EMPTY ||+          (state == State::TOMBSTONE && e.key() == key)) {+        if (state == State::EMPTY) {+          e.setKey(key);+          ++used_;+          DCHECK_LE(used_, capacity_);+        }+        e.setValue(value);+        e.setValid();+        setSize(size() + 1);+        DCHECK_LE(size(), used_);+        return true;+      }+      index = (index + 1) & mask;+    }+    CHECK(false) << "No available slots";+    folly::assume_unreachable();+  }++  void erase(Iterator& it) {+    DCHECK_NE(it, end());+    Elem& e = elem_[it.index_];+    erase_internal(e);+  }++  bool erase(Key key) {+    size_t index = writer_find(key);+    if (index == capacity_) {+      return false;+    }+    Elem& e = elem_[index];+    erase_internal(e);+    return true;+  }++  FOLLY_ALWAYS_INLINE Iterator find(Key key) const {+    size_t index = reader_find(key);+    return Iterator(*this, index);+  }++  FOLLY_ALWAYS_INLINE bool contains(Key key) const {+    return reader_find(key) < capacity_;+  }++ private:+  FOLLY_ALWAYS_INLINE size_t hash(Key key) const {+    size_t mask = capacity_ - 1;+    size_t index = std::hash<Key>()(key) & mask;+    DCHECK_LT(index, capacity_);+    return index;+  }++  void setSize(size_t size) { size_.store(size, std::memory_order_release); }++  FOLLY_ALWAYS_INLINE size_t reader_find(Key key) const {+    return find_internal(key);+  }++  size_t writer_find(Key key) { return find_internal(key); }++  FOLLY_ALWAYS_INLINE size_t find_internal(Key key) const {+    if (!empty()) {+      size_t index = hash(key);+      auto attempts = capacity_;+      size_t mask = capacity_ - 1;+      while (attempts--) {+        Elem& e = elem_[index];+        auto state = e.state();+        if (state == State::VALID && e.key() == key) {+          return index;+        }+        if (state == State::EMPTY) {+          break;+        }+        index = (index + 1) & mask;+      }+    }+    return capacity_;+  }++  void erase_internal(Elem& e) {+    e.erase();+    DCHECK_GT(size(), 0);+    setSize(size() - 1);+  }++  /// Elem+  class Elem {+    std::atomic<State> state_;+    Key key_;+    std::atomic<Value> value_;++   public:+    Elem() : state_(State::EMPTY) {}++    FOLLY_ALWAYS_INLINE State state() const {+      return state_.load(std::memory_order_acquire);+    }++    FOLLY_ALWAYS_INLINE bool valid() const { return state() == State::VALID; }++    FOLLY_ALWAYS_INLINE Key key() const { return key_; }++    FOLLY_ALWAYS_INLINE Value value() const {+      return value_.load(std::memory_order_relaxed);+    }++    void setKey(Key key) { key_ = key; }++    void setValue(Value value) {+      value_.store(value, std::memory_order_relaxed);+    }++    void setValid() { state_.store(State::VALID, std::memory_order_release); }++    void erase() { state_.store(State::TOMBSTONE, std::memory_order_release); }+  }; // Elem++ public:+  /// Iterator+  class Iterator {+    Elem* elem_;+    size_t capacity_;+    size_t index_;++   public:+    FOLLY_ALWAYS_INLINE Key key() const {+      DCHECK_LT(index_, capacity_);+      Elem& e = elem_[index_];+      return e.key();+    }++    FOLLY_ALWAYS_INLINE Value value() const {+      DCHECK_LT(index_, capacity_);+      Elem& e = elem_[index_];+      return e.value();+    }++    FOLLY_ALWAYS_INLINE Iterator& operator++() {+      DCHECK_LT(index_, capacity_);+      ++index_;+      next();+      return *this;+    }++    FOLLY_ALWAYS_INLINE bool operator==(const Iterator& o) const {+      DCHECK(elem_ == o.elem_ || elem_ == nullptr || o.elem_ == nullptr);+      DCHECK_EQ(capacity_, o.capacity_);+      DCHECK_LE(index_, capacity_);+      return index_ == o.index_;+    }++    FOLLY_ALWAYS_INLINE bool operator!=(const Iterator& o) const {+      return !(*this == o);+    }++   private:+    friend class SingleWriterFixedHashMap;++    explicit Iterator(const SingleWriterFixedHashMap& m, size_t i = 0)+        : elem_(i == m.capacity_ ? nullptr : m.elem_.get()),+          capacity_(m.capacity_),+          index_(i) {+      if (index_ < capacity_) {+        next();+      }+    }++    FOLLY_ALWAYS_INLINE void next() {+      while (index_ < capacity_ && !elem_[index_].valid()) {+        ++index_;+      }+    }+  }; // Iterator+}; // SingleWriterFixedHashMap++} // namespace folly
+ folly/folly/concurrency/container/atomic_grow_array.h view
@@ -0,0 +1,589 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <atomic>+#include <cstddef>+#include <new>+#include <thread>++#include <folly/CPortability.h>+#include <folly/ConstexprMath.h>+#include <folly/Likely.h>+#include <folly/ScopeGuard.h>+#include <folly/container/span.h>+#include <folly/lang/Align.h>+#include <folly/lang/Bits.h>+#include <folly/lang/New.h>++namespace folly {++/// atomic_grow_array_policy_default+///+/// A default or example policy for use with atomic_grow_array.+template <typename Item>+struct atomic_grow_array_policy_default {+  std::size_t grow(+      std::size_t /* const curr */, std::size_t const index) const noexcept {+    return nextPowTwo(index + 1);+  }+  Item make() const noexcept(noexcept(Item())) { return Item(); }+};++/// atomic_grow_array+///+/// A specialized data structure roughly modeling an infinite heap-allocated+/// array. The array is of course not actually infinite in size, but indexed+/// access at any index is always permitted. The container features both+/// reference-stability and iterator-stability.+///+/// Supports fast concurrent `operator[](size_t)`, which returns a reference to+/// the element at the given position. Indexed access is wait-free, modulo the+/// allocator algorithm and modulo element constructors. Indexed access has and+/// is intended to have a fast, minimal instruction sequence.+///+/// The array capacity may only grow and not shrink. When array capacity grows,+/// the array size grows to fill the capacity. This means that all the elements+/// that would be required to fill the capacity are allocated and constructed+/// during growth. Moreover, multiple threads may race to grow the capacity, in+/// which case only one thread wins the race - the threads losing the race then+/// destroy all elements they created in that round of racing to grow the array+/// capacity.+///+/// With the default policy featuring power-of-two exponential growth, the+/// number of outstanding allocations is:+///   log2(capacity)+/// And the outstanding allocations' sizes sum to:+///   log2(capacity) * 2 * sizeof(void*) <--- array-segment metadata+///   capacity * 2 * sizeof(void*) <--- array-segment pointers list+///   capacity * sizeof(value_type) <--- array-segment elements slab+/// Modulo allocator size-classes, of which this container makes no attempt to+/// take advantage, and value-type alignment.+template <+    typename Item,+    typename Policy = atomic_grow_array_policy_default<Item>>+class atomic_grow_array : private Policy {+ public:+  using size_type = std::size_t;+  using value_type = Item;++  using pointer_span = span<value_type* const>;+  using const_pointer_span = span<value_type const* const>;++  class iterator;+  class const_iterator;++ private:+  static constexpr bool is_nothrow_grow_v =+      noexcept(FOLLY_DECLVAL(Policy const&).grow(0, 0)) &&+      noexcept(FOLLY_DECLVAL(Policy const&).make()) &&+      noexcept(::operator new(0));++  struct array;++  struct end_tag {};++  template <bool>+  class basic_view;++  template <bool Const, typename Down>+  class basic_iterator {+   private:+    template <bool, typename>+    friend class basic_iterator;+    template <bool>+    friend class basic_view;++    using self = basic_iterator;+    using down = Down;+    friend down;++    template <typename T>+    using maybe_add_const_t = conditional_t<Const, T const, T>;++    array const* array_{};+    size_type index_{};++    basic_iterator(array const* const a, size_type const i) noexcept+        : array_{a}, index_{i} {}+    basic_iterator(array const* const a, end_tag) noexcept+        : array_{a}, index_{a ? a->size : 0} {}++    template <+        bool ThatC,+        typename ThatDown,+        bool ThisC = Const,+        typename = std::enable_if_t<!ThatC && ThisC>>+    explicit basic_iterator(basic_iterator<ThatC, ThatDown> that) noexcept+        : array_{that.array_}, index_{that.index_} {}++    down& as_down() noexcept { return static_cast<down&>(*this); }+    down const& as_down() const noexcept {+      return static_cast<down const&>(*this);+    }++   public:+    using iterator_category = std::random_access_iterator_tag;+    using value_type = atomic_grow_array::value_type;+    using difference_type = std::ptrdiff_t;+    using pointer = maybe_add_const_t<value_type>*;+    using reference = maybe_add_const_t<value_type>&;++    basic_iterator() = default; // produces an invalid iterator++    down& operator++() noexcept { return ++index_, as_down(); }+    down operator++(int) noexcept { return down{array_, index_++}; }+    down& operator+=(difference_type const n) noexcept {+      return index_ += n, as_down();+    }+    down operator+(difference_type const n) noexcept {+      return down{as_down()} += n;+    }+    down& operator-=(difference_type const n) noexcept {+      return index_ -= n, as_down();+    }+    down operator-(difference_type const n) noexcept {+      return down{as_down()} -= n;+    }+    friend difference_type operator-(down const lhs, down const rhs) noexcept {+      return lhs.index_ - rhs.index_;+    }+    friend bool operator==(down const lhs, down const rhs) noexcept {+      return lhs.index_ == rhs.index_;+    }+    friend bool operator!=(down const lhs, down const rhs) noexcept {+      return lhs.index_ != rhs.index_;+    }+    friend bool operator<(down const lhs, down const rhs) noexcept {+      return lhs.index < rhs.index_;+    }+    friend bool operator<=(down const lhs, down const rhs) noexcept {+      return lhs.index <= rhs.index_;+    }+    friend bool operator>(down const lhs, down const rhs) noexcept {+      return lhs.index > rhs.index_;+    }+    friend bool operator>=(down const lhs, down const rhs) noexcept {+      return lhs.index >= rhs.index_;+    }+    reference operator*() noexcept { return *array_->list[index_]; }+    reference operator[](difference_type const n) { return *(*this + n); }+  };++  template <bool Const>+  class basic_view {+   private:+    friend atomic_grow_array;++    template <typename T>+    using maybe_add_const_t = conditional_t<Const, T const, T>;++    using up = atomic_grow_array;++    array const* array_{};++    explicit basic_view(array const* arr) noexcept : array_{arr} {}++    template <+        bool ThatC,+        bool ThisC = Const,+        typename = std::enable_if_t<!ThatC && ThisC>>+    explicit basic_view(basic_view<ThatC> that) noexcept+        : array_{that.array_} {}++   public:+    using value_type = typename atomic_grow_array::value_type;+    using size_type = typename atomic_grow_array::size_type;+    using reference = maybe_add_const_t<value_type>&;+    using const_reference = value_type const&;+    using pointer = maybe_add_const_t<value_type>*;+    using const_pointer = value_type const*;+    using iterator = conditional_t<Const, up::const_iterator, up::iterator>;+    using const_iterator = up::const_iterator;++    basic_view() = default; // produces an invalid view++    iterator begin() noexcept { return iterator{array_, 0}; }+    const_iterator begin() const noexcept { return iterator{array_, 0}; }+    const_iterator cbegin() const noexcept { return iterator{array_, 0}; }+    iterator end() noexcept { return iterator{array_, end_tag{}}; }+    const_iterator end() const noexcept { return iterator{array_, end_tag{}}; }+    const_iterator cend() const noexcept { return iterator{array_, end_tag{}}; }++    size_type size() const noexcept { return array_ ? array_->size : 0; }+    bool empty() const noexcept { return !size(); }++    reference operator[](size_type index) noexcept {+      return *array_->list[index];+    }+    const_reference operator[](size_type index) const noexcept {+      return *array_->list[index];+    }++    span<pointer const> as_ptr_span() noexcept {+      using type = span<pointer const>;+      return array_ ? type{array_->list, array_->size} : type{};+    }+    span<const_pointer const> as_ptr_span() const noexcept {+      using type = span<const_pointer const>;+      return array_ ? type{array_->list, array_->size} : type{};+    }+  };++ public:+  atomic_grow_array() = default;+  explicit atomic_grow_array(Policy const& policy_) //+      noexcept(noexcept(Policy{policy_}))+      : Policy{policy_} {}+  ~atomic_grow_array() { reset(); }++  Policy const& policy() const noexcept {+    return static_cast<Policy const&>(*this);+  }++  /// size+  ///+  /// A recent value of the true size.+  ///+  /// Always a lower bound of - ie, never larger than - the true size.+  ///+  /// Example:+  ///+  ///   atomic_grow_array& array = /* ... */;+  ///   for (size_t i = 0; i < array.size(); ++i) {+  ///     do_something_with(array[i]);+  ///   }+  size_t size() const noexcept { return size_.load(mo_acquire); }++  /// empty+  ///+  /// Equivalent to size() == 0.+  bool empty() const noexcept { return size() == 0; }++  /// operator[]+  ///+  /// A reference to the element at the given index.+  ///+  /// Every index is always valid, modulo system memory size of course.+  ///+  /// The principal meaning is that it is not necessary to configure a capacity+  /// limit in all deployed environments, to monitor the accuracies of those+  /// capacity limit in all deployed environments, and to update those capacity+  /// limits as their accuracies suffer.+  ///+  /// Intended for dense indexed access patterns and not for sparse indexed+  /// access patterns. For the sparse case, at large sizes, a concurrent+  /// unordered-map would have much less memory overhead and much less growth+  /// compute overhead. The reason is that the memory overhead and growth+  /// compute overhead in this data structure, with the default policy, is+  /// proportional to the maximum index accessed, while for an unordered-map it+  /// would be proportional to the number of accessed indices.+  ///+  /// Indexed access during element construction is forbidden but undiagnosed.+  /// One likely scenario is stack overflow; another is memory exhaustion.+  FOLLY_ALWAYS_INLINE value_type& operator[](size_type const index) //+      noexcept(is_nothrow_grow_v) {+    auto const x = index < size_.load(mo_acquire);+    auto const p = FOLLY_LIKELY(x) ? array_.load(mo_acquire) : at_slow(index);+    return *p->list[index];+  }++  /// iterator+  ///+  /// An iterator type used by view.+  class iterator : private basic_iterator<false, iterator> {+    using base = basic_iterator<false, iterator>;+    friend base;+    friend const_iterator;+    template <bool>+    friend class basic_view;++   public:+    using typename base::difference_type;+    using typename base::iterator_category;+    using typename base::pointer;+    using typename base::reference;+    using typename base::value_type;++    using base::base;+    using base::operator++;+    using base::operator+;+    using base::operator+=;+    using base::operator-;+    using base::operator-=;+    using base::operator*;+    using base::operator[];+  };++  /// const_iterator+  ///+  /// An iterator type used by view and const_view.+  class const_iterator : private basic_iterator<true, const_iterator> {+    using base = basic_iterator<true, const_iterator>;+    friend base;+    template <bool>+    friend class basic_view;++   public:+    using typename base::difference_type;+    using typename base::iterator_category;+    using typename base::pointer;+    using typename base::reference;+    using typename base::value_type;++    using base::base;+    using base::operator++;+    using base::operator+;+    using base::operator+=;+    using base::operator-;+    using base::operator-=;+    using base::operator*;+    using base::operator[];++    /* implicit */ const_iterator(iterator that) noexcept : base{that} {}+  };++  /// view+  ///+  /// Models std::ranges::range.+  ///+  /// Gives a view over all of the elements available at the time the view was+  /// created. If the array capacity is later increased, the view will not cover+  /// the new elements but it and its iterators will all remain valid.+  class view : private basic_view<false> {+    friend atomic_grow_array;+    using base = basic_view<false>;++   public:+    using typename base::const_iterator;+    using typename base::const_reference;+    using typename base::iterator;+    using typename base::reference;+    using typename base::size_type;+    using typename base::value_type;++    using base::as_ptr_span;+    using base::base;+    using base::begin;+    using base::cbegin;+    using base::cend;+    using base::empty;+    using base::end;+    using base::size;+    using base::operator[];+  };++  /// const_view+  ///+  /// Models std::ranges::range.+  ///+  /// Gives a view over all of the elements available at the time the view was+  /// created. If the array capacity is later increased, the view will not cover+  /// the new elements but it and its iterators will all remain valid.+  class const_view : private basic_view<true> {+    friend atomic_grow_array;+    using base = basic_view<true>;++   public:+    using typename base::const_iterator;+    using typename base::const_reference;+    using typename base::iterator;+    using typename base::reference;+    using typename base::size_type;+    using typename base::value_type;++    using base::as_ptr_span;+    using base::base;+    using base::begin;+    using base::cbegin;+    using base::cend;+    using base::empty;+    using base::end;+    using base::size;+    using base::operator[];++    /* implicit */ const_view(view that) noexcept : base{that} {}+  };++  /// as_view+  ///+  /// Example:+  ///+  ///   atomic_grow_array& array = /* ... */;+  ///   for (auto& item : array.as_view()) {+  ///     do_something_with(item);+  ///   }+  ///+  /// If the atomic_grow_array is grown after the call to as_view(), the+  /// returned view will provide access only to the size and elements at the+  /// time it was created and not to a later size or to elements created later.+  ///+  /// Notes:+  /// * This exists since the choice is for atomic_grow_array not to model a+  ///   range directly. Such a range could not be as performant and could have+  ///   surprising behavior, such as this expression maybe evaluating to false:+  ///     (array.begin() == array.end()) == (array.begin() == array.end())+  /// * May be more performant than repeatedly indexing with operator[] up until+  ///   size(), even when size() is gotten once and then cached.+  view as_view() noexcept { return view{array_.load(mo_acquire)}; }+  const_view as_view() const noexcept { return view{array_.load(mo_acquire)}; }++  /// as_ptr_span+  ///+  /// Convenience wrapper for view::as_ptr_span.+  pointer_span as_ptr_span() noexcept { return as_view().as_ptr_span(); }+  const_pointer_span as_ptr_span() const noexcept {+    return as_view().as_ptr_span();+  }++ private:+  static constexpr auto mo_acquire = std::memory_order_acquire;+  static constexpr auto mo_release = std::memory_order_release;+  static constexpr auto mo_acq_rel = std::memory_order_acq_rel;++  //  prefix of layout structure+  //+  //  missing alignment and suffix because there are two variable-sized array+  //  fields+  struct array {+    array* next{}; // maybe null; const+    size_type size{}; // size != 0; size > (next ? next->size : 0); const+    value_type* list[]; // value_type* list[size]; const+    // value_type slab[size - (next ? next->size : 0)]; non-const+  };++  FOLLY_NOINLINE array* at_slow(size_type const index) //+      noexcept(is_nothrow_grow_v) {+    //  uses optimistic concurrency in order to avoid the space cost of any+    //  embedded mutex or the possible contention or deadlock from any global+    //  mutex or global mutex slab+    //+    //  deadlock from a global mutex or global mutex slab may occur if the+    //  value_type type constructor may also access the global mutex or global+    //  mutex slab, whether directly or indirectly+    array* p = array_.load(mo_acquire);+    array* q = nullptr;+    size_type const size = policy().grow(p ? p->size : 0, index);+    assert(index < size);+    do {+      if (p && index < p->size) {+        return p;+      }+      //  the race begins here+      q = new_array(size, p);+      if (!q) {+        //  the race is lost early+        continue;+      }+      //  this c/x only need success-release/failure-acquire, but c+++      //  implementations have trouble with that; so, success-acq-rel+      //  see: folly::atomic_compare_exchange_strong_explicit+      if (array_.compare_exchange_strong(p, q, mo_acq_rel, mo_acquire)) {+        //  the race is won+        size_.store(size, mo_release);+        return q;+      }+      //  the race is lost+      del_array(q);+    } while (1);+  }++  static constexpr size_type array_align() {+    return folly::constexpr_max(folly::max_align_v, alignof(value_type));+  }+  static size_type array_size(size_type const size, size_type const base) {+    constexpr auto a = array_align();+    return //+        folly::constexpr_ceil(sizeof(array) + size * sizeof(value_type*), a) ++        folly::constexpr_ceil((size - base) * sizeof(value_type), a);+  }+  static value_type* array_slab(array* const curr) {+    return reinterpret_cast<value_type*>(folly::constexpr_ceil(+        reinterpret_cast<uintptr_t>(&curr->list[curr->size]),+        static_cast<uintptr_t>(array_align())));+  }++  array* new_array(size_type const size, array*& next) {+    auto const base = next ? next->size : 0;+    assert(size > base);+    array* curr = static_cast<array*>(+        operator_new(array_size(size, base), std::align_val_t{array_align()}));+    auto rollback = folly::makeGuard([&] { del_array(curr); });+    curr->size = size;+    curr->next = next;+    auto const slab = array_slab(curr);+    //  copy pointers to all pre-existing elements; cannot throw+    for (size_type i = 0; i < base; ++i) {+      curr->list[i] = next->list[i];+    }+    //  zero-initialize to new elements; cannot throw+    for (size_type i = base; i < size; ++i) {+      curr->list[i] = nullptr;+    }+    //  initialize new elements and the pointers to them; may throw+    for (size_type i = base; i < size; ++i) {+      //  detect race losses early+      //  just need release, but acquire for consistency with c/x in at_slow+      if (auto const p = array_.load(std::memory_order_acquire); p != next) {+        next = p;+        return nullptr;+      }+      //  no race loss yet+      curr->list[i] = ::new (&slab[i - base]) value_type(policy().make());+    }+    rollback.dismiss();+    return curr;+  }++  void del_array(array* const curr) {+    assert(curr);+    auto size = curr->size;+    auto const next = curr->next;+    auto const base = next ? next->size : 0;+    assert(size > base);+    //  skip past zero-initialized pointers at the end since their corresponding+    //  elements were never created - this situation arises when initialization+    //  of an element fails within new_array and the rollback calls del_array+    while (size > base && !curr->list[size - 1]) {+      --size;+    }+    //  destroy elements owned by this array only, and not elements owned by any+    //  other arrays - those elements will be destroyed by del_array calls on+    //  their owning arrays+    for (size_type i = 0; i < size - base; ++i) {+      curr->list[size - 1 - i]->~value_type();+    }+    operator_delete(+        static_cast<void*>(curr),+        array_size(curr->size, base),+        std::align_val_t{array_align()});+  }++  void reset() {+    auto curr = array_.load(mo_acquire);+    while (curr) {+      auto const next = curr->next;+      del_array(curr);+      curr = next;+    }+  }++  std::atomic<size_type> size_{0};+  std::atomic<array*> array_{nullptr};+};++} // namespace folly
+ folly/folly/concurrency/detail/AtomicSharedPtr-detail.h view
@@ -0,0 +1,216 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <limits.h>+#include <atomic>+#include <memory>++#include <folly/lang/SafeAssert.h>++#if defined(__GLIBCXX__)++namespace folly {+namespace detail {++// This implementation is specific to libstdc++, now accepting+// diffs for other libraries.++// Specifically, this adds support for two things:+// 1) incrementing/decrementing the shared count by more than 1 at a time+// 2) Getting the thing the shared_ptr points to, which may be different from+//    the aliased pointer.++class shared_ptr_internals {+ public:+  template <typename T, typename... Args>+  static std::shared_ptr<T> make_ptr(Args&&... args) {+    return std::make_shared<T>(std::forward<Args...>(args...));+  }+  typedef std::__shared_count<std::_S_atomic> shared_count;+  typedef std::_Sp_counted_base<std::_S_atomic> counted_base;+  template <typename T>+  using CountedPtr = std::shared_ptr<T>;++  template <typename T>+  static counted_base* get_counted_base(const std::shared_ptr<T>& bar);++  static void inc_shared_count(counted_base* base, long count);++  template <typename T>+  static void release_shared(counted_base* base, long count);++  template <typename T>+  static T* get_shared_ptr(counted_base* base);++  template <typename T>+  static T* release_ptr(std::shared_ptr<T>& p);++  template <typename T>+  static std::shared_ptr<T> get_shared_ptr_from_counted_base(+      counted_base* base, bool inc = true);++ private:+  /* Accessors for private members using explicit template instantiation */+  struct access_shared_ptr {+    typedef shared_count std::__shared_ptr<const void, std::_S_atomic>::*type;+    friend type fieldPtr(access_shared_ptr);+  };++  struct access_base {+    typedef counted_base* shared_count::*type;+    friend type fieldPtr(access_base);+  };++  struct access_use_count {+    typedef _Atomic_word counted_base::*type;+    friend type fieldPtr(access_use_count);+  };++  struct access_weak_count {+    typedef _Atomic_word counted_base::*type;+    friend type fieldPtr(access_weak_count);+  };++  struct access_counted_ptr_ptr {+    typedef const void* std::_Sp_counted_ptr<const void*, std::_S_atomic>::*+        type;+    friend type fieldPtr(access_counted_ptr_ptr);+  };++  struct access_shared_ptr_ptr {+    typedef const void* std::__shared_ptr<const void, std::_S_atomic>::*type;+    friend type fieldPtr(access_shared_ptr_ptr);+  };++  struct access_refcount {+    typedef shared_count std::__shared_ptr<const void, std::_S_atomic>::*type;+    friend type fieldPtr(access_refcount);+  };++  template <typename Tag, typename Tag::type M>+  struct Rob {+    friend typename Tag::type fieldPtr(Tag) { return M; }+  };+};++template struct shared_ptr_internals::Rob<+    shared_ptr_internals::access_shared_ptr,+    &std::__shared_ptr<const void, std::_S_atomic>::_M_refcount>;+template struct shared_ptr_internals::Rob<+    shared_ptr_internals::access_base,+    &shared_ptr_internals::shared_count::_M_pi>;+template struct shared_ptr_internals::Rob<+    shared_ptr_internals::access_use_count,+    &shared_ptr_internals::counted_base::_M_use_count>;+template struct shared_ptr_internals::Rob<+    shared_ptr_internals::access_weak_count,+    &shared_ptr_internals::counted_base::_M_weak_count>;+template struct shared_ptr_internals::Rob<+    shared_ptr_internals::access_counted_ptr_ptr,+    &std::_Sp_counted_ptr<const void*, std::_S_atomic>::_M_ptr>;+template struct shared_ptr_internals::Rob<+    shared_ptr_internals::access_shared_ptr_ptr,+    &std::__shared_ptr<const void, std::_S_atomic>::_M_ptr>;+template struct shared_ptr_internals::Rob<+    shared_ptr_internals::access_refcount,+    &std::__shared_ptr<const void, std::_S_atomic>::_M_refcount>;++template <typename T>+inline shared_ptr_internals::counted_base*+shared_ptr_internals::get_counted_base(const std::shared_ptr<T>& bar) {+  // reinterpret_pointer_cast<const void>+  // Not quite C++ legal, but explicit template instantiation access to+  // private members requires full type name (i.e. shared_ptr<const void>, not+  // shared_ptr<T>)+  const std::shared_ptr<const void>& ptr(+      reinterpret_cast<const std::shared_ptr<const void>&>(bar));+  return (ptr.*fieldPtr(access_shared_ptr{})).*fieldPtr(access_base{});+}++inline void shared_ptr_internals::inc_shared_count(+    counted_base* base, long count) {+  // Check that we don't exceed the maximum number of atomic_shared_ptrs.+  // Consider setting EXTERNAL_COUNT lower if this CHECK is hit.+  FOLLY_SAFE_CHECK(+      base->_M_get_use_count() + count < INT_MAX, "atomic_shared_ptr overflow");+  __gnu_cxx::__atomic_add_dispatch(+      &(base->*fieldPtr(access_use_count{})), static_cast<int>(count));+}++template <typename T>+inline void shared_ptr_internals::release_shared(+    counted_base* base, long count) {+  // If count == 1, this is equivalent to base->_M_release()+  if (__gnu_cxx::__exchange_and_add_dispatch(+          &(base->*fieldPtr(access_use_count{})), -static_cast<int>(count)) ==+      count) {+    base->_M_dispose();++    if (__gnu_cxx::__exchange_and_add_dispatch(+            &(base->*fieldPtr(access_weak_count{})), -1) == 1) {+      base->_M_destroy();+    }+  }+}++template <typename T>+inline T* shared_ptr_internals::get_shared_ptr(counted_base* base) {+  // See if this was a make_shared allocation+  auto inplace = base->_M_get_deleter(typeid(std::_Sp_make_shared_tag));+  if (inplace) {+    return (T*)inplace;+  }+  // Could also be a _Sp_counted_deleter, but the layout is the same+  using derived_type = std::_Sp_counted_ptr<const void*, std::_S_atomic>;+  auto ptr = reinterpret_cast<derived_type*>(base);+  return (T*)(ptr->*fieldPtr(access_counted_ptr_ptr{}));+}++template <typename T>+inline T* shared_ptr_internals::release_ptr(std::shared_ptr<T>& p) {+  auto res = p.get();+  std::shared_ptr<const void>& ptr(+      reinterpret_cast<std::shared_ptr<const void>&>(p));+  ptr.*fieldPtr(access_shared_ptr_ptr{}) = nullptr;+  (ptr.*fieldPtr(access_refcount{})).*fieldPtr(access_base{}) = nullptr;+  return res;+}++template <typename T>+inline std::shared_ptr<T>+shared_ptr_internals::get_shared_ptr_from_counted_base(+    counted_base* base, bool inc) {+  if (!base) {+    return nullptr;+  }+  std::shared_ptr<const void> newp;+  if (inc) {+    inc_shared_count(base, 1);+  }+  newp.*fieldPtr(access_shared_ptr_ptr{}) =+      get_shared_ptr<const void>(base); // _M_ptr+  (newp.*fieldPtr(access_refcount{})).*fieldPtr(access_base{}) = base;+  // reinterpret_pointer_cast<T>+  auto res = reinterpret_cast<std::shared_ptr<T>*>(&newp);+  return std::move(*res);+}++} // namespace detail+} // namespace folly++#endif // defined(__GLIBCXX__)
+ folly/folly/concurrency/detail/ConcurrentHashMap-detail.h view
@@ -0,0 +1,2032 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <algorithm>+#include <atomic>+#include <mutex>+#include <new>++#include <folly/ScopeGuard.h>+#include <folly/container/HeterogeneousAccess.h>+#include <folly/container/detail/F14Mask.h>+#include <folly/lang/Exception.h>+#include <folly/synchronization/Hazptr.h>++#ifdef __aarch64__+#include <arm_acle.h>+#include <arm_neon.h>+#elif FOLLY_SSE_PREREQ(4, 2) && !FOLLY_MOBILE+#include <nmmintrin.h>+#endif++namespace folly {++namespace detail {++namespace concurrenthashmap {++enum class InsertType {+  DOES_NOT_EXIST, // insert/emplace operations.  If key exists, return false.+  MUST_EXIST, // assign operations.  If key does not exist, return false.+  ANY, // insert_or_assign.+  MATCH, // assign_if_equal (not in std).  For concurrent maps, a+         // way to atomically change a value if equal to some other+         // value.+  MATCH_OR_DOES_NOT_EXIST, // behaves like MATCH if key exists, inserts if key+                           // does not exist.+};++template <+    typename KeyType,+    typename ValueType,+    typename Allocator,+    template <typename>+    class Atom,+    typename Enabled = void>+class ValueHolder {+ public:+  typedef std::pair<const KeyType, ValueType> value_type;++  explicit ValueHolder(const ValueHolder& other) : item_(other.item_) {}++  template <typename Arg, typename... Args>+  ValueHolder(std::piecewise_construct_t, Arg&& k, Args&&... args)+      : item_(+            std::piecewise_construct,+            std::forward_as_tuple(std::forward<Arg>(k)),+            std::forward_as_tuple(std::forward<Args>(args)...)) {}+  value_type& getItem() { return item_; }++ private:+  value_type item_;+};++// If the ValueType is not copy constructible, we can instead add+// an extra indirection.  Adds more allocations / deallocations and+// pulls in an extra cacheline.+template <+    typename KeyType,+    typename ValueType,+    typename Allocator,+    template <typename>+    class Atom>+class ValueHolder<+    KeyType,+    ValueType,+    Allocator,+    Atom,+    std::enable_if_t<+        !std::is_nothrow_copy_constructible<ValueType>::value ||+        !std::is_nothrow_copy_constructible<KeyType>::value>> {+  typedef std::pair<const KeyType, ValueType> value_type;++  struct CountedItem {+    value_type kv_;+    Atom<uint32_t> numlinks_{1}; // Number of incoming links++    template <typename Arg, typename... Args>+    CountedItem(std::piecewise_construct_t, Arg&& k, Args&&... args)+        : kv_(std::piecewise_construct,+              std::forward_as_tuple(std::forward<Arg>(k)),+              std::forward_as_tuple(std::forward<Args>(args)...)) {}++    value_type& getItem() { return kv_; }++    void acquireLink() {+      uint32_t count = numlinks_.fetch_add(1, std::memory_order_release);+      DCHECK_GE(count, 1u);+    }++    bool releaseLink() {+      uint32_t count = numlinks_.load(std::memory_order_acquire);+      DCHECK_GE(count, 1u);+      if (count > 1) {+        count = numlinks_.fetch_sub(1, std::memory_order_acq_rel);+      }+      return count == 1;+    }+  }; // CountedItem+  // Back to ValueHolder specialization++  CountedItem* item_; // Link to unique key-value item.++ public:+  explicit ValueHolder(const ValueHolder& other) {+    DCHECK(other.item_);+    item_ = other.item_;+    item_->acquireLink();+  }++  ValueHolder& operator=(const ValueHolder&) = delete;++  template <typename Arg, typename... Args>+  ValueHolder(std::piecewise_construct_t, Arg&& k, Args&&... args) {+    item_ = (CountedItem*)Allocator().allocate(sizeof(CountedItem));+    auto g = makeGuard([&] {+      Allocator().deallocate((uint8_t*)item_, sizeof(CountedItem));+    });+    new (item_) CountedItem(+        std::piecewise_construct,+        std::forward<Arg>(k),+        std::forward<Args>(args)...);+    g.dismiss();+  }++  ~ValueHolder() {+    DCHECK(item_);+    if (item_->releaseLink()) {+      item_->~CountedItem();+      Allocator().deallocate((uint8_t*)item_, sizeof(CountedItem));+    }+  }++  value_type& getItem() {+    DCHECK(item_);+    return item_->getItem();+  }+}; // ValueHolder specialization++template <typename Node, typename Allocator>+struct AllocNodeGuard : NonCopyableNonMovable {+  Allocator alloc;+  Node* node{};++  void dismiss() { node = nullptr; }+  Node* release() { return std::exchange(node, nullptr); }++  template <typename... Arg>+  explicit AllocNodeGuard(Allocator alloc_, Arg&&... arg)+      : alloc{std::move(alloc_)}, node{(Node*)alloc_.allocate(sizeof(Node))} {+    auto guard = makeGuard([&] {+      alloc.deallocate((uint8_t*)node, sizeof(Node));+    });+    new (node) Node(std::forward<Arg>(arg)...);+    guard.dismiss();+  }++  ~AllocNodeGuard() {+    if (node) {+      node->~Node();+      alloc.deallocate((uint8_t*)node, sizeof(Node));+    }+  }++  template <typename... Arg>+  static Node* make(Allocator alloc_, Arg&&... arg) {+    return AllocNodeGuard(alloc_, std::forward<Arg>(arg)...).release();+  }+};++// hazptr deleter that can use an allocator.+template <typename Allocator>+class HazptrDeleter {+ public:+  template <typename Node>+  void operator()(Node* node) {+    node->~Node();+    Allocator().deallocate((uint8_t*)node, sizeof(Node));+  }+};++class HazptrTableDeleter {+  size_t count_;++ public:+  HazptrTableDeleter(size_t count) : count_(count) {}+  HazptrTableDeleter() = default;+  template <typename Table>+  void operator()(Table* table) {+    table->destroy(count_);+  }+};++namespace bucket {++template <+    typename KeyType,+    typename ValueType,+    typename Allocator,+    template <typename> class Atom = std::atomic>+class NodeT+    : public hazptr_obj_base_linked<+          NodeT<KeyType, ValueType, Allocator, Atom>,+          Atom,+          concurrenthashmap::HazptrDeleter<Allocator>> {+ public:+  typedef std::pair<const KeyType, ValueType> value_type;++  explicit NodeT(hazptr_obj_cohort<Atom>* cohort, NodeT* other)+      : item_(other->item_) {+    init(cohort);+  }++  template <typename Arg, typename... Args>+  NodeT(hazptr_obj_cohort<Atom>* cohort, Arg&& k, Args&&... args)+      : item_(+            std::piecewise_construct,+            std::forward<Arg>(k),+            std::forward<Args>(args)...) {+    init(cohort);+  }++  void release() { this->unlink(); }++  value_type& getItem() { return item_.getItem(); }++  template <typename S>+  void push_links(bool m, S& s) {+    if (m) {+      auto p = next_.load(std::memory_order_acquire);+      if (p) {+        s.push(p);+      }+    }+  }++  Atom<NodeT*> next_{nullptr};++ private:+  void init(hazptr_obj_cohort<Atom>* cohort) {+    DCHECK(cohort);+    this->set_deleter( // defined in hazptr_obj+        concurrenthashmap::HazptrDeleter<Allocator>());+    this->set_cohort_tag(cohort); // defined in hazptr_obj+    this->acquire_link_safe(); // defined in hazptr_obj_base_linked+  }++  ValueHolder<KeyType, ValueType, Allocator, Atom> item_;+};++template <+    typename KeyType,+    typename ValueType,+    uint8_t ShardBits = 0,+    typename HashFn = std::hash<KeyType>,+    typename KeyEqual = std::equal_to<KeyType>,+    typename Allocator = std::allocator<uint8_t>,+    template <typename> class Atom = std::atomic,+    class Mutex = std::mutex>+class alignas(64) BucketTable {+ public:+  // Slightly higher than 1.0, in case hashing to shards isn't+  // perfectly balanced, reserve(size) will still work without+  // rehashing.+  static constexpr float kDefaultLoadFactor = 1.05f;+  typedef std::pair<const KeyType, ValueType> value_type;++  using Node =+      concurrenthashmap::bucket::NodeT<KeyType, ValueType, Allocator, Atom>;+  using InsertType = concurrenthashmap::InsertType;+  class Iterator;++  BucketTable(+      size_t initial_buckets,+      float load_factor,+      size_t max_size,+      hazptr_obj_cohort<Atom>* cohort)+      : load_factor_(load_factor), max_size_(max_size) {+    DCHECK(cohort);+    initial_buckets = folly::nextPowTwo(initial_buckets);+    DCHECK(+        max_size_ == 0 ||+        (isPowTwo(max_size_) &&+         (folly::popcount(max_size_ - 1) + ShardBits <= 32)));+    auto buckets = Buckets::create(initial_buckets, cohort);+    buckets_.store(buckets, std::memory_order_release);+    load_factor_nodes_ =+        to_integral(static_cast<float>(initial_buckets) * load_factor_);+    bucket_count_.store(initial_buckets, std::memory_order_relaxed);+  }++  ~BucketTable() {+    auto buckets = buckets_.load(std::memory_order_relaxed);+    // To catch use-after-destruction bugs in user code.+    buckets_.store(nullptr, std::memory_order_release);+    // We can delete and not retire() here, since users must have+    // their own synchronization around destruction.+    auto count = bucket_count_.load(std::memory_order_relaxed);+    buckets->unlink_and_reclaim_nodes(count);+    buckets->destroy(count);+  }++  size_t size() { return size_.load(std::memory_order_acquire); }++  void clearSize() { size_.store(0, std::memory_order_release); }++  void incSize() {+    auto sz = size_.load(std::memory_order_relaxed);+    size_.store(sz + 1, std::memory_order_release);+  }++  void decSize() {+    auto sz = size_.load(std::memory_order_relaxed);+    DCHECK_GT(sz, 0);+    size_.store(sz - 1, std::memory_order_release);+  }++  bool empty() { return size() == 0; }++  template <typename MatchFunc, typename K, typename... Args>+  bool insert(+      Iterator& it,+      size_t h,+      const K& k,+      InsertType type,+      MatchFunc match,+      hazptr_obj_cohort<Atom>* cohort,+      Args&&... args) {+    return doInsert(+        it, h, k, type, match, nullptr, cohort, std::forward<Args>(args)...);+  }++  template <typename MatchFunc, typename K, typename... Args>+  bool insert(+      Iterator& it,+      size_t h,+      const K& k,+      InsertType type,+      MatchFunc match,+      Node* cur,+      hazptr_obj_cohort<Atom>* cohort) {+    return doInsert(it, h, k, type, match, cur, cohort, cur);+  }++  // Must hold lock.+  void rehash(size_t bucket_count, hazptr_obj_cohort<Atom>* cohort) {+    auto oldcount = bucket_count_.load(std::memory_order_relaxed);+    // bucket_count must be a power of 2+    DCHECK_EQ(bucket_count & (bucket_count - 1), 0);+    if (bucket_count <= oldcount) {+      return; // Rehash only if expanding.+    }+    auto buckets = buckets_.load(std::memory_order_relaxed);+    DCHECK(buckets); // Use-after-destruction by user.+    auto newbuckets = Buckets::create(bucket_count, cohort);+    load_factor_nodes_ =+        to_integral(static_cast<float>(bucket_count) * load_factor_);+    for (size_t i = 0; i < oldcount; i++) {+      auto bucket = &buckets->buckets_[i]();+      auto node = bucket->load(std::memory_order_relaxed);+      if (!node) {+        continue;+      }+      auto h = HashFn()(node->getItem().first);+      auto idx = getIdx(bucket_count, h);+      // Reuse as long a chain as possible from the end.  Since the+      // nodes don't have previous pointers, the longest last chain+      // will be the same for both the previous hashmap and the new one,+      // assuming all the nodes hash to the same bucket.+      auto lastrun = node;+      auto lastidx = idx;+      auto last = node->next_.load(std::memory_order_relaxed);+      for (; last != nullptr;+           last = last->next_.load(std::memory_order_relaxed)) {+        auto k = getIdx(bucket_count, HashFn()(last->getItem().first));+        if (k != lastidx) {+          lastidx = k;+          lastrun = last;+        }+      }+      // Set longest last run in new bucket, incrementing the refcount.+      lastrun->acquire_link(); // defined in hazptr_obj_base_linked+      newbuckets->buckets_[lastidx]().store(lastrun, std::memory_order_relaxed);+      // Clone remaining nodes+      for (; node != lastrun;+           node = node->next_.load(std::memory_order_relaxed)) {+        auto newnode = (Node*)Allocator().allocate(sizeof(Node));+        new (newnode) Node(cohort, node);+        auto k = getIdx(bucket_count, HashFn()(node->getItem().first));+        auto prevhead = &newbuckets->buckets_[k]();+        newnode->next_.store(prevhead->load(std::memory_order_relaxed));+        prevhead->store(newnode, std::memory_order_relaxed);+      }+    }++    auto oldbuckets = buckets_.load(std::memory_order_relaxed);+    DCHECK(oldbuckets); // Use-after-destruction by user.+    seqlock_.fetch_add(1, std::memory_order_release);+    bucket_count_.store(bucket_count, std::memory_order_release);+    buckets_.store(newbuckets, std::memory_order_release);+    seqlock_.fetch_add(1, std::memory_order_release);+    oldbuckets->retire(concurrenthashmap::HazptrTableDeleter(oldcount));+  }++  template <typename K>+  bool find(Iterator& res, size_t h, const K& k) {+    auto& hazcurr = res.hazptrs_[1];+    auto& haznext = res.hazptrs_[2];+    size_t bcount;+    Buckets* buckets;+    getBucketsAndCount(bcount, buckets, res.hazptrs_[0]);++    auto idx = getIdx(bcount, h);+    auto prev = &buckets->buckets_[idx]();+    auto node = hazcurr.protect(*prev);+    while (node) {+      if (KeyEqual()(k, node->getItem().first)) {+        res.setNode(node, buckets, bcount, idx);+        return true;+      }+      node = haznext.protect(node->next_);+      hazcurr.swap(haznext);+    }+    return false;+  }++  template <typename K, typename MatchFunc>+  std::size_t erase(size_t h, const K& key, Iterator* iter, MatchFunc match) {+    Node* node{nullptr};+    {+      std::lock_guard g(m_);++      size_t bcount = bucket_count_.load(std::memory_order_relaxed);+      auto buckets = buckets_.load(std::memory_order_relaxed);+      DCHECK(buckets); // Use-after-destruction by user.+      auto idx = getIdx(bcount, h);+      auto head = &buckets->buckets_[idx]();+      node = head->load(std::memory_order_relaxed);+      Node* prev = nullptr;+      while (node) {+        if (KeyEqual()(key, node->getItem().first)) {+          if (!match(node->getItem().second)) {+            return 0;+          }+          auto next = node->next_.load(std::memory_order_relaxed);+          if (next) {+            next->acquire_link(); // defined in hazptr_obj_base_linked+          }+          if (prev) {+            prev->next_.store(next, std::memory_order_release);+          } else {+            // Must be head of list.+            head->store(next, std::memory_order_release);+          }++          if (iter) {+            iter->hazptrs_[0].reset_protection(buckets);+            iter->setNode(+                node->next_.load(std::memory_order_acquire),+                buckets,+                bcount,+                idx);+            iter->next();+          }+          decSize();+          break;+        }+        prev = node;+        node = node->next_.load(std::memory_order_relaxed);+      }+    }+    // Delete the node while not under the lock.+    if (node) {+      node->release();+      return 1;+    }++    return 0;+  }++  void clear(hazptr_obj_cohort<Atom>* cohort) {+    size_t bcount;+    Buckets* buckets;+    {+      std::lock_guard g(m_);+      bcount = bucket_count_.load(std::memory_order_relaxed);+      auto newbuckets = Buckets::create(bcount, cohort);+      buckets = buckets_.load(std::memory_order_relaxed);+      buckets_.store(newbuckets, std::memory_order_release);+      clearSize();+    }+    DCHECK(buckets); // Use-after-destruction by user.+    buckets->retire(concurrenthashmap::HazptrTableDeleter(bcount));+  }++  void max_load_factor(float factor) {+    std::lock_guard g(m_);+    load_factor_ = factor;+    load_factor_nodes_ =+        bucket_count_.load(std::memory_order_relaxed) * load_factor_;+  }++  Iterator cbegin() {+    Iterator res;+    size_t bcount;+    Buckets* buckets;+    getBucketsAndCount(bcount, buckets, res.hazptrs_[0]);+    res.setNode(nullptr, buckets, bcount, 0);+    res.next();+    return res;+  }++  Iterator cend() { return Iterator(nullptr); }++ private:+  // Could be optimized to avoid an extra pointer dereference by+  // allocating buckets_ at the same time.+  class Buckets+      : public hazptr_obj_base<+            Buckets,+            Atom,+            concurrenthashmap::HazptrTableDeleter> {+    using BucketRoot = hazptr_root<Node, Atom>;++    Buckets() {}+    ~Buckets() {}++   public:+    static Buckets* create(size_t count, hazptr_obj_cohort<Atom>* cohort) {+      auto buf =+          Allocator().allocate(sizeof(Buckets) + sizeof(BucketRoot) * count);+      auto buckets = new (buf) Buckets();+      DCHECK(cohort);+      buckets->set_cohort_tag(cohort); // defined in hazptr_obj+      for (size_t i = 0; i < count; i++) {+        new (&buckets->buckets_[i]) BucketRoot;+      }+      return buckets;+    }++    void destroy(size_t count) {+      for (size_t i = 0; i < count; i++) {+        buckets_[i].~BucketRoot();+      }+      this->~Buckets();+      Allocator().deallocate(+          (uint8_t*)this, sizeof(BucketRoot) * count + sizeof(*this));+    }++    void unlink_and_reclaim_nodes(size_t count) {+      for (size_t i = 0; i < count; i++) {+        auto node = buckets_[i]().load(std::memory_order_relaxed);+        if (node) {+          buckets_[i]().store(nullptr, std::memory_order_relaxed);+          while (node) {+            auto next = node->next_.load(std::memory_order_relaxed);+            if (next) {+              node->next_.store(nullptr, std::memory_order_relaxed);+            }+            node->unlink_and_reclaim_unchecked();+            node = next;+          }+        }+      }+    }++    BucketRoot buckets_[0];+  };++ public:+  class Iterator {+   public:+    FOLLY_ALWAYS_INLINE Iterator()+        : hazptrs_(make_hazard_pointer_array<3, Atom>()) {}+    FOLLY_ALWAYS_INLINE explicit Iterator(std::nullptr_t) : hazptrs_() {}+    FOLLY_ALWAYS_INLINE ~Iterator() {}++    void setNode(+        Node* node, Buckets* buckets, size_t bucket_count, uint64_t idx) {+      node_ = node;+      buckets_ = buckets;+      idx_ = idx;+      bucket_count_ = bucket_count;+    }++    const value_type& operator*() const {+      DCHECK(node_);+      return node_->getItem();+    }++    const value_type* operator->() const {+      DCHECK(node_);+      return &(node_->getItem());+    }++    const Iterator& operator++() {+      DCHECK(node_);+      node_ = hazptrs_[2].protect(node_->next_);+      hazptrs_[1].swap(hazptrs_[2]);+      if (!node_) {+        ++idx_;+        next();+      }+      return *this;+    }++    void next() {+      while (!node_) {+        if (idx_ >= bucket_count_) {+          break;+        }+        DCHECK(buckets_);+        node_ = hazptrs_[1].protect(buckets_->buckets_[idx_]());+        if (node_) {+          break;+        }+        ++idx_;+      }+    }++    bool operator==(const Iterator& o) const { return node_ == o.node_; }++    bool operator!=(const Iterator& o) const { return !(*this == o); }++    Iterator& operator=(const Iterator& o) = delete;++    Iterator& operator=(Iterator&& o) noexcept {+      if (this != &o) {+        hazptrs_ = std::move(o.hazptrs_);+        node_ = std::exchange(o.node_, nullptr);+        buckets_ = std::exchange(o.buckets_, nullptr);+        bucket_count_ = std::exchange(o.bucket_count_, 0);+        idx_ = std::exchange(o.idx_, 0);+      }+      return *this;+    }++    Iterator(const Iterator& o) = delete;++    Iterator(Iterator&& o) noexcept+        : hazptrs_(std::move(o.hazptrs_)),+          node_(std::exchange(o.node_, nullptr)),+          buckets_(std::exchange(o.buckets_, nullptr)),+          bucket_count_(std::exchange(o.bucket_count_, 0)),+          idx_(std::exchange(o.idx_, 0)) {}++    // These are accessed directly from the functions above+    hazptr_array<3, Atom> hazptrs_;++   private:+    Node* node_{nullptr};+    Buckets* buckets_{nullptr};+    size_t bucket_count_{0};+    uint64_t idx_{0};+  };++ private:+  // Shards have already used low ShardBits of the hash.+  // Shift it over to use fresh bits.+  uint64_t getIdx(size_t bucket_count, size_t hash) {+    return (hash >> ShardBits) & (bucket_count - 1);+  }+  void getBucketsAndCount(+      size_t& bcount, Buckets*& buckets, hazptr_holder<Atom>& hazptr) {+    while (true) {+      auto seqlock = seqlock_.load(std::memory_order_acquire);+      bcount = bucket_count_.load(std::memory_order_acquire);+      buckets = hazptr.protect(buckets_);+      auto seqlock2 = seqlock_.load(std::memory_order_acquire);+      if (!(seqlock & 1) && (seqlock == seqlock2)) {+        break;+      }+    }+    DCHECK(buckets) << "Use-after-destruction by user.";+  }++  template <typename MatchFunc, typename K, typename... Args>+  bool doInsert(+      Iterator& it,+      size_t h,+      const K& k,+      InsertType type,+      MatchFunc match,+      Node* cur,+      hazptr_obj_cohort<Atom>* cohort,+      Args&&... args) {+    std::unique_lock g(m_);++    size_t bcount = bucket_count_.load(std::memory_order_relaxed);+    auto buckets = buckets_.load(std::memory_order_relaxed);+    // Check for rehash needed for DOES_NOT_EXIST+    if (size() >= load_factor_nodes_ &&+        (type == InsertType::DOES_NOT_EXIST ||+         type == InsertType::MATCH_OR_DOES_NOT_EXIST)) {+      if (max_size_ && size() << 1 > max_size_) {+        // Would exceed max size.+        throw_exception<std::bad_alloc>();+      }+      rehash(bcount << 1, cohort);+      buckets = buckets_.load(std::memory_order_relaxed);+      bcount = bucket_count_.load(std::memory_order_relaxed);+    }++    DCHECK(buckets) << "Use-after-destruction by user.";+    auto idx = getIdx(bcount, h);+    auto head = &buckets->buckets_[idx]();+    auto node = head->load(std::memory_order_relaxed);+    auto headnode = node;+    auto prev = head;+    auto& hazbuckets = it.hazptrs_[0];+    auto& haznode = it.hazptrs_[1];+    hazbuckets.reset_protection(buckets);+    bool matched = false;+    while (node) {+      // Is the key found?+      if (KeyEqual()(k, node->getItem().first)) {+        it.setNode(node, buckets, bcount, idx);+        haznode.reset_protection(node);+        if (type == InsertType::MATCH ||+            type == InsertType::MATCH_OR_DOES_NOT_EXIST) {+          if (!match(node->getItem().second)) {+            return false;+          }+          matched = true;+        }+        if (type == InsertType::DOES_NOT_EXIST ||+            (type == InsertType::MATCH_OR_DOES_NOT_EXIST && !matched)) {+          return false;+        } else {+          if (!cur) {+            cur = AllocNodeGuard<Node, Allocator>::make(+                Allocator(), cohort, std::forward<Args>(args)...);+          }+          auto next = node->next_.load(std::memory_order_relaxed);+          cur->next_.store(next, std::memory_order_relaxed);+          if (next) {+            next->acquire_link(); // defined in hazptr_obj_base_linked+          }+          prev->store(cur, std::memory_order_release);+          it.setNode(cur, buckets, bcount, idx);+          haznode.reset_protection(cur);+          g.unlock();+          // Release not under lock.+          node->release();+          return true;+        }+      }++      prev = &node->next_;+      node = node->next_.load(std::memory_order_relaxed);+    }+    if (type != InsertType::DOES_NOT_EXIST &&+        (type != InsertType::MATCH_OR_DOES_NOT_EXIST || matched) &&+        type != InsertType::ANY) {+      haznode.reset_protection();+      hazbuckets.reset_protection();+      return false;+    }+    // Node not found, check for rehash on ANY+    if (size() >= load_factor_nodes_ && type == InsertType::ANY) {+      if (max_size_ && size() << 1 > max_size_) {+        // Would exceed max size.+        throw_exception<std::bad_alloc>();+      }+      rehash(bcount << 1, cohort);++      // Reload correct bucket.+      buckets = buckets_.load(std::memory_order_relaxed);+      DCHECK(buckets); // Use-after-destruction by user.+      bcount <<= 1;+      hazbuckets.reset_protection(buckets);+      idx = getIdx(bcount, h);+      head = &buckets->buckets_[idx]();+      headnode = head->load(std::memory_order_relaxed);+    }++    // We found a slot to put the node.+    incSize();+    if (!cur) {+      // InsertType::ANY+      // OR DOES_NOT_EXIST, but only in the try_emplace case+      DCHECK(+          type == InsertType::ANY || type == InsertType::DOES_NOT_EXIST ||+          (type == InsertType::MATCH_OR_DOES_NOT_EXIST && !matched));+      cur = AllocNodeGuard<Node, Allocator>::make(+          Allocator(), cohort, std::forward<Args>(args)...);+    }+    cur->next_.store(headnode, std::memory_order_relaxed);+    head->store(cur, std::memory_order_release);+    it.setNode(cur, buckets, bcount, idx);+    haznode.reset_protection(cur);+    return true;+  }++  Mutex m_;+  float load_factor_;+  size_t load_factor_nodes_;+  Atom<size_t> size_{0};+  size_t const max_size_;++  // Fields needed for read-only access, on separate cacheline.+  alignas(64) Atom<Buckets*> buckets_{nullptr};+  std::atomic<uint64_t> seqlock_{0};+  Atom<size_t> bucket_count_;+};++} // namespace bucket++#if (FOLLY_SSE_PREREQ(4, 2) || FOLLY_AARCH64) && \+    FOLLY_F14_VECTOR_INTRINSICS_AVAILABLE++namespace simd {++using folly::f14::detail::DenseMaskIter;+using folly::f14::detail::FirstEmptyInMask;+using folly::f14::detail::FullMask;+using folly::f14::detail::MaskType;+using folly::f14::detail::SparseMaskIter;++using folly::hazptr_obj_base;+using folly::hazptr_obj_cohort;++template <+    typename KeyType,+    typename ValueType,+    typename Allocator,+    template <typename> class Atom = std::atomic>+class NodeT+    : public hazptr_obj_base<+          NodeT<KeyType, ValueType, Allocator, Atom>,+          Atom,+          HazptrDeleter<Allocator>> {+ public:+  typedef std::pair<const KeyType, ValueType> value_type;++  template <typename Arg, typename... Args>+  NodeT(hazptr_obj_cohort<Atom>* cohort, Arg&& k, Args&&... args)+      : item_(+            std::piecewise_construct,+            std::forward_as_tuple(std::forward<Arg>(k)),+            std::forward_as_tuple(std::forward<Args>(args)...)) {+    init(cohort);+  }++  value_type& getItem() { return item_; }++ private:+  void init(hazptr_obj_cohort<Atom>* cohort) {+    DCHECK(cohort);+    this->set_deleter( // defined in hazptr_obj+        HazptrDeleter<Allocator>());+    this->set_cohort_tag(cohort); // defined in hazptr_obj+  }++  value_type item_;+};++constexpr std::size_t kRequiredVectorAlignment =+    constexpr_max(std::size_t{16}, alignof(max_align_t));++template <+    typename KeyType,+    typename ValueType,+    uint8_t ShardBits = 0,+    typename HashFn = std::hash<KeyType>,+    typename KeyEqual = std::equal_to<KeyType>,+    typename Allocator = std::allocator<uint8_t>,+    template <typename> class Atom = std::atomic,+    class Mutex = std::mutex>+class alignas(64) SIMDTable {+ public:+  using Node =+      concurrenthashmap::simd::NodeT<KeyType, ValueType, Allocator, Atom>;++ private:+  using HashPair = std::pair<std::size_t, std::size_t>;+  struct alignas(kRequiredVectorAlignment) Chunk {+    static constexpr unsigned kCapacity = 14;+    static constexpr unsigned kDesiredCapacity = 12;++    static constexpr MaskType kFullMask = FullMask<kCapacity>::value;++   private:+    // Non-empty tags have their top bit set.++    // tags [0,8)+    Atom<uint64_t> tags_low_;++    // tags_hi_ holds tags [8,14), hostedOverflowCount and outboundOverflowCount++    // hostedOverflowCount: the number of values in this chunk that were placed+    // because they overflowed their desired chunk.++    // outboundOverflowCount: num values that would have been placed into this+    // chunk if there had been space, including values that also overflowed+    // previous full chunks.  This value saturates; once it becomes 255 it no+    // longer increases nor decreases.++    // Note: more bits can be used for outboundOverflowCount if this+    // optimization becomes useful+    Atom<uint64_t> tags_hi_;++    std::array<aligned_storage_for_t<Atom<Node*>>, kCapacity> rawItems_;++   public:+    void clear() {+      for (size_t i = 0; i < kCapacity; i++) {+        item(i).store(nullptr, std::memory_order_relaxed);+      }+      tags_low_.store(0, std::memory_order_relaxed);+      tags_hi_.store(0, std::memory_order_relaxed);+    }++    std::size_t tag(std::size_t index) const {+      std::size_t off = index % 8;+      const Atom<uint64_t>& tag_src = off == index ? tags_low_ : tags_hi_;+      uint64_t tags = tag_src.load(std::memory_order_relaxed);+      tags >>= (off * 8);+      return tags & 0xff;+    }++    void setTag(std::size_t index, std::size_t tag) {+      std::size_t off = index % 8;+      Atom<uint64_t>& old_tags = off == index ? tags_low_ : tags_hi_;+      uint64_t new_tags = old_tags.load(std::memory_order_relaxed);+      uint64_t mask = 0xffULL << (off * 8);+      new_tags = (new_tags & ~mask) | (tag << (off * 8));+      old_tags.store(new_tags, std::memory_order_release);+    }++    void setNodeAndTag(std::size_t index, Node* node, std::size_t tag) {+      FOLLY_SAFE_DCHECK(+          index < kCapacity && (tag == 0x0 || (tag >= 0x80 && tag <= 0xff)),+          "");+      item(index).store(node, std::memory_order_release);+      setTag(index, tag);+    }++    void clearNodeAndTag(std::size_t index) {+      setNodeAndTag(index, nullptr, 0);+    }++#ifdef __aarch64__++    ////////+    // Tag filtering using NEON intrinsics++    SparseMaskIter tagMatchIter(std::size_t needle) const {+      FOLLY_SAFE_DCHECK(needle >= 0x80 && needle < 0x100, "");+      uint64_t low = tags_low_.load(std::memory_order_acquire);+      uint64_t hi = tags_hi_.load(std::memory_order_acquire);+      uint8x16_t needleV = vdupq_n_u8(static_cast<uint8_t>(needle));+      uint64x2_t vec;+      vec[0] = low;+      vec[1] = hi;+      auto eqV = vceqq_u8(vreinterpretq_u8_u64(vec), needleV);+      uint8x8_t maskV = vshrn_n_u16(vreinterpretq_u16_u8(eqV), 4);+      uint64_t mask = vget_lane_u64(vreinterpret_u64_u8(maskV), 0) & kFullMask;+      return SparseMaskIter{mask};+    }++    MaskType occupiedMask() const {+      uint64_t low = tags_low_.load(std::memory_order_relaxed);+      uint64_t hi = tags_hi_.load(std::memory_order_relaxed);+      uint64x2_t vec;+      vec[0] = low;+      vec[1] = hi;+      // signed shift extends top bit to all bits+      auto occupiedV =+          vreinterpretq_u8_s8(vshrq_n_s8(vreinterpretq_s8_u64(vec), 7));+      uint8x8_t maskV = vshrn_n_u16(vreinterpretq_u16_u8(occupiedV), 4);+      return vget_lane_u64(vreinterpret_u64_u8(maskV), 0) & kFullMask;+    }++#else++    ////////+    // Tag filtering using SSE2 intrinsics++    SparseMaskIter tagMatchIter(std::size_t needle) const {+      FOLLY_SAFE_DCHECK(needle >= 0x80 && needle < 0x100, "");+      uint64_t low = tags_low_.load(std::memory_order_acquire);+      uint64_t hi = tags_hi_.load(std::memory_order_acquire);+      auto tagV = _mm_set_epi64x(hi, low);+      auto needleV = _mm_set1_epi8(static_cast<uint8_t>(needle));+      auto eqV = _mm_cmpeq_epi8(tagV, needleV);+      auto mask = _mm_movemask_epi8(eqV) & kFullMask;+      return SparseMaskIter{mask};+    }++    MaskType occupiedMask() const {+      uint64_t low = tags_low_.load(std::memory_order_relaxed);+      uint64_t hi = tags_hi_.load(std::memory_order_relaxed);+      auto tagV = _mm_set_epi64x(hi, low);+      return _mm_movemask_epi8(tagV) & kFullMask;+    }++#endif++    DenseMaskIter occupiedIter() const {+      // Currently only invoked when relaxed semantics are sufficient.+      return DenseMaskIter{nullptr /*unused*/, occupiedMask()};+    }++    FirstEmptyInMask firstEmpty() const {+      return FirstEmptyInMask{occupiedMask() ^ kFullMask};+    }++    Atom<Node*>* itemAddr(std::size_t i) const {+      return static_cast<Atom<Node*>*>(+          const_cast<void*>(static_cast<void const*>(&rawItems_[i])));+    }++    Atom<Node*>& item(size_t i) { return *std::launder(itemAddr(i)); }++    static constexpr uint64_t kOutboundOverflowIndex = 7 * 8;+    static constexpr uint64_t kSaturatedOutboundOverflowCount = 0xffULL+        << kOutboundOverflowIndex;+    static constexpr uint64_t kOutboundOverflowOperand = 0x1ULL+        << kOutboundOverflowIndex;++    unsigned outboundOverflowCount() const {+      uint64_t count = tags_hi_.load(std::memory_order_relaxed);+      return count >> kOutboundOverflowIndex;+    }++    void incrOutboundOverflowCount() {+      uint64_t count = tags_hi_.load(std::memory_order_relaxed);+      if (count < kSaturatedOutboundOverflowCount) {+        tags_hi_.store(+            count + kOutboundOverflowOperand, std::memory_order_relaxed);+      }+    }++    void decrOutboundOverflowCount() {+      uint64_t count = tags_hi_.load(std::memory_order_relaxed);+      if (count < kSaturatedOutboundOverflowCount) {+        tags_hi_.store(+            count - kOutboundOverflowOperand, std::memory_order_relaxed);+      }+    }++    static constexpr uint64_t kHostedOverflowIndex = 6 * 8;+    static constexpr uint64_t kHostedOverflowOperand = 0x10ULL+        << kHostedOverflowIndex;++    unsigned hostedOverflowCount() const {+      uint64_t control = tags_hi_.load(std::memory_order_relaxed);+      return (control >> 52) & 0xf;+    }++    void incrHostedOverflowCount() {+      tags_hi_.fetch_add(kHostedOverflowOperand, std::memory_order_relaxed);+    }++    void decrHostedOverflowCount() {+      tags_hi_.fetch_sub(kHostedOverflowOperand, std::memory_order_relaxed);+    }+  };++  class Chunks : public hazptr_obj_base<Chunks, Atom, HazptrTableDeleter> {+    Chunks() {}+    ~Chunks() {}++   public:+    static Chunks* create(size_t count, hazptr_obj_cohort<Atom>* cohort) {+      auto buf = Allocator().allocate(sizeof(Chunks) + sizeof(Chunk) * count);+      auto chunks = new (buf) Chunks();+      DCHECK(cohort);+      chunks->set_cohort_tag(cohort); // defined in hazptr_obj+      for (size_t i = 0; i < count; i++) {+        new (&chunks->chunks_[i]) Chunk;+        chunks->chunks_[i].clear();+      }+      return chunks;+    }++    void destroy(size_t count) {+      for (size_t i = 0; i < count; i++) {+        chunks_[i].~Chunk();+      }+      this->~Chunks();+      Allocator().deallocate(+          (uint8_t*)this, sizeof(Chunk) * count + sizeof(*this));+    }++    void reclaim_nodes(size_t count) {+      for (size_t i = 0; i < count; i++) {+        Chunk& chunk = chunks_[i];+        auto occupied = chunk.occupiedIter();+        while (occupied.hasNext()) {+          auto idx = occupied.next();+          chunk.setTag(idx, 0);+          Node* node =+              chunk.item(idx).exchange(nullptr, std::memory_order_relaxed);+          // Tags and node ptrs should be in sync at this point.+          DCHECK(node);+          node->retire();+        }+      }+    }++    Chunk* getChunk(size_t index, size_t ccount) {+      DCHECK(isPowTwo(ccount));+      return &chunks_[index & (ccount - 1)];+    }++   private:+    Chunk chunks_[0];+  };++ public:+  static constexpr float kDefaultLoadFactor =+      Chunk::kDesiredCapacity / (float)Chunk::kCapacity;++  typedef std::pair<const KeyType, ValueType> value_type;++  using InsertType = concurrenthashmap::InsertType;++  class Iterator {+   public:+    FOLLY_ALWAYS_INLINE Iterator()+        : hazptrs_(make_hazard_pointer_array<2, Atom>()) {}+    FOLLY_ALWAYS_INLINE explicit Iterator(std::nullptr_t) : hazptrs_() {}+    FOLLY_ALWAYS_INLINE ~Iterator() {}++    void setNode(+        Node* node,+        Chunks* chunks,+        size_t chunk_count,+        uint64_t chunk_idx,+        uint64_t tag_idx) {+      DCHECK(chunk_idx < chunk_count || chunk_idx == 0);+      DCHECK(isPowTwo(chunk_count));+      node_ = node;+      chunks_ = chunks;+      chunk_count_ = chunk_count;+      chunk_idx_ = chunk_idx;+      tag_idx_ = tag_idx;+    }++    const value_type& operator*() const {+      DCHECK(node_);+      return node_->getItem();+    }++    const value_type* operator->() const {+      DCHECK(node_);+      return &(node_->getItem());+    }++    const Iterator& operator++() {+      DCHECK(node_);+      ++tag_idx_;+      findNextNode();+      return *this;+    }++    void next() {+      if (node_) {+        return;+      }+      findNextNode();+    }++    bool operator==(const Iterator& o) const { return node_ == o.node_; }++    bool operator!=(const Iterator& o) const { return !(*this == o); }++    Iterator& operator=(const Iterator& o) = delete;++    Iterator& operator=(Iterator&& o) noexcept {+      if (this != &o) {+        hazptrs_ = std::move(o.hazptrs_);+        node_ = std::exchange(o.node_, nullptr);+        chunks_ = std::exchange(o.chunks_, nullptr);+        chunk_count_ = std::exchange(o.chunk_count_, 0);+        chunk_idx_ = std::exchange(o.chunk_idx_, 0);+        tag_idx_ = std::exchange(o.tag_idx_, 0);+      }+      return *this;+    }++    Iterator(const Iterator& o) = delete;++    Iterator(Iterator&& o) noexcept+        : hazptrs_(std::move(o.hazptrs_)),+          node_(std::exchange(o.node_, nullptr)),+          chunks_(std::exchange(o.chunks_, nullptr)),+          chunk_count_(std::exchange(o.chunk_count_, 0)),+          chunk_idx_(std::exchange(o.chunk_idx_, 0)),+          tag_idx_(std::exchange(o.tag_idx_, 0)) {}++    // These are accessed directly from the functions above+    hazptr_array<2, Atom> hazptrs_;++   private:+    void findNextNode() {+      do {+        if (tag_idx_ >= Chunk::kCapacity) {+          tag_idx_ = 0;+          ++chunk_idx_;+        }+        if (chunk_idx_ >= chunk_count_) {+          node_ = nullptr;+          break;+        }+        DCHECK(chunks_);+        // Note that iteration could also be implemented with tag filtering+        node_ = hazptrs_[1].protect(+            chunks_->getChunk(chunk_idx_, chunk_count_)->item(tag_idx_));+        if (node_) {+          break;+        }+        ++tag_idx_;+      } while (true);+    }++    Node* node_{nullptr};+    Chunks* chunks_{nullptr};+    size_t chunk_count_{0};+    uint64_t chunk_idx_{0};+    uint64_t tag_idx_{0};+  };++  SIMDTable(+      size_t initial_size,+      float load_factor,+      size_t max_size,+      hazptr_obj_cohort<Atom>* cohort)+      : load_factor_(load_factor),+        max_size_(max_size),+        chunks_(nullptr),+        chunk_count_(0) {+    DCHECK(cohort);+    DCHECK(+        max_size_ == 0 ||+        (isPowTwo(max_size_) &&+         (folly::popcount(max_size_ - 1) + ShardBits <= 32)));+    DCHECK(load_factor_ > 0.0);+    load_factor_ = std::min<float>(load_factor_, 1.0);+    rehash(initial_size, cohort);+  }++  ~SIMDTable() {+    auto chunks = chunks_.load(std::memory_order_relaxed);+    // To catch use-after-destruction bugs in user code.+    chunks_.store(nullptr, std::memory_order_release);+    // We can delete and not retire() here, since users must have+    // their own synchronization around destruction.+    auto count = chunk_count_.load(std::memory_order_relaxed);+    chunks->reclaim_nodes(count);+    chunks->destroy(count);+  }++  size_t size() { return size_.load(std::memory_order_acquire); }++  void clearSize() { size_.store(0, std::memory_order_release); }++  void incSize() {+    auto sz = size_.load(std::memory_order_relaxed);+    size_.store(sz + 1, std::memory_order_release);+  }++  void decSize() {+    auto sz = size_.load(std::memory_order_relaxed);+    DCHECK_GT(sz, 0);+    size_.store(sz - 1, std::memory_order_release);+  }++  bool empty() { return size() == 0; }++  template <typename MatchFunc, typename K, typename... Args>+  bool insert(+      Iterator& it,+      size_t h,+      const K& k,+      InsertType type,+      MatchFunc match,+      hazptr_obj_cohort<Atom>* cohort,+      Args&&... args) {+    Node* node;+    Chunks* chunks;+    size_t ccount, chunk_idx, tag_idx;++    auto hp = splitHash(h);++    std::unique_lock g(m_);++    if (!prepare_insert(+            it,+            k,+            type,+            match,+            cohort,+            chunk_idx,+            tag_idx,+            node,+            chunks,+            ccount,+            hp)) {+      return false;+    }++    auto cur = AllocNodeGuard<Node, Allocator>::make(+        Allocator(), cohort, std::forward<Args>(args)...);++    if (!node) {+      std::tie(chunk_idx, tag_idx) =+          findEmptyInsertLocation(chunks, ccount, hp);+      incSize();+    }++    Chunk* chunk = chunks->getChunk(chunk_idx, ccount);+    chunk->setNodeAndTag(tag_idx, cur, hp.second);+    it.setNode(cur, chunks, ccount, chunk_idx, tag_idx);+    it.hazptrs_[1].reset_protection(cur);++    g.unlock();+    // Retire not under lock+    if (node) {+      node->retire();+    }+    return true;+  }++  template <typename MatchFunc, typename K, typename... Args>+  bool insert(+      Iterator& it,+      size_t h,+      const K& k,+      InsertType type,+      MatchFunc match,+      Node* cur,+      hazptr_obj_cohort<Atom>* cohort) {+    DCHECK(cur != nullptr);+    Node* node;+    Chunks* chunks;+    size_t ccount, chunk_idx, tag_idx;++    auto hp = splitHash(h);++    std::unique_lock g(m_);++    if (!prepare_insert(+            it,+            k,+            type,+            match,+            cohort,+            chunk_idx,+            tag_idx,+            node,+            chunks,+            ccount,+            hp)) {+      return false;+    }++    if (!node) {+      std::tie(chunk_idx, tag_idx) =+          findEmptyInsertLocation(chunks, ccount, hp);+      incSize();+    }++    Chunk* chunk = chunks->getChunk(chunk_idx, ccount);+    chunk->setNodeAndTag(tag_idx, cur, hp.second);+    it.setNode(cur, chunks, ccount, chunk_idx, tag_idx);+    it.hazptrs_[1].reset_protection(cur);++    g.unlock();+    // Retire not under lock+    if (node) {+      node->retire();+    }+    return true;+  }++  void rehash(size_t size, hazptr_obj_cohort<Atom>* cohort) {+    size_t new_chunk_count = size == 0 ? 0 : (size - 1) / Chunk::kCapacity + 1;+    rehash_internal(folly::nextPowTwo(new_chunk_count), cohort);+  }++  template <typename K>+  bool find(Iterator& res, size_t h, const K& k) {+    auto& hazz = res.hazptrs_[1];+    auto hp = splitHash(h);+    size_t ccount;+    Chunks* chunks;+    getChunksAndCount(ccount, chunks, res.hazptrs_[0]);++    size_t step = probeDelta(hp);+    auto& chunk_idx = hp.first;+    for (size_t tries = 0; tries < ccount; ++tries) {+      Chunk* chunk = chunks->getChunk(chunk_idx, ccount);+      auto hits = chunk->tagMatchIter(hp.second);+      while (hits.hasNext()) {+        size_t tag_idx = hits.next();+        Node* node = hazz.protect(chunk->item(tag_idx));+        if (FOLLY_LIKELY(node && KeyEqual()(k, node->getItem().first))) {+          chunk_idx = chunk_idx & (ccount - 1);+          res.setNode(node, chunks, ccount, chunk_idx, tag_idx);+          return true;+        }+        hazz.reset_protection();+      }++      if (FOLLY_LIKELY(chunk->outboundOverflowCount() == 0)) {+        break;+      }+      chunk_idx += step;+    }+    return false;+  }++  template <typename K, typename MatchFunc>+  std::size_t erase(size_t h, const K& key, Iterator* iter, MatchFunc match) {+    const HashPair hp = splitHash(h);++    std::unique_lock g(m_);++    size_t ccount = chunk_count_.load(std::memory_order_relaxed);+    auto chunks = chunks_.load(std::memory_order_relaxed);+    DCHECK(chunks); // Use-after-destruction by user.+    size_t chunk_idx, tag_idx;++    Node* node = find_internal(key, hp, chunks, ccount, chunk_idx, tag_idx);++    if (!node) {+      return 0;+    }++    if (!match(node->getItem().second)) {+      return 0;+    }++    Chunk* chunk = chunks->getChunk(chunk_idx, ccount);++    // Decrement any overflow counters+    if (chunk->hostedOverflowCount() != 0) {+      size_t index = hp.first;+      size_t delta = probeDelta(hp);+      bool preferredChunk = true;+      while (true) {+        Chunk* overflowChunk = chunks->getChunk(index, ccount);+        if (chunk == overflowChunk) {+          if (!preferredChunk) {+            overflowChunk->decrHostedOverflowCount();+          }+          break;+        }+        overflowChunk->decrOutboundOverflowCount();+        preferredChunk = false;+        index += delta;+      }+    }++    chunk->clearNodeAndTag(tag_idx);++    decSize();+    if (iter) {+      iter->hazptrs_[0].reset_protection(chunks);+      iter->setNode(nullptr, chunks, ccount, chunk_idx, tag_idx + 1);+      iter->next();+    }+    // Retire the node while not under the lock.+    g.unlock();+    node->retire();+    return 1;+  }++  void clear(hazptr_obj_cohort<Atom>* cohort) {+    size_t ccount;+    Chunks* chunks;+    {+      std::lock_guard g(m_);+      ccount = chunk_count_.load(std::memory_order_relaxed);+      auto newchunks = Chunks::create(ccount, cohort);+      chunks = chunks_.load(std::memory_order_relaxed);+      chunks_.store(newchunks, std::memory_order_release);+      clearSize();+    }+    DCHECK(chunks); // Use-after-destruction by user.+    chunks->reclaim_nodes(ccount);+    chunks->retire(HazptrTableDeleter(ccount));+  }++  void max_load_factor(float factor) {+    DCHECK(factor > 0.0);+    if (factor > 1.0) {+      throw_exception<std::invalid_argument>("load factor must be <= 1.0");+    }+    std::lock_guard g(m_);+    load_factor_ = factor;+    auto ccount = chunk_count_.load(std::memory_order_relaxed);+    grow_threshold_ = ccount * Chunk::kCapacity * load_factor_;+  }++  Iterator cbegin() {+    Iterator res;+    size_t ccount;+    Chunks* chunks;+    getChunksAndCount(ccount, chunks, res.hazptrs_[0]);+    res.setNode(nullptr, chunks, ccount, 0, 0);+    res.next();+    return res;+  }++  Iterator cend() { return Iterator(nullptr); }++ private:+  static HashPair splitHash(std::size_t hash) {+#ifdef __aarch64__+    std::size_t c = __crc32cd(0, hash);+#else+    std::size_t c = _mm_crc32_u64(0, hash);+#endif+    size_t tag = (c >> 24) | 0x80;+    hash += c;+    return std::make_pair(hash, tag);+  }++  static size_t probeDelta(HashPair hp) { return 2 * hp.second + 1; }++  // Must hold lock.+  template <typename K>+  Node* find_internal(+      const K& k,+      const HashPair& hp,+      Chunks* chunks,+      size_t ccount,+      size_t& chunk_idx,+      size_t& tag_idx) {+    // must be called with mutex held+    size_t step = probeDelta(hp);+    chunk_idx = hp.first;++    for (size_t tries = 0; tries < ccount; ++tries) {+      Chunk* chunk = chunks->getChunk(chunk_idx, ccount);+      auto hits = chunk->tagMatchIter(hp.second);+      while (hits.hasNext()) {+        tag_idx = hits.next();+        Node* node = chunk->item(tag_idx).load(std::memory_order_acquire);+        if (FOLLY_LIKELY(node && KeyEqual()(k, node->getItem().first))) {+          chunk_idx = (chunk_idx & (ccount - 1));+          return node;+        }+      }+      if (FOLLY_LIKELY(chunk->outboundOverflowCount() == 0)) {+        break;+      }+      chunk_idx += step;+    }+    return nullptr;+  }++  template <typename MatchFunc, typename K, typename... Args>+  bool prepare_insert(+      Iterator& it,+      const K& k,+      InsertType type,+      MatchFunc match,+      hazptr_obj_cohort<Atom>* cohort,+      size_t& chunk_idx,+      size_t& tag_idx,+      Node*& node,+      Chunks*& chunks,+      size_t& ccount,+      const HashPair& hp) {+    ccount = chunk_count_.load(std::memory_order_relaxed);+    chunks = chunks_.load(std::memory_order_relaxed);++    if (size() >= grow_threshold_ &&+        (type == InsertType::DOES_NOT_EXIST ||+         type == InsertType::MATCH_OR_DOES_NOT_EXIST)) {+      if (max_size_ && size() << 1 > max_size_) {+        // Would exceed max size.+        throw_exception<std::bad_alloc>();+      }+      rehash_internal(ccount << 1, cohort);+      ccount = chunk_count_.load(std::memory_order_relaxed);+      chunks = chunks_.load(std::memory_order_relaxed);+    }++    DCHECK(chunks); // Use-after-destruction by user.+    node = find_internal(k, hp, chunks, ccount, chunk_idx, tag_idx);++    it.hazptrs_[0].reset_protection(chunks);+    if (node) {+      it.hazptrs_[1].reset_protection(node);+      it.setNode(node, chunks, ccount, chunk_idx, tag_idx);+      if (type == InsertType::MATCH ||+          type == InsertType::MATCH_OR_DOES_NOT_EXIST) {+        if (!match(node->getItem().second)) {+          return false;+        }+      } else if (type == InsertType::DOES_NOT_EXIST) {+        return false;+      }+    } else {+      if (type != InsertType::DOES_NOT_EXIST &&+          type != InsertType::MATCH_OR_DOES_NOT_EXIST &&+          type != InsertType::ANY) {+        it.hazptrs_[0].reset_protection();+        return false;+      }+      // Already checked for rehash on DOES_NOT_EXIST, now check on ANY+      if (size() >= grow_threshold_ && type == InsertType::ANY) {+        if (max_size_ && size() << 1 > max_size_) {+          // Would exceed max size.+          throw_exception<std::bad_alloc>();+        }+        rehash_internal(ccount << 1, cohort);+        ccount = chunk_count_.load(std::memory_order_relaxed);+        chunks = chunks_.load(std::memory_order_relaxed);+        DCHECK(chunks); // Use-after-destruction by user.+        it.hazptrs_[0].reset_protection(chunks);+      }+    }+    return true;+  }++  void rehash_internal(+      size_t new_chunk_count, hazptr_obj_cohort<Atom>* cohort) {+    DCHECK(isPowTwo(new_chunk_count));+    auto old_chunk_count = chunk_count_.load(std::memory_order_relaxed);+    if (old_chunk_count >= new_chunk_count) {+      return;+    }+    auto new_chunks = Chunks::create(new_chunk_count, cohort);+    auto old_chunks = chunks_.load(std::memory_order_relaxed);+    grow_threshold_ =+        to_integral(new_chunk_count * Chunk::kCapacity * load_factor_);++    for (size_t i = 0; i < old_chunk_count; i++) {+      DCHECK(old_chunks); // Use-after-destruction by user.+      Chunk* oldchunk = old_chunks->getChunk(i, old_chunk_count);+      auto occupied = oldchunk->occupiedIter();+      while (occupied.hasNext()) {+        auto idx = occupied.next();+        Node* node = oldchunk->item(idx).load(std::memory_order_relaxed);+        size_t new_chunk_idx;+        size_t new_tag_idx;+        auto h = HashFn()(node->getItem().first);+        auto hp = splitHash(h);+        std::tie(new_chunk_idx, new_tag_idx) =+            findEmptyInsertLocation(new_chunks, new_chunk_count, hp);+        Chunk* newchunk = new_chunks->getChunk(new_chunk_idx, new_chunk_count);+        newchunk->setNodeAndTag(new_tag_idx, node, hp.second);+      }+    }++    seqlock_.fetch_add(1, std::memory_order_release);+    chunk_count_.store(new_chunk_count, std::memory_order_release);+    chunks_.store(new_chunks, std::memory_order_release);+    seqlock_.fetch_add(1, std::memory_order_release);+    if (old_chunks) {+      old_chunks->retire(HazptrTableDeleter(old_chunk_count));+    }+  }++  void getChunksAndCount(+      size_t& ccount, Chunks*& chunks, hazptr_holder<Atom>& hazptr) {+    while (true) {+      auto seqlock = seqlock_.load(std::memory_order_acquire);+      ccount = chunk_count_.load(std::memory_order_acquire);+      chunks = hazptr.protect(chunks_);+      auto seqlock2 = seqlock_.load(std::memory_order_acquire);+      if (!(seqlock & 1) && (seqlock == seqlock2)) {+        break;+      }+    }+    DCHECK(chunks);+  }++  std::pair<size_t, size_t> findEmptyInsertLocation(+      Chunks* chunks, size_t ccount, const HashPair& hp) {+    size_t chunk_idx = hp.first;+    Chunk* dst_chunk = chunks->getChunk(chunk_idx, ccount);+    auto firstEmpty = dst_chunk->firstEmpty();++    if (!firstEmpty.hasIndex()) {+      size_t delta = probeDelta(hp);+      do {+        dst_chunk->incrOutboundOverflowCount();+        chunk_idx += delta;+        dst_chunk = chunks->getChunk(chunk_idx, ccount);+        firstEmpty = dst_chunk->firstEmpty();+      } while (!firstEmpty.hasIndex());+      dst_chunk->incrHostedOverflowCount();+    }+    size_t dst_tag_idx = firstEmpty.index();+    return std::make_pair(chunk_idx & (ccount - 1), dst_tag_idx);+  }++  Mutex m_;+  float load_factor_; // ceil of 1.0+  size_t grow_threshold_;+  Atom<size_t> size_{0};+  size_t const max_size_;++  // Fields needed for read-only access, on separate cacheline.+  alignas(64) Atom<Chunks*> chunks_{nullptr};+  std::atomic<uint64_t> seqlock_{0};+  Atom<size_t> chunk_count_;+};+} // namespace simd++#endif // FOLLY_SSE_PREREQ(4, 2) && FOLLY_F14_VECTOR_INTRINSICS_AVAILABLE++} // namespace concurrenthashmap++/* A Segment is a single shard of the ConcurrentHashMap.+ * All writes take the lock, while readers are all wait-free.+ * Readers always proceed in parallel with the single writer.+ *+ *+ * Possible additional optimizations:+ *+ * * insert / erase could be lock / wait free.  Would need to be+ *   careful that assign and rehash don't conflict (possibly with+ *   reader/writer lock, or microlock per node or per bucket, etc).+ *   Java 8 goes halfway, and does lock per bucket, except for the+ *   first item, that is inserted with a CAS (which is somewhat+ *   specific to java having a lock per object)+ *+ * * I tried using trylock() and find() to warm the cache for insert()+ *   and erase() similar to Java 7, but didn't have much luck.+ *+ * * We could order elements using split ordering, for faster rehash,+ *   and no need to ever copy nodes.  Note that a full split ordering+ *   including dummy nodes increases the memory usage by 2x, but we+ *   could split the difference and still require a lock to set bucket+ *   pointers.+ */+template <+    typename KeyType,+    typename ValueType,+    uint8_t ShardBits = 0,+    typename HashFn = std::hash<KeyType>,+    typename KeyEqual = std::equal_to<KeyType>,+    typename Allocator = std::allocator<uint8_t>,+    template <typename> class Atom = std::atomic,+    class Mutex = std::mutex,+    template <+        typename,+        typename,+        uint8_t,+        typename,+        typename,+        typename,+        template <typename>+        class,+        class>+    class Impl = concurrenthashmap::bucket::BucketTable>+class alignas(64) ConcurrentHashMapSegment {+  using ImplT = Impl<+      KeyType,+      ValueType,+      ShardBits,+      HashFn,+      KeyEqual,+      Allocator,+      Atom,+      Mutex>;++ public:+  typedef KeyType key_type;+  typedef ValueType mapped_type;+  typedef std::pair<const KeyType, ValueType> value_type;+  typedef std::size_t size_type;++  using InsertType = concurrenthashmap::InsertType;+  using Iterator = typename ImplT::Iterator;+  using Node = typename ImplT::Node;+  static constexpr float kDefaultLoadFactor = ImplT::kDefaultLoadFactor;++  ConcurrentHashMapSegment(+      size_t initial_buckets,+      float load_factor,+      size_t max_size,+      hazptr_obj_cohort<Atom>* cohort)+      : impl_(initial_buckets, load_factor, max_size, cohort), cohort_(cohort) {+    DCHECK(cohort);+  }++  ~ConcurrentHashMapSegment() = default;++  size_t size() { return impl_.size(); }++  bool empty() { return impl_.empty(); }++  template <typename Key>+  bool insert(Iterator& it, size_t h, std::pair<Key, mapped_type>&& foo) {+    return insert(it, h, std::move(foo.first), std::move(foo.second));+  }++  template <typename Key, typename Value>+  bool insert(Iterator& it, size_t h, Key&& k, Value&& v) {+    concurrenthashmap::AllocNodeGuard<Node, Allocator> g(+        Allocator(), cohort_, std::forward<Key>(k), std::forward<Value>(v));+    auto res = insert_internal(+        it,+        h,+        g.node->getItem().first,+        InsertType::DOES_NOT_EXIST,+        [](const ValueType&) { return false; },+        g.node);+    if (res) {+      g.dismiss();+    }+    return res;+  }++  template <typename Key, typename... Args>+  bool try_emplace(Iterator& it, size_t h, Key&& k, Args&&... args) {+    // Note: first key is only ever compared.  Second is moved in to+    // create the node, and the first key is never touched again.+    return insert_internal(+        it,+        h,+        std::forward<Key>(k),+        InsertType::DOES_NOT_EXIST,+        [](const ValueType&) { return false; },+        std::forward<Key>(k),+        std::forward<Args>(args)...);+  }++  template <typename... Args>+  bool emplace(Iterator& it, size_t h, const KeyType& k, Node* node) {+    return insert_internal(+        it,+        h,+        k,+        InsertType::DOES_NOT_EXIST,+        [](const ValueType&) { return false; },+        node);+  }++  template <typename Key, typename Value>+  bool insert_or_assign(Iterator& it, size_t h, Key&& k, Value&& v) {+    concurrenthashmap::AllocNodeGuard<Node, Allocator> g(+        Allocator(), cohort_, std::forward<Key>(k), std::forward<Value>(v));+    auto res = insert_internal(+        it,+        h,+        g.node->getItem().first,+        InsertType::ANY,+        [](const ValueType&) { return false; },+        g.node);+    if (res) {+      g.dismiss();+    }+    return res;+  }++  template <typename Key, typename Value, typename Predicate>+  bool insert_or_assign_if(+      Iterator& it, size_t h, Key&& k, Value&& desired, Predicate&& predicate) {+    concurrenthashmap::AllocNodeGuard<Node, Allocator> g(+        Allocator(),+        cohort_,+        std::forward<Key>(k),+        std::forward<Value>(desired));+    auto res = insert_internal(+        it,+        h,+        g.node->getItem().first,+        InsertType::MATCH_OR_DOES_NOT_EXIST,+        std::forward<Predicate>(predicate),+        g.node);+    if (res) {+      g.dismiss();+    }+    return res;+  }++  template <typename Key, typename Value>+  bool assign(Iterator& it, size_t h, Key&& k, Value&& v) {+    concurrenthashmap::AllocNodeGuard<Node, Allocator> g(+        Allocator(), cohort_, std::forward<Key>(k), std::forward<Value>(v));+    auto res = insert_internal(+        it,+        h,+        g.node->getItem().first,+        InsertType::MUST_EXIST,+        [](const ValueType&) { return false; },+        g.node);+    if (res) {+      g.dismiss();+    }+    return res;+  }++  template <typename Key, typename Value, typename Predicate>+  bool assign_if(+      Iterator& it, size_t h, Key&& k, Value&& desired, Predicate&& predicate) {+    concurrenthashmap::AllocNodeGuard<Node, Allocator> g(+        Allocator(),+        cohort_,+        std::forward<Key>(k),+        std::forward<Value>(desired));+    auto res = insert_internal(+        it,+        h,+        g.node->getItem().first,+        InsertType::MATCH,+        std::forward<Predicate>(predicate),+        g.node);+    if (res) {+      g.dismiss();+    }+    return res;+  }++  template <typename Key, typename Value>+  bool assign_if_equal(+      Iterator& it,+      size_t h,+      Key&& k,+      const ValueType& expected,+      Value&& desired) {+    return assign_if(+        it,+        h,+        std::forward<Key>(k),+        std::forward<Value>(desired),+        [&expected](const ValueType& v) { return v == expected; });+  }++  template <typename MatchFunc, typename K, typename... Args>+  bool insert_internal(+      Iterator& it,+      size_t h,+      const K& k,+      InsertType type,+      MatchFunc match,+      Args&&... args) {+    return impl_.insert(+        it, h, k, type, match, cohort_, std::forward<Args>(args)...);+  }++  template <typename MatchFunc, typename K, typename... Args>+  bool insert_internal(+      Iterator& it,+      size_t h,+      const K& k,+      InsertType type,+      MatchFunc match,+      Node* cur) {+    return impl_.insert(it, h, k, type, match, cur, cohort_);+  }++  // Must hold lock.+  void rehash(size_t bucket_count) {+    impl_.rehash(folly::nextPowTwo(bucket_count), cohort_);+  }++  template <typename K>+  bool find(Iterator& res, size_t h, const K& k) {+    return impl_.find(res, h, k);+  }++  // Listed separately because we need a prev pointer.+  template <typename K>+  size_type erase(size_t h, const K& key) {+    return erase_internal(h, key, nullptr, [](const ValueType&) {+      return true;+    });+  }++  template <typename K, typename Predicate>+  size_type erase_key_if(size_t h, const K& key, Predicate&& predicate) {+    return erase_internal(h, key, nullptr, std::forward<Predicate>(predicate));+  }++  template <typename K, typename MatchFunc>+  size_type erase_internal(+      size_t h, const K& key, Iterator* iter, MatchFunc match) {+    return impl_.erase(h, key, iter, match);+  }++  // Unfortunately because we are reusing nodes on rehash, we can't+  // have prev pointers in the bucket chain.  We have to start the+  // search from the bucket.+  //+  // This is a small departure from standard stl containers: erase may+  // throw if hash or key_eq functions throw.+  void erase(Iterator& res, Iterator& pos, size_t h) {+    erase_internal(h, pos->first, &res, [](const ValueType&) { return true; });+    // Invalidate the iterator.+    pos = cend();+  }++  void clear() { impl_.clear(cohort_); }++  void max_load_factor(float factor) { impl_.max_load_factor(factor); }++  Iterator cbegin() { return impl_.cbegin(); }++  Iterator cend() { return impl_.cend(); }++ private:+  ImplT impl_;+  hazptr_obj_cohort<Atom>* cohort_;+};+} // namespace detail+} // namespace folly
+ folly/folly/concurrency/memory/AtomicReadMostlyMainPtr.cpp view
@@ -0,0 +1,43 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#include <folly/concurrency/memory/AtomicReadMostlyMainPtr.h>++#include <folly/executors/InlineExecutor.h>++namespace folly {+namespace detail {++namespace {+struct FailingExecutor : folly::Executor {+  // We shouldn't be invoking any callbacks.+  void add(Func func) override {+    LOG(DFATAL)+        << "Added an RCU callback to the AtomicReadMostlyMainPtr executor.";+    InlineExecutor::instance().add(std::move(func));+  }+};+} // namespace++// *All* modifications of *all* AtomicReadMostlyMainPtrs use the same mutex and+// domain. The first of these just shrinks the size of the individual objects a+// little, but the second is necessary for correctness; we want to support+// arbitrarily many AtomicReadMostlyMainPtrs.+Indestructible<std::mutex> atomicReadMostlyMu;+Indestructible<folly::rcu_domain> atomicReadMostlyDomain(new FailingExecutor);++} // namespace detail+} // namespace folly
+ folly/folly/concurrency/memory/AtomicReadMostlyMainPtr.h view
@@ -0,0 +1,191 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <atomic>+#include <cstdint>+#include <memory>+#include <mutex>++#include <folly/Indestructible.h>+#include <folly/concurrency/memory/ReadMostlySharedPtr.h>+#include <folly/synchronization/Rcu.h>++namespace folly {++namespace detail {+extern Indestructible<std::mutex> atomicReadMostlyMu;+extern Indestructible<rcu_domain> atomicReadMostlyDomain;+} // namespace detail++/*+ * What atomic_shared_ptr is to shared_ptr, AtomicReadMostlyMainPtr is to+ * ReadMostlyMainPtr; it allows racy conflicting accesses to one. This gives+ * true shared_ptr-like semantics, including reclamation at the point where the+ * last pointer to an object goes away.+ *+ * It's about the same speed (slightly slower) as ReadMostlyMainPtr. The most+ * significant feature they share is avoiding reader-reader contention and+ * atomic RMWs in the absence of writes.+ */+template <typename T>+class AtomicReadMostlyMainPtr {+ public:+  AtomicReadMostlyMainPtr() : curMainPtrIndex_(0) {}++  explicit AtomicReadMostlyMainPtr(std::shared_ptr<T> ptr)+      : curMainPtrIndex_(0) {+    mainPtrs_[0] = ReadMostlyMainPtr<T>{std::move(ptr)};+  }++  void operator=(std::shared_ptr<T> desired) { store(std::move(desired)); }++  bool is_lock_free() const { return false; }++  ReadMostlySharedPtr<T> load(+      std::memory_order order = std::memory_order_seq_cst) const {+    detail::atomicReadMostlyDomain->lock();+    // Synchronization point with the store in storeLocked().+    auto index = curMainPtrIndex_.load(order);+    auto result = mainPtrs_[index].getShared();+    detail::atomicReadMostlyDomain->unlock();+    return result;+  }++  void store(+      std::shared_ptr<T> ptr,+      std::memory_order order = std::memory_order_seq_cst) {+    std::shared_ptr<T> old;+    {+      std::lock_guard lg(*detail::atomicReadMostlyMu);+      old = exchangeLocked(std::move(ptr), order);+    }+    // If ~T() runs (triggered by the shared_ptr refcount decrement), it's here,+    // after dropping the lock. This avoids a possible (albeit esoteric)+    // deadlock if ~T() modifies the AtomicReadMostlyMainPtr that used to point+    // to it.+  }++  std::shared_ptr<T> exchange(+      std::shared_ptr<T> ptr,+      std::memory_order order = std::memory_order_seq_cst) {+    std::lock_guard lg(*detail::atomicReadMostlyMu);+    return exchangeLocked(std::move(ptr), order);+  }++  bool compare_exchange_weak(+      std::shared_ptr<T>& expected,+      const std::shared_ptr<T>& desired,+      std::memory_order successOrder = std::memory_order_seq_cst,+      std::memory_order failureOrder = std::memory_order_seq_cst) {+    return compare_exchange_strong(+        expected, desired, successOrder, failureOrder);+  }++  bool compare_exchange_strong(+      std::shared_ptr<T>& expected,+      const std::shared_ptr<T>& desired,+      std::memory_order successOrder = std::memory_order_seq_cst,+      std::memory_order failureOrder = std::memory_order_seq_cst) {+    // See the note at the end of store; we need to defer any destruction we+    // might trigger until after the lock is released.+    // This is not actually needed down the success path (the reference passed+    // in as expected is another pointer to the same object, so we won't+    // decrement the refcount to 0), but "never decrement a refcount while+    // holding a lock" is an easier rule to keep in our heads, and costs us+    // nothing.+    std::shared_ptr<T> prev;+    std::shared_ptr<T> expectedDup;+    {+      std::lock_guard lg(*detail::atomicReadMostlyMu);+      auto index = curMainPtrIndex_.load(failureOrder);+      ReadMostlyMainPtr<T>& oldMain = mainPtrs_[index];+      if (oldMain.get() != expected.get()) {+        expectedDup = std::move(expected);+        expected = oldMain.getStdShared();+        return false;+      }+      prev = exchangeLocked(desired, successOrder);+    }+    return true;+  }++ private:+  // Must hold the global mutex.+  std::shared_ptr<T> exchangeLocked(+      std::shared_ptr<T> ptr,+      std::memory_order order = std::memory_order_seq_cst) {+    // This is where the tricky bits happen; all modifications of the mainPtrs_+    // and index happen here. We maintain the invariant that, on entry to this+    // method, all read-side critical sections in progress are using the version+    // indicated by curMainPtrIndex_, and the other version is nulled out.+    // (Readers can still hold a ReadMostlySharedPtr to the thing the old+    // version used to point to; they just can't access the old version to get+    // that handle any more).+    auto index = curMainPtrIndex_.load(std::memory_order_relaxed);+    ReadMostlyMainPtr<T>& oldMain = mainPtrs_[index];+    ReadMostlyMainPtr<T>& newMain = mainPtrs_[1 - index];+    DCHECK(newMain.get() == nullptr)+        << "Invariant should ensure that at most one version is non-null";+    newMain.reset(std::move(ptr));+    // If order is acq_rel, it should degrade to just release, and if acquire to+    // relaxed, since this is a store rather than an RMW. (Of course, this is+    // such a slow method that we don't really care, but precision is its own+    // reward. If TSAN one day understands asymmetric barriers, this will also+    // improve its error detection here). We get our "acquire-y-ness" from the+    // mutex.+    auto realOrder =+        (order == std::memory_order_acq_rel ? std::memory_order_release+             : order == std::memory_order_acquire+             ? std::memory_order_relaxed+             : order);+    // After this, read-side critical sections can access both versions, but+    // new ones will use newMain.+    // This is also synchronization point with loads.+    curMainPtrIndex_.store(1 - index, realOrder);+    // Wait for all read-side critical sections using oldMain to finish.+    detail::atomicReadMostlyDomain->synchronize();+    // We've reestablished the first half of the invariant (all readers are+    // using newMain), now let's establish the other one (that the other pointer+    // is null).+    auto result = oldMain.getStdShared();+    oldMain.reset();+    return result;+  }++  // The right way to think of this implementation is as an+  // std::atomic<ReadMostlyMainPtr<T>*>, protected by RCU. There's only two+  // tricky parts:+  // 1. We give ourselves our own RCU domain, and synchronize on modification,+  //    so that we don't do any batching of deallocations. This gives+  //    shared_ptr-like eager reclamation semantics.+  // 2. Instead of putting the ReadMostlyMainPtrs on the heap, we keep them as+  //    part of the same object to improve locality.++  // Really, just a 0/1 index. This is also the synchronization point for memory+  // orders.+  std::atomic<uint8_t> curMainPtrIndex_;++  // Both the ReadMostlyMainPtrs themselves and the domain have nontrivial+  // indirections even on the read path, and asymmetric barriers on the write+  // path. Some of these could be fused as a later optimization, at the cost of+  // having to put more tricky threading primitives in this class that are+  // currently abstracted out by those.+  ReadMostlyMainPtr<T> mainPtrs_[2];+};++} // namespace folly
+ folly/folly/concurrency/memory/PrimaryPtr.h view
@@ -0,0 +1,338 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <memory>+#include <mutex>++#include <folly/Function.h>+#include <folly/futures/Cleanup.h>+#include <folly/futures/Future.h>++#include <glog/logging.h>++namespace folly {++template <typename T>+class EnablePrimaryFromThis;++template <typename T>+class PrimaryPtr;++template <typename T>+class PrimaryPtrRef;++namespace detail {+struct publicallyDerivedFromEnablePrimaryFromThis_fn {+  template <class T>+  void operator()(const EnablePrimaryFromThis<T>&) const {}+};+} // namespace detail++template <class T>+constexpr bool is_enable_master_from_this_v = folly::+    is_invocable_v<detail::publicallyDerivedFromEnablePrimaryFromThis_fn, T>;++template <typename T>+using is_enable_master_from_this =+    std::bool_constant<is_enable_master_from_this_v<T>>;++/**+ * EnablePrimaryFromThis provides an object with appropriate access to the+ * functionality of the PrimaryPtr holding this.+ */+template <typename T>+class EnablePrimaryFromThis {+  // initializes members when the PrimaryPtr for this is constructed+  //+  // used by the PrimaryPtr for this, to invoke the EnablePrimaryFromThis base+  // of T, if it exists.+  template <+      class O,+      class Master,+      std::enable_if_t<is_enable_master_from_this_v<O>, int> = 0>+  static void set(EnablePrimaryFromThis<O>* that, Master& m) {+    that->outerPtrWeak_ = m.outerPtrWeak_;+  }++  template <+      class O,+      class Master,+      std::enable_if_t<!is_enable_master_from_this_v<O>, int> = 0>+  static void set(O*, Master&) {}++ public:+  // Gets a non-owning reference to the pointer. PrimaryPtr::join() and the+  // PrimaryPtr::cleanup() work do *NOT* wait for outstanding PrimaryPtrRef+  // objects to be released.+  PrimaryPtrRef<T> masterRefFromThis() {+    return PrimaryPtrRef<T>(outerPtrWeak_);+  }++  // Gets a non-owning const reference to the pointer. PrimaryPtr::join() and+  // the PrimaryPtr::cleanup() work do *NOT* wait for outstanding PrimaryPtrRef+  // objects to be released.+  PrimaryPtrRef<const T> masterRefFromThis() const {+    return PrimaryPtrRef<const T>(outerPtrWeak_);+  }++  // Attempts to lock a pointer. Returns null if pointer is not set or if+  // PrimaryPtr::join() was called or the PrimaryPtr::cleanup() task was started+  // (even if the call to PrimaryPtr::join() hasn't returned yet and the+  // PrimaryPtr::cleanup() task has not completed yet).+  std::shared_ptr<T> masterLockFromThis() {+    if (auto outerPtr = outerPtrWeak_.lock()) {+      return *outerPtr;+    }+    return nullptr;+  }++  // Attempts to lock a pointer. Returns null if pointer is not set or if+  // PrimaryPtr::join() was called or the PrimaryPtr::cleanup() task was started+  // (even if the call to PrimaryPtr::join() hasn't returned yet and the+  // PrimaryPtr::cleanup() task has not completed yet).+  std::shared_ptr<T const> masterLockFromThis() const {+    if (!*this) {+      return nullptr;+    }+    if (auto outerPtr = outerPtrWeak_.lock()) {+      return *outerPtr;+    }+    return nullptr;+  }++ private:+  template <class>+  friend class PrimaryPtr;++  std::weak_ptr<std::shared_ptr<T>> outerPtrWeak_;+};++/**+ * PrimaryPtr should be used to achieve deterministic destruction of objects+ * with shared ownership. Once an object is managed by a PrimaryPtr, shared_ptrs+ * can be obtained pointing to that object. However destroying those shared_ptrs+ * will never call the object destructor inline. To destroy the object, join()+ * method must be called on PrimaryPtr or the task returned from cleanup() must+ * be completed, which will wait for all shared_ptrs to be released and then+ * call the object destructor on the caller supplied execution context.+ */+template <typename T>+class PrimaryPtr {+  // retrieves nested cleanup() work from innerPtr_. Called when the PrimaryPtr+  // cleanup() task has finished waiting for outstanding references+  //+  template <class Cleanup, std::enable_if_t<is_cleanup_v<Cleanup>, int> = 0>+  static folly::SemiFuture<folly::Unit> getCleanup(Cleanup* cleanup) {+    return std::move(*cleanup).cleanup();+  }++  template <class O, std::enable_if_t<!is_cleanup_v<O>, int> = 0>+  static folly::SemiFuture<folly::Unit> getCleanup(O*) {+    return folly::makeSemiFuture();+  }++ public:+  PrimaryPtr() = delete;+  template <class T2, class Deleter>+  PrimaryPtr(std::unique_ptr<T2, Deleter> ptr) {+    set(std::move(ptr));+  }++  explicit PrimaryPtr(std::nullptr_t) {}++  ~PrimaryPtr() {+    if (*this) {+      LOG(FATAL) << "PrimaryPtr has to be joined explicitly.";+    }+  }++  PrimaryPtr(PrimaryPtr<T>&&) = default;++  void swap(PrimaryPtr<T>& other) noexcept {+    PrimaryPtr<T> temp = std::move(other);+    other = std::move(*this);+    *this = std::move(temp);+  }++  PrimaryPtr<T> exchange(PrimaryPtr<T>&& newVal) noexcept {+    PrimaryPtr<T> oldVal = std::move(*this);+    *this = std::move(newVal);+    return oldVal;+  }++  explicit operator bool() const { return !!innerPtr_; }++  // Attempts to lock a pointer. Returns null if pointer is not set or if join()+  // was called or the cleanup() task was started (even if the call to join()+  // hasn't returned yet and the cleanup() task has not completed yet).+  std::shared_ptr<T> lock() const {+    if (auto outerPtr = outerPtrWeak_.lock()) {+      return *outerPtr;+    }+    return nullptr;+  }++  // Waits until all the refereces obtained via lock() are released. Then+  // destroys the object in the current thread.+  // Can not be called concurrently with set().+  void join() {+    if (!*this) {+      return;+    }+    this->cleanup().get();+  }++  // Returns: a SemiFuture that waits until all the refereces obtained via+  // lock() are released. Then destroys the object on the Executor provided to+  // the SemiFuture.+  //+  // The returned SemiFuture must run to completion before calling set()+  //+  folly::SemiFuture<folly::Unit> cleanup() {+    return folly::makeSemiFuture()+        // clear outerPtrShared_ after cleanup is started+        // to disable further calls to lock().+        // then wait for outstanding references.+        .deferValue([this](folly::Unit) {+          if (!this->outerPtrShared_) {+            LOG(FATAL)+                << "Cleanup already run - lock() was previouly disabled.";+          }+          this->outerPtrShared_.reset();+          return std::move(this->unreferenced_);+        })+        // start cleanup tasks+        .deferValue([this](folly::Unit) { return getCleanup(innerPtr_.get()); })+        .defer([this](folly::Try<folly::Unit> r) {+          if (r.hasException()) {+            LOG(FATAL) << "Cleanup actions must be noexcept.";+          }+          this->innerPtr_.reset();+        });+  }++  // Sets the pointer. Can not be called concurrently with lock() or join() or+  // ref() or while the SemiFuture returned from cleanup() is running.+  template <class T2, class Deleter>+  void set(std::unique_ptr<T2, Deleter> ptr) {+    if (*this) {+      LOG(FATAL) << "PrimaryPtr has to be joined before being set.";+    }++    if (!ptr) {+      return;+    }++    auto rawPtr = ptr.get();+    innerPtr_ = std::unique_ptr<T, folly::Function<void(T*)>>{+        ptr.release(),+        [d = ptr.get_deleter(), rawPtr](T*) mutable { d(rawPtr); }};++    auto [referencesPromise, referencesFuture] =+        folly::makePromiseContract<folly::Unit>();+    unreferenced_ = std::move(referencesFuture);++    // The deleter object needs to be copyable in std::shared_ptr on some+    // platform. To work around this limitation we can slightly tweak the+    // semantics of deleter copy constructor and check we always use this+    // object at most once.+    class LastReference {+     public:+      LastReference(Promise<Unit>&& p) : p_(std::move(p)) {}+      LastReference(LastReference&&) = default;+      LastReference(LastReference& other) : LastReference(std::move(other)) {}+      void operator()(T*) {+        DCHECK(!p_.isFulfilled());+        p_.setValue();+      }++     private:+      Promise<Unit> p_;+    };+    auto innerPtrShared = std::shared_ptr<T>(+        innerPtr_.get(), LastReference{std::move(referencesPromise)});++    outerPtrWeak_ = outerPtrShared_ =+        std::make_shared<std::shared_ptr<T>>(innerPtrShared);++    // attaches optional EnablePrimaryFromThis base of innerPtr_ to this+    // PrimaryPtr+    EnablePrimaryFromThis<T>::set(innerPtr_.get(), *this);+  }++  // Gets a non-owning reference to the pointer. join() and the cleanup() work+  // do *NOT* wait for outstanding PrimaryPtrRef objects to be released.+  PrimaryPtrRef<T> ref() const { return PrimaryPtrRef<T>(outerPtrWeak_); }++ private:+  // Making this private for now since non-null PrimaryPtr's must be explicitly+  // joined before destruction.+  PrimaryPtr<T>& operator=(PrimaryPtr<T>&& other) = default;++  template <class>+  friend class EnablePrimaryFromThis;+  friend class PrimaryPtrRef<T>;++  folly::SemiFuture<folly::Unit> unreferenced_;+  std::shared_ptr<std::shared_ptr<T>> outerPtrShared_;+  std::weak_ptr<std::shared_ptr<T>> outerPtrWeak_;+  std::unique_ptr<T, folly::Function<void(T*)>> innerPtr_;+};++template <typename T>+void swap(PrimaryPtr<T>& x, PrimaryPtr<T>& y) noexcept {+  x.swap(y);+}++template <typename T>+PrimaryPtr<T> exchange(PrimaryPtr<T>& x, PrimaryPtr<T>&& newVal) noexcept {+  return x.exchange(std::move(newVal));+}++/**+ * PrimaryPtrRef is a non-owning reference to the pointer. PrimaryPtr::join()+ * and the PrimaryPtr::cleanup() work do *NOT* wait for outstanding+ * PrimaryPtrRef objects to be released.+ */+template <typename T>+class PrimaryPtrRef {+ public:+  PrimaryPtrRef() = default;++  // Attempts to lock a pointer. Returns null if pointer is not set or if+  // join() was called or cleanup() work was started (even if the call to join()+  // hasn't returned yet or the cleanup() work has not completed yet).+  std::shared_ptr<T> lock() const {+    if (auto outerPtr = outerPtrWeak_.lock()) {+      return *outerPtr;+    }+    return nullptr;+  }++ private:+  template <class>+  friend class EnablePrimaryFromThis;+  template <class>+  friend class PrimaryPtr;+  /* implicit */ PrimaryPtrRef(std::weak_ptr<std::shared_ptr<T>> outerPtrWeak)+      : outerPtrWeak_(std::move(outerPtrWeak)) {}++  std::weak_ptr<std::shared_ptr<T>> outerPtrWeak_;+};++} // namespace folly
+ folly/folly/concurrency/memory/ReadMostlySharedPtr.h view
@@ -0,0 +1,511 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <atomic>++#include <folly/Function.h>+#include <folly/concurrency/memory/TLRefCount.h>++namespace folly {++template <typename T, typename RefCount>+class ReadMostlyMainPtr;+template <typename T, typename RefCount>+class ReadMostlyWeakPtr;+template <typename T, typename RefCount>+class ReadMostlySharedPtr;+template <typename RefCount>+class ReadMostlyMainPtrDeleter;++using DefaultRefCount = TLRefCount;++namespace detail {++template <typename RefCount = DefaultRefCount>+class ReadMostlySharedPtrCore {+ public:+  std::shared_ptr<const void> getShared() { return ptr_; }++  bool incref() { return ++count_ > 0; }++  void decref() {+    if (--count_ == 0) {+      ptr_.reset();++      decrefWeak();+    }+  }++  void increfWeak() {+    auto value = ++weakCount_;+    DCHECK_GT(value, 0);+  }++  void decrefWeak() {+    if (--weakCount_ == 0) {+      delete this;+    }+  }++  size_t useCount() const { return *count_; }++  ~ReadMostlySharedPtrCore() noexcept {+    assert(*count_ == 0);+    assert(*weakCount_ == 0);+  }++ private:+  template <typename T, typename RefCount2>+  friend class folly::ReadMostlyMainPtr;+  friend class ReadMostlyMainPtrDeleter<RefCount>;++  explicit ReadMostlySharedPtrCore(std::shared_ptr<const void> ptr)+      : ptr_(std::move(ptr)) {}++  RefCount count_;+  RefCount weakCount_;+  std::shared_ptr<const void> ptr_;+};++} // namespace detail++template <typename T, typename RefCount = DefaultRefCount>+class ReadMostlyMainPtr {+ public:+  ReadMostlyMainPtr() {}++  explicit ReadMostlyMainPtr(std::shared_ptr<T> ptr) { reset(std::move(ptr)); }++  ReadMostlyMainPtr(const ReadMostlyMainPtr&) = delete;+  ReadMostlyMainPtr& operator=(const ReadMostlyMainPtr&) = delete;++  ReadMostlyMainPtr(ReadMostlyMainPtr&& other) noexcept {+    *this = std::move(other);+  }++  ReadMostlyMainPtr& operator=(ReadMostlyMainPtr&& other) noexcept {+    std::swap(impl_, other.impl_);+    std::swap(ptrRaw_, other.ptrRaw_);+    return *this;+  }++  bool operator==(const ReadMostlyMainPtr<T, RefCount>& other) const {+    return get() == other.get();+  }++  bool operator==(T* other) const { return get() == other; }++  bool operator==(const ReadMostlySharedPtr<T, RefCount>& other) const {+    return get() == other.get();+  }++  ~ReadMostlyMainPtr() noexcept { reset(); }++  void reset() noexcept {+    if (impl_) {+      ptrRaw_ = nullptr;+      impl_->count_.useGlobal();+      impl_->weakCount_.useGlobal();+      impl_->decref();+      impl_ = nullptr;+    }+  }++  void reset(std::shared_ptr<T> ptr) {+    reset();+    if (ptr) {+      ptrRaw_ = ptr.get();+      impl_ = new detail::ReadMostlySharedPtrCore<RefCount>(std::move(ptr));+    }+  }++  T* get() const { return ptrRaw_; }++  std::shared_ptr<T> getStdShared() const {+    if (impl_) {+      return {impl_->getShared(), ptrRaw_};+    } else {+      return {};+    }+  }++  T& operator*() const { return *get(); }++  T* operator->() const { return get(); }++  ReadMostlySharedPtr<T, RefCount> getShared() const {+    return ReadMostlySharedPtr<T, RefCount>(*this);+  }++  explicit operator bool() const { return impl_ != nullptr; }++ private:+  template <typename U, typename RefCount2>+  friend class ReadMostlyWeakPtr;+  template <typename U, typename RefCount2>+  friend class ReadMostlySharedPtr;+  friend class ReadMostlyMainPtrDeleter<RefCount>;++  detail::ReadMostlySharedPtrCore<RefCount>* impl_{nullptr};+  T* ptrRaw_{nullptr};+};++template <typename T, typename RefCount = DefaultRefCount>+class ReadMostlyWeakPtr {+ public:+  ReadMostlyWeakPtr() {}++  ReadMostlyWeakPtr(const ReadMostlyWeakPtr& other) { *this = other; }++  ReadMostlyWeakPtr(ReadMostlyWeakPtr&& other) noexcept {+    *this = std::move(other);+  }++  template <+      typename T2,+      typename = std::enable_if_t<std::is_convertible<T2*, T*>::value>>+  ReadMostlyWeakPtr(const ReadMostlyWeakPtr<T2, RefCount>& other) {+    *this = other;+  }++  template <+      typename T2,+      typename = std::enable_if_t<std::is_convertible<T2*, T*>::value>>+  ReadMostlyWeakPtr(ReadMostlyWeakPtr<T2, RefCount>&& other) noexcept {+    *this = std::move(other);+  }++  template <+      typename T2,+      typename = std::enable_if_t<std::is_convertible<T2*, T*>::value>>+  explicit ReadMostlyWeakPtr(const ReadMostlyMainPtr<T2, RefCount>& other) {+    *this = other;+  }++  template <+      typename T2,+      typename = std::enable_if_t<std::is_convertible<T2*, T*>::value>>+  explicit ReadMostlyWeakPtr(const ReadMostlySharedPtr<T2, RefCount>& other) {+    *this = other;+  }++  ReadMostlyWeakPtr& operator=(const ReadMostlyWeakPtr& other) {+    reset(other.impl_, other.ptrRaw_);+    return *this;+  }++  ReadMostlyWeakPtr& operator=(ReadMostlyWeakPtr&& other) noexcept {+    std::swap(impl_, other.impl_);+    std::swap(ptrRaw_, other.ptrRaw_);+    return *this;+  }++  template <+      typename T2,+      typename = std::enable_if_t<std::is_convertible<T2*, T*>::value>>+  ReadMostlyWeakPtr& operator=(const ReadMostlyWeakPtr<T2, RefCount>& other) {+    reset(other.impl_, other.ptrRaw_);+    return *this;+  }++  template <+      typename T2,+      typename = std::enable_if_t<std::is_convertible<T2*, T*>::value>>+  ReadMostlyWeakPtr& operator=(+      ReadMostlyWeakPtr<T2, RefCount>&& other) noexcept {+    reset();+    impl_ = std::exchange(other.impl_, nullptr);+    ptrRaw_ = std::exchange(other.ptrRaw_, nullptr);+    return *this;+  }++  template <+      typename T2,+      typename = std::enable_if_t<std::is_convertible<T2*, T*>::value>>+  ReadMostlyWeakPtr& operator=(const ReadMostlyMainPtr<T2, RefCount>& mainPtr) {+    reset(mainPtr.impl_, mainPtr.ptrRaw_);+    return *this;+  }++  template <+      typename T2,+      typename = std::enable_if_t<std::is_convertible<T2*, T*>::value>>+  ReadMostlyWeakPtr& operator=(+      const ReadMostlySharedPtr<T2, RefCount>& mainPtr) {+    reset(mainPtr.impl_, mainPtr.ptrRaw_);+    return *this;+  }++  ~ReadMostlyWeakPtr() noexcept { reset(nullptr, nullptr); }++  ReadMostlySharedPtr<T, RefCount> lock() {+    return ReadMostlySharedPtr<T, RefCount>(*this);+  }++ private:+  template <typename U, typename RefCount2>+  friend class ReadMostlyWeakPtr;+  template <typename U, typename RefCount2>+  friend class ReadMostlySharedPtr;++  void reset(detail::ReadMostlySharedPtrCore<RefCount>* impl, T* ptrRaw) {+    if (impl_ == impl) {+      return;+    }++    if (impl_) {+      impl_->decrefWeak();+    }+    impl_ = impl;+    ptrRaw_ = ptrRaw;+    if (impl_) {+      impl_->increfWeak();+    }+  }++  detail::ReadMostlySharedPtrCore<RefCount>* impl_{nullptr};+  T* ptrRaw_{nullptr};+};++template <typename T, typename RefCount = DefaultRefCount>+class ReadMostlySharedPtr {+ public:+  ReadMostlySharedPtr() {}++  ReadMostlySharedPtr(const ReadMostlySharedPtr& other) { *this = other; }++  ReadMostlySharedPtr(ReadMostlySharedPtr&& other) noexcept {+    *this = std::move(other);+  }++  template <+      typename T2,+      typename = std::enable_if_t<std::is_convertible<T2*, T*>::value>>+  ReadMostlySharedPtr(const ReadMostlySharedPtr<T2, RefCount>& other) {+    *this = other;+  }++  template <+      typename T2,+      typename = std::enable_if_t<std::is_convertible<T2*, T*>::value>>+  ReadMostlySharedPtr(ReadMostlySharedPtr<T2, RefCount>&& other) noexcept {+    *this = std::move(other);+  }++  template <+      typename T2,+      typename = std::enable_if_t<std::is_convertible<T2*, T*>::value>>+  explicit ReadMostlySharedPtr(const ReadMostlyWeakPtr<T2, RefCount>& other) {+    *this = other;+  }++  // Generally, this shouldn't be used.+  template <+      typename T2,+      typename = std::enable_if_t<std::is_convertible<T2*, T*>::value>>+  explicit ReadMostlySharedPtr(const ReadMostlyMainPtr<T2, RefCount>& other) {+    *this = other;+  }++  ReadMostlySharedPtr& operator=(const ReadMostlySharedPtr& other) {+    reset(other.impl_, other.ptrRaw_);+    return *this;+  }++  ReadMostlySharedPtr& operator=(ReadMostlySharedPtr&& other) noexcept {+    std::swap(impl_, other.impl_);+    std::swap(ptrRaw_, other.ptrRaw_);+    return *this;+  }++  template <+      typename T2,+      typename = std::enable_if_t<std::is_convertible<T2*, T*>::value>>+  ReadMostlySharedPtr& operator=(+      const ReadMostlySharedPtr<T2, RefCount>& other) {+    reset(other.impl_, other.ptrRaw_);+    return *this;+  }++  template <+      typename T2,+      typename = std::enable_if_t<std::is_convertible<T2*, T*>::value>>+  ReadMostlySharedPtr& operator=(+      ReadMostlySharedPtr<T2, RefCount>&& other) noexcept {+    reset();+    impl_ = std::exchange(other.impl_, nullptr);+    ptrRaw_ = std::exchange(other.ptrRaw_, nullptr);+    return *this;+  }++  template <+      typename T2,+      typename = std::enable_if_t<std::is_convertible<T2*, T*>::value>>+  ReadMostlySharedPtr& operator=(const ReadMostlyWeakPtr<T2, RefCount>& other) {+    reset(other.impl_, other.ptrRaw_);+    return *this;+  }++  template <+      typename T2,+      typename = std::enable_if_t<std::is_convertible<T2*, T*>::value>>+  ReadMostlySharedPtr& operator=(const ReadMostlyMainPtr<T2, RefCount>& other) {+    reset(other.impl_, other.ptrRaw_);+    return *this;+  }++  ~ReadMostlySharedPtr() noexcept { reset(nullptr, nullptr); }++  bool operator==(const ReadMostlyMainPtr<T, RefCount>& other) const {+    return get() == other.get();+  }++  bool operator==(T* other) const { return get() == other; }++  bool operator==(const ReadMostlySharedPtr<T, RefCount>& other) const {+    return get() == other.get();+  }++  void reset() { reset(nullptr, nullptr); }++  T* get() const { return ptrRaw_; }++  std::shared_ptr<T> getStdShared() const {+    if (impl_) {+      return {impl_->getShared(), ptrRaw_};+    } else {+      return {};+    }+  }++  T& operator*() const { return *get(); }++  T* operator->() const { return get(); }++  size_t use_count() const { return impl_->useCount(); }++  bool unique() const { return use_count() == 1; }++  explicit operator bool() const { return impl_ != nullptr; }++ private:+  template <typename U, typename RefCount2>+  friend class ReadMostlyWeakPtr;+  template <typename U, typename RefCount2>+  friend class ReadMostlySharedPtr;++  void reset(detail::ReadMostlySharedPtrCore<RefCount>* impl, T* ptrRaw) {+    if (impl_ == impl) {+      return;+    }++    if (impl_) {+      impl_->decref();+      impl_ = nullptr;+      ptrRaw_ = nullptr;+    }++    if (impl && impl->incref()) {+      impl_ = impl;+      ptrRaw_ = ptrRaw;+    }+  }++  T* ptrRaw_{nullptr};+  detail::ReadMostlySharedPtrCore<RefCount>* impl_{nullptr};+};++/**+ * This can be used to destroy multiple ReadMostlyMainPtrs at once.+ */+template <typename RefCount = DefaultRefCount>+class ReadMostlyMainPtrDeleter {+ public:+  ~ReadMostlyMainPtrDeleter() noexcept {+    RefCount::useGlobal(refCounts_);+    for (auto& decref : decrefs_) {+      decref();+    }+  }++  template <typename T>+  void add(ReadMostlyMainPtr<T, RefCount> ptr) noexcept {+    if (!ptr.impl_) {+      return;+    }++    refCounts_.push_back(&ptr.impl_->count_);+    refCounts_.push_back(&ptr.impl_->weakCount_);+    decrefs_.push_back([impl = ptr.impl_] { impl->decref(); });+    ptr.impl_ = nullptr;+    ptr.ptrRaw_ = nullptr;+  }++ private:+  std::vector<RefCount*> refCounts_;+  std::vector<folly::Function<void()>> decrefs_;+};++template <typename T, typename RefCount>+inline bool operator==(+    const ReadMostlyMainPtr<T, RefCount>& ptr, std::nullptr_t) {+  return ptr.get() == nullptr;+}++template <typename T, typename RefCount>+inline bool operator==(+    std::nullptr_t, const ReadMostlyMainPtr<T, RefCount>& ptr) {+  return ptr.get() == nullptr;+}++template <typename T, typename RefCount>+inline bool operator==(+    const ReadMostlySharedPtr<T, RefCount>& ptr, std::nullptr_t) {+  return ptr.get() == nullptr;+}++template <typename T, typename RefCount>+inline bool operator==(+    std::nullptr_t, const ReadMostlySharedPtr<T, RefCount>& ptr) {+  return ptr.get() == nullptr;+}++template <typename T, typename RefCount>+inline bool operator!=(+    const ReadMostlyMainPtr<T, RefCount>& ptr, std::nullptr_t) {+  return !(ptr == nullptr);+}++template <typename T, typename RefCount>+inline bool operator!=(+    std::nullptr_t, const ReadMostlyMainPtr<T, RefCount>& ptr) {+  return !(ptr == nullptr);+}++template <typename T, typename RefCount>+inline bool operator!=(+    const ReadMostlySharedPtr<T, RefCount>& ptr, std::nullptr_t) {+  return !(ptr == nullptr);+}++template <typename T, typename RefCount>+inline bool operator!=(+    std::nullptr_t, const ReadMostlySharedPtr<T, RefCount>& ptr) {+  return !(ptr == nullptr);+}+} // namespace folly
+ folly/folly/concurrency/memory/TLRefCount.h view
@@ -0,0 +1,228 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <folly/ThreadLocal.h>+#include <folly/synchronization/AsymmetricThreadFence.h>+#include <folly/synchronization/detail/Sleeper.h>++namespace folly {++class TLRefCount {+ public:+  using Int = int64_t;++  TLRefCount()+      : localCount_([&]() { return LocalRefCount(*this); }),+        collectGuard_(this, [](void*) {}) {}++  ~TLRefCount() noexcept {+    assert(globalCount_.load() == 0);+    assert(state_.load() == State::GLOBAL);+  }++  // This can't increment from 0.+  Int operator++() noexcept {+    auto& localCount = *localCount_;++    if (++localCount) {+      return 42;+    }++    if (state_.load() == State::GLOBAL_TRANSITION) {+      std::lock_guard lg(globalMutex_);+    }++    assert(state_.load() == State::GLOBAL);++    auto value = globalCount_.load();+    do {+      if (value == 0) {+        return 0;+      }+    } while (!globalCount_.compare_exchange_weak(value, value + 1));++    return value + 1;+  }++  Int operator--() noexcept {+    auto& localCount = *localCount_;++    if (--localCount) {+      return 42;+    }++    if (state_.load() == State::GLOBAL_TRANSITION) {+      std::lock_guard lg(globalMutex_);+    }++    assert(state_.load() == State::GLOBAL);++    return globalCount_-- - 1;+  }++  Int operator*() const {+    if (state_ != State::GLOBAL) {+      return 42;+    }+    return globalCount_.load();+  }++  void useGlobal() noexcept {+    std::array<TLRefCount*, 1> ptrs{{this}};+    useGlobal(ptrs);+  }++  template <typename Container>+  static void useGlobal(const Container& refCountPtrs) {+#ifdef FOLLY_SANITIZE_THREAD+    // TSAN has a limitation for the number of locks held concurrently, so it's+    // safer to call useGlobal() serially.+    if (refCountPtrs.size() > 1) {+      for (auto refCountPtr : refCountPtrs) {+        refCountPtr->useGlobal();+      }+      return;+    }+#endif++    std::vector<std::unique_lock<std::mutex>> lgs_;+    for (auto refCountPtr : refCountPtrs) {+      lgs_.emplace_back(refCountPtr->globalMutex_);++      refCountPtr->state_ = State::GLOBAL_TRANSITION;+    }++    asymmetric_thread_fence_heavy(std::memory_order_seq_cst);++    for (auto refCountPtr : refCountPtrs) {+      std::weak_ptr<void> collectGuardWeak = refCountPtr->collectGuard_;++      // Make sure we can't create new LocalRefCounts+      refCountPtr->collectGuard_.reset();++      while (!collectGuardWeak.expired()) {+        auto accessor = refCountPtr->localCount_.accessAllThreads();+        for (auto& count : accessor) {+          count.collect();+        }+      }++      refCountPtr->state_ = State::GLOBAL;+    }+  }++ private:+  using AtomicInt = std::atomic<Int>;++  enum class State {+    LOCAL,+    GLOBAL_TRANSITION,+    GLOBAL,+  };++  class LocalRefCount {+   public:+    explicit LocalRefCount(TLRefCount& refCount) : refCount_(refCount) {+      std::lock_guard lg(refCount.globalMutex_);++      collectGuard_ = refCount.collectGuard_;+    }++    ~LocalRefCount() { collect(); }++    void collect() {+      {+        std::lock_guard lg(collectMutex_);++        if (!collectGuard_) {+          return;+        }++        collectCount_ = count_.load();+        refCount_.globalCount_.fetch_add(collectCount_);+        collectGuard_.reset();+      }+      // Once we exit collect(), it's possible TLRefCount may be deleted by our+      // user since the global count may reach zero. We must therefore ensure+      // that the thread corresponding to this LocalRefCount is not still+      // executing the update() function. We wait on inUpdate_ to ensure this.+      // We won't have to worry about further update() calls beyond this point,+      // because the state is already non-LOCAL. We also don't need to worry+      // about if a thread is in an update() call but have not gotten around to+      // setting inUpdate_ to true yet, because then count_ has also not been+      // updated and we couldn't reach global zero in that case.+      folly::detail::Sleeper sleeper;+      while (inUpdate_.load(std::memory_order_acquire)) {+        sleeper.wait();+      }+    }++    bool operator++() { return update(1); }++    bool operator--() { return update(-1); }++   private:+    bool update(Int delta) {+      if (FOLLY_UNLIKELY(refCount_.state_.load() != State::LOCAL)) {+        return false;+      }++      // This is equivalent to atomic fetch_add. We know that this operation+      // is always performed from a single thread.+      // asymmetric_thread_fence_light() makes things faster than atomic+      // fetch_add on platforms with native support.+      auto count = count_.load(std::memory_order_relaxed) + delta;+      inUpdate_.store(true, std::memory_order_relaxed);+      SCOPE_EXIT {+        inUpdate_.store(false, std::memory_order_release);+      };+      count_.store(count, std::memory_order_release);++      asymmetric_thread_fence_light(std::memory_order_seq_cst);++      if (FOLLY_UNLIKELY(refCount_.state_.load() != State::LOCAL)) {+        std::lock_guard lg(collectMutex_);++        if (collectGuard_) {+          return true;+        }+        if (collectCount_ != count) {+          return false;+        }+      }++      return true;+    }++    AtomicInt count_{0};+    std::atomic<bool> inUpdate_{false};+    TLRefCount& refCount_;++    std::mutex collectMutex_;+    Int collectCount_{0};+    std::shared_ptr<void> collectGuard_;+  };++  std::atomic<State> state_{State::LOCAL};+  folly::ThreadLocal<LocalRefCount, TLRefCount> localCount_;+  std::atomic<int64_t> globalCount_{1};+  std::mutex globalMutex_;+  std::shared_ptr<void> collectGuard_;+};++} // namespace folly
+ folly/folly/container/Access.h view
@@ -0,0 +1,57 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <folly/functional/Invoke.h>++namespace folly {++namespace access {++/// size_fn+/// size+///+/// Invokes unqualified size with std::size in scope.+FOLLY_CREATE_FREE_INVOKER_SUITE(size, std);++/// empty_fn+/// empty+///+/// Invokes unqualified empty with std::empty in scope.+FOLLY_CREATE_FREE_INVOKER_SUITE(empty, std);++/// data_fn+/// data+///+/// Invokes unqualified data with std::data in scope.+FOLLY_CREATE_FREE_INVOKER_SUITE(data, std);++/// begin_fn+/// begin+///+/// Invokes unqualified begin with std::begin in scope.+FOLLY_CREATE_FREE_INVOKER_SUITE(begin, std);++/// end_fn+/// end+///+/// Invokes unqualified end with std::end in scope.+FOLLY_CREATE_FREE_INVOKER_SUITE(end, std);++} // namespace access++} // namespace folly
+ folly/folly/container/Array.h view
@@ -0,0 +1,87 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+/**+ * Helper functions to create std::arrays.+ *+ * @file container/Array.h+ * @refcode folly/docs/examples/folly/container/Array.cpp+ */++#pragma once++#include <array>+#include <type_traits>+#include <utility>++#include <folly/CPortability.h>+#include <folly/Traits.h>+#include <folly/Utility.h>++namespace folly {++namespace array_detail {+template <class T>+using is_ref_wrapper = is_instantiation_of<std::reference_wrapper, T>;++template <typename T>+using not_ref_wrapper =+    folly::Negation<is_ref_wrapper<typename std::decay<T>::type>>;++template <typename D, typename...>+struct return_type_helper {+  using type = D;+};+template <typename... TList>+struct return_type_helper<void, TList...> {+  static_assert(+      folly::Conjunction<not_ref_wrapper<TList>...>::value,+      "TList cannot contain reference_wrappers when D is void");+  using type = typename std::common_type<TList...>::type;+};++template <typename D, typename... TList>+using return_type = std::+    array<typename return_type_helper<D, TList...>::type, sizeof...(TList)>;+} // namespace array_detail++/// Constructs a std::array with the given argument list.+///+/// @param t  The values to be put in the array.+template <typename D = void, typename... TList>+constexpr array_detail::return_type<D, TList...> make_array(TList&&... t) {+  using value_type =+      typename array_detail::return_type_helper<D, TList...>::type;+  return {{static_cast<value_type>(std::forward<TList>(t))...}};+}++namespace array_detail {+template <typename MakeItem, std::size_t... Index>+FOLLY_ERASE constexpr auto make_array_with_(+    MakeItem const& make, std::index_sequence<Index...>) {+  return std::array<decltype(make(0)), sizeof...(Index)>{{make(Index)...}};+}+} // namespace array_detail++/// Generates a std::array<..., Size> with elements m(i) for i in [0, Size).+///+/// @tparam Size  The size of the array+/// @param make  The generator that makes the array elements. ret[i] = make(i)+template <std::size_t Size, typename MakeItem>+constexpr auto make_array_with(MakeItem const& make) {+  return array_detail::make_array_with_(make, std::make_index_sequence<Size>{});+}++} // namespace folly
+ folly/folly/container/BitIterator.h view
@@ -0,0 +1,203 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++/**+ * BitIterator+ *    Wrapper around an iterator over an integral type that iterates+ *    over its underlying bits in MSb to LSb order+ *+ * findFirstSet(BitIterator begin, BitIterator end)+ *    return a BitIterator pointing to the first 1 bit in [begin, end), or+ *    end if all bits in [begin, end) are 0+ */++#pragma once++#include <cassert>+#include <cinttypes>+#include <cstdint>+#include <cstring>+#include <iterator>+#include <limits>+#include <type_traits>++#include <boost/iterator/iterator_adaptor.hpp>++#include <folly/Portability.h>+#include <folly/container/detail/BitIteratorDetail.h>+#include <folly/lang/Bits.h>++namespace folly {++/**+ * Fast bit iteration facility.+ */++template <class BaseIter>+class BitIterator;+template <class BaseIter>+BitIterator<BaseIter> findFirstSet(+    BitIterator<BaseIter>, BitIterator<BaseIter>);+/**+ * Wrapper around an iterator over an integer type that iterates+ * over its underlying bits in LSb to MSb order.+ *+ * BitIterator models the same iterator concepts as the base iterator.+ */+template <class BaseIter>+class BitIterator : public bititerator_detail::BitIteratorBase<BaseIter>::type {+ public:+  /**+   * Return the number of bits in an element of the underlying iterator.+   */+  static unsigned int bitsPerBlock() {+    return std::numeric_limits<typename std::make_unsigned<+        typename std::iterator_traits<BaseIter>::value_type>::type>::digits;+  }++  /**+   * Construct a BitIterator that points at a given bit offset (default 0)+   * in iter.+   */+  explicit BitIterator(const BaseIter& iter, size_t bitOff = 0)+      : bititerator_detail::BitIteratorBase<BaseIter>::type(iter),+        bitOffset_(bitOff) {+    assert(bitOffset_ < bitsPerBlock());+  }++  size_t bitOffset() const { return bitOffset_; }++  void advanceToNextBlock() {+    bitOffset_ = 0;+    ++this->base_reference();+  }++  BitIterator& operator=(const BaseIter& other) {+    this->~BitIterator();+    new (this) BitIterator(other);+    return *this;+  }++ private:+  friend class boost::iterator_core_access;+  friend BitIterator findFirstSet<>(BitIterator, BitIterator);++  typedef bititerator_detail::BitReference<+      typename std::iterator_traits<BaseIter>::reference,+      typename std::iterator_traits<BaseIter>::value_type>+      BitRef;++  void advanceInBlock(size_t n) {+    bitOffset_ += n;+    assert(bitOffset_ < bitsPerBlock());+  }++  BitRef dereference() const {+    return BitRef(*this->base_reference(), bitOffset_);+  }++  void advance(ssize_t n) {+    size_t bpb = bitsPerBlock();+    ssize_t blocks = n / ssize_t(bpb);+    bitOffset_ += n % bpb;+    if (bitOffset_ >= bpb) {+      bitOffset_ -= bpb;+      ++blocks;+    }+    this->base_reference() += blocks;+  }++  void increment() {+    if (++bitOffset_ == bitsPerBlock()) {+      advanceToNextBlock();+    }+  }++  void decrement() {+    if (bitOffset_-- == 0) {+      bitOffset_ = bitsPerBlock() - 1;+      --this->base_reference();+    }+  }++  bool equal(const BitIterator& other) const {+    return (+        bitOffset_ == other.bitOffset_ &&+        this->base_reference() == other.base_reference());+  }++  ssize_t distance_to(const BitIterator& other) const {+    return ssize_t(+        (other.base_reference() - this->base_reference()) * bitsPerBlock() ++        other.bitOffset_ - bitOffset_);+  }++  size_t bitOffset_;+};++/**+ * Helper function, so you can write+ * auto bi = makeBitIterator(container.begin());+ */+template <class BaseIter>+BitIterator<BaseIter> makeBitIterator(const BaseIter& iter) {+  return BitIterator<BaseIter>(iter);+}++/**+ * Find first bit set in a range of bit iterators.+ * 4.5x faster than the obvious std::find(begin, end, true);+ */+template <class BaseIter>+BitIterator<BaseIter> findFirstSet(+    BitIterator<BaseIter> begin, BitIterator<BaseIter> end) {+  // shortcut to avoid ugly static_cast<>+  static const typename std::iterator_traits<BaseIter>::value_type one = 1;++  while (begin.base() != end.base()) {+    typename std::iterator_traits<BaseIter>::value_type v = *begin.base();+    // mask out the bits that don't matter (< begin.bitOffset)+    v &= ~((one << begin.bitOffset()) - 1);+    size_t firstSet = findFirstSet(v);+    if (firstSet) {+      --firstSet; // now it's 0-based+      assert(firstSet >= begin.bitOffset());+      begin.advanceInBlock(firstSet - begin.bitOffset());+      return begin;+    }+    begin.advanceToNextBlock();+  }++  // now begin points to the same block as end+  if (end.bitOffset() != 0) { // assume end is dereferenceable+    typename std::iterator_traits<BaseIter>::value_type v = *begin.base();+    // mask out the bits that don't matter (< begin.bitOffset)+    v &= ~((one << begin.bitOffset()) - 1);+    // mask out the bits that don't matter (>= end.bitOffset)+    v &= (one << end.bitOffset()) - 1;+    size_t firstSet = findFirstSet(v);+    if (firstSet) {+      --firstSet; // now it's 0-based+      assert(firstSet >= begin.bitOffset());+      begin.advanceInBlock(firstSet - begin.bitOffset());+      return begin;+    }+  }++  return end;+}++} // namespace folly
+ folly/folly/container/Enumerate.h view
@@ -0,0 +1,162 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <iterator>+#include <memory>++#include <folly/CPortability.h>+#include <folly/portability/SysTypes.h>++/**+ * Similar to Python's enumerate(), folly::enumerate() can be used to+ * iterate a range with a for-range loop, and it also allows to+ * retrieve the count of iterations so far. Can be used in constexpr+ * context.+ *+ * For example:+ *+ * for (auto&& [index, element] : folly::enumerate(vec)) {+ *   // index is a const reference to a size_t containing the iteration count.+ *   // element is a reference to the type contained within vec, mutable+ *   // unless vec is const.+ * }+ *+ * If the binding is const, the element reference is too.+ *+ * for (const auto&& [index, element] : folly::enumerate(vec)) {+ *   // element is always a const reference.+ * }+ *+ * It can also be used as follows:+ *+ * for (auto&& it : folly::enumerate(vec)) {+ *   // *it is a reference to the current element. Mutable unless vec is const.+ *   // it->member can be used as well.+ *   // it.index contains the iteration count.+ * }+ *+ * As before, const auto&& it can also be used.+ */++namespace folly {++namespace detail {++template <class T>+struct MakeConst {+  using type = const T;+};+template <class T>+struct MakeConst<T&> {+  using type = const T&;+};+template <class T>+struct MakeConst<T*> {+  using type = const T*;+};++template <class Iterator>+class Enumerator {+ public:+  constexpr explicit Enumerator(Iterator it) : it_(std::move(it)) {}++  class Proxy {+   public:+    using difference_type = ssize_t;+    using value_type = typename std::iterator_traits<Iterator>::value_type;+    using reference = typename std::iterator_traits<Iterator>::reference;+    using pointer = typename std::iterator_traits<Iterator>::pointer;+    using iterator_category = std::input_iterator_tag;++    FOLLY_ALWAYS_INLINE constexpr explicit Proxy(const Enumerator& e)+        : index(e.idx_), element(*e.it_) {}++    // Non-const Proxy: Forward constness from Iterator.+    FOLLY_ALWAYS_INLINE constexpr reference operator*() { return element; }+    FOLLY_ALWAYS_INLINE constexpr pointer operator->() {+      return std::addressof(element);+    }++    // Const Proxy: Force const references.+    FOLLY_ALWAYS_INLINE constexpr typename MakeConst<reference>::type+    operator*() const {+      return element;+    }+    FOLLY_ALWAYS_INLINE constexpr typename MakeConst<pointer>::type operator->()+        const {+      return std::addressof(element);+    }++   public:+    const size_t index;+    reference element;+  };++  FOLLY_ALWAYS_INLINE constexpr Proxy operator*() const { return Proxy(*this); }++  FOLLY_ALWAYS_INLINE constexpr Enumerator& operator++() {+    ++it_;+    ++idx_;+    return *this;+  }++  template <typename OtherIterator>+  FOLLY_ALWAYS_INLINE constexpr bool operator==(+      const Enumerator<OtherIterator>& rhs) const {+    return it_ == rhs.it_;+  }++  template <typename OtherIterator>+  FOLLY_ALWAYS_INLINE constexpr bool operator!=(+      const Enumerator<OtherIterator>& rhs) const {+    return !(it_ == rhs.it_);+  }++ private:+  template <typename OtherIterator>+  friend class Enumerator;++  Iterator it_;+  size_t idx_ = 0;+};++template <class Range>+class RangeEnumerator {+  Range r_;+  using BeginIteratorType = decltype(std::declval<Range>().begin());+  using EndIteratorType = decltype(std::declval<Range>().end());++ public:+  constexpr explicit RangeEnumerator(Range&& r) : r_(std::forward<Range>(r)) {}++  constexpr Enumerator<BeginIteratorType> begin() {+    return Enumerator<BeginIteratorType>(r_.begin());+  }+  constexpr Enumerator<EndIteratorType> end() {+    return Enumerator<EndIteratorType>(r_.end());+  }+};++} // namespace detail++template <class Range>+constexpr detail::RangeEnumerator<Range> enumerate(Range&& r) {+  return detail::RangeEnumerator<Range>(std::forward<Range>(r));+}++} // namespace folly
+ folly/folly/container/EvictingCacheMap.h view
@@ -0,0 +1,749 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <algorithm>+#include <exception>+#include <functional>++#include <boost/intrusive/list.hpp>+#include <boost/iterator/iterator_adaptor.hpp>++#include <folly/container/F14Set.h>+#include <folly/container/HeterogeneousAccess.h>+#include <folly/lang/Exception.h>++namespace folly {++/**+ * A general purpose LRU evicting cache designed to support constant time+ * set/get/insert/erase ops. The only required configuration parameter is the+ * `maxSize`, which is the maximum number of entries held by the cache, which+ * is also dynamically changeable. Insertion will evict (and destroy with ~TKey+ * and ~TValue) existing entries in LRU order as needed to keep number of+ * entries less than maxSize. When automatic eviction is triggered, the+ * minimum number of evictions is `clearSize`, which is configurable with a+ * default of 1. If a callback is specified with setPruneHook, it is invoked+ * for each eviction. However, the prune hook cannot manage object lifetimes+ * because it is not invoked on erase nor cache destruction.+ *+ * This is NOT a thread-safe implementation.+ *+ * Iterators and references are only invalidated when the referenced entry+ * might have been removed (pruned or erased), like std::map.+ *+ * NOTE: maxSize==0 is a special case that disables automatic evictions.+ * prune() can be used for manually trimming down the number of entries.+ *+ * Implementaion: Maintains a doubly linked list (`lru_`) of entry nodes in+ * LRU order, which are also connected to hash table index (`index_`). The+ * access order is maintained on the list by moving an element to the front+ * of list on a get, and adding to the front on insert. Assuming quality+ * hashing, set/get are both constant time operations.+ *+ * NOTE: Previous versions of this structure used a hash table size that was+ * fixed at creation time, but that limitation is no longer present.+ */+template <+    class TKey,+    class TValue,+    class THash = HeterogeneousAccessHash<TKey>,+    class TKeyEqual = HeterogeneousAccessEqualTo<TKey>>+class EvictingCacheMap {+ private:+  // typedefs for brevity+  struct Node;+  struct NodeList;+  struct KeyHasher;+  struct KeyValueEqual;+  using NodeMap = F14VectorSet<Node*, KeyHasher, KeyValueEqual>;+  using TPair = std::pair<const TKey, TValue>;++ public:+  using PruneHookCall = std::function<void(TKey, TValue&&)>;++  // iterator base : returns TPair on dereference+  template <typename Value, typename TIterator>+  class iterator_base+      : public boost::iterator_adaptor<+            iterator_base<Value, TIterator>,+            TIterator,+            Value,+            boost::bidirectional_traversal_tag> {+   public:+    iterator_base() {}++    explicit iterator_base(TIterator it)+        : iterator_base::iterator_adaptor_(it) {}++    template <+        typename V,+        typename I,+        std::enable_if_t<+            std::is_same<V const, Value>::value &&+                std::is_convertible<I, TIterator>::value,+            int> = 0>+    /* implicit */ iterator_base(iterator_base<V, I> const& other)+        : iterator_base::iterator_adaptor_(other.base()) {}++    Value& dereference() const { return this->base_reference()->pr; }+  };++  // iterators+  using iterator = iterator_base<TPair, typename NodeList::iterator>;+  using const_iterator =+      iterator_base<const TPair, typename NodeList::const_iterator>;+  using reverse_iterator =+      iterator_base<TPair, typename NodeList::reverse_iterator>;+  using const_reverse_iterator =+      iterator_base<const TPair, typename NodeList::const_reverse_iterator>;++  // public type aliases for convenience+  using key_type = TKey;+  using mapped_type = TValue;+  using hasher = THash;++  /*+   * Approximate size of memory used by each entry added to the cache,+   * including the shallow bits (sizeof) of TKey and TValue, but not the deep+   * bits. Using 128 (bytes per chunk) / 10 (avg entries per chunk) as+   * approximate F14 index entry size.+   */+  static constexpr std::size_t kApproximateEntryMemUsage = 13 + sizeof(Node);++ private:+  template <typename K, typename T>+  using EnableHeterogeneousFind = std::enable_if_t<+      detail::EligibleForHeterogeneousFind<TKey, THash, TKeyEqual, K>::value,+      T>;++  template <typename K, typename T>+  using EnableHeterogeneousInsert = std::enable_if_t<+      detail::EligibleForHeterogeneousInsert<TKey, THash, TKeyEqual, K>::value,+      T>;++  template <typename K>+  using IsIter = Disjunction<+      std::is_same<iterator, remove_cvref_t<K>>,+      std::is_same<const_iterator, remove_cvref_t<K>>>;++  template <typename K, typename T>+  using EnableHeterogeneousErase = std::enable_if_t<+      detail::EligibleForHeterogeneousFind<+          TKey,+          THash,+          TKeyEqual,+          std::conditional_t<IsIter<K>::value, TKey, K>>::value &&+          !IsIter<K>::value,+      T>;++ public:+  /**+   * Construct a EvictingCacheMap+   * @param maxSize maximum size of the cache map.  Once the map size exceeds+   *     maxSize, the map will begin to evict.+   * @param clearSize the number of elements to clear at a time when automatic+   *     eviction on insert is triggered.+   */+  explicit EvictingCacheMap(+      std::size_t maxSize,+      std::size_t clearSize = 1,+      const THash& keyHash = THash(),+      const TKeyEqual& keyEqual = TKeyEqual())+      : keyHash_(keyHash),+        keyEqual_(keyEqual),+        index_(maxSize + /*transient*/ 1, keyHash_, keyEqual_),+        maxSize_(maxSize),+        clearSize_(clearSize) {}++  EvictingCacheMap(const EvictingCacheMap&) = delete;+  EvictingCacheMap& operator=(const EvictingCacheMap&) = delete;+  EvictingCacheMap(EvictingCacheMap&&) = default;+  EvictingCacheMap& operator=(EvictingCacheMap&&) = default;++  ~EvictingCacheMap() { assert(lru_.size() == index_.size()); }++  /**+   * Adjust the max size of EvictingCacheMap, evicting as needed to ensure the+   * new max is not exceeded.+   *+   * Calling this function with an arugment of 0 removes the limit on the cache+   * size and elements are not evicted unless clients explicitly call prune.+   *+   * @param maxSize new maximum size of the cache map.+   * @param pruneHook eviction callback to use INSTEAD OF the configured one+   */+  void setMaxSize(size_t maxSize, PruneHookCall pruneHook = nullptr) {+    if (maxSize != 0 && maxSize < size()) {+      // Prune the excess elements with our new constraints.+      prune(std::max(size() - maxSize, clearSize_), pruneHook);+    }+    maxSize_ = maxSize;+  }++  std::size_t getMaxSize() const { return maxSize_; }++  void setClearSize(std::size_t clearSize) { clearSize_ = clearSize; }++  /**+   * Check for existence of a specific key in the map.  This operation has+   *     no effect on LRU order.+   * @param key key to search for+   * @return true if exists, false otherwise+   */+  bool exists(const TKey& key) const { return existsImpl(key); }++  template <typename K, EnableHeterogeneousFind<K, int> = 0>+  bool exists(const K& key) const {+    return existsImpl(key);+  }++  /**+   * Get the value associated with a specific key.  This function always+   *     promotes a found value to the head of the LRU.+   * @param key key associated with the value+   * @return the value if it exists+   * @throw std::out_of_range exception of the key does not exist+   */+  TValue& get(const TKey& key) { return getImpl(key); }++  template <typename K, EnableHeterogeneousFind<K, int> = 0>+  TValue& get(const K& key) {+    return getImpl(key);+  }++  /**+   * Get the iterator associated with a specific key.  This function always+   *     promotes a found value to the head of the LRU.+   * @param key key to associate with value+   * @return the iterator of the object (a std::pair of const TKey, TValue) or+   *     end() if it does not exist+   */+  iterator find(const TKey& key) { return findImpl(*this, key); }++  template <typename K, EnableHeterogeneousFind<K, int> = 0>+  iterator find(const K& key) {+    return findImpl(*this, key);+  }++  /**+   * Get the value associated with a specific key.  This function never+   *     promotes a found value to the head of the LRU.+   * @param key key associated with the value+   * @return the value if it exists+   * @throw std::out_of_range exception of the key does not exist+   */+  const TValue& getWithoutPromotion(const TKey& key) const {+    return getWithoutPromotionImpl(*this, key);+  }++  template <typename K, EnableHeterogeneousFind<K, int> = 0>+  const TValue& getWithoutPromotion(const K& key) const {+    return getWithoutPromotionImpl(*this, key);+  }++  TValue& getWithoutPromotion(const TKey& key) {+    return getWithoutPromotionImpl(*this, key);+  }++  template <typename K, EnableHeterogeneousFind<K, int> = 0>+  TValue& getWithoutPromotion(const K& key) {+    return getWithoutPromotionImpl(*this, key);+  }++  /**+   * Get the iterator associated with a specific key.  This function never+   *     promotes a found value to the head of the LRU.+   * @param key key to associate with value+   * @return the iterator of the object (a std::pair of const TKey, TValue) or+   *     end() if it does not exist+   */+  const_iterator findWithoutPromotion(const TKey& key) const {+    return findWithoutPromotionImpl(*this, key);+  }++  template <typename K, EnableHeterogeneousFind<K, int> = 0>+  const_iterator findWithoutPromotion(const K& key) const {+    return findWithoutPromotionImpl(*this, key);+  }++  iterator findWithoutPromotion(const TKey& key) {+    return findWithoutPromotionImpl(*this, key);+  }++  template <typename K, EnableHeterogeneousFind<K, int> = 0>+  iterator findWithoutPromotion(const K& key) {+    return findWithoutPromotionImpl(*this, key);+  }++  /**+   * Erase the key-value pair associated with key if it exists. Prune hook+   * is not called unless one passed in here.+   * @param key key associated with the value+   * @param eraseHook callback to use with erased entry (similar to a prune+   * hook)+   * @return true if the key existed and was erased, else false+   */+  bool erase(const TKey& key, PruneHookCall eraseHook = nullptr) {+    return eraseKeyImpl(key, eraseHook);+  }++  template <typename K, EnableHeterogeneousErase<K, int> = 0>+  bool erase(const K& key, PruneHookCall eraseHook = nullptr) {+    return eraseKeyImpl(key, eraseHook);+  }++  /**+   * Erase the key-value pair associated with pos. Prune hook is not called+   * unless one passed in here.+   * @param pos iterator to the element to be erased+   * @param eraseHook callback to use with erased entry (similar to a prune+   * hook)+   * @return iterator to the following element or end() if pos was the last+   *     element+   */+  iterator erase(const_iterator pos, PruneHookCall eraseHook = nullptr) {+    return iterator(+        eraseImpl(const_cast<Node*>(&(*pos.base())), pos.base(), eraseHook));+  }++  /**+   * Set a key-value pair in the dictionary+   * @param key key to associate with value+   * @param value value to associate with the key+   * @param promote boolean flag indicating whether or not to move something+   *     to the front of an LRU.  This only really matters if you're setting+   *     a value that already exists.+   * @param pruneHook eviction callback to use INSTEAD OF the configured one+   */+  void set(+      const TKey& key,+      TValue&& value,+      bool promote = true,+      PruneHookCall pruneHook = nullptr) {+    setImpl(key, std::move(value), promote, pruneHook);+  }++  void set(+      const TKey& key,+      const TValue& value,+      bool promote = true,+      PruneHookCall pruneHook = nullptr) {+    TValue tmp{value}; // can't yet rely on temporary materialization+    setImpl(key, std::move(tmp), promote, pruneHook);+  }++  template <typename K, EnableHeterogeneousInsert<K, int> = 0>+  void set(+      const K& key,+      TValue&& value,+      bool promote = true,+      PruneHookCall pruneHook = nullptr) {+    setImpl(key, std::move(value), promote, pruneHook);+  }++  template <typename K, EnableHeterogeneousInsert<K, int> = 0>+  void set(+      const K& key,+      const TValue& value,+      bool promote = true,+      PruneHookCall pruneHook = nullptr) {+    TValue tmp{value}; // can't yet rely on temporary materialization+    setImpl(key, std::move(tmp), promote, pruneHook);+  }++  /**+   * Insert a new key-value pair in the dictionary if no element exists for key+   * @param key key to associate with value+   * @param value value to associate with the key+   * @param pruneHook eviction callback to use INSTEAD OF the configured one+   * @return a pair consisting of an iterator to the inserted element (or to the+   *     element that prevented the insertion) and a bool denoting whether the+   *     insertion took place.+   */+  std::pair<iterator, bool> insert(+      const TKey& key, TValue&& value, PruneHookCall pruneHook = nullptr) {+    return insertImpl(key, std::move(value), pruneHook);+  }++  std::pair<iterator, bool> insert(+      const TKey& key, const TValue& value, PruneHookCall pruneHook = nullptr) {+    TValue tmp{value}; // can't yet rely on temporary materialization+    return insertImpl(key, std::move(tmp), pruneHook);+  }++  template <typename K, EnableHeterogeneousInsert<K, int> = 0>+  std::pair<iterator, bool> insert(+      const K& key, TValue&& value, PruneHookCall pruneHook = nullptr) {+    return insertImpl(key, std::move(value), pruneHook);+  }++  template <typename K, EnableHeterogeneousInsert<K, int> = 0>+  std::pair<iterator, bool> insert(+      const K& key, const TValue& value, PruneHookCall pruneHook = nullptr) {+    TValue tmp{value}; // can't yet rely on temporary materialization+    return insertImpl(key, std::move(tmp), pruneHook);+  }++  /**+   * Emplace a new key-value pair in the dictionary if no element exists for+   * key, utilizing the configured prunehook+   * @param key key to associate with value+   * @param args args to construct TValue in place, to associate with the key+   * @return a pair consisting of an iterator to the inserted element (or to the+   *     element that prevented the insertion) and a bool denoting whether the+   *     insertion took place.+   */+  template <typename K, typename... Args>+  std::pair<iterator, bool> try_emplace(const K& key, Args&&... args) {+    return emplaceWithPruneHook<K, Args...>(+        key, std::forward<Args>(args)..., nullptr);+  }++  /**+   * Emplace a new key-value pair in the dictionary if no element exists for key+   * @param key key to associate with value+   * @param args args to construct TValue in place, to associate with the key+   * @param pruneHook eviction callback to use INSTEAD OF the configured one+   * @return a pair consisting of an iterator to the inserted element (or to the+   *     element that prevented the insertion) and a bool denoting whether the+   *     insertion took place.+   */+  template <typename K, typename... Args>+  std::pair<iterator, bool> emplaceWithPruneHook(+      const K& key, Args&&... args, PruneHookCall pruneHook) {+    return insertImpl<K>(+        std::make_unique<Node>(+            std::piecewise_construct, key, std::forward<Args>(args)...),+        pruneHook);+  }++  /**+   * Get the number of elements in the dictionary+   * @return the size of the dictionary+   */+  std::size_t size() const {+    assert(index_.size() == lru_.size());+    return index_.size();+  }++  /**+   * Typical empty function+   * @return true if empty, false otherwise+   */+  bool empty() const { return index_.empty(); }++  /**+   * Remove all entries (as if all evicted)+   * @param pruneHook eviction callback to use INSTEAD OF the configured one+   */+  void clear(PruneHookCall pruneHook = nullptr) { prune(size(), pruneHook); }++  /**+   * Set the prune hook, which is the function invoked on the key and value+   *     on each eviction. An operation will throw if the pruneHook throws.+   *     Note that this prune hook is not automatically called on entries+   *     explicitly erase()ed nor on remaining entries at destruction time.+   * @param pruneHook eviction callback to set as default, or nullptr to clear+   */+  void setPruneHook(PruneHookCall pruneHook) { pruneHook_ = pruneHook; }++  PruneHookCall getPruneHook() { return pruneHook_; }++  /**+   * Prune the minimum of pruneSize and size() from the back of the LRU.+   * Will throw if pruneHook throws.+   * @param pruneSize minimum number of elements to prune+   * @param pruneHook eviction callback to use INSTEAD OF the configured one+   */+  void prune(std::size_t pruneSize, PruneHookCall pruneHook = nullptr) {+    auto& ph = (nullptr == pruneHook) ? pruneHook_ : pruneHook;++    for (std::size_t i = 0; i < pruneSize && !lru_.empty(); i++) {+      auto* node = &(*lru_.rbegin());+      std::unique_ptr<Node> node_owner(node);++      lru_.erase(lru_.iterator_to(*node));+      index_.erase(node);+      if (ph) {+        // NOTE: might throw, so we are in an exception-safe state+        ph(node->pr.first, std::move(node->pr.second));+      }+    }+  }++  // Iterators and such+  iterator begin() { return iterator(lru_.begin()); }+  iterator end() { return iterator(lru_.end()); }+  const_iterator begin() const { return const_iterator(lru_.begin()); }+  const_iterator end() const { return const_iterator(lru_.end()); }++  const_iterator cbegin() const { return const_iterator(lru_.cbegin()); }+  const_iterator cend() const { return const_iterator(lru_.cend()); }++  reverse_iterator rbegin() { return reverse_iterator(lru_.rbegin()); }+  reverse_iterator rend() { return reverse_iterator(lru_.rend()); }++  const_reverse_iterator rbegin() const {+    return const_reverse_iterator(lru_.rbegin());+  }+  const_reverse_iterator rend() const {+    return const_reverse_iterator(lru_.rend());+  }++  const_reverse_iterator crbegin() const {+    return const_reverse_iterator(lru_.crbegin());+  }+  const_reverse_iterator crend() const {+    return const_reverse_iterator(lru_.crend());+  }++ private:+  struct Node+      : public boost::intrusive::list_base_hook<+            boost::intrusive::link_mode<boost::intrusive::safe_link>> {+    template <typename K>+    Node(const K& key, TValue&& value) : pr(key, std::move(value)) {}++    template <typename Key, typename... Args>+    explicit Node(std::piecewise_construct_t, Key&& k, Args&&... args)+        : pr(std::piecewise_construct,+             std::forward_as_tuple(std::forward<Key>(k)),+             std::forward_as_tuple(std::forward<Args>(args)...)) {}+    TPair pr;+  };+  using NodePtr = Node*;++  // NOTE: deriving from boost::intrusive::list is likely discouraged. This is+  // simply an alternative to an ugly explicit move operator for+  // EvictingCacheMap. Change to that if this derivation proves problematic.+  struct NodeList : public boost::intrusive::list<Node> {+    NodeList() {}+    NodeList& operator=(NodeList&& that) noexcept {+      // Clear the moved-from rather than swap, for consistency with NodeMap+      clear_nodes();+      // Now invoke base class move operator without using static_cast+      boost::intrusive::list<Node>& this_parent = *this;+      boost::intrusive::list<Node>&& that_parent = std::move(that);+      this_parent = std::move(that_parent);+      return *this;+    }+    NodeList(NodeList&& that) noexcept { *this = std::move(that); }+    ~NodeList() {+      // Adds leak-free final destruction to the intrusive container+      clear_nodes();+    }++   private:+    void clear_nodes() {+      boost::intrusive::list<Node>::clear_and_dispose([](Node* ptr) {+        delete ptr;+      });+    }+  };++  struct KeyHasher : THash {+    static_assert(std::is_nothrow_copy_constructible_v<THash>);+    template <typename K>+    static inline constexpr bool nx =+        is_nothrow_invocable_v<THash const&, K const&>;++    using is_transparent = void;+    using folly_is_avalanching = IsAvalanchingHasher<THash, TKey>;++    using THash::THash;++    explicit KeyHasher(THash const& that) noexcept : THash(that) {}++    template <typename K>+    std::size_t operator()(const K& key) const noexcept(nx<K>) {+      return THash::operator()(key);+    }+    std::size_t operator()(const NodePtr& node) const noexcept(nx<TKey>) {+      return THash::operator()(node->pr.first);+    }+  };++  struct KeyValueEqual : private TKeyEqual {+    static_assert(std::is_nothrow_copy_constructible_v<TKeyEqual>);+    template <typename L, typename R>+    static inline constexpr bool nx =+        is_nothrow_invocable_v<TKeyEqual const&, L const&, R const&>;++    using is_transparent = void;++    using TKeyEqual::TKeyEqual;++    explicit KeyValueEqual(TKeyEqual const& that) noexcept : TKeyEqual(that) {}++    template <typename K>+    bool operator()(const K& lhs, const NodePtr& rhs) const+        noexcept(nx<K, TKey>) {+      return TKeyEqual::operator()(lhs, rhs->pr.first);+    }+    template <typename K>+    bool operator()(const NodePtr& lhs, const K& rhs) const+        noexcept(nx<TKey, K>) {+      return TKeyEqual::operator()(lhs->pr.first, rhs);+    }+    bool operator()(const NodePtr& lhs, const NodePtr& rhs) const+        noexcept(nx<TKey, TKey>) {+      return TKeyEqual::operator()(lhs->pr.first, rhs->pr.first);+    }+  };++  template <typename K>+  bool existsImpl(const K& key) const {+    return findInIndex(key) != nullptr;+  }++  template <typename K>+  TValue& getImpl(const K& key) {+    auto it = findImpl(*this, key);+    if (it == end()) {+      throw_exception<std::out_of_range>("Key does not exist");+    }+    return it->second;+  }++  template <typename Self>+  using self_iterator_t =+      std::conditional_t<std::is_const<Self>::value, const_iterator, iterator>;++  template <typename Self, typename K>+  static auto findImpl(Self& self, const K& key) {+    Node* ptr = self.findInIndex(key);+    if (!ptr) {+      return self.end();+    }+    self.lru_.splice(self.lru_.begin(), self.lru_, self.lru_.iterator_to(*ptr));+    return self_iterator_t<Self>(self.lru_.iterator_to(*ptr));+  }++  template <typename Self, typename K>+  static auto& getWithoutPromotionImpl(Self& self, const K& key) {+    auto it = self.findWithoutPromotion(key);+    if (it == self.end()) {+      throw_exception<std::out_of_range>("Key does not exist");+    }+    return it->second;+  }++  template <typename Self, typename K>+  static auto findWithoutPromotionImpl(Self& self, const K& key) {+    Node* ptr = self.findInIndex(key);+    return ptr+        ? self_iterator_t<Self>(self.lru_.iterator_to(*ptr))+        : self.end();+  }++  typename NodeList::iterator eraseImpl(+      Node* ptr,+      typename NodeList::const_iterator base_iter,+      PruneHookCall eraseHook) {+    std::unique_ptr<Node> node_owner(ptr);+    index_.erase(ptr);+    auto next_base_iter = lru_.erase(base_iter);+    if (eraseHook) {+      // NOTE: might throw, so we are in an exception-safe state+      eraseHook(ptr->pr.first, std::move(ptr->pr.second));+    }+    return next_base_iter;+  }++  template <typename K>+  bool eraseKeyImpl(const K& key, PruneHookCall eraseHook) {+    Node* ptr = findInIndex(key);+    if (ptr) {+      eraseImpl(ptr, lru_.iterator_to(*ptr), eraseHook);+      return true;+    }+    return false;+  }++  template <typename K>+  void setImpl(+      const K& key, TValue&& value, bool promote, PruneHookCall pruneHook) {+    Node* ptr = findInIndex(key);+    if (ptr) {+      ptr->pr.second = std::move(value);+      if (promote) {+        lru_.splice(lru_.begin(), lru_, lru_.iterator_to(*ptr));+      }+    } else {+      auto node = new Node(key, std::move(value));+      index_.insert(node);+      lru_.push_front(*node);++      // no evictions if maxSize_ is 0 i.e. unlimited capacity+      if (maxSize_ > 0 && size() > maxSize_) {+        prune(clearSize_, pruneHook);+      }+    }+  }++  template <typename K>+  auto insertImpl(const K& key, TValue&& value, PruneHookCall pruneHook) {+    auto node_owner = std::make_unique<Node>(key, std::move(value));+    return insertImpl<K>(std::move(node_owner), std::move(pruneHook));+  }++  template <typename K>+  auto insertImpl(std::unique_ptr<Node> nodeOwner, PruneHookCall pruneHook) {+    Node* node = nodeOwner.get();+    {+      auto pair = index_.insert(node);+      if (!pair.second) {+        // No change. Abandon/destroy new node.+        return std::pair<iterator, bool>(lru_.iterator_to(**pair.first), false);+      }++      // upcoming prune might invalidate iterator+      assert(*pair.first == node);+    }++    // Complete insertion+    lru_.push_front(*nodeOwner.release());++    // no evictions if maxSize_ is 0 i.e. unlimited capacity+    if (maxSize_ > 0 && size() > maxSize_) {+      prune(clearSize_, pruneHook);+    }++    return std::pair<iterator, bool>(lru_.iterator_to(*node), true);+  }++  template <typename K>+  Node* findInIndex(const K& key) const {+    auto it = index_.find(key);+    if (it != index_.end()) {+      return *it;+    } else {+      return nullptr;+    }+  }++  PruneHookCall pruneHook_;+  KeyHasher keyHash_;+  KeyValueEqual keyEqual_;+  NodeMap index_;+  NodeList lru_;+  std::size_t maxSize_;+  std::size_t clearSize_;+};++} // namespace folly
+ folly/folly/container/F14Map-fwd.h view
@@ -0,0 +1,109 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <utility>++#include <folly/container/detail/F14Defaults.h>+#include <folly/memory/MemoryResource.h>++namespace folly {+template <+    typename Key,+    typename Mapped,+    typename Hasher = f14::DefaultHasher<Key>,+    typename KeyEqual = f14::DefaultKeyEqual<Key>,+    typename Alloc = f14::DefaultAlloc<std::pair<Key const, Mapped>>>+class F14NodeMap;++template <+    typename Key,+    typename Mapped,+    typename Hasher = f14::DefaultHasher<Key>,+    typename KeyEqual = f14::DefaultKeyEqual<Key>,+    typename Alloc = f14::DefaultAlloc<std::pair<Key const, Mapped>>>+class F14ValueMap;++template <+    typename Key,+    typename Mapped,+    typename Hasher = f14::DefaultHasher<Key>,+    typename KeyEqual = f14::DefaultKeyEqual<Key>,+    typename Alloc = f14::DefaultAlloc<std::pair<Key const, Mapped>>>+class F14VectorMap;++template <+    typename Key,+    typename Mapped,+    typename Hasher = f14::DefaultHasher<Key>,+    typename KeyEqual = f14::DefaultKeyEqual<Key>,+    typename Alloc = f14::DefaultAlloc<std::pair<Key const, Mapped>>>+class F14FastMap;++#if FOLLY_HAS_MEMORY_RESOURCE+namespace pmr {+template <+    typename Key,+    typename Mapped,+    typename Hasher = f14::DefaultHasher<Key>,+    typename KeyEqual = f14::DefaultKeyEqual<Key>>+using F14NodeMap = folly::F14NodeMap<+    Key,+    Mapped,+    Hasher,+    KeyEqual,+    std::pmr::polymorphic_allocator<std::pair<Key const, Mapped>>>;++template <+    typename Key,+    typename Mapped,+    typename Hasher = f14::DefaultHasher<Key>,+    typename KeyEqual = f14::DefaultKeyEqual<Key>>+using F14ValueMap = folly::F14ValueMap<+    Key,+    Mapped,+    Hasher,+    KeyEqual,+    std::pmr::polymorphic_allocator<std::pair<Key const, Mapped>>>;++template <+    typename Key,+    typename Mapped,+    typename Hasher = f14::DefaultHasher<Key>,+    typename KeyEqual = f14::DefaultKeyEqual<Key>>+using F14VectorMap = folly::F14VectorMap<+    Key,+    Mapped,+    Hasher,+    KeyEqual,+    std::pmr::polymorphic_allocator<std::pair<Key const, Mapped>>>;++template <+    typename Key,+    typename Mapped,+    typename Hasher = f14::DefaultHasher<Key>,+    typename KeyEqual = f14::DefaultKeyEqual<Key>>+using F14FastMap = folly::F14FastMap<+    Key,+    Mapped,+    Hasher,+    KeyEqual,+    std::pmr::polymorphic_allocator<std::pair<Key const, Mapped>>>;+} // namespace pmr+#endif // FOLLY_HAS_MEMORY_RESOURCE++} // namespace folly
+ folly/folly/container/F14Map.h view
@@ -0,0 +1,2066 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++/**+ * F14NodeMap, F14ValueMap, and F14VectorMap+ *+ * F14FastMap conditionally works like F14ValueMap or F14VectorMap+ *+ * See F14.md+ */++#include <cstddef>+#include <initializer_list>+#include <stdexcept>+#include <tuple>++#include <folly/Portability.h>+#include <folly/Range.h>+#include <folly/Traits.h>+#include <folly/container/View.h>+#include <folly/lang/Exception.h>+#include <folly/lang/SafeAssert.h>++#include <folly/container/F14Map-fwd.h>+#include <folly/container/Iterator.h>+#include <folly/container/detail/F14Policy.h>+#include <folly/container/detail/F14Table.h>+#include <folly/container/detail/Util.h>++// If !FOLLY_F14_VECTOR_INTRINSICS_AVAILABLE, fallback definitions are exported+// in this file+#include <folly/container/detail/F14MapFallback.h> // IWYU pragma: export++namespace folly {++#if FOLLY_F14_VECTOR_INTRINSICS_AVAILABLE++//////// Common case for supported platforms++namespace f14 {+namespace detail {++template <typename Policy>+class F14BasicMap {+  template <typename K, typename T>+  using EnableHeterogeneousFind = std::enable_if_t<+      ::folly::detail::EligibleForHeterogeneousFind<+          typename Policy::Key,+          typename Policy::Hasher,+          typename Policy::KeyEqual,+          K>::value,+      T>;++  template <typename K, typename T>+  using EnableHeterogeneousInsert = std::enable_if_t<+      ::folly::detail::EligibleForHeterogeneousInsert<+          typename Policy::Key,+          typename Policy::Hasher,+          typename Policy::KeyEqual,+          K>::value,+      T>;++  template <typename K>+  using IsIter = Disjunction<+      std::is_same<typename Policy::Iter, remove_cvref_t<K>>,+      std::is_same<typename Policy::ConstIter, remove_cvref_t<K>>>;++  template <typename K, typename T>+  using EnableHeterogeneousErase = std::enable_if_t<+      ::folly::detail::EligibleForHeterogeneousFind<+          typename Policy::Key,+          typename Policy::Hasher,+          typename Policy::KeyEqual,+          std::conditional_t<IsIter<K>::value, typename Policy::Key, K>>::+              value &&+          !IsIter<K>::value,+      T>;++ public:+  //// PUBLIC - Member types++  using key_type = typename Policy::Key;+  using mapped_type = typename Policy::Mapped;+  using value_type = typename Policy::Value;+  using size_type = std::size_t;+  using difference_type = std::ptrdiff_t;+  using hash_token_type = F14HashToken;+  using hasher = typename Policy::Hasher;+  using key_equal = typename Policy::KeyEqual;+  using hashed_key_type = F14HashedKey<key_type, hasher, key_equal>;+  using allocator_type = typename Policy::Alloc;+  using reference = value_type&;+  using const_reference = value_type const&;+  using pointer = typename Policy::AllocTraits::pointer;+  using const_pointer = typename Policy::AllocTraits::const_pointer;+  using iterator = typename Policy::Iter;+  using const_iterator = typename Policy::ConstIter;++ private:+  using ItemIter = typename Policy::ItemIter;++ public:+  //// PUBLIC - Member functions++  F14BasicMap() noexcept(Policy::kDefaultConstructIsNoexcept) : table_{} {}++  explicit F14BasicMap(+      std::size_t initialCapacity,+      hasher const& hash = hasher{},+      key_equal const& eq = key_equal{},+      allocator_type const& alloc = allocator_type{})+      : table_{initialCapacity, hash, eq, alloc} {}++  explicit F14BasicMap(std::size_t initialCapacity, allocator_type const& alloc)+      : F14BasicMap(initialCapacity, hasher{}, key_equal{}, alloc) {}++  explicit F14BasicMap(+      std::size_t initialCapacity,+      hasher const& hash,+      allocator_type const& alloc)+      : F14BasicMap(initialCapacity, hash, key_equal{}, alloc) {}++  explicit F14BasicMap(allocator_type const& alloc)+      : F14BasicMap(0, hasher{}, key_equal{}, alloc) {}++  template <typename InputIt>+  F14BasicMap(+      InputIt first,+      InputIt last,+      std::size_t initialCapacity = 0,+      hasher const& hash = hasher{},+      key_equal const& eq = key_equal{},+      allocator_type const& alloc = allocator_type{})+      : table_{initialCapacity, hash, eq, alloc} {+    initialInsert(std::move(first), std::move(last), initialCapacity);+  }++  template <typename InputIt>+  F14BasicMap(+      InputIt first,+      InputIt last,+      std::size_t initialCapacity,+      allocator_type const& alloc)+      : table_{initialCapacity, hasher{}, key_equal{}, alloc} {+    initialInsert(std::move(first), std::move(last), initialCapacity);+  }++  template <typename InputIt>+  F14BasicMap(+      InputIt first,+      InputIt last,+      std::size_t initialCapacity,+      hasher const& hash,+      allocator_type const& alloc)+      : table_{initialCapacity, hash, key_equal{}, alloc} {+    initialInsert(std::move(first), std::move(last), initialCapacity);+  }++  F14BasicMap(F14BasicMap const& rhs) = default;++  F14BasicMap(F14BasicMap const& rhs, allocator_type const& alloc)+      : table_{rhs.table_, alloc} {}++  F14BasicMap(F14BasicMap&& rhs) = default;++  F14BasicMap(F14BasicMap&& rhs, allocator_type const& alloc) noexcept(+      Policy::kAllocIsAlwaysEqual)+      : table_{std::move(rhs.table_), alloc} {}++  /* implicit */ F14BasicMap(+      std::initializer_list<value_type> init,+      std::size_t initialCapacity = 0,+      hasher const& hash = hasher{},+      key_equal const& eq = key_equal{},+      allocator_type const& alloc = allocator_type{})+      : table_{initialCapacity, hash, eq, alloc} {+    initialInsert(init.begin(), init.end(), initialCapacity);+  }++  F14BasicMap(+      std::initializer_list<value_type> init,+      std::size_t initialCapacity,+      allocator_type const& alloc)+      : table_{initialCapacity, hasher{}, key_equal{}, alloc} {+    initialInsert(init.begin(), init.end(), initialCapacity);+  }++  F14BasicMap(+      std::initializer_list<value_type> init,+      std::size_t initialCapacity,+      hasher const& hash,+      allocator_type const& alloc)+      : table_{initialCapacity, hash, key_equal{}, alloc} {+    initialInsert(init.begin(), init.end(), initialCapacity);+  }++  F14BasicMap& operator=(F14BasicMap const&) = default;++  F14BasicMap& operator=(F14BasicMap&&) = default;++  F14BasicMap& operator=(std::initializer_list<value_type> ilist) {+    clear();+    bulkInsert(ilist.begin(), ilist.end(), true);+    return *this;+  }++  /// Get the allocator for this container.+  /// @methodset Allocator+  allocator_type get_allocator() const noexcept { return table_.alloc(); }++  //// PUBLIC - Iterators++  /// Get an iterator to the beginning+  /// @methodset Iterators+  iterator begin() noexcept { return table_.makeIter(table_.begin()); }+  const_iterator begin() const noexcept { return cbegin(); }+  /// Get an iterator to the beginning+  /// @methodset Iterators+  const_iterator cbegin() const noexcept {+    return table_.makeConstIter(table_.begin());+  }++  /// Get an iterator to the end+  /// @methodset Iterators+  iterator end() noexcept { return table_.makeIter(table_.end()); }+  const_iterator end() const noexcept { return cend(); }+  /// Get an iterator to the end+  /// @methodset Iterators+  const_iterator cend() const noexcept {+    return table_.makeConstIter(table_.end());+  }++  //// PUBLIC - Capacity++  /// Check if this container has any elements.+  /// @methodset Capacity+  bool empty() const noexcept { return table_.empty(); }++  /// Number of elements in this container.+  /// @methodset Capacity+  std::size_t size() const noexcept { return table_.size(); }++  /// The maximum size of this container.+  /// @methodset Capacity+  std::size_t max_size() const noexcept { return table_.max_size(); }++  //// PUBLIC - Modifiers++  /**+   * Remove all elements.+   * @methodset Modifiers+   *+   * Frees heap-allocated memory; bucket_count is returned to 0.+   */+  void clear() noexcept { table_.clear(); }++  /**+   * Add a single element.+   * @overloadbrief Add elements.+   * @methodset Modifiers+   */+  std::pair<iterator, bool> insert(value_type const& value) {+    return emplace(value);+  }++  /// Add a single element, of a heterogeneous type.+  template <typename P>+  std::enable_if_t<+      std::is_constructible<value_type, P&&>::value,+      std::pair<iterator, bool>>+  insert(P&& value) {+    return emplace(std::forward<P>(value));+  }++  /**+   * Add a single element, of a heterogeneous type.+   *+   * TODO(T31574848): Work around libstdc++ versions (e.g., GCC < 6) with no+   * implementation of N4387 ("perfect initialization" for pairs and tuples).+   */+  template <typename U1, typename U2>+  std::enable_if_t<+      std::is_constructible<key_type, U1 const&>::value &&+          std::is_constructible<mapped_type, U2 const&>::value,+      std::pair<iterator, bool>>+  insert(std::pair<U1, U2> const& value) {+    return emplace(value);+  }++  /**+   * Add a single element, of a heterogeneous type.+   *+   * TODO(T31574848): Work around libstdc++ versions (e.g., GCC < 6) with no+   * implementation of N4387 ("perfect initialization" for pairs and tuples).+   */+  template <typename U1, typename U2>+  std::enable_if_t<+      std::is_constructible<key_type, U1&&>::value &&+          std::is_constructible<mapped_type, U2&&>::value,+      std::pair<iterator, bool>>+  insert(std::pair<U1, U2>&& value) {+    return emplace(std::move(value));+  }++  /// Add a single element.+  std::pair<iterator, bool> insert(value_type&& value) {+    return emplace(std::move(value));+  }++  // std::unordered_map's hinted insertion API is misleading.  No+  // implementation I've seen actually uses the hint.  Code restructuring+  // by the caller to use the hinted API is at best unnecessary, and at+  // worst a pessimization.  It is used, however, so we provide it.++  /// Add a single element, with a locational hint.+  iterator insert(const_iterator /*hint*/, value_type const& value) {+    return insert(value).first;+  }++  /// Add a single element, of heterogeneous type, with a locational hint.+  template <typename P>+  std::enable_if_t<std::is_constructible<value_type, P&&>::value, iterator>+  insert(const_iterator /*hint*/, P&& value) {+    return insert(std::forward<P>(value)).first;+  }++  /// Add a single element, with a locational hint.+  iterator insert(const_iterator /*hint*/, value_type&& value) {+    return insert(std::move(value)).first;+  }++  /// Emplace with hint.+  /// @methodset Modifiers+  template <class... Args>+  iterator emplace_hint(const_iterator /*hint*/, Args&&... args) {+    return emplace(std::forward<Args>(args)...).first;+  }++ private:+  template <class InputIt>+  FOLLY_ALWAYS_INLINE void bulkInsert(+      InputIt first, InputIt last, bool autoReserve) {+    if (autoReserve) {+      auto n = std::distance(first, last);+      if (n == 0) {+        return;+      }+      table_.reserveForInsert(n);+    }+    while (first != last) {+      insert(*first);+      ++first;+    }+  }++  template <class InputIt>+  void initialInsert(InputIt first, InputIt last, std::size_t initialCapacity) {+    FOLLY_SAFE_DCHECK(empty() && bucket_count() >= initialCapacity, "");++    // It's possible that there are a lot of duplicates in first..last and+    // so we will oversize ourself.  The common case, however, is that+    // we can avoid a lot of rehashing if we pre-expand.  The behavior+    // is easy to disable at a particular call site by asking for an+    // initialCapacity of 1.+    bool autoReserve =+        std::is_base_of<+            std::random_access_iterator_tag,+            typename std::iterator_traits<InputIt>::iterator_category>::value &&+        initialCapacity == 0;+    bulkInsert(std::move(first), std::move(last), autoReserve);+  }++ public:+  /// Add elements+  template <class InputIt>+  void insert(InputIt first, InputIt last) {+    // Bulk reserve is a heuristic choice, so it can backfire.  We restrict+    // ourself to situations that mimic bulk construction without an+    // explicit initialCapacity.+    bool autoReserve =+        std::is_base_of<+            std::random_access_iterator_tag,+            typename std::iterator_traits<InputIt>::iterator_category>::value &&+        bucket_count() == 0;+    bulkInsert(std::move(first), std::move(last), autoReserve);+  }++  /// Add elements from an initializer list+  void insert(std::initializer_list<value_type> ilist) {+    insert(ilist.begin(), ilist.end());+  }++  /// Insert if the key is missing, overwrite using operator= if present.+  /// @methodset Modifiers+  template <typename M>+  std::pair<iterator, bool> insert_or_assign(key_type const& key, M&& obj) {+    auto rv = try_emplace(key, std::forward<M>(obj));+    if (!rv.second) {+      rv.first->second = std::forward<M>(obj);+    }+    return rv;+  }++  template <typename M>+  std::pair<iterator, bool> insert_or_assign(key_type&& key, M&& obj) {+    auto rv = try_emplace(std::move(key), std::forward<M>(obj));+    if (!rv.second) {+      rv.first->second = std::forward<M>(obj);+    }+    return rv;+  }++  template <typename M>+  std::pair<iterator, bool> insert_or_assign(+      F14HashToken const& token, key_type const& key, M&& obj) {+    auto rv = try_emplace_token(token, key, std::forward<M>(obj));+    if (!rv.second) {+      rv.first->second = std::forward<M>(obj);+    }+    return rv;+  }++  template <typename M>+  std::pair<iterator, bool> insert_or_assign(+      F14HashToken const& token, key_type&& key, M&& obj) {+    auto rv = try_emplace_token(token, std::move(key), std::forward<M>(obj));+    if (!rv.second) {+      rv.first->second = std::forward<M>(obj);+    }+    return rv;+  }++  template <typename M>+  iterator insert_or_assign(+      const_iterator /*hint*/, key_type const& key, M&& obj) {+    return insert_or_assign(key, std::forward<M>(obj)).first;+  }++  template <typename M>+  iterator insert_or_assign(const_iterator /*hint*/, key_type&& key, M&& obj) {+    return insert_or_assign(std::move(key), std::forward<M>(obj)).first;+  }++  template <typename K, typename M>+  EnableHeterogeneousInsert<K, std::pair<iterator, bool>> insert_or_assign(+      K&& key, M&& obj) {+    auto rv = try_emplace(std::forward<K>(key), std::forward<M>(obj));+    if (!rv.second) {+      rv.first->second = std::forward<M>(obj);+    }+    return rv;+  }++  template <typename K, typename M>+  EnableHeterogeneousInsert<K, std::pair<iterator, bool>> insert_or_assign(+      F14HashToken const& token, K&& key, M&& obj) {+    auto rv =+        try_emplace_token(token, std::forward<K>(key), std::forward<M>(obj));+    if (!rv.second) {+      rv.first->second = std::forward<M>(obj);+    }+    return rv;+  }++ private:+  template <typename Arg>+  using UsableAsKey = ::folly::detail::+      EligibleForHeterogeneousFind<key_type, hasher, key_equal, Arg>;++ public:+  /**+   * Add an element by constructing it in-place.+   * @overloadbrief Add elements in-place.+   * @methodset Modifiers+   */+  template <typename... Args>+  std::pair<iterator, bool> emplace(Args&&... args) {+    auto rv =+        folly::detail::callWithExtractedKey<key_type, mapped_type, UsableAsKey>(+            table_.alloc(),+            [&](auto&&... inner) {+              return table_.tryEmplaceValue(+                  std::forward<decltype(inner)>(inner)...);+            },+            std::forward<Args>(args)...);+    return std::make_pair(table_.makeIter(rv.first), rv.second);+  }++  /**+   * Add an element by constructing it in-place.+   * @methodset Modifiers+   *+   * Does nothing if the key already exists.+   */+  template <typename... Args>+  std::pair<iterator, bool> try_emplace(key_type const& key, Args&&... args) {+    auto rv = table_.tryEmplaceValue(+        key,+        std::piecewise_construct,+        std::forward_as_tuple(key),+        std::forward_as_tuple(std::forward<Args>(args)...));+    return std::make_pair(table_.makeIter(rv.first), rv.second);+  }++  template <typename... Args>+  std::pair<iterator, bool> try_emplace(key_type&& key, Args&&... args) {+    auto rv = table_.tryEmplaceValue(+        key,+        std::piecewise_construct,+        std::forward_as_tuple(std::move(key)),+        std::forward_as_tuple(std::forward<Args>(args)...));+    return std::make_pair(table_.makeIter(rv.first), rv.second);+  }++  /// @copydoc try_emplace+  template <typename... Args>+  std::pair<iterator, bool> try_emplace_token(+      F14HashToken const& token, key_type const& key, Args&&... args) {+    auto rv = table_.tryEmplaceValueWithToken(+        token,+        key,+        std::piecewise_construct,+        std::forward_as_tuple(key),+        std::forward_as_tuple(std::forward<Args>(args)...));+    return std::make_pair(table_.makeIter(rv.first), rv.second);+  }++  template <typename... Args>+  std::pair<iterator, bool> try_emplace_token(+      F14HashToken const& token, key_type&& key, Args&&... args) {+    auto rv = table_.tryEmplaceValueWithToken(+        token,+        key,+        std::piecewise_construct,+        std::forward_as_tuple(std::move(key)),+        std::forward_as_tuple(std::forward<Args>(args)...));+    return std::make_pair(table_.makeIter(rv.first), rv.second);+  }++  template <typename... Args>+  iterator try_emplace(+      const_iterator /*hint*/, key_type const& key, Args&&... args) {+    auto rv = table_.tryEmplaceValue(+        key,+        std::piecewise_construct,+        std::forward_as_tuple(key),+        std::forward_as_tuple(std::forward<Args>(args)...));+    return table_.makeIter(rv.first);+  }++  template <typename... Args>+  iterator try_emplace(+      const_iterator /*hint*/, key_type&& key, Args&&... args) {+    auto rv = table_.tryEmplaceValue(+        key,+        std::piecewise_construct,+        std::forward_as_tuple(std::move(key)),+        std::forward_as_tuple(std::forward<Args>(args)...));+    return table_.makeIter(rv.first);+  }++  template <typename K, typename... Args>+  EnableHeterogeneousInsert<K, std::pair<iterator, bool>> try_emplace(+      K&& key, Args&&... args) {+    auto rv = table_.tryEmplaceValue(+        key,+        std::piecewise_construct,+        std::forward_as_tuple(std::forward<K>(key)),+        std::forward_as_tuple(std::forward<Args>(args)...));+    return std::make_pair(table_.makeIter(rv.first), rv.second);+  }++  template <typename K, typename... Args>+  EnableHeterogeneousInsert<K, std::pair<iterator, bool>> try_emplace_token(+      F14HashToken const& token, K&& key, Args&&... args) {+    auto rv = table_.tryEmplaceValueWithToken(+        token,+        key,+        std::piecewise_construct,+        std::forward_as_tuple(std::forward<K>(key)),+        std::forward_as_tuple(std::forward<Args>(args)...));+    return std::make_pair(table_.makeIter(rv.first), rv.second);+  }++  /**+   * Remove element at a specific position (iterator).+   * @overloadbrief Remove elements.+   * @methodset Modifiers+   */+  FOLLY_ALWAYS_INLINE iterator erase(const_iterator pos) {+    return eraseInto(pos, variadic_noop);+  }++  /**+   * Remove element at a specific position (iterator).+   *+   * This form avoids ambiguity when key_type has a templated constructor+   * that accepts const_iterator+   */+  FOLLY_ALWAYS_INLINE iterator erase(iterator pos) {+    return eraseInto(pos, variadic_noop);+  }++  /// Remove a range of elements.+  iterator erase(const_iterator first, const_iterator last) {+    return eraseInto(first, last, variadic_noop);+  }++  /// Remove a specific key.+  size_type erase(key_type const& key) { return eraseInto(key, variadic_noop); }++  /// Remove a key, using a heterogeneous representation.+  template <typename K>+  EnableHeterogeneousErase<K, size_type> erase(K const& key) {+    return eraseInto(key, variadic_noop);+  }++ protected:+  template <typename BeforeDestroy>+  FOLLY_ALWAYS_INLINE void tableEraseIterInto(+      ItemIter pos, BeforeDestroy&& beforeDestroy) {+    table_.eraseIterInto(pos, [&](value_type&& v) {+      auto p = Policy::moveValue(v);+      beforeDestroy(std::move(p.first), std::move(p.second));+    });+  }++  template <typename K, typename BeforeDestroy>+  FOLLY_ALWAYS_INLINE std::size_t tableEraseKeyInto(+      K const& key, BeforeDestroy&& beforeDestroy) {+    return table_.eraseKeyInto(key, [&](value_type&& v) {+      auto p = Policy::moveValue(v);+      beforeDestroy(std::move(p.first), std::move(p.second));+    });+  }++ public:+  /**+   * Callback-erase a single iterator.+   * @overloadbrief Erase with pre-destruction callback.+   * @methodset Modifiers+   *+   * eraseInto contains the same overloads as erase but provides+   * an additional callback argument which is called with an rvalue+   * reference (not const) to the key and an rvalue reference to the+   * mapped value directly before it is destroyed. This can be used+   * to extract an entry out of a F14Map while avoiding a copy.+   */+  template <typename BeforeDestroy>+  FOLLY_ALWAYS_INLINE iterator+  eraseInto(const_iterator pos, BeforeDestroy&& beforeDestroy) {+    // If we are inlined then gcc and clang can optimize away all of the+    // work of itemPos.advance() if our return value is discarded.+    auto itemPos = table_.unwrapIter(pos);+    tableEraseIterInto(itemPos, beforeDestroy);+    itemPos.advanceLikelyDead();+    return table_.makeIter(itemPos);+  }++  /// This form avoids ambiguity when key_type has a templated constructor+  /// that accepts const_iterator+  template <typename BeforeDestroy>+  FOLLY_ALWAYS_INLINE iterator+  eraseInto(iterator pos, BeforeDestroy&& beforeDestroy) {+    const_iterator cpos{pos};+    return eraseInto(cpos, beforeDestroy);+  }++  /// Callback-erase a range of values.+  template <typename BeforeDestroy>+  iterator eraseInto(+      const_iterator first,+      const_iterator last,+      BeforeDestroy&& beforeDestroy) {+    auto itemFirst = table_.unwrapIter(first);+    auto itemLast = table_.unwrapIter(last);+    while (itemFirst != itemLast) {+      tableEraseIterInto(itemFirst, beforeDestroy);+      itemFirst.advance();+    }+    return table_.makeIter(itemFirst);+  }++  template <typename BeforeDestroy>+  size_type eraseInto(key_type const& key, BeforeDestroy&& beforeDestroy) {+    return tableEraseKeyInto(key, beforeDestroy);+  }++  /// Callback-erase a specific key, using a heterogeneous representation.+  template <typename K, typename BeforeDestroy>+  EnableHeterogeneousErase<K, size_type> eraseInto(+      K const& key, BeforeDestroy&& beforeDestroy) {+    return tableEraseKeyInto(key, beforeDestroy);+  }++  //// PUBLIC - Lookup++  /// Get a value for a key+  /// @methodset Element Access+  FOLLY_ALWAYS_INLINE mapped_type& at(key_type const& key) {+    return at(*this, key);+  }++  FOLLY_ALWAYS_INLINE mapped_type const& at(key_type const& key) const {+    return at(*this, key);+  }++  template <typename K>+  EnableHeterogeneousFind<K, mapped_type&> at(K const& key) {+    return at(*this, key);+  }++  template <typename K>+  EnableHeterogeneousFind<K, mapped_type const&> at(K const& key) const {+    return at(*this, key);+  }++  /// Get a value for a key; create the value if it doesn't already exist+  /// @methodset Element Access+  mapped_type& operator[](key_type const& key) {+    return try_emplace(key).first->second;+  }++  mapped_type& operator[](key_type&& key) {+    return try_emplace(std::move(key)).first->second;+  }++  template <typename K>+  EnableHeterogeneousInsert<K, mapped_type&> operator[](K&& key) {+    return try_emplace(std::forward<K>(key)).first->second;+  }++  /// @overloadbrief Number of elements matching the given key.+  /// @methodset Lookup+  FOLLY_ALWAYS_INLINE size_type count(key_type const& key) const {+    return contains(key) ? 1 : 0;+  }++  template <typename K>+  FOLLY_ALWAYS_INLINE EnableHeterogeneousFind<K, size_type> count(+      K const& key) const {+    return contains(key) ? 1 : 0;+  }++  /**+   * @overloadbrief Prehash a key.+   * @methodset Lookup+   *+   * prehash(key) does the work of evaluating hash_function()(key)+   * (including additional bit-mixing for non-avalanching hash functions),+   * and wraps the result of that work in a token for later reuse.+   *+   * The returned token may be used at any time, may be used more than+   * once, and may be used in other F14 sets and maps.  Tokens are+   * transferrable between any F14 containers (maps and sets) with the+   * same key_type and equal hash_function()s.+   *+   * Hash tokens are not hints -- it is a bug to call any method on this+   * class with a token t and key k where t isn't the result of a call+   * to prehash(k2) with k2 == k.+   */+  F14HashToken prehash(key_type const& key) const {+    return table_.prehash(key);+  }+  /// @copydoc prehash+  F14HashToken prehash(key_type const& key, std::size_t hash) const {+    return table_.prehash(key, hash);+  }++  /// @copydoc prehash+  template <typename K>+  EnableHeterogeneousFind<K, F14HashToken> prehash(K const& key) const {+    return table_.prehash(key);+  }+  /// @copydoc prehash+  template <typename K>+  EnableHeterogeneousFind<K, F14HashToken> prehash(+      K const& key, std::size_t hash) const {+    return table_.prehash(key, hash);+  }++  /**+   * @overloadbrief Prefetch cachelines associated with a key.+   * @methodset Lookup+   *+   * prefetch(token) begins prefetching the first steps of looking for key into+   * the local CPU cache.+   *+   * Example Scenario: Loading 2 values from a cold map.+   * You have a map that is cold, meaning it is out of the local CPU cache,+   * and you want to load two values from the map. This can be extended to+   * load N values, but we're loading 2 for simplicity.+   *+   * When the map is cold the dominating factor in the latency is loading the+   * cache line of the entry into the local CPU cache. Using prehash() will+   * issue these cache line fetches in parallel.  That means that by the time we+   * finish map.find(token1, key1) the cache lines needed by map.find(token2,+   * key2) may already be in the local CPU cache. In the best case this will+   * half the latency.+   *+   * It is always okay to call prefetch() before a find() or other lookup+   * operation, as it only prefetches cache lines that are guaranteed to be+   * needed by the lookup.+   *+   *   std::pair<iterator, iterator> find2(+   *       auto& map, key_type const& key1, key_type const& key2) {+   *     auto const token1 = map.prehash(key1);+   *     map.prefetch(token1);+   *     auto const token2 = map.prehash(key2);+   *     map.prefetch(token2);+   *     return std::make_pair(map.find(token1, key1), map.find(token2, key2));+   *   }+   */+  void prefetch(F14HashToken const& token) const { table_.prefetch(token); }++  /// @overloadbrief Get the iterator for a key.+  /// @methodset Lookup+  FOLLY_ALWAYS_INLINE iterator find(key_type const& key) {+    return table_.makeIter(table_.find(key));+  }++  /// Get the iterator for a key.+  FOLLY_ALWAYS_INLINE const_iterator find(key_type const& key) const {+    return table_.makeConstIter(table_.find(key));+  }++  FOLLY_ALWAYS_INLINE iterator+  find(F14HashToken const& token, key_type const& key) {+    return table_.makeIter(table_.find(token, key));+  }++  FOLLY_ALWAYS_INLINE const_iterator+  find(F14HashToken const& token, key_type const& key) const {+    return table_.makeConstIter(table_.find(token, key));+  }++  template <typename K>+  FOLLY_ALWAYS_INLINE EnableHeterogeneousFind<K, iterator> find(K const& key) {+    return table_.makeIter(table_.find(key));+  }++  template <typename K>+  FOLLY_ALWAYS_INLINE EnableHeterogeneousFind<K, const_iterator> find(+      K const& key) const {+    return table_.makeConstIter(table_.find(key));+  }++  template <typename K>+  FOLLY_ALWAYS_INLINE EnableHeterogeneousFind<K, iterator> find(+      F14HashToken const& token, K const& key) {+    return table_.makeIter(table_.find(token, key));+  }++  template <typename K>+  FOLLY_ALWAYS_INLINE EnableHeterogeneousFind<K, const_iterator> find(+      F14HashToken const& token, K const& key) const {+    return table_.makeConstIter(table_.find(token, key));+  }++  /**+   * @overloadbrief Checks if the container contains an element with the+   * specific key.+   * @methodset Lookup+   */+  FOLLY_ALWAYS_INLINE bool contains(key_type const& key) const {+    return !table_.find(key).atEnd();+  }++  template <typename K>+  FOLLY_ALWAYS_INLINE EnableHeterogeneousFind<K, bool> contains(+      K const& key) const {+    return !table_.find(key).atEnd();+  }++  FOLLY_ALWAYS_INLINE bool contains(+      F14HashToken const& token, key_type const& key) const {+    return !table_.find(token, key).atEnd();+  }++  template <typename K>+  FOLLY_ALWAYS_INLINE EnableHeterogeneousFind<K, bool> contains(+      F14HashToken const& token, K const& key) const {+    return !table_.find(token, key).atEnd();+  }++  /// @overloadbrief Returns the range of elements matching a specific key.+  /// @methodset Lookup+  std::pair<iterator, iterator> equal_range(key_type const& key) {+    return equal_range(*this, key);+  }++  std::pair<const_iterator, const_iterator> equal_range(+      key_type const& key) const {+    return equal_range(*this, key);+  }++  template <typename K>+  EnableHeterogeneousFind<K, std::pair<iterator, iterator>> equal_range(+      K const& key) {+    return equal_range(*this, key);+  }++  template <typename K>+  EnableHeterogeneousFind<K, std::pair<const_iterator, const_iterator>>+  equal_range(K const& key) const {+    return equal_range(*this, key);+  }++  //// PUBLIC - Bucket interface++  /// The number of buckets in this container.+  /// @methodset Bucket interface+  std::size_t bucket_count() const noexcept { return table_.bucket_count(); }++  /// The maximum number of buckets for this container.+  /// @methodset Bucket interface+  std::size_t max_bucket_count() const noexcept {+    return table_.max_bucket_count();+  }++  //// PUBLIC - Hash policy++  /// Load factor of the underlying hashtable.+  /// @methodset Hash policy+  float load_factor() const noexcept { return table_.load_factor(); }++  /**+   * @overloadbrief Load factor control.+   * Get the maximum load factor for this container.+   * @methodset Hash policy+   */+  float max_load_factor() const noexcept { return table_.max_load_factor(); }++  /// Set the maximum load factor for this container.+  /// @methodset Hash policy+  void max_load_factor(float v) { table_.max_load_factor(v); }++  /**+   * Rehash this container.+   * @methodset Hash policy+   *+   * This function is provided for compliance with C++'s requirements for+   * hashtables, but is no better than a simple `reserve` call for F14.+   *+   * @param bucketCapacity  The desired capacity across all buckets.+   */+  void rehash(std::size_t bucketCapacity) {+    // The standard's rehash() requires understanding the max load factor,+    // which is easy to get wrong.  Since we don't actually allow adjustment+    // of max_load_factor there is no difference.+    reserve(bucketCapacity);+  }++  /**+   * Pre-allocate space for at least this many elements.+   * @methodset Capacity+   *+   * @param capacity  The number of elements to pre-allocate space for.+   */+  void reserve(std::size_t capacity) { table_.reserve(capacity); }++  //// PUBLIC - Observers++  /// Get the hasher.+  /// @methodset Observers+  hasher hash_function() const { return table_.hasher(); }++  /// Get the key_equal.+  /// @methodset Observers+  key_equal key_eq() const { return table_.keyEqual(); }++  //// PUBLIC - F14 Extensions++  /**+   * Checks for a value using operator==+   * @methodset Lookup+   *+   * containsEqualValue returns true iff there is an element in the map+   * that compares equal to value using operator==.  It is undefined+   * behavior to call this function if operator== on key_type can ever+   * return true when the same keys passed to key_eq() would return false+   * (the opposite is allowed).+   */+  bool containsEqualValue(value_type const& value) const {+    auto it = table_.findMatching(value.first, [&](auto& key) {+      return value.first == key;+    });+    return !it.atEnd() && value.second == table_.valueAtItem(it.citem()).second;+  }++  /// Get memory footprint, not including sizeof(*this).+  /// @methodset Capacity+  std::size_t getAllocatedMemorySize() const {+    return table_.getAllocatedMemorySize();+  }++  /**+   * In-depth memory analysis.+   * @methodset Capacity+   *+   * Enumerates classes of allocated memory blocks currently owned+   * by this table, calling visitor(allocationSize, allocationCount).+   * This can be used to get a more accurate indication of memory footprint+   * than getAllocatedMemorySize() if you have some way of computing the+   * internal fragmentation of the allocator, such as JEMalloc's nallocx.+   * The visitor might be called twice with the same allocationSize. The+   * visitor's computation should produce the same result for visitor(8,+   * 2) as for two calls to visitor(8, 1), for example.  The visitor may+   * be called with a zero allocationCount.+   */+  template <typename V>+  void visitAllocationClasses(V&& visitor) const {+    return table_.visitAllocationClasses(visitor);+  }++  /**+   * Visit contiguous ranges of elements.+   * @methodset Iterators+   *+   * Calls visitor with two value_type const*, b and e, such that every+   * entry in the table is included in exactly one of the ranges [b,e).+   * This can be used to efficiently iterate elements in bulk when crossing+   * an API boundary that supports contiguous blocks of items.+   */+  template <typename V>+  void visitContiguousRanges(V&& visitor) const;++  /// Get stats.+  /// @methodset Hash policy+  F14TableStats computeStats() const noexcept { return table_.computeStats(); }++ private:+  template <typename Self, typename K>+  FOLLY_ALWAYS_INLINE static auto& at(Self& self, K const& key) {+    auto iter = self.find(key);+    if (iter == self.end()) {+      throw_exception<std::out_of_range>("at() did not find key");+    }+    return iter->second;+  }++  template <typename Self, typename K>+  static auto equal_range(Self& self, K const& key) {+    auto first = self.find(key);+    auto last = first;+    if (last != self.end()) {+      ++last;+    }+    return std::make_pair(first, last);+  }++ protected:+  F14Table<Policy> table_;+};+} // namespace detail+} // namespace f14++template <+    typename Key,+    typename Mapped,+    typename Hasher,+    typename KeyEqual,+    typename Alloc>+class F14ValueMap+    : public f14::detail::F14BasicMap<f14::detail::MapPolicyWithDefaults<+          f14::detail::ValueContainerPolicy,+          Key,+          Mapped,+          Hasher,+          KeyEqual,+          Alloc>> {+ protected:+  using Policy = f14::detail::MapPolicyWithDefaults<+      f14::detail::ValueContainerPolicy,+      Key,+      Mapped,+      Hasher,+      KeyEqual,+      Alloc>;++ private:+  using Super = f14::detail::F14BasicMap<Policy>;++ public:+  using typename Super::value_type;++  F14ValueMap() = default;++  using Super::Super;++  F14ValueMap& operator=(std::initializer_list<value_type> ilist) {+    Super::operator=(ilist);+    return *this;+  }++  /// Swaps contained objects with another F14Map+  /// @methodset Modifiers+  void swap(F14ValueMap& rhs) noexcept(Policy::kSwapIsNoexcept) {+    this->table_.swap(rhs.table_);+  }++  /**+   * Visit contiguous ranges of elements.+   * @methodset Iterators+   *+   * Calls visitor with two value_type const*, b and e, such that every+   * entry in the table is included in exactly one of the ranges [b,e).+   * This can be used to efficiently iterate elements in bulk when crossing+   * an API boundary that supports contiguous blocks of items.+   */+  template <typename V>+  void visitContiguousRanges(V&& visitor) const {+    this->table_.visitContiguousItemRanges(visitor);+  }+};+#endif // FOLLY_F14_VECTOR_INTRINSICS_AVAILABLE++template <+    typename InputIt,+    typename Hasher = f14::DefaultHasher<iterator_key_type_t<InputIt>>,+    typename KeyEqual = f14::DefaultKeyEqual<iterator_key_type_t<InputIt>>,+    typename Alloc = f14::DefaultAlloc<iterator_value_type_t<InputIt>>,+    typename = detail::RequireInputIterator<InputIt>,+    // Next two constraints are necessary to disambiguate from next constructor+    typename = detail::RequireNotAllocator<Hasher>,+    typename = detail::RequireNotAllocator<KeyEqual>,+    typename = detail::RequireAllocator<Alloc>>+F14ValueMap(+    InputIt, InputIt, std::size_t = {}, Hasher = {}, KeyEqual = {}, Alloc = {})+    -> F14ValueMap<+        iterator_key_type_t<InputIt>,+        iterator_mapped_type_t<InputIt>,+        Hasher,+        KeyEqual,+        Alloc>;++template <+    typename InputIt,+    typename Alloc,+    typename = detail::RequireInputIterator<InputIt>,+    typename = detail::RequireAllocator<Alloc>>+F14ValueMap(InputIt, InputIt, std::size_t, Alloc)+    -> F14ValueMap<+        iterator_key_type_t<InputIt>,+        iterator_mapped_type_t<InputIt>,+        f14::DefaultHasher<iterator_key_type_t<InputIt>>,+        f14::DefaultKeyEqual<iterator_key_type_t<InputIt>>,+        Alloc>;++template <+    typename InputIt,+    typename Hasher,+    typename Alloc,+    typename = detail::RequireInputIterator<InputIt>,+    typename = detail::RequireNotAllocator<Hasher>,+    typename = detail::RequireAllocator<Alloc>>+F14ValueMap(InputIt, InputIt, std::size_t, Hasher, Alloc)+    -> F14ValueMap<+        iterator_key_type_t<InputIt>,+        iterator_mapped_type_t<InputIt>,+        Hasher,+        f14::DefaultKeyEqual<iterator_key_type_t<InputIt>>,+        Alloc>;++template <+    typename Key,+    typename Mapped,+    typename Hasher = f14::DefaultHasher<Key>,+    typename KeyEqual = f14::DefaultKeyEqual<Key>,+    typename Alloc = f14::DefaultAlloc<std::pair<const Key, Mapped>>,+    typename = detail::RequireNotAllocator<Hasher>,+    typename = detail::RequireNotAllocator<KeyEqual>,+    typename = detail::RequireAllocator<Alloc>>+F14ValueMap(+    std::initializer_list<std::pair<Key, Mapped>>,+    std::size_t = {},+    Hasher = {},+    KeyEqual = {},+    Alloc = {}) -> F14ValueMap<Key, Mapped, Hasher, KeyEqual, Alloc>;++template <+    typename Key,+    typename Mapped,+    typename Alloc,+    typename = detail::RequireAllocator<Alloc>>+F14ValueMap(std::initializer_list<std::pair<Key, Mapped>>, std::size_t, Alloc)+    -> F14ValueMap<+        Key,+        Mapped,+        f14::DefaultHasher<Key>,+        f14::DefaultKeyEqual<Key>,+        Alloc>;++template <+    typename Key,+    typename Mapped,+    typename Hasher,+    typename Alloc,+    typename = detail::RequireAllocator<Alloc>>+F14ValueMap(+    std::initializer_list<std::pair<Key, Mapped>>, std::size_t, Hasher, Alloc)+    -> F14ValueMap<Key, Mapped, Hasher, f14::DefaultKeyEqual<Key>, Alloc>;++#if FOLLY_F14_VECTOR_INTRINSICS_AVAILABLE+template <+    typename Key,+    typename Mapped,+    typename Hasher,+    typename KeyEqual,+    typename Alloc>+class F14NodeMap+    : public f14::detail::F14BasicMap<f14::detail::MapPolicyWithDefaults<+          f14::detail::NodeContainerPolicy,+          Key,+          Mapped,+          Hasher,+          KeyEqual,+          Alloc>> {+ protected:+  using Policy = f14::detail::MapPolicyWithDefaults<+      f14::detail::NodeContainerPolicy,+      Key,+      Mapped,+      Hasher,+      KeyEqual,+      Alloc>;++ private:+  using Super = f14::detail::F14BasicMap<Policy>;++ public:+  using typename Super::value_type;++  F14NodeMap() = default;++  using Super::Super;++  F14NodeMap& operator=(std::initializer_list<value_type> ilist) {+    Super::operator=(ilist);+    return *this;+  }++  /// @methodset Modifiers+  void swap(F14NodeMap& rhs) noexcept(Policy::kSwapIsNoexcept) {+    this->table_.swap(rhs.table_);+  }++  /**+   * Visit contiguous ranges of elements.+   * @methodset Iterators+   *+   * Calls visitor with two value_type const*, b and e, such that every+   * entry in the table is included in exactly one of the ranges [b,e).+   * This can be used to efficiently iterate elements in bulk when crossing+   * an API boundary that supports contiguous blocks of items.+   */+  template <typename V>+  void visitContiguousRanges(V&& visitor) const {+    this->table_.visitItems([&](typename Policy::Item ptr) {+      value_type const* b = std::addressof(*ptr);+      visitor(b, b + 1);+    });+  }++  // TODO extract and node_handle insert+};+#endif // FOLLY_F14_VECTOR_INTRINSICS_AVAILABLE++template <+    typename InputIt,+    typename Hasher = f14::DefaultHasher<iterator_key_type_t<InputIt>>,+    typename KeyEqual = f14::DefaultKeyEqual<iterator_key_type_t<InputIt>>,+    typename Alloc = f14::DefaultAlloc<iterator_value_type_t<InputIt>>,+    typename = detail::RequireInputIterator<InputIt>,+    typename = detail::RequireNotAllocator<Hasher>,+    typename = detail::RequireNotAllocator<KeyEqual>,+    typename = detail::RequireAllocator<Alloc>>+F14NodeMap(+    InputIt, InputIt, std::size_t = {}, Hasher = {}, KeyEqual = {}, Alloc = {})+    -> F14NodeMap<+        iterator_key_type_t<InputIt>,+        iterator_mapped_type_t<InputIt>,+        Hasher,+        KeyEqual,+        Alloc>;++template <+    typename InputIt,+    typename Alloc,+    typename = detail::RequireInputIterator<InputIt>,+    typename = detail::RequireAllocator<Alloc>>+F14NodeMap(InputIt, InputIt, std::size_t, Alloc)+    -> F14NodeMap<+        iterator_key_type_t<InputIt>,+        iterator_mapped_type_t<InputIt>,+        f14::DefaultHasher<iterator_key_type_t<InputIt>>,+        f14::DefaultKeyEqual<iterator_key_type_t<InputIt>>,+        Alloc>;++template <+    typename InputIt,+    typename Hasher,+    typename Alloc,+    typename = detail::RequireInputIterator<InputIt>,+    typename = detail::RequireNotAllocator<Hasher>,+    typename = detail::RequireAllocator<Alloc>>+F14NodeMap(InputIt, InputIt, std::size_t, Hasher, Alloc)+    -> F14NodeMap<+        iterator_key_type_t<InputIt>,+        iterator_mapped_type_t<InputIt>,+        Hasher,+        f14::DefaultKeyEqual<iterator_key_type_t<InputIt>>,+        Alloc>;++template <+    typename Key,+    typename Mapped,+    typename Hasher = f14::DefaultHasher<Key>,+    typename KeyEqual = f14::DefaultKeyEqual<Key>,+    typename Alloc = f14::DefaultAlloc<std::pair<const Key, Mapped>>,+    typename = detail::RequireNotAllocator<Hasher>,+    typename = detail::RequireNotAllocator<KeyEqual>,+    typename = detail::RequireAllocator<Alloc>>+F14NodeMap(+    std::initializer_list<std::pair<Key, Mapped>>,+    std::size_t = {},+    Hasher = {},+    KeyEqual = {},+    Alloc = {}) -> F14NodeMap<Key, Mapped, Hasher, KeyEqual, Alloc>;++template <+    typename Key,+    typename Mapped,+    typename Alloc,+    typename = detail::RequireAllocator<Alloc>>+F14NodeMap(std::initializer_list<std::pair<Key, Mapped>>, std::size_t, Alloc)+    -> F14NodeMap<+        Key,+        Mapped,+        f14::DefaultHasher<Key>,+        f14::DefaultKeyEqual<Key>,+        Alloc>;++template <+    typename Key,+    typename Mapped,+    typename Hasher,+    typename Alloc,+    typename = detail::RequireAllocator<Alloc>>+F14NodeMap(+    std::initializer_list<std::pair<Key, Mapped>>, std::size_t, Hasher, Alloc)+    -> F14NodeMap<Key, Mapped, Hasher, f14::DefaultKeyEqual<Key>, Alloc>;++#if FOLLY_F14_VECTOR_INTRINSICS_AVAILABLE+namespace f14 {+namespace detail {+template <+    typename Key,+    typename Mapped,+    typename Hasher,+    typename KeyEqual,+    typename Alloc,+    typename EligibleForPerturbedInsertionOrder>+class F14VectorMapImpl+    : public F14BasicMap<MapPolicyWithDefaults<+          VectorContainerPolicy,+          Key,+          Mapped,+          Hasher,+          KeyEqual,+          Alloc,+          EligibleForPerturbedInsertionOrder>> {+ protected:+  using Policy = MapPolicyWithDefaults<+      VectorContainerPolicy,+      Key,+      Mapped,+      Hasher,+      KeyEqual,+      Alloc,+      EligibleForPerturbedInsertionOrder>;++ private:+  using Super = F14BasicMap<Policy>;++  template <typename K>+  using IsIter = Disjunction<+      std::is_same<typename Policy::Iter, remove_cvref_t<K>>,+      std::is_same<typename Policy::ConstIter, remove_cvref_t<K>>,+      std::is_same<typename Policy::ReverseIter, remove_cvref_t<K>>,+      std::is_same<typename Policy::ConstReverseIter, remove_cvref_t<K>>>;++  template <typename K, typename T>+  using EnableHeterogeneousVectorErase = std::enable_if_t<+      ::folly::detail::EligibleForHeterogeneousFind<+          Key,+          Hasher,+          KeyEqual,+          std::conditional_t<IsIter<K>::value, Key, K>>::value &&+          !IsIter<K>::value,+      T>;++ public:+  using typename Super::const_iterator;+  using typename Super::iterator;+  using typename Super::key_type;+  using typename Super::mapped_type;+  using typename Super::value_type;++  F14VectorMapImpl() = default;++  // inherit constructors+  using Super::Super;++  F14VectorMapImpl& operator=(std::initializer_list<value_type> ilist) {+    Super::operator=(ilist);+    return *this;+  }++  /// @methodset Iterators+  iterator begin() { return this->table_.linearBegin(this->size()); }+  /// @methodset Iterators+  const_iterator begin() const { return cbegin(); }+  /// @methodset Iterators+  const_iterator cbegin() const {+    return this->table_.linearBegin(this->size());+  }++  /// @methodset Iterators+  iterator end() { return this->table_.linearEnd(); }+  /// @methodset Iterators+  const_iterator end() const { return cend(); }+  /// @methodset Iterators+  const_iterator cend() const { return this->table_.linearEnd(); }++ private:+  template <typename BeforeDestroy>+  void eraseUnderlying(+      typename Policy::ItemIter underlying, BeforeDestroy&& beforeDestroy) {+    Alloc& a = this->table_.alloc();+    auto values = this->table_.values_;++    // Remove the ptr from the base table and destroy the value.+    auto index = underlying.item();+    // The item still needs to be hashable during this call, so we must destroy+    // the value _afterwards_.+    this->tableEraseIterInto(underlying, beforeDestroy);+    Policy::AllocTraits::destroy(a, std::addressof(values[index]));++    // move the last element in values_ down and fix up the inbound index+    auto tailIndex = this->size();+    if (tailIndex != index) {+      auto tail = this->table_.find(+          VectorContainerIndexSearch{static_cast<uint32_t>(tailIndex)});+      tail.item() = index;+      auto p = std::addressof(values[index]);+      assume(p != nullptr);+      this->table_.transfer(a, std::addressof(values[tailIndex]), p, 1);+    }+  }++  template <typename K, typename BeforeDestroy>+  std::size_t eraseUnderlyingKey(K const& key, BeforeDestroy&& beforeDestroy) {+    auto underlying = this->table_.find(key);+    if (underlying.atEnd()) {+      return 0;+    } else {+      eraseUnderlying(underlying, beforeDestroy);+      return 1;+    }+  }++ public:+  /**+   * Remove element at a specific position (iterator).+   * @overloadbrief Remove elements.+   * @methodset Modifiers+   */+  FOLLY_ALWAYS_INLINE iterator erase(const_iterator pos) {+    return eraseInto(pos, variadic_noop);+  }++  // This form avoids ambiguity when key_type has a templated constructor+  // that accepts const_iterator+  FOLLY_ALWAYS_INLINE iterator erase(iterator pos) {+    return eraseInto(pos, variadic_noop);+  }++  /// Remove a range of elements.+  iterator erase(const_iterator first, const_iterator last) {+    return eraseInto(first, last, variadic_noop);+  }++  /// Remove a specific key.+  std::size_t erase(key_type const& key) {+    return eraseInto(key, variadic_noop);+  }++  /// Remove a key, using a heterogeneous representation.+  template <typename K>+  EnableHeterogeneousVectorErase<K, std::size_t> erase(K const& key) {+    return eraseInto(key, variadic_noop);+  }++  /**+   * Callback-erase a single iterator.+   * @overloadbrief Erase with pre-destruction callback.+   * @methodset Modifiers+   *+   * Like erase, but with an additional callback argument which is called with+   * an rvalue reference to the item directly before it is destroyed. This can+   * be used to extract an item out of a F14Set while avoiding a copy.+   */+  template <typename BeforeDestroy>+  FOLLY_ALWAYS_INLINE iterator+  eraseInto(const_iterator pos, BeforeDestroy&& beforeDestroy) {+    auto index = this->table_.iterToIndex(pos);+    auto underlying = this->table_.find(VectorContainerIndexSearch{index});+    eraseUnderlying(underlying, beforeDestroy);+    return index == 0 ? end() : this->table_.indexToIter(index - 1);+  }++  // This form avoids ambiguity when key_type has a templated constructor+  // that accepts const_iterator+  template <typename BeforeDestroy>+  FOLLY_ALWAYS_INLINE iterator+  eraseInto(iterator pos, BeforeDestroy&& beforeDestroy) {+    const_iterator cpos{pos};+    return eraseInto(cpos, beforeDestroy);+  }++  /// Callback-erase a range of values.+  template <typename BeforeDestroy>+  iterator eraseInto(+      const_iterator first,+      const_iterator last,+      BeforeDestroy&& beforeDestroy) {+    while (first != last) {+      first = eraseInto(first, beforeDestroy);+    }+    auto index = this->table_.iterToIndex(first);+    return index == 0 ? end() : this->table_.indexToIter(index - 1);+  }++  /// Callback-erase a specific key.+  template <typename BeforeDestroy>+  std::size_t eraseInto(key_type const& key, BeforeDestroy&& beforeDestroy) {+    return eraseUnderlyingKey(key, beforeDestroy);+  }++  /// Callback-erase a specific key, using a heterogeneous representation.+  template <typename K, typename BeforeDestroy>+  EnableHeterogeneousVectorErase<K, std::size_t> eraseInto(+      K const& key, BeforeDestroy&& beforeDestroy) {+    return eraseUnderlyingKey(key, beforeDestroy);+  }++  /**+   * Visit contiguous ranges of elements.+   * @methodset Iterators+   *+   * Calls visitor with two value_type const*, b and e, such that every+   * entry in the table is included in exactly one of the ranges [b,e).+   * This can be used to efficiently iterate elements in bulk when crossing+   * an API boundary that supports contiguous blocks of items.+   */+  template <typename V>+  void visitContiguousRanges(V&& visitor) const {+    auto n = this->table_.size();+    if (n > 0) {+      value_type const* b = std::addressof(this->table_.values_[0]);+      visitor(b, b + n);+    }+  }+};+} // namespace detail+} // namespace f14++template <+    typename Key,+    typename Mapped,+    typename Hasher,+    typename KeyEqual,+    typename Alloc>+class F14VectorMap+    : public f14::detail::F14VectorMapImpl<+          Key,+          Mapped,+          Hasher,+          KeyEqual,+          Alloc,+          std::false_type> {+  using Super = f14::detail::+      F14VectorMapImpl<Key, Mapped, Hasher, KeyEqual, Alloc, std::false_type>;++ public:+  using typename Super::const_iterator;+  using typename Super::iterator;+  using typename Super::value_type;+  using reverse_iterator = typename Super::Policy::ReverseIter;+  using const_reverse_iterator = typename Super::Policy::ConstReverseIter;++  F14VectorMap() = default;++  // inherit constructors+  using Super::Super;++  F14VectorMap& operator=(std::initializer_list<value_type> ilist) {+    Super::operator=(ilist);+    return *this;+  }++  /// @methodset Modifiers+  void swap(F14VectorMap& rhs) noexcept(Super::Policy::kSwapIsNoexcept) {+    this->table_.swap(rhs.table_);+  }++  // ITERATION ORDER+  //+  // Deterministic iteration order for insert-only workloads is part of+  // F14VectorMap's supported API: iterator is LIFO and reverse_iterator+  // is FIFO.+  //+  // If there have been no calls to erase() then iterator and+  // const_iterator enumerate entries in the opposite of insertion order.+  // begin()->first is the key most recently inserted.  reverse_iterator+  // and reverse_const_iterator, therefore, enumerate in LIFO (insertion)+  // order for insert-only workloads.  Deterministic iteration order is+  // only guaranteed if no keys were removed since the last time the+  // map was empty.  Iteration order is preserved across rehashes and+  // F14VectorMap copies and moves.+  //+  // iterator uses LIFO order so that erasing while iterating with begin()+  // and end() is safe using the erase(it++) idiom, which is supported+  // by std::map and std::unordered_map.  erase(iter) invalidates iter+  // and all iterators before iter in the non-reverse iteration order.+  // Every successful erase invalidates all reverse iterators.+  //+  // No erase is provided for reverse_iterator or const_reverse_iterator+  // to make it harder to shoot yourself in the foot by erasing while+  // reverse-iterating.  You can write that as map.erase(map.iter(riter))+  // if you really need it.++  /// @methodset Iterators+  reverse_iterator rbegin() { return this->table_.values_; }+  /// @methodset Iterators+  const_reverse_iterator rbegin() const { return crbegin(); }+  /// @methodset Iterators+  const_reverse_iterator crbegin() const { return this->table_.values_; }++  /// @methodset Iterators+  reverse_iterator rend() { return this->table_.values_ + this->table_.size(); }+  /// @methodset Iterators+  const_reverse_iterator rend() const { return crend(); }+  /// @methodset Iterators+  const_reverse_iterator crend() const {+    return this->table_.values_ + this->table_.size();+  }++  /// Explicit conversions between iterator and reverse_iterator+  /// @methodset Iterators+  iterator iter(reverse_iterator riter) { return this->table_.iter(riter); }+  const_iterator iter(const_reverse_iterator riter) const {+    return this->table_.iter(riter);+  }++  /// @copydoc iter+  reverse_iterator riter(iterator it) { return this->table_.riter(it); }+  const_reverse_iterator riter(const_iterator it) const {+    return this->table_.riter(it);+  }++  friend Range<const_reverse_iterator> tag_invoke(+      order_preserving_reinsertion_view_fn, F14VectorMap const& c) noexcept {+    return {c.rbegin(), c.rend()};+  }+};+#endif // FOLLY_F14_VECTOR_INTRINSICS_AVAILABLE++template <+    typename InputIt,+    typename Hasher = f14::DefaultHasher<iterator_key_type_t<InputIt>>,+    typename KeyEqual = f14::DefaultKeyEqual<iterator_key_type_t<InputIt>>,+    typename Alloc = f14::DefaultAlloc<iterator_value_type_t<InputIt>>,+    typename = detail::RequireInputIterator<InputIt>,+    typename = detail::RequireNotAllocator<Hasher>,+    typename = detail::RequireNotAllocator<KeyEqual>,+    typename = detail::RequireAllocator<Alloc>>+F14VectorMap(+    InputIt, InputIt, std::size_t = {}, Hasher = {}, KeyEqual = {}, Alloc = {})+    -> F14VectorMap<+        iterator_key_type_t<InputIt>,+        iterator_mapped_type_t<InputIt>,+        Hasher,+        KeyEqual,+        Alloc>;++template <+    typename InputIt,+    typename Alloc,+    typename = detail::RequireInputIterator<InputIt>,+    typename = detail::RequireAllocator<Alloc>>+F14VectorMap(InputIt, InputIt, std::size_t, Alloc)+    -> F14VectorMap<+        iterator_key_type_t<InputIt>,+        iterator_mapped_type_t<InputIt>,+        f14::DefaultHasher<iterator_key_type_t<InputIt>>,+        f14::DefaultKeyEqual<iterator_key_type_t<InputIt>>,+        Alloc>;++template <+    typename InputIt,+    typename Hasher,+    typename Alloc,+    typename = detail::RequireInputIterator<InputIt>,+    typename = detail::RequireNotAllocator<Hasher>,+    typename = detail::RequireAllocator<Alloc>>+F14VectorMap(InputIt, InputIt, std::size_t, Hasher, Alloc)+    -> F14VectorMap<+        iterator_key_type_t<InputIt>,+        iterator_mapped_type_t<InputIt>,+        Hasher,+        f14::DefaultKeyEqual<iterator_key_type_t<InputIt>>,+        Alloc>;++template <+    typename Key,+    typename Mapped,+    typename Hasher = f14::DefaultHasher<Key>,+    typename KeyEqual = f14::DefaultKeyEqual<Key>,+    typename Alloc = f14::DefaultAlloc<std::pair<const Key, Mapped>>,+    typename = detail::RequireNotAllocator<Hasher>,+    typename = detail::RequireNotAllocator<KeyEqual>,+    typename = detail::RequireAllocator<Alloc>>+F14VectorMap(+    std::initializer_list<std::pair<Key, Mapped>>,+    std::size_t = {},+    Hasher = {},+    KeyEqual = {},+    Alloc = {}) -> F14VectorMap<Key, Mapped, Hasher, KeyEqual, Alloc>;++template <+    typename Key,+    typename Mapped,+    typename Alloc,+    typename = detail::RequireAllocator<Alloc>>+F14VectorMap(std::initializer_list<std::pair<Key, Mapped>>, std::size_t, Alloc)+    -> F14VectorMap<+        Key,+        Mapped,+        f14::DefaultHasher<Key>,+        f14::DefaultKeyEqual<Key>,+        Alloc>;++template <+    typename Key,+    typename Mapped,+    typename Hasher,+    typename Alloc,+    typename = detail::RequireAllocator<Alloc>>+F14VectorMap(+    std::initializer_list<std::pair<Key, Mapped>>, std::size_t, Hasher, Alloc)+    -> F14VectorMap<Key, Mapped, Hasher, f14::DefaultKeyEqual<Key>, Alloc>;++#if FOLLY_F14_VECTOR_INTRINSICS_AVAILABLE+/**+ * F14FastMap is, under the hood, either an F14ValueMap or an F14VectorMap.+ * F14FastMap chooses which of these two representations to use based on the+ * size of a node.+ */+template <+    typename Key,+    typename Mapped,+    typename Hasher,+    typename KeyEqual,+    typename Alloc>+class F14FastMap+    : public std::conditional_t<+          (sizeof(std::pair<Key const, Mapped>) < 24),+          F14ValueMap<Key, Mapped, Hasher, KeyEqual, Alloc>,+          f14::detail::F14VectorMapImpl<+              Key,+              Mapped,+              Hasher,+              KeyEqual,+              Alloc,+              std::true_type>> {+  using Super = std::conditional_t<+      sizeof(std::pair<Key const, Mapped>) < 24,+      F14ValueMap<Key, Mapped, Hasher, KeyEqual, Alloc>,+      f14::detail::F14VectorMapImpl<+          Key,+          Mapped,+          Hasher,+          KeyEqual,+          Alloc,+          std::true_type>>;++ public:+  using typename Super::value_type;++  F14FastMap() = default;++  using Super::Super;++  F14FastMap& operator=(std::initializer_list<value_type> ilist) {+    Super::operator=(ilist);+    return *this;+  }++  /// @methodset Modifiers+  void swap(F14FastMap& rhs) noexcept(Super::Policy::kSwapIsNoexcept) {+    this->table_.swap(rhs.table_);+  }+};+#endif // if FOLLY_F14_VECTOR_INTRINSICS_AVAILABLE++template <+    typename InputIt,+    typename Hasher = f14::DefaultHasher<iterator_key_type_t<InputIt>>,+    typename KeyEqual = f14::DefaultKeyEqual<iterator_key_type_t<InputIt>>,+    typename Alloc = f14::DefaultAlloc<iterator_value_type_t<InputIt>>,+    typename = detail::RequireInputIterator<InputIt>,+    typename = detail::RequireNotAllocator<Hasher>,+    typename = detail::RequireNotAllocator<KeyEqual>,+    typename = detail::RequireAllocator<Alloc>>+F14FastMap(+    InputIt, InputIt, std::size_t = {}, Hasher = {}, KeyEqual = {}, Alloc = {})+    -> F14FastMap<+        iterator_key_type_t<InputIt>,+        iterator_mapped_type_t<InputIt>,+        Hasher,+        KeyEqual,+        Alloc>;++template <+    typename InputIt,+    typename Alloc,+    typename = detail::RequireInputIterator<InputIt>,+    typename = detail::RequireAllocator<Alloc>>+F14FastMap(InputIt, InputIt, std::size_t, Alloc)+    -> F14FastMap<+        iterator_key_type_t<InputIt>,+        iterator_mapped_type_t<InputIt>,+        f14::DefaultHasher<iterator_key_type_t<InputIt>>,+        f14::DefaultKeyEqual<iterator_key_type_t<InputIt>>,+        Alloc>;++template <+    typename InputIt,+    typename Hasher,+    typename Alloc,+    typename = detail::RequireInputIterator<InputIt>,+    typename = detail::RequireNotAllocator<Hasher>,+    typename = detail::RequireAllocator<Alloc>>+F14FastMap(InputIt, InputIt, std::size_t, Hasher, Alloc)+    -> F14FastMap<+        iterator_key_type_t<InputIt>,+        iterator_mapped_type_t<InputIt>,+        Hasher,+        f14::DefaultKeyEqual<iterator_key_type_t<InputIt>>,+        Alloc>;++template <+    typename Key,+    typename Mapped,+    typename Hasher = f14::DefaultHasher<Key>,+    typename KeyEqual = f14::DefaultKeyEqual<Key>,+    typename Alloc = f14::DefaultAlloc<std::pair<const Key, Mapped>>,+    typename = detail::RequireNotAllocator<Hasher>,+    typename = detail::RequireNotAllocator<KeyEqual>,+    typename = detail::RequireAllocator<Alloc>>+F14FastMap(+    std::initializer_list<std::pair<Key, Mapped>>,+    std::size_t = {},+    Hasher = {},+    KeyEqual = {},+    Alloc = {}) -> F14FastMap<Key, Mapped, Hasher, KeyEqual, Alloc>;++template <+    typename Key,+    typename Mapped,+    typename Alloc,+    typename = detail::RequireAllocator<Alloc>>+F14FastMap(std::initializer_list<std::pair<Key, Mapped>>, std::size_t, Alloc)+    -> F14FastMap<+        Key,+        Mapped,+        f14::DefaultHasher<Key>,+        f14::DefaultKeyEqual<Key>,+        Alloc>;++template <+    typename Key,+    typename Mapped,+    typename Hasher,+    typename Alloc,+    typename = detail::RequireAllocator<Alloc>>+F14FastMap(+    std::initializer_list<std::pair<Key, Mapped>>, std::size_t, Hasher, Alloc)+    -> F14FastMap<Key, Mapped, Hasher, f14::DefaultKeyEqual<Key>, Alloc>;++} // namespace folly++namespace folly {+namespace f14 {+namespace detail {+template <typename M>+bool mapsEqual(M const& lhs, M const& rhs) {+  if (lhs.size() != rhs.size()) {+    return false;+  }+  for (auto& kv : lhs) {+    if (!rhs.containsEqualValue(kv)) {+      return false;+    }+  }+  return true;+}+} // namespace detail+} // namespace f14++template <typename K, typename M, typename H, typename E, typename A>+bool operator==(+    F14ValueMap<K, M, H, E, A> const& lhs,+    F14ValueMap<K, M, H, E, A> const& rhs) {+  return mapsEqual(lhs, rhs);+}++template <typename K, typename M, typename H, typename E, typename A>+bool operator!=(+    F14ValueMap<K, M, H, E, A> const& lhs,+    F14ValueMap<K, M, H, E, A> const& rhs) {+  return !(lhs == rhs);+}++template <typename K, typename M, typename H, typename E, typename A>+bool operator==(+    F14NodeMap<K, M, H, E, A> const& lhs,+    F14NodeMap<K, M, H, E, A> const& rhs) {+  return mapsEqual(lhs, rhs);+}++template <typename K, typename M, typename H, typename E, typename A>+bool operator!=(+    F14NodeMap<K, M, H, E, A> const& lhs,+    F14NodeMap<K, M, H, E, A> const& rhs) {+  return !(lhs == rhs);+}++template <typename K, typename M, typename H, typename E, typename A>+bool operator==(+    F14VectorMap<K, M, H, E, A> const& lhs,+    F14VectorMap<K, M, H, E, A> const& rhs) {+  return mapsEqual(lhs, rhs);+}++template <typename K, typename M, typename H, typename E, typename A>+bool operator!=(+    F14VectorMap<K, M, H, E, A> const& lhs,+    F14VectorMap<K, M, H, E, A> const& rhs) {+  return !(lhs == rhs);+}++template <typename K, typename M, typename H, typename E, typename A>+bool operator==(+    F14FastMap<K, M, H, E, A> const& lhs,+    F14FastMap<K, M, H, E, A> const& rhs) {+  return mapsEqual(lhs, rhs);+}++template <typename K, typename M, typename H, typename E, typename A>+bool operator!=(+    F14FastMap<K, M, H, E, A> const& lhs,+    F14FastMap<K, M, H, E, A> const& rhs) {+  return !(lhs == rhs);+}++template <typename K, typename M, typename H, typename E, typename A>+void swap(+    F14ValueMap<K, M, H, E, A>& lhs,+    F14ValueMap<K, M, H, E, A>& rhs) noexcept(noexcept(lhs.swap(rhs))) {+  lhs.swap(rhs);+}++template <typename K, typename M, typename H, typename E, typename A>+void swap(+    F14NodeMap<K, M, H, E, A>& lhs,+    F14NodeMap<K, M, H, E, A>& rhs) noexcept(noexcept(lhs.swap(rhs))) {+  lhs.swap(rhs);+}++template <typename K, typename M, typename H, typename E, typename A>+void swap(+    F14VectorMap<K, M, H, E, A>& lhs,+    F14VectorMap<K, M, H, E, A>& rhs) noexcept(noexcept(lhs.swap(rhs))) {+  lhs.swap(rhs);+}++template <typename K, typename M, typename H, typename E, typename A>+void swap(+    F14FastMap<K, M, H, E, A>& lhs,+    F14FastMap<K, M, H, E, A>& rhs) noexcept(noexcept(lhs.swap(rhs))) {+  lhs.swap(rhs);+}++template <+    typename K,+    typename M,+    typename H,+    typename E,+    typename A,+    typename Pred>+std::size_t erase_if(F14ValueMap<K, M, H, E, A>& c, Pred pred) {+  return f14::detail::erase_if_impl(c, pred);+}++template <+    typename K,+    typename M,+    typename H,+    typename E,+    typename A,+    typename Pred>+std::size_t erase_if(F14NodeMap<K, M, H, E, A>& c, Pred pred) {+  return f14::detail::erase_if_impl(c, pred);+}++template <+    typename K,+    typename M,+    typename H,+    typename E,+    typename A,+    typename Pred>+std::size_t erase_if(F14VectorMap<K, M, H, E, A>& c, Pred pred) {+  return f14::detail::erase_if_impl(c, pred);+}++template <+    typename K,+    typename M,+    typename H,+    typename E,+    typename A,+    typename Pred>+std::size_t erase_if(F14FastMap<K, M, H, E, A>& c, Pred pred) {+  return f14::detail::erase_if_impl(c, pred);+}++} // namespace folly
+ folly/folly/container/F14Set-fwd.h view
@@ -0,0 +1,83 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <folly/container/detail/F14Defaults.h>+#include <folly/memory/MemoryResource.h>++namespace folly {+template <+    typename Key,+    typename Hasher = f14::DefaultHasher<Key>,+    typename KeyEqual = f14::DefaultKeyEqual<Key>,+    typename Alloc = f14::DefaultAlloc<Key>>+class F14NodeSet;++template <+    typename Key,+    typename Hasher = f14::DefaultHasher<Key>,+    typename KeyEqual = f14::DefaultKeyEqual<Key>,+    typename Alloc = f14::DefaultAlloc<Key>>+class F14ValueSet;++template <+    typename Key,+    typename Hasher = f14::DefaultHasher<Key>,+    typename KeyEqual = f14::DefaultKeyEqual<Key>,+    typename Alloc = f14::DefaultAlloc<Key>>+class F14VectorSet;++template <+    typename Key,+    typename Hasher = f14::DefaultHasher<Key>,+    typename KeyEqual = f14::DefaultKeyEqual<Key>,+    typename Alloc = f14::DefaultAlloc<Key>>+class F14FastSet;++#if FOLLY_HAS_MEMORY_RESOURCE+namespace pmr {+template <+    typename Key,+    typename Hasher = f14::DefaultHasher<Key>,+    typename KeyEqual = f14::DefaultKeyEqual<Key>>+using F14NodeSet = folly::+    F14NodeSet<Key, Hasher, KeyEqual, std::pmr::polymorphic_allocator<Key>>;++template <+    typename Key,+    typename Hasher = f14::DefaultHasher<Key>,+    typename KeyEqual = f14::DefaultKeyEqual<Key>>+using F14ValueSet = folly::+    F14ValueSet<Key, Hasher, KeyEqual, std::pmr::polymorphic_allocator<Key>>;++template <+    typename Key,+    typename Hasher = f14::DefaultHasher<Key>,+    typename KeyEqual = f14::DefaultKeyEqual<Key>>+using F14VectorSet = folly::+    F14VectorSet<Key, Hasher, KeyEqual, std::pmr::polymorphic_allocator<Key>>;++template <+    typename Key,+    typename Hasher = f14::DefaultHasher<Key>,+    typename KeyEqual = f14::DefaultKeyEqual<Key>>+using F14FastSet = folly::+    F14FastSet<Key, Hasher, KeyEqual, std::pmr::polymorphic_allocator<Key>>;+} // namespace pmr+#endif // FOLLY_HAS_MEMORY_RESOURCE++} // namespace folly
+ folly/folly/container/F14Set.h view
@@ -0,0 +1,1588 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++//+// Docs: https://fburl.com/fbcref_f14nodeset+//++#pragma once++/**+ * F14NodeSet, F14ValueSet, and F14VectorSet+ *+ * F14FastSet conditionally works like F14ValueSet or F14VectorSet+ *+ * See F14.md+ */++#include <cstddef>+#include <initializer_list>+#include <tuple>++#include <folly/Portability.h>+#include <folly/container/View.h>+#include <folly/lang/SafeAssert.h>++#include <folly/container/F14Set-fwd.h>+#include <folly/container/Iterator.h>+#include <folly/container/detail/F14Policy.h>+#include <folly/container/detail/F14SetFallback.h>+#include <folly/container/detail/F14Table.h>+#include <folly/container/detail/Util.h>++namespace folly {++#if FOLLY_F14_VECTOR_INTRINSICS_AVAILABLE++//////// Common case for supported platforms++namespace f14 {+namespace detail {++template <typename Policy>+class F14BasicSet {+  template <typename K, typename T>+  using EnableHeterogeneousFind = std::enable_if_t<+      ::folly::detail::EligibleForHeterogeneousFind<+          typename Policy::Value,+          typename Policy::Hasher,+          typename Policy::KeyEqual,+          K>::value,+      T>;++  template <typename K, typename T>+  using EnableHeterogeneousInsert = std::enable_if_t<+      ::folly::detail::EligibleForHeterogeneousInsert<+          typename Policy::Value,+          typename Policy::Hasher,+          typename Policy::KeyEqual,+          K>::value,+      T>;++  template <typename K>+  using IsIter = std::is_same<typename Policy::Iter, remove_cvref_t<K>>;++  template <typename K, typename T>+  using EnableHeterogeneousErase = std::enable_if_t<+      ::folly::detail::EligibleForHeterogeneousFind<+          typename Policy::Value,+          typename Policy::Hasher,+          typename Policy::KeyEqual,+          std::conditional_t<IsIter<K>::value, typename Policy::Value, K>>::+              value &&+          !IsIter<K>::value,+      T>;++ public:+  //// PUBLIC - Member types++  using key_type = typename Policy::Value;+  using value_type = key_type;+  using size_type = std::size_t;+  using difference_type = std::ptrdiff_t;+  using hash_token_type = F14HashToken;+  using hasher = typename Policy::Hasher;+  using key_equal = typename Policy::KeyEqual;+  using hashed_key_type = F14HashedKey<key_type, hasher, key_equal>;+  using allocator_type = typename Policy::Alloc;+  using reference = value_type&;+  using const_reference = value_type const&;+  using pointer = typename Policy::AllocTraits::pointer;+  using const_pointer = typename Policy::AllocTraits::const_pointer;+  using iterator = typename Policy::Iter;+  using const_iterator = iterator;++ private:+  using ItemIter = typename Policy::ItemIter;++ public:+  //// PUBLIC - Member functions++  F14BasicSet() noexcept(Policy::kDefaultConstructIsNoexcept) : table_{} {}++  explicit F14BasicSet(+      std::size_t initialCapacity,+      hasher const& hash = hasher{},+      key_equal const& eq = key_equal{},+      allocator_type const& alloc = allocator_type{})+      : table_{initialCapacity, hash, eq, alloc} {}++  explicit F14BasicSet(std::size_t initialCapacity, allocator_type const& alloc)+      : F14BasicSet(initialCapacity, hasher{}, key_equal{}, alloc) {}++  explicit F14BasicSet(+      std::size_t initialCapacity,+      hasher const& hash,+      allocator_type const& alloc)+      : F14BasicSet(initialCapacity, hash, key_equal{}, alloc) {}++  explicit F14BasicSet(allocator_type const& alloc)+      : F14BasicSet(0, hasher{}, key_equal{}, alloc) {}++  template <typename InputIt>+  F14BasicSet(+      InputIt first,+      InputIt last,+      std::size_t initialCapacity = 0,+      hasher const& hash = hasher{},+      key_equal const& eq = key_equal{},+      allocator_type const& alloc = allocator_type{})+      : table_{initialCapacity, hash, eq, alloc} {+    initialInsert(first, last, initialCapacity);+  }++  template <typename InputIt>+  F14BasicSet(+      InputIt first,+      InputIt last,+      std::size_t initialCapacity,+      allocator_type const& alloc)+      : table_{initialCapacity, hasher{}, key_equal{}, alloc} {+    initialInsert(first, last, initialCapacity);+  }++  template <typename InputIt>+  F14BasicSet(+      InputIt first,+      InputIt last,+      std::size_t initialCapacity,+      hasher const& hash,+      allocator_type const& alloc)+      : table_{initialCapacity, hash, key_equal{}, alloc} {+    initialInsert(first, last, initialCapacity);+  }++  F14BasicSet(F14BasicSet const& rhs) = default;++  F14BasicSet(F14BasicSet const& rhs, allocator_type const& alloc)+      : table_(rhs.table_, alloc) {}++  F14BasicSet(F14BasicSet&& rhs) = default;++  F14BasicSet(F14BasicSet&& rhs, allocator_type const& alloc) noexcept(+      Policy::kAllocIsAlwaysEqual)+      : table_{std::move(rhs.table_), alloc} {}++  /* implicit */ F14BasicSet(+      std::initializer_list<value_type> init,+      std::size_t initialCapacity = 0,+      hasher const& hash = hasher{},+      key_equal const& eq = key_equal{},+      allocator_type const& alloc = allocator_type{})+      : table_{initialCapacity, hash, eq, alloc} {+    initialInsert(init.begin(), init.end(), initialCapacity);+  }++  F14BasicSet(+      std::initializer_list<value_type> init,+      std::size_t initialCapacity,+      allocator_type const& alloc)+      : table_{initialCapacity, hasher{}, key_equal{}, alloc} {+    initialInsert(init.begin(), init.end(), initialCapacity);+  }++  F14BasicSet(+      std::initializer_list<value_type> init,+      std::size_t initialCapacity,+      hasher const& hash,+      allocator_type const& alloc)+      : table_{initialCapacity, hash, key_equal{}, alloc} {+    initialInsert(init.begin(), init.end(), initialCapacity);+  }++  F14BasicSet& operator=(F14BasicSet const&) = default;++  F14BasicSet& operator=(F14BasicSet&&) = default;++  F14BasicSet& operator=(std::initializer_list<value_type> ilist) {+    clear();+    bulkInsert(ilist.begin(), ilist.end(), true);+    return *this;+  }++  /// Get the allocator for this container.+  allocator_type get_allocator() const noexcept { return table_.alloc(); }++  //// PUBLIC - Iterators++  /// @methodset Iterators+  iterator begin() noexcept { return cbegin(); }+  const_iterator begin() const noexcept { return cbegin(); }+  /// @methodset Iterators+  const_iterator cbegin() const noexcept {+    return table_.makeIter(table_.begin());+  }++  /// @methodset Iterators+  iterator end() noexcept { return cend(); }+  const_iterator end() const noexcept { return cend(); }+  /// @methodset Iterators+  const_iterator cend() const noexcept { return table_.makeIter(table_.end()); }++  //// PUBLIC - Capacity++  /**+   * Check if this container has any elements.+   * @methodset Capacity+   */+  bool empty() const noexcept { return table_.empty(); }++  /**+   * Number of elements in this container.+   * @methodset Capacity+   */+  std::size_t size() const noexcept { return table_.size(); }++  /**+   * The maximum size of this container.+   * @methodset Capacity+   */+  std::size_t max_size() const noexcept { return table_.max_size(); }++  //// PUBLIC - Modifiers++  /**+   * Remove all elements.+   *+   * Frees heap-allocated memory; bucket_count is returned to 0.+   *+   * @methodset Modifiers+   */+  void clear() noexcept { table_.clear(); }++  /**+   * Add a single element.+   *+   * @overloadbrief Add elements.+   * @methodset Modifiers+   */+  std::pair<iterator, bool> insert(value_type const& value) {+    return emplace(value);+  }++  /// Add a single element.+  std::pair<iterator, bool> insert(value_type&& value) {+    return emplace(std::move(value));+  }++  /**+   * Add a single element, with a locational hint.+   *+   * std::unordered_set's hinted insertion API is misleading.  No implementation+   * I've seen actually uses the hint.  Code restructuring by the caller to use+   * the hinted API is at best unnecessary, and at worst a pessimization.  It is+   * used, however, so we provide it.+   */+  iterator insert(const_iterator /*hint*/, value_type const& value) {+    return insert(value).first;+  }++  /// Add a single element, with a locational hint.+  iterator insert(const_iterator /*hint*/, value_type&& value) {+    return insert(std::move(value)).first;+  }++  /// Add a single element, of a heterognous type.+  template <typename K>+  EnableHeterogeneousInsert<K, std::pair<iterator, bool>> insert(K&& value) {+    return emplace(std::forward<K>(value));+  }++ private:+  template <class InputIt>+  FOLLY_ALWAYS_INLINE void bulkInsert(+      InputIt first, InputIt last, bool autoReserve) {+    if (autoReserve) {+      auto n = std::distance(first, last);+      if (n == 0) {+        return;+      }+      table_.reserveForInsert(n);+    }+    while (first != last) {+      insert(*first);+      ++first;+    }+  }++  template <class InputIt>+  void initialInsert(InputIt first, InputIt last, std::size_t initialCapacity) {+    FOLLY_SAFE_DCHECK(empty() && bucket_count() >= initialCapacity, "");++    // It's possible that there are a lot of duplicates in first..last and+    // so we will oversize ourself.  The common case, however, is that+    // we can avoid a lot of rehashing if we pre-expand.  The behavior+    // is easy to disable at a particular call site by asking for an+    // initialCapacity of 1.+    bool autoReserve =+        std::is_base_of<+            std::random_access_iterator_tag,+            typename std::iterator_traits<InputIt>::iterator_category>::value &&+        initialCapacity == 0;+    bulkInsert(first, last, autoReserve);+  }++ public:+  /// Add a range of elements.+  template <class InputIt>+  void insert(InputIt first, InputIt last) {+    // Bulk reserve is a heuristic choice, so it can backfire.  We restrict+    // ourself to situations that mimic bulk construction without an+    // explicit initialCapacity.+    bool autoReserve =+        std::is_base_of<+            std::random_access_iterator_tag,+            typename std::iterator_traits<InputIt>::iterator_category>::value &&+        bucket_count() == 0;+    bulkInsert(first, last, autoReserve);+  }++  /// Add elements from an initializer list.+  void insert(std::initializer_list<value_type> ilist) {+    insert(ilist.begin(), ilist.end());+  }++ private:+  template <typename Arg>+  using UsableAsKey = ::folly::detail::+      EligibleForHeterogeneousFind<key_type, hasher, key_equal, Arg>;++ public:+  /**+   * Add an element by constructing it in-place.+   *+   * @overloadbrief Add elements in-place.+   * @methodset Modifiers+   */+  template <class... Args>+  std::pair<iterator, bool> emplace(Args&&... args) {+    auto rv = folly::detail::callWithConstructedKey<key_type, UsableAsKey>(+        table_.alloc(),+        [&](auto&&... inner) {+          return table_.tryEmplaceValue(+              std::forward<decltype(inner)>(inner)...);+        },+        std::forward<Args>(args)...);+    return std::make_pair(table_.makeIter(rv.first), rv.second);+  }++  // Emplace with prehash token+  template <class... Args>+  std::pair<iterator, bool> emplace_token(+      F14HashToken const& token, Args&&... args) {+    auto rv = folly::detail::callWithConstructedKey<key_type, UsableAsKey>(+        table_.alloc(),+        [&](auto&&... inner) {+          return table_.tryEmplaceValueWithToken(+              token, std::forward<decltype(inner)>(inner)...);+        },+        std::forward<Args>(args)...);+    return std::make_pair(table_.makeIter(rv.first), rv.second);+  }++  /// Emplace with hint.+  template <class... Args>+  iterator emplace_hint(const_iterator /*hint*/, Args&&... args) {+    return emplace(std::forward<Args>(args)...).first;+  }++  /**+   * Remove element at a specific position (iterator).+   * @overloadbrief Remove elements.+   * @methodset Modifiers+   */+  FOLLY_ALWAYS_INLINE iterator erase(const_iterator pos) {+    return eraseInto(pos, variadic_noop);+  }++  /// Remove a range of elements.+  iterator erase(const_iterator first, const_iterator last) {+    return eraseInto(first, last, variadic_noop);+  }++  /// Remove a specific key.+  size_type erase(key_type const& key) { return eraseInto(key, variadic_noop); }++  /// Remove a key, using a heterogeneous representation.+  template <typename K>+  EnableHeterogeneousErase<K, size_type> erase(K const& key) {+    return eraseInto(key, variadic_noop);+  }++  /**+   * Callback-erase a single iterator.+   *+   * Like erase, but with an additional callback argument which is called with+   * an rvalue reference to the item directly before it is destroyed. This can+   * be used to extract an item out of a F14Set while avoiding a copy.+   *+   * @overloadbrief Erase with pre-destruction callback.+   * @methodset Modifiers+   */+  template <typename BeforeDestroy>+  FOLLY_ALWAYS_INLINE iterator+  eraseInto(const_iterator pos, BeforeDestroy&& beforeDestroy) {+    auto itemPos = table_.unwrapIter(pos);+    table_.eraseIterInto(itemPos, beforeDestroy);++    // If we are inlined then gcc and clang can optimize away all of the+    // work of ++pos if the caller discards it.+    itemPos.advanceLikelyDead();+    return table_.makeIter(itemPos);+  }++  /// Callback-erase a range of values.+  template <typename BeforeDestroy>+  iterator eraseInto(+      const_iterator first,+      const_iterator last,+      BeforeDestroy&& beforeDestroy) {+    while (first != last) {+      first = eraseInto(first, beforeDestroy);+    }+    return first;+  }++  /// Callback-erase a specific key.+  template <typename BeforeDestroy>+  size_type eraseInto(key_type const& key, BeforeDestroy&& beforeDestroy) {+    return table_.eraseKeyInto(key, beforeDestroy);+  }++  /// Callback-erase a specific key, using a heterogeneous representation.+  template <typename K, typename BeforeDestroy>+  EnableHeterogeneousErase<K, size_type> eraseInto(+      K const& key, BeforeDestroy&& beforeDestroy) {+    return table_.eraseKeyInto(key, beforeDestroy);+  }++  //// PUBLIC - Lookup++  /**+   * Number of elements matching the given key.+   * @methodset Lookup+   */+  FOLLY_ALWAYS_INLINE size_type count(key_type const& key) const {+    return contains(key) ? 1 : 0;+  }++  template <typename K>+  FOLLY_ALWAYS_INLINE EnableHeterogeneousFind<K, size_type> count(+      K const& key) const {+    return contains(key) ? 1 : 0;+  }++  /**+   * @overloadbrief Prehash a key.+   * @methodset Lookup+   *+   * prehash(key) does the work of evaluating hash_function()(key)+   * (including additional bit-mixing for non-avalanching hash functions),+   * and wraps the result of that work in a token for later reuse.+   *+   * The returned token may be used at any time, may be used more than+   * once, and may be used in other F14 sets and maps.  Tokens are+   * transferrable between any F14 containers (maps and sets) with the+   * same key_type and equal hash_function()s.+   *+   * Hash tokens are not hints -- it is a bug to call any method on this+   * class with a token t and key k where t isn't the result of a call+   * to prehash(k2) with k2 == k.+   */+  F14HashToken prehash(key_type const& key) const {+    return table_.prehash(key);+  }+  /// @copydoc prehash+  F14HashToken prehash(key_type const& key, std::size_t hash) const {+    return table_.prehash(key, hash);+  }++  /// @copydoc prehash+  template <typename K>+  EnableHeterogeneousFind<K, F14HashToken> prehash(K const& key) const {+    return table_.prehash(key);+  }+  /// @copydoc prehash+  template <typename K>+  EnableHeterogeneousFind<K, F14HashToken> prehash(+      K const& key, std::size_t hash) const {+    return table_.prehash(key, hash);+  }++  /**+   * @overloadbrief Prefetch cachelines associated with a key.+   * @methodset Lookup+   *+   * prefetch(token) begins prefetching the first steps of looking for key into+   * the local CPU cache.+   *+   * Example Scenario: Loading 2 values from a cold map.+   * You have a map that is cold, meaning it is out of the local CPU cache,+   * and you want to load two values from the map. This can be extended to+   * load N values, but we're loading 2 for simplicity.+   *+   * When the map is cold the dominating factor in the latency is loading the+   * cache line of the entry into the local CPU cache. Using prehash() will+   * issue these cache line fetches in parallel.  That means that by the time we+   * finish map.find(token1, key1) the cache lines needed by map.find(token2,+   * key2) may already be in the local CPU cache. In the best case this will+   * half the latency.+   *+   * It is always okay to call prefetch() before a find() or other lookup+   * operation, as it only prefetches cache lines that are guaranteed to be+   * needed by the lookup.+   *+   *   std::pair<iterator, iterator> find2(+   *       auto& set, key_type const& key1, key_type const& key2) {+   *     auto const token1 = set.prehash(key1);+   *     set.prefetch(token1);+   *     auto const token2 = set.prehash(key2);+   *     set.prefetch(token2);+   *     return std::make_pair(set.find(token1, key1), set.find(token2, key2));+   *   }+   */+  void prefetch(F14HashToken const& token) const { table_.prefetch(token); }++  /**+   * @overloadbrief Get the iterator for a key.+   * @methodset Lookup+   */+  FOLLY_ALWAYS_INLINE iterator find(key_type const& key) {+    return const_cast<F14BasicSet const*>(this)->find(key);+  }++  FOLLY_ALWAYS_INLINE const_iterator find(key_type const& key) const {+    return table_.makeIter(table_.find(key));+  }++  FOLLY_ALWAYS_INLINE iterator+  find(F14HashToken const& token, key_type const& key) {+    return const_cast<F14BasicSet const*>(this)->find(token, key);+  }++  FOLLY_ALWAYS_INLINE const_iterator+  find(F14HashToken const& token, key_type const& key) const {+    return table_.makeIter(table_.find(token, key));+  }++  template <typename K>+  FOLLY_ALWAYS_INLINE EnableHeterogeneousFind<K, iterator> find(K const& key) {+    return const_cast<F14BasicSet const*>(this)->find(key);+  }++  template <typename K>+  FOLLY_ALWAYS_INLINE EnableHeterogeneousFind<K, const_iterator> find(+      K const& key) const {+    return table_.makeIter(table_.find(key));+  }++  template <typename K>+  FOLLY_ALWAYS_INLINE EnableHeterogeneousFind<K, iterator> find(+      F14HashToken const& token, K const& key) {+    return const_cast<F14BasicSet const*>(this)->find(token, key);+  }++  template <typename K>+  FOLLY_ALWAYS_INLINE EnableHeterogeneousFind<K, const_iterator> find(+      F14HashToken const& token, K const& key) const {+    return table_.makeIter(table_.find(token, key));+  }++  /**+   * @overloadbrief Checks if the container contains an element with the+   * specific key.+   * @methodset Lookup+   */+  FOLLY_ALWAYS_INLINE bool contains(key_type const& key) const {+    return !table_.find(key).atEnd();+  }++  template <typename K>+  FOLLY_ALWAYS_INLINE EnableHeterogeneousFind<K, bool> contains(+      K const& key) const {+    return !table_.find(key).atEnd();+  }++  FOLLY_ALWAYS_INLINE bool contains(+      F14HashToken const& token, key_type const& key) const {+    return !table_.find(token, key).atEnd();+  }++  template <typename K>+  FOLLY_ALWAYS_INLINE EnableHeterogeneousFind<K, bool> contains(+      F14HashToken const& token, K const& key) const {+    return !table_.find(token, key).atEnd();+  }++  /**+   * @overloadbrief Returns the range of elements matching a specific key.+   * @methodset Lookup+   */+  std::pair<iterator, iterator> equal_range(key_type const& key) {+    return equal_range(*this, key);+  }++  std::pair<const_iterator, const_iterator> equal_range(+      key_type const& key) const {+    return equal_range(*this, key);+  }++  template <typename K>+  EnableHeterogeneousFind<K, std::pair<iterator, iterator>> equal_range(+      K const& key) {+    return equal_range(*this, key);+  }++  template <typename K>+  EnableHeterogeneousFind<K, std::pair<const_iterator, const_iterator>>+  equal_range(K const& key) const {+    return equal_range(*this, key);+  }++  //// PUBLIC - Bucket interface++  /**+   * The number of buckets in this container.+   * @methodset Bucket interface+   */+  std::size_t bucket_count() const noexcept { return table_.bucket_count(); }++  /**+   * The maximum number of buckets for this container.+   * @methodset Bucket interface+   */+  std::size_t max_bucket_count() const noexcept {+    return table_.max_bucket_count();+  }++  //// PUBLIC - Hash policy++  /**+   * Load factor of the underlying hashtable.+   * @methodset Hash policy+   */+  float load_factor() const noexcept { return table_.load_factor(); }++  /**+   * @overloadbrief Load factor control.+   * Get the maximum load factor for this container.+   * @methodset Hash policy+   */+  float max_load_factor() const noexcept { return table_.max_load_factor(); }++  /**+   * Set the maximum load factor for this container.+   * @methodset Hash policy+   */+  void max_load_factor(float v) { table_.max_load_factor(v); }++  /**+   * Rehash this container.+   *+   * This function is provided for compliance with C++'s requirements for+   * hashtables, but is no better than a simple `reserve` call for F14.+   *+   * @param bucketCapacity  The desired capacity across all buckets.+   *+   * @methodset Hash policy+   */+  void rehash(std::size_t bucketCapacity) {+    // The standard's rehash() requires understanding the max load factor,+    // which is easy to get wrong.  Since we don't actually allow adjustment+    // of max_load_factor there is no difference.+    reserve(bucketCapacity);+  }++  /**+   * Pre-allocate space for at least this many elements.+   *+   * @param capacity  The number of elements to pre-allocate space for.+   *+   * @methodset Capacity+   */+  void reserve(std::size_t capacity) { table_.reserve(capacity); }++  //// PUBLIC - Observers++  /**+   * Get the hasher.+   * @methodset Observers+   */+  hasher hash_function() const { return table_.hasher(); }++  /**+   * Get the key_equal.+   * @methodset Observers+   */+  key_equal key_eq() const { return table_.keyEqual(); }++  //// PUBLIC - F14 Extensions++  /**+   * Checks for a value using operator==+   *+   * returns true iff there is an element in the set+   * that compares equal to key using operator==.  It is undefined+   * behavior to call this function if operator== on key_type can ever+   * return true when the same keys passed to key_eq() would return false+   * (the opposite is allowed).  When using the default key_eq this function+   * is equivalent to contains().+   *+   * @methodset Lookup+   */+  bool containsEqualValue(value_type const& value) const {+    return !table_.findMatching(value, [&](auto& k) { return value == k; })+                .atEnd();+  }++  /**+   * Get memory footprint, not including sizeof(*this).+   * @methodset Capacity+   */+  std::size_t getAllocatedMemorySize() const {+    return table_.getAllocatedMemorySize();+  }++  /**+   * In-depth memory analysis.+   *+   * Enumerates classes of allocated memory blocks currently owned+   * by this table, calling visitor(allocationSize, allocationCount).+   * This can be used to get a more accurate indication of memory footprint+   * than getAllocatedMemorySize() if you have some way of computing the+   * internal fragmentation of the allocator, such as JEMalloc's nallocx.+   * The visitor might be called twice with the same allocationSize. The+   * visitor's computation should produce the same result for visitor(8,+   * 2) as for two calls to visitor(8, 1), for example.  The visitor may+   * be called with a zero allocationCount.+   *+   * @methodset Capacity+   */+  template <typename V>+  void visitAllocationClasses(V&& visitor) const {+    return table_.visitAllocationClasses(visitor);+  }++  /**+   * Visit contiguous ranges of elements.+   *+   * Calls visitor with two value_type const*, b and e, such that every+   * entry in the table is included in exactly one of the ranges [b,e).+   * This can be used to efficiently iterate elements in bulk when crossing+   * an API boundary that supports contiguous blocks of items.+   *+   * @methodset Iterators+   */+  template <typename V>+  void visitContiguousRanges(V&& visitor) const;++  /**+   * Get stats.+   * @methodset Hash policy+   */+  F14TableStats computeStats() const noexcept { return table_.computeStats(); }++ private:+  template <typename Self, typename K>+  static auto equal_range(Self& self, K const& key) {+    auto first = self.find(key);+    auto last = first;+    if (last != self.end()) {+      ++last;+    }+    return std::make_pair(first, last);+  }++ protected:+  F14Table<Policy> table_;+};+} // namespace detail+} // namespace f14++template <typename Key, typename Hasher, typename KeyEqual, typename Alloc>+class F14ValueSet+    : public f14::detail::F14BasicSet<f14::detail::SetPolicyWithDefaults<+          f14::detail::ValueContainerPolicy,+          Key,+          Hasher,+          KeyEqual,+          Alloc>> {+ protected:+  friend struct F14ValueSetTester;+  using Policy = f14::detail::SetPolicyWithDefaults<+      f14::detail::ValueContainerPolicy,+      Key,+      Hasher,+      KeyEqual,+      Alloc>;++ private:+  using Super = f14::detail::F14BasicSet<Policy>;++ public:+  using typename Super::value_type;++  F14ValueSet() = default;++  using Super::Super;++  F14ValueSet& operator=(std::initializer_list<value_type> ilist) {+    Super::operator=(ilist);+    return *this;+  }++  void swap(F14ValueSet& rhs) noexcept(Policy::kSwapIsNoexcept) {+    this->table_.swap(rhs.table_);+  }++  template <typename V>+  void visitContiguousRanges(V&& visitor) const {+    this->table_.visitContiguousItemRanges(std::forward<V>(visitor));+  }+};+#endif // FOLLY_F14_VECTOR_INTRINSICS_AVAILABLE++template <+    typename InputIt,+    typename Hasher = f14::DefaultHasher<iterator_value_type_t<InputIt>>,+    typename KeyEqual = f14::DefaultKeyEqual<iterator_value_type_t<InputIt>>,+    typename Alloc = f14::DefaultAlloc<iterator_value_type_t<InputIt>>,+    typename = detail::RequireInputIterator<InputIt>,+    typename = detail::RequireNotAllocator<Hasher>,+    typename = detail::RequireNotAllocator<KeyEqual>,+    typename = detail::RequireAllocator<Alloc>>+F14ValueSet(+    InputIt, InputIt, std::size_t = {}, Hasher = {}, KeyEqual = {}, Alloc = {})+    -> F14ValueSet<iterator_value_type_t<InputIt>, Hasher, KeyEqual, Alloc>;++template <+    typename InputIt,+    typename Alloc,+    typename = detail::RequireInputIterator<InputIt>,+    typename = detail::RequireAllocator<Alloc>>+F14ValueSet(InputIt, InputIt, std::size_t, Alloc)+    -> F14ValueSet<+        iterator_value_type_t<InputIt>,+        f14::DefaultHasher<iterator_value_type_t<InputIt>>,+        f14::DefaultKeyEqual<iterator_value_type_t<InputIt>>,+        Alloc>;++template <+    typename InputIt,+    typename Hasher,+    typename Alloc,+    typename = detail::RequireInputIterator<InputIt>,+    typename = detail::RequireNotAllocator<Hasher>,+    typename = detail::RequireAllocator<Alloc>>+F14ValueSet(InputIt, InputIt, std::size_t, Hasher, Alloc)+    -> F14ValueSet<+        iterator_value_type_t<InputIt>,+        Hasher,+        f14::DefaultKeyEqual<iterator_value_type_t<InputIt>>,+        Alloc>;++template <+    typename Key,+    typename Hasher = f14::DefaultHasher<Key>,+    typename KeyEqual = f14::DefaultKeyEqual<Key>,+    typename Alloc = f14::DefaultAlloc<Key>,+    typename = detail::RequireNotAllocator<Hasher>,+    typename = detail::RequireNotAllocator<KeyEqual>,+    typename = detail::RequireAllocator<Alloc>>+F14ValueSet(+    std::initializer_list<Key>,+    std::size_t = {},+    Hasher = {},+    KeyEqual = {},+    Alloc = {}) -> F14ValueSet<Key, Hasher, KeyEqual, Alloc>;++template <+    typename Key,+    typename Alloc,+    typename = detail::RequireAllocator<Alloc>>+F14ValueSet(std::initializer_list<Key>, std::size_t, Alloc)+    -> F14ValueSet<+        Key,+        f14::DefaultHasher<Key>,+        f14::DefaultKeyEqual<Key>,+        Alloc>;++template <+    typename Key,+    typename Hasher,+    typename Alloc,+    typename = detail::RequireAllocator<Alloc>>+F14ValueSet(std::initializer_list<Key>, std::size_t, Hasher, Alloc)+    -> F14ValueSet<Key, Hasher, f14::DefaultKeyEqual<Key>, Alloc>;++#if FOLLY_F14_VECTOR_INTRINSICS_AVAILABLE+template <typename Key, typename Hasher, typename KeyEqual, typename Alloc>+class F14NodeSet+    : public f14::detail::F14BasicSet<f14::detail::SetPolicyWithDefaults<+          f14::detail::NodeContainerPolicy,+          Key,+          Hasher,+          KeyEqual,+          Alloc>> {+ protected:+  using Policy = f14::detail::SetPolicyWithDefaults<+      f14::detail::NodeContainerPolicy,+      Key,+      Hasher,+      KeyEqual,+      Alloc>;++ private:+  using Super = f14::detail::F14BasicSet<Policy>;++ public:+  using typename Super::value_type;++  F14NodeSet() = default;++  using Super::Super;++  F14NodeSet& operator=(std::initializer_list<value_type> ilist) {+    Super::operator=(ilist);+    return *this;+  }++  void swap(F14NodeSet& rhs) noexcept(Policy::kSwapIsNoexcept) {+    this->table_.swap(rhs.table_);+  }++  template <typename V>+  void visitContiguousRanges(V&& visitor) const {+    this->table_.visitItems([&](typename Policy::Item ptr) {+      value_type const* b = std::addressof(*ptr);+      visitor(b, b + 1);+    });+  }+};+#endif // FOLLY_F14_VECTOR_INTRINSICS_AVAILABLE++template <+    typename InputIt,+    typename Hasher = f14::DefaultHasher<iterator_value_type_t<InputIt>>,+    typename KeyEqual = f14::DefaultKeyEqual<iterator_value_type_t<InputIt>>,+    typename Alloc = f14::DefaultAlloc<iterator_value_type_t<InputIt>>,+    typename = detail::RequireInputIterator<InputIt>,+    typename = detail::RequireNotAllocator<Hasher>,+    typename = detail::RequireNotAllocator<KeyEqual>,+    typename = detail::RequireAllocator<Alloc>>+F14NodeSet(+    InputIt, InputIt, std::size_t = {}, Hasher = {}, KeyEqual = {}, Alloc = {})+    -> F14NodeSet<iterator_value_type_t<InputIt>, Hasher, KeyEqual, Alloc>;++template <+    typename InputIt,+    typename Alloc,+    typename = detail::RequireInputIterator<InputIt>,+    typename = detail::RequireAllocator<Alloc>>+F14NodeSet(InputIt, InputIt, std::size_t, Alloc)+    -> F14NodeSet<+        iterator_value_type_t<InputIt>,+        f14::DefaultHasher<iterator_value_type_t<InputIt>>,+        f14::DefaultKeyEqual<iterator_value_type_t<InputIt>>,+        Alloc>;++template <+    typename InputIt,+    typename Hasher,+    typename Alloc,+    typename = detail::RequireInputIterator<InputIt>,+    typename = detail::RequireNotAllocator<Hasher>,+    typename = detail::RequireAllocator<Alloc>>+F14NodeSet(InputIt, InputIt, std::size_t, Hasher, Alloc)+    -> F14NodeSet<+        iterator_value_type_t<InputIt>,+        Hasher,+        f14::DefaultKeyEqual<iterator_value_type_t<InputIt>>,+        Alloc>;++template <+    typename Key,+    typename Hasher = f14::DefaultHasher<Key>,+    typename KeyEqual = f14::DefaultKeyEqual<Key>,+    typename Alloc = f14::DefaultAlloc<Key>,+    typename = detail::RequireNotAllocator<Hasher>,+    typename = detail::RequireNotAllocator<KeyEqual>,+    typename = detail::RequireAllocator<Alloc>>+F14NodeSet(+    std::initializer_list<Key>,+    std::size_t = {},+    Hasher = {},+    KeyEqual = {},+    Alloc = {}) -> F14NodeSet<Key, Hasher, KeyEqual, Alloc>;++template <+    typename Key,+    typename Alloc,+    typename = detail::RequireAllocator<Alloc>>+F14NodeSet(std::initializer_list<Key>, std::size_t, Alloc)+    -> F14NodeSet<+        Key,+        f14::DefaultHasher<Key>,+        f14::DefaultKeyEqual<Key>,+        Alloc>;++template <+    typename Key,+    typename Hasher,+    typename Alloc,+    typename = detail::RequireAllocator<Alloc>>+F14NodeSet(std::initializer_list<Key>, std::size_t, Hasher, Alloc)+    -> F14NodeSet<Key, Hasher, f14::DefaultKeyEqual<Key>, Alloc>;++#if FOLLY_F14_VECTOR_INTRINSICS_AVAILABLE+namespace f14 {+namespace detail {+template <+    typename Key,+    typename Hasher,+    typename KeyEqual,+    typename Alloc,+    typename EligibleForPerturbedInsertionOrder>+class F14VectorSetImpl+    : public F14BasicSet<SetPolicyWithDefaults<+          VectorContainerPolicy,+          Key,+          Hasher,+          KeyEqual,+          Alloc,+          EligibleForPerturbedInsertionOrder>> {+ protected:+  using Policy = SetPolicyWithDefaults<+      VectorContainerPolicy,+      Key,+      Hasher,+      KeyEqual,+      Alloc,+      EligibleForPerturbedInsertionOrder>;++ private:+  using Super = F14BasicSet<Policy>;++  template <typename K>+  using IsIter = Disjunction<+      std::is_same<typename Policy::Iter, remove_cvref_t<K>>,+      std::is_same<typename Policy::ReverseIter, remove_cvref_t<K>>>;++  template <typename K, typename T>+  using EnableHeterogeneousVectorErase = std::enable_if_t<+      ::folly::detail::EligibleForHeterogeneousFind<+          typename Policy::Value,+          typename Policy::Hasher,+          typename Policy::KeyEqual,+          std::conditional_t<IsIter<K>::value, typename Policy::Value, K>>::+              value &&+          !IsIter<K>::value,+      T>;++ public:+  using typename Super::const_iterator;+  using typename Super::iterator;+  using typename Super::key_type;+  using typename Super::value_type;++  F14VectorSetImpl() = default;++  using Super::Super;++  F14VectorSetImpl& operator=(std::initializer_list<value_type> ilist) {+    Super::operator=(ilist);+    return *this;+  }++  iterator begin() { return cbegin(); }+  const_iterator begin() const { return cbegin(); }+  const_iterator cbegin() const {+    return this->table_.linearBegin(this->size());+  }++  iterator end() { return cend(); }+  const_iterator end() const { return cend(); }+  const_iterator cend() const { return this->table_.linearEnd(); }++ private:+  template <typename BeforeDestroy>+  void eraseUnderlying(+      typename Policy::ItemIter underlying, BeforeDestroy&& beforeDestroy) {+    Alloc& a = this->table_.alloc();+    auto values = this->table_.values_;++    // destroy the value and remove the ptr from the base table+    auto index = underlying.item();+    this->table_.eraseIterInto(underlying, beforeDestroy);+    Policy::AllocTraits::destroy(a, std::addressof(values[index]));++    // move the last element in values_ down and fix up the inbound index+    auto tailIndex = this->size();+    if (tailIndex != index) {+      auto tail = this->table_.find(+          VectorContainerIndexSearch{static_cast<uint32_t>(tailIndex)});+      tail.item() = index;+      auto p = std::addressof(values[index]);+      assume(p != nullptr);+      this->table_.transfer(a, std::addressof(values[tailIndex]), p, 1);+    }+  }++  template <typename K, typename BeforeDestroy>+  std::size_t eraseUnderlyingKey(K const& key, BeforeDestroy&& beforeDestroy) {+    auto underlying = this->table_.find(key);+    if (underlying.atEnd()) {+      return 0;+    } else {+      eraseUnderlying(underlying, beforeDestroy);+      return 1;+    }+  }++ public:+  FOLLY_ALWAYS_INLINE iterator erase(const_iterator pos) {+    return eraseInto(pos, variadic_noop);+  }++  iterator erase(const_iterator first, const_iterator last) {+    return eraseInto(first, last, variadic_noop);+  }++  std::size_t erase(key_type const& key) {+    return eraseInto(key, variadic_noop);+  }++  template <typename K>+  EnableHeterogeneousVectorErase<K, std::size_t> erase(K const& key) {+    return eraseInto(key, variadic_noop);+  }++  template <typename BeforeDestroy>+  FOLLY_ALWAYS_INLINE iterator+  eraseInto(const_iterator pos, BeforeDestroy&& beforeDestroy) {+    auto underlying = this->table_.find(+        VectorContainerIndexSearch{this->table_.iterToIndex(pos)});+    eraseUnderlying(underlying, beforeDestroy);+    return ++pos;+  }++  template <typename BeforeDestroy>+  iterator eraseInto(+      const_iterator first,+      const_iterator last,+      BeforeDestroy&& beforeDestroy) {+    while (first != last) {+      first = eraseInto(first, beforeDestroy);+    }+    return first;+  }++  template <typename BeforeDestroy>+  std::size_t eraseInto(key_type const& key, BeforeDestroy&& beforeDestroy) {+    return eraseUnderlyingKey(key, beforeDestroy);+  }++  template <typename K, typename BeforeDestroy>+  EnableHeterogeneousVectorErase<K, std::size_t> eraseInto(+      K const& key, BeforeDestroy&& beforeDestroy) {+    return eraseUnderlyingKey(key, beforeDestroy);+  }++  template <typename V>+  void visitContiguousRanges(V&& visitor) const {+    auto n = this->table_.size();+    if (n > 0) {+      value_type const* b = std::addressof(this->table_.values_[0]);+      visitor(b, b + n);+    }+  }+};+} // namespace detail+} // namespace f14++template <typename Key, typename Hasher, typename KeyEqual, typename Alloc>+class F14VectorSet+    : public f14::detail::+          F14VectorSetImpl<Key, Hasher, KeyEqual, Alloc, std::false_type> {+  using Super = f14::detail::+      F14VectorSetImpl<Key, Hasher, KeyEqual, Alloc, std::false_type>;++ public:+  using typename Super::const_iterator;+  using typename Super::iterator;+  using typename Super::value_type;+  using reverse_iterator = typename Super::Policy::ReverseIter;+  using const_reverse_iterator = reverse_iterator;++  F14VectorSet() = default;++  using Super::Super;++  F14VectorSet& operator=(std::initializer_list<value_type> ilist) {+    Super::operator=(ilist);+    return *this;+  }++  void swap(F14VectorSet& rhs) noexcept(Super::Policy::kSwapIsNoexcept) {+    this->table_.swap(rhs.table_);+  }++  // ITERATION ORDER+  //+  // Deterministic iteration order for insert-only workloads is part of+  // F14VectorSet's supported API: iterator is LIFO and reverse_iterator+  // is FIFO.+  //+  // If there have been no calls to erase() then iterator and+  // const_iterator enumerate entries in the opposite of insertion order.+  // begin()->first is the key most recently inserted.  reverse_iterator+  // and reverse_const_iterator, therefore, enumerate in LIFO (insertion)+  // order for insert-only workloads.  Deterministic iteration order is+  // only guaranteed if no keys were removed since the last time the+  // set was empty.  Iteration order is preserved across rehashes and+  // F14VectorSet copies and moves.+  //+  // iterator uses LIFO order so that erasing while iterating with begin()+  // and end() is safe using the erase(it++) idiom, which is supported+  // by std::set and std::unordered_set.  erase(iter) invalidates iter+  // and all iterators before iter in the non-reverse iteration order.+  // Every successful erase invalidates all reverse iterators.+  //+  // No erase is provided for reverse_iterator (AKA const_reverse_iterator)+  // to make it harder to shoot yourself in the foot by erasing while+  // reverse-iterating.  You can write that as set.erase(set.iter(riter))+  // if you need it.++  reverse_iterator rbegin() { return this->table_.values_; }+  const_reverse_iterator rbegin() const { return crbegin(); }+  const_reverse_iterator crbegin() const { return this->table_.values_; }++  reverse_iterator rend() { return this->table_.values_ + this->table_.size(); }+  const_reverse_iterator rend() const { return crend(); }+  const_reverse_iterator crend() const {+    return this->table_.values_ + this->table_.size();+  }++  // explicit conversions between iterator and reverse_iterator+  iterator iter(reverse_iterator riter) { return this->table_.iter(riter); }+  const_iterator iter(const_reverse_iterator riter) const {+    return this->table_.iter(riter);+  }++  reverse_iterator riter(iterator it) { return this->table_.riter(it); }+  const_reverse_iterator riter(const_iterator it) const {+    return this->table_.riter(it);+  }++  friend Range<const_reverse_iterator> tag_invoke(+      order_preserving_reinsertion_view_fn, F14VectorSet const& c) noexcept {+    return {c.rbegin(), c.rend()};+  }+};+#endif // FOLLY_F14_VECTOR_INTRINSICS_AVAILABLE++template <+    typename InputIt,+    typename Hasher = f14::DefaultHasher<iterator_value_type_t<InputIt>>,+    typename KeyEqual = f14::DefaultKeyEqual<iterator_value_type_t<InputIt>>,+    typename Alloc = f14::DefaultAlloc<iterator_value_type_t<InputIt>>,+    typename = detail::RequireInputIterator<InputIt>,+    typename = detail::RequireNotAllocator<Hasher>,+    typename = detail::RequireNotAllocator<KeyEqual>,+    typename = detail::RequireAllocator<Alloc>>+F14VectorSet(+    InputIt, InputIt, std::size_t = {}, Hasher = {}, KeyEqual = {}, Alloc = {})+    -> F14VectorSet<iterator_value_type_t<InputIt>, Hasher, KeyEqual, Alloc>;++template <+    typename InputIt,+    typename Alloc,+    typename = detail::RequireInputIterator<InputIt>,+    typename = detail::RequireAllocator<Alloc>>+F14VectorSet(InputIt, InputIt, std::size_t, Alloc)+    -> F14VectorSet<+        iterator_value_type_t<InputIt>,+        f14::DefaultHasher<iterator_value_type_t<InputIt>>,+        f14::DefaultKeyEqual<iterator_value_type_t<InputIt>>,+        Alloc>;++template <+    typename InputIt,+    typename Hasher,+    typename Alloc,+    typename = detail::RequireInputIterator<InputIt>,+    typename = detail::RequireNotAllocator<Hasher>,+    typename = detail::RequireAllocator<Alloc>>+F14VectorSet(InputIt, InputIt, std::size_t, Hasher, Alloc)+    -> F14VectorSet<+        iterator_value_type_t<InputIt>,+        Hasher,+        f14::DefaultKeyEqual<iterator_value_type_t<InputIt>>,+        Alloc>;++template <+    typename Key,+    typename Hasher = f14::DefaultHasher<Key>,+    typename KeyEqual = f14::DefaultKeyEqual<Key>,+    typename Alloc = f14::DefaultAlloc<Key>,+    typename = detail::RequireNotAllocator<Hasher>,+    typename = detail::RequireNotAllocator<KeyEqual>,+    typename = detail::RequireAllocator<Alloc>>+F14VectorSet(+    std::initializer_list<Key>,+    std::size_t = {},+    Hasher = {},+    KeyEqual = {},+    Alloc = {}) -> F14VectorSet<Key, Hasher, KeyEqual, Alloc>;++template <+    typename Key,+    typename Alloc,+    typename = detail::RequireAllocator<Alloc>>+F14VectorSet(std::initializer_list<Key>, std::size_t, Alloc)+    -> F14VectorSet<+        Key,+        f14::DefaultHasher<Key>,+        f14::DefaultKeyEqual<Key>,+        Alloc>;++template <+    typename Key,+    typename Hasher,+    typename Alloc,+    typename = detail::RequireAllocator<Alloc>>+F14VectorSet(std::initializer_list<Key>, std::size_t, Hasher, Alloc)+    -> F14VectorSet<Key, Hasher, f14::DefaultKeyEqual<Key>, Alloc>;++#if FOLLY_F14_VECTOR_INTRINSICS_AVAILABLE+template <typename Key, typename Hasher, typename KeyEqual, typename Alloc>+class F14FastSet+    : public std::conditional_t<+          sizeof(Key) < 24,+          F14ValueSet<Key, Hasher, KeyEqual, Alloc>,+          f14::detail::+              F14VectorSetImpl<Key, Hasher, KeyEqual, Alloc, std::true_type>> {+  using Super = std::conditional_t<+      sizeof(Key) < 24,+      F14ValueSet<Key, Hasher, KeyEqual, Alloc>,+      f14::detail::+          F14VectorSetImpl<Key, Hasher, KeyEqual, Alloc, std::true_type>>;++ public:+  using typename Super::value_type;++  F14FastSet() = default;++  using Super::Super;++  F14FastSet& operator=(std::initializer_list<value_type> ilist) {+    Super::operator=(ilist);+    return *this;+  }++  void swap(F14FastSet& rhs) noexcept(Super::Policy::kSwapIsNoexcept) {+    this->table_.swap(rhs.table_);+  }+};+#endif // FOLLY_F14_VECTOR_INTRINSICS_AVAILABLE++template <+    typename InputIt,+    typename Hasher = f14::DefaultHasher<iterator_value_type_t<InputIt>>,+    typename KeyEqual = f14::DefaultKeyEqual<iterator_value_type_t<InputIt>>,+    typename Alloc = f14::DefaultAlloc<iterator_value_type_t<InputIt>>,+    typename = detail::RequireInputIterator<InputIt>,+    typename = detail::RequireNotAllocator<Hasher>,+    typename = detail::RequireNotAllocator<KeyEqual>,+    typename = detail::RequireAllocator<Alloc>>+F14FastSet(+    InputIt, InputIt, std::size_t = {}, Hasher = {}, KeyEqual = {}, Alloc = {})+    -> F14FastSet<iterator_value_type_t<InputIt>, Hasher, KeyEqual, Alloc>;++template <+    typename InputIt,+    typename Alloc,+    typename = detail::RequireInputIterator<InputIt>,+    typename = detail::RequireAllocator<Alloc>>+F14FastSet(InputIt, InputIt, std::size_t, Alloc)+    -> F14FastSet<+        iterator_value_type_t<InputIt>,+        f14::DefaultHasher<iterator_value_type_t<InputIt>>,+        f14::DefaultKeyEqual<iterator_value_type_t<InputIt>>,+        Alloc>;++template <+    typename InputIt,+    typename Hasher,+    typename Alloc,+    typename = detail::RequireInputIterator<InputIt>,+    typename = detail::RequireNotAllocator<Hasher>,+    typename = detail::RequireAllocator<Alloc>>+F14FastSet(InputIt, InputIt, std::size_t, Hasher, Alloc)+    -> F14FastSet<+        iterator_value_type_t<InputIt>,+        Hasher,+        f14::DefaultKeyEqual<iterator_value_type_t<InputIt>>,+        Alloc>;++template <+    typename Key,+    typename Hasher = f14::DefaultHasher<Key>,+    typename KeyEqual = f14::DefaultKeyEqual<Key>,+    typename Alloc = f14::DefaultAlloc<Key>,+    typename = detail::RequireNotAllocator<Hasher>,+    typename = detail::RequireNotAllocator<KeyEqual>,+    typename = detail::RequireAllocator<Alloc>>+F14FastSet(+    std::initializer_list<Key>,+    std::size_t = {},+    Hasher = {},+    KeyEqual = {},+    Alloc = {}) -> F14FastSet<Key, Hasher, KeyEqual, Alloc>;++template <+    typename Key,+    typename Alloc,+    typename = detail::RequireAllocator<Alloc>>+F14FastSet(std::initializer_list<Key>, std::size_t, Alloc)+    -> F14FastSet<+        Key,+        f14::DefaultHasher<Key>,+        f14::DefaultKeyEqual<Key>,+        Alloc>;++template <+    typename Key,+    typename Hasher,+    typename Alloc,+    typename = detail::RequireAllocator<Alloc>>+F14FastSet(std::initializer_list<Key>, std::size_t, Hasher, Alloc)+    -> F14FastSet<Key, Hasher, f14::DefaultKeyEqual<Key>, Alloc>;++} // namespace folly++namespace folly {+namespace f14 {+namespace detail {+template <typename S>+bool setsEqual(S const& lhs, S const& rhs) {+  if (lhs.size() != rhs.size()) {+    return false;+  }+  for (auto& k : lhs) {+    if (!rhs.containsEqualValue(k)) {+      return false;+    }+  }+  return true;+}+} // namespace detail+} // namespace f14++template <typename K, typename H, typename E, typename A>+bool operator==(+    F14ValueSet<K, H, E, A> const& lhs, F14ValueSet<K, H, E, A> const& rhs) {+  return setsEqual(lhs, rhs);+}++template <typename K, typename H, typename E, typename A>+bool operator!=(+    F14ValueSet<K, H, E, A> const& lhs, F14ValueSet<K, H, E, A> const& rhs) {+  return !(lhs == rhs);+}++template <typename K, typename H, typename E, typename A>+bool operator==(+    F14NodeSet<K, H, E, A> const& lhs, F14NodeSet<K, H, E, A> const& rhs) {+  return setsEqual(lhs, rhs);+}++template <typename K, typename H, typename E, typename A>+bool operator!=(+    F14NodeSet<K, H, E, A> const& lhs, F14NodeSet<K, H, E, A> const& rhs) {+  return !(lhs == rhs);+}++template <typename K, typename H, typename E, typename A>+bool operator==(+    F14VectorSet<K, H, E, A> const& lhs, F14VectorSet<K, H, E, A> const& rhs) {+  return setsEqual(lhs, rhs);+}++template <typename K, typename H, typename E, typename A>+bool operator!=(+    F14VectorSet<K, H, E, A> const& lhs, F14VectorSet<K, H, E, A> const& rhs) {+  return !(lhs == rhs);+}++template <typename K, typename H, typename E, typename A>+bool operator==(+    F14FastSet<K, H, E, A> const& lhs, F14FastSet<K, H, E, A> const& rhs) {+  return setsEqual(lhs, rhs);+}++template <typename K, typename H, typename E, typename A>+bool operator!=(+    F14FastSet<K, H, E, A> const& lhs, F14FastSet<K, H, E, A> const& rhs) {+  return !(lhs == rhs);+}++template <typename K, typename H, typename E, typename A>+void swap(F14ValueSet<K, H, E, A>& lhs, F14ValueSet<K, H, E, A>& rhs) noexcept(+    noexcept(lhs.swap(rhs))) {+  lhs.swap(rhs);+}++template <typename K, typename H, typename E, typename A>+void swap(F14NodeSet<K, H, E, A>& lhs, F14NodeSet<K, H, E, A>& rhs) noexcept(+    noexcept(lhs.swap(rhs))) {+  lhs.swap(rhs);+}++template <typename K, typename H, typename E, typename A>+void swap(+    F14VectorSet<K, H, E, A>& lhs,+    F14VectorSet<K, H, E, A>& rhs) noexcept(noexcept(lhs.swap(rhs))) {+  lhs.swap(rhs);+}++template <typename K, typename H, typename E, typename A>+void swap(F14FastSet<K, H, E, A>& lhs, F14FastSet<K, H, E, A>& rhs) noexcept(+    noexcept(lhs.swap(rhs))) {+  lhs.swap(rhs);+}++template <typename K, typename H, typename E, typename A, typename Pred>+std::size_t erase_if(F14ValueSet<K, H, E, A>& c, Pred pred) {+  return f14::detail::erase_if_impl(c, pred);+}++template <typename K, typename H, typename E, typename A, typename Pred>+std::size_t erase_if(F14NodeSet<K, H, E, A>& c, Pred pred) {+  return f14::detail::erase_if_impl(c, pred);+}++template <typename K, typename H, typename E, typename A, typename Pred>+std::size_t erase_if(F14VectorSet<K, H, E, A>& c, Pred pred) {+  return f14::detail::erase_if_impl(c, pred);+}++template <typename K, typename H, typename E, typename A, typename Pred>+std::size_t erase_if(F14FastSet<K, H, E, A>& c, Pred pred) {+  return f14::detail::erase_if_impl(c, pred);+}++} // namespace folly
+ folly/folly/container/FBVector.h view
@@ -0,0 +1,1729 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++/*+ * Nicholas Ormrod      (njormrod)+ * Andrei Alexandrescu  (aalexandre)+ *+ * FBVector is Facebook's drop-in implementation of std::vector. It has special+ * optimizations for use with relocatable types and jemalloc.+ */++#pragma once++//=============================================================================+// headers++#include <algorithm>+#include <cassert>+#include <iterator>+#include <memory>+#include <stdexcept>+#include <type_traits>+#include <utility>++#include <folly/FormatTraits.h>+#include <folly/Likely.h>+#include <folly/ScopeGuard.h>+#include <folly/Traits.h>+#include <folly/lang/CheckedMath.h>+#include <folly/lang/Exception.h>+#include <folly/lang/Hint.h>+#include <folly/memory/Malloc.h>++//=============================================================================+// forward declaration++namespace folly {+template <class T, class Allocator = std::allocator<T>>+class fbvector;+} // namespace folly++//=============================================================================+// unrolling++#define FOLLY_FBV_UNROLL_PTR(first, last, OP)     \+  do {                                            \+    for (; (last) - (first) >= 4; (first) += 4) { \+      OP(((first) + 0));                          \+      OP(((first) + 1));                          \+      OP(((first) + 2));                          \+      OP(((first) + 3));                          \+    }                                             \+    for (; (first) != (last); ++(first))          \+      OP((first));                                \+  } while (0)++//=============================================================================+///////////////////////////////////////////////////////////////////////////////+//                                                                           //+//                              fbvector class                               //+//                                                                           //+///////////////////////////////////////////////////////////////////////////////++namespace folly {++namespace detail {+inline void* thunk_return_nullptr() {+  return nullptr;+}+} // namespace detail++template <class T, class Allocator>+class fbvector {+  //===========================================================================+  //---------------------------------------------------------------------------+  // implementation+ private:+  typedef std::allocator_traits<Allocator> A;++  struct Impl : public Allocator {+    // typedefs+    typedef typename A::pointer pointer;+    typedef typename A::size_type size_type;++    // data+    pointer b_, e_, z_;++    // constructors+    Impl() : Allocator(), b_(nullptr), e_(nullptr), z_(nullptr) {}+    /* implicit */ Impl(const Allocator& alloc)+        : Allocator(alloc), b_(nullptr), e_(nullptr), z_(nullptr) {}+    /* implicit */ Impl(Allocator&& alloc)+        : Allocator(std::move(alloc)), b_(nullptr), e_(nullptr), z_(nullptr) {}++    /* implicit */ Impl(size_type n, const Allocator& alloc = Allocator())+        : Allocator(alloc) {+      init(n);+    }++    Impl(Impl&& other) noexcept+        : Allocator(std::move(other)),+          b_(other.b_),+          e_(other.e_),+          z_(other.z_) {+      other.b_ = other.e_ = other.z_ = nullptr;+    }++    // destructor+    ~Impl() { destroy(); }++    // allocation+    // note that 'allocate' and 'deallocate' are inherited from Allocator+    T* D_allocate(size_type n) {+      if constexpr (kUsingStdAllocator) {+        return static_cast<T*>(checkedMalloc(n * sizeof(T)));+      } else {+        return std::allocator_traits<Allocator>::allocate(*this, n);+      }+    }++    void D_deallocate(T* p, size_type n) noexcept {+      if constexpr (kUsingStdAllocator) {+        free(p);+      } else {+        std::allocator_traits<Allocator>::deallocate(*this, p, n);+      }+    }++    // helpers+    void swapData(Impl& other) {+      std::swap(b_, other.b_);+      std::swap(e_, other.e_);+      std::swap(z_, other.z_);+    }++    // data ops+    inline void destroy() noexcept {+      if (b_) {+        // THIS DISPATCH CODE IS DUPLICATED IN fbvector::D_destroy_range_a.+        // It has been inlined here for speed. It calls the static fbvector+        //  methods to perform the actual destruction.+        if constexpr (kUsingStdAllocator) {+          S_destroy_range(b_, e_);+        } else {+          S_destroy_range_a(*this, b_, e_);+        }++        D_deallocate(b_, size_type(z_ - b_));+      }+    }++    void init(size_type n) {+      if (FOLLY_UNLIKELY(n == 0)) {+        b_ = e_ = z_ = nullptr;+      } else {+        size_type sz = folly::goodMallocSize(n * sizeof(T)) / sizeof(T);+        b_ = D_allocate(sz);+        e_ = b_;+        z_ = b_ + sz;+      }+    }++    void set(pointer newB, size_type newSize, size_type newCap) {+      z_ = newB + newCap;+      e_ = newB + newSize;+      b_ = newB;+    }++    void reset(size_type newCap) {+      destroy();+      auto rollback = makeGuard([&] { init(0); });+      init(newCap);+      rollback.dismiss();+    }+    void reset() { // same as reset(0)+      destroy();+      b_ = e_ = z_ = nullptr;+    }+  } impl_;++  static void swap(Impl& a, Impl& b) {+    using std::swap;+    if constexpr (!kUsingStdAllocator) {+      swap(static_cast<Allocator&>(a), static_cast<Allocator&>(b));+    }+    a.swapData(b);+  }++  //===========================================================================+  //---------------------------------------------------------------------------+  // types and constants+ public:+  typedef T value_type;+  typedef value_type& reference;+  typedef const value_type& const_reference;+  typedef T* iterator;+  typedef const T* const_iterator;+  typedef size_t size_type;+  typedef typename std::make_signed<size_type>::type difference_type;+  typedef Allocator allocator_type;+  typedef typename A::pointer pointer;+  typedef typename A::const_pointer const_pointer;+  typedef std::reverse_iterator<iterator> reverse_iterator;+  typedef std::reverse_iterator<const_iterator> const_reverse_iterator;++ private:+  static constexpr bool should_pass_by_value =+      std::is_trivially_copyable<T>::value &&+      sizeof(T) <= 16; // don't force large structures to be passed by value+  typedef typename std::conditional<should_pass_by_value, T, const T&>::type VT;+  typedef typename std::conditional<should_pass_by_value, T, T&&>::type MT;++  static constexpr bool kUsingStdAllocator =+      std::is_same<Allocator, std::allocator<T>>::value;+  typedef std::bool_constant<+      kUsingStdAllocator || A::propagate_on_container_move_assignment::value>+      moveIsSwap;++  //===========================================================================+  //---------------------------------------------------------------------------+  // allocator helpers++  //---------------------------------------------------------------------------+  // allocate++  T* M_allocate(size_type n) { return impl_.D_allocate(n); }++  //---------------------------------------------------------------------------+  // deallocate++  void M_deallocate(T* p, size_type n) noexcept { impl_.D_deallocate(p, n); }++  //---------------------------------------------------------------------------+  // construct++  // GCC is very sensitive to the exact way that construct is called. For+  //  that reason there are several different specializations of construct.++  template <typename U, typename... Args>+  void M_construct(U* p, Args&&... args) {+    if constexpr (kUsingStdAllocator) {+      new (p) U(std::forward<Args>(args)...);+    } else {+      std::allocator_traits<Allocator>::construct(+          impl_, p, std::forward<Args>(args)...);+    }+  }++  template <typename U, typename... Args>+  static void S_construct(U* p, Args&&... args) {+    new (p) U(std::forward<Args>(args)...);+  }++  template <typename U, typename... Args>+  static void S_construct_a(Allocator& a, U* p, Args&&... args) {+    std::allocator_traits<Allocator>::construct(+        a, p, std::forward<Args>(args)...);+  }++  // scalar optimization+  // TODO we can expand this optimization to: default copyable and assignable+  template <+      typename U,+      typename Enable = typename std::enable_if<std::is_scalar<U>::value>::type>+  void M_construct(U* p, U arg) {+    if constexpr (kUsingStdAllocator) {+      *p = arg;+    } else {+      std::allocator_traits<Allocator>::construct(impl_, p, arg);+    }+  }++  template <+      typename U,+      typename Enable = typename std::enable_if<std::is_scalar<U>::value>::type>+  static void S_construct(U* p, U arg) {+    *p = arg;+  }++  template <+      typename U,+      typename Enable = typename std::enable_if<std::is_scalar<U>::value>::type>+  static void S_construct_a(Allocator& a, U* p, U arg) {+    std::allocator_traits<Allocator>::construct(a, p, arg);+  }++  // const& optimization+  template <+      typename U,+      typename Enable =+          typename std::enable_if<!std::is_scalar<U>::value>::type>+  void M_construct(U* p, const U& value) {+    if constexpr (kUsingStdAllocator) {+      new (p) U(value);+    } else {+      std::allocator_traits<Allocator>::construct(impl_, p, value);+    }+  }++  template <+      typename U,+      typename Enable =+          typename std::enable_if<!std::is_scalar<U>::value>::type>+  static void S_construct(U* p, const U& value) {+    new (p) U(value);+  }++  template <+      typename U,+      typename Enable =+          typename std::enable_if<!std::is_scalar<U>::value>::type>+  static void S_construct_a(Allocator& a, U* p, const U& value) {+    std::allocator_traits<Allocator>::construct(a, p, value);+  }++  //---------------------------------------------------------------------------+  // destroy++  void M_destroy(T* p) noexcept {+    if constexpr (kUsingStdAllocator) {+      if constexpr (!std::is_trivially_destructible<T>::value) {+        p->~T();+      }+    } else {+      std::allocator_traits<Allocator>::destroy(impl_, p);+    }+  }++  //===========================================================================+  //---------------------------------------------------------------------------+  // algorithmic helpers+ private:+  //---------------------------------------------------------------------------+  // destroy_range++  // wrappers+  void M_destroy_range_e(T* pos) noexcept {+    D_destroy_range_a(pos, impl_.e_);+    impl_.e_ = pos;+  }++  // dispatch+  // THIS DISPATCH CODE IS DUPLICATED IN IMPL. SEE IMPL FOR DETAILS.+  void D_destroy_range_a(T* first, T* last) noexcept {+    if constexpr (kUsingStdAllocator) {+      S_destroy_range(first, last);+    } else {+      S_destroy_range_a(impl_, first, last);+    }+  }++  // allocator+  static void S_destroy_range_a(Allocator& a, T* first, T* last) noexcept {+    for (; first != last; ++first) {+      std::allocator_traits<Allocator>::destroy(a, first);+    }+  }++  // optimized+  static void S_destroy_range(T* first, T* last) noexcept {+    if constexpr (!std::is_trivially_destructible<T>::value) {+#define FOLLY_FBV_OP(p) (p)->~T()+      // EXPERIMENTAL DATA on fbvector<vector<int>> (where each vector<int> has+      //  size 0), were vector<int> to be relocatable.+      // The unrolled version seems to work faster for small to medium sized+      //  fbvectors. It gets a 10% speedup on fbvectors of size 1024, 64, and+      //  16.+      // The simple loop version seems to work faster for large fbvectors. The+      //  unrolled version is about 6% slower on fbvectors on size 16384.+      // The two methods seem tied for very large fbvectors. The unrolled+      //  version is about 0.5% slower on size 262144.++      // for (; first != last; ++first) first->~T();+      FOLLY_FBV_UNROLL_PTR(first, last, FOLLY_FBV_OP);+#undef FOLLY_FBV_OP+    }+  }++  //---------------------------------------------------------------------------+  // uninitialized_fill_n++  // wrappers+  void M_uninitialized_fill_n_e(size_type sz) {+    D_uninitialized_fill_n_a(impl_.e_, sz);+    impl_.e_ += sz;+  }++  void M_uninitialized_fill_n_e(size_type sz, VT value) {+    D_uninitialized_fill_n_a(impl_.e_, sz, value);+    impl_.e_ += sz;+  }++  // dispatch+  void D_uninitialized_fill_n_a(T* dest, size_type sz) {+    if constexpr (kUsingStdAllocator) {+      S_uninitialized_fill_n(dest, sz);+    } else {+      S_uninitialized_fill_n_a(impl_, dest, sz);+    }+  }++  void D_uninitialized_fill_n_a(T* dest, size_type sz, VT value) {+    if constexpr (kUsingStdAllocator) {+      S_uninitialized_fill_n(dest, sz, value);+    } else {+      S_uninitialized_fill_n_a(impl_, dest, sz, value);+    }+  }++  // allocator+  template <typename... Args>+  static void S_uninitialized_fill_n_a(+      Allocator& a, T* dest, size_type sz, Args&&... args) {+    auto b = dest;+    T* e = nullptr;+    if (!folly::checked_add(&e, dest, sz)) {+      throw_exception<std::length_error>("FBVector exceeded max size.");+    }+    auto rollback = makeGuard([&] { S_destroy_range_a(a, dest, b); });+    for (; b != e; ++b) {+      std::allocator_traits<Allocator>::construct(+          a, b, std::forward<Args>(args)...);+    }+    rollback.dismiss();+  }++  // optimized+  static void S_uninitialized_fill_n(T* dest, size_type n) {+    if constexpr (folly::IsZeroInitializable<T>::value) {+      if (FOLLY_LIKELY(n != 0)) {+        T* sz = nullptr;+        if (!folly::checked_add(&sz, dest, n)) {+          throw_exception<std::length_error>("FBVector exceeded max size.");+        }+        std::memset((void*)dest, 0, sizeof(T) * n);+      }+    } else {+      auto b = dest;+      T* e = nullptr;+      if (!folly::checked_add(&e, dest, n)) {+        throw_exception<std::length_error>("FBVector exceeded max size.");+      }+      auto rollback = makeGuard([&] {+        --b;+        for (; b >= dest; --b) {+          b->~T();+        }+      });+      for (; b != e; ++b) {+        S_construct(b);+      }+      rollback.dismiss();+    }+  }++  static void S_uninitialized_fill_n(T* dest, size_type n, const T& value) {+    auto b = dest;+    T* e = nullptr;+    if (!folly::checked_add(&e, dest, n)) {+      throw_exception<std::length_error>("FBVector exceeded max size.");+    }+    auto rollback = makeGuard([&] { S_destroy_range(dest, b); });+    for (; b != e; ++b) {+      S_construct(b, value);+    }+    rollback.dismiss();+  }++  //---------------------------------------------------------------------------+  // uninitialized_copy++  // it is possible to add an optimization for the case where+  // It = move(T*) and IsRelocatable<T> and Is0Initializeable<T>++  // wrappers+  template <typename It>+  void M_uninitialized_copy_e(It first, It last) {+    D_uninitialized_copy_a(impl_.e_, first, last);+    impl_.e_ += std::distance(first, last);+  }++  template <typename It>+  void M_uninitialized_move_e(It first, It last) {+    D_uninitialized_move_a(impl_.e_, first, last);+    impl_.e_ += std::distance(first, last);+  }++  // dispatch+  template <typename It>+  void D_uninitialized_copy_a(T* dest, It first, It last) {+    if constexpr (kUsingStdAllocator) {+      if constexpr (std::is_trivially_copyable<T>::value) {+        S_uninitialized_copy_bits(dest, first, last);+      } else {+        S_uninitialized_copy(dest, first, last);+      }+    } else {+      S_uninitialized_copy_a(impl_, dest, first, last);+    }+  }++  template <typename It>+  void D_uninitialized_move_a(T* dest, It first, It last) {+    D_uninitialized_copy_a(+        dest, std::make_move_iterator(first), std::make_move_iterator(last));+  }++  // allocator+  template <typename It>+  static void S_uninitialized_copy_a(Allocator& a, T* dest, It first, It last) {+    auto b = dest;+    auto rollback = makeGuard([&] { S_destroy_range_a(a, dest, b); });+    for (; first != last; ++first, ++b) {+      std::allocator_traits<Allocator>::construct(a, b, *first);+    }+    rollback.dismiss();+  }++  // optimized+  template <typename It>+  static void S_uninitialized_copy(T* dest, It first, It last) {+    auto b = dest;+    auto rollback = makeGuard([&] { S_destroy_range(dest, b); });+    for (; first != last; ++first, ++b) {+      S_construct(b, *first);+    }+    rollback.dismiss();+  }++  static void S_uninitialized_copy_bits(+      T* dest, const T* first, const T* last) {+    if (last != first) {+      std::memcpy((void*)dest, (void*)first, (last - first) * sizeof(T));+    }+  }++  static void S_uninitialized_copy_bits(+      T* dest, std::move_iterator<T*> first, std::move_iterator<T*> last) {+    T* bFirst = first.base();+    T* bLast = last.base();+    if (bLast != bFirst) {+      std::memcpy((void*)dest, (void*)bFirst, (bLast - bFirst) * sizeof(T));+    }+  }++  template <typename It>+  static void S_uninitialized_copy_bits(T* dest, It first, It last) {+    S_uninitialized_copy(dest, first, last);+  }++  //---------------------------------------------------------------------------+  // copy_n++  // This function is "unsafe": it assumes that the iterator can be advanced at+  //  least n times. However, as a private function, that unsafety is managed+  //  wholly by fbvector itself.++  template <typename It>+  static It S_copy_n(T* dest, It first, size_type n) {+    auto e = dest + n;+    for (; dest != e; ++dest, ++first) {+      *dest = *first;+    }+    return first;+  }++  static const T* S_copy_n(T* dest, const T* first, size_type n) {+    if constexpr (std::is_trivially_copyable<T>::value) {+      std::memcpy((void*)dest, (void*)first, n * sizeof(T));+      return first + n;+    } else {+      return S_copy_n<const T*>(dest, first, n);+    }+  }++  static std::move_iterator<T*> S_copy_n(+      T* dest, std::move_iterator<T*> mIt, size_type n) {+    if constexpr (std::is_trivially_copyable<T>::value) {+      T* first = mIt.base();+      std::memcpy((void*)dest, (void*)first, n * sizeof(T));+      return std::make_move_iterator(first + n);+    } else {+      return S_copy_n<std::move_iterator<T*>>(dest, mIt, n);+    }+  }++  //===========================================================================+  //---------------------------------------------------------------------------+  // relocation helpers+ private:+  // Relocation is divided into three parts:+  //+  //  1: relocate_move+  //     Performs the actual movement of data from point a to point b.+  //+  //  2: relocate_done+  //     Destroys the old data.+  //+  //  3: relocate_undo+  //     Destoys the new data and restores the old data.+  //+  // The three steps are used because there may be an exception after part 1+  //  has completed. If that is the case, then relocate_undo can nullify the+  //  initial move. Otherwise, relocate_done performs the last bit of tidying+  //  up.+  //+  // The relocation trio may use either memcpy, move, or copy. It is decided+  //  by the following case statement:+  //+  //  IsRelocatable && kUsingStdAllocator    -> memcpy+  //  has_nothrow_move && kUsingStdAllocator -> move+  //  cannot copy                            -> move+  //  default                                -> copy+  //+  // If the class is non-copyable then it must be movable. However, if the+  //  move constructor is not noexcept, i.e. an error could be thrown, then+  //  relocate_undo will be unable to restore the old data, for fear of a+  //  second exception being thrown. This is a known and unavoidable+  //  deficiency. In lieu of a strong exception guarantee, relocate_undo does+  //  the next best thing: it provides a weak exception guarantee by+  //  destroying the new data, but leaving the old data in an indeterminate+  //  state. Note that that indeterminate state will be valid, since the+  //  old data has not been destroyed; it has merely been the source of a+  //  move, which is required to leave the source in a valid state.++  // wrappers+  void M_relocate(T* newB) {+    relocate_move(newB, impl_.b_, impl_.e_);+    relocate_done(newB, impl_.b_, impl_.e_);+  }++  // dispatch type trait+  typedef std::bool_constant<+      folly::IsRelocatable<T>::value && kUsingStdAllocator>+      relocate_use_memcpy;++  typedef std::bool_constant<+      (std::is_nothrow_move_constructible<T>::value && kUsingStdAllocator) ||+      !std::is_copy_constructible<T>::value>+      relocate_use_move;++  // move+  void relocate_move(T* dest, T* first, T* last) {+    relocate_move_or_memcpy(dest, first, last, relocate_use_memcpy());+  }++  void relocate_move_or_memcpy(T* dest, T* first, T* last, std::true_type) {+    if (first != nullptr) {+      std::memcpy((void*)dest, (void*)first, (last - first) * sizeof(T));+    }+  }++  void relocate_move_or_memcpy(T* dest, T* first, T* last, std::false_type) {+    relocate_move_or_copy(dest, first, last, relocate_use_move());+  }++  void relocate_move_or_copy(T* dest, T* first, T* last, std::true_type) {+    D_uninitialized_move_a(dest, first, last);+  }++  void relocate_move_or_copy(T* dest, T* first, T* last, std::false_type) {+    D_uninitialized_copy_a(dest, first, last);+  }++  // done+  void relocate_done(T* /*dest*/, T* first, T* last) noexcept {+    if constexpr (folly::IsRelocatable<T>::value && kUsingStdAllocator) {+      // used memcpy; data has been relocated, do not call destructor+    } else {+      D_destroy_range_a(first, last);+    }+  }++  // undo+  void relocate_undo(T* dest, T* first, T* last) noexcept {+    if constexpr (folly::IsRelocatable<T>::value && kUsingStdAllocator) {+      // used memcpy, old data is still valid, nothing to do+    } else if constexpr (+        std::is_nothrow_move_constructible<T>::value && kUsingStdAllocator) {+      // noexcept move everything back, aka relocate_move+      relocate_move(first, dest, dest + (last - first));+    } else if constexpr (!std::is_copy_constructible<T>::value) {+      // weak guarantee+      D_destroy_range_a(dest, dest + (last - first));+    } else {+      // used copy, old data is still valid+      D_destroy_range_a(dest, dest + (last - first));+    }+  }++  //===========================================================================+  //---------------------------------------------------------------------------+  // construct/copy/destroy+ public:+  fbvector() = default;++  explicit fbvector(const Allocator& a) : impl_(a) {}++  explicit fbvector(size_type n, const Allocator& a = Allocator())+      : impl_(n, a) {+    M_uninitialized_fill_n_e(n);+  }++  fbvector(size_type n, VT value, const Allocator& a = Allocator())+      : impl_(n, a) {+    M_uninitialized_fill_n_e(n, value);+  }++  template <+      class It,+      class Category = typename std::iterator_traits<It>::iterator_category>+  fbvector(It first, It last, const Allocator& a = Allocator())+      : fbvector(first, last, a, Category()) {}++  fbvector(const fbvector& other)+      : impl_(+            other.size(),+            A::select_on_container_copy_construction(other.impl_)) {+    M_uninitialized_copy_e(other.begin(), other.end());+  }++  fbvector(fbvector&& other) noexcept : impl_(std::move(other.impl_)) {}++  fbvector(const fbvector& other, const Allocator& a)+      : fbvector(other.begin(), other.end(), a) {}++  /* may throw */ fbvector(fbvector&& other, const Allocator& a) : impl_(a) {+    if (impl_ == other.impl_) {+      impl_.swapData(other.impl_);+    } else {+      impl_.init(other.size());+      M_uninitialized_move_e(other.begin(), other.end());+    }+  }++  fbvector(std::initializer_list<T> il, const Allocator& a = Allocator())+      : fbvector(il.begin(), il.end(), a) {}++  ~fbvector() = default; // the cleanup occurs in impl_++  fbvector& operator=(const fbvector& other) {+    if (FOLLY_UNLIKELY(this == &other)) {+      return *this;+    }++    if constexpr (+        !kUsingStdAllocator &&+        A::propagate_on_container_copy_assignment::value) {+      if (impl_ != other.impl_) {+        // can't use other's different allocator to clean up self+        impl_.reset();+      }+      (Allocator&)impl_ = (Allocator&)other.impl_;+    }++    assign(other.begin(), other.end());+    return *this;+  }++  fbvector& operator=(fbvector&& other) {+    if (FOLLY_UNLIKELY(this == &other)) {+      return *this;+    }+    moveFrom(std::move(other), moveIsSwap());+    return *this;+  }++  fbvector& operator=(std::initializer_list<T> il) {+    assign(il.begin(), il.end());+    return *this;+  }++  template <+      class It,+      class Category = typename std::iterator_traits<It>::iterator_category>+  void assign(It first, It last) {+    assign(first, last, Category());+  }++  void assign(size_type n, VT value) {+    if (n > capacity()) {+      // Not enough space. Do not reserve in place, since we will+      // discard the old values anyways.+      if (dataIsInternalAndNotVT(value)) {+        T copy(std::move(value));+        impl_.reset(n);+        M_uninitialized_fill_n_e(n, copy);+      } else {+        impl_.reset(n);+        M_uninitialized_fill_n_e(n, value);+      }+    } else if (n <= size()) {+      auto newE = impl_.b_ + n;+      std::fill(impl_.b_, newE, value);+      M_destroy_range_e(newE);+    } else {+      std::fill(impl_.b_, impl_.e_, value);+      M_uninitialized_fill_n_e(n - size(), value);+    }+  }++  void assign(std::initializer_list<T> il) { assign(il.begin(), il.end()); }++  allocator_type get_allocator() const noexcept { return impl_; }++ private:+  // contract dispatch for iterator types fbvector(It first, It last)+  template <class ForwardIterator>+  fbvector(+      ForwardIterator first,+      ForwardIterator last,+      const Allocator& a,+      std::forward_iterator_tag)+      : impl_(size_type(std::distance(first, last)), a) {+    M_uninitialized_copy_e(first, last);+  }++  template <class InputIterator>+  fbvector(+      InputIterator first,+      InputIterator last,+      const Allocator& a,+      std::input_iterator_tag)+      : impl_(a) {+    for (; first != last; ++first) {+      emplace_back(*first);+    }+  }++  // contract dispatch for allocator movement in operator=(fbvector&&)+  void moveFrom(fbvector&& other, std::true_type) { swap(impl_, other.impl_); }+  void moveFrom(fbvector&& other, std::false_type) {+    if (impl_ == other.impl_) {+      impl_.swapData(other.impl_);+    } else {+      impl_.reset(other.size());+      M_uninitialized_move_e(other.begin(), other.end());+    }+  }++  // contract dispatch for iterator types in assign(It first, It last)+  template <class ForwardIterator>+  void assign(+      ForwardIterator first, ForwardIterator last, std::forward_iterator_tag) {+    const auto newSize = size_type(std::distance(first, last));+    if (newSize > capacity()) {+      impl_.reset(newSize);+      M_uninitialized_copy_e(first, last);+    } else if (newSize <= size()) {+      auto newEnd = std::copy(first, last, impl_.b_);+      M_destroy_range_e(newEnd);+    } else {+      auto mid = S_copy_n(impl_.b_, first, size());+      M_uninitialized_copy_e<decltype(last)>(mid, last);+    }+  }++  template <class InputIterator>+  void assign(+      InputIterator first, InputIterator last, std::input_iterator_tag) {+    auto p = impl_.b_;+    for (; first != last && p != impl_.e_; ++first, ++p) {+      *p = *first;+    }+    if (p != impl_.e_) {+      M_destroy_range_e(p);+    } else {+      for (; first != last; ++first) {+        emplace_back(*first);+      }+    }+  }++  // contract dispatch for aliasing under VT optimization+  bool dataIsInternalAndNotVT(const T& t) {+    if constexpr (should_pass_by_value) {+      return false;+    } else {+      return dataIsInternal(t);+    }+  }+  bool dataIsInternal(const T& t) {+    return FOLLY_UNLIKELY(+        impl_.b_ <= std::addressof(t) && std::addressof(t) < impl_.e_);+  }++  //===========================================================================+  //---------------------------------------------------------------------------+  // iterators+ public:+  iterator begin() noexcept { return impl_.b_; }+  const_iterator begin() const noexcept { return impl_.b_; }+  iterator end() noexcept { return impl_.e_; }+  const_iterator end() const noexcept { return impl_.e_; }+  reverse_iterator rbegin() noexcept { return reverse_iterator(end()); }+  const_reverse_iterator rbegin() const noexcept {+    return const_reverse_iterator(end());+  }+  reverse_iterator rend() noexcept { return reverse_iterator(begin()); }+  const_reverse_iterator rend() const noexcept {+    return const_reverse_iterator(begin());+  }++  const_iterator cbegin() const noexcept { return impl_.b_; }+  const_iterator cend() const noexcept { return impl_.e_; }+  const_reverse_iterator crbegin() const noexcept {+    return const_reverse_iterator(end());+  }+  const_reverse_iterator crend() const noexcept {+    return const_reverse_iterator(begin());+  }++  //===========================================================================+  //---------------------------------------------------------------------------+  // capacity+ public:+  size_type size() const noexcept { return size_type(impl_.e_ - impl_.b_); }++  size_type max_size() const noexcept {+    // good luck gettin' there+    return ~size_type(0);+  }++  void resize(size_type n) {+    if (n <= size()) {+      M_destroy_range_e(impl_.b_ + n);+    } else {+      reserve(n);+      M_uninitialized_fill_n_e(n - size());+    }+  }++  void resize(size_type n, VT t) {+    if (n <= size()) {+      M_destroy_range_e(impl_.b_ + n);+    } else if (dataIsInternalAndNotVT(t) && n > capacity()) {+      T copy(t);+      reserve(n);+      M_uninitialized_fill_n_e(n - size(), copy);+    } else {+      reserve(n);+      M_uninitialized_fill_n_e(n - size(), t);+    }+  }++  size_type capacity() const noexcept { return size_type(impl_.z_ - impl_.b_); }++  bool empty() const noexcept { return impl_.b_ == impl_.e_; }++  void reserve(size_type n) {+    if (n <= capacity()) {+      return;+    }+    if (impl_.b_ && reserve_in_place(n)) {+      return;+    }++    auto newCap = folly::goodMallocSize(n * sizeof(T)) / sizeof(T);+    auto newB = M_allocate(newCap);+    {+      auto rollback = makeGuard([&] { M_deallocate(newB, newCap); });+      M_relocate(newB);+      rollback.dismiss();+    }+    if (impl_.b_) {+      M_deallocate(impl_.b_, size_type(impl_.z_ - impl_.b_));+    }+    impl_.z_ = newB + newCap;+    impl_.e_ = newB + (impl_.e_ - impl_.b_);+    impl_.b_ = newB;+  }++  void shrink_to_fit() noexcept {+    if (empty()) {+      impl_.reset();+      return;+    }++    auto const newCapacityBytes = folly::goodMallocSize(size() * sizeof(T));+    auto const newCap = newCapacityBytes / sizeof(T);+    auto const oldCap = capacity();++    if (newCap >= oldCap) {+      return;+    }++    void* p = impl_.b_;+    // xallocx() will shrink to precisely newCapacityBytes (which was generated+    // by goodMallocSize()) if it successfully shrinks in place.+    if ((usingJEMalloc() && kUsingStdAllocator) &&+        newCapacityBytes >= folly::jemallocMinInPlaceExpandable &&+        xallocx(p, newCapacityBytes, 0, 0) == newCapacityBytes) {+      impl_.z_ += newCap - oldCap;+    } else {+      T* newB = static_cast<T*>(catch_exception(+          [&] { return M_allocate(newCap); }, //+          &detail::thunk_return_nullptr));+      if (!newB) {+        return;+      }+      if (!catch_exception(+              [&] { return M_relocate(newB), true; },+              [&] { return M_deallocate(newB, newCap), false; })) {+        return;+      }+      if (impl_.b_) {+        M_deallocate(impl_.b_, size_type(impl_.z_ - impl_.b_));+      }+      impl_.z_ = newB + newCap;+      impl_.e_ = newB + (impl_.e_ - impl_.b_);+      impl_.b_ = newB;+    }+  }++ private:+  bool reserve_in_place(size_type n) {+    if (!kUsingStdAllocator || !usingJEMalloc()) {+      return false;+    }++    // jemalloc can never grow in place blocks smaller than 4096 bytes.+    if ((impl_.z_ - impl_.b_) * sizeof(T) <+        folly::jemallocMinInPlaceExpandable) {+      return false;+    }++    auto const newCapacityBytes = folly::goodMallocSize(n * sizeof(T));+    void* p = impl_.b_;+    if (xallocx(p, newCapacityBytes, 0, 0) == newCapacityBytes) {+      impl_.z_ = impl_.b_ + newCapacityBytes / sizeof(T);+      return true;+    }+    return false;+  }++  //===========================================================================+  //---------------------------------------------------------------------------+  // element access+ public:+  reference operator[](size_type n) {+    assert(n < size());+    return impl_.b_[n];+  }+  const_reference operator[](size_type n) const {+    assert(n < size());+    return impl_.b_[n];+  }+  const_reference at(size_type n) const {+    if (FOLLY_UNLIKELY(n >= size())) {+      throw_exception<std::out_of_range>(+          "fbvector: index is greater than size.");+    }+    return (*this)[n];+  }+  reference at(size_type n) {+    auto const& cThis = *this;+    return const_cast<reference>(cThis.at(n));+  }+  reference front() {+    assert(!empty());+    return *impl_.b_;+  }+  const_reference front() const {+    assert(!empty());+    return *impl_.b_;+  }+  reference back() {+    assert(!empty());+    return impl_.e_[-1];+  }+  const_reference back() const {+    assert(!empty());+    return impl_.e_[-1];+  }++  //===========================================================================+  //---------------------------------------------------------------------------+  // data access+ public:+  T* data() noexcept { return impl_.b_; }+  const T* data() const noexcept { return impl_.b_; }++  //===========================================================================+  //---------------------------------------------------------------------------+  // modifiers (common)+ public:+  template <class... Args>+  reference emplace_back(Args&&... args) {+    if (impl_.e_ != impl_.z_) {+      M_construct(impl_.e_, std::forward<Args>(args)...);+      ++impl_.e_;+    } else {+      emplace_back_aux(std::forward<Args>(args)...);+    }+    return back();+  }++  void push_back(const T& value) {+    if (impl_.e_ != impl_.z_) {+      M_construct(impl_.e_, value);+      ++impl_.e_;+    } else {+      emplace_back_aux(value);+    }+  }++  void push_back(T&& value) {+    if (impl_.e_ != impl_.z_) {+      M_construct(impl_.e_, std::move(value));+      ++impl_.e_;+    } else {+      emplace_back_aux(std::move(value));+    }+  }++  void pop_back() {+    assert(!empty());+    --impl_.e_;+    M_destroy(impl_.e_);+  }++  void swap(fbvector& other) noexcept {+    if constexpr (+        !kUsingStdAllocator && A::propagate_on_container_swap::value) {+      swap(impl_, other.impl_);+    } else {+      impl_.swapData(other.impl_);+    }+  }++  void clear() noexcept { M_destroy_range_e(impl_.b_); }++ private:+  // std::vector implements a similar function with a different growth+  //  strategy: empty() ? 1 : capacity() * 2.+  //+  // fbvector grows differently on two counts:+  //+  // (1) initial size+  //     Instead of growing to size 1 from empty, fbvector allocates at least+  //     64 bytes. You may still use reserve to reserve a lesser amount of+  //     memory.+  // (2) 1.5x+  //     For medium-sized vectors, the growth strategy is 1.5x. See the docs+  //     for details.+  //     This does not apply to very small or very large fbvectors. This is a+  //     heuristic.+  //     A nice addition to fbvector would be the capability of having a user-+  //     defined growth strategy, probably as part of the allocator.+  //++  size_type computePushBackCapacity() const {+    if (capacity() == 0) {+      return std::max(64 / sizeof(T), size_type(1));+    }+    if (capacity() < folly::jemallocMinInPlaceExpandable / sizeof(T)) {+      return capacity() * 2;+    }+    if (capacity() > 4096 * 32 / sizeof(T)) {+      return capacity() * 2;+    }+    return (capacity() * 3 + 1) / 2;+  }++  static bool emplace_back_aux_xallocx(+      size_type& byte_sz,+      size_type current_size,+      size_type push_back_capacity,+      pointer& z,+      pointer b) {+    byte_sz = folly::goodMallocSize(push_back_capacity * sizeof(T));+    if (kUsingStdAllocator && usingJEMalloc() &&+        ((z - b) * sizeof(T) >= folly::jemallocMinInPlaceExpandable)) {+      // Try to reserve in place.+      // Ask xallocx to allocate in place at least size()+1 and at most sz+      //  space.+      // xallocx will allocate as much as possible within that range, which+      //  is the best possible outcome: if sz space is available, take it all,+      //  otherwise take as much as possible. If nothing is available, then+      //  fail.+      // In this fashion, we never relocate if there is a possibility of+      //  expanding in place, and we never reallocate by less than the desired+      //  amount unless we cannot expand further. Hence we will not reallocate+      //  sub-optimally twice in a row (modulo the blocking memory being freed).+      size_type lower =+          folly::goodMallocSize(sizeof(T) + current_size * sizeof(T));+      size_type upper = byte_sz;+      size_type extra = upper - lower;++      void* p = b;+      size_t actual;++      if ((actual = xallocx(p, lower, extra, 0)) >= lower) {+        z = b + actual / sizeof(T);+        return true;+      }+    }++    return false;+  }++  template <class... Args>+  void emplace_back_aux(Args&&... args) {+    // Try to reserve in place.+    size_type byte_sz;+    if (emplace_back_aux_xallocx(+            byte_sz, size(), computePushBackCapacity(), impl_.z_, impl_.b_)) {+      M_construct(impl_.e_, std::forward<Args>(args)...);+      ++impl_.e_;+      return;+    }++    // Reallocation failed. Perform a manual relocation.+    size_type sz = byte_sz / sizeof(T);+    auto newB = M_allocate(sz);+    auto newE = newB + size();+    {+      auto rollback1 = makeGuard([&] { M_deallocate(newB, sz); });+      if constexpr (folly::IsRelocatable<T>::value && kUsingStdAllocator) {+        // For linear memory access, relocate before construction.+        // By the test condition, relocate is noexcept.+        // Note that there is no cleanup to do if M_construct throws - that's+        //  one of the beauties of relocation.+        // Benchmarks for this code have high variance, and seem to be close.+        relocate_move(newB, impl_.b_, impl_.e_);+        M_construct(newE, std::forward<Args>(args)...);+        ++newE;+      } else {+        M_construct(newE, std::forward<Args>(args)...);+        ++newE;+        auto rollback2 = makeGuard([&] { M_destroy(newE - 1); });+        M_relocate(newB);+        rollback2.dismiss();+      }+      rollback1.dismiss();+    }+    if (impl_.b_) {+      M_deallocate(impl_.b_, size());+    }+    impl_.b_ = newB;+    impl_.e_ = newE;+    impl_.z_ = newB + sz;+  }++  //===========================================================================+  //---------------------------------------------------------------------------+  // modifiers (erase)+ public:+  iterator erase(const_iterator position) {+    return erase(position, position + 1);+  }++  iterator erase(const_iterator first, const_iterator last) {+    assert(isValid(first) && isValid(last));+    assert(first <= last);+    if (first != last) {+      if (last == end()) {+        M_destroy_range_e((iterator)first);+      } else {+        if constexpr (folly::IsRelocatable<T>::value && kUsingStdAllocator) {+          D_destroy_range_a((iterator)first, (iterator)last);+          if (last - first >= cend() - last) {+            std::memcpy((void*)first, (void*)last, (cend() - last) * sizeof(T));+          } else {+            std::memmove(+                (void*)first, (void*)last, (cend() - last) * sizeof(T));+          }+          impl_.e_ -= (last - first);+        } else {+          std::copy(+              std::make_move_iterator((iterator)last),+              std::make_move_iterator(end()),+              (iterator)first);+          auto newEnd = impl_.e_ - std::distance(first, last);+          M_destroy_range_e(newEnd);+        }+      }+    }+    return (iterator)first;+  }++  //===========================================================================+  //---------------------------------------------------------------------------+  // modifiers (insert)+ private: // we have the private section first because it defines some macros+  bool isValid(const_iterator it) { return cbegin() <= it && it <= cend(); }++  size_type computeInsertCapacity(size_type n) {+    size_type nc = std::max(computePushBackCapacity(), size() + n);+    size_type ac = folly::goodMallocSize(nc * sizeof(T)) / sizeof(T);+    return ac;+  }++  //---------------------------------------------------------------------------+  //+  // make_window takes an fbvector, and creates an uninitialized gap (a+  //  window) at the given position, of the given size. The fbvector must+  //  have enough capacity.+  //+  // Explanation by picture.+  //+  //    123456789______+  //        ^+  //        make_window here of size 3+  //+  //    1234___56789___+  //+  // If something goes wrong and the window must be destroyed, use+  //  undo_window to provide a weak exception guarantee. It destroys+  //  the right ledge.+  //+  //    1234___________+  //+  //---------------------------------------------------------------------------+  //+  // wrap_frame takes an inverse window and relocates an fbvector around it.+  //  The fbvector must have at least as many elements as the left ledge.+  //+  // Explanation by picture.+  //+  //        START+  //    fbvector:             inverse window:+  //    123456789______       _____abcde_______+  //                          [idx][ n ]+  //+  //        RESULT+  //    _______________       12345abcde6789___+  //+  //---------------------------------------------------------------------------+  //+  // insert_use_fresh_memory returns true iff the fbvector should use a fresh+  //  block of memory for the insertion. If the fbvector does not have enough+  //  spare capacity, then it must return true. Otherwise either true or false+  //  may be returned.+  //+  //---------------------------------------------------------------------------+  //+  // These three functions, make_window, wrap_frame, and+  //  insert_use_fresh_memory, can be combined into a uniform interface.+  // Since that interface involves a lot of case-work, it is built into+  //  some macros: FOLLY_FBVECTOR_INSERT_(PRE|START|TRY|END)+  // Macros are used in an attempt to let GCC perform better optimizations,+  //  especially control flow optimization.+  //++  //---------------------------------------------------------------------------+  // window++  void make_window(iterator position, size_type n) {+    // The result is guaranteed to be non-negative, so use an unsigned type:+    size_type tail = size_type(std::distance(position, impl_.e_));++    if (tail <= n) {+      relocate_move(position + n, position, impl_.e_);+      relocate_done(position + n, position, impl_.e_);+      impl_.e_ += n;+    } else {+      if constexpr (folly::IsRelocatable<T>::value && kUsingStdAllocator) {+        compiler_may_unsafely_assume(position != nullptr);+        std::memmove((void*)(position + n), (void*)position, tail * sizeof(T));+        impl_.e_ += n;+      } else {+        D_uninitialized_move_a(impl_.e_, impl_.e_ - n, impl_.e_);+        {+          auto rollback = makeGuard([&] {+            D_destroy_range_a(impl_.e_ - n, impl_.e_ + n);+            impl_.e_ -= n;+          });+          std::copy_backward(+              std::make_move_iterator(position),+              std::make_move_iterator(impl_.e_ - n),+              impl_.e_);+          rollback.dismiss();+        }+        impl_.e_ += n;+        D_destroy_range_a(position, position + n);+      }+    }+  }++  void undo_window(iterator position, size_type n) noexcept {+    D_destroy_range_a(position + n, impl_.e_);+    impl_.e_ = position;+  }++  //---------------------------------------------------------------------------+  // frame++  void wrap_frame(T* ledge, size_type idx, size_type n) {+    assert(size() >= idx);+    assert(n != 0);++    relocate_move(ledge, impl_.b_, impl_.b_ + idx);+    {+      auto rollback = makeGuard([&] { //+        relocate_undo(ledge, impl_.b_, impl_.b_ + idx);+      });+      relocate_move(ledge + idx + n, impl_.b_ + idx, impl_.e_);+      rollback.dismiss();+    }+    relocate_done(ledge, impl_.b_, impl_.b_ + idx);+    relocate_done(ledge + idx + n, impl_.b_ + idx, impl_.e_);+  }++  //---------------------------------------------------------------------------+  // use fresh?++  bool insert_use_fresh(bool at_end, size_type n) {+    if (at_end) {+      if (size() + n <= capacity()) {+        return false;+      }+      if (reserve_in_place(size() + n)) {+        return false;+      }+      return true;+    }++    if (size() + n > capacity()) {+      return true;+    }++    return false;+  }++  //---------------------------------------------------------------------------+  // interface++  template <+      typename IsInternalFunc,+      typename InsertInternalFunc,+      typename ConstructFunc,+      typename DestroyFunc>+  iterator do_real_insert(+      const_iterator cpos,+      size_type n,+      IsInternalFunc&& isInternalFunc,+      InsertInternalFunc&& insertInternalFunc,+      ConstructFunc&& constructFunc,+      DestroyFunc&& destroyFunc) {+    if (n == 0) {+      return iterator(cpos);+    }+    bool at_end = cpos == cend();+    bool fresh = insert_use_fresh(at_end, n);+    if (!at_end) {+      if (!fresh && isInternalFunc()) {+        // check for internal data (technically not required by the standard)+        return insertInternalFunc();+      }+      assert(isValid(cpos));+    }+    T* position = const_cast<T*>(cpos);+    size_type idx = size_type(std::distance(impl_.b_, position));+    T* b;+    size_type newCap; /* intentionally uninitialized */++    if (fresh) {+      newCap = computeInsertCapacity(n);+      b = M_allocate(newCap);+    } else {+      if (!at_end) {+        make_window(position, n);+      } else {+        impl_.e_ += n;+      }+      b = impl_.b_;+    }++    T* start = b + idx;+    {+      auto rollback = makeGuard([&] {+        if (fresh) {+          M_deallocate(b, newCap);+        } else {+          if (!at_end) {+            undo_window(position, n);+          } else {+            impl_.e_ -= n;+          }+        }+      });+      // construct the inserted elements+      constructFunc(start);+      rollback.dismiss();+    }++    if (fresh) {+      {+        auto rollback = makeGuard([&] {+          // delete the inserted elements (exception has been thrown)+          destroyFunc(start);+          M_deallocate(b, newCap);+        });+        wrap_frame(b, idx, n);+        rollback.dismiss();+      }+      if (impl_.b_) {+        M_deallocate(impl_.b_, capacity());+      }+      impl_.set(b, size() + n, newCap);+      return impl_.b_ + idx;+    } else {+      return position;+    }+  }++ public:+  template <class... Args>+  iterator emplace(const_iterator cpos, Args&&... args) {+    return do_real_insert(+        cpos,+        1,+        [&] { return false; },+        [&] { return iterator{}; },+        [&](iterator start) {+          M_construct(start, std::forward<Args>(args)...);+        },+        [&](iterator start) { M_destroy(start); });+  }++  iterator insert(const_iterator cpos, const T& value) {+    return do_real_insert(+        cpos,+        1,+        [&] { return dataIsInternal(value); },+        [&] { return insert(cpos, T(value)); },+        [&](iterator start) { M_construct(start, value); },+        [&](iterator start) { M_destroy(start); });+  }++  iterator insert(const_iterator cpos, T&& value) {+    return do_real_insert(+        cpos,+        1,+        [&] { return dataIsInternal(value); },+        [&] { return insert(cpos, T(std::move(value))); },+        [&](iterator start) { M_construct(start, std::move(value)); },+        [&](iterator start) { M_destroy(start); });+  }++  iterator insert(const_iterator cpos, size_type n, VT value) {+    return do_real_insert(+        cpos,+        n,+        [&] { return dataIsInternalAndNotVT(value); },+        [&] { return insert(cpos, n, T(value)); },+        [&](iterator start) { D_uninitialized_fill_n_a(start, n, value); },+        [&](iterator start) { D_destroy_range_a(start, start + n); });+  }++  template <+      class It,+      class Category = typename std::iterator_traits<It>::iterator_category>+  iterator insert(const_iterator cpos, It first, It last) {+    return insert(cpos, first, last, Category());+  }++  iterator insert(const_iterator cpos, std::initializer_list<T> il) {+    return insert(cpos, il.begin(), il.end());+  }++  //---------------------------------------------------------------------------+  // insert dispatch for iterator types+ private:+  template <class FIt>+  iterator insert(+      const_iterator cpos, FIt first, FIt last, std::forward_iterator_tag) {+    size_type n = size_type(std::distance(first, last));+    return do_real_insert(+        cpos,+        n,+        [&] { return false; },+        [&] { return iterator{}; },+        [&](iterator start) { D_uninitialized_copy_a(start, first, last); },+        [&](iterator start) { D_destroy_range_a(start, start + n); });+  }++  template <class IIt>+  iterator insert(+      const_iterator cpos, IIt first, IIt last, std::input_iterator_tag) {+    T* position = const_cast<T*>(cpos);+    assert(isValid(position));+    size_type idx = std::distance(begin(), position);++    fbvector storage(+        std::make_move_iterator(position),+        std::make_move_iterator(end()),+        A::select_on_container_copy_construction(impl_));+    M_destroy_range_e(position);+    for (; first != last; ++first) {+      emplace_back(*first);+    }+    insert(+        cend(),+        std::make_move_iterator(storage.begin()),+        std::make_move_iterator(storage.end()));+    return impl_.b_ + idx;+  }++  //===========================================================================+  //---------------------------------------------------------------------------+  // lexicographical functions+ public:+  bool operator==(const fbvector& other) const {+    return size() == other.size() && std::equal(begin(), end(), other.begin());+  }++  bool operator!=(const fbvector& other) const { return !(*this == other); }++  bool operator<(const fbvector& other) const {+    return std::lexicographical_compare(+        begin(), end(), other.begin(), other.end());+  }++  bool operator>(const fbvector& other) const { return other < *this; }++  bool operator<=(const fbvector& other) const { return !(*this > other); }++  bool operator>=(const fbvector& other) const { return !(*this < other); }++  //===========================================================================+  //---------------------------------------------------------------------------+  // friends+ private:+  template <class _T, class _A>+  friend _T* relinquish(fbvector<_T, _A>&);++  template <class _T, class _A>+  friend void attach(fbvector<_T, _A>&, _T* data, size_t sz, size_t cap);++}; // class fbvector++//=============================================================================+//-----------------------------------------------------------------------------+// specialized functions++template <class T, class A>+void swap(fbvector<T, A>& lhs, fbvector<T, A>& rhs) noexcept {+  lhs.swap(rhs);+}++//=============================================================================+//-----------------------------------------------------------------------------+// other++namespace detail {++// Format support.+template <class T, class A>+struct IndexableTraits<fbvector<T, A>>+    : public IndexableTraitsSeq<fbvector<T, A>> {};++} // namespace detail++template <class T, class A>+void compactResize(fbvector<T, A>* v, size_t sz) {+  v->resize(sz);+  v->shrink_to_fit();+}++// DANGER+//+// relinquish and attach are not a members function specifically so that it is+//  awkward to call them. It is very easy to shoot yourself in the foot with+//  these functions.+//+// If you call relinquish, then it is your responsibility to free the data+//  and the storage, both of which may have been generated in a non-standard+//  way through the fbvector's allocator.+//+// If you call attach, it is your responsibility to ensure that the fbvector+//  is fresh (size and capacity both zero), and that the supplied data is+//  capable of being manipulated by the allocator.+// It is acceptable to supply a stack pointer IF:+//  (1) The vector's data does not outlive the stack pointer. This includes+//      extension of the data's life through a move operation.+//  (2) The pointer has enough capacity that the vector will never be+//      relocated.+//  (3) Insert is not called on the vector; these functions have leeway to+//      relocate the vector even if there is enough capacity.+//  (4) A stack pointer is compatible with the fbvector's allocator.+//++template <class T, class A>+T* relinquish(fbvector<T, A>& v) {+  T* ret = v.data();+  v.impl_.b_ = v.impl_.e_ = v.impl_.z_ = nullptr;+  return ret;+}++template <class T, class A>+void attach(fbvector<T, A>& v, T* data, size_t sz, size_t cap) {+  assert(v.data() == nullptr);+  v.impl_.b_ = data;+  v.impl_.e_ = data + sz;+  v.impl_.z_ = data + cap;+}++template <+    class InputIt,+    class Allocator =+        std::allocator<typename std::iterator_traits<InputIt>::value_type>>+fbvector(InputIt, InputIt, Allocator = Allocator())+    -> fbvector<typename std::iterator_traits<InputIt>::value_type, Allocator>;++template <class T, class A, class U>+void erase(fbvector<T, A>& v, U value) {+  v.erase(std::remove(v.begin(), v.end(), value), v.end());+}++template <class T, class A, class Predicate>+void erase_if(fbvector<T, A>& v, Predicate predicate) {+  v.erase(std::remove_if(v.begin(), v.end(), std::ref(predicate)), v.end());+}+} // namespace folly
+ folly/folly/container/Foreach-inl.h view
@@ -0,0 +1,316 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#include <cassert>+#include <cstdint>+#include <initializer_list>+#include <iterator>+#include <tuple>+#include <type_traits>+#include <utility>++#include <folly/Portability.h>+#include <folly/Traits.h>+#include <folly/Utility.h>+#include <folly/container/Access.h>+#include <folly/functional/Invoke.h>++namespace folly {++namespace for_each_detail {++namespace adl {++/* using override */+using std::get;++/**+ * The adl_ functions below lookup the function name in the namespace of the+ * type of the object being passed into the function. If no function with that+ * name exists for the passed object then the default std:: versions are going+ * to be called+ */+template <std::size_t Index, typename Type>+auto adl_get(Type&& instance) -> decltype(get<Index>(std::declval<Type>())) {+  return get<Index>(std::forward<Type>(instance));+}++} // namespace adl++/**+ * Enable if the tuple supports fetching via a member get<>()+ */+template <typename T>+using EnableIfMemberGetFound =+    void_t<decltype(std::declval<T>().template get<0>())>;+template <typename, typename T>+struct IsMemberGetFound : std::bool_constant<!require_sizeof<T>> {};+template <typename T>+struct IsMemberGetFound<EnableIfMemberGetFound<T>, T> : std::true_type {};++/**+ * A get that tries member get<> first and if that is not found tries ADL get<>.+ * This mechanism is as found in the structured bindings proposal here 11.5.3.+ * http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2017/n4659.pdf+ */+template <+    std::size_t Index,+    typename Type,+    std::enable_if_t<!IsMemberGetFound<void, Type>::value, int> = 0>+auto get_impl(Type&& instance)+    -> decltype(adl::adl_get<Index>(static_cast<Type&&>(instance))) {+  return adl::adl_get<Index>(static_cast<Type&&>(instance));+}+template <+    std::size_t Index,+    typename Type,+    std::enable_if_t<IsMemberGetFound<void, Type>::value, int> = 0>+auto get_impl(Type&& instance)+    -> decltype(static_cast<Type&&>(instance).template get<Index>()) {+  return static_cast<Type&&>(instance).template get<Index>();+}++/**+ * Check if the sequence is a tuple+ */+template <typename Type, typename T = typename std::decay<Type>::type>+using EnableIfTuple = void_t<+    decltype(get_impl<0>(std::declval<T>())),+    decltype(std::tuple_size<T>::value)>;+template <typename, typename T>+struct IsTuple : std::bool_constant<!require_sizeof<T>> {};+template <typename T>+struct IsTuple<EnableIfTuple<T>, T> : std::true_type {};++/**+ * Check if the sequence is a range+ */+template <typename Type, typename T = typename std::decay<Type>::type>+using EnableIfRange = void_t<+    decltype(access::begin(std::declval<T&>())),+    decltype(access::end(std::declval<T&>()))>;+template <typename, typename T>+struct IsRange : std::bool_constant<!require_sizeof<T>> {};+template <typename T>+struct IsRange<EnableIfRange<T>, T> : std::true_type {};++struct TupleTag {};+struct RangeTag {};++/**+ * Should ideally check if it is a tuple and if not return void, but msvc fails+ */+template <typename Sequence>+using SequenceTag =+    std::conditional_t<IsRange<void, Sequence>::value, RangeTag, TupleTag>;++struct BeginAddTag {};+struct IndexingTag {};++template <typename Func, typename Item, typename Iter>+using ForEachImplTag = std::conditional_t<+    is_invocable_v<Func, Item, index_constant<0>, Iter>,+    index_constant<3>,+    std::conditional_t<+        is_invocable_v<Func, Item, index_constant<0>>,+        index_constant<2>,+        std::conditional_t<+            is_invocable_v<Func, Item>,+            index_constant<1>,+            void>>>;++template <+    typename Func,+    typename... Args,+    std::enable_if_t<is_invocable_r_v<LoopControl, Func, Args...>, int> = 0>+LoopControl invoke_returning_loop_control(Func&& f, Args&&... a) {+  return static_cast<Func&&>(f)(static_cast<Args&&>(a)...);+}+template <+    typename Func,+    typename... Args,+    std::enable_if_t<!is_invocable_r_v<LoopControl, Func, Args...>, int> = 0>+LoopControl invoke_returning_loop_control(Func&& f, Args&&... a) {+  static_assert(+      std::is_void<invoke_result_t<Func, Args...>>::value,+      "return either LoopControl or void");+  return static_cast<Func&&>(f)(static_cast<Args&&>(a)...), loop_continue;+}++/**+ * Implementations for the runtime function+ */+template <typename Sequence, typename Func>+void for_each_range_impl(index_constant<3>, Sequence&& range, Func& func) {+  auto first = access::begin(range);+  auto last = access::end(range);+  for (auto index = std::size_t{0}; first != last; ++index) {+    auto next = std::next(first);+    auto control = invoke_returning_loop_control(func, *first, index, first);+    if (loop_break == control) {+      break;+    }+    first = next;+  }+}+template <typename Sequence, typename Func>+void for_each_range_impl(index_constant<2>, Sequence&& range, Func& func) {+  // make a three arg adaptor for the function passed in so that the main+  // implementation function can be used+  auto three_arg_adaptor =+      [&func](auto&& ele, auto index, auto) -> decltype(auto) {+    return func(std::forward<decltype(ele)>(ele), index);+  };+  for_each_range_impl(+      index_constant<3>{}, std::forward<Sequence>(range), three_arg_adaptor);+}++template <typename Sequence, typename Func>+void for_each_range_impl(index_constant<1>, Sequence&& range, Func& func) {+  // make a three argument adaptor for the function passed in that just ignores+  // the second and third argument+  auto three_arg_adaptor = [&func](auto&& ele, auto, auto) -> decltype(auto) {+    return func(std::forward<decltype(ele)>(ele));+  };+  for_each_range_impl(+      index_constant<3>{}, std::forward<Sequence>(range), three_arg_adaptor);+}++/**+ * Handlers for iteration+ */+template <typename Sequence, typename Func, std::size_t... Indices>+void for_each_tuple_impl(+    std::index_sequence<Indices...>, Sequence&& seq, Func& func) {+  using _ = int[];++  // unroll the loop in an initializer list construction parameter expansion+  // pack+  auto control = loop_continue;++  // cast to void to ignore the result; use the int[] initialization to do the+  // loop execution, the ternary conditional will decide whether or not to+  // evaluate the result+  //+  // if func does not return loop-control, expect the optimizer to see through+  // invoke_returning_loop_control always returning loop_continue+  void(_{(+      ((control == loop_continue)+           ? (control = invoke_returning_loop_control(+                  func,+                  get_impl<Indices>(std::forward<Sequence>(seq)),+                  index_constant<Indices>{}))+           : (loop_continue)),+      0)...});+}++/**+ * The two top level compile time loop iteration functions handle the dispatch+ * based on the number of arguments the passed in function can be passed, if 2+ * arguments can be passed then the implementation dispatches work further to+ * the implementation classes above. If not then an adaptor is constructed+ * which is passed on to the 2 argument specialization, which then in turn+ * forwards implementation to the implementation classes above+ */+template <typename Sequence, typename Func>+void for_each_tuple_impl(index_constant<2>, Sequence&& seq, Func& func) {+  // pass the length as an index sequence to the implementation as an+  // optimization over manual template "tail recursion" unrolling+  using size = std::tuple_size<typename std::decay<Sequence>::type>;+  for_each_tuple_impl(+      std::make_index_sequence<size::value>{},+      std::forward<Sequence>(seq),+      func);+}+template <typename Sequence, typename Func>+void for_each_tuple_impl(index_constant<1>, Sequence&& seq, Func& func) {+  // make an adaptor for the function passed in, in case it can only be passed+  // on argument+  auto two_arg_adaptor = [&func](auto&& ele, auto) -> decltype(auto) {+    return func(std::forward<decltype(ele)>(ele));+  };+  for_each_tuple_impl(+      index_constant<2>{}, std::forward<Sequence>(seq), two_arg_adaptor);+}++/**+ * Top level handlers for the for_each loop, with one overload for tuples and+ * one overload for ranges+ *+ * This implies that if type is both a range and a tuple, it is treated as a+ * range rather than as a tuple+ */+template <typename Sequence, typename Func>+void for_each_impl(TupleTag, Sequence&& range, Func& func) {+  using type = decltype(get_impl<0>(std::declval<Sequence>()));+  using tag = ForEachImplTag<Func, type, void>;+  static_assert(!std::is_same<tag, void>::value, "unknown invocability");+  for_each_tuple_impl(tag{}, std::forward<Sequence>(range), func);+}+template <typename Sequence, typename Func>+void for_each_impl(RangeTag, Sequence&& range, Func& func) {+  using iter = decltype(access::begin(std::declval<Sequence>()));+  using type = decltype(*std::declval<iter>());+  using tag = ForEachImplTag<Func, type, iter>;+  static_assert(!std::is_same<tag, void>::value, "unknown invocability");+  for_each_range_impl(tag{}, std::forward<Sequence>(range), func);+}++template <typename Sequence, typename Index>+decltype(auto) fetch_impl(IndexingTag, Sequence&& sequence, Index&& index) {+  return std::forward<Sequence>(sequence)[std::forward<Index>(index)];+}+template <typename Sequence, typename Index>+decltype(auto) fetch_impl(BeginAddTag, Sequence&& sequence, Index index) {+  return *(access::begin(std::forward<Sequence>(sequence)) + index);+}++template <typename Sequence, typename Index>+decltype(auto) fetch_impl(TupleTag, Sequence&& sequence, Index index) {+  return get_impl<index>(std::forward<Sequence>(sequence));+}+template <typename Sequence, typename Index>+decltype(auto) fetch_impl(RangeTag, Sequence&& sequence, Index&& index) {+  using iter = decltype(access::begin(std::declval<Sequence>()));+  using iter_traits = std::iterator_traits<remove_cvref_t<iter>>;+  using iter_cat = typename iter_traits::iterator_category;+  using tag = std::conditional_t<+      std::is_same<iter_cat, std::random_access_iterator_tag>::value,+      BeginAddTag,+      IndexingTag>;+  return fetch_impl(+      tag{}, std::forward<Sequence>(sequence), std::forward<Index>(index));+}++} // namespace for_each_detail++template <typename Sequence, typename Func>+constexpr Func for_each(Sequence&& sequence, Func func) {+  namespace fed = for_each_detail;+  using tag = fed::SequenceTag<Sequence>;+  fed::for_each_impl(tag{}, std::forward<Sequence>(sequence), func);+  return func;+}++template <typename Sequence, typename Index>+constexpr decltype(auto) fetch(Sequence&& sequence, Index&& index) {+  namespace fed = for_each_detail;+  using tag = fed::SequenceTag<Sequence>;+  return for_each_detail::fetch_impl(+      tag{}, std::forward<Sequence>(sequence), std::forward<Index>(index));+}++} // namespace folly
+ folly/folly/container/Foreach.h view
@@ -0,0 +1,209 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <folly/Portability.h>+#include <folly/Preprocessor.h>++#include <type_traits>++namespace folly {++/**+ * @function for_each+ *+ * folly::for_each is a generalized iteration algorithm. Example:+ *+ *  auto one = std::make_tuple(1, 2, 3);+ *  auto two = std::vector<int>{1, 2, 3};+ *  auto func = [](auto element, auto index) {+ *    cout << index << " : " << element << endl;+ *  };+ *  folly::for_each(one, func);+ *  folly::for_each(two, func);+ *+ * The for_each function allows iteration through sequences, these can either be+ * runtime sequences (i.e. entities for which std::begin and std::end work) or+ * compile time sequences (as deemed by the presence of std::tuple_length<> and+ * member get<> or ADL get<> functions).+ *+ * If a sequence type is both a runtime sequence (aka range) and a compile-time+ * sequence (aka tuple), then it is treated as a range in preference to a tuple.+ * An example of such a type is std::array.+ *+ * The function is made to provide a convenient library based alternative to the+ * proposal p0589r0, which aims to generalize the range based for loop even+ * further to work with compile time sequences.+ *+ * A drawback of using range based for loops is that sometimes you do not have+ * access to the index within the range. This provides easy access to that, even+ * with compile time sequences.+ *+ * And breaking out is easy:+ *+ *  auto range_one = std::vector<int>{1, 2, 3};+ *  auto range_two = std::make_tuple(1, 2, 3);+ *  auto func = [](auto ele, auto index) {+ *    cout << "Element at index " << index << " : " << ele;+ *    if (index == 1) {+ *      return folly::loop_break;+ *    }+ *    return folly::loop_continue;+ *  };+ *  folly::for_each(range_one, func);+ *  folly::for_each(range_two, func);+ *+ * A simple use case would be when using futures, if the user was doing calls to+ * n servers then they would accept the callback with the futures like this:+ *+ *  auto vec = std::vector<std::future<int>>{request_one(), ...};+ *  when_all(vec.begin(), vec.end()).then([](auto futures) {+ *    folly::for_each(futures, [](auto& fut) { ... });+ *  });+ *+ * Now when this code switches to use tuples instead of the runtime std::vector,+ * then the loop does not need to change, the code will still work just fine:+ *+ *  when_all(future_one, future_two, future_three).then([](auto futures) {+ *    folly::for_each(futures, [](auto& fut) { ... });+ *  });+ */+template <typename Range, typename Func>+constexpr Func for_each(Range&& range, Func func);++/**+ * The user should return loop_break and loop_continue if they want to iterate+ * in such a way that they can preemptively stop the loop and break out when+ * certain conditions are met.+ */+namespace for_each_detail {+enum class LoopControl : bool { BREAK, CONTINUE };+} // namespace for_each_detail++constexpr auto loop_break = for_each_detail::LoopControl::BREAK;+constexpr auto loop_continue = for_each_detail::LoopControl::CONTINUE;++/**+ * Utility method to help access elements of a sequence with one uniform+ * interface.+ *+ * This can be useful for example when you are looping through a sequence and+ * want to modify another sequence based on the information in the current+ * sequence:+ *+ *  auto range_one = std::make_tuple(1, 2, 3);+ *  auto range_two = std::make_tuple(4, 5, 6);+ *  folly::for_each(range_one, [&range_two](auto ele, auto index) {+ *    folly::fetch(range_two, index) = ele;+ *  });+ *+ * For ranges, this works by first trying to use the iterator class if the+ * iterator has been marked to be a random access iterator. This should be+ * inspectable via the std::iterator_traits traits class. If the iterator class+ * is not present or is not a random access iterator then the implementation+ * falls back to trying to use the indexing operator (operator[]) to fetch the+ * required element.+ */+template <typename Sequence, typename Index>+constexpr decltype(auto) fetch(Sequence&& sequence, Index&& index);++} // namespace folly++/**+ * Everything below is deprecated.+ */++/*+ * Form a local variable name from "FOR_EACH_" x __LINE__, so that+ * FOR_EACH can be nested without creating shadowed declarations.+ */+#define _FE_ANON(x) FB_CONCATENATE(FOR_EACH_, FB_CONCATENATE(x, __LINE__))++/*+ * If you just want the element values, please use:+ *+ *    for (auto&& element : collection)+ *+ * If you need access to the iterators please write an explicit iterator loop+ */+#define FOR_EACH(i, c)                                                     \+  if (bool _FE_ANON(s1_) = false) {                                        \+  } else                                                                   \+    for (auto&& _FE_ANON(s2_) = (c); !_FE_ANON(s1_); _FE_ANON(s1_) = true) \+      for (auto i = _FE_ANON(s2_).begin(); i != _FE_ANON(s2_).end(); ++i)++/*+ * If you just want the element values, please use this (ranges-v3) construct:+ *+ *    for (auto&& element : collection | views::reverse)+ *+ * If you need access to the iterators please write an explicit iterator loop+ */+#define FOR_EACH_R(i, c)                                                   \+  if (bool _FE_ANON(s1_) = false) {                                        \+  } else                                                                   \+    for (auto&& _FE_ANON(s2_) = (c); !_FE_ANON(s1_); _FE_ANON(s1_) = true) \+      for (auto i = _FE_ANON(s2_).rbegin(); i != _FE_ANON(s2_).rend(); ++i)++namespace folly {+namespace detail {++/**+ * notThereYet helps the FOR_EACH_RANGE macro by opportunistically+ * using "<" instead of "!=" whenever available when checking for loop+ * termination. This makes e.g. examples such as FOR_EACH_RANGE (i,+ * 10, 5) execute zero iterations instead of looping virtually+ * forever. At the same time, some iterator types define "!=" but not+ * "<". The notThereYet function will dispatch differently for those.+ *+ * The code below uses `<` for a conservative subset of types for which+ * it is known to be valid.+ */++template <class T, class U>+typename std::enable_if<+    (std::is_arithmetic<T>::value && std::is_arithmetic<U>::value) ||+        (std::is_pointer<T>::value && std::is_pointer<U>::value),+    bool>::type+notThereYet(T& iter, const U& end) {+  return iter < end;+}++template <class T, class U>+typename std::enable_if<+    !((std::is_arithmetic<T>::value && std::is_arithmetic<U>::value) ||+      (std::is_pointer<T>::value && std::is_pointer<U>::value)),+    bool>::type+notThereYet(T& iter, const U& end) {+  return iter != end;+}++} // namespace detail+} // namespace folly++/*+ * Look at the Ranges-v3 views and you'll probably find an easier way to build+ * the view you want but the equivalent is roughly:+ *+ *    for (auto& element : make_subrange(begin, end))+ */+#define FOR_EACH_RANGE(i, begin, end)          \+  for (auto i = (true ? (begin) : (end));      \+       ::folly::detail::notThereYet(i, (end)); \+       ++i)++#include <folly/container/Foreach-inl.h>
+ folly/folly/container/HeterogeneousAccess-fwd.h view
@@ -0,0 +1,33 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++namespace folly {++template <typename T, typename Enable = void>+struct HeterogeneousAccessEqualTo;++template <typename T, typename Enable = void>+struct HeterogeneousAccessHash;++template <typename CharT>+struct TransparentStringEqualTo;++template <typename CharT>+struct TransparentStringHash;++} // namespace folly
+ folly/folly/container/HeterogeneousAccess.h view
@@ -0,0 +1,166 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <functional>+#include <string>+#include <string_view>++#include <folly/Range.h>+#include <folly/Traits.h>+#include <folly/container/HeterogeneousAccess-fwd.h>+#include <folly/hash/Hash.h>+#include <folly/hash/rapidhash.h>++namespace folly {++// folly::HeterogeneousAccessEqualTo<T>, and+// folly::HeterogeneousAccessHash<T> are functors suitable as defaults+// for containers that support heterogeneous access.  When possible, they+// will be marked as transparent.  When no transparent implementation+// is available then they fall back to std::equal_to and std::hash+// respectively.  Since the fallbacks are not marked as transparent,+// heterogeneous lookup won't be available in that case.  A corresponding+// HeterogeneousAccessLess<T> could be easily added if desired.+//+// If T can be implicitly converted to a StringPiece or+// to a Range<T::value_type const*> that is hashable, then+// HeterogeneousAccess{EqualTo,Hash}<T> will be transparent without any+// additional work.  In practice this is true for T that can be convered to+// StringPiece or Range<IntegralType const*>.  This includes std::string,+// std::string_view (when available), std::array, folly::Range,+// std::vector, and folly::small_vector.+//+// Additional specializations of HeterogeneousAccess*<T> should go in+// the header that declares T.  Don't forget to typedef is_transparent to+// void and folly_is_avalanching to std::true_type in the specializations.++template <typename T, typename Enable>+struct HeterogeneousAccessEqualTo : std::equal_to<T> {};++template <typename T, typename Enable>+struct HeterogeneousAccessHash : std::hash<T> {+  using folly_is_avalanching = IsAvalanchingHasher<std::hash<T>, T>;+};++//////// strings++namespace detail {++template <typename T, typename Enable = void>+struct ValueTypeForTransparentConversionToRange {+  using type = char;+};++// We assume that folly::hasher<folly::Range<T const*>> won't be enabled+// when it would be lower quality than std::hash<U> for a U that is+// convertible to folly::Range<T const*>.+template <typename T>+struct ValueTypeForTransparentConversionToRange<+    T,+    void_t<+        decltype(std::declval<hasher<Range<typename T::value_type const*>>>()(+            std::declval<Range<typename T::value_type const*>>()))>> {+  using type = std::remove_const_t<typename T::value_type>;+};++template <typename T>+using TransparentlyConvertibleToRange = std::is_convertible<+    T,+    Range<typename ValueTypeForTransparentConversionToRange<T>::type const*>>;++template <typename T>+struct TransparentRangeEqualTo {+  using is_transparent = void;++  template <typename U1, typename U2>+  bool operator()(U1 const& lhs, U2 const& rhs) const {+    return Range<T const*>{lhs} == Range<T const*>{rhs};+  }++  // This overload is not required for functionality, but+  // guarantees that replacing std::equal_to<std::string> with+  // HeterogeneousAccessEqualTo<std::string> is truly zero overhead+  bool operator()(std::string const& lhs, std::string const& rhs) const {+    return lhs == rhs;+  }+};++template <typename T>+struct TransparentRangeHash {+  using is_transparent = void;+  using folly_is_avalanching = std::true_type;++  template <typename U>+  std::size_t operator()(U const& stringish) const {+    return hasher<Range<T const*>>{}(Range<T const*>{stringish});+  }+};++template <>+struct TransparentRangeHash<char> {+  using is_transparent = void;+  using folly_is_avalanching = std::true_type;++  template <typename U>+  std::size_t operator()(U const& stringish) const {+    auto sp = StringPiece{stringish};+    return static_cast<std::size_t>(+        folly::hash::rapidhashNano(sp.data(), sp.size()));+  }+};++template <+    typename TableKey,+    typename Hasher,+    typename KeyEqual,+    typename ArgKey>+struct EligibleForHeterogeneousFind+    : Conjunction<+          is_transparent<Hasher>,+          is_transparent<KeyEqual>,+          is_invocable<Hasher, ArgKey const&>,+          is_invocable<KeyEqual, ArgKey const&, TableKey const&>> {};++template <+    typename TableKey,+    typename Hasher,+    typename KeyEqual,+    typename ArgKey>+using EligibleForHeterogeneousInsert = Conjunction<+    EligibleForHeterogeneousFind<TableKey, Hasher, KeyEqual, ArgKey>,+    std::is_constructible<TableKey, ArgKey>>;++} // namespace detail++template <typename T>+struct HeterogeneousAccessEqualTo<+    T,+    std::enable_if_t<detail::TransparentlyConvertibleToRange<T>::value>>+    : detail::TransparentRangeEqualTo<+          typename detail::ValueTypeForTransparentConversionToRange<T>::type> {+};++template <typename T>+struct HeterogeneousAccessHash<+    T,+    std::enable_if_t<detail::TransparentlyConvertibleToRange<T>::value>>+    : detail::TransparentRangeHash<+          typename detail::ValueTypeForTransparentConversionToRange<T>::type> {+};++} // namespace folly
+ folly/folly/container/IntrusiveHeap.h view
@@ -0,0 +1,268 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++/**+ * Skew heap [1] implementation using top-down meld.+ *+ * [1] D.D. Sleator, R.E. Tarjan, Self-Adjusting Heaps,+ * SIAM Journal of Computing, 15(1): 52-69, 1986.+ * http://www.cs.cmu.edu/~sleator/papers/adjusting-heaps.pdf+ */++#pragma once++#include <functional>+#include <utility>++#include <boost/intrusive/parent_from_member.hpp>+#include <boost/noncopyable.hpp>+#include <glog/logging.h>+#include <folly/Portability.h>++namespace folly {++/**+ * Base class for items to be inserted into IntrusiveHeap<..., Tag> storing+ * pointers for internal use.+ */+template <class Tag = void>+class IntrusiveHeapNode : private boost::noncopyable {+ public:+  bool isLinked() const { return parent_ != kUnlinked; }++ private:+  template <class, class, class, class>+  friend class IntrusiveHeap;+  friend class IntrusiveHeapTest;++  static IntrusiveHeapNode* const kUnlinked;++  /**+   * If this is in a heap, (parent_ == nullptr) <=> (this == heap_.root_).+   * Otherwise, parent_ is kUnlinked.+   */+  IntrusiveHeapNode* parent_ = kUnlinked;++  /**+   * If this is in a heap, left_ and right_ point to subheaps or nullptr.+   * Otherwise, these are undefined.+   */+  IntrusiveHeapNode* left_;+  IntrusiveHeapNode* right_;+};++template <class Tag>+IntrusiveHeapNode<Tag>* const IntrusiveHeapNode<Tag>::kUnlinked =+    reinterpret_cast<IntrusiveHeapNode*>(1);++template <class T, class Tag>+struct DerivedNodeTraits {+  static IntrusiveHeapNode<Tag>* asNode(T* x) { return x; }+  static T* asT(IntrusiveHeapNode<Tag>* n) { return static_cast<T*>(n); }+};++template <class T, class Tag, IntrusiveHeapNode<Tag> T::*PtrToMember>+struct MemberNodeTraits {+  static IntrusiveHeapNode<Tag>* asNode(T* x) { return &(x->*PtrToMember); }+  static T* asT(IntrusiveHeapNode<Tag>* n) {+    return boost::intrusive::get_parent_from_member(n, PtrToMember);+  }+};++/**+ * IntrusiveHeap implements a skew heap with intrusive pointers to provide+ * O(log(n)) operations on any node in the heap with no separately allocated+ * node type.+ *+ * - To be inserted into an IntrusiveHeap<T, Compare, Tag>, T must inherit from+ *   IntrusiveHeapNode<Tag>, or have a member of type IntrusiveHeapNode<Tag> and+ *   use MemberNodeTraits.+ *+ * - An instance of T may only be included in one IntrusiveHeap for each Tag+ *   type. It may be included in more than one IntrusiveHeap by inheriting from+ *   IntrusiveHeapNode again with a different tag type, or by using composition+ *   with different members.+ */+template <+    class T,+    class Compare = std::less<>,+    class Tag = void,+    class NodeTraitsType = DerivedNodeTraits<T, Tag>>+class IntrusiveHeap {+ public:+  using Node = IntrusiveHeapNode<Tag>;+  using NodeTraits = NodeTraitsType;+  using Value = T;++  IntrusiveHeap() {}++  IntrusiveHeap(const IntrusiveHeap&) = delete;+  IntrusiveHeap& operator=(const IntrusiveHeap&) = delete;++  IntrusiveHeap(IntrusiveHeap&& other) noexcept+      : root_(std::exchange(other.root_, nullptr)) {}+  IntrusiveHeap& operator=(IntrusiveHeap&&) = delete;++  /**+   * Returns a pointer to the maximum value in the heap, or nullptr if the heap+   * is empty.+   */+  T* top() const { return root_ != nullptr ? asT(root_) : nullptr; }++  bool empty() const { return root_ == nullptr; }++  /**+   * Removes the maximum value from the heap and returns it, or nullptr if the+   * heap is empty.+   */+  T* pop() {+    if (root_ == nullptr) {+      return nullptr;+    }+    Node* top = root_;+    merge(top->left_, top->right_, nullptr, &root_);+    top->parent_ = Node::kUnlinked;+    return asT(top);+  }++  /**+   * Visits all items in the heap.+   */+  template <class Visitor>+  void visit(const Visitor& visitor) const {+    visit(visitor, root_);+  }++  /**+   * Updates the heap to reflect a change in a given value.+   */+  void update(T* x) {+    erase(x);+    push(x);+  }++  void push(T* x) {+    DCHECK(x);+    auto n = asNode(x);+    DCHECK(!n->isLinked());+    n->parent_ = nullptr;+    n->left_ = nullptr;+    n->right_ = nullptr;+    merge(n, root_, nullptr, &root_);+  }++  void erase(T* x) {+    auto n = asNode(x);+    DCHECK(n->isLinked());+    DCHECK(contains(x));+    auto parent = n->parent_;+    Node** out;+    if (parent == nullptr) {+      out = &root_;+    } else if (parent->left_ == n) {+      out = &parent->left_;+    } else {+      DCHECK_EQ(parent->right_, n);+      out = &parent->right_;+    }++    merge(n->left_, n->right_, parent, out);+    n->parent_ = Node::kUnlinked;+  }++  /**+   * Check whether this node is included in this heap. Primarily meant for+   * assertions, as containment should be externally tracked.+   */+  bool contains(const T* x) const {+    DCHECK(x);+    auto n = asNode(x);+    while (n->parent_) {+      n = n->parent_;+    }+    return n == root_;+  }++  /**+   * Moves the contents of other into *this.+   */+  void merge(IntrusiveHeap other) {+    merge(root_, other.root_, nullptr, &root_);+  }++ private:+  friend class IntrusiveHeapTest;++  template <class Visitor>+  static void visit(const Visitor& visitor, Node* x) {+    for (; x != nullptr; x = x->right_) {+      visitor(asT(x));+      visit(visitor, x->left_);+    }+  }++  /**+   * Merges two subtrees, assigns a parent, populates *out with new subtree.+   */+  FOLLY_ALWAYS_INLINE static void merge(+      Node* a, Node* b, Node* parent, Node** out) {+    DCHECK(out);+    if (a == nullptr || b == nullptr) {+      *out = a ? a : b;+      if (*out) {+        (*out)->parent_ = parent;+      }+      return;+    }+    do {+      Node* grandparent = parent;+      if (compare(a, b)) {+        parent = b;+      } else {+        parent = a;+        a = b;+      }+      b = parent->right_;+      *out = parent;+      out = &parent->left_;+      parent->right_ = parent->left_;+      parent->parent_ = grandparent;+      DCHECK(a);+    } while (b != nullptr);+    *out = a;+    a->parent_ = parent;+  }++  static Node* asNode(T* x) { return NodeTraits::asNode(x); }++  static const Node* asNode(const T* x) {+    return NodeTraits::asNode(const_cast<T*>(x));+  }++  static T* asT(Node* n) { return NodeTraits::asT(n); }++  static const T* asT(const Node* n) {+    return NodeTraits::asT(const_cast<Node*>(n));+  }++  static bool compare(const Node* a, const Node* b) {+    return Compare()(*asT(a), *asT(b));+  }++  Node* root_ = nullptr;+};++} // namespace folly
+ folly/folly/container/IntrusiveList.h view
@@ -0,0 +1,131 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++/*+ * This file contains convenience aliases that make boost::intrusive::list+ * easier to use.+ */++#include <boost/intrusive/list.hpp>++namespace folly {++/**+ * An auto-unlink intrusive list hook.+ */+using IntrusiveListHook = boost::intrusive::list_member_hook<+    boost::intrusive::link_mode<boost::intrusive::auto_unlink>>;++/**+ * An intrusive list.+ *+ * An IntrusiveList always uses an auto-unlink hook.+ * Beware that IntrusiveList::size() is an O(n) operation, since it has to walk+ * the entire list.+ *+ * Example usage:+ *+ *   class Foo {+ *     // Note that the listHook member variable needs to be visible+ *     // to the code that defines the IntrusiveList instantiation.+ *     // The list hook can be made public, or you can make the other class a+ *     // friend.+ *     IntrusiveListHook listHook;+ *   };+ *+ *   using FooList = IntrusiveList<Foo, &Foo::listHook>;+ *+ *   Foo *foo = new Foo();+ *   FooList myList;+ *   myList.push_back(*foo);+ *+ * Note that each IntrusiveListHook can only be part of a single list at any+ * given time.  If you need the same object to be stored in two lists at once,+ * you need to use two different IntrusiveListHook member variables.+ *+ * The elements stored in the list must contain an IntrusiveListHook member+ * variable.+ */+template <typename T, IntrusiveListHook T::*PtrToMember>+using IntrusiveList = boost::intrusive::list<+    T,+    boost::intrusive::member_hook<T, IntrusiveListHook, PtrToMember>,+    boost::intrusive::constant_time_size<false>>;++/**+ * A safe-link intrusive list hook.+ */+using SafeIntrusiveListHook = boost::intrusive::list_member_hook<+    boost::intrusive::link_mode<boost::intrusive::safe_link>>;++/**+ * A safe intrusive list.+ *+ * This is like IntrusiveList but always uses the safe-link hook which ensures+ * that the hook is initialised to an unlinked state on construction and reset+ * an unlinked state upon removing it from a list.+ */+template <typename T, SafeIntrusiveListHook T::*PtrToMember>+using SafeIntrusiveList = boost::intrusive::list<+    T,+    boost::intrusive::member_hook<T, SafeIntrusiveListHook, PtrToMember>,+    boost::intrusive::constant_time_size<false>>;++/**+ * An intrusive list with const-time size() method.+ *+ * A CountedIntrusiveList always uses a safe-link hook.+ * CountedIntrusiveList::size() is an O(1) operation. Users of this type+ * of lists need to remove a member from a list by calling one of the+ * methods on the list (e.g., erase(), pop_front(), etc.), rather than+ * calling unlink on the member's list hook. Given references to a+ * list and a member, a constant-time removal operation can be+ * accomplished by list.erase(list.iterator_to(member)). Also, when a+ * member is destroyed, it is NOT automatically removed from the list.+ *+ * Example usage:+ *+ *   class Foo {+ *     // Note that the listHook member variable needs to be visible+ *     // to the code that defines the CountedIntrusiveList instantiation.+ *     // The list hook can be made public, or you can make the other class a+ *     // friend.+ *     SafeIntrusiveListHook listHook;+ *   };+ *+ *   using FooList = CountedIntrusiveList<Foo, &Foo::listHook> FooList;+ *+ *   Foo *foo = new Foo();+ *   FooList myList;+ *   myList.push_back(*foo);+ *   myList.pop_front();+ *+ * Note that each SafeIntrusiveListHook can only be part of a single list at any+ * given time.  If you need the same object to be stored in two lists at once,+ * you need to use two different SafeIntrusiveListHook member variables.+ *+ * The elements stored in the list must contain an SafeIntrusiveListHook member+ * variable.+ */+template <typename T, SafeIntrusiveListHook T::*PtrToMember>+using CountedIntrusiveList = boost::intrusive::list<+    T,+    boost::intrusive::member_hook<T, SafeIntrusiveListHook, PtrToMember>,+    boost::intrusive::constant_time_size<true>>;++} // namespace folly
+ folly/folly/container/Iterator.h view
@@ -0,0 +1,843 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <functional>+#include <iterator>+#include <memory>+#include <tuple>+#include <type_traits>+#include <utility>++#include <folly/Traits.h>+#include <folly/Utility.h>+#include <folly/container/Access.h>+#include <folly/functional/Invoke.h>+#include <folly/lang/RValueReferenceWrapper.h>++namespace folly {++//  iterator_has_known_distance_v+//+//  Whether std::distance over a pair of iterators is reasonably known to give+//  the distance without advancing the iterators or copies of them.+template <typename Iter, typename SentinelIter>+inline constexpr bool iterator_has_known_distance_v =+    !require_sizeof<Iter> || !require_sizeof<SentinelIter>;+template <typename Iter>+inline constexpr bool iterator_has_known_distance_v<Iter, Iter> =+    std::is_base_of<+        std::random_access_iterator_tag,+        typename std::iterator_traits<Iter>::iterator_category>::value;++//  range_has_known_distance_v+//+//  Whether std::distance over the begin and end iterators is reasonably known+//  to give the distance without advancing the iterators or copies of them.+//+//  Useful for conditionally reserving memory in advance of iterating the range.+//+//  Note: Many use-cases are better served by range-v3 or std::ranges.+//+//  Example:+//+//      std::vector<result_type> results;+//      auto elems = /* some range */;+//      auto const elemsb = folly::access::begin(elems);+//      auto const elemse = folly::access::end(elems);+//+//      if constexpr (range_has_known_distance_v<decltype(elems)>) {+//        auto const dist = std::distance(elemsb, elemse);+//        results.reserve(static_cast<std::size_t>(dist));+//      }+//+//      for (auto elemsi = elemsb; elemsi != elemsi; ++i) {+//        results.push_back(do_work(*elemsi));+//      }+//      return results;+template <typename Range>+inline constexpr bool range_has_known_distance_v =+    iterator_has_known_distance_v<+        invoke_result_t<access::begin_fn, Range>,+        invoke_result_t<access::end_fn, Range>>;++//  iterator_category_t+//+//  Extracts iterator_category from an iterator.+template <typename Iter>+using iterator_category_t =+    typename std::iterator_traits<Iter>::iterator_category;++namespace detail {++template <typename Iter, typename Category, typename = void>+inline constexpr bool iterator_category_matches_v_ =+    !require_sizeof<Iter> || !require_sizeof<Category>;+template <typename Iter, typename Category>+inline constexpr bool iterator_category_matches_v_<+    Iter,+    Category,+    void_t<iterator_category_t<Iter>>> =+    std::is_convertible<iterator_category_t<Iter>, Category>::value;++} // namespace detail++//  iterator_category_matches_v+//+//  Whether an iterator's category matches Category (std::input_iterator_tag,+//  std::output_iterator_tag, etc). Defined for non-iterator types as well.+//+//  Useful for containers deduction guides implementation.+template <typename Iter, typename Category>+inline constexpr bool iterator_category_matches_v =+    detail::iterator_category_matches_v_<Iter, Category>;++//  iterator_value_type_t+//+//  Extracts a value type from an iterator.+template <typename Iter>+using iterator_value_type_t = typename std::iterator_traits<Iter>::value_type;++//  iterator_reference_type_t+//+//  Extracts reference from an iterator (C++20 iter_reference_t backported)+template <typename Iter>+using iterator_reference_t = decltype(*std::declval<Iter&>());++//  iterator_key_type_t+//+//  Extracts a key type from an iterator, leverages the knowledge that+//  key/value containers usually use std::pair<const K, V> as a value_type.+template <typename Iter>+using iterator_key_type_t =+    remove_cvref_t<typename iterator_value_type_t<Iter>::first_type>;++//  iterator_mapped_type_t+//+//  Extracts a mapped type from an iterator.+template <typename Iter>+using iterator_mapped_type_t =+    typename iterator_value_type_t<Iter>::second_type;++/**+ * Argument tuple for variadic emplace/constructor calls. Stores arguments by+ * (decayed) value. Restores original argument types with reference qualifiers+ * and adornments at unpack time to emulate perfect forwarding.+ *+ * Uses inheritance instead of a type alias to std::tuple so that emplace+ * iterators with implicit unpacking disabled can distinguish between+ * emplace_args and std::tuple parameters.+ *+ * @seealso folly::make_emplace_args+ * @seealso folly::get_emplace_arg+ */+template <typename... Args>+struct emplace_args : public std::tuple<std::decay_t<Args>...> {+  using storage_type = std::tuple<std::decay_t<Args>...>;+  using storage_type::storage_type;+};++/**+ * Pack arguments in a tuple for assignment to a folly::emplace_iterator,+ * folly::front_emplace_iterator, or folly::back_emplace_iterator. The+ * iterator's operator= will unpack the tuple and pass the unpacked arguments+ * to the container's emplace function, which in turn forwards the arguments to+ * the (multi-argument) constructor of the target class.+ *+ * Argument tuples generated with folly::make_emplace_args will be unpacked+ * before being passed to the container's emplace function, even for iterators+ * where implicit_unpack is set to false (so they will not implicitly unpack+ * std::pair or std::tuple arguments to operator=).+ *+ * Arguments are copied (lvalues) or moved (rvalues). To avoid copies and moves,+ * wrap references using std::ref(), std::cref(), and folly::rref(). Beware of+ * dangling references, especially references to temporary objects created with+ * folly::rref().+ *+ * Note that an argument pack created with folly::make_emplace_args is different+ * from an argument pack created with std::make_pair or std::make_tuple.+ * Specifically, passing a std::pair&& or std::tuple&& to an emplace iterator's+ * operator= will pass rvalue references to all fields of that tuple to the+ * container's emplace function, while passing an emplace_args&& to operator=+ * will cast those field references to the exact argument types as passed to+ * folly::make_emplace_args previously. If all arguments have been wrapped by+ * std::reference_wrappers or folly::rvalue_reference_wrappers, the result will+ * be the same as if the container's emplace function had been called directly+ * (perfect forwarding), with no temporary copies of the arguments.+ *+ * @seealso folly::rref+ *+ * @example+ *   class Widget { Widget(int, int); };+ *   std::vector<Widget> makeWidgets(const std::vector<int>& in) {+ *     std::vector<Widget> out;+ *     std::transform(+ *         in.begin(),+ *         in.end(),+ *         folly::back_emplacer(out),+ *         [](int i) { return folly::make_emplace_args(i, i); });+ *     return out;+ *   }+ */+template <typename... Args>+emplace_args<Args...> make_emplace_args(Args&&... args) noexcept(+    noexcept(emplace_args<Args...>(std::forward<Args>(args)...))) {+  return emplace_args<Args...>(std::forward<Args>(args)...);+}++namespace detail {+template <typename Arg>+decltype(auto) unwrap_emplace_arg(Arg&& arg) noexcept {+  return std::forward<Arg>(arg);+}+template <typename Arg>+decltype(auto) unwrap_emplace_arg(std::reference_wrapper<Arg> arg) noexcept {+  return arg.get();+}+template <typename Arg>+decltype(auto) unwrap_emplace_arg(+    folly::rvalue_reference_wrapper<Arg> arg) noexcept {+  return std::move(arg).get();+}+} // namespace detail++/**+ * Getter function for unpacking a single emplace argument.+ *+ * Calling get_emplace_arg on an emplace_args rvalue reference results in+ * perfect forwarding of the original input types. A special case are+ * std::reference_wrapper and folly::rvalue_reference_wrapper objects within+ * folly::emplace_args. These are also unwrapped so that the bare reference is+ * returned.+ *+ * std::get is not a customization point in the standard library, so the+ * cleanest solution was to define our own getter function.+ */+template <size_t I, typename... Args>+decltype(auto) get_emplace_arg(emplace_args<Args...>&& args) noexcept {+  using Out = std::tuple<Args...>;+  return detail::unwrap_emplace_arg(+      std::forward<std::tuple_element_t<I, Out>>(std::get<I>(args)));+}+template <size_t I, typename... Args>+decltype(auto) get_emplace_arg(emplace_args<Args...>& args) noexcept {+  return detail::unwrap_emplace_arg(std::get<I>(args));+}+template <size_t I, typename... Args>+decltype(auto) get_emplace_arg(const emplace_args<Args...>& args) noexcept {+  return detail::unwrap_emplace_arg(std::get<I>(args));+}+template <size_t I, typename Args>+decltype(auto) get_emplace_arg(Args&& args) noexcept {+  return std::get<I>(std::move(args));+}+template <size_t I, typename Args>+decltype(auto) get_emplace_arg(Args& args) noexcept {+  return std::get<I>(args);+}+template <size_t I, typename Args>+decltype(auto) get_emplace_arg(const Args& args) noexcept {+  return std::get<I>(args);+}++namespace detail {+/**+ * Emplace implementation class for folly::emplace_iterator.+ */+template <typename Container>+struct Emplace {+  Emplace(Container& c, typename Container::iterator i)+      : container(std::addressof(c)), iter(std::move(i)) {}+  template <typename... Args>+  void emplace(Args&&... args) {+    iter = container->emplace(iter, std::forward<Args>(args)...);+    ++iter;+  }+  Container* container;+  typename Container::iterator iter;+};++/**+ * Emplace implementation class for folly::hint_emplace_iterator.+ */+template <typename Container>+struct EmplaceHint {+  EmplaceHint(Container& c, typename Container::iterator i)+      : container(std::addressof(c)), iter(std::move(i)) {}+  template <typename... Args>+  void emplace(Args&&... args) {+    iter = container->emplace_hint(iter, std::forward<Args>(args)...);+    ++iter;+  }+  Container* container;+  typename Container::iterator iter;+};++/**+ * Emplace implementation class for folly::front_emplace_iterator.+ */+template <typename Container>+struct EmplaceFront {+  explicit EmplaceFront(Container& c) : container(std::addressof(c)) {}+  template <typename... Args>+  void emplace(Args&&... args) {+    container->emplace_front(std::forward<Args>(args)...);+  }+  Container* container;+};++/**+ * Emplace implementation class for folly::back_emplace_iterator.+ */+template <typename Container>+struct EmplaceBack {+  explicit EmplaceBack(Container& c) : container(std::addressof(c)) {}+  template <typename... Args>+  void emplace(Args&&... args) {+    container->emplace_back(std::forward<Args>(args)...);+  }+  Container* container;+};++/**+ * Generic base class and implementation of all emplace iterator classes.+ *+ * Uses the curiously recurring template pattern (CRTP) to cast `this*` to+ * `Derived*`; i.e., to implement covariant return types in a generic manner.+ */+template <typename Derived, typename EmplaceImpl, bool implicit_unpack>+class emplace_iterator_base;++/**+ * Partial specialization of emplace_iterator_base with implicit unpacking+ * disabled.+ */+template <typename Derived, typename EmplaceImpl>+class emplace_iterator_base<Derived, EmplaceImpl, false>+    : protected EmplaceImpl /* protected implementation inheritance */ {+ public:+  // Iterator traits.+  using iterator_category = std::output_iterator_tag;+  using value_type = void;+  using difference_type = void;+  using pointer = void;+  using reference = void;+  using container_type =+      std::remove_reference_t<decltype(*EmplaceImpl::container)>;++  using EmplaceImpl::EmplaceImpl;++  /**+   * Canonical output operator. Forwards single argument straight to container's+   * emplace function.+   */+  template <typename T>+  Derived& operator=(T&& arg) {+    this->emplace(std::forward<T>(arg));+    return static_cast<Derived&>(*this);+  }++  /**+   * Special output operator for packed arguments. Unpacks args and performs+   * variadic call to container's emplace function.+   */+  template <typename... Args>+  Derived& operator=(emplace_args<Args...>& args) {+    return unpackAndEmplace(args, std::index_sequence_for<Args...>{});+  }+  template <typename... Args>+  Derived& operator=(const emplace_args<Args...>& args) {+    return unpackAndEmplace(args, std::index_sequence_for<Args...>{});+  }+  template <typename... Args>+  Derived& operator=(emplace_args<Args...>&& args) {+    return unpackAndEmplace(+        std::move(args), std::index_sequence_for<Args...>{});+  }++  // No-ops.+  Derived& operator*() { return static_cast<Derived&>(*this); }+  Derived& operator++() { return static_cast<Derived&>(*this); }+  Derived& operator++(int) { return static_cast<Derived&>(*this); }++  // We need all of these explicit defaults because the custom operator=+  // overloads disable implicit generation of these functions.+  emplace_iterator_base(const emplace_iterator_base&) = default;+  emplace_iterator_base(emplace_iterator_base&&) noexcept = default;+  emplace_iterator_base& operator=(emplace_iterator_base&) = default;+  emplace_iterator_base& operator=(const emplace_iterator_base&) = default;+  emplace_iterator_base& operator=(emplace_iterator_base&&) noexcept = default;++ protected:+  template <typename Args, std::size_t... I>+  Derived& unpackAndEmplace(Args& args, std::index_sequence<I...>) {+    this->emplace(get_emplace_arg<I>(args)...);+    return static_cast<Derived&>(*this);+  }+  template <typename Args, std::size_t... I>+  Derived& unpackAndEmplace(const Args& args, std::index_sequence<I...>) {+    this->emplace(get_emplace_arg<I>(args)...);+    return static_cast<Derived&>(*this);+  }+  template <typename Args, std::size_t... I>+  Derived& unpackAndEmplace(Args&& args, std::index_sequence<I...>) {+    this->emplace(get_emplace_arg<I>(std::move(args))...);+    return static_cast<Derived&>(*this);+  }+};++/**+ * Partial specialization of emplace_iterator_base with implicit unpacking+ * enabled.+ *+ * Uses inheritance rather than SFINAE. operator= requires a single argument,+ * which makes it very tricky to use std::enable_if or similar.+ */+template <typename Derived, typename EmplaceImpl>+class emplace_iterator_base<Derived, EmplaceImpl, true>+    : public emplace_iterator_base<Derived, EmplaceImpl, false> {+ private:+  using Base = emplace_iterator_base<Derived, EmplaceImpl, false>;++ public:+  using Base::Base;+  using Base::operator=;++  /**+   * Special output operator for arguments packed into a std::pair. Unpacks+   * the pair and performs variadic call to container's emplace function.+   */+  template <typename... Args>+  Derived& operator=(std::pair<Args...>& args) {+    return this->unpackAndEmplace(args, std::index_sequence_for<Args...>{});+  }+  template <typename... Args>+  Derived& operator=(const std::pair<Args...>& args) {+    return this->unpackAndEmplace(args, std::index_sequence_for<Args...>{});+  }+  template <typename... Args>+  Derived& operator=(std::pair<Args...>&& args) {+    return this->unpackAndEmplace(+        std::move(args), std::index_sequence_for<Args...>{});+  }++  /**+   * Special output operator for arguments packed into a std::tuple. Unpacks+   * the tuple and performs variadic call to container's emplace function.+   */+  template <typename... Args>+  Derived& operator=(std::tuple<Args...>& args) {+    return this->unpackAndEmplace(args, std::index_sequence_for<Args...>{});+  }+  template <typename... Args>+  Derived& operator=(const std::tuple<Args...>& args) {+    return this->unpackAndEmplace(args, std::index_sequence_for<Args...>{});+  }+  template <typename... Args>+  Derived& operator=(std::tuple<Args...>&& args) {+    return this->unpackAndEmplace(+        std::move(args), std::index_sequence_for<Args...>{});+  }++  // We need all of these explicit defaults because the custom operator=+  // overloads disable implicit generation of these functions.+  emplace_iterator_base(const emplace_iterator_base&) = default;+  emplace_iterator_base(emplace_iterator_base&&) noexcept = default;+  emplace_iterator_base& operator=(emplace_iterator_base&) = default;+  emplace_iterator_base& operator=(const emplace_iterator_base&) = default;+  emplace_iterator_base& operator=(emplace_iterator_base&&) noexcept = default;+};++/**+ * Concrete instantiation of emplace_iterator_base. All emplace iterator+ * classes; folly::emplace_iterator, folly::hint_emplace_iterator,+ * folly::front_emplace_iterator, and folly::back_emplace_iterator; are just+ * type aliases of this class.+ *+ * It is not possible to alias emplace_iterator_base directly, because type+ * aliases cannot be used for CRTP.+ */+template <+    template <typename>+    class EmplaceImplT,+    typename Container,+    bool implicit_unpack>+class emplace_iterator_impl+    : public emplace_iterator_base<+          emplace_iterator_impl<EmplaceImplT, Container, implicit_unpack>,+          EmplaceImplT<Container>,+          implicit_unpack> {+ private:+  using Base = emplace_iterator_base<+      emplace_iterator_impl,+      EmplaceImplT<Container>,+      implicit_unpack>;++ public:+  using Base::Base;+  using Base::operator=;++  // We need all of these explicit defaults because the custom operator=+  // overloads disable implicit generation of these functions.+  emplace_iterator_impl(const emplace_iterator_impl&) = default;+  emplace_iterator_impl(emplace_iterator_impl&&) noexcept = default;+  emplace_iterator_impl& operator=(emplace_iterator_impl&) = default;+  emplace_iterator_impl& operator=(const emplace_iterator_impl&) = default;+  emplace_iterator_impl& operator=(emplace_iterator_impl&&) noexcept = default;+};+} // namespace detail++/**+ * Behaves just like std::insert_iterator except that it calls emplace()+ * instead of insert(). Uses perfect forwarding.+ */+template <typename Container, bool implicit_unpack = true>+using emplace_iterator =+    detail::emplace_iterator_impl<detail::Emplace, Container, implicit_unpack>;++/**+ * Behaves just like std::insert_iterator except that it calls emplace_hint()+ * instead of insert(). Uses perfect forwarding.+ */+template <typename Container, bool implicit_unpack = true>+using hint_emplace_iterator = detail::+    emplace_iterator_impl<detail::EmplaceHint, Container, implicit_unpack>;++/**+ * Behaves just like std::front_insert_iterator except that it calls+ * emplace_front() instead of insert(). Uses perfect forwarding.+ */+template <typename Container, bool implicit_unpack = true>+using front_emplace_iterator = detail::+    emplace_iterator_impl<detail::EmplaceFront, Container, implicit_unpack>;++/**+ * Behaves just like std::back_insert_iterator except that it calls+ * emplace_back() instead of insert(). Uses perfect forwarding.+ */+template <typename Container, bool implicit_unpack = true>+using back_emplace_iterator = detail::+    emplace_iterator_impl<detail::EmplaceBack, Container, implicit_unpack>;++/**+ * Convenience function to construct a folly::emplace_iterator, analogous to+ * std::inserter().+ *+ * Setting implicit_unpack to false will disable implicit unpacking of+ * single std::pair and std::tuple arguments to the iterator's operator=. That+ * may be desirable in case of constructors that expect a std::pair or+ * std::tuple argument.+ */+template <bool implicit_unpack = true, typename Container>+emplace_iterator<Container, implicit_unpack> emplacer(+    Container& c, typename Container::iterator i) {+  return emplace_iterator<Container, implicit_unpack>(c, std::move(i));+}++/**+ * Convenience function to construct a folly::hint_emplace_iterator, analogous+ * to std::inserter().+ *+ * Setting implicit_unpack to false will disable implicit unpacking of+ * single std::pair and std::tuple arguments to the iterator's operator=. That+ * may be desirable in case of constructors that expect a std::pair or+ * std::tuple argument.+ */+template <bool implicit_unpack = true, typename Container>+hint_emplace_iterator<Container, implicit_unpack> hint_emplacer(+    Container& c, typename Container::iterator i) {+  return hint_emplace_iterator<Container, implicit_unpack>(c, std::move(i));+}++/**+ * Convenience function to construct a folly::front_emplace_iterator, analogous+ * to std::front_inserter().+ *+ * Setting implicit_unpack to false will disable implicit unpacking of+ * single std::pair and std::tuple arguments to the iterator's operator=. That+ * may be desirable in case of constructors that expect a std::pair or+ * std::tuple argument.+ */+template <bool implicit_unpack = true, typename Container>+front_emplace_iterator<Container, implicit_unpack> front_emplacer(+    Container& c) {+  return front_emplace_iterator<Container, implicit_unpack>(c);+}++/**+ * Convenience function to construct a folly::back_emplace_iterator, analogous+ * to std::back_inserter().+ *+ * Setting implicit_unpack to false will disable implicit unpacking of+ * single std::pair and std::tuple arguments to the iterator's operator=. That+ * may be desirable in case of constructors that expect a std::pair or+ * std::tuple argument.+ */+template <bool implicit_unpack = true, typename Container>+back_emplace_iterator<Container, implicit_unpack> back_emplacer(Container& c) {+  return back_emplace_iterator<Container, implicit_unpack>(c);+}++namespace detail {++// An accepted way to make operator-> work+// https://quuxplusone.github.io/blog/2019/02/06/arrow-proxy/+template <typename Ref>+struct arrow_proxy {+  Ref res;+  Ref* operator->() { return &res; }++  explicit arrow_proxy(Ref* ref) : res(*ref) {}+};++struct index_iterator_access_at {+  template <typename Container, typename Index>+  constexpr decltype(auto) operator()(Container& container, Index index) const {+    if constexpr (folly::is_tag_invocable_v<+                      index_iterator_access_at,+                      Container&,+                      Index>) {+      return folly::tag_invoke(*this, container, index);+    } else {+      return container[index];+    }+  }+};++} // namespace detail++FOLLY_DEFINE_CPO(detail::index_iterator_access_at, index_iterator_access_at)++/**+ * index_iterator+ *+ * An iterator class for random access data structures that provide an+ * access by index via `operator[](size_type)`.+ *+ * Requires a `value_type` defined in a container (we cannot+ * get the value type from reference).+ *+ * Example:+ *  class Container {+ *   public:+ *    using value_type = <*>;  // we need value_type to be defined.+ *    using iterator = folly::index_iterator<Container>;+ *    using const_iterator = folly::index_iterator<const Container>;+ *    using reverse_iterator = std::reverse_iterator<iterator>;+ *    using const_reverse_iterator = std::reverse_iterator<const_iterator>;+ *+ *    some_ref_type  operator[](std::size_t index);+ *    some_cref_type operator[](std::size_t index) const;+ *   ...+ *  };+ *+ *  Note that `some_ref_type` can be any proxy reference, as long as the+ *  algorithms support that (for example from range-v3).+ *+ * NOTE: if `operator[]` doesn't work for you for some reason+ *       you can specify:+ *+ * ```+ *  friend some_ref_type tag_invoke(+ *    folly::cpo_t<index_iterator_access_at>,+ *    Container& c,+ *    std::size_t index);+ *+ *  friend some_cref_type tag_invoke(+ *    folly::cpo_t<index_iterator_access_at>,+ *    const Container& c,+ *    std::size_t index);+ * ```+ **/++template <typename Container>+class index_iterator {+  template <typename T>+  using get_size_type_t = typename std::remove_cv_t<T>::size_type;++  template <typename T>+  using get_difference_type_t = typename std::remove_cv_t<T>::difference_type;++  template <typename IndexType>+  constexpr static decltype(auto) get_reference_by_index(+      Container& container, IndexType index) {+    return index_iterator_access_at(container, index);+  }++ public:+  // index iterator specific types++  using container_type = Container;+  using size_type = detected_or_t<std::size_t, get_size_type_t, Container>;++  // iterator types++  using value_type = typename std::remove_const_t<container_type>::value_type;+  using iterator_category = std::random_access_iterator_tag;+  using reference = decltype(get_reference_by_index(+      FOLLY_DECLVAL(container_type&), size_type{}));+  using difference_type =+      detected_or_t<std::ptrdiff_t, get_difference_type_t, Container>;++  using pointer = std::conditional_t<+      std::is_reference<reference>::value,+      std::remove_reference_t<reference>*,+      detail::arrow_proxy<reference>>;++  static_assert(+      std::is_signed<difference_type>::value, "difference_type must be signed");++  // accessors++  // instance of `index_iterator_accessor`+  container_type* get_container() const { return container_; }+  difference_type get_index() const { return index_; }++  constexpr index_iterator() = default;++  constexpr index_iterator(container_type& container, size_type index)+      : container_(&container), index_(index) {}++  // converting constructors --++  template <+      typename OtherContainer,+      typename = std::enable_if_t<+          std::is_same<std::remove_const_t<container_type>, OtherContainer>::+              value &&+          std::is_const<container_type>::value>>+  /* implicit */ constexpr index_iterator(index_iterator<OtherContainer> other)+      : container_(other.get_container()), index_(other.get_index()) {}++  // access ---++  constexpr reference operator*() const {+    return get_reference_by_index(*container_, index_);+  }++  pointer operator->() const {+    // It's equivalent to pointer{&**this} but compiler stops+    // compilation on taking an address of a temporary.+    // In this case `arrow_proxy` will copy the temporary and there is no+    // issue.+    auto&& ref = **this;+    pointer res{&ref};+    return res;+  }++  constexpr reference operator[](difference_type n) const {+    return *(*this + n);+  }++  // operator++/--++  constexpr index_iterator& operator++() {+    ++index_;+    return *this;+  }++  constexpr index_iterator operator++(int) {+    auto tmp = *this;+    ++*this;+    return tmp;+  }++  constexpr index_iterator& operator--() {+    --index_;+    return *this;+  }++  constexpr index_iterator operator--(int) {+    auto tmp = *this;+    --*this;+    return tmp;+  }++  // operator+/-++  constexpr index_iterator& operator+=(difference_type n) {+    auto signed_index = static_cast<difference_type>(index_) + n;+    index_ = static_cast<size_type>(signed_index);+    return *this;+  }++  constexpr index_iterator& operator-=(difference_type n) {+    index_ += -n;+    return *this;+  }++  constexpr friend index_iterator operator+(+      index_iterator x, difference_type n) {+    return x += n;+  }++  constexpr friend index_iterator operator+(+      difference_type n, index_iterator x) {+    return x + n;+  }++  constexpr friend index_iterator operator-(+      index_iterator x, difference_type n) {+    return x -= n;+  }++  constexpr friend difference_type operator-(+      index_iterator x, index_iterator y) {+    assert(x.container_ == y.container_);+    return static_cast<difference_type>(x.index_) -+        static_cast<difference_type>(y.index_);+  }++  // comparisons+  friend constexpr bool operator==(+      const index_iterator& x, const index_iterator& y) {+    assert(x.container_ == y.container_);+    return x.index_ == y.index_;+  }++  friend constexpr bool operator!=(+      const index_iterator& x, const index_iterator& y) {+    return !(x == y);+  }++  friend constexpr bool operator<(+      const index_iterator& x, const index_iterator& y) {+    assert(x.container_ == y.container_);+    return x.index_ < y.index_;+  }++  friend constexpr bool operator<=(+      const index_iterator& x, const index_iterator& y) {+    return !(y < x);+  }++  friend constexpr bool operator>=(+      const index_iterator& x, const index_iterator& y) {+    return !(x < y);+  }++  friend constexpr bool operator>(+      const index_iterator& x, const index_iterator& y) {+    return y < x;+  }++ private:+  container_type* container_ = nullptr;+  size_type index_ = 0;+};++} // namespace folly
+ folly/folly/container/MapUtil.h view
@@ -0,0 +1,385 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++/**+ * MapUtil provides convenience functions to get a value from a map.+ *+ * @refcode folly/docs/examples/folly/MapUtil.cpp+ * @file MapUtil.h+ */++#pragma once++#include <tuple>++#include <fmt/format.h>++#include <folly/Optional.h>+#include <folly/Range.h>+#include <folly/functional/Invoke.h>++namespace folly {++/**+ * Given a map and a key, return the value corresponding to the key in the map,+ * or a given default value if the key doesn't exist in the map.+ */+template <typename Map, typename Key = typename Map::key_type>+typename Map::mapped_type get_default(const Map& map, const Key& key) {+  auto pos = map.find(key);+  return (pos != map.end()) ? (pos->second) : (typename Map::mapped_type{});+}+template <+    class Map,+    typename Key = typename Map::key_type,+    typename Value = typename Map::mapped_type,+    typename std::enable_if<!is_invocable_v<Value>>::type* = nullptr>+typename Map::mapped_type get_default(+    const Map& map, const Key& key, Value&& dflt) {+  using M = typename Map::mapped_type;+  auto pos = map.find(key);+  return (pos != map.end())+      ? pos->second+      : static_cast<M>(static_cast<Value&&>(dflt));+}++/**+ * Give a map and a key, return the value corresponding to the key in the map,+ * or a given default value if the key doesn't exist in the map.+ */+template <+    class Map,+    typename Key = typename Map::key_type,+    typename Func,+    typename = typename std::enable_if<+        is_invocable_r_v<typename Map::mapped_type, Func>>::type>+typename Map::mapped_type get_default(+    const Map& map, const Key& key, Func&& dflt) {+  auto pos = map.find(key);+  return pos != map.end() ? pos->second : dflt();+}++/**+ * Given a map and a key, return the value corresponding to the key in the map,+ * or throw an exception of the specified type.+ */+template <+    class E = std::out_of_range,+    class Map,+    typename Key = typename Map::key_type>+const typename Map::mapped_type& get_or_throw(+    const Map& map,+    const Key& key,+    const StringPiece& exceptionStrPrefix = StringPiece()) {+  auto pos = map.find(key);+  if (pos != map.end()) {+    return pos->second;+  }+  throw_exception<E>(fmt::format("{}{}", exceptionStrPrefix, key));+}++template <+    class E = std::out_of_range,+    class Map,+    typename Key = typename Map::key_type>+typename Map::mapped_type& get_or_throw(+    Map& map,+    const Key& key,+    const StringPiece& exceptionStrPrefix = StringPiece()) {+  auto pos = map.find(key);+  if (pos != map.end()) {+    return pos->second;+  }+  throw_exception<E>(fmt::format("{}{}", exceptionStrPrefix, key));+}++/**+ * Given a map and a key, return a Optional<V> if the key exists and None if the+ * key does not exist in the map.+ */+template <+    template <typename> class Optional = folly::Optional,+    class Map,+    typename Key = typename Map::key_type>+Optional<typename Map::mapped_type> get_optional(+    const Map& map, const Key& key) {+  auto pos = map.find(key);+  if (pos != map.end()) {+    return Optional<typename Map::mapped_type>(pos->second);+  } else {+    return {};+  }+}++/**+ * Given a map and a key, return a reference to the value corresponding to the+ * key in the map, or the given default reference if the key doesn't exist in+ * the map.+ */+template <class Map, typename Key = typename Map::key_type>+const typename Map::mapped_type& get_ref_default(+    const Map& map, const Key& key, const typename Map::mapped_type& dflt) {+  auto pos = map.find(key);+  return (pos != map.end() ? pos->second : dflt);+}++/**+ * Passing a temporary default value returns a dangling reference when it is+ * returned. Lifetime extension is broken by the indirection.+ * The caller must ensure that the default value outlives the reference returned+ * by get_ref_default().+ */+template <class Map, typename Key = typename Map::key_type>+const typename Map::mapped_type& get_ref_default(+    const Map& map, const Key& key, typename Map::mapped_type&& dflt) = delete;++template <class Map, typename Key = typename Map::key_type>+const typename Map::mapped_type& get_ref_default(+    const Map& map, const Key& key, const typename Map::mapped_type&& dflt) =+    delete;++/**+ * Given a map and a key, return a reference to the value corresponding to the+ * key in the map, or the given default reference if the key doesn't exist in+ * the map.+ */+template <+    class Map,+    typename Key = typename Map::key_type,+    typename Func,+    typename = typename std::enable_if<+        is_invocable_r_v<const typename Map::mapped_type&, Func>>::type,+    typename = typename std::enable_if<+        std::is_reference<invoke_result_t<Func>>::value>::type>+const typename Map::mapped_type& get_ref_default(+    const Map& map, const Key& key, Func&& dflt) {+  auto pos = map.find(key);+  return (pos != map.end() ? pos->second : dflt());+}++/**+ * @brief Given a map and a key, return a pointer to the value corresponding to+ * the key in the map, or nullptr if the key doesn't exist in the map.+ */+template <class Map, typename Key = typename Map::key_type>+auto get_ptr(const Map& map, const Key& key) {+  auto pos = map.find(key);+  return (pos != map.end() ? &pos->second : nullptr);+}+template <class Map, typename Key = typename Map::key_type>+const typename Map::mapped_type* FOLLY_NULLABLE+get_ptr(const Map* FOLLY_NULLABLE map, const Key& key) {+  return map ? get_ptr(*map, key) : nullptr;+}++/**+ * Non-const overload of the above.+ */+template <class Map, typename Key = typename Map::key_type>+auto get_ptr(Map& map, const Key& key) {+  auto pos = map.find(key);+  return (pos != map.end() ? &pos->second : nullptr);+}++template <class Map, typename Key = typename Map::key_type>+typename Map::mapped_type* FOLLY_NULLABLE+get_ptr(Map* FOLLY_NULLABLE map, const Key& key) {+  return map ? get_ptr(*map, key) : nullptr;+}++/**+ * Same as `get_ptr` but for `find` variants that search for two keys at once.+ */+template <class Map, typename Key = typename Map::key_type>+std::pair<const typename Map::mapped_type*, const typename Map::mapped_type*>+get_ptr2(const Map& map, const Key& key0, const Key& key1) {+  const auto& iter_pair = map.find(key0, key1);+  auto iter0 = iter_pair.first;+  auto iter1 = iter_pair.second;+  auto end = map.end();+  return std::make_pair(+      iter0 != end ? &iter0->second : nullptr,+      iter1 != end ? &iter1->second : nullptr);+}++/**+ * Same as `get_ptr` but for `find` variants that search for two keys at once.+ */+template <class Map, typename Key = typename Map::key_type>+std::pair<typename Map::mapped_type*, typename Map::mapped_type*> get_ptr2(+    Map& map, const Key& key0, const Key& key1) {+  const auto& iter_pair = map.find(key0, key1);+  auto iter0 = iter_pair.first;+  auto iter1 = iter_pair.second;+  auto end = map.end();+  return std::make_pair(+      iter0 != end ? &iter0->second : nullptr,+      iter1 != end ? &iter1->second : nullptr);+}++// TODO: Remove the return type computations when clang 3.5 and gcc 5.1 are+// the minimum supported versions.+namespace detail {+template <+    class T,+    size_t pathLength,+    class = typename std::enable_if<(pathLength > 0)>::type>+struct NestedMapType {+  using type =+      typename NestedMapType<std::remove_pointer_t<T>, pathLength - 1>::type::+          mapped_type;+};++template <class T>+struct NestedMapType<T, 1> {+  using type = typename T::mapped_type;+};++template <typename... KeysDefault>+struct DefaultType;++template <typename Default>+struct DefaultType<Default> {+  using type = Default;+};++template <typename Key, typename... KeysDefault>+struct DefaultType<Key, KeysDefault...> {+  using type = typename DefaultType<KeysDefault...>::type;+};++template <class... KeysDefault>+auto extract_default(const KeysDefault&... keysDefault) ->+    typename DefaultType<KeysDefault...>::type const& {+  return std::get<sizeof...(KeysDefault) - 1>(std::tie(keysDefault...));+}+} // namespace detail++/**+ * Given a map of maps and a path of keys, return a Optional<V> if the nested+ * key exists and None if the nested keys does not exist in the map.+ */+template <+    template <typename> class Optional = folly::Optional,+    class Map,+    class Key1,+    class Key2,+    class... Keys>+auto get_optional(+    const Map& map, const Key1& key1, const Key2& key2, const Keys&... keys)+    -> Optional<+        typename detail::NestedMapType<Map, 2 + sizeof...(Keys)>::type> {+  auto pos = map.find(key1);+  if (pos != map.end()) {+    return get_optional<Optional>(pos->second, key2, keys...);+  } else {+    return {};+  }+}++/**+ * Given a map of maps and a path of keys, return a pointer to the nested value,+ * or nullptr if the key doesn't exist in the map.+ */+template <class Map, class Key1, class Key2, class... Keys>+auto get_ptr(+    const Map& map, const Key1& key1, const Key2& key2, const Keys&... keys) ->+    typename detail::NestedMapType<Map, 2 + sizeof...(Keys)>::type+    const* FOLLY_NULLABLE {+  auto pos = map.find(key1);+  return pos != map.end() ? get_ptr(pos->second, key2, keys...) : nullptr;+}++template <class Map, class Key1, class Key2, class... Keys>+auto get_ptr(+    const Map* FOLLY_NULLABLE map,+    const Key1& key1,+    const Key2& key2,+    const Keys&... keys) ->+    typename detail::NestedMapType<Map, 2 + sizeof...(Keys)>::type+    const* FOLLY_NULLABLE {+  return map ? get_ptr(*map, key1, key2, keys...) : nullptr;+}++template <class Map, class Key1, class Key2, class... Keys>+auto get_ptr(Map& map, const Key1& key1, const Key2& key2, const Keys&... keys)+    -> typename detail::NestedMapType<Map, 2 + sizeof...(Keys)>::+        type* FOLLY_NULLABLE {+  auto pos = map.find(key1);+  return pos != map.end() ? get_ptr(pos->second, key2, keys...) : nullptr;+}++template <class Map, class Key1, class Key2, class... Keys>+auto get_ptr(+    Map* FOLLY_NULLABLE map,+    const Key1& key1,+    const Key2& key2,+    const Keys&... keys) ->+    typename detail::NestedMapType<Map, 2 + sizeof...(Keys)>::+        type* FOLLY_NULLABLE {+  return map ? get_ptr(*map, key1, key2, keys...) : nullptr;+}++/**+ * Given a map and a path of keys, return the value corresponding to the nested+ * value, or a given default value if the path doesn't exist in the map.+ * The default value is the last parameter, and is copied when returned.+ */+template <+    class Map,+    class Key1,+    class Key2,+    class... KeysDefault,+    typename = typename std::enable_if<sizeof...(KeysDefault) != 0>::type>+auto get_default(+    const Map& map,+    const Key1& key1,+    const Key2& key2,+    const KeysDefault&... keysDefault) ->+    typename detail::NestedMapType<Map, 1 + sizeof...(KeysDefault)>::type {+  if (const auto* ptr = get_ptr(map, key1)) {+    return get_default(*ptr, key2, keysDefault...);+  }+  return detail::extract_default(keysDefault...);+}++/**+ * Given a map and a path of keys, return a reference to the value corresponding+ * to the nested value, or the given default reference if the path doesn't exist+ * in the map.+ * The default value is the last parameter, and must be a lvalue reference.+ */+template <+    class Map,+    class Key1,+    class Key2,+    class... KeysDefault,+    typename = typename std::enable_if<sizeof...(KeysDefault) != 0>::type,+    typename = typename std::enable_if<std::is_lvalue_reference<+        typename detail::DefaultType<KeysDefault...>::type>::value>::type>+auto get_ref_default(+    const Map& map,+    const Key1& key1,+    const Key2& key2,+    KeysDefault&&... keysDefault) ->+    typename detail::NestedMapType<Map, 1 + sizeof...(KeysDefault)>::type+    const& {+  if (const auto* ptr = get_ptr(map, key1)) {+    return get_ref_default(*ptr, key2, keysDefault...);+  }+  return detail::extract_default(keysDefault...);+}+} // namespace folly
+ folly/folly/container/Merge.h view
@@ -0,0 +1,90 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++/*+ * folly::merge() is an implementation of std::merge with one additonal+ * guarantee: if the input ranges overlap, the order that values *from the two+ * different ranges* appear in the output is well defined (std::merge only+ * guarantees relative ordering is maintained within a single input range).+ * This semantic is very useful when the output container removes duplicates+ * (such as std::map) to guarantee that elements from b override elements from+ * a.+ *+ * ex. Let's say we have two vector<pair<int, int>> as input, and we are+ * merging into a vector<pair<int, int>>. The comparator is returns true if the+ * first argument has a lesser 'first' value in the pair.+ *+ * a = {{1, 1}, {2, 2}, {3, 3}};+ * b = {{1, 2}, {2, 3}};+ *+ * folly::merge<...>(a.begin(), a.end(), b.begin(), b.end(), outputIter) is+ * guaranteed to produce {{1, 1}, {1, 2}, {2, 2}, {2, 3}, {3, 3}}. That is,+ * if comp(it_a, it_b) == comp(it_b, it_a) == false, we first insert the element+ * from a.+ */++#pragma once++#include <algorithm>++namespace folly {++template <class InputIt1, class InputIt2, class OutputIt, class Compare>+OutputIt merge(+    InputIt1 first1,+    InputIt1 last1,+    InputIt2 first2,+    InputIt2 last2,+    OutputIt d_first,+    Compare comp) {+  for (; first1 != last1; ++d_first) {+    if (first2 == last2) {+      return std::copy(first1, last1, d_first);+    }+    if (comp(*first2, *first1)) {+      *d_first = *first2;+      ++first2;+    } else {+      *d_first = *first1;+      ++first1;+    }+  }+  return std::copy(first2, last2, d_first);+}++template <class InputIt1, class InputIt2, class OutputIt>+OutputIt merge(+    InputIt1 first1,+    InputIt1 last1,+    InputIt2 first2,+    InputIt2 last2,+    OutputIt d_first) {+  for (; first1 != last1; ++d_first) {+    if (first2 == last2) {+      return std::copy(first1, last1, d_first);+    }+    if (*first2 < *first1) {+      *d_first = *first2;+      ++first2;+    } else {+      *d_first = *first1;+      ++first1;+    }+  }+  return std::copy(first2, last2, d_first);+}++} // namespace folly
+ folly/folly/container/RegexMatchCache.cpp view
@@ -0,0 +1,531 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#include <folly/container/RegexMatchCache.h>++#include <folly/portability/Windows.h>++#include <ostream>++#include <boost/regex.hpp>+#include <fmt/format.h>+#include <glog/logging.h>++#include <folly/MapUtil.h>+#include <folly/String.h>+#include <folly/container/Reserve.h>+#include <folly/ssl/OpenSSLHash.h>+#include <folly/synchronization/AtomicUtil.h>++namespace folly {++static std::string quote(std::string_view const s) {+  return fmt::format("\"{}\"", cEscape<std::string>(s));+}++RegexMatchCacheKey::data_type RegexMatchCacheKey::init(+    std::string_view const regex) noexcept {+  data_type data;+  folly::ssl::OpenSSLHash::sha256(range(data), StringPiece(regex));+  return data;+}++class RegexMatchCache::RegexObject {+ private:+  boost::regex object;++ public:+  explicit RegexObject(std::string_view const regex)+      : object{std::string(regex)} {}++  bool operator()(std::string_view const string) const {+    return boost::regex_match(std::string(string), object);+  }+};++void RegexMatchCache::repair() noexcept {+  stringQueueReverse_.clear();+  stringQueueForward_.clear();+  for (auto& [match, entry] : cacheMatchToRegex_) {+    entry.regexes.reset();+  }+  cacheRegexToMatch_.clear();+  regexVector_.clear();+}++RegexMatchCache::KeyMap::~KeyMap() = default;++void RegexMatchCache::InspectView::print(std::ostream& o) const {+  auto regexToString = [&](auto const& regex) {+    return quote(keys_.lookup(regex));+  };+  o << "cache-regex-to-match[" << ref_.cacheRegexToMatch_.size()+    << "]:" << std::endl;+  for (auto const& [regex, entry] : ref_.cacheRegexToMatch_) {+    o << "  " << regexToString(regex) << ":" << std::endl;+    for (auto const match : entry.matches) {+      o << "    " << quote(*match) << std::endl;+    }+  }+  o << "cache-match-to-regex[" << ref_.cacheMatchToRegex_.size()+    << "]:" << std::endl;+  for (auto const& [match, entry] : ref_.cacheMatchToRegex_) {+    o << "  " << quote(*match) << ":" << std::endl;+    for (auto const regexi : entry.regexes.as_index_set_view()) {+      auto const regex = ref_.regexVector_.value_at_index(regexi);+      o << "    " << regexToString(*regex) << std::endl;+    }+  }+  o << "string-queue-forward[" << ref_.stringQueueForward_.size()+    << "]:" << std::endl;+  for (auto const& [string, entry] : ref_.stringQueueForward_) {+    o << "  " << quote(*string) << ":" << std::endl;+    for (auto const regexi : entry.regexes.as_index_set_view()) {+      auto const regex = ref_.regexVector_.value_at_index(regexi);+      o << "    " << regexToString(*regex) << std::endl;+    }+  }+  o << "string-queue-reverse[" << ref_.stringQueueReverse_.size()+    << "]:" << std::endl;+  for (auto const& [regex, entry] : ref_.stringQueueReverse_) {+    o << "  " << regexToString(*regex) << ":" << std::endl;+    for (auto const string : entry.strings) {+      o << "    " << quote(*string) << std::endl;+    }+  }+}++struct RegexMatchCache::ConsistencyReportMatcher::state {+  std::unordered_map<regex_key, RegexObject> cache;+};++RegexMatchCache::ConsistencyReportMatcher::ConsistencyReportMatcher()+    : state_{std::make_unique<state>()} {}++RegexMatchCache::ConsistencyReportMatcher::~ConsistencyReportMatcher() =+    default;++bool RegexMatchCache::ConsistencyReportMatcher::match(+    KeyMap const& keys, regex_key const regex, string_pointer const string) {+  auto const [iter, inserted] =+      state_->cache.try_emplace(regex, keys.lookup(regex));+  return iter->second(*string);+}++RegexMatchCache::RegexMatchCache() noexcept = default;++RegexMatchCache::~RegexMatchCache() = default;++std::vector<std::string_view> RegexMatchCache::getRegexList(+    KeyMap const& keys) const {+  std::vector<std::string_view> result;+  result.reserve(cacheRegexToMatch_.size());+  for (auto const& [regex, entry] : cacheRegexToMatch_) {+    result.push_back(keys.lookup(regex));+  }+  return result;+}++std::vector<RegexMatchCache::string_pointer> RegexMatchCache::getStringList()+    const {+  std::vector<string_pointer> result;+  result.reserve(cacheMatchToRegex_.size());+  for (auto const& [match, entry] : cacheMatchToRegex_) {+    result.push_back(match);+  }+  return result;+}++void RegexMatchCache::consistency(+    ConsistencyReportMatcher& matcher,+    KeyMap const& keys,+    FunctionRef<void(std::string)> const report) const {+  auto const q = [](std::string_view const s) { return quote(s); };+  auto const h = report;++  auto const r = [&](regex_key const r) { return keys.lookup(r); };++  if (cacheRegexToMatch_.empty() || cacheMatchToRegex_.empty()) {+    if (!stringQueueForward_.empty()) {+      h("string-queue-forward not empty");+    }+    if (!stringQueueReverse_.empty()) {+      h("string-queue-reverse not empty");+    }+  }++  //  check that caches are accurate+  //  check that caches are bidi-consistent+  //  check that missing cache entries are found in string-queues+  for (auto const& [regex, rtmentry] : cacheRegexToMatch_) {+    auto const regexs = r(regex);+    auto const regexi = regexVector_.index_of_value(&regex);+    for (auto const& [match, mtrentry] : cacheMatchToRegex_) {+      auto const rtmcontains = rtmentry.matches.contains(match);+      auto const mtrcontains = mtrentry.regexes.get_value(regexi);+      if (rtmcontains && !mtrcontains) {+        h(fmt::format( //+            "cache-regex-to-match[{}] wild {}",+            q(regexs),+            q(*match)));+      }+      if (mtrcontains && !rtmcontains) {+        h(fmt::format( //+            "cache-match-to-regex[{}] wild {}",+            q(*match),+            q(regexs)));+      }+      auto const result = matcher.match(keys, regex, match);+      auto const queues = result && (!rtmcontains || !mtrcontains);+      auto const sqfptr =+          !queues ? nullptr : get_ptr(stringQueueForward_, match);+      auto const sqfhas = sqfptr && sqfptr->regexes.get_value(regexi);+      auto const sqrptr =+          !queues ? nullptr : get_ptr(stringQueueReverse_, &regex);+      auto const sqrhas = sqrptr && sqrptr->strings.contains(match);+      if (rtmcontains && !result) {+        h(fmt::format( //+            "cache-regex-to-match[{}] wild {}",+            q(regexs),+            q(*match)));+      }+      if (result && !rtmcontains) {+        if (!sqfhas || !sqrhas) {+          h(fmt::format( //+              "cache-regex-to-match[{}] missing {}",+              q(regexs),+              q(*match)));+        }+      }+      if (mtrcontains && !result) {+        h(fmt::format( //+            "cache-match-to-regex[{}] wild {}",+            q(*match),+            q(regexs)));+      }+      if (result && !mtrcontains) {+        if (!sqfhas || !sqrhas) {+          h(fmt::format( //+              "cache-match-to-regex[{}] missing {}",+              q(*match),+              q(regexs)));+        }+      }+    }+  }++  //  check that string-queues are bidi-consistent+  //  check that string-queue keys are subsets of caches+  //  check that string-queue entries are not in caches+  for (auto const& [string, entry] : stringQueueForward_) {+    auto const mtrptr = get_ptr(cacheMatchToRegex_, string);+    if (!mtrptr) {+      h(fmt::format( //+          "string-queue-forward has string[{}]",+          q(*string)));+    }+    for (auto const regexi : entry.regexes.as_index_set_view()) {+      auto const regex = regexVector_.value_at_index(regexi);+      auto const sqrptr = get_ptr(stringQueueReverse_, regex);+      if (!sqrptr) {+        h(fmt::format( //+            "string-queue-reverse none regex[{}]",+            q(r(*regex))));+      } else if (!sqrptr->strings.contains(string)) {+        h(fmt::format( //+            "string-queue-reverse[{}] none string[{}]",+            q(r(*regex)),+            q(*string)));+      }+      auto const mtrhas = mtrptr && mtrptr->regexes.get_value(regexi);+      auto const rtmptr = get_ptr(cacheRegexToMatch_, *regex);+      auto const rtmhas = rtmptr && rtmptr->matches.count(string);+      if (mtrhas || rtmhas) {+        h(fmt::format( //+            "string-queue-forward[{}] has regex[{}]",+            q(*string),+            q(r(*regex))));+      }+    }+  }+  for (auto const& [regex, entry] : stringQueueReverse_) {+    auto const regexi = regexVector_.index_of_value(regex);+    auto const rtmptr = get_ptr(cacheRegexToMatch_, *regex);+    for (auto const string : entry.strings) {+      auto const sqfptr = get_ptr(stringQueueForward_, string);+      if (!sqfptr) {+        h(fmt::format( //+            "string-queue-forward none string[{}]",+            q(*string)));+      } else if (!sqfptr->regexes.get_value(regexi)) {+        h(fmt::format( //+            "string-queue-forward[{}] none regex[{}]",+            q(*string),+            q(r(*regex))));+      }+      auto const mtrptr = get_ptr(cacheMatchToRegex_, string);+      auto const mtrhas = mtrptr && mtrptr->regexes.get_value(regexi);+      auto const rtmhas = rtmptr && rtmptr->matches.count(string);+      if (mtrhas || rtmhas) {+        h(fmt::format( //+            "string-queue-reverse[{}] has string[{}]",+            q(r(*regex)),+            q(*string)));+      }+    }+  }+}++bool RegexMatchCache::hasRegex(regex_key const& regex) const noexcept {+  return cacheRegexToMatch_.contains(regex);+}++void RegexMatchCache::addRegex(regex_key const& regex) {+  auto const [rtmiter, rtminserted] = cacheRegexToMatch_.try_emplace(regex);+  if (!rtminserted) {+    return;+  }+  auto guard = makeGuard(std::bind(&RegexMatchCache::repair, this));+  auto const regexp = &rtmiter->first;+  auto const regexi = regexVector_.insert_value(regexp).first;+  if (cacheMatchToRegex_.empty()) {+    guard.dismiss();+    return;+  }+  auto const [sqriter, sqrinserted] = stringQueueReverse_.try_emplace(regexp);+  CHECK(sqrinserted) << "string already in string-queue-reverse";+  auto& sqrentry = sqriter->second;+  for (auto const& [string, mtrentry] : cacheMatchToRegex_) {+    stringQueueForward_[string].regexes.set_value(regexi, true);+    sqrentry.strings.insert(string);+  }+  guard.dismiss();+}++void RegexMatchCache::eraseRegex(regex_key const& regex) {+  auto const rtmiter = cacheRegexToMatch_.find(regex);+  if (rtmiter == cacheRegexToMatch_.end()) {+    return;+  }+  auto guard = makeGuard(std::bind(&RegexMatchCache::repair, this));+  auto const regexp = &rtmiter->first;+  auto const regexi = regexVector_.index_of_value(regexp);+  for (auto const match : rtmiter->second.matches) {+    get_ptr(cacheMatchToRegex_, match)->regexes.set_value(regexi, false);+  }+  auto const sqriter = stringQueueReverse_.find(regexp);+  if (sqriter != stringQueueReverse_.end()) {+    for (auto const string : sqriter->second.strings) {+      auto const sqfiter = stringQueueForward_.find(string);+      CHECK(sqfiter != stringQueueForward_.end());+      sqfiter->second.regexes.set_value(regexi, false);+      if (sqfiter->second.regexes.as_index_set_view().empty()) {+        stringQueueForward_.erase(sqfiter);+      }+    }+    stringQueueReverse_.erase(sqriter);+  }+  regexVector_.erase_value(regexp);+  cacheRegexToMatch_.erase(rtmiter);+  guard.dismiss();+}++bool RegexMatchCache::hasString(string_pointer const string) const noexcept {+  return //+      cacheMatchToRegex_.contains(string) ||+      stringQueueForward_.contains(string);+}++void RegexMatchCache::addString(string_pointer const string) {+  //  return-early if already added+  if (!cacheMatchToRegex_.try_emplace(string).second) {+    return;+  }+  if (cacheRegexToMatch_.empty()) {+    return;+  }+  auto guard = makeGuard(std::bind(&RegexMatchCache::repair, this));+  auto const [sqfiter, sqfinserted] = stringQueueForward_.try_emplace(string);+  CHECK(sqfinserted) << "string already in string-queue-forward";++  //  add to string-queue-forward and string-queue-reverse+  auto& sqfentry = sqfiter->second;+  for (auto const& [regex, entry] : cacheRegexToMatch_) {+    auto regexi = regexVector_.index_of_value(&regex);+    sqfentry.regexes.set_value(regexi, true);+    stringQueueReverse_[&regex].strings.insert(string);+  }+  guard.dismiss();+}++void RegexMatchCache::eraseString(string_pointer const string) {+  auto guard = makeGuard(std::bind(&RegexMatchCache::repair, this));++  //  erase from string-queue-forward and string-queue-reverse+  auto const sqfiter = stringQueueForward_.find(string);+  if (sqfiter != stringQueueForward_.end()) {+    for (auto const regexi : sqfiter->second.regexes.as_index_set_view()) {+      auto const regexp = regexVector_.value_at_index(regexi);+      auto const sqriter = stringQueueReverse_.find(regexp);+      sqriter->second.strings.erase(string);+      if (sqriter->second.strings.empty()) {+        stringQueueReverse_.erase(sqriter);+      }+    }+    stringQueueForward_.erase(sqfiter);+  }++  //  erase from cache-regex-to-match and cache-match-to-regex+  auto const mtriter = cacheMatchToRegex_.find(string);+  if (mtriter != cacheMatchToRegex_.end()) {+    for (auto const regexi : mtriter->second.regexes.as_index_set_view()) {+      auto const regex = regexVector_.value_at_index(regexi);+      get_ptr(cacheRegexToMatch_, *regex)->matches.erase(string);+    }+    cacheMatchToRegex_.erase(mtriter);+  }++  guard.dismiss();+}++std::vector<std::string const*> RegexMatchCache::findMatchesUncached(+    std::string_view const regex) const {+  std::vector<std::string const*> result;+  RegexObject robject{regex};+  for (auto const& [string, _] : cacheMatchToRegex_) {+    if (robject(*string)) {+      result.push_back(string);+    }+  }+  return result;+}++bool RegexMatchCache::isReadyToFindMatches(+    regex_key const& regex) const noexcept {+  auto const rtmiter = cacheRegexToMatch_.find(regex);+  return //+      rtmiter != cacheRegexToMatch_.end() &&+      !stringQueueReverse_.contains(&rtmiter->first);+}++void RegexMatchCache::prepareToFindMatches(regex_key_and_view const& regex) {+  auto guard = makeGuard(std::bind(&RegexMatchCache::repair, this));+  auto const [rtmiter, inserted] = cacheRegexToMatch_.try_emplace(regex);+  auto const regexp = &rtmiter->first;+  auto& rtmentry = rtmiter->second;+  auto const [regexi, rvinserted] = regexVector_.insert_value(regexp);+  CHECK_EQ(rvinserted, inserted);++  if (inserted) {+    //  evaluate new regex over matches+    CHECK(!stringQueueReverse_.contains(regexp));+    if (cacheMatchToRegex_.empty()) {+      CHECK(stringQueueForward_.empty());+      CHECK(stringQueueReverse_.empty());+      guard.dismiss();+      return;+    }+    RegexObject robject{regex};+    for (auto& [string, mtrentry] : cacheMatchToRegex_) {+      if (robject(*string)) {+        rtmentry.matches.insert(string);+        mtrentry.regexes.set_value(regexi, true);+      }+    }+  } else {+    //  evaluate old regex over queue+    auto const sqriter = stringQueueReverse_.find(regexp);+    if (sqriter == stringQueueReverse_.end()) {+      //  was actually ready-to-find-matches for regex+      guard.dismiss();+      return;+    }+    auto const strings = std::move(sqriter->second.strings);+    CHECK(!strings.empty());+    stringQueueReverse_.erase(sqriter);+    RegexObject robject{regex};+    for (auto const string : strings) {+      auto const sqfiter = stringQueueForward_.find(string);+      CHECK(sqfiter != stringQueueForward_.end());+      CHECK(sqfiter->second.regexes.get_value(regexi));+      sqfiter->second.regexes.set_value(regexi, false);+      if (sqfiter->second.regexes.as_index_set_view().empty()) {+        stringQueueForward_.erase(sqfiter);+      }++      auto const mtriter = cacheMatchToRegex_.find(string);+      CHECK(mtriter != cacheMatchToRegex_.end());+      auto& mtrentry = mtriter->second;+      if (robject(*string)) {+        rtmentry.matches.insert(string);+        mtrentry.regexes.set_value(regexi, true);+      }+    }+  }+  guard.dismiss();+}++RegexMatchCache::FindMatchesUnsafeResult RegexMatchCache::findMatchesUnsafe(+    regex_key const& regex, time_point const now) const {+  if (kIsDebug && !isReadyToFindMatches(regex)) {+    throw std::logic_error("not ready to find matches");+  }+  auto const& rtmentry = cacheRegexToMatch_.at(regex);+  atomic_fetch_modify(+      rtmentry.accessed_at,+      [now](auto const val) { return std::max(now, val); },+      std::memory_order_relaxed);+  return {rtmentry.matches};+}++std::vector<RegexMatchCache::string_pointer> RegexMatchCache::findMatches(+    regex_key const& regex, time_point const now) const {+  auto const matches = findMatchesUnsafe(regex, now);+  return {matches.begin(), matches.end()};+}++bool RegexMatchCache::hasItemsToPurge(time_point const expiry) const noexcept {+  for (auto const& [regex, entry] : cacheRegexToMatch_) {+    auto const accessed_at = entry.accessed_at.load(std::memory_order_relaxed);+    if (accessed_at <= expiry) {+      return true;+    }+  }+  return false;+}++void RegexMatchCache::clear() {+  std::exchange(stringQueueReverse_, {});+  std::exchange(stringQueueForward_, {});+  std::exchange(cacheMatchToRegex_, {});+  std::exchange(cacheRegexToMatch_, {});+  std::exchange(regexVector_, {});+}++void RegexMatchCache::purge(time_point const expiry) {+  std::vector<regex_key> regexes;+  for (auto const& [regex, entry] : cacheRegexToMatch_) {+    auto const accessed_at = entry.accessed_at.load(std::memory_order_relaxed);+    if (accessed_at <= expiry) {+      regexes.push_back(regex);+    }+  }+  for (auto const& regex : regexes) {+    eraseRegex(regex);+  }+}++} // namespace folly
+ folly/folly/container/RegexMatchCache.h view
@@ -0,0 +1,702 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <array>+#include <cassert>+#include <chrono>+#include <iosfwd>+#include <string>+#include <string_view>+#include <tuple>+#include <type_traits>+#include <unordered_map>+#include <vector>++#include <folly/Chrono.h>+#include <folly/Function.h>+#include <folly/container/F14Map.h>+#include <folly/container/F14Set.h>+#include <folly/container/Reserve.h>+#include <folly/container/span.h>+#include <folly/lang/Bits.h>++namespace folly {++/// RegexMatchCacheDynamicBitset+///+/// A dynamic bitset for use within, and optimized for, RegexMatchCache.+/// * Small, having the same size and alignment as a pointer.+/// * Optimistically non-allocating, using in-situ storage for small bitsets.+///+/// Intended for use only within RegexMatchCache.+///+/// Incomplete as a generic container.+class RegexMatchCacheDynamicBitset {+ private:+  template <typename Word>+  struct bit_span {+    Word* data;+    size_t size;++    bit_span(Word* const data_, size_t const size_) noexcept+        : data{data_}, size{size_} {}+    bit_span(bit_span const&) = default;+    bit_span& operator=(bit_span const&) = default;++    auto as_tuple() const noexcept { return std::tuple{data, size}; }++    friend bool operator==(bit_span const& a, bit_span const& b) noexcept {+      return a.as_tuple() == b.as_tuple();+    }+    friend bool operator!=(bit_span const& a, bit_span const& b) noexcept {+      return a.as_tuple() != b.as_tuple();+    }+  };++ public:+  RegexMatchCacheDynamicBitset() = default;++  RegexMatchCacheDynamicBitset(RegexMatchCacheDynamicBitset const&) = delete;+  RegexMatchCacheDynamicBitset(RegexMatchCacheDynamicBitset&& that) noexcept+      : data_{std::exchange(that.data_, {})} {}++  ~RegexMatchCacheDynamicBitset() { reset_(); }++  void operator=(RegexMatchCacheDynamicBitset const&) = delete;+  RegexMatchCacheDynamicBitset& operator=(+      RegexMatchCacheDynamicBitset&& that) noexcept {+    reset_();+    data_ = std::exchange(that.data_, {});+    return *this;+  }++  bool get_value(size_t const index) const noexcept {+    auto data = get_bit_span_();+    if (!(index < data.size)) {+      return false;+    }+    return get_value_(data, index);+  }++  void set_value(size_t const index, bool const value) {+    constexpr auto wordbits = sizeof(uintptr_t) * 8;+    auto data = get_bit_span_();++    if (!(index < data.size) ||+        (data.size == wordbits && index == wordbits - 1)) {+      if (!value) {+        return;+      }+      data = reserve_(index);+    }+    assert(index < data.size);+    set_value_(data, index, value);+  }++  void reset() noexcept { reset_(); }++  class index_set_view {+   private:+    friend RegexMatchCacheDynamicBitset;+    bit_span<uintptr_t const> bitset_;++    explicit index_set_view(RegexMatchCacheDynamicBitset const& bitset) noexcept+        : bitset_{bitset.get_bit_span_()} {}++   public:+    using value_type = size_t;++    class const_iterator {+     public:+      using value_type = size_t;+      using difference_type = ptrdiff_t;+      using pointer = void;+      using iterator_category = std::forward_iterator_tag;++      struct reference {+       private:+        friend class const_iterator;++        size_t const index_;++        explicit reference(size_t const index) noexcept : index_{index} {}++       public:+        operator size_t() const noexcept { return index_; }+      };++     private:+      using self = const_iterator;++      bit_span<uintptr_t const> const data_;+      size_t index_;++      size_t ceil_valid_index(size_t index) const noexcept {+        constexpr auto wordbits = sizeof(uintptr_t) * 8;+        while (index < data_.size) {+          auto const wordidx = index / wordbits;+          auto const wordoff = index % wordbits;+          if (auto const word = data_.data[wordidx] >> wordoff) {+            return index + findFirstSet(word) - 1;+          }+          index = (wordidx + 1) * wordbits;+        }+        return index;+      }++     public:+      const_iterator(+          bit_span<uintptr_t const> const data, size_t const index) noexcept+          : data_{data}, index_{ceil_valid_index(index)} {}++      reference operator*() const noexcept { return reference{index_}; }+      const_iterator& operator++() noexcept {+        index_ = ceil_valid_index(index_ + 1);+        return *this;+      }++      friend bool operator==(self const& a, self const& b) noexcept {+        return a.index_ == b.index_;+      }+      friend bool operator!=(self const& a, self const& b) noexcept {+        return a.index_ != b.index_;+      }+    };++    const_iterator begin() const noexcept { return const_iterator{bitset_, 0}; }+    const_iterator end() const noexcept {+      return const_iterator{bitset_, bitset_.size};+    }++    bool empty() const noexcept { return begin() == end(); }+  };++  index_set_view as_index_set_view() const noexcept {+    return index_set_view{*this};+  }++ private:+  bool has_capacity_(size_t const index) const noexcept {+    constexpr auto wordbits = sizeof(uintptr_t) * 8;+    auto const buf = get_bit_span_();+    return index < buf.size && !(buf.size == wordbits && index == wordbits - 1);+  }++  bit_span<uintptr_t> reserve_(size_t const index) {+    assert(!has_capacity_(index));+    constexpr auto wordbits = sizeof(uintptr_t) * 8;+    constexpr auto minsize = wordbits * 2; // min growth from in-situ to on-heap+    auto const newsize = std::max(strictNextPowTwo(index), minsize);+    assert(newsize >= minsize);+    assert(newsize % wordbits == 0);+    auto const newdata = new uintptr_t[newsize / 8];+    auto const buf = get_bit_span_();+    auto const buf2size = nextPowTwo(buf.size);+    std::memcpy(newdata, buf.data, buf2size / 8);+    std::memset(newdata + buf2size / wordbits, 0, (newsize - buf2size) / 8);+    if (!(to_signed(data_) < 0)) {+      auto const data = new bit_span<uintptr_t>{newdata, newsize};+      assert(!(reinterpret_cast<uintptr_t>(data) & 1));+      data_ = (reinterpret_cast<uintptr_t>(data) >> 1) | ~(~uintptr_t(0) >> 1);+      return *data;+    } else {+      auto const data = reinterpret_cast<bit_span<uintptr_t>*>(data_ << 1);+      delete[] data->data;+      *data = {newdata, newsize};+      return *data;+    }+  }++  void reset_() {+    if (!(to_signed(data_) < 0)) {+      data_ = 0;+    } else {+      auto const data = reinterpret_cast<bit_span<uintptr_t>*>(data_ << 1);+      delete[] data->data;+      delete data;+      data_ = 0;+    }+  }++  template <typename Word>+  static bool get_value_(+      bit_span<Word> const buf, size_t const index) noexcept {+    assert(index < buf.size);+    constexpr auto wordbits = sizeof(Word) * 8;+    auto const wordidx = index / wordbits;+    auto const wordoff = index % wordbits;+    auto const mask = Word(1) << wordoff;+    auto& word = buf.data[wordidx];+    return word & mask;+  }++  template <typename Word>+  static void set_value_(+      bit_span<Word> const buf, size_t const index, bool const value) noexcept {+    assert(index < buf.size);+    constexpr auto wordbits = sizeof(Word) * 8;+    assert(buf.size != wordbits || index != wordbits - 1);+    auto const wordidx = index / wordbits;+    auto const wordoff = index % wordbits;+    auto const mask = Word(1) << wordoff;+    auto& word = buf.data[wordidx];+    word = value ? word | mask : word & ~mask;+  }++  bit_span<uintptr_t const> get_bit_span_() const noexcept {+    if (!(to_signed(data_) < 0)) {+      return {&data_, sizeof(data_) * 8};+    } else {+      return *reinterpret_cast<bit_span<uintptr_t const> const*>(data_ << 1);+    }+  }++  bit_span<uintptr_t> get_bit_span_() noexcept {+    if (!(to_signed(data_) < 0)) {+      return {&data_, sizeof(data_) * 8};+    } else {+      return *reinterpret_cast<bit_span<uintptr_t> const*>(data_ << 1);+    }+  }++  uintptr_t data_{};+};++/// RegexMatchCacheIndexedVector+///+/// An indexed vector, which is a vector for which the index of any element can+/// be found efficiently.+///+/// Intended for use only within RegexMatchCache.+///+/// Incomplete as a generic container.+template <typename Value>+class RegexMatchCacheIndexedVector {+ public:+  size_t size() const noexcept { return forward_.size(); }++  bool contains_index(size_t index) const noexcept {+    return reverse_.contains(index);+  }++  bool contains_value(Value const& value) const noexcept {+    return forward_.contains(value);+  }++  std::pair<size_t, bool> insert_value(Value const& value) {+    auto [iter, inserted] = forward_.try_emplace(value);+    if (inserted) {+      auto rollback_forward = makeGuard([&, iter_ = iter] {+        forward_.erase(iter_);+      });+      if (free_.capacity() < forward_.size()) {+        grow_capacity_by(free_, forward_.size() - free_.size());+      }+      assert(!(free_.capacity() < forward_.size()));+      auto const from_free = !free_.empty();+      auto const index = from_free ? free_.back() : forward_.size() - 1;+      from_free ? free_.pop_back() : void();+      iter->second = index;+      auto rollback_free = makeGuard([&] {+        from_free ? free_.push_back(index) : void();+      });+      assert(!reverse_.contains(index));+      reverse_[index] = value;+      rollback_free.dismiss();+      rollback_forward.dismiss();+    }+    return {iter->second, inserted};+  }++  bool erase_value(Value const& value) noexcept {+    auto iter = forward_.find(value);+    if (iter == forward_.end()) {+      return false;+    }+    assert(free_.size() < free_.capacity());+    auto index = iter->second;+    free_.push_back(index);+    forward_.erase(iter);+    reverse_.erase(index);+    return true;+  }++  void clear() noexcept {+    reverse_.clear();+    forward_.clear();+    free_.clear();+  }++  Value const& value_at_index(size_t index) const { return reverse_.at(index); }++  size_t index_of_value(Value const& value) const { return forward_.at(value); }++  class forward_view {+   private:+    friend RegexMatchCacheIndexedVector;+    using map_t = folly::F14FastMap<Value, size_t>;+    map_t const& map;+    explicit forward_view(map_t const& map_) noexcept : map{map_} {}++   public:+    using value_type = typename map_t::value_type;+    using size_type = typename map_t::size_type;+    using iterator = typename map_t::const_iterator;++    size_t size() const noexcept { return map.size(); }+    iterator begin() const noexcept { return map.begin(); }+    iterator end() const noexcept { return map.end(); }+  };++  forward_view as_forward_view() const noexcept {+    return forward_view{forward_};+  }++ private:+  std::vector<size_t> free_;+  folly::F14FastMap<Value, size_t> forward_;+  folly::F14FastMap<size_t, Value> reverse_;+};++/// RegexMatchCacheKey+///+/// A key derived from a string. Used with RegexMatchCache.+///+/// Intended for use only with RegexMatchCache.+///+/// Incomplete as a generic facility.+class RegexMatchCacheKey {+ private:+  using self = RegexMatchCacheKey;++  static inline constexpr size_t data_size = 32;+  static inline constexpr size_t data_align = alignof(size_t);++  using data_type = std::array<unsigned char, data_size>;++  alignas(data_align) data_type const data_;++  static data_type init(std::string_view regex) noexcept;++  template <typename T, size_t E, typename V = std::remove_cv_t<T>>+  static constexpr bool is_span_compatible_v = //+      !std::is_volatile_v<T> && //+      std::is_integral_v<V> && //+      std::is_unsigned_v<V> && //+      !std::is_same_v<bool, V> && //+      !std::is_same_v<char, V> && //+      alignof(V) <= data_align && //+      (E == data_size / sizeof(T) || E == dynamic_extent);++ public:+  explicit RegexMatchCacheKey(std::string_view regex) noexcept+      : data_{init(regex)} {}++  template <+      typename T,+      std::size_t E,+      std::enable_if_t<is_span_compatible_v<T, E>, int> = 0>+  explicit operator span<T const, E>() const noexcept {+    return {reinterpret_cast<T const*>(data_.data()), E};+  }++  friend auto operator==(self const& a, self const& b) noexcept {+    return a.data_ == b.data_;+  }+  friend auto operator!=(self const& a, self const& b) noexcept {+    return a.data_ != b.data_;+  }+};++} // namespace folly++namespace std {++template <>+struct hash<::folly::RegexMatchCacheKey> {+  using folly_is_avalanching = std::true_type;++  size_t operator()(::folly::RegexMatchCacheKey const& key) const noexcept {+    return ::folly::span<size_t const>{key}[0];+  }+};++} // namespace std++namespace folly {++/// RegexMatchCacheKeyAndView+///+/// A composite key and view derived from a string. Used with RegexMatchCache.+///+/// Intended for use only with RegexMatchCache.+///+/// Incomplete as a generic facility.+class RegexMatchCacheKeyAndView {+ public:+  using regex_key = RegexMatchCacheKey;++  regex_key const key;+  std::string_view const view;++  explicit RegexMatchCacheKeyAndView(std::string_view regex) noexcept+      : key{regex}, view{regex} {}++  /* implicit */ operator RegexMatchCacheKey const&() const noexcept {+    return key;+  }+  /* implicit */ operator std::string_view const&() const noexcept {+    return view;+  }++ private:+  RegexMatchCacheKeyAndView(+      regex_key const& k, std::string_view const v) noexcept+      : key{k}, view{v} {}+};++/// RegexMatchCache+///+/// A cache around boost::regex_match(string, regex).+///+/// For efficiency, assumes several constraints and makes several guarantees.+///+/// The data structure owns regexes but does not own strings. The lifetimes of+/// all strings in the cache must surround their additions to the cache and+/// their subsequent removals from the cache or destruction of the cache.+///+/// The data structure is in two parts:+/// * A bidirectional match-cache contains all known matches.+/// * a bidirectional string-queue contains unknown, hypothetical matches.+///+/// Cached lookup operates only over the match-cache. When the string-queue for+/// a given regex is not empty, that regex is said to be uncoalesced. Cached+/// lookups are not permitted for an uncoalesced regex; that regex must first be+/// coalesced.+///+/// Addition of a string adds the string to the string-queue corresponding to+/// all known regexes. It does not perform any regex-match operations.+///+/// Addition and coalesce of a regex performs regex-matches for that regex only.+/// The string-queue for the given regex is removed and all elements matched+/// against the regex, and matching strings are added to the match-cache.+///+/// Lookup must follow a pattern like this:+///+///    if (!cache.isReadyToFindMatches(regex)) { // const+///      cache.prepareToFindMatches(regex); // non-const+///    }+///    auto matches = cache.findMatches(regex); // const+///+/// This is to support concurrent lookups, where the cache is protected by a+/// shared mutex.+///+/// The data structure is exception-safe in a sense. If an exception is thrown+/// within any non-const member function and escapes, the data structure may+/// purge all cached regexes while leaving all strings. In most such member+/// functions, only a memory-allocation failure would cause an exception to be+/// thrown. But in prepareToFindMatches, the provided regex may be syntactically+/// invalid and parsing it may throw, or it may be pathological and evaluating+/// it over a string may throw. In any event, the resolution is to clear out all+/// added regexes and to leave only the added strings. The reason is that this+/// resolution is simple and likely to be correct, while any other mechanism+/// would be complex and would be likely to have bugs.+class RegexMatchCache {+ public:+  using clock = folly::chrono::coarse_steady_clock;+  using time_point = clock::time_point;++  using regex_key = RegexMatchCacheKey;+  using regex_key_and_view = RegexMatchCacheKeyAndView;++ private:+  using regex_pointer = regex_key const*;+  using string_pointer = std::string const*;++  class RegexObject;++  struct RegexToMatchEntry : MoveOnly {+    mutable std::atomic<time_point> accessed_at{};++    folly::F14VectorSet<string_pointer> matches;+  };++  struct MatchToRegexEntry : MoveOnly {+    RegexMatchCacheDynamicBitset regexes;+  };++  struct StringQueueForwardEntry : MoveOnly {+    RegexMatchCacheDynamicBitset regexes;+  };++  struct StringQueueReverseEntry : MoveOnly {+    folly::F14VectorSet<string_pointer> strings;+  };++  RegexMatchCacheIndexedVector<regex_pointer> regexVector_;++  /// cacheRegexToMatch_+  ///+  /// A match-cache map from regexes to the sets of matching strings.+  ///+  /// The set of matching strings for a given regex may be incomplete. This+  /// happens when strings are added to the universe but have not yet been+  /// coalesced for the given regex. The set of uncoalesced strings for a+  /// given regex is in stringQueueReverse_.+  ///+  /// For each regex, includes a last-accessed-at timestamp. This timestamp+  /// is used when purging old regexes from the cache, for the caller's own+  /// definition of old.+  folly::F14NodeMap<regex_key, RegexToMatchEntry> cacheRegexToMatch_;++  /// cacheMatchToRegex_+  ///+  /// A match-cache map from strings to the sets of matching regexes.+  ///+  /// The set of matching regexes for a given string may be incomplete. This+  /// happens when strings are added to the universe but have not yet been+  /// coalesced for all regexes in the universe. The set of regexes for which+  /// a given string has not yet been coalesced is in stringQueueForward_.+  folly::F14FastMap<string_pointer, MatchToRegexEntry> cacheMatchToRegex_;++  /// stringQueueForward_+  ///+  /// A pending-coalesce map from strings to regexes for which the strings have+  /// not yet been coalesced, that is, for which it is not yet known that the+  /// strings do or do not match the given regexes.+  ///+  /// In a steady-state when all strings have been coalesced for all regexes,+  /// this map would be empty.+  folly::F14FastMap<string_pointer, StringQueueForwardEntry>+      stringQueueForward_;++  /// stringQueueReverse_+  ///+  /// A pending-coalesce map from regexes to strings which have not yet been+  /// coalesced for the given regex, that is, for which it is not yet known that+  /// the strings do or do not match the given regexes.+  ///+  /// In a steady-state when all strings have been coalesced for all regexes,+  /// this map would be empty.+  folly::F14FastMap<regex_pointer, StringQueueReverseEntry> stringQueueReverse_;++  void repair() noexcept;++ public:+  class KeyMap {+   public:+    using regex_key = RegexMatchCacheKey;+    using regex_key_and_view = RegexMatchCacheKeyAndView;++    virtual ~KeyMap() = 0;++    virtual std::string_view lookup(regex_key const& regex) const = 0;+  };++  class InspectView {+    friend RegexMatchCache;++   private:+    RegexMatchCache const& ref_;+    KeyMap const& keys_;++    explicit InspectView(+        RegexMatchCache const& ref, KeyMap const& keys) noexcept+        : ref_{ref}, keys_{keys} {}++    void print(std::ostream& o) const;++   public:+    friend std::ostream& operator<<(std::ostream& o, InspectView const view) {+      return (view.print(o), o);+    }+  };++  class ConsistencyReportMatcher {+   private:+    struct state;+    std::unique_ptr<state> state_;++   public:+    using regex_key = RegexMatchCache::regex_key;+    using regex_key_and_view = RegexMatchCache::regex_key_and_view;+    using string_pointer = RegexMatchCache::string_pointer;++    ConsistencyReportMatcher();+    virtual ~ConsistencyReportMatcher();++    virtual bool match(+        KeyMap const& keys, regex_key regex, string_pointer string);+  };++  class FindMatchesUnsafeResult {+   private:+    friend class RegexMatchCache;++    using map_t = folly::F14VectorSet<string_pointer>;++    map_t const& matches_;++    /* implicit */ FindMatchesUnsafeResult(map_t const& matches) noexcept+        : matches_{matches} {}++   public:+    using value_type = map_t::value_type;++    auto size() const noexcept { return matches_.size(); }+    auto begin() const noexcept { return matches_.begin(); }+    auto end() const noexcept { return matches_.end(); }+  };++  RegexMatchCache() noexcept;+  ~RegexMatchCache();++  std::vector<std::string_view> getRegexList(KeyMap const& keys) const;+  std::vector<string_pointer> getStringList() const;+  InspectView inspect(KeyMap const& keys) const noexcept {+    return InspectView{*this, keys};+  }+  void consistency(+      ConsistencyReportMatcher& crcache,+      KeyMap const& keys,+      FunctionRef<void(std::string)> report) const;++  bool hasRegex(regex_key const& regex) const noexcept;+  void addRegex(regex_key const& regex);+  void eraseRegex(regex_key const& regex);++  bool hasString(string_pointer string) const noexcept;+  void addString(string_pointer string);+  void eraseString(string_pointer string);++  std::vector<string_pointer> findMatchesUncached(std::string_view regex) const;++  bool isReadyToFindMatches(regex_key const& regex) const noexcept;+  void prepareToFindMatches(regex_key_and_view const& regex);+  FindMatchesUnsafeResult findMatchesUnsafe(+      regex_key const& regex, time_point now) const;+  std::vector<string_pointer> findMatches(+      regex_key const& regex, time_point now) const;++  bool hasItemsToPurge(time_point expiry) const noexcept;++  void clear();+  void purge(time_point expiry);+};++} // namespace folly
+ folly/folly/container/Reserve.h view
@@ -0,0 +1,109 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <stdexcept>++#include <folly/Likely.h>+#include <folly/Traits.h>+#include <folly/Utility.h>+#include <folly/lang/Exception.h>++namespace folly {++namespace detail {++template <typename C>+using detect_capacity = decltype(FOLLY_DECLVAL(C).capacity());++template <typename C>+using detect_bucket_count = decltype(FOLLY_DECLVAL(C).bucket_count());++template <typename C>+using detect_max_load_factor = decltype(FOLLY_DECLVAL(C).max_load_factor());++template <typename C, typename... A>+using detect_reserve = decltype(FOLLY_DECLVAL(C).reserve(FOLLY_DECLVAL(A)...));++template <typename C>+using container_detect_reserve =+    detect_reserve<C, typename remove_cvref_t<C>::size_type>;++} // namespace detail++/**+ * Avoids quadratic behavior that could arise from c.reserve(c.size() + N).+ *+ * May reserve more than N in order to effect geometric growth.  Useful when+ * c.size() is unknown, but more elements must be added by discrete operations.+ * For example: N emplace calls in a loop, or a series of range inserts, the sum+ * of their sizes being N.  Behaves like reserve() if the container is empty.+ */+struct grow_capacity_by_fn {+  template <typename C>+  constexpr void operator()(C& c, typename C::size_type const n) const {+    const size_t sz = c.size();++    if (FOLLY_UNLIKELY(c.max_size() - sz < n)) {+      folly::throw_exception<std::length_error>("max_size exceeded");+    }++    if constexpr (folly::is_detected_v<detail::detect_capacity, C&>) {+      if (sz + n <= c.capacity()) {+        return;+      }+    } else if constexpr (+        folly::is_detected_v<detail::detect_bucket_count, C&> &&+        folly::is_detected_v<detail::detect_max_load_factor, C&>) {+      if (sz + n <= c.bucket_count() * c.max_load_factor()) {+        return;+      }+    } else {+      static_assert(folly::always_false<C>, "unexpected container type");+    }++    auto const ra = sz * 2;+    auto const rb = sz + n;+    c.reserve(rb < ra ? ra : rb);+  }+};++inline constexpr grow_capacity_by_fn grow_capacity_by{};++/**+ * Useful when writing generic code that handles containers.+ *+ * Examples:+ *  - std::unordered_map provides reserve(), but std::map does not+ *  - std::vector provides reserve(), but std::deque and std::list do not+ */+struct reserve_if_available_fn {+  template <typename C>+  constexpr auto operator()(C& c, typename C::size_type const n) const+      noexcept(!folly::is_detected_v<detail::container_detect_reserve, C&>) {+    constexpr auto match =+        folly::is_detected_v<detail::container_detect_reserve, C&>;+    if constexpr (match) {+      c.reserve(n);+    }+    return std::bool_constant<match>{};+  }+};++inline constexpr reserve_if_available_fn reserve_if_available{};++} // namespace folly
+ folly/folly/container/SparseByteSet.h view
@@ -0,0 +1,123 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <cassert>+#include <cstdint>++#include <folly/CPortability.h>++namespace folly {++/***+ *  SparseByteSet+ *+ *  A special-purpose data structure representing a set of bytes.+ *  May have better performance than std::bitset<256>, depending on workload.+ *+ *  Operations:+ *  - add(byte)+ *  - remove(byte)+ *  - contains(byte)+ *  - clear()+ *+ *  Performance:+ *  - The entire capacity of the set is inline; the set never allocates.+ *  - The constructor zeros only the first two bytes of the object.+ *  - add and contains both run in constant time w.r.t. the size of the set.+ *    Constant time - not amortized constant - and with small constant factor.+ *+ *  This data structure is ideal for on-stack use.+ *+ *  Aho, Hopcroft, and Ullman refer to this trick in "The Design and Analysis+ *  of Computer Algorithms" (1974), but the best description is here:+ *  http://research.swtch.com/sparse+ *  http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.30.7319+ */+class SparseByteSet {+ public:+  //  There are this many possible values:+  static constexpr uint16_t kCapacity = 256;++  //  No init of byte-arrays required!+  SparseByteSet() : size_(0) {}++  /***+   *  add(byte)+   *+   *  O(1), non-amortized.+   */+  inline bool add(uint8_t i) {+    bool r = !contains(i);+    if (r) {+      assert(size_ < kCapacity);+      dense_[size_] = i;+      sparse_[i] = uint8_t(size_);+      size_++;+    }+    return r;+  }++  /***+   *  remove(byte)+   *+   *  O(1), non-amortized.+   */+  inline bool remove(uint8_t i) {+    bool r = contains(i);+    if (r) {+      if (dense_[size_ - 1] != i) {+        uint8_t last_element = dense_[size_ - 1];+        dense_[sparse_[i]] = last_element;+        sparse_[last_element] = sparse_[i];+      }+      --size_;+    }+    return r;+  }++  /***+   *  contains(byte)+   *+   *  O(1), non-amortized.+   */+  inline bool contains(uint8_t i) const FOLLY_DISABLE_MEMORY_SANITIZER {+    return sparse_[i] < size_ && dense_[sparse_[i]] == i;+  }++  /***+   *  clear()+   *+   *  O(1), non-amortized.+   */+  inline void clear() { size_ = 0; }++  /***+   *  size()+   *+   *  O(1), non-amortized.+   */+  inline uint16_t size() { return size_; }++ private:+  uint16_t size_; // can't use uint8_t because it would overflow if all+                  // possible values were inserted.+  uint8_t sparse_[kCapacity];+  uint8_t dense_[kCapacity];+};++} // namespace folly
+ folly/folly/container/StdBitset.h view
@@ -0,0 +1,81 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <folly/CPortability.h>+#include <folly/Portability.h>++#include <bitset>+#include <cassert>++namespace folly {++struct std_bitset_find_first_fn {+  /// Return the index of the first set bit in a bitset, or bitset.size() if+  /// none.+  template <size_t N>+  FOLLY_ALWAYS_INLINE size_t+  operator()(const std::bitset<N>& bitset) const noexcept {+    // equivalent to #if defined(__GLIBCXX__)+    if constexpr (kIsGlibcxx) {+      // GNU provides non-standard (its a hold over from the original SGI+      // implementation) _Find_first(), which efficiently returns the index of+      // the first set bit.+      return bitset._Find_first();+    }++    for (size_t i = 0; i < bitset.size(); ++i) {+      if (bitset[i]) {+        return i;+      }+    }++    return bitset.size();+  }+};++inline constexpr std_bitset_find_first_fn std_bitset_find_first{};++struct std_bitset_find_next_fn {+  /// Return the index of the first set bit in a bitset after the given index,+  /// or bitset.size() if none.+  template <size_t N>+  FOLLY_ALWAYS_INLINE size_t+  operator()(const std::bitset<N>& bitset, size_t prev) const noexcept {+    assert(prev < bitset.size());++    // equivalent to #if defined(__GLIBCXX__)+    if constexpr (kIsGlibcxx) {+      // GNU provides non-standard (its a hold over from the original SGI+      // implementation) _Find_next(), which given an index, efficiently returns+      // the index of the first set bit after the index.+      return bitset._Find_next(prev);+    }++    for (size_t i = prev + 1; i < bitset.size(); ++i) {+      if (bitset[i]) {+        return i;+      }+    }++    return bitset.size();+  }+};++inline constexpr std_bitset_find_next_fn std_bitset_find_next{};++} // namespace folly
+ folly/folly/container/View.h view
@@ -0,0 +1,81 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <folly/Portability.h>+#include <folly/functional/Invoke.h>+#include <folly/lang/CustomizationPoint.h>++namespace folly {++//  order_preserving_reinsertion_view_fn+//  order_preserving_reinsertion_view+//+//  Extension point for containers to provide an order such that if entries are+//  inserted into a new instance in that order, iteration order of the new+//  instance matches the original's. This can be useful for containers that have+//  defined but non-FIFO iteration order, such as F14Vector*.+//+//  Should return an iterable view (a type that provides begin() and end()).+//+//  Containers should provide overloads via tag-invoke.+struct order_preserving_reinsertion_view_fn {+ private:+  using fn = order_preserving_reinsertion_view_fn;++ public:+  template <typename Container>+  FOLLY_ERASE constexpr auto operator()(Container const& container) const+      noexcept(is_nothrow_tag_invocable_v<fn, Container const&>)+          -> tag_invoke_result_t<fn, Container const&> {+    return tag_invoke(*this, container);+  }+};+FOLLY_DEFINE_CPO(+    order_preserving_reinsertion_view_fn, order_preserving_reinsertion_view)++//  order_preserving_reinsertion_view_or_default_fn+//  order_preserving_reinsertion_view_or_default+//+//  If a tag-invoke extension of order_preserving_reinsertion_view is available+//  over the given argument, forwards to that. Otherwise, returns the argument.+struct order_preserving_reinsertion_view_or_default_fn {+ private:+  using fn = order_preserving_reinsertion_view_fn;++ public:+  template <+      typename Container,+      std::enable_if_t<is_tag_invocable_v<fn, Container const&>, int> = 0>+  FOLLY_ERASE constexpr auto operator()(Container const& container) const+      noexcept(is_nothrow_tag_invocable_v<fn, Container const&>)+          -> tag_invoke_result_t<fn, Container const&> {+    return tag_invoke(fn{}, container);+  }+  template <+      typename Container,+      std::enable_if_t<!is_tag_invocable_v<fn, Container const&>, int> = 0>+  FOLLY_ERASE constexpr Container const& operator()(+      Container const& container) const noexcept {+    return container;+  }+};+FOLLY_DEFINE_CPO(+    order_preserving_reinsertion_view_or_default_fn,+    order_preserving_reinsertion_view_or_default)++} // namespace folly
+ folly/folly/container/WeightedEvictingCacheMap.h view
@@ -0,0 +1,651 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <new>+#include <type_traits>+#include <folly/container/EvictingCacheMap.h>++namespace folly {++/**+ * A variant of EvictingCacheMap that assigns weights to entries and+ * evicts entries in LRU order to ensure the total weight of all entries+ * stays below some set maximum. ImplicitlyWeighted means this variant+ * derives the weights from the key-values using a chosen function. TWeightFn+ * must be a type implementing `size_t operator()(const TKey&, const Tvalue&)`+ *+ * Example usage: if TKey and TValue are std::string, the weight could be the+ * sum of the string sizes (already stored in the key and value) so that the+ * total weight approximates the total memory usage.+ *+ * Also consider WeightedEvictingCacheMap below, which tracks weights+ * explicitly, along with keys and values.+ *+ * TValue must be either movable or copyable. TKey must be copyable.+ *+ * IMPORTANT NOTES:+ * * Returned references, pointers, or iterators are potentially invalid+ * after any pruning operation (set, insert, etc. that might increase total+ * weight), or any set/insert on the same key (which are allowed to create a+ * new entry or modify an existing entry becoming obsolete).+ * * Operations are more restrictive than EvictingCacheMap to reduce the+ * risk of modifying a value in a way that changes its weight without proper+ * tracking. TValue can be a const type if appropriate.+ * * This is NOT a thread-safe structure.+ * * For simplicity, functions taking a key implicitly inherit type+ * constraints from EvictingCacheMap. (Must either match TKey or+ * EligibleForHeterogeneousFind/Insert.)+ *+ * This implementation has not been highly optimized and is a wrapper around+ * EvictingCacheMap.+ */+template <+    class TKey,+    class TValue,+    class TWeightFn,+    class THash = HeterogeneousAccessHash<TKey>,+    class TKeyEqual = HeterogeneousAccessEqualTo<TKey>>+class ImplicitlyWeightedEvictingCacheMap {+ private: // typedefs+  using ECM = EvictingCacheMap<TKey, TValue, THash, TKeyEqual>;++ public:+  using PruneHookCall = std::function<void(TKey, TValue&&)>;++  explicit ImplicitlyWeightedEvictingCacheMap(+      std::size_t maxTotalWeight,+      const TWeightFn& weightFn = TWeightFn(),+      const THash& keyHash = THash(),+      const TKeyEqual& keyEqual = TKeyEqual())+      : ecm_(/* no max size*/ 0, 1, keyHash, keyEqual),+        weightFn_(weightFn),+        maxTotalWeight_(maxTotalWeight),+        currentTotalWeight_(0) {+    setupPruneHook();+  }++  // Like EvictingCacheMap+  ImplicitlyWeightedEvictingCacheMap(+      const ImplicitlyWeightedEvictingCacheMap&) = delete;+  ImplicitlyWeightedEvictingCacheMap& operator=(+      const ImplicitlyWeightedEvictingCacheMap&) = delete;+  ImplicitlyWeightedEvictingCacheMap& operator=(+      ImplicitlyWeightedEvictingCacheMap&& that) {+    // Put moved-from in a valid but empty state+    ecm_ = std::move(that.ecm_);+    that.ecm_.clear();+    weightFn_ = std::move(that.weightFn_);+    that.weightFn_ = TWeightFn();+    maxTotalWeight_ = std::move(that.maxTotalWeight_);+    that.maxTotalWeight_ = 0;+    currentTotalWeight_ = std::move(that.currentTotalWeight_);+    that.currentTotalWeight_ = 0;+    // Set prune hook for this 'this' (not that this)+    setupPruneHook();+    return *this;+  }+  ImplicitlyWeightedEvictingCacheMap(ImplicitlyWeightedEvictingCacheMap&& that)+      : ecm_(/* no max size*/ 0) {+    *this = std::move(that);+  }++  ~ImplicitlyWeightedEvictingCacheMap() {+#ifndef NDEBUG+    // Verify remaining total weight is properly tracked. This can break if+    // improperly mutating the TValues that changes the result of TWeightFn.+    std::size_t actual_total_weight = 0;+    for (auto& e : ecm_) {+      actual_total_weight += weightFn_(e.first, e.second);+    }+    assert(actual_total_weight == currentTotalWeight_);+#endif+  }++  static constexpr std::size_t kApproximateEntryMemUsage =+      ECM::kApproximateEntryMemUsage;++  // iterators. NOTE: mutable iterators not included because of challenges and+  // confusion over writes affecting the implicit weights and whether entries+  // are promoted or potentially invalidated due to pruning.+  using const_iterator = typename ECM::const_iterator;+  using const_reverse_iterator = typename ECM::const_reverse_iterator;++  const_iterator begin() const { return ecm_.begin(); }+  const_iterator end() const { return ecm_.end(); }++  const_reverse_iterator rbegin() const { return ecm_.rbegin(); }+  const_reverse_iterator rend() const { return ecm_.rend(); }++  /**+   * Get the number of elements in the dictionary+   * @return the size of the dictionary+   */+  std::size_t size() const { return ecm_.size(); }++  /**+   * Typical empty function+   * @return true if empty, false otherwise+   */+  bool empty() const { return ecm_.empty(); }++  /**+   * Returns total weight of all entries currently in the cache.+   */+  std::size_t getCurrentTotalWeight() const { return currentTotalWeight_; }++  /**+   * Returns the maximum allowed total weight of all entries in+   * the cache.+   */+  std::size_t getMaxTotalWeight() const { return maxTotalWeight_; }++  /**+   * Sets the maximum allowed total weight of all entries in+   * the cache, evicting entries as needed for the new limit.+   */+  void setMaxTotalWeight(std::size_t newMaxTotalWeight) {+    maxTotalWeight_ = newMaxTotalWeight;+    pruneToMaxTotalWeight();+  }++  /**+   * Check for existence of a specific key in the map.  This operation has+   *     no effect on LRU order.+   * @param key key to search for+   * @return true if exists, false otherwise+   */+  template <typename K>+  bool exists(const K& key) const {+    return ecm_.exists(key);+  }++  /**+   * Get the value associated with a specific key.  This function always+   * promotes a found value to the head of the LRU. The returned reference+   * is const and might be inavlidated by many subsequent operations. See+   * IMPORTANT NOTES above. See also replace().+   *+   * @param key key to search for+   * @return the value if it exists+   * @throw std::out_of_range exception of the key does not exist+   */+  template <typename K>+  const TValue& get(const K& key) {+    return ecm_.get(key);+  }++  // Same but without LRU promotion+  template <typename K>+  const TValue& getWithoutPromotion(const K& key) const {+    return ecm_.getWithoutPromotion(key);+  }++  /**+   * Set a key-value pair in the dictionary with a given weight. If its weight+   * is more than maxTotalWeight, the entry is inserted anyway and all other+   * entries are evicted, so that get() after set() always succeeds. The+   * structure can be temporarily over max weight until the next modification.+   * The new or modified entry is always inserted at or promoted to the head+   * of the LRU.+   *+   * @param key key to associate with value+   * @param value value to associate with the key+   */+  template <typename K>+  void set(const K& key, TValue&& value) {+    std::size_t new_weight = weightFn_(key, value);+    std::size_t old_weight = 0;+    auto it = find(key); // Does promotion+    if (it != end()) {+      using TMutableValue = std::remove_const_t<TValue>;++      // Existing entry+      old_weight = weightFn_(it->first, it->second);+      auto ptr = const_cast<TMutableValue*>(&it->second);+      // Overwrite+      // Would be this, but need to work around+      // const values, which don't allow move-assignment:+      //   it->second = std::move(value);+      // Below hack is OK because we reserve the right to "entirely new+      // entry" semantics for set() (invalidate pointers/iterators/etc.) but+      // optimize with "replace in place" implementation.+      ptr->~TValue();+      new (ptr) TValue(std::move(value));+    } else {+      // No existing entry+      ecm_.insert(key, std::move(value));+    }+    // Protect the entry we just put at the head of the LRU+    entryWeightUpdated(old_weight, new_weight, /*protect_one*/ true);+  }++  template <typename K>+  void set(const K& key, const TValue& value) {+    TValue tmp{value}; // can't yet rely on C++17 temporary materialization+    set(key, std::move(tmp));+  }++  // TODO: insert() functions, which would refuse insert if new entry weight+  // exceeds max (or existing entry for key)+  // template <typename K>+  // std::pair<const_iterator, bool> insert(const K& key, TValue&& value) {}++  /**+   * Erases any entry with given key or iterator.+   *+   * @param key_or_it key to search for or iterator+   */+  template <typename KI>+  void erase(const KI& key_or_it) {+    // Use prune hook as erase hook to keep total weight updated+    ecm_.erase(key_or_it, ecm_.getPruneHook());+  }++  /**+   * Get the iterator associated with a specific key.  This function always+   * promotes a found value to the head of the LRU. See IMPORTANT NOTES above+   * for why this only returns a const_iterator. See also replace().+   * @param key key to associate with value+   * @return the const_iterator of the object (a std::pair of const TKey,+   *     TValue) or end() if it does not exist+   */+  template <typename K>+  const_iterator find(const K& key) {+    return const_iterator(ecm_.find(key).base());+  }++  // Same but without LRU promotion+  template <typename K>+  const_iterator findWithoutPromotion(const K& key) const {+    return ecm_.findWithoutPromotion(key);+  }++  /**+   * Replace the value associated with an entry from a const_iterator, in a+   * safe way that tracks any modification to the weight. Entries are evicted+   * so that max total weight it not exceeded, except this function protects+   * the entry at the head of the LRU list from eviction. Thus, for find()+   * immediately followed by replace(), the modified entry is protected+   * against eviction even if exceeding max total weight (like set(), iterator+   * remains valid). The iterator also remains valid if the weight does not+   * increase. Otherwise, the iterator is potentially invalidated by eviction+   * during this operation.+   *+   * If TValue is a const type, this function is invalid.+   * @param it const_iterator for the entry to modify, which must come from+   * this cache map+   * @param value replacement value+   */+  void replace(const_iterator it, TValue&& value) {+    assert(it != end());+    size_t old_weight = weightFn_(it->first, it->second);+    size_t new_weight = weightFn_(it->first, value);+    // Overwrite in place+    const_cast<TValue&>(it->second) = std::move(value);+    // Evict as needed (possibly including this entry, unless it's LRU head)+    entryWeightUpdated(old_weight, new_weight, /*protect_one*/ true);+  }++  void replace(const_iterator it, const TValue& value) {+    TValue tmp{value}; // can't yet rely on C++17 temporary materialization+    replace(it, std::move(tmp));+  }++  /**+   * Clear the cache to an empty state.+   */+  void clear() {+    ecm_.clear();+    assert(currentTotalWeight_ == 0);+  }++  /**+   * Set the prune hook, which is the function invoked on the key and value+   *     on each eviction. An operation will throw if the pruneHook throws.+   *     Note that this prune hook is not automatically called on entries+   *     explicitly erase()ed nor on remaining entries at destruction time.+   * @param pruneHook eviction callback to set as default, or nullptr to clear+   */+  void setPruneHook(PruneHookCall pruneHook) { pruneHook_ = pruneHook; }++ private: // fns+  void setupPruneHook() {+    ecm_.setPruneHook([this](const TKey& key, TValue&& value) {+      std::size_t weight = weightFn_(key, value);+      assert(currentTotalWeight_ >= weight);+      currentTotalWeight_ -= weight;+      if (pruneHook_) {+        pruneHook_(key, std::move(value));+      }+    });+  }++  void entryWeightUpdated(+      std::size_t old_weight,+      std::size_t new_weight,+      bool protect_one = false) {+    assert(old_weight <= currentTotalWeight_);+    currentTotalWeight_ += new_weight - old_weight;+    pruneToMaxTotalWeight(protect_one);+  }++  void pruneToMaxTotalWeight(bool protect_one = false) {+    // NOTE: Avoid infinite loop even in the case of weight tracking bug+    size_t min_count = protect_one ? 1 : 0;+    while (currentTotalWeight_ > maxTotalWeight_ && ecm_.size() > min_count) {+      ecm_.prune(1);+    }+  }++  template <class _TKey, class _TValue, class _THash, class _TKeyEqual>+  friend class WeightedEvictingCacheMap;++ private: // data+  PruneHookCall pruneHook_;+  ECM ecm_;+  TWeightFn weightFn_;+  std::size_t maxTotalWeight_;+  std::size_t currentTotalWeight_;+};++/**+ * A variant of EvictingCacheMap that tracks weights for entries and+ * evicts entries in LRU order to ensure the total weight of all entries+ * stays below some set maximum. Weights are stored as a size_t with each+ * entry.+ *+ * Example usage: if TKey is std::string and TValue is some large, complex+ * object type, the weight could be the estimated memory size of the key+ * and complex object. Tracking the weight explicitly minimizes costly+ * recomputation of the estimated memory size. Thus, the total weight of all+ * entries approximates the total memory usage.+ *+ * TValue must be either movable or copyable. TKey must be copyable.+ *+ * IMPORTANT NOTES:+ * * Returned references, pointers, or iterators are potentially invalid+ * after any pruning operation (set, insert, etc. that might increase total+ * weight), or any set/insert on the same key (which are allowed to create a+ * new entry or modify an existing entry becoming obsolete).+ * * This is NOT a thread-safe structure.+ * * For simplicity, functions taking a key implicitly inherit type+ * constraints from EvictingCacheMap. (Must either match TKey or+ * EligibleForHeterogeneousFind/Insert.)+ *+ * This implementation has not been highly optimized.+ */+template <+    class TKey,+    class TValue,+    class THash = HeterogeneousAccessHash<TKey>,+    class TKeyEqual = HeterogeneousAccessEqualTo<TKey>>+class WeightedEvictingCacheMap {+ public: // types+  struct ValueAndWeight {+    /*implicit*/ ValueAndWeight(TValue&& _value, std::size_t _weight)+        : value(std::move(_value)), weight(_weight) {}+    // Value is mutable through non-const iterator+    TValue value;+    // Weight is not mutable to ensure proper tracking. See updateWeight().+    const std::size_t weight;+  };++ private: // types+  struct WeightFn {+    std::size_t operator()(const TKey& /*key*/, const ValueAndWeight& p) {+      return p.weight;+    }+  };+  using IWECM = ImplicitlyWeightedEvictingCacheMap<+      TKey,+      ValueAndWeight,+      WeightFn,+      THash,+      TKeyEqual>;++ public:+  using PruneHookCall = std::function<void(TKey, TValue&&, size_t)>;++  explicit WeightedEvictingCacheMap(+      std::size_t maxTotalWeight,+      const THash& keyHash = THash(),+      const TKeyEqual& keyEqual = TKeyEqual())+      : iwecm_(maxTotalWeight, WeightFn(), keyHash, keyEqual) {}++  // Like EvictingCacheMap+  WeightedEvictingCacheMap(const WeightedEvictingCacheMap&) = delete;+  WeightedEvictingCacheMap& operator=(const WeightedEvictingCacheMap&) = delete;+  WeightedEvictingCacheMap(WeightedEvictingCacheMap&&) = default;+  WeightedEvictingCacheMap& operator=(WeightedEvictingCacheMap&&) = default;++  static constexpr std::size_t kApproximateEntryMemUsage =+      IWECM::kApproximateEntryMemUsage;++  // iterators that dereference to ValueAndWeight+  using iterator = typename IWECM::ECM::iterator;+  using reverse_iterator = typename IWECM::ECM::reverse_iterator;+  using const_iterator = typename IWECM::const_iterator;+  using const_reverse_iterator = typename IWECM::const_reverse_iterator;++  iterator begin() { return iwecm_.ecm_.begin(); }+  iterator end() { return iwecm_.ecm_.end(); }++  const_iterator begin() const { return cbegin(); }+  const_iterator end() const { return cend(); }++  const_iterator cbegin() const { return iwecm_.begin(); }+  const_iterator cend() const { return iwecm_.end(); }++  reverse_iterator rbegin() { return iwecm_.ecm_.rbegin(); }+  reverse_iterator rend() { return iwecm_.ecm_.rend(); }++  const_reverse_iterator rbegin() const { return crbegin(); }+  const_reverse_iterator rend() const { return crend(); }++  const_reverse_iterator crbegin() const { return iwecm_.rbegin(); }+  const_reverse_iterator crend() const { return iwecm_.rend(); }++  /**+   * Get the number of elements in the dictionary+   * @return the size of the dictionary+   */+  std::size_t size() const { return iwecm_.size(); }++  /**+   * Typical empty function+   * @return true if empty, false otherwise+   */+  bool empty() const { return iwecm_.empty(); }++  /**+   * Returns total weight of all entries currently in the cache.+   */+  std::size_t getCurrentTotalWeight() const {+    return iwecm_.getCurrentTotalWeight();+  }++  /**+   * Returns the maximum allowed total weight of all entries in+   * the cache.+   */+  std::size_t getMaxTotalWeight() const { return iwecm_.getMaxTotalWeight(); }++  /**+   * Sets the maximum allowed total weight of all entries in+   * the cache, evicting entries as needed for the new limit.+   */+  void setMaxTotalWeight(std::size_t newMaxTotalWeight) {+    iwecm_.setMaxTotalWeight(newMaxTotalWeight);+  }++  /**+   * Check for existence of a specific key in the map.  This operation has+   *     no effect on LRU order.+   * @param key key to search for+   * @return true if exists, false otherwise+   */+  template <typename K>+  bool exists(const K& key) const {+    return iwecm_.exists(key);+  }++  /**+   * Get the value associated with a specific key.  This function always+   * promotes a found value to the head of the LRU. The TValue can be+   * modified in place through the reference, keeping in mind the reference+   * can easily be invalidated (IMPORTANT NOTES above).+   * @param key key to search for+   * @return the value if it exists+   * @throw std::out_of_range exception of the key does not exist+   */+  template <typename K>+  TValue& get(const K& key) {+    return const_cast<TValue&>(iwecm_.get(key).value);+  }+  // Same but without LRU promotion+  template <typename K>+  TValue& getWithoutPromotion(const K& key) {+    return const_cast<TValue&>(iwecm_.getWithoutPromotion(key).value);+  }+  template <typename K>+  const TValue& getWithoutPromotion(const K& key) const {+    return iwecm_.getWithoutPromotion(key).value;+  }++  /**+   * Set a key-value pair in the dictionary with a given weight. If its weight+   * is more than maxTotalWeight, the entry is inserted anyway and all other+   * entries are evicted, so that get() after set() always succeeds. The+   * structure can be temporarily over max weight until the next modification.+   * The new or modified entry is always inserted at or promoted to the head+   * of the LRU.+   *+   * @param key key to associate with value+   * @param value value to associate with the key+   * @param weight weight to associate with the key-value entry+   */+  template <typename K>+  void set(const K& key, TValue&& value, std::size_t weight) {+    ValueAndWeight tmp{std::move(value), weight};+    return iwecm_.set(key, std::move(tmp));+  }++  template <typename K>+  void set(const K& key, const TValue& value, std::size_t weight) {+    TValue tmp{value}; // can't yet rely on C++17 temporary materialization+    return set(key, std::move(tmp), weight);+  }++  // TODO: insert() functions, which would refuse insert if new entry weight+  // exceeds max (or existing entry for key)+  // template <typename K>+  // std::pair<const_iterator, bool> insert(const K& key, TValue&& value) {}++  /**+   * Erases any entry with given key or iterator.+   *+   * @param key_or_it key to search for or iterator+   */+  template <typename KI>+  void erase(const KI& key_or_it) {+    iwecm_.erase(key_or_it);+  }++  /**+   * Get the iterator associated with a specific key. This function always+   * promotes a found value to the head of the LRU. Although values can be+   * modified through iterators, weights are const. See updateWeight().+   * @param key key to associate with value+   * @return the iterator of std::pair<const TKey, ValueAndWeight>, or+   *     end() if it does not exist+   */+  template <typename K>+  iterator find(const K& key) {+    return iwecm_.ecm_.find(key);+  }++  // Same but without LRU promotion+  template <typename K>+  iterator findWithoutPromotion(const K& key) {+    return iwecm_.ecm_.findWithoutPromotion(key);+  }+  template <typename K>+  const_iterator findWithoutPromotion(const K& key) const {+    return iwecm_.findWithoutPromotion(key);+  }++  /**+   * Overwrite the weight associated with a specific entry. As usual, entries+   * are evicted so that max total weight it not exceeded, except this+   * function protects the entry at the head of the LRU list from eviction.+   * Thus, for find() immediately followed by updateWeight(), the modified+   * entry is protected against eviction even if exceeding max total weight+   * (like set(), iterator remains valid). The iterator also remains valid if+   * the weight does not increase. Otherwise, the iterator is potentially+   * invalidated by eviction during this operation.+   *+   * @param it iterator for the entry to modify, which must come from+   * this cache map+   * @param new_weight the updated weight to assign+   */+  void updateWeight(const_iterator it, std::size_t new_weight) {+    updateWeightImpl(it, new_weight);+  }+  void updateWeight(iterator it, std::size_t new_weight) {+    updateWeightImpl(it, new_weight);+  }++  /**+   * Clear the cache to an empty state.+   */+  void clear() { iwecm_.clear(); }++  /**+   * Set the prune hook, which is the function invoked on the key and value+   *     on each eviction. An operation will throw if the pruneHook throws.+   *     Note that this prune hook is not automatically called on entries+   *     explicitly erase()ed nor on remaining entries at destruction time.+   * @param pruneHook eviction callback to set as default, or nullptr to clear+   */+  void setPruneHook(PruneHookCall pruneHook) {+    iwecm_.setPruneHook([pruneHook = std::move(pruneHook)](+                            const TKey& key, ValueAndWeight&& valueAndWeight) {+      if (!pruneHook) {+        return;+      }+      pruneHook(key, std::move(valueAndWeight.value), valueAndWeight.weight);+    });+  }++ private:+  // Like IWECM::replace+  template <typename It>+  void updateWeightImpl(It it, std::size_t new_weight) {+    assert(it != end());+    size_t old_weight = it->second.weight;+    // Overwrite in place+    const_cast<std::size_t&>(it->second.weight) = new_weight;+    // Evict as needed (possibly including this entry, unless it's LRU head)+    iwecm_.entryWeightUpdated(old_weight, new_weight, /*protect_one*/ true);+  }++  PruneHookCall pruneHook_;+  IWECM iwecm_;+};++} // namespace folly
+ folly/folly/container/detail/BitIteratorDetail.h view
@@ -0,0 +1,87 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <iterator>+#include <type_traits>++#include <boost/iterator/iterator_adaptor.hpp>++#include <folly/portability/SysTypes.h>++namespace folly {++template <class BaseIter>+class BitIterator;++namespace bititerator_detail {++// Reference to a bit.+// Templatize on both parent reference and value types to capture+// const-ness correctly and to work with the case where Ref is a+// reference-like type (not T&), just like our BitReference here.+template <class Ref, class Value>+class BitReference {+ public:+  BitReference(Ref r, size_t bit) : ref_(r), bit_(bit) {}++  /* implicit */ operator bool() const { return ref_ & (one_ << bit_); }++  BitReference& operator=(bool b) {+    if (b) {+      set();+    } else {+      clear();+    }+    return *this;+  }++  void set() { ref_ |= (one_ << bit_); }++  void clear() { ref_ &= ~(one_ << bit_); }++  void flip() { ref_ ^= (one_ << bit_); }++ private:+  // shortcut to avoid writing static_cast everywhere+  const static Value one_ = 1;++  Ref ref_;+  size_t bit_;+};++template <class BaseIter>+struct BitIteratorBase {+  static_assert(+      std::is_integral<+          typename std::iterator_traits<BaseIter>::value_type>::value,+      "BitIterator may only be used with integral types");+  typedef boost::iterator_adaptor<+      BitIterator<BaseIter>, // Derived+      BaseIter, // Base+      bool, // Value+      typename std::iterator_traits<+          BaseIter>::iterator_category, // CategoryOrTraversal+      bititerator_detail::BitReference<+          typename std::iterator_traits<BaseIter>::reference,+          typename std::iterator_traits<BaseIter>::value_type>, // Reference+      ssize_t>+      type;+};++} // namespace bititerator_detail+} // namespace folly
+ folly/folly/container/detail/BoolWrapper.h view
@@ -0,0 +1,38 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++namespace folly {+namespace detail {++class BoolWrapper {+ public:+  constexpr /* implicit */ BoolWrapper(bool value = false) noexcept+      : value_(value) {}++  constexpr /* implicit */ operator bool() const noexcept { return value_; }++  constexpr BoolWrapper operator!() const noexcept {+    return BoolWrapper(!value_);+  }++ private:+  bool value_;+};++} // namespace detail+} // namespace folly
+ folly/folly/container/detail/F14Defaults.h view
@@ -0,0 +1,34 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <memory>++#include <folly/container/HeterogeneousAccess-fwd.h>++namespace folly {+namespace f14 {+template <typename T>+using DefaultHasher = HeterogeneousAccessHash<T>;++template <typename T>+using DefaultKeyEqual = HeterogeneousAccessEqualTo<T>;++template <typename T>+using DefaultAlloc = std::allocator<T>;+} // namespace f14+} // namespace folly
+ folly/folly/container/detail/F14IntrinsicsAvailability.h view
@@ -0,0 +1,75 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <folly/Portability.h>++#if defined(FOLLY_F14_FALLBACK_DISABLED) && FOLLY_F14_FALLBACK_DISABLED == 1+#define FOLLY_F14_VECTOR_INTRINSICS_CONFIGURED 1+#elif !FOLLY_MOBILE+#define FOLLY_F14_VECTOR_INTRINSICS_CONFIGURED 1+#else+#define FOLLY_F14_VECTOR_INTRINSICS_CONFIGURED 0+#endif++// F14 has been implemented for SSE2 and NEON (so far).+//+// This platform detection is a bit of a mess because it combines the+// detection of supported platforms (FOLLY_SSE >= 2 || FOLLY_NEON) with+// the selection of platforms on which we want to use it.+//+// Currently no 32-bit ARM versions are desired because we don't want to+// need a separate build for chips that don't have NEON.  AARCH64 support+// is enabled for non-mobile platforms, but on mobile platforms there+// are downstream iteration order effects that have not yet been resolved.+//+// If FOLLY_F14_VECTOR_INTRINSICS_AVAILABLE differs across compilation+// units the program will fail to link due to a missing definition of+// folly::container::detail::F14LinkCheck<X>::check() for some X.+#if (FOLLY_SSE >= 2 || (FOLLY_NEON && FOLLY_AARCH64) || FOLLY_RISCV64) && \+    FOLLY_F14_VECTOR_INTRINSICS_CONFIGURED &&                             \+    !(defined(FOLLY_F14_FORCE_FALLBACK) && FOLLY_F14_FORCE_FALLBACK)+#define FOLLY_F14_VECTOR_INTRINSICS_AVAILABLE 1+#else+#define FOLLY_F14_VECTOR_INTRINSICS_AVAILABLE 0+#endif++#if FOLLY_SSE_PREREQ(4, 2) || FOLLY_ARM_FEATURE_CRC32+#define FOLLY_F14_CRC_INTRINSIC_AVAILABLE 1+#else+#define FOLLY_F14_CRC_INTRINSIC_AVAILABLE 0+#endif++namespace folly {+namespace f14 {+namespace detail {++enum class F14IntrinsicsMode { None, Simd, SimdAndCrc };++static constexpr F14IntrinsicsMode getF14IntrinsicsMode() {+#if !FOLLY_F14_VECTOR_INTRINSICS_AVAILABLE+  return F14IntrinsicsMode::None;+#elif !FOLLY_F14_CRC_INTRINSIC_AVAILABLE+  return F14IntrinsicsMode::Simd;+#else+  return F14IntrinsicsMode::SimdAndCrc;+#endif+}++} // namespace detail+} // namespace f14+} // namespace folly
+ folly/folly/container/detail/F14MapFallback.h view
@@ -0,0 +1,718 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <algorithm>+#include <type_traits>+#include <unordered_map>++#include <folly/Optional.h>+#include <folly/lang/Assume.h>++#include <folly/container/detail/F14Table.h>+#include <folly/container/detail/Util.h>++/**+ * This file is intended to be included only by F14Map.h. It contains fallback+ * implementations of F14Map types for platforms that do not support the+ * required SIMD instructions, based on std::unordered_map.+ */++#if !FOLLY_F14_VECTOR_INTRINSICS_AVAILABLE++namespace folly {++namespace f14 {+namespace detail {+template <typename K, typename M, typename H, typename E, typename A>+class F14BasicMap : public std::unordered_map<K, M, H, E, A> {+  using Super = std::unordered_map<K, M, H, E, A>;++  template <typename K2, typename T>+  using EnableHeterogeneousFind = std::enable_if_t<+      ::folly::detail::EligibleForHeterogeneousFind<K, H, E, K2>::value,+      T>;++  template <typename K2, typename T>+  using EnableHeterogeneousInsert = std::enable_if_t<+      ::folly::detail::EligibleForHeterogeneousInsert<K, H, E, K2>::value,+      T>;++  template <typename K2>+  using IsIter = Disjunction<+      std::is_same<typename Super::iterator, remove_cvref_t<K2>>,+      std::is_same<typename Super::const_iterator, remove_cvref_t<K2>>>;++  template <typename K2, typename T>+  using EnableHeterogeneousErase = std::enable_if_t<+      ::folly::detail::EligibleForHeterogeneousFind<+          K,+          H,+          E,+          std::conditional_t<IsIter<K2>::value, K, K2>>::value &&+          !IsIter<K2>::value,+      T>;++ public:+  using typename Super::const_iterator;+  using typename Super::hasher;+  using typename Super::iterator;+  using typename Super::key_equal;+  using typename Super::key_type;+  using typename Super::mapped_type;+  using typename Super::pointer;+  using typename Super::size_type;+  using typename Super::value_type;++  F14BasicMap() = default;++  using Super::Super;++  //// PUBLIC - Modifiers++  std::pair<iterator, bool> insert(value_type const& value) {+    return emplace(value);+  }++  template <typename P>+  std::enable_if_t<+      std::is_constructible<value_type, P&&>::value,+      std::pair<iterator, bool>>+  insert(P&& value) {+    return emplace(std::forward<P>(value));+  }++  // TODO(T31574848): Work around libstdc++ versions (e.g., GCC < 6) with no+  // implementation of N4387 ("perfect initialization" for pairs and tuples).+  template <typename U1, typename U2>+  std::enable_if_t<+      std::is_constructible<key_type, U1 const&>::value &&+          std::is_constructible<mapped_type, U2 const&>::value,+      std::pair<iterator, bool>>+  insert(std::pair<U1, U2> const& value) {+    return emplace(value);+  }++  // TODO(T31574848)+  template <typename U1, typename U2>+  std::enable_if_t<+      std::is_constructible<key_type, U1&&>::value &&+          std::is_constructible<mapped_type, U2&&>::value,+      std::pair<iterator, bool>>+  insert(std::pair<U1, U2>&& value) {+    return emplace(std::move(value));+  }++  std::pair<iterator, bool> insert(value_type&& value) {+    return emplace(std::move(value));+  }++  iterator insert(const_iterator /*hint*/, value_type const& value) {+    return insert(value).first;+  }++  template <typename P>+  std::enable_if_t<std::is_constructible<value_type, P&&>::value, iterator>+  insert(const_iterator /*hint*/, P&& value) {+    return insert(std::forward<P>(value)).first;+  }++  iterator insert(const_iterator /*hint*/, value_type&& value) {+    return insert(std::move(value)).first;+  }++  template <class... Args>+  iterator emplace_hint(const_iterator /*hint*/, Args&&... args) {+    return emplace(std::forward<Args>(args)...).first;+  }++  template <class InputIt>+  void insert(InputIt first, InputIt last) {+    while (first != last) {+      insert(*first);+      ++first;+    }+  }++  void insert(std::initializer_list<value_type> ilist) {+    insert(ilist.begin(), ilist.end());+  }++  template <typename M2>+  std::pair<iterator, bool> insert_or_assign(key_type const& key, M2&& obj) {+    auto rv = try_emplace(key, std::forward<M2>(obj));+    if (!rv.second) {+      rv.first->second = std::forward<M2>(obj);+    }+    return rv;+  }++  template <typename M2>+  std::pair<iterator, bool> insert_or_assign(key_type&& key, M2&& obj) {+    auto rv = try_emplace(std::move(key), std::forward<M2>(obj));+    if (!rv.second) {+      rv.first->second = std::forward<M2>(obj);+    }+    return rv;+  }++  template <typename M2>+  iterator insert_or_assign(+      const_iterator /*hint*/, key_type const& key, M2&& obj) {+    return insert_or_assign(key, std::forward<M2>(obj)).first;+  }++  template <typename M2>+  iterator insert_or_assign(const_iterator /*hint*/, key_type&& key, M2&& obj) {+    return insert_or_assign(std::move(key), std::forward<M2>(obj)).first;+  }++  template <typename K2, typename M2>+  EnableHeterogeneousInsert<K2, std::pair<iterator, bool>> insert_or_assign(+      K2&& key, M2&& obj) {+    auto rv = try_emplace(std::forward<K2>(key), std::forward<M2>(obj));+    if (!rv.second) {+      rv.first->second = std::forward<M2>(obj);+    }+    return rv;+  }++ private:+  template <typename Arg>+  using UsableAsKey = ::folly::detail::+      EligibleForHeterogeneousFind<key_type, hasher, key_equal, Arg>;++ public:+  template <typename... Args>+  std::pair<iterator, bool> emplace(Args&&... args) {+    auto alloc = this->get_allocator();+    return folly::detail::+        callWithExtractedKey<key_type, mapped_type, UsableAsKey>(+            alloc,+            [&](auto& key, auto&&... inner) {+              auto it = find(key);+              if (it != this->end()) {+                return std::make_pair(it, false);+              }+              auto rv = Super::emplace(std::forward<decltype(inner)>(inner)...);+              FOLLY_SAFE_DCHECK(+                  rv.second, "post-find emplace should always insert");+              return rv;+            },+            std::forward<Args>(args)...);+  }++  template <typename... Args>+  std::pair<iterator, bool> try_emplace(key_type const& key, Args&&... args) {+    return emplace(+        std::piecewise_construct,+        std::forward_as_tuple(key),+        std::forward_as_tuple(std::forward<Args>(args)...));+  }++  template <typename... Args>+  std::pair<iterator, bool> try_emplace(key_type&& key, Args&&... args) {+    return emplace(+        std::piecewise_construct,+        std::forward_as_tuple(std::move(key)),+        std::forward_as_tuple(std::forward<Args>(args)...));+  }++  template <typename... Args>+  iterator try_emplace(+      const_iterator /*hint*/, key_type const& key, Args&&... args) {+    return emplace(+               std::piecewise_construct,+               std::forward_as_tuple(key),+               std::forward_as_tuple(std::forward<Args>(args)...))+        .first;+  }++  template <typename... Args>+  iterator try_emplace(+      const_iterator /*hint*/, key_type&& key, Args&&... args) {+    return emplace(+               std::piecewise_construct,+               std::forward_as_tuple(std::move(key)),+               std::forward_as_tuple(std::forward<Args>(args)...))+        .first;+  }++  template <typename K2, typename... Args>+  EnableHeterogeneousInsert<K2, std::pair<iterator, bool>> try_emplace(+      K2&& key, Args&&... args) {+    return emplace(+        std::piecewise_construct,+        std::forward_as_tuple(std::forward<K2>(key)),+        std::forward_as_tuple(std::forward<Args>(args)...));+  }++  using Super::erase;++  template <typename K2>+  EnableHeterogeneousErase<K2, size_type> erase(K2 const& key) {+    auto it = find(key);+    if (it != this->end()) {+      erase(it);+      return 1;+    } else {+      return 0;+    }+  }++  //// PUBLIC - Lookup++ private:+  template <typename K2>+  struct BottomKeyEqual {+    [[noreturn]] bool operator()(K2 const&, K2 const&) const {+      assume_unreachable();+    }+  };++  template <typename Self, typename K2>+  static auto findLocal(Self& self, K2 const& key)+      -> folly::Optional<decltype(self.begin(0))> {+    if (self.empty()) {+      return none;+    }+    using A2 = typename std::allocator_traits<A>::template rebind_alloc<+        std::pair<K2 const, M>>;+    using E2 = BottomKeyEqual<K2>;+    // this is exceedingly wicked!+    auto slot =+        reinterpret_cast<std::unordered_map<K2, M, H, E2, A2> const&>(self)+            .bucket(key);+    auto b = self.begin(slot);+    auto e = self.end(slot);+    while (b != e) {+      if (self.key_eq()(key, b->first)) {+        return b;+      }+      ++b;+    }+    FOLLY_SAFE_DCHECK(+        self.size() > 3 ||+            std::none_of(+                self.begin(),+                self.end(),+                [&](auto const& kv) { return self.key_eq()(key, kv.first); }),+        "");+    return none;+  }++  template <typename Self, typename K2>+  static auto& atImpl(Self& self, K2 const& key) {+    auto it = findLocal(self, key);+    if (!it) {+      throw_exception<std::out_of_range>("at() did not find key");+    }+    return (*it)->second;+  }++ public:+  mapped_type& at(key_type const& key) { return Super::at(key); }++  mapped_type const& at(key_type const& key) const { return Super::at(key); }++  template <typename K2>+  EnableHeterogeneousFind<K2, mapped_type&> at(K2 const& key) {+    return atImpl(*this, key);+  }++  template <typename K2>+  EnableHeterogeneousFind<K2, mapped_type const&> at(K2 const& key) const {+    return atImpl(*this, key);+  }++  using Super::operator[];++  template <typename K2>+  EnableHeterogeneousInsert<K2, mapped_type&> operator[](K2&& key) {+    return try_emplace(std::forward<K2>(key)).first->second;+  }++  size_type count(key_type const& key) const { return Super::count(key); }++  template <typename K2>+  EnableHeterogeneousFind<K2, size_type> count(K2 const& key) const {+    return !findLocal(*this, key) ? 0 : 1;+  }++  bool contains(key_type const& key) const { return count(key) != 0; }++  template <typename K2>+  EnableHeterogeneousFind<K2, bool> contains(K2 const& key) const {+    return count(key) != 0;+  }++ private:+  template <typename Iter, typename LocalIter>+  static std::+      enable_if_t<std::is_constructible<Iter, LocalIter const&>::value, Iter>+      fromLocal(LocalIter const& src, int = 0) {+    return Iter(src);+  }++  template <typename Iter, typename LocalIter>+  static std::+      enable_if_t<!std::is_constructible<Iter, LocalIter const&>::value, Iter>+      fromLocal(LocalIter const& src) {+    Iter dst;+    static_assert(sizeof(dst) <= sizeof(src));+    std::memcpy(std::addressof(dst), std::addressof(src), sizeof(dst));+    FOLLY_SAFE_CHECK(+        std::addressof(*src) == std::addressof(*dst),+        "ABI-assuming local_iterator to iterator conversion failed");+    return dst;+  }++  template <typename Iter, typename Self, typename K2>+  static Iter findImpl(Self& self, K2 const& key) {+    auto optLocalIt = findLocal(self, key);+    if (!optLocalIt) {+      return self.end();+    } else {+      return fromLocal<Iter>(*optLocalIt);+    }+  }++ public:+  iterator find(key_type const& key) { return Super::find(key); }++  const_iterator find(key_type const& key) const { return Super::find(key); }++  template <typename K2>+  EnableHeterogeneousFind<K2, iterator> find(K2 const& key) {+    return findImpl<iterator>(*this, key);+  }++  template <typename K2>+  EnableHeterogeneousFind<K2, const_iterator> find(K2 const& key) const {+    return findImpl<const_iterator>(*this, key);+  }++ private:+  template <typename Self, typename K2>+  static auto equalRangeImpl(Self& self, K2 const& key) {+    auto first = self.find(key);+    auto last = first;+    if (last != self.end()) {+      ++last;+    }+    return std::make_pair(first, last);+  }++ public:+  using Super::equal_range;++  template <typename K2>+  EnableHeterogeneousFind<K2, std::pair<iterator, iterator>> equal_range(+      K2 const& key) {+    return equalRangeImpl(*this, key);+  }++  template <typename K2>+  EnableHeterogeneousFind<K2, std::pair<const_iterator, const_iterator>>+  equal_range(K2 const& key) const {+    return equalRangeImpl(*this, key);+  }++  //// PUBLIC - F14 Extensions++  template <typename BeforeDestroy>+  iterator eraseInto(const_iterator pos, BeforeDestroy&& beforeDestroy) {+    iterator it = erase(pos, pos);+    FOLLY_SAFE_CHECK(std::addressof(*it) == std::addressof(*pos), "");+    return eraseInto(it, beforeDestroy);+  }++  template <typename BeforeDestroy>+  iterator eraseInto(iterator pos, BeforeDestroy&& beforeDestroy) {+    const_iterator prev{pos};+    ++pos;+    auto nh = this->extract(prev);+    FOLLY_SAFE_CHECK(!nh.empty(), "");+    beforeDestroy(std::move(nh.key()), std::move(nh.mapped()));+    return pos;+  }++  template <typename BeforeDestroy>+  iterator eraseInto(+      const_iterator first,+      const_iterator last,+      BeforeDestroy&& beforeDestroy) {+    iterator pos = erase(first, first);+    FOLLY_SAFE_CHECK(std::addressof(*pos) == std::addressof(*first), "");+    while (pos != last) {+      pos = eraseInto(pos, beforeDestroy);+    }+    return pos;+  }++ private:+  template <typename K2, typename BeforeDestroy>+  size_type eraseIntoImpl(K2 const& key, BeforeDestroy& beforeDestroy) {+    auto it = find(key);+    if (it != this->end()) {+      eraseInto(it, beforeDestroy);+      return 1;+    } else {+      return 0;+    }+  }++ public:+  template <typename BeforeDestroy>+  size_type eraseInto(key_type const& key, BeforeDestroy&& beforeDestroy) {+    return eraseIntoImpl(key, beforeDestroy);+  }++  template <typename K2, typename BeforeDestroy>+  EnableHeterogeneousErase<K2, size_type> eraseInto(+      K2 const& key, BeforeDestroy&& beforeDestroy) {+    return eraseIntoImpl(key, beforeDestroy);+  }++  bool containsEqualValue(value_type const& value) const {+    // bucket isn't valid if bucket_count is zero+    if (this->empty()) {+      return false;+    }+    auto slot = this->bucket(value.first);+    auto e = this->end(slot);+    for (auto b = this->begin(slot); b != e; ++b) {+      if (b->first == value.first) {+        return b->second == value.second;+      }+    }+    return false;+  }++  // exact for libstdc++, approximate for others+  std::size_t getAllocatedMemorySize() const {+    std::size_t rv = 0;+    visitAllocationClasses([&](std::size_t bytes, std::size_t n) {+      rv += bytes * n;+    });+    return rv;+  }++  // exact for libstdc++, approximate for others+  template <typename V>+  void visitAllocationClasses(V&& visitor) const {+    auto bc = this->bucket_count();+    if (bc > 1) {+      visitor(bc * sizeof(pointer), 1);+    }+    if (this->size() > 0) {+      visitor(sizeof(StdNodeReplica<K, value_type, H>), this->size());+    }+  }++  template <typename V>+  void visitContiguousRanges(V&& visitor) const {+    for (value_type const& entry : *this) {+      value_type const* b = std::addressof(entry);+      visitor(b, b + 1);+    }+  }++  /// F14HashToken interface+  template <typename V>+  std::pair<iterator, bool> insert_or_assign(+      F14HashToken const&, key_type const& key, V&& obj) {+    return insert_or_assign(key, std::forward<V>(obj));+  }++  template <typename V>+  std::pair<iterator, bool> insert_or_assign(+      F14HashToken const&, key_type&& key, V&& obj) {+    return insert_or_assign(std::move(key), std::forward<V>(obj));+  }++  template <typename K2, typename V>+  EnableHeterogeneousInsert<K2, std::pair<iterator, bool>> insert_or_assign(+      F14HashToken const&, K2&& key, V&& obj) {+    return insert_or_assign(std::forward<K2>(key), std::forward<V>(obj));+  }++  template <typename... Args>+  std::pair<iterator, bool> try_emplace_token(+      F14HashToken const&, key_type const& key, Args&&... args) {+    return try_emplace(key, std::forward<Args>(args)...);+  }++  template <typename... Args>+  std::pair<iterator, bool> try_emplace_token(+      F14HashToken const&, key_type&& key, Args&&... args) {+    return try_emplace(std::move(key), std::forward<Args>(args)...);+  }++  template <typename K2, typename... Args>+  EnableHeterogeneousInsert<K2, std::pair<iterator, bool>> try_emplace_token(+      F14HashToken const&, K2&& key, Args&&... args) {+    return try_emplace(std::forward<K2>(key), std::forward<Args>(args)...);+  }++  F14HashToken prehash(key_type const&) const {+    return {}; // Ignored.+  }+  F14HashToken prehash(key_type const&, std::size_t) const {+    return {}; // Ignored.+  }++  template <typename K2>+  EnableHeterogeneousFind<K2, F14HashToken> prehash(K2 const&) const {+    return {}; // Ignored.+  }+  template <typename K2>+  EnableHeterogeneousFind<K2, F14HashToken> prehash(+      K2 const&, std::size_t) const {+    return {}; // Ignored.+  }++  void prefetch(F14HashToken const&) const {}++  iterator find(F14HashToken const&, key_type const& key) { return find(key); }++  const_iterator find(F14HashToken const&, key_type const& key) const {+    return find(key);+  }++  template <typename K2>+  EnableHeterogeneousFind<K2, iterator> find(+      F14HashToken const&, K2 const& key) {+    return find(key);+  }++  template <typename K2>+  EnableHeterogeneousFind<K2, const_iterator> find(+      F14HashToken const&, K2 const& key) const {+    return find(key);+  }++  bool contains(F14HashToken const&, key_type const& key) const {+    return contains(key);+  }++  template <typename K2>+  EnableHeterogeneousFind<K2, bool> contains(+      F14HashToken const&, K2 const& key) const {+    return contains(key);+  }+};+} // namespace detail+} // namespace f14++template <+    typename Key,+    typename Mapped,+    typename Hasher,+    typename KeyEqual,+    typename Alloc>+class F14ValueMap+    : public f14::detail::F14BasicMap<Key, Mapped, Hasher, KeyEqual, Alloc> {+  using Super = f14::detail::F14BasicMap<Key, Mapped, Hasher, KeyEqual, Alloc>;++ public:+  using typename Super::value_type;++  F14ValueMap() = default;++  using Super::Super;++  F14ValueMap& operator=(std::initializer_list<value_type> ilist) {+    Super::operator=(ilist);+    return *this;+  }+};++template <+    typename Key,+    typename Mapped,+    typename Hasher,+    typename KeyEqual,+    typename Alloc>+class F14NodeMap+    : public f14::detail::F14BasicMap<Key, Mapped, Hasher, KeyEqual, Alloc> {+  using Super = f14::detail::F14BasicMap<Key, Mapped, Hasher, KeyEqual, Alloc>;++ public:+  using typename Super::value_type;++  F14NodeMap() = default;++  using Super::Super;++  F14NodeMap& operator=(std::initializer_list<value_type> ilist) {+    Super::operator=(ilist);+    return *this;+  }+};++template <+    typename Key,+    typename Mapped,+    typename Hasher,+    typename KeyEqual,+    typename Alloc>+class F14VectorMap+    : public f14::detail::F14BasicMap<Key, Mapped, Hasher, KeyEqual, Alloc> {+  using Super = f14::detail::F14BasicMap<Key, Mapped, Hasher, KeyEqual, Alloc>;++ public:+  using typename Super::const_iterator;+  using typename Super::iterator;+  using typename Super::value_type;++  F14VectorMap() = default;++  using Super::Super;++  F14VectorMap& operator=(std::initializer_list<value_type> ilist) {+    Super::operator=(ilist);+    return *this;+  }+};++template <+    typename Key,+    typename Mapped,+    typename Hasher,+    typename KeyEqual,+    typename Alloc>+class F14FastMap+    : public f14::detail::F14BasicMap<Key, Mapped, Hasher, KeyEqual, Alloc> {+  using Super = f14::detail::F14BasicMap<Key, Mapped, Hasher, KeyEqual, Alloc>;++ public:+  using typename Super::value_type;++  F14FastMap() = default;++  using Super::Super;++  F14FastMap& operator=(std::initializer_list<value_type> ilist) {+    Super::operator=(ilist);+    return *this;+  }+};++} // namespace folly++#endif // !if FOLLY_F14_VECTOR_INTRINSICS_AVAILABLE
+ folly/folly/container/detail/F14Mask.h view
@@ -0,0 +1,235 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <algorithm>+#include <cstdint>++#include <folly/Bits.h>+#include <folly/ConstexprMath.h>+#include <folly/Likely.h>+#include <folly/Portability.h>+#include <folly/container/detail/F14IntrinsicsAvailability.h>+#include <folly/lang/Assume.h>+#include <folly/lang/SafeAssert.h>++#if FOLLY_F14_VECTOR_INTRINSICS_AVAILABLE++namespace folly {+namespace f14 {+namespace detail {++template <typename T>+FOLLY_ALWAYS_INLINE static unsigned findFirstSetNonZero(T mask) {+  assume(mask != 0);+  if (sizeof(mask) == sizeof(unsigned)) {+    return __builtin_ctz(static_cast<unsigned>(mask));+  } else {+    return __builtin_ctzll(mask);+  }+}++#if FOLLY_NEON+using MaskType = uint64_t;++constexpr unsigned kMaskSpacing = 4;+#else // FOLLY_SSE >= 2 || FOLLY_RISCV64+using MaskType = uint32_t;++constexpr unsigned kMaskSpacing = 1;+#endif++template <unsigned BitCount>+struct FullMask {+  static constexpr MaskType value =+      (FullMask<BitCount - 1>::value << kMaskSpacing) + 1;+};++template <>+struct FullMask<1> : std::integral_constant<MaskType, 1> {};++#if FOLLY_ARM+// Mask iteration is different for ARM because that is the only platform+// for which the mask is bigger than a register.++// Iterates a mask, optimized for the case that only a few bits are set+class SparseMaskIter {+  static_assert(kMaskSpacing == 4);++  uint32_t interleavedMask_;++ public:+  explicit SparseMaskIter(MaskType mask)+      : interleavedMask_{static_cast<uint32_t>(((mask >> 32) << 2) | mask)} {}++  bool hasNext() { return interleavedMask_ != 0; }++  unsigned next() {+    FOLLY_SAFE_DCHECK(hasNext(), "");+    unsigned i = findFirstSetNonZero(interleavedMask_);+    interleavedMask_ &= (interleavedMask_ - 1);+    return ((i >> 2) | (i << 2)) & 0xf;+  }+};++// Iterates a mask, optimized for the case that most bits are set+class DenseMaskIter {+  static_assert(kMaskSpacing == 4);++  std::size_t count_;+  unsigned index_;+  uint8_t const* tags_;++ public:+  explicit DenseMaskIter(uint8_t const* tags, MaskType mask) {+    if (mask == 0) {+      count_ = 0;+    } else {+      count_ = popcount(static_cast<uint32_t>(((mask >> 32) << 2) | mask));+      if (FOLLY_LIKELY((mask & 1) != 0)) {+        index_ = 0;+      } else {+        index_ = findFirstSetNonZero(mask) / kMaskSpacing;+      }+      tags_ = tags;+    }+  }++  bool hasNext() { return count_ > 0; }++  unsigned next() {+    auto rv = index_;+    --count_;+    if (count_ > 0) {+      do {+        ++index_;+      } while ((tags_[index_] & 0x80) == 0);+    }+    FOLLY_SAFE_DCHECK(index_ < 16, "");+    return rv;+  }+};++#else+// Iterates a mask, optimized for the case that only a few bits are set+class SparseMaskIter {+  MaskType mask_;++ public:+  explicit SparseMaskIter(MaskType mask) : mask_{mask} {}++  bool hasNext() { return mask_ != 0; }++  unsigned next() {+    FOLLY_SAFE_DCHECK(hasNext(), "");+    unsigned i = findFirstSetNonZero(mask_);+    mask_ &= (mask_ - 1);+    return i / kMaskSpacing;+  }+};++// Iterates a mask, optimized for the case that most bits are set+class DenseMaskIter {+  MaskType mask_;+  unsigned index_{0};++ public:+  explicit DenseMaskIter(uint8_t const*, MaskType mask) : mask_{mask} {}++  bool hasNext() { return mask_ != 0; }++  unsigned next() {+    FOLLY_SAFE_DCHECK(hasNext(), "");+    if (FOLLY_LIKELY((mask_ & 1) != 0)) {+      mask_ >>= kMaskSpacing;+      return index_++;+    } else {+      unsigned s = findFirstSetNonZero(mask_);+      unsigned rv = index_ + (s / kMaskSpacing);+      mask_ >>= (s + kMaskSpacing);+      index_ = rv + 1;+      return rv;+    }+  }+};+#endif++// Iterates a mask, returning pairs of [begin,end) index covering blocks+// of set bits+class MaskRangeIter {+  MaskType mask_;+  unsigned shift_{0};++ public:+  explicit MaskRangeIter(MaskType mask) {+    // If kMaskSpacing is > 1 then there will be empty bits even for+    // contiguous ranges.  Fill them in.+    mask_ = mask * ((1 << kMaskSpacing) - 1);+  }++  bool hasNext() { return mask_ != 0; }++  std::pair<unsigned, unsigned> next() {+    FOLLY_SAFE_DCHECK(hasNext(), "");+    auto s = shift_;+    unsigned b = findFirstSetNonZero(mask_);+    unsigned e = findFirstSetNonZero(~(mask_ | (mask_ - 1)));+    mask_ >>= e;+    shift_ = s + e;+    return std::make_pair((s + b) / kMaskSpacing, (s + e) / kMaskSpacing);+  }+};++// Holds the result of an index query that has an optional result,+// interpreting a mask of 0 to be the empty answer and the index of the+// last set bit to be the non-empty answer+class LastOccupiedInMask {+  MaskType mask_;++ public:+  explicit LastOccupiedInMask(MaskType mask) : mask_{mask} {}++  bool hasIndex() const { return mask_ != 0; }++  unsigned index() const {+    assume(mask_ != 0);+    return (findLastSet(mask_) - 1) / kMaskSpacing;+  }+};++// Holds the result of an index query that has an optional result,+// interpreting a mask of 0 to be the empty answer and the index of the+// first set bit to be the non-empty answer+class FirstEmptyInMask {+  MaskType mask_;++ public:+  explicit FirstEmptyInMask(MaskType mask) : mask_{mask} {}++  bool hasIndex() const { return mask_ != 0; }++  unsigned index() const {+    FOLLY_SAFE_DCHECK(mask_ != 0, "");+    return findFirstSetNonZero(mask_) / kMaskSpacing;+  }+};++} // namespace detail+} // namespace f14+} // namespace folly++#endif
+ folly/folly/container/detail/F14Policy.h view
@@ -0,0 +1,1526 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <memory>+#include <new>+#include <type_traits>+#include <utility>++#include <folly/Memory.h>+#include <folly/Portability.h>+#include <folly/Traits.h>+#include <folly/Unit.h>+#include <folly/container/HeterogeneousAccess.h>+#include <folly/container/detail/F14Table.h>+#include <folly/hash/Hash.h>+#include <folly/lang/Align.h>+#include <folly/lang/SafeAssert.h>+#include <folly/memory/Malloc.h>++#if FOLLY_F14_VECTOR_INTRINSICS_AVAILABLE++namespace folly {+namespace f14 {+namespace detail {++template <typename Ptr>+using NonConstPtr = typename std::pointer_traits<Ptr>::template rebind<+    std::remove_const_t<typename std::pointer_traits<Ptr>::element_type>>;++template <typename KeyType, typename MappedType>+using MapValueType = std::pair<KeyType const, MappedType>;++template <typename KeyType, typename MappedTypeOrVoid>+using SetOrMapValueType = std::conditional_t<+    std::is_same<MappedTypeOrVoid, void>::value,+    KeyType,+    MapValueType<KeyType, MappedTypeOrVoid>>;++template <typename T>+using IsNothrowMoveAndDestroy = Conjunction<+    std::is_nothrow_move_constructible<T>,+    std::is_nothrow_destructible<T>>;++// Used to enable EBO for Hasher, KeyEqual, and Alloc.  std::tuple of+// all empty objects is empty in libstdc++ but not libc++.+template <+    char Tag,+    typename T,+    bool Inherit = std::is_empty<T>::value && !std::is_final<T>::value>+struct ObjectHolder {+  T value_;++  template <typename... Args>+  ObjectHolder(Args&&... args) : value_{std::forward<Args>(args)...} {}++  T& operator*() { return value_; }+  T const& operator*() const { return value_; }+};++template <char Tag, typename T>+struct ObjectHolder<Tag, T, true> : T {+  template <typename... Args>+  ObjectHolder(Args&&... args) : T{std::forward<Args>(args)...} {}++  T& operator*() { return *this; }+  T const& operator*() const { return *this; }+};++// Policy provides the functionality of hasher, key_equal, and+// allocator_type.  In addition, it can add indirection to the values+// contained in the base table by defining a non-trivial value() method.+//+// To facilitate stateful implementations it is guaranteed that there+// will be a 1:1 relationship between BaseTable and Policy instance:+// policies will only be copied when their owning table is copied, and+// they will only be moved when their owning table is moved.+//+// Key equality will have the user-supplied search key as its first+// argument and the table contents as its second.  Heterogeneous lookup+// should be handled on the first argument.+//+// Item is the data stored inline in the hash table's chunks.  The policy+// controls how this is mapped to the corresponding Value.+//+// The policies defined in this file work for either set or map types.+// Most of the functionality is identical. A few methods detect the+// collection type by checking to see if MappedType is void, and then use+// SFINAE to select the appropriate implementation.+template <+    typename KeyType,+    typename MappedTypeOrVoid,+    typename HasherOrVoid,+    typename KeyEqualOrVoid,+    typename AllocOrVoid,+    typename ItemType>+struct FOLLY_MSVC_DECLSPEC(empty_bases) BasePolicy+    : private ObjectHolder<+          'H',+          Defaulted<HasherOrVoid, DefaultHasher<KeyType>>>,+      private ObjectHolder<+          'E',+          Defaulted<KeyEqualOrVoid, DefaultKeyEqual<KeyType>>>,+      private ObjectHolder<+          'A',+          Defaulted<+              AllocOrVoid,+              DefaultAlloc<SetOrMapValueType<KeyType, MappedTypeOrVoid>>>> {+  //////// user-supplied types++  using Key = KeyType;+  using Mapped = MappedTypeOrVoid;+  using Value = SetOrMapValueType<Key, Mapped>;+  using Item = ItemType;+  using Hasher = Defaulted<HasherOrVoid, DefaultHasher<Key>>;+  using KeyEqual = Defaulted<KeyEqualOrVoid, DefaultKeyEqual<Key>>;+  using Alloc = Defaulted<AllocOrVoid, DefaultAlloc<Value>>;+  using AllocTraits = std::allocator_traits<Alloc>;++  using ByteAlloc = typename AllocTraits::template rebind_alloc<uint8_t>;+  using ByteAllocTraits = typename std::allocator_traits<ByteAlloc>;+  using BytePtr = typename ByteAllocTraits::pointer;++  //////// info about user-supplied types++  static_assert(+      std::is_same<typename AllocTraits::value_type, Value>::value,+      "wrong allocator value_type");++ private:+  using HasherHolder = ObjectHolder<'H', Hasher>;+  using KeyEqualHolder = ObjectHolder<'E', KeyEqual>;+  using AllocHolder = ObjectHolder<'A', Alloc>;++  // emulate c++17's std::allocator_traits<A>::is_always_equal++  template <typename A, typename = void>+  struct AllocIsAlwaysEqual : std::is_empty<A> {};++  template <typename A>+  struct AllocIsAlwaysEqual<A, typename A::is_always_equal>+      : A::is_always_equal {};++ public:+  static constexpr bool kAllocIsAlwaysEqual = AllocIsAlwaysEqual<Alloc>::value;++  static constexpr bool kDefaultConstructIsNoexcept =+      std::is_nothrow_default_constructible<Hasher>::value &&+      std::is_nothrow_default_constructible<KeyEqual>::value &&+      std::is_nothrow_default_constructible<Alloc>::value;++  static constexpr bool kSwapIsNoexcept = kAllocIsAlwaysEqual &&+      std::is_nothrow_swappable_v<Hasher> &&+      std::is_nothrow_swappable_v<KeyEqual>;++  static constexpr bool isAvalanchingHasher() {+    return IsAvalanchingHasher<Hasher, Key>::value;+  }++  static constexpr bool shouldAssume32BitHash() {+    return ShouldAssume32BitHash<Hasher>::value;+  }++  //////// internal types and constants++  using InternalSizeType = std::size_t;++  // if false, F14Table will be smaller but F14Table::begin() won't work+  static constexpr bool kEnableItemIteration = true;++  using Chunk = F14Chunk<Item>;+  using ChunkPtr = typename std::pointer_traits<+      typename AllocTraits::pointer>::template rebind<Chunk>;+  using ItemIter = F14ItemIter<ChunkPtr>;++  static constexpr bool kIsMap = !std::is_same<Key, Value>::value;+  static_assert(+      kIsMap == !std::is_void<MappedTypeOrVoid>::value,+      "Assumption for the kIsMap check violated.");++  using MappedOrBool = std::conditional_t<kIsMap, Mapped, bool>;++  // if true, bucket_count() after reserve(n) will be as close as possible+  // to n for multi-chunk tables+  static constexpr bool kContinuousCapacity = false;++  //////// methods++  BasePolicy(Hasher const& hasher, KeyEqual const& keyEqual, Alloc const& alloc)+      : HasherHolder{hasher}, KeyEqualHolder{keyEqual}, AllocHolder{alloc} {}++  BasePolicy(BasePolicy const& rhs)+      : HasherHolder{rhs.hasher()},+        KeyEqualHolder{rhs.keyEqual()},+        AllocHolder{+            AllocTraits::select_on_container_copy_construction(rhs.alloc())} {}++  BasePolicy(BasePolicy const& rhs, Alloc const& alloc)+      : HasherHolder{rhs.hasher()},+        KeyEqualHolder{rhs.keyEqual()},+        AllocHolder{alloc} {}++  BasePolicy(BasePolicy&& rhs) noexcept+      : HasherHolder{std::move(rhs.hasher())},+        KeyEqualHolder{std::move(rhs.keyEqual())},+        AllocHolder{std::move(rhs.alloc())} {}++  BasePolicy(BasePolicy&& rhs, Alloc const& alloc) noexcept+      : HasherHolder{std::move(rhs.hasher())},+        KeyEqualHolder{std::move(rhs.keyEqual())},+        AllocHolder{alloc} {}++ private:+  template <typename Src>+  void maybeAssignAlloc(std::true_type, Src&& src) {+    alloc() = std::forward<Src>(src);+  }++  template <typename Src>+  void maybeAssignAlloc(std::false_type, Src&&) {}++  template <typename A>+  void maybeSwapAlloc(std::true_type, A& rhs) {+    using std::swap;+    swap(alloc(), rhs);+  }++  template <typename A>+  void maybeSwapAlloc(std::false_type, A&) {}++ public:+  BasePolicy& operator=(BasePolicy const& rhs) {+    hasher() = rhs.hasher();+    keyEqual() = rhs.keyEqual();+    maybeAssignAlloc(+        typename AllocTraits::propagate_on_container_copy_assignment{},+        rhs.alloc());+    return *this;+  }++  BasePolicy& operator=(BasePolicy&& rhs) noexcept {+    hasher() = std::move(rhs.hasher());+    keyEqual() = std::move(rhs.keyEqual());+    maybeAssignAlloc(+        typename AllocTraits::propagate_on_container_move_assignment{},+        std::move(rhs.alloc()));+    return *this;+  }++  void swapBasePolicy(BasePolicy& rhs) {+    using std::swap;+    swap(hasher(), rhs.hasher());+    swap(keyEqual(), rhs.keyEqual());+    maybeSwapAlloc(+        typename AllocTraits::propagate_on_container_swap{}, rhs.alloc());+  }++  Hasher& hasher() { return *static_cast<HasherHolder&>(*this); }+  Hasher const& hasher() const {+    return *static_cast<HasherHolder const&>(*this);+  }+  KeyEqual& keyEqual() { return *static_cast<KeyEqualHolder&>(*this); }+  KeyEqual const& keyEqual() const {+    return *static_cast<KeyEqualHolder const&>(*this);+  }+  Alloc& alloc() { return *static_cast<AllocHolder&>(*this); }+  Alloc const& alloc() const { return *static_cast<AllocHolder const&>(*this); }++  template <typename K>+  std::size_t computeKeyHash(K const& key) const {+    static_assert(+        isAvalanchingHasher() == IsAvalanchingHasher<Hasher, K>::value);+    static_assert(+        !isAvalanchingHasher() ||+            sizeof(decltype(hasher()(key))) >= sizeof(std::size_t),+        "hasher is not avalanching if it doesn't return enough bits");+    return hasher()(key);+  }++  Key const& keyForValue(Key const& v) const { return v; }+  Key const& keyForValue(std::pair<Key const, MappedOrBool> const& p) const {+    return p.first;+  }+  Key const& keyForValue(std::pair<Key&&, MappedOrBool&&> const& p) const {+    return p.first;+  }++  // map's choice of pair<K const, T> as value_type is unfortunate,+  // because it means we either need a proxy iterator, a pointless key+  // copy when moving items during rehash, or some sort of UB hack.+  //+  // This code implements the hack.  Use moveValue(v) instead of+  // std::move(v) as the source of a move construction.  enable_if_t is+  // used so that this works for maps while being a no-op for sets.+  template <typename Dummy = int>+  static std::pair<Key&&, MappedOrBool&&> moveValue(+      std::pair<Key const, MappedOrBool>& value,+      std::enable_if_t<kIsMap, Dummy> = 0) {+    return {std::move(const_cast<Key&>(value.first)), std::move(value.second)};+  }++  template <typename Dummy = int>+  static Value&& moveValue(Value& value, std::enable_if_t<!kIsMap, Dummy> = 0) {+    return std::move(value);+  }++  template <typename P>+  bool beforeBuild(+      std::size_t /*size*/, std::size_t /*capacity*/, P&& /*rhs*/) {+    return false;+  }++  template <typename P>+  void afterBuild(+      bool /*undoState*/,+      bool /*success*/,+      std::size_t /*size*/,+      std::size_t /*capacity*/,+      P&& /*rhs*/) {}++  std::size_t alignedAllocSize(std::size_t n) const {+    if (kRequiredVectorAlignment <= alignof(max_align_t) ||+        std::is_same<ByteAlloc, std::allocator<uint8_t>>::value) {+      return n;+    } else {+      return n + kRequiredVectorAlignment;+    }+  }++  bool beforeRehash(+      std::size_t /*size*/,+      std::size_t /*oldCapacity*/,+      std::size_t /*newCapacity*/,+      std::size_t chunkAllocSize,+      BytePtr& outChunkAllocation) {+    outChunkAllocation =+        allocateOverAligned<ByteAlloc, kRequiredVectorAlignment>(+            ByteAlloc{alloc()}, chunkAllocSize);+    return false;+  }++  void afterRehash(+      bool /*undoState*/,+      bool /*success*/,+      std::size_t /*size*/,+      std::size_t /*oldCapacity*/,+      std::size_t /*newCapacity*/,+      BytePtr chunkAllocation,+      std::size_t chunkAllocSize) {+    // on success, this will be the old allocation, on failure the new one+    if (chunkAllocation != nullptr) {+      deallocateOverAligned<ByteAlloc, kRequiredVectorAlignment>(+          ByteAlloc{alloc()}, chunkAllocation, chunkAllocSize);+    }+  }++  void beforeClear(std::size_t /*size*/, std::size_t /*capacity*/) {}++  void afterClear(std::size_t /*size*/, std::size_t /*capacity*/) {}++  void beforeReset(std::size_t /*size*/, std::size_t /*capacity*/) {}++  void afterReset(+      std::size_t /*size*/,+      std::size_t /*capacity*/,+      BytePtr chunkAllocation,+      std::size_t chunkAllocSize) {+    deallocateOverAligned<ByteAlloc, kRequiredVectorAlignment>(+        ByteAlloc{alloc()}, chunkAllocation, chunkAllocSize);+  }++  void prefetchValue(Item const&) const {+    // Subclass should disable with prefetchBeforeRehash(),+    // prefetchBeforeCopy(), and prefetchBeforeDestroy().  if they don't+    // override this method, because neither gcc nor clang can figure+    // out that DenseMaskIter with an empty body can be elided.+    FOLLY_SAFE_DCHECK(false, "should be disabled");+  }++  void afterDestroyWithoutDeallocate(Value* addr, std::size_t n) {+    if (kIsLibrarySanitizeAddress) {+      memset(static_cast<void*>(addr), 0x66, sizeof(Value) * n);+    }+  }+};++// BaseIter is a convenience for concrete set and map implementations+template <typename ValuePtr, typename Item>+class BaseIter {+ private:+  using pointee = typename std::pointer_traits<ValuePtr>::element_type;++ public:+  using iterator_category = std::forward_iterator_tag;+  using value_type = std::remove_const_t<pointee>;+  using difference_type = std::ptrdiff_t;+  using pointer = ValuePtr;+  using reference = pointee&;++ protected:+  using Chunk = F14Chunk<Item>;+  using ChunkPtr =+      typename std::pointer_traits<ValuePtr>::template rebind<Chunk>;+  using ItemIter = F14ItemIter<ChunkPtr>;++  using ValueConstPtr = typename std::pointer_traits<ValuePtr>::template rebind<+      std::add_const_t<typename std::pointer_traits<ValuePtr>::element_type>>;+};++//////// ValueContainer++template <+    typename Key,+    typename Mapped,+    typename HasherOrVoid,+    typename KeyEqualOrVoid,+    typename AllocOrVoid>+class ValueContainerPolicy;++template <typename ValuePtr>+using ValueContainerIteratorBase = BaseIter<+    ValuePtr,+    std::remove_const_t<typename std::pointer_traits<ValuePtr>::element_type>>;++template <typename ValuePtr>+class ValueContainerIterator : public ValueContainerIteratorBase<ValuePtr> {+  using Super = ValueContainerIteratorBase<ValuePtr>;+  using ItemIter = typename Super::ItemIter;+  using ValueConstPtr = typename Super::ValueConstPtr;++ public:+  using pointer = typename Super::pointer;+  using reference = typename Super::reference;+  using value_type = typename Super::value_type;++  ValueContainerIterator() = default;+  ValueContainerIterator(ValueContainerIterator const&) = default;+  ValueContainerIterator(ValueContainerIterator&&) = default;+  ValueContainerIterator& operator=(ValueContainerIterator const&) = default;+  ValueContainerIterator& operator=(ValueContainerIterator&&) = default;+  ~ValueContainerIterator() = default;++  /*implicit*/ operator ValueContainerIterator<ValueConstPtr>() const {+    return ValueContainerIterator<ValueConstPtr>{underlying_};+  }++  reference operator*() const { return underlying_.item(); }++  pointer operator->() const {+    return std::pointer_traits<pointer>::pointer_to(**this);+  }++  ValueContainerIterator& operator++() {+    underlying_.advance();+    return *this;+  }++  ValueContainerIterator operator++(int) {+    auto cur = *this;+    ++*this;+    return cur;+  }++  friend bool operator==(+      ValueContainerIterator const& lhs, ValueContainerIterator const& rhs) {+    return lhs.underlying_ == rhs.underlying_;+  }+  friend bool operator!=(+      ValueContainerIterator const& lhs, ValueContainerIterator const& rhs) {+    return !(lhs == rhs);+  }++ private:+  ItemIter underlying_;++  explicit ValueContainerIterator(ItemIter const& underlying)+      : underlying_{underlying} {}++  template <typename K, typename M, typename H, typename E, typename A>+  friend class ValueContainerPolicy;++  template <typename P>+  friend class ValueContainerIterator;+};++template <+    typename Key,+    typename MappedTypeOrVoid,+    typename HasherOrVoid,+    typename KeyEqualOrVoid,+    typename AllocOrVoid>+class ValueContainerPolicy+    : public BasePolicy<+          Key,+          MappedTypeOrVoid,+          HasherOrVoid,+          KeyEqualOrVoid,+          AllocOrVoid,+          SetOrMapValueType<Key, MappedTypeOrVoid>> {+ public:+  using Super = BasePolicy<+      Key,+      MappedTypeOrVoid,+      HasherOrVoid,+      KeyEqualOrVoid,+      AllocOrVoid,+      SetOrMapValueType<Key, MappedTypeOrVoid>>;+  using Alloc = typename Super::Alloc;+  using AllocTraits = typename Super::AllocTraits;+  using Item = typename Super::Item;+  using ItemIter = typename Super::ItemIter;+  using Value = typename Super::Value;+  using KeyEqual = typename Super::KeyEqual;+  using Hasher = typename Super::Hasher;+  using Mapped = typename Super::Mapped;++  static constexpr bool kDefaultConstructIsNoexcept =+      Super::kDefaultConstructIsNoexcept;+  static constexpr bool kAllocIsAlwaysEqual = Super::kAllocIsAlwaysEqual;+  static constexpr bool kEnableItemIteration = Super::kEnableItemIteration;+  static constexpr bool kContinuousCapacity = Super::kContinuousCapacity;+  static constexpr auto isAvalanchingHasher = Super::isAvalanchingHasher;+  static constexpr bool kSwapIsNoexcept = Super::kSwapIsNoexcept;++ private:+  using ByteAlloc = typename Super::ByteAlloc;++  using Super::kIsMap;++ public:+  using ConstIter = ValueContainerIterator<typename AllocTraits::const_pointer>;+  using Iter = std::conditional_t<+      kIsMap,+      ValueContainerIterator<typename AllocTraits::pointer>,+      ConstIter>;++  //////// F14Table policy++  static constexpr bool prefetchBeforeRehash() { return false; }++  static constexpr bool prefetchBeforeCopy() { return false; }++  static constexpr bool prefetchBeforeDestroy() { return false; }++  static constexpr bool destroyItemOnClear() {+    return !std::is_trivially_destructible<Item>::value ||+        !AllocatorHasDefaultObjectDestroy<Alloc, Item>::value;+  }++  // inherit constructors+  using Super::Super;++  void swapPolicy(ValueContainerPolicy& rhs) { this->swapBasePolicy(rhs); }++  using Super::keyForValue;+  static_assert(+      std::is_same<Item, Value>::value,+      "Item and Value should be the same type for ValueContainerPolicy.");++  std::size_t computeItemHash(Item const& item) const {+    return this->computeKeyHash(keyForValue(item));+  }++  template <typename K>+  bool keyMatchesItem(K const& key, Item const& item) const {+    return this->keyEqual()(key, keyForValue(item));+  }++  Value const& buildArgForItem(Item const& item) const& { return item; }++  // buildArgForItem(Item&)&& is used when moving between unequal allocators+  decltype(auto) buildArgForItem(Item& item) && {+    return Super::moveValue(item);+  }++  Value const& valueAtItem(Item const& item) const { return item; }++  Value&& valueAtItemForExtract(Item& item) { return std::move(item); }++  template <typename Table, typename... Args>+  void constructValueAtItem(Table&&, Item* itemAddr, Args&&... args) {+    Alloc& a = this->alloc();+    // GCC < 6 doesn't use the fact that itemAddr came from a reference+    // to avoid a null-check in the placement new.  folly::assume-ing it+    // here gets rid of that branch.  The branch is very predictable,+    // but spoils some further optimizations.  All clang versions that+    // compile folly seem to be okay.+    //+    // TODO(T31574848): clean up assume-s used to optimize placement new+    assume(itemAddr != nullptr);+    AllocTraits::construct(a, itemAddr, std::forward<Args>(args)...);+  }++  template <typename T>+  std::enable_if_t<IsNothrowMoveAndDestroy<T>::value>+  complainUnlessNothrowMoveAndDestroy() {}++  template <typename T>+  [[deprecated(+      "mark {key_type,mapped_type} {move constructor,destructor} noexcept, or use F14Node* if they aren't")]] std::+      enable_if_t<!IsNothrowMoveAndDestroy<T>::value>+      complainUnlessNothrowMoveAndDestroy() {}++  void moveItemDuringRehash(Item* itemAddr, Item& src) {+    complainUnlessNothrowMoveAndDestroy<Key>();+    complainUnlessNothrowMoveAndDestroy<lift_unit_t<MappedTypeOrVoid>>();++    constructValueAtItem(0, itemAddr, Super::moveValue(src));+    if (destroyItemOnClear()) {+      if (kIsMap) {+        // Laundering in the standard is only described as a solution+        // for changes to const fields due to the creation of a new+        // object lifetime (destroy and then placement new in the same+        // location), but it seems highly likely that it will also cause+        // the compiler to drop such assumptions that are violated due+        // to our UB const_cast in moveValue.+        destroyItem(*std::launder(std::addressof(src)));+      } else {+        destroyItem(src);+      }+    }+  }++  void destroyItem(Item& item) noexcept {+    Alloc& a = this->alloc();+    auto ptr = std::addressof(item);+    AllocTraits::destroy(a, ptr);+    this->afterDestroyWithoutDeallocate(ptr, 1);+  }++  template <typename V>+  void visitPolicyAllocationClasses(+      std::size_t chunkAllocSize,+      std::size_t /*size*/,+      std::size_t /*capacity*/,+      V&& visitor) const {+    if (chunkAllocSize > 0) {+      visitor(+          allocationBytesForOverAligned<ByteAlloc, kRequiredVectorAlignment>(+              chunkAllocSize),+          1);+    }+  }++  //////// F14BasicMap/Set policy++  FOLLY_ALWAYS_INLINE Iter makeIter(ItemIter const& underlying) const {+    return Iter{underlying};+  }+  ConstIter makeConstIter(ItemIter const& underlying) const {+    return ConstIter{underlying};+  }+  ItemIter const& unwrapIter(ConstIter const& iter) const {+    return iter.underlying_;+  }+};++//////// NodeContainer++template <+    typename Key,+    typename Mapped,+    typename HasherOrVoid,+    typename KeyEqualOrVoid,+    typename AllocOrVoid>+class NodeContainerPolicy;++template <typename ValuePtr>+class NodeContainerIterator : public BaseIter<ValuePtr, NonConstPtr<ValuePtr>> {+  using Super = BaseIter<ValuePtr, NonConstPtr<ValuePtr>>;+  using ItemIter = typename Super::ItemIter;+  using ValueConstPtr = typename Super::ValueConstPtr;++ public:+  using pointer = typename Super::pointer;+  using reference = typename Super::reference;+  using value_type = typename Super::value_type;++  NodeContainerIterator() = default;+  NodeContainerIterator(NodeContainerIterator const&) = default;+  NodeContainerIterator(NodeContainerIterator&&) = default;+  NodeContainerIterator& operator=(NodeContainerIterator const&) = default;+  NodeContainerIterator& operator=(NodeContainerIterator&&) = default;+  ~NodeContainerIterator() = default;++  /*implicit*/ operator NodeContainerIterator<ValueConstPtr>() const {+    return NodeContainerIterator<ValueConstPtr>{underlying_};+  }++  reference operator*() const { return *underlying_.item(); }++  pointer operator->() const {+    return std::pointer_traits<pointer>::pointer_to(**this);+  }++  NodeContainerIterator& operator++() {+    underlying_.advance();+    return *this;+  }++  NodeContainerIterator operator++(int) {+    auto cur = *this;+    ++*this;+    return cur;+  }++  friend bool operator==(+      NodeContainerIterator const& lhs, NodeContainerIterator const& rhs) {+    return lhs.underlying_ == rhs.underlying_;+  }+  friend bool operator!=(+      NodeContainerIterator const& lhs, NodeContainerIterator const& rhs) {+    return !(lhs == rhs);+  }++ private:+  ItemIter underlying_;++  explicit NodeContainerIterator(ItemIter const& underlying)+      : underlying_{underlying} {}++  template <typename K, typename M, typename H, typename E, typename A>+  friend class NodeContainerPolicy;++  template <typename P>+  friend class NodeContainerIterator;+};++template <+    typename Key,+    typename MappedTypeOrVoid,+    typename HasherOrVoid,+    typename KeyEqualOrVoid,+    typename AllocOrVoid>+class NodeContainerPolicy+    : public BasePolicy<+          Key,+          MappedTypeOrVoid,+          HasherOrVoid,+          KeyEqualOrVoid,+          AllocOrVoid,+          typename std::allocator_traits<Defaulted<+              AllocOrVoid,+              DefaultAlloc<std::conditional_t<+                  std::is_void<MappedTypeOrVoid>::value,+                  Key,+                  MapValueType<Key, MappedTypeOrVoid>>>>>::pointer> {+ public:+  using Super = BasePolicy<+      Key,+      MappedTypeOrVoid,+      HasherOrVoid,+      KeyEqualOrVoid,+      AllocOrVoid,+      typename std::allocator_traits<Defaulted<+          AllocOrVoid,+          DefaultAlloc<std::conditional_t<+              std::is_void<MappedTypeOrVoid>::value,+              Key,+              MapValueType<Key, MappedTypeOrVoid>>>>>::pointer>;+  using Alloc = typename Super::Alloc;+  using AllocTraits = typename Super::AllocTraits;+  using Item = typename Super::Item;+  using ItemIter = typename Super::ItemIter;+  using Value = typename Super::Value;++ private:+  using ByteAlloc = typename Super::ByteAlloc;++  using Super::kIsMap;++ public:+  using ConstIter = NodeContainerIterator<typename AllocTraits::const_pointer>;+  using Iter = std::conditional_t<+      kIsMap,+      NodeContainerIterator<typename AllocTraits::pointer>,+      ConstIter>;++  //////// F14Table policy++  static constexpr bool prefetchBeforeRehash() { return true; }++  static constexpr bool prefetchBeforeCopy() { return true; }++  static constexpr bool prefetchBeforeDestroy() {+    return !std::is_trivially_destructible<Value>::value;+  }++  static constexpr bool destroyItemOnClear() { return true; }++  // inherit constructors+  using Super::Super;++  void swapPolicy(NodeContainerPolicy& rhs) { this->swapBasePolicy(rhs); }++  using Super::keyForValue;++  std::size_t computeItemHash(Item const& item) const {+    return this->computeKeyHash(keyForValue(*item));+  }++  template <typename K>+  bool keyMatchesItem(K const& key, Item const& item) const {+    return this->keyEqual()(key, keyForValue(*item));+  }++  Value const& buildArgForItem(Item const& item) const& { return *item; }++  // buildArgForItem(Item&)&& is used when moving between unequal allocators+  decltype(auto) buildArgForItem(Item& item) && {+    return Super::moveValue(*item);+  }++  Value const& valueAtItem(Item const& item) const { return *item; }++  Value&& valueAtItemForExtract(Item& item) { return std::move(*item); }++  template <typename Table, typename... Args>+  void constructValueAtItem(Table&&, Item* itemAddr, Args&&... args) {+    Alloc& a = this->alloc();+    // TODO(T31574848): clean up assume-s used to optimize placement new+    assume(itemAddr != nullptr);+    new (itemAddr) Item{AllocTraits::allocate(a, 1)};+    auto p = std::addressof(**itemAddr);+    // TODO(T31574848): clean up assume-s used to optimize placement new+    assume(p != nullptr);+    auto rollback = makeGuard([&] { AllocTraits::deallocate(a, p, 1); });+    AllocTraits::construct(a, p, std::forward<Args>(args)...);+    rollback.dismiss();+  }++  void moveItemDuringRehash(Item* itemAddr, Item& src) {+    // This is basically *itemAddr = src; src = nullptr, but allowing+    // for fancy pointers.+    // TODO(T31574848): clean up assume-s used to optimize placement new+    assume(itemAddr != nullptr);+    new (itemAddr) Item{std::move(src)};+    src = nullptr;+    src.~Item();+  }++  void prefetchValue(Item const& item) const {+    prefetchAddr(std::addressof(*item));+  }++  template <typename T>+  std::enable_if_t<std::is_nothrow_destructible<T>::value>+  complainUnlessNothrowDestroy() {}++  template <typename T>+  [[deprecated("Mark key and mapped type destructor nothrow")]] std::+      enable_if_t<!std::is_nothrow_destructible<T>::value>+      complainUnlessNothrowDestroy() {}++  void destroyItem(Item& item) noexcept {+    complainUnlessNothrowDestroy<Key>();+    complainUnlessNothrowDestroy<lift_unit_t<MappedTypeOrVoid>>();+    if (item != nullptr) {+      Alloc& a = this->alloc();+      AllocTraits::destroy(a, std::addressof(*item));+      AllocTraits::deallocate(a, item, 1);+    }+    item.~Item();+  }++  template <typename V>+  void visitPolicyAllocationClasses(+      std::size_t chunkAllocSize,+      std::size_t size,+      std::size_t /*capacity*/,+      V&& visitor) const {+    if (chunkAllocSize > 0) {+      visitor(+          allocationBytesForOverAligned<ByteAlloc, kRequiredVectorAlignment>(+              chunkAllocSize),+          1);+    }+    if (size > 0) {+      visitor(sizeof(Value), size);+    }+  }++  //////// F14BasicMap/Set policy++  FOLLY_ALWAYS_INLINE Iter makeIter(ItemIter const& underlying) const {+    return Iter{underlying};+  }+  ConstIter makeConstIter(ItemIter const& underlying) const {+    return Iter{underlying};+  }+  ItemIter const& unwrapIter(ConstIter const& iter) const {+    return iter.underlying_;+  }+};++//////// VectorContainer++template <+    typename Key,+    typename MappedTypeOrVoid,+    typename HasherOrVoid,+    typename KeyEqualOrVoid,+    typename AllocOrVoid,+    typename EligibleForPerturbedInsertionOrder>+class VectorContainerPolicy;++template <typename ValuePtr>+class VectorContainerIterator : public BaseIter<ValuePtr, uint32_t> {+  using Super = BaseIter<ValuePtr, uint32_t>;+  using ValueConstPtr = typename Super::ValueConstPtr;++ public:+  using pointer = typename Super::pointer;+  using reference = typename Super::reference;+  using value_type = typename Super::value_type;++  VectorContainerIterator() = default;+  VectorContainerIterator(VectorContainerIterator const&) = default;+  VectorContainerIterator(VectorContainerIterator&&) = default;+  VectorContainerIterator& operator=(VectorContainerIterator const&) = default;+  VectorContainerIterator& operator=(VectorContainerIterator&&) = default;+  ~VectorContainerIterator() = default;++  /*implicit*/ operator VectorContainerIterator<ValueConstPtr>() const {+    return VectorContainerIterator<ValueConstPtr>{current_, lowest_};+  }++  reference operator*() const { return *current_; }++  pointer operator->() const { return current_; }++  VectorContainerIterator& operator++() {+    if (FOLLY_UNLIKELY(current_ == lowest_)) {+      current_ = nullptr;+    } else {+      --current_;+    }+    return *this;+  }++  VectorContainerIterator operator++(int) {+    auto cur = *this;+    ++*this;+    return cur;+  }++  friend bool operator==(+      VectorContainerIterator const& lhs, VectorContainerIterator const& rhs) {+    return lhs.current_ == rhs.current_;+  }+  friend bool operator!=(+      VectorContainerIterator const& lhs, VectorContainerIterator const& rhs) {+    return !(lhs == rhs);+  }++ private:+  ValuePtr current_;+  ValuePtr lowest_;++  explicit VectorContainerIterator(ValuePtr current, ValuePtr lowest)+      : current_(current), lowest_(lowest) {}++  std::size_t index() const { return current_ - lowest_; }++  template <+      typename K,+      typename M,+      typename H,+      typename E,+      typename A,+      typename P>+  friend class VectorContainerPolicy;++  template <typename P>+  friend class VectorContainerIterator;+};++struct VectorContainerIndexSearch {+  uint32_t index_;+};++template <+    typename Key,+    typename MappedTypeOrVoid,+    typename HasherOrVoid,+    typename KeyEqualOrVoid,+    typename AllocOrVoid,+    typename EligibleForPerturbedInsertionOrder>+class VectorContainerPolicy+    : public BasePolicy<+          Key,+          MappedTypeOrVoid,+          HasherOrVoid,+          KeyEqualOrVoid,+          AllocOrVoid,+          uint32_t> {+ public:+  using Super = BasePolicy<+      Key,+      MappedTypeOrVoid,+      HasherOrVoid,+      KeyEqualOrVoid,+      AllocOrVoid,+      uint32_t>;+  using Value = typename Super::Value;+  using Alloc = typename Super::Alloc;+  using AllocTraits = typename Super::AllocTraits;+  using ByteAlloc = typename Super::ByteAlloc;+  using ByteAllocTraits = typename Super::ByteAllocTraits;+  using BytePtr = typename Super::BytePtr;+  using Hasher = typename Super::Hasher;+  using Item = typename Super::Item;+  using ItemIter = typename Super::ItemIter;+  using KeyEqual = typename Super::KeyEqual;++  using Super::kAllocIsAlwaysEqual;++ private:+  using Super::kIsMap;++ public:+  static constexpr bool kEnableItemIteration = false;++  static constexpr bool kContinuousCapacity = true;++  using InternalSizeType = Item;++  using ConstIter =+      VectorContainerIterator<typename AllocTraits::const_pointer>;+  using Iter = std::conditional_t<+      kIsMap,+      VectorContainerIterator<typename AllocTraits::pointer>,+      ConstIter>;+  using ConstReverseIter = typename AllocTraits::const_pointer;+  using ReverseIter = std::+      conditional_t<kIsMap, typename AllocTraits::pointer, ConstReverseIter>;++  using ValuePtr = typename AllocTraits::pointer;++  //////// F14Table policy++  static constexpr bool prefetchBeforeRehash() { return true; }++  static constexpr bool prefetchBeforeCopy() { return false; }++  static constexpr bool prefetchBeforeDestroy() { return false; }++  static constexpr bool destroyItemOnClear() { return false; }++ private:+  static constexpr bool valueIsTriviallyCopyable() {+    return AllocatorHasDefaultObjectConstruct<Alloc, Value, Value>::value &&+        AllocatorHasDefaultObjectDestroy<Alloc, Value>::value &&+        std::is_trivially_copyable<Value>::value;+  }++ public:+  VectorContainerPolicy(+      Hasher const& hasher, KeyEqual const& keyEqual, Alloc const& alloc)+      : Super{hasher, keyEqual, alloc} {}++  VectorContainerPolicy(VectorContainerPolicy const& rhs) : Super{rhs} {+    // values_ will get allocated later to do the copy+  }++  VectorContainerPolicy(VectorContainerPolicy const& rhs, Alloc const& alloc)+      : Super{rhs, alloc} {+    // values_ will get allocated later to do the copy+  }++  VectorContainerPolicy(VectorContainerPolicy&& rhs) noexcept+      : Super{std::move(rhs)}, values_{rhs.values_} {+    rhs.values_ = nullptr;+  }++  VectorContainerPolicy(+      VectorContainerPolicy&& rhs, Alloc const& alloc) noexcept+      : Super{std::move(rhs), alloc} {+    if (kAllocIsAlwaysEqual || this->alloc() == rhs.alloc()) {+      // common case+      values_ = rhs.values_;+      rhs.values_ = nullptr;+    } else {+      // table must be constructed in new memory+      values_ = nullptr;+    }+  }++  VectorContainerPolicy& operator=(VectorContainerPolicy const& rhs) {+    if (this != &rhs) {+      FOLLY_SAFE_DCHECK(values_ == nullptr, "");+      Super::operator=(rhs);+    }+    return *this;+  }++  VectorContainerPolicy& operator=(VectorContainerPolicy&& rhs) noexcept {+    if (this != &rhs) {+      FOLLY_SAFE_DCHECK(values_ == nullptr, "");+      bool transfer =+          AllocTraits::propagate_on_container_move_assignment::value ||+          kAllocIsAlwaysEqual || this->alloc() == rhs.alloc();+      Super::operator=(std::move(rhs));+      if (transfer) {+        values_ = rhs.values_;+        rhs.values_ = nullptr;+      }+    }+    return *this;+  }++  void swapPolicy(VectorContainerPolicy& rhs) {+    using std::swap;+    this->swapBasePolicy(rhs);+    swap(values_, rhs.values_);+  }++  template <typename K>+  std::size_t computeKeyHash(K const& key) const {+    static_assert(+        Super::isAvalanchingHasher() == IsAvalanchingHasher<Hasher, K>::value);+    return this->hasher()(key);+  }++  std::size_t computeKeyHash(VectorContainerIndexSearch const& key) const {+    return computeItemHash(key.index_);+  }++  using Super::keyForValue;++  std::size_t computeItemHash(Item const& item) const {+    return this->computeKeyHash(keyForValue(values_[item]));+  }++  bool keyMatchesItem(+      VectorContainerIndexSearch const& key, Item const& item) const {+    return key.index_ == item;+  }++  template <typename K>+  bool keyMatchesItem(K const& key, Item const& item) const {+    return this->keyEqual()(key, keyForValue(values_[item]));+  }++  Key const& keyForValue(VectorContainerIndexSearch const& arg) const {+    return keyForValue(values_[arg.index_]);+  }++  VectorContainerIndexSearch buildArgForItem(Item const& item) const {+    return {item};+  }++  Value const& valueAtItem(Item const& item) const { return values_[item]; }++  Value&& valueAtItemForExtract(Item& item) { return std::move(values_[item]); }++  template <typename Table>+  void constructValueAtItem(+      Table&&, Item* itemAddr, VectorContainerIndexSearch arg) {+    *itemAddr = arg.index_;+  }++  template <typename Table, typename... Args>+  void constructValueAtItem(Table&& table, Item* itemAddr, Args&&... args) {+    Alloc& a = this->alloc();+    auto size = static_cast<InternalSizeType>(table.size());+    FOLLY_SAFE_DCHECK(+        table.size() < std::numeric_limits<InternalSizeType>::max(), "");+    *itemAddr = size;+    auto dst = std::addressof(values_[size]);+    // TODO(T31574848): clean up assume-s used to optimize placement new+    assume(dst != nullptr);+    AllocTraits::construct(a, dst, std::forward<Args>(args)...);++    constexpr bool perturb = FOLLY_F14_PERTURB_INSERTION_ORDER;+    if (EligibleForPerturbedInsertionOrder::value && perturb &&+        !tlsPendingSafeInserts()) {+      // Pick a random victim. We have to do this post-construction+      // because the item and tag are already set in the table before+      // calling constructValueAtItem, so if there is a tag collision+      // find may evaluate values_[size] during the search.+      auto i = static_cast<InternalSizeType>(tlsMinstdRand(size + 1));+      if (i != size) {+        auto& lhsItem = *itemAddr;+        auto rhsIter = table.find(+            VectorContainerIndexSearch{static_cast<InternalSizeType>(i)});+        FOLLY_SAFE_DCHECK(!rhsIter.atEnd(), "");+        auto& rhsItem = rhsIter.item();+        FOLLY_SAFE_DCHECK(lhsItem == size, "");+        FOLLY_SAFE_DCHECK(rhsItem == i, "");++        aligned_storage_for_t<Value> tmp;+        Value* tmpValue = static_cast<Value*>(static_cast<void*>(&tmp));+        transfer(a, std::addressof(values_[i]), tmpValue, 1);+        transfer(+            a, std::addressof(values_[size]), std::addressof(values_[i]), 1);+        transfer(a, tmpValue, std::addressof(values_[size]), 1);+        lhsItem = i;+        rhsItem = size;+      }+    }+  }++  void moveItemDuringRehash(Item* itemAddr, Item& src) { *itemAddr = src; }++  void prefetchValue(Item const& item) const {+    prefetchAddr(std::addressof(values_[item]));+  }++  void destroyItem(Item&) noexcept {}++  template <typename T>+  std::enable_if_t<IsNothrowMoveAndDestroy<T>::value>+  complainUnlessNothrowMoveAndDestroy() {}++  template <typename T>+  [[deprecated(+      "mark {key_type,mapped_type} {move constructor,destructor} noexcept, or use F14Node* if they aren't")]] std::+      enable_if_t<!IsNothrowMoveAndDestroy<T>::value>+      complainUnlessNothrowMoveAndDestroy() {}++  void transfer(Alloc& a, Value* src, Value* dst, std::size_t n) {+    complainUnlessNothrowMoveAndDestroy<Key>();+    complainUnlessNothrowMoveAndDestroy<lift_unit_t<MappedTypeOrVoid>>();++    auto origSrc = src;+    if (valueIsTriviallyCopyable()) {+      std::memcpy(+          static_cast<void*>(dst),+          static_cast<void const*>(src),+          n * sizeof(Value));+    } else {+      for (std::size_t i = 0; i < n; ++i, ++src, ++dst) {+        // TODO(T31574848): clean up assume-s used to optimize placement new+        assume(dst != nullptr);+        AllocTraits::construct(a, dst, Super::moveValue(*src));+        if (kIsMap) {+          AllocTraits::destroy(a, std::launder(src));+        } else {+          AllocTraits::destroy(a, src);+        }+      }+    }+    this->afterDestroyWithoutDeallocate(origSrc, n);+  }++  template <typename P, typename V>+  bool beforeBuildImpl(std::size_t size, P&& rhs, V const& constructorArgFor) {+    Alloc& a = this->alloc();++    FOLLY_SAFE_DCHECK(values_ != nullptr, "");++    auto src = std::addressof(rhs.values_[0]);+    Value* dst = std::addressof(values_[0]);++    if (valueIsTriviallyCopyable()) {+      std::memcpy(+          static_cast<void*>(dst),+          static_cast<void const*>(src),+          size * sizeof(Value));+    } else {+      for (std::size_t i = 0; i < size; ++i, ++src, ++dst) {+        try {+          // TODO(T31574848): clean up assume-s used to optimize placement new+          assume(dst != nullptr);+          AllocTraits::construct(a, dst, constructorArgFor(*src));+        } catch (...) {+          for (Value* cleanup = std::addressof(values_[0]); cleanup != dst;+               ++cleanup) {+            AllocTraits::destroy(a, cleanup);+          }+          throw;+        }+      }+    }+    return true;+  }++  bool beforeBuild(+      std::size_t size,+      std::size_t /*capacity*/,+      VectorContainerPolicy const& rhs) {+    return beforeBuildImpl(size, rhs, [](Value const& v) { return v; });+  }++  bool beforeBuild(+      std::size_t size, std::size_t /*capacity*/, VectorContainerPolicy&& rhs) {+    return beforeBuildImpl(size, rhs, [](Value& v) {+      return Super::moveValue(v);+    });+  }++  template <typename P>+  void afterBuild(+      bool /*undoState*/,+      bool success,+      std::size_t /*size*/,+      std::size_t /*capacity*/,+      P&& /*rhs*/) {+    // buildArgForItem can be used to construct a new item trivially,+    // so no failure between beforeBuild and afterBuild should be possible+    FOLLY_SAFE_DCHECK(success, "");+  }++ private:+  // Returns the byte offset of the first Value in a unified allocation+  // that first holds prefixBytes of data, where prefixBytes comes from+  // Chunk storage and may be only 4-byte aligned due to sub-chunk+  // allocation.+  static std::size_t valuesOffset(std::size_t prefixBytes) {+    FOLLY_SAFE_DCHECK((prefixBytes % alignof(Item)) == 0, "");+    if (alignof(Value) > alignof(Item)) {+      prefixBytes = -(-prefixBytes & ~(alignof(Value) - 1));+    }+    FOLLY_SAFE_DCHECK((prefixBytes % alignof(Value)) == 0, "");+    return prefixBytes;+  }++  // Returns the total number of bytes that should be allocated to store+  // prefixBytes of Chunks and valueCapacity values.+  static std::size_t allocSize(+      std::size_t prefixBytes, std::size_t valueCapacity) {+    return valuesOffset(prefixBytes) + sizeof(Value) * valueCapacity;+  }++ public:+  ValuePtr beforeRehash(+      std::size_t size,+      std::size_t oldCapacity,+      std::size_t newCapacity,+      std::size_t chunkAllocSize,+      BytePtr& outChunkAllocation) {+    FOLLY_SAFE_DCHECK(+        size <= oldCapacity && ((values_ == nullptr) == (oldCapacity == 0)) &&+            newCapacity > 0 &&+            newCapacity <= (std::numeric_limits<Item>::max)(),+        "");++    outChunkAllocation =+        allocateOverAligned<ByteAlloc, kRequiredVectorAlignment>(+            ByteAlloc{Super::alloc()}, allocSize(chunkAllocSize, newCapacity));++    ValuePtr before = values_;+    ValuePtr after = std::pointer_traits<ValuePtr>::pointer_to(+        *static_cast<Value*>(static_cast<void*>(+            &*outChunkAllocation + valuesOffset(chunkAllocSize))));++    if (size > 0) {+      Alloc& a = this->alloc();+      transfer(a, std::addressof(before[0]), std::addressof(after[0]), size);+    }++    values_ = after;+    return before;+  }++  FOLLY_NOINLINE void afterFailedRehash(ValuePtr state, std::size_t size) {+    // state holds the old storage+    Alloc& a = this->alloc();+    if (size > 0) {+      transfer(a, std::addressof(values_[0]), std::addressof(state[0]), size);+    }+    values_ = state;+  }++  void afterRehash(+      ValuePtr state,+      bool success,+      std::size_t size,+      std::size_t oldCapacity,+      std::size_t newCapacity,+      BytePtr chunkAllocation,+      std::size_t chunkAllocSize) {+    if (!success) {+      afterFailedRehash(state, size);+    }++    // on success, chunkAllocation is the old allocation, on failure it is the+    // new one+    if (chunkAllocation != nullptr) {+      deallocateOverAligned<ByteAlloc, kRequiredVectorAlignment>(+          ByteAlloc{Super::alloc()},+          chunkAllocation,+          allocSize(chunkAllocSize, (success ? oldCapacity : newCapacity)));+    }+  }++  void beforeClear(std::size_t size, std::size_t capacity) {+    FOLLY_SAFE_DCHECK(+        size <= capacity && ((values_ == nullptr) == (capacity == 0)), "");+    Alloc& a = this->alloc();+    for (std::size_t i = 0; i < size; ++i) {+      AllocTraits::destroy(a, std::addressof(values_[i]));+    }+  }++  void beforeReset(std::size_t size, std::size_t capacity) {+    beforeClear(size, capacity);+  }++  void afterReset(+      std::size_t /*size*/,+      std::size_t capacity,+      BytePtr chunkAllocation,+      std::size_t chunkAllocSize) {+    if (chunkAllocation != nullptr) {+      deallocateOverAligned<ByteAlloc, kRequiredVectorAlignment>(+          ByteAlloc{Super::alloc()},+          chunkAllocation,+          allocSize(chunkAllocSize, capacity));+      values_ = nullptr;+    }+  }++  template <typename V>+  void visitPolicyAllocationClasses(+      std::size_t chunkAllocSize,+      std::size_t /*size*/,+      std::size_t capacity,+      V&& visitor) const {+    FOLLY_SAFE_DCHECK((chunkAllocSize == 0) == (capacity == 0), "");+    if (chunkAllocSize > 0) {+      visitor(+          allocationBytesForOverAligned<ByteAlloc, kRequiredVectorAlignment>(+              allocSize(chunkAllocSize, capacity)),+          1);+    }+  }++  // Iterator stuff++  Iter linearBegin(std::size_t size) const {+    return size > 0+        ? Iter{values_ + size - 1, values_}+        : Iter{nullptr, nullptr};+  }++  Iter linearEnd() const { return Iter{nullptr, nullptr}; }++  //////// F14BasicMap/Set policy++  Iter makeIter(ItemIter const& underlying) const {+    if (underlying.atEnd()) {+      return linearEnd();+    } else {+      assume(values_ + underlying.item() != nullptr);+      assume(values_ != nullptr);+      return Iter{values_ + underlying.item(), values_};+    }+  }++  ConstIter makeConstIter(ItemIter const& underlying) const {+    return makeIter(underlying);+  }++  Item iterToIndex(ConstIter const& iter) const {+    auto n = iter.index();+    assume(n <= std::numeric_limits<Item>::max());+    return static_cast<Item>(n);+  }++  Iter indexToIter(Item index) const { return Iter{values_ + index, values_}; }++  Iter iter(ReverseIter it) { return Iter{it, values_}; }++  ConstIter iter(ConstReverseIter it) const { return ConstIter{it, values_}; }++  ReverseIter riter(Iter it) { return it.current_; }++  ConstReverseIter riter(ConstIter it) const { return it.current_; }++  ValuePtr values_{nullptr};+};++template <+    template <typename, typename, typename, typename, typename, typename...>+    class Policy,+    typename Key,+    typename Mapped,+    typename Hasher,+    typename KeyEqual,+    typename Alloc,+    typename... Args>+using MapPolicyWithDefaults = Policy<+    Key,+    Mapped,+    VoidDefault<Hasher, DefaultHasher<Key>>,+    VoidDefault<KeyEqual, DefaultKeyEqual<Key>>,+    VoidDefault<Alloc, DefaultAlloc<std::pair<Key const, Mapped>>>,+    Args...>;++template <+    template <typename, typename, typename, typename, typename, typename...>+    class Policy,+    typename Key,+    typename Hasher,+    typename KeyEqual,+    typename Alloc,+    typename... Args>+using SetPolicyWithDefaults = Policy<+    Key,+    void,+    VoidDefault<Hasher, DefaultHasher<Key>>,+    VoidDefault<KeyEqual, DefaultKeyEqual<Key>>,+    VoidDefault<Alloc, DefaultAlloc<Key>>,+    Args...>;++} // namespace detail+} // namespace f14+} // namespace folly++#endif // FOLLY_F14_VECTOR_INTRINSICS_AVAILABLE
+ folly/folly/container/detail/F14SetFallback.h view
@@ -0,0 +1,529 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <algorithm>+#include <type_traits>+#include <unordered_set>++#include <folly/container/detail/F14Table.h>+#include <folly/container/detail/Util.h>++/**+ * This file is intended to be included only by F14Set.h. It contains fallback+ * implementations of F14Set types for platforms that do not support the+ * required SIMD instructions, based on std::unordered_set.+ */++#if !FOLLY_F14_VECTOR_INTRINSICS_AVAILABLE++namespace folly {++namespace f14 {+namespace detail {+template <typename KeyType, typename Hasher, typename KeyEqual, typename Alloc>+class F14BasicSet+    : public std::unordered_set<KeyType, Hasher, KeyEqual, Alloc> {+  using Super = std::unordered_set<KeyType, Hasher, KeyEqual, Alloc>;++ public:+  using typename Super::allocator_type;+  using typename Super::const_iterator;+  using typename Super::hasher;+  using typename Super::iterator;+  using typename Super::key_equal;+  using typename Super::key_type;+  using typename Super::pointer;+  using typename Super::size_type;+  using typename Super::value_type;++ private:+  template <typename K, typename T>+  using EnableHeterogeneousFind = std::enable_if_t<+      ::folly::detail::+          EligibleForHeterogeneousFind<key_type, hasher, key_equal, K>::value,+      T>;++  template <typename K, typename T>+  using EnableHeterogeneousInsert = std::enable_if_t<+      ::folly::detail::+          EligibleForHeterogeneousInsert<key_type, hasher, key_equal, K>::value,+      T>;++  template <typename K>+  using IsIter = Disjunction<+      std::is_same<iterator, remove_cvref_t<K>>,+      std::is_same<const_iterator, remove_cvref_t<K>>>;++  template <typename K, typename T>+  using EnableHeterogeneousErase = std::enable_if_t<+      ::folly::detail::EligibleForHeterogeneousFind<+          key_type,+          hasher,+          key_equal,+          std::conditional_t<IsIter<K>::value, key_type, K>>::value &&+          !IsIter<K>::value,+      T>;++ public:+  F14BasicSet() = default;++  using Super::Super;++  //// PUBLIC - Modifiers++  using Super::insert;++  template <typename K>+  EnableHeterogeneousInsert<K, std::pair<iterator, bool>> insert(K&& value) {+    return emplace(std::forward<K>(value));+  }++  template <class InputIt>+  void insert(InputIt first, InputIt last) {+    while (first != last) {+      insert(*first);+      ++first;+    }+  }++ private:+  template <typename Arg>+  using UsableAsKey = ::folly::detail::+      EligibleForHeterogeneousFind<key_type, hasher, key_equal, Arg>;++ public:+  template <class... Args>+  std::pair<iterator, bool> emplace(Args&&... args) {+    auto a = this->get_allocator();+    return folly::detail::callWithConstructedKey<key_type, UsableAsKey>(+        a,+        [&](auto const&, auto&& key) {+          if (!std::is_same<key_type, remove_cvref_t<decltype(key)>>::value) {+            // this is a heterogeneous emplace+            auto it = find(key);+            if (it != this->end()) {+              return std::make_pair(it, false);+            }+            auto rv = Super::emplace(std::forward<decltype(key)>(key));+            FOLLY_SAFE_DCHECK(+                rv.second, "post-find emplace should always insert");+            return rv;+          } else {+            return Super::emplace(std::forward<decltype(key)>(key));+          }+        },+        std::forward<Args>(args)...);+  }++  template <class... Args>+  iterator emplace_hint(const_iterator /*hint*/, Args&&... args) {+    return emplace(std::forward<Args>(args)...).first;+  }++  using Super::erase;++  template <typename K>+  EnableHeterogeneousErase<K, size_type> erase(K const& key) {+    auto it = find(key);+    if (it != this->end()) {+      erase(it);+      return 1;+    } else {+      return 0;+    }+  }++  //// PUBLIC - Lookup++ private:+  // BottomKeyEqual must have same size, alignment, emptiness, and finality as+  // KeyEqual+  struct BottomKeyEqualEmpty {};+  template <size_t S, size_t A>+  struct BottomKeyEqualNonEmpty {+    alignas(A) char data[S];+  };+  using BottomKeyEqualBase = conditional_t<+      std::is_empty<KeyEqual>::value,+      BottomKeyEqualEmpty,+      BottomKeyEqualNonEmpty<sizeof(KeyEqual), alignof(KeyEqual)>>;+  template <bool IsFinal, typename K>+  struct BottomKeyEqualCond : BottomKeyEqualBase {+    [[noreturn]] bool operator()(K const&, K const&) const {+      assume_unreachable();+    }+  };+  template <typename K>+  struct BottomKeyEqualCond<true, K> final : BottomKeyEqualCond<false, K> {};+  template <typename K>+  using BottomKeyEqual = BottomKeyEqualCond<+      std::is_final<KeyEqual>::value || std::is_union<KeyEqual>::value,+      K>;+  using BottomTest = BottomKeyEqual<char>;+  static_assert(sizeof(BottomTest) == sizeof(KeyEqual), "mismatch size");+  static_assert(alignof(BottomTest) == alignof(KeyEqual), "mismatch align");+  static_assert(+      std::is_empty<BottomTest>::value == std::is_empty<KeyEqual>::value,+      "mismatch is-empty");+  static_assert(+      (std::is_final<BottomTest>::value || std::is_union<BottomTest>::value) ==+          (std::is_final<KeyEqual>::value || std::is_union<KeyEqual>::value),+      "mismatch is-final");++  template <typename Iter, typename LocalIter>+  static std::+      enable_if_t<std::is_constructible<Iter, LocalIter const&>::value, Iter>+      fromLocal(LocalIter const& src, int = 0) {+    return Iter(src);+  }++  template <typename Iter, typename LocalIter>+  static std::+      enable_if_t<!std::is_constructible<Iter, LocalIter const&>::value, Iter>+      fromLocal(LocalIter const& src) {+    Iter dst;+    static_assert(sizeof(dst) <= sizeof(src));+    std::memcpy(std::addressof(dst), std::addressof(src), sizeof(dst));+    FOLLY_SAFE_CHECK(+        std::addressof(*src) == std::addressof(*dst),+        "ABI-assuming local_iterator to iterator conversion failed");+    return dst;+  }++  template <typename Iter, typename Self, typename K>+  static Iter findImpl(Self& self, K const& key) {+    if (self.empty()) {+      return self.end();+    }+    using A = typename std::allocator_traits<+        allocator_type>::template rebind_alloc<K>;+    using E = BottomKeyEqual<K>;+    // this is exceedingly wicked!+    auto slot =+        reinterpret_cast<std::unordered_set<K, hasher, E, A> const&>(self)+            .bucket(key);+    auto b = self.begin(slot);+    auto e = self.end(slot);+    while (b != e) {+      if (self.key_eq()(key, *b)) {+        return fromLocal<Iter>(b);+      }+      ++b;+    }+    FOLLY_SAFE_DCHECK(+        self.size() > 3 ||+            std::none_of(+                self.begin(),+                self.end(),+                [&](auto const& k) { return self.key_eq()(key, k); }),+        "");+    return self.end();+  }++ public:+  using Super::count;++  template <typename K>+  EnableHeterogeneousFind<K, size_type> count(K const& key) const {+    return contains(key) ? 1 : 0;+  }++  using Super::find;++  template <typename K>+  EnableHeterogeneousFind<K, iterator> find(K const& key) {+    return findImpl<iterator>(*this, key);+  }++  template <typename K>+  EnableHeterogeneousFind<K, const_iterator> find(K const& key) const {+    return findImpl<const_iterator>(*this, key);+  }++  bool contains(key_type const& key) const { return find(key) != this->end(); }++  template <typename K>+  EnableHeterogeneousFind<K, bool> contains(K const& key) const {+    return find(key) != this->end();+  }++ private:+  template <typename Self, typename K>+  static auto equalRangeImpl(Self& self, K const& key) {+    auto first = self.find(key);+    auto last = first;+    if (last != self.end()) {+      ++last;+    }+    return std::make_pair(first, last);+  }++ public:+  using Super::equal_range;++  template <typename K>+  EnableHeterogeneousFind<K, std::pair<iterator, iterator>> equal_range(+      K const& key) {+    return equalRangeImpl(*this, key);+  }++  template <typename K>+  EnableHeterogeneousFind<K, std::pair<const_iterator, const_iterator>>+  equal_range(K const& key) const {+    return equalRangeImpl(*this, key);+  }++  //// PUBLIC - F14 Extensions++ private:+  // converts const_iterator to iterator when they are different types+  // such as in libstdc+++  template <typename... Args>+  iterator citerToIter(const_iterator cit, Args&&...) {+    iterator it = erase(cit, cit);+    FOLLY_SAFE_CHECK(std::addressof(*it) == std::addressof(*cit), "");+    return it;+  }++  // converts const_iterator to iterator when they are the same type+  // such as in libc+++  iterator citerToIter(iterator it) { return it; }++ public:+  template <typename BeforeDestroy>+  iterator eraseInto(const_iterator pos, BeforeDestroy&& beforeDestroy) {+    iterator next = citerToIter(pos);+    ++next;+    auto nh = this->extract(pos);+    if (!nh.empty()) {+      beforeDestroy(std::move(nh.value()));+    }+    return next;+  }++  template <typename BeforeDestroy>+  iterator eraseInto(+      const_iterator first,+      const_iterator last,+      BeforeDestroy&& beforeDestroy) {+    iterator pos = citerToIter(first);+    while (pos != last) {+      pos = eraseInto(pos, beforeDestroy);+    }+    return pos;+  }++ private:+  template <typename K, typename BeforeDestroy>+  size_type eraseIntoImpl(K const& key, BeforeDestroy& beforeDestroy) {+    auto it = find(key);+    if (it != this->end()) {+      eraseInto(it, beforeDestroy);+      return 1;+    } else {+      return 0;+    }+  }++ public:+  template <typename BeforeDestroy>+  size_type eraseInto(key_type const& key, BeforeDestroy&& beforeDestroy) {+    return eraseIntoImpl(key, beforeDestroy);+  }++  template <typename K, typename BeforeDestroy>+  EnableHeterogeneousErase<K, size_type> eraseInto(+      K const& key, BeforeDestroy&& beforeDestroy) {+    return eraseIntoImpl(key, beforeDestroy);+  }++  bool containsEqualValue(value_type const& value) const {+    // bucket is only valid if bucket_count is non-zero+    if (this->empty()) {+      return false;+    }+    auto slot = this->bucket(value);+    auto e = this->end(slot);+    for (auto b = this->begin(slot); b != e; ++b) {+      if (*b == value) {+        return true;+      }+    }+    return false;+  }++  // exact for libstdc++, approximate for others+  std::size_t getAllocatedMemorySize() const {+    std::size_t rv = 0;+    visitAllocationClasses([&](std::size_t bytes, std::size_t n) {+      rv += bytes * n;+    });+    return rv;+  }++  // exact for libstdc++, approximate for others+  template <typename V>+  void visitAllocationClasses(V&& visitor) const {+    auto bc = this->bucket_count();+    if (bc > 1) {+      visitor(bc * sizeof(pointer), 1);+    }+    if (this->size() > 0) {+      visitor(+          sizeof(StdNodeReplica<key_type, value_type, hasher>), this->size());+    }+  }++  template <typename V>+  void visitContiguousRanges(V&& visitor) const {+    for (value_type const& entry : *this) {+      value_type const* b = std::addressof(entry);+      visitor(b, b + 1);+    }+  }++  /// F14HashToken interface+  template <class... Args>+  std::pair<iterator, bool> emplace_token(F14HashToken const&, Args&&... args) {+    return emplace(std::forward<Args>(args)...);+  }++  F14HashToken prehash(key_type const& /*key*/) const {+    return {}; // Ignored.+  }+  F14HashToken prehash(key_type const& /*key*/, std::size_t /*hash*/) const {+    return {}; // Ignored.+  }++  template <typename K>+  EnableHeterogeneousFind<K, F14HashToken> prehash(K const& /*key*/) const {+    return {};+  }+  template <typename K>+  EnableHeterogeneousFind<K, F14HashToken> prehash(+      K const& /*key*/, std::size_t /*hash*/) const {+    return {};+  }++  void prefetch(F14HashToken const& /*token*/) const {}++  iterator find(F14HashToken const&, key_type const& key) { return find(key); }++  const_iterator find(F14HashToken const&, key_type const& key) const {+    return find(key);+  }++  template <typename K>+  EnableHeterogeneousFind<K, iterator> find(F14HashToken const&, K const& key) {+    return find(key);+  }++  template <typename K>+  EnableHeterogeneousFind<K, const_iterator> find(+      F14HashToken const&, K const& key) const {+    return find(key);+  }++  bool contains(F14HashToken const&, key_type const& key) const {+    return find(key) != this->end();+  }++  template <typename K>+  EnableHeterogeneousFind<K, bool> contains(+      F14HashToken const&, K const& key) const {+    return find(key) != this->end();+  }+};+} // namespace detail+} // namespace f14++template <typename Key, typename Hasher, typename KeyEqual, typename Alloc>+class F14NodeSet+    : public f14::detail::F14BasicSet<Key, Hasher, KeyEqual, Alloc> {+  using Super = f14::detail::F14BasicSet<Key, Hasher, KeyEqual, Alloc>;++ public:+  using typename Super::value_type;++  F14NodeSet() = default;++  using Super::Super;++  F14NodeSet& operator=(std::initializer_list<value_type> ilist) {+    Super::operator=(ilist);+    return *this;+  }+};++template <typename Key, typename Hasher, typename KeyEqual, typename Alloc>+class F14ValueSet+    : public f14::detail::F14BasicSet<Key, Hasher, KeyEqual, Alloc> {+  using Super = f14::detail::F14BasicSet<Key, Hasher, KeyEqual, Alloc>;++ public:+  using typename Super::value_type;++  F14ValueSet() : Super() {}++  using Super::Super;++  F14ValueSet& operator=(std::initializer_list<value_type> ilist) {+    Super::operator=(ilist);+    return *this;+  }+};++template <typename Key, typename Hasher, typename KeyEqual, typename Alloc>+class F14VectorSet+    : public f14::detail::F14BasicSet<Key, Hasher, KeyEqual, Alloc> {+  using Super = f14::detail::F14BasicSet<Key, Hasher, KeyEqual, Alloc>;++ public:+  using typename Super::value_type;++  F14VectorSet() = default;++  using Super::Super;++  F14VectorSet& operator=(std::initializer_list<value_type> ilist) {+    Super::operator=(ilist);+    return *this;+  }+};++template <typename Key, typename Hasher, typename KeyEqual, typename Alloc>+class F14FastSet+    : public f14::detail::F14BasicSet<Key, Hasher, KeyEqual, Alloc> {+  using Super = f14::detail::F14BasicSet<Key, Hasher, KeyEqual, Alloc>;++ public:+  using typename Super::value_type;++  F14FastSet() = default;++  using Super::Super;++  F14FastSet& operator=(std::initializer_list<value_type> ilist) {+    Super::operator=(ilist);+    return *this;+  }+};++} // namespace folly++#endif // !if FOLLY_F14_VECTOR_INTRINSICS_AVAILABLE
+ folly/folly/container/detail/F14Table.cpp view
@@ -0,0 +1,71 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#include <folly/container/detail/F14Table.h>++#include <atomic>+#include <chrono>++namespace folly {+namespace f14 {+namespace detail {++// If you get a link failure that leads you here, your build has varying+// compiler flags across compilation units in a way that would break F14.+// SIMD (SSE2 or NEON) needs to be either on everywhere or off everywhere+// that uses F14.  If SIMD is on then hardware CRC needs to be enabled+// everywhere or disabled everywhere.+void F14LinkCheck<getF14IntrinsicsMode()>::check() noexcept {}++//// Debug and ASAN stuff++bool tlsPendingSafeInserts(std::ptrdiff_t delta) {+  static std::atomic<size_t> value_non_tl{0};+  static thread_local std::atomic<size_t> value_tl{0};+  auto& value = kIsDebug || kIsLibrarySanitizeAddress ? value_tl : value_non_tl;++  FOLLY_SAFE_DCHECK(delta >= -1, "");+  std::size_t v = value.load(std::memory_order_acquire);+  if (delta > 0 || (delta == -1 && v > 0)) {+    v += delta;+    v = std::min(std::numeric_limits<std::size_t>::max() / 2, v);+    value.store(v, std::memory_order_release);+  }+  return v != 0;+}++std::size_t tlsMinstdRand(std::size_t n) {+  static std::atomic<uint32_t> state_non_tl{0};+  static thread_local std::atomic<uint32_t> state_tl{0};+  auto& state = kIsDebug || kIsLibrarySanitizeAddress ? state_tl : state_non_tl;++  FOLLY_SAFE_DCHECK(n > 0, "");++  auto s = state.load(std::memory_order_acquire);+  if (s == 0) {+    uint64_t seed = static_cast<uint64_t>(+        std::chrono::steady_clock::now().time_since_epoch().count());+    s = hash::twang_32from64(seed);+  }++  s = static_cast<uint32_t>((s * uint64_t{48271}) % uint64_t{2147483647});+  state.store(s, std::memory_order_release);+  return std::size_t{s} % n;+}++} // namespace detail+} // namespace f14+} // namespace folly
+ folly/folly/container/detail/F14Table.h view
@@ -0,0 +1,2832 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <cstddef>+#include <cstdint>+#include <cstring>++#include <array>+#include <iterator>+#include <limits>+#include <memory>+#include <new>+#include <string_view>+#include <type_traits>+#include <utility>+#include <vector>++#include <folly/Bits.h>+#include <folly/ConstexprMath.h>+#include <folly/Likely.h>+#include <folly/Memory.h>+#include <folly/Portability.h>+#include <folly/ScopeGuard.h>+#include <folly/Traits.h>+#include <folly/functional/Invoke.h>+#include <folly/lang/Align.h>+#include <folly/lang/Assume.h>+#include <folly/lang/Exception.h>+#include <folly/lang/Pretty.h>+#include <folly/lang/SafeAssert.h>+#include <folly/portability/Builtins.h>++#include <folly/container/HeterogeneousAccess.h>+#include <folly/container/detail/F14Defaults.h>+#include <folly/container/detail/F14IntrinsicsAvailability.h>+#include <folly/container/detail/F14Mask.h>++#if FOLLY_ARM_FEATURE_NEON_SVE_BRIDGE+#include <arm_neon_sve_bridge.h> // @manual+#include <arm_sve.h>+#endif++#if __has_include(<concepts>)+#include <concepts>+#endif++#if FOLLY_F14_VECTOR_INTRINSICS_AVAILABLE++#if FOLLY_F14_CRC_INTRINSIC_AVAILABLE+#if FOLLY_NEON+#include <arm_acle.h> // __crc32cd+#else+#include <nmmintrin.h> // _mm_crc32_u64+#endif+#else+#ifdef _WIN32+#include <intrin.h> // _mul128 in fallback bit mixer+#endif+#endif++#if FOLLY_NEON+#include <arm_neon.h> // uint8x16t intrinsics+#if FOLLY_ARM_FEATURE_NEON_SVE_BRIDGE+#include <arm_neon_sve_bridge.h> // @manual+#include <arm_sve.h>+#endif+#elif FOLLY_SSE >= 2 // SSE2+#include <emmintrin.h> // _mm_set1_epi8+#include <immintrin.h> // __m128i intrinsics+#include <xmmintrin.h> // _mm_prefetch+#endif++#ifndef FOLLY_F14_PERTURB_INSERTION_ORDER+#define FOLLY_F14_PERTURB_INSERTION_ORDER folly::kIsDebug+#endif++#else // FOLLY_F14_VECTOR_INTRINSICS_AVAILABLE++#ifndef FOLLY_F14_PERTURB_INSERTION_ORDER+#define FOLLY_F14_PERTURB_INSERTION_ORDER false+#endif++#endif // FOLLY_F14_VECTOR_INTRINSICS_AVAILABLE++namespace folly {++struct F14TableStats {+  char const* policy;+  std::size_t size{0};+  std::size_t valueSize{0};+  std::size_t bucketCount{0};+  std::size_t chunkCount{0};+  std::vector<std::size_t> chunkOccupancyHisto;+  std::vector<std::size_t> chunkOutboundOverflowHisto;+  std::vector<std::size_t> chunkHostedOverflowHisto;+  std::vector<std::size_t> keyProbeLengthHisto;+  std::vector<std::size_t> missProbeLengthHisto;+  std::size_t totalBytes{0};+  std::size_t overheadBytes{0};++ private:+  template <typename T>+  static auto computeHelper(T const* m) -> decltype(m->computeStats()) {+    return m->computeStats();+  }++  static F14TableStats computeHelper(...) { return {}; }++ public:+  template <typename T>+  static F14TableStats compute(T const& m) {+    return computeHelper(&m);+  }+};++namespace f14 {+namespace detail {++template <F14IntrinsicsMode>+struct F14LinkCheck {};++template <>+struct F14LinkCheck<getF14IntrinsicsMode()> {+  // The purpose of this method is to trigger a link failure if+  // compilation flags vary across compilation units.  The definition+  // is in F14Table.cpp, so only one of F14LinkCheck<None>::check,+  // F14LinkCheck<Simd>::check, or F14LinkCheck<SimdAndCrc>::check will+  // be available at link time.+  //+  // To cause a link failure the function must be invoked in code that+  // is not optimized away, so we call it on a couple of cold paths+  // (exception handling paths in copy construction and rehash).  LTO may+  // remove it entirely, but that's fine.+  static void check() noexcept;+};++bool tlsPendingSafeInserts(std::ptrdiff_t delta = 0);+std::size_t tlsMinstdRand(std::size_t n);++#if defined(_LIBCPP_VERSION)++template <typename K, typename V, typename H>+struct StdNodeReplica {+  void* next;+  std::size_t hash;+  V value;+};++#else++template <typename H>+struct StdIsFastHash : std::true_type {};+template <>+struct StdIsFastHash<std::hash<long double>> : std::false_type {};+template <typename... Args>+struct StdIsFastHash<std::hash<std::basic_string<Args...>>> : std::false_type {+};+template <typename... Args>+struct StdIsFastHash<std::hash<std::basic_string_view<Args...>>>+    : std::false_type {};++// mimic internal node of unordered containers in STL to estimate the size+template <typename K, typename V, typename H, typename Enable = void>+struct StdNodeReplica {+  void* next;+  V value;+};+template <typename K, typename V, typename H>+struct StdNodeReplica<+    K,+    V,+    H,+    std::enable_if_t<+        !StdIsFastHash<H>::value || !is_nothrow_invocable_v<H, K>>> {+  void* next;+  V value;+  std::size_t hash;+};++#endif++template <class Container, class Predicate>+typename Container::size_type erase_if_impl(+    Container& c, Predicate& predicate) {+  auto const old_size = c.size();+  for (auto i = c.begin(), last = c.end(); i != last;) {+    auto prev = i++;+    if (predicate(*prev)) {+      c.erase(prev);+    }+  }+  return old_size - c.size();+}++} // namespace detail+} // namespace f14++#if FOLLY_F14_VECTOR_INTRINSICS_AVAILABLE+namespace f14 {+namespace detail {+template <typename Policy>+class F14Table;+} // namespace detail+} // namespace f14++class F14HashToken final {+ public:+  constexpr F14HashToken() = default;++  friend constexpr bool operator==(+      F14HashToken const& a, F14HashToken const& b) noexcept {+    return a.hp_.first == b.hp_.first; // processed hash but not tag+  }+  friend constexpr bool operator!=(+      F14HashToken const& a, F14HashToken const& b) noexcept {+    return !(a == b);+  }++ private:+  using HashPair = std::pair<std::size_t, std::size_t>;++  constexpr explicit F14HashToken(HashPair hp) noexcept : hp_(hp) {}+  constexpr explicit operator HashPair() const noexcept { return hp_; }++  HashPair hp_;++  template <typename Policy>+  friend class f14::detail::F14Table;++  template <typename Key, typename Hasher, typename KeyEqual>+  friend class F14HashedKey;+};++#else+class F14HashToken final {+  friend constexpr bool operator==(+      F14HashToken const&, F14HashToken const&) noexcept {+    return true;+  }+  friend constexpr bool operator!=(+      F14HashToken const&, F14HashToken const&) noexcept {+    return false;+  }+};+#endif++#if defined(__cpp_concepts) && __cpp_concepts && __has_include(<concepts>)+static_assert(std::regular<F14HashToken>);+#endif++namespace f14 {+namespace detail {++// Detection for folly_assume_32bit_hash++template <typename Hasher, typename Void = void>+struct ShouldAssume32BitHash : std::bool_constant<!require_sizeof<Hasher>> {};++template <typename Hasher>+struct ShouldAssume32BitHash<+    Hasher,+    void_t<typename Hasher::folly_assume_32bit_hash>>+    : std::bool_constant<Hasher::folly_assume_32bit_hash::value> {};++//////// hash helpers++// Hash values are used to compute the desired position, which is the+// chunk index at which we would like to place a value (if there is no+// overflow), and the tag, which is an additional 7 bits of entropy.+//+// The standard's definition of hash function quality only refers to+// the probability of collisions of the entire hash value, not to the+// probability of collisions of the results of shifting or masking the+// hash value.  Some hash functions, however, provide this stronger+// guarantee (not quite the same as the definition of avalanching,+// but similar).+//+// If the user-supplied hasher is an avalanching one (each bit of the+// hash value has a 50% chance of being the same for differing hash+// inputs), then we can just take 7 bits of the hash value for the tag+// and the rest for the desired position.  Avalanching hashers also+// let us map hash value to array index position with just a bitmask+// without risking clumping.  (Many hash tables just accept the risk+// and do it regardless.)+//+// std::hash<std::string> avalanches in all implementations we've+// examined: libstdc++-v3 uses MurmurHash2, and libc++ uses CityHash+// or MurmurHash2.  The other std::hash specializations, however, do not+// have this property.  std::hash for integral and pointer values is the+// identity function on libstdc++-v3 and libc++, in particular.  In our+// experience it is also fairly common for user-defined specializations+// of std::hash to combine fields in an ad-hoc way that does not evenly+// distribute entropy among the bits of the result (a + 37 * b, for+// example, where a and b are integer fields).+//+// For hash functions we don't trust to avalanche, we repair things by+// applying a bit mixer to the user-supplied hash.++#if FOLLY_F14_VECTOR_INTRINSICS_AVAILABLE+#if FOLLY_X64 || FOLLY_AARCH64 || FOLLY_RISCV64+// 64-bit+template <typename Hasher, typename Key>+std::pair<std::size_t, std::size_t> splitHashImpl(std::size_t hash) {+  static_assert(sizeof(std::size_t) == sizeof(uint64_t));+  std::size_t tag;+  if (!IsAvalanchingHasher<Hasher, Key>::value) {+#if FOLLY_F14_CRC_INTRINSIC_AVAILABLE+#if FOLLY_SSE_PREREQ(4, 2)+    // SSE4.2 CRC+    std::size_t c = _mm_crc32_u64(0, hash);+    tag = (c >> 24) | 0x80;+    hash += c;+#else+    // CRC is optional on armv8 (-march=armv8-a+crc), standard on armv8.1+    std::size_t c = __crc32cd(0, hash);+    tag = (c >> 24) | 0x80;+    hash += c;+#endif+#else+    // The mixer below is not fully avalanching for all 64 bits of+    // output, but looks quite good for bits 18..63 and puts plenty+    // of entropy even lower when considering multiple bits together+    // (like the tag).  Importantly, when under register pressure it+    // uses fewer registers, instructions, and immediate constants+    // than the alternatives, resulting in compact code that is more+    // easily inlinable.  In one instantiation a modified Murmur mixer+    // was 48 bytes of assembly (even after using the same multiplicand+    // for both steps) and this one was 27 bytes, for example.+    auto const kMul = 0xc4ceb9fe1a85ec53ULL;+#ifdef _WIN32+    __int64 signedHi;+    __int64 signedLo = _mul128(+        static_cast<__int64>(hash), static_cast<__int64>(kMul), &signedHi);+    auto hi = static_cast<uint64_t>(signedHi);+    auto lo = static_cast<uint64_t>(signedLo);+#else+    auto hi = static_cast<uint64_t>(+        (static_cast<unsigned __int128>(hash) * kMul) >> 64);+    auto lo = hash * kMul;+#endif+    hash = hi ^ lo;+    hash *= kMul;+    tag = ((hash >> 15) & 0x7f) | 0x80;+    hash >>= 22;+#endif+  } else {+    // F14 uses the bottom bits of the hash to form the index, so for maps+    // with less than 16.7 million entries, it's safe to have a 32-bit hash,+    // and use the bottom 24 bits for the index and leave the top 8 for the+    // tag.+    //+    // | 0x80 sets the top bit in the tag.+    // We need to avoid 0 tag for a non-empty value.+    // In some places we also rely on the top bit+    // being 1 for all non-empty values.+    if (ShouldAssume32BitHash<Hasher>::value) {+      tag = ((hash >> 24) | 0x80) & 0xFF;+      // Explicitly mask off the top 32-bits so that the compiler can+      // optimize away whatever is populating the top 32-bits, which is likely+      // just the lower 32-bits duplicated.+      hash = hash & 0xFFFF'FFFF;+    } else {+      tag = (hash >> 56) | 0x80;+    }+  }+  return std::make_pair(hash, tag);+}+#else+// 32-bit+template <typename Hasher, typename Key>+std::pair<std::size_t, std::size_t> splitHashImpl(std::size_t hash) {+  static_assert(sizeof(std::size_t) == sizeof(uint32_t));+  uint8_t tag;+  if (!IsAvalanchingHasher<Hasher, Key>::value) {+#if FOLLY_F14_CRC_INTRINSIC_AVAILABLE+#if FOLLY_SSE_PREREQ(4, 2)+    // SSE4.2 CRC+    auto c = _mm_crc32_u32(0, hash);+    tag = static_cast<uint8_t>(~(c >> 25));+    hash += c;+#else+    auto c = __crc32cw(0, hash);+    tag = static_cast<uint8_t>(~(c >> 25));+    hash += c;+#endif+#else+    // finalizer for 32-bit murmur2+    hash ^= hash >> 13;+    hash *= 0x5bd1e995;+    hash ^= hash >> 15;+    tag = static_cast<uint8_t>(~(hash >> 25));+#endif+  } else {+    // | 0x80 sets the top bit in the tag.+    // We need to avoid 0 tag for a non-empty value.+    // In some places we also rely on the top bit+    // being 1 for all non-empty values.+    tag = (hash >> 24) | 0x80;+  }+  return std::make_pair(hash, tag);+}+#endif+#endif+} // namespace detail+} // namespace f14++template <+    typename TKeyType,+    typename Hasher = f14::DefaultHasher<TKeyType>,+    typename KeyEqual = f14::DefaultKeyEqual<TKeyType>>+class F14HashedKey final {+ private:+  template <typename K>+  using EligibleForHeterogeneousCompare =+      detail::EligibleForHeterogeneousFind<TKeyType, Hasher, KeyEqual, K>;++  template <typename K, typename T>+  using EnableHeterogeneousCompare =+      std::enable_if_t<EligibleForHeterogeneousCompare<K>::value, T>;++  static constexpr void checkTemplateParamContract() {+    static_assert(is_constexpr_default_constructible_v<Hasher>);+    static_assert(is_constexpr_default_constructible_v<KeyEqual>);+    static_assert(std::is_trivially_copyable_v<Hasher>);+    static_assert(std::is_trivially_copyable_v<KeyEqual>);+    static_assert(std::is_empty_v<Hasher>);+    static_assert(std::is_empty_v<KeyEqual>);+    // When `Hasher` or `KeyEqual` is not transparent, `F14HashedKey` will+    // behave like `TKeyType` without any performance effect, it is most likely+    // not what is expected.+    static_assert(is_transparent_v<Hasher>);+    static_assert(is_transparent_v<KeyEqual>);+  }++ public:+#if FOLLY_F14_VECTOR_INTRINSICS_AVAILABLE+  template <typename... Args>+  explicit F14HashedKey(Args&&... args)+      : key_(std::forward<Args>(args)...),+        hash_(f14::detail::splitHashImpl<Hasher, TKeyType>(Hasher{}(key_))) {+    checkTemplateParamContract();+  }+#else+  F14HashedKey() = delete;+#endif++  const TKeyType& getKey() const { return key_; }+  const F14HashToken& getHashToken() const { return hash_; }+  // We want the conversion to the key to be implicit - the hashed key should+  // seamlessly behave as the key itself.+  /* implicit */ operator const TKeyType&() const { return key_; }+  explicit operator const F14HashToken&() const { return hash_; }++  template <typename T>+  using IsRangeConvertible =+      std::enable_if_t<detail::TransparentlyConvertibleToRange<T>::value>;++  template <typename T>+  using RangeT =+      Range<typename detail::ValueTypeForTransparentConversionToRange<+          T>::type const*>;++  template <typename K = TKeyType, typename Enable = IsRangeConvertible<K>>+  constexpr explicit operator RangeT<K>() const {+    return key_;+  }++  friend bool operator==(const F14HashedKey& a, const F14HashedKey& b) {+    return KeyEqual{}(a.key_, b.key_);+  }+  friend bool operator!=(const F14HashedKey& a, const F14HashedKey& b) {+    return !(a == b);+  }+  friend bool operator==(const F14HashedKey& a, const TKeyType& b) {+    return KeyEqual{}(a.key_, b);+  }+  friend bool operator!=(const F14HashedKey& a, const TKeyType& b) {+    return !(a == b);+  }+  friend bool operator==(const TKeyType& a, const F14HashedKey& b) {+    return KeyEqual{}(a, b.key_);+  }+  friend bool operator!=(const TKeyType& a, const F14HashedKey& b) {+    return !(a == b);+  }+  template <typename K>+  friend EnableHeterogeneousCompare<K, bool> operator==(+      const F14HashedKey& a, const K& b) {+    return KeyEqual{}(a.key_, b);+  }+  template <typename K>+  friend EnableHeterogeneousCompare<K, bool> operator!=(+      const F14HashedKey& a, const K& b) {+    return !(a == b);+  }+  template <typename K>+  friend EnableHeterogeneousCompare<K, bool> operator==(+      const K& a, const F14HashedKey& b) {+    return KeyEqual{}(a, b.key_);+  }+  template <typename K>+  friend EnableHeterogeneousCompare<K, bool> operator!=(+      const K& a, const F14HashedKey& b) {+    return !(a == b);+  }++ private:+  TKeyType key_;+  F14HashToken hash_;+};++#if FOLLY_F14_VECTOR_INTRINSICS_AVAILABLE+namespace f14 {+namespace detail {++//// Defaults should be selected using void+template <typename Arg, typename Default>+using VoidDefault =+    std::conditional_t<std::is_same<Arg, Default>::value, void, Arg>;++template <typename Arg, typename Default>+using Defaulted =+    std::conditional_t<std::is_same<Arg, void>::value, Default, Arg>;++////////////////++/// Prefetch the first cache line of the object at ptr.+template <typename T>+FOLLY_ALWAYS_INLINE static void prefetchAddr(T const* ptr) {+#ifndef _WIN32+  FOLLY_PUSH_WARNING+  FOLLY_GNU_DISABLE_WARNING("-Warray-bounds")+  /// The argument ptr is permitted to be wild, since wild pointers are allowed+  /// for prefetching on every architecture. While this behavior is technically+  /// undefined (forbidden) in C and C++, we need this behavior in order to+  /// avoid extra cost in the callers. Recent versions of GCC warn when they+  /// detect uses of pointers which may be wild. So we suppress the warning.+  __builtin_prefetch(static_cast<void const*>(ptr));+  FOLLY_POP_WARNING+#elif FOLLY_NEON+  __prefetch(static_cast<void const*>(ptr));+#elif FOLLY_SSE >= 2+  _mm_prefetch(+      static_cast<char const*>(static_cast<void const*>(ptr)), _MM_HINT_T0);+#endif+}++#if FOLLY_NEON+using TagVector = uint8x16_t;+#elif FOLLY_SSE >= 2+using TagVector = __m128i;+#elif FOLLY_HAVE_INT128_T+using TagVector = __uint128_t;+#endif++// We could use unaligned loads to relax this requirement, but that+// would be both a performance penalty and require a bulkier packed+// ItemIter format+constexpr std::size_t kRequiredVectorAlignment =+    constexpr_max(std::size_t{16}, alignof(max_align_t));++struct alignas(kRequiredVectorAlignment) F14EmptyTagVector {+  std::array<uint8_t, 15> bytes_{};+  uint8_t marker_{255}; // coincides with outboundOverflowCount_+};++FOLLY_EXPORT inline F14EmptyTagVector& getF14EmptyTagVector() noexcept {+  static constexpr F14EmptyTagVector instance;+  auto const raw = reinterpret_cast<uintptr_t>(&instance);+  FOLLY_SAFE_DCHECK(+      (raw % kRequiredVectorAlignment) == 0,+      raw,+      " not aligned to ",+      kRequiredVectorAlignment);+  return const_cast<F14EmptyTagVector&>(instance);+}++template <typename ItemType>+struct alignas(kRequiredVectorAlignment) F14Chunk {+  using Item = ItemType;++  // For our 16 byte vector alignment (and assuming alignof(Item) >=+  // 4) kCapacity of 14 is the most space efficient.  Slightly smaller+  // or larger capacities can help with cache alignment in a couple of+  // cases without wasting too much space, but once the items are larger+  // then we're unlikely to get much benefit anyway.  The only case we+  // optimize is using kCapacity of 12 for 4 byte items, which makes the+  // chunk take exactly 1 cache line, and adding 16 bytes of padding for+  // 16 byte items so that a chunk takes exactly 4 cache lines.+  static constexpr unsigned kCapacity = sizeof(Item) == 4 ? 12 : 14;++  static constexpr unsigned kDesiredCapacity = kCapacity - 2;++  static constexpr unsigned kAllocatedCapacity =+      kCapacity + (sizeof(Item) == 16 ? 1 : 0);++  // If kCapacity == 12 then we get 16 bits of capacityScale by using+  // tag 12 and 13, otherwise we only get 4 bits of control_+  static constexpr std::size_t kCapacityScaleBits = kCapacity == 12 ? 16 : 4;+  static constexpr std::size_t kCapacityScaleShift = kCapacityScaleBits - 4;++  static constexpr MaskType kFullMask = FullMask<kCapacity>::value;++  static constexpr std::uint8_t kOutboundOverflowMax = 254;+  static constexpr std::uint8_t kOutboundOverflowEmpty = 255;++  // Non-empty tags have their top bit set.  tags_ array might be bigger+  // than kCapacity to keep alignment of first item.+  std::array<uint8_t, 14> tags_;++  // Bits 0..3 of chunk 0 record the scaling factor between the number of+  // chunks and the max size without rehash.  Bits 4-7 in any chunk are a+  // 4-bit counter of the number of values in this chunk that were placed+  // because they overflowed their desired chunk (hostedOverflowCount).+  uint8_t control_;++  // The number of values that would have been placed into this chunk if+  // there had been space, including values that also overflowed previous+  // full chunks.  This value saturates; once it becomes 254 it no longer+  // increases nor decreases.+  uint8_t outboundOverflowCount_;++  std::array<aligned_storage_for_t<Item>, kAllocatedCapacity> rawItems_;++  FOLLY_EXPORT static F14Chunk* getSomeEmptyInstance() noexcept {+    return reinterpret_cast<F14Chunk*>(&getF14EmptyTagVector());+  }++  static bool isEmptyInstance(F14Chunk const* const chunk) noexcept {+    auto const empty = reinterpret_cast<F14EmptyTagVector const*>(chunk);+    return empty->marker_ == kOutboundOverflowEmpty;+  }++  void clear() {+    // tags_ = {}; control_ = 0; outboundOverflowCount_ = 0;++    // gcc < 6 doesn't exploit chunk alignment to generate the optimal+    // SSE clear from memset.  This is very hot code, so it is worth+    // handling that case specially.+    std::memset(&tags_[0], '\0', 16);+  }++  void copyOverflowInfoFrom(F14Chunk const& rhs) {+    FOLLY_SAFE_DCHECK(!isEmptyInstance(&rhs));+    FOLLY_SAFE_DCHECK(hostedOverflowCount() == 0, "");+    control_ += static_cast<uint8_t>(rhs.control_ & 0xf0);+    outboundOverflowCount_ = rhs.outboundOverflowCount_;+  }++  unsigned hostedOverflowCount() const { return control_ >> 4; }++  static constexpr uint8_t kIncrHostedOverflowCount = 0x10;+  static constexpr uint8_t kDecrHostedOverflowCount =+      static_cast<uint8_t>(-0x10);++  void adjustHostedOverflowCount(uint8_t op) { control_ += op; }++  bool eof() const { return capacityScale(this) != 0; }++  static std::size_t capacityScale(F14Chunk const* const chunk) {+    auto const empty = reinterpret_cast<F14EmptyTagVector const*>(chunk);+    if constexpr (kCapacityScaleBits == 4) {+      return empty->bytes_[offsetof(F14Chunk, control_)] & 0xf;+    } else {+      uint16_t v;+      std::memcpy(&v, &empty->bytes_[12], 2);+      return v;+    }+  }++  void setCapacityScale(std::size_t scale) {+    FOLLY_SAFE_DCHECK(+        !isEmptyInstance(this) && scale > 0 &&+            scale < (std::size_t{1} << kCapacityScaleBits),+        "");+    if constexpr (kCapacityScaleBits == 4) {+      control_ = static_cast<uint8_t>((control_ & ~0xf) | scale);+    } else {+      uint16_t v = static_cast<uint16_t>(scale);+      std::memcpy(&tags_[12], &v, 2);+    }+  }++  void markEof(std::size_t scale) {+    folly::assume(control_ == 0);+    setCapacityScale(scale);+  }++  unsigned outboundOverflowCount() const { return outboundOverflowCount_; }++  void incrOutboundOverflowCount() {+    if (outboundOverflowCount_ != kOutboundOverflowMax) {+      ++outboundOverflowCount_;+    }+  }++  void decrOutboundOverflowCount() {+    FOLLY_SAFE_DCHECK(outboundOverflowCount_ != 0);+    if (outboundOverflowCount_ != kOutboundOverflowMax) {+      --outboundOverflowCount_;+    }+  }++  std::size_t tag(std::size_t index) const { return tags_[index]; }++  void setTag(std::size_t index, std::size_t tag) {+    FOLLY_SAFE_DCHECK(!isEmptyInstance(this) && tag >= 0x80 && tag <= 0xff, "");+    FOLLY_SAFE_CHECK(tags_[index] == 0, "");+    tags_[index] = static_cast<uint8_t>(tag);+  }++  void clearTag(std::size_t index) {+    FOLLY_SAFE_CHECK((tags_[index] & 0x80) != 0, "");+    tags_[index] = 0;+  }++#if FOLLY_NEON++  ////////+  // Tag filtering using NEON/SVE intrinsics++#if FOLLY_ARM_FEATURE_NEON_SVE_BRIDGE++  SparseMaskIter tagMatchIter(uint8x16_t needleV, svbool_t pred) const {+    svuint8_t tagV = svld1_u8(pred, &tags_[0]);+    auto eqV =+        svset_neonq_u8(svundef_u8(), vceqq_u8(svget_neonq(tagV), needleV));+    // preserve only bits 0 and 4 of each byte+    eqV = svand_n_u8_x(pred, eqV, 17);+    // get info from every byte into the bottom half of every uint16_t+    // by shifting right 4, then round to get it into a 64-bit vector+    uint8x8_t maskV = vshrn_n_u16(vreinterpretq_u16_u8(svget_neonq(eqV)), 4);+    uint64_t mask = vreinterpret_u64_u8(maskV)[0];+    return SparseMaskIter(mask);+  }++#else++  SparseMaskIter tagMatchIter(uint8x16_t needleV) const {+    uint8x16_t tagV = vld1q_u8(&tags_[0]);+    auto eqV = vceqq_u8(tagV, needleV);+    // get info from every byte into the bottom half of every uint16_t+    // by shifting right 4, then round to get it into a 64-bit vector+    uint8x8_t maskV = vshrn_n_u16(vreinterpretq_u16_u8(eqV), 4);+    uint64_t mask = vget_lane_u64(vreinterpret_u64_u8(maskV), 0) & kFullMask;+    return SparseMaskIter(mask);+  }++#endif++  MaskType occupiedMask() const {+    uint8x16_t tagV = vld1q_u8(&tags_[0]);+    // signed shift extends top bit to all bits+    auto occupiedV =+        vreinterpretq_u8_s8(vshrq_n_s8(vreinterpretq_s8_u8(tagV), 7));+    uint8x8_t maskV = vshrn_n_u16(vreinterpretq_u16_u8(occupiedV), 4);+    return vget_lane_u64(vreinterpret_u64_u8(maskV), 0) & kFullMask;+  }+#elif FOLLY_SSE >= 2+  ////////+  // Tag filtering using SSE2 intrinsics++  TagVector const* tagVector() const {+    return static_cast<TagVector const*>(static_cast<void const*>(&tags_[0]));+  }++  SparseMaskIter tagMatchIter(__m128i needleV) const {+    auto tagV = _mm_load_si128(tagVector());++    auto eqV = _mm_cmpeq_epi8(tagV, needleV);+    auto mask = _mm_movemask_epi8(eqV) & kFullMask;+    return SparseMaskIter{mask};+  }++  MaskType occupiedMask() const {+    auto tagV = _mm_load_si128(tagVector());+    return _mm_movemask_epi8(tagV) & kFullMask;+  }+#elif FOLLY_HAVE_INT128_T+  ////////+  // Tag filtering using plain C/C++++  SparseMaskIter tagMatchIter(std::size_t needle) const {+    FOLLY_SAFE_DCHECK(needle >= 0x80 && needle < 0x100, "");+    auto tagV = static_cast<uint8_t const*>(&tags_[0]);+    MaskType mask = 0;+    FOLLY_PRAGMA_UNROLL_N(16)+    for (auto i = 0u; i < kCapacity; i++) {+      mask |= ((tagV[i] == static_cast<uint8_t>(needle)) ? 1 : 0) << i;+    }+    return SparseMaskIter{mask & kFullMask};+  }++  MaskType occupiedMask() const {+    auto tagV = static_cast<uint8_t const*>(&tags_[0]);+    MaskType mask = 0;+    FOLLY_PRAGMA_UNROLL_N(16)+    for (auto i = 0u; i < kCapacity; i++) {+      mask |= ((tagV[i] & 0x80) ? 1 : 0) << i;+    }+    return mask & kFullMask;+  }+#endif++  DenseMaskIter occupiedIter() const {+    return DenseMaskIter{&tags_[0], occupiedMask()};+  }++  MaskRangeIter occupiedRangeIter() const {+    return MaskRangeIter{occupiedMask()};+  }++  LastOccupiedInMask lastOccupied() const {+    return LastOccupiedInMask{occupiedMask()};+  }++  FirstEmptyInMask firstEmpty() const {+    return FirstEmptyInMask{occupiedMask() ^ kFullMask};+  }++  bool occupied(std::size_t index) const {+    FOLLY_SAFE_DCHECK(tags_[index] == 0 || (tags_[index] & 0x80) != 0, "");+    return tags_[index] != 0;+  }++  /// Permitted to return a wild pointer, which is allowed for prefetching on+  /// every architecture. This behavior is technically undefined (forbidden) in+  /// C and C++, but we violate the rule in order to avoid extra cost in the+  /// prefetch paths. The wild pointer that may be returned is whatever follows+  /// any empty-instance global in the memory of any DSO.+  Item* itemAddr(std::size_t i) const {+    return static_cast<Item*>(+        const_cast<void*>(static_cast<void const*>(&rawItems_[i])));+  }++  Item& item(std::size_t i) {+    FOLLY_SAFE_DCHECK(this->occupied(i), "");+    compiler_may_unsafely_assume(this != getSomeEmptyInstance());+    return *std::launder(itemAddr(i));+  }++  Item const& citem(std::size_t i) const {+    FOLLY_SAFE_DCHECK(this->occupied(i), "");+    return *std::launder(itemAddr(i));+  }++  static F14Chunk& owner(Item& item, std::size_t index) {+    auto rawAddr =+        static_cast<uint8_t*>(static_cast<void*>(std::addressof(item))) -+        offsetof(F14Chunk, rawItems_) - index * sizeof(Item);+    auto chunkAddr = static_cast<F14Chunk*>(static_cast<void*>(rawAddr));+    FOLLY_SAFE_DCHECK(std::addressof(item) == chunkAddr->itemAddr(index), "");+    return *chunkAddr;+  }+};++////////////////++// PackedChunkItemPtr points to an Item in an F14Chunk, allowing both the+// Item& and its index to be recovered.  It sorts by the address of the+// item, and it only works for items that are in a properly-aligned chunk.++// generic form, not actually packed+template <typename Ptr>+class PackedChunkItemPtr {+ public:+  PackedChunkItemPtr(Ptr p, std::size_t i) noexcept : ptr_{p}, index_{i} {+    FOLLY_SAFE_DCHECK(ptr_ != nullptr || index_ == 0, "");+  }++  Ptr ptr() const { return ptr_; }++  std::size_t index() const { return index_; }++  bool operator<(PackedChunkItemPtr const& rhs) const {+    FOLLY_SAFE_DCHECK(ptr_ != rhs.ptr_ || index_ == rhs.index_, "");+    return ptr_ < rhs.ptr_;+  }++  bool operator==(PackedChunkItemPtr const& rhs) const {+    FOLLY_SAFE_DCHECK(ptr_ != rhs.ptr_ || index_ == rhs.index_, "");+    return ptr_ == rhs.ptr_;+  }++  bool operator!=(PackedChunkItemPtr const& rhs) const {+    return !(*this == rhs);+  }++ private:+  Ptr ptr_;+  std::size_t index_;+};++// Bare pointer form, packed into a uintptr_t.  Uses only bits wasted by+// alignment, so it works on 32-bit and 64-bit platforms+template <typename T>+class PackedChunkItemPtr<T*> {+  static_assert((alignof(F14Chunk<T>) % 16) == 0);++  // Chunks are 16-byte aligned, so we can maintain a packed pointer to a+  // chunk item by packing the 4-bit item index into the least significant+  // bits of a pointer to the chunk itself.  This makes ItemIter::pack+  // more expensive, however, since it has to compute the chunk address.+  //+  // Chunk items have varying alignment constraints, so it would seem+  // to be that we can't do a similar trick while using only bit masking+  // operations on the Item* itself.  It happens to be, however, that if+  // sizeof(Item) is not a multiple of 16 then we can recover a portion+  // of the index bits from the knowledge that the Item-s are stored in+  // an array that is itself 16-byte aligned.+  //+  // If kAlignBits is the number of trailing zero bits in sizeof(Item)+  // (up to 4), then we can borrow those bits to store kAlignBits of the+  // index directly.  We can recover (4 - kAlignBits) bits of the index+  // from the item pointer itself, by defining/observing that+  //+  // A = kAlignBits                  (A <= 4)+  //+  // S = (sizeof(Item) % 16) >> A    (shifted-away bits are all zero)+  //+  // R = (itemPtr % 16) >> A         (shifted-away bits are all zero)+  //+  // M = 16 >> A+  //+  // itemPtr % 16   = (index * sizeof(Item)) % 16+  //+  // (R * 2^A) % 16 = (index * (sizeof(Item) % 16)) % 16+  //+  // (R * 2^A) % 16 = (index * 2^A * S) % 16+  //+  // R % M          = (index * S) % M+  //+  // S is relatively prime with M, so a multiplicative inverse is easy+  // to compute+  //+  // Sinv = S^(M - 1) % M+  //+  // (R * Sinv) % M = index % M+  //+  // This lets us recover the bottom bits of the index.  When sizeof(T)+  // is 8-byte aligned kSizeInverse will always be 1.  When sizeof(T)+  // is 4-byte aligned kSizeInverse will be either 1 or 3.++  // returns pow(x, y) % m+  static constexpr uintptr_t powerMod(uintptr_t x, uintptr_t y, uintptr_t m) {+    return y == 0 ? 1 : (x * powerMod(x, y - 1, m)) % m;+  }++  static constexpr uintptr_t kIndexBits = 4;+  static constexpr uintptr_t kIndexMask = (uintptr_t{1} << kIndexBits) - 1;++  static constexpr uintptr_t kAlignBits = constexpr_min(+      uintptr_t{4}, constexpr_find_first_set(uintptr_t{sizeof(T)}) - 1);++  static constexpr uintptr_t kAlignMask = (uintptr_t{1} << kAlignBits) - 1;++  static constexpr uintptr_t kModulus = uintptr_t{1}+      << (kIndexBits - kAlignBits);+  static constexpr uintptr_t kSizeInverse =+      powerMod(sizeof(T) >> kAlignBits, kModulus - 1, kModulus);++ public:+  PackedChunkItemPtr(T* p, std::size_t i) noexcept {+    uintptr_t encoded = i >> (kIndexBits - kAlignBits);+    assume((encoded & ~kAlignMask) == 0);+    raw_ = reinterpret_cast<uintptr_t>(p) | encoded;+    FOLLY_SAFE_DCHECK(p == ptr(), "");+    FOLLY_SAFE_DCHECK(i == index(), "");+  }++  T* ptr() const { return reinterpret_cast<T*>(raw_ & ~kAlignMask); }++  std::size_t index() const {+    auto encoded = (raw_ & kAlignMask) << (kIndexBits - kAlignBits);+    auto deduced =+        ((raw_ >> kAlignBits) * kSizeInverse) & (kIndexMask >> kAlignBits);+    return encoded | deduced;+  }++  bool operator<(PackedChunkItemPtr const& rhs) const {+    return raw_ < rhs.raw_;+  }+  bool operator==(PackedChunkItemPtr const& rhs) const {+    return raw_ == rhs.raw_;+  }+  bool operator!=(PackedChunkItemPtr const& rhs) const {+    return !(*this == rhs);+  }++ private:+  uintptr_t raw_;+};++template <typename ChunkPtr>+class F14ItemIter {+ private:+  using Chunk = typename std::pointer_traits<ChunkPtr>::element_type;++ public:+  using Item = typename Chunk::Item;+  using ItemPtr = typename std::pointer_traits<ChunkPtr>::template rebind<Item>;+  using ItemConstPtr =+      typename std::pointer_traits<ChunkPtr>::template rebind<Item const>;++  using Packed = PackedChunkItemPtr<ItemPtr>;++  //// PUBLIC++  F14ItemIter() noexcept : itemPtr_{nullptr}, index_{0} {}++  // default copy and move constructors and assignment operators are correct++  explicit F14ItemIter(Packed const& packed)+      : itemPtr_{packed.ptr()}, index_{packed.index()} {}++  F14ItemIter(ChunkPtr chunk, std::size_t index)+      : itemPtr_{std::pointer_traits<ItemPtr>::pointer_to(chunk->item(index))},+        index_{index} {+    FOLLY_SAFE_DCHECK(index < Chunk::kCapacity, "");+    assume(+        std::pointer_traits<ItemPtr>::pointer_to(chunk->item(index)) !=+        nullptr);+    assume(itemPtr_ != nullptr);+  }++  FOLLY_ALWAYS_INLINE void advanceImpl(bool checkEof, bool likelyDead) {+    auto c = chunk();++    // common case is packed entries+    while (index_ > 0) {+      --index_;+      --itemPtr_;+      if (FOLLY_LIKELY(c->occupied(index_))) {+        return;+      }+    }++    // It's fairly common for an iterator to be advanced and then become+    // dead, for example in the return value from erase(iter) or in+    // the last step of a loop.  We'd like to make sure that the entire+    // advance() method can be eliminated by the compiler's dead code+    // elimination pass.  To do that it must eliminate the loops, which+    // requires it to prove that they have no side effects.  It's easy+    // to show that there are no escaping stores, but at the moment+    // compilers also consider an infinite loop to be a side effect.+    // (There are parts of the standard that would allow them to treat+    // this as undefined behavior, but at the moment they don't exploit+    // those clauses.)+    //+    // The following loop should really be a while loop, which would+    // save a register, some instructions, and a conditional branch,+    // but by writing it as a for loop the compiler can prove to itself+    // that it will eventually terminate.  (No matter that even if the+    // loop executed in a single cycle it would take about 200 years to+    // run all 2^64 iterations.)+    //+    // https://gcc.gnu.org/bugzilla/show_bug.cgi?id=82776 has the bug we+    // filed about the issue.  while (true) {+    for (std::size_t i = 1; !likelyDead || i != 0; ++i) {+      if (checkEof) {+        // exhausted the current chunk+        if (FOLLY_UNLIKELY(c->eof())) {+          FOLLY_SAFE_DCHECK(index_ == 0, "");+          itemPtr_ = nullptr;+          return;+        }+      } else {+        FOLLY_SAFE_DCHECK(!c->eof(), "");+      }+      --c;+      auto last = c->lastOccupied();+      if (checkEof && !likelyDead) {+        prefetchAddr(&*c - 1);+      }+      if (FOLLY_LIKELY(last.hasIndex())) {+        index_ = last.index();+        itemPtr_ = std::pointer_traits<ItemPtr>::pointer_to(c->item(index_));+        return;+      }+    }+  }++  void precheckedAdvance() { advanceImpl(false, false); }++  FOLLY_ALWAYS_INLINE void advance() { advanceImpl(true, false); }++  FOLLY_ALWAYS_INLINE void advanceLikelyDead() { advanceImpl(true, true); }++  ChunkPtr chunk() const {+    return std::pointer_traits<ChunkPtr>::pointer_to(+        Chunk::owner(*itemPtr_, index_));+  }++  std::size_t index() const { return index_; }++  Item* itemAddr() const { return std::addressof(*itemPtr_); }+  Item& item() const { return *itemPtr_; }+  Item const& citem() const { return *itemPtr_; }++  bool atEnd() const { return itemPtr_ == nullptr; }++  Packed pack() const { return Packed{itemPtr_, static_cast<uint8_t>(index_)}; }++  bool operator==(F14ItemIter const& rhs) const {+    // this form makes iter == end() into a single null check after inlining+    // and constant propagation+    return itemPtr_ == rhs.itemPtr_;+  }++  bool operator!=(F14ItemIter const& rhs) const { return !(*this == rhs); }++ private:+  ItemPtr itemPtr_;+  std::size_t index_;+};++////////////////++struct PackedSizeAndChunkShift {+ private:+  // F14Table's chunk count is a power of 2 that is at least 1 and at+  // most 1 << 63. In order to encode it efficiently, we store its base 2+  // logarithm `n`, so that (1UL << n) is the chunkCount. Despite+  // needing only 6 bits to encode the shift, we reserve 8 bits to+  // avoid extra masking, leaving 56 bits for the size, which is+  // plenty.++  // We use the least significant bits of+  // packedSizeAndChunkShift_ for the shift so that size access+  // is just a single shift and shift access is a normal 8-bit load.+  static constexpr uint32_t kSizeShift = 8;+  static constexpr uint32_t kChunkCountShiftMask = (1 << kSizeShift) - 1;+  uint64_t packedSizeAndChunkShift_{0};++ public:+  // subtract 1 because we can't represent 1 << 64, triggering UBSAN.+  static constexpr uint8_t kMaxSupportedChunkShift =+      std::numeric_limits<uint64_t>::digits - 1;++  static constexpr std::size_t kMaxSize =+      (size_t(1) << (std::numeric_limits<std::size_t>::digits - kSizeShift)) -+      1;++  uint64_t size() const noexcept {+    return packedSizeAndChunkShift_ >> kSizeShift;+  }++  uint8_t chunkShift() const noexcept {+    return packedSizeAndChunkShift_ & kChunkCountShiftMask;+  }++  std::size_t chunkCount() const noexcept {+    const auto chunkCountShift = chunkShift();+    return std::size_t(1) << chunkCountShift;+  }++  void setSize(std::size_t sz) noexcept {+    packedSizeAndChunkShift_ =+        chunkShift() | (static_cast<uint64_t>(sz) << kSizeShift);+  }++  void setChunkCount(std::size_t newCount) {+    FOLLY_SAFE_DCHECK(+        folly::isPowTwo(newCount), newCount); // note that this forbids 0!+    const auto shift = findFirstSet(newCount) - 1; // firstSet is 1-based.+    packedSizeAndChunkShift_ =+        (static_cast<uint64_t>(size()) << kSizeShift) | shift;+    FOLLY_SAFE_DCHECK(chunkCount() == newCount, "");+  }++  void swap(PackedSizeAndChunkShift& rhs) noexcept {+    std::swap(packedSizeAndChunkShift_, rhs.packedSizeAndChunkShift_);+  }+};++struct UnpackedSizeAndChunkShift {+ private:+  // Simple implementation for 32-bit systems: just use two words.+  std::size_t size_ = 0;+  uint8_t chunkShift_ = 0;++ public:+  // subtract 1 because we can't represent 1 << 32, triggering UBSAN.+  static constexpr uint8_t kMaxSupportedChunkShift =+      std::numeric_limits<std::size_t>::digits - 1;++  static constexpr std::size_t kMaxSize =+      std::numeric_limits<std::size_t>::max();++  uint64_t size() const noexcept { return size_; }++  uint8_t chunkShift() const noexcept { return chunkShift_; }++  std::size_t chunkCount() const noexcept {+    const auto chunkCountShift = chunkShift();+    return std::size_t(1) << chunkCountShift;+  }++  void setSize(std::size_t sz) noexcept { size_ = sz; }++  void setChunkCount(std::size_t newCount) {+    FOLLY_SAFE_DCHECK(+        folly::isPowTwo(newCount), newCount); // note that this forbids 0!+    const auto shift = findFirstSet(newCount) - 1; // firstSet is 1-based.+    FOLLY_SAFE_DCHECK(shift <= std::numeric_limits<uint8_t>::max(), "");+    chunkShift_ = static_cast<uint8_t>(shift);+    FOLLY_SAFE_DCHECK(chunkCount() == newCount, "");+  }++  void swap(UnpackedSizeAndChunkShift& rhs) noexcept {+    std::swap(size_, rhs.size_);+    std::swap(chunkShift_, rhs.chunkShift_);+  }+};++using SizeAndChunkShift = std::conditional_t<+    sizeof(uint64_t) == sizeof(std::size_t),+    PackedSizeAndChunkShift,+    UnpackedSizeAndChunkShift>;++template <typename ItemIter, bool EnablePackedItemIter>+struct SizeAndChunkShiftAndPackedBegin {+ private:+  SizeAndChunkShift sizeAndChunkShift_;++  typename ItemIter::Packed packedBegin_{ItemIter{}.pack()};++ public:+  auto size() const { return sizeAndChunkShift_.size(); }++  auto chunkShift() const { return sizeAndChunkShift_.chunkShift(); }++  auto chunkCount() const { return sizeAndChunkShift_.chunkCount(); }++  void setSize(uint64_t sz) { sizeAndChunkShift_.setSize(sz); }++  void incrementSize() { sizeAndChunkShift_.setSize(size() + 1); }++  void decrementSize() { sizeAndChunkShift_.setSize(size() - 1); }++  void setChunkCount(std::size_t count) {+    sizeAndChunkShift_.setChunkCount(count);+  }++  typename ItemIter::Packed& packedBegin() { return packedBegin_; }++  typename ItemIter::Packed const& packedBegin() const { return packedBegin_; }++  void swap(SizeAndChunkShiftAndPackedBegin& rhs) noexcept {+    sizeAndChunkShift_.swap(rhs.sizeAndChunkShift_);+    std::swap(packedBegin_, rhs.packedBegin_);+  }+};++template <typename ItemIter>+struct SizeAndChunkShiftAndPackedBegin<ItemIter, false> {+ private:+  SizeAndChunkShift sizeAndChunkShift_;++ public:+  auto size() const { return sizeAndChunkShift_.size(); }++  auto chunkShift() const { return sizeAndChunkShift_.chunkShift(); }++  auto chunkCount() const { return sizeAndChunkShift_.chunkCount(); }++  void setSize(uint64_t sz) { sizeAndChunkShift_.setSize(sz); }++  void incrementSize() { sizeAndChunkShift_.setSize(size() + 1); }++  void decrementSize() { sizeAndChunkShift_.setSize(size() - 1); }++  void setChunkCount(std::size_t count) {+    sizeAndChunkShift_.setChunkCount(count);+  }++  [[noreturn]] typename ItemIter::Packed& packedBegin() {+    assume_unreachable();+  }++  [[noreturn]] typename ItemIter::Packed const& packedBegin() const {+    assume_unreachable();+  }+};++template <typename Policy>+class F14Table : public Policy {+ public:+  using Item = typename Policy::Item;++  using value_type = typename Policy::Value;+  using allocator_type = typename Policy::Alloc;++ private:+  using Alloc = typename Policy::Alloc;+  using AllocTraits = typename Policy::AllocTraits;+  using Hasher = typename Policy::Hasher;+  using InternalSizeType = typename Policy::InternalSizeType;+  using KeyEqual = typename Policy::KeyEqual;++  using Policy::kAllocIsAlwaysEqual;+  using Policy::kContinuousCapacity;+  using Policy::kDefaultConstructIsNoexcept;+  using Policy::kEnableItemIteration;+  using Policy::kSwapIsNoexcept;++  using Policy::destroyItemOnClear;+  using Policy::isAvalanchingHasher;+  using Policy::prefetchBeforeCopy;+  using Policy::prefetchBeforeDestroy;+  using Policy::prefetchBeforeRehash;+  using Policy::shouldAssume32BitHash;++  using ByteAlloc = typename AllocTraits::template rebind_alloc<uint8_t>;+  using BytePtr = typename std::allocator_traits<ByteAlloc>::pointer;++  using Chunk = F14Chunk<Item>;+  using ChunkPtr =+      typename std::pointer_traits<BytePtr>::template rebind<Chunk>;++  using HashPair = typename F14HashToken::HashPair;++ public:+  using ItemIter = F14ItemIter<ChunkPtr>;++ private:+  //////// begin fields++  ChunkPtr chunks_{Chunk::getSomeEmptyInstance()};+  SizeAndChunkShiftAndPackedBegin<ItemIter, kEnableItemIteration>+      sizeAndChunkShiftAndPackedBegin_;++  //////// end fields++  auto chunkShift() const {+    return sizeAndChunkShiftAndPackedBegin_.chunkShift();+  }++  auto chunkCount() const {+    return sizeAndChunkShiftAndPackedBegin_.chunkCount();+  }++  std::size_t moduloByChunkCount(std::size_t index) const {+#ifdef __BMI2__+    // TODO: remove once the compiler selects BZHI on its own.+    // see codegen for: codeSize_find_F14value, defined in:+    // folly/container/test/F14SmallOverheads.cpp+    return _bzhi_u64(index, chunkShift());+#else+    return index & ((std::size_t(1) << chunkShift()) - 1);+#endif+  }++  void swapContents(F14Table& rhs) noexcept {+    using std::swap;+    swap(chunks_, rhs.chunks_);+    swap(+        sizeAndChunkShiftAndPackedBegin_, rhs.sizeAndChunkShiftAndPackedBegin_);+  }++ public:+  // Equivalent to F14Table(0, ...), but implemented separately to avoid forcing+  // a reserve() instantiation in the common case.+  F14Table() noexcept(Policy::kDefaultConstructIsNoexcept)+      : Policy{Hasher{}, KeyEqual{}, Alloc{}} {}++  F14Table(+      std::size_t initialCapacity,+      Hasher const& hasher,+      KeyEqual const& keyEqual,+      Alloc const& alloc)+      : Policy{hasher, keyEqual, alloc} {+    debugModeOnReserve(initialCapacity);+    initialReserve(initialCapacity);+  }++  F14Table(F14Table const& rhs) : Policy{rhs} { buildFromF14Table(rhs); }++  F14Table(F14Table const& rhs, Alloc const& alloc) : Policy{rhs, alloc} {+    buildFromF14Table(rhs);+  }++  F14Table(F14Table&& rhs) noexcept(+      std::is_nothrow_move_constructible<Hasher>::value &&+      std::is_nothrow_move_constructible<KeyEqual>::value &&+      std::is_nothrow_move_constructible<Alloc>::value)+      : Policy{std::move(rhs)} {+    swapContents(rhs);+  }++  F14Table(F14Table&& rhs, Alloc const& alloc) noexcept(kAllocIsAlwaysEqual)+      : Policy{std::move(rhs), alloc} {+    // if-constexpr allows avoiding dependence on usable Hasher etc.+    if constexpr (kAllocIsAlwaysEqual) {+      // move storage (common case)+      swapContents(rhs);+    } else if (this->alloc() == rhs.alloc()) {+      // move storage (common case)+      swapContents(rhs);+    } else {+      // new storage because allocators unequal, move values (rare case)+      buildFromF14Table(std::move(rhs));+    }+  }++  F14Table& operator=(F14Table const& rhs) {+    if (this != &rhs) {+      reset();+      static_cast<Policy&>(*this) = rhs;+      buildFromF14Table(rhs);+    }+    return *this;+  }++  F14Table& operator=(F14Table&& rhs) noexcept(+      std::is_nothrow_move_assignable<Hasher>::value &&+      std::is_nothrow_move_assignable<KeyEqual>::value &&+      (kAllocIsAlwaysEqual ||+       (AllocTraits::propagate_on_container_move_assignment::value &&+        std::is_nothrow_move_assignable<Alloc>::value))) {+    if (this != &rhs) {+      reset();+      static_cast<Policy&>(*this) = std::move(rhs);+      // if-constexpr allows avoiding dependence on usable Hasher etc.+      if constexpr (+          AllocTraits::propagate_on_container_move_assignment::value ||+          kAllocIsAlwaysEqual) {+        // move storage (common case)+        swapContents(rhs);+      } else if (this->alloc() == rhs.alloc()) {+        // move storage (common case)+        swapContents(rhs);+      } else {+        // new storage because allocators unequal, move values (rare case)+        buildFromF14Table(std::move(rhs));+      }+    }+    return *this;+  }++  ~F14Table() { reset(); }++  void swap(F14Table& rhs) noexcept(kSwapIsNoexcept) {+    // If propagate_on_container_swap is false and allocators are+    // not equal, the only way to accomplish a swap would be to do+    // dynamic allocation and then move (or swap) each contained value.+    // AllocatorAwareContainer-s are not supposed to attempt this, but+    // rather are supposed to have undefined behavior in that case.+    FOLLY_SAFE_CHECK(+        AllocTraits::propagate_on_container_swap::value ||+            kAllocIsAlwaysEqual || this->alloc() == rhs.alloc(),+        "swap is undefined for unequal non-propagating allocators");+    this->swapPolicy(rhs);+    swapContents(rhs);+  }++ private:+  static HashPair splitHash(std::size_t hash) {+    return f14::detail::splitHashImpl<Hasher, value_type>(hash);+  }+  //////// memory management helpers++  static std::size_t computeCapacity(+      std::size_t chunkCount, std::size_t scale) {+    FOLLY_SAFE_DCHECK(!(chunkCount > 1 && scale == 0), "");+    FOLLY_SAFE_DCHECK(+        scale < (std::size_t{1} << Chunk::kCapacityScaleBits), "");+    FOLLY_SAFE_DCHECK((chunkCount & (chunkCount - 1)) == 0, "");+    return (((chunkCount - 1) >> Chunk::kCapacityScaleShift) + 1) * scale;+  }++  std::pair<std::size_t, std::size_t> computeChunkCountAndScale(+      std::size_t desiredCapacity,+      bool continuousSingleChunkCapacity,+      bool continuousMultiChunkCapacity) const {+    if (desiredCapacity <= Chunk::kCapacity) {+      // we can go to 100% capacity in a single chunk with no problem+      if (!continuousSingleChunkCapacity) {+        if (desiredCapacity <= 2) {+          desiredCapacity = 2;+        } else if (desiredCapacity <= 6) {+          desiredCapacity = 6;+        } else {+          desiredCapacity = Chunk::kCapacity;+        }+      }+      auto rv = std::make_pair(std::size_t{1}, desiredCapacity);+      FOLLY_SAFE_DCHECK(+          computeCapacity(rv.first, rv.second) == desiredCapacity, "");+      return rv;+    } else {+      std::size_t minChunks =+          (desiredCapacity - 1) / Chunk::kDesiredCapacity + 1;+      std::size_t chunkPow = findLastSet(minChunks - 1);+      if (chunkPow == 8 * sizeof(std::size_t)) {+        throw_exception<std::bad_alloc>();+      }++      std::size_t chunkCount = std::size_t{1} << chunkPow;++      // Let cc * scale be the actual capacity.+      // cc = ((chunkCount - 1) >> kCapacityScaleShift) + 1.+      // If chunkPow >= kCapacityScaleShift, then cc = chunkCount >>+      // kCapacityScaleShift = 1 << (chunkPow - kCapacityScaleShift),+      // otherwise it equals 1 = 1 << 0.  Let cc = 1 << ss.+      std::size_t ss = chunkPow >= Chunk::kCapacityScaleShift+          ? chunkPow - Chunk::kCapacityScaleShift+          : 0;++      std::size_t scale;+      if (continuousMultiChunkCapacity) {+        // (1 << ss) * scale >= desiredCapacity+        scale = ((desiredCapacity - 1) >> ss) + 1;+      } else {+        // (1 << ss) * scale == chunkCount * kDesiredCapacity+        scale = Chunk::kDesiredCapacity << (chunkPow - ss);+      }++      std::size_t actualCapacity = computeCapacity(chunkCount, scale);+      FOLLY_SAFE_DCHECK(actualCapacity >= desiredCapacity, "");+      if (actualCapacity > max_size()) {+        throw_exception<std::bad_alloc>();+      }++      return std::make_pair(chunkCount, scale);+    }+  }++  static std::size_t chunkAllocSize(+      std::size_t chunkCount, std::size_t capacityScale) {+    FOLLY_SAFE_DCHECK(chunkCount > 0, "");+    FOLLY_SAFE_DCHECK(!(chunkCount > 1 && capacityScale == 0), "");+    if (chunkCount == 1) {+      static_assert(offsetof(Chunk, rawItems_) == 16);+      return 16 + sizeof(Item) * computeCapacity(1, capacityScale);+    } else {+      return sizeof(Chunk) * chunkCount;+    }+  }++  ChunkPtr initializeChunks(+      BytePtr raw, std::size_t chunkCount, std::size_t capacityScale) {+    static_assert(std::is_trivial<Chunk>::value, "F14Chunk should be POD");+    auto chunks = static_cast<Chunk*>(static_cast<void*>(&*raw));+    for (std::size_t i = 0; i < chunkCount; ++i) {+      chunks[i].clear();+    }+    chunks[0].markEof(capacityScale);+    return std::pointer_traits<ChunkPtr>::pointer_to(*chunks);+  }++  std::size_t itemCount() const noexcept {+    if (chunkShift() == 0) {+      return computeCapacity(+          1, Chunk::capacityScale(access::to_address(chunks_)));+    } else {+      return chunkCount() * Chunk::kCapacity;+    }+  }++ public:+  ItemIter begin() const noexcept {+    FOLLY_SAFE_DCHECK(kEnableItemIteration, "");+    return ItemIter{sizeAndChunkShiftAndPackedBegin_.packedBegin()};+  }++  ItemIter end() const noexcept { return ItemIter{}; }++  bool empty() const noexcept { return size() == 0; }++  auto size() const noexcept { return sizeAndChunkShiftAndPackedBegin_.size(); }++  std::size_t max_size() const noexcept {+    auto& a = this->alloc();+    return std::min<std::size_t>(+        {SizeAndChunkShift::kMaxSize,+         std::numeric_limits<InternalSizeType>::max(),+         AllocTraits::max_size(a)});+  }++  std::size_t bucket_count() const noexcept {+    return computeCapacity(+        chunkCount(), Chunk::capacityScale(access::to_address(chunks_)));+  }++  std::size_t max_bucket_count() const noexcept { return max_size(); }++  float load_factor() const noexcept {+    return empty()+        ? 0.0f+        : static_cast<float>(size()) / static_cast<float>(bucket_count());+  }++  float max_load_factor() const noexcept { return 1.0f; }++  void max_load_factor(float) noexcept {+    // Probing hash tables can't run load factors >= 1 (unlike chaining+    // tables).  In addition, we have measured that there is little or+    // no performance advantage to running a smaller load factor (cache+    // locality losses outweigh the small reduction in probe lengths,+    // often making it slower).  Therefore, we've decided to just fix+    // max_load_factor at 1.0f regardless of what the user requests.+    // This has an additional advantage that we don't have to store it.+    // Taking alignment into consideration this makes every F14 table+    // 8 bytes smaller, and is part of the reason an empty F14NodeMap+    // is almost half the size of an empty std::unordered_map (32 vs+    // 56 bytes).+    //+    // I don't have a strong opinion on whether we should remove this+    // method or leave a stub, let ngbronson or xshi know if you have a+    // compelling argument either way.+  }++ private:+  // Our probe strategy is to advance through additional chunks with+  // a stride that is key-specific.  This is called double hashing,+  // and is a well known and high quality probing strategy.  So long as+  // the stride and the chunk count are relatively prime, we will visit+  // every chunk once and then return to the original chunk, letting us+  // detect and end the cycle.  The chunk count is a power of two, so+  // we can satisfy the relatively prime part by choosing an odd stride.+  // We've already computed a high quality secondary hash value for the+  // tag, so we just use it for the second probe hash as well.+  //+  // At the maximum load factor of 12/14, expected probe length for a+  // find hit is 1.041, with 99% of keys found in the first three chunks.+  // Expected probe length for a find miss (or insert) is 1.275, with a+  // p99 probe length of 4 (fewer than 1% of failing find look at 5 or+  // more chunks).+  //+  // This code is structured so you can try various ways of encoding+  // the current probe state.  For example, at the moment the probe's+  // state is the position in the cycle and the resulting chunk index is+  // computed from that inside probeCurrentIndex.  We could also make the+  // probe state the chunk index, and then increment it by hp.second *+  // 2 + 1 in probeAdvance.  Wrapping can be applied early or late as+  // well.  This particular code seems to be easier for the optimizer+  // to understand.+  //+  // We could also implement probing strategies that resulted in the same+  // tour for every key initially assigned to a chunk (linear probing or+  // quadratic), but that results in longer probe lengths.  In particular,+  // the cache locality wins of linear probing are not worth the increase+  // in probe lengths (extra work and less branch predictability) in+  // our experiments.++  std::size_t probeDelta(HashPair hp) const { return 2 * hp.second + 1; }++  // TRICKY!  It may seem strange to have a std::size_t needle and narrow+  // it at the last moment, rather than making HashPair::second be a+  // uint8_t, but the latter choice sometimes leads to a performance+  // problem.+  //+  // On architectures with SSE2 but not AVX2, _mm_set1_epi8 expands+  // to multiple instructions.  One of those is a MOVD of either 4 or+  // 8 byte width.  Only the bottom byte of that move actually affects+  // the result, but if a 1-byte needle has been spilled then this will+  // be a 4 byte load.  GCC 5.5 has been observed to reload needle+  // (or perhaps fuse a reload and part of a previous static_cast)+  // needle using a MOVZX with a 1 byte load in parallel with the MOVD.+  // This combination causes a failure of store-to-load forwarding,+  // which has a big performance penalty (60 nanoseconds per find on+  // a microbenchmark).  Keeping needle >= 4 bytes avoids the problem+  // and also happens to result in slightly more compact assembly.++  FOLLY_ALWAYS_INLINE auto loadNeedleV(std::size_t needle) const {+#if FOLLY_NEON+    return vdupq_n_u8(static_cast<uint8_t>(needle));+#elif FOLLY_SSE >= 2+    return _mm_set1_epi8(static_cast<uint8_t>(needle));+#else+    return needle;+#endif+  }++  enum class Prefetch { DISABLED, ENABLED };++  template <typename K>+  FOLLY_ALWAYS_INLINE ItemIter+  findImpl(HashPair hp, K const& key, Prefetch prefetch) const {+    FOLLY_SAFE_DCHECK(hp.second >= 0x80 && hp.second < 0x100, "");+#if FOLLY_ARM_FEATURE_NEON_SVE_BRIDGE+    svbool_t pred = svwhilelt_b8_u32(0, chunks_->kCapacity);+#endif+    std::size_t index = hp.first;+    std::size_t step = probeDelta(hp);+    auto needleV = loadNeedleV(hp.second);+    for (std::size_t tries = chunkCount(); tries > 0;) {+      ChunkPtr chunk = chunks_ + moduloByChunkCount(index);+      if (prefetch == Prefetch::ENABLED && sizeof(Chunk) > 64) {+        prefetchAddr(chunk->itemAddr(8));+      }+#if FOLLY_ARM_FEATURE_NEON_SVE_BRIDGE+      auto hits = chunk->tagMatchIter(needleV, pred);+#else+      auto hits = chunk->tagMatchIter(needleV);+#endif+      while (hits.hasNext()) {+        auto i = hits.next();+        if (FOLLY_LIKELY(this->keyMatchesItem(key, chunk->item(i)))) {+          // Tag match and key match were both successful.  The chance+          // of a false tag match is 1/128 for each key in the chunk+          // (with a proper hash function).+          return ItemIter{chunk, i};+        }+      }+      if (FOLLY_LIKELY(chunk->outboundOverflowCount() == 0)) {+        // No keys that wanted to be placed in this chunk were denied+        // entry, so our search is over.  This is the common case.+        break;+      }+      --tries;+      index += step;+    }+    // Loop exit because tries is exhausted is rare, but possible.+    // That means that for every chunk there is currently a key present+    // in the map that visited that chunk on its probe search but ended+    // up somewhere else, and we have searched every chunk.+    return ItemIter{};+  }++  template <typename K>+  HashPair computeHash(K const& key) const {+    return splitHash(this->computeKeyHash(key));+  }++  template <typename HKKey, typename HKHasher, typename HKEqual>+  HashPair computeHash(+      F14HashedKey<HKKey, HKHasher, HKEqual> const& hashedKey) const {+    static_assert(std::is_same_v<HKHasher, Hasher>);+    static_assert(std::is_same_v<HKEqual, KeyEqual>);+    return static_cast<HashPair>(hashedKey.getHashToken());+  }++ public:+  // prehash()/prefetch() split the work of find(key) into three calls, enabling+  // you to manually implement loop pipelining for hot bulk lookups. prehash()+  // computes the hash and prefetch() prefetches the first computed memory+  // location, and the two-arg find(F14HashToken, K) performs the rest of the+  // search.+  template <typename K>+  F14HashToken prehash(K const& key) const {+    return F14HashToken{computeHash(key)};+  }++  template <typename K>+  F14HashToken prehash(K const& key, std::size_t hash) const {+    FOLLY_SAFE_DCHECK(hash == this->computeKeyHash(key));+    return F14HashToken{splitHash(hash)};+  }++  void prefetch(F14HashToken const& token) const {+    FOLLY_SAFE_DCHECK(chunks_ != nullptr, "");+    ChunkPtr firstChunk = chunks_ + moduloByChunkCount(token.hp_.first);+    prefetchAddr(firstChunk);+  }++  template <typename K>+  FOLLY_ALWAYS_INLINE ItemIter find(K const& key) const {+    auto hp = computeHash(key);+    return findImpl(hp, key, Prefetch::ENABLED);+  }++  template <typename K>+  FOLLY_ALWAYS_INLINE ItemIter+  find(F14HashToken const& token, K const& key) const {+    FOLLY_SAFE_DCHECK(computeHash(key) == static_cast<HashPair>(token), "");+    return findImpl(static_cast<HashPair>(token), key, Prefetch::DISABLED);+  }++  // Searches for a key using a key predicate that is a refinement+  // of key equality.  func(k) should return true only if k is equal+  // to key according to key_eq(), but is allowed to apply additional+  // constraints.+  template <typename K, typename F>+  FOLLY_ALWAYS_INLINE ItemIter findMatching(K const& key, F&& func) const {+    auto hp = computeHash(key);+#if FOLLY_ARM_FEATURE_NEON_SVE_BRIDGE+    svbool_t pred = svwhilelt_b8_u32(0, chunks_->kCapacity);+#endif+    std::size_t index = hp.first;+    auto needleV = loadNeedleV(hp.second);+    std::size_t step = probeDelta(hp);+    for (std::size_t tries = chunkCount(); tries > 0; --tries) {+      ChunkPtr chunk = chunks_ + moduloByChunkCount(index);+      if (sizeof(Chunk) > 64) {+        prefetchAddr(chunk->itemAddr(8));+      }+#if FOLLY_ARM_FEATURE_NEON_SVE_BRIDGE+      auto hits = chunk->tagMatchIter(needleV, pred);+#else+      auto hits = chunk->tagMatchIter(needleV);+#endif+      while (hits.hasNext()) {+        auto i = hits.next();+        if (FOLLY_LIKELY(+                func(this->keyForValue(this->valueAtItem(chunk->item(i)))))) {+          return ItemIter{chunk, i};+        }+      }+      if (FOLLY_LIKELY(chunk->outboundOverflowCount() == 0)) {+        break;+      }+      index += step;+    }+    return ItemIter{};+  }++ private:+  void adjustSizeAndBeginAfterInsert(ItemIter iter) {+    if constexpr (kEnableItemIteration) {+      // packedBegin is the max of all valid ItemIter::pack()+      auto packed = iter.pack();+      if (sizeAndChunkShiftAndPackedBegin_.packedBegin() < packed) {+        sizeAndChunkShiftAndPackedBegin_.packedBegin() = packed;+      }+    }++    sizeAndChunkShiftAndPackedBegin_.incrementSize();+  }++  // Ignores hp if pos.chunk()->hostedOverflowCount() == 0+  void eraseBlank(ItemIter iter, HashPair hp) {+    iter.chunk()->clearTag(iter.index());++    if (iter.chunk()->hostedOverflowCount() != 0) {+      // clean up+      std::size_t index = hp.first;+      std::size_t delta = probeDelta(hp);+      uint8_t hostedOp = 0;+      while (true) {+        ChunkPtr chunk = chunks_ + moduloByChunkCount(index);+        if (chunk == iter.chunk()) {+          chunk->adjustHostedOverflowCount(hostedOp);+          break;+        }+        chunk->decrOutboundOverflowCount();+        hostedOp = Chunk::kDecrHostedOverflowCount;+        index += delta;+      }+    }+  }++  void adjustSizeAndBeginBeforeErase(ItemIter iter) {+    sizeAndChunkShiftAndPackedBegin_.decrementSize();+    if constexpr (kEnableItemIteration) {+      if (iter.pack() == sizeAndChunkShiftAndPackedBegin_.packedBegin()) {+        if (size() == 0) {+          iter = ItemIter{};+        } else {+          iter.precheckedAdvance();+        }+        sizeAndChunkShiftAndPackedBegin_.packedBegin() = iter.pack();+      }+    }+  }++  template <typename... Args>+  void insertAtBlank(ItemIter pos, HashPair hp, Args&&... args) {+    try {+      auto dst = pos.itemAddr();+      this->constructValueAtItem(*this, dst, std::forward<Args>(args)...);+    } catch (...) {+      eraseBlank(pos, hp);+      throw;+    }+    adjustSizeAndBeginAfterInsert(pos);+  }++  ItemIter allocateTag(uint8_t* fullness, HashPair hp) {+    ChunkPtr chunk;+    std::size_t index = hp.first;+    std::size_t delta = probeDelta(hp);+    uint8_t hostedOp = 0;+    while (true) {+      index = moduloByChunkCount(index);+      chunk = chunks_ + index;+      if (FOLLY_LIKELY(fullness[index] < Chunk::kCapacity)) {+        break;+      }+      chunk->incrOutboundOverflowCount();+      hostedOp = Chunk::kIncrHostedOverflowCount;+      index += delta;+    }+    unsigned itemIndex = fullness[index]++;+    FOLLY_SAFE_DCHECK(!chunk->occupied(itemIndex), "");+    chunk->setTag(itemIndex, hp.second);+    chunk->adjustHostedOverflowCount(hostedOp);+    return ItemIter{chunk, itemIndex};+  }++  ChunkPtr lastOccupiedChunk() const {+    FOLLY_SAFE_DCHECK(size() > 0, "");+    if constexpr (kEnableItemIteration) {+      return begin().chunk();+    } else {+      return chunks_ + chunkCount() - 1;+    }+  }++  template <typename T>+  void directBuildFrom(T&& src) {+    FOLLY_SAFE_DCHECK(src.size() > 0 && chunkShift() == src.chunkShift(), "");++    // We use std::forward<T> to allow portions of src to be moved out by+    // either beforeBuild or afterBuild, but we are just relying on good+    // behavior of our Policy superclass to ensure that any particular+    // field of this is a donor at most once.++    auto undoState =+        this->beforeBuild(src.size(), bucket_count(), std::forward<T>(src));+    bool success = false;+    SCOPE_EXIT {+      this->afterBuild(+          undoState, success, src.size(), bucket_count(), std::forward<T>(src));+    };++    // Copy can fail part-way through if a Value copy constructor throws.+    // Failing afterBuild is limited in its cleanup power in this case,+    // because it can't enumerate the items that were actually copied.+    // Fortunately we can divide the situation into cases where all of+    // the state is owned by the table itself (F14Node and F14Value),+    // for which clearImpl() can do partial cleanup, and cases where all+    // of the values are owned by the policy (F14Vector), in which case+    // partial failure should not occur.  Sorry for the subtle invariants+    // in the Policy API.++    if (std::is_trivially_copyable<Item>::value &&+        !this->destroyItemOnClear() && itemCount() == src.itemCount()) {+      FOLLY_SAFE_DCHECK(chunkShift() == src.chunkShift(), "");++      auto scale = Chunk::capacityScale(access::to_address(chunks_));++      // most happy path+      auto n = chunkAllocSize(chunkCount(), scale);+      std::memcpy(&chunks_[0], &src.chunks_[0], n);+      sizeAndChunkShiftAndPackedBegin_.setSize(src.size());+      if constexpr (kEnableItemIteration) {+        auto srcBegin = src.begin();+        sizeAndChunkShiftAndPackedBegin_.packedBegin() =+            ItemIter{+                chunks_ + (srcBegin.chunk() - src.chunks_), srcBegin.index()}+                .pack();+      }+      if constexpr (kContinuousCapacity) {+        // capacityScale might not match even if itemCount matches+        chunks_->setCapacityScale(scale);+      }+    } else {+      // Happy path, no rehash but pack items toward bottom of chunk+      // and use copy constructor.  Don't try to optimize by using+      // lastOccupiedChunk() because there may be higher unoccupied chunks+      // with the overflow bit set.+      auto srcChunk = &src.chunks_[chunkCount() - 1];+      Chunk* dstChunk = &chunks_[chunkCount() - 1];+      do {+        dstChunk->copyOverflowInfoFrom(*srcChunk);++        auto iter = srcChunk->occupiedIter();+        if (prefetchBeforeCopy()) {+          for (auto piter = iter; piter.hasNext();) {+            this->prefetchValue(srcChunk->citem(piter.next()));+          }+        }++        std::size_t dstI = 0;+        for (; iter.hasNext(); ++dstI) {+          auto srcI = iter.next();+          auto&& srcArg =+              std::forward<T>(src).buildArgForItem(srcChunk->item(srcI));+          auto dst = dstChunk->itemAddr(dstI);+          this->constructValueAtItem(+              0, dst, std::forward<decltype(srcArg)>(srcArg));+          dstChunk->setTag(dstI, srcChunk->tag(srcI));+          sizeAndChunkShiftAndPackedBegin_.incrementSize();+        }++        --srcChunk;+        --dstChunk;+      } while (size() != src.size());++      // reset doesn't care about packedBegin, so we don't fix it until the end+      if constexpr (kEnableItemIteration) {+        std::size_t maxChunkIndex = src.lastOccupiedChunk() - src.chunks_;+        sizeAndChunkShiftAndPackedBegin_.packedBegin() =+            ItemIter{+                chunks_ + maxChunkIndex,+                chunks_[maxChunkIndex].lastOccupied().index()}+                .pack();+      }+    }++    success = true;+  }++  template <typename T>+  void rehashBuildFrom(T&& src) {+    FOLLY_SAFE_DCHECK(src.chunkCount() > chunkCount(), "");++    // 1 byte per chunk means < 1 bit per value temporary overhead+    std::array<uint8_t, 256> stackBuf;+    uint8_t* fullness;+    auto cc = chunkCount();+    if (cc <= stackBuf.size()) {+      fullness = stackBuf.data();+    } else {+      ByteAlloc a{this->alloc()};+      fullness = &*std::allocator_traits<ByteAlloc>::allocate(a, cc);+    }+    SCOPE_EXIT {+      if (cc > stackBuf.size()) {+        ByteAlloc a{this->alloc()};+        std::allocator_traits<ByteAlloc>::deallocate(+            a,+            std::pointer_traits<typename std::allocator_traits<+                ByteAlloc>::pointer>::pointer_to(*fullness),+            cc);+      }+    };+    std::memset(fullness, '\0', cc);++    // We use std::forward<T> to allow portions of src to be moved out by+    // either beforeBuild or afterBuild, but we are just relying on good+    // behavior of our Policy superclass to ensure that any particular+    // field of this is a donor at most once.++    // Exception safety requires beforeBuild to happen after all of the+    // allocate() calls.+    auto undoState =+        this->beforeBuild(src.size(), bucket_count(), std::forward<T>(src));+    bool success = false;+    SCOPE_EXIT {+      this->afterBuild(+          undoState, success, src.size(), bucket_count(), std::forward<T>(src));+    };++    // The current table is at a valid state at all points for policies+    // in which non-trivial values are owned by the main table (F14Node+    // and F14Value), so reset() will clean things up properly if we+    // fail partway through.  For the case that the policy manages value+    // lifecycle (F14Vector) then nothing after beforeBuild can throw and+    // we don't have to worry about partial failure.++    std::size_t srcChunkIndex = src.lastOccupiedChunk() - src.chunks_;+    while (true) {+      auto srcChunk = &src.chunks_[srcChunkIndex];+      auto iter = srcChunk->occupiedIter();+      if (prefetchBeforeRehash()) {+        for (auto piter = iter; piter.hasNext();) {+          this->prefetchValue(srcChunk->item(piter.next()));+        }+      }+      if (srcChunk->hostedOverflowCount() == 0) {+        // all items are in their preferred chunk (no probing), so we+        // don't need to compute any hash values+        while (iter.hasNext()) {+          auto i = iter.next();+          auto& srcItem = srcChunk->item(i);+          auto&& srcArg = std::forward<T>(src).buildArgForItem(srcItem);+          HashPair hp{srcChunkIndex, srcChunk->tag(i)};+          insertAtBlank(+              allocateTag(fullness, hp),+              hp,+              std::forward<decltype(srcArg)>(srcArg));+        }+      } else {+        // any chunk's items might be in here+        while (iter.hasNext()) {+          auto i = iter.next();+          auto& srcItem = srcChunk->item(i);+          auto&& srcArg = std::forward<T>(src).buildArgForItem(srcItem);+          auto const& srcKey = src.keyForValue(srcArg);+          auto hp = computeHash(srcKey);+          FOLLY_SAFE_CHECK(hp.second == srcChunk->tag(i), "");+          insertAtBlank(+              allocateTag(fullness, hp),+              hp,+              std::forward<decltype(srcArg)>(srcArg));+        }+      }+      if (srcChunkIndex == 0) {+        break;+      }+      --srcChunkIndex;+    }++    success = true;+  }++  template <typename T>+  FOLLY_NOINLINE void buildFromF14Table(T&& src) {+    FOLLY_SAFE_DCHECK(bucket_count() == 0, "");+    if (src.size() == 0) {+      return;+    }++    // Use the source's capacity, unless it is oversized.+    auto upperLimit = computeChunkCountAndScale(src.size(), false, false);+    auto ccas = std::make_pair(+        src.chunkCount(),+        Chunk::capacityScale(access::to_address(src.chunks_)));+    FOLLY_SAFE_DCHECK(+        ccas.first >= upperLimit.first,+        "rounded chunk count can't be bigger than actual");+    if (ccas.first > upperLimit.first || ccas.second > upperLimit.second) {+      ccas = upperLimit;+    }+    rehashImpl(0, 1, 0, ccas.first, ccas.second);++    try {+      if (chunkShift() == src.chunkShift()) {+        directBuildFrom(std::forward<T>(src));+      } else {+        rehashBuildFrom(std::forward<T>(src));+      }+    } catch (...) {+      reset();+      F14LinkCheck<getF14IntrinsicsMode()>::check();+      throw;+    }+  }++  void maybeRehash(std::size_t desiredCapacity, bool attemptExact) {+    auto origChunkCount = chunkCount();+    auto origCapacityScale = Chunk::capacityScale(access::to_address(chunks_));+    auto origCapacity = computeCapacity(origChunkCount, origCapacityScale);++    std::size_t newChunkCount;+    std::size_t newCapacityScale;+    std::tie(newChunkCount, newCapacityScale) = computeChunkCountAndScale(+        desiredCapacity, attemptExact, kContinuousCapacity && attemptExact);+    auto newCapacity = computeCapacity(newChunkCount, newCapacityScale);++    if (origCapacity != newCapacity) {+      rehashImpl(+          size(),+          origChunkCount,+          origCapacityScale,+          newChunkCount,+          newCapacityScale);+    }+  }++  void reserveImpl(std::size_t requestedCapacity) {+    const size_t targetCapacity =+        std::max<std::size_t>(requestedCapacity, size());+    if (targetCapacity == 0) {+      reset();+      return;+    }++    // Special case reserve(n) for n <= size() (pseudo "shrink_to_fit")+    if (requestedCapacity <= size()) {+      maybeRehash(targetCapacity, /*attemptExact*/ true);+      return;+    }++    auto origCapacity = bucket_count();++    // Never shrink in order to avoid O(n^2) behavior of repeated reserves+    if (targetCapacity <= origCapacity) {+      return;+    }++    // Large increase? Good chance the capacity is exactly right+    bool attemptExact =+        targetCapacity > origCapacity + ((origCapacity + 7) / 8);+    maybeRehash(targetCapacity, attemptExact);+  }++  FOLLY_NOINLINE void reserveForInsertImpl(+      std::size_t capacityMinusOne,+      std::size_t origChunkCount,+      std::size_t origCapacityScale,+      std::size_t origCapacity) {+    FOLLY_SAFE_DCHECK(capacityMinusOne >= size(), "");+    std::size_t capacity = capacityMinusOne + 1;++    // we want to grow by between 2^0.5 and 2^1.5 ending at a "good"+    // size, so we grow by 2^0.5 and then round up++    // 1.01101_2 = 1.40625+    std::size_t minGrowth = origCapacity + (origCapacity >> 2) ++        (origCapacity >> 3) + (origCapacity >> 5);+    capacity = std::max<std::size_t>(capacity, minGrowth);++    std::size_t newChunkCount;+    std::size_t newCapacityScale;+    std::tie(newChunkCount, newCapacityScale) =+        computeChunkCountAndScale(capacity, false, false);++    FOLLY_SAFE_DCHECK(+        computeCapacity(newChunkCount, newCapacityScale) > origCapacity, "");++    rehashImpl(+        size(),+        origChunkCount,+        origCapacityScale,+        newChunkCount,+        newCapacityScale);+  }++  void initialReserve(std::size_t desiredCapacity) {+    FOLLY_SAFE_DCHECK(size() == 0, "");+    FOLLY_SAFE_DCHECK(chunkShift() == 0, "");+    FOLLY_SAFE_DCHECK(!!chunks_);+    FOLLY_SAFE_DCHECK(Chunk::isEmptyInstance(access::to_address(chunks_)), "");+    if (desiredCapacity == 0) {+      return;+    }++    std::size_t newChunkCount;+    std::size_t newCapacityScale;+    std::tie(newChunkCount, newCapacityScale) = computeChunkCountAndScale(+        desiredCapacity, /*attemptExact=*/true, kContinuousCapacity);+    auto newCapacity = computeCapacity(newChunkCount, newCapacityScale);+    auto newAllocSize = chunkAllocSize(newChunkCount, newCapacityScale);++    BytePtr rawAllocation;+    auto undoState =+        this->beforeRehash(0, 0, newCapacity, newAllocSize, rawAllocation);++    chunks_ = initializeChunks(rawAllocation, newChunkCount, newCapacityScale);++    sizeAndChunkShiftAndPackedBegin_.setChunkCount(newChunkCount);++    this->afterRehash(+        std::move(undoState), true, 0, 0, newCapacity, nullptr, 0);+  }++  void rehashImpl(+      std::size_t origSize,+      std::size_t origChunkCount,+      std::size_t origCapacityScale,+      std::size_t newChunkCount,+      std::size_t newCapacityScale) {+    auto origChunks = chunks_;+    auto origCapacity = computeCapacity(origChunkCount, origCapacityScale);+    auto origAllocSize = chunkAllocSize(origChunkCount, origCapacityScale);+    auto newCapacity = computeCapacity(newChunkCount, newCapacityScale);+    auto newAllocSize = chunkAllocSize(newChunkCount, newCapacityScale);++    BytePtr rawAllocation;+    auto undoState = this->beforeRehash(+        origSize, origCapacity, newCapacity, newAllocSize, rawAllocation);+    chunks_ = initializeChunks(rawAllocation, newChunkCount, newCapacityScale);++    sizeAndChunkShiftAndPackedBegin_.setChunkCount(newChunkCount);++    bool success = false;+    SCOPE_EXIT {+      // this SCOPE_EXIT reverts chunks_ and chunkShift if necessary+      BytePtr finishedRawAllocation = nullptr;+      std::size_t finishedAllocSize = 0;+      if (FOLLY_LIKELY(success)) {+        if (origCapacity > 0) {+          finishedRawAllocation = std::pointer_traits<BytePtr>::pointer_to(+              *static_cast<uint8_t*>(static_cast<void*>(&*origChunks)));+          finishedAllocSize = origAllocSize;+        }+      } else {+        finishedRawAllocation = rawAllocation;+        finishedAllocSize = newAllocSize;+        chunks_ = origChunks;+        sizeAndChunkShiftAndPackedBegin_.setChunkCount(origChunkCount);+        F14LinkCheck<getF14IntrinsicsMode()>::check();+      }++      this->afterRehash(+          std::move(undoState),+          success,+          origSize,+          origCapacity,+          newCapacity,+          finishedRawAllocation,+          finishedAllocSize);+    };++    if (origSize == 0) {+      // nothing to do+    } else if (origChunkCount == 1 && newChunkCount == 1) {+      // no mask, no chunk scan, no hash computation, no probing+      auto srcChunk = origChunks;+      auto dstChunk = chunks_;+      std::size_t srcI = 0;+      std::size_t dstI = 0;+      while (dstI < origSize) {+        if (FOLLY_LIKELY(srcChunk->occupied(srcI))) {+          dstChunk->setTag(dstI, srcChunk->tag(srcI));+          this->moveItemDuringRehash(+              dstChunk->itemAddr(dstI), srcChunk->item(srcI));+          ++dstI;+        }+        ++srcI;+      }+      if constexpr (kEnableItemIteration) {+        sizeAndChunkShiftAndPackedBegin_.packedBegin() =+            ItemIter{dstChunk, dstI - 1}.pack();+      }+    } else {+      // 1 byte per chunk means < 1 bit per value temporary overhead+      std::array<uint8_t, 256> stackBuf;+      uint8_t* fullness;+      if (newChunkCount <= stackBuf.size()) {+        fullness = stackBuf.data();+      } else {+        ByteAlloc a{this->alloc()};+        // may throw+        fullness =+            &*std::allocator_traits<ByteAlloc>::allocate(a, newChunkCount);+      }+      std::memset(fullness, '\0', newChunkCount);+      SCOPE_EXIT {+        if (newChunkCount > stackBuf.size()) {+          ByteAlloc a{this->alloc()};+          std::allocator_traits<ByteAlloc>::deallocate(+              a,+              std::pointer_traits<typename std::allocator_traits<+                  ByteAlloc>::pointer>::pointer_to(*fullness),+              newChunkCount);+        }+      };++      auto srcChunk = origChunks + origChunkCount - 1;+      std::size_t remaining = origSize;+      while (remaining > 0) {+        auto iter = srcChunk->occupiedIter();+        if (prefetchBeforeRehash()) {+          for (auto piter = iter; piter.hasNext();) {+            this->prefetchValue(srcChunk->item(piter.next()));+          }+        }+        while (iter.hasNext()) {+          --remaining;+          auto srcI = iter.next();+          Item& srcItem = srcChunk->item(srcI);+          auto hp = splitHash(+              this->computeItemHash(const_cast<Item const&>(srcItem)));+          FOLLY_SAFE_CHECK(hp.second == srcChunk->tag(srcI), "");++          auto dstIter = allocateTag(fullness, hp);+          this->moveItemDuringRehash(dstIter.itemAddr(), srcItem);+        }+        --srcChunk;+      }++      if constexpr (kEnableItemIteration) {+        // this code replaces size invocations of adjustSizeAndBeginAfterInsert+        std::size_t i = chunkCount() - 1;+        while (fullness[i] == 0) {+          --i;+        }+        sizeAndChunkShiftAndPackedBegin_.packedBegin() =+            ItemIter{chunks_ + i, std::size_t{fullness[i]} - 1}.pack();+      }+    }++    success = true;+  }++  // Randomization to help expose bugs when running tests in debug or+  // sanitizer builds++  FOLLY_ALWAYS_INLINE void debugModeOnReserve(std::size_t capacity) {+    if constexpr (kIsLibrarySanitizeAddress || kIsDebug) {+      if (capacity > size()) {+        tlsPendingSafeInserts(static_cast<std::ptrdiff_t>(capacity - size()));+      }+    }+  }++  void debugModeSpuriousRehash() {+    auto cc = chunkCount();+    auto ss = Chunk::capacityScale(access::to_address(chunks_));+    rehashImpl(size(), cc, ss, cc, ss);+  }++  FOLLY_ALWAYS_INLINE void debugModeBeforeInsert() {+    // When running under ASAN, we add a spurious rehash with 1/size()+    // probability before every insert.  This means that finding reference+    // stability problems for F14Value and F14Vector is much more likely.+    // The most common pattern that causes this is+    //+    //   auto& ref = map[k1]; map[k2] = foo(ref);+    //+    // One way to fix this is to call map.reserve(N) before such a+    // sequence, where N is the number of keys that might be inserted+    // within the section that retains references plus the existing size.+    if constexpr (kIsLibrarySanitizeAddress) {+      if (!tlsPendingSafeInserts() && size() > 0 &&+          tlsMinstdRand(size()) == 0) {+        debugModeSpuriousRehash();+      }+    }+  }++  FOLLY_ALWAYS_INLINE void debugModeAfterInsert() {+    if constexpr (kIsLibrarySanitizeAddress || kIsDebug) {+      tlsPendingSafeInserts(-1);+    }+  }++  FOLLY_ALWAYS_INLINE void debugModePerturbSlotInsertOrder(+      ChunkPtr chunk, std::size_t& itemIndex) {+    FOLLY_SAFE_DCHECK(!chunk->occupied(itemIndex), "");+    constexpr bool perturbSlot = FOLLY_F14_PERTURB_INSERTION_ORDER;+    if (perturbSlot && !tlsPendingSafeInserts()) {+      std::size_t e = chunkShift() == 0 ? bucket_count() : Chunk::kCapacity;+      std::size_t i = itemIndex + tlsMinstdRand(e - itemIndex);+      if (!chunk->occupied(i)) {+        itemIndex = i;+      }+    }+  }++ public:+  // user has no control over max_load_factor++  void rehash(std::size_t capacity) { reserve(capacity); }++  void reserve(std::size_t capacity) {+    // We want to support the pattern+    //   map.reserve(map.size() + 2); auto& r1 = map[k1]; auto& r2 = map[k2];+    debugModeOnReserve(capacity);+    FOLLY_SAFE_DCHECK(!!chunks_);+    if (Chunk::isEmptyInstance(access::to_address(chunks_))) {+      initialReserve(capacity);+    } else {+      reserveImpl(capacity);+    }+  }++  void reserveForInsert(size_t incoming = 1) {+    FOLLY_SAFE_DCHECK(incoming > 0, "");++    auto needed = size() + incoming;+    auto chunkCount_ = chunkCount();+    auto scale = Chunk::capacityScale(access::to_address(chunks_));+    auto existing = computeCapacity(chunkCount_, scale);+    if (needed - 1 >= existing) {+      reserveForInsertImpl(needed - 1, chunkCount_, scale, existing);+    }+  }++  // Returns pos,true if construct, pos,false if found.  key is only used+  // during the search; all constructor args for an inserted value come+  // from args...  key won't be accessed after args are touched.+  template <typename K, typename... Args>+  std::pair<ItemIter, bool> tryEmplaceValue(K const& key, Args&&... args) {+    const auto hp = computeHash(key);+    return tryEmplaceValueImpl(hp, key, std::forward<Args>(args)...);+  }++  template <typename K, typename... Args>+  std::pair<ItemIter, bool> tryEmplaceValueWithToken(+      F14HashToken const& token, K const& key, Args&&... args) {+    FOLLY_SAFE_DCHECK(computeHash(key) == static_cast<HashPair>(token), "");+    return tryEmplaceValueImpl(+        static_cast<HashPair>(token), key, std::forward<Args>(args)...);+  }++  template <typename K, typename... Args>+  std::pair<ItemIter, bool> tryEmplaceValueImpl(+      HashPair hp, K const& key, Args&&... args) {+    if (size() > 0) {+      auto existing = findImpl(hp, key, Prefetch::ENABLED);+      if (!existing.atEnd()) {+        return std::make_pair(existing, false);+      }+    }++    debugModeBeforeInsert();++    reserveForInsert();++    std::size_t index = hp.first;+    ChunkPtr chunk = chunks_ + moduloByChunkCount(index);+    auto firstEmpty = chunk->firstEmpty();++    if (!firstEmpty.hasIndex()) {+      std::size_t delta = probeDelta(hp);+      do {+        chunk->incrOutboundOverflowCount();+        index += delta;+        chunk = chunks_ + moduloByChunkCount(index);+        firstEmpty = chunk->firstEmpty();+      } while (!firstEmpty.hasIndex());+      chunk->adjustHostedOverflowCount(Chunk::kIncrHostedOverflowCount);+    }+    std::size_t itemIndex = firstEmpty.index();++    debugModePerturbSlotInsertOrder(chunk, itemIndex);++    chunk->setTag(itemIndex, hp.second);+    ItemIter iter{chunk, itemIndex};++    // insertAtBlank will clear the tag if the constructor throws+    insertAtBlank(iter, hp, std::forward<Args>(args)...);++    debugModeAfterInsert();++    return std::make_pair(iter, true);+  }++ private:+  template <bool Reset>+  void clearImpl() noexcept {+    FOLLY_SAFE_DCHECK(!!chunks_);+    if (Chunk::isEmptyInstance(access::to_address(chunks_))) {+      FOLLY_SAFE_DCHECK(empty() && bucket_count() == 0, "");+      return;+    }++    // turn clear into reset if the table is >= 16 chunks so that+    // we don't get too low a load factor+    bool willReset = Reset || chunkCount() >= 16;++    auto origSize = size();+    auto origCapacity = bucket_count();+    if (willReset) {+      this->beforeReset(origSize, origCapacity);+    } else {+      this->beforeClear(origSize, origCapacity);+    }++    if (!empty()) {+      if (destroyItemOnClear()) {+        for (std::size_t ci = 0; ci < chunkCount(); ++ci) {+          ChunkPtr chunk = chunks_ + ci;+          auto iter = chunk->occupiedIter();+          if (prefetchBeforeDestroy()) {+            for (auto piter = iter; piter.hasNext();) {+              this->prefetchValue(chunk->item(piter.next()));+            }+          }+          while (iter.hasNext()) {+            this->destroyItem(chunk->item(iter.next()));+          }+        }+      }+      if (!willReset) {+        // It's okay to do this in a separate loop because we only do it+        // when the chunk count is small.  That avoids a branch when we+        // are promoting a clear to a reset for a large table.+        auto scale = Chunk::capacityScale(access::to_address(chunks_));+        for (std::size_t ci = 0; ci < chunkCount(); ++ci) {+          chunks_[ci].clear();+        }+        chunks_[0].markEof(scale);+      }+      if constexpr (kEnableItemIteration) {+        sizeAndChunkShiftAndPackedBegin_.packedBegin() = ItemIter{}.pack();+      }+      sizeAndChunkShiftAndPackedBegin_.setSize(0);+    }++    if (willReset) {+      BytePtr rawAllocation = std::pointer_traits<BytePtr>::pointer_to(+          *static_cast<uint8_t*>(static_cast<void*>(&*chunks_)));+      std::size_t rawSize = chunkAllocSize(+          chunkCount(), Chunk::capacityScale(access::to_address(chunks_)));++      chunks_ = Chunk::getSomeEmptyInstance();+      sizeAndChunkShiftAndPackedBegin_.setChunkCount(1);++      this->afterReset(origSize, origCapacity, rawAllocation, rawSize);+    } else {+      this->afterClear(origSize, origCapacity);+    }+  }++  void eraseImpl(ItemIter pos, HashPair hp) {+    this->destroyItem(pos.item());+    adjustSizeAndBeginBeforeErase(pos);+    eraseBlank(pos, hp);+  }++ public:+  // The item needs to still be hashable during this call.  If you want+  // to intercept the value before it is destroyed (to extract it, for+  // example), do so in the beforeDestroy callback.+  template <typename BeforeDestroy>+  void eraseIterInto(ItemIter pos, BeforeDestroy&& beforeDestroy) {+    HashPair hp{};+    if (pos.chunk()->hostedOverflowCount() != 0) {+      hp = splitHash(this->computeItemHash(pos.citem()));+    }+    beforeDestroy(this->valueAtItemForExtract(pos.item()));+    eraseImpl(pos, hp);+  }++  template <typename K, typename BeforeDestroy>+  std::size_t eraseKeyInto(K const& key, BeforeDestroy&& beforeDestroy) {+    if (FOLLY_UNLIKELY(size() == 0)) {+      return 0;+    }+    auto hp = computeHash(key);+    auto iter = findImpl(hp, key, Prefetch::ENABLED);+    if (!iter.atEnd()) {+      beforeDestroy(this->valueAtItemForExtract(iter.item()));+      eraseImpl(iter, hp);+      return 1;+    } else {+      return 0;+    }+  }++  void clear() noexcept {+    if constexpr (kIsLibrarySanitizeAddress) {+      // force recycling of heap memory+      auto bc = bucket_count();+      reset();+      try {+        reserveImpl(bc);+      } catch (std::bad_alloc const&) {+        // ASAN mode only, keep going+      }+    } else {+      clearImpl<false>();+    }+  }++  // Like clear(), but always frees all dynamic storage allocated+  // by the table.+  void reset() noexcept { clearImpl<true>(); }++  // Get memory footprint, not including sizeof(*this).+  std::size_t getAllocatedMemorySize() const {+    std::size_t sum = 0;+    visitAllocationClasses([&sum](std::size_t bytes, std::size_t n) {+      sum += bytes * n;+    });+    return sum;+  }++  // Enumerates classes of allocated memory blocks currently owned+  // by this table, calling visitor(allocationSize, allocationCount).+  // This can be used to get a more accurate indication of memory footprint+  // than getAllocatedMemorySize() if you have some way of computing the+  // internal fragmentation of the allocator, such as JEMalloc's nallocx.+  // The visitor might be called twice with the same allocationSize. The+  // visitor's computation should produce the same result for visitor(8,+  // 2) as for two calls to visitor(8, 1), for example.  The visitor may+  // be called with a zero allocationCount.+  template <typename V>+  void visitAllocationClasses(V&& visitor) const {+    auto scale = Chunk::capacityScale(access::to_address(chunks_));+    this->visitPolicyAllocationClasses(+        scale == 0 ? 0 : chunkAllocSize(chunkCount(), scale),+        size(),+        bucket_count(),+        visitor);+  }++  // visitor should take an Item const&+  template <typename V>+  void visitItems(V&& visitor) const {+    if (empty()) {+      return;+    }+    std::size_t maxChunkIndex = lastOccupiedChunk() - chunks_;+    auto chunk = &chunks_[0];+    for (std::size_t i = 0; i <= maxChunkIndex; ++i, ++chunk) {+      auto iter = chunk->occupiedIter();+      if (prefetchBeforeCopy()) {+        for (auto piter = iter; piter.hasNext();) {+          this->prefetchValue(chunk->citem(piter.next()));+        }+      }+      while (iter.hasNext()) {+        visitor(chunk->citem(iter.next()));+      }+    }+  }++  // visitor should take two Item const*+  template <typename V>+  void visitContiguousItemRanges(V&& visitor) const {+    if (empty()) {+      return;+    }+    std::size_t maxChunkIndex = lastOccupiedChunk() - chunks_;+    auto chunk = &chunks_[0];+    for (std::size_t i = 0; i <= maxChunkIndex; ++i, ++chunk) {+      for (auto iter = chunk->occupiedRangeIter(); iter.hasNext();) {+        auto be = iter.next();+        FOLLY_SAFE_DCHECK(+            chunk->occupied(be.first) && chunk->occupied(be.second - 1), "");+        Item const* b = chunk->itemAddr(be.first);+        visitor(b, b + (be.second - be.first));+      }+    }+  }++ private:+  static std::size_t& histoAt(+      std::vector<std::size_t>& histo, std::size_t index) {+    if (histo.size() <= index) {+      histo.resize(index + 1);+    }+    return histo.at(index);+  }++ public:+  // Expensive+  F14TableStats computeStats() const {+    F14TableStats stats;++    if constexpr (kIsDebug && kEnableItemIteration) {+      // validate iteration+      std::size_t n = 0;+      ItemIter prev;+      for (auto iter = begin(); iter != end(); iter.advance()) {+        FOLLY_SAFE_DCHECK(n == 0 || iter.pack() < prev.pack(), "");+        ++n;+        prev = iter;+      }+      FOLLY_SAFE_DCHECK(n == size(), "");+    }++    FOLLY_SAFE_DCHECK(+        (Chunk::isEmptyInstance(access::to_address(chunks_))) ==+            (bucket_count() == 0),+        "");++    std::size_t n1 = 0;+    std::size_t n2 = 0;+    auto cc = bucket_count() == 0 ? 0 : chunkCount();+    for (std::size_t ci = 0; ci < cc; ++ci) {+      ChunkPtr chunk = chunks_ + ci;+      FOLLY_SAFE_DCHECK(chunk->eof() == (ci == 0), "");++      auto iter = chunk->occupiedIter();++      std::size_t chunkOccupied = 0;+      for (auto piter = iter; piter.hasNext(); piter.next()) {+        ++chunkOccupied;+      }+      n1 += chunkOccupied;++      histoAt(stats.chunkOccupancyHisto, chunkOccupied)++;+      histoAt(+          stats.chunkOutboundOverflowHisto, chunk->outboundOverflowCount())++;+      histoAt(stats.chunkHostedOverflowHisto, chunk->hostedOverflowCount())++;++      while (iter.hasNext()) {+        auto ii = iter.next();+        ++n2;++        {+          auto& item = chunk->citem(ii);+          auto hp = splitHash(this->computeItemHash(item));+          FOLLY_SAFE_DCHECK(chunk->tag(ii) == hp.second, "");++          std::size_t dist = 1;+          std::size_t index = hp.first;+          std::size_t delta = probeDelta(hp);+          while (moduloByChunkCount(index) != ci) {+            index += delta;+            ++dist;+          }++          histoAt(stats.keyProbeLengthHisto, dist)++;+        }++        // misses could have any tag, so we do the dumb but accurate+        // thing and just try them all+        for (std::size_t ti = 0; ti < 256; ++ti) {+          uint8_t tag = static_cast<uint8_t>(ti == 0 ? 1 : 0);+          HashPair hp{ci, tag};++          std::size_t dist = 1;+          std::size_t index = hp.first;+          std::size_t delta = probeDelta(hp);+          for (std::size_t tries = 0; tries >> chunkShift() == 0 &&+               chunks_[moduloByChunkCount(index)].outboundOverflowCount() != 0;+               ++tries) {+            index += delta;+            ++dist;+          }++          histoAt(stats.missProbeLengthHisto, dist)++;+        }+      }+    }++    FOLLY_SAFE_DCHECK(n1 == size(), "");+    FOLLY_SAFE_DCHECK(n2 == size(), "");++    stats.policy = pretty_name<Policy>();+    stats.size = size();+    stats.valueSize = sizeof(value_type);+    stats.bucketCount = bucket_count();+    stats.chunkCount = cc;++    stats.totalBytes = sizeof(*this) + getAllocatedMemorySize();+    stats.overheadBytes = stats.totalBytes - size() * sizeof(value_type);++    return stats;+  }+};+} // namespace detail+} // namespace f14++#endif // FOLLY_F14_VECTOR_INTRINSICS_AVAILABLE++namespace f14 {+namespace test {+inline void disableInsertOrderRandomization() {+  if constexpr (kIsLibrarySanitizeAddress || kIsDebug) {+    detail::tlsPendingSafeInserts(static_cast<std::ptrdiff_t>(+        (std::numeric_limits<std::size_t>::max)() / 2));+  }+}+} // namespace test+} // namespace f14+} // namespace folly
+ folly/folly/container/detail/Util.h view
@@ -0,0 +1,303 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <memory>+#include <tuple>+#include <type_traits>+#include <utility>++#include <folly/Traits.h>+#include <folly/container/Iterator.h>+#include <folly/functional/ApplyTuple.h>++// Utility functions for container implementors++namespace folly {+namespace detail {++template <typename KeyType, typename Alloc>+struct TemporaryEmplaceKey {+  TemporaryEmplaceKey(TemporaryEmplaceKey const&) = delete;+  TemporaryEmplaceKey(TemporaryEmplaceKey&&) = delete;++  template <typename... Args>+  TemporaryEmplaceKey(Alloc& a, std::tuple<Args...>&& args) : alloc_(a) {+    auto p = &value();+    apply(+        [&, p](auto&&... inner) {+          std::allocator_traits<Alloc>::construct(+              alloc_, p, std::forward<decltype(inner)>(inner)...);+        },+        std::move(args));+  }++  ~TemporaryEmplaceKey() {+    std::allocator_traits<Alloc>::destroy(alloc_, &value());+  }++  KeyType& value() { return *static_cast<KeyType*>(static_cast<void*>(&raw_)); }++  Alloc& alloc_;+  std::aligned_storage_t<sizeof(KeyType), alignof(KeyType)> raw_;+};++// A map's emplace(args...) function takes arguments that can be used to+// construct a pair<key_type const, mapped_type>, but that construction+// only needs to take place if the key is not present in the container.+// callWithExtractedKey helps to handle this efficiently by looking for a+// reference to the key within the args list.  If the search is successful+// then the search can be performed without constructing any temporaries.+// If the search is not successful then callWithExtractedKey constructs+// a temporary key_type and a new argument list suitable for constructing+// the entire value_type if necessary.+//+// callWithExtractedKey(a, f, args...) will call f(k, args'...), where+// k is the key and args'... is an argument list that can be used to+// construct a pair of key and mapped value.  Note that this means f gets+// the key twice.+//+// In some cases a temporary key must be constructed.  This is accomplished+// with std::allocator_traits<>::construct, and the temporary will be+// destroyed with std::allocator_traits<>::destroy.  Using the allocator's+// construct method reduces unnecessary copies for pmr allocators.+//+// callWithExtractedKey supports heterogeneous lookup with the UsableAsKey+// template parameter.  If a single key argument of type K is found in+// args... then it will be passed directly to f if it is either KeyType or+// if UsableAsKey<remove_cvref_t<K>>::value is true.  If you don't care+// about heterogeneous lookup you can just pass a single-arg template+// that extends std::false_type.++// TODO(T31574848): We can remove the std::enable_if_t once we no longer+// target platforms without N4387 ("perfect initialization" for pairs+// and tuples).  libstdc++ at gcc-6.1.0 is the first release that contains+// the improved set of pair constructors.+template <+    typename KeyType,+    typename MappedType,+    typename Func,+    typename UsableKeyType,+    typename Arg1,+    typename Arg2,+    std::enable_if_t<+        std::is_constructible<+            std::pair<KeyType const, MappedType>,+            Arg1&&,+            Arg2&&>::value,+        int> = 0>+auto callWithKeyAndPairArgs(+    Func&& f,+    UsableKeyType const& key,+    std::tuple<Arg1>&& first_args,+    std::tuple<Arg2>&& second_args) {+  return f(+      key,+      std::forward<Arg1>(std::get<0>(first_args)),+      std::forward<Arg2>(std::get<0>(second_args)));+}++template <+    typename KeyType,+    typename MappedType,+    typename Func,+    typename UsableKeyType,+    typename... Args1,+    typename... Args2>+auto callWithKeyAndPairArgs(+    Func&& f,+    UsableKeyType const& key,+    std::tuple<Args1...>&& first_args,+    std::tuple<Args2...>&& second_args) {+  return f(+      key,+      std::piecewise_construct,+      std::move(first_args),+      std::move(second_args));+}++template <typename>+using ExactKeyMatchOnly = std::false_type;++template <+    typename KeyType,+    typename MappedType,+    template <typename> class UsableAsKey = ExactKeyMatchOnly,+    typename Alloc,+    typename Func,+    typename Arg1,+    typename... Args2,+    std::enable_if_t<+        std::is_same<remove_cvref_t<Arg1>, KeyType>::value ||+            UsableAsKey<remove_cvref_t<Arg1>>::value,+        int> = 0>+auto callWithExtractedKey(+    Alloc&,+    Func&& f,+    std::piecewise_construct_t,+    std::tuple<Arg1>&& first_args,+    std::tuple<Args2...>&& second_args) {+  // we found a usable key in the args :)+  auto const& key = std::get<0>(first_args);+  return callWithKeyAndPairArgs<KeyType, MappedType>(+      std::forward<Func>(f),+      key,+      std::tuple<Arg1&&>(std::move(first_args)),+      std::tuple<Args2&&...>(std::move(second_args)));+}++template <+    typename KeyType,+    typename MappedType,+    template <typename> class UsableAsKey = ExactKeyMatchOnly,+    typename Alloc,+    typename Func,+    typename... Args1,+    typename... Args2>+auto callWithExtractedKey(+    Alloc& a,+    Func&& f,+    std::piecewise_construct_t,+    std::tuple<Args1...>&& first_args,+    std::tuple<Args2...>&& second_args) {+  // we will need to materialize a temporary key :(+  TemporaryEmplaceKey<KeyType, Alloc> key(+      a, std::tuple<Args1&&...>(std::move(first_args)));+  return callWithKeyAndPairArgs<KeyType, MappedType>(+      std::forward<Func>(f),+      const_cast<KeyType const&>(key.value()),+      std::forward_as_tuple(std::move(key.value())),+      std::tuple<Args2&&...>(std::move(second_args)));+}++template <+    typename KeyType,+    typename MappedType,+    template <typename> class UsableAsKey = ExactKeyMatchOnly,+    typename Alloc,+    typename Func>+auto callWithExtractedKey(Alloc& a, Func&& f) {+  return callWithExtractedKey<KeyType, MappedType, UsableAsKey>(+      a,+      std::forward<Func>(f),+      std::piecewise_construct,+      std::tuple<>{},+      std::tuple<>{});+}++template <+    typename KeyType,+    typename MappedType,+    template <typename> class UsableAsKey = ExactKeyMatchOnly,+    typename Alloc,+    typename Func,+    typename U1,+    typename U2>+auto callWithExtractedKey(Alloc& a, Func&& f, U1&& x, U2&& y) {+  return callWithExtractedKey<KeyType, MappedType, UsableAsKey>(+      a,+      std::forward<Func>(f),+      std::piecewise_construct,+      std::forward_as_tuple(std::forward<U1>(x)),+      std::forward_as_tuple(std::forward<U2>(y)));+}++template <+    typename KeyType,+    typename MappedType,+    template <typename> class UsableAsKey = ExactKeyMatchOnly,+    typename Alloc,+    typename Func,+    typename U1,+    typename U2>+auto callWithExtractedKey(Alloc& a, Func&& f, std::pair<U1, U2> const& p) {+  return callWithExtractedKey<KeyType, MappedType, UsableAsKey>(+      a,+      std::forward<Func>(f),+      std::piecewise_construct,+      std::forward_as_tuple(p.first),+      std::forward_as_tuple(p.second));+}++template <+    typename KeyType,+    typename MappedType,+    template <typename> class UsableAsKey = ExactKeyMatchOnly,+    typename Alloc,+    typename Func,+    typename U1,+    typename U2>+auto callWithExtractedKey(Alloc& a, Func&& f, std::pair<U1, U2>&& p) {+  // std::move(p.first) is wrong because if U1 is an lvalue reference the+  // result will incorrectly be an rvalue ref.  static_cast here allows+  // proper ref collapsing+  return callWithExtractedKey<KeyType, MappedType, UsableAsKey>(+      a,+      std::forward<Func>(f),+      std::piecewise_construct,+      std::forward_as_tuple(static_cast<U1&&>(p.first)),+      std::forward_as_tuple(static_cast<U2&&>(p.second)));+}++// callWithConstructedKey is the set container analogue of+// callWithExtractedKey++template <+    typename KeyType,+    template <typename> class UsableAsKey = ExactKeyMatchOnly,+    typename Alloc,+    typename Func,+    typename Arg,+    std::enable_if_t<+        std::is_same<remove_cvref_t<Arg>, KeyType>::value ||+            UsableAsKey<remove_cvref_t<Arg>>::value,+        int> = 0>+auto callWithConstructedKey(Alloc&, Func&& f, Arg&& arg) {+  // we found a usable key in the args :)+  auto const& key = arg;+  return f(key, std::forward<Arg>(arg));+}++template <+    typename KeyType,+    template <typename> class UsableAsKey = ExactKeyMatchOnly,+    typename Alloc,+    typename Func,+    typename... Args>+auto callWithConstructedKey(Alloc& a, Func&& f, Args&&... args) {+  // we will need to materialize a temporary key :(+  TemporaryEmplaceKey<KeyType, Alloc> key(+      a, std::forward_as_tuple(std::forward<Args>(args)...));+  return f(const_cast<KeyType const&>(key.value()), std::move(key.value()));+}++// Traits to simplify deduction guides implementation for containers.++// SFINAE constraint to test whether a type is an allocator according to+// is_allocator trait.+template <typename T>+using RequireAllocator = std::enable_if_t<is_allocator_v<T>, T>;++template <typename T>+using RequireNotAllocator = std::enable_if_t<!is_allocator_v<T>, T>;++template <typename T>+using RequireInputIterator =+    std::enable_if_t<iterator_category_matches_v<T, std::input_iterator_tag>>;++} // namespace detail+} // namespace folly
+ folly/folly/container/detail/tape_detail.h view
@@ -0,0 +1,134 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <folly/Portability.h>+#include <folly/Range.h>+#include <folly/container/Iterator.h>+#include <folly/container/range_traits.h>+#include <folly/lang/Hint.h>+#include <folly/memory/UninitializedMemoryHacks.h>++#include <cstddef>+#include <iterator>+#include <memory>+#include <string>+#include <string_view>+#include <type_traits>+#include <utility>+#include <vector>++namespace folly {+namespace detail {++template <typename Container, bool = is_contiguous_range_v<Container>>+struct tape_reference_traits {+  using iterator = typename Container::const_iterator;+  using reference = Range<iterator>;++  static constexpr reference make(iterator f, iterator l) {+    return reference{f, l};+  }+};++template <typename Container>+struct tape_reference_traits<Container, true> {+  using iterator = typename Container::const_iterator;+  using value_type = typename std::iterator_traits<iterator>::value_type;+  using reference = Range<const value_type*>;++  static constexpr auto* get_address(iterator it) {+    // std::to_address is only available since C++20+    if constexpr (std::is_pointer_v<iterator>) {+      return it;+    } else {+      return it.operator->();+    }+  }++  static constexpr reference make(iterator f, iterator l) {+    return reference{get_address(f), get_address(l)};+  }+};++template <typename R>+using get_range_const_iterator_t =+    decltype(std::cbegin(std::declval<const R&>()));++struct fake_type {};++template <typename R>+using maybe_range_const_iterator_t =+    detected_or_t<fake_type*, get_range_const_iterator_t, R>;++template <typename R>+using maybe_range_value_t =+    iterator_value_type_t<maybe_range_const_iterator_t<R>>;++// This is a big function to inline but it's used insie a big function too+template <typename I, typename S>+auto compute_total_tape_len_if_possible(I f, S l) {+  using success = std::pair<std::size_t, std::size_t>;+  using failure = fake_type;+  if constexpr (!iterator_category_matches_v<I, std::forward_iterator_tag>) {+    return failure{};+  }+  // We have to special case StringPiece to special case `const char*` and+  // `char[]`+  else if constexpr (+      std::is_convertible_v<iterator_value_type_t<I>, folly::StringPiece>) {+    std::size_t records_size = 0U;+    std::size_t flat_size = 0U;++    for (I i = f; i != l; ++i) {+      ++records_size;+      flat_size += folly::StringPiece(*i).size();+    }+    return success{records_size, flat_size};+  } else if constexpr (!range_has_known_distance_v<iterator_value_type_t<I>>) {+    return failure{};+  } else {+    std::size_t records_size = 0U;+    std::size_t flat_size = 0U;++    for (I i = f; i != l; ++i) {+      ++records_size;+      flat_size +=+          static_cast<std::size_t>(std::distance(std::begin(*i), std::end(*i)));+    }+    return success{records_size, flat_size};+  }+}++template <typename Container, typename I, typename S>+void append_range_unsafe(Container& c, I f, S l) {+  if constexpr (+      !iterator_category_matches_v<I, std::random_access_iterator_tag> ||+      !std::is_trivially_copy_constructible_v<iterator_value_type_t<I>> ||+      !(is_instantiation_of_v<std::vector, Container> ||+        is_instantiation_of_v<std::basic_string, Container>)) {+    c.insert(c.end(), f, l);+  } else {+    folly::compiler_may_unsafely_assume(l >= f);+    auto old_size = c.size();+    detail::unsafeVectorSetLargerSize(c, c.size() + (l - f));+    std::copy(f, l, c.begin() + old_size);+  }+}++} // namespace detail+} // namespace folly
+ folly/folly/container/heap_vector_types.h view
@@ -0,0 +1,1703 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++/*+ * This header defines two new containers, heap_vector_set and heap_vector_map+ * classes. These containers are designed to be a drop-in replacement of+ * sorted_vector_set and sorted_vector_map. Similarly to sorted_vector_map/set,+ * heap_vector_map/set models AssociativeContainers. Below, we list important+ * differences from std::set and std::map (also documented in+ * folly/sorted_vector_types.h):+ *+ *   - insert() and erase() invalidate iterators and references.+ *   - erase(iterator) returns an iterator pointing to the next valid element.+ *   - insert() and erase() are O(N)+ *   - our iterators model RandomAccessIterator+ *   - heap_vector_map::value_type is pair<K,V>, not pair<const K,V>.+ *     (This is basically because we want to store the value_type in+ *     std::vector<>, which requires it to be Assignable.)+ *   - insert() single key variants, emplace(), and emplace_hint() only provide+ *     the strong exception guarantee (unchanged when exception is thrown) when+ *     std::is_nothrow_move_constructible<value_type>::value is true.+ *+ * heap_vector_map/set have exactly the same size as sorted_vector_map/set.+ * These containers utilizes a vector container (e.g. std::vector) to store the+ * values. Heap containers (similarly to sorted containers) have no additional+ * memory overhead. They lay out the data in an optimal way w.r.t locality (see+ * https://algorithmica.org/en/eytzinger), called eytzinger or heap order. For+ * example in a sorted_vector_set, the underlying vector contains:+ *              index    0   1   2   3   4   5   6   7   8   9+ *       vector[index]   0, 10, 20, 30, 40, 50, 60, 70, 80, 90+ * while in a heap_vector_set, the underlying vector contains:+ *              index    0   1   2   3   4   5   6   7   8   9+ *       vector[index]  60, 30, 80, 10, 50, 70, 90,  0, 20, 40+ * Lookup elements in sorted vector containers relies on binary search,+ * std::lower_bound. While in heap containers, lookup operation has two+ * benefits:+ *+ * 1. Cache locality, the container is traversed sequentially instead of binary+ * search that jumps around the sorted vector.+ * 2. The branches in a binary search are mispredicted resulting in hardware+ * penalty while using heap lookup search the branch can be avoided by using+ * cmov instruction. We observerd look up operations are up to 2X faster than+ * sorted_vector_map.+ *+ * However, Insertion/deletion operations are much slower. If insertions and+ * deletions are rare operations for your use case then heap containers might+ * be the right choice for you. Also, to minimize impact of insertions while+ * creating heap containers, we recommend not to insert element by element+ * instead first collect elements in a vector, then construct the heap map from+ * it.+ *+ * Another substantial trade off, inorder traversal of heap container elements+ * is slower than sorted vector containers. This is expected as heap map needs+ * to jump around to access map elements in order. A remedy is to use underlying+ * vector iterators. This works only when the order is irrelevant. For example,+ * using heap container iterator:+ *         for (auto& e: HeapSet)+ *           std::cout << e << ", ";+ * Prints:  0, 10, 20, 30, 40, 50, 60, 70, 80, 90+ * and using underlying vector container iterator:+ *         for (auto& e : HeapSet.iterate())+ *           std::cout << e << ", ";+ * Prints:  60, 30, 80, 10, 50, 70, 90,  0, 20, 40+ * The latter loop is the fastest traversal.+ *+ * Finally The main benefit of heap containers is a compact representation+ * that achieves fast random lookup. Use this map when lookup is the+ * dominant operation and at the same time saving memory is important.+ */++#pragma once++#include <algorithm>+#include <cassert>+#include <functional>+#include <initializer_list>+#include <iterator>+#include <memory>+#include <stdexcept>+#include <type_traits>+#include <utility>+#include <vector>++#include <folly/Range.h>+#include <folly/ScopeGuard.h>+#include <folly/Traits.h>+#include <folly/Utility.h>+#include <folly/container/Iterator.h>+#include <folly/functional/Invoke.h>+#include <folly/lang/Exception.h>+#include <folly/memory/MemoryResource.h>+#include <folly/portability/Builtins.h>+#include <folly/small_vector.h>++namespace folly {+template <+    typename Key,+    typename Value,+    typename Compare,+    typename Allocator,+    typename GrowthPolicy,+    typename Container>+class heap_vector_map;++namespace detail {++namespace heap_vector_detail {++/*+ * Heap Containers Helper Functions+ * ---------------------------------+ * Terminology:+ * - offset means the index at which element is stored in the container vector,+ *   following the heap order.+ * - index means the element rank following the container compare order.+ *+ * Introduction:+ * Heap order can be constructed from a sorted input vector using below+ * naive algorithm.+ *+ *     heapify(input, output, index = 0, offset = 1) {+ *       if (offset <= size) {+ *         index = heapify(input, output, i, 2 * offset);+ *         output[offset - 1] = input[index++];+ *         index = heapify(input, output, i, 2 * offset + 1);+ *       }+ *       return index;+ *     }+ *+ * Helper functions below implement efficient algorithms to:+ *  - Find offsets of the smallest/greatest elements.+ *  - Given an offset, calculate the offset where next/previous element is+ *  stored if it exists.+ *  - Given a start and an end offsets, calculate distance between their+ *  corresponding indexes.+ *  - Fast inplace heapification.+ *  - Insert a new element while preserving heap order.+ *  - Delete an element and preserve heap order.+ *  - find lower/upper bound of a key following container compare order.+ */++// Returns the offset of the smallest element in the heap container.+// The samllest element is stored at:+//     vector[2^n - 1] where 2^n <= size < 2^(n+1).+// firstOffset returns 2^n - 1 if size > 0, 0 otherwise+template <typename size_type>+size_type firstOffset(size_type size) {+  if (size) {+    return (1+            << (CHAR_BIT * sizeof(unsigned long) -+                __builtin_clzl((unsigned long)size) - 1)) -+        1;+  } else {+    return 0;+  }+}++// Returns the offset of greatest element in heap container.+// the greatest element is storted at:+//     vector[2^n - 2] if 2^n - 1 <= size < 2^(n+1)+// lastOffset returns 2^n - 2 if size > 0, 0 otherwise+template <typename size_type>+size_type lastOffset(size_type size) {+  if (size) {+    return ((size & (size + 1)) == 0 ? size : firstOffset(size)) - 1;+  }+  return 0;+}++// Returns the offset of the next element. It is calculated based on the+// size of the map.+// To simplify implementation, offset must be 1-based+// return value is also 1-based.+template <typename size_type>+size_type next(size_type offset, size_type size) {+  auto next = (2 * offset);+  if (next >= size) {+    return offset >> __builtin_ffsl((unsigned long)~offset);+  } else {+    next += 1;+    for (auto n = 2 * next; n <= size; n *= 2) {+      next = n;+    }+    return next;+  }+}++// Returns the offset of the previous element. It is calculated based on+// the size of the map.+// To simplify implementation, offset must be 1-based+// return value is also 1-based.+template <typename size_type>+size_type prev(size_type offset, size_type size) {+  auto prev = 2 * offset;+  if (prev <= size) {+    for (auto p = 2 * prev + 1; p <= size; p = 2 * p + 1) {+      if (2 * p >= size) {+        return p;+      }+    }+    return prev;+  }+  return offset >> __builtin_ffsl((unsigned long)offset);+}++// To avoid scanning all offsets, skip offsets that cannot be within the+// range of offset1 and offset2.+// Note We could compute least common ancestor of offset1 and offset2+// to further minimize scanning. However it is not profitable when container+// is relatively small.+template <typename size_type>+size_type getStartOffsetToScan(size_type offset1, size_type offset2) {+  while ((offset1 & (offset1 - 1)) != 0) {+    offset1 >>= 1;+  }+  if (offset1 > 1) {+    while ((offset2 & (offset2 - 1)) != 0) {+      offset2 >>= 1;+    }+    offset1 = std::min(offset1, offset2);+  }+  return offset1;+}++// Given a start and end offsets, returns the distance between+// their corresponding indexes.+template <typename Container, typename size_type>+typename Container::difference_type distance(+    Container& cont, size_type start, size_type end) {+  using difference_type = typename Container::difference_type;+  difference_type dist = 0;+  size_type size = cont.size();+  // To simplify logic base start and end from one.+  start++;+  end++;+  std::function<bool(size_type, size_type)> calculateDistance =+      [&](size_type offset, size_type lb) {+        if (offset > size)+          return false;+        for (; offset <= size; offset <<= 1)+          ;+        offset >>= 1;+        for (; offset > lb; offset >>= 1) {+          if (offset == start) {+            if (dist) {+              dist *= -1;+              return true;+            }+            dist = 1;+          } else if (offset == end) {+            if (dist) {+              return true;+            }+            dist = 1;+          } else if (dist) {+            dist++;+          }+          if (calculateDistance(2 * offset + 1, offset)) {+            return true;+          }+        }+        return false;+      };+  auto offset = getStartOffsetToScan(start, end);+  calculateDistance(offset, size_type(0));+  // Handle start == end()+  if (start > size) {+    dist *= -1;+  }++  return dist;+}++// Returns the offset for each index in heap container+// for example if size = 7 then+//       index     0  1  2  3  4  5  6+//     offsets = { 3, 1, 4, 0, 5, 2, 6 }+// The smallest element (index = 0) of heap container is stored at cont[3] and+// so on.+template <typename size_type, typename Offsets>+void getOffsets(size_type size, Offsets& offsets) {+  size_type i = 0;+  size_type offset = 0;+  size_type index = size;+  do {+    for (size_type o = offset; o < size; o = 2 * o + 2) {+      offsets[i++] = o;+    }+    offset = offsets[--i];+    offsets[--index] = offset;+    offset = 2 * offset + 1;+  } while (i || offset < size);+}++// Inplace conversion of a sorted vector to heap layout+// This algorithm utilizes circular swaps to position each element in its heap+// order offset in the vector. For example, given a sorted vector below:+//     cont = { 0, 10, 20, 30, 40, 50, 60, 70 }+// getOffsets returns:+//       index     0  1  2  3  4  5  6  7+//     offsets = { 4, 2, 6, 1, 3, 5, 7, 0 }+//+// The algorithm moves elements circularly:+// cont[4]->cont[0]->cont[7]->cont[6]->cont[2]->cont[1]->cont[3]-> cont[4]+// cont[5] remains inplace+// returns:+// cont = { 40, 20, 60, 10, 30, 50, 70, 0 }+template <class Container>+void heapify(Container& cont) {+  using size_type = typename Container::size_type;+  size_type size = cont.size();+  std::vector<size_type> offsets;+  offsets.resize(size);+  getOffsets(size, offsets);++  std::function<void(size_type, size_type)> rotate =+      [&](size_type next, size_type index) {+        std::vector<size_type> worklist;+        while (index != next) {+          worklist.push_back(next);+          next = offsets[next];+        }+        while (!worklist.empty()) {+          auto cur = worklist.back();+          worklist.pop_back();+          cont[offsets[cur]] = std::move(cont[cur]);+          offsets[cur] = size;+        }+      };++  for (size_type index = 0; index < size; index++) {+    // already moved+    if (offsets[index] == size) {+      continue;+    }+    size_type next = offsets[index];+    if (next == index) {+      continue;+    }+    // Subtlety: operator[] returns a Container::reference. Because+    // Container::reference can be a proxy, using bare `auto` is not+    // sufficient to remove the "reference nature" of+    // Container::reference and force a move out of the container;+    // instead, we need Container::value_type.+    typename Container::value_type tmp = std::move(cont[index]);+    rotate(next, index);+    cont[next] = std::move(tmp);+  }+}++// Below helper functions to implement inplace insertion/deletion.++// Returns the sequence of offsets that need to be moved. This sequence+// is the range between size-1 and the offset of the inserted element.+// There are two cases: {size - 1, ..., offset} or {offset, ..., size - 1}+// For example if 45 is inserted at offset == 0 and size == 9.+// Before insertion:+//     element    0 10   20 30 40     50 60 70+//     offset     7  3    1  4  0      5  2  6+// After inserting 45:+//     element    0 10   20 30 40 45  50 60 70+//     offset     7  3    8  1  4  0   5  2  6+// This function returns:+//     offsets = { 8, 1, 4, 0 }+template <typename size_type, typename Offsets>+bool getOffsetRange(+    size_type size, Offsets& offsets, size_type offset, size_type current) {+  for (; current <= size; current <<= 1) {+    if (getOffsetRange(size, offsets, offset, 2 * current + 1)) {+      return true;+    }+    if (offsets.empty()) {+      if (offset == current || size == current) {+        // Start recording offsets that need to be moved.+        offsets.push_back(current - 1);+      }+    } else {+      // record offset+      offsets.push_back(current - 1);+      if (offset == current || size == current) {+        // Stop recording offsets+        return true;+      }+    }+  }+  return false;+}++template <typename size_type, typename Offsets>+void getOffsetRange(size_type size, Offsets& offsets, size_type offset) {+  auto start = getStartOffsetToScan(offset, size);+  getOffsetRange(size, offsets, offset, start);+}++// Insert a new element in heap order+// Assumption: the inserted element is already pushed at the back of the+// container vector (i.e. located at vector[size - 1]).+template <typename size_type, class Container>+size_type insert(size_type offset, Container& cont) {+  size_type size = cont.size();+  if (size == 1) {+    return 0;+  }+  size_type adjust = 1;+  if (offset == size - 1) {+    adjust = 0;+    auto last = lastOffset(size);+    if (last == offset) {+      return offset;+    }+    offset = last;+  }+  std::vector<size_type> offsets;+  offsets.reserve(size);+  getOffsetRange(size, offsets, offset + 1);+  typename Container::value_type v = std::move(cont[size - 1]);+  if (offsets[0] != offset) {+    for (size_type i = 1, e = offsets.size(); i < e; ++i) {+      cont[offsets[i - 1]] = std::move(cont[offsets[i]]);+    }+    cont[offset] = std::move(v);+    return offset;+  }+  for (size_type i = offsets.size() - 1; i > adjust; --i) {+    cont[offsets[i]] = std::move(cont[offsets[i - 1]]);+  }+  cont[offsets[adjust]] = std::move(v);+  return offsets[adjust];+}++// Erase one element and preserve heap order+template <typename size_type, class Container>+size_type erase(size_type offset, Container& cont) {+  size_type size = cont.size();+  if (offset + 1 == size) {+    auto ret = next(offset + 1, size);+    cont.resize(size - 1);+    return ret ? ret - 1 : size - 1;+  }+  std::vector<size_type> offsets;+  offsets.reserve(size);+  getOffsetRange(size, offsets, offset + 1);+  if (offsets[0] == offset) {+    for (size_type i = 1, e = offsets.size(); i < e; i++) {+      cont[offsets[i - 1]] = std::move(cont[offsets[i]]);+    }+  } else {+    for (size_type i = offsets.size() - 1; i > 0; --i) {+      cont[offsets[i]] = std::move(cont[offsets[i - 1]]);+    }+  }+  cont.resize(size - 1);+  return offset;+}++// Search lower bound in a container sorted in heap order.+// To speed up lower_bound for small containers, peel four iterations and use+// reverse compare to exit quickly.+// The branch inside the loop is converted to a cmov by the compiler. cmov are+// more efficient when the branch is unpredictable.+template <typename Compare, typename RCompare, typename Container>+typename Container::size_type lower_bound(+    Container& cont, Compare cmp, RCompare reverseCmp) {+  using size_type = typename Container::size_type;+  size_type size = cont.size();+  auto last = size;+  size_type offset = 0;+  if (size) {+    if (cmp(cont[offset])) {+      offset = 2 * offset + 2;+    } else {+      if (!reverseCmp(cont[offset])) {+        return offset;+      }+      last = offset;+      offset = 2 * offset + 1;+    }+    if (offset < size) {+      if (cmp(cont[offset])) {+        offset = 2 * offset + 2;+      } else {+        if (!reverseCmp(cont[offset])) {+          return offset;+        }+        last = offset;+        offset = 2 * offset + 1;+      }+      if (offset < size) {+        if (cmp(cont[offset])) {+          offset = 2 * offset + 2;+        } else {+          if (!reverseCmp(cont[offset])) {+            return offset;+          }+          last = offset;+          offset = 2 * offset + 1;+        }+        if (offset < size) {+          if (cmp(cont[offset])) {+            offset = 2 * offset + 2;+          } else {+            if (!reverseCmp(cont[offset])) {+              return offset;+            }+            last = offset;+            offset = 2 * offset + 1;+          }+          for (; offset < size; offset++) {+            if (cmp(cont[offset])) {+              offset = 2 * offset + 1;+            } else {+              last = offset;+              offset = 2 * offset;+            }+          }+        }+      }+    }+  }+  return last;+}++template <typename Compare, typename Container>+typename Container::size_type upper_bound(Container& cont, Compare cmp) {+  using size_type = typename Container::size_type;+  auto size = cont.size();+  auto last = size;+  for (size_type offset = 0; offset < size; offset++) {+    if (!cmp(cont[offset])) {+      offset = 2 * offset + 1;+    } else {+      last = offset;+      offset = 2 * offset;+    }+  }+  return last;+}++// Helper functions below are similar to sorted containers. Wherever+// applicable renamed to heap containers.+template <typename, typename Compare, typename Key, typename T>+struct heap_vector_enable_if_is_transparent {};++template <typename Compare, typename Key, typename T>+struct heap_vector_enable_if_is_transparent<+    void_t<typename Compare::is_transparent>,+    Compare,+    Key,+    T> {+  using type = T;+};++// This wrapper goes around a GrowthPolicy and provides iterator+// preservation semantics, but only if the growth policy is not the+// default (i.e. nothing).+template <class Policy>+struct growth_policy_wrapper : private Policy {+  template <class Container, class Iterator>+  Iterator increase_capacity(Container& c, Iterator desired_insertion) {+    using diff_t = typename Container::difference_type;+    diff_t d = desired_insertion - c.begin();+    Policy::increase_capacity(c);+    return c.begin() + d;+  }+};+template <>+struct growth_policy_wrapper<void> {+  template <class Container, class Iterator>+  Iterator increase_capacity(Container&, Iterator it) {+    return it;+  }+};++template <class OurContainer, class Container, class InputIterator>+void bulk_insert(+    OurContainer& sorted,+    Container& cont,+    InputIterator first,+    InputIterator last) {+  assert(first != last);++  auto const prev_size = cont.size();+  cont.insert(cont.end(), first, last);+  auto const middle = cont.begin() + prev_size;++  auto const& cmp(sorted.value_comp());+  if (!std::is_sorted(middle, cont.end(), cmp)) {+    std::sort(middle, cont.end(), cmp);+  }+  if (middle != cont.begin() && !cmp(*(middle - 1), *middle)) {+    std::inplace_merge(cont.begin(), middle, cont.end(), cmp);+  }+  cont.erase(+      std::unique(+          cont.begin(),+          cont.end(),+          [&](typename OurContainer::value_type const& a,+              typename OurContainer::value_type const& b) {+            return !cmp(a, b) && !cmp(b, a);+          }),+      cont.end());+  heapify(cont);+}++template <typename Container, typename Compare>+bool is_sorted_unique(Container const& container, Compare const& comp) {+  if (container.empty()) {+    return true;+  }+  auto const e = container.end();+  for (auto a = container.begin(), b = std::next(a); b != e; ++a, ++b) {+    if (!comp(*a, *b)) {+      return false;+    }+  }+  return true;+}++template <typename Container, typename Compare>+Container&& as_sorted_unique(Container&& container, Compare const& comp) {+  std::sort(container.begin(), container.end(), comp);+  container.erase(+      std::unique(+          container.begin(),+          container.end(),+          [&](auto const& a, auto const& b) {+            return !comp(a, b) && !comp(b, a);+          }),+      container.end());+  return static_cast<Container&&>(container);+}++// class value_compare_map is used to compare map elements.+template <class Compare>+struct value_compare_map : Compare {+  template <typename... value_type>+  auto operator()(const value_type&... a) const+      noexcept(is_nothrow_invocable_v<const Compare&, decltype((a.first))...>)+          -> invoke_result_t<const Compare&, decltype((a.first))...> {+    return Compare::operator()(a.first...);+  }++  template <typename value_type>+  const auto& getKey(const value_type& a) const noexcept {+    return a.first;+  }++  explicit value_compare_map(const Compare& c) noexcept(+      std::is_nothrow_copy_constructible<Compare>::value)+      : Compare(c) {}+};++// wrapper class value_compare_set for set elements.+template <class Compare>+struct value_compare_set : Compare {+  using Compare::operator();++  template <typename value_type>+  value_type& getKey(value_type& a) const noexcept {+    return a;+  }++  explicit value_compare_set(const Compare& c) noexcept(+      std::is_nothrow_copy_constructible<Compare>::value)+      : Compare(c) {}+};++/**+ * A heap_vector_container is a container similar to std::set<>, but+ * implemented as a heap array with std::vector<>.+ * This class contains shared implementation between set and map. It+ * fully implements set methods and used as base class for map.+ *+ * @tparam T               Data type to store+ * @tparam Compare         Comparison function that imposes a+ *                              strict weak ordering over instances of T+ * @tparam Allocator       allocation policy+ * @tparam GrowthPolicy    policy object to control growth+ * @tparam Container       underlying vector where elements are stored+ * @tparam KeyT            key type, for set it is same as T.+ * @tparam ValueCompare    wrapper class to compare Container::value_type+ *+ */+template <+    class T,+    class Compare = std::less<T>,+    class Allocator = std::allocator<T>,+    class GrowthPolicy = void,+    class Container = std::vector<T, Allocator>,+    class KeyT = T,+    class ValueCompare = value_compare_set<Compare>>+class heap_vector_container : growth_policy_wrapper<GrowthPolicy> {+ protected:+  growth_policy_wrapper<GrowthPolicy>& get_growth_policy() { return *this; }++  template <typename K, typename V, typename C = Compare>+  using if_is_transparent =+      _t<heap_vector_enable_if_is_transparent<void, C, K, V>>;++  struct EBO;++ public:+  using key_type = KeyT;+  using value_type = T;+  using key_compare = Compare;+  using value_compare = ValueCompare;+  using allocator_type = Allocator;+  using container_type = Container;+  using pointer = typename Container::pointer;+  using reference = typename Container::reference;+  using const_reference = typename Container::const_reference;+  using difference_type = typename Container::difference_type;+  using size_type = typename Container::size_type;++  // Defines inorder iterator for heap set.+  template <typename Iter>+  struct heap_iterator {+    using iterator_category = std::random_access_iterator_tag;+    using size_type = typename Container::size_type;+    using difference_type =+        typename std::iterator_traits<Iter>::difference_type;+    using value_type = typename std::iterator_traits<Iter>::value_type;+    using pointer = typename std::iterator_traits<Iter>::pointer;+    using reference = typename std::iterator_traits<Iter>::reference;++    heap_iterator() = default;+    template <typename C>+    heap_iterator(Iter ptr, C* cont) {+      ptr_ = ptr;+      cont_ = const_cast<Container*>(cont);+    }++    template <+        typename I2,+        typename = typename std::enable_if<+            std::is_same<typename Container::iterator, I2>::value>::type>+    /* implicit */ heap_iterator(const heap_iterator<I2>& rawIterator)+        : ptr_(rawIterator.ptr_), cont_(rawIterator.cont_) {}++    ~heap_iterator() = default;++    heap_iterator(const heap_iterator& rawIterator) = default;++    heap_iterator& operator=(const heap_iterator& rawIterator) = default;+    heap_iterator& operator=(Iter ptr) {+      assert(+          (ptr - cont_->begin()) >= 0 &&+          (ptr - cont_->begin()) <= (difference_type)cont_->size());+      ptr_ = ptr;+      return (*this);+    }++    bool operator==(const heap_iterator& rawIterator) const {+      return ptr_ == rawIterator.ptr_;+    }+    bool operator!=(const heap_iterator& rawIterator) const {+      return !operator==(rawIterator);+    }++    heap_iterator& operator+=(const difference_type& movement) {+      size_type offset = ptr_ - cont_->begin() + 1;+      auto size = cont_->size();+      if (movement < 0) {+        difference_type i = 0;++        if (offset - 1 == size) {+          // handle --end()+          offset = heap_vector_detail::lastOffset(size) + 1;+          i = -1;+        }+        for (; i > movement; i--) {+          offset = heap_vector_detail::prev(offset, size);+        }+      } else {+        for (difference_type i = 0; i < movement; i++) {+          offset = heap_vector_detail::next(offset, size);+        }+      }+      ptr_ = cont_->begin() + (offset == 0 ? cont_->size() : offset - 1);+      return (*this);+    }++    heap_iterator& operator-=(const difference_type& movement) {+      return operator+=(-movement);+    }+    heap_iterator& operator++() { return operator+=(1); }+    heap_iterator& operator--() { return operator-=(1); }+    heap_iterator operator++(int) {+      auto temp(*this);+      operator+=(1);+      return temp;+    }+    heap_iterator operator--(int) {+      auto temp(*this);+      operator-=(1);+      return temp;+    }+    heap_iterator operator+(const difference_type& movement) {+      auto temp(*this);+      temp += movement;+      return temp;+    }+    heap_iterator operator+(const difference_type& movement) const {+      auto temp(*this);+      temp += movement;+      return temp;+    }++    heap_iterator operator-(const difference_type& movement) {+      auto temp(*this);+      temp -= movement;+      return temp;+    }++    heap_iterator operator-(const difference_type& movement) const {+      auto temp(*this);+      temp -= movement;+      return temp;+    }++    difference_type operator-(const heap_iterator& rawIterator) {+      assert(cont_ == rawIterator.cont_);+      size_type offset0 = ptr_ - cont_->begin();+      size_type offset1 = rawIterator.ptr_ - cont_->begin();+      if (offset1 == offset0)+        return 0;+      return heap_vector_detail::distance(*cont_, offset1, offset0);+    }++    difference_type operator-(const heap_iterator& rawIterator) const {+      assert(cont_ == rawIterator.cont_);+      size_type offset0 = ptr_ - cont_->begin();+      size_type offset1 = rawIterator.ptr_ - cont_->begin();+      if (offset1 == offset0)+        return 0;+      return heap_vector_detail::distance(*cont_, offset1, offset0);+    }++    reference operator*() const { return *ptr_; }+    pointer operator->() const {+      if constexpr (std::is_pointer_v<Iter>) {+        return ptr_;+      } else {+        return ptr_.operator->();+      }+    }++   protected:+    template <typename I2>+    friend struct heap_iterator;++    template <+        typename T2,+        typename Compare2,+        typename Allocator2,+        typename GrowthPolicy2,+        typename Container2,+        typename KeyT2,+        typename ValueCompare2>+    friend class heap_vector_container;++    template <+        typename Key2,+        typename Value2,+        typename Compare2,+        typename Allocator2,+        typename GrowthPolicy2,+        typename Container2>+    friend class ::folly::heap_vector_map;++    Iter ptr_;+    Container* cont_;+  };++  using iterator = heap_iterator<typename Container::iterator>;+  using const_iterator = heap_iterator<typename Container::const_iterator>;+  using reverse_iterator = std::reverse_iterator<iterator>;+  using const_reverse_iterator = std::reverse_iterator<const_iterator>;++  heap_vector_container() : m_(value_compare(Compare()), Allocator()) {}++  heap_vector_container(const heap_vector_container&) = default;++  heap_vector_container(+      const heap_vector_container& other, const Allocator& alloc)+      : m_(other.m_, alloc) {}++  heap_vector_container(heap_vector_container&&) = default;++  heap_vector_container(+      heap_vector_container&& other,+      const Allocator&+          alloc) noexcept(std::+                              is_nothrow_constructible<+                                  EBO,+                                  EBO&&,+                                  const Allocator&>::value)+      : m_(std::move(other.m_), alloc) {}++  explicit heap_vector_container(const Allocator& alloc)+      : m_(value_compare(Compare()), alloc) {}++  explicit heap_vector_container(+      const Compare& comp, const Allocator& alloc = Allocator())+      : m_(value_compare(comp), alloc) {}++  template <class InputIterator>+  explicit heap_vector_container(+      InputIterator first,+      InputIterator last,+      const Compare& comp = Compare(),+      const Allocator& alloc = Allocator())+      : m_(value_compare(comp), alloc) {+    insert(first, last);+  }++  template <class InputIterator>+  heap_vector_container(+      InputIterator first, InputIterator last, const Allocator& alloc)+      : m_(value_compare(Compare()), alloc) {+    insert(first, last);+  }++  /* implicit */ heap_vector_container(+      std::initializer_list<value_type> list,+      const Compare& comp = Compare(),+      const Allocator& alloc = Allocator())+      : m_(value_compare(comp), alloc) {+    insert(list.begin(), list.end());+  }++  heap_vector_container(+      std::initializer_list<value_type> list, const Allocator& alloc)+      : m_(value_compare(Compare()), alloc) {+    insert(list.begin(), list.end());+  }++  // Construct a heap_vector_container by stealing the storage of a prefilled+  // container. The container need not be sorted already. This supports+  // bulk construction of heap_vector_container with zero allocations, not+  // counting those performed by the caller.+  // Note that `heap_vector_container(const Container& container)` is not+  // provided, since the purpose of this constructor is to avoid an unnecessary+  // copy.+  explicit heap_vector_container(+      Container&& container,+      const Compare& comp =+          Compare()) noexcept(std::+                                  is_nothrow_constructible<+                                      EBO,+                                      value_compare,+                                      Container&&>::value)+      : heap_vector_container(+            sorted_unique,+            heap_vector_detail::as_sorted_unique(+                std::move(container), value_compare(comp)),+            comp) {}++  // Construct a heap_vector_container by stealing the storage of a prefilled+  // container. Its elements must be sorted and unique, as sorted_unique_t+  // hints. Supports bulk construction of heap_vector_container with zero+  // allocations, not counting those performed by the caller.+  // Note that `heap_vector_container(sorted_unique_t, const Container&+  // container)` is not provided, since the purpose of this constructor is to+  // avoid an extra copy.+  heap_vector_container(+      sorted_unique_t /* unused */,+      Container&& container,+      const Compare& comp =+          Compare()) noexcept(std::+                                  is_nothrow_constructible<+                                      EBO,+                                      value_compare,+                                      Container&&>::value)+      : m_(value_compare(comp), std::move(container)) {+    assert(heap_vector_detail::is_sorted_unique(m_.cont_, value_comp()));+    heap_vector_detail::heapify(m_.cont_);+  }++  Allocator get_allocator() const { return m_.cont_.get_allocator(); }++  const Container& get_container() const noexcept { return m_.cont_; }++  /**+   * Directly swap the container. Similar to swap()+   */+  void swap_container(Container& newContainer) {+    heap_vector_detail::as_sorted_unique(newContainer, value_comp());+    heap_vector_detail::heapify(newContainer);+    using std::swap;+    swap(m_.cont_, newContainer);+  }+  void swap_container(sorted_unique_t, Container& newContainer) {+    assert(heap_vector_detail::is_sorted_unique(newContainer, value_comp()));+    heap_vector_detail::heapify(newContainer);+    using std::swap;+    swap(m_.cont_, newContainer);+  }++  heap_vector_container& operator=(const heap_vector_container& other) =+      default;++  heap_vector_container& operator=(heap_vector_container&& other) = default;++  heap_vector_container& operator=(std::initializer_list<value_type> ilist) {+    clear();+    insert(ilist.begin(), ilist.end());+    return *this;+  }++  key_compare key_comp() const { return m_; }+  value_compare value_comp() const { return m_; }++  iterator begin() {+    if (size()) {+      return iterator(+          m_.cont_.begin() + heap_vector_detail::firstOffset(size()),+          &m_.cont_);+    }+    return iterator(m_.cont_.begin(), &m_.cont_);+  }+  iterator end() { return iterator(m_.cont_.end(), &m_.cont_); }+  const_iterator cbegin() const {+    if (size()) {+      return const_iterator(+          m_.cont_.cbegin() + heap_vector_detail::firstOffset(size()),+          &m_.cont_);+    }+    return const_iterator(m_.cont_.cbegin(), &m_.cont_);+  }+  const_iterator begin() const {+    if (size()) {+      return const_iterator(+          m_.cont_.cbegin() + heap_vector_detail::firstOffset(size()),+          &m_.cont_);+    }+    return const_iterator(m_.cont_.begin(), &m_.cont_);+  }+  const_iterator cend() const {+    return const_iterator(m_.cont_.cend(), &m_.cont_);+  }+  const_iterator end() const {+    return const_iterator(m_.cont_.end(), &m_.cont_);+  }+  reverse_iterator rbegin() { return reverse_iterator(end()); }+  reverse_iterator rend() { return reverse_iterator(begin()); }+  const_reverse_iterator rbegin() const {+    return const_reverse_iterator(end());+  }+  const_reverse_iterator rend() const {+    return const_reverse_iterator(begin());+  }+  const_reverse_iterator crbegin() const {+    return const_reverse_iterator(end());+  }+  const_reverse_iterator crend() const {+    return const_reverse_iterator(begin());+  }++  void clear() { return m_.cont_.clear(); }+  size_type size() const { return m_.cont_.size(); }+  size_type max_size() const { return m_.cont_.max_size(); }+  bool empty() const { return m_.cont_.empty(); }+  void reserve(size_type s) { return m_.cont_.reserve(s); }+  void shrink_to_fit() { m_.cont_.shrink_to_fit(); }+  size_type capacity() const { return m_.cont_.capacity(); }+  const value_type* data() const noexcept { return m_.cont_.data(); }++  std::pair<iterator, bool> insert(const value_type& value) {+    iterator it = lower_bound(m_.getKey(value));+    if (it == end() || value_comp()(value, *it)) {+      auto offset = it.ptr_ - m_.cont_.begin();+      get_growth_policy().increase_capacity(*this, it);+      m_.cont_.push_back(value);+      offset = heap_vector_detail::insert(offset, m_.cont_);+      it = m_.cont_.begin() + offset;+      return std::make_pair(it, true);+    }+    return std::make_pair(it, false);+  }++  std::pair<iterator, bool> insert(value_type&& value) {+    iterator it = lower_bound(m_.getKey(value));+    if (it == end() || value_comp()(value, *it)) {+      auto offset = it.ptr_ - m_.cont_.begin();+      get_growth_policy().increase_capacity(*this, it);+      m_.cont_.push_back(std::move(value));+      offset = heap_vector_detail::insert(offset, m_.cont_);+      it = m_.cont_.begin() + offset;+      return std::make_pair(it, true);+    }+    return std::make_pair(it, false);+  }+  /* There is no benefit of using hint. Keep it for compatibility+   * Ignore and insert */+  iterator insert(const_iterator /* hint */, const value_type& value) {+    return insert(value).first;+  }++  iterator insert(const_iterator /* hint */, value_type&& value) {+    return insert(std::move(value)).first;+  }++  template <class InputIterator>+  void insert(InputIterator first, InputIterator last) {+    if (first == last) {+      return;+    }+    if (iterator_has_known_distance_v<InputIterator, InputIterator> &&+        std::distance(first, last) == 1) {+      insert(*first);+      return;+    }+    std::sort(m_.cont_.begin(), m_.cont_.end(), value_comp());+    heap_vector_detail::bulk_insert(*this, m_.cont_, first, last);+  }++  void insert(std::initializer_list<value_type> ilist) {+    insert(ilist.begin(), ilist.end());+  }+  // emplace isn't better than insert for heap_vector_container, but aids+  // compatibility+  template <typename... Args>+  std::pair<iterator, bool> emplace(Args&&... args) {+    std::aligned_storage_t<sizeof(value_type), alignof(value_type)> b;+    auto* p = static_cast<value_type*>(static_cast<void*>(&b));+    auto a = get_allocator();+    std::allocator_traits<allocator_type>::construct(+        a, p, std::forward<Args>(args)...);+    auto g = makeGuard([&]() {+      std::allocator_traits<allocator_type>::destroy(a, p);+    });+    return insert(std::move(*p));+  }++  std::pair<iterator, bool> emplace(const value_type& value) {+    return insert(value);+  }++  std::pair<iterator, bool> emplace(value_type&& value) {+    return insert(std::move(value));+  }++  // emplace_hint isn't better than insert for heap_vector_container, but aids+  // compatibility+  template <typename... Args>+  iterator emplace_hint(const_iterator /* hint */, Args&&... args) {+    return emplace(std::forward<Args>(args)...).first;+  }++  iterator emplace_hint(const_iterator hint, const value_type& value) {+    return insert(hint, value);+  }++  iterator emplace_hint(const_iterator hint, value_type&& value) {+    return insert(hint, std::move(value));+  }++  size_type erase(const key_type& key) {+    iterator it = find(key);+    if (it == end()) {+      return 0;+    }+    heap_vector_detail::erase(it.ptr_ - m_.cont_.begin(), m_.cont_);+    return 1;+  }++  iterator erase(const_iterator it) {+    auto offset =+        heap_vector_detail::erase(it.ptr_ - m_.cont_.begin(), m_.cont_);+    iterator ret = end();+    ret = m_.cont_.begin() + offset;+    return ret;+  }++  iterator erase(const_iterator first, const_iterator last) {+    if (first == last) {+      return end();+    }+    auto dist = last - first;+    if (dist <= 0) {+      return end();+    }+    if (dist == 1) {+      return erase(first);+    }+    auto it = m_.cont_.begin() + (first - begin());+    std::sort(m_.cont_.begin(), m_.cont_.end(), value_comp());+    it = m_.cont_.erase(it, it + dist);+    heap_vector_detail::heapify(m_.cont_);+    return begin() + (it - m_.cont_.begin());+  }++  iterator find(const key_type& key) { return find_(*this, key); }++  const_iterator find(const key_type& key) const { return find_(*this, key); }++  template <typename K>+  if_is_transparent<K, iterator> find(const K& key) {+    return find_(*this, key);+  }++  template <typename K>+  if_is_transparent<K, const_iterator> find(const K& key) const {+    return find_(*this, key);+  }++  size_type count(const key_type& key) const {+    return find(key) == end() ? 0 : 1;+  }++  template <typename K>+  if_is_transparent<K, size_type> count(const K& key) const {+    return find(key) == end() ? 0 : 1;+  }++  bool contains(const key_type& key) const { return find(key) != end(); }++  template <typename K>+  if_is_transparent<K, bool> contains(const K& key) const {+    return find(key) != end();+  }++  iterator lower_bound(const key_type& key) { return lower_bound(*this, key); }++  const_iterator lower_bound(const key_type& key) const {+    return lower_bound(*this, key);+  }++  template <typename K>+  if_is_transparent<K, iterator> lower_bound(const K& key) {+    return lower_bound(*this, key);+  }++  template <typename K>+  if_is_transparent<K, const_iterator> lower_bound(const K& key) const {+    return lower_bound(*this, key);+  }++  iterator upper_bound(const key_type& key) { return upper_bound(*this, key); }++  const_iterator upper_bound(const key_type& key) const {+    return upper_bound(*this, key);+  }++  template <typename K>+  if_is_transparent<K, iterator> upper_bound(const K& key) {+    return upper_bound(*this, key);+  }++  template <typename K>+  if_is_transparent<K, const_iterator> upper_bound(const K& key) const {+    return upper_bound(*this, key);+  }++  std::pair<iterator, iterator> equal_range(const key_type& key) {+    return {lower_bound(key), upper_bound(key)};+  }++  std::pair<const_iterator, const_iterator> equal_range(+      const key_type& key) const {+    return {lower_bound(key), upper_bound(key)};+  }++  template <typename K>+  if_is_transparent<K, std::pair<iterator, iterator>> equal_range(+      const K& key) {+    return {lower_bound(key), upper_bound(key)};+  }++  template <typename K>+  if_is_transparent<K, std::pair<const_iterator, const_iterator>> equal_range(+      const K& key) const {+    return {lower_bound(key), upper_bound(key)};+  }++  void swap(heap_vector_container& o) noexcept(+      std::is_nothrow_swappable<Compare>::value &&+      noexcept(std::declval<Container&>().swap(std::declval<Container&>()))) {+    using std::swap; // Allow ADL for swap(); fall back to std::swap().+    Compare& a = m_;+    Compare& b = o.m_;+    swap(a, b);+    m_.cont_.swap(o.m_.cont_);+  }++  bool operator==(const heap_vector_container& other) const {+    return m_.cont_ == other.m_.cont_;+  }++  bool operator!=(const heap_vector_container& other) const {+    return !operator==(other);+  }++  bool operator<(const heap_vector_container& other) const {+    return std::lexicographical_compare(+        begin(), end(), other.begin(), other.end(), value_comp());+  }+  bool operator>(const heap_vector_container& other) const {+    return other < *this;+  }+  bool operator<=(const heap_vector_container& other) const {+    return !operator>(other);+  }+  bool operator>=(const heap_vector_container& other) const {+    return !operator<(other);+  }++  // Use underlying vector iterators to quickly traverse heap container.+  // Note elements are traversed following the heap order, i.e., memory+  // storage order.+  Range<typename Container::iterator> iterate() noexcept {+    return Range<typename Container::iterator>(+        m_.cont_.begin(), m_.cont_.end());+  }++  const Range<typename Container::const_iterator> iterate() const noexcept {+    return Range<typename Container::const_iterator>(+        m_.cont_.begin(), m_.cont_.end());+  }++ protected:+  // This is to get the empty base optimization+  struct EBO : value_compare {+    explicit EBO(const value_compare& c, const Allocator& alloc) noexcept(+        std::is_nothrow_default_constructible<Container>::value)+        : value_compare(c), cont_(alloc) {}+    EBO(const EBO& other, const Allocator& alloc) noexcept(+        std::is_nothrow_constructible<+            Container,+            const Container&,+            const Allocator&>::value)+        : value_compare(static_cast<const value_compare&>(other)),+          cont_(other.cont_, alloc) {}+    EBO(EBO&& other, const Allocator& alloc) noexcept(+        std::is_nothrow_constructible<+            Container,+            Container&&,+            const Allocator&>::value)+        : value_compare(static_cast<value_compare&&>(other)),+          cont_(std::move(other.cont_), alloc) {}+    EBO(const Compare& c, Container&& cont) noexcept(+        std::is_nothrow_move_constructible<Container>::value)+        : value_compare(c), cont_(std::move(cont)) {}+    Container cont_;+  } m_;++  template <typename Self>+  using self_iterator_t = typename std::+      conditional<std::is_const<Self>::value, const_iterator, iterator>::type;++  template <typename Self, typename K>+  static self_iterator_t<Self> find_(Self& self, K const& key) {+    self_iterator_t<Self> end = self.end();+    self_iterator_t<Self> it = self.lower_bound(key);+    if (it == end || !self.key_comp()(key, self.m_.getKey(*it))) {+      return it;+    }+    return end;+  }++  template <typename Self, typename K>+  static self_iterator_t<Self> lower_bound(Self& self, K const& key) {+    auto c = self.key_comp();+    auto cmp = [&](auto const& a) { return c(self.m_.getKey(a), key); };+    auto reverseCmp = [&](auto const& a) { return c(key, self.m_.getKey(a)); };+    auto offset =+        heap_vector_detail::lower_bound(self.m_.cont_, cmp, reverseCmp);+    self_iterator_t<Self> ret = self.end();+    ret = self.m_.cont_.begin() + offset;+    return ret;+  }++  template <typename Self, typename K>+  static self_iterator_t<Self> upper_bound(Self& self, K const& key) {+    auto c = self.key_comp();+    auto cmp = [&](auto const& a) { return c(key, self.m_.getKey(a)); };+    auto offset = heap_vector_detail::upper_bound(self.m_.cont_, cmp);+    self_iterator_t<Self> ret = self.end();+    ret = self.m_.cont_.begin() + offset;+    return ret;+  }+};++} // namespace heap_vector_detail++} // namespace detail++/* heap_vector_set is a specialization of heap_vector_container+ *+ * @tparam T               Data type to store+ * @tparam Compare         Comparison function that imposes a+ *                              strict weak ordering over instances of T+ * @tparam Allocator       allocation policy+ * @tparam GrowthPolicy    policy object to control growth+ * @tparam Container       underlying vector where elements are stored+ */+template <+    class T,+    class Compare = std::less<T>,+    class Allocator = std::allocator<T>,+    class GrowthPolicy = void,+    class Container = std::vector<T, Allocator>>+class heap_vector_set+    : public detail::heap_vector_detail::heap_vector_container<+          T,+          Compare,+          Allocator,+          GrowthPolicy,+          Container,+          T,+          detail::heap_vector_detail::value_compare_set<Compare>> {+ private:+  using heap_vector_container =+      detail::heap_vector_detail::heap_vector_container<+          T,+          Compare,+          Allocator,+          GrowthPolicy,+          Container,+          T,+          detail::heap_vector_detail::value_compare_set<Compare>>;++ public:+  using heap_vector_container::heap_vector_container;+};++// Swap function that can be found using ADL.+template <class T, class C, class A, class G>+inline void swap(+    heap_vector_set<T, C, A, G>& a, heap_vector_set<T, C, A, G>& b) noexcept {+  return a.swap(b);+}++#if FOLLY_HAS_MEMORY_RESOURCE++namespace pmr {++template <+    class T,+    class Compare = std::less<T>,+    class GrowthPolicy = void,+    class Container = std::vector<T, std::pmr::polymorphic_allocator<T>>>+using heap_vector_set = folly::heap_vector_set<+    T,+    Compare,+    std::pmr::polymorphic_allocator<T>,+    GrowthPolicy,+    Container>;++} // namespace pmr++#endif++//////////////////////////////////////////////////////////////////////++/**+ * A heap_vector_map based on heap layout.+ *+ * @tparam Key           Key type+ * @tparam Value         Value type+ * @tparam Compare       Function that can compare key types and impose+ *                            a strict weak ordering over them.+ * @tparam Allocator     allocation policy+ * @tparam GrowthPolicy  policy object to control growth+ *+ */++template <+    class Key,+    class Value,+    class Compare = std::less<Key>,+    class Allocator = std::allocator<std::pair<Key, Value>>,+    class GrowthPolicy = void,+    class Container = std::vector<std::pair<Key, Value>, Allocator>>+class heap_vector_map+    : public detail::heap_vector_detail::heap_vector_container<+          typename Container::value_type,+          Compare,+          Allocator,+          GrowthPolicy,+          Container,+          Key,+          detail::heap_vector_detail::value_compare_map<Compare>> {+ public:+  using key_type = Key;+  using mapped_type = Value;+  using value_type = typename Container::value_type;+  using key_compare = Compare;+  using allocator_type = Allocator;+  using container_type = Container;+  using pointer = typename Container::pointer;+  using reference = typename Container::reference;+  using const_reference = typename Container::const_reference;+  using difference_type = typename Container::difference_type;+  using size_type = typename Container::size_type;+  using value_compare = detail::heap_vector_detail::value_compare_map<Compare>;++ protected:+  using heap_vector_container =+      detail::heap_vector_detail::heap_vector_container<+          value_type,+          key_compare,+          Allocator,+          GrowthPolicy,+          Container,+          key_type,+          value_compare>;+  using heap_vector_container::get_growth_policy;+  using heap_vector_container::m_;++ public:+  using iterator = typename heap_vector_container::iterator;+  using const_iterator = typename heap_vector_container::const_iterator;+  using reverse_iterator = std::reverse_iterator<iterator>;+  using const_reverse_iterator = std::reverse_iterator<const_iterator>;++  // Since heap_vector_container methods are publicly available through+  // inheritance, just expose method used within this class.+  using heap_vector_container::end;+  using heap_vector_container::find;+  using heap_vector_container::heap_vector_container;+  using heap_vector_container::key_comp;+  using heap_vector_container::lower_bound;++  mapped_type& at(const key_type& key) {+    iterator it = find(key);+    if (it != end()) {+      return it->second;+    }+    throw_exception<std::out_of_range>("heap_vector_map::at");+  }++  const mapped_type& at(const key_type& key) const {+    const_iterator it = find(key);+    if (it != end()) {+      return it->second;+    }+    throw_exception<std::out_of_range>("heap_vector_map::at");+  }++  mapped_type& operator[](const key_type& key) {+    iterator it = lower_bound(key);+    if (it == end() || key_comp()(key, it->first)) {+      auto offset = it.ptr_ - m_.cont_.begin();+      get_growth_policy().increase_capacity(*this, it);+      m_.cont_.emplace_back(key, mapped_type());+      offset = detail::heap_vector_detail::insert(offset, m_.cont_);+      it = m_.cont_.begin() + offset;+      return it->second;+    }+    return it->second;+  }+};++// Swap function that can be found using ADL.+template <class K, class V, class C, class A, class G>+inline void swap(+    heap_vector_map<K, V, C, A, G>& a,+    heap_vector_map<K, V, C, A, G>& b) noexcept {+  return a.swap(b);+}++#if FOLLY_HAS_MEMORY_RESOURCE++namespace pmr {++template <+    class Key,+    class Value,+    class Compare = std::less<Key>,+    class GrowthPolicy = void,+    class Container = std::vector<+        std::pair<Key, Value>,+        std::pmr::polymorphic_allocator<std::pair<Key, Value>>>>+using heap_vector_map = folly::heap_vector_map<+    Key,+    Value,+    Compare,+    std::pmr::polymorphic_allocator<std::pair<Key, Value>>,+    GrowthPolicy,+    Container>;++} // namespace pmr++#endif++// Specialize heap_vector_map to integral key type and std::less comparaison.+// small_heap_map achieve a very fast find for small map < 200 elements.+template <+    typename Key,+    typename Value,+    typename SizeType = uint32_t,+    class Container = folly::small_vector<+        std::pair<Key, Value>,+        0,+        folly::small_vector_policy::policy_size_type<SizeType>>,+    typename = std::enable_if_t<+        std::is_integral<Key>::value || std::is_enum<Key>::value>>+class small_heap_vector_map+    : public folly::heap_vector_map<+          Key,+          Value,+          std::less<Key>,+          typename Container::allocator_type,+          void,+          Container> {+ public:+  using key_type = Key;+  using mapped_type = Value;+  using value_type = typename Container::value_type;+  using key_compare = std::less<Key>;+  using allocator_type = typename Container::allocator_type;+  using container_type = Container;+  using pointer = typename Container::pointer;+  using reference = typename Container::reference;+  using const_reference = typename Container::const_reference;+  using difference_type = typename Container::difference_type;+  using size_type = typename Container::size_type;+  using value_compare =+      detail::heap_vector_detail::value_compare_map<std::less<Key>>;++ private:+  using heap_vector_map = folly::+      heap_vector_map<Key, Value, key_compare, allocator_type, void, Container>;++  using heap_vector_map::m_;++ public:+  using iterator = typename heap_vector_map::iterator;+  using const_iterator = typename heap_vector_map::const_iterator;+  using reverse_iterator = std::reverse_iterator<iterator>;+  using const_reverse_iterator = std::reverse_iterator<const_iterator>;++  using heap_vector_map::begin;+  using heap_vector_map::heap_vector_map;+  using heap_vector_map::size;+  iterator find(const key_type key) {+    auto offset = find_(*this, key);+    iterator ret = begin();+    ret = m_.cont_.begin() + offset;+    return ret;+  }++  const_iterator find(const key_type key) const {+    auto offset = find_(*this, key);+    const_iterator ret = begin();+    ret = m_.cont_.begin() + offset;+    return ret;+  }++ private:+  template <typename Self, typename K>+  static inline size_type find_(Self& self, K const key) {+    auto size = self.size();+    if (!size) {+      return 0;+    }+    auto& cont = self.m_.cont_;+    size_type offset = 1;+    auto cur_k = self.m_.getKey(cont[0]);+    for (int i = 0; i < 6; i++) {+      auto o = offset;+      offset = 2 * offset;+      if (cur_k <= key) {+        ++offset;+        if (cur_k == key) {+          return o - 1;+        }+      }+      if (offset > size) {+        return size;+      }+      cur_k = self.m_.getKey(cont[offset - 1]);+    }+    while (true) {+      auto lt = cur_k < key;+      if (cur_k == key) {+        return offset - 1;+      }+      offset = 2 * offset + lt;+      if (offset > size)+        return size;+      cur_k = self.m_.getKey(cont[offset - 1]);+    }+    return size;+  }+};++} // namespace folly
+ folly/folly/container/range_traits.h view
@@ -0,0 +1,81 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <type_traits>++#include <folly/Traits.h>+#include <folly/Utility.h>++#if defined(__cpp_lib_ranges)+#include <ranges>+#endif++namespace folly {++namespace detail {++// clang-format off+template <+    typename R,+    typename S = typename R::size_type,+    typename V = typename R::value_type,+    typename TD = decltype(FOLLY_DECLVAL(R&).data()),+    typename TCD = decltype(FOLLY_DECLVAL(R const&).data()),+    typename TCS = decltype(FOLLY_DECLVAL(R const&).size())>+using is_contiguous_range_fallback_impl_ = std::bool_constant<(true+    && (std::is_same_v<V*, TD> || std::is_same_v<V const*, TD>)+    && std::is_same_v<V const*, TCD>+    && std::is_same_v<S, TCS>)>;+// clang-format on++template <+    typename R,+    bool RUser = std::is_union_v<R> || std::is_class_v<R>>+constexpr bool is_contiguous_range_v_ = //+    !require_sizeof<conditional_t<RUser, R, int>>+#if defined(__cpp_lib_ranges)+    || std::ranges::contiguous_range<R>+#endif+    || is_bounded_array_v<R> //+    || detected_or_t<std::false_type, is_contiguous_range_fallback_impl_, R>{};++} // namespace detail++//  is_contiguous_range_v+//+//  True when any of:+//  * std::ranges::contiguous_range holds, if available+//  * is_bounded_array_v holds+//  * certain conditions using member types or type aliases size_type and+//    value_type and member functions size() and data()+//  Otherwise false.+//+//  Necessarily true if the given type is any of:+//  * T[S], ie a bounded array+//  * std::array+//  * std::basic_string+//  * std::basic_string_view+//  * std::span+//  * std::vector+//+//  Rejects incomplete class/union types, even if they would be accepted when+//  completed.+template <typename R>+constexpr bool is_contiguous_range_v = detail::is_contiguous_range_v_<R>;++} // namespace folly
+ folly/folly/container/small_vector.h view
@@ -0,0 +1,1530 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++/*+ * For high-level documentation and usage examples see+ * folly/docs/small_vector.md+ */++#pragma once++#include <algorithm>+#include <cassert>+#include <cstdlib>+#include <cstring>+#include <iterator>+#include <memory>+#include <stdexcept>+#include <type_traits>+#include <utility>++#include <boost/operators.hpp>++#include <folly/ConstexprMath.h>+#include <folly/FormatTraits.h>+#include <folly/Likely.h>+#include <folly/Portability.h>+#include <folly/ScopeGuard.h>+#include <folly/Traits.h>+#include <folly/functional/Invoke.h>+#include <folly/hash/Hash.h>+#include <folly/lang/Align.h>+#include <folly/lang/Assume.h>+#include <folly/lang/CheckedMath.h>+#include <folly/lang/Exception.h>+#include <folly/memory/Malloc.h>+#include <folly/memory/SanitizeLeak.h>+#include <folly/portability/Malloc.h>++#if (FOLLY_X64 || FOLLY_PPC64 || FOLLY_AARCH64 || FOLLY_RISCV64)+#define FOLLY_SV_PACK_ATTR FOLLY_PACK_ATTR+#define FOLLY_SV_PACK_PUSH FOLLY_PACK_PUSH+#define FOLLY_SV_PACK_POP FOLLY_PACK_POP+#else+#define FOLLY_SV_PACK_ATTR+#define FOLLY_SV_PACK_PUSH+#define FOLLY_SV_PACK_POP+#endif++// Ignore shadowing warnings within this file, so includers can use -Wshadow.+FOLLY_PUSH_WARNING+FOLLY_GNU_DISABLE_WARNING("-Wshadow")++namespace folly {++namespace small_vector_policy {++namespace detail {++struct item_size_type {+  template <typename T>+  using get = typename T::size_type;+  template <typename T>+  struct set {+    using size_type = T;+  };+};++struct item_in_situ_only {+  template <typename T>+  using get = typename T::in_situ_only;+  template <typename T>+  struct set {+    using in_situ_only = T;+  };+};++template <template <typename> class F, typename... T>+constexpr size_t last_matching_() {+  bool const values[] = {is_detected_v<F, T>..., false};+  for (size_t i = 0; i < sizeof...(T); ++i) {+    auto const j = sizeof...(T) - 1 - i;+    if (values[j]) {+      return j;+    }+  }+  return sizeof...(T);+}++template <size_t M, typename I, typename... P>+struct merge_ //+    : I::template set<typename I::template get<+          type_pack_element_t<sizeof...(P) - M, P...>>> {};+template <typename I, typename... P>+struct merge_<0, I, P...> {};+template <typename I, typename... P>+using merge =+    merge_<sizeof...(P) - last_matching_<I::template get, P...>(), I, P...>;++} // namespace detail++template <typename... Policy>+struct merge //+    : detail::merge<detail::item_size_type, Policy...>,+      detail::merge<detail::item_in_situ_only, Policy...> {};++template <typename SizeType>+struct policy_size_type {+  using size_type = SizeType;+};++template <bool Value>+struct policy_in_situ_only {+  using in_situ_only = std::bool_constant<Value>;+};++} // namespace small_vector_policy++//////////////////////////////////////////////////////////////////////++template <class T, std::size_t M, class P>+class small_vector;++//////////////////////////////////////////////////////////////////////++namespace detail {++namespace small_vector_detail {++/*+ * Move objects in memory to the right into some uninitialized memory, where+ * the region overlaps. Then call create() for each hole in reverse order.+ *+ * This doesn't just use std::move_backward because move_backward only works+ * if all the memory is initialized to type T already.+ *+ * The create function should return a reference type, to avoid+ * extra copies and moves for non-trivial types.+ */+template <class T, class Create>+typename std::enable_if<!std::is_trivially_copyable_v<T>>::type+moveObjectsRightAndCreate(+    T* const first,+    T* const lastConstructed,+    T* const realLast,+    Create&& create) {+  if (lastConstructed == realLast) {+    return;+  }++  T* out = realLast;+  T* in = lastConstructed;+  {+    auto rollback = makeGuard([&] {+      // We want to make sure the same stuff is uninitialized memory+      // if we exit via an exception (this is to make sure we provide+      // the basic exception safety guarantee for insert functions).+      if (out < lastConstructed) {+        out = lastConstructed - 1;+      }+      std::destroy(out + 1, realLast);+    });+    // Decrement the pointers only when it is known that the resulting pointer+    // is within the boundaries of the object. Decrementing past the beginning+    // of the object is UB. Note that this is asymmetric wrt forward iteration,+    // as past-the-end pointers are explicitly allowed.+    for (; in != first && out > lastConstructed;) {+      // Out must be decremented before an exception can be thrown so that+      // the rollback guard knows where to start.+      --out;+      new (out) T(std::move(*(--in)));+    }+    for (; in != first;) {+      --out;+      *out = std::move(*(--in));+    }+    for (; out > lastConstructed;) {+      --out;+      new (out) T(create());+    }+    for (; out != first;) {+      --out;+      *out = create();+    }+    rollback.dismiss();+  }+}++// Specialization for trivially copyable types.  The call to+// std::move_backward here will just turn into a memmove.+// This must only be used with trivially copyable types because some of the+// memory may be uninitialized, and std::move_backward() won't work when it+// can't memmove().+template <class T, class Create>+typename std::enable_if<std::is_trivially_copyable_v<T>>::type+moveObjectsRightAndCreate(+    T* const first,+    T* const lastConstructed,+    T* const realLast,+    Create&& create) {+  std::move_backward(first, lastConstructed, realLast);+  T* const end = first - 1;+  T* out = first + (realLast - lastConstructed) - 1;+  for (; out != end; --out) {+    *out = create();+  }+}++/*+ * Populate a region of memory using `op' to construct elements.  If+ * anything throws, undo what we did.+ */+template <class T, class Function>+void populateMemForward(T* mem, std::size_t n, Function const& op) {+  std::size_t idx = 0;+  {+    auto rollback = makeGuard([&] { std::destroy_n(mem, idx); });+    for (size_t i = 0; i < n; ++i) {+      op(&mem[idx]);+      ++idx;+    }+    rollback.dismiss();+  }+}++/*+ * Copies `fromSize` elements from `from' to `to', where `to' is only+ * initialized up to `toSize`, but has enough storage for `fromSize'. If+ * `toSize' > `fromSize', the extra elements are destructed.+ */+template <class Iterator1, class Iterator2>+void partiallyUninitializedCopy(+    Iterator1 from, size_t fromSize, Iterator2 to, size_t toSize) {+  const size_t minSize = std::min(fromSize, toSize);+  std::copy(from, from + minSize, to);+  if (fromSize > toSize) {+    std::uninitialized_copy(from + minSize, from + fromSize, to + minSize);+  } else {+    std::destroy(to + minSize, to + toSize);+  }+}++} // namespace small_vector_detail++template <class SizeType, bool ShouldUseHeap, bool AlwaysUseHeap>+struct IntegralSizePolicyBase {+  typedef SizeType InternalSizeType;++  IntegralSizePolicyBase() : size_(0) {}++ protected:+  static constexpr std::size_t policyMaxSize() { return SizeType(~kClearMask); }++  std::size_t doSize() const {+    return AlwaysUseHeap ? size_ : size_ & ~kClearMask;+  }++  std::size_t isExtern() const { return AlwaysUseHeap || kExternMask & size_; }++  void setExtern(bool b) {+    if (AlwaysUseHeap) {+      return;+    }+    if (b) {+      size_ |= kExternMask;+    } else {+      size_ &= ~kExternMask;+    }+  }++  std::size_t isHeapifiedCapacity() const {+    return AlwaysUseHeap || kCapacityMask & size_;+  }++  void setHeapifiedCapacity(bool b) {+    if (AlwaysUseHeap) {+      return;+    }+    if (b) {+      size_ |= kCapacityMask;+    } else {+      size_ &= ~kCapacityMask;+    }+  }+  void setSize(std::size_t sz) {+    assert(sz <= policyMaxSize());+    size_ = AlwaysUseHeap ? sz : (kClearMask & size_) | SizeType(sz);+  }++  void incrementSize(std::size_t n) {+    // We can safely increment size without overflowing into mask bits because+    // we always check new size is less than maxPolicySize (see+    // makeSizeInternal). To be sure, added assertion to verify it.+    assert(doSize() + n <= policyMaxSize());+    size_ += SizeType(n);+  }+  std::size_t getInternalSize() { return size_; }++  void swapSizePolicy(IntegralSizePolicyBase& o) { std::swap(size_, o.size_); }++  void resetSizePolicy() { size_ = 0; }++ protected:+  static bool constexpr kShouldUseHeap = ShouldUseHeap || AlwaysUseHeap;+  static bool constexpr kAlwaysUseHeap = AlwaysUseHeap;++ private:+  // We reserve two most significant bits of size_.+  static SizeType constexpr kExternMask =+      kShouldUseHeap ? SizeType(1) << (sizeof(SizeType) * 8 - 1) : 0;++  static SizeType constexpr kCapacityMask =+      kShouldUseHeap ? SizeType(1) << (sizeof(SizeType) * 8 - 2) : 0;++  static SizeType constexpr kClearMask =+      kShouldUseHeap ? SizeType(3) << (sizeof(SizeType) * 8 - 2) : 0;++  SizeType size_;+};++template <class SizeType, bool ShouldUseHeap, bool AlwaysUseHeap>+struct IntegralSizePolicy;++template <class SizeType, bool AlwaysUseHeap>+struct IntegralSizePolicy<SizeType, true, AlwaysUseHeap>+    : public IntegralSizePolicyBase<SizeType, true, AlwaysUseHeap> {+ public:+  /*+   * Move a range to a range of uninitialized memory.  Assumes the+   * ranges don't overlap.+   */+  template <class T>+  typename std::enable_if<!std::is_trivially_copyable_v<T>>::type+  moveToUninitialized(T* first, T* last, T* out) {+    std::size_t idx = 0;+    {+      auto rollback = makeGuard([&] {+        // Even for callers trying to give the strong guarantee+        // (e.g. push_back) it's ok to assume here that we don't have to+        // move things back and that it was a copy constructor that+        // threw: if someone throws from a move constructor the effects+        // are unspecified.+        std::destroy_n(out, idx);+      });+      for (; first != last; ++first, ++idx) {+        new (&out[idx]) T(std::move(*first));+      }+      rollback.dismiss();+    }+  }++  // Specialization for trivially copyable types.+  template <class T>+  typename std::enable_if<std::is_trivially_copyable_v<T>>::type+  moveToUninitialized(T* first, T* last, T* out) {+    std::memmove(+        static_cast<void*>(out),+        static_cast<void const*>(first),+        (last - first) * sizeof *first);+  }++  /*+   * Move a range to a range of uninitialized memory. Assumes the+   * ranges don't overlap. Inserts an element at out + pos using+   * emplaceFunc(). out will contain (end - begin) + 1 elements on success and+   * none on failure. If emplaceFunc() throws [begin, end) is unmodified.+   */+  template <class T, class EmplaceFunc>+  void moveToUninitializedEmplace(+      T* begin, T* end, T* out, SizeType pos, EmplaceFunc&& emplaceFunc) {+    // Must be called first so that if it throws [begin, end) is unmodified.+    // We have to support the strong exception guarantee for emplace_back().+    emplaceFunc(out + pos);+    // move old elements to the left of the new one+    FOLLY_PUSH_WARNING+    FOLLY_MSVC_DISABLE_WARNING(4702) {+      auto rollback = makeGuard([&] { //+        std::destroy_at(out + pos);+      });+      if (begin) {+        this->moveToUninitialized(begin, begin + pos, out);+      }+      rollback.dismiss();+    }+    // move old elements to the right of the new one+    {+      auto rollback = makeGuard([&] { std::destroy_n(out, pos + 1); });+      if (begin + pos < end) {+        this->moveToUninitialized(begin + pos, end, out + pos + 1);+      }+      rollback.dismiss();+    }+    FOLLY_POP_WARNING+  }+};++template <class SizeType, bool AlwaysUseHeap>+struct IntegralSizePolicy<SizeType, false, AlwaysUseHeap>+    : public IntegralSizePolicyBase<SizeType, false, AlwaysUseHeap> {+ public:+  template <class T>+  void moveToUninitialized(T* /*first*/, T* /*last*/, T* /*out*/) {+    assume_unreachable();+  }+  template <class T, class EmplaceFunc>+  void moveToUninitializedEmplace(+      T* /* begin */,+      T* /* end */,+      T* /* out */,+      SizeType /* pos */,+      EmplaceFunc&& /* emplaceFunc */) {+    assume_unreachable();+  }+};++/*+ * If you're just trying to use this class, ignore everything about+ * this next small_vector_base class thing.+ *+ * The purpose of this junk is to minimize sizeof(small_vector<>)+ * and allow specifying the template parameters in whatever order is+ * convenient for the user.  There's a few extra steps here to try+ * to keep the error messages at least semi-reasonable.+ *+ * Apologies for all the black magic.+ */+template <class Value, std::size_t RequestedMaxInline, class InPolicy>+struct small_vector_base {+  static_assert(!std::is_integral<InPolicy>::value, "legacy");+  using Policy = small_vector_policy::merge<+      small_vector_policy::policy_size_type<size_t>,+      small_vector_policy::policy_in_situ_only<false>,+      conditional_t<std::is_void<InPolicy>::value, tag_t<>, InPolicy>>;++  /*+   * Make the real policy base classes.+   */+  typedef IntegralSizePolicy<+      typename Policy::size_type,+      !Policy::in_situ_only::value,+      RequestedMaxInline == 0>+      ActualSizePolicy;++  /*+   * Now inherit from them all.  This is done in such a convoluted+   * way to make sure we get the empty base optimization on all these+   * types to keep sizeof(small_vector<>) minimal.+   */+  typedef boost::totally_ordered1<+      small_vector<Value, RequestedMaxInline, InPolicy>,+      ActualSizePolicy>+      type;+};++namespace small_vector_detail {++inline void* unshiftPointer(void* p, size_t sizeBytes) {+  return static_cast<char*>(p) - sizeBytes;+}++} // namespace small_vector_detail++namespace small_vector_detail {++inline void* shiftPointer(void* p, size_t sizeBytes) {+  return static_cast<char*>(p) + sizeBytes;+}++} // namespace small_vector_detail++// No backward compatibility using declarations needed+} // namespace detail++//////////////////////////////////////////////////////////////////////+template <class Value, std::size_t RequestedMaxInline = 1, class Policy = void>+class small_vector+    : public detail::small_vector_base<Value, RequestedMaxInline, Policy>::+          type {+  typedef typename detail::+      small_vector_base<Value, RequestedMaxInline, Policy>::type BaseType;+  typedef typename BaseType::InternalSizeType InternalSizeType;++  /*+   * Figure out the max number of elements we should inline.  (If+   * the user asks for less inlined elements than we can fit unioned+   * into our value_type*, we will inline more than they asked.)+   */+  static constexpr auto kSizeOfValuePtr = sizeof(Value*);+  static constexpr auto kSizeOfValue = sizeof(Value);+  static constexpr std::size_t MaxInline{+      RequestedMaxInline == 0+          ? 0+          : constexpr_max(kSizeOfValuePtr / kSizeOfValue, RequestedMaxInline)};++ public:+  typedef std::size_t size_type;+  typedef Value value_type;+  typedef std::allocator<Value> allocator_type;+  typedef value_type& reference;+  typedef value_type const& const_reference;+  typedef value_type* iterator;+  typedef value_type* pointer;+  typedef value_type const* const_iterator;+  typedef value_type const* const_pointer;+  typedef std::ptrdiff_t difference_type;++  typedef std::reverse_iterator<iterator> reverse_iterator;+  typedef std::reverse_iterator<const_iterator> const_reverse_iterator;++  small_vector() = default;+  // Allocator is unused here. It is taken in for compatibility with std::vector+  // interface, but it will be ignored.+  small_vector(const std::allocator<Value>&) {}++  small_vector(small_vector const& o) {+    if constexpr (kShouldCopyWholeInlineStorageTrivial) {+      if (!o.isExtern()) {+        copyWholeInlineStorageTrivial(o);+        return;+      }+    }++    auto n = o.size();+    makeSize(n);+    {+      auto rollback = makeGuard([&] { freeHeap(); });+      std::uninitialized_copy(o.begin(), o.begin() + n, begin());+      rollback.dismiss();+    }+    this->setSize(n);+  }++  small_vector(small_vector&& o) noexcept(+      std::is_nothrow_move_constructible<Value>::value) {+    if (o.isExtern()) {+      this->u.pdata_.heap_ = o.u.pdata_.heap_;+      o.u.pdata_.heap_ = nullptr;+      this->swapSizePolicy(o);+      if (kHasInlineCapacity) {+        this->u.setCapacity(o.u.getCapacity());+      }+    } else {+      if constexpr (kShouldCopyWholeInlineStorageTrivial) {+        copyWholeInlineStorageTrivial(o);+        o.resetSizePolicy();+      } else if constexpr (IsRelocatable<Value>::value) {+        moveInlineStorageRelocatable(std::move(o));+      } else {+        auto n = o.size();+        std::uninitialized_copy(+            std::make_move_iterator(o.begin()),+            std::make_move_iterator(o.end()),+            begin());+        this->setSize(n);+        o.clear();+      }+    }+  }++  small_vector(std::initializer_list<value_type> il) {+    constructImpl(il.begin(), il.end(), std::false_type());+  }++  explicit small_vector(size_type n) {+    FOLLY_PUSH_WARNING+    FOLLY_GCC_DISABLE_WARNING("-Warray-bounds")+    doConstruct(n, [&](void* p) { new (p) value_type(); });+    FOLLY_POP_WARNING+  }++  small_vector(size_type n, value_type const& t) {+    FOLLY_PUSH_WARNING+    FOLLY_GCC_DISABLE_WARNING("-Warray-bounds")+    doConstruct(n, [&](void* p) { new (p) value_type(t); });+    FOLLY_POP_WARNING+  }++  template <class Arg>+  explicit small_vector(Arg arg1, Arg arg2) {+    // Forward using std::is_arithmetic to get to the proper+    // implementation; this disambiguates between the iterators and+    // (size_t, value_type) meaning for this constructor.+    constructImpl(arg1, arg2, std::is_arithmetic<Arg>());+  }++  ~small_vector() { destroy(); }++  small_vector& operator=(small_vector const& o) {+    if (FOLLY_LIKELY(this != &o)) {+      if constexpr (kShouldCopyWholeInlineStorageTrivial) {+        if (!this->isExtern() && !o.isExtern()) {+          copyWholeInlineStorageTrivial(o);+          return *this;+        }+      }+      if (o.size() < capacity()) {+        const size_t oSize = o.size();+        detail::small_vector_detail::partiallyUninitializedCopy(+            o.begin(), oSize, begin(), size());+        this->setSize(oSize);+      } else {+        assign(o.begin(), o.end());+      }+    }+    return *this;+  }++  small_vector& operator=(small_vector&& o) noexcept(+      std::is_nothrow_move_constructible<Value>::value) {+    if (FOLLY_LIKELY(this != &o)) {+      // If either is external, reduce to the default-constructed case for this,+      // since there is nothing that we can move in-place.+      if (this->isExtern() || o.isExtern()) {+        reset();+      }++      if (!o.isExtern()) {+        if constexpr (kShouldCopyWholeInlineStorageTrivial) {+          copyWholeInlineStorageTrivial(o);+          o.resetSizePolicy();+        } else if constexpr (IsRelocatable<Value>::value) {+          std::destroy_n(u.buffer(), size());+          moveInlineStorageRelocatable(std::move(o));+        } else {+          const size_t oSize = o.size();+          detail::small_vector_detail::partiallyUninitializedCopy(+              std::make_move_iterator(o.u.buffer()),+              oSize,+              this->u.buffer(),+              size());+          this->setSize(oSize);+          o.clear();+        }+      } else {+        this->u.pdata_.heap_ = o.u.pdata_.heap_;+        o.u.pdata_.heap_ = nullptr;+        // this was already reset above, so it's empty and internal.+        this->swapSizePolicy(o);+        if (kHasInlineCapacity) {+          this->u.setCapacity(o.u.getCapacity());+        }+      }+    }+    return *this;+  }++  bool operator==(small_vector const& o) const {+    return size() == o.size() && std::equal(begin(), end(), o.begin());+  }++  bool operator<(small_vector const& o) const {+    return std::lexicographical_compare(begin(), end(), o.begin(), o.end());+  }++#if FOLLY_CPLUSPLUS >= 202002L && defined(__cpp_lib_three_way_comparison)+  template <typename U = value_type>+  friend auto operator<=>(const small_vector& lhs, const small_vector& rhs)+      -> decltype(std::declval<const U&>() <=> std::declval<const U&>()) {+    return std::lexicographical_compare_three_way(+        lhs.begin(), lhs.end(), rhs.begin(), rhs.end());+  }+#endif // FOLLY_CPLUSPLUS >= 202002L && defined(__cpp_lib_three_way_comparison)++  static constexpr size_type max_size() {+    return !BaseType::kShouldUseHeap+        ? static_cast<size_type>(MaxInline)+        : BaseType::policyMaxSize();+  }++  allocator_type get_allocator() const { return {}; }++  size_type size() const { return this->doSize(); }+  bool empty() const { return !size(); }++  iterator begin() { return data(); }+  iterator end() { return data() + size(); }+  const_iterator begin() const { return data(); }+  const_iterator end() const { return data() + size(); }+  const_iterator cbegin() const { return begin(); }+  const_iterator cend() const { return end(); }++  reverse_iterator rbegin() { return reverse_iterator(end()); }+  reverse_iterator rend() { return reverse_iterator(begin()); }++  const_reverse_iterator rbegin() const {+    return const_reverse_iterator(end());+  }++  const_reverse_iterator rend() const {+    return const_reverse_iterator(begin());+  }++  const_reverse_iterator crbegin() const { return rbegin(); }+  const_reverse_iterator crend() const { return rend(); }++  /*+   * Usually one of the simplest functions in a Container-like class+   * but a bit more complex here.  We have to handle all combinations+   * of in-place vs. heap between this and o.+   */+  void swap(small_vector& o) noexcept(+      std::is_nothrow_move_constructible<Value>::value &&+      std::is_nothrow_swappable_v<Value>) {+    using std::swap; // Allow ADL on swap for our value_type.++    if (this->isExtern() && o.isExtern()) {+      this->swapSizePolicy(o);++      // Cannot use std::swap() because pdata_ is packed.+      auto* tmp = u.pdata_.heap_;+      u.pdata_.heap_ = o.u.pdata_.heap_;+      o.u.pdata_.heap_ = tmp;++      if (kHasInlineCapacity) {+        const auto currentCapacity = this->u.getCapacity();+        this->setCapacity(o.u.getCapacity());+        o.u.setCapacity(currentCapacity);+      }++      return;+    }++    if (!this->isExtern() && !o.isExtern()) {+      auto& oldSmall = size() < o.size() ? *this : o;+      auto& oldLarge = size() < o.size() ? o : *this;++      for (size_type i = 0; i < oldSmall.size(); ++i) {+        swap(oldSmall[i], oldLarge[i]);+      }++      size_type i = oldSmall.size();+      const size_type ci = i;+      {+        auto rollback = makeGuard([&] {+          oldSmall.setSize(i);+          std::destroy(oldLarge.begin() + i, oldLarge.end());+          oldLarge.setSize(ci);+        });+        for (; i < oldLarge.size(); ++i) {+          auto addr = oldSmall.begin() + i;+          new (addr) value_type(std::move(oldLarge[i]));+          std::destroy_at(oldLarge.data() + i);+        }+        rollback.dismiss();+      }+      oldSmall.setSize(i);+      oldLarge.setSize(ci);+      return;+    }++    // isExtern != o.isExtern()+    auto& oldExtern = o.isExtern() ? o : *this;+    auto& oldIntern = o.isExtern() ? *this : o;++    auto oldExternCapacity = oldExtern.capacity();+    auto oldExternHeap = oldExtern.u.pdata_.heap_;++    auto buff = oldExtern.u.buffer();+    size_type i = 0;+    {+      auto rollback = makeGuard([&] {+        std::destroy_n(buff, i);+        std::destroy(oldIntern.begin() + i, oldIntern.end());+        oldIntern.resetSizePolicy();+        oldExtern.u.pdata_.heap_ = oldExternHeap;+        oldExtern.setCapacity(oldExternCapacity);+      });+      for (; i < oldIntern.size(); ++i) {+        new (&buff[i]) value_type(std::move(oldIntern[i]));+        std::destroy_at(oldIntern.data() + i);+      }+      rollback.dismiss();+    }+    oldIntern.u.pdata_.heap_ = oldExternHeap;+    this->swapSizePolicy(o);+    oldIntern.setCapacity(oldExternCapacity);+  }++  void resize(size_type sz) {+    if (sz <= size()) {+      downsize(sz);+      return;+    }+    auto extra = sz - size();+    makeSize(sz);+    detail::small_vector_detail::populateMemForward(+        begin() + size(), extra, [&](void* p) { new (p) value_type(); });+    this->incrementSize(extra);+  }++  void resize(size_type sz, value_type const& v) {+    if (sz < size()) {+      FOLLY_PUSH_WARNING+      FOLLY_GCC_DISABLE_WARNING("-Warray-bounds")+      erase(begin() + sz, end());+      FOLLY_POP_WARNING+      return;+    }+    auto extra = sz - size();+    makeSize(sz);+    detail::small_vector_detail::populateMemForward(+        begin() + size(), extra, [&](void* p) { new (p) value_type(v); });+    this->incrementSize(extra);+  }++  value_type* data() noexcept {+    return this->isExtern() ? u.heap() : u.buffer();+  }++  value_type const* data() const noexcept {+    return this->isExtern() ? u.heap() : u.buffer();+  }++  template <class... Args>+  iterator emplace(const_iterator p, Args&&... args) {+    if (p == cend()) {+      emplace_back(std::forward<Args>(args)...);+      return end() - 1;+    }++    /*+     * We implement emplace at places other than at the back with a+     * temporary for exception safety reasons.  It is possible to+     * avoid having to do this, but it becomes hard to maintain the+     * basic exception safety guarantee (unless you respond to a copy+     * constructor throwing by clearing the whole vector).+     *+     * The reason for this is that otherwise you have to destruct an+     * element before constructing this one in its place---if the+     * constructor throws, you either need a nothrow default+     * constructor or a nothrow copy/move to get something back in the+     * "gap", and the vector requirements don't guarantee we have any+     * of these.  Clearing the whole vector is a legal response in+     * this situation, but it seems like this implementation is easy+     * enough and probably better.+     */+    return insert(p, value_type(std::forward<Args>(args)...));+  }++  void reserve(size_type sz) { makeSize(sz); }++  size_type capacity() const {+    struct Unreachable {+      size_t operator()(void*) const { assume_unreachable(); }+    };+    using AllocationSizeOrUnreachable =+        conditional_t<kMustTrackHeapifiedCapacity, Unreachable, AllocationSize>;+    if (this->isExtern()) {+      if (hasCapacity()) {+        return u.getCapacity();+      }+      return AllocationSizeOrUnreachable{}(u.pdata_.heap_) / sizeof(value_type);+    }+    return MaxInline;+  }++  void shrink_to_fit() {+    if (!this->isExtern()) {+      return;+    }++    small_vector tmp(begin(), end());+    tmp.swap(*this);+  }++  template <class... Args>+  reference emplace_back(Args&&... args) {+    auto isize_ = this->getInternalSize();+    if (isize_ < MaxInline) {+      new (u.buffer() + isize_) value_type(std::forward<Args>(args)...);+      this->incrementSize(1);+      return *(u.buffer() + isize_);+    }+    if (!BaseType::kShouldUseHeap) {+      throw_exception<std::length_error>("max_size exceeded in small_vector");+    }+    auto currentSize = size();+    auto currentCapacity = capacity();+    if (currentCapacity == currentSize) {+      // Any of args may be references into the vector.+      // When we are reallocating, we have to be careful to construct the new+      // element before modifying the data in the old buffer.+      makeSize(+          currentSize + 1,+          [&](void* p) { new (p) value_type(std::forward<Args>(args)...); },+          currentSize);+    } else {+      // We know the vector is stored in the heap.+      new (u.heap() + currentSize) value_type(std::forward<Args>(args)...);+    }+    this->incrementSize(1);+    return *(u.heap() + currentSize);+  }++  void push_back(value_type&& t) { emplace_back(std::move(t)); }++  void push_back(value_type const& t) { emplace_back(t); }++  void pop_back() {+    // ideally this would be implemented in terms of erase(end() - 1) to reuse+    // the higher-level abstraction, but neither Clang or GCC are able to+    // optimize it away. if you change this, please verify (with disassembly)+    // that the generated code on -O3 (and ideally -O2) stays short+    downsize(size() - 1);+  }++  iterator insert(const_iterator constp, value_type&& t) {+    iterator p = unconst(constp);+    if (p == end()) {+      push_back(std::move(t));+      return end() - 1;+    }++    auto offset = p - begin();+    auto currentSize = size();+    if (capacity() == currentSize) {+      makeSize(+          currentSize + 1,+          [&t](void* ptr) { new (ptr) value_type(std::move(t)); },+          offset);+      this->incrementSize(1);+    } else {+      detail::small_vector_detail::moveObjectsRightAndCreate(+          data() + offset,+          data() + currentSize,+          data() + currentSize + 1,+          [&]() mutable -> value_type&& { return std::move(t); });+      this->incrementSize(1);+    }+    return begin() + offset;+  }++  iterator insert(const_iterator p, value_type const& t) {+    // Make a copy and forward to the rvalue value_type&& overload+    // above.+    //+    // std::move() is necessary to avoid an MSVC compiler bug which will+    // issue this warning when used with unsigned int:+    // warning C4717 : 'folly::small_vector<unsigned int,192,void>::insert':+    // recursive on all control paths, function will cause runtime stack+    // overflow+    return insert(p, std::move(value_type(t)));+  }++  iterator insert(const_iterator pos, size_type n, value_type const& val) {+    auto offset = pos - begin();+    if (n != 0) {+      auto currentSize = size();+      makeSize(currentSize + n);+      detail::small_vector_detail::moveObjectsRightAndCreate(+          data() + offset,+          data() + currentSize,+          data() + currentSize + n,+          [&]() mutable -> value_type const& { return val; });+      this->incrementSize(n);+    }+    return begin() + offset;+  }++  template <class Arg>+  iterator insert(const_iterator p, Arg arg1, Arg arg2) {+    // Forward using std::is_arithmetic to get to the proper+    // implementation; this disambiguates between the iterators and+    // (size_t, value_type) meaning for this function.+    return insertImpl(unconst(p), arg1, arg2, std::is_arithmetic<Arg>());+  }++  iterator insert(const_iterator p, std::initializer_list<value_type> il) {+    return insert(p, il.begin(), il.end());+  }++  iterator erase(const_iterator q) {+    // ideally this would be implemented in terms of erase(q, q + 1) to reuse+    // the higher-level abstraction, but neither Clang or GCC are able to+    // optimize it away. if you change this, please verify (with disassembly)+    // that the generated code on -O3 (and ideally -O2) stays short+    std::move(unconst(q) + 1, end(), unconst(q));+    downsize(size() - 1);+    return unconst(q);+  }++  iterator erase(const_iterator q1, const_iterator q2) {+    if (q1 == q2) {+      return unconst(q1);+    }+    std::move(unconst(q2), end(), unconst(q1));+    downsize(size() - std::distance(q1, q2));+    return unconst(q1);+  }++  void clear() {+    // ideally this would be implemented in terms of erase(begin(), end()) to+    // reuse the higher-level abstraction, but neither Clang or GCC are able to+    // optimize it away. if you change this, please verify (with disassembly)+    // that the generated code on -O3 (and ideally -O2) stays short+    downsize(0);+  }++  template <class Arg>+  void assign(Arg first, Arg last) {+    clear();+    insert(end(), first, last);+  }++  void assign(std::initializer_list<value_type> il) {+    assign(il.begin(), il.end());+  }++  void assign(size_type n, const value_type& t) {+    clear();+    insert(end(), n, t);+  }++  reference front() {+    assert(!empty());+    return *begin();+  }+  reference back() {+    assert(!empty());+    return *(end() - 1);+  }+  const_reference front() const {+    assert(!empty());+    return *begin();+  }+  const_reference back() const {+    assert(!empty());+    return *(end() - 1);+  }++  reference operator[](size_type i) {+    assert(i < size());+    return *(begin() + i);+  }++  const_reference operator[](size_type i) const {+    assert(i < size());+    return *(begin() + i);+  }++  reference at(size_type i) {+    if (i >= size()) {+      throw_exception<std::out_of_range>("index out of range");+    }+    return (*this)[i];+  }++  const_reference at(size_type i) const {+    if (i >= size()) {+      throw_exception<std::out_of_range>("index out of range");+    }+    return (*this)[i];+  }++ private:+  static iterator unconst(const_iterator it) {+    return const_cast<iterator>(it);+  }++  void downsize(size_type sz) {+    assert(sz <= size());+    std::destroy(begin() + sz, end());+    this->setSize(sz);+  }++  void copyWholeInlineStorageTrivial(small_vector const& o) {+    static_assert(std::is_trivially_copyable_v<Value>);+    FOLLY_PUSH_WARNING+    FOLLY_GCC_DISABLE_WARNING("-Warray-bounds")+    std::copy(o.u.buffer(), o.u.buffer() + MaxInline, u.buffer());+    FOLLY_POP_WARNING+    this->setSize(o.size());+  }++  void moveInlineStorageRelocatable(small_vector&& o) {+    static_assert(IsRelocatable<Value>::value);+    const auto n = o.size();+    FOLLY_PUSH_WARNING+    FOLLY_GCC_DISABLE_WARNING("-Wclass-memaccess")+    if constexpr (kMayCopyWholeInlineStorage) {+      std::memcpy(u.buffer(), o.u.buffer(), MaxInline * kSizeOfValue);+    } else {+      std::memcpy(u.buffer(), o.u.buffer(), n * kSizeOfValue);+    }+    FOLLY_POP_WARNING+    this->setSize(n);+    o.resetSizePolicy();+  }++  void reset() {+    clear();+    freeHeap();+    this->resetSizePolicy();+  }++  // The std::false_type argument is part of disambiguating the+  // iterator insert functions from integral types (see insert().)+  template <class It>+  iterator insertImpl(iterator pos, It first, It last, std::false_type) {+    if (first == last) {+      return pos;+    }+    using categ = typename std::iterator_traits<It>::iterator_category;+    using it_ref = typename std::iterator_traits<It>::reference;+    if (std::is_same<categ, std::input_iterator_tag>::value) {+      auto offset = pos - begin();+      while (first != last) {+        pos = insert(pos, *first++);+        ++pos;+      }+      return begin() + offset;+    }++    auto const distance = std::distance(first, last);+    auto const offset = pos - begin();+    auto currentSize = size();+    assert(distance >= 0);+    assert(offset >= 0);+    makeSize(currentSize + distance);+    detail::small_vector_detail::moveObjectsRightAndCreate(+        data() + offset,+        data() + currentSize,+        data() + currentSize + distance,+        [&, in = last]() mutable -> it_ref { return *--in; });+    this->incrementSize(distance);+    return begin() + offset;+  }++  iterator insertImpl(+      iterator pos, size_type n, const value_type& val, std::true_type) {+    // The true_type means this should call the size_t,value_type+    // overload.  (See insert().)+    return insert(pos, n, val);+  }++  void destroy() {+    std::destroy(begin(), end());+    freeHeap();+  }++  // The std::false_type argument came from std::is_arithmetic as part+  // of disambiguating an overload (see the comment in the+  // constructor).+  template <class It>+  void constructImpl(It first, It last, std::false_type) {+    typedef typename std::iterator_traits<It>::iterator_category categ;+    if (std::is_same<categ, std::input_iterator_tag>::value) {+      // With iterators that only allow a single pass, we can't really+      // do anything sane here.+      auto rollback = makeGuard([&] { destroy(); });+      while (first != last) {+        emplace_back(*first++);+      }+      rollback.dismiss();+      return;+    }+    size_type distance = std::distance(first, last);+    if (distance <= MaxInline) {+      this->incrementSize(distance);+      detail::small_vector_detail::populateMemForward(+          u.buffer(), distance, [&](void* p) { new (p) value_type(*first++); });+      return;+    }+    makeSize(distance);+    this->incrementSize(distance);+    {+      auto rollback = makeGuard([&] { freeHeap(); });+      detail::small_vector_detail::populateMemForward(+          u.heap(), distance, [&](void* p) { new (p) value_type(*first++); });+      rollback.dismiss();+    }+  }++  template <typename InitFunc>+  void doConstruct(size_type n, InitFunc&& func) {+    makeSize(n);+    assert(size() == 0);+    this->incrementSize(n);+    {+      auto rollback = makeGuard([&] { freeHeap(); });+      detail::small_vector_detail::populateMemForward(+          data(), n, std::forward<InitFunc>(func));+      rollback.dismiss();+    }+  }++  // The true_type means we should forward to the size_t,value_type+  // overload.+  void constructImpl(size_type n, value_type const& val, std::true_type) {+    FOLLY_PUSH_WARNING+    FOLLY_GCC_DISABLE_WARNING("-Warray-bounds")+    doConstruct(n, [&](void* p) { new (p) value_type(val); });+    FOLLY_POP_WARNING+  }++  /*+   * Compute the size after growth.+   */+  size_type computeNewSize() const {+    size_t c = capacity();+    if (!checked_mul(&c, c, size_t(3))) {+      throw_exception<std::length_error>(+          "Requested new size exceeds size representable by size_type");+    }+    c = (c / 2) + 1;+    return static_cast<size_type>(std::min<size_t>(c, max_size()));+  }++  void makeSize(size_type newSize) {+    if (newSize <= capacity()) {+      return;+    }+    makeSizeInternal(newSize, false, [](void*) { assume_unreachable(); }, 0);+  }++  template <typename EmplaceFunc>+  void makeSize(size_type newSize, EmplaceFunc&& emplaceFunc, size_type pos) {+    assert(size() == capacity());+    makeSizeInternal(+        newSize, true, std::forward<EmplaceFunc>(emplaceFunc), pos);+  }++  /*+   * Ensure we have a large enough memory region to be size `newSize'.+   * Will move/copy elements if we are spilling to heap_ or needed to+   * allocate a new region, but if resized in place doesn't initialize+   * anything in the new region.  In any case doesn't change size().+   * Supports insertion of new element during reallocation by given+   * pointer to new element and position of new element.+   * NOTE: If reallocation is not needed, insert must be false,+   * because we only know how to emplace elements into new memory.+   */+  template <typename EmplaceFunc>+  void makeSizeInternal(+      size_type newSize,+      bool insert,+      EmplaceFunc&& emplaceFunc,+      size_type pos) {+    if (newSize > max_size()) {+      throw_exception<std::length_error>("max_size exceeded in small_vector");+    }+    assert(this->kShouldUseHeap);+    // This branch isn't needed for correctness, but allows the optimizer to+    // skip generating code for the rest of this function in in-situ-only+    // small_vectors.+    if (!this->kShouldUseHeap) {+      return;+    }++    newSize = std::max(newSize, computeNewSize());++    size_t needBytes = newSize;+    if (!checked_mul(&needBytes, needBytes, sizeof(value_type))) {+      throw_exception<std::length_error>(+          "Requested new size exceeds size representable by size_type");+    }+    // If the capacity isn't explicitly stored inline, but the heap+    // allocation is grown to over some threshold, we should store+    // a capacity at the front of the heap allocation.+    const bool heapifyCapacity =+        !kHasInlineCapacity && needBytes >= kHeapifyCapacityThreshold;+    const size_t allocationExtraBytes =+        heapifyCapacity ? kHeapifyCapacitySize : 0;+    size_t needAllocSizeBytes = needBytes;+    if (!checked_add(+            &needAllocSizeBytes, needAllocSizeBytes, allocationExtraBytes)) {+      throw_exception<std::length_error>(+          "Requested new size exceeds size representable by size_type");+    }+    const size_t goodAllocationSizeBytes = goodMallocSize(needAllocSizeBytes);+    const size_t goodAllocationNewCapacity =+        (goodAllocationSizeBytes - allocationExtraBytes) / sizeof(value_type);+    const size_t newCapacity = std::min(goodAllocationNewCapacity, max_size());+    // Make sure that the allocation request has a size computable from the+    // capacity, instead of using goodAllocationSizeBytes, so that we can do+    // sized deallocation. If goodMallocSize() gives us extra bytes that are not+    // a multiple of the value size we cannot use them anyway.+    const size_t sizeBytes =+        newCapacity * sizeof(value_type) + allocationExtraBytes;+    void* newh = checkedMalloc(sizeBytes);+    value_type* newp = static_cast<value_type*>(+        heapifyCapacity+            ? detail::small_vector_detail::shiftPointer(+                  newh, kHeapifyCapacitySize)+            : newh);++    {+      auto rollback = makeGuard([&] { //+        sizedFree(newh, sizeBytes);+      });+      if (insert) {+        // move and insert the new element+        this->moveToUninitializedEmplace(+            begin(), end(), newp, pos, std::forward<EmplaceFunc>(emplaceFunc));+      } else {+        // move without inserting new element+        if (data()) {+          this->moveToUninitialized(begin(), end(), newp);+        }+      }+      rollback.dismiss();+    }+    annotate_object_leaked(newh);+    std::destroy(begin(), end());+    freeHeap();+    // Store shifted pointer if capacity is heapified+    u.pdata_.heap_ = newp;+    this->setHeapifiedCapacity(heapifyCapacity);+    this->setExtern(true);+    this->setCapacity(newCapacity);+  }++  /*+   * This will set the capacity field, stored inline in the storage_ field+   * if there is sufficient room to store it.+   */+  void setCapacity(size_type newCapacity) {+    assert(this->isExtern());+    if (hasCapacity()) {+      assert(newCapacity < std::numeric_limits<InternalSizeType>::max());+      u.setCapacity(newCapacity);+    }+  }++ private:+  // These internal classes are packed to minimize total memory usage,+  // however, it is important that we don't pack the class as a whole+  // otherwise the inline storage may not have the correct alignment+  // for the value type.+  FOLLY_SV_PACK_PUSH+  struct HeapPtrWithCapacity {+    value_type* heap_;+    InternalSizeType capacity_;++    InternalSizeType getCapacity() const { return capacity_; }+    void setCapacity(InternalSizeType c) { capacity_ = c; }+    size_t allocationExtraBytes() const { return 0; }+  } FOLLY_SV_PACK_ATTR;+  FOLLY_SV_PACK_POP++  FOLLY_SV_PACK_PUSH+  struct HeapPtr {+    // heap[-kHeapifyCapacitySize] contains capacity+    value_type* heap_;++    InternalSizeType getCapacity() const {+      return heap_+          ? *static_cast<InternalSizeType*>(+                detail::small_vector_detail::unshiftPointer(+                    heap_, kHeapifyCapacitySize))+          : 0;+    }+    void setCapacity(InternalSizeType c) {+      *static_cast<InternalSizeType*>(+          detail::small_vector_detail::unshiftPointer(+              heap_, kHeapifyCapacitySize)) = c;+    }+    size_t allocationExtraBytes() const { return kHeapifyCapacitySize; }+  } FOLLY_SV_PACK_ATTR;+  FOLLY_SV_PACK_POP++  static constexpr size_t kMaxInlineNonZero = MaxInline ? MaxInline : 1u;+  typedef aligned_storage_for_t<value_type[kMaxInlineNonZero]>+      InlineStorageDataType;++  typedef typename std::conditional<+      sizeof(value_type) * MaxInline != 0,+      InlineStorageDataType,+      char>::type InlineStorageType;++  // If the storage is small enough, it is usually faster to copy it entirely,+  // instead of just size() values, to make the loop fixed-size and+  // unrollable. Limit is half of a cache line, to minimize probability of+  // crossing a cache line and thus introducing an unnecessary cache miss.+  static constexpr bool kMayCopyWholeInlineStorage =+      sizeof(InlineStorageType) <= hardware_constructive_interference_size / 2;++  static constexpr bool kShouldCopyWholeInlineStorageTrivial =+      std::is_trivially_copyable_v<Value> && kMayCopyWholeInlineStorage;++  static bool constexpr kHasInlineCapacity = !BaseType::kAlwaysUseHeap &&+      sizeof(HeapPtrWithCapacity) < sizeof(InlineStorageType);++  // This value should we multiple of word size.+  static size_t constexpr kHeapifyCapacitySize = sizeof(+      typename std::+          aligned_storage<sizeof(InternalSizeType), alignof(value_type)>::type);++  struct AllocationSize {+    auto operator()(void* ptr) const {+      (void)ptr;+#if defined(FOLLY_HAVE_MALLOC_USABLE_SIZE)+      return malloc_usable_size(ptr);+#endif+      // it is important that this method not return a size_t if we can't call+      // malloc_usable_size! kMustTrackHeapifiedCapacity uses the deduced return+      // type of this function in order to decide whether small_vector must+      // track its own capacity or not.+    }+  };++  static bool constexpr kMustTrackHeapifiedCapacity =+      BaseType::kAlwaysUseHeap ||+      !is_invocable_r_v<size_t, AllocationSize, void*>;++  // Threshold to control capacity heapifying.+  static size_t constexpr kHeapifyCapacityThreshold =+      (kMustTrackHeapifiedCapacity ? 0 : 100) * kHeapifyCapacitySize;++  static bool constexpr kAlwaysHasCapacity =+      kHasInlineCapacity || kMustTrackHeapifiedCapacity;++  typedef typename std::+      conditional<kHasInlineCapacity, HeapPtrWithCapacity, HeapPtr>::type+          PointerType;++  bool hasCapacity() const {+    return kAlwaysHasCapacity || !kHeapifyCapacityThreshold ||+        this->isHeapifiedCapacity();+  }++  void freeHeap() {+    if (!this->isExtern() || !u.pdata_.heap_) {+      return;+    }++    if (hasCapacity()) {+      auto extraBytes = u.pdata_.allocationExtraBytes();+      auto vp = detail::small_vector_detail::unshiftPointer(+          u.pdata_.heap_, extraBytes);+      annotate_object_collected(vp);+      sizedFree(vp, u.getCapacity() * sizeof(value_type) + extraBytes);+    } else {+      auto vp = u.pdata_.heap_;+      annotate_object_collected(vp);+      free(vp);+    }+  }++  union Data {+    explicit Data() { pdata_.heap_ = nullptr; }++    PointerType pdata_;+    InlineStorageType storage_;++    value_type* buffer() noexcept {+      void* vp = &storage_;+      return static_cast<value_type*>(vp);+    }+    value_type const* buffer() const noexcept {+      return const_cast<Data*>(this)->buffer();+    }+    value_type* heap() noexcept { return pdata_.heap_; }+    value_type const* heap() const noexcept { return pdata_.heap_; }++    InternalSizeType getCapacity() const { return pdata_.getCapacity(); }+    void setCapacity(InternalSizeType c) { pdata_.setCapacity(c); }++  } u;+};++//////////////////////////////////////////////////////////////////////++// Basic guarantee only, or provides the nothrow guarantee iff T has a+// nothrow move or copy constructor.+template <class T, std::size_t MaxInline, class P>+void swap(small_vector<T, MaxInline, P>& a, small_vector<T, MaxInline, P>& b) {+  a.swap(b);+}++template <class T, std::size_t MaxInline, class P, class U>+void erase(small_vector<T, MaxInline, P>& v, U value) {+  v.erase(std::remove(v.begin(), v.end(), value), v.end());+}++template <class T, std::size_t MaxInline, class P, class Predicate>+void erase_if(small_vector<T, MaxInline, P>& v, Predicate predicate) {+  v.erase(std::remove_if(v.begin(), v.end(), std::ref(predicate)), v.end());+}++//////////////////////////////////////////////////////////////////////++namespace detail {++// Format support.+template <class T, size_t M, class P>+struct IndexableTraits<small_vector<T, M, P>>+    : public IndexableTraitsSeq<small_vector<T, M, P>> {};++} // namespace detail++template <typename>+struct is_small_vector : std::false_type {};++template <class Value, size_t N, class Policy>+struct is_small_vector<small_vector<Value, N, Policy>> : std::true_type {};++template <typename T>+inline constexpr bool is_small_vector_v = is_small_vector<T>::value;++} // namespace folly++FOLLY_POP_WARNING++#undef FOLLY_SV_PACK_ATTR+#undef FOLLY_SV_PACK_PUSH+#undef FOLLY_SV_PACK_POP++namespace std {++template <class T, std::size_t M, class P>+struct hash<folly::small_vector<T, M, P>> {+  size_t operator()(const folly::small_vector<T, M, P>& v) const {+    return folly::hash::hash_range(v.begin(), v.end());+  }+};++} // namespace std
+ folly/folly/container/sorted_vector_types.h view
@@ -0,0 +1,1752 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++/*+ * This header defines two classes that very nearly model+ * AssociativeContainer (but not quite).  These implement set-like and+ * map-like behavior on top of a sorted vector, instead of using+ * rb-trees like std::set and std::map.+ *+ * This is potentially useful in cases where the number of elements in+ * the set or map is small, or when you want to avoid using more+ * memory than necessary and insertions/deletions are much more rare+ * than lookups (these classes have O(N) insertions/deletions).+ *+ * In the interest of using these in conditions where the goal is to+ * minimize memory usage, they support a GrowthPolicy parameter, which+ * is a class defining a single function called increase_capacity,+ * which will be called whenever we are about to insert something: you+ * can then decide to call reserve() based on the current capacity()+ * and size() of the passed in vector-esque Container type.  An+ * example growth policy that grows one element at a time:+ *+ *    struct OneAtATimePolicy {+ *      template <class Container>+ *      void increase_capacity(Container& c) {+ *        if (c.size() == c.capacity()) {+ *          c.reserve(c.size() + 1);+ *        }+ *      }+ *    };+ *+ *    typedef sorted_vector_set<int,+ *                              std::less<int>,+ *                              std::allocator<int>,+ *                              OneAtATimePolicy>+ *            OneAtATimeIntSet;+ *+ * Important differences from std::set and std::map:+ *   - insert() and erase() invalidate iterators and references.+       erase(iterator) returns an iterator pointing to the next valid element.+ *   - insert() and erase() are O(N)+ *   - our iterators model RandomAccessIterator+ *   - sorted_vector_map::value_type is pair<K,V>, not pair<const K,V>.+ *     (This is basically because we want to store the value_type in+ *     std::vector<>, which requires it to be Assignable.)+ *   - insert() single key variants, emplace(), and emplace_hint() only provide+ *     the strong exception guarantee (unchanged when exception is thrown) when+ *     std::is_nothrow_move_constructible<value_type>::value is true.+ */++#pragma once++#include <algorithm>+#include <cassert>+#include <initializer_list>+#include <iterator>+#include <memory>+#include <stdexcept>+#include <type_traits>+#include <utility>+#include <vector>++#include <folly/ScopeGuard.h>+#include <folly/Traits.h>+#include <folly/Utility.h>+#include <folly/lang/Access.h>+#include <folly/lang/Exception.h>+#include <folly/memory/MemoryResource.h>+#include <folly/small_vector.h>++namespace folly {++//////////////////////////////////////////////////////////////////////++namespace detail {++template <typename, typename Compare, typename Key, typename T>+struct sorted_vector_enable_if_is_transparent {};++template <typename Compare, typename Key, typename T>+struct sorted_vector_enable_if_is_transparent<+    void_t<typename Compare::is_transparent>,+    Compare,+    Key,+    T> {+  using type = T;+};++// This wrapper goes around a GrowthPolicy and provides iterator+// preservation semantics, but only if the growth policy is not the+// default (i.e. nothing).+template <class Policy>+struct growth_policy_wrapper : private Policy {+  template <class Container, class Iterator>+  Iterator increase_capacity(Container& c, Iterator desired_insertion) {+    typedef typename Container::difference_type diff_t;+    diff_t d = desired_insertion - c.begin();+    Policy::increase_capacity(c);+    return c.begin() + d;+  }+};+template <>+struct growth_policy_wrapper<void> {+  template <class Container, class Iterator>+  Iterator increase_capacity(Container&, Iterator it) {+    return it;+  }+};++template <class OurContainer, class Vector, class GrowthPolicy, class Value>+typename OurContainer::iterator insert_with_hint(+    OurContainer& sorted,+    Vector& cont,+    typename OurContainer::const_iterator hint,+    Value&& value,+    GrowthPolicy& po) {+  const typename OurContainer::value_compare& cmp(sorted.value_comp());+  if (hint == cont.end() || cmp(value, *hint)) {+    if (hint == cont.begin() || cmp(*(hint - 1), value)) {+      hint = po.increase_capacity(cont, hint);+      return cont.emplace(hint, std::forward<Value>(value));+    } else {+      return sorted.emplace(std::forward<Value>(value)).first;+    }+  }++  if (cmp(*hint, value)) {+    if (hint + 1 == cont.end() || cmp(value, *(hint + 1))) {+      hint = po.increase_capacity(cont, hint + 1);+      return cont.emplace(hint, std::forward<Value>(value));+    } else {+      return sorted.emplace(std::forward<Value>(value)).first;+    }+  }++  // Value and *hint did not compare, so they are equal keys.+  return sorted.begin() + std::distance(sorted.cbegin(), hint);+}++template <typename Iterator, typename Compare>+bool is_sorted_unique(Iterator begin, Iterator end, Compare const& comp) {+  if (begin == end) {+    return true;+  }+  for (auto next = std::next(begin); next != end; ++begin, ++next) {+    if (!comp(*begin, *next)) {+      return false;+    }+  }+  return true;+}++template <typename Container, typename Compare>+Container&& as_sorted_unique(Container&& container, Compare const& comp) {+  std::sort(container.begin(), container.end(), comp);+  container.erase(+      std::unique(+          container.begin(),+          container.end(),+          [&](auto const& a, auto const& b) {+            return !comp(a, b) && !comp(b, a);+          }),+      container.end());+  return static_cast<Container&&>(container);+}++template <typename Container, typename Compare>+class DirectMutationGuard {+ public:+  DirectMutationGuard(+      Container& container, const Compare& comp, bool isSortedUnique)+      : container_(container), comp_(comp), isSortedUnique_(isSortedUnique) {}++  ~DirectMutationGuard() noexcept(false) {+    if (isSortedUnique_) {+      assert(detail::is_sorted_unique(+          container_.begin(), container_.end(), comp_));+      return;+    }+    as_sorted_unique(container_, comp_);+  }++  Container& get() { return container_; }++ private:+  Container& container_;+  const Compare comp_;+  const bool isSortedUnique_;+};++template <class OurContainer, class Vector, class InputIterator>+void bulk_insert(+    OurContainer& sorted,+    Vector& cont,+    InputIterator first,+    InputIterator last,+    bool range_is_sorted_unique = false) {+  // Prevent deref of middle where middle == cont.end().+  if (first == last) {+    return;+  }++  auto const prev_size = cont.size();+  cont.insert(cont.end(), first, last);+  auto const middle = cont.begin() + prev_size;++  auto const& cmp(sorted.value_comp());+  if (range_is_sorted_unique) {+    assert(is_sorted_unique(middle, cont.end(), cmp));+  } else if (!std::is_sorted(middle, cont.end(), cmp)) {+    std::sort(middle, cont.end(), cmp);+  }++  // We do not need to consider elements strictly smaller than the smallest new+  // element in merge/unique.+  auto merge_begin = middle;+  while (merge_begin != cont.begin() && !cmp(*(merge_begin - 1), *middle)) {+    --merge_begin;+  }++  if (merge_begin != middle) {+    std::inplace_merge(cont.begin(), middle, cont.end(), cmp);+  } else if (range_is_sorted_unique) {+    // Old and new elements are already disjoint and unique. This includes the+    // case when cont is initially empty.+    return;+  }++  cont.erase(+      std::unique(+          merge_begin,+          cont.end(),+          [&](typename OurContainer::value_type const& a,+              typename OurContainer::value_type const& b) {+            return !cmp(a, b);+          }),+      cont.end());+}++} // namespace detail++//////////////////////////////////////////////////////////////////////++/**+ * A sorted_vector_set is a container similar to std::set<>, but+ * implemented as a sorted array with std::vector<>.+ *+ * @tparam T               Data type to store+ * @tparam Compare         Comparison function that imposes a+ *                              strict weak ordering over instances of T+ * @tparam Allocator       allocation policy+ * @tparam GrowthPolicy    policy object to control growth+ */+template <+    class T,+    class Compare = std::less<T>,+    class Allocator = std::allocator<T>,+    class GrowthPolicy = void,+    class Container = std::vector<T, Allocator>>+class sorted_vector_set : detail::growth_policy_wrapper<GrowthPolicy> {+  detail::growth_policy_wrapper<GrowthPolicy>& get_growth_policy() {+    return *this;+  }++  template <typename K, typename V, typename C = Compare>+  using if_is_transparent =+      _t<detail::sorted_vector_enable_if_is_transparent<void, C, K, V>>;++  struct EBO;++ public:+  typedef T value_type;+  typedef T key_type;+  typedef Compare key_compare;+  typedef Compare value_compare;+  typedef Allocator allocator_type;+  typedef Container container_type;++  typedef typename Container::pointer pointer;+  typedef typename Container::reference reference;+  typedef typename Container::const_reference const_reference;+  typedef typename Container::const_pointer const_pointer;+  /*+   * XXX: Our normal iterator ought to also be a constant iterator+   * (cf. Defect Report 103 for std::set), but this is a bit more of a+   * pain.+   */+  typedef typename Container::iterator iterator;+  typedef typename Container::const_iterator const_iterator;+  typedef typename Container::difference_type difference_type;+  typedef typename Container::size_type size_type;+  typedef typename Container::reverse_iterator reverse_iterator;+  typedef typename Container::const_reverse_iterator const_reverse_iterator;+  typedef detail::DirectMutationGuard<Container, value_compare>+      direct_mutation_guard;++  sorted_vector_set() : m_(Compare(), Allocator()) {}++  sorted_vector_set(const sorted_vector_set&) = default;++  sorted_vector_set(const sorted_vector_set& other, const Allocator& alloc)+      : m_(other.m_, alloc) {}++  sorted_vector_set(sorted_vector_set&&) = default;++  sorted_vector_set(sorted_vector_set&& other, const Allocator& alloc) noexcept(+      std::is_nothrow_constructible<EBO, EBO&&, const Allocator&>::value)+      : m_(std::move(other.m_), alloc) {}++  explicit sorted_vector_set(const Allocator& alloc) : m_(Compare(), alloc) {}++  explicit sorted_vector_set(+      const Compare& comp, const Allocator& alloc = Allocator())+      : m_(comp, alloc) {}++  template <class InputIterator>+  sorted_vector_set(+      InputIterator first,+      InputIterator last,+      const Compare& comp = Compare(),+      const Allocator& alloc = Allocator())+      : m_(comp, alloc) {+    // This is linear if [first, last) is already sorted (and if we+    // can figure out the distance between the two iterators).+    insert(first, last);+  }++  template <class InputIterator>+  sorted_vector_set(+      InputIterator first, InputIterator last, const Allocator& alloc)+      : m_(Compare(), alloc) {+    // This is linear if [first, last) is already sorted (and if we+    // can figure out the distance between the two iterators).+    insert(first, last);+  }++  /* implicit */ sorted_vector_set(+      std::initializer_list<value_type> list,+      const Compare& comp = Compare(),+      const Allocator& alloc = Allocator())+      : m_(comp, alloc) {+    insert(list.begin(), list.end());+  }++  sorted_vector_set(+      std::initializer_list<value_type> list, const Allocator& alloc)+      : m_(Compare(), alloc) {+    insert(list.begin(), list.end());+  }++  // Construct a sorted_vector_set by stealing the storage of a prefilled+  // container. The container need not be sorted already. This supports+  // bulk construction of sorted_vector_set with zero allocations, not counting+  // those performed by the caller. (The iterator range constructor performs at+  // least one allocation).+  //+  // Note that `sorted_vector_set(const Container& container)` is not provided,+  // since the purpose of this constructor is to avoid an unnecessary copy.+  explicit sorted_vector_set(+      Container&& container, const Compare& comp = Compare())+      : sorted_vector_set(+            sorted_unique,+            detail::as_sorted_unique(std::move(container), comp),+            comp) {}++  // Construct a sorted_vector_set by stealing the storage of a prefilled+  // container. Its elements must be sorted and unique, as sorted_unique_t+  // hints. Supports bulk construction of sorted_vector_set with zero+  // allocations, not counting those performed by the caller. (The iterator+  // range constructor performs at least one allocation).+  //+  // Note that `sorted_vector_set(sorted_unique_t, const Container& container)`+  // is not provided, since the purpose of this constructor is to avoid an extra+  // copy.+  sorted_vector_set(+      sorted_unique_t,+      Container&& container,+      const Compare& comp =+          Compare()) noexcept(std::+                                  is_nothrow_constructible<+                                      EBO,+                                      const Compare&,+                                      Container&&>::value)+      : m_(comp, std::move(container)) {+    assert(detail::is_sorted_unique(+        m_.cont_.begin(), m_.cont_.end(), value_comp()));+  }++  Allocator get_allocator() const { return m_.cont_.get_allocator(); }++  const Container& get_container() const noexcept { return m_.cont_; }++  /**+   * Directly mutate the container.+   *+   * Get a guarded reference to the underlying container for direct mutation.+   * sorted_unique_t signals that user will make sure that after the+   * modification the container will have its values as sorted-unique+   * (conforming to container's value_comp). Violating this assumption will+   * result in undefined behavior.+   *+   * This function is not safe to use concurrently with other functions.+   */+  direct_mutation_guard get_container_for_direct_mutation(+      sorted_unique_t) noexcept {+    return direct_mutation_guard{+        m_.cont_, value_comp(), /* range_is_sorted_unique */ true};+  }++  /**+   * Directly mutate the container.+   *+   * Get a guarded reference to the underlying container for direct mutation.+   * The container will initially be sorted and unique. You are not required to+   * maintain the sorted-unique invariant while mutating. When the guard is+   * released, it will sort and unique-ify the container.+   *+   * This function is not safe to use concurrently with other functions.+   */+  direct_mutation_guard get_container_for_direct_mutation() noexcept {+    return direct_mutation_guard{+        m_.cont_, value_comp(), /* range_is_sorted_unique */ false};+  }++  /**+   * Directly swap the container. Similar to swap()+   */+  void swap_container(Container& newContainer) {+    detail::as_sorted_unique(newContainer, value_comp());+    using std::swap;+    swap(m_.cont_, newContainer);+  }+  void swap_container(sorted_unique_t, Container& newContainer) {+    assert(detail::is_sorted_unique(+        newContainer.begin(), newContainer.end(), value_comp()));+    using std::swap;+    swap(m_.cont_, newContainer);+  }++  sorted_vector_set& operator=(const sorted_vector_set& other) = default;++  sorted_vector_set& operator=(sorted_vector_set&& other) = default;++  sorted_vector_set& operator=(std::initializer_list<value_type> ilist) {+    clear();+    insert(ilist.begin(), ilist.end());+    return *this;+  }++  key_compare key_comp() const { return m_; }+  value_compare value_comp() const { return m_; }++  iterator begin() { return m_.cont_.begin(); }+  iterator end() { return m_.cont_.end(); }+  const_iterator cbegin() const { return m_.cont_.cbegin(); }+  const_iterator begin() const { return m_.cont_.begin(); }+  const_iterator cend() const { return m_.cont_.cend(); }+  const_iterator end() const { return m_.cont_.end(); }+  reverse_iterator rbegin() { return m_.cont_.rbegin(); }+  reverse_iterator rend() { return m_.cont_.rend(); }+  const_reverse_iterator rbegin() const { return m_.cont_.rbegin(); }+  const_reverse_iterator rend() const { return m_.cont_.rend(); }++  void clear() { return m_.cont_.clear(); }+  size_type size() const { return m_.cont_.size(); }+  size_type max_size() const { return m_.cont_.max_size(); }+  bool empty() const { return m_.cont_.empty(); }+  void reserve(size_type s) { return m_.cont_.reserve(s); }+  void shrink_to_fit() { m_.cont_.shrink_to_fit(); }+  size_type capacity() const { return m_.cont_.capacity(); }++  std::pair<iterator, bool> insert(const value_type& value) {+    iterator it = lower_bound(value);+    if (it == end() || value_comp()(value, *it)) {+      it = get_growth_policy().increase_capacity(m_.cont_, it);+      return std::make_pair(m_.cont_.emplace(it, value), true);+    }+    return std::make_pair(it, false);+  }++  std::pair<iterator, bool> insert(value_type&& value) {+    iterator it = lower_bound(value);+    if (it == end() || value_comp()(value, *it)) {+      it = get_growth_policy().increase_capacity(m_.cont_, it);+      return std::make_pair(m_.cont_.emplace(it, std::move(value)), true);+    }+    return std::make_pair(it, false);+  }++  iterator insert(const_iterator hint, const value_type& value) {+    return detail::insert_with_hint(+        *this, m_.cont_, hint, value, get_growth_policy());+  }++  iterator insert(const_iterator hint, value_type&& value) {+    return detail::insert_with_hint(+        *this, m_.cont_, hint, std::move(value), get_growth_policy());+  }++  template <class InputIterator>+  void insert(InputIterator first, InputIterator last) {+    detail::bulk_insert(*this, m_.cont_, first, last);+  }++  // If [first, last) is known to be sorted and unique according to the+  // comparator (for example if the range comes from a sorted container of the+  // same type) this version can save unnecessary operations, especially if+  // *this is empty.+  template <class InputIterator>+  void insert(sorted_unique_t, InputIterator first, InputIterator last) {+    detail::bulk_insert(+        *this, m_.cont_, first, last, /* range_is_sorted_unique */ true);+  }++  void insert(std::initializer_list<value_type> ilist) {+    insert(ilist.begin(), ilist.end());+  }++  // emplace isn't better than insert for sorted_vector_set, but aids+  // compatibility+  template <typename... Args>+  std::pair<iterator, bool> emplace(Args&&... args) {+    std::aligned_storage_t<sizeof(value_type), alignof(value_type)> b;+    value_type* p = static_cast<value_type*>(static_cast<void*>(&b));+    auto a = get_allocator();+    std::allocator_traits<allocator_type>::construct(+        a, p, std::forward<Args>(args)...);+    auto g = makeGuard([&]() {+      std::allocator_traits<allocator_type>::destroy(a, p);+    });+    return insert(std::move(*p));+  }++  std::pair<iterator, bool> emplace(const value_type& value) {+    return insert(value);+  }++  std::pair<iterator, bool> emplace(value_type&& value) {+    return insert(std::move(value));+  }++  // emplace_hint isn't better than insert for sorted_vector_set, but aids+  // compatibility+  template <typename... Args>+  iterator emplace_hint(const_iterator hint, Args&&... args) {+    std::aligned_storage_t<sizeof(value_type), alignof(value_type)> b;+    value_type* p = static_cast<value_type*>(static_cast<void*>(&b));+    auto a = get_allocator();+    std::allocator_traits<allocator_type>::construct(+        a, p, std::forward<Args>(args)...);+    auto g = makeGuard([&]() {+      std::allocator_traits<allocator_type>::destroy(a, p);+    });+    return insert(hint, std::move(*p));+  }++  iterator emplace_hint(const_iterator hint, const value_type& value) {+    return insert(hint, value);+  }++  iterator emplace_hint(const_iterator hint, value_type&& value) {+    return insert(hint, std::move(value));+  }++  size_type erase(const key_type& key) {+    iterator it = find(key);+    if (it == end()) {+      return 0;+    }+    m_.cont_.erase(it);+    return 1;+  }++  iterator erase(const_iterator it) { return m_.cont_.erase(it); }++  iterator erase(const_iterator first, const_iterator last) {+    return m_.cont_.erase(first, last);+  }++  template <class Predicate>+  friend size_type erase_if(sorted_vector_set& container, Predicate predicate) {+    auto& c = container.m_.cont_;+    const auto preEraseSize = c.size();+    c.erase(std::remove_if(c.begin(), c.end(), std::ref(predicate)), c.end());+    return preEraseSize - c.size();+  }++  iterator find(const key_type& key) { return find_(*this, key); }++  const_iterator find(const key_type& key) const { return find_(*this, key); }++  template <typename K>+  if_is_transparent<K, iterator> find(const K& key) {+    return find_(*this, key);+  }++  template <typename K>+  if_is_transparent<K, const_iterator> find(const K& key) const {+    return find_(*this, key);+  }++  size_type count(const key_type& key) const {+    return find(key) == end() ? 0 : 1;+  }++  std::pair<iterator, iterator> find(+      const key_type& key1, const key_type& key2) {+    if (key_comp()(key2, key1)) {+      auto iterators = find2_(*this, key2, key1);+      access::swap(iterators.first, iterators.second);+      return iterators;+    } else {+      return find2_(*this, key1, key2);+    }+  }++  std::pair<const_iterator, const_iterator> find(+      const key_type& key1, const key_type& key2) const {+    if (key_comp()(key2, key1)) {+      auto iterators = find2_(*this, key2, key1);+      access::swap(iterators.first, iterators.second);+      return iterators;+    } else {+      return find2_(*this, key1, key2);+    }+  }++  template <typename K>+  std::pair<if_is_transparent<K, iterator>, if_is_transparent<K, iterator>>+  find(const K& key1, const K& key2) {+    if (key_comp()(key2, key1)) {+      auto iterators = find2_(*this, key2, key1);+      access::swap(iterators.first, iterators.second);+      return iterators;+    } else {+      return find2_(*this, key1, key2);+    }+  }++  template <typename K>+  std::pair<+      if_is_transparent<K, const_iterator>,+      if_is_transparent<K, const_iterator>>+  find(const K& key1, const K& key2) const {+    if (key_comp()(key2, key1)) {+      auto iterators = find2_(*this, key2, key1);+      access::swap(iterators.first, iterators.second);+      return iterators;+    } else {+      return find2_(*this, key1, key2);+    }+  }++  template <typename K>+  if_is_transparent<K, size_type> count(const K& key) const {+    return find(key) == end() ? 0 : 1;+  }++  bool contains(const key_type& key) const { return find(key) != end(); }++  template <typename K>+  if_is_transparent<K, bool> contains(const K& key) const {+    return find(key) != end();+  }++  iterator lower_bound(const key_type& key) {+    return std::lower_bound(begin(), end(), key, key_comp());+  }++  const_iterator lower_bound(const key_type& key) const {+    return std::lower_bound(begin(), end(), key, key_comp());+  }++  template <typename K>+  if_is_transparent<K, iterator> lower_bound(const K& key) {+    return std::lower_bound(begin(), end(), key, key_comp());+  }++  template <typename K>+  if_is_transparent<K, const_iterator> lower_bound(const K& key) const {+    return std::lower_bound(begin(), end(), key, key_comp());+  }++  iterator upper_bound(const key_type& key) {+    return std::upper_bound(begin(), end(), key, key_comp());+  }++  const_iterator upper_bound(const key_type& key) const {+    return std::upper_bound(begin(), end(), key, key_comp());+  }++  template <typename K>+  if_is_transparent<K, iterator> upper_bound(const K& key) {+    return std::upper_bound(begin(), end(), key, key_comp());+  }++  template <typename K>+  if_is_transparent<K, const_iterator> upper_bound(const K& key) const {+    return std::upper_bound(begin(), end(), key, key_comp());+  }++  std::pair<iterator, iterator> equal_range(const key_type& key) {+    return std::equal_range(begin(), end(), key, key_comp());+  }++  std::pair<const_iterator, const_iterator> equal_range(+      const key_type& key) const {+    return std::equal_range(begin(), end(), key, key_comp());+  }++  template <typename K>+  if_is_transparent<K, std::pair<iterator, iterator>> equal_range(+      const K& key) {+    return std::equal_range(begin(), end(), key, key_comp());+  }++  template <typename K>+  if_is_transparent<K, std::pair<const_iterator, const_iterator>> equal_range(+      const K& key) const {+    return std::equal_range(begin(), end(), key, key_comp());+  }++  void swap(sorted_vector_set& o) noexcept(+      std::is_nothrow_swappable_v<Compare> &&+      noexcept(std::declval<Container&>().swap(o.m_.cont_))) {+    using std::swap; // Allow ADL for swap(); fall back to std::swap().+    Compare& a = m_;+    Compare& b = o.m_;+    swap(a, b);+    m_.cont_.swap(o.m_.cont_);+  }++  bool operator==(const sorted_vector_set& other) const {+    return other.m_.cont_ == m_.cont_;+  }+  bool operator!=(const sorted_vector_set& other) const {+    return !operator==(other);+  }++  bool operator<(const sorted_vector_set& other) const {+    return m_.cont_ < other.m_.cont_;+  }+  bool operator>(const sorted_vector_set& other) const { return other < *this; }+  bool operator<=(const sorted_vector_set& other) const {+    return !operator>(other);+  }+  bool operator>=(const sorted_vector_set& other) const {+    return !operator<(other);+  }++#if FOLLY_CPLUSPLUS >= 202002L && defined(__cpp_impl_three_way_comparison)+  template <typename U = Container>+  friend auto operator<=>(+      const sorted_vector_set& lhs, const sorted_vector_set& rhs)+      -> decltype(std::declval<const U&>() <=> std::declval<const U&>()) {+    return lhs.m_.cont_ <=> rhs.m_.cont_;+  }+#endif // FOLLY_CPLUSPLUS >= 202002L && defined(__cpp_impl_three_way_comparison)++  const value_type* data() const noexcept { return m_.cont_.data(); }++ private:+  /*+   * This structure derives from the comparison object in order to+   * make use of the empty base class optimization if our comparison+   * functor is an empty class (usual case).+   *+   * Wrapping up this member like this is better than deriving from+   * the Compare object ourselves (there are some perverse edge cases+   * involving virtual functions).+   *+   * More info:  http://www.cantrip.org/emptyopt.html+   */+  struct EBO : Compare {+    explicit EBO(const Compare& c, const Allocator& alloc) noexcept(+        std::is_nothrow_default_constructible<Container>::value)+        : Compare(c), cont_(alloc) {}+    EBO(const EBO& other, const Allocator& alloc) noexcept(+        std::is_nothrow_constructible<+            Container,+            const Container&,+            const Allocator&>::value)+        : Compare(static_cast<const Compare&>(other)),+          cont_(other.cont_, alloc) {}+    EBO(EBO&& other, const Allocator& alloc) noexcept(+        std::is_nothrow_constructible<+            Container,+            Container&&,+            const Allocator&>::value)+        : Compare(static_cast<Compare&&>(other)),+          cont_(std::move(other.cont_), alloc) {}+    EBO(const Compare& c, Container&& cont) noexcept(+        std::is_nothrow_move_constructible<Container>::value)+        : Compare(c), cont_(std::move(cont)) {}+    Container cont_;+  } m_;++  template <typename Self>+  using self_iterator_t = _t<+      std::conditional<std::is_const<Self>::value, const_iterator, iterator>>;++  template <typename Self, typename K>+  static self_iterator_t<Self> find_(Self& self, K const& key) {+    auto end = self.end();+    auto it = self.lower_bound(key);+    if (it == end || !self.key_comp()(key, *it)) {+      return it;+    }+    return end;+  }+  template <typename Self, typename K>+  static std::pair<self_iterator_t<Self>, self_iterator_t<Self>> lower_bound2_(+      Self& self, K const& key1, K const& key2) {+    auto len = self.size();+    auto first = self.begin(), second = self.begin();+    auto c = self.key_comp();+    assert(!c(key2, key1));+    while (true) {+      if (len == 0) {+        return std::make_pair(first, first);+      }+      auto half = len / 2;+      auto middle = first + half;+      if (c(*middle, key1)) {+        first = middle + 1;+        half = len - half - 1;+      } else if (c(*middle, key2)) {+        second = middle + (len & 1);+        len = half;+        break;+      }+      len = half;+    }+    while (len) {+      auto half = len / 2;+      auto middle1 = first + half;+      auto middle2 = second + half;+      if (c(*middle1, key1)) {+        first = middle1 + (len & 1);+      }+      if (c(*middle2, key2)) {+        second = middle2 + (len & 1);+      }+      len = half;+    }+    return std::make_pair(first, second);+  }++  template <typename Self, typename K>+  static std::pair<self_iterator_t<Self>, self_iterator_t<Self>> find2_(+      Self& self, K const& key1, K const& key2) {+    auto end = self.end();+    auto its = lower_bound2_(self, key1, key2);+    if (its.second != end) {+      if (self.key_comp()(key1, *its.first)) {+        its.first = end;+      }+      if (self.key_comp()(key2, *its.second)) {+        its.second = end;+      }+    } else if (its.first != end && self.key_comp()(key1, *its.first)) {+      its.first = end;+    }+    return its;+  }+};++// Swap function that can be found using ADL.+template <class T, class C, class A, class G>+inline void swap(+    sorted_vector_set<T, C, A, G>& a, sorted_vector_set<T, C, A, G>& b) {+  return a.swap(b);+}++template <typename T>+inline constexpr bool is_sorted_vector_set_v =+    is_instantiation_of_v<sorted_vector_set, T>;++template <typename T>+struct is_sorted_vector_set : std::bool_constant<is_sorted_vector_set_v<T>> {};++template <+    class T,+    size_t N = 1,+    class Compare = std::less<T>,+    class Allocator = std::allocator<T>,+    class GrowthPolicy = void,+    class SmallVectorPolicy = void>+using small_sorted_vector_set = sorted_vector_set<+    T,+    Compare,+    Allocator,+    GrowthPolicy,+    folly::small_vector<T, N, SmallVectorPolicy>>;++template <typename T>+inline constexpr bool is_small_sorted_vector_set_v =+    is_sorted_vector_set_v<T> && is_small_vector_v<typename T::container_type>;++template <typename T>+struct is_small_sorted_vector_set+    : std::bool_constant<is_small_sorted_vector_set_v<T>> {};++#if FOLLY_HAS_MEMORY_RESOURCE++namespace pmr {++template <+    class T,+    class Compare = std::less<T>,+    class GrowthPolicy = void,+    class Container = std::vector<T, std::pmr::polymorphic_allocator<T>>>+using sorted_vector_set = folly::sorted_vector_set<+    T,+    Compare,+    std::pmr::polymorphic_allocator<T>,+    GrowthPolicy,+    Container>;++} // namespace pmr++#endif++//////////////////////////////////////////////////////////////////////++/**+ * A sorted_vector_map is similar to a sorted_vector_set but stores+ * <key,value> pairs instead of single elements.+ *+ * @tparam Key           Key type+ * @tparam Value         Value type+ * @tparam Compare       Function that can compare key types and impose+ *                            a strict weak ordering over them.+ * @tparam Allocator     allocation policy+ * @tparam GrowthPolicy  policy object to control growth+ */+template <+    class Key,+    class Value,+    class Compare = std::less<Key>,+    class Allocator = std::allocator<std::pair<Key, Value>>,+    class GrowthPolicy = void,+    class Container = std::vector<std::pair<Key, Value>, Allocator>>+class sorted_vector_map : detail::growth_policy_wrapper<GrowthPolicy> {+  detail::growth_policy_wrapper<GrowthPolicy>& get_growth_policy() {+    return *this;+  }++  template <typename K, typename V, typename C = Compare>+  using if_is_transparent =+      _t<detail::sorted_vector_enable_if_is_transparent<void, C, K, V>>;++  struct EBO;++ public:+  typedef Key key_type;+  typedef Value mapped_type;+  typedef typename Container::value_type value_type;+  typedef Compare key_compare;+  typedef Allocator allocator_type;+  typedef Container container_type;++  struct value_compare : private Compare {+    bool operator()(const value_type& a, const value_type& b) const {+      return Compare::operator()(a.first, b.first);+    }++   protected:+    friend class sorted_vector_map;+    explicit value_compare(const Compare& c) : Compare(c) {}+  };++  typedef typename Container::pointer pointer;+  typedef typename Container::const_pointer const_pointer;+  typedef typename Container::reference reference;+  typedef typename Container::const_reference const_reference;+  typedef typename Container::iterator iterator;+  typedef typename Container::const_iterator const_iterator;+  typedef typename Container::difference_type difference_type;+  typedef typename Container::size_type size_type;+  typedef typename Container::reverse_iterator reverse_iterator;+  typedef typename Container::const_reverse_iterator const_reverse_iterator;+  typedef detail::DirectMutationGuard<Container, value_compare>+      direct_mutation_guard;++  sorted_vector_map() noexcept(+      std::is_nothrow_constructible<EBO, value_compare, Allocator>::value)+      : m_(value_compare(Compare()), Allocator()) {}++  sorted_vector_map(const sorted_vector_map&) = default;++  sorted_vector_map(const sorted_vector_map& other, const Allocator& alloc)+      : m_(other.m_, alloc) {}++  sorted_vector_map(sorted_vector_map&&) = default;++  sorted_vector_map(sorted_vector_map&& other, const Allocator& alloc) noexcept(+      std::is_nothrow_constructible<EBO, EBO&&, const Allocator&>::value)+      : m_(std::move(other.m_), alloc) {}++  explicit sorted_vector_map(const Allocator& alloc)+      : m_(value_compare(Compare()), alloc) {}++  explicit sorted_vector_map(+      const Compare& comp, const Allocator& alloc = Allocator())+      : m_(value_compare(comp), alloc) {}++  template <class InputIterator>+  explicit sorted_vector_map(+      InputIterator first,+      InputIterator last,+      const Compare& comp = Compare(),+      const Allocator& alloc = Allocator())+      : m_(value_compare(comp), alloc) {+    insert(first, last);+  }++  template <class InputIterator>+  sorted_vector_map(+      InputIterator first, InputIterator last, const Allocator& alloc)+      : m_(value_compare(Compare()), alloc) {+    insert(first, last);+  }++  /* implicit */ sorted_vector_map(+      std::initializer_list<value_type> list,+      const Compare& comp = Compare(),+      const Allocator& alloc = Allocator())+      : m_(value_compare(comp), alloc) {+    insert(list.begin(), list.end());+  }++  sorted_vector_map(+      std::initializer_list<value_type> list, const Allocator& alloc)+      : m_(value_compare(Compare()), alloc) {+    insert(list.begin(), list.end());+  }++  // Construct a sorted_vector_map by stealing the storage of a prefilled+  // container. The container need not be sorted already. This supports+  // bulk construction of sorted_vector_map with zero allocations, not counting+  // those performed by the caller. (The iterator range constructor performs at+  // least one allocation).+  //+  // Note that `sorted_vector_map(const Container& container)` is not provided,+  // since the purpose of this constructor is to avoid an unnecessary copy.+  explicit sorted_vector_map(+      Container&& container, const Compare& comp = Compare())+      : sorted_vector_map(+            sorted_unique,+            detail::as_sorted_unique(std::move(container), value_compare(comp)),+            comp) {}++  // Construct a sorted_vector_map by stealing the storage of a prefilled+  // container. Its elements must be sorted and unique, as sorted_unique_t+  // hints. Supports bulk construction of sorted_vector_map with zero+  // allocations, not counting those performed by the caller. (The iterator+  // range constructor performs at least one allocation).+  //+  // Note that `sorted_vector_map(sorted_unique_t, const Container& container)`+  // is not provided, since the purpose of this constructor is to avoid an extra+  // copy.+  sorted_vector_map(+      sorted_unique_t,+      Container&& container,+      const Compare& comp =+          Compare()) noexcept(std::+                                  is_nothrow_constructible<+                                      EBO,+                                      value_compare,+                                      Container&&>::value)+      : m_(value_compare(comp), std::move(container)) {+    assert(detail::is_sorted_unique(+        m_.cont_.begin(), m_.cont_.end(), value_comp()));+  }++  Allocator get_allocator() const { return m_.cont_.get_allocator(); }++  const Container& get_container() const noexcept { return m_.cont_; }++  /**+   * Directly mutate the container.+   *+   * Get a guarded reference to the underlying container for direct mutation.+   * sorted_unique_t signals that user will make sure that after the+   * modification the container will have its values as sorted-unique+   * (conforming to container's value_comp). Violating this assumption will+   * result in undefined behavior.+   *+   * This function is not safe to use concurrently with other functions.+   */+  direct_mutation_guard get_container_for_direct_mutation(+      sorted_unique_t) noexcept {+    return direct_mutation_guard{+        m_.cont_, value_comp(), /* range_is_sorted_unique */ true};+  }++  /**+   * Directly mutate the container.+   *+   * Get a guarded reference to the underlying container for direct mutation.+   * The container will initially be sorted and unique. You are not required to+   * maintain the sorted-unique invariant while mutating. When the guard is+   * released, it will sort and unique-ify the container.+   *+   * This function is not safe to use concurrently with other functions.+   */+  direct_mutation_guard get_container_for_direct_mutation() noexcept {+    return direct_mutation_guard{+        m_.cont_, value_comp(), /* range_is_sorted_unique */ false};+  }++  /**+   * Directly swap the container. Similar to swap()+   */+  void swap_container(Container& newContainer) {+    detail::as_sorted_unique(newContainer, value_comp());+    using std::swap;+    swap(m_.cont_, newContainer);+  }+  void swap_container(sorted_unique_t, Container& newContainer) {+    assert(detail::is_sorted_unique(+        newContainer.begin(), newContainer.end(), value_comp()));+    using std::swap;+    swap(m_.cont_, newContainer);+  }++  sorted_vector_map& operator=(const sorted_vector_map& other) = default;++  sorted_vector_map& operator=(sorted_vector_map&& other) = default;++  sorted_vector_map& operator=(std::initializer_list<value_type> ilist) {+    clear();+    insert(ilist.begin(), ilist.end());+    return *this;+  }++  key_compare key_comp() const { return m_; }+  value_compare value_comp() const { return m_; }++  iterator begin() { return m_.cont_.begin(); }+  iterator end() { return m_.cont_.end(); }+  const_iterator cbegin() const { return m_.cont_.cbegin(); }+  const_iterator begin() const { return m_.cont_.begin(); }+  const_iterator cend() const { return m_.cont_.cend(); }+  const_iterator end() const { return m_.cont_.end(); }+  reverse_iterator rbegin() { return m_.cont_.rbegin(); }+  reverse_iterator rend() { return m_.cont_.rend(); }+  const_reverse_iterator crbegin() const { return m_.cont_.crbegin(); }+  const_reverse_iterator rbegin() const { return m_.cont_.rbegin(); }+  const_reverse_iterator crend() const { return m_.cont_.crend(); }+  const_reverse_iterator rend() const { return m_.cont_.rend(); }++  void clear() { return m_.cont_.clear(); }+  size_type size() const { return m_.cont_.size(); }+  size_type max_size() const { return m_.cont_.max_size(); }+  bool empty() const { return m_.cont_.empty(); }+  void reserve(size_type s) { return m_.cont_.reserve(s); }+  void shrink_to_fit() { m_.cont_.shrink_to_fit(); }+  size_type capacity() const { return m_.cont_.capacity(); }++  std::pair<iterator, bool> insert(const value_type& value) {+    iterator it = lower_bound(value.first);+    if (it == end() || value_comp()(value, *it)) {+      it = get_growth_policy().increase_capacity(m_.cont_, it);+      return std::make_pair(m_.cont_.emplace(it, value), true);+    }+    return std::make_pair(it, false);+  }++  std::pair<iterator, bool> insert(value_type&& value) {+    iterator it = lower_bound(value.first);+    if (it == end() || value_comp()(value, *it)) {+      it = get_growth_policy().increase_capacity(m_.cont_, it);+      return std::make_pair(m_.cont_.emplace(it, std::move(value)), true);+    }+    return std::make_pair(it, false);+  }++  iterator insert(const_iterator hint, const value_type& value) {+    return detail::insert_with_hint(+        *this, m_.cont_, hint, value, get_growth_policy());+  }++  iterator insert(const_iterator hint, value_type&& value) {+    return detail::insert_with_hint(+        *this, m_.cont_, hint, std::move(value), get_growth_policy());+  }++  template <class InputIterator>+  void insert(InputIterator first, InputIterator last) {+    detail::bulk_insert(*this, m_.cont_, first, last);+  }++  // If [first, last) is known to be sorted and unique according to the+  // comparator (for example if the range comes from a sorted container of the+  // same type) this version can save unnecessary operations, especially if+  // *this is empty.+  template <class InputIterator>+  void insert(sorted_unique_t, InputIterator first, InputIterator last) {+    detail::bulk_insert(+        *this, m_.cont_, first, last, /* range_is_sorted_unique */ true);+  }++  void insert(std::initializer_list<value_type> ilist) {+    insert(ilist.begin(), ilist.end());+  }++  // emplace isn't better than insert for sorted_vector_map, but aids+  // compatibility+  template <typename... Args>+  std::pair<iterator, bool> emplace(Args&&... args) {+    std::aligned_storage_t<sizeof(value_type), alignof(value_type)> b;+    value_type* p = static_cast<value_type*>(static_cast<void*>(&b));+    auto a = get_allocator();+    std::allocator_traits<allocator_type>::construct(+        a, p, std::forward<Args>(args)...);+    auto g = makeGuard([&]() {+      std::allocator_traits<allocator_type>::destroy(a, p);+    });+    return insert(std::move(*p));+  }++  std::pair<iterator, bool> emplace(const value_type& value) {+    return insert(value);+  }++  std::pair<iterator, bool> emplace(value_type&& value) {+    return insert(std::move(value));+  }++  // emplace_hint isn't better than insert for sorted_vector_set, but aids+  // compatibility+  template <typename... Args>+  iterator emplace_hint(const_iterator hint, Args&&... args) {+    std::aligned_storage_t<sizeof(value_type), alignof(value_type)> b;+    value_type* p = static_cast<value_type*>(static_cast<void*>(&b));+    auto a = get_allocator();+    std::allocator_traits<allocator_type>::construct(+        a, p, std::forward<Args>(args)...);+    auto g = makeGuard([&]() {+      std::allocator_traits<allocator_type>::destroy(a, p);+    });+    return insert(hint, std::move(*p));+  }++  iterator emplace_hint(const_iterator hint, const value_type& value) {+    return insert(hint, value);+  }++  iterator emplace_hint(const_iterator hint, value_type&& value) {+    return insert(hint, std::move(value));+  }++  template <typename... Args>+  std::pair<iterator, bool> try_emplace(key_type&& k, Args&&... args) {+    return try_emplace_impl(std::move(k), std::forward<Args>(args)...);+  }++  template <typename... Args>+  std::pair<iterator, bool> try_emplace(const key_type& k, Args&&... args) {+    return try_emplace_impl(k, std::forward<Args>(args)...);+  }++  template <typename M>+  std::pair<iterator, bool> insert_or_assign(const key_type& k, M&& obj) {+    auto itAndInserted = try_emplace(k, std::forward<M>(obj));+    if (!itAndInserted.second) {+      itAndInserted.first->second = std::forward<M>(obj);+    }+    return itAndInserted;+  }++  template <typename M>+  std::pair<iterator, bool> insert_or_assign(key_type&& k, M&& obj) {+    auto itAndInserted = try_emplace(std::move(k), std::forward<M>(obj));+    if (!itAndInserted.second) {+      itAndInserted.first->second = std::forward<M>(obj);+    }+    return itAndInserted;+  }++  template <class M>+  iterator insert_or_assign(const_iterator hint, const key_type& k, M&& obj) {+    return insert_or_assign_impl(hint, k, std::forward<M>(obj));+  }++  template <class M>+  iterator insert_or_assign(const_iterator hint, key_type&& k, M&& obj) {+    return insert_or_assign_impl(hint, std::move(k), std::forward<M>(obj));+  }++  size_type erase(const key_type& key) {+    iterator it = find(key);+    if (it == end()) {+      return 0;+    }+    m_.cont_.erase(it);+    return 1;+  }++  iterator erase(const_iterator it) { return m_.cont_.erase(it); }++  iterator erase(const_iterator first, const_iterator last) {+    return m_.cont_.erase(first, last);+  }++  template <class Predicate>+  friend size_type erase_if(sorted_vector_map& container, Predicate predicate) {+    auto& c = container.m_.cont_;+    const auto preEraseSize = c.size();+    c.erase(std::remove_if(c.begin(), c.end(), std::ref(predicate)), c.end());+    return preEraseSize - c.size();+  }++  iterator find(const key_type& key) { return find_(*this, key); }++  const_iterator find(const key_type& key) const { return find_(*this, key); }++  template <typename K>+  if_is_transparent<K, iterator> find(const K& key) {+    return find_(*this, key);+  }++  template <typename K>+  if_is_transparent<K, const_iterator> find(const K& key) const {+    return find_(*this, key);+  }++  std::pair<iterator, iterator> find(+      const key_type& key1, const key_type& key2) {+    if (key_comp()(key2, key1)) {+      auto iterators = find2_(*this, key2, key1);+      access::swap(iterators.first, iterators.second);+      return iterators;+    } else {+      return find2_(*this, key1, key2);+    }+  }++  std::pair<const_iterator, const_iterator> find(+      const key_type& key1, const key_type& key2) const {+    if (key_comp()(key2, key1)) {+      auto iterators = find2_(*this, key2, key1);+      access::swap(iterators.first, iterators.second);+      return iterators;+    } else {+      return find2_(*this, key1, key2);+    }+  }++  template <typename K>+  std::pair<if_is_transparent<K, iterator>, if_is_transparent<K, iterator>>+  find(const K& key1, const K& key2) {+    if (key_comp()(key2, key1)) {+      auto iterators = find2_(*this, key2, key1);+      access::swap(iterators.first, iterators.second);+      return iterators;+    } else {+      return find2_(*this, key1, key2);+    }+  }++  template <typename K>+  std::pair<+      if_is_transparent<K, const_iterator>,+      if_is_transparent<K, const_iterator>>+  find(const K& key1, const K& key2) const {+    if (key_comp()(key2, key1)) {+      auto iterators = find2_(*this, key2, key1);+      access::swap(iterators.first, iterators.second);+      return iterators;+    } else {+      return find2_(*this, key1, key2);+    }+  }++  mapped_type& at(const key_type& key) {+    iterator it = find(key);+    if (it != end()) {+      return it->second;+    }+    throw_exception<std::out_of_range>("sorted_vector_map::at");+  }++  const mapped_type& at(const key_type& key) const {+    const_iterator it = find(key);+    if (it != end()) {+      return it->second;+    }+    throw_exception<std::out_of_range>("sorted_vector_map::at");+  }++  size_type count(const key_type& key) const {+    return find(key) == end() ? 0 : 1;+  }++  template <typename K>+  if_is_transparent<K, size_type> count(const K& key) const {+    return find(key) == end() ? 0 : 1;+  }++  bool contains(const key_type& key) const { return find(key) != end(); }++  template <typename K>+  if_is_transparent<K, bool> contains(const K& key) const {+    return find(key) != end();+  }++  iterator lower_bound(const key_type& key) { return lower_bound(*this, key); }++  const_iterator lower_bound(const key_type& key) const {+    return lower_bound(*this, key);+  }++  template <typename K>+  if_is_transparent<K, iterator> lower_bound(const K& key) {+    return lower_bound(*this, key);+  }++  template <typename K>+  if_is_transparent<K, const_iterator> lower_bound(const K& key) const {+    return lower_bound(*this, key);+  }++  iterator upper_bound(const key_type& key) { return upper_bound(*this, key); }++  const_iterator upper_bound(const key_type& key) const {+    return upper_bound(*this, key);+  }++  template <typename K>+  if_is_transparent<K, iterator> upper_bound(const K& key) {+    return upper_bound(*this, key);+  }++  template <typename K>+  if_is_transparent<K, const_iterator> upper_bound(const K& key) const {+    return upper_bound(*this, key);+  }++  std::pair<iterator, iterator> equal_range(const key_type& key) {+    return equal_range(*this, key);+  }++  std::pair<const_iterator, const_iterator> equal_range(+      const key_type& key) const {+    return equal_range(*this, key);+  }++  template <typename K>+  if_is_transparent<K, std::pair<iterator, iterator>> equal_range(+      const K& key) {+    return equal_range(*this, key);+  }++  template <typename K>+  if_is_transparent<K, std::pair<const_iterator, const_iterator>> equal_range(+      const K& key) const {+    return equal_range(*this, key);+  }++  // Nothrow as long as swap() on the Compare type is nothrow.+  void swap(sorted_vector_map& o) {+    using std::swap; // Allow ADL for swap(); fall back to std::swap().+    Compare& a = m_;+    Compare& b = o.m_;+    swap(a, b);+    m_.cont_.swap(o.m_.cont_);+  }++  mapped_type& operator[](const key_type& key) {+    iterator it = lower_bound(key);+    if (it == end() || key_comp()(key, it->first)) {+      return insert(it, value_type(key, mapped_type()))->second;+    }+    return it->second;+  }++  bool operator==(const sorted_vector_map& other) const {+    return m_.cont_ == other.m_.cont_;+  }+  bool operator!=(const sorted_vector_map& other) const {+    return !operator==(other);+  }++  bool operator<(const sorted_vector_map& other) const {+    return m_.cont_ < other.m_.cont_;+  }+  bool operator>(const sorted_vector_map& other) const { return other < *this; }+  bool operator<=(const sorted_vector_map& other) const {+    return !operator>(other);+  }+  bool operator>=(const sorted_vector_map& other) const {+    return !operator<(other);+  }++#if FOLLY_CPLUSPLUS >= 202002L && defined(__cpp_impl_three_way_comparison)+  template <typename U = Container>+  friend auto operator<=>(+      const sorted_vector_map& lhs, const sorted_vector_map& rhs)+      -> decltype(std::declval<const U&>() <=> std::declval<const U&>()) {+    return lhs.m_.cont_ <=> rhs.m_.cont_;+  }+#endif // FOLLY_CPLUSPLUS >= 202002L && defined(__cpp_impl_three_way_comparison)++  const value_type* data() const noexcept { return m_.cont_.data(); }++ private:+  // This is to get the empty base optimization; see the comment in+  // sorted_vector_set.+  struct EBO : value_compare {+    explicit EBO(const value_compare& c, const Allocator& alloc) noexcept(+        std::is_nothrow_default_constructible<Container>::value)+        : value_compare(c), cont_(alloc) {}+    EBO(const EBO& other, const Allocator& alloc) noexcept(+        std::is_nothrow_constructible<+            Container,+            const Container&,+            const Allocator&>::value)+        : value_compare(static_cast<const value_compare&>(other)),+          cont_(other.cont_, alloc) {}+    EBO(EBO&& other, const Allocator& alloc) noexcept(+        std::is_nothrow_constructible<+            Container,+            Container&&,+            const Allocator&>::value)+        : value_compare(static_cast<value_compare&&>(other)),+          cont_(std::move(other.cont_), alloc) {}+    EBO(const Compare& c, Container&& cont) noexcept(+        std::is_nothrow_move_constructible<Container>::value)+        : value_compare(c), cont_(std::move(cont)) {}+    Container cont_;+  } m_;++  template <typename Self>+  using self_iterator_t = _t<+      std::conditional<std::is_const<Self>::value, const_iterator, iterator>>;++  template <typename Self, typename K>+  static self_iterator_t<Self> find_(Self& self, K const& key) {+    auto end = self.end();+    auto it = self.lower_bound(key);+    if (it == end || !self.key_comp()(key, it->first)) {+      return it;+    }+    return end;+  }++  template <typename Self, typename K>+  static self_iterator_t<Self> lower_bound(Self& self, K const& key) {+    auto f = [c = self.key_comp()](auto const& a, K const& b) {+      return c(a.first, b);+    };+    return std::lower_bound(self.begin(), self.end(), key, f);+  }++  template <typename Self, typename K>+  static self_iterator_t<Self> upper_bound(Self& self, K const& key) {+    auto f = [c = self.key_comp()](K const& a, auto const& b) {+      return c(a, b.first);+    };+    return std::upper_bound(self.begin(), self.end(), key, f);+  }++  template <typename Self, typename K>+  static std::pair<self_iterator_t<Self>, self_iterator_t<Self>> equal_range(+      Self& self, K const& key) {+    // Note: std::equal_range can't be passed a functor that takes+    // argument types different from the iterator value_type, so we+    // have to do this.+    return {lower_bound(self, key), upper_bound(self, key)};+  }++  template <typename Self, typename K>+  static std::pair<self_iterator_t<Self>, self_iterator_t<Self>> lower_bound2_(+      Self& self, K const& key1, K const& key2) {+    auto len = self.size();+    auto first = self.begin(), second = self.begin();+    auto c = self.key_comp();+    assert(!c(key2, key1));+    while (true) {+      if (len == 0) {+        return std::make_pair(first, first);+      }+      auto half = len / 2;+      auto middle = first + half;+      if (c(middle->first, key1)) {+        first = middle + 1;+        half = len - half - 1;+      } else if (c(middle->first, key2)) {+        second = middle + (len & 1);+        len = half;+        break;+      }+      len = half;+    }+    while (len) {+      auto half = len / 2;+      auto middle1 = first + half;+      auto middle2 = second + half;+      if (c(middle1->first, key1)) {+        first = middle1 + (len & 1);+      }+      if (c(middle2->first, key2)) {+        second = middle2 + (len & 1);+      }+      len = half;+    }+    return std::make_pair(first, second);+  }++  template <typename Self, typename K>+  static std::pair<self_iterator_t<Self>, self_iterator_t<Self>> find2_(+      Self& self, K const& key1, K const& key2) {+    auto end = self.end();+    auto its = lower_bound2_(self, key1, key2);+    if (its.second != end) {+      if (self.key_comp()(key1, its.first->first)) {+        its.first = end;+      }+      if (self.key_comp()(key2, its.second->first)) {+        its.second = end;+      }+    } else if (its.first != end && self.key_comp()(key1, its.first->first)) {+      its.first = end;+    }+    return its;+  }++  template <typename K, typename... Args>+  std::pair<iterator, bool> try_emplace_impl(K&& key, Args&&... args) {+    iterator it = lower_bound(key);+    if (it == end() || key_comp()(key, it->first)) {+      return std::make_pair(+          emplace_hint(+              it,+              std::piecewise_construct,+              std::forward_as_tuple(std::forward<K>(key)),+              std::forward_as_tuple(std::forward<Args>(args)...)),+          true);+    }+    return std::make_pair(it, false);+  }++  template <class K, class M>+  iterator insert_or_assign_impl(const_iterator hint, K&& k, M&& obj) {+    if (hint == end() || key_comp()(k, hint->first)) {+      if (hint == begin() || key_comp()((hint - 1)->first, k)) {+        auto it = get_growth_policy().increase_capacity(m_.cont_, hint);+        return m_.cont_.emplace(+            it, std::make_pair(std::forward<K>(k), std::forward<M>(obj)));+      } else {+        return insert_or_assign(std::forward<K>(k), std::forward<M>(obj)).first;+      }+    }++    if (key_comp()(hint->first, k)) {+      if (hint + 1 == end() || key_comp()(k, (hint + 1)->first)) {+        auto it = get_growth_policy().increase_capacity(m_.cont_, hint + 1);+        return m_.cont_.emplace(+            it, std::make_pair(std::forward<K>(k), std::forward<M>(obj)));+      } else {+        return insert_or_assign(std::forward<K>(k), std::forward<M>(obj)).first;+      }+    }++    // Value and *hint did not compare, so they are equal keys.+    auto it = begin() + std::distance(cbegin(), hint);+    it->second = std::forward<M>(obj);+    return it;+  }+};++// Swap function that can be found using ADL.+template <class K, class V, class C, class A, class G>+inline void swap(+    sorted_vector_map<K, V, C, A, G>& a, sorted_vector_map<K, V, C, A, G>& b) {+  return a.swap(b);+}++template <typename T>+inline constexpr bool is_sorted_vector_map_v =+    is_instantiation_of_v<sorted_vector_map, T>;++template <typename T>+struct is_sorted_vector_map : std::bool_constant<is_sorted_vector_map_v<T>> {};++template <+    class Key,+    class Value,+    size_t N = 1,+    class Compare = std::less<Key>,+    class Allocator = std::allocator<std::pair<Key, Value>>,+    class GrowthPolicy = void,+    class SmallVectorPolicy = void>+using small_sorted_vector_map = sorted_vector_map<+    Key,+    Value,+    Compare,+    Allocator,+    GrowthPolicy,+    folly::small_vector<std::pair<Key, Value>, N, SmallVectorPolicy>>;++template <typename T>+inline constexpr bool is_small_sorted_vector_map_v =+    is_sorted_vector_map_v<T> && is_small_vector_v<typename T::container_type>;++template <typename T>+struct is_small_sorted_vector_map+    : std::bool_constant<is_small_sorted_vector_map_v<T>> {};++#if FOLLY_HAS_MEMORY_RESOURCE++namespace pmr {++template <+    class Key,+    class Value,+    class Compare = std::less<Key>,+    class GrowthPolicy = void,+    class Container = std::vector<+        std::pair<Key, Value>,+        std::pmr::polymorphic_allocator<std::pair<Key, Value>>>>+using sorted_vector_map = folly::sorted_vector_map<+    Key,+    Value,+    Compare,+    std::pmr::polymorphic_allocator<std::pair<Key, Value>>,+    GrowthPolicy,+    Container>;++} // namespace pmr++#endif++//////////////////////////////////////////////////////////////////////++} // namespace folly
+ folly/folly/container/span.h view
@@ -0,0 +1,408 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <array>+#include <cassert>+#include <cstddef>+#include <limits>+#include <type_traits>++#include <folly/CppAttributes.h>+#include <folly/Portability.h>+#include <folly/Traits.h>+#include <folly/Utility.h>+#include <folly/container/Access.h>+#include <folly/container/Iterator.h>+#include <folly/functional/Invoke.h>+#include <folly/portability/Constexpr.h>++#if __cpp_lib_span >= 202002L+#include <span>+#endif++namespace folly {++namespace detail {++namespace fallback_span {++inline constexpr auto dynamic_extent = std::size_t(-1);++template <std::size_t N>+struct span_extent {+  constexpr span_extent() = default;+  explicit constexpr span_extent(+      [[maybe_unused]] std::size_t const e) noexcept {+    assert(e == N);+  }+  constexpr span_extent(span_extent const&) = default;+  constexpr span_extent& operator=(span_extent const&) = default;++  /* implicit */ constexpr operator std::size_t() const noexcept { return N; }+};++template <>+struct span_extent<dynamic_extent> {+  std::size_t extent{};++  constexpr span_extent() = default;+  explicit constexpr span_extent(std::size_t const e) noexcept : extent{e} {}+  constexpr span_extent(span_extent const&) = default;+  constexpr span_extent& operator=(span_extent const&) = default;++  /* implicit */ constexpr operator std::size_t() const noexcept {+    return extent;+  }+};++/// span+///+/// mimic: std::span, C++20+template <typename T, std::size_t Extent = dynamic_extent>+class span {+ public:+  static_assert(!std::is_reference_v<T>);+  static_assert(!std::is_void_v<T>);+  static_assert(!std::is_function_v<T>);+  static_assert(sizeof(T) < size_t(-1));+  static_assert(!std::is_abstract_v<T>);++  using element_type = T;+  using value_type = std::remove_cv_t<element_type>;+  using size_type = std::size_t;+  using difference_type = std::ptrdiff_t;+  using pointer = element_type*;+  using const_pointer = element_type const*;+  using reference = element_type&;+  using const_reference = element_type const&;+  using iterator = pointer;+  using reverse_iterator = std::reverse_iterator<iterator>;++  static inline constexpr std::size_t extent = Extent;++ private:+  template <bool C>+  using if_ = std::enable_if_t<C, int>;++  template <typename U, typename V = std::remove_cv_t<U>>+  static inline constexpr bool array_element_match_v =+      std::is_same_v<V, value_type> && std::is_convertible_v<U*, pointer>;++  template <+      typename Rng,+      typename Size = remove_cvref_t<invoke_result_t<access::size_fn, Rng>>,+      typename Data = invoke_result_t<access::data_fn, Rng>,+      typename U = std::remove_pointer_t<Data>>+  static constexpr bool is_range_v =+      !std::is_same_v<bool, Size> && std::is_unsigned_v<Size> &&+      std::is_pointer_v<Data> && array_element_match_v<U>;++  static constexpr size_type subspan_extent(+      size_type const offset, size_type const count) {+    // clang-format off+    return+        count != dynamic_extent ? count :+        extent != dynamic_extent ? extent - offset :+        dynamic_extent;+    // clang-format on+  }++  pointer data_;+  [[FOLLY_ATTR_NO_UNIQUE_ADDRESS]] span_extent<extent> extent_;++ public:+  template <size_type E = extent, if_<E == dynamic_extent || E == 0> = 0>+  constexpr span() noexcept : data_{}, extent_{} {}++  constexpr span(pointer const first, size_type const count)+      : data_{first}, extent_{count} {}++  constexpr span(pointer const first, pointer const last)+      : data_{first}, extent_{to_unsigned(last - first)} {+    assert(!(last < first));+  }++  template <+      std::size_t N,+      std::size_t E = extent,+      if_<E == dynamic_extent || E == N> = 0>+  /* implicit */ constexpr span(type_t<element_type> (&arr)[N]) noexcept+      : data_{arr}, extent_{N} {}++  template <+      typename U,+      std::size_t N,+      std::size_t E = extent,+      if_<E == dynamic_extent || E == N> = 0,+      if_<array_element_match_v<U>> = 0>+  /* implicit */ constexpr span(std::array<U, N>& arr) noexcept+      : data_{arr.data()}, extent_{N} {}++  template <+      typename U,+      std::size_t N,+      std::size_t E = extent,+      if_<E == dynamic_extent || E == N> = 0,+      if_<array_element_match_v<U const>> = 0>+  /* implicit */ constexpr span(std::array<U, N> const& arr) noexcept+      : data_{arr.data()}, extent_{N} {}++  template <typename Rng, if_<is_range_v<Rng&>> = 0>+  /* implicit */ constexpr span(Rng&& range)+      : data_{access::data(range)}, extent_{access::size(range)} {}++  constexpr span(span const&) = default;++  constexpr span& operator=(span const&) = default;++  constexpr pointer data() const noexcept { return data_; }+  constexpr size_type size() const noexcept { return extent_; }+  constexpr size_type size_bytes() const noexcept {+    return size() * sizeof(element_type);+  }+  constexpr bool empty() const noexcept { return size() == 0; }++  constexpr iterator begin() const noexcept { return data_; }+  constexpr iterator end() const noexcept { return data_ + size(); }+  constexpr reverse_iterator rbegin() const noexcept {+    return std::make_reverse_iterator(end());+  }+  constexpr reverse_iterator rend() const noexcept {+    return std::make_reverse_iterator(begin());+  }++  constexpr reference front() const {+    assert(!empty());+    return data_[0];+  }+  constexpr reference back() const {+    assert(!empty());+    return data_[size() - 1];+  }+  constexpr reference operator[](size_type const idx) const {+    assert(idx < size());+    return data_[idx];+  }++  template <+      size_type Offset,+      size_type Count = dynamic_extent,+      typename...,+      size_type E = subspan_extent(Offset, Count)>+  constexpr span<element_type, E> subspan() const {+    static_assert(!(Extent < Offset));+    static_assert(Count == dynamic_extent || !(extent - Offset < Count));+    assert(!(size() < Offset));+    assert(Count == dynamic_extent || !(size() - Offset < Count));+    return {data_ + Offset, Count == dynamic_extent ? size() - Offset : Count};+  }++  constexpr span<element_type, dynamic_extent> subspan(+      size_type const offset, size_type const count = dynamic_extent) const {+    assert(!(extent < offset));+    assert(count == dynamic_extent || !(extent - offset < count));+    assert(!(size() < offset));+    assert(count == dynamic_extent || !(size() - offset < count));+    return {data_ + offset, count == dynamic_extent ? size() - offset : count};+  }++  template <size_type Count>+  constexpr span<element_type, Count> first() const {+    static_assert(!(extent < Count));+    assert(!(size() < Count));+    return {data_, Count};+  }+  constexpr span<element_type, dynamic_extent> first(+      size_type const count) const {+    assert(!(extent < count));+    assert(!(size() < count));+    return {data_, count};+  }++  template <size_type Count>+  constexpr span<element_type, Count> last() const {+    static_assert(!(extent < Count));+    assert(!(size() < Count));+    return {data_ + size() - Count, Count};+  }+  constexpr span<element_type, dynamic_extent> last(+      size_type const count) const {+    assert(!(extent < count));+    assert(!(size() < count));+    return {data_ + size() - count, count};+  }+};++template <typename T, typename EndOrSize>+span(T*, EndOrSize) -> span<T>;++template <typename T, std::size_t N>+span(T (&)[N]) -> span<T, N>;++template <typename T, std::size_t N>+span(std::array<T, N>&) -> span<T, N>;++template <typename T, std::size_t N>+span(const std::array<T, N>&) -> span<const T, N>;++template <typename R>+span(R&&) -> span<std::remove_reference_t<+              iterator_reference_t<decltype(std::begin(std::declval<R&>()))>>>;++} // namespace fallback_span++} // namespace detail++#if __cpp_lib_span >= 202002L++using std::dynamic_extent;+using std::span;++#else++using detail::fallback_span::dynamic_extent;+using detail::fallback_span::span;++#endif++namespace detail {++struct span_cast_impl_fn {+  template <+      template <typename, std::size_t>+      class Span,+      typename U,+      typename T,+      std::size_t Extent>+  constexpr auto operator()(Span<T, Extent> in, U* castData) const {+    assert(+        static_cast<void const*>(in.data()) ==+        static_cast<void const*>(castData));++    // check alignment+    if (!folly::is_constant_evaluated_or(true)) {+      assert(reinterpret_cast<std::uintptr_t>(in.data()) % sizeof(U) == 0);+    }++    if constexpr (Extent == dynamic_extent) {+      assert(in.size() * sizeof(T) % sizeof(U) == 0);+      return Span<U, dynamic_extent>(+          castData, in.size() * sizeof(T) / sizeof(U));+    } else {+      static_assert(Extent * sizeof(T) % sizeof(U) == 0);+      constexpr std::size_t kResSize = Extent * sizeof(T) / sizeof(U);+      return Span<U, kResSize>(castData, kResSize);+    }+  }+};++inline constexpr span_cast_impl_fn span_cast_impl;++} // namespace detail++/// static_span_cast+/// static_span_cast_fn+/// reinterpret_span_cast+/// reinterpret_span_cast_fn+/// const_span_cast+/// const_span_cast_fn+///+/// Casts a span to a different span. The result is a span referring to the same+/// region in memory but as a different type.+///+/// Example:+///+///   std::span<std::byte> bytes = ...+///   std::span<int> ints = folly::reinterpret_span_cast<int>(bytes);++template <typename U>+struct static_span_cast_fn {+  template <typename T, std::size_t Extent>+  constexpr auto operator()(detail::fallback_span::span<T, Extent> in) const {+    return detail::span_cast_impl(in, static_cast<U*>(in.data()));+  }+#if __cpp_lib_span >= 202002L+  template <typename T, std::size_t Extent>+  constexpr auto operator()(std::span<T, Extent> in) const {+    return detail::span_cast_impl(in, static_cast<U*>(in.data()));+  }+#endif+};+template <typename U>+inline constexpr static_span_cast_fn<U> static_span_cast;++template <typename U>+struct reinterpret_span_cast_fn {+  template <typename T, std::size_t Extent>+  constexpr auto operator()(detail::fallback_span::span<T, Extent> in) const {+    return detail::span_cast_impl(in, reinterpret_cast<U*>(in.data()));+  }+#if __cpp_lib_span >= 202002L+  template <typename T, std::size_t Extent>+  constexpr auto operator()(std::span<T, Extent> in) const {+    return detail::span_cast_impl(in, reinterpret_cast<U*>(in.data()));+  }+#endif+};+template <typename U>+inline constexpr reinterpret_span_cast_fn<U> reinterpret_span_cast;++template <typename U>+struct const_span_cast_fn {+  template <typename T, std::size_t Extent>+  constexpr auto operator()(detail::fallback_span::span<T, Extent> in) const {+    return detail::span_cast_impl(in, const_cast<U*>(in.data()));+  }+#if __cpp_lib_span >= 202002L+  template <typename T, std::size_t Extent>+  constexpr auto operator()(std::span<T, Extent> in) const {+    return detail::span_cast_impl(in, const_cast<U*>(in.data()));+  }+#endif+};+template <typename U>+inline constexpr const_span_cast_fn<U> const_span_cast;++namespace detail {++namespace fallback_span {++/// as_bytes+///+/// mimic: std::as_bytes, C++20+template <typename T, std::size_t Extent>+auto as_bytes(span<T, Extent> s) noexcept {+  return reinterpret_span_cast<std::byte const>(s);+}++/// as_writable_bytes+///+/// mimic: std::as_writable_bytes, C++20+template <+    typename T,+    std::size_t Extent,+    std::enable_if_t<!std::is_const_v<T>, int> = 0>+auto as_writable_bytes(span<T, Extent> s) noexcept {+  return reinterpret_span_cast<std::byte>(s);+}++} // namespace fallback_span++} // namespace detail++} // namespace folly
+ folly/folly/container/tape.h view
@@ -0,0 +1,612 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <folly/Portability.h>+#include <folly/Range.h>+#include <folly/container/Iterator.h>+#include <folly/container/detail/tape_detail.h>+#include <folly/memory/UninitializedMemoryHacks.h>++#include <algorithm>+#include <cassert>+#include <initializer_list>+#include <iterator>+#include <numeric>+#include <string_view>+#include <type_traits>+#include <vector>++#if defined(__cpp_lib_ranges)+#include <ranges>+#endif++namespace folly {++#if defined(__cpp_lib_ranges)+#define FOLLY_TAPE_CONTAINER_REQUIRES std::ranges::random_access_range+#else+#define FOLLY_TAPE_CONTAINER_REQUIRES typename+#endif++/* # Tape+ *+ * A container adapter, that builds a version of `vector<vector>` on top of a+ * random access underlying container.+ *+ * Instead of having a container of containers it's more efficient to have+ * a single container and store where the separators are.+ *+ * [string second string third string]+ *  ^      ^             ^+ *+ * One subrange of internal elements we call a `record`.+ *+ * You can `push` a new `record` or pop one from the back.+ * We also support an `erase` like `std::vector` but `insert` only for one+ * element. (there is no reason for limitation, except it's not implemented).+ *+ * NOTE: for when you don't have the `record` ready, you can use a+ * `record_builder` interface.+ *+ * Existing `records` can be accessed by index.+ * Existing `records` cannot be mutated, except for the last record (see record+ * builder).+ *+ * ## tape<tape>+ *+ * tape<tape> is supported, though not all of the APIs.+ * More apis can be implemented if/when needed.+ * Use `record_builder`.+ *+ * ## PERFORMANCE CHARACTERISTICS (folly/container/test/tape_bench):+ *+ * Reading (cache miss):+ * Container performs much better for access than vector<vector>/vector<string>+ * for cases where the data is out of cache.+ * If the data is in cache, reading is roughly the same.+ *+ * Construction+ * If you know for a fact that all the elements are fitting into SSO buffer,+ * and you always have complete records (not building) then `tape` does not help+ * you, or can even be a slignt regression.+ *+ * Otherwise tape can give you good speedups, especially if you need to+ * `push_back` on individual records.+ *+ * Potential future perf improvements.+ * * it is possible to do a tape with one allocation for both metada and+ *   data (in special cases).+ * * when converting indexes to pointers, compiler has to shift.+ *   For contigious containers we can store offsets in bytes.+ *+ * ## Exception safety+ * We provide only basic exception safety: the object is destructible or+ * assignable.+ * `std::bad_alloc` is assumed to never happen (a function that uses malloc can+ * be marked noexcept).+ *+ * ## NAME TAPE+ *+ * Name tape is taken from a lecture by Alexander Stepanov but we are not 100%+ * sure if this is the container he had in mind.+ */+template <FOLLY_TAPE_CONTAINER_REQUIRES Container>+class tape;++// string_tape - a common usecase.+using string_tape = tape<std::vector<char>>;++template <FOLLY_TAPE_CONTAINER_REQUIRES Container>+class tape {+  using ref_traits = detail::tape_reference_traits<Container>;++ public:+  using container_type = Container;++  using const_reference = typename ref_traits::reference;+  using reference = const_reference;++  // value_type for tape does not make much sense.+  // The best we found is to make reference type to be value type.+  // This does not quite make sense but works well enough.+  using value_type = const_reference;+  using scalar_value_type = detail::maybe_range_value_t<container_type>;++  using size_type = typename Container::size_type;+  using difference_type = typename Container::difference_type;++  using iterator = folly::index_iterator<const tape>;+  using const_iterator = iterator;+  using reverse_iterator = std::reverse_iterator<iterator>;+  using const_reverse_iterator = std::reverse_iterator<const_iterator>;++  // concepts ------++  template <typename I>+  static constexpr bool iterator_of_scalars =+      std::is_convertible_v<iterator_value_type_t<I>, scalar_value_type>;++  template <typename I>+  static constexpr bool range_of_scalars =+      iterator_of_scalars<detail::maybe_range_const_iterator_t<I>>;++  template <typename I>+  static constexpr bool iterator_of_records =+      range_of_scalars<iterator_value_type_t<I>>;++  template <typename I>+  static constexpr bool range_of_records =+      iterator_of_records<detail::maybe_range_const_iterator_t<I>>;++  // rule of 5+  tape(const tape&) = default;+  tape& operator=(const tape&) = default;+  ~tape() = default;+  tape(tape&&) noexcept;+  tape& operator=(tape&&) noexcept;++  // constructors -----+  tape() noexcept = default;++  template <+      typename I,+      typename S,+      typename = std::enable_if_t<iterator_of_records<I>>>+  explicit tape(I f, S l) {+    range_constructor(f, l);+  }++  template <+      typename R,+      typename = std::enable_if_t<+          std::is_convertible_v<R, const_reference> || // const char*+          range_of_records<R>>>+  explicit tape(std::initializer_list<R> il) {+    range_constructor(il.begin(), il.end());+  }++  // access ------++  [[nodiscard]] const_reference operator[](size_type i) const noexcept {+    return ref_traits::make(+        data_.begin() + markers_[i], data_.begin() + markers_[i + 1]);+  }++  [[nodiscard]] const_reference at(size_type i) const {+    if (FOLLY_UNLIKELY(i >= size())) {+      // libc++ doesn't provide index. This helps optimizations.+      throw std::out_of_range("tape");+    }+    return operator[](i);+  }++  [[nodiscard]] bool empty() const noexcept { return size() == 0; }+  [[nodiscard]] size_type size() const noexcept { return markers_.size() - 1; }+  [[nodiscard]] size_type size_flat() const noexcept { return data_.size(); }++  [[nodiscard]] const_reference front() const noexcept { return operator[](0); }+  [[nodiscard]] const_reference back() const noexcept {+    return operator[](size() - 1);+  }++  // iterators ----++  [[nodiscard]] const_iterator begin() const noexcept { return {*this, 0}; }+  [[nodiscard]] const_iterator cbegin() const noexcept { return begin(); }++  [[nodiscard]] const_iterator end() const noexcept { return {*this, size()}; }+  [[nodiscard]] const_iterator cend() const noexcept { return end(); }++  [[nodiscard]] auto rbegin() const noexcept {+    return const_reverse_iterator{end()};+  }+  [[nodiscard]] auto crbegin() const noexcept { return rbegin(); }++  [[nodiscard]] auto rend() const noexcept {+    return const_reverse_iterator{begin()};+  }+  [[nodiscard]] auto crend() const noexcept { return rend(); }++  // push / emplace_back --------++  template <typename I, typename S>+  auto push_back(I f, S l) -> std::enable_if_t<iterator_of_scalars<I>> {+    data_.insert(data_.end(), f, l);+    markers_.push_back(static_cast<difference_type>(data_.size()));+  }++  template <typename R>+  auto push_back(R&& r)+      -> std::enable_if_t<+          range_of_scalars<R> &&+          !std::is_convertible_v<R, const_reference>> // handle \0 separately+  {+    push_back(std::begin(r), std::end(r));+  }++  void push_back(const_reference r) { push_back(r.begin(), r.end()); }++  void push_back(std::initializer_list<scalar_value_type> r) {+    push_back(r.begin(), r.end());+  }++  void emplace_back() { push_back({}); }++  template <typename... Args>+  void emplace_back(Args&&... args) {+    push_back(std::forward<Args>(args)...);+  }++  // push_back_unsafe --------+  // like push_back but requires you to have enough capacity for added range.+  // happened to give a 2x performance improvements on certain benchmarks.++  // requires to have enough capacity+  template <typename I, typename S>+  auto push_back_unsafe(I f, S l) -> std::enable_if_t<iterator_of_scalars<I>> {+    // basic exception guarantee is preserved here.+    detail::append_range_unsafe(data_, f, l);+    markers_.push_back(static_cast<difference_type>(data_.size()));+  }++  template <typename R>+  auto push_back_unsafe(R&& r)+      -> std::enable_if_t<+          range_of_scalars<R> &&+          !std::is_convertible_v<R, const_reference>> // handle \0 separately+  {+    push_back_unsafe(std::begin(r), std::end(r));+  }++  void push_back_unsafe(const_reference r) {+    push_back_unsafe(r.begin(), r.end());+  }++  // record builder (constructing last record) -------++  class record_builder;++  // get a record builder.+  // new_record_builder starts a builder for a new record.+  // last_record_builder allows you to append/mutate the last record.+  [[nodiscard]] record_builder new_record_builder();+  [[nodiscard]] record_builder last_record_builder();++  // insert one record ----------++  template <typename I, typename S>+  auto insert(const_iterator pos, I f, S l)+      -> std::enable_if_t<iterator_of_scalars<I>, iterator>;++  template <typename R>+  auto insert(const_iterator pos, R&& r)+      -> std::enable_if_t<+          range_of_scalars<R> && !std::is_convertible_v<R, const_reference>,+          iterator> {+    return insert(pos, std::begin(r), std::end(r));+  }++  iterator insert(+      const_iterator pos, std::initializer_list<scalar_value_type> r) {+    return insert(pos, r.begin(), r.end());+  }++  iterator insert(const_iterator pos, const_reference r) {+    return insert(pos, r.begin(), r.end());+  }++  // capacity ------+  void reserve(size_type records, size_type elements) {+    markers_.reserve(records + 1);+    data_.reserve(elements);+  }++  // assumes that 1 element per record. This is likely to help a bit.+  void reserve(size_type records) {+    markers_.reserve(records + 1);+    data_.reserve(records);+  }++  void shrink_to_fit() {+    markers_.shrink_to_fit();+    data_.shrink_to_fit();+  }++  // resize/clear -------++  // same args as for push_back/emplace back are accepted+  template <typename... Args>+  void resize(size_type new_size, const Args&... args);++  void clear() noexcept {+    markers_.resize(1);+    data_.clear();+  }++  // erase -------++  void pop_back() noexcept {+    assert(!empty());+    data_.resize(data_.size() - back().size());+    markers_.pop_back();+  }++  // note: same behaviour as for std::vector, erasing end() is UB+  iterator erase(const_iterator pos) {+    assert(pos != end());+    return erase(pos, pos + 1);+  }++  iterator erase(const_iterator f, const_iterator l);++  // ordering --------++  friend bool operator==(const tape& x, const tape& y) {+    return x.markers_ == y.markers_ && x.data_ == y.data_;+  }++  friend bool operator!=(const tape& x, const tape& y) { return !(x == y); }++  friend bool operator<(const tape& x, const tape& y) {+    return std::lexicographical_compare(x.begin(), x.end(), y.begin(), y.end());+  }++  friend bool operator>(const tape& x, const tape& y) { return y < x; }+  friend bool operator<=(const tape& x, const tape& y) { return !(y < x); }+  friend bool operator>=(const tape& x, const tape& y) { return !(x < y); }++  folly::Range<const difference_type*> markers() const { return markers_; }+  reference scalars() const {+    return ref_traits::make(data_.begin(), data_.end());+  }++ private:+  template <typename I, typename S>+  void range_constructor(I f, S l);++  // NOTE: using container difference_type might be too much here but,+  // on the other hand, there should be reasonably few items on the tape and+  // this makes interface simpler.+  std::vector<difference_type> markers_ = {0};+  container_type data_;+};++// Provides a way to construct a last record similar+// to how you would `std::vector`.+// Typical workflow is you `push_back` a bunch of individual elements and then+// `commit()`.+template <FOLLY_TAPE_CONTAINER_REQUIRES Container>+class tape<Container>::record_builder {+ public:+  record_builder(const record_builder&) = delete;+  record_builder(record_builder&&) = delete;+  record_builder& operator=(const record_builder&) = delete;+  record_builder& operator=(record_builder&&) = delete;++  using iterator = typename container_type::iterator;+  using const_iterator = typename container_type::const_iterator;+  using reference = typename std::iterator_traits<iterator>::reference;+  using const_reference =+      typename std::iterator_traits<const_iterator>::reference;+  using size_type = typename container_type::size_type;+  using difference_type =+      typename std::iterator_traits<iterator>::difference_type;++  // mutators ---++  void push_back(scalar_value_type x) { self_->data_.push_back(std::move(x)); }++  template <typename... Args>+  reference emplace_back(Args&&... args) {+    self_->data_.emplace_back(std::forward<Args>(args)...);+    // cannot rely on the container doing the right thing here.+    return self_->data_.back();+  }++  // constructed record is added to the tape.+  void commit() { self_->markers_.push_back(self_->data_.size()); }++  // discards elements of the constructed record. (automatic on destruction)+  void abort() { self_->data_.resize(self_->markers_.back()); }++  // iterators -----++  [[nodiscard]] iterator begin() noexcept {+    return self_->data_.begin() + self_->markers_.back();+  }+  [[nodiscard]] const_iterator begin() const noexcept {+    return self_->data_.cbegin() + self_->markers_.back();+  }+  [[nodiscard]] const_iterator cbegin() const noexcept { return begin(); }++  [[nodiscard]] iterator end() noexcept { return self_->data_.end(); }+  [[nodiscard]] const_iterator end() const noexcept {+    return self_->data_.cend();+  }+  [[nodiscard]] const_iterator cend() const noexcept { return end(); }++  // sometimes functions (like fmt) optimize for vector back inserter.+  // so better expose that.+  [[nodiscard]] auto back_inserter() noexcept {+    return std::back_inserter(self_->data_);+  }++  // access ---++  [[nodiscard]] bool empty() const noexcept { return begin() == end(); }++  [[nodiscard]] size_type size() const noexcept {+    return static_cast<size_type>(end() - begin());+  }++  [[nodiscard]] reference operator[](size_type i) noexcept {+    return begin()[static_cast<difference_type>(i)];+  }++  [[nodiscard]] const_reference operator[](size_type i) const noexcept {+    return begin()[static_cast<difference_type>(i)];+  }++  [[nodiscard]] reference at(size_type i) {+    if (FOLLY_UNLIKELY(i >= size())) {+      // libc++ doesn't provide index. This helps optimizations.+      throw std::out_of_range("tape::scoped_record_builder");+    }+    return operator[](i);+  }++  [[nodiscard]] const_reference at(size_type i) const {+    if (FOLLY_UNLIKELY(i >= size())) {+      // libc++ doesn't provide index. This helps optimizations.+      throw std::out_of_range("tape::scoped_record_builder");+    }+    return operator[](i);+  }++  [[nodiscard]] reference back() { return self_->data_.back(); }+  [[nodiscard]] const_reference back() const { return self_->data_.back(); }++  ~record_builder() noexcept { abort(); }++ private:+  friend class tape;++  explicit record_builder(tape& self) : self_(&self) {}++  tape* self_;+};++template <FOLLY_TAPE_CONTAINER_REQUIRES Container>+auto tape<Container>::new_record_builder() -> record_builder {+  return record_builder{*this};+}++template <FOLLY_TAPE_CONTAINER_REQUIRES Container>+auto tape<Container>::last_record_builder() -> record_builder {+  assert(!empty());+  markers_.pop_back();+  return new_record_builder();+}++// tape methods -----++template <FOLLY_TAPE_CONTAINER_REQUIRES Container>+tape<Container>::tape(tape&& x) noexcept+    : markers_(std::move(x.markers_)), data_(std::move(x.data_)) {+  // we assume that allocations never fail+  x.markers_ = {0};+  x.data_.clear();+}++template <FOLLY_TAPE_CONTAINER_REQUIRES Container>+tape<Container>& tape<Container>::operator=(tape&& x) noexcept {+  if (this != &x) {+    markers_ = std::move(x.markers_);+    data_ = std::move(x.data_);+  }+  // we assume that allocations never fail+  x.markers_ = {0};+  x.data_.clear();+  return *this;+}++template <FOLLY_TAPE_CONTAINER_REQUIRES Container>+template <typename I, typename S>+void tape<Container>::range_constructor(I f, S l) {+  if constexpr (auto maybe = detail::compute_total_tape_len_if_possible(f, l);+                std::is_same_v<decltype(maybe), detail::fake_type>) {+    while (f != l) {+      push_back(*f);+      ++f;+    }+  } else {+    auto [nrecords, total_len] = maybe;+    reserve(nrecords, total_len);++    while (f != l) {+      push_back_unsafe(*f);+      ++f;+    }+  }+}++template <FOLLY_TAPE_CONTAINER_REQUIRES Container>+template <typename... Args>+void tape<Container>::resize(size_type new_size, const Args&... args) {+  if (new_size >= size()) {+    new_size -= size();+    while (new_size--) {+      emplace_back(args...);+    }+    return;+  }++  data_.resize(markers_[new_size]);+  markers_.resize(new_size + 1);+}++template <FOLLY_TAPE_CONTAINER_REQUIRES Container>+template <typename I, typename S>+auto tape<Container>::insert(const_iterator pos, I f, S l)+    -> std::enable_if_t<iterator_of_scalars<I>, iterator> {+  auto data_pos = data_.begin() + markers_[pos.get_index()];+  size_type old_size = data_.size();+  data_.insert(data_pos, f, l);++  auto inserted_len = static_cast<difference_type>(data_.size() - old_size);++  difference_type start = markers_[pos.get_index()];++  auto markers_tail =+      markers_.insert(markers_.begin() + pos.get_index(), start);+  ++markers_tail;++  std::transform(+      markers_tail, markers_.end(), markers_tail, [&](difference_type m) {+        return m + inserted_len;+      });++  // both tape* and index stayed the same+  return pos;+}++template <FOLLY_TAPE_CONTAINER_REQUIRES Container>+auto tape<Container>::erase(const_iterator f, const_iterator l) -> iterator {+  difference_type from = f.get_index();+  difference_type to = l.get_index();++  auto markers_f = markers_.begin() + from;+  auto markers_l = markers_.begin() + to;+  auto data_f = data_.begin() + *markers_f;+  auto data_l = data_.begin() + *markers_l;++  std::ptrdiff_t removed_length = data_l - data_f;+  std::transform(markers_l, markers_.end(), markers_l, [&](difference_type m) {+    return m - removed_length;+  });++  markers_.erase(markers_f, markers_l);+  data_.erase(data_f, data_l);++  // both tape* and index stayed the same+  return f;+}++#undef FOLLY_TAPE_CONTAINER_REQUIRES++} // namespace folly
+ folly/folly/container/test/F14TestUtil.h view
@@ -0,0 +1,118 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <cstddef>+#include <ostream>+#include <type_traits>+#include <vector>++#include <folly/container/detail/F14Policy.h>+#include <folly/container/detail/F14Table.h>++namespace folly {+namespace f14 {++struct Histo {+  std::vector<std::size_t> const& data;+};++inline std::ostream& operator<<(std::ostream& xo, Histo const& histo) {+  xo << "[";+  size_t sum = 0;+  for (auto v : histo.data) {+    sum += v;+  }+  auto const dsum = static_cast<double>(sum);+  size_t partial = 0;+  for (size_t i = 0; i < histo.data.size(); ++i) {+    if (i > 0) {+      xo << ", ";+    }+    partial += histo.data[i];+    if (histo.data[i] > 0) {+      xo << i << ": " << histo.data[i] << " ("+         << (static_cast<double>(partial) * 100.0 / dsum) << "%)";+    }+  }+  xo << "]";+  return xo;+}++inline double expectedProbe(std::vector<std::size_t> const& probeLengths) {+  std::size_t sum = 0;+  std::size_t count = 0;+  for (std::size_t i = 1; i < probeLengths.size(); ++i) {+    sum += i * probeLengths[i];+    count += probeLengths[i];+  }+  return static_cast<double>(sum) / static_cast<double>(count);+}++// Returns i such that probeLengths elements 0 to i (inclusive) account+// for at least 99% of the samples.+inline std::size_t p99Probe(std::vector<std::size_t> const& probeLengths) {+  std::size_t count = 0;+  for (std::size_t i = 1; i < probeLengths.size(); ++i) {+    count += probeLengths[i];+  }+  std::size_t rv = probeLengths.size();+  std::size_t suffix = 0;+  while ((suffix + probeLengths[rv - 1]) * 100 <= count) {+    --rv;+  }+  return rv;+}++inline std::ostream& operator<<(std::ostream& xo, F14TableStats const& stats) {+  xo << "{ " << std::endl;+  xo << "  policy: " << stats.policy << std::endl;+  xo << "  size: " << stats.size << std::endl;+  xo << "  valueSize: " << stats.valueSize << std::endl;+  xo << "  bucketCount: " << stats.bucketCount << std::endl;+  xo << "  chunkCount: " << stats.chunkCount << std::endl;+  xo << "  chunkOccupancyHisto" << Histo{stats.chunkOccupancyHisto}+     << std::endl;+  xo << "  chunkOutboundOverflowHisto"+     << Histo{stats.chunkOutboundOverflowHisto} << std::endl;+  xo << "  chunkHostedOverflowHisto" << Histo{stats.chunkHostedOverflowHisto}+     << std::endl;+  xo << "  keyProbeLengthHisto" << Histo{stats.keyProbeLengthHisto}+     << std::endl;+  xo << "  missProbeLengthHisto" << Histo{stats.missProbeLengthHisto}+     << std::endl;+  xo << "  totalBytes: " << stats.totalBytes << std::endl;+  xo << "  valueBytes: " << (stats.size * stats.valueSize) << std::endl;+  xo << "  overheadBytes: " << stats.overheadBytes << std::endl;+  if (stats.size > 0) {+    xo << "  overheadBytesPerKey: "+       << (static_cast<double>(stats.overheadBytes) /+           static_cast<double>(stats.size))+       << std::endl;+  }+  xo << "}";+  return xo;+}++template <typename Container>+std::vector<typename std::decay_t<Container>::value_type> asVector(+    const Container& c) {+  return {c.begin(), c.end()};+}++} // namespace f14+} // namespace folly
+ folly/folly/container/test/TrackingTypes.h view
@@ -0,0 +1,584 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <cstddef>+#include <limits>+#include <memory>+#include <ostream>++#include <folly/Function.h>+#include <folly/hash/Hash.h>+#include <folly/lang/SafeAssert.h>+#include <folly/portability/Asm.h>++namespace folly {+namespace test {++struct MoveOnlyTestInt {+  int x;+  bool destroyed{false};++  MoveOnlyTestInt() noexcept : x(0) {}+  /* implicit */ MoveOnlyTestInt(int x0) : x(x0) {}+  MoveOnlyTestInt(MoveOnlyTestInt&& rhs) noexcept : x(rhs.x) {}+  MoveOnlyTestInt(MoveOnlyTestInt const&) = delete;+  MoveOnlyTestInt& operator=(MoveOnlyTestInt&& rhs) noexcept {+    FOLLY_SAFE_CHECK(!rhs.destroyed, "");+    x = rhs.x;+    return *this;+  }+  MoveOnlyTestInt& operator=(MoveOnlyTestInt const&) = delete;++  ~MoveOnlyTestInt() {+    FOLLY_SAFE_CHECK(!destroyed, "");+    destroyed = true;+    asm_volatile_memory(); // try to keep compiler from eliding the store+  }++  bool operator==(MoveOnlyTestInt const& rhs) const {+    FOLLY_SAFE_CHECK(!destroyed, "");+    FOLLY_SAFE_CHECK(!rhs.destroyed, "");+    return x == rhs.x && destroyed == rhs.destroyed;+  }+  bool operator!=(MoveOnlyTestInt const& rhs) const { return !(*this == rhs); }+};++struct ThrowOnCopyTestInt {+  int x{0};++  ThrowOnCopyTestInt() {}++  [[noreturn]] ThrowOnCopyTestInt(const ThrowOnCopyTestInt& other)+      : x(other.x) {+    throw std::exception{};+  }++  ThrowOnCopyTestInt& operator=(const ThrowOnCopyTestInt&) {+    throw std::exception{};+  }++  bool operator==(const ThrowOnCopyTestInt& other) const {+    return x == other.x;+  }++  bool operator!=(const ThrowOnCopyTestInt& other) const {+    return !(x == other.x);+  }+};++struct PermissiveConstructorTestInt {+  int x;++  PermissiveConstructorTestInt() noexcept : x(0) {}+  /* implicit */ PermissiveConstructorTestInt(int x0) : x(x0) {}++  template <typename T>+  /* implicit */ PermissiveConstructorTestInt(T&& src)+      : x(std::forward<T>(src)) {}++  PermissiveConstructorTestInt(PermissiveConstructorTestInt&& rhs) noexcept+      : x(rhs.x) {}+  PermissiveConstructorTestInt(PermissiveConstructorTestInt const&) = delete;+  PermissiveConstructorTestInt& operator=(+      PermissiveConstructorTestInt&& rhs) noexcept {+    x = rhs.x;+    return *this;+  }+  PermissiveConstructorTestInt& operator=(PermissiveConstructorTestInt const&) =+      delete;++  bool operator==(PermissiveConstructorTestInt const& rhs) const {+    return x == rhs.x;+  }+  bool operator!=(PermissiveConstructorTestInt const& rhs) const {+    return !(*this == rhs);+  }+};++// Tracked is implicitly constructible across tags+struct Counts {+  uint64_t copyConstruct{0};+  uint64_t moveConstruct{0};+  uint64_t copyConvert{0};+  uint64_t moveConvert{0};+  uint64_t copyAssign{0};+  uint64_t moveAssign{0};+  uint64_t defaultConstruct{0};+  uint64_t destroyed{0};++  explicit Counts(+      uint64_t copConstr = 0,+      uint64_t movConstr = 0,+      uint64_t copConv = 0,+      uint64_t movConv = 0,+      uint64_t copAssign = 0,+      uint64_t movAssign = 0,+      uint64_t def = 0,+      uint64_t destr = 0)+      : copyConstruct{copConstr},+        moveConstruct{movConstr},+        copyConvert{copConv},+        moveConvert{movConv},+        copyAssign{copAssign},+        moveAssign{movAssign},+        defaultConstruct{def},+        destroyed{destr} {}++  int64_t liveCount() const {+    return copyConstruct + moveConstruct + copyConvert + moveConvert ++        defaultConstruct - destroyed;+  }++  // dist ignores destroyed count+  uint64_t dist(Counts const& rhs) const {+    auto d = [](uint64_t x, uint64_t y) { return (x - y) * (x - y); };+    return d(copyConstruct, rhs.copyConstruct) ++        d(moveConstruct, rhs.moveConstruct) + d(copyConvert, rhs.copyConvert) ++        d(moveConvert, rhs.moveConvert) + d(copyAssign, rhs.copyAssign) ++        d(moveAssign, rhs.moveAssign) ++        d(defaultConstruct, rhs.defaultConstruct);+  }++  bool operator==(Counts const& rhs) const {+    return dist(rhs) == 0 && destroyed == rhs.destroyed;+  }+  bool operator!=(Counts const& rhs) const { return !(*this == rhs); }+};++inline std::ostream& operator<<(std::ostream& xo, Counts const& counts) {+  xo << "[";+  std::string glue = "";+  if (counts.copyConstruct > 0) {+    xo << glue << counts.copyConstruct << " copy";+    glue = ", ";+  }+  if (counts.moveConstruct > 0) {+    xo << glue << counts.moveConstruct << " move";+    glue = ", ";+  }+  if (counts.copyConvert > 0) {+    xo << glue << counts.copyConvert << " copy convert";+    glue = ", ";+  }+  if (counts.moveConvert > 0) {+    xo << glue << counts.moveConvert << " move convert";+    glue = ", ";+  }+  if (counts.copyAssign > 0) {+    xo << glue << counts.copyAssign << " copy assign";+    glue = ", ";+  }+  if (counts.moveAssign > 0) {+    xo << glue << counts.moveAssign << " move assign";+    glue = ", ";+  }+  if (counts.defaultConstruct > 0) {+    xo << glue << counts.defaultConstruct << " default construct";+    glue = ", ";+  }+  if (counts.destroyed > 0) {+    xo << glue << counts.destroyed << " destroyed";+    glue = ", ";+  }+  xo << "]";+  return xo;+}++inline Counts& sumCounts() {+  static thread_local Counts value{};+  return value;+}++template <int Tag>+struct Tracked {+  static_assert(Tag <= 5, "Need to extend Tracked<Tag> in TestUtil.cpp");++  static Counts& counts() {+    static thread_local Counts value{};+    return value;+  }++  uint64_t val_;++  Tracked() : val_{0} {+    sumCounts().defaultConstruct++;+    counts().defaultConstruct++;+  }+  /* implicit */ Tracked(uint64_t const& val) : val_{val} {+    sumCounts().copyConvert++;+    counts().copyConvert++;+  }+  /* implicit */ Tracked(uint64_t&& val) : val_{val} {+    sumCounts().moveConvert++;+    counts().moveConvert++;+  }+  Tracked(Tracked const& rhs) : val_{rhs.val_} {+    sumCounts().copyConstruct++;+    counts().copyConstruct++;+  }+  Tracked(Tracked&& rhs) noexcept : val_{rhs.val_} {+    sumCounts().moveConstruct++;+    counts().moveConstruct++;+  }+  Tracked& operator=(Tracked const& rhs) {+    val_ = rhs.val_;+    sumCounts().copyAssign++;+    counts().copyAssign++;+    return *this;+  }+  Tracked& operator=(Tracked&& rhs) noexcept {+    val_ = rhs.val_;+    sumCounts().moveAssign++;+    counts().moveAssign++;+    return *this;+  }++  template <int T>+  /* implicit */ Tracked(Tracked<T> const& rhs) : val_{rhs.val_} {+    sumCounts().copyConvert++;+    counts().copyConvert++;+  }++  template <int T>+  /* implicit */ Tracked(Tracked<T>&& rhs) : val_{rhs.val_} {+    sumCounts().moveConvert++;+    counts().moveConvert++;+  }++  ~Tracked() {+    sumCounts().destroyed++;+    counts().destroyed++;+  }++  bool operator==(Tracked const& rhs) const { return val_ == rhs.val_; }+  bool operator!=(Tracked const& rhs) const { return !(*this == rhs); }+};++template <int Tag>+struct TransparentTrackedHash {+  using is_transparent = void;++  size_t operator()(Tracked<Tag> const& tracked) const {+    return tracked.val_ ^ Tag;+  }+  size_t operator()(uint64_t v) const { return v ^ Tag; }+};++template <int Tag>+struct TransparentTrackedEqual {+  using is_transparent = void;++  uint64_t unwrap(Tracked<Tag> const& v) const { return v.val_; }+  uint64_t unwrap(uint64_t v) const { return v; }++  template <typename A, typename B>+  bool operator()(A const& lhs, B const& rhs) const {+    return unwrap(lhs) == unwrap(rhs);+  }+};++inline size_t& testAllocatedMemorySize() {+  static thread_local size_t value{0};+  return value;+}++inline size_t& testAllocatedBlockCount() {+  static thread_local size_t value{0};+  return value;+}++inline size_t& testAllocationCount() {+  static thread_local size_t value{0};+  return value;+}++inline size_t& testAllocationMaxCount() {+  static thread_local size_t value{std::numeric_limits<std::size_t>::max()};+  return value;+}++inline void limitTestAllocations(std::size_t allocationsBeforeException = 0) {+  testAllocationMaxCount() = testAllocationCount() + allocationsBeforeException;+}++inline void unlimitTestAllocations() {+  testAllocationMaxCount() = std::numeric_limits<std::size_t>::max();+}++inline void resetTracking() {+  sumCounts() = Counts{};+  Tracked<0>::counts() = Counts{};+  Tracked<1>::counts() = Counts{};+  Tracked<2>::counts() = Counts{};+  Tracked<3>::counts() = Counts{};+  Tracked<4>::counts() = Counts{};+  Tracked<5>::counts() = Counts{};+  testAllocatedMemorySize() = 0;+  testAllocatedBlockCount() = 0;+  testAllocationCount() = 0;+  testAllocationMaxCount() = std::numeric_limits<std::size_t>::max();+}++template <class T>+class SwapTrackingAlloc {+ public:+  using Alloc = std::allocator<T>;+  using AllocTraits = std::allocator_traits<Alloc>;+  using value_type = typename AllocTraits::value_type;++  using pointer = typename AllocTraits::pointer;+  using const_pointer = typename AllocTraits::const_pointer;+  using reference = value_type&;+  using const_reference = value_type const&;+  using size_type = typename AllocTraits::size_type;++  using propagate_on_container_swap = std::true_type;+  using propagate_on_container_copy_assignment = std::true_type;+  using propagate_on_container_move_assignment = std::true_type;++  SwapTrackingAlloc() {}++  template <class U>+  /* implicit */ SwapTrackingAlloc(SwapTrackingAlloc<U> const& other) noexcept+      : a_(other.a_), t_(other.t_) {}++  template <class U>+  SwapTrackingAlloc& operator=(SwapTrackingAlloc<U> const& other) noexcept {+    a_ = other.a_;+    t_ = other.t_;+    return *this;+  }++  template <class U>+  /* implicit */ SwapTrackingAlloc(SwapTrackingAlloc<U>&& other) noexcept+      : a_(std::move(other.a_)), t_(std::move(other.t_)) {}++  template <class U>+  SwapTrackingAlloc& operator=(SwapTrackingAlloc<U>&& other) noexcept {+    a_ = std::move(other.a_);+    t_ = std::move(other.t_);+    return *this;+  }++  T* allocate(size_t n) {+    if (testAllocationCount() >= testAllocationMaxCount()) {+      throw std::bad_alloc();+    }+    ++testAllocationCount();+    testAllocatedMemorySize() += n * sizeof(T);+    ++testAllocatedBlockCount();+    std::size_t extra =+        std::max<std::size_t>(1, sizeof(std::size_t) / sizeof(T));+    T* p = a_.allocate(extra + n);+    void* raw = static_cast<void*>(p);+    *static_cast<std::size_t*>(raw) = n;+    return p + extra;+  }+  void deallocate(T* p, size_t n) {+    testAllocatedMemorySize() -= n * sizeof(T);+    --testAllocatedBlockCount();+    std::size_t extra =+        std::max<std::size_t>(1, sizeof(std::size_t) / sizeof(T));+    std::size_t check;+    void* raw = static_cast<void*>(p - extra);+    check = *static_cast<std::size_t*>(raw);+    FOLLY_SAFE_CHECK(check == n, "");+    a_.deallocate(p - extra, n + extra);+  }++ private:+  std::allocator<T> a_;+  Tracked<0> t_;++  template <class U>+  friend class SwapTrackingAlloc;+};++template <class T>+void swap(SwapTrackingAlloc<T>&, SwapTrackingAlloc<T>&) noexcept {+  // For argument dependent lookup:+  // This function will be called if the custom swap functions of a container+  // is used. Otherwise, std::swap() will do 1 move construct and 2 move+  // assigns which will get tracked by t_.+}++template <class T1, class T2>+bool operator==(SwapTrackingAlloc<T1> const&, SwapTrackingAlloc<T2> const&) {+  return true;+}++template <class T1, class T2>+bool operator!=(SwapTrackingAlloc<T1> const&, SwapTrackingAlloc<T2> const&) {+  return false;+}++template <class T>+class GenericAlloc {+ public:+  using value_type = T;++  using pointer = T*;+  using const_pointer = T const*;+  using reference = T&;+  using const_reference = T const&;+  using size_type = std::size_t;++  using propagate_on_container_swap = std::true_type;+  using propagate_on_container_copy_assignment = std::true_type;+  using propagate_on_container_move_assignment = std::true_type;++  using AllocBytesFunc = folly::Function<void*(std::size_t)>;+  using DeallocBytesFunc = folly::Function<void(void*, std::size_t)>;++  GenericAlloc() = delete;++  template <typename A, typename D>+  GenericAlloc(A&& alloc, D&& dealloc)+      : alloc_{std::make_shared<AllocBytesFunc>(std::forward<A>(alloc))},+        dealloc_{std::make_shared<DeallocBytesFunc>(std::forward<D>(dealloc))} {+  }++  template <class U>+  /* implicit */ GenericAlloc(GenericAlloc<U> const& other) noexcept+      : alloc_{other.alloc_}, dealloc_{other.dealloc_} {}++  template <class U>+  GenericAlloc& operator=(GenericAlloc<U> const& other) noexcept {+    alloc_ = other.alloc_;+    dealloc_ = other.dealloc_;+    return *this;+  }++  template <class U>+  /* implicit */ GenericAlloc(GenericAlloc<U>&& other) noexcept+      : alloc_(std::move(other.alloc_)), dealloc_(std::move(other.dealloc_)) {}++  template <class U>+  GenericAlloc& operator=(GenericAlloc<U>&& other) noexcept {+    alloc_ = std::move(other.alloc_);+    dealloc_ = std::move(other.dealloc_);+    return *this;+  }++  T* allocate(size_t n) { return static_cast<T*>((*alloc_)(n * sizeof(T))); }+  void deallocate(T* p, size_t n) {+    (*dealloc_)(static_cast<void*>(p), n * sizeof(T));+  }++  template <typename U>+  bool operator==(GenericAlloc<U> const& rhs) const {+    return alloc_ == rhs.alloc_;+  }++  template <typename U>+  bool operator!=(GenericAlloc<U> const& rhs) const {+    return !(*this == rhs);+  }++ private:+  std::shared_ptr<AllocBytesFunc> alloc_;+  std::shared_ptr<DeallocBytesFunc> dealloc_;++  template <class U>+  friend class GenericAlloc;+};++template <typename T>+class GenericEqual {+ public:+  using EqualFunc = folly::Function<bool(T const&, T const&)>;++  GenericEqual() = delete;++  template <typename E>+  /* implicit */ GenericEqual(E&& equal)+      : equal_{std::make_shared<EqualFunc>(std::forward<E>(equal))} {}++  bool operator()(T const& lhs, T const& rhs) const {+    return (*equal_)(lhs, rhs);+  }++ private:+  std::shared_ptr<EqualFunc> equal_;+};++template <typename T>+class GenericHasher {+ public:+  using HasherFunc = folly::Function<std::size_t(T const&)>;++  GenericHasher() = delete;++  template <typename H>+  /* implicit */ GenericHasher(H&& hasher)+      : hasher_{std::make_shared<HasherFunc>(std::forward<H>(hasher))} {}++  std::size_t operator()(T const& val) const { return (*hasher_)(val); }++ private:+  std::shared_ptr<HasherFunc> hasher_;+};++struct HashFirst {+  template <typename P>+  std::size_t operator()(P const& p) const {+    return folly::Hash{}(p.first);+  }+};++struct EqualFirst {+  template <typename P>+  bool operator()(P const& lhs, P const& rhs) const {+    return lhs.first == rhs.first;+  }+};++} // namespace test+} // namespace folly++namespace std {+template <>+struct hash<folly::test::MoveOnlyTestInt> {+  std::size_t operator()(folly::test::MoveOnlyTestInt const& val) const {+    FOLLY_SAFE_CHECK(!val.destroyed, "");+    return val.x;+  }+};++template <>+struct hash<folly::test::ThrowOnCopyTestInt> {+  std::size_t operator()(folly::test::ThrowOnCopyTestInt const& val) const {+    return val.x;+  }+};++template <>+struct hash<folly::test::PermissiveConstructorTestInt> {+  std::size_t operator()(+      folly::test::PermissiveConstructorTestInt const& val) const {+    return val.x;+  }+};++template <int Tag>+struct hash<folly::test::Tracked<Tag>> {+  size_t operator()(folly::test::Tracked<Tag> const& tracked) const {+    return tracked.val_ ^ Tag;+  }+};+} // namespace std
+ folly/folly/container/vector_bool.h view
@@ -0,0 +1,48 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <memory>+#include <vector>++#include <folly/container/detail/BoolWrapper.h>+#include <folly/memory/MemoryResource.h>++namespace folly {++/// Convenience alias to use instead of `std::vector<bool>` to avoid infamous+/// `std::vector<bool>` specialization.+///+/// Usage example:+///+///     folly::vector_bool<> vec = {false, true};+///     assert(vec[0] == false);+///     assert(vec[1] == true);+template <template <class> typename Allocator = std::allocator>+using vector_bool = std::vector<+    folly::detail::BoolWrapper, //+    Allocator<folly::detail::BoolWrapper>>;++#if FOLLY_HAS_MEMORY_RESOURCE+namespace pmr {++using vector_bool = vector_bool<std::pmr::polymorphic_allocator>;++} // namespace pmr+#endif // FOLLY_HAS_MEMORY_RESOURCE++} // namespace folly
+ folly/folly/coro/Accumulate-inl.h view
@@ -0,0 +1,43 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#if FOLLY_HAS_COROUTINES++namespace folly {+namespace coro {++template <typename Reference, typename Value, typename Output>+Task<Output> accumulate(+    AsyncGenerator<Reference, Value> generator, Output init) {+  return accumulate(std::move(generator), std::move(init), std::plus{});+}++template <+    typename Reference,+    typename Value,+    typename Output,+    typename BinaryOp>+Task<Output> accumulate(+    AsyncGenerator<Reference, Value> generator, Output init, BinaryOp op) {+  while (auto next = co_await generator.next()) {+    init = op(std::move(init), std::move(next).value());+  }+  co_return init;+}+} // namespace coro+} // namespace folly++#endif // FOLLY_HAS_COROUTINES
+ folly/folly/coro/Accumulate.h view
@@ -0,0 +1,58 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <folly/coro/AsyncGenerator.h>+#include <folly/coro/Coroutine.h>+#include <folly/coro/Task.h>++#if FOLLY_HAS_COROUTINES++namespace folly {+namespace coro {++// Accumulate the values from an input stream into a single value given+// an optional binary accumulation operation, similar to std::accumulate.+//+// The input is a stream of values.+//+// The output is a Task containing the result of the accumulation+//+// Example:+//   AsyncGenerator<int> stream();+//+//   Task<void> consumer() {+//     auto sum = co_await accumulate(stream(), 0, std::plus{});+//   }+template <typename Reference, typename Value, typename Output>+Task<Output> accumulate(+    AsyncGenerator<Reference, Value> generator, Output init);++template <+    typename Reference,+    typename Value,+    typename Output,+    typename BinaryOp>+Task<Output> accumulate(+    AsyncGenerator<Reference, Value> generator, Output init, BinaryOp op);++} // namespace coro+} // namespace folly++#endif // FOLLY_HAS_COROUTINES++#include <folly/coro/Accumulate-inl.h>
+ folly/folly/coro/AsyncGenerator.h view
@@ -0,0 +1,885 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++//+// Docs: https://fburl.com/fbcref_asyncgenerator+//++#pragma once++#include <folly/CancellationToken.h>+#include <folly/ExceptionWrapper.h>+#include <folly/Traits.h>+#include <folly/Try.h>+#include <folly/coro/AutoCleanup-fwd.h>+#include <folly/coro/Coroutine.h>+#include <folly/coro/CurrentExecutor.h>+#include <folly/coro/Invoke.h>+#include <folly/coro/Result.h>+#include <folly/coro/ScopeExit.h>+#include <folly/coro/ViaIfAsync.h>+#include <folly/coro/WithAsyncStack.h>+#include <folly/coro/WithCancellation.h>+#include <folly/coro/detail/Malloc.h>+#include <folly/coro/detail/ManualLifetime.h>+#include <folly/lang/SafeAlias-fwd.h>+#include <folly/tracing/AsyncStack.h>++#include <glog/logging.h>++#include <iterator>+#include <type_traits>++#if FOLLY_HAS_COROUTINES++namespace folly {+namespace coro {+namespace detail {++template <typename Reference, typename Value, bool RequiresCleanup>+class AsyncGeneratorPromise;++} // namespace detail++/**+ * The AsyncGenerator class represents a sequence of asynchronously produced+ * values where the values are produced by a coroutine.+ *+ * Values are produced by using the 'co_yield' keyword and the coroutine can+ * also consume other asynchronous operations using the 'co_await' keyword.+ * The end of the sequence is indicated by executing 'co_return;' either+ * explicitly or by letting execution run off the end of the coroutine.+ *+ * Reference Type+ * --------------+ * The first template parameter controls the 'reference' type.+ * i.e. the type returned when you dereference the iterator using operator*().+ * This type is typically specified as an actual reference type.+ * eg. 'const T&' (non-mutable), 'T&' (mutable)  or 'T&&' (movable) depending+ * what access you want your consumers to have to the yielded values.+ *+ * It's also possible to specify the 'Reference' template parameter as a value+ * type. In this case the generator takes a copy of the yielded value (either+ * copied or move-constructed) and you get a copy of this value every time+ * you dereference the iterator with '*iter'.+ * This can be expensive for types that are expensive to copy, but can provide+ * a small performance win for types that are cheap to copy (like built-in+ * integer types).+ *+ * Value Type+ * ----------+ * The second template parameter is optional, but if specified can be used as+ * the value-type that should be used to take a copy of the value returned by+ * the Reference type.+ * By default this type is the same as 'Reference' type stripped of qualifiers+ * and references. However, in some cases it can be a different type.+ * For example, if the 'Reference' type was a non-reference proxy type.+ *+ * Example:+ *+ *     AsyncGenerator<std::tuple<const K&, V&>, std::tuple<K, V>> getItems() {+ *         auto firstMap = co_await getFirstMap();+ *         for (auto&& [k, v] : firstMap) {+ *           co_yield {k, v};+ *         }+ *         auto secondMap = co_await getSecondMap();+ *         for (auto&& [k, v] : secondMap) {+ *           co_yield {k, v};+ *         }+ *      }+ *+ * This is mostly useful for generic algorithms that need to take copies of+ * elements of the sequence.+ *+ * Executor Affinity+ * -----------------+ * An AsyncGenerator coroutine has similar executor-affinity to that of the+ * folly::coro::Task coroutine type. Every time a consumer requests a new value+ * from the generator using 'co_await ++it' the generator inherits the caller's+ * current executor. The coroutine will ensure that it always resumes on the+ * associated executor when resuming from `co_await' expression until it hits+ * the next 'co_yield' or 'co_return' statement.+ * Note that the executor can potentially change at a 'co_yield' statement if+ * the next element of the sequence is requested from a consumer coroutine that+ * is associated with a different executor.+ *+ * Example: Writing an async generator.+ *+ *       folly::coro::AsyncGenerator<Record&&> getRecordsAsync() {+ *         auto resultSet = executeQuery(someQuery);+ *         for (;;) {+ *           auto resultSetPage = co_await resultSet.nextPage();+ *           if (resultSetPage.empty()) break;+ *           for (auto& row : resultSetPage) {+ *             co_yield Record{row.get("name"), row.get("email")};+ *           }+ *         }+ *  }+ *+ * Example: Consuming items from an async generator+ *+ *       folly::coro::Task<void> consumer() {+ *         auto records = getRecordsAsync();+ *         while (auto item = co_await records.next()) {+ *           auto&& record = *item;+ *           process(record);+ *         }+ *       }+ *+ * Async Cleanup+ * -------------+ * When the template parameter RequiresCleanup is true, the owner of an+ * AsyncGenerator is responsible for awaiting cleanup() before the generator+ * object's destructor is called. That allows to use folly::coro::co_scope_exit+ * awaitables inside AsyncGenerator, which are asynchronously executed when+ * cleanup() is awaited. Note that the AsyncGenerator coroutine frame is+ * destroyed before co_scope_exit awaitables are executed.+ *+ * There is an alias CleanableAsyncGenerator for AsyncGenerator with+ * RequiresCleanup set to true.+ *+ * Drain safety+ * ------------+ * One significant difference between AsyncGenerator and folly::coro::Task is+ * that AsyncGenerator may be destroyed between next() calls - i.e. destroyed+ * without being fully drained.+ *+ * For example:+ *+ *   AsyncGenerator<int> gen() {+ *     SCOPE_EXIT {+ *       LOG(INFO) << "Step 4";+ *     };+ *     LOG(INFO) << "Step 1";+ *     co_yield 41;+ *     SCOPE_EXIT {+ *       LOG(INFO) << "Step 3";+ *     };+ *     LOG(INFO) << "Step 2";+ *     co_yield 42;+ *     SCOPE_EXIT {+ *       LOG(INFO) << "Never reached";+ *     };+ *     LOG(INFO) << "Never reached";+ *     co_yield 43;+ *   }+ *+ *   {+ *     AsyncGenerator<int> g = gen();+ *     while (auto next = co_await g.next()) {+ *       LOG(INFO) << *next;+ *       if (*next == 42) {+ *         break;+ *         // ^^^ this may trigger generator destruction before it is drained.+ *       }+ *     }+ *   }+ *+ * This means that when writing an AsyncGenerator, you should always document+ * whether such AsyncGenerator requires draining before destruction (drain+ * unsafe). When possible you should always aim to make AsyncGenerator not+ * require draining before destruction (drain safe).+ *+ * If an AsyncGenerator is drain unsafe, always mention this in the+ * documentation and ideally include some assertions that help detect cases+ * where such AsyncGenerator is destroyed without being fully drained.+ *+ * Example:+ *+ *   AsyncGenerator<int> gen() {+ *     auto drainGuard = makeGuard([] { LOG(FATAL) << "I shall be drained!"; });+ *     co_yield 41;+ *     co_yield 42;+ *     co_yield 43;+ *     drainGuard.dismiss();+ *   }+ */+template <+    typename Reference,+    typename Value = remove_cvref_t<Reference>,+    bool RequiresCleanup = false>+class FOLLY_NODISCARD AsyncGenerator {+  static_assert(+      std::is_constructible<Value, Reference>::value,+      "AsyncGenerator 'value_type' must be constructible from a 'reference'.");++ public:+  using promise_type =+      detail::AsyncGeneratorPromise<Reference, Value, RequiresCleanup>;+  // Standard `AsyncGenerator` coros can easily capture references & other+  // unsafe aliasing.+  //+  // Future: Implement a `coro/safe` generator wrapper, like+  // `async_closure_gen`.+  using folly_private_safe_alias_t = safe_alias_constant<safe_alias::unsafe>;++ private:+  using handle_t = coroutine_handle<promise_type>;++ public:+  using value_type = Value;+  using reference = Reference;+  using pointer = std::add_pointer_t<Reference>;++ public:+  AsyncGenerator() noexcept : coro_() {}++  AsyncGenerator(AsyncGenerator&& other) noexcept+      : coro_(std::exchange(other.coro_, {})) {}++  ~AsyncGenerator() {+    if (coro_) {+      if constexpr (RequiresCleanup) {+        LOG(FATAL) << "cleanup() hasn't been called!";+      }++      coro_.destroy();+    }+  }++  class CleanupSemiAwaitable;++  class FOLLY_NODISCARD CleanupAwaitable {+   public:+    bool await_ready() noexcept { return !scopeExit_; }++    template <typename Promise>+    FOLLY_NOINLINE auto await_suspend(+        coroutine_handle<Promise> continuation) noexcept {+      asyncFrame_.setReturnAddress();+      scopeExit_.promise().setContext(+          continuation, &asyncFrame_, executor_.get_alias());+      if constexpr (detail::promiseHasAsyncFrame_v<Promise>) {+        folly::pushAsyncStackFrameCallerCallee(+            continuation.promise().getAsyncFrame(), asyncFrame_);+        return scopeExit_;+      } else {+        folly::resumeCoroutineWithNewAsyncStackRoot(scopeExit_);+      }+    }++    void await_resume() noexcept {}++   private:+    friend CleanupSemiAwaitable;++    CleanupAwaitable(+        coroutine_handle<detail::ScopeExitTaskPromiseBase> scopeExit,+        folly::Executor::KeepAlive<> executor) noexcept+        : scopeExit_{scopeExit}, executor_{std::move(executor)} {}++    friend CleanupAwaitable tag_invoke(+        cpo_t<co_withAsyncStack>, CleanupAwaitable awaitable) noexcept {+      return std::move(awaitable);+    }++    coroutine_handle<detail::ScopeExitTaskPromiseBase> scopeExit_;+    folly::AsyncStackFrame asyncFrame_;+    folly::Executor::KeepAlive<> executor_;+  };++  class FOLLY_NODISCARD CleanupSemiAwaitable {+   public:+    CleanupAwaitable viaIfAsync(Executor::KeepAlive<> executor) noexcept {+      return CleanupAwaitable{scopeExit_, std::move(executor)};+    }++    using folly_private_safe_alias_t = safe_alias_constant<safe_alias::unsafe>;++   private:+    friend AsyncGenerator;++    explicit CleanupSemiAwaitable(+        coroutine_handle<detail::ScopeExitTaskPromiseBase> scopeExit) noexcept+        : scopeExit_{scopeExit} {}++    coroutine_handle<detail::ScopeExitTaskPromiseBase> scopeExit_;+  };++  CleanupSemiAwaitable cleanup() && {+    static_assert(RequiresCleanup);+    if (coro_) {+      SCOPE_EXIT {+        std::exchange(coro_, {}).destroy();+      };+      return CleanupSemiAwaitable{coro_.promise().scopeExit_};+    } else {+      return CleanupSemiAwaitable{{}};+    }+  }++  AsyncGenerator& operator=(AsyncGenerator&& other) noexcept {+    auto oldCoro = std::exchange(coro_, std::exchange(other.coro_, {}));+    if (oldCoro) {+      CHECK(!RequiresCleanup) << "cleanup() hasn't been called!";+      oldCoro.destroy();+    }+    return *this;+  }++  void swap(AsyncGenerator& other) noexcept { std::swap(coro_, other.coro_); }++  class NextAwaitable;+  class NextSemiAwaitable;++  class NextResult {+   public:+    NextResult() noexcept : hasValue_(false) {}++    NextResult(NextResult&& other) noexcept : hasValue_(other.hasValue_) {+      if (hasValue_) {+        value_.construct(std::move(other.value_).get());+      }+    }++    ~NextResult() {+      if (hasValue_) {+        value_.destruct();+      }+    }++    NextResult& operator=(NextResult&& other) {+      if (&other != this) {+        if (has_value()) {+          hasValue_ = false;+          value_.destruct();+        }++        if (other.has_value()) {+          value_.construct(std::move(other.value_).get());+          hasValue_ = true;+        }+      }+      return *this;+    }++    bool has_value() const noexcept { return hasValue_; }++    explicit operator bool() const noexcept { return has_value(); }++    decltype(auto) value() & {+      DCHECK(has_value());+      return value_.get();+    }++    decltype(auto) value() && {+      DCHECK(has_value());+      return std::move(value_).get();+    }++    decltype(auto) value() const& {+      DCHECK(has_value());+      return value_.get();+    }++    decltype(auto) value() const&& {+      DCHECK(has_value());+      return std::move(value_).get();+    }++    decltype(auto) operator*() & { return value(); }++    decltype(auto) operator*() && { return std::move(*this).value(); }++    decltype(auto) operator*() const& { return value(); }++    decltype(auto) operator*() const&& { return std::move(*this).value(); }++    decltype(auto) operator->() {+      DCHECK(has_value());+      auto&& x = value_.get();+      return std::addressof(x);+    }++    decltype(auto) operator->() const {+      DCHECK(has_value());+      auto&& x = value_.get();+      return std::addressof(x);+    }++   private:+    friend NextAwaitable;+    explicit NextResult(handle_t coro) noexcept : hasValue_(true) {+      value_.construct(coro.promise().getRvalue());+    }++    detail::ManualLifetime<Reference> value_;+    bool hasValue_ = false;+  };++  class NextAwaitable {+   public:+    bool await_ready() noexcept { return !coro_; }++    template <typename Promise>+    FOLLY_NOINLINE auto await_suspend(+        coroutine_handle<Promise> continuation) noexcept {+      auto& promise = coro_.promise();++      promise.setContinuation(continuation);+      promise.clearValue();++      auto& asyncFrame = promise.getAsyncFrame();+      asyncFrame.setReturnAddress();++      if constexpr (detail::promiseHasAsyncFrame_v<Promise>) {+        folly::pushAsyncStackFrameCallerCallee(+            continuation.promise().getAsyncFrame(), asyncFrame);+        return coro_;+      } else {+        folly::resumeCoroutineWithNewAsyncStackRoot(coro_);+      }+    }++    NextResult await_resume() {+      if (!coro_) {+        return NextResult{};+      } else if (!coro_.promise().hasValue()) {+        coro_.promise().throwIfException();+        return NextResult{};+      } else {+        return NextResult{coro_};+      }+    }++    folly::Try<NextResult> await_resume_try() noexcept {+      if (coro_) {+        if (coro_.promise().hasValue()) {+          return folly::Try<NextResult>(NextResult{coro_});+        } else if (coro_.promise().hasException()) {+          return folly::Try<NextResult>(+              std::move(coro_.promise().getException()));+        }+      }+      return folly::Try<NextResult>(NextResult{});+    }++   private:+    friend NextSemiAwaitable;+    explicit NextAwaitable(handle_t coro) noexcept : coro_(coro) {}++    friend NextAwaitable tag_invoke(+        cpo_t<co_withAsyncStack>, NextAwaitable awaitable) noexcept {+      return NextAwaitable{awaitable.coro_};+    }++    handle_t coro_;+  };++  class NextSemiAwaitable {+   public:+    NextAwaitable viaIfAsync(Executor::KeepAlive<> executor) noexcept {+      if (coro_) {+        coro_.promise().setExecutor(std::move(executor));+      }+      return NextAwaitable{coro_};+    }++    friend NextSemiAwaitable co_withCancellation(+        CancellationToken cancelToken, NextSemiAwaitable&& awaitable) {+      if (awaitable.coro_) {+        awaitable.coro_.promise().setCancellationToken(std::move(cancelToken));+      }+      return NextSemiAwaitable{std::exchange(awaitable.coro_, {})};+    }++    using folly_private_safe_alias_t = safe_alias_constant<safe_alias::unsafe>;++   private:+    friend AsyncGenerator;++    explicit NextSemiAwaitable(handle_t coro) noexcept : coro_(coro) {}++    handle_t coro_;+  };++  NextSemiAwaitable next() noexcept {+    DCHECK(!coro_ || !coro_.done());+    return NextSemiAwaitable{coro_};+  }++  template <typename F, typename... A, typename F_, typename... A_>+  friend AsyncGenerator tag_invoke(+      tag_t<co_invoke_fn>, tag_t<AsyncGenerator, F, A...>, F_ f, A_... a) {+    if constexpr (RequiresCleanup) {+      auto&& [fScoped, r] = co_await co_scope_exit(+          [](auto&&, auto&& gen) { return std::move(gen).cleanup(); },+          static_cast<F&&>(f),+          AsyncGenerator{});+      r = invoke(static_cast<F&&>(fScoped), static_cast<A&&>(a)...);+      while (true) {+        co_yield co_result(co_await co_awaitTry(r.next()));+      }+    } else {+      auto r = invoke(static_cast<F&&>(f), static_cast<A&&>(a)...);+      while (true) {+        co_yield co_result(co_await co_awaitTry(r.next()));+      }+    }+  }++ private:+  friend promise_type;++  explicit AsyncGenerator(coroutine_handle<promise_type> coro) noexcept+      : coro_(coro) {}++  coroutine_handle<promise_type> coro_;+};++template <typename Reference, typename Value = remove_cvref_t<Reference>>+using CleanableAsyncGenerator =+    AsyncGenerator<Reference, Value, true /* RequiresCleanup */>;++namespace detail {++template <bool RequiresCleanup>+struct BaseAsyncGeneratorPromise {};++template <>+struct BaseAsyncGeneratorPromise<true> {+  coroutine_handle<ScopeExitTaskPromiseBase> scopeExit_;+};++template <typename Reference, typename Value, bool RequiresCleanup = false>+class AsyncGeneratorPromise final+    : public ExtendedCoroutinePromise,+      BaseAsyncGeneratorPromise<RequiresCleanup> {+  class YieldAwaiter {+   public:+    bool await_ready() noexcept { return false; }+    coroutine_handle<> await_suspend(+        coroutine_handle<AsyncGeneratorPromise> h) noexcept {+      AsyncGeneratorPromise& promise = h.promise();+      // Pop AsyncStackFrame first as clearContext() clears the frame state.+      folly::popAsyncStackFrameCallee(promise.getAsyncFrame());+      promise.clearContext();+      if (promise.hasException()) {+        auto [handle, frame] =+            promise.continuation_.getErrorHandle(promise.getException());+        return handle.getHandle();+      }+      return promise.continuation_.getHandle();+    }+    void await_resume() noexcept {}+  };++ public:+  template <typename... Args>+  AsyncGeneratorPromise(Args&... args) {+    if constexpr (RequiresCleanup) {+      scheduleAutoCleanupIfNeeded(+          coroutine_handle<AsyncGeneratorPromise>::from_promise(*this),+          args...);+    }+  }++  ~AsyncGeneratorPromise() {+    switch (state_) {+      case State::VALUE:+        folly::coro::detail::deactivate(value_);+        break;+      case State::EXCEPTION_WRAPPER:+        folly::coro::detail::deactivate(exceptionWrapper_);+        break;+      case State::DONE:+      case State::INVALID:+        break;+    }+  }++  static void* operator new(std::size_t size) {+    return ::folly_coro_async_malloc(size);+  }++  static void operator delete(void* ptr, std::size_t size) {+    ::folly_coro_async_free(ptr, size);+  }++  AsyncGenerator<Reference, Value, RequiresCleanup>+  get_return_object() noexcept {+    return AsyncGenerator<Reference, Value, RequiresCleanup>{+        coroutine_handle<AsyncGeneratorPromise>::from_promise(*this)};+  }++  suspend_always initial_suspend() noexcept { return {}; }++  YieldAwaiter final_suspend() noexcept {+    DCHECK(!hasValue());+    return {};+  }++  YieldAwaiter yield_value(Reference&& value) noexcept(+      std::is_nothrow_move_constructible<Reference>::value) {+    DCHECK(state_ == State::INVALID);+    folly::coro::detail::activate(value_, static_cast<Reference&&>(value));+    state_ = State::VALUE;+    return YieldAwaiter{};+  }++  /// In the case where 'Reference' is not actually a reference-type we+  /// allow implicit conversion from the co_yield argument to Reference.+  /// However, we don't want to allow this for cases where 'Reference' _is_+  /// a reference because this could result in the reference binding to a+  /// temporary that results from an implicit conversion.+  template <+      typename U,+      std::enable_if_t<+          !std::is_reference_v<Reference> &&+              std::is_convertible_v<U&&, Reference>,+          int> = 0>+  YieldAwaiter yield_value(U&& value) noexcept(+      std::is_nothrow_constructible_v<Reference, U>) {+    DCHECK(state_ == State::INVALID);+    folly::coro::detail::activate(value_, static_cast<U&&>(value));+    state_ = State::VALUE;+    return {};+  }++  YieldAwaiter yield_value(co_error&& error) noexcept {+    DCHECK(state_ == State::INVALID);+    folly::coro::detail::activate(+        exceptionWrapper_, std::move(error.exception()));+    state_ = State::EXCEPTION_WRAPPER;+    return {};+  }++  YieldAwaiter yield_value(co_result<Value>&& res) noexcept {+    if (res.result().hasValue()) {+      return yield_value(std::move(res.result().value()));+    } else if (res.result().hasException()) {+      return yield_value(co_error(res.result().exception()));+    } else {+      return_void();+      return {};+    }+  }++  YieldAwaiter yield_value(+      co_result<typename AsyncGenerator<Reference, Value, RequiresCleanup>::+                    NextResult>&& res) noexcept {+    DCHECK(+        res.result().hasValue() ||+        (res.result().hasException() && res.result().exception()));+    if (res.result().hasException()) {+      return yield_value(co_error(res.result().exception()));+    } else if (res.result().hasValue()) {+      if (res.result()->has_value()) {+        return yield_value(std::move(res.result()->value()));+      } else {+        return_void();+        return {};+      }+    }+    return yield_value(co_error(UsingUninitializedTry{}));+  }++  variant_awaitable<YieldAwaiter, ready_awaitable<>> await_transform(+      co_safe_point_t) noexcept {+    if (cancelToken_.isCancellationRequested()) {+      return yield_value(co_cancelled);+    }+    return ready_awaitable<>{};+  }++  void unhandled_exception() noexcept {+    DCHECK(state_ == State::INVALID);+    folly::coro::detail::activate(exceptionWrapper_, current_exception());+    state_ = State::EXCEPTION_WRAPPER;+  }++  void return_void() noexcept {+    DCHECK(state_ == State::INVALID);+    state_ = State::DONE;+  }++  // FIXME: Much of this class is currently copy-pasted from `TaskPromiseBase`,+  // Refactor this to use that, so as to avoid `co_await` behavior divergence.++  template <+      typename Awaitable,+      std::enable_if_t<!must_await_immediately_v<Awaitable>, int> = 0>+  auto await_transform(Awaitable&& awaitable) {+    bypassExceptionThrowing_ =+        bypassExceptionThrowing_ == BypassExceptionThrowing::REQUESTED+        ? BypassExceptionThrowing::ACTIVE+        : BypassExceptionThrowing::INACTIVE;++    return folly::coro::co_withAsyncStack(folly::coro::co_viaIfAsync(+        executor_.get_alias(),+        folly::coro::co_withCancellation(+            cancelToken_, static_cast<Awaitable&&>(awaitable))));+  }+  template <+      typename Awaitable,+      std::enable_if_t<must_await_immediately_v<Awaitable>, int> = 0>+  auto await_transform(Awaitable awaitable) {+    bypassExceptionThrowing_ =+        bypassExceptionThrowing_ == BypassExceptionThrowing::REQUESTED+        ? BypassExceptionThrowing::ACTIVE+        : BypassExceptionThrowing::INACTIVE;++    return folly::coro::co_withAsyncStack(folly::coro::co_viaIfAsync(+        executor_.get_alias(),+        folly::coro::co_withCancellation(+            cancelToken_,+            mustAwaitImmediatelyUnsafeMover(std::move(awaitable))())));+  }++  template <typename Awaitable>+  auto await_transform(NothrowAwaitable<Awaitable> awaitable) {+    bypassExceptionThrowing_ = BypassExceptionThrowing::REQUESTED;+    return await_transform(+        mustAwaitImmediatelyUnsafeMover(awaitable.unwrap())());+  }++  auto await_transform(folly::coro::co_current_executor_t) noexcept {+    return ready_awaitable<folly::Executor*>{executor_.get()};+  }++  auto await_transform(folly::coro::co_current_cancellation_token_t) noexcept {+    return ready_awaitable<const folly::CancellationToken&>{cancelToken_};+  }++  void setCancellationToken(folly::CancellationToken cancelToken) noexcept {+    // Only keep the first cancellation token.+    // ie. the inner-most cancellation scope of the consumer's calling+    // context.+    if (!hasCancelTokenOverride_) {+      cancelToken_ = std::move(cancelToken);+      hasCancelTokenOverride_ = true;+    }+  }++  void setExecutor(folly::Executor::KeepAlive<> executor) noexcept {+    DCHECK(executor);+    executor_ = std::move(executor);+  }++  void setContinuation(ExtendedCoroutineHandle continuation) noexcept {+    continuation_ = continuation;+  }++  bool hasException() const noexcept {+    return state_ == State::EXCEPTION_WRAPPER;+  }++  folly::exception_wrapper& getException() noexcept {+    DCHECK(hasException());+    return exceptionWrapper_.get();+  }++  void throwIfException() {+    if (state_ == State::EXCEPTION_WRAPPER) {+      exceptionWrapper_.get().throw_exception();+    }+  }++  decltype(auto) getRvalue() noexcept {+    DCHECK(hasValue());+    return std::move(value_).get();+  }++  void clearValue() noexcept {+    if (hasValue()) {+      state_ = State::INVALID;+      folly::coro::detail::deactivate(value_);+    } else {+      CHECK(state_ != State::DONE)+          << "Using generator after receiving completion.";+      CHECK(state_ != State::EXCEPTION_WRAPPER)+          << "Using generator after receiving exception.";+    }+  }++  bool hasValue() const noexcept { return state_ == State::VALUE; }++  folly::AsyncStackFrame& getAsyncFrame() noexcept { return asyncFrame_; }++  std::pair<ExtendedCoroutineHandle, AsyncStackFrame*> getErrorHandle(+      exception_wrapper& ex) final {+    if (bypassExceptionThrowing_ == BypassExceptionThrowing::ACTIVE) {+      auto yieldAwaiter = yield_value(co_error(std::move(ex)));+      DCHECK(!yieldAwaiter.await_ready());+      return {+          yieldAwaiter.await_suspend(+              coroutine_handle<AsyncGeneratorPromise>::from_promise(*this)),+          // yieldAwaiter.await_suspend pops a frame+          getAsyncFrame().getParentFrame()};+    }+    return {+        coroutine_handle<AsyncGeneratorPromise>::from_promise(*this), nullptr};+  }++ private:+  friend AsyncGenerator<Reference, Value, RequiresCleanup>;++  void clearContext() noexcept {+    executor_ = {};+    cancelToken_ = {};+    hasCancelTokenOverride_ = false;+    asyncFrame_ = {};+  }++  friend coroutine_handle<ScopeExitTaskPromiseBase> tag_invoke(+      cpo_t<co_attachScopeExit>,+      AsyncGeneratorPromise& p,+      coroutine_handle<ScopeExitTaskPromiseBase> scopeExit) noexcept {+    static_assert(+        RequiresCleanup,+        "Only CleanableAsyncGenerator (AsyncGenerator with RequiresCleanup"+        " template parameter set to true) supports attaching co_scope_exit");+    return std::exchange(p.scopeExit_, scopeExit);+  }++  enum class State : std::uint8_t {+    INVALID,+    VALUE,+    EXCEPTION_WRAPPER,+    DONE,+  };++  ExtendedCoroutineHandle continuation_;+  folly::AsyncStackFrame asyncFrame_;+  folly::Executor::KeepAlive<> executor_;+  folly::CancellationToken cancelToken_;+  union {+    ManualLifetime<folly::exception_wrapper> exceptionWrapper_;+    ManualLifetime<Reference> value_;+  };+  State state_ = State::INVALID;+  bool hasCancelTokenOverride_ = false;++  enum class BypassExceptionThrowing : uint8_t {+    INACTIVE,+    ACTIVE,+    REQUESTED,+  } bypassExceptionThrowing_{BypassExceptionThrowing::INACTIVE};+};++} // namespace detail++template <typename Reference, typename Value>+auto tag_invoke(+    cpo_t<co_cleanup>, CleanableAsyncGenerator<Reference, Value>&& gen) {+  return std::move(gen).cleanup();+}++} // namespace coro++} // namespace folly++#endif
+ folly/folly/coro/AsyncPipe.h view
@@ -0,0 +1,297 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <folly/Try.h>+#include <folly/coro/AsyncGenerator.h>+#include <folly/coro/Coroutine.h>+#include <folly/coro/Invoke.h>+#include <folly/coro/SmallUnboundedQueue.h>+#include <folly/coro/ViaIfAsync.h>+#include <folly/fibers/Semaphore.h>++#include <memory>+#include <utility>++#if FOLLY_HAS_COROUTINES++namespace folly {+namespace coro {++// An AsyncGenerator with a write end+//+// Usage:+//   auto pipe = AsyncPipe<T>::create();+//   pipe.second.write(std::move(val1));+//   auto val2 = co_await pipe.first.next();+//+//  write() returns false if the read end has been destroyed (unless+//  SingleProducer is disabled, in which case this behavior is undefined).+//  The generator is completed when the write end is destroyed or on close()+//  close() can also be passed an exception, which is thrown when read.+//+//  An optional onClosed callback can be passed to create(). This callback will+//  be called either when the generator is destroyed by the consumer, or when+//  the pipe is closed by the publisher (whichever comes first). The onClosed+//  callback may destroy the AsyncPipe object inline, and must not call close()+//  on the AsyncPipe object inline. If an onClosed callback is specified and the+//  publisher would like to destroy the pipe outside of the callback, it must+//  first close the pipe.+//+//  If SingleProducer is disabled, AsyncPipe's write() method (but not its+//  close() method) becomes thread-safe. close() must be sequenced after all+//  write()s in this mode.++template <+    typename T,+    bool SingleProducer = true,+    template <typename, bool, bool> typename QueueType = SmallUnboundedQueue>+class AsyncPipe {+ public:+  ~AsyncPipe() {+    CHECK(!onClosed_ || onClosed_->wasInvokeRequested())+        << "If an onClosed callback is specified and the generator still "+        << "exists, the publisher must explicitly close the pipe prior to "+        << "destruction.";+    std::move(*this).close();+  }++  AsyncPipe(AsyncPipe&& pipe) noexcept {+    queue_ = std::move(pipe.queue_);+    onClosed_ = std::move(pipe.onClosed_);+  }++  AsyncPipe& operator=(AsyncPipe&& pipe) {+    if (this != &pipe) {+      CHECK(!onClosed_ || onClosed_->wasInvokeRequested())+          << "If an onClosed callback is specified and the generator still "+          << "exists, the publisher must explicitly close the pipe prior to "+          << "destruction.";+      std::move(*this).close();+      queue_ = std::move(pipe.queue_);+      onClosed_ = std::move(pipe.onClosed_);+    }+    return *this;+  }++  static std::pair<folly::coro::AsyncGenerator<T&&>, AsyncPipe> create(+      folly::Function<void()> onClosed = nullptr) {+    auto queue = std::make_shared<Queue>();+    auto cancellationSource = std::optional<folly::CancellationSource>();+    auto onClosedCallback = std::unique_ptr<OnClosedCallback>();+    if (onClosed != nullptr) {+      cancellationSource.emplace();+      onClosedCallback = std::make_unique<OnClosedCallback>(+          *cancellationSource, std::move(onClosed));+    }+    auto guard =+        folly::makeGuard([cancellationSource = std::move(cancellationSource)] {+          if (cancellationSource) {+            cancellationSource->requestCancellation();+          }+        });+    return {+        folly::coro::co_invoke(+            [queue,+             guard = std::move(guard)]() -> folly::coro::AsyncGenerator<T&&> {+              while (true) {+                co_yield co_result(co_await co_nothrow(queue->dequeue()));+              }+            }),+        AsyncPipe(queue, std::move(onClosedCallback))};+  }++  template <typename U = T>+  bool write(U&& val) {+    if (auto queue = queue_.lock()) {+      queue->enqueue(folly::Try<T>(std::forward<U>(val)));+      return true;+    }+    return false;+  }++  void close(folly::exception_wrapper ew) && {+    if (auto queue = queue_.lock()) {+      queue->enqueue(folly::Try<T>(std::move(ew)));+      queue_.reset();+    }+    if (onClosed_ != nullptr) {+      onClosed_->requestInvoke();+      onClosed_.reset();+    }+  }++  void close() && {+    if (auto queue = queue_.lock()) {+      queue->enqueue(folly::Try<T>());+      queue_.reset();+    }+    if (onClosed_ != nullptr) {+      onClosed_->requestInvoke();+      onClosed_.reset();+    }+  }++  bool isClosed() const { return queue_.expired(); }++ private:+  using Queue = QueueType<folly::Try<T>, SingleProducer, true>;++  class OnClosedCallback {+   public:+    OnClosedCallback(+        folly::CancellationSource cancellationSource,+        folly::Function<void()> onClosedFunc)+        : cancellationSource_(std::move(cancellationSource)),+          cancellationCallback_(+              cancellationSource_.getToken(), std::move(onClosedFunc)) {}++    void requestInvoke() { cancellationSource_.requestCancellation(); }++    bool wasInvokeRequested() {+      return cancellationSource_.isCancellationRequested();+    }++   private:+    folly::CancellationSource cancellationSource_;+    folly::CancellationCallback cancellationCallback_;+  };++  explicit AsyncPipe(+      std::weak_ptr<Queue> queue, std::unique_ptr<OnClosedCallback> onClosed)+      : queue_(std::move(queue)), onClosed_(std::move(onClosed)) {}++  std::weak_ptr<Queue> queue_;+  std::unique_ptr<OnClosedCallback> onClosed_;+};++// Bounded variant of AsyncPipe which buffers a fixed number of writes+// before blocking new attempts to write until the buffer is drained.+//+// Usage:+//   auto [generator, pipe] = BoundedAsyncPipe<T>::create(/* tokens */ 10);+//   co_await pipe.write(std::move(entry));+//   auto entry = co_await generator.next().value();+//+// write() is a coroutine which only blocks when+// no capacity is remaining. write() returns false if the read-end has been+// destroyed or was destroyed while blocking, only throwing OperationCanceled+// if the parent coroutine was canceled while blocking.+//+// try_write() is offered which will never block, but will return false+// and not write if no capacity is remaining or the read end is already+// destroyed.+//+// close() functions the same as AsyncPipe, and must be invoked before+// destruction if an onClose callback is attached.+template <+    typename T,+    bool SingleProducer = true,+    template <typename, bool, bool> typename QueueType = SmallUnboundedQueue>+class BoundedAsyncPipe {+ public:+  using Pipe = AsyncPipe<T, SingleProducer, QueueType>;++  static std::pair<AsyncGenerator<T&&>, BoundedAsyncPipe> create(+      size_t tokens, folly::Function<void()> onClosed = nullptr) {+    auto [generator, pipe] = Pipe::create(std::move(onClosed));++    auto semaphore = std::make_shared<folly::fibers::Semaphore>(tokens);++    folly::CancellationSource cancellationSource;+    auto cancellationToken = cancellationSource.getToken();+    auto guard = folly::makeGuard(+        [cancellationSource = std::move(cancellationSource)]() {+          cancellationSource.requestCancellation();+        });++    auto signalingGenerator = co_invoke(+        [generator_2 = std::move(generator),+         guard = std::move(guard),+         semaphore]() mutable -> folly::coro::AsyncGenerator<T&&> {+          while (true) {+            auto itemTry = co_await co_awaitTry(generator_2.next());+            semaphore->signal();+            co_yield co_result(std::move(itemTry));+          }+        });+    return std::pair<AsyncGenerator<T&&>, BoundedAsyncPipe>(+        std::move(signalingGenerator),+        BoundedAsyncPipe(+            std::move(pipe),+            std::move(semaphore),+            std::move(cancellationToken)));+  }++  template <typename U = T>+  folly::coro::Task<bool> write(U&& u) {+    auto parentToken = co_await co_current_cancellation_token;++    auto waitResult = co_await co_awaitTry(co_withCancellation(+        folly::CancellationToken::merge(+            std::move(parentToken), cancellationToken_),+        semaphore_->co_wait()));+    if (cancellationToken_.isCancellationRequested()) {+      // eagerly return false if the read-end was destroyed instead of throwing+      // OperationCanceled, to have uniform behavior when the generator is+      // destroyed+      co_return false;+    } else if (waitResult.hasException()) {+      co_yield co_error(std::move(waitResult).exception());+    }++    co_return pipe_.write(std::forward<U>(u));+  }++  template <typename U = T>+  bool try_write(U&& u) {+    bool available = semaphore_->try_wait();+    if (!available) {+      return false;+    }+    return pipe_.write(std::forward<U>(u));+  }++  size_t getAvailableSpace() { return semaphore_->getAvailableTokens(); }++  size_t getOccupiedSpace() {+    return semaphore_->getCapacity() - getAvailableSpace();+  }++  void close(exception_wrapper&& w) && { std::move(pipe_).close(std::move(w)); }+  void close() && { std::move(pipe_).close(); }++  bool isClosed() const { return pipe_.isClosed(); }++ private:+  BoundedAsyncPipe(+      Pipe&& pipe,+      std::shared_ptr<folly::fibers::Semaphore> semaphore,+      folly::CancellationToken cancellationToken)+      : pipe_(std::move(pipe)),+        semaphore_(std::move(semaphore)),+        cancellationToken_(std::move(cancellationToken)) {}++  Pipe pipe_;+  std::shared_ptr<folly::fibers::Semaphore> semaphore_;+  folly::CancellationToken cancellationToken_;+};++} // namespace coro+} // namespace folly++#endif // FOLLY_HAS_COROUTINES
+ folly/folly/coro/AsyncScope.h view
@@ -0,0 +1,423 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++//+// Docs: https://fburl.com/fbcref_asyncscope+//++#pragma once++#include <folly/CancellationToken.h>+#include <folly/ExceptionWrapper.h>+#include <folly/coro/Coroutine.h>+#include <folly/coro/CurrentExecutor.h>+#include <folly/coro/Task.h>+#include <folly/coro/detail/Barrier.h>+#include <folly/coro/detail/BarrierTask.h>+#include <folly/futures/Future.h>+#include <folly/portability/SourceLocation.h>+#include <folly/synchronization/RelaxedAtomic.h>++#include <glog/logging.h>++#include <atomic>+#include <cassert>+#include <optional>++#if FOLLY_HAS_COROUTINES++namespace folly {+namespace coro {++/**+ * The AsyncScope class is used to allow you to start a dynamic, unbounded+ * number of tasks, which can all run concurrently, and then later wait for+ * completion of all of the tasks.+ *+ * Tasks added to an AsyncScope must have a void or folly::Unit result-type+ * and must handle any errors prior to completing.+ *+ * @refcode folly/docs/examples/folly/coro/AsyncScope.cpp+ * @class folly::coro::AsyncScope+ */+//+// Example:+//    folly::coro::Task<void> process(Event event) {+//      try {+//        co_await do_processing(event.data);+//      } catch (...) {+//        LOG(ERROR) << "Processing event failed";+//      }+//    }+//+//    folly::coro::AsyncScope scope;+//    scope.add(co_withExecutor(folly::getGlobalCPUExecutor(), process(ev1)));+//    scope.add(co_withExecutor(folly::getGlobalCPUExecutor(), process(ev2)));+//    scope.add(co_withExecutor(folly::getGlobalCPUExecutor(), process(ev3)));+//    co_await scope.joinAsync();+//+class AsyncScope {+ public:+  AsyncScope() noexcept;++  // @param throwOnJoin If true, will throw last unhandled exception (if any)+  //                    on joinAsync. Default behavior is to ignore the+  //                    exception in opt builds and FATAL in dev builds+  explicit AsyncScope(bool throwOnJoin) noexcept;++  // Destroy the AsyncScope object.+  //+  // NOTE: If you have called add() on this scope then you _must_+  // call either cleanup() or joinAsync() and wait until the that operation+  // completes before calling the destructor.+  ~AsyncScope();++  /**+   * Query the number of tasks added to the scope that have not yet completed.+   */+  std::size_t remaining() const noexcept;++  /**+   * Start the specified task/awaitable by co_awaiting it.+   *+   * Exceptions+   * ----------+   * IMPORTANT: Tasks submitted to the AsyncScope by calling .add() must+   * ensure they do not complete with an exception. Exceptions propagating+   * from the 'co_await awaitable' expression are logged using DFATAL.+   *+   * To avoid this occurring you should make sure to catch and handle any+   * exceptions within the task being started here.+   *+   * Interaction with cleanup/joinAsync+   * ----------------------------------+   * It is invalid to call add() once the joinAsync() or cleanup()+   * operations have completed.+   *+   * This generally means that it is unsafe to call .add() once cleanup()+   * has started as it may be racing with completion of cleanup().+   *+   * The exception to this rule is for cases where you know you are running+   * within a task that has been started with .add() and thus you know that+   * cleanup() will not yet have completed.+   *+   * Passing folly::coro::Task+   * -------------------------+   * NOTE: You cannot pass a folly::coro::Task to this method.+   * You must first call co_withExecutor() to specify which executor the task+   * should run on.+   */+  // returnAddress customize entry point to async stack (useful if this is+  // called from async code already). If not set will default to+  // FOLLY_ASYNC_STACK_RETURN_ADDRESS()+  template <typename Awaitable>+  void add(Awaitable&& awaitable, void* returnAddress = nullptr);++  template <typename Awaitable>+  void addWithSourceLoc(+      Awaitable&& awaitable,+      void* returnAddress = nullptr,+      source_location sourceLocation = source_location::current());++  /**+   * Asynchronously wait for all started tasks to complete.+   *+   * Either call this method _or_ cleanup() to join the work.+   * It is invalid to call both of them.+   */+  Task<void> joinAsync() noexcept;++  /**+   * Asynchronously cleanup all started tasks.+   *+   * If you have previuosly called add() then you must call cleanup()+   * and wait for the retuned future to complete before the AsyncScope+   * object destructs.+   */+  SemiFuture<Unit> cleanup() noexcept;++ private:+  template <typename Awaitable>+  static detail::DetachedBarrierTask addImpl(+      Awaitable awaitable,+      bool throwOnJoin,+      folly::exception_wrapper& maybeException,+      std::optional<source_location> source,+      std::atomic<bool>& exceptionRaised) {+    static_assert(+        std::is_void_v<await_result_t<Awaitable>> ||+            std::is_same_v<await_result_t<Awaitable>, folly::Unit>,+        "Result of the task would be discarded. Make sure task result is either void or folly::Unit.");++    exception_wrapper exn;+    try {+      if constexpr (detail::is_awaitable_try<Awaitable>) {+        auto ret = co_await co_awaitTry(std::move(awaitable));+        if (ret.hasException()) {+          exn = std::move(ret.exception());+        }+      } else {+        co_await std::move(awaitable);+      }+    } catch (...) {+      // not-awaitable-try awaitables and not-noexcept-copy-constructible values+      exn = exception_wrapper(std::current_exception());+    }+    if (exn && !exn.get_exception<OperationCancelled>()) {+      if (throwOnJoin) {+        LOG(ERROR)+            << (source.has_value() ? sourceLocationToString(source.value())+                                   : "")+            << "Unhandled exception thrown from task added to AsyncScope: "+            << exn;+        if (!exceptionRaised.exchange(true)) {+          maybeException = std::move(exn);+        }+      } else {+        LOG(DFATAL)+            << (source.has_value() ? sourceLocationToString(source.value())+                                   : "")+            << "Unhandled exception thrown from task added to AsyncScope: "+            << exn;+      }+    }+  }++  detail::Barrier barrier_{1};+  relaxed_atomic<bool> anyTasksStarted_{false};+  relaxed_atomic<bool> joinStarted_{false};+  relaxed_atomic<bool> joined_{false};+  bool throwOnJoin_{false};+  folly::exception_wrapper maybeException_;+  std::atomic<bool> exceptionRaised_{false};+};++inline AsyncScope::AsyncScope() noexcept : throwOnJoin_(false) {}++inline AsyncScope::AsyncScope(bool throwOnJoin) noexcept+    : throwOnJoin_(throwOnJoin) {}++inline AsyncScope::~AsyncScope() {+  CHECK(!anyTasksStarted_ || joined_)+      << "AsyncScope::cleanup() not yet complete";+}++inline std::size_t AsyncScope::remaining() const noexcept {+  const std::size_t count = barrier_.remaining();+  return joinStarted_ ? count : (count > 1 ? count - 1 : 0);+}++template <typename Awaitable>+FOLLY_NOINLINE inline void AsyncScope::add(+    Awaitable&& awaitable, void* returnAddress) {+  CHECK(!joined_)+      << "It is invalid to add() more work after work has been joined";+  anyTasksStarted_ = true;+  addImpl(+      static_cast<Awaitable&&>(awaitable),+      throwOnJoin_,+      maybeException_,+      std::nullopt,+      exceptionRaised_)+      .start(+          &barrier_,+          returnAddress ? returnAddress : FOLLY_ASYNC_STACK_RETURN_ADDRESS());+}++template <typename Awaitable>+FOLLY_NOINLINE inline void AsyncScope::addWithSourceLoc(+    Awaitable&& awaitable,+    void* returnAddress,+    source_location sourceLocation) {+  CHECK(!joined_)+      << "It is invalid to add() more work after work has been joined";+  anyTasksStarted_ = true;+  addImpl(+      static_cast<Awaitable&&>(awaitable),+      throwOnJoin_,+      maybeException_,+      sourceLocation,+      exceptionRaised_)+      .start(+          &barrier_,+          returnAddress ? returnAddress : FOLLY_ASYNC_STACK_RETURN_ADDRESS());+}++inline Task<void> AsyncScope::joinAsync() noexcept {+  assert(!joinStarted_ && "It is invalid to join a scope multiple times");+  joinStarted_ = true;+  co_await barrier_.arriveAndWait();+  joined_ = true;+  if (maybeException_) {+    co_yield co_error{std::move(maybeException_)};+  }+}++inline folly::SemiFuture<folly::Unit> AsyncScope::cleanup() noexcept {+  return joinAsync().semi();+}++/**+ * A cancellable version of AsyncScope. Work added to this scope will be+ * provided a cancellation token for cancelling during join.+ *+ * See add() and cancelAndJoinAsync() for more information.+ *+ * Note: Task and AsyncGenerator will ignore the internal cancellation+ * signal if they already have a cancellation token (i.e. if someone has already+ * called co_withCancellation on them.)+ * If you need an external cancellation signal as well, pass that token to this+ * constructor or to add() instead of attaching it to the Awaitable.+ *+ * @refcode+ * folly/docs/examples/folly/coro/CancellableAsyncScope.cpp+ * @class folly::coro::CancellableAsyncScope+ */+class CancellableAsyncScope {+ public:+  CancellableAsyncScope() noexcept+      : cancellationToken_(cancellationSource_.getToken()) {}+  explicit CancellableAsyncScope(bool throwOnJoin) noexcept+      : cancellationToken_(cancellationSource_.getToken()),+        scope_(throwOnJoin) {}+  explicit CancellableAsyncScope(CancellationToken&& token)+      : cancellationToken_(CancellationToken::merge(+            cancellationSource_.getToken(), std::move(token))) {}+  CancellableAsyncScope(CancellationToken&& token, bool throwOnJoin)+      : cancellationToken_(CancellationToken::merge(+            cancellationSource_.getToken(), std::move(token))),+        scope_(throwOnJoin) {}++  /**+   * Query the number of tasks added to the scope that have not yet completed.+   */+  std::size_t remaining() const noexcept { return scope_.remaining(); }++  /**+   * Start the specified task/awaitable by co_awaiting it. The awaitable will be+   * provided a cancellation token to respond to cancelAndJoinAsync() in the+   * future.+   *+   * An additional cancellation token may be passed in to apply to the+   * awaitable; it will be merged with the internal token.+   *+   * Note that cancellation is cooperative, your task must handle cancellation+   * in order to have any effect.+   *+   * See the documentation on AsyncScope::add.+   */+  template <typename Awaitable>+  FOLLY_NOINLINE void add(+      Awaitable&& awaitable,+      std::optional<CancellationToken> token = std::nullopt,+      void* returnAddress = nullptr) {+    scope_.add(+        co_withCancellation(+            token ? CancellationToken::merge(*token, cancellationToken_)+                  : cancellationToken_,+            static_cast<Awaitable&&>(awaitable)),+        returnAddress ? returnAddress : FOLLY_ASYNC_STACK_RETURN_ADDRESS());+  }++  template <typename Awaitable>+  FOLLY_NOINLINE void addWithSourceLoc(+      Awaitable&& awaitable,+      std::optional<CancellationToken> token,+      void* returnAddress = nullptr,+      source_location sourceLocation = source_location::current()) {+    scope_.addWithSourceLoc(+        co_withCancellation(+            token ? CancellationToken::merge(*token, cancellationToken_)+                  : cancellationToken_,+            static_cast<Awaitable&&>(awaitable)),+        returnAddress ? returnAddress : FOLLY_ASYNC_STACK_RETURN_ADDRESS(),+        sourceLocation);+  }++  template <typename Awaitable>+  void addWithSourceLoc(+      Awaitable&& awaitable,+      source_location sourceLocation = source_location::current()) {+    addWithSourceLoc(+        std::forward<Awaitable>(awaitable),+        std::nullopt,+        nullptr,+        std::move(sourceLocation));+  }++  /**+   * Schedules the given task on the current executor and adds it to the+   * AsyncScope. The task will be provided a cancellation token to respond to+   * cancelAndJoinAsync() in the future.+   *+   * Note that cancellation is cooperative, your task must handle cancellation+   * in order to have any effect.+   */+  template <class T>+  folly::coro::Task<void> co_schedule(folly::coro::Task<T>&& task) {+    add(co_withExecutor(co_await co_current_executor, std::move(task)));+  }++  /**+   * Request cancellation for all started tasks that accepted a+   * CancellationToken in add().+   */+  void requestCancellation() const noexcept {+    cancellationSource_.requestCancellation();+  }++  /**+   * Query if cancellation was requested on the tasks added to this AsyncScope.+   *+   * This will return true if either cancellation was requested using+   * `requestCancellation()` method of this scope OR if cancellation is+   * requested on the token passed into the constructor.+   */+  bool isScopeCancellationRequested() const noexcept {+    return cancellationToken_.isCancellationRequested();+  }++  /**+   * Request cancellation and asynchronously wait for all started tasks to+   * complete.+   *+   * Either call this method, _or_ joinAsync() to join the work. It is invalid+   * to call both of them.+   */+  Task<void> cancelAndJoinAsync() noexcept {+    requestCancellation();+    co_await joinAsync();+  }++  /**+   * Asynchronously wait for all started tasks to complete without requesting+   * cancellation.+   *+   * Either call this method _or_ cancelAndJoinAsync() to join the+   * work. It is invalid to call both of them.+   */+  Task<void> joinAsync() noexcept { co_await scope_.joinAsync(); }++ private:+  folly::CancellationSource cancellationSource_;+  CancellationToken cancellationToken_;+  AsyncScope scope_;+};++} // namespace coro+} // namespace folly++#endif // FOLLY_HAS_COROUTINES
+ folly/folly/coro/AsyncStack.h view
@@ -0,0 +1,78 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#include <folly/Executor.h>+#include <folly/coro/Coroutine.h>+#include <folly/coro/WithAsyncStack.h>+#include <folly/tracing/AsyncStack.h>++#include <utility>+#include <vector>++#if FOLLY_HAS_COROUTINES++namespace folly {+namespace coro {++class AsyncStackTraceAwaitable {+  class Awaiter {+   public:+    bool await_ready() const { return false; }++    template <typename Promise>+    bool await_suspend(coroutine_handle<Promise> h) noexcept {+      initialFrame_ = &h.promise().getAsyncFrame();+      return false;+    }++    FOLLY_NOINLINE std::vector<std::uintptr_t> await_resume() {+      static constexpr size_t maxFrames = 100;+      std::array<std::uintptr_t, maxFrames> result;++      result[0] =+          reinterpret_cast<std::uintptr_t>(FOLLY_ASYNC_STACK_RETURN_ADDRESS());+      auto numFrames = getAsyncStackTraceFromInitialFrame(+          initialFrame_, result.data() + 1, maxFrames - 1);++      return std::vector<std::uintptr_t>(+          std::make_move_iterator(result.begin()),+          std::make_move_iterator(result.begin()) + numFrames + 1);+    }++   private:+    folly::AsyncStackFrame* initialFrame_;+  };++ public:+  AsyncStackTraceAwaitable viaIfAsync(+      const folly::Executor::KeepAlive<>&) const noexcept {+    return {};+  }++  Awaiter operator co_await() const noexcept { return {}; }++  friend AsyncStackTraceAwaitable tag_invoke(+      cpo_t<co_withAsyncStack>, AsyncStackTraceAwaitable awaitable) noexcept {+    return awaitable;+  }+};++inline constexpr AsyncStackTraceAwaitable co_current_async_stack_trace = {};++} // namespace coro+} // namespace folly++#endif // FOLLY_HAS_COROUTINES
+ folly/folly/coro/AutoCleanup-fwd.h view
@@ -0,0 +1,53 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <folly/coro/Cleanup.h>+#include <folly/coro/Coroutine.h>++#if FOLLY_HAS_COROUTINES++namespace folly::coro {++template <typename T, typename CleanupFn = co_cleanup_fn>+class AutoCleanup;++template <typename T>+struct is_auto_cleanup : std::false_type {};++template <typename T>+constexpr bool is_auto_cleanup_v = is_auto_cleanup<T>::value;++template <typename T, typename CleanupFn>+struct is_auto_cleanup<AutoCleanup<T, CleanupFn>> : std::true_type {};++namespace detail {+template <typename Promise, typename... Args>+void scheduleAutoCleanup(coroutine_handle<Promise> coro, Args&... args);+} // namespace detail++template <typename Promise, typename... Args>+void scheduleAutoCleanupIfNeeded(+    coroutine_handle<Promise> coro, Args&... args) {+  if constexpr ((is_auto_cleanup_v<Args> || ...)) {+    detail::scheduleAutoCleanup(coro, args...);+  }+}++} // namespace folly::coro++#endif
+ folly/folly/coro/AutoCleanup.h view
@@ -0,0 +1,161 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <folly/coro/AutoCleanup-fwd.h>+#include <folly/coro/Collect.h>+#include <folly/coro/Task.h>++#if FOLLY_HAS_COROUTINES++namespace folly::coro {++namespace detail {+template <typename T>+struct ScopeExitArg {+  explicit ScopeExitArg(T&) {}++  folly::coro::Task<> cleanup() && { co_return; }++  void update(T&) {}+};+} // namespace detail++/// The user can use AutoCleanup to wrap arguments passed to a+/// CleanableAsyncGenerator. When the coroutine promise of+/// CleanableAsyncGenerator is created it will automatically attach+/// co_scope_exit task that performs async cleanup for all the arguments wrapped+/// in AutoCleanup. This allows to ensure cleanup of the arguments even when+/// next() of the CleanableAsyncGenerator is never co_awaited.+///+/// Example usage:+///  folly::coro::CleanableAsyncGenerator<std::pair<int, int>> zip(+///      folly::coro::AutoCleanup<folly::coro::CleanableAsyncGenerator<int>> a,+///      folly::coro::AutoCleanup<folly::coro::CleanableAsyncGenerator<int>> b+///  ) {+///    while (true) {+///      auto x = co_await a->next();+///      if (!x) {+///        break;+///      }+///      auto y = co_await b->next();+///      if (!y) {+///        break;+///      }+///      co_yield std::make_pair(*x, *y);+///    }+///  }+///+/// In the example above, a and b will be cleaned up automatically when+/// cleanup() of the zip generator is co_awaited, even if next() of the zip+/// generator have been never co_awaited.++template <typename T, typename CleanupFn>+class AutoCleanup : MoveOnly {+ public:+  using type = T;+  using cleanup_fn = CleanupFn;++  explicit AutoCleanup(T&& object, CleanupFn cleanupFn = CleanupFn{}) noexcept+      : ptr_{std::addressof(object)}, cleanupFn_{std::move(cleanupFn)} {}++  AutoCleanup(const AutoCleanup&) = delete;++  AutoCleanup(AutoCleanup&& other) noexcept+      : ptr_{std::exchange(other.ptr_, nullptr)},+        cleanupFn_{std::move(other.cleanupFn_)} {}++  ~AutoCleanup() { DCHECK(!kIsDebug || ptr_ == nullptr || scheduled_); }++  AutoCleanup& operator=(const AutoCleanup&) = delete;++  AutoCleanup& operator=(AutoCleanup&&) = delete;++  std::add_lvalue_reference_t<T> operator*() const+      noexcept(noexcept(*std::declval<T*>())) {+    return *get();+  }++  T* operator->() const noexcept { return get(); }++  T* get() const noexcept {+    DCHECK(!kIsDebug || scheduled_);+    return ptr_;+  }++ private:+  using BoolIfDebug = conditional_t<kIsDebug, bool, std::false_type>;++  T* ptr_;+  CleanupFn cleanupFn_;+  [[FOLLY_ATTR_NO_UNIQUE_ADDRESS]] BoolIfDebug scheduled_{};++  friend struct detail::ScopeExitArg<AutoCleanup<T, CleanupFn>>;+};++namespace detail {++template <typename T, typename CleanupFn>+struct ScopeExitArg<AutoCleanup<T, CleanupFn>> {+  T object;+  CleanupFn cleanupFn;++  explicit ScopeExitArg(AutoCleanup<T, CleanupFn>& autoCleanup)+      : object{std::move(*autoCleanup.ptr_)},+        cleanupFn{autoCleanup.cleanupFn_} {}++  auto cleanup() && { return cleanupFn(std::move(object)); }++  void update(AutoCleanup<T, CleanupFn>& autoCleanup) {+    autoCleanup.ptr_ = std::addressof(object);+    if constexpr (kIsDebug) {+      DCHECK(!autoCleanup.scheduled_);+      autoCleanup.scheduled_ = true;+    }+  }+};++template <typename Promise, typename Action, typename... Args>+auto attachScopeExit(+    coroutine_handle<Promise> coro, Action&& action, Args&&... args) {+  auto scopeExitAwaiter = co_viaIfAsync(+      nullptr,+      co_scope_exit(+          static_cast<Action&&>(action), static_cast<Args&&>(args)...));+  auto ready = scopeExitAwaiter.await_ready();+  DCHECK(!ready);+  auto suspend = scopeExitAwaiter.await_suspend(coro);+  DCHECK(!suspend);+  return scopeExitAwaiter.await_resume();+}++template <typename Promise, typename... Args>+void scheduleAutoCleanup(coroutine_handle<Promise> coro, Args&... args) {+  auto result = attachScopeExit(+      coro,+      [](auto&&... scopeExitArgs) -> Task<> {+        co_await collectAll(std::move(scopeExitArgs).cleanup()...);+      },+      detail::ScopeExitArg<Args>(args)...);+  std::apply([&](auto&... objs) { ((objs.update(args)), ...); }, result);+}++} // namespace detail++} // namespace folly::coro++#endif
+ folly/folly/coro/AwaitImmediately.h view
@@ -0,0 +1,239 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <type_traits>++#include <folly/Utility.h>++#if FOLLY_HAS_COROUTINES++namespace folly::coro {++// ## What are immediately-awaitable types, and how should I handle them?+//+// When `must_await_immediately_v<A> == true`, this indicates that the+// awaitable or semiawaitable `A` should be `co_await`ed in the full-expression+// that created it.  For an example, see `NowTask.h`.  Using immediate+// awaitables reduces the risk of lifetime bugs.+//+// To create a new immediately-awaitable type, follow this protocol:+//   - Derive from `public AddMustAwaitImmediately<YourBase>`.+//   - Implement `getUnsafeMover(ForMustAwaitImmediately) && noexcept`, but+//     first read the ENTIRE docblock of `mustAwaitImmediatelyUnsafeMover()`+//     with the notes on object slicing and `noexcept` behavior.+//+// To handle immediately-awaitables, `folly::coro` APIs must follow these rules:+//+//  (1) NEVER expose non-static member functions for actions that consume the+//      (semi)awaitable.  For example, `task.scheduleOn()` was removed in favor+//      of `co_withExecutor()`, `.start()` and `.semi()` are unsafe, etc.+//+//      INSTEAD: Use static member functions, or ADL CPOs, which take the+//      (semi)awaitable by-value.+//+//  (2) If your API must also support await-by-reference for types like+//      `Baton`, then you must bifurcate the API on the return value of+//      `must_await_immediately_v`.  Grep for examples.+//         - `true`: Pass-by-value+//         - `false`: Pass-by-forwarding reference+//+//  (3) Immediately-awaitable types are immovable, but you may need to move+//      them internally in your library implementation.  For this, you can use+//      the callable from `mustAwaitImmediatelyUnsafeMover(std::move(t))`.  For+//      example, you can move the mover into another coroutine, and then invoke+//      its `operator()` to reconstitute the (semi)awaitable.  Most often, you+//      will invoke the mover inline.+//+//      DANGER: You should NOT use this to move immovable types outside of+//      `folly::coro` library internals, where the lifetime safety is assured+//      via pass-by-value from (1) or (2).+//+// Caveat: If you encounter a public `folly::coro` API that is does not yet+// handle immediately-awaitable types, and simply takes the awaitable by `&&`,+// please fix it via one of these paths:+//   - If possible, switch it to take all (semi)awaitables by-value.+//   - If not, branch the API as in (2) above.+//   - Ask for help in the Coroutines group.++namespace detail {++template <typename T>+using must_await_immediately_of_ =+    typename T::folly_private_must_await_immediately_t;++template <typename Void, typename T>+struct must_await_immediately_ {+  static_assert(+      require_sizeof<T>, "`must_await_immediately_t` on incomplete type");+  using type = std::false_type;+};++template <>+struct must_await_immediately_<void, void> {+  using type = std::false_type;+};++template <typename T>+struct must_await_immediately_<void_t<must_await_immediately_of_<T>>, T> {+  // We _could_ do an "is `T` immovable" check, but the cost/benefit seems low.+  // That would only guard against a users wrongly adding this to their type:+  //   folly_private_must_await_immediately_t = std::true_type+  // instead of inheriting from `AddMustAwaitImmediately<>`.+  using type = must_await_immediately_of_<T>;+};++} // namespace detail++template <typename T>+using must_await_immediately_t =+    typename detail::must_await_immediately_<void, T>::type;++template <typename T>+inline constexpr bool must_await_immediately_v =+    must_await_immediately_t<T>::value;++// To make a (semi)awaitable immediate, have it publicly inherit from+// `AddMustAwaitImmediately<InnerSemiAwaitable>`.  It is templated on the+// "inner type" so that all bases are distinct in a tower-of-wrappers, like+// `TryAwaitable<NowTask<T>>`.  Repeating a base would break the empty basea+// optimization (EBO).+template <typename InnerT>+struct AddMustAwaitImmediately : public InnerT {+  using folly_private_must_await_immediately_t = std::true_type;++  using InnerT::InnerT;++  // Avoiding `NonCopyableNonMovable` to avoid breaking EBO.+  ~AddMustAwaitImmediately() = default;+  AddMustAwaitImmediately(AddMustAwaitImmediately&&) = delete;+  AddMustAwaitImmediately& operator=(AddMustAwaitImmediately&&) = delete;+  AddMustAwaitImmediately(const AddMustAwaitImmediately&) = delete;+  AddMustAwaitImmediately& operator=(const AddMustAwaitImmediately&) = delete;+};++// See `mustAwaitImmediatelyUnsafeMover()` for the docs.  In short, for an+// `Outer` that is immovable, this stores an unwrapped `Inner` (semi)awaitable+// that can reconstitute `Outer` on `operator()`.+//+// DANGER: Before returning this class from your `getUnsafeMover()`, you must+// review "A note on object slicing" and make sure your usage isn't affected.+template <typename Outer, typename InnerMover>+class MustAwaitImmediatelyUnsafeMover {+ private:+  InnerMover mover_;++ public:+  // `Outer*` is just for type deduction and should be `nullptr`.+  MustAwaitImmediatelyUnsafeMover(Outer*, InnerMover m) noexcept+      : mover_(std::move(m)) {+    // See mustAwaitImmediatelyUnsafeMover docblock+    static_assert(std::is_nothrow_move_constructible_v<InnerMover>);+    // See "A note on object slicing" below+    static_assert(+        sizeof(Outer) == sizeof(decltype(FOLLY_DECLVAL(InnerMover)())));+  }+  Outer operator()() && noexcept {+    // See mustAwaitImmediatelyUnsafeMover docblock+    static_assert(noexcept(Outer{std::move(mover_)()}));+    return Outer{std::move(mover_)()};+  }+};++// Analog of `MustAwaitImmediatelyUnsafeMover` for movable (semi)awaitables.+template <typename T>+struct NoOpMover {+ private:+  T t_;++ public:+  explicit NoOpMover(T t) noexcept : t_(std::move(t)) {+    // See mustAwaitImmediatelyUnsafeMover docblock+    static_assert(std::is_nothrow_move_constructible_v<T>);+  }+  T operator()() && noexcept { return std::move(t_); }+};++// Overload tag / passkey for the customizable method `getUnsafeMover`.+struct ForMustAwaitImmediately {};++namespace detail {+template <typename T>+using unsafe_mover_for_must_await_immediately_t =+    decltype(FOLLY_DECLVAL(T).getUnsafeMover(ForMustAwaitImmediately{}));+}++// After taking an immediately-(semi)awaitable by-value, this lets+// `folly::coro` libraries move it internally.  They MUST make sure not to let+// the awaitable be used outside of its original full-expression.+//+// This wraps `getUnsafeMover` for types that implement it, and provides+// a no-op fallback for those that don't. Required semantics:+//+//  - It's a destructive operation -- hence the r-value qualifier+//  - It takes ownership of the internals of `awaitable`.+//  - It returns a mover value (never a reference), whose `operator() &&` is a+//    single-use operation that returns a new awaitable equivalent to the+//    original `awaitable` that was passed in & moved out.+//+// ## A note on object slicing -- for `getUnsafeMover` implementations+//+// The default `getUnsafeMover()` implementations return `NoOpMover` or+// `MustAwaitImmediatelyUnsafeMover`.  The net effect is that the mover+// stores an unwrapped, inner type that is movable (like `Task`), and+// its `operator()` reconstitues the original "outer" type.+//+// This is fine for type-only wrappers.  But, object slicing is a danger for+// wrappers that affect the object's lifetime management or layout. For example:+//   - If "outer" adds a new member, this would be discarded.  The current+//     `getUnsafeMover`s compare before/after `sizeof`.  Do that in new ones!+//   - If the wrapper customizes destruction, move, copy, or assignment, then+//     the wrapping/unwrapping will cause the custom logic will run at an+//     unexpected time.  Such wrapper types MUST customize `getUnsafeMover` to+//     return a custom mover that handles this correctly.+//+// ## A note on `noexcept` discipline+//+// This `static_assert`s that constructing **and** using a mover is `noexcept`.+// This could, of course, be relaxed via `noexcept(noexcept(...))` logic, but+// IMO no `folly::coro` awaitables SHOULD have throwing move ctors, so this+// requirement is Actually Fine (until proven otherwise).+template <+    typename Awaitable,+    typename DetectRes = detected_or<+        NoOpMover<Awaitable>,+        detail::unsafe_mover_for_must_await_immediately_t,+        Awaitable>>+// CAREFUL: Passing by `&&` can violate the immediately-awaitable restriction!+typename DetectRes::type mustAwaitImmediatelyUnsafeMover(+    Awaitable&& awaitable) noexcept {+  static_assert(noexcept(FOLLY_DECLVAL(typename DetectRes::type&&)()));+  if constexpr (DetectRes::value_t::value) {+    static_assert(noexcept(static_cast<Awaitable&&>(awaitable).getUnsafeMover(+        ForMustAwaitImmediately{})));+    return static_cast<Awaitable&&>(awaitable).getUnsafeMover(+        ForMustAwaitImmediately{});+  } else {+    static_assert(+        std::is_nothrow_constructible_v<NoOpMover<Awaitable>, Awaitable&&>);+    return NoOpMover<Awaitable>{static_cast<Awaitable&&>(awaitable)};+  }+}++} // namespace folly::coro++#endif
+ folly/folly/coro/AwaitResult.h view
@@ -0,0 +1,158 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <folly/coro/ViaIfAsync.h>+#include <folly/result/try.h>++#if FOLLY_HAS_RESULT++namespace folly::coro {++/// `co_await_result` is the `result<T>` analog of the older `co_awaitTry`.+///+/// In a `folly::coro` async coroutine, use `co_await_result` like so:+///+///   result<int> res = co_await co_await_result(taskReturningInt());+///   if (auto* ex = get_exception<MyError>(res)) {+///     /* handle ex */+///   } else {+///     sum += co_await co_ready(res); // efficiently propagate unhandled error+///   }+///+/// Contrast that with related async coro vocabulary:+///  - `co_yield co_result(r)` from `Result.h` -- propagate `result<T>` or+///    `Try<T>` to the awaiter of the current coro.+///  - `auto& v = co_await co_ready(r)` from `Ready.h` -- given a `result<T>`,+///    unpack the value, or propagate any error to our awaiter.+///+/// The purpose of `co_await_result` is to handle errors from a child task via+/// `result<T>`, rather than through `try {} catch {}`.  Some reasons to do so:+///   - Your error-handling APIs (logging, retry, etc) use `result<T>`.+///   - You wish to avoid the ~microsecond cost of thrown exceptions,+///     applicable only when your error path is hot, and the child uses+///     `co_yield` instead of `throw` to propagate exceptions.++namespace detail {++template <typename Awaiter>+using detect_await_resume_result =+    decltype(FOLLY_DECLVAL(Awaiter).await_resume_result());++template <typename Awaiter>+constexpr bool is_awaiter_result =+    is_detected_v<detect_await_resume_result, Awaiter>;++template <typename Awaitable>+constexpr bool is_awaitable_result =+    is_awaiter_result<awaiter_type_t<Awaitable>>;++// On the happy path, this uses the dedicated `await_resume_result()` protocol,+// if it's supported by the awaiter.+//+// As fallback, this reuses the `await_resume_try()` machinery in the hope that+// the compiler will be able to optimize away the `Try` -> `result` conversion.+//+// The reasons to support the dedicated protocol are (1) better semantics, and+// (2) a data flow that's easier for the compiler to optimize.  Specifically:+//+//   - `await_resume_result()` cleanly handles `Task<V&>`, whereas `Try`+//     doesn't support storing references, and the caller of `co_awaitTry` has+//     to deal with `Try<std::reference_wrapper<V>>`.  See the test in+//     `TaskOfLvalueReferenceAsTry`.+//+//   - `await_resume_result()` implementations can explicitly avoid the "empty+//     `Try`" pitfall, which is something that gets converted to a+//     `UsingUninitializedTry` error by the `try_to_result()` fallback.+//+//   - Falling back to `await_resume_try()` can incur an extra move-copy, which+//     may not always optimize away.+template <typename Awaitable>+class ResultAwaiter {+ private:+  static_assert(is_awaitable_try<Awaitable> || is_awaitable_result<Awaitable>);++  using Awaiter = awaiter_type_t<Awaitable>;+  Awaiter awaiter_;++ public:+  explicit ResultAwaiter(Awaitable&& awaiter)+      : awaiter_(get_awaiter(static_cast<Awaitable&&>(awaiter))) {}++  // clang-format off+  auto await_ready() FOLLY_DETAIL_FORWARD_BODY(awaiter_.await_ready())++  template <typename Promise>+  auto await_suspend(coroutine_handle<Promise> coro)+      FOLLY_DETAIL_FORWARD_BODY(awaiter_.await_suspend(coro))+      // clang-format on++      template <+          typename Awaiter2 = Awaiter,+          typename Result =+              decltype(FOLLY_DECLVAL(Awaiter2&).await_resume_result())>+      Result await_resume() noexcept(noexcept(awaiter_.await_resume_result())) {+    return awaiter_.await_resume_result();+  }++  template <+      typename Awaiter2 = Awaiter,+      typename Result =+          decltype(try_to_result(FOLLY_DECLVAL(Awaiter2&).await_resume_try()))>+  Result await_resume() noexcept(+      noexcept(try_to_result(awaiter_.await_resume_try())))+    requires(!is_awaitable_result<Awaitable>)+  {+    return try_to_result(awaiter_.await_resume_try());+  }+};++template <typename T>+class [[FOLLY_ATTR_CLANG_CORO_AWAIT_ELIDABLE]] ResultAwaitable+    : public CommutativeWrapperAwaitable<ResultAwaitable, T> {+ public:+  using CommutativeWrapperAwaitable<ResultAwaitable, T>::+      CommutativeWrapperAwaitable;++  template <+      typename Self,+      std::enable_if_t<+          std::is_same_v<remove_cvref_t<Self>, ResultAwaitable>,+          int> = 0,+      typename T2 = like_t<Self, T>,+      std::enable_if_t<is_awaitable_v<T2>, int> = 0>+  friend ResultAwaiter<T2> operator co_await(Self && self) {+    return ResultAwaiter<T2>{static_cast<Self&&>(self).inner_};+  }++  using folly_private_noexcept_awaitable_t = std::true_type;+};++} // namespace detail++// IMPORTANT: If you need an `Awaitable&&` overload, you must bifurcate this+// API on `must_await_immediately_v`, see `co_awaitTry` for an example.+template <typename Awaitable>+detail::ResultAwaitable<Awaitable> co_await_result(+    [[FOLLY_ATTR_CLANG_CORO_AWAIT_ELIDABLE_ARGUMENT]] Awaitable awaitable) {+  return detail::ResultAwaitable<Awaitable>{+      mustAwaitImmediatelyUnsafeMover(std::move(awaitable))()};+}++} // namespace folly::coro++#endif
+ folly/folly/coro/Baton.cpp view
@@ -0,0 +1,69 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#include <folly/coro/Baton.h>++#include <folly/coro/Coroutine.h>+#include <folly/synchronization/AtomicUtil.h>++#include <cassert>+#include <utility>++#if FOLLY_HAS_COROUTINES++using namespace folly::coro;++Baton::~Baton() {+  // Should not be any waiting coroutines when the baton is destructed.+  // Caller should ensure the baton is posted before destructing.+  assert(+      state_.load(std::memory_order_relaxed) == static_cast<void*>(this) ||+      state_.load(std::memory_order_relaxed) == nullptr);+}++void Baton::post() noexcept {+  void* const signalledState = static_cast<void*>(this);+  void* oldValue = state_.exchange(signalledState, std::memory_order_acq_rel);+  if (oldValue != signalledState) {+    // We are the first thread to set the state to signalled and there is+    // a waiting coroutine. We are responsible for resuming it.+    WaitOperation* awaiter = static_cast<WaitOperation*>(oldValue);+    while (awaiter != nullptr) {+      std::exchange(awaiter, awaiter->next_)->awaitingCoroutine_.resume();+    }+  }+}++bool Baton::waitImpl(WaitOperation* awaiter) const noexcept {+  // Try to push the awaiter onto the front of the queue of waiters.+  const auto signalledState = static_cast<const void*>(this);+  void* oldValue = state_.load(std::memory_order_acquire);+  do {+    if (oldValue == signalledState) {+      // Already in the signalled state, don't enqueue it.+      return false;+    }+    awaiter->next_ = static_cast<WaitOperation*>(oldValue);+  } while (!folly::atomic_compare_exchange_weak_explicit(+      &state_,+      &oldValue,+      awaiter,+      std::memory_order_release,+      std::memory_order_acquire));+  return true;+}++#endif // FOLLY_HAS_COROUTINES
+ folly/folly/coro/Baton.h view
@@ -0,0 +1,156 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <atomic>++#include <folly/Try.h>+#include <folly/coro/Coroutine.h>++#if FOLLY_HAS_COROUTINES++namespace folly {+namespace coro {++/// A baton is a synchronisation primitive for coroutines that allows a+/// coroutine to co_await the baton and suspend until the baton is posted by+/// some thread via a call to .post().+///+/// This primitive is typically used in the construction of larger library types+/// rather than directly in user code.+///+/// As a primitive, this is not cancellation-aware.+///+/// The Baton supports being awaited by multiple coroutines at a time. If the+/// baton is not ready at the time it is awaited then an awaiting coroutine+/// suspends. All suspended coroutines waiting for the baton to be posted will+/// be resumed when some thread next calls .post().+///+/// Example usage:+///+///   folly::coro::Baton baton;+///   std::string sharedValue;+///+///   folly::coro::Task<void> consumer()+///   {+///     // Wait until the baton is posted.+///     co_await baton;+///+///     // Now safe to read shared state.+///     std::cout << sharedValue << std::cout;+///   }+///+///   void producer()+///   {+///     // Write to shared state+///     sharedValue = "some result";+///+///     // Publish the value by 'posting' the baton.+///     // This will resume the consumer if it was currently suspended.+///     baton.post();+///   }+class Baton {+ public:+  class WaitOperation;++  /// Initialise the Baton to either the signalled or non-signalled state.+  explicit Baton(bool initiallySignalled = false) noexcept;++  ~Baton();++  /// Query whether the Baton is currently in the signalled state.+  bool ready() const noexcept;++  /// Asynchronously wait for the Baton to enter the signalled state.+  ///+  /// The returned object must be co_awaited from a coroutine. If the Baton+  /// is already signalled then the awaiting coroutine will continue without+  /// suspending. Otherwise, if the Baton is not yet signalled then the+  /// awaiting coroutine will suspend execution and will be resumed when some+  /// thread later calls post().+  [[nodiscard]] WaitOperation operator co_await() const noexcept;++  /// Set the Baton to the signalled state if it is not already signalled.+  ///+  /// This will resume any coroutines that are currently suspended waiting+  /// for the Baton inside 'co_await baton'.+  void post() noexcept;++  /// Atomically reset the baton back to the non-signalled state.+  ///+  /// This is a no-op if the baton was already in the non-signalled state.+  void reset() noexcept;++  class WaitOperation {+   public:+    explicit WaitOperation(const Baton& baton) noexcept : baton_(baton) {}++    bool await_ready() const noexcept { return baton_.ready(); }++    bool await_suspend(coroutine_handle<> awaitingCoroutine) noexcept {+      awaitingCoroutine_ = awaitingCoroutine;+      return baton_.waitImpl(this);+    }++    void await_resume() noexcept {}++    // Awaiting a baton doesn't throw, so supporting `co_awaitTry` here only+    // serves to simplify generic code.+    folly::Try<void> await_resume_try() noexcept { return {}; }++   protected:+    friend class Baton;++    const Baton& baton_;+    coroutine_handle<> awaitingCoroutine_;+    WaitOperation* next_;+  };++ private:+  // Try to register the awaiter as+  bool waitImpl(WaitOperation* awaiter) const noexcept;++  // this  - Baton is in the signalled/posted state.+  // other - Baton is not signalled/posted and this is a pointer to the head+  //         of a potentially empty linked-list of Awaiter nodes that were+  //         waiting for the baton to become signalled.+  mutable std::atomic<void*> state_;+};++inline Baton::Baton(bool initiallySignalled) noexcept+    : state_(initiallySignalled ? static_cast<void*>(this) : nullptr) {}++inline bool Baton::ready() const noexcept {+  return state_.load(std::memory_order_acquire) ==+      static_cast<const void*>(this);+}++inline Baton::WaitOperation Baton::operator co_await() const noexcept {+  return Baton::WaitOperation{*this};+}++inline void Baton::reset() noexcept {+  // Transition from 'signalled' (ie. 'this') to not-signalled (ie. nullptr).+  void* oldState = this;+  (void)state_.compare_exchange_strong(+      oldState, nullptr, std::memory_order_acq_rel, std::memory_order_relaxed);+}++} // namespace coro+} // namespace folly++#endif // FOLLY_HAS_COROUTINES
+ folly/folly/coro/BlockingWait.h view
@@ -0,0 +1,460 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <folly/Try.h>+#include <folly/coro/AwaitImmediately.h>+#include <folly/coro/Coroutine.h>+#include <folly/coro/Task.h>+#include <folly/coro/Traits.h>+#include <folly/coro/ViaIfAsync.h>+#include <folly/coro/WithAsyncStack.h>+#include <folly/coro/detail/Malloc.h>+#include <folly/coro/detail/Traits.h>+#include <folly/executors/ManualExecutor.h>+#include <folly/fibers/Baton.h>+#include <folly/synchronization/Baton.h>+#include <folly/tracing/AsyncStack.h>++#include <cassert>+#include <exception>+#include <type_traits>+#include <utility>++#if FOLLY_HAS_COROUTINES++namespace folly {+namespace coro {++namespace detail {++template <typename T>+class BlockingWaitTask;++class BlockingWaitPromiseBase {+  struct FinalAwaiter {+    bool await_ready() noexcept { return false; }+    template <typename Promise>+    void await_suspend(coroutine_handle<Promise> coro) noexcept {+      BlockingWaitPromiseBase& promise = coro.promise();+      folly::deactivateAsyncStackFrame(promise.getAsyncFrame());+      promise.baton_.post();+    }+    void await_resume() noexcept {}+  };++ public:+  BlockingWaitPromiseBase() noexcept = default;++  static void* operator new(std::size_t size) {+    return ::folly_coro_async_malloc(size);+  }++  static void operator delete(void* ptr, std::size_t size) {+    ::folly_coro_async_free(ptr, size);+  }++  suspend_always initial_suspend() { return {}; }++  FinalAwaiter final_suspend() noexcept { return {}; }++  template <typename Awaitable>+  decltype(auto) await_transform(Awaitable&& awaitable) {+    return folly::coro::co_withAsyncStack(static_cast<Awaitable&&>(awaitable));+  }++  bool done() const noexcept { return baton_.ready(); }++  void wait() noexcept { baton_.wait(); }++  folly::AsyncStackFrame& getAsyncFrame() noexcept { return asyncFrame_; }++ private:+  folly::fibers::Baton baton_;+  folly::AsyncStackFrame asyncFrame_;+};++template <typename T>+class BlockingWaitPromise final : public BlockingWaitPromiseBase {+ public:+  BlockingWaitPromise() noexcept = default;++  ~BlockingWaitPromise() = default;++  BlockingWaitTask<T> get_return_object() noexcept;++  void unhandled_exception() noexcept {+    result_->emplaceException(folly::exception_wrapper{current_exception()});+  }++  template <+      typename U = T,+      std::enable_if_t<std::is_convertible<U, T>::value, int> = 0>+  void return_value(U&& value) noexcept(+      std::is_nothrow_constructible<T, U&&>::value) {+    result_->emplace(static_cast<U&&>(value));+  }++  void setTry(folly::Try<T>* result) noexcept { result_ = &result; }++ private:+  folly::Try<T>* result_;+};++template <typename T>+class BlockingWaitPromise<T&> final : public BlockingWaitPromiseBase {+ public:+  BlockingWaitPromise() noexcept = default;++  ~BlockingWaitPromise() = default;++  BlockingWaitTask<T&> get_return_object() noexcept;++  void unhandled_exception() noexcept {+    result_->emplaceException(folly::exception_wrapper{current_exception()});+  }++  auto yield_value(T&& value) noexcept {+    result_->emplace(std::ref(value));+    return final_suspend();+  }++  auto yield_value(T& value) noexcept {+    result_->emplace(std::ref(value));+    return final_suspend();+  }++  void return_void() {+    // This should never be reachable.+    // The coroutine should either have suspended at co_yield or should have+    // thrown an exception and skipped over the implicit co_return and+    // gone straight to unhandled_exception().+    std::abort();+  }++  void setTry(folly::Try<std::reference_wrapper<T>>* result) noexcept {+    result_ = result;+  }++ private:+  folly::Try<std::reference_wrapper<T>>* result_;+};++template <>+class BlockingWaitPromise<void> final : public BlockingWaitPromiseBase {+ public:+  BlockingWaitPromise() = default;++  BlockingWaitTask<void> get_return_object() noexcept;++  void return_void() noexcept {}++  void unhandled_exception() noexcept {+    result_->emplaceException(exception_wrapper{current_exception()});+  }++  void setTry(folly::Try<void>* result) noexcept { result_ = result; }++ private:+  folly::Try<void>* result_;+};++template <typename T>+class BlockingWaitTask {+ public:+  using promise_type = BlockingWaitPromise<T>;+  using handle_t = coroutine_handle<promise_type>;++  explicit BlockingWaitTask(handle_t coro) noexcept : coro_(coro) {}++  BlockingWaitTask(BlockingWaitTask&& other) noexcept+      : coro_(std::exchange(other.coro_, {})) {}++  BlockingWaitTask& operator=(BlockingWaitTask&& other) noexcept = delete;++  ~BlockingWaitTask() {+    if (coro_) {+      coro_.destroy();+    }+  }++  FOLLY_NOINLINE T get(folly::AsyncStackFrame& parentFrame) && {+    folly::Try<detail::lift_lvalue_reference_t<T>> result;+    auto& promise = coro_.promise();+    promise.setTry(&result);++    auto& asyncFrame = promise.getAsyncFrame();+    asyncFrame.setParentFrame(parentFrame);+    asyncFrame.setReturnAddress();+    {+      RequestContextScopeGuard guard{RequestContext::saveContext()};+      folly::resumeCoroutineWithNewAsyncStackRoot(coro_);+    }+    promise.wait();+    return std::move(result).value();+  }++  FOLLY_NOINLINE T getVia(+      folly::DrivableExecutor* executor,+      folly::AsyncStackFrame& parentFrame) && {+    folly::Try<detail::lift_lvalue_reference_t<T>> result;+    auto& promise = coro_.promise();+    promise.setTry(&result);++    auto& asyncFrame = promise.getAsyncFrame();+    asyncFrame.setReturnAddress();+    asyncFrame.setParentFrame(parentFrame);++    executor->add(+        [coro = coro_, rctx = RequestContext::saveContext()]() mutable {+          RequestContextScopeGuard guard{std::move(rctx)};+          folly::resumeCoroutineWithNewAsyncStackRoot(coro);+        });+    while (!promise.done()) {+      executor->drive();+    }+    return std::move(result).value();+  }++ private:+  handle_t coro_;+};++template <typename T>+inline BlockingWaitTask<T>+BlockingWaitPromise<T>::get_return_object() noexcept {+  return BlockingWaitTask<T>{+      coroutine_handle<BlockingWaitPromise<T>>::from_promise(*this)};+}++template <typename T>+inline BlockingWaitTask<T&>+BlockingWaitPromise<T&>::get_return_object() noexcept {+  return BlockingWaitTask<T&>{+      coroutine_handle<BlockingWaitPromise<T&>>::from_promise(*this)};+}++inline BlockingWaitTask<void>+BlockingWaitPromise<void>::get_return_object() noexcept {+  return BlockingWaitTask<void>{+      coroutine_handle<BlockingWaitPromise<void>>::from_promise(*this)};+}++template <+    typename Awaitable,+    typename Result = await_result_t<Awaitable>,+    std::enable_if_t<std::is_void<Result>::value, int> = 0>+BlockingWaitTask<void> makeRefBlockingWaitTask(Awaitable&& awaitable) {+  co_await static_cast<Awaitable&&>(awaitable);+}++template <+    typename Awaitable,+    typename Result = await_result_t<Awaitable>,+    std::enable_if_t<!std::is_void<Result>::value, int> = 0>+auto makeRefBlockingWaitTask(Awaitable&& awaitable)+    -> BlockingWaitTask<std::add_lvalue_reference_t<Result>> {+  co_yield co_await static_cast<Awaitable&&>(awaitable);+}++class BlockingWaitExecutor final : public folly::DrivableExecutor {+ public:+  ~BlockingWaitExecutor() override {+    while (keepAliveCount_.load() > 0) {+      drive();+    }+  }++  void add(Func func) override {+    bool empty;+    {+      auto wQueue = queue_.wlock();+      empty = wQueue->empty();+      wQueue->push_back(std::move(func));+    }+    if (empty) {+      baton_.post();+    }+  }++  void drive() override {+    baton_.wait();+    baton_.reset();++    folly::fibers::runInMainContext([&]() {+      std::vector<Func> funcs;+      queue_.swap(funcs);+      for (auto& func : funcs) {+        std::exchange(func, nullptr)();+      }+    });+  }++ private:+  bool keepAliveAcquire() noexcept override {+    auto keepAliveCount =+        keepAliveCount_.fetch_add(1, std::memory_order_relaxed);+    DCHECK(keepAliveCount >= 0);+    return true;+  }++  void keepAliveRelease() noexcept override {+    auto keepAliveCount = keepAliveCount_.load(std::memory_order_relaxed);+    do {+      DCHECK(keepAliveCount > 0);+      if (keepAliveCount == 1) {+        add([this] {+          // the final count *must* be released from this executor or else if we+          // are mid-destructor we have a data race+          keepAliveCount_.fetch_sub(1, std::memory_order_relaxed);+        });+        return;+      }+    } while (!keepAliveCount_.compare_exchange_weak(+        keepAliveCount,+        keepAliveCount - 1,+        std::memory_order_release,+        std::memory_order_relaxed));+  }++  folly::Synchronized<std::vector<Func>> queue_;+  fibers::Baton baton_;++  std::atomic<ssize_t> keepAliveCount_{0};+};++} // namespace detail++/// blocking_wait_fn+///+/// Awaits co_awaits the passed awaitable and blocks the current thread until+/// the await operation completes.+///+/// Useful for launching an asynchronous operation from the top-level main()+/// function or from unit-tests.+///+/// WARNING:+/// Avoid using this function within any code that might run on the thread+/// of an executor as this can potentially lead to deadlock if the operation+/// you are waiting on needs to do some work on that executor in order to+/// complete.+struct blocking_wait_fn {+  template <typename Awaitable>+  FOLLY_NOINLINE auto operator()(Awaitable&& awaitable) const+      -> detail::decay_rvalue_reference_t<await_result_t<Awaitable>> {+    folly::AsyncStackFrame frame;+    frame.setReturnAddress();++    folly::AsyncStackRoot stackRoot;+    stackRoot.setNextRoot(folly::tryGetCurrentAsyncStackRoot());+    stackRoot.setStackFrameContext();+    stackRoot.setTopFrame(frame);++    return static_cast<std::add_rvalue_reference_t<await_result_t<Awaitable>>>(+        detail::makeRefBlockingWaitTask(static_cast<Awaitable&&>(awaitable))+            .get(frame));+  }++  template <+      typename SemiAwaitable,+      std::enable_if_t<!must_await_immediately_v<SemiAwaitable>, int> = 0>+  FOLLY_NOINLINE auto operator()(+      SemiAwaitable&& awaitable, folly::DrivableExecutor* executor) const+      -> detail::decay_rvalue_reference_t<semi_await_result_t<SemiAwaitable>> {+    folly::AsyncStackFrame frame;+    frame.setReturnAddress();++    folly::AsyncStackRoot stackRoot;+    stackRoot.setNextRoot(folly::tryGetCurrentAsyncStackRoot());+    stackRoot.setStackFrameContext();+    stackRoot.setTopFrame(frame);++    return static_cast<+        std::add_rvalue_reference_t<semi_await_result_t<SemiAwaitable>>>(+        detail::makeRefBlockingWaitTask(+            folly::coro::co_viaIfAsync(+                folly::getKeepAliveToken(executor),+                static_cast<SemiAwaitable&&>(awaitable)))+            .getVia(executor, frame));+  }+  template <+      typename SemiAwaitable,+      std::enable_if_t<must_await_immediately_v<SemiAwaitable>, int> = 0>+  FOLLY_NOINLINE auto operator()(+      SemiAwaitable awaitable, folly::DrivableExecutor* executor) const+      -> detail::decay_rvalue_reference_t<semi_await_result_t<SemiAwaitable>> {+    folly::AsyncStackFrame frame;+    frame.setReturnAddress();++    folly::AsyncStackRoot stackRoot;+    stackRoot.setNextRoot(folly::tryGetCurrentAsyncStackRoot());+    stackRoot.setStackFrameContext();+    stackRoot.setTopFrame(frame);++    return static_cast<+        std::add_rvalue_reference_t<semi_await_result_t<SemiAwaitable>>>(+        detail::makeRefBlockingWaitTask(+            folly::coro::co_viaIfAsync(+                folly::getKeepAliveToken(executor),+                mustAwaitImmediatelyUnsafeMover(std::move(awaitable))()))+            .getVia(executor, frame));+  }++  template <+      typename SemiAwaitable,+      std::enable_if_t<!is_awaitable_v<SemiAwaitable>, int> = 0,+      std::enable_if_t<!must_await_immediately_v<SemiAwaitable>, int> = 0>+  auto operator()(SemiAwaitable&& awaitable) const+      -> detail::decay_rvalue_reference_t<semi_await_result_t<SemiAwaitable>> {+    std::exception_ptr eptr;+    {+      detail::BlockingWaitExecutor executor;+      try {+        return operator()(static_cast<SemiAwaitable&&>(awaitable), &executor);+      } catch (...) {+        eptr = current_exception();+      }+    }+    std::rethrow_exception(eptr);+  }+  template <+      typename SemiAwaitable,+      std::enable_if_t<!is_awaitable_v<SemiAwaitable>, int> = 0,+      std::enable_if_t<must_await_immediately_v<SemiAwaitable>, int> = 0>+  auto operator()(SemiAwaitable awaitable) const+      -> detail::decay_rvalue_reference_t<semi_await_result_t<SemiAwaitable>> {+    std::exception_ptr eptr;+    {+      detail::BlockingWaitExecutor executor;+      try {+        return operator()(+            mustAwaitImmediatelyUnsafeMover(std::move(awaitable))(), &executor);+      } catch (...) {+        eptr = current_exception();+      }+    }+    std::rethrow_exception(eptr);+  }+};+inline constexpr blocking_wait_fn blocking_wait{};+static constexpr blocking_wait_fn const& blockingWait =+    blocking_wait; // backcompat++} // namespace coro+} // namespace folly++#endif // FOLLY_HAS_COROUTINES
+ folly/folly/coro/BoundedQueue.h view
@@ -0,0 +1,157 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <folly/MPMCQueue.h>+#include <folly/ProducerConsumerQueue.h>+#include <folly/coro/Task.h>+#include <folly/fibers/Semaphore.h>++#if FOLLY_HAS_COROUTINES++namespace folly {+namespace coro {++// A coroutine version of bounded queue with given capacity. Both enqueue and+// dequeue are async awaitable.+template <typename T, bool SingleProducer = false, bool SingleConsumer = false>+class BoundedQueue {+  static constexpr bool kSPSC = SingleProducer && SingleConsumer;++ public:+  explicit BoundedQueue(uint32_t capacity)+      : queue_(+            kSPSC ? capacity + 1 // One more extra space because usable space of+                                 // ProducerConsumerQueue used below is (size-1)+                  : capacity),+        enqueueSemaphore_{capacity},+        dequeueSemaphore_{0} {}++  BoundedQueue(const BoundedQueue&) = delete;+  BoundedQueue& operator=(const BoundedQueue&) = delete;++  template <typename U = T>+  folly::coro::Task<void> enqueue(U&& item) {+    co_await folly::coro::co_nothrow(enqueueSemaphore_.co_wait());+    enqueueReady(std::forward<U>(item));+    dequeueSemaphore_.signal();+  }++  template <typename U = T>+  bool try_enqueue(U&& item) {+    auto waitSuccess = enqueueSemaphore_.try_wait();+    if (!waitSuccess) {+      return false;+    }+    enqueueReady(std::forward<U>(item));+    dequeueSemaphore_.signal();+    return true;+  }++  // Dequeue a value from the queue.+  // Note that this operation can be safely cancelled by requesting cancellation+  // on the awaiting coroutine's associated CancellationToken.+  // If the operation is successfully cancelled then it will complete with+  // an error of type folly::OperationCancelled.+  // WARNING: It is not safe to wrap this with folly::coro::timeout(). Wrap with+  // folly::coro::timeoutNoDiscard(), or use co_try_dequeue_for() instead.+  folly::coro::Task<T> dequeue() {+    co_await folly::coro::co_nothrow(dequeueSemaphore_.co_wait());+    T item;+    dequeueReady(item);+    enqueueSemaphore_.signal();+    co_return item;+  }++  // Try to dequeue a value from the queue with a timeout. The operation will+  // either successfully dequeue an item from the queue, or else be cancelled+  // and complete with an error of type folly::OperationCancelled.+  template <typename Duration>+  folly::coro::Task<T> co_try_dequeue_for(Duration timeout) {+    co_await folly::coro::co_nothrow(+        dequeueSemaphore_.co_try_wait_for(timeout));+    T item;+    dequeueReady(item);+    enqueueSemaphore_.signal();+    co_return item;+  }++  folly::coro::Task<void> dequeue(T& item) {+    co_await folly::coro::co_nothrow(dequeueSemaphore_.co_wait());+    dequeueReady(item);+    enqueueSemaphore_.signal();+  }++  std::optional<T> try_dequeue() {+    T item;+    if (try_dequeue(item)) {+      return item;+    }+    return std::nullopt;+  }++  bool try_dequeue(T& item) {+    auto waitSuccess = dequeueSemaphore_.try_wait();+    if (!waitSuccess) {+      return false;+    }+    dequeueReady(item);+    enqueueSemaphore_.signal();+    return true;+  }++  bool empty() const { return queue_.isEmpty(); }++  size_t size() const {+    if constexpr (kSPSC) {+      return queue_.sizeGuess();+    } else {+      return queue_.size();+    }+  }++ private:+  template <typename U = T>+  void enqueueReady(U&& item) {+    if constexpr (kSPSC) {+      CHECK(queue_.write(std::forward<U>(item)));+    } else {+      // Cannot use write() because the thread that acquired the next ticket may+      // not have completed the read yet.+      CHECK(queue_.writeIfNotFull(std::forward<U>(item)));+    }+  }++  void dequeueReady(T& item) {+    if constexpr (kSPSC) {+      CHECK(queue_.read(item));+    } else {+      // Cannot use read() because the thread that acquired the next ticket may+      // not have completed the write yet.+      CHECK(queue_.readIfNotEmpty(item));+    }+  }++  std::conditional_t<kSPSC, ProducerConsumerQueue<T>, MPMCQueue<T>> queue_;+  folly::fibers::Semaphore enqueueSemaphore_;+  folly::fibers::Semaphore dequeueSemaphore_;+};++} // namespace coro+} // namespace folly++#endif // FOLLY_HAS_COROUTINES
+ folly/folly/coro/Cleanup.h view
@@ -0,0 +1,50 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <folly/functional/Invoke.h>++#if FOLLY_HAS_COROUTINES++namespace folly::coro {++/// A customization point that allows to provide an async cleanup function for a+/// type. folly::coro::AutoCleanup uses co_cleanup_fn as the default cleanup+/// function, so it is enough to define co_cleanup for a type to be able to use+/// it with AutoCleanup.+struct co_cleanup_fn {+  template <+      typename T,+      std::enable_if_t<+          folly::is_tag_invocable_v<co_cleanup_fn, T&&> &&+              !std::is_lvalue_reference_v<T>,+          int> = 0>+  auto operator()(T&& object) const+      noexcept(folly::is_nothrow_tag_invocable_v<co_cleanup_fn, T&&>)+          -> folly::tag_invoke_result_t<co_cleanup_fn, T&&> {+    return folly::tag_invoke(co_cleanup_fn{}, std::forward<T>(object));+  }++  template <typename T>+  void operator()(T& object) = delete;+};++FOLLY_DEFINE_CPO(co_cleanup_fn, co_cleanup)++} // namespace folly::coro++#endif
+ folly/folly/coro/Collect-inl.h view
@@ -0,0 +1,1184 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#include <atomic>+#include <utility>++#include <folly/CancellationToken.h>+#include <folly/ExceptionWrapper.h>+#include <folly/coro/AsyncPipe.h>+#include <folly/coro/AsyncScope.h>+#include <folly/coro/Mutex.h>+#include <folly/coro/detail/Barrier.h>+#include <folly/coro/detail/BarrierTask.h>+#include <folly/coro/detail/CurrentAsyncFrame.h>+#include <folly/coro/detail/Helpers.h>++#if FOLLY_HAS_COROUTINES++namespace folly {+namespace coro {+namespace detail {++template <typename T>+T&& getValueOrUnit(Try<T>&& value) {+  assert(value.hasValue());+  return std::move(value).value();+}++inline Unit getValueOrUnit([[maybe_unused]] Try<void>&& value) {+  assert(value.hasValue());+  return Unit{};+}++template <+    typename InputRange,+    typename Make,+    typename Iter = invoke_result_t<access::begin_fn, InputRange&>,+    typename Elem = remove_cvref_t<decltype(*std::declval<Iter&>())>,+    typename RTask = invoke_result_t<Make&, Elem, std::size_t>>+std::vector<RTask> collectMakeInnerTaskVec(InputRange& awaitables, Make& make) {+  std::vector<RTask> tasks;++  auto abegin = access::begin(awaitables);+  auto aend = access::end(awaitables);++  if constexpr (is_invocable_v<folly::access::size_fn, InputRange&>) {+    tasks.reserve(static_cast<std::size_t>(folly::access::size(awaitables)));+  } else if constexpr (range_has_known_distance_v<InputRange&>) {+    tasks.reserve(static_cast<std::size_t>(std::distance(abegin, aend)));+  }++  std::size_t index = 0;+  for (auto aiter = abegin; aiter != aend; ++aiter) {+    tasks.push_back(make(std::move(*aiter), index++));+  }++  return tasks;+}++template <typename SemiAwaitableMover, typename Result>+BarrierTask makeCollectAllTryTask(+    Executor::KeepAlive<> executor,+    const CancellationToken& cancelToken,+    SemiAwaitableMover&& mover,+    Try<Result>& result) {+  try {+    if constexpr (std::is_void_v<Result>) {+      co_await co_viaIfAsync(+          std::move(executor),+          co_withCancellation(+              cancelToken, static_cast<SemiAwaitableMover&&>(mover)()));+      result.emplace();+    } else {+      result.emplace(co_await co_viaIfAsync(+          std::move(executor),+          co_withCancellation(+              cancelToken, static_cast<SemiAwaitableMover&&>(mover)())));+    }+  } catch (...) {+    result.emplaceException(current_exception());+  }+}++template <+    typename Ret,+    typename... SemiAwaitables,+    size_t... Indices,+    typename... SemiAwaitablesMovers>+Ret collectAllTryImpl(+    tag_t<Ret, SemiAwaitables...>,+    std::index_sequence<Indices...>,+    SemiAwaitablesMovers... movers) {+  static_assert(sizeof...(Indices) == sizeof...(SemiAwaitables));+  static_assert(sizeof...(Indices) == sizeof...(SemiAwaitablesMovers));+  if constexpr (sizeof...(SemiAwaitables) == 0) {+    co_return std::tuple<>{};+  } else {+    const Executor::KeepAlive<> executor = co_await co_current_executor;+    const CancellationToken& cancelToken =+        co_await co_current_cancellation_token;++    std::tuple<collect_all_try_component_t<SemiAwaitables>...> results;++    folly::coro::detail::BarrierTask tasks[sizeof...(SemiAwaitables)] = {+        makeCollectAllTryTask(+            executor.get_alias(),+            cancelToken,+            static_cast<SemiAwaitablesMovers&&>(movers),+            std::get<Indices>(results))...,+    };++    folly::coro::detail::Barrier barrier{sizeof...(SemiAwaitables) + 1};++    auto& asyncFrame = co_await detail::co_current_async_stack_frame;++    // Use std::initializer_list to ensure that the sub-tasks are launched+    // in the order they appear in the parameter pack.++    // Save the initial context and restore it after starting each task+    // as the task may have modified the context before suspending and we+    // want to make sure the next task is started with the same initial+    // context.+    const auto context = RequestContext::saveContext();+    (void)std::initializer_list<int>{(+        tasks[Indices].start(&barrier, asyncFrame),+        RequestContext::setContext(context),+        0)...};++    // Wait for all of the sub-tasks to finish execution.+    // Should be safe to avoid an executor transition here even if the+    // operation completes asynchronously since all of the child tasks+    // should already have transitioned to the correct executor due to+    // the use of co_viaIfAsync() within makeCollectAllTryTask().+    co_await UnsafeResumeInlineSemiAwaitable{barrier.arriveAndWait()};++    co_return results;+  }+}++template <+    typename Ret,+    typename... SemiAwaitables,+    size_t... Indices,+    typename... SemiFns>+Ret collectAllImpl(+    tag_t<Ret, SemiAwaitables...>,+    std::index_sequence<Indices...>,+    // `semiFns()` is the immovable, must-await-immediately `SemiAwaitable`+    SemiFns... semiFns) {+  if constexpr (sizeof...(SemiAwaitables) == 0) {+    co_return std::tuple<>{};+  } else {+    const Executor::KeepAlive<> executor = co_await co_current_executor;+    const CancellationToken& parentCancelToken =+        co_await co_current_cancellation_token;++    const CancellationSource cancelSource;+    const CancellationToken cancelToken =+        CancellationToken::merge(parentCancelToken, cancelSource.getToken());++    exception_wrapper firstException;++    auto makeTask = [&](auto&& fn, auto& result) -> BarrierTask {+      using await_result =+          semi_await_result_t<decltype(static_cast<decltype(fn)>(fn)())>;+      try {+        if constexpr (std::is_void_v<await_result>) {+          co_await co_viaIfAsync(+              executor.get_alias(),+              co_withCancellation(+                  cancelToken, static_cast<decltype(fn)>(fn)()));+          result.emplace();+        } else {+          result.emplace(co_await co_viaIfAsync(+              executor.get_alias(),+              co_withCancellation(+                  cancelToken, static_cast<decltype(fn)>(fn)())));+        }+      } catch (...) {+        if (!cancelSource.requestCancellation()) {+          // This was the first failure, remember its error.+          firstException = exception_wrapper{current_exception()};+        }+      }+    };++    std::tuple<collect_all_try_component_t<SemiAwaitables>...> results;++    folly::coro::detail::BarrierTask tasks[sizeof...(SemiAwaitables)] = {+        makeTask(+            static_cast<SemiFns&&>(semiFns), std::get<Indices>(results))...,+    };++    folly::coro::detail::Barrier barrier{sizeof...(SemiAwaitables) + 1};++    // Save the initial context and restore it after starting each task+    // as the task may have modified the context before suspending and we+    // want to make sure the next task is started with the same initial+    // context.+    const auto context = RequestContext::saveContext();++    auto& asyncFrame = co_await detail::co_current_async_stack_frame;++    // Use std::initializer_list to ensure that the sub-tasks are launched+    // in the order they appear in the parameter pack.+    (void)std::initializer_list<int>{(+        tasks[Indices].start(&barrier, asyncFrame),+        RequestContext::setContext(context),+        0)...};++    // Wait for all of the sub-tasks to finish execution.+    // Should be safe to avoid an executor transition here even if the+    // operation completes asynchronously since all of the child tasks+    // should already have transitioned to the correct executor due to+    // the use of co_viaIfAsync() within makeBarrierTask().+    co_await UnsafeResumeInlineSemiAwaitable{barrier.arriveAndWait()};++    if (firstException) {+      co_yield co_error(std::move(firstException));+    }++    co_return std::tuple<collect_all_component_t<SemiAwaitables>...>{+        getValueOrUnit(std::get<Indices>(std::move(results)))...};+  }+}++template <typename InputRange, typename IsTry, typename AsyncScope>+auto makeUnorderedAsyncGeneratorImpl(+    AsyncScope& scope, InputRange awaitables, IsTry) {+  using Item =+      async_generator_from_awaitable_range_item_t<InputRange, IsTry::value>;+  return [](AsyncScope& scopeParam,+            InputRange awaitablesParam) -> AsyncGenerator<Item&&> {+    auto [results, pipe] = AsyncPipe<Item, false>::create();+    struct SharedState {+      explicit SharedState(AsyncPipe<Item, false>&& p) : pipe(std::move(p)) {}++      AsyncPipe<Item, false> pipe;+      const CancellationSource cancelSource;+    };+    auto sharedState = std::make_shared<SharedState>(std::move(pipe));+    auto cancelToken = sharedState->cancelSource.getToken();++    auto guard = folly::makeGuard([&] {+      sharedState->cancelSource.requestCancellation();+    });+    auto ex = co_await co_current_executor;+    size_t expected = 0;+    // Save the initial context and restore it after starting each task+    // as the task may have modified the context before suspending and we+    // want to make sure the next task is started with the same initial+    // context.+    const auto context = RequestContext::saveContext();++    for (auto&& semiAwaitable : static_cast<InputRange&&>(awaitablesParam)) {+      auto task = [](auto semiAwaitableParam, auto state) -> Task<void> {+        auto result = co_await co_awaitTry(std::move(semiAwaitableParam));+        if (!result.hasValue() && !IsTry::value) {+          state->cancelSource.requestCancellation();+        }+        state->pipe.write(std::move(result));+      }(static_cast<decltype(semiAwaitable)&&>(semiAwaitable), sharedState);+      if constexpr (std::is_same_v<AsyncScope, folly::coro::AsyncScope>) {+        scopeParam.add(co_withExecutor(+            ex, co_withCancellation(cancelToken, std::move(task))));+      } else {+        static_assert(std::is_same_v<AsyncScope, CancellableAsyncScope>);+        scopeParam.add(co_withExecutor(ex, std::move(task)), cancelToken);+      }+      ++expected;+      RequestContext::setContext(context);+    }++    while (expected > 0) {+      CancellationCallback cancelCallback(+          co_await co_current_cancellation_token,+          [&]() noexcept { sharedState->cancelSource.requestCancellation(); });++      if constexpr (!IsTry::value) {+        auto result = co_await co_awaitTry(results.next());+        if (result.hasValue() && result->has_value()) {+          co_yield std::move(**result);+          if (--expected) {+            continue;+          }+          result.emplace(); // completion result+        }+        guard.dismiss();+        co_yield co_result(std::move(result));+      } else {+        // Prevent AsyncPipe from receiving cancellation so we get the right+        // number of OperationCancelled.+        auto result = co_await co_withCancellation({}, results.next());+        co_yield std::move(*result);+        if (--expected == 0) {+          guard.dismiss();+          co_return;+        }+      }+    }+  }(scope, std::move(awaitables));+}++template <typename... SemiAwaitables, size_t... Indices>+auto collectAnyImpl(+    std::index_sequence<Indices...>, SemiAwaitables&&... awaitables)+    -> folly::coro::Task<std::pair<+        std::size_t,+        folly::Try<collect_any_component_t<SemiAwaitables...>>>> {+  const CancellationToken& parentCancelToken =+      co_await co_current_cancellation_token;+  const CancellationSource cancelSource;+  const CancellationToken cancelToken =+      CancellationToken::merge(parentCancelToken, cancelSource.getToken());++  std::pair<std::size_t, folly::Try<collect_any_component_t<SemiAwaitables...>>>+      firstCompletion;+  firstCompletion.first = size_t(-1);+  co_await folly::coro::collectAll(folly::coro::co_withCancellation(+      cancelToken,+      folly::coro::co_invoke(+          [&, aw = static_cast<SemiAwaitables&&>(awaitables)]() mutable+          -> folly::coro::Task<void> {+            auto result = co_await folly::coro::co_awaitTry(+                static_cast<SemiAwaitables&&>(aw));+            if (!cancelSource.requestCancellation()) {+              // This is first entity to request cancellation.+              firstCompletion.first = Indices;+              firstCompletion.second = std::move(result);+            }+          }))...);++  co_return firstCompletion;+}++template <typename... SemiAwaitables, size_t... Indices>+auto collectAnyWithoutExceptionImpl(+    std::index_sequence<Indices...>, SemiAwaitables&&... awaitables)+    -> folly::coro::Task<std::pair<+        std::size_t,+        folly::Try<detail::collect_any_component_t<SemiAwaitables...>>>> {+  const CancellationToken& parentCancelToken =+      co_await co_current_cancellation_token;+  const CancellationSource cancelSource;+  const CancellationToken cancelToken =+      CancellationToken::merge(parentCancelToken, cancelSource.getToken());++  constexpr std::size_t nAwaitables = sizeof...(SemiAwaitables);+  std::atomic<std::size_t> nAwaited = 1;+  std::pair<std::size_t, folly::Try<collect_any_component_t<SemiAwaitables...>>>+      firstValueOrLastException;+  firstValueOrLastException.first = std::numeric_limits<size_t>::max();+  co_await folly::coro::collectAll(folly::coro::co_withCancellation(+      cancelToken, [&]() -> folly::coro::Task<void> {+        auto result = co_await folly::coro::co_awaitTry(+            std::forward<SemiAwaitables>(awaitables));+        if ((result.hasValue() ||+             nAwaited.fetch_add(1, std::memory_order_relaxed) == nAwaitables) &&+            !cancelSource.requestCancellation()) {+          firstValueOrLastException.first = Indices;+          firstValueOrLastException.second = std::move(result);+        }+      }())...);++  co_return firstValueOrLastException;+}++template <typename... SemiAwaitables, size_t... Indices>+auto collectAnyNoDiscardImpl(+    std::index_sequence<Indices...>, SemiAwaitables&&... awaitables)+    -> folly::coro::Task<+        std::tuple<collect_all_try_component_t<SemiAwaitables>...>> {+  const CancellationSource cancelSource;+  const CancellationToken cancelToken = CancellationToken::merge(+      co_await co_current_cancellation_token, cancelSource.getToken());++  std::tuple<collect_all_try_component_t<SemiAwaitables>...> results;+  co_await folly::coro::collectAll(folly::coro::co_withCancellation(+      cancelToken, folly::coro::co_invoke([&]() -> folly::coro::Task<void> {+        auto result = co_await folly::coro::co_awaitTry(+            std::forward<SemiAwaitables>(awaitables));+        cancelSource.requestCancellation();+        std::get<Indices>(results) = std::move(result);+      }))...);++  co_return results;+}++} // namespace detail++template <typename... SemiAwaitables>+auto collectAll(SemiAwaitables... awaitables)+    -> detail::CollectAllTask<SemiAwaitables...> {+  return detail::collectAllImpl(+      tag<detail::CollectAllTask<SemiAwaitables...>, SemiAwaitables...>,+      std::make_index_sequence<sizeof...(SemiAwaitables)>{},+      mustAwaitImmediatelyUnsafeMover(+          static_cast<SemiAwaitables&&>(awaitables))...);+}++template <typename... SemiAwaitables>+auto collectAllTry(SemiAwaitables... awaitables)+    -> detail::CollectAllTryTask<SemiAwaitables...> {+  return detail::collectAllTryImpl(+      tag<detail::CollectAllTryTask<SemiAwaitables...>, SemiAwaitables...>,+      std::make_index_sequence<sizeof...(SemiAwaitables)>{},+      mustAwaitImmediatelyUnsafeMover(+          static_cast<SemiAwaitables&&>(awaitables))...);+}++template <+    typename InputRange,+    std::enable_if_t<+        !std::is_void_v<+            semi_await_result_t<detail::range_reference_t<InputRange>>>,+        int>>+auto collectAllRange(InputRange awaitables)+    -> folly::coro::Task<std::vector<detail::collect_all_range_component_t<+        detail::range_reference_t<InputRange>>>> {+  const folly::Executor::KeepAlive<> executor = co_await co_current_executor;+  const CancellationSource cancelSource;+  const CancellationToken cancelToken = CancellationToken::merge(+      co_await co_current_cancellation_token, cancelSource.getToken());++  std::vector<detail::collect_all_try_range_component_t<+      detail::range_reference_t<InputRange>>>+      tryResults;++  exception_wrapper firstException;++  using awaitable_type = remove_cvref_t<detail::range_reference_t<InputRange>>;+  auto makeTask = [&](awaitable_type semiAwaitable, std::size_t index)+      -> detail::BarrierTask {+    assert(index < tryResults.size());++    try {+      tryResults[index].emplace(co_await co_viaIfAsync(+          executor.get_alias(),+          co_withCancellation(cancelToken, std::move(semiAwaitable))));+    } catch (...) {+      if (!cancelSource.requestCancellation()) {+        firstException = exception_wrapper{current_exception()};+      }+    }+  };++  auto tasks = detail::collectMakeInnerTaskVec(awaitables, makeTask);++  tryResults.resize(tasks.size());++  // Save the initial context and restore it after starting each task+  // as the task may have modified the context before suspending and we+  // want to make sure the next task is started with the same initial+  // context.+  const auto context = RequestContext::saveContext();++  auto& asyncFrame = co_await detail::co_current_async_stack_frame;++  // Launch the tasks and wait for them all to finish.+  {+    detail::Barrier barrier{tasks.size() + 1};+    for (auto&& task : tasks) {+      task.start(&barrier, asyncFrame);+      RequestContext::setContext(context);+    }+    co_await detail::UnsafeResumeInlineSemiAwaitable{barrier.arriveAndWait()};+  }++  // Check if there were any exceptions and rethrow the first one.+  if (firstException) {+    co_yield co_error(std::move(firstException));+  }++  std::vector<detail::collect_all_range_component_t<+      detail::range_reference_t<InputRange>>>+      results;+  results.reserve(tryResults.size());+  for (auto& result : tryResults) {+    results.emplace_back(std::move(result).value());+  }++  co_return results;+}++template <+    typename InputRange,+    std::enable_if_t<+        std::is_void_v<+            semi_await_result_t<detail::range_reference_t<InputRange>>>,+        int>>+auto collectAllRange(InputRange awaitables) -> folly::coro::Task<void> {+  const folly::Executor::KeepAlive<> executor = co_await co_current_executor;+  const CancellationSource cancelSource;+  const CancellationToken cancelToken = CancellationToken::merge(+      co_await co_current_cancellation_token, cancelSource.getToken());++  exception_wrapper firstException;++  using awaitable_type = remove_cvref_t<detail::range_reference_t<InputRange>>;+  auto makeTask =+      [&](awaitable_type semiAwaitable, std::size_t) -> detail::BarrierTask {+    try {+      co_await co_viaIfAsync(+          executor.get_alias(),+          co_withCancellation(cancelToken, std::move(semiAwaitable)));+    } catch (...) {+      if (!cancelSource.requestCancellation()) {+        firstException = exception_wrapper{current_exception()};+      }+    }+  };++  auto tasks = detail::collectMakeInnerTaskVec(awaitables, makeTask);++  // Save the initial context and restore it after starting each task+  // as the task may have modified the context before suspending and we+  // want to make sure the next task is started with the same initial+  // context.+  const auto context = RequestContext::saveContext();++  auto& asyncFrame = co_await detail::co_current_async_stack_frame;++  // Launch the tasks and wait for them all to finish.+  {+    detail::Barrier barrier{tasks.size() + 1};+    for (auto&& task : tasks) {+      task.start(&barrier, asyncFrame);+      RequestContext::setContext(context);+    }+    co_await detail::UnsafeResumeInlineSemiAwaitable{barrier.arriveAndWait()};+  }++  // Check if there were any exceptions and rethrow the first one.+  if (firstException) {+    co_yield co_error(std::move(firstException));+  }+}++template <typename InputRange>+auto collectAllTryRange(InputRange awaitables)+    -> folly::coro::Task<std::vector<detail::collect_all_try_range_component_t<+        detail::range_reference_t<InputRange>>>> {+  std::vector<detail::collect_all_try_range_component_t<+      detail::range_reference_t<InputRange>>>+      results;++  const folly::Executor::KeepAlive<> executor =+      folly::getKeepAliveToken(co_await co_current_executor);++  const CancellationToken& cancelToken = co_await co_current_cancellation_token;++  using awaitable_type = remove_cvref_t<detail::range_reference_t<InputRange>>;+  auto makeTask = [&](awaitable_type semiAwaitable, std::size_t index)+      -> detail::BarrierTask {+    assert(index < results.size());+    auto& result = results[index];+    try {+      using await_result = semi_await_result_t<awaitable_type>;+      if constexpr (std::is_void_v<await_result>) {+        co_await co_viaIfAsync(+            executor.get_alias(),+            co_withCancellation(cancelToken, std::move(semiAwaitable)));+        result.emplace();+      } else {+        result.emplace(co_await co_viaIfAsync(+            executor.get_alias(),+            co_withCancellation(cancelToken, std::move(semiAwaitable))));+      }+    } catch (...) {+      result.emplaceException(current_exception());+    }+  };++  auto tasks = detail::collectMakeInnerTaskVec(awaitables, makeTask);++  // Now that we know how many tasks there are, allocate that+  // many Try objects to store the results before we start+  // executing the tasks.+  results.resize(tasks.size());++  // Save the initial context and restore it after starting each task+  // as the task may have modified the context before suspending and we+  // want to make sure the next task is started with the same initial+  // context.+  const auto context = RequestContext::saveContext();++  auto& asyncFrame = co_await detail::co_current_async_stack_frame;++  // Launch the tasks and wait for them all to finish.+  {+    detail::Barrier barrier{tasks.size() + 1};+    for (auto&& task : tasks) {+      task.start(&barrier, asyncFrame);+      RequestContext::setContext(context);+    }+    co_await detail::UnsafeResumeInlineSemiAwaitable{barrier.arriveAndWait()};+  }++  co_return results;+}++template <+    typename InputRange,+    std::enable_if_t<+        std::is_void_v<+            semi_await_result_t<detail::range_reference_t<InputRange>>>,+        int>>+auto collectAllWindowed(InputRange awaitables, std::size_t maxConcurrency)+    -> folly::coro::Task<void> {+  assert(maxConcurrency > 0);++  const folly::Executor::KeepAlive<> executor = co_await co_current_executor;+  const CancellationSource cancelSource;+  const CancellationToken cancelToken = CancellationToken::merge(+      co_await co_current_cancellation_token, cancelSource.getToken());++  exception_wrapper firstException;++  const auto trySetFirstException = [&](exception_wrapper&& e) noexcept {+    if (!cancelSource.requestCancellation()) {+      // This is first entity to request cancellation.+      firstException = std::move(e);+    }+  };++  auto iter = access::begin(awaitables);+  const auto iterEnd = access::end(awaitables);++  using iterator_t = decltype(iter);+  using awaitable_t = typename std::iterator_traits<iterator_t>::value_type;++  folly::coro::Mutex mutex;++  exception_wrapper iterationException;++  auto makeWorker = [&]() -> detail::BarrierTask {+    auto lock =+        co_await co_viaIfAsync(executor.get_alias(), mutex.co_scoped_lock());++    while (!iterationException && iter != iterEnd) {+      std::optional<awaitable_t> awaitable;+      try {+        awaitable.emplace(*iter);+        ++iter;+      } catch (...) {+        iterationException = exception_wrapper{current_exception()};+        cancelSource.requestCancellation();+      }++      if (!awaitable) {+        co_return;+      }++      lock.unlock();++      try {+        co_await co_viaIfAsync(+            executor.get_alias(),+            co_withCancellation(cancelToken, std::move(*awaitable)));+      } catch (...) {+        trySetFirstException(exception_wrapper{current_exception()});+      }++      lock =+          co_await co_viaIfAsync(executor.get_alias(), mutex.co_scoped_lock());+    }+  };++  std::vector<detail::BarrierTask> workerTasks;++  detail::Barrier barrier{1};++  // Save the initial context and restore it after starting each task+  // as the task may have modified the context before suspending and we+  // want to make sure the next task is started with the same initial+  // context.+  const auto context = RequestContext::saveContext();++  auto& asyncFrame = co_await detail::co_current_async_stack_frame;++  try {+    auto lock = co_await mutex.co_scoped_lock();++    while (!iterationException && iter != iterEnd &&+           workerTasks.size() < maxConcurrency) {+      // Unlock the mutex before starting the worker so that+      // it can consume as many results synchronously as it can before+      // returning here and letting us spawn another task.+      // This can avoid spawning more worker coroutines than is necessary+      // to consume all of the awaitables.+      lock.unlock();++      workerTasks.push_back(makeWorker());+      barrier.add(1);+      workerTasks.back().start(&barrier, asyncFrame);++      RequestContext::setContext(context);++      lock = co_await mutex.co_scoped_lock();+    }+  } catch (...) {+    if (workerTasks.empty()) {+      iterationException = exception_wrapper{current_exception()};+    }+  }++  co_await detail::UnsafeResumeInlineSemiAwaitable{barrier.arriveAndWait()};++  if (auto& ex = iterationException ? iterationException : firstException) {+    co_yield co_error(std::move(ex));+  }+}++template <+    typename InputRange,+    std::enable_if_t<+        !std::is_void_v<+            semi_await_result_t<detail::range_reference_t<InputRange>>>,+        int>>+auto collectAllWindowed(InputRange awaitables, std::size_t maxConcurrency)+    -> folly::coro::Task<std::vector<detail::collect_all_range_component_t<+        detail::range_reference_t<InputRange>>>> {+  assert(maxConcurrency > 0);++  const folly::Executor::KeepAlive<> executor = co_await co_current_executor;++  const CancellationToken& parentCancelToken =+      co_await co_current_cancellation_token;+  const CancellationSource cancelSource;+  const CancellationToken cancelToken =+      CancellationToken::merge(parentCancelToken, cancelSource.getToken());++  exception_wrapper firstException;++  auto trySetFirstException = [&](exception_wrapper&& e) noexcept {+    if (!cancelSource.requestCancellation()) {+      // This is first entity to request cancellation.+      firstException = std::move(e);+    }+  };++  auto iter = access::begin(awaitables);+  const auto iterEnd = access::end(awaitables);++  using iterator_t = decltype(iter);+  using awaitable_t = typename std::iterator_traits<iterator_t>::value_type;++  folly::coro::Mutex mutex;++  std::vector<detail::collect_all_try_range_component_t<+      detail::range_reference_t<InputRange>>>+      tryResults;++  exception_wrapper iterationException;++  auto makeWorker = [&]() -> detail::BarrierTask {+    auto lock =+        co_await co_viaIfAsync(executor.get_alias(), mutex.co_scoped_lock());++    while (!iterationException && iter != iterEnd) {+      const std::size_t thisIndex = tryResults.size();+      std::optional<awaitable_t> awaitable;+      try {+        tryResults.emplace_back();+        awaitable.emplace(*iter);+        ++iter;+      } catch (...) {+        iterationException = exception_wrapper{current_exception()};+        cancelSource.requestCancellation();+      }++      if (!awaitable) {+        co_return;+      }++      lock.unlock();++      detail::collect_all_try_range_component_t<+          detail::range_reference_t<InputRange>>+          tryResult;++      try {+        tryResult.emplace(co_await co_viaIfAsync(+            executor.get_alias(),+            co_withCancellation(+                cancelToken, static_cast<awaitable_t&&>(*awaitable))));+      } catch (...) {+        trySetFirstException(exception_wrapper{current_exception()});+      }++      lock =+          co_await co_viaIfAsync(executor.get_alias(), mutex.co_scoped_lock());++      try {+        tryResults[thisIndex] = std::move(tryResult);+      } catch (...) {+        trySetFirstException(exception_wrapper{current_exception()});+      }+    }+  };++  std::vector<detail::BarrierTask> workerTasks;++  detail::Barrier barrier{1};++  exception_wrapper workerCreationException;++  // Save the initial context and restore it after starting each task+  // as the task may have modified the context before suspending and we+  // want to make sure the next task is started with the same initial+  // context.+  const auto context = RequestContext::saveContext();++  auto& asyncFrame = co_await detail::co_current_async_stack_frame;++  try {+    auto lock = co_await mutex.co_scoped_lock();++    while (!iterationException && iter != iterEnd &&+           workerTasks.size() < maxConcurrency) {+      // Unlock the mutex before starting the worker so that+      // it can consume as many results synchronously as it can before+      // returning here and letting us spawn another task.+      // This can avoid spawning more worker coroutines than is necessary+      // to consume all of the awaitables.+      lock.unlock();++      workerTasks.push_back(makeWorker());+      barrier.add(1);+      workerTasks.back().start(&barrier, asyncFrame);++      RequestContext::setContext(context);++      lock = co_await mutex.co_scoped_lock();+    }+  } catch (...) {+    // Only a fatal error if we failed to create any worker tasks.+    if (workerTasks.empty()) {+      // No need to synchronise here. There are no concurrent tasks running.+      iterationException = exception_wrapper{current_exception()};+    }+  }++  co_await detail::UnsafeResumeInlineSemiAwaitable{barrier.arriveAndWait()};++  if (auto& ex = iterationException ? iterationException : firstException) {+    co_yield co_error(std::move(ex));+  }++  std::vector<detail::collect_all_range_component_t<+      detail::range_reference_t<InputRange>>>+      results;+  results.reserve(tryResults.size());++  for (auto&& tryResult : tryResults) {+    assert(tryResult.hasValue());+    results.emplace_back(std::move(tryResult).value());+  }++  co_return results;+}++template <typename InputRange>+auto collectAllTryWindowed(InputRange awaitables, std::size_t maxConcurrency)+    -> folly::coro::Task<std::vector<detail::collect_all_try_range_component_t<+        detail::range_reference_t<InputRange>>>> {+  assert(maxConcurrency > 0);++  std::vector<detail::collect_all_try_range_component_t<+      detail::range_reference_t<InputRange>>>+      results;++  exception_wrapper iterationException;++  folly::coro::Mutex mutex;++  const Executor::KeepAlive<> executor = co_await co_current_executor;+  const CancellationToken& cancelToken = co_await co_current_cancellation_token;++  auto iter = access::begin(awaitables);+  const auto iterEnd = access::end(awaitables);++  using iterator_t = decltype(iter);+  using awaitable_t = typename std::iterator_traits<iterator_t>::value_type;+  using result_t = semi_await_result_t<awaitable_t>;++  auto makeWorker = [&]() -> detail::BarrierTask {+    auto lock =+        co_await co_viaIfAsync(executor.get_alias(), mutex.co_scoped_lock());++    while (!iterationException && iter != iterEnd) {+      const std::size_t thisIndex = results.size();+      std::optional<awaitable_t> awaitable;++      try {+        results.emplace_back();+        awaitable.emplace(*iter);+        ++iter;+      } catch (...) {+        iterationException = exception_wrapper{current_exception()};+      }++      if (!awaitable) {+        co_return;+      }++      lock.unlock();++      detail::collect_all_try_range_component_t<+          detail::range_reference_t<InputRange>>+          result;++      try {+        if constexpr (std::is_void_v<result_t>) {+          co_await co_viaIfAsync(+              executor.get_alias(),+              co_withCancellation(cancelToken, std::move(*awaitable)));+          result.emplace();+        } else {+          result.emplace(co_await co_viaIfAsync(+              executor.get_alias(),+              co_withCancellation(cancelToken, std::move(*awaitable))));+        }+      } catch (...) {+        result.emplaceException(current_exception());+      }++      lock =+          co_await co_viaIfAsync(executor.get_alias(), mutex.co_scoped_lock());++      try {+        results[thisIndex] = std::move(result);+      } catch (...) {+        results[thisIndex].emplaceException(current_exception());+      }+    }+  };++  std::vector<detail::BarrierTask> workerTasks;++  detail::Barrier barrier{1};++  // Save the initial context and restore it after starting each task+  // as the task may have modified the context before suspending and we+  // want to make sure the next task is started with the same initial+  // context.+  const auto context = RequestContext::saveContext();++  auto& asyncFrame = co_await detail::co_current_async_stack_frame;++  try {+    auto lock = co_await mutex.co_scoped_lock();+    while (!iterationException && iter != iterEnd &&+           workerTasks.size() < maxConcurrency) {+      // Unlock the mutex before starting the child operation so that+      // it can consume as many results synchronously as it can before+      // returning here and letting us potentially spawn another task.+      // This can avoid spawning more worker coroutines than is necessary+      // to consume all of the awaitables.+      lock.unlock();++      workerTasks.push_back(makeWorker());+      barrier.add(1);+      workerTasks.back().start(&barrier, asyncFrame);++      RequestContext::setContext(context);++      lock = co_await mutex.co_scoped_lock();+    }+  } catch (...) {+    // Failure to create a worker is an error if we failed+    // to create _any_ workers. As long as we created one then+    // the algorithm should still be able to make forward progress.+    if (workerTasks.empty()) {+      iterationException = exception_wrapper{current_exception()};+    }+  }++  co_await detail::UnsafeResumeInlineSemiAwaitable{barrier.arriveAndWait()};++  if (iterationException) {+    co_yield co_error(std::move(iterationException));+  }++  co_return results;+}++template <typename InputRange>+auto makeUnorderedAsyncGenerator(AsyncScope& scope, InputRange awaitables)+    -> AsyncGenerator<detail::async_generator_from_awaitable_range_item_t<+        InputRange,+        false>&&> {+  return detail::makeUnorderedAsyncGeneratorImpl(+      scope, std::move(awaitables), std::bool_constant<false>{});+}++template <typename InputRange>+auto makeUnorderedTryAsyncGenerator(AsyncScope& scope, InputRange awaitables)+    -> AsyncGenerator<detail::async_generator_from_awaitable_range_item_t<+        InputRange,+        true>&&> {+  return detail::makeUnorderedAsyncGeneratorImpl(+      scope, std::move(awaitables), std::bool_constant<true>{});+}++template <typename InputRange>+auto makeUnorderedAsyncGenerator(+    CancellableAsyncScope& scope, InputRange awaitables)+    -> AsyncGenerator<detail::async_generator_from_awaitable_range_item_t<+        InputRange,+        false>&&> {+  return detail::makeUnorderedAsyncGeneratorImpl(+      scope, std::move(awaitables), std::bool_constant<false>{});+}++template <typename InputRange>+auto makeUnorderedTryAsyncGenerator(+    CancellableAsyncScope& scope, InputRange awaitables)+    -> AsyncGenerator<detail::async_generator_from_awaitable_range_item_t<+        InputRange,+        true>&&> {+  return detail::makeUnorderedAsyncGeneratorImpl(+      scope, std::move(awaitables), std::bool_constant<true>{});+}++template <typename SemiAwaitable, typename... SemiAwaitables>+auto collectAny(SemiAwaitable&& awaitable, SemiAwaitables&&... awaitables)+    -> folly::coro::Task<std::pair<+        std::size_t,+        folly::Try<detail::collect_any_component_t<+            SemiAwaitable,+            SemiAwaitables...>>>> {+  return detail::collectAnyImpl(+      std::make_index_sequence<sizeof...(SemiAwaitables) + 1>{},+      static_cast<SemiAwaitable&&>(awaitable),+      static_cast<SemiAwaitables&&>(awaitables)...);+}++template <typename... SemiAwaitables>+auto collectAnyWithoutException(SemiAwaitables&&... awaitables)+    -> folly::coro::Task<std::pair<+        std::size_t,+        folly::Try<detail::collect_any_component_t<SemiAwaitables...>>>> {+  return detail::collectAnyWithoutExceptionImpl(+      std::make_index_sequence<sizeof...(SemiAwaitables)>{},+      static_cast<SemiAwaitables&&>(awaitables)...);+}++template <typename... SemiAwaitables>+auto collectAnyNoDiscard(SemiAwaitables&&... awaitables)+    -> folly::coro::Task<std::tuple<detail::collect_all_try_component_t<+        remove_cvref_t<SemiAwaitables>>...>> {+  return detail::collectAnyNoDiscardImpl(+      std::make_index_sequence<sizeof...(SemiAwaitables)>{},+      static_cast<SemiAwaitables&&>(awaitables)...);+}++template <typename InputRange>+auto collectAnyRange(InputRange awaitables)+    -> folly::coro::Task<std::pair<+        size_t,+        folly::Try<detail::collect_all_range_component_t<+            detail::range_reference_t<InputRange>>>>> {+  const CancellationToken& parentCancelToken =+      co_await co_current_cancellation_token;+  const CancellationSource cancelSource;+  const CancellationToken cancelToken =+      CancellationToken::merge(parentCancelToken, cancelSource.getToken());++  std::pair<+      size_t,+      folly::Try<detail::collect_all_range_component_t<+          detail::range_reference_t<InputRange>>>>+      firstCompletion;+  firstCompletion.first = size_t(-1);++  using awaitable_type = remove_cvref_t<detail::range_reference_t<InputRange>>;+  auto makeTask = [&](awaitable_type semiAwaitable, size_t index)+      -> folly::coro::Task<void> {+    auto result = co_await folly::coro::co_awaitTry(std::move(semiAwaitable));+    if (!cancelSource.requestCancellation()) {+      // This is first entity to request cancellation.+      firstCompletion.first = index;+      firstCompletion.second = std::move(result);+    }+  };++  auto tasks = detail::collectMakeInnerTaskVec(awaitables, makeTask);++  co_await folly::coro::co_withCancellation(+      cancelToken, folly::coro::collectAllRange(detail::MoveRange(tasks)));++  co_return firstCompletion;+}++template <typename InputRange>+auto collectAnyWithoutExceptionRange(InputRange awaitables)+    -> folly::coro::Task<std::pair<+        size_t,+        folly::Try<detail::collect_all_range_component_t<+            detail::range_reference_t<InputRange>>>>> {+  const CancellationToken& parentCancelToken =+      co_await co_current_cancellation_token;+  const CancellationSource cancelSource;+  const CancellationToken cancelToken =+      CancellationToken::merge(parentCancelToken, cancelSource.getToken());++  size_t nAwaitables;+  std::atomic<std::size_t> nAwaited = 1;+  std::pair<+      size_t,+      folly::Try<detail::collect_all_range_component_t<+          detail::range_reference_t<InputRange>>>>+      firstValueOrLastException;+  firstValueOrLastException.first = std::numeric_limits<size_t>::max();++  using awaitable_type = remove_cvref_t<detail::range_reference_t<InputRange>>;+  auto makeTask = [&](awaitable_type semiAwaitable, size_t index)+      -> folly::coro::Task<void> {+    auto result = co_await folly::coro::co_awaitTry(std::move(semiAwaitable));+    if ((result.hasValue() ||+         nAwaited.fetch_add(1, std::memory_order_relaxed) == nAwaitables) &&+        !cancelSource.requestCancellation()) {+      firstValueOrLastException.first = index;+      firstValueOrLastException.second = std::move(result);+    }+  };++  auto tasks = detail::collectMakeInnerTaskVec(awaitables, makeTask);+  nAwaitables = tasks.size();+  co_await folly::coro::co_withCancellation(+      cancelToken, folly::coro::collectAllRange(detail::MoveRange(tasks)));++  co_return firstValueOrLastException;+}++template <typename InputRange>+auto collectAnyNoDiscardRange(InputRange awaitables)+    -> folly::coro::Task<std::vector<detail::collect_all_try_range_component_t<+        detail::range_reference_t<InputRange>>>> {+  const CancellationToken& parentCancelToken =+      co_await co_current_cancellation_token;+  const CancellationSource cancelSource;+  const CancellationToken cancelToken =+      CancellationToken::merge(parentCancelToken, cancelSource.getToken());++  std::vector<detail::collect_all_try_range_component_t<+      detail::range_reference_t<InputRange>>>+      results;++  using awaitable_type = remove_cvref_t<detail::range_reference_t<InputRange>>;+  auto makeTask = [&](awaitable_type semiAwaitable, size_t index)+      -> folly::coro::Task<void> {+    auto result = co_await folly::coro::co_awaitTry(std::move(semiAwaitable));+    cancelSource.requestCancellation();+    results[index] = std::move(result);+  };++  auto tasks = detail::collectMakeInnerTaskVec(awaitables, makeTask);++  results.resize(tasks.size());+  co_await folly::coro::co_withCancellation(+      cancelToken, folly::coro::collectAllRange(detail::MoveRange(tasks)));++  co_return results;+}++} // namespace coro+} // namespace folly++#endif // FOLLY_HAS_COROUTINES
+ folly/folly/coro/Collect.h view
@@ -0,0 +1,658 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <folly/Try.h>+#include <folly/Unit.h>+#include <folly/container/Access.h>+#include <folly/container/Iterator.h>+#include <folly/coro/AsyncGenerator.h>+#include <folly/coro/AsyncScope.h>+#include <folly/coro/Coroutine.h>+#include <folly/coro/Task.h>+#include <folly/coro/ViaIfAsync.h>+#include <folly/coro/detail/PickTaskWrapper.h>+#include <folly/coro/detail/Traits.h>+// `collectAll(coroFutureInt())` makes a `SafeTask`+#include <folly/coro/safe/SafeTask.h>+// `collectAll(memberTask())` makes a `NowTask`+#include <folly/coro/safe/NowTask.h>++#include <functional>+#include <iterator>+#include <tuple>+#include <type_traits>++#if FOLLY_HAS_COROUTINES++namespace folly {+namespace coro {+namespace detail {++template <typename SemiAwaitable>+using collect_all_try_component_t = folly::Try<decay_rvalue_reference_t<+    lift_lvalue_reference_t<semi_await_result_t<SemiAwaitable>>>>;++template <typename SemiAwaitable>+using collect_all_component_t =+    decay_rvalue_reference_t<lift_unit_t<semi_await_result_t<SemiAwaitable>>>;++template <typename SemiAwaitable>+using collect_all_range_component_t = decay_rvalue_reference_t<+    lift_lvalue_reference_t<lift_unit_t<semi_await_result_t<SemiAwaitable>>>>;++template <typename SemiAwaitable>+using collect_all_try_range_component_t =+    collect_all_try_component_t<SemiAwaitable>;++template <typename... SemiAwaitables>+using collect_any_component_t = std::common_type_t<+    decay_rvalue_reference_t<semi_await_result_t<SemiAwaitables>>...>;++template <typename Range>+using range_iterator_t = decltype(access::begin(std::declval<Range&>()));++template <typename Iterator>+using iterator_reference_t = typename std::iterator_traits<Iterator>::reference;++template <typename Range>+using range_reference_t = iterator_reference_t<range_iterator_t<Range>>;++// A bare-bones std::range implementation that is similar to ranges::views::move+template <typename Container>+class MoveRange {+ public:+  explicit MoveRange(Container& container) : container_(container) {}++  auto begin() { return std::make_move_iterator(container_.begin()); }+  auto end() { return std::make_move_iterator(container_.end()); }++ private:+  Container& container_;+};++// Future: Apply `AsNoexcept` to the task if the entire collection process is+// noexcept-awaitable.  This would require reworking the implementation a bit,+// since e.g. `CancellationToken::merge` can throw `bad_alloc`.+template <typename... SemiAwaitables>+using CollectAllTask = PickTaskWrapper<+    std::tuple<collect_all_component_t<remove_cvref_t<SemiAwaitables>>...>,+    std::min({safe_alias::maybe_value, safe_alias_of_v<SemiAwaitables>...}),+    (must_await_immediately_v<SemiAwaitables> || ...)>;++template <typename... SemiAwaitables>+using CollectAllTryTask = PickTaskWrapper<+    std::tuple<collect_all_try_component_t<remove_cvref_t<SemiAwaitables>>...>,+    std::min({safe_alias::maybe_value, safe_alias_of_v<SemiAwaitables>...}),+    (must_await_immediately_v<SemiAwaitables> || ...)>;++} // namespace detail++///////////////////////////////////////////////////////////////////////////+// collectAll(SemiAwaitable<Ts>...) -> SemiAwaitable<std::tuple<Ts...>>+//+// The collectAll() function can be used to concurrently co_await on multiple+// SemiAwaitable objects and continue once they are all complete.+//+// collectAll() accepts an arbitrary number of SemiAwaitable objects and returns+// a SemiAwaitable object that will complete with a std::tuple of the results.+//+// When the returned SemiAwaitable object is co_awaited it will launch+// a new coroutine for awaiting each input awaitable in-turn.+//+// Note that coroutines for awaiting the input awaitables of later arguments+// will not be launched until the prior coroutine reaches its first suspend+// point. This means that awaiting multiple sub-tasks that all complete+// synchronously will still execute them sequentially on the current thread.+//+// If any of the input operations complete with an exception then it will+// request cancellation of any outstanding tasks and the whole collectAll()+// operation will complete with an exception once all of the operations+// have completed.  Any partial results will be discarded. If multiple+// operations fail with an exception then the exception from the first task+// to fail will be rethrown and subsequent errors are discarded.+//+// If you need to know which operation failed or you want to handle partial+// failures then you can use the folly::coro::collectAllTry() instead which+// returns a tuple of Try<T> objects instead of a tuple of values.+//+// Example: Serially awaiting multiple operations (slower)+//   folly::coro::Task<Foo> doSomething();+//   folly::coro::Task<Bar> doSomethingElse();+//+//   Foo result1 = co_await doSomething();+//   Bar result2 = co_await doSomethingElse();+//+// Example: Concurrently awaiting multiple operations (faster) C++17-only.+//   auto [result1, result2] =+//       co_await folly::coro::collectAll(doSomething(), doSomethingElse());+//+template <typename... SemiAwaitables>+// Do NOT take awaitables by-reference, that would break `NowTask` safety.+auto collectAll(SemiAwaitables... awaitables)+    -> detail::CollectAllTask<SemiAwaitables...>;++///////////////////////////////////////////////////////////////////////////+// collectAllTry(SemiAwaitable<Ts>...)+//    -> SemiAwaitable<std::tuple<Try<Ts>...>>+//+// Like the collectAll() function, the collectAllTry() function can be used to+// concurrently await multiple input SemiAwaitable objects.+//+// The collectAllTry() function differs from collectAll() in that it produces a+// tuple of Try<T> objects rather than a tuple of the values.+// This allows the caller to inspect the success/failure of individual+// operations and handle partial failures but has a less-convenient interface+// than collectAll().+//+// It also differs in that failure of one subtask does _not_ request+// cancellation of the other subtasks.+//+// Example: Handling partial failure with collectAllTry()+//    folly::coro::Task<Foo> doSomething();+//    folly::coro::Task<Bar> doSomethingElse();+//+//    auto [result1, result2] = co_await folly::coro::collectAllTry(+//        doSomething(), doSomethingElse());+//+//    if (result1.hasValue()) {+//      Foo& foo = result1.value();+//      process(foo);+//    } else {+//      logError("doSomething() failed", result1.exception());+//    }+//+//    if (result2.hasValue()) {+//      Bar& bar = result2.value();+//      process(bar);+//    } else {+//      logError("doSomethingElse() failed", result2.exception());+//    }+//+template <typename... SemiAwaitables>+auto collectAllTry(SemiAwaitables... awaitables)+    -> detail::CollectAllTryTask<SemiAwaitables...>;++////////////////////////////////////////////////////////////////////////+// collectAllRange(RangeOf<SemiAwaitable<T>>&&)+//   -> SemiAwaitable<std::vector<T>>+//+// The collectAllRange() function can be used to concurrently await a collection+// of SemiAwaitable objects, returning a std::vector of the individual results+// in the same order as the input once all operations have completed.+//+// If any of the operations fail with an exception then requests cancellation of+// any outstanding operations and the entire operation fails with an exception,+// discarding any partial results. If more than one operation fails with an+// exception then the exception from task that failed first (in time) is+// rethrown. Other results and exceptions are discarded.+//+// If you need to be able to distinguish which operation failed or handle+// partial failures then use collectAllTryRange() instead.+//+// Note that the expression `*it` must be SemiAwaitable.+// This typically means that containers of Task<T> must be adapted to produce+// moved-elements by applying the ranges::views::move transform.+// e.g.+//+//   std::vector<Task<T>> tasks = ...;+//   std::vector<T> vals = co_await collectAllRange(tasks |+//   ranges::views::move);+//+template <+    typename InputRange,+    std::enable_if_t<+        !std::is_void_v<+            semi_await_result_t<detail::range_reference_t<InputRange>>>,+        int> = 0>+auto collectAllRange(InputRange awaitables)+    -> folly::coro::Task<std::vector<detail::collect_all_range_component_t<+        detail::range_reference_t<InputRange>>>>;+template <+    typename InputRange,+    std::enable_if_t<+        std::is_void_v<+            semi_await_result_t<detail::range_reference_t<InputRange>>>,+        int> = 0>+auto collectAllRange(InputRange awaitables) -> folly::coro::Task<void>;++////////////////////////////////////////////////////////////////////////////+// collectAllTryRange(RangeOf<SemiAwaitable<T>>&&)+//    -> SemiAwaitable<std::vector<folly::Try<T>>>+//+// The collectAllTryRange() function can be used to concurrently await a+// collection of SemiAwaitable objects and produces a std::vector of+// Try<T> objects in the same order as the input once all of the input+// operations have completed.+//+// The success/failure of individual results can be inspected by calling+// .hasValue() or .hasException() on the elements of the returned vector.+template <typename InputRange>+auto collectAllTryRange(InputRange awaitables)+    -> folly::coro::Task<std::vector<detail::collect_all_try_range_component_t<+        detail::range_reference_t<InputRange>>>>;++// collectAllRange()/collectAllTryRange() overloads that simplifies the+// common-case where an rvalue std::vector<SemiAwaitable> is passed.+//+// This avoids the caller needing to pipe the input through ranges::views::move+// transform to force the elements to be rvalue-references since the+// std::vector<T>::reference type is T& rather than T&& and some awaitables,+// such as Task<U>, are not lvalue awaitable.+template <typename SemiAwaitable>+auto collectAllRange(std::vector<SemiAwaitable> awaitables)+    -> decltype(collectAllRange(detail::MoveRange(awaitables))) {+  co_return co_await collectAllRange(detail::MoveRange(awaitables));+}++template <typename SemiAwaitable>+auto collectAllTryRange(std::vector<SemiAwaitable> awaitables)+    -> decltype(collectAllTryRange(detail::MoveRange(awaitables))) {+  co_return co_await collectAllTryRange(detail::MoveRange(awaitables));+}++namespace detail {+template <typename InputRange, bool IsTry>+using async_generator_from_awaitable_range_item_t = conditional_t<+    IsTry,+    collect_all_try_range_component_t<range_reference_t<InputRange>>,+    collect_all_range_component_t<range_reference_t<InputRange>>>;+}++////////////////////////////////////////////////////////////////////////////+// makeUnorderedAsyncGenerator(AsyncScope&,+// RangeOf<SemiAwaitable<T>>&&) -> AsyncGenerator<T&&>+// makeUnorderedTryAsyncGenerator(AsyncScope&,+// RangeOf<SemiAwaitable<T>>&&) -> AsyncGenerator<Try<T>&&>++// Returns an AsyncGenerator that yields results of passed-in awaitables in+// order of completion.+// Destroying or cancelling the AsyncGenerator cancels the remaining awaitables.+//+// makeUnorderedAsyncGenerator cancels all remaining+// awaitables when any of them fail with an exception. Any results obtained+// before the failure are still returned via the generator, then the first+// exception in time. makeUnorderedTryAsyncGenerator does not+// cancel awaitables when one fails, and yields all results even when cancelled.+//+// Awaitables are attached to the passed-in AsyncScope.++template <typename InputRange>+auto makeUnorderedAsyncGenerator(AsyncScope& scope, InputRange awaitables)+    -> AsyncGenerator<detail::async_generator_from_awaitable_range_item_t<+        InputRange,+        false>&&>;+template <typename InputRange>+auto makeUnorderedTryAsyncGenerator(AsyncScope& scope, InputRange awaitables)+    -> AsyncGenerator<detail::async_generator_from_awaitable_range_item_t<+        InputRange,+        true>&&>;++template <typename SemiAwaitable>+auto makeUnorderedAsyncGenerator(+    AsyncScope& scope, std::vector<SemiAwaitable> awaitables)+    -> decltype(makeUnorderedAsyncGenerator(+        scope, detail::MoveRange(awaitables))) {+  auto gen = makeUnorderedAsyncGenerator(scope, detail::MoveRange(awaitables));+  while (true) {+    co_yield co_result(co_await co_awaitTry(gen.next()));+  }+}+template <typename SemiAwaitable>+auto makeUnorderedTryAsyncGenerator(+    AsyncScope& scope, std::vector<SemiAwaitable> awaitables)+    -> decltype(makeUnorderedTryAsyncGenerator(+        scope, detail::MoveRange(awaitables))) {+  auto gen =+      makeUnorderedTryAsyncGenerator(scope, detail::MoveRange(awaitables));+  while (true) {+    co_yield co_result(co_await co_awaitTry(gen.next()));+  }+}++// Can also be used with CancellableAsyncScope++template <typename InputRange>+auto makeUnorderedAsyncGenerator(+    CancellableAsyncScope& scope, InputRange awaitables)+    -> AsyncGenerator<detail::async_generator_from_awaitable_range_item_t<+        InputRange,+        false>&&>;+template <typename InputRange>+auto makeUnorderedTryAsyncGenerator(+    CancellableAsyncScope& scope, InputRange awaitables)+    -> AsyncGenerator<detail::async_generator_from_awaitable_range_item_t<+        InputRange,+        true>&&>;++template <typename SemiAwaitable>+auto makeUnorderedAsyncGenerator(+    CancellableAsyncScope& scope, std::vector<SemiAwaitable> awaitables)+    -> decltype(makeUnorderedAsyncGenerator(+        scope, detail::MoveRange(awaitables))) {+  auto gen = makeUnorderedAsyncGenerator(scope, detail::MoveRange(awaitables));+  while (true) {+    co_yield co_result(co_await co_awaitTry(gen.next()));+  }+}+template <typename SemiAwaitable>+auto makeUnorderedTryAsyncGenerator(+    CancellableAsyncScope& scope, std::vector<SemiAwaitable> awaitables)+    -> decltype(makeUnorderedTryAsyncGenerator(+        scope, detail::MoveRange(awaitables))) {+  auto gen =+      makeUnorderedTryAsyncGenerator(scope, detail::MoveRange(awaitables));+  while (true) {+    co_yield co_result(co_await co_awaitTry(gen.next()));+  }+}++///////////////////////////////////////////////////////////////////////////////+// collectAllWindowed(RangeOf<SemiAwaitable<T>>&&, size_t maxConcurrency)+//   -> SemiAwaitable<std::vector<T>>+//+// collectAllWindowed(RangeOf<SemiAwaitable<void>>&&, size_t maxConcurrency)+//   -> SemiAwaitable<void>+//+// Await each of the input awaitables in the range, allowing at most+// 'maxConcurrency' of these input awaitables to be concurrently awaited+// at any one point in time.+//+// If any of the input awaitables fail with an exception then requests+// cancellation of any incomplete operations and fails the whole+// operation with an exception. If multiple input awaitables fail with+// an exception then the exception from the first task to fail (in time)+// will be rethrown and the rest of the results will be discarded.+//+// If there is an exception thrown while iterating over the input-range then+// it will still guarantee that any prior awaitables in the input-range will+// run to completion before completing the collectAllWindowed() operation with+// the exception thrown during iteration.+//+// The resulting std::vector will contain the results in the corresponding+// order of their respective awaitables in the input range.+template <+    typename InputRange,+    std::enable_if_t<+        std::is_void_v<+            semi_await_result_t<detail::range_reference_t<InputRange>>>,+        int> = 0>+auto collectAllWindowed(InputRange awaitables, std::size_t maxConcurrency)+    -> folly::coro::Task<void>;+template <+    typename InputRange,+    std::enable_if_t<+        !std::is_void_v<+            semi_await_result_t<detail::range_reference_t<InputRange>>>,+        int> = 0>+auto collectAllWindowed(InputRange awaitables, std::size_t maxConcurrency)+    -> folly::coro::Task<std::vector<detail::collect_all_range_component_t<+        detail::range_reference_t<InputRange>>>>;++///////////////////////////////////////////////////////////////////////////////+// collectAllTryWindowed(RangeOf<SemiAwaitable<T>>&, size_t maxConcurrency)+//   -> SemiAwaitable<std::vector<folly::Try<T>>>+//+// Concurrently awaits a collection of awaitable with bounded concurrency,+// producing a vector of Try values containing each of the results.+//+// The resulting std::vector will contain the results in the corresponding+// order of their respective awaitables in the input range.+//+// Note that the whole operation may still complete with an exception if+// iterating over the awaitables fails with an exception (eg. if you pass+// a Generator<Task<T>&&> and the generator throws an exception).+template <typename InputRange>+auto collectAllTryWindowed(InputRange awaitables, std::size_t maxConcurrency)+    -> folly::coro::Task<std::vector<detail::collect_all_try_range_component_t<+        detail::range_reference_t<InputRange>>>>;++// collectAllWindowed()/collectAllTryWindowed() overloads that simplify the+// use of these functions with std::vector<SemiAwaitable>.+template <typename SemiAwaitable>+auto collectAllWindowed(+    std::vector<SemiAwaitable> awaitables, std::size_t maxConcurrency)+    -> decltype(collectAllWindowed(+        detail::MoveRange(awaitables), maxConcurrency)) {+  co_return co_await collectAllWindowed(+      detail::MoveRange(awaitables), maxConcurrency);+}++template <typename SemiAwaitable>+auto collectAllTryWindowed(+    std::vector<SemiAwaitable> awaitables, std::size_t maxConcurrency)+    -> decltype(collectAllTryWindowed(+        detail::MoveRange(awaitables), maxConcurrency)) {+  co_return co_await collectAllTryWindowed(+      detail::MoveRange(awaitables), maxConcurrency);+}++///////////////////////////////////////////////////////////////////////////+// collectAny(SemiAwaitable<Ts>...) -> SemiAwaitable<+//   std::pair<std::size_t, folly::Try<std::common_type<Ts...>>>>+//+// The collectAny() function can be used to concurrently co_await on multiple+// SemiAwaitable objects, get the result and index of the first one completing,+// cancel the remaining ones and continue once they are completed.+//+// collectAny() accepts a positive number of SemiAwaitable objects and+// returns a SemiAwaitable object that will complete with a pair containing the+// result of the first one to complete and its index.+//+// collectAny() is built on top of collectAll(), be aware of the coroutine+// starting behavior described in collectAll() documentation.+//+// The result of the first SemiAwaitable is going to be returned, whether it+// is a value or an exception. Any result of the remaining SemiAwaitables will+// be discarded, independently of whether it's a value or an exception.+//+// Example:+//   folly::coro::Task<Foo> getDataOneWay();+//   folly::coro::Task<Foo> getDataAnotherWay();+//+//   std::pair<std::size_t, Try<Foo>> result = co_await folly::coro::collectAny(+//       getDataOneWay(), getDataAnotherWay());+//+template <typename SemiAwaitable, typename... SemiAwaitables>+auto collectAny(SemiAwaitable&& awaitable, SemiAwaitables&&... awaitables)+    -> folly::coro::Task<std::pair<+        std::size_t,+        folly::Try<detail::collect_any_component_t<+            SemiAwaitable,+            SemiAwaitables...>>>>;++///////////////////////////////////////////////////////////////////////////+// collectAnyWithoutException(SemiAwaitable<Ts>...)+//    -> SemiAwaitable<std::pair<std::size_t, folly::Try<T>>>+//+// The collectAnyWithoutException() function is similar to collectAny() in that+// it co_awaits multiple SemiAwaitables and cancels any outstanding operations+// when complete. Unlike collectAny(), it returns the first success, or the last+// exception if all of the SemiAwaitables fail.+//+// collectAnyWithoutException() is built on top of collectAll(), be aware of the+// coroutine starting behavior described in collectAll() documentation.+//+// The result of the first successful SemiAwaitable, or the the exception from+// the last SemiAwaitable is returned if none are successful. Any result of the+// remaining SemiAwaitables will be discarded, independently of whether it's a+// value or an exception.+//+// Example:+//   folly::coro::Task<Foo> getDataOneWay();+//   folly::coro::Task<Foo> getDataAnotherWay();+//+//   std::pair<std::size_t, Try<Foo>> result =+//       co_await folly::coro::collectAnyWithoutException(+//           getDataOneWay(), getDataAnotherWay());+//+template <typename... SemiAwaitables>+auto collectAnyWithoutException(SemiAwaitables&&... awaitables)+    -> folly::coro::Task<std::pair<+        std::size_t,+        folly::Try<detail::collect_any_component_t<SemiAwaitables...>>>>;++///////////////////////////////////////////////////////////////////////////+// collectAnyNoDiscard(SemiAwaitable<Ts>...) ->+//   SemiAwaitable<std::tuple<folly::Try<Ts>...>>+//+// The collectAnyNoDiscard() function is similar to collectAny() in that it+// co_awaits multiple SemiAwaitables and cancels any outstanding operations once+// at least one has finished. Unlike collectAny(), it returns results from *all*+// SemiAwaitables, including folly::OperationCancelled for operations that were+// cancelled.+//+// collectAnyNoDiscard() is built on top of collectAll(), be aware of the+// coroutine starting behavior described in collectAll() documentation.+//+// The returned tuple contains the results of all the SemiAwaitables.+//+// Example:+//   folly::coro::Task<Foo> getDataOneWay();+//   folly::coro::Task<Bar> getDataAnotherWay();+//+//   std::tuple<folly::Try<Foo>, folly::Try<Bar>> result = co_await+//      folly::coro::collectAnyNoDiscard(getDataOneWay(), getDataAnotherWay());+//+template <typename... SemiAwaitables>+auto collectAnyNoDiscard(SemiAwaitables&&... awaitables)+    -> folly::coro::Task<std::tuple<detail::collect_all_try_component_t<+        remove_cvref_t<SemiAwaitables>>...>>;++///////////////////////////////////////////////////////////////////////////+// collectAnyRange(RangeOf<SemiAwaitable<T>>&&)+//   -> SemiAwaitable<std::pair<std::size_t, folly::Try<T>>>+//+// The collectAnyRange() function can be used to concurrently co_await on+// multiple SemiAwaitable objects, get the result and index of the first one+// completing, cancel the remaining ones and continue once they are completed.+//+// collectAnyRange() accepts zero or more SemiAwaitable objects and+// returns a SemiAwaitable object that will complete with a pair containing the+// result of the first one to complete and its index.+//+// collectAnyRange() is built on top of collectAllRange(), be aware of the+// coroutine starting behavior described in collectAll() documentation.+//+// The result of the first SemiAwaitable is going to be returned, whether it+// is a value or an exception. Any result of the remaining SemiAwaitables will+// be discarded, independently of whether it's a value or an exception.+//+// e.g.+//+//   std::vector<Task<T>> tasks = ...;+//   std::pair<size_t, Try<T>> result = co_await collectAnyRange(tasks |+//       ranges::views::move);+//+template <typename InputRange>+auto collectAnyRange(InputRange awaitables)+    -> folly::coro::Task<std::pair<+        size_t,+        folly::Try<detail::collect_all_range_component_t<+            detail::range_reference_t<InputRange>>>>>;++///////////////////////////////////////////////////////////////////////////+// collectAnyWithoutExceptionRange(RangeOf<SemiAwaitable<T>>&&)+//   -> SemiAwaitable<std::pair<std::size_t, folly::Try<T>>>+//+// The collectAnyWithoutExceptionRange() function is similar to+// collectAnyRange() in that it co_awaits multiple SemiAwaitables and cancels+// any outstanding operations when complete. Unlike collectAnyRange(), it+// returns the first success, or the last exception if all of the SemiAwaitables+// fail.+//+// collectAnyWithoutExceptionRange() is built on top of collectAllRange(), be+// aware of the coroutine starting behavior described in collectAll()+// documentation.+//+// The result of the first successful SemiAwaitable, or the the exception from+// the last SemiAwaitable is returned if none are successful. Any result of the+// remaining SemiAwaitables will be discarded, independently of whether it's a+// value or an exception.+//+// Example:+//   std::vector<Task<T>> tasks = ...;+//   std::pair<size_t, Try<T>> result =+//       co_await collectAnyWithoutExceptionRange(tasks | ranges::views::move);+//+template <typename InputRange>+auto collectAnyWithoutExceptionRange(InputRange awaitables)+    -> folly::coro::Task<std::pair<+        size_t,+        folly::Try<detail::collect_all_range_component_t<+            detail::range_reference_t<InputRange>>>>>;++///////////////////////////////////////////////////////////////////////////+// collectAnyNoDiscardRange(RangeOf<SemiAwaitable<T>>&&)+//    -> SemiAwaitable<std::vector<folly::Try<T>>>+//+// The collectAnyNoDiscardRange() function is similar to collectAnyRange() in+// that it co_awaits multiple SemiAwaitables and cancels any outstanding+// operations once at least one has finished. Unlike collectAnyRange(), it+// returns results from *all* SemiAwaitables, including+// folly::OperationCancelled for operations that were cancelled.+//+// collectAnyNoDiscardRange() is built on top of collectAllRange(), be aware of+// the coroutine starting behavior described in collectAll() documentation.+//+// The success/failure of individual results can be inspected by calling+// .hasValue() or .hasException() on the elements of the returned vector.+//+// Example:+//   folly::coro::Task<Foo> getDataOneWay();+//   folly::coro::Task<Foo> getDataAnotherWay();+//+//   std::vector<folly::Try<Foo>> result = co_await+//      folly::coro::collectAnyNoDiscard(getDataOneWay(), getDataAnotherWay());+//+template <typename InputRange>+auto collectAnyNoDiscardRange(InputRange awaitables)+    -> folly::coro::Task<std::vector<detail::collect_all_try_range_component_t<+        detail::range_reference_t<InputRange>>>>;++// collectAnyRange()/collectAnyWithoutExceptionRange()/collectAnyNoDiscardRange()+// overloads that simplifies the common-case where an rvalue+// std::vector<SemiAwaitable> is passed.+//+// This avoids the caller needing to pipe the input through ranges::views::move+// transform to force the elements to be rvalue-references since the+// std::vector<T>::reference type is T& rather than T&& and some awaitables,+// such as Task<U>, are not lvalue awaitable.+template <typename SemiAwaitable>+auto collectAnyRange(std::vector<SemiAwaitable> awaitables)+    -> decltype(collectAnyRange(detail::MoveRange(awaitables))) {+  co_return co_await collectAnyRange(detail::MoveRange(awaitables));+}+template <typename SemiAwaitable>+auto collectAnyWithoutExceptionRange(std::vector<SemiAwaitable> awaitables)+    -> decltype(collectAnyWithoutExceptionRange(+        detail::MoveRange(awaitables))) {+  co_return co_await collectAnyWithoutExceptionRange(+      detail::MoveRange(awaitables));+}+template <typename SemiAwaitable>+auto collectAnyNoDiscardRange(std::vector<SemiAwaitable> awaitables)+    -> decltype(collectAnyNoDiscardRange(detail::MoveRange(awaitables))) {+  co_return co_await collectAnyNoDiscardRange(detail::MoveRange(awaitables));+}++} // namespace coro+} // namespace folly++#endif // FOLLY_HAS_COROUTINES++#include <folly/coro/Collect-inl.h>
+ folly/folly/coro/Concat-inl.h view
@@ -0,0 +1,40 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#include <folly/coro/Coroutine.h>++namespace folly {+namespace coro {++template <+    typename HReference,+    typename... TReference,+    typename HValue,+    typename... TValue>+AsyncGenerator<HReference, HValue> concat(+    AsyncGenerator<HReference, HValue> head,+    AsyncGenerator<TReference, TValue>... tail) {+  static_assert((std::is_same_v<decltype(head), decltype(tail)> && ...));+  using list = AsyncGenerator<HReference, HValue>[];+  for (auto& gen : list{std::move(head), std::move(tail)...}) {+    while (auto val = co_await gen.next()) {+      co_yield std::move(val).value();+    }+  }+}++} // namespace coro+} // namespace folly
+ folly/folly/coro/Concat.h view
@@ -0,0 +1,62 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <folly/coro/AsyncGenerator.h>+#include <folly/coro/Coroutine.h>++#if FOLLY_HAS_COROUTINES++namespace folly {+namespace coro {++// Concatenate the values from multiple streams into a single stream such+// that each stream is exhausted before the next one begins.+//+// The input is a variadic list of AsyncGenerators, where each input has the+// same Reference and Value types.+//+// The output is a single AsyncGenerator over all of the input generators.+//+// Example:+//  AsyncGenerator<int> stream();+//+//  Task<int> consumer() {+//    auto values = concat(stream(), stream(), stream());+//+//    int result = 0;+//    while (auto item = co_await values.next()) {+//      result += *item;+//    }+//+//    return result;+//  }+template <+    typename HReference,+    typename... TReference,+    typename HValue,+    typename... TValue>+AsyncGenerator<HReference, HValue> concat(+    AsyncGenerator<HReference, HValue> head,+    AsyncGenerator<TReference, TValue>... tail);++} // namespace coro+} // namespace folly++#endif // FOLLY_HAS_COROUTINES++#include <folly/coro/Concat-inl.h>
+ folly/folly/coro/Coroutine.h view
@@ -0,0 +1,367 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <type_traits>++#if __has_include(<variant>)+#include <variant>+#endif++#include <folly/Portability.h>+#include <folly/Utility.h>++#if FOLLY_HAS_COROUTINES++// libc++'s <coroutine> header only provides its declarations for C++20 and+// above, so we need to fall back to <experimental/coroutine> when building with+// C++17.+#if (__has_include(<coroutine>) && !defined(LLVM_COROUTINES)) || defined(__cpp_impl_coroutine)+#define FOLLY_USE_STD_COROUTINE 1+#else+#define FOLLY_USE_STD_COROUTINE 0+#endif++#if FOLLY_USE_STD_COROUTINE+#include <coroutine>+#else+#include <experimental/coroutine>+#endif++#endif // FOLLY_HAS_COROUTINES++//  A place for foundational vocabulary types.+//+//  This header reexports the foundational vocabulary coroutine-helper types+//  from the standard, and exports several new foundational vocabulary types+//  as well.+//+//  Types which are non-foundational and non-vocabulary should go elsewhere.++#if FOLLY_HAS_COROUTINES++namespace folly {+class exception_wrapper;+struct AsyncStackFrame;+} // namespace folly++namespace folly::coro {++#if FOLLY_USE_STD_COROUTINE+namespace impl = std;+#else+namespace impl = std::experimental;+#endif++using impl::coroutine_handle;+using impl::coroutine_traits;+using impl::noop_coroutine;+using impl::noop_coroutine_handle;+using impl::noop_coroutine_promise;+using impl::suspend_always;+using impl::suspend_never;++//  ready_awaitable+//+//  An awaitable which is immediately ready with a value. Suspension is no-op.+//  Resumption returns the value.+//+//  The value type is permitted to be a reference.+template <typename T = void>+class ready_awaitable {+  static_assert(!std::is_void<T>::value, "base template unsuitable for void");++ public:+  explicit ready_awaitable(T value) //+      noexcept(noexcept(T(FOLLY_DECLVAL(T&&))))+      : value_(static_cast<T&&>(value)) {}++  bool await_ready() noexcept { return true; }+  void await_suspend(coroutine_handle<>) noexcept {}+  T await_resume() noexcept(noexcept(T(FOLLY_DECLVAL(T&&)))) {+    return static_cast<T&&>(value_);+  }++ private:+  T value_;+};++//  ready_awaitable+//+//  An awaitable type which is immediately ready. Suspension is a no-op.+template <>+class ready_awaitable<void> {+ public:+  ready_awaitable() noexcept = default;++  bool await_ready() noexcept { return true; }+  void await_suspend(coroutine_handle<>) noexcept {}+  void await_resume() noexcept {}+};++namespace detail {++//  await_suspend_return_coroutine_fn+//  await_suspend_return_coroutine+//+//  The special member await_suspend has three forms, differing in their return+//  types. It may return void, bool, or coroutine_handle<>. This invokes member+//  await_suspend on the argument, conspiring always to return coroutine_handle+//  no matter the underlying form of member await_suspend on the argument.+struct await_suspend_return_coroutine_fn {+  template <typename A, typename P>+  coroutine_handle<> operator()(A& a, coroutine_handle<P> coro) const+      noexcept(noexcept(a.await_suspend(coro))) {+    using result = decltype(a.await_suspend(coro));+    if constexpr (std::is_same<void, result>::value) {+      a.await_suspend(coro);+      return noop_coroutine();+    } else if constexpr (std::is_same<bool, result>::value) {+      return a.await_suspend(coro) ? noop_coroutine() : coro;+    } else {+      return a.await_suspend(coro);+    }+  }+};+inline constexpr await_suspend_return_coroutine_fn+    await_suspend_return_coroutine{};++} // namespace detail++#if __has_include(<variant>)++//  variant_awaitable+//+//  An awaitable type which is backed by one of several possible underlying+//  awaitables.+template <typename... A>+class variant_awaitable : private std::variant<A...> {+ private:+  using base = std::variant<A...>;++  template <typename Visitor>+  auto visit(Visitor v) {+    return std::visit(v, static_cast<base&>(*this));+  }++ public:+  // imports the base-class constructors wholesale for implementation simplicity+  using base::base; // assume there are no valueless-by-exception instances++  auto await_ready() noexcept(+      (noexcept(FOLLY_DECLVAL(A&).await_ready()) && ...)) {+    return visit([&](auto& a) { return a.await_ready(); });+  }+  template <typename P>+  auto await_suspend(coroutine_handle<P> coro) noexcept(+      (noexcept(FOLLY_DECLVAL(A&).await_suspend(coro)) && ...)) {+    auto impl = detail::await_suspend_return_coroutine;+    return visit([&](auto& a) { return impl(a, coro); });+  }+  auto await_resume() noexcept(+      (noexcept(FOLLY_DECLVAL(A&).await_resume()) && ...)) {+    return visit([&](auto& a) { return a.await_resume(); });+  }+};++#endif // __has_include(<variant>)++//  ----++namespace detail {++struct detect_promise_return_object_eager_conversion_ {+  struct promise_type {+    struct return_object {+      /* implicit */ return_object(promise_type& p) noexcept : promise{&p} {+        promise->object = this;+      }+      ~return_object() {+        if (promise) {+          promise->object = nullptr;+        }+      }++      promise_type* promise;+    };++    ~promise_type() {+      if (object) {+        object->promise = nullptr;+      }+    }++    suspend_never initial_suspend() const noexcept { return {}; }+    suspend_never final_suspend() const noexcept { return {}; }+    void unhandled_exception() {}++    return_object get_return_object() noexcept { return {*this}; }+    void return_void() {}++    return_object* object = nullptr;+  };++  /* implicit */ detect_promise_return_object_eager_conversion_(+      promise_type::return_object const& o) noexcept+      : eager{!!o.promise} {}+  //  letting the coroutine type be trivially-copyable makes the coroutine crash+  //  under clang; to work around, provide an empty but not trivial destructor+  ~detect_promise_return_object_eager_conversion_() {}++  bool eager = false;++  static detect_promise_return_object_eager_conversion_ go() noexcept {+    // FIXME: when building against Apple SDKs using c++17, we hit this all over+    // the place on complex testing infrastructure for iOS. Since it's not clear+    // how to fix the issue properly right now, force ignore this warnings and+    // unblock expected/optional coroutines. This should be removed once the+    // build config is changed to use -Wno-deprecated-experimental-coroutine.+    FOLLY_PUSH_WARNING+#if defined(__clang__) && \+    (13 < __clang_major__ && __clang_major__ < 17 - defined(__APPLE__))+    FOLLY_CLANG_DISABLE_WARNING("-Wdeprecated-experimental-coroutine")+#endif+    co_return;+    FOLLY_POP_WARNING+  }+};++} // namespace detail++//  detect_promise_return_object_eager_conversion+//+//  Returns true if the compiler implements coroutine promise return-object+//  conversion eagerly and returns false if the compiler defers conversion.+//+//  It is expected that the caller holds the promise return-object until the+//  promise is fulfilled, even when it is not the same type as the coroutine.+//+//    auto ret = promise.get_return_object();+//    initial-suspend, etc...+//    return ret;+//+//  But this expected behavior was, mistakenly, never specified.+//+//  Some compilers misbehave, where the caller holds precisely the coroutine+//  type by converting the promise return-object eagerly when it is of some+//  type different from the coroutine type.+//+//    coro-type ret = promise.get_return_object();+//    initial-suspend, etc...+//    return ret;+//+//  Known behaviors are as follows:+//  * For msvc, conversion is eager for vs < 2019 update 16.5 (msc ver 1925) and+//    is deferred for vs >= 2019 update 16.5 (msc ver 1925).+//    References:+//      https://developercommunity.visualstudio.com/t/c-coroutine-get-return-object-converted-too-early/222420+//  * For g++, conversion is deferred.+//  * For clang++, conversion is eager for 15 <= llvm < 17 and is deferred for+//    llvm < 15 or llvm >= 17.+//    References:+//      https://reviews.llvm.org/D117087+//      https://github.com/llvm/llvm-project/issues/56532+//      https://reviews.llvm.org/D145639+//+//  Meta sometimes uses llvm patched to have deferred conversion where the+//  corresponding upstream implements eager conversion. So version numbers do+//  not tell the whole story.+//+//  This function detects which behavior the compiler implements at a mix of+//  compile time and run time, depending on the compiler. It is only necessary+//  to do the runtime detection for llvm but, conveniently, llvm is able to do+//  full heap-allocation elision ("HALO") and optimize the detection down to a+//  constant.+//+//  TODO: Remove this detection once the behavior is specified.+inline bool detect_promise_return_object_eager_conversion() {+  using coro = detail::detect_promise_return_object_eager_conversion_;+  constexpr auto t = kMscVer && kMscVer < 1925;+  constexpr auto f = (kGnuc && !kIsClang) || (kMscVer >= 1925);+  return t ? true : f ? false : coro::go().eager;+}++class ExtendedCoroutineHandle;++// Extended promise interface folly::coro types are expected to implement+class ExtendedCoroutinePromise {+ public:+  // Types may provide a more efficient resumption path when they know they will+  // be receiving an error result from the awaitee.+  // If they do, they might also update the active stack frame.+  virtual std::pair<ExtendedCoroutineHandle, AsyncStackFrame*> getErrorHandle(+      exception_wrapper&) = 0;++ protected:+  ~ExtendedCoroutinePromise() = default;+};++// Extended version of coroutine_handle<void>+// Assumes (and enforces) assumption that coroutine_handle is a pointer+class ExtendedCoroutineHandle {+ public:+  template <typename Promise>+  /*implicit*/ ExtendedCoroutineHandle(+      coroutine_handle<Promise> handle) noexcept+      : basic_(handle), extended_(fromBasic(handle)) {}++  /*implicit*/ ExtendedCoroutineHandle(coroutine_handle<> handle) noexcept+      : basic_(handle) {}++  template <+      typename Promise,+      std::enable_if_t<+          std::is_base_of_v<ExtendedCoroutinePromise, Promise>,+          int> = 0>+  /*implicit*/ ExtendedCoroutineHandle(Promise* p) noexcept+      : basic_(coroutine_handle<Promise>::from_promise(*p)), extended_(p) {}++  ExtendedCoroutineHandle() noexcept = default;++  void resume() { basic_.resume(); }++  void destroy() { basic_.destroy(); }++  coroutine_handle<> getHandle() const noexcept { return basic_; }++  std::pair<ExtendedCoroutineHandle, AsyncStackFrame*> getErrorHandle(+      exception_wrapper& ex) {+    if (extended_) {+      return extended_->getErrorHandle(ex);+    }+    return {basic_, nullptr};+  }++  explicit operator bool() const noexcept { return !!basic_; }++ private:+  template <typename Promise>+  static auto fromBasic(coroutine_handle<Promise> handle) noexcept {+    if constexpr (std::is_convertible_v<Promise*, ExtendedCoroutinePromise*>) {+      return static_cast<ExtendedCoroutinePromise*>(&handle.promise());+    } else {+      return nullptr;+    }+  }++  coroutine_handle<> basic_;+  ExtendedCoroutinePromise* extended_{nullptr};+};++} // namespace folly::coro++#endif
+ folly/folly/coro/CurrentExecutor.h view
@@ -0,0 +1,198 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <utility>++#include <folly/Executor.h>+#include <folly/coro/Coroutine.h>+#include <folly/coro/ViaIfAsync.h>+#include <folly/coro/WithAsyncStack.h>+#include <folly/io/async/Request.h>++#if FOLLY_HAS_COROUTINES++namespace folly {+namespace coro {++namespace detail {+struct co_current_executor_ {+  enum class secret_ { token_ };+  explicit constexpr co_current_executor_(secret_) {}+};+} // namespace detail++using co_current_executor_t = detail::co_current_executor_;++// Special placeholder object that can be 'co_await'ed from within a Task<T>+// or an AsyncGenerator<T> to obtain the current folly::Executor associated+// with the current coroutine.+//+// Note that for a folly::Task the executor will remain the same throughout+// the lifetime of the coroutine. For a folly::AsyncGenerator<T> the current+// executor may change when resuming from a co_yield suspend-point.+//+// Example:+//   folly::coro::Task<void> example() {+//     Executor* e = co_await folly::coro::co_current_executor;+//     e->add([] { do_something(); });+//   }+inline constexpr co_current_executor_t co_current_executor{+    co_current_executor_t::secret_::token_};++namespace detail {++class co_reschedule_on_current_executor_ {+  class AwaiterBase {+   public:+    explicit AwaiterBase(folly::Executor::KeepAlive<> executor) noexcept+        : executor_(std::move(executor)) {}++    bool await_ready() noexcept { return false; }++    void await_resume() noexcept {}++   protected:+    folly::Executor::KeepAlive<> executor_;+  };++ public:+  class StackAwareAwaiter : public AwaiterBase {+   public:+    using AwaiterBase::AwaiterBase;++    template <typename Promise>+    void await_suspend(coroutine_handle<Promise> coro) noexcept {+      await_suspend_impl(coro, coro.promise().getAsyncFrame());+    }++   private:+    FOLLY_CORO_AWAIT_SUSPEND_NONTRIVIAL_ATTRIBUTES void await_suspend_impl(+        coroutine_handle<> coro, AsyncStackFrame& frame) {+      auto& stackRoot = *frame.getStackRoot();+      folly::deactivateAsyncStackFrame(frame);+      try {+        executor_->add(+            [coro, &frame, ctx = RequestContext::saveContext()]() mutable {+              RequestContextScopeGuard contextScope{std::move(ctx)};+              folly::resumeCoroutineWithNewAsyncStackRoot(coro, frame);+            });+      } catch (...) {+        folly::activateAsyncStackFrame(stackRoot, frame);+        throw;+      }+    }+  };++  class Awaiter : public AwaiterBase {+   public:+    using AwaiterBase::AwaiterBase;++    FOLLY_CORO_AWAIT_SUSPEND_NONTRIVIAL_ATTRIBUTES void await_suspend(+        coroutine_handle<> coro) {+      executor_->add([coro, ctx = RequestContext::saveContext()]() mutable {+        RequestContextScopeGuard contextScope{std::move(ctx)};+        coro.resume();+      });+    }++    friend StackAwareAwaiter tag_invoke(+        cpo_t<co_withAsyncStack>, Awaiter awaiter) {+      return StackAwareAwaiter{std::move(awaiter.executor_)};+    }+  };++  friend Awaiter co_viaIfAsync(+      folly::Executor::KeepAlive<> executor,+      co_reschedule_on_current_executor_) {+    return Awaiter{std::move(executor)};+  }+};++} // namespace detail++using co_reschedule_on_current_executor_t =+    detail::co_reschedule_on_current_executor_;++// A SemiAwaitable object that allows you to reschedule the current coroutine+// onto the currently associated executor.+//+// This can be used as a form of cooperative multi-tasking for coroutines that+// wish to provide fair access to the execution resources. eg. to periodically+// give up their current execution slot to allow other tasks to run.+//+// Example:+//   folly::coro::Task<void> doCpuIntensiveWorkFairly() {+//     for (int i = 0; i < 1'000'000; ++i) {+//       // Periodically reschedule to the executor.+//       if ((i % 1024) == 1023) {+//         co_await folly::coro::co_reschedule_on_current_executor;+//       }+//       doSomeWork(i);+//     }+//   }+inline constexpr co_reschedule_on_current_executor_t+    co_reschedule_on_current_executor;++namespace detail {+struct co_current_cancellation_token_ {+  enum class secret_ { token_ };+  explicit constexpr co_current_cancellation_token_(secret_) {}+};+} // namespace detail++using co_current_cancellation_token_t = detail::co_current_cancellation_token_;++inline constexpr co_current_cancellation_token_t co_current_cancellation_token{+    co_current_cancellation_token_t::secret_::token_};++//  co_safe_point_t+//  co_safe_point+//+//  A semi-awaitable type and value which, when awaited in an async coroutine+//  supporting safe-points, causes a safe-point to be reached.+//+//  Example:+//+//    co_await co_safe_point; // a safe-point is reached+//+//  At this safe-point:+//  - If cancellation has been requested then the coroutine is terminated with+//    cancellation.+//  - To aid overall system concurrency, the coroutine may be rescheduled onto+//    the current executor.+//  - Otherwise, the coroutine is resumed.+//+//  Recommended for use wherever cancellation is checked and handled via early+//  termination.+//+//  Technical note: behavior is typically implemented in some overload+//  of await_transform in the coroutine's promise type, or in the awaitable+//  or awaiter it returns. Example:+//+//      struct /* some coroutine type */ {+//        struct promise_type {+//          /* some awaiter */ await_transform(co_safe_point_t) noexcept;+//        };+//      };+class co_safe_point_t final {};+inline constexpr co_safe_point_t co_safe_point{};++} // namespace coro+} // namespace folly++#endif // FOLLY_HAS_COROUTINES
+ folly/folly/coro/DetachOnCancel.h view
@@ -0,0 +1,94 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <type_traits>++#include <folly/coro/Baton.h>+#include <folly/coro/Coroutine.h>+#include <folly/coro/Invoke.h>+#include <folly/coro/Task.h>+#include <folly/coro/Traits.h>+#include <folly/coro/detail/Helpers.h>++#if FOLLY_HAS_COROUTINES++namespace folly {+namespace coro {+/**+ * detachOnCancel is used to handle operations that are hard to be cancelled. A+ * typical use case is: The caller starts a task with timeout (in this case, 1+ * sec timeout). The task itself launches a long running job and the job doesn't+ * handle cancellation (sleep_for in this example). The caller has timeout and+ * the cancellation is propagated to the task. The detachOnCancel detects the+ * cancellation and return immediately. However, the background task still runs+ * until the thread join.+ *+ * \refcode folly/docs/examples/folly/coro/DetachOnCancel.cpp+ *+ * It is important to manage the scope of each variable. If the long running+ * task references any variable that is created in the scope of detachOnCancel,+ * then the result may be freed and the long running task may trigger+ * use-after-free error.+ */+template <typename Awaitable>+Task<semi_await_result_t<Awaitable>> detachOnCancel(Awaitable awaitable) {+  auto posted = std::make_unique<std::atomic<bool>>(false);+  Baton baton;+  Try<detail::lift_lvalue_reference_t<semi_await_result_t<Awaitable>>> result;++  {+    auto t = co_invoke(+        [awaitable_2 = std::move(+             awaitable)]() mutable -> Task<semi_await_result_t<Awaitable>> {+          co_return co_await std::move(awaitable_2);+        });+    co_withExecutor(co_await co_current_executor, std::move(t))+        .startInlineUnsafe(+            [postedPtr = posted.get(), &baton, &result](auto&& r) {+              std::unique_ptr<std::atomic<bool>> p(postedPtr);+              if (!p->exchange(true, std::memory_order_acq_rel)) {+                p.release();+                tryAssign(result, std::move(r));+                baton.post();+              }+            },+            co_await co_current_cancellation_token);+  }++  {+    CancellationCallback cancelCallback(+        co_await co_current_cancellation_token, [&posted, &baton, &result] {+          if (!posted->exchange(true, std::memory_order_acq_rel)) {+            posted.release();+            result.emplaceException(folly::OperationCancelled{});+            baton.post();+          }+        });+    co_await baton;+  }++  if (result.hasException()) {+    co_yield folly::coro::co_error(result.exception());+  }++  co_return std::move(result).value();+}+} // namespace coro+} // namespace folly++#endif // FOLLY_HAS_COROUTINES
+ folly/folly/coro/Filter-inl.h view
@@ -0,0 +1,35 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#if FOLLY_HAS_COROUTINES++namespace folly {+namespace coro {++template <typename FilterFn, typename Reference, typename Value>+AsyncGenerator<Reference, Value> filter(+    AsyncGenerator<Reference, Value> source, FilterFn filterFn) {+  while (auto item = co_await source.next()) {+    if (invoke(filterFn, item.value())) {+      co_yield std::move(item).value();+    }+  }+}++} // namespace coro+} // namespace folly++#endif // FOLLY_HAS_COROUTINES
+ folly/folly/coro/Filter.h view
@@ -0,0 +1,48 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <folly/coro/AsyncGenerator.h>+#include <folly/coro/Coroutine.h>++#if FOLLY_HAS_COROUTINES++namespace folly {+namespace coro {++// Filter the Values from an input stream using an unary predicate.+//+// The input is a stream of Values.+//+// The output is a stream of Values that satisfy the predicate.+//+// Example:+//   AsyncGenerator<int> getAllNumbers();+//+//   AsyncGenerator<int> getEvenNumbers(AsyncGenerator<int> allNumbers) {+//     return filter(getAllNumbers(), [](int i){ return i % 2 == 0; });+//   }+template <typename FilterFn, typename Reference, typename Value>+AsyncGenerator<Reference, Value> filter(+    AsyncGenerator<Reference, Value> source, FilterFn filterFn);++} // namespace coro+} // namespace folly++#endif // FOLLY_HAS_COROUTINES++#include <folly/coro/Filter-inl.h>
+ folly/folly/coro/FutureUtil.h view
@@ -0,0 +1,110 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <folly/CancellationToken.h>+#include <folly/coro/Baton.h>+#include <folly/coro/Coroutine.h>+#include <folly/coro/CurrentExecutor.h>+#include <folly/coro/Invoke.h>+#include <folly/coro/Task.h>+#include <folly/coro/Traits.h>+#include <folly/futures/Future.h>++#if FOLLY_HAS_COROUTINES++namespace folly {+namespace coro {++// Converts the given SemiAwaitable to a Task (without starting it)+struct ToTaskFn {+  template <typename SemiAwaitable>+  Task<semi_await_result_t<SemiAwaitable>> operator()(SemiAwaitable a) const {+    co_return co_await std::move(a);+  }+  template <typename SemiAwaitable>+  Task<semi_await_result_t<SemiAwaitable>> operator()(+      std::reference_wrapper<SemiAwaitable> a) const {+    co_return co_await a.get();+  }+  Task<void> operator()(folly::Future<Unit> a) const {+    co_yield co_result(co_await co_awaitTry(std::move(a)));+  }+  Task<void> operator()(folly::SemiFuture<Unit> a) const {+    co_yield co_result(co_await co_awaitTry(std::move(a)));+  }+};+inline constexpr ToTaskFn toTask{};++template <typename V>+Task<drop_unit_t<V>> toTaskInterruptOnCancel(folly::Future<V> f) {+  bool cancelled{false};+  Baton baton;+  Try<V> result;+  f.setCallback_(+      [&result, &baton](Executor::KeepAlive<>&&, Try<V>&& t) {+        result = std::move(t);+        baton.post();+      },+      // No user logic runs in the callback, we can avoid the cost of switching+      // the context.+      /* context */ nullptr);++  {+    CancellationCallback cancelCallback(+        co_await co_current_cancellation_token, [&]() noexcept {+          cancelled = true;+          f.cancel();+        });+    co_await baton;+  }+  if (cancelled) {+    co_yield co_cancelled;+  }+  co_yield co_result(std::move(result));+}++template <typename V>+Task<drop_unit_t<V>> toTaskInterruptOnCancel(folly::SemiFuture<V> f) {+  auto ex = co_await co_current_executor;+  co_await co_nothrow(toTaskInterruptOnCancel(std::move(f).via(ex)));+}++// Converts the given SemiAwaitable to a SemiFuture (without starting it)+template <typename SemiAwaitable>+folly::SemiFuture<+    lift_unit_t<semi_await_result_t<remove_reference_wrapper_t<SemiAwaitable>>>>+toSemiFuture(SemiAwaitable&& a) {+  return toTask(std::forward<SemiAwaitable>(a)).semi();+}++// Converts the given SemiAwaitable to a Future, starting it on the Executor+template <typename SemiAwaitable>+folly::Future<+    lift_unit_t<semi_await_result_t<remove_reference_wrapper_t<SemiAwaitable>>>>+toFuture(SemiAwaitable&& a, Executor::KeepAlive<> ex) {+  auto excopy = ex;+  return co_withExecutor(+             std::move(excopy), toTask(std::forward<SemiAwaitable>(a)))+      .start()+      .via(std::move(ex));+}++} // namespace coro+} // namespace folly++#endif // FOLLY_HAS_COROUTINES
+ folly/folly/coro/Generator.h view
@@ -0,0 +1,282 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <cassert>+#include <exception>+#include <type_traits>+#include <utility>++#include <folly/coro/Coroutine.h>+#include <folly/coro/Invoke.h>+#include <folly/lang/Exception.h>++#if FOLLY_HAS_COROUTINES++namespace folly {+namespace coro {++template <typename T>+class Generator {+ public:+  class promise_type final {+   public:+    promise_type() noexcept+        : m_value(nullptr),+          m_exception(nullptr),+          m_root(this),+          m_parentOrLeaf(this) {}++    promise_type(const promise_type&) = delete;+    promise_type(promise_type&&) = delete;++    auto get_return_object() noexcept { return Generator<T>{*this}; }++    suspend_always initial_suspend() noexcept { return {}; }++    suspend_always final_suspend() noexcept { return {}; }++    void unhandled_exception() noexcept { m_exception = current_exception(); }++    void return_void() noexcept {}++    suspend_always yield_value(T& value) noexcept {+      m_value = std::addressof(value);+      return {};+    }++    suspend_always yield_value(T&& value) noexcept {+      m_value = std::addressof(value);+      return {};+    }++    auto yield_value(Generator&& generator) noexcept {+      return yield_value(generator);+    }++    auto yield_value(Generator& generator) noexcept {+      struct awaitable {+        awaitable(promise_type* childPromise) : m_childPromise(childPromise) {}++        bool await_ready() noexcept { return this->m_childPromise == nullptr; }++        void await_suspend(coroutine_handle<promise_type>) noexcept {}++        void await_resume() {+          if (this->m_childPromise != nullptr) {+            this->m_childPromise->throw_if_exception();+          }+        }++       private:+        promise_type* m_childPromise;+      };++      if (generator.m_promise != nullptr) {+        m_root->m_parentOrLeaf = generator.m_promise;+        generator.m_promise->m_root = m_root;+        generator.m_promise->m_parentOrLeaf = this;+        generator.m_promise->resume();++        // NB: This branch looks like a (premature?) optimization for empty+        // generators, and until proven otherwise in benchmarks, it may be+        // advantageous to simply return `awaitable{generator.m_promise}`.+        if (!generator.m_promise->is_complete() ||+            generator.m_promise->m_exception != nullptr) {+          return awaitable{generator.m_promise};+        }++        m_root->m_parentOrLeaf = this;+      }++      return awaitable{nullptr};+    }++    // Don't allow any use of 'co_await' inside the Generator+    // coroutine.+    template <typename U>+    void await_transform(U&& value) = delete;++    void destroy() noexcept {+      coroutine_handle<promise_type>::from_promise(*this).destroy();+    }++    void throw_if_exception() {+      if (m_exception != nullptr) {+        std::rethrow_exception(std::move(m_exception));+      }+    }++    bool is_complete() noexcept {+      return coroutine_handle<promise_type>::from_promise(*this).done();+    }++    T& value() noexcept {+      assert(this == m_root);+      assert(!is_complete());+      return *(m_parentOrLeaf->m_value);+    }++    void pull() noexcept {+      assert(this == m_root);+      assert(!m_parentOrLeaf->is_complete());++      m_parentOrLeaf->resume();++      while (m_parentOrLeaf != this && m_parentOrLeaf->is_complete()) {+        m_parentOrLeaf = m_parentOrLeaf->m_parentOrLeaf;+        m_parentOrLeaf->resume();+      }+    }++   private:+    void resume() noexcept {+      coroutine_handle<promise_type>::from_promise(*this).resume();+    }++    std::add_pointer_t<T> m_value;+    std::exception_ptr m_exception;++    promise_type* m_root;++    // If this is the promise of the root generator then this field+    // is a pointer to the leaf promise.+    // For non-root generators this is a pointer to the parent promise.+    promise_type* m_parentOrLeaf;+  };++  Generator() noexcept : m_promise(nullptr) {}++  Generator(promise_type& promise) noexcept : m_promise(&promise) {}++  Generator(Generator&& other) noexcept : m_promise(other.m_promise) {+    other.m_promise = nullptr;+  }++  Generator(const Generator& other) = delete;+  Generator& operator=(const Generator& other) = delete;++  ~Generator() {+    if (m_promise != nullptr) {+      m_promise->destroy();+    }+  }++  Generator& operator=(Generator&& other) noexcept {+    if (this != &other) {+      if (m_promise != nullptr) {+        m_promise->destroy();+      }++      m_promise = other.m_promise;+      other.m_promise = nullptr;+    }++    return *this;+  }++  class iterator {+   public:+    using iterator_category = std::input_iterator_tag;+    // What type should we use for counting elements of a potentially infinite+    // sequence?+    using difference_type = std::ptrdiff_t;+    using value_type = std::remove_reference_t<T>;+    using reference = std::conditional_t<std::is_reference_v<T>, T, T&>;+    using pointer = std::add_pointer_t<T>;++    iterator() noexcept : m_promise(nullptr) {}++    explicit iterator(promise_type* promise) noexcept : m_promise(promise) {}++    bool operator==(const iterator& other) const noexcept {+      return m_promise == other.m_promise;+    }++    bool operator!=(const iterator& other) const noexcept {+      return m_promise != other.m_promise;+    }++    iterator& operator++() {+      assert(m_promise != nullptr);+      assert(!m_promise->is_complete());++      m_promise->pull();+      if (m_promise->is_complete()) {+        auto* temp = m_promise;+        m_promise = nullptr;+        temp->throw_if_exception();+      }++      return *this;+    }++    void operator++(int) { (void)operator++(); }++    reference operator*() const noexcept {+      assert(m_promise != nullptr);+      return static_cast<reference>(m_promise->value());+    }++    pointer operator->() const noexcept { return std::addressof(operator*()); }++   private:+    promise_type* m_promise;+  };++  iterator begin() {+    if (m_promise != nullptr) {+      m_promise->pull();+      if (!m_promise->is_complete()) {+        return iterator(m_promise);+      }++      m_promise->throw_if_exception();+    }++    return iterator(nullptr);+  }++  iterator end() noexcept { return iterator(nullptr); }++  void swap(Generator& other) noexcept {+    std::swap(m_promise, other.m_promise);+  }++  template <typename F, typename... A, typename F_, typename... A_>+  friend Generator tag_invoke(+      tag_t<co_invoke_fn>, tag_t<Generator, F, A...>, F_ f, A_... a) {+    auto&& r = invoke(static_cast<F&&>(f), static_cast<A&&>(a)...);+    for (auto&& v : r) {+      co_yield std::move(v);+    }+  }++ private:+  friend class promise_type;++  promise_type* m_promise;+};++template <typename T>+void swap(Generator<T>& a, Generator<T>& b) noexcept {+  a.swap(b);+}+} // namespace coro+} // namespace folly++#endif // FOLLY_HAS_COROUTINES
+ folly/folly/coro/GmockHelpers.h view
@@ -0,0 +1,256 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <atomic>+#include <type_traits>++#include <folly/coro/Coroutine.h>+#include <folly/coro/GtestHelpers.h>+#include <folly/coro/Result.h>+#include <folly/coro/Task.h>+#include <folly/portability/GMock.h>++#if FOLLY_HAS_COROUTINES++namespace folly {+namespace coro {+namespace gmock_helpers {++// This helper function is intended for use in GMock implementations where the+// implementation of the method is a coroutine lambda.+//+// The GMock framework internally always takes a copy of an action/lambda+// before invoking it to prevent cases where invoking the method might end+// up destroying itself.+//+// However, this is problematic for coroutine-lambdas-with-captures as the+// return-value from invoking a coroutine lambda will typically capture a+// reference to the copy of the lambda which will immediately become a dangling+// reference as soon as the mocking framework returns that value to the caller.+//+// Use this action-factory instead of Invoke() when passing coroutine-lambdas+// to mock definitions to ensure that a copy of the lambda is kept alive until+// the coroutine completes. It does this by invoking the lambda using the+// folly::coro::co_invoke() helper instead of directly invoking the lambda.+//+//+// Example:+//   using namespace ::testing+//   using namespace folly::coro::gmock_helpers;+//+//   MockFoo mock;+//   int fooCallCount = 0;+//+//   EXPECT_CALL(mock, foo(_))+//     .WillRepeatedly(CoInvoke(+//         [&](int x) -> folly::coro::Task<int> {+//           ++fooCallCount;+//           co_return x + 1;+//         }));+//+template <typename F>+auto CoInvoke(F&& f) {+  return ::testing::Invoke([f = static_cast<F&&>(f)](auto&&... a) {+    return co_invoke(f, static_cast<decltype(a)>(a)...);+  });+}++// Member function overload+template <class Class, typename MethodPtr>+auto CoInvoke(Class* obj_ptr, MethodPtr method_ptr) {+  return ::testing::Invoke([=](auto&&... a) {+    return co_invoke(method_ptr, obj_ptr, static_cast<decltype(a)>(a)...);+  });+}++// CoInvoke variant that does not pass arguments to callback function.+//+// Example:+//   using namespace ::testing+//   using namespace folly::coro::gmock_helpers;+//+//   MockFoo mock;+//   int fooCallCount = 0;+//+//   EXPECT_CALL(mock, foo(_))+//     .WillRepeatedly(CoInvokeWithoutArgs(+//         [&]() -> folly::coro::Task<int> {+//           ++fooCallCount;+//           co_return 42;+//         }));+template <typename F>+auto CoInvokeWithoutArgs(F&& f) {+  return ::testing::InvokeWithoutArgs([f = static_cast<F&&>(f)]() {+    return co_invoke(f);+  });+}++// Member function overload+template <class Class, typename MethodPtr>+auto CoInvokeWithoutArgs(Class* obj_ptr, MethodPtr method_ptr) {+  return ::testing::InvokeWithoutArgs([=]() {+    return co_invoke(method_ptr, obj_ptr);+  });+}++namespace detail {+template <typename Fn>+auto makeCoAction(Fn&& fn) {+  static_assert(+      std::is_copy_constructible_v<remove_cvref_t<Fn>>,+      "Fn should be copyable to allow calling mocked call multiple times.");++  using Ret = std::invoke_result_t<remove_cvref_t<Fn>&&>;+  return ::testing::InvokeWithoutArgs(+      [fn = std::forward<Fn>(fn)]() mutable -> Ret { return co_invoke(fn); });+}++// Helper class to capture a ByMove return value for mocked coroutine function.+// Adds a test failure if it is moved twice like:+//    .WillRepeatedly(CoReturnByMove...)+template <typename R>+struct OnceForwarder {+  static_assert(std::is_reference_v<R>);+  using V = remove_cvref_t<R>;++  explicit OnceForwarder(R r) noexcept(std::is_nothrow_constructible_v<V>)+      : val_(static_cast<R>(r)) {}++  R operator()() noexcept {+    auto performedPreviously =+        performed_.exchange(true, std::memory_order_relaxed);+    if (performedPreviously) {+      terminate_with<std::runtime_error>(+          "a CoReturnByMove action must be performed only once");+    }+    return static_cast<R>(val_);+  }++ private:+  V val_;+  std::atomic<bool> performed_ = false;+};++// Allow to return a value by providing a convertible value.+// This works similarly to Return(x):+// MOCK_METHOD1(Method, T(U));+// EXPECT_CALL(mock, Method(_)).WillOnce(Return(F()));+// should work as long as F is convertible to T.+template <typename T>+class CoReturnImpl {+ public:+  explicit CoReturnImpl(T&& value) : value_(std::move(value)) {}++  template <typename Result, typename ArgumentTuple>+  Result Perform(const ArgumentTuple& /* unused */) const {+    return [](T value) -> Result { co_return value; }(T(value_));+  }++ private:+  T value_;+};++template <typename T>+class CoReturnByMoveImpl {+ public:+  explicit CoReturnByMoveImpl(std::shared_ptr<OnceForwarder<T&&>> forwarder)+      : forwarder_(std::move(forwarder)) {}++  template <typename Result, typename ArgumentTuple>+  Result Perform(const ArgumentTuple& /* unused */) const {+    return [](std::shared_ptr<OnceForwarder<T&&>> forwarder) -> Result {+      co_return (*forwarder)();+    }(forwarder_);+  }++ private:+  std::shared_ptr<OnceForwarder<T&&>> forwarder_;+};++} // namespace detail++// Helper functions to adapt CoRoutines enabled functions to be mocked using+// gMock. CoReturn and CoThrows are gMock Action types that mirror the Return+// and Throws Action types used in EXPECT_CALL|ON_CALL invocations.+//+// Example:+//   using namespace ::testing+//   using namespace folly::coro::gmock_helpers;+//+//   MockFoo mock;+//   std::string result = "abc";+//+//   EXPECT_CALL(mock, co_foo(_))+//     .WillRepeatedly(CoReturn(result));+//+//   // For Task<void> return types.+//   EXPECT_CALL(mock, co_bar(_))+//     .WillRepeatedly(CoReturn());+//+//   // For returning by move.+//   EXPECT_CALL(mock, co_bar(_))+//     .WillRepeatedly(CoReturnByMove(std::move(result)));+//+//   // For returning by move.+//   EXPECT_CALL(mock, co_bar(_))+//     .WillRepeatedly(CoReturnByMove(std::make_unique(result)));+//+//+//  EXPECT_CALL(mock, co_foo(_))+//     .WillRepeatedly(CoThrow<std::string>(std::runtime_error("error")));+template <typename T>+auto CoReturn(T ret) {+  return ::testing::MakePolymorphicAction(+      detail::CoReturnImpl<T>(std::move(ret)));+}++inline auto CoReturn() {+  return ::testing::InvokeWithoutArgs([]() -> Task<> { co_return; });+}++template <typename T>+auto CoReturnByMove(T&& ret) {+  static_assert(+      !std::is_lvalue_reference_v<decltype(ret)>,+      "the argument must be passed as non-const rvalue-ref");+  static_assert(+      !std::is_const_v<T>,+      "the argument must be passed as non-const rvalue-ref");++  auto ptr = std::make_shared<detail::OnceForwarder<T&&>>(std::move(ret));++  return ::testing::MakePolymorphicAction(+      detail::CoReturnByMoveImpl<T>(std::move(ptr)));+}++template <typename T, typename Ex>+auto CoThrow(Ex&& e) {+  return detail::makeCoAction([ex = std::forward<Ex>(e)]() -> Task<T> {+    co_yield co_error(ex);+  });+}++} // namespace gmock_helpers+} // namespace coro+} // namespace folly++#define CO_ASSERT_THAT(value, matcher) \+  CO_ASSERT_PRED_FORMAT1(              \+      ::testing::internal::MakePredicateFormatterFromMatcher(matcher), value)++#endif // FOLLY_HAS_COROUTINES
+ folly/folly/coro/GtestHelpers.h view
@@ -0,0 +1,339 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <folly/coro/BlockingWait.h>+#include <folly/coro/Coroutine.h>+#include <folly/coro/Task.h>+#include <folly/debugging/exception_tracer/SmartExceptionTracer.h>+#include <folly/portability/GTest.h>++#if FOLLY_HAS_COROUTINES++namespace folly {+namespace detail {++template <typename Out>+inline auto gtestLogCurrentException(Out&& out) {+  auto ew = exception_wrapper(std::current_exception());+#ifdef FOLLY_HAVE_SMART_EXCEPTION_TRACER+  auto trace = folly::exception_tracer::getAsyncTrace(ew);+  out << ew << ", async stack trace: " << trace;+#else+  out << ew;+#endif+}++} // namespace detail+} // namespace folly++/**+ * This is based on the GTEST_TEST_ macro from gtest-internal.h. It seems that+ * gtest doesn't yet support coro tests, so this macro adds a way to define a+ * test case written as a coroutine using folly::coro::Task. It will be called+ * using folly::coro::blockingWait().+ *+ * Note that you cannot use ASSERT macros in coro tests. See below for+ * CO_ASSERT_*.+ */+#define CO_TEST_(                                                              \+    test_suite_name,                                                           \+    test_name,                                                                 \+    parent_class,                                                              \+    parent_id,                                                                 \+    body_coro_t,                                                               \+    unwrap_body)                                                               \+  static_assert(                                                               \+      sizeof(GTEST_STRINGIFY_(test_suite_name)) > 1,                           \+      "test_suite_name must not be empty");                                    \+  static_assert(                                                               \+      sizeof(GTEST_STRINGIFY_(test_name)) > 1, "test_name must not be empty"); \+  class GTEST_TEST_CLASS_NAME_(test_suite_name, test_name)                     \+      : public parent_class {                                                  \+   public:                                                                     \+    GTEST_TEST_CLASS_NAME_(test_suite_name, test_name)() = default;            \+    ~GTEST_TEST_CLASS_NAME_(test_suite_name, test_name)() override = default;  \+    GTEST_TEST_CLASS_NAME_(test_suite_name, test_name)                         \+    (const GTEST_TEST_CLASS_NAME_(test_suite_name, test_name) &) = delete;     \+    GTEST_TEST_CLASS_NAME_(test_suite_name, test_name) & operator=(            \+        const GTEST_TEST_CLASS_NAME_(test_suite_name, test_name) &) =          \+        delete; /* NOLINT */                                                   \+    GTEST_TEST_CLASS_NAME_(test_suite_name, test_name)                         \+    (GTEST_TEST_CLASS_NAME_(test_suite_name, test_name) &&) noexcept = delete; \+    GTEST_TEST_CLASS_NAME_(test_suite_name, test_name) & operator=(            \+        GTEST_TEST_CLASS_NAME_(test_suite_name, test_name) &&) noexcept =      \+        delete; /* NOLINT */                                                   \+                                                                               \+   private:                                                                    \+    void TestBody() override;                                                  \+    body_coro_t co_TestBody();                                                 \+    static ::testing::TestInfo* const test_info_ [[maybe_unused]];             \+  };                                                                           \+                                                                               \+  ::testing::TestInfo* const GTEST_TEST_CLASS_NAME_(                           \+      test_suite_name, test_name)::test_info_ =                                \+      ::testing::internal::MakeAndRegisterTestInfo(                            \+          #test_suite_name,                                                    \+          #test_name,                                                          \+          nullptr,                                                             \+          nullptr,                                                             \+          ::testing::internal::CodeLocation(__FILE__, __LINE__),               \+          (parent_id),                                                         \+          ::testing::internal::SuiteApiResolver<                               \+              parent_class>::GetSetUpCaseOrSuite(__FILE__, __LINE__),          \+          ::testing::internal::SuiteApiResolver<                               \+              parent_class>::GetTearDownCaseOrSuite(__FILE__, __LINE__),       \+          new ::testing::internal::TestFactoryImpl<GTEST_TEST_CLASS_NAME_(     \+              test_suite_name, test_name)>);                                   \+  void GTEST_TEST_CLASS_NAME_(test_suite_name, test_name)::TestBody() {        \+    unwrap_body(co_TestBody);                                                  \+  }                                                                            \+  body_coro_t GTEST_TEST_CLASS_NAME_(test_suite_name, test_name)::co_TestBody()++#define CO_UNWRAP_BODY(body)                                    \+  try {                                                         \+    folly::coro::blockingWait(body());                          \+  } catch (...) {                                               \+    folly::detail::gtestLogCurrentException(GTEST_LOG_(ERROR)); \+    throw;                                                      \+  }++/**                                    \+ * TEST() for coro tests.              \+ */+#define CO_TEST(test_case_name, test_name)  \+  CO_TEST_(                                 \+      test_case_name,                       \+      test_name,                            \+      ::testing::Test,                      \+      ::testing::internal::GetTestTypeId(), \+      folly::coro::Task<void>,              \+      CO_UNWRAP_BODY)++/**+ * TEST_F() for coro tests.+ */+#define CO_TEST_F(test_fixture, test_name)            \+  CO_TEST_(                                           \+      test_fixture,                                   \+      test_name,                                      \+      test_fixture,                                   \+      ::testing::internal::GetTypeId<test_fixture>(), \+      folly::coro::Task<void>,                        \+      CO_UNWRAP_BODY)++#define CO_TEST_P(test_suite_name, test_name)                                  \+  class GTEST_TEST_CLASS_NAME_(test_suite_name, test_name)                     \+      : public test_suite_name {                                               \+   public:                                                                     \+    GTEST_TEST_CLASS_NAME_(test_suite_name, test_name)() {}                    \+    void TestBody() override;                                                  \+    folly::coro::Task<void> co_TestBody();                                     \+                                                                               \+   private:                                                                    \+    static int AddToRegistry() {                                               \+      ::testing::UnitTest::GetInstance()                                       \+          ->parameterized_test_registry()                                      \+          .GetTestSuitePatternHolder<test_suite_name>(                         \+              GTEST_STRINGIFY_(test_suite_name),                               \+              ::testing::internal::CodeLocation(__FILE__, __LINE__))           \+          ->AddTestPattern(                                                    \+              GTEST_STRINGIFY_(test_suite_name),                               \+              GTEST_STRINGIFY_(test_name),                                     \+              new ::testing::internal::TestMetaFactory<GTEST_TEST_CLASS_NAME_( \+                  test_suite_name, test_name)>(),                              \+              ::testing::internal::CodeLocation(__FILE__, __LINE__));          \+      return 0;                                                                \+    }                                                                          \+    static int gtest_registering_dummy_ [[maybe_unused]];                      \+    GTEST_TEST_CLASS_NAME_(test_suite_name, test_name)                         \+    (const GTEST_TEST_CLASS_NAME_(test_suite_name, test_name) &) = delete;     \+    GTEST_TEST_CLASS_NAME_(test_suite_name, test_name) & operator=(            \+        const GTEST_TEST_CLASS_NAME_(test_suite_name, test_name) &) =          \+        delete; /* NOLINT */                                                   \+  };                                                                           \+  int GTEST_TEST_CLASS_NAME_(                                                  \+      test_suite_name, test_name)::gtest_registering_dummy_ =                  \+      GTEST_TEST_CLASS_NAME_(test_suite_name, test_name)::AddToRegistry();     \+  void GTEST_TEST_CLASS_NAME_(test_suite_name, test_name)::TestBody() {        \+    try {                                                                      \+      folly::coro::blockingWait(co_TestBody());                                \+    } catch (...) {                                                            \+      folly::detail::gtestLogCurrentException(GTEST_LOG_(ERROR));              \+      throw;                                                                   \+    }                                                                          \+  }                                                                            \+  folly::coro::Task<void> GTEST_TEST_CLASS_NAME_(                              \+      test_suite_name, test_name)::co_TestBody()++#define CO_TYPED_TEST(CaseName, TestName)                                     \+  static_assert(                                                              \+      sizeof(GTEST_STRINGIFY_(TestName)) > 1, "test-name must not be empty"); \+  template <typename gtest_TypeParam_>                                        \+  class GTEST_TEST_CLASS_NAME_(CaseName, TestName)                            \+      : public CaseName<gtest_TypeParam_> {                                   \+   private:                                                                   \+    typedef CaseName<gtest_TypeParam_> TestFixture;                           \+    typedef gtest_TypeParam_ TypeParam;                                       \+    void TestBody() override;                                                 \+    folly::coro::Task<void> co_TestBody();                                    \+  };                                                                          \+  static bool gtest_##CaseName##_##TestName##_registered_ [[maybe_unused]] =  \+      ::testing::internal::TypeParameterizedTest<                             \+          CaseName,                                                           \+          ::testing::internal::TemplateSel<GTEST_TEST_CLASS_NAME_(            \+              CaseName, TestName)>,                                           \+          GTEST_TYPE_PARAMS_(CaseName)>::                                     \+          Register(                                                           \+              "",                                                             \+              ::testing::internal::CodeLocation(__FILE__, __LINE__),          \+              GTEST_STRINGIFY_(CaseName),                                     \+              GTEST_STRINGIFY_(TestName),                                     \+              0,                                                              \+              ::testing::internal::GenerateNames<                             \+                  GTEST_NAME_GENERATOR_(CaseName),                            \+                  GTEST_TYPE_PARAMS_(CaseName)>());                           \+  template <typename gtest_TypeParam_>                                        \+  void GTEST_TEST_CLASS_NAME_(                                                \+      CaseName, TestName)<gtest_TypeParam_>::TestBody() {                     \+    try {                                                                     \+      folly::coro::blockingWait(co_TestBody());                               \+    } catch (...) {                                                           \+      folly::detail::gtestLogCurrentException(GTEST_LOG_(ERROR));             \+      throw;                                                                  \+    }                                                                         \+  }                                                                           \+  template <typename gtest_TypeParam_>                                        \+  folly::coro::Task<void> GTEST_TEST_CLASS_NAME_(                             \+      CaseName, TestName)<gtest_TypeParam_>::co_TestBody()++#define CO_TYPED_TEST_P(SuiteName, TestName)                      \+  namespace GTEST_SUITE_NAMESPACE_(SuiteName) {                   \+  template <typename gtest_TypeParam_>                            \+  class TestName : public SuiteName<gtest_TypeParam_> {           \+   private:                                                       \+    typedef SuiteName<gtest_TypeParam_> TestFixture;              \+    typedef gtest_TypeParam_ TypeParam;                           \+    void TestBody() override;                                     \+    folly::coro::Task<> co_TestBody();                            \+  };                                                              \+  [[maybe_unused]] static bool gtest_##TestName##_defined_ =      \+      GTEST_TYPED_TEST_SUITE_P_STATE_(SuiteName).AddTestName(     \+          __FILE__,                                               \+          __LINE__,                                               \+          GTEST_STRINGIFY_(SuiteName),                            \+          GTEST_STRINGIFY_(TestName));                            \+  }                                                               \+  template <typename gtest_TypeParam_>                            \+  void GTEST_SUITE_NAMESPACE_(                                    \+      SuiteName)::TestName<gtest_TypeParam_>::TestBody() {        \+    try {                                                         \+      folly::coro::blockingWait(co_TestBody());                   \+    } catch (...) {                                               \+      folly::detail::gtestLogCurrentException(GTEST_LOG_(ERROR)); \+      throw;                                                      \+    }                                                             \+  }                                                               \+  template <typename gtest_TypeParam_>                            \+  folly::coro::Task<void> GTEST_SUITE_NAMESPACE_(                 \+      SuiteName)::TestName<gtest_TypeParam_>::co_TestBody()++/**+ * Coroutine versions of GTests's Assertion predicate macros. Use these in place+ * of ASSERT_* in CO_TEST or coroutine functions.+ */+#define CO_GTEST_FATAL_FAILURE_(message) \+  co_return GTEST_MESSAGE_(message, ::testing::TestPartResult::kFatalFailure)++#define CO_ASSERT_PRED_FORMAT1(pred_format, v1) \+  GTEST_PRED_FORMAT1_(pred_format, v1, CO_GTEST_FATAL_FAILURE_)+#define CO_ASSERT_PRED_FORMAT2(pred_format, v1, v2) \+  GTEST_PRED_FORMAT2_(pred_format, v1, v2, CO_GTEST_FATAL_FAILURE_)++#define CO_ASSERT_TRUE(condition) \+  GTEST_TEST_BOOLEAN_(            \+      (condition), #condition, false, true, CO_GTEST_FATAL_FAILURE_)+#define CO_ASSERT_FALSE(condition) \+  GTEST_TEST_BOOLEAN_(             \+      !(condition), #condition, true, false, CO_GTEST_FATAL_FAILURE_)++#if defined(GTEST_IS_NULL_LITERAL_)+#define CO_ASSERT_EQ(val1, val2)                                            \+  CO_ASSERT_PRED_FORMAT2(                                                   \+      ::testing::internal::EqHelper<GTEST_IS_NULL_LITERAL_(val1)>::Compare, \+      val1,                                                                 \+      val2)+#else+#define CO_ASSERT_EQ(val1, val2) \+  CO_ASSERT_PRED_FORMAT2(::testing::internal::EqHelper::Compare, val1, val2)+#endif++#define CO_ASSERT_NE(val1, val2) \+  CO_ASSERT_PRED_FORMAT2(::testing::internal::CmpHelperNE, val1, val2)+#define CO_ASSERT_LE(val1, val2) \+  CO_ASSERT_PRED_FORMAT2(::testing::internal::CmpHelperLE, val1, val2)+#define CO_ASSERT_LT(val1, val2) \+  CO_ASSERT_PRED_FORMAT2(::testing::internal::CmpHelperLT, val1, val2)+#define CO_ASSERT_GE(val1, val2) \+  CO_ASSERT_PRED_FORMAT2(::testing::internal::CmpHelperGE, val1, val2)+#define CO_ASSERT_GT(val1, val2) \+  CO_ASSERT_PRED_FORMAT2(::testing::internal::CmpHelperGT, val1, val2)++#define CO_ASSERT_THROW(statement, expected_exception) \+  GTEST_TEST_THROW_(statement, expected_exception, CO_GTEST_FATAL_FAILURE_)+#define CO_ASSERT_NO_THROW(statement) \+  GTEST_TEST_NO_THROW_(statement, CO_GTEST_FATAL_FAILURE_)+#define CO_ASSERT_ANY_THROW(statement) \+  GTEST_TEST_ANY_THROW_(statement, CO_GTEST_FATAL_FAILURE_)++/**+ * coroutine version of FAIL() which is defined as GTEST_FAIL()+ * GTEST_FATAL_FAILURE_("Failed")+ */+#define CO_FAIL() CO_GTEST_FATAL_FAILURE_("Failed")++/**+ * Coroutine version of SKIP() which is defined as GTEST_SKIP()+ */+#define CO_SKIP(message) \+  co_return GTEST_MESSAGE_(message, ::testing::TestPartResult::kSkip)++/**+ * Coroutine version of SKIP_IF()+ */+#define CO_SKIP_IF(expr, message) \+  GTEST_AMBIGUOUS_ELSE_BLOCKER_   \+  if (!(expr)) {                  \+  } else                          \+    CO_SKIP(message)++/**+ * Coroutine version of SUCCEED() which is defined as GTEST_SUCCEED()+ */+#define CO_SUCCEED(message) \+  co_return GTEST_MESSAGE_(message, ::testing::TestPartResult::kSuccess)++/**+ * Coroutine version+ */+#define CO_SUCCEED_IF(expr, message) \+  GTEST_AMBIGUOUS_ELSE_BLOCKER_      \+  if (!(expr)) {                     \+  } else                             \+    CO_SUCCEED(message)++#endif // FOLLY_HAS_COROUTINES
+ folly/folly/coro/Invoke.h view
@@ -0,0 +1,99 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <folly/functional/Invoke.h>+#include <folly/lang/CustomizationPoint.h>++namespace folly {+namespace coro {++//  co_invoke+//+//  This utility callable is a safe way to instantiate a coroutine using a+//  coroutine callable. It guarantees that the callable and the arguments+//  outlive the coroutine which invocation returns. Otherwise, the callable+//  and the arguments are not safe to be used within the coroutine body.+//+//  For example, if the callable is a lambda with captures, the captures would+//  not otherwise be safe to use in the coroutine body without using co_invoke.+//+//  Models invoke for any callable which returns a coroutine type which declares+//  support for co_invoke, including:+//    * AsyncGenerator<...>+//    * Task<...>+//+//  Like invoke in that the callable is invoked with the cvref-qual with which+//  it is passed to co_invoke and the arguments are passed with the cvref-quals+//  with which they were passed to co_invoke.+//+//  Different from invoke in that the callable and all arguments are decay-+//  copied and it is the copies that are held for the lifetime of the coroutine+//  and used in the invocation, whereas invoke merely forwards them directly in+//  the invocation without first constructing any values copied from them.+//+//  Example:+//+//      auto gen = co_invoke([range]() -> AsyncGenerator<T> {+//        for (auto value : range) {+//          co_yield co_await make<T>(value);+//        }+//      });+//+//  Example:+//+//      auto task = co_invoke([name, dob]() -> Task<T> {+//        co_return co_await make<T>(name, dob);+//      });+//+//  A word of caution. The callable and each argument is decay-copied by the+//  customizations. No effort is made to coalesce copies when copies would have+//  been made with direct invocation.+//+//      string name = "foobar"; // will be copied twice+//      auto task = co_invoke([](string n) -> Task<T> {+//        co_return co_await make<T>(n);+//      }, name); // passed as &+//+//      string name = "foobar"; // will be moved twice and copied zero times+//      auto task = co_invoke([](string n) -> Task<T> {+//        co_return co_await make<T>(n);+//      }, std::move(name)); // passed as &&+struct co_invoke_fn {+  template <typename F, typename... A>+  FOLLY_ERASE constexpr auto+  operator()(F&& f, A&&... a) const noexcept(noexcept(tag_invoke(+      tag<co_invoke_fn>,+      tag<invoke_result_t<F, A...>, F, A...>,+      static_cast<F&&>(f),+      static_cast<A&&>(a)...)))+      -> decltype(tag_invoke(+          tag<co_invoke_fn>,+          tag<invoke_result_t<F, A...>, F, A...>,+          static_cast<F&&>(f),+          static_cast<A&&>(a)...)) {+    return tag_invoke(+        tag<co_invoke_fn>,+        tag<invoke_result_t<F, A...>, F, A...>,+        static_cast<F&&>(f),+        static_cast<A&&>(a)...);+  }+};+FOLLY_DEFINE_CPO(co_invoke_fn, co_invoke)++} // namespace coro+} // namespace folly
+ folly/folly/coro/Merge-inl.h view
@@ -0,0 +1,393 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#include <exception>+#include <memory>++#include <folly/CancellationToken.h>+#include <folly/Executor.h>+#include <folly/ScopeGuard.h>+#include <folly/coro/Baton.h>+#include <folly/coro/Mutex.h>+#include <folly/coro/Task.h>+#include <folly/coro/ViaIfAsync.h>+#include <folly/coro/WithCancellation.h>+#include <folly/coro/detail/Barrier.h>+#include <folly/coro/detail/BarrierTask.h>+#include <folly/coro/detail/CurrentAsyncFrame.h>+#include <folly/coro/detail/Helpers.h>++#if FOLLY_HAS_COROUTINES++namespace folly {+namespace coro {+namespace detail {++enum class CallbackRecordSelector { Invalid, Value, None, Error };++constexpr inline std::in_place_index_t<0> const callback_record_value{};+constexpr inline std::in_place_index_t<1> const callback_record_none{};+constexpr inline std::in_place_index_t<2> const callback_record_error{};++//+// CallbackRecord records the result of a single invocation of a callback.+//+// This is very related to Try and expected, but this also records None in+// addition to Value and Error results.+//+// When the callback supports multiple overloads of Value then T would be+// something like a variant<tuple<..>, ..>+//+// When the callback supports multiple overloads of Error then all the errors+// are coerced to folly::exception_wrapper+//+template <class T>+class CallbackRecord {+  static void clear(CallbackRecord* that) {+    auto selector =+        std::exchange(that->selector_, CallbackRecordSelector::Invalid);+    if (selector == CallbackRecordSelector::Value) {+      detail::deactivate(that->value_);+    } else if (selector == CallbackRecordSelector::Error) {+      detail::deactivate(that->error_);+    }+  }+  template <class OtherReference>+  static void convert_variant(+      CallbackRecord* that, const CallbackRecord<OtherReference>& other) {+    if (other.hasValue()) {+      detail::activate(that->value_, other.value_.get());+    } else if (other.hasError()) {+      detail::activate(that->error_, other.error_.get());+    }+    that->selector_ = other.selector_;+  }+  template <class OtherReference>+  static void convert_variant(+      CallbackRecord* that, CallbackRecord<OtherReference>&& other) {+    if (other.hasValue()) {+      detail::activate(that->value_, std::move(other.value_).get());+    } else if (other.hasError()) {+      detail::activate(that->error_, std::move(other.error_).get());+    }+    that->selector_ = other.selector_;+  }++ public:+  ~CallbackRecord() { clear(this); }++  CallbackRecord() noexcept : selector_(CallbackRecordSelector::Invalid) {}++  template <class V>+  CallbackRecord(const std::in_place_index_t<0>&, V&& v) noexcept(+      std::is_nothrow_constructible_v<T, V>)+      : CallbackRecord() {+    detail::activate(value_, std::forward<V>(v));+    selector_ = CallbackRecordSelector::Value;+  }+  explicit CallbackRecord(const std::in_place_index_t<1>&) noexcept+      : selector_(CallbackRecordSelector::None) {}+  CallbackRecord(+      const std::in_place_index_t<2>&, folly::exception_wrapper e) noexcept+      : CallbackRecord() {+    detail::activate(error_, std::move(e));+    selector_ = CallbackRecordSelector::Error;+  }++  CallbackRecord(CallbackRecord&& other) noexcept(+      std::is_nothrow_move_constructible_v<T>)+      : CallbackRecord() {+    convert_variant(this, std::move(other));+  }++  CallbackRecord& operator=(CallbackRecord&& other) noexcept(+      std::is_nothrow_move_constructible_v<T>) {+    if (&other != this) {+      clear(this);+      convert_variant(this, std::move(other));+    }+    return *this;+  }++  template <class U>+  CallbackRecord(CallbackRecord<U>&& other) noexcept(+      std::is_nothrow_constructible_v<T, U>)+      : CallbackRecord() {+    convert_variant(this, std::move(other));+  }++  bool hasNone() const noexcept {+    return selector_ == CallbackRecordSelector::None;+  }++  bool hasError() const noexcept {+    return selector_ == CallbackRecordSelector::Error;+  }++  decltype(auto) error() & {+    DCHECK(hasError());+    return error_.get();+  }++  decltype(auto) error() && {+    DCHECK(hasError());+    return std::move(error_).get();+  }++  decltype(auto) error() const& {+    DCHECK(hasError());+    return error_.get();+  }++  decltype(auto) error() const&& {+    DCHECK(hasError());+    return std::move(error_).get();+  }++  bool hasValue() const noexcept {+    return selector_ == CallbackRecordSelector::Value;+  }++  decltype(auto) value() & {+    DCHECK(hasValue());+    return value_.get();+  }++  decltype(auto) value() && {+    DCHECK(hasValue());+    return std::move(value_).get();+  }++  decltype(auto) value() const& {+    DCHECK(hasValue());+    return value_.get();+  }++  decltype(auto) value() const&& {+    DCHECK(hasValue());+    return std::move(value_).get();+  }++  explicit operator bool() const noexcept {+    return selector_ != CallbackRecordSelector::Invalid;+  }++ private:+  union {+    detail::ManualLifetime<T> value_;+    detail::ManualLifetime<folly::exception_wrapper> error_;+  };+  CallbackRecordSelector selector_;+};++template <typename Reference, typename Value, typename GeneratorType>+AsyncGenerator<Reference, Value> mergeImpl(+    folly::Executor::KeepAlive<> executor, GeneratorType sources) {+  struct SharedState {+    explicit SharedState(folly::Executor::KeepAlive<> executor_)+        : executor(std::move(executor_)) {}++    const folly::Executor::KeepAlive<> executor;+    const folly::CancellationSource cancelSource;+    coro::Mutex mutex;+    coro::Baton recordPublished;+    coro::Baton recordConsumed;+    coro::Baton allTasksCompleted;+    detail::CallbackRecord<Reference> record;+  };++  auto makeConsumerTask =+      [](std::shared_ptr<SharedState> state,+         GeneratorType sources_) -> Task<void> {+    auto makeWorkerTask =+        [](std::shared_ptr<SharedState> state_,+           AsyncGenerator<Reference, Value> generator)+        -> detail::DetachedBarrierTask {+      exception_wrapper ex;+      auto cancelToken = state_->cancelSource.getToken();+      try {+        while (auto item = co_await co_viaIfAsync(+                   state_->executor.get_alias(),+                   co_withCancellation(cancelToken, generator.next()))) {+          // We have a new value to emit in the merged stream.+          {+            auto lock = co_await co_viaIfAsync(+                state_->executor.get_alias(), state_->mutex.co_scoped_lock());++            if (cancelToken.isCancellationRequested()) {+              // Consumer has detached and doesn't want any more values.+              // Discard this value.+              break;+            }++            // Publish the value.+            state_->record = detail::CallbackRecord<Reference>{+                detail::callback_record_value, *std::move(item)};+            state_->recordPublished.post();++            // Wait until the consumer is finished with it.+            co_await co_viaIfAsync(+                state_->executor.get_alias(), state_->recordConsumed);+            state_->recordConsumed.reset();++            // Clear the result before releasing the lock.+            state_->record = {};+          }++          if (cancelToken.isCancellationRequested()) {+            break;+          }+        }+      } catch (...) {+        ex = exception_wrapper{current_exception()};+      }++      if (ex) {+        state_->cancelSource.requestCancellation();++        auto lock = co_await co_viaIfAsync(+            state_->executor.get_alias(), state_->mutex.co_scoped_lock());+        if (!state_->record.hasError()) {+          state_->record = detail::CallbackRecord<Reference>{+              detail::callback_record_error, std::move(ex)};+          state_->recordPublished.post();+        }+      }+    };++    detail::Barrier barrier{1};++    auto& asyncFrame = co_await detail::co_current_async_stack_frame;++    // Save the initial context and restore it after starting each task+    // as the task may have modified the context before suspending and we+    // want to make sure the next task is started with the same initial+    // context.+    const auto context = RequestContext::saveContext();++    exception_wrapper ex;+    try {+      while (auto item = co_await sources_.next()) {+        if (state->cancelSource.isCancellationRequested()) {+          break;+        }+        makeWorkerTask(state, *std::move(item)).start(&barrier, asyncFrame);+        RequestContext::setContext(context);+      }+    } catch (...) {+      ex = exception_wrapper{current_exception()};+    }++    if (ex) {+      state->cancelSource.requestCancellation();++      auto lock = co_await co_viaIfAsync(+          state->executor.get_alias(), state->mutex.co_scoped_lock());+      if (!state->record.hasError()) {+        state->record = detail::CallbackRecord<Reference>{+            detail::callback_record_error, std::move(ex)};+        state->recordPublished.post();+      }+    }++    // Wait for all worker tasks to finish consuming the entirety of their+    // input streams.+    co_await detail::UnsafeResumeInlineSemiAwaitable{barrier.arriveAndWait()};++    // Guaranteed there are no more concurrent producers trying to acquire+    // the mutex here.+    if (!state->record.hasError()) {+      // Stream not yet been terminated with an error.+      // Terminate the stream with the 'end()' signal.+      assert(!state->record.hasValue());+      state->record =+          detail::CallbackRecord<Reference>{detail::callback_record_none};+      state->recordPublished.post();+    }+  };++  auto state = std::make_shared<SharedState>(executor);++  SCOPE_EXIT {+    state->cancelSource.requestCancellation();+    // Make sure we resume the worker thread so that it has a chance to notice+    // that cancellation has been requested.+    state->recordConsumed.post();+  };++  // Start a task that consumes the stream of input streams.+  co_withExecutor(executor, makeConsumerTask(state, std::move(sources)))+      .start(+          [state](auto&&) { state->allTasksCompleted.post(); },+          state->cancelSource.getToken());++  // Consume values produced by the input streams.+  while (true) {+    if (!state->recordPublished.ready()) {+      folly::CancellationCallback cb{+          co_await co_current_cancellation_token,+          [&] { state->cancelSource.requestCancellation(); }};+      co_await state->recordPublished;+    }+    state->recordPublished.reset();++    if (state->record.hasValue()) {+      // next value+      co_yield std::move(state->record).value();+      state->recordConsumed.post();+    } else {+      // We're closing the output stream. In the spirit of structured+      // concurrency, let's make sure to not leave any background tasks behind.+      co_await state->allTasksCompleted;++      if (state->record.hasError()) {+        std::move(state->record).error().throw_exception();+      } else {+        // none+        assert(state->record.hasNone());+        break;+      }+    }+  }+}+} // namespace detail++template <typename Reference, typename Value>+AsyncGenerator<Reference, Value> merge(+    folly::Executor::KeepAlive<> executor,+    AsyncGenerator<AsyncGenerator<Reference, Value>&&> sources) {+  return detail::mergeImpl<+      Reference,+      Value,+      AsyncGenerator<AsyncGenerator<Reference, Value>&&>>(+      std::move(executor), std::move(sources));+}++template <typename Reference, typename Value>+AsyncGenerator<Reference, Value> merge(+    folly::Executor::KeepAlive<> executor,+    AsyncGenerator<AsyncGenerator<Reference, Value>> sources) {+  return detail::mergeImpl<+      Reference,+      Value,+      AsyncGenerator<AsyncGenerator<Reference, Value>>>(+      std::move(executor), std::move(sources));+}++} // namespace coro+} // namespace folly++#endif // FOLLY_HAS_COROUTINES
+ folly/folly/coro/Merge.h view
@@ -0,0 +1,77 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <folly/coro/AsyncGenerator.h>+#include <folly/coro/Coroutine.h>++#if FOLLY_HAS_COROUTINES++namespace folly {+namespace coro {++// Merge the results of a number of input streams.+//+// The 'executor' parameter specifies the execution context to+// be used for awaiting each value from the sources.+// The 'sources' parameter represents an async-stream of async-streams.+// The resulting generator merges the results from each of the streams+// produced by 'sources', interleaving them in the order that the values+// are produced.+//+// The resulting stream will terminate when the end of the 'sources' stream has+// been reached and the ends of all of the input streams it produced have been+// reached.+//+// On exception or cancellation, cancels remaining input streams and 'sources',+// discards any remaining values, and produces an exception (if an input stream+// produced an exception) or end-of-stream (if next() call was cancelled).+//+// Structured concurrency: if the output stream produced an empty value+// (end-of-stream) or an exception, it's guaranteed that 'sources' and all input+// generators have been destroyed.+// If the output stream is destroyed early (before reaching end-of-stream or+// exception), the remaining input generators are cancelled and detached; beware+// of use-after-free.+//+// Normally cancelling output stream's next() call cancels the stream, discards+// any remaining values, and returns an end-of-stream. But there are caveats:+//  * If there's an item ready to be delivered, next() call returns it without+//    checking for cancellation. So if input streams are fast, and next() is+//    called infrequently, cancellation may go unprocessed indefinitely unless+//    you also check for cancellation on your side (which you should probably do+//    anyway unless you're calling next() in a tight loop).+//  * It's possible that the cancelled next() registers the cancellation but+//    returns a value anyway (if it was produced at just the right moment). Then+//    a later next() call would return end-of-stream even if it was called with+//    a different, non-cancelled cancellation token.+template <typename Reference, typename Value>+AsyncGenerator<Reference, Value> merge(+    folly::Executor::KeepAlive<> executor,+    AsyncGenerator<AsyncGenerator<Reference, Value>&&> sources);++template <typename Reference, typename Value>+AsyncGenerator<Reference, Value> merge(+    folly::Executor::KeepAlive<> executor,+    AsyncGenerator<AsyncGenerator<Reference, Value>> sources);++} // namespace coro+} // namespace folly++#endif // FOLLY_HAS_COROUTINES++#include <folly/coro/Merge-inl.h>
+ folly/folly/coro/Mutex.cpp view
@@ -0,0 +1,108 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#include <folly/coro/Mutex.h>++#include <cassert>++#if FOLLY_HAS_COROUTINES++using namespace folly::coro;++Mutex::~Mutex() {+  // Check there are no waiters waiting to acquire the lock.+  assert(+      state_.load(std::memory_order_relaxed) == unlockedState() ||+      state_.load(std::memory_order_relaxed) == nullptr);+  assert(waiters_ == nullptr);+}++void Mutex::unlock() noexcept {+  assert(state_.load(std::memory_order_relaxed) != unlockedState());++  auto* waitersHead = waiters_;+  if (waitersHead == nullptr) {+    void* currentState = state_.load(std::memory_order_relaxed);+    if (currentState == nullptr) {+      // Looks like there are no waiters waiting to acquire the lock.+      // Try to unlock it - use a compare-exchange to decide the race between+      // unlocking the mutex and another thread enqueueing another waiter.+      const bool releasedLock = state_.compare_exchange_strong(+          currentState,+          unlockedState(),+          std::memory_order_release,+          std::memory_order_relaxed);+      if (releasedLock) {+        return;+      }+    }++    // There are some awaiters that have been newly queued.+    // Dequeue them and reverse their order from LIFO to FIFO.+    currentState = state_.exchange(nullptr, std::memory_order_acquire);++    assert(currentState != unlockedState());+    assert(currentState != nullptr);++    auto* waiter = static_cast<LockAwaiter*>(currentState);+    do {+      auto* temp = waiter->next_;+      waiter->next_ = waitersHead;+      waitersHead = waiter;+      waiter = temp;+    } while (waiter != nullptr);+  }++  assert(waitersHead != nullptr);++  waiters_ = waitersHead->next_;++  waitersHead->awaitingCoroutine_.resume();+}++bool Mutex::lockAsyncImpl(LockAwaiter* awaiter) {+  void* oldValue = state_.load(std::memory_order_relaxed);+  while (true) {+    if (oldValue == unlockedState()) {+      // It looks like the mutex is currently unlocked.+      // Try to acquire it synchronously.+      void* newValue = nullptr;+      if (state_.compare_exchange_weak(+              oldValue,+              newValue,+              std::memory_order_acquire,+              std::memory_order_relaxed)) {+        // Acquired synchronously, don't suspend.+        return false;+      }+    } else {+      // It looks like the mutex is currently locked.+      // Try to queue this waiter to the list of waiters.+      void* newValue = awaiter;+      awaiter->next_ = static_cast<LockAwaiter*>(oldValue);+      if (state_.compare_exchange_weak(+              oldValue,+              newValue,+              std::memory_order_release,+              std::memory_order_relaxed)) {+        // Queued waiter successfully. Awaiting coroutine should suspend.+        return true;+      }+    }+  }+}++#endif
+ folly/folly/coro/Mutex.h view
@@ -0,0 +1,244 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <folly/Executor.h>+#include <folly/coro/Coroutine.h>+#include <folly/coro/ViaIfAsync.h>++#include <atomic>+#include <mutex>+#include <type_traits>++#if FOLLY_HAS_COROUTINES++namespace folly {+namespace coro {++/// A mutex that can be locked asynchronously using 'co_await'.+///+/// Ownership of the mutex is not tied to any particular thread.+/// This allows the coroutine owning the lock to transition from one thread+/// to another while holding the lock and then perform the unlock() operation+/// on another thread.+///+/// This mutex guarantees a FIFO scheduling algorithm - coroutines acquire the+/// lock in the order that they execute the 'co_await mutex.co_lock()'+/// operation.+///+/// Note that you cannot use std::scoped_lock/std::lock_guard to acquire the+/// lock as the lock must be acquired with use of 'co_await' which cannot be+/// used in a constructor.+///+/// You can still use the std::scoped_lock/std::lock_guard in conjunction with+/// std::adopt_lock to automatically unlock the mutex when the current scope+/// exits after having locked the mutex using either 'co_await m.co_lock()'+/// or 'm.try_lock()'.+///+/// You can also attempt to acquire the lock using std::unique_lock in+/// conjunction with std::try_to_lock.+///+/// For example:+///   folly::coro::Mutex m;+///   folly::Executor& executor;+///+///   folly::coro::Task<> asyncScopedLockExample()+///   {+///     std::unique_lock<folly::coro::Mutex> lock = co_await m.co_scoped_lock();+///     ...+///   }+///+///   folly::coro::Task<> asyncManualLockAndUnlock()+///   {+///     co_await m.co_lock(executor);+///     ...+///     m.unlock();+///   }+///+///   void nonAsyncTryLock()+///   {+///     if (m.try_lock())+///     {+///       // Once the lock is acquired you can pass ownership of the lock to+///       // a std::lock_guard object.+///       std::lock_guard<folly::coro::Mutex> lock{m, std::adopt_lock};+///       ...+///     }+///   }+///+///   void nonAsyncScopedTryLock()+///   {+///     std::unique_lock<folly::coro::Mutex> lock{m, std::try_to_lock};+///     if (lock)+///     {+///       ...+///     }+///   }+class Mutex {+  class ScopedLockAwaiter;+  class LockAwaiter;+  template <typename Awaiter>+  class LockOperation;++ public:+  /// Construct a new async mutex that is initially unlocked.+  Mutex() noexcept : state_(unlockedState()), waiters_(nullptr) {}++  Mutex(const Mutex&) = delete;+  Mutex(Mutex&&) = delete;+  Mutex& operator=(const Mutex&) = delete;+  Mutex& operator=(Mutex&&) = delete;++  ~Mutex();++  /// Try to lock the mutex synchronously.+  ///+  /// Returns true if the lock was able to be acquired synchronously, false+  /// if the lock could not be acquired because it was already locked.+  ///+  /// If this method returns true then the caller is responsible for ensuring+  /// that unlock() is called to release the lock.+  bool try_lock() noexcept {+    void* oldValue = unlockedState();+    return state_.compare_exchange_strong(+        oldValue,+        nullptr,+        std::memory_order_acquire,+        std::memory_order_relaxed);+  }++  /// Lock the mutex asynchronously, returning an RAII object that will release+  /// the lock at the end of the scope.+  ///+  /// You must co_await the return value to wait until the lock is acquired.+  ///+  /// Chain a call to .viaIfAsync() to specify the executor to resume on when+  /// the lock is eventually acquired in the case that the lock could not be+  /// acquired synchronously. Note that the executor will be passed implicitly+  /// if awaiting from a Task or AsyncGenerator coroutine. The awaiting+  /// coroutine will continue without suspending if the lock could be acquired+  /// synchronously.+  [[nodiscard]] LockOperation<ScopedLockAwaiter> co_scoped_lock() noexcept;++  /// Lock the mutex asynchronously.+  ///+  /// You must co_await the return value to wait until the lock is acquired.+  ///+  /// Chain a call to .viaIfAsync() to specify the executor to resume on when+  /// the lock is eventually acquired in the case that the lock could not be+  /// acquired synchronously. The awaiting coroutine will continue without+  /// suspending if the lock could be acquired synchronously.+  ///+  /// Once the 'co_await m.co_lock()' operation completes, the awaiting+  /// coroutine is responsible for ensuring that .unlock() is called to release+  /// the lock.+  ///+  /// Consider using co_scoped_lock() instead to obtain a std::scoped_lock+  /// that handles releasing the lock at the end of the scope.+  [[nodiscard]] LockOperation<LockAwaiter> co_lock() noexcept;++  /// Unlock the mutex.+  ///+  /// If there are other coroutines waiting to lock the mutex then this will+  /// schedule the resumption of the next coroutine in the queue.+  void unlock() noexcept;++ private:+  using folly_coro_aware_mutex = std::true_type;++  class LockAwaiter {+   public:+    explicit LockAwaiter(Mutex& mutex) noexcept : mutex_(mutex) {}++    bool await_ready() noexcept { return mutex_.try_lock(); }++    bool await_suspend(coroutine_handle<> awaitingCoroutine) noexcept {+      awaitingCoroutine_ = awaitingCoroutine;+      return mutex_.lockAsyncImpl(this);+    }++    void await_resume() noexcept {}++   protected:+    Mutex& mutex_;++   private:+    friend Mutex;++    coroutine_handle<> awaitingCoroutine_;+    LockAwaiter* next_;+  };++  class ScopedLockAwaiter : public LockAwaiter {+   public:+    using LockAwaiter::LockAwaiter;++    std::unique_lock<Mutex> await_resume() noexcept {+      return std::unique_lock<Mutex>{mutex_, std::adopt_lock};+    }+  };++  template <typename Awaiter>+  class LockOperation {+   public:+    explicit LockOperation(Mutex& mutex) noexcept : mutex_(mutex) {}++    auto viaIfAsync(folly::Executor::KeepAlive<> executor) const {+      return folly::coro::co_viaIfAsync(std::move(executor), Awaiter{mutex_});+    }++   private:+    Mutex& mutex_;+  };++  // Special value for state_ that indicates the mutex is not locked.+  void* unlockedState() noexcept { return this; }++  // Try to lock the mutex.+  //+  // Returns true if the lock could not be acquired synchronously and awaiting+  // coroutine should suspend. In this case the coroutine will be resumed later+  // once it acquires the mutex. Returns false if the lock was acquired+  // synchronously and the awaiting coroutine should continue without+  // suspending.+  bool lockAsyncImpl(LockAwaiter* awaiter);++  // This contains either:+  // - this    => Not locked+  // - nullptr => Locked, no newly queued waiters (ie. empty list of waiters)+  // - other   => Pointer to first LockAwaiter* in a linked-list of newly+  //              queued awaiters in LIFO order.+  std::atomic<void*> state_;++  // Linked-list of waiters in FIFO order.+  // Only the current lock holder is allowed to access this member.+  LockAwaiter* waiters_;+};++inline Mutex::LockOperation<Mutex::ScopedLockAwaiter>+Mutex::co_scoped_lock() noexcept {+  return LockOperation<ScopedLockAwaiter>{*this};+}++inline Mutex::LockOperation<Mutex::LockAwaiter> Mutex::co_lock() noexcept {+  return LockOperation<LockAwaiter>{*this};+}++} // namespace coro+} // namespace folly++#endif // FOLLY_HAS_COROUTINES
+ folly/folly/coro/Noexcept.h view
@@ -0,0 +1,299 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <folly/coro/TaskWrapper.h>+#include <folly/coro/ViaIfAsync.h>++#if FOLLY_HAS_COROUTINES++/// ## When to use this+///+/// Use `AsNoexcept<>` only with APIs that only take coroutines that MUST NOT+/// throw when awaited -- like `co_cleanup()` or async scopes.  If your code+/// compiles without `AsNoexcept<>`, you do not need it!+///+/// ## This is probably not the utility you are looking for!+///+///   - This is not related to `co_nothrow`, which is a perf optimization for+///     coroutines with hot exceptions.  `co_await co_nothrow(foo())` means+///     "when `foo()` throws, exit the current coro and pass the exception to+///     whatever is awaiting me".  This saves the ~usec overhead of rethrow.+///+///   - If your project likes to avoid exceptions, that is not a great reason+///     to reflexively make all your coros `AsNoexcept<>`, for these reasons:+///       * Since `AsNoexcept<>` is implemented as a wrapper, it may reduce+///         your build speed.+///       * Coro frame allocation & construction can still throw (unless you+///         also mark the coro function `noexcept`).+///       * Emitting the `noexcept` -> `std::terminate` offramp can sometimes+///         be a pessimization compared to normal exception propagation.+///+///   - We do not provide a helper like `co_await co_fatalOnThrow(...)`, since+///     most callsites should either handle the exception (possibly with+///     `co_awaitTry`), or let it fly.+///+/// ## Why does this even exist, what's wrong with `noexcept`?+///+/// Some `folly::coro` libraries require `AsNoexcept<>` is to clearly signal a+/// **firm contract** between the API and the user-supplied coroutine.  This is+/// only appropriate in situations similar to sync destructors, where the API+/// has no good recourse in case of a thrown exception.+///+/// We need a special wrapper type because marking a coroutine function+/// `noexcept` says nothing about whether awaiting the resulting coroutine can+/// throw.  Rather, it makes fatal any exception thrown during the construction+/// of the coroutine object itself (i.e.  a `bad_alloc` for the frame, or+/// errors copying/moving the args).+///+/// ## How exactly does `AsNoexcept<>` work?+///+/// `Noexcept.h` lets you mark coroutine types as `noexcept_awaitable_v`:+///+///   []() -> AsNoexcept<Task<T>, OnCancel(defaultT())> { co_return ...; }+///+/// This function creates a coroutine whose awaitable is that of the inner+/// task, but wrapped with `detail::NoexceptAwaitable<...>`.+//+/// The latter is an awaitable-wrapper similar to `co_awaitTry()`, except that+/// it terminates the program if `someAwaitable()` resumes with a thrown+/// exception.  So, both of these will never throw, but the former returns a+/// `Try` while latter returns an unwrapped value:+///+///   co_await co_awaitTry(intTask())  // `Try<int>`+///   co_await detail::NoexceptAwaitable<int, OnCancel(0)>{intTask()} // `int`+///+/// Both the coroutine `AsNoexcept<Task<...>, ...>` and the preceding 2+/// awaitables return `true` for `noexcept_awaitable_v`.+///+/// `AsNoexcept<>` / `NoexceptAwaitable<>` compose properly with other coro-+/// and awaitable-wrappers.  But, not all combinations make sense -- see the+/// test, and/or extend it if needed.  For example, the outer wrapper is+/// useless in `NoexceptAwaitable<...>(co_awaitTry(...))`, since exceptions+/// would already have been routed into a `Try`.++namespace folly::coro {++struct TerminateOnCancel {};+inline constexpr TerminateOnCancel terminateOnCancel{};++template <typename T>+struct OnCancel {+  T privateVal_; // only `public` to make this a structural type+  T onCancelDefaultValue() const noexcept {+    static_assert(std::is_nothrow_copy_constructible_v<T>);+    return privateVal_;+  }+  consteval explicit OnCancel(T t) : privateVal_{std::move(t)} {}+};++template <>+struct OnCancel<void> {+  void onCancelDefaultValue() const noexcept {}+  consteval explicit OnCancel() = default;+};++namespace detail {++template <typename Awaitable, auto CancelCfg>+class NoexceptAwaiter {+ private:+  using Awaiter = awaiter_type_t<Awaitable>;+  Awaiter awaiter_;++ public:+  explicit NoexceptAwaiter(Awaitable&& awaiter)+      : awaiter_(get_awaiter(static_cast<Awaitable&&>(awaiter))) {}++  auto await_ready() noexcept -> decltype(awaiter_.await_ready()) {+    // As of this writing, all `await_ready` in `folly::coro` are `noexcept`.+    // If this is legitimately triggered, then we can decide the right policy.+    static_assert(noexcept(awaiter_.await_ready()));+    return awaiter_.await_ready();+  }++  // `noexcept` forces any rethrown exceptions to `std::terminate`+  auto await_resume() noexcept -> decltype(awaiter_.await_resume()) {+    if constexpr (std::is_same_v<decltype(CancelCfg), TerminateOnCancel>) {+      return awaiter_.await_resume();+    } else {+      try {+        return awaiter_.await_resume();+      } catch (const OperationCancelled&) {+        // IMPORTANT: If you want to extend this protocol to pull out a default+        // value from the awaiter, be sure to add this assert:+        // static_assert(noexcept(CancelCfg.onCancelDefaultValue(awaiter_)));+        return CancelCfg.onCancelDefaultValue();+      }+    }+  }++  // `noexcept` here as well, because the underlying awaitable might+  // have a throwing `await_suspend`, and those exceptions propagate+  // to the parent coro promise, bypassing `await_resume`.+  // Demo: https://godbolt.org/z/Edfj8P8be+  template <typename Promise>+  auto await_suspend(coroutine_handle<Promise> coro) noexcept+      -> decltype(awaiter_.await_suspend(coro)) {+    return awaiter_.await_suspend(coro);+  }+};++template <typename, auto>+class NoexceptAwaitable;++template <auto CancelCfg>+struct NoexceptAwaitableWithCancelCfg {+  template <typename T>+  using apply = NoexceptAwaitable<T, CancelCfg>;+};++template <typename T, auto CancelCfg>+class [[FOLLY_ATTR_CLANG_CORO_AWAIT_ELIDABLE]] NoexceptAwaitable+    : public CommutativeWrapperAwaitable<+          NoexceptAwaitableWithCancelCfg<CancelCfg>::template apply,+          T> {+ public:+  using CommutativeWrapperAwaitable<+      NoexceptAwaitableWithCancelCfg<CancelCfg>::template apply,+      T>::CommutativeWrapperAwaitable;++  template <typename T2 = T, std::enable_if_t<is_awaitable_v<T2>, int> = 0>+  NoexceptAwaiter<T, CancelCfg> operator co_await() && {+    return NoexceptAwaiter<T, CancelCfg>{std::move(this->inner_)};+  }++  using folly_private_noexcept_awaitable_t = std::true_type;+};++} // namespace detail++#if FOLLY_HAS_IMMOVABLE_COROUTINES++template <typename Inner, auto CancelCfg>+class AsNoexcept;+// NB: While it'd be prettier to have `AsNoexcept` branch on whether the inner+// task has an executor, a separate template is much simpler.+template <typename Inner, auto CancelCfg>+class AsNoexceptWithExecutor;++namespace detail {+template <typename Inner, auto CancelCfg>+struct AsNoexceptWithExecutorCfg {+  using InnerTaskWithExecutorT = Inner;+  using WrapperTaskT = AsNoexcept<+      typename Inner::folly_private_task_without_executor_t,+      CancelCfg>;+  template <typename Awaitable> // library-internal, meant to be by-rref+  static inline auto wrapAwaitable(Awaitable&& awaitable) noexcept {+    // Assert can be removed, I was concerned if we accidentally double-wrap+    static_assert(!noexcept_awaitable_v<Awaitable>);+    return detail::NoexceptAwaitable<Awaitable, CancelCfg>{+        mustAwaitImmediatelyUnsafeMover(static_cast<Awaitable&&>(awaitable))()};+  }+};+template <typename Inner, auto CancelCfg>+using AsNoexceptWithExecutorBase = TaskWithExecutorWrapperCrtp<+    AsNoexceptWithExecutor<Inner, CancelCfg>,+    AsNoexceptWithExecutorCfg<Inner, CancelCfg>>;+} // namespace detail++template <typename Inner, auto CancelCfg = OnCancel<void>{}>+class FOLLY_NODISCARD AsNoexceptWithExecutor final+    : public detail::AsNoexceptWithExecutorBase<Inner, CancelCfg> {+ protected:+  using detail::AsNoexceptWithExecutorBase<Inner, CancelCfg>::+      AsNoexceptWithExecutorBase;++ public:+  using folly_private_noexcept_awaitable_t = std::true_type;+};++namespace detail {++template <typename... BaseArgs>+class AsNoexceptTaskPromiseWrapper final+    : public TaskPromiseWrapper<BaseArgs...> {};++template <typename Inner, auto CancelCfg>+struct AsNoexceptCfg {+  using ValueT = semi_await_result_t<Inner>;+  using InnerTaskT = Inner;+  using TaskWithExecutorT = AsNoexceptWithExecutor<+      decltype(co_withExecutor(+          FOLLY_DECLVAL(Executor::KeepAlive<>), FOLLY_DECLVAL(Inner))),+      CancelCfg>;+  using PromiseT = AsNoexceptTaskPromiseWrapper<+      ValueT,+      AsNoexcept<Inner, CancelCfg>,+      typename folly::coro::coroutine_traits<Inner>::promise_type>;+  template <typename Awaitable> // library-internal, meant to be by-rref+  static inline auto wrapAwaitable(Awaitable&& awaitable) noexcept {+    // Assert can be removed, I was concerned if we accidentally double-wrap+    static_assert(!noexcept_awaitable_v<Awaitable>);+    return detail::NoexceptAwaitable<Awaitable, CancelCfg>{+        static_cast<Awaitable&&>(awaitable)};+  }+};++template <typename Inner, auto CancelCfg>+using AsNoexceptBase = TaskWrapperCrtp<+    AsNoexcept<Inner, CancelCfg>,+    AsNoexceptCfg<Inner, CancelCfg>>;++// CAUTION: `as_noexcept_rewrapper` gives you the power to wrap and unwrap+// `AsNoexcept`, so you must be extremely careful to preserve behavior:+//   - The unwrapped task must be rewrapped before awaiting.+//   - You must not wrap any other task.++template <typename>+struct as_noexcept_rewrapper {+  static inline constexpr bool as_noexcept_wrapped = false;+  static auto wrap_with(auto fn) { return fn(); }+};++template <typename Inner, auto Cfg>+struct as_noexcept_rewrapper<AsNoexcept<Inner, Cfg>> {+  static inline constexpr bool as_noexcept_wrapped = true;+  static Inner unwrapTask(AsNoexcept<Inner, Cfg>&& t) {+    return std::move(t).unwrapTask();+  }+  static auto wrap_with(auto fn) {+    return AsNoexcept<decltype(fn()), Cfg>{fn()};+  }+};++} // namespace detail++template <typename Inner, auto CancelCfg = OnCancel<void>{}>+class FOLLY_CORO_TASK_ATTRS AsNoexcept final+    : public detail::AsNoexceptBase<Inner, CancelCfg> {+ protected:+  using detail::AsNoexceptBase<Inner, CancelCfg>::AsNoexceptBase;++  template <typename> // Can unwrap and re-wrap (construct)+  friend struct detail::as_noexcept_rewrapper;++ public:+  using folly_private_noexcept_awaitable_t = std::true_type;+};++#endif // FOLLY_HAS_IMMOVABLE_COROUTINES++} // namespace folly::coro++#endif // FOLLY_HAS_COROUTINES
+ folly/folly/coro/Promise.h view
@@ -0,0 +1,334 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <utility>++#include <folly/CancellationToken.h>+#include <folly/Try.h>+#include <folly/coro/Baton.h>+#include <folly/coro/Coroutine.h>+#include <folly/futures/Promise.h>+#include <folly/lang/SafeAlias-fwd.h>+#include <folly/synchronization/RelaxedAtomic.h>++#if FOLLY_HAS_COROUTINES++namespace folly::coro {+template <typename T>+class Promise;+template <typename T>+class Future;++// Creates promise and associated unfulfilled future+template <typename T>+std::pair<Promise<T>, Future<T>> makePromiseContract();++// Creates fulfilled future+template <typename T>+Future<remove_cvref_t<T>> makeFuture(T&&);+template <typename T>+Future<T> makeFuture(exception_wrapper&&);+Future<void> makeFuture();++namespace detail {+template <typename T>+struct PromiseState {+  PromiseState() = default;++  Try<T> result;+  // Must be exchanged to true before setting result+  folly::relaxed_atomic<bool> fulfilled{false};+  // Must be posted after setting result+  coro::Baton ready;+};+} // namespace detail++template <typename T>+class Promise {+ public:+  /**+   * Construct an empty Promise.+   *+   * This object is not valid use until you initialize it with move assignment.+   */+  Promise() = default;++  Promise(Promise&& other) noexcept+      : ct_(std::move(other.ct_)),+        state_(std::exchange(other.state_, nullptr)) {}+  Promise& operator=(Promise&& other) noexcept {+    if (this != &other && state_ && !state_->fulfilled) {+      setException(BrokenPromise{tag<T>});+    }+    ct_ = std::move(other.ct_);+    state_ = std::exchange(other.state_, nullptr);+    return *this;+  }+  Promise(const Promise&) = delete;+  Promise& operator=(const Promise&) = delete;++  ~Promise() {+    if (state_ && !state_->fulfilled) {+      setException(BrokenPromise{tag<T>});+    }+  }++  bool valid() const noexcept { return state_; }++  bool isFulfilled() const noexcept { return state_ && state_->fulfilled; }++  template <typename... Args>+  void setValue(Args&&... args) {+    trySetValue(std::forward<Args>(args)...);+  }++  template <typename... Args>+  void setException(Args&&... args) {+    trySetException(std::forward<Args>(args)...);+  }++  void setResult(Try<T>&& result) { trySetResult(std::move(result)); }++  /**+   * Fulfills the promise with a value if not already fulfilled.+   * @returns Whether the fulfillment took place.+   */+  template <typename... Args>+  bool trySetValue(Args&&... args) {+    DCHECK(state_);+    if (state_->fulfilled.exchange(true)) {+      return false;+    }+    if constexpr (std::is_void_v<T>) {+      static_assert(sizeof...(Args) == 0);+    } else {+      state_->result.emplace(std::forward<Args>(args)...);+    }+    state_->ready.post();+    return true;+  }++  /**+   * Fulfills the promise with an exception if not already fulfilled.+   * @returns Whether the fulfillment took place.+   */+  template <typename... Args>+  bool trySetException(Args&&... args) {+    DCHECK(state_);+    if (state_->fulfilled.exchange(true)) {+      return false;+    }+    state_->result.emplaceException(std::forward<Args>(args)...);+    state_->ready.post();+    return true;+  }++  /**+   * Fulfills the promise with a Try if not already fulfilled.+   * @returns Whether the fulfillment took place.+   */+  bool trySetResult(Try<T>&& result) {+    DCHECK(state_);+    if (state_->fulfilled.exchange(true)) {+      return false;+    }+    state_->result = std::move(result);+    state_->ready.post();+    return true;+  }++  /**+   * Fulfills the promise with a value/Try returned from calling func if not+   * already fulfilled.+   *+   * If either the call to func or the result's constructor completes with an+   * exception then the exception is caught and stored as the result.+   *+   * @returns Whether the fulfillment took place.+   */+  template <typename Func>+  bool trySetWith(Func&& func) {+    DCHECK(state_);+    if (state_->fulfilled.exchange(true)) {+      return false;+    }+    try {+      state_->result = Try<T>(std::forward<Func>(func)());+    } catch (...) {+      state_->result.emplaceException(current_exception());+    }+    state_->ready.post();+    return true;+  }++  /**+   * Fulfills the promise with an exception returned from calling func if not+   * already fulfilled.+   *+   * If either the call to func or the result's constructor completes with an+   * exception then the exception is caught and stored as the result.+   *+   * @returns Whether the fulfillment took place.+   */+  template <typename Func>+  bool trySetExceptionWith(Func&& func) {+    DCHECK(state_);+    if (state_->fulfilled.exchange(true)) {+      return false;+    }+    try {+      state_->result.emplaceException(std::forward<Func>(func)());+    } catch (...) {+      state_->result.emplaceException(current_exception());+    }+    state_->ready.post();+    return true;+  }++  const CancellationToken& getCancellationToken() const { return ct_; }++ private:+  Promise(CancellationToken ct, detail::PromiseState<T>& state)+      : ct_(std::move(ct)), state_(&state) {}++  CancellationToken ct_;+  detail::PromiseState<T>* state_{nullptr};++  friend std::pair<Promise<T>, Future<T>> makePromiseContract<T>();+};++template <typename T>+class Future {+ public:+  /**+   * Construct an empty Future.+   *+   * This object is not valid use until you initialize it with move assignment.+   */+  Future() = default;++  Future(Future&&) noexcept = default;+  Future& operator=(Future&&) noexcept = default;+  Future(const Future&) = delete;+  Future& operator=(const Future&) = delete;++  class WaitOperation : private Baton::WaitOperation {+   public:+    explicit WaitOperation(Future& future) noexcept+        : Baton::WaitOperation(future.state_->ready),+          future_(future),+          cb_(std::move(future.ct_), [&] { future_.cancel(); }) {}++    using Baton::WaitOperation::await_ready;+    using Baton::WaitOperation::await_suspend;++    T await_resume() {+      if constexpr (!std::is_void_v<T>) {+        return std::move(future_.state_->result.value());+      } else {+        future_.state_->result.throwIfFailed();+      }+    }++    folly::Try<T> await_resume_try() {+      return std::move(future_.state_->result);+    }++   private:+    Future& future_;+    CancellationCallback cb_;+  };++  [[nodiscard]] WaitOperation operator co_await() && noexcept {+    return WaitOperation{*this};+  }++  bool isReady() const noexcept { return state_->ready.ready(); }++  friend Future co_withCancellation(+      folly::CancellationToken ct, Future&& future) noexcept {+    if (!std::exchange(future.hasCancelTokenOverride_, true)) {+      future.ct_ = std::move(ct);+    }+    return std::move(future);+  }++  using folly_private_safe_alias_t = safe_alias_of<T>;++ private:+  Future(CancellationSource cs, detail::PromiseState<T>& state)+      : cs_(std::move(cs)), state_(&state) {}++  void cancel() {+    if (!state_->fulfilled.exchange(true)) {+      cs_.requestCancellation();+      state_->result.emplaceException(OperationCancelled{});+      state_->ready.post();+    }+  }++  CancellationSource cs_;+  detail::PromiseState<T>* state_{nullptr};+  // The token inherited when the future is awaited+  CancellationToken ct_;+  bool hasCancelTokenOverride_{false};++  friend std::pair<Promise<T>, Future<T>> makePromiseContract<T>();+};++/**+ * makePromiseContract can help you migrating your non-coroutine code base to+ * coroutine. If your code already uses Future/SemiFuture, you don't need this+ * tool. A common use case is with async callback functions. In the example, we+ * can pass a callback function into the legacy code sleepAndNotify and+ * sleepAndNotify sets the promise on completion. Consider to use detachOnCancel+ * with this makePromiseContract to handle long running (longer than your+ * timeout) tasks that don't handle cancellation properly.+ *+ * \refcode folly/docs/examples/folly/coro/Promise.cpp+ */+template <typename T>+std::pair<Promise<T>, Future<T>> makePromiseContract() {+  auto [cs, data] = CancellationSource::create(+      folly::detail::WithDataTag<detail::PromiseState<T>>{});+  return {+      Promise<T>{cs.getToken(), std::get<0>(*data)},+      Future<T>{std::move(cs), std::get<0>(*data)}};+}++template <typename T>+Future<remove_cvref_t<T>> makeFuture(T&& t) {+  auto [promise, future] = makePromiseContract<remove_cvref_t<T>>();+  promise.setValue(std::forward<T>(t));+  return std::move(future);+}+template <typename T>+Future<T> makeFuture(exception_wrapper&& ex) {+  auto [promise, future] = makePromiseContract<T>();+  promise.setException(std::move(ex));+  return std::move(future);+}+inline Future<void> makeFuture() {+  auto [promise, future] = makePromiseContract<void>();+  promise.setValue();+  return std::move(future);+}++} // namespace folly::coro++#endif
+ folly/folly/coro/Ready.h view
@@ -0,0 +1,114 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <folly/Executor.h>+#include <folly/coro/Result.h>+#include <folly/coro/WithAsyncStack.h>++/// Use `co_ready` to "await" synchronous coroutine types from inside async+/// coroutines like `coro::Task`. For example:+///+///   result<int> getN();+///   int n = co_await co_ready(getN());+///+/// Also see `co_await_result` (`AwaitResult.h`) and `co_result` (`Result.h`).+///+/// If you need to optimize away ALL exception throwing in **async** code,+/// `co_ready` is not your top choice.  In a `Task` coro:+///+///   auto v = co_await co_nothrow(asyncMayError()); // best practice+///   auto v = co_await co_ready(co_await_result(asyncMayError())); // too long+///+/// However, when you are calling synchronous `result` functions, or need to+/// efficiently handle **some** async errors, `co_ready` is your friend:+///+///   auto res = syncResultFn(); // or `co_await co_await_result(asyncFn())`+///   if (auto* ex = get_exception<MyError>(res)) {+///     /* handle ex */+///   } else {+///     auto v = co_await co_ready(std::move(res)); // propagate unhandled+///   }+///+/// This pattern has a few good properties:+///   - Easy error handling -- extracts the value from its argument, or+///     short-circuit any error to current coro's awaiter.+///   - Unlike `catch (const std::exception& ex)`, won't catch (and therefore+///     break) cancellation.+///   - The error path is MUCH more efficient (3-30 nanoseconds) than+///     `value_or_throw()` (1 microsecond).+///+/// We don't support `co_await syncResultFn()` to avoids confusion about which+/// parts of the code are sync vs async.  The distinction is critical, since+/// one must not hold non-coro mutexes across async suspend points.+///+/// Future:+///   - Adding `std::ref` / `std::cref`, and possibly `folly::rref` variants of+///     this (as in `result.h`) might improve performance in hot code.+///   - The current implementation is `result`-only.  If you have a need, it+///     would be fine to add the analogous specialization for `Try`.  Just be+///     mindful of its two warts: empty state and empty `exception_wrapper`.++namespace folly::coro {++template <typename>+class co_ready;++#if FOLLY_HAS_RESULT++template <typename T>+class co_ready<result<T>> {+ private:+  result<T> res_;++ public:+  explicit co_ready(result<T>&& res) : res_(std::move(res)) {}++  bool await_ready() const noexcept { return res_.has_value(); }++  auto await_resume() noexcept -> decltype(std::move(res_).value_or_throw()) {+    return std::move(res_).value_or_throw();+  }++  template <typename Promise>+  auto await_suspend(+      std::coroutine_handle<Promise> awaitingCoroutine) noexcept {+    auto& promise = awaitingCoroutine.promise();+    // We have to use the legacy API because (1) `folly::coro` internals still+    // model cancellation as an exception, (2) to use `co_cancelled` here we'd+    // have to check `res_` for `OperationCancelled` which can cost 50-100ns+.+    auto awaiter = promise.yield_value(co_error(+        std::move(res_).non_value().get_legacy_error_or_cancellation()));+    return awaiter.await_suspend(awaitingCoroutine);+  }++  friend auto co_viaIfAsync(+      const Executor::KeepAlive<>&, co_ready&& r) noexcept {+    return std::move(r);+  }++  friend auto tag_invoke(cpo_t<co_withAsyncStack>, co_ready&& r) noexcept {+    return std::move(r);+  }+};++template <typename T>+co_ready(result<T>&&) -> co_ready<result<T>>;++#endif // FOLLY_HAS_RESULT++} // namespace folly::coro
+ folly/folly/coro/Result.h view
@@ -0,0 +1,92 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <cassert>+#include <type_traits>++#include <folly/ExceptionWrapper.h>+#include <folly/OperationCancelled.h>+#include <folly/Try.h>+#include <folly/result/result.h>++namespace folly {+namespace coro {++class co_error final {+ public:+  template <+      typename... A,+      std::enable_if_t<+          sizeof...(A) && std::is_constructible<exception_wrapper, A...>::value,+          int> = 0>+  explicit co_error(A&&... a) noexcept(+      std::is_nothrow_constructible<exception_wrapper, A...>::value)+      : ex_(static_cast<A&&>(a)...) {+    assert(ex_);+  }++  const exception_wrapper& exception() const { return ex_; }++  exception_wrapper& exception() { return ex_; }++ private:+  exception_wrapper ex_;+};++template <typename T>+class co_result final {+ public:+  explicit co_result(Try<T>&& result) noexcept(+      std::is_nothrow_move_constructible<T>::value)+      : result_(std::move(result)) {+    assert(!result_.hasException() || result_.exception());+  }++#if FOLLY_HAS_RESULT+  // Covered in `AwaitResultTest.cpp`, unlike the rest of this file, which is+  // covered in `TaskTest.cpp`.+  template <std::same_as<folly::result<T>> U> // no implicit ctors for `result`+  explicit co_result(U result) noexcept(+      std::is_nothrow_move_constructible<T>::value)+      : co_result(result_to_try(std::move(result))) {}+#endif++  const Try<T>& result() const { return result_; }++  Try<T>& result() { return result_; }++ private:+  Try<T> result_;+};++#if FOLLY_HAS_RESULT+template <typename T>+co_result(result<T>) -> co_result<T>;+#endif++class co_cancelled_t final {+ public:+  /* implicit */ operator co_error() const {+    return co_error(OperationCancelled{});+  }+};++inline constexpr co_cancelled_t co_cancelled{};++} // namespace coro+} // namespace folly
+ folly/folly/coro/Retry.h view
@@ -0,0 +1,336 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++//+// Docs: https://fburl.com/fbcref_coro_retry+//++#pragma once++#include <folly/CancellationToken.h>+#include <folly/ConstexprMath.h>+#include <folly/ExceptionWrapper.h>+#include <folly/Random.h>+#include <folly/Try.h>+#include <folly/coro/Coroutine.h>+#include <folly/coro/Result.h>+#include <folly/coro/Sleep.h>+#include <folly/coro/Task.h>+#include <folly/coro/Traits.h>++#include <cstdint>+#include <random>+#include <utility>++#if FOLLY_HAS_COROUTINES++/**+ * \file coro/Retry.h+ *+ * Coroutine implementation of futures/Retrying.h+ *+ * This file provides utility functions (with building blocks) to build retry+ * logic. There are three function families:+ * - retryWhen: try to a func (which produces an awaitable); if fail (with an+ * non folly::OperationCancelled exception), then wait on a delay func (which+ * produces an awaitable too). This retry logic can run **forever**, so it is+ * not recommended to use it directly. This the auxiliary function to help build+ * retryN and retryWithExponentialBackoff.+ * - retryN: try the func with limited N times+ * - retryWithExponentialBackoff: the retries will be restarted with+ * exponiential backoff.+ *+ * \refcode folly/docs/examples/folly/coro/Retry.cpp+ */++namespace folly::coro {++/// Execute a given asynchronous operation returned by func(),+/// retrying it on failure, if desired, after awaiting+/// retryDelay(error).+///+/// If 'func()' operation succeeds or completes with OperationCancelled+/// then completes immediately with that result.+///+/// Otherwise, if it fails with an error then the function+/// 'retryDelay()' is invoked with the exception_wrapper for+/// the error and must return another Task<void>.+///+/// If this task completes successfully or completes with then it will retry+/// the func() operation, otherwise if it completes with an+/// error then the whole operation will complete with that error.+///+/// This allows you to do some asynchronous work between retries (such as+/// sleeping for a given duration, but could be some reparatory work in+/// response to particular errors) and the retry will be scheduled once+/// the retryDelay() operation completes successfully.+template <typename Func, typename RetryDelayFunc>+auto retryWhen(Func func, RetryDelayFunc retryDelay)+    -> Task<semi_await_result_t<invoke_result_t<Func&>>> {+  while (true) {+    exception_wrapper error;+    try {+      auto result = co_await folly::coro::co_awaitTry(func());+      if (result.hasValue()) {+        co_return std::move(result).value();+      } else {+        assert(result.hasException());+        error = std::move(result.exception());+      }+    } catch (...) {+      error = exception_wrapper(current_exception());+    }++    if (error.is_compatible_with<folly::OperationCancelled>()) {+      co_yield folly::coro::co_error(std::move(error));+    }++    Try<void> retryResult =+        co_await folly::coro::co_awaitTry(retryDelay(std::move(error)));+    if (retryResult.hasException()) {+      /// Failure (or cancellation) of retryDelay() indicates we should stop+      /// retrying.+      co_yield folly::coro::co_error(std::move(retryResult.exception()));+    }++    /// Otherwise we go around the loop again.+  }+}++namespace detail {++template <typename Decider>+class RetryImmediatelyWithLimit {+ public:+  template <typename Decider2>+  explicit RetryImmediatelyWithLimit(+      uint32_t maxRetries, Decider2&& decider) noexcept+      : retriesRemaining_(maxRetries),+        decider_(static_cast<Decider2&&>(decider)) {}++  Task<void> operator()(exception_wrapper&& ew) & {+    if (retriesRemaining_ == 0 || !decider_(ew)) {+      co_yield folly::coro::co_error(std::move(ew));+    }++    const auto& cancelToken = co_await co_current_cancellation_token;+    if (cancelToken.isCancellationRequested()) {+      co_yield folly::coro::co_error(OperationCancelled{});+    }++    --retriesRemaining_;+  }++ private:+  uint32_t retriesRemaining_;+  Decider decider_;+};++struct AlwaysRetry {+  bool operator()(const folly::exception_wrapper&) const noexcept {+    return true;+  }+};++} // namespace detail++/// Executes the operation returned by func(), retrying it up to+/// 'maxRetries' times on failure with no delay between retries.+template <typename Func, typename Decider>+auto retryN(uint32_t maxRetries, Func&& func, Decider&& decider) {+  return folly::coro::retryWhen(+      static_cast<Func&&>(func),+      detail::RetryImmediatelyWithLimit<remove_cvref_t<Decider>>{+          maxRetries, static_cast<Decider&&>(decider)});+}++template <typename Func>+auto retryN(uint32_t maxRetries, Func&& func) {+  return folly::coro::retryN(+      maxRetries, static_cast<Func&&>(func), detail::AlwaysRetry{});+}++namespace detail {++template <typename URNG, typename Decider>+class ExponentialBackoffWithJitter {+ public:+  template <typename URNG2, typename Decider2>+  explicit ExponentialBackoffWithJitter(+      Timekeeper* tk,+      uint32_t maxRetries,+      Duration minBackoff,+      Duration maxBackoff,+      double relativeJitterStdDev,+      URNG2&& rng,+      Decider2&& decider) noexcept+      : timeKeeper_(tk),+        maxRetries_(maxRetries),+        retryCount_(0),+        minBackoff_(minBackoff),+        maxBackoff_(maxBackoff),+        relativeJitterStdDev_(relativeJitterStdDev),+        randomGen_(static_cast<URNG2&&>(rng)),+        decider_(static_cast<Decider2&&>(decider)) {}++  Task<void> operator()(exception_wrapper&& ew) & {+    using dist = std::normal_distribution<double>;++    if (retryCount_ == maxRetries_ || !decider_(ew)) {+      co_yield folly::coro::co_error(std::move(ew));+    }++    ++retryCount_;++    /// The jitter will be a value between [e^-stdev]+    const auto jitter = relativeJitterStdDev_ > 0+        ? std::exp(dist{0., relativeJitterStdDev_}(randomGen_))+        : 1.;+    // TODO T186551522 Calculate backoff in microseconds.+    const auto backoffNominal =+        Duration(folly::constexpr_clamp_cast<Duration::rep>(+            jitter * minBackoff_.count() * std::pow(2, retryCount_ - 1u)));++    const Duration backoff = std::clamp(+        backoffNominal, minBackoff_, std::max(minBackoff_, maxBackoff_));++    co_await folly::coro::sleep(backoff, timeKeeper_);++    /// Check to see if we were cancelled during the sleep.+    const auto& cancelToken = co_await co_current_cancellation_token;+    if (cancelToken.isCancellationRequested()) {+      co_yield folly::coro::co_cancelled;+    }+  }++ private:+  Timekeeper* timeKeeper_;+  const uint32_t maxRetries_;+  uint32_t retryCount_;+  const Duration minBackoff_;+  const Duration maxBackoff_;+  const double relativeJitterStdDev_;+  URNG randomGen_;+  Decider decider_;+};++} // namespace detail++/// Executes the operation returned from 'func()', retrying it on failure+/// up to 'maxRetries' times, with an exponential backoff, doubling the backoff+/// on average for each retry, applying some random jitter, up to the specified+/// maximum backoff, passing each error to decider to decide whether to retry or+/// not.+template <typename Func, typename URNG, typename Decider>+auto retryWithExponentialBackoff(+    uint32_t maxRetries,+    Duration minBackoff,+    Duration maxBackoff,+    double relativeJitterStdDev,+    Timekeeper* timeKeeper,+    URNG&& rng,+    Func&& func,+    Decider&& decider) {+  return folly::coro::retryWhen(+      static_cast<Func&&>(func),+      detail::ExponentialBackoffWithJitter<+          remove_cvref_t<URNG>,+          remove_cvref_t<Decider>>{+          timeKeeper,+          maxRetries,+          minBackoff,+          maxBackoff,+          relativeJitterStdDev,+          static_cast<URNG&&>(rng),+          static_cast<Decider&&>(decider)});+}++template <typename Func, typename URNG>+auto retryWithExponentialBackoff(+    uint32_t maxRetries,+    Duration minBackoff,+    Duration maxBackoff,+    double relativeJitterStdDev,+    Timekeeper* timeKeeper,+    URNG&& rng,+    Func&& func) {+  return folly::coro::retryWithExponentialBackoff(+      maxRetries,+      minBackoff,+      maxBackoff,+      relativeJitterStdDev,+      timeKeeper,+      static_cast<URNG&&>(rng),+      static_cast<Func&&>(func),+      detail::AlwaysRetry{});+}++template <typename Func>+auto retryWithExponentialBackoff(+    uint32_t maxRetries,+    Duration minBackoff,+    Duration maxBackoff,+    double relativeJitterStdDev,+    Timekeeper* timeKeeper,+    Func&& func) {+  return folly::coro::retryWithExponentialBackoff(+      maxRetries,+      minBackoff,+      maxBackoff,+      relativeJitterStdDev,+      timeKeeper,+      ThreadLocalPRNG(),+      static_cast<Func&&>(func));+}++template <typename Func>+auto retryWithExponentialBackoff(+    uint32_t maxRetries,+    Duration minBackoff,+    Duration maxBackoff,+    double relativeJitterStdDev,+    Func&& func) {+  return folly::coro::retryWithExponentialBackoff(+      maxRetries,+      minBackoff,+      maxBackoff,+      relativeJitterStdDev,+      static_cast<Timekeeper*>(nullptr),+      static_cast<Func&&>(func));+}++template <typename Func, typename Decider>+auto retryWithExponentialBackoff(+    uint32_t maxRetries,+    Duration minBackoff,+    Duration maxBackoff,+    double relativeJitterStdDev,+    Func&& func,+    Decider&& decider) {+  return folly::coro::retryWithExponentialBackoff(+      maxRetries,+      minBackoff,+      maxBackoff,+      relativeJitterStdDev,+      static_cast<Timekeeper*>(nullptr),+      ThreadLocalPRNG(),+      static_cast<Func&&>(func),+      static_cast<Decider&&>(decider));+}++} // namespace folly::coro++#endif // FOLLY_HAS_COROUTINES
+ folly/folly/coro/RustAdaptors.h view
@@ -0,0 +1,186 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <folly/CancellationToken.h>+#include <folly/Executor.h>+#include <folly/Optional.h>+#include <folly/coro/AsyncGenerator.h>+#include <folly/coro/Task.h>+#include <folly/futures/Future.h>+#include <folly/synchronization/Baton.h>++#if FOLLY_HAS_COROUTINES++namespace folly {+namespace coro {++template <typename T>+class PollFuture final : private Executor {+ public:+  using Poll = Optional<lift_unit_t<T>>;+  using Waker = Function<void()>;++  explicit PollFuture(Task<T> task) {+    Executor* self = this;+    co_withExecutor(makeKeepAlive(self), std::move(task))+        .start(+            [&](Try<T>&& result) noexcept {+              // Rust doesn't support exceptions+              DCHECK(!result.hasException());+              if constexpr (!std::is_same_v<T, void>) {+                result_ = std::move(result).value();+              } else {+                result_ = unit;+              }+            },+            cancellationSource_.getToken());+  }++  explicit PollFuture(SemiFuture<lift_unit_t<T>> future) {+    Executor* self = this;+    std::move(future)+        .via(makeKeepAlive(self))+        .setCallback_([&](Executor::KeepAlive<>&&, Try<T>&& result) mutable {+          result_ = std::move(result).value();+        });+  }++  ~PollFuture() override {+    cancellationSource_.requestCancellation();+    if (keepAliveCount_.load(std::memory_order_relaxed) > 0) {+      folly::Baton<> b;+      while (!poll([&] { b.post(); })) {+        b.wait();+        b.reset();+      }+    }+  }++  Poll poll(Waker waker) {+    while (true) {+      std::queue<Func> funcs;+      {+        auto wQueueAndWaker = queueAndWaker_.wlock();+        if (wQueueAndWaker->funcs.empty()) {+          wQueueAndWaker->waker = std::move(waker);+          break;+        }++        std::swap(funcs, wQueueAndWaker->funcs);+      }++      while (!funcs.empty()) {+        funcs.front()();+        funcs.pop();+      }+    }++    if (keepAliveCount_.load(std::memory_order_relaxed) == 0) {+      return std::move(result_);+    }+    return none;+  }++ private:+  void add(Func func) override {+    auto waker = [&] {+      auto wQueueAndWaker = queueAndWaker_.wlock();+      wQueueAndWaker->funcs.push(std::move(func));+      return std::exchange(wQueueAndWaker->waker, {});+    }();+    if (waker) {+      waker();+    }+  }++  bool keepAliveAcquire() noexcept override {+    auto keepAliveCount =+        keepAliveCount_.fetch_add(1, std::memory_order_relaxed);+    DCHECK(keepAliveCount > 0);+    return true;+  }++  void keepAliveRelease() noexcept override {+    auto keepAliveCount = keepAliveCount_.load(std::memory_order_relaxed);+    do {+      DCHECK(keepAliveCount > 0);+      if (keepAliveCount == 1) {+        add([this] {+          // the final count *must* be released from this executor so that we+          // don't race with poll.+          keepAliveCount_.fetch_sub(1, std::memory_order_relaxed);+        });+        return;+      }+    } while (!keepAliveCount_.compare_exchange_weak(+        keepAliveCount,+        keepAliveCount - 1,+        std::memory_order_release,+        std::memory_order_relaxed));+  }++  struct QueueAndWaker {+    std::queue<Func> funcs;+    Waker waker;+  };+  Synchronized<QueueAndWaker> queueAndWaker_;+  std::atomic<ssize_t> keepAliveCount_{1};+  Optional<lift_unit_t<T>> result_;+  CancellationSource cancellationSource_;+};++template <typename T>+class PollStream {+ public:+  using Poll = Optional<Optional<T>>;+  using Waker = Function<void()>;++  explicit PollStream(AsyncGenerator<T> asyncGenerator)+      : asyncGenerator_(std::move(asyncGenerator)) {}++  Poll poll(Waker waker) {+    if (!nextFuture_) {+      nextFuture_.emplace(getNext());+    }++    auto nextPoll = nextFuture_->poll(std::move(waker));+    if (!nextPoll) {+      return none;+    }++    nextFuture_.reset();+    return nextPoll;+  }++ private:+  Task<Optional<T>> getNext() {+    auto next = co_await asyncGenerator_.next();+    if (next) {+      co_return std::move(next).value();+    }+    co_return none;+  }++  AsyncGenerator<T> asyncGenerator_;+  Optional<PollFuture<Optional<T>>> nextFuture_;+};++} // namespace coro+} // namespace folly++#endif // FOLLY_HAS_COROUTINES
+ folly/folly/coro/ScopeExit.h view
@@ -0,0 +1,362 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++//+// Docs: https://fburl.com/fbcref_coro_scopeexit+//++#pragma once++#include <folly/tracing/AsyncStack.h>++#include <folly/ExceptionWrapper.h>+#include <folly/Executor.h>+#include <folly/ScopeGuard.h>+#include <folly/coro/Coroutine.h>+#include <folly/coro/Traits.h>+#include <folly/coro/ViaIfAsync.h>+#include <folly/functional/Invoke.h>+#include <folly/lang/Assume.h>+#include <folly/lang/CustomizationPoint.h>++#if FOLLY_HAS_COROUTINES++namespace folly {+namespace coro {+namespace detail {+struct AttachScopeExitFn {+  /// Dispatches to a custom implementation using tag_invoke()+  template <+      typename ParentPromise,+      typename ChildPromise,+      std::enable_if_t<+          folly::is_tag_invocable_v<+              AttachScopeExitFn,+              ParentPromise&,+              coroutine_handle<ChildPromise>>,+          int> = 0>+  auto operator()(+      ParentPromise& parent, coroutine_handle<ChildPromise> action) const+      noexcept(folly::is_nothrow_tag_invocable_v<+               AttachScopeExitFn,+               ParentPromise&,+               coroutine_handle<ChildPromise>>)+          -> folly::tag_invoke_result_t<+              AttachScopeExitFn,+              ParentPromise&,+              coroutine_handle<ChildPromise>> {+    return folly::tag_invoke(AttachScopeExitFn{}, parent, action);+  }+};++/// co_attachScopeExit extension point opts the parent coroutine type into+/// handling ScopeExitTasks and executing them at the end of the parent+/// coroutine's scope.+///+/// There are two important steps the parent coroutine must take:+/// 1. It must store the provided ScopeExitTask coroutine handle and return the+/// latest previously attached ScopeExitTask handle (or an empty handle if this+/// one is the first).+/// 2. On destruction of the parent coroutine, the context of the latest stored+/// ScopeExitTask coroutine must be set by calling setContext(...) on its+/// promise object, then the ScopeExitTask coroutine must be executed.+/// The continuation passed to the setContext(...) call will be resumed after+/// the executing the last (the first attached) coroutine in the ScopeExitTask+/// chain. NOTE: The user must not pop the async frame if it is passed to the+/// setContext(...) call, it will be popped by the last ScopeExitTask coroutine+/// in the chain instead.+FOLLY_DEFINE_CPO(AttachScopeExitFn, co_attachScopeExit)++template <typename... Args>+class ScopeExitTask;++class ScopeExitTaskPromiseBase {+ public:+  class FinalAwaiter {+   public:+    bool await_ready() noexcept { return false; }++    template <typename Promise>+    FOLLY_CORO_AWAIT_SUSPEND_NONTRIVIAL_ATTRIBUTES coroutine_handle<>+    await_suspend(coroutine_handle<Promise> coro) noexcept {+      SCOPE_EXIT {+        coro.destroy();+      };++      ScopeExitTaskPromiseBase& promise = coro.promise();+      DCHECK(promise.continuation_);+      DCHECK(promise.parentAsyncFrame_);+      DCHECK(promise.executor_);+      if (promise.next_) {+        promise.next_.promise().setContext(+            promise.continuation_,+            promise.parentAsyncFrame_,+            promise.executor_.get_alias(),+            std::move(promise.error_));+        return promise.next_;+      }++      /// If we reached this point, then this ScopeExitTask is the final one to+      /// be executed on the parent task, and we can now pop the parent's async+      /// frame before calling the original parent's continuation.+      folly::popAsyncStackFrameCallee(*promise.parentAsyncFrame_);+      if (promise.error_) {+        auto [handle, frame] =+            promise.continuation_.getErrorHandle(promise.error_);+        return handle.getHandle();+      }+      return promise.continuation_.getHandle();+    }++    [[noreturn]] void await_resume() noexcept { folly::assume_unreachable(); }+  };++  void setContext(+      ExtendedCoroutineHandle continuation,+      folly::AsyncStackFrame* asyncFrame,+      folly::Executor::KeepAlive<> executor,+      folly::exception_wrapper error = {}) {+    continuation_ = continuation;+    parentAsyncFrame_ = asyncFrame;+    executor_ = std::move(executor);+    error_ = std::move(error);+  }++  suspend_always initial_suspend() noexcept { return {}; }++  FinalAwaiter final_suspend() noexcept { return {}; }++  template <typename Awaitable>+  auto await_transform(Awaitable&& awaitable) {+    return folly::coro::co_withAsyncStack(folly::coro::co_viaIfAsync(+        executor_.get_alias(), static_cast<Awaitable&&>(awaitable)));+  }++  folly::AsyncStackFrame& getAsyncFrame() noexcept {+    return *parentAsyncFrame_;+  }++  [[noreturn]] void unhandled_exception() noexcept {+    /// Since ScopeExitTasks execute after the parent coroutine has completed,+    /// we are unable to propagate exceptions back to the caller. Similar to+    /// throwing another exception while unwinding an exception, we opt to+    /// terminate here by throwing within a noexcept frame.+    rethrow_current_exception();+  }++  void return_void() noexcept {}++ protected:+  template <typename... Args>+  friend class ScopeExitTask;++  ExtendedCoroutineHandle continuation_;+  folly::AsyncStackFrame* parentAsyncFrame_;+  folly::Executor::KeepAlive<> executor_;+  folly::exception_wrapper error_;+  coroutine_handle<ScopeExitTaskPromiseBase> next_;+};++template <typename... Args>+class ScopeExitTaskPromise : public ScopeExitTaskPromiseBase {+ public:+  template <typename Action>+  explicit ScopeExitTaskPromise(Action&&, Args&... args) noexcept+      : args_(args...) {}++  ScopeExitTask<Args...> get_return_object() noexcept;++ private:+  friend class ScopeExitTask<Args...>;++  std::tuple<Args&...> args_;+};++template <typename... Args>+class [[nodiscard]] ScopeExitTask {+ public:+  using promise_type = ScopeExitTaskPromise<Args...>;++ private:+  class Awaiter;+  using handle_t = coroutine_handle<promise_type>;++ public:+  explicit ScopeExitTask(handle_t coro) noexcept : coro_(coro) {}++  ~ScopeExitTask() {+    /// Failing to await this Task is likely a bug+    DCHECK(!coro_);+  }++  ScopeExitTask(ScopeExitTask&& t) noexcept+      : coro_(std::exchange(t.coro_, {})) {}++  friend auto co_viaIfAsync(Executor::KeepAlive<>, ScopeExitTask&& t) noexcept {+    DCHECK(t.coro_);+    return Awaiter{std::exchange(t.coro_, {})};+  }++  /// We explicitly do not handle co_withCancellation, as these tasks are+  /// designed to always run at the end of their parent coroutine.++ private:+  class Awaiter {+   public:+    explicit Awaiter(handle_t coro) noexcept : coro_(coro) {}++    Awaiter(Awaiter&& other) noexcept : coro_(std::exchange(other.coro_, {})) {}++    Awaiter(const Awaiter&) = delete;++    ~Awaiter() {+      /// The coro will destroy itself in the FinalAwaiter, before continuing+      /// the next continuation+      DCHECK(!coro_);+    }++    bool await_ready() const noexcept { return false; }++    template <typename Promise>+    bool await_suspend(coroutine_handle<Promise> parent) noexcept {+      auto& promise = coro_.promise();+      auto& parentPromise = parent.promise();++      /// Calling co_attachScopeExit here inserts the ScopeExit coroutine handle+      /// as the parent's continuation, and sets the ScopeExit's continuation as+      /// the parents.+      ///+      /// Before:+      /// Parent FinalAwaiter -> Parent's continuation+      ///+      /// After one scope exit:+      /// Parent FinalAwaiter -> ScopeExit1 -> Parent's Continuation+      /// After two scope exits:+      /// Parent FinalAwaiter -> ScopeExit2 -> ScopeExit1 -> Parent's+      /// continuation+      ///+      /// This ensures that the scope exit coroutines are executed in reverse+      /// order to when they were attached in the parent.+      ///+      /// Since each ScopeExitTask runs as a continuation at the end of the+      /// parent coroutine's scope without popping the async stack to the+      /// caller, we must run within the parent's async frame. In order to+      /// guarantee correctness, the parent must defer responsibility of popping+      /// the async stack frame to the final scope exit continuation.+      promise.next_ = co_attachScopeExit(+          parentPromise,+          coroutine_handle<ScopeExitTaskPromiseBase>::from_promise(+              coro_.promise()));++      return false;+    }++    std::tuple<Args&...> await_resume() noexcept {+      /// The coro will destroy itself in the FinalAwaiter+      handle_t coro = std::exchange(coro_, {});+      return std::move(coro.promise().args_);+    }++   private:+    friend Awaiter tag_invoke(cpo_t<co_withAsyncStack>, Awaiter&& t) noexcept {+      return std::move(t);+    }++    handle_t coro_;+  };++  handle_t coro_;+};++template <typename... Args>+inline ScopeExitTask<Args...>+ScopeExitTaskPromise<Args...>::get_return_object() noexcept {+  return ScopeExitTask<Args...>{+      coroutine_handle<ScopeExitTaskPromise>::from_promise(*this)};+}++} // namespace detail++class co_scope_exit_fn {+  /// Use a static helper as we do not wish to pass the implicit `this` pointer+  /// to the promise constructor+  ///+  /// TODO: It's not mandatory to elide copy/move of args into the coroutine+  /// frame today, which makes using some types, like AsyncScope, annoying. For+  /// non-copyable, non-moveable types, you must wrap the type in a+  /// std::unique_ptr.+  ///+  /// We might be able to work around this by storing the arguments in the+  /// promise type, rather than on the coroutine frame.+  template <typename Action, typename... Args>+  static detail::ScopeExitTask<Args...> coScopeExitImpl(+      Action action, Args... args) {+    co_await std::move(action)(std::move(args)...);+  }++ public:+  template <typename Action, typename... Args>+  detail::ScopeExitTask<std::decay_t<Args>...> operator()(+      Action&& action, Args&&... args) const {+    return coScopeExitImpl(+        static_cast<Action&&>(action), static_cast<Args&&>(args)...);+  }+};++/// co_scope_exit is a utility function that allows you to associate+/// continuations which execute at the end of the coroutine, just before+/// resuming the caller.+///+/// The first argument is a Task-returning callable. The subsequent arguments+/// are optional state that can be used within the exit coroutine. The cleanup+/// action will assume ownership of the provided state by copying the state+/// inside the exit coroutine.+///+/// If you need access to the state in both the parent coroutine *and* in the+/// exit coroutine, you can receive l-values to the captured state as return+/// values. See the example below.+///+/// If you attach multiple co_scope_exit coroutines, they will be executed in+/// reverse order to the order in which they were registered.+///+/// CAUTION: The body of the co_scope_exit coroutine runs *after* the parent+/// coroutine has already been destroyed. This means that any local variables in+/// the coroutine body will no longer be accessible. Do not capture references+/// to any locals in the exit coroutine, or else you will hit undefined+/// behavior. Any state you wish to pass to the scope exit coroutine should be+/// passed as an argument to co_scope_exit.+///+/// Example:+/// folly::coro::Task<> doSomethingComplicated(std::vector<int> inputs) {+///   auto&& [scope] = co_await folly::coro::co_scope_exit(+///       [](auto scope) -> folly::coro::Task<> {+///         co_await scope.joinAsync();+///       }, std::make_unique<AsyncScope>());+///+///   // Do some complicated, potentially throwing work using the AsyncScope+///   auto ex = co_await co_current_executor;+///   asyncScope->add(co_withExecutor(ex, someTask(std::move(inputs))));+/// }+///+/// The body of the coroutine passed to co_scope_exit will be executed when the+/// parent task completes, either when the parent completes with a result, or+/// due to an unhandled exception.+inline constexpr co_scope_exit_fn co_scope_exit{};++} // namespace coro+} // namespace folly++#endif // FOLLY_HAS_COROUTINES
+ folly/folly/coro/SerialQueueRunner.cpp view
@@ -0,0 +1,102 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#include <folly/coro/SerialQueueRunner.h>++#include <functional>+#include <stdexcept>+#include <utility>++#if FOLLY_HAS_COROUTINES++namespace folly::coro {++void SerialQueueRunner::add(Work task) {+  std::unique_lock lock{mut_};+  if (done_) {+    throw std::runtime_error("add after done");+  }+  tasks_.push_back(std::move(task));+  if (baton_) {+    baton_->post();+  }+}++void SerialQueueRunner::done() {+  std::unique_lock lock{mut_};+  if (done_) {+    throw std::runtime_error("add after done");+  }+  done_ = true;+  if (baton_) {+    baton_->post();+  }+}++Task<> SerialQueueRunner::run() {+  if (running_.exchange(true, std::memory_order_relaxed)) {+    co_yield co_error{+        make_exception_wrapper<std::runtime_error>("multiple calls to run")};+  }+  while (true) {+    auto [done, tasks] = co_await pull();+    for (auto& task : tasks) {+      auto res = co_await co_awaitTry(std::move(task));+      exception_wrapper exn =+          res.hasException() ? std::move(res.exception()) : exception_wrapper();+      if (!exn_ && exn && !exn.get_exception<OperationCancelled>()) {+        exn_ = std::move(exn);+      }+    }+    if (done) {+      break;+    }+  }+  if (exn_) {+    co_yield co_error{std::move(exn_)};+  }+}++void SerialQueueRunner::cancel() {+  std::unique_lock lock{mut_};+  if (!done_) {+    done();+  }+}++Task<> SerialQueueRunner::await() {+  CancellationCallback cb{+      co_await co_current_cancellation_token,+      std::bind(&SerialQueueRunner::cancel, this)};+  co_await *baton_;+}++Task<SerialQueueRunner::PullResult> SerialQueueRunner::pull() {+  std::unique_lock lock{mut_};+  if (!done_ && tasks_.empty()) {+    folly::coro::Baton baton;+    baton_ = &baton;+    lock.unlock();+    co_await await();+    lock.lock();+    baton_ = nullptr;+  }+  co_return std::pair{done_, std::move(tasks_)};+}++} // namespace folly::coro++#endif
+ folly/folly/coro/SerialQueueRunner.h view
@@ -0,0 +1,73 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <atomic>+#include <mutex>+#include <queue>+#include <tuple>+#include <vector>++#include <folly/ExceptionWrapper.h>+#include <folly/coro/Baton.h>+#include <folly/coro/Task.h>++#if FOLLY_HAS_COROUTINES++namespace folly::coro {++/// SerialQueueRunner+///+/// Runs coroutine work items submitted via add() in sequence, i.e. with no+/// overlapping.+///+/// Different from scheduling via SequencedExecutor, which runs the *parts* of+/// the work items between resume- and suspend-points in sequence, but which+/// overlaps work items.+class SerialQueueRunner {+ private:+  using Work = Task<>;++ public:+  void add(Work task); // task must not throw when awaited!+  void done();++  Task<> run();++ private:+  using PullResult = std::pair<bool, std::vector<Work>>;++  struct Mutex : std::mutex {+    // to suppress lint advice about holding lock objects alive across co_await+    using folly_coro_aware_mutex = void;+  };++  void cancel();+  Task<> await();+  Task<PullResult> pull();++  Mutex mut_{};+  Baton* baton_{};+  std::vector<Work> tasks_{};+  bool done_{};+  std::atomic<bool> running_{};+  exception_wrapper exn_{};+};++} // namespace folly::coro++#endif
+ folly/folly/coro/SharedLock.h view
@@ -0,0 +1,167 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <memory>+#include <mutex>+#include <type_traits>+#include <utility>++#include <glog/logging.h>++#include <folly/Portability.h>+#include <folly/coro/Coroutine.h>+#include <folly/coro/Task.h>+#include <folly/synchronization/Lock.h>++#if FOLLY_HAS_COROUTINES++namespace folly {+namespace coro {++namespace detail {++template <typename Mutex, typename Policy>+class FOLLY_NODISCARD LockBase {+ public:+  static_assert(std::is_same_v<+                bool,+                invoke_result_t<typename Policy::try_lock_fn, Mutex&>>);++  LockBase() noexcept : mutex_(nullptr), locked_(false) {}++  explicit LockBase(Mutex& mutex, std::defer_lock_t) noexcept+      : mutex_(std::addressof(mutex)), locked_(false) {}++  explicit LockBase(Mutex& mutex, std::adopt_lock_t) noexcept+      : mutex_(std::addressof(mutex)), locked_(true) {}++  explicit LockBase(Mutex& mutex, std::try_to_lock_t) noexcept(+      noexcept(typename Policy::try_lock_fn{}(mutex)))+      : mutex_(std::addressof(mutex)),+        locked_(typename Policy::try_lock_fn{}(mutex)) {}++  LockBase(LockBase&& other) noexcept+      : mutex_(std::exchange(other.mutex_, nullptr)),+        locked_(std::exchange(other.locked_, false)) {}++  LockBase(const LockBase&) = delete;+  LockBase& operator=(const LockBase&) = delete;++  ~LockBase() {+    if (locked_) {+      typename Policy::unlock_fn{}(*mutex_);+    }+  }++  LockBase& operator=(LockBase&& other) noexcept {+    LockBase temp(std::move(other));+    swap(temp);+    return *this;+  }++  Mutex* mutex() const noexcept { return mutex_; }++  Mutex* release() noexcept {+    locked_ = false;+    return std::exchange(mutex_, nullptr);+  }++  bool owns_lock() const noexcept { return locked_; }++  explicit operator bool() const noexcept { return owns_lock(); }++  bool try_lock() noexcept(noexcept(typename Policy::try_lock_fn{}(*mutex_))) {+    DCHECK(!locked_);+    DCHECK(mutex_ != nullptr);+    locked_ = typename Policy::try_lock{}(*mutex_);+    return locked_;+  }++  void unlock() noexcept(noexcept(typename Policy::unlock_fn{}(*mutex_))) {+    DCHECK(locked_);+    locked_ = false;+    typename Policy::unlock_fn{}(*mutex_);+  }++  void swap(LockBase& other) noexcept {+    std::swap(mutex_, other.mutex_);+    std::swap(locked_, other.locked_);+  }++ protected:+  Mutex* mutex_;+  bool locked_;+};++struct lock_policy_shared {+  using try_lock_fn = access::try_lock_shared_fn;+  using unlock_fn = access::unlock_shared_fn;+};+struct lock_policy_upgrade {+  using try_lock_fn = access::try_lock_upgrade_fn;+  using unlock_fn = access::unlock_upgrade_fn;+};+} // namespace detail++/// This type mirrors the interface of std::shared_lock as much as possible.+///+/// The main difference between this type and std::shared_lock is that this+/// type is designed to be used with asynchronous shared-mutex types where+/// the lock acquisition is an asynchronous operation.+///+/// TODO: Actually implement the .co_lock() method on this class.+///+/// Workaround for now is to use:+///   SharedLock<SharedMutex> lock{mutex, std::defer_lock};+///   ...+///   lock = co_await lock.mutex()->co_scoped_lock_shared();+template <typename Mutex>+class SharedLock : public detail::LockBase<Mutex, detail::lock_policy_shared> {+ public:+  using detail::LockBase<Mutex, detail::lock_policy_shared>::LockBase;+};++template <typename Mutex, typename... A>+explicit SharedLock(Mutex&, A const&...) -> SharedLock<Mutex>;++template <typename Mutex>+class UpgradeLock+    : public detail::LockBase<Mutex, detail::lock_policy_upgrade> {+ public:+  using detail::LockBase<Mutex, detail::lock_policy_upgrade>::LockBase;+};++template <typename Mutex, typename... A>+explicit UpgradeLock(Mutex&, A const&...) -> UpgradeLock<Mutex>;++/// Async version of the folly::transition_lock+/// TODO: add more transition policies beyond just from upgrade to exclusive+template <typename Mutex>+folly::coro::Task<std::unique_lock<Mutex>> co_transition_lock(+    UpgradeLock<Mutex>& lock) {+  if (lock.owns_lock()) {+    co_return co_await lock.release()->co_scoped_unlock_upgrade_and_lock();+  } else {+    co_return std::unique_lock<Mutex>{*lock.release(), std::defer_lock};+  }+}++} // namespace coro+} // namespace folly++#endif // FOLLY_HAS_COROUTINES
+ folly/folly/coro/SharedMutex.cpp view
@@ -0,0 +1,284 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#include <folly/coro/SharedMutex.h>++#if FOLLY_HAS_COROUTINES++using namespace folly::coro;++SharedMutexFair::~SharedMutexFair() {+  assert(state_.lock()->lockedFlagAndReaderCount_ == kUnlocked);+  assert(state_.lock()->waitersHead_ == nullptr);+}++bool SharedMutexFair::try_lock() noexcept {+  auto lock = state_.lock();+  if (lock->lockedFlagAndReaderCount_ == kUnlocked) {+    lock->lockedFlagAndReaderCount_ = kExclusiveLockFlag;+    return true;+  }+  return false;+}++bool SharedMutexFair::try_lock_shared() noexcept {+  auto lock = state_.lock();+  if (canLockShared(*lock)) {+    lock->lockedFlagAndReaderCount_ += kSharedLockCountIncrement;+    // check for potential overflow+    assert(lock->lockedFlagAndReaderCount_ >= kSharedLockCountIncrement);+    return true;+  }+  return false;+}++bool SharedMutexFair::try_lock_upgrade() noexcept {+  auto lock = state_.lock();+  if (canLockUpgrade(*lock)) {+    lock->lockedFlagAndReaderCount_ |= kUpgradeLockFlag;+    return true;+  }+  return false;+}++bool SharedMutexFair::try_unlock_upgrade_and_lock() noexcept {+  auto lock = state_.lock();+  assert(lock->lockedFlagAndReaderCount_ & kUpgradeLockFlag);+  // skip the line and perform the upgrade as long as there is+  // no outstanding shared locks+  if (lock->lockedFlagAndReaderCount_ == kUpgradeLockFlag) {+    lock->lockedFlagAndReaderCount_ = kExclusiveLockFlag;+    return true;+  }+  return false;+}++void SharedMutexFair::unlock() noexcept {+  LockAwaiterBase* awaitersToResume = nullptr;+  {+    auto lockedState = state_.lock();+    assert(lockedState->lockedFlagAndReaderCount_ == kExclusiveLockFlag);+    lockedState->lockedFlagAndReaderCount_ = kUnlocked;+    awaitersToResume = getWaitersToResume(*lockedState, LockType::EXCLUSIVE);+  }++  resumeWaiters(awaitersToResume);+}++void SharedMutexFair::unlock_shared() noexcept {+  LockAwaiterBase* awaitersToResume = nullptr;+  {+    auto lockedState = state_.lock();+    assert(lockedState->lockedFlagAndReaderCount_ >= kSharedLockCountIncrement);+    lockedState->lockedFlagAndReaderCount_ -= kSharedLockCountIncrement;+    awaitersToResume = getWaitersToResume(*lockedState, LockType::SHARED);+  }++  resumeWaiters(awaitersToResume);+}++void SharedMutexFair::unlock_upgrade() noexcept {+  LockAwaiterBase* awaitersToResume = nullptr;+  {+    auto lockedState = state_.lock();+    assert(lockedState->lockedFlagAndReaderCount_ & kUpgradeLockFlag);+    lockedState->lockedFlagAndReaderCount_ &= ~kUpgradeLockFlag;+    awaitersToResume = getWaitersToResume(*lockedState, LockType::UPGRADE);+  }++  resumeWaiters(awaitersToResume);+}++// `getWaitersToResume` can sometimes perform long reader scan to+// find readers to resume. But the amortized cost of it is still O(1),+// as we never scan the same waiter more than once (except the head).+SharedMutexFair::LockAwaiterBase* SharedMutexFair::getWaitersToResume(+    SharedMutexFair::State& state, LockType prevLockType) noexcept {+  // to keep the state transition code concise we only modify+  // the mutex's waitersHead_ pointer, and depend on the SCOPE_EXIT+  // to ensure the consistency of the state+  SCOPE_EXIT {+    if (state.waitersHead_ == nullptr) {+      state.waitersTailNext_ = &state.waitersHead_;+    }+  };++  // the state transition is a function of the lock type of the+  // previous lock (what just got unlocked), the waiter(s) and the current mutex+  // state (outstanding locks)+  if (state.upgrader_ != nullptr) {+    // there is an active upgrade lock holder waiting to upgrade to exclusive+    // prioritize the lock transfer over other lock acquisition waiters+    assert(state.lockedFlagAndReaderCount_ & kUpgradeLockFlag);+    if (state.lockedFlagAndReaderCount_ == kUpgradeLockFlag) {+      auto* waiter = std::exchange(state.upgrader_, nullptr);+      // there can only be an active upgrade lock holder waiting to transfer to+      // exclusive+      assert(waiter->nextAwaiter_ == nullptr);+      state.lockedFlagAndReaderCount_ = kExclusiveLockFlag;+      return waiter;+    } else {+      // drain the readers and do not grant any locks to other waiters+      return nullptr;+    }+  }++  // there is no active upgrader; process the pending lock acquisition requests+  // in order+  auto* head = state.waitersHead_;+  if (head == nullptr) {+    // there is no waiters to resume+    return nullptr;+  }++  // There is no pending lock transfers. The mutex can only be unlock_* into one+  // of the following states:+  // - unlocked (from unlock)+  // - shared locked (from unlock_shared or unlock_upgrade)+  // - upgrade and shared locked (from unlock_shared)+  assert(state.lockedFlagAndReaderCount_ != kExclusiveLockFlag);+  if (head->lockType_ == LockType::EXCLUSIVE) {+    if (state.lockedFlagAndReaderCount_ == kUnlocked) {+      // transition to exclusively locked state+      state.waitersHead_ = std::exchange(head->nextAwaiter_, nullptr);+      state.lockedFlagAndReaderCount_ = kExclusiveLockFlag;+      --state.waitingWriterCount_;+      return head;+    }+  } else if (+      head->lockType_ != LockType::UPGRADE ||+      (state.lockedFlagAndReaderCount_ & kUpgradeLockFlag) == 0) {+    // Now the next waiter is either a reader or an upgrader, and the mutex+    // state ensures that the next waiter can always be resumed.+    // The only case that we can't resume the next head (and skip scanning)+    // is when the head is an upgrader and the mutex is in an upgrade locked+    // state. There is nothing to resume in that case (no readers will be queued+    // to begin with).+    state.waitersHead_ = scanReadersAndUpgrader(head, state, prevLockType);+    return head;+  }+  return nullptr;+}++void SharedMutexFair::resumeWaiters(LockAwaiterBase* awaiters) noexcept {+  while (awaiters != nullptr) {+    std::exchange(awaiters, awaiters->nextAwaiter_)->resume();+  }+}++// Scan for a run of SHARED and UPGRADE lock types and return the next+// waiter+SharedMutexFair::LockAwaiterBase* SharedMutexFair::scanReadersAndUpgrader(+    SharedMutexFair::LockAwaiterBase* head,+    SharedMutexFair::State& lockedState,+    LockType prevLockType) noexcept {+  SharedMutexFair::LockAwaiterBase* last =+      nullptr; // tail of the waiters to be resumed+  size_t& state = lockedState.lockedFlagAndReaderCount_;+  // Scan for a continuous run of SHARED and UPGRADE lock types+  while (head != nullptr) {+    if (head->lockType_ == LockType::SHARED) {+      state += kSharedLockCountIncrement;+      // check for potential overflow+      assert(state >= kSharedLockCountIncrement);+    } else if (+        head->lockType_ == LockType::UPGRADE &&+        (state & kUpgradeLockFlag) == 0) {+      state |= kUpgradeLockFlag;+    } else {+      break;+    }+    last = head;+    head = head->nextAwaiter_;+  }+  assert(last != nullptr);++  auto* newWaiterHead = head;+  auto* prev = last;+  if (prevLockType == LockType::EXCLUSIVE) {+    // Do a long reader scan up until the next exclusive lock waiter+    // iff someone just unlocked an exclusive lock.+    // e.g. when the waiter list looks like+    // U1 U2 U3 U4 S1 W S2+    // , and someone just unlocked an exclusive lock+    // we unlock U1 _and_ S1.+    // However, doing long reader scan every time someone unlocks any lock+    // is wasteful. Readers can only be blocked when the mutex has an active+    // exclusive lock or there are writers waiting ahead of it. In other words,+    // a reader can only be blocked when there is one or more writers ahead of+    // it. The writer can be holding an active lock or simply waiting ahead of+    // the reader.+    //+    // So we only need to do the long reader scan when an exclusive lock is+    // released, and we do a long reader scan up until the next exclusive+    // waiter. If any other lock (shared or upgrade) is released, all the+    // readers are blocked right now should remain blocked (as they are blocked+    // one or more writers ahead of them).+    //+    // We never want to scan past an exclusive waiter, as it would lead to+    // writer starvation and makes this mutex "unfair" or reader-prioritized. In+    // this way, we can keep the amortized cost of `getWaitersToResume` be O(1).+    //+    // Notice that previous lock type being EXCLUSIVE is different from mutex+    // state being UNLOCKED, as the first condition is more restrictive. E.g. if+    // the mutex enters UNLOCKED from unlock_upgrade(), and the waiter list+    // looks like "U U W S W U ... " there is no point to scan the waiter list+    // for readers, as either there is no blocked readers to begin with, or+    // they are blocked by another writer ahead of them, and should remain+    // blocked. The waiter list would never look like "U U S" at the time of+    // unlock_upgrade() because the "S" should never be blocked to begin with.+    //+    // This behavior should not lead to starvation+    // - it does not starve writers because we do not scan pass "W"+    // - it does not starve upgraders because these upgraders are not blocked+    //   by the readers anyway+    // - it does not block lock transition (upgrade -> exclusive) because+    //   lock transition is handled with highest priority, and no readers+    //   are granted when a lock transition is in progress, when it is trying+    //   to drain all the readers+    // E.g. the waiter list might look like U1 U2 S1 U3 S2 W1 S3+    // After calling this function, the `head` should point to U1 S1 S2+    // and the return value should point to U2 U3 W1 S3+    while (head != nullptr) {+      if (head->lockType_ == LockType::SHARED) {+        assert(head != newWaiterHead);+        assert(prev != last);+        state += kSharedLockCountIncrement;+        // check for potential overflow+        assert(state >= kSharedLockCountIncrement);+        prev->nextAwaiter_ = head->nextAwaiter_;+        last->nextAwaiter_ = head;+        last = head;+        head = head->nextAwaiter_;+        if (head == nullptr) {+          // if we skipped ahead and resumed the last waiter+          // we need to update the waiter tail pointer+          lockedState.waitersTailNext_ = &prev->nextAwaiter_;+        }+      } else if (head->lockType_ == LockType::UPGRADE) {+        // skip the upgrade waiter+        prev = head;+        head = head->nextAwaiter_;+      } else {+        break;+      }+    }+  }++  last->nextAwaiter_ = nullptr;+  return newWaiterHead;+}+#endif
+ folly/folly/coro/SharedMutex.h view
@@ -0,0 +1,605 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <atomic>+#include <cassert>+#include <limits>+#include <mutex>+#include <utility>++#include <folly/Executor.h>+#include <folly/SpinLock.h>+#include <folly/Synchronized.h>+#include <folly/coro/Coroutine.h>+#include <folly/coro/SharedLock.h>+#include <folly/coro/ViaIfAsync.h>++#if FOLLY_HAS_COROUTINES++namespace folly {+namespace coro {++/// The folly::coro::SharedMutexFair class provides a thread synchronisation+/// primitive that allows a coroutine to asynchronously acquire a lock on the+/// mutex.+///+/// The mutex supports three kinds of locks:+/// - exclusive-lock - Also known as a write-lock.+///                    While an exclusive lock is held, no other thread will be+///                    able to acquire either an exclusive lock or a shared+///                    lock until the exclusive lock is released.+/// - shared-lock    - Also known as a read-lock.+///                    The mutex permits multiple shared locks to be held+///                    concurrently but does not permit shared locks to be held+///                    concurrently with exclusive locks.+/// - upgrade-lock   - When an upgrade lock is held, others can still acquire+///                    shared locks but no exclusive lock, or upgrade lock.+///                    An upgrade lock can be later upgraded to an exclusive+///                    lock atomically after all the outstanding shared locks+///                    are released.+///+/// This mutex employs a fair lock acquisition strategy that attempts to process+/// locks in a mostly FIFO order in which they arrive at the mutex.+/// This means that if the mutex is currently read-locked and some coroutine+/// tries to acquire a write-lock, that subsequent read-lock attempts will+/// be queued up behind the write-lock, allowing the write-lock to be acquired+/// in a bounded amount of time.+///+/// One implication of this strategy is that it is not safe to unconditionally+/// acquire a new read-lock while already holding a read-lock, since it's+/// possible that this could lead to deadlock if there was another coroutine+/// that was currently waiting on a write-lock.+///+/// Notably, lock transition (e.g. upgrade an upgrade lock to an exclusive lock)+/// does not respect the FIFO order and is eager. This means a pending lock+/// transition will be processed as soon as possible. This is to avoid deadlock+/// in following scenario+/// 1. coroutine A has the upgrade lock+/// 2. coroutine B is waiting for an exclusive lock+/// 3. coroutine A tries to upgrade the lock to exclusive+/// Coroutine A and B would deadlock if we process the lock transition+/// (operation #3) in FIFO order. The readers will not be starved because they+/// are not blocked by the upgrade state to begin with. The writers/upgraders+/// will not be starved because they cannot acquire the lock anyway.+///+/// The locks acquired by this mutex do not have thread affinity. A coroutine+/// can acquire the lock on one thread and release the lock on another thread.+///+/// Example usage:+///+///  class AsyncStringSet {+///    mutable folly::coro::SharedMutexFair mutex_;+///    std::unordered_set<std::string> values_;+///+///    AsyncStringSet() = default;+///+///    folly::coro::Task<bool> insert(std::string value) {+///      auto lock = co_await mutex_.co_scoped_lock();+///      co_return values_.insert(value).second;+///    }+///+///    folly::coro::Task<bool> remove(std::string value) {+///      auto lock = co_await mutex_.co_scoped_lock();+///      co_return values_.erase(value) > 0;+///    }+///+///    folly::coro::Task<bool> contains(std::string value) const {+///      auto lock = co_await mutex_.co_scoped_lock_shared();+///      co_return values_.count(value) > 0;+///    }+///  };+class SharedMutexFair : private folly::NonCopyableNonMovable {+  template <typename Awaiter>+  class LockOperation;+  class LockAwaiter;+  class ScopedLockAwaiter;+  class LockSharedAwaiter;+  class ScopedLockSharedAwaiter;+  class LockUpgradeAwaiter;+  class ScopedLockUpgradeAwaiter;+  class UnlockUpgradeAndLockAwaiter;+  class ScopedUnlockUpgradeAndLockAwaiter;++ public:+  SharedMutexFair() noexcept = default;++  ~SharedMutexFair();++  /// Try to acquire an exclusive lock on the mutex synchronously.+  ///+  /// If this returns true then the exclusive lock was acquired synchronously+  /// and the caller is responsible for calling .unlock() later to release+  /// the exclusive lock. If this returns false then the lock was not acquired.+  ///+  /// Consider using a std::unique_lock to ensure the lock is released at the+  /// end of a scope.+  bool try_lock() noexcept;++  /// Try to acquire a shared lock on the mutex synchronously.+  ///+  /// If this returns true then the shared lock was acquired synchronously+  /// and the caller is responsible for calling .unlock_shared() later to+  /// release the shared lock.+  bool try_lock_shared() noexcept;++  /// Try to acquire an upgrade lock on the mutex synchronously.+  ///+  /// If this returns true then the upgrade lock was acquired synchronously+  /// and the caller is responsible for calling .unlock_upgrade() later to+  /// release the upgrade lock.+  bool try_lock_upgrade() noexcept;++  /// Asynchronously acquire an exclusive lock on the mutex.+  ///+  /// Returns a SemiAwaitable<void> type that requires the caller to inject+  /// an executor by calling .viaIfAsync(executor) and then co_awaiting the+  /// result to wait for the lock to be acquired. Note that if the caller is+  /// awaiting the lock operation within a folly::coro::Task then the current+  /// executor will be injected implicitly without needing to call+  /// .viaIfAsync().+  ///+  /// If the lock was acquired synchronously then the awaiting coroutine+  /// continues on the current thread without suspending.+  /// If the lock could not be acquired synchronously then the awaiting+  /// coroutine is suspended and later resumed on the specified executor when+  /// the lock becomes available.+  ///+  /// After this operation completes, the caller is responsible for calling+  /// .unlock() to release the lock.+  [[nodiscard]] LockOperation<LockAwaiter> co_lock() noexcept;++  /// Asynchronously acquire an exclusive lock on the mutex and return an object+  /// that will release the lock when it goes out of scope.+  ///+  /// Returns a SemiAwaitable<std::unique_lock<SharedMutexFair>> that, once+  /// associated with an executor using .viaIfAsync(), must be co_awaited to+  /// wait for the lock to be acquired.+  ///+  /// If the lock could be acquired immediately then the coroutine continues+  /// execution without suspending. Otherwise, the coroutine is suspended and+  /// will later be resumed on the specified executor once the lock has been+  /// acquired.+  [[nodiscard]] LockOperation<ScopedLockAwaiter> co_scoped_lock() noexcept;++  /// Asynchronously acquire a shared lock on the mutex.+  ///+  /// Returns a SemiAwaitable<void> type that requires the caller to inject+  /// an executor by calling .viaIfAsync(executor) and then co_awaiting the+  /// result to wait for the lock to be acquired. Note that if the caller is+  /// awaiting the lock operation within a folly::coro::Task then the current+  /// executor will be injected implicitly without needing to call+  /// .viaIfAsync().+  ///+  /// If the lock was acquired synchronously then the awaiting coroutine+  /// continues on the current thread without suspending.+  /// If the lock could not be acquired synchronously then the awaiting+  /// coroutine is suspended and later resumed on the specified executor when+  /// the lock becomes available.+  ///+  /// After this operation completes, the caller is responsible for calling+  /// .unlock_shared() to release the lock.+  [[nodiscard]] LockOperation<LockSharedAwaiter> co_lock_shared() noexcept;++  /// Asynchronously acquire a shared lock on the mutex and return an object+  /// that will release the lock when it goes out of scope.+  ///+  /// Returns a SemiAwaitable<std::shared_lock<SharedMutexFair>> that, once+  /// associated with an executor using .viaIfAsync(), must be co_awaited to+  /// wait for the lock to be acquired.+  ///+  /// If the lock could be acquired immediately then the coroutine continues+  /// execution without suspending. Otherwise, the coroutine is suspended and+  /// will later be resumed on the specified executor once the lock has been+  /// acquired.+  [[nodiscard]] LockOperation<ScopedLockSharedAwaiter>+  co_scoped_lock_shared() noexcept;++  /// Asynchronously acquire an upgrade lock on the mutex.+  ///+  /// Returns a SemiAwaitable<void> type that requires the caller to inject+  /// an executor by calling .viaIfAsync(executor) and then co_awaiting the+  /// result to wait for the lock to be acquired. Note that if the caller is+  /// awaiting the lock operation within a folly::coro::Task then the current+  /// executor will be injected implicitly without needing to call+  /// .viaIfAsync().+  ///+  /// If the lock was acquired synchronously then the awaiting coroutine+  /// continues on the current thread without suspending.+  /// If the lock could not be acquired synchronously then the awaiting+  /// coroutine is suspended and later resumed on the specified executor when+  /// the lock becomes available.+  ///+  /// After this operation completes, the caller is responsible for calling+  /// .unlock_upgrade() to release the lock.+  [[nodiscard]] LockOperation<LockUpgradeAwaiter> co_lock_upgrade() noexcept;++  /// Asynchronously acquire an upgrade lock on the mutex and return an object+  /// that will release the lock when it goes out of scope.+  ///+  /// Returns a SemiAwaitable<UpgradeLock<SharedMutexFair>> that, once+  /// associated with an executor using .viaIfAsync(), must be co_awaited to+  /// wait for the lock to be acquired.+  ///+  /// If the lock could be acquired immediately then the coroutine continues+  /// execution without suspending. Otherwise, the coroutine is suspended and+  /// will later be resumed on the specified executor once the lock has been+  /// acquired.+  [[nodiscard]] LockOperation<ScopedLockUpgradeAwaiter>+  co_scoped_lock_upgrade() noexcept;++  /// Asynchronously transition the currently held upgrade lock to exclusive.+  ///+  /// Returns a SemiAwaitable<void> type that requires the caller to inject+  /// an executor by calling .viaIfAsync(executor) and then co_awaiting the+  /// result to wait for the lock to be acquired. Note that if the caller is+  /// awaiting the lock operation within a folly::coro::Task then the current+  /// executor will be injected implicitly without needing to call+  /// .viaIfAsync().+  ///+  /// If the lock was transitioned synchronously then the awaiting coroutine+  /// continues on the current thread without suspending.+  /// If the lock could not be transitioned synchronously then the awaiting+  /// coroutine is suspended and later resumed on the specified executor when+  /// the lock becomes available.+  ///+  /// After this operation completes, the caller is responsible for calling+  /// .unlock() to release the lock.+  [[nodiscard]] LockOperation<UnlockUpgradeAndLockAwaiter>+  co_unlock_upgrade_and_lock() noexcept;++  /// Asynchronously transfer the currently held upgrade lock to exclusive+  /// and return an object that will release the exclusive lock when it+  /// goes out of scope.+  ///+  /// Notice that if the upgrade lock is acquired using+  /// `co_scoped_lock_upgrade()`, one should transfer the lock via+  /// `co_transition_lock(coro::UpgradeLock<coro::SharedMutex>&)` to avoid+  /// double unlock. This method is mostly useful if the original upgrade+  /// lock is acquired manually via `co_await mutex.co_lock_upgrade();`.+  ///+  /// Returns a SemiAwaitable<std::unique_lock<SharedMutexFair>> that, once+  /// associated with an executor using .viaIfAsync(), must be co_awaited to+  /// wait for the lock to be acquired.+  ///+  /// If the lock could be acquired immediately then the coroutine continues+  /// execution without suspending. Otherwise, the coroutine is suspended and+  /// will later be resumed on the specified executor once the lock has been+  /// acquired.+  [[nodiscard]] LockOperation<ScopedUnlockUpgradeAndLockAwaiter>+  co_scoped_unlock_upgrade_and_lock() noexcept;++  /// Release the exclusive lock.+  ///+  /// This will resume the next coroutine(s) waiting to acquire the lock, if+  /// any.+  void unlock() noexcept;++  /// Release a shared lock.+  ///+  /// If this is the last shared lock then this will resume the next+  /// coroutine(s) waiting to acquire the lock, if any.+  void unlock_shared() noexcept;++  /// Release an upgrade lock.+  ///+  /// This will resume the next coroutine(s) waiting to acquire an exclusive+  /// lock or an upgrade lock, if any.+  void unlock_upgrade() noexcept;++  /// Try to atomically transition an upgrade lock to an exclusive lock+  /// synchronously.+  ///+  /// If this returns true then the lock was acquired synchronously+  /// and the caller is responsible for calling .unlock() later to+  /// release the lock. Otherwise, the caller remains responsible for calling+  /// .unlock_upgrade() later to release the upgrade lock.+  bool try_unlock_upgrade_and_lock() noexcept;++ private:+  using folly_coro_aware_mutex = std::true_type;++  enum class LockType : std::uint8_t { EXCLUSIVE, UPGRADE, SHARED };++  class LockAwaiterBase {+   protected:+    friend class SharedMutexFair;++    explicit LockAwaiterBase(SharedMutexFair& mutex, LockType lockType) noexcept+        : mutex_(&mutex), nextAwaiter_(nullptr), lockType_(lockType) {}++    void resume() noexcept { continuation_.resume(); }++    SharedMutexFair* mutex_;+    LockAwaiterBase* nextAwaiter_;+    coroutine_handle<> continuation_;+    LockType lockType_;+  };++  class LockAwaiter : public LockAwaiterBase {+   public:+    explicit LockAwaiter(SharedMutexFair& mutex) noexcept+        : LockAwaiterBase(mutex, LockType::EXCLUSIVE) {}++    bool await_ready() noexcept { return mutex_->try_lock(); }++    FOLLY_CORO_AWAIT_SUSPEND_NONTRIVIAL_ATTRIBUTES bool await_suspend(+        coroutine_handle<> continuation) noexcept {+      auto lock = mutex_->state_.lock();++      // Exclusive lock can only be acquired if it's currently unlocked.+      if (lock->lockedFlagAndReaderCount_ == kUnlocked) {+        lock->lockedFlagAndReaderCount_ = kExclusiveLockFlag;+        return false;+      }++      // Append to the end of the waiters queue.+      continuation_ = continuation;+      ++lock->waitingWriterCount_;+      *lock->waitersTailNext_ = this;+      lock->waitersTailNext_ = &nextAwaiter_;+      return true;+    }++    void await_resume() noexcept {}+  };++  class LockSharedAwaiter : public LockAwaiterBase {+   public:+    explicit LockSharedAwaiter(SharedMutexFair& mutex) noexcept+        : LockAwaiterBase(mutex, LockType::SHARED) {}++    bool await_ready() noexcept { return mutex_->try_lock_shared(); }++    FOLLY_CORO_AWAIT_SUSPEND_NONTRIVIAL_ATTRIBUTES bool await_suspend(+        coroutine_handle<> continuation) noexcept {+      auto lock = mutex_->state_.lock();++      if (canLockShared(*lock)) {+        lock->lockedFlagAndReaderCount_ += kSharedLockCountIncrement;+        // check for potential overflow+        assert(lock->lockedFlagAndReaderCount_ >= kSharedLockCountIncrement);+        return false;+      }++      // Lock not available immediately.+      // Queue up for later resumption.+      continuation_ = continuation;+      *lock->waitersTailNext_ = this;+      lock->waitersTailNext_ = &nextAwaiter_;+      return true;+    }++    void await_resume() noexcept {}+  };++  class LockUpgradeAwaiter : public LockAwaiterBase {+   public:+    explicit LockUpgradeAwaiter(SharedMutexFair& mutex) noexcept+        : LockAwaiterBase(mutex, LockType::UPGRADE) {}++    bool await_ready() noexcept { return mutex_->try_lock_upgrade(); }++    FOLLY_CORO_AWAIT_SUSPEND_NONTRIVIAL_ATTRIBUTES bool await_suspend(+        coroutine_handle<> continuation) noexcept {+      auto lock = mutex_->state_.lock();++      if (canLockUpgrade(*lock)) {+        lock->lockedFlagAndReaderCount_ |= kUpgradeLockFlag;+        return false;+      }++      continuation_ = continuation;+      *lock->waitersTailNext_ = this;+      lock->waitersTailNext_ = &nextAwaiter_;+      return true;+    }++    void await_resume() noexcept {}+  };++  class UnlockUpgradeAndLockAwaiter : public LockAwaiterBase {+   public:+    explicit UnlockUpgradeAndLockAwaiter(SharedMutexFair& mutex) noexcept+        : LockAwaiterBase(mutex, LockType::EXCLUSIVE) {}++    bool await_ready() noexcept {+      return mutex_->try_unlock_upgrade_and_lock();+    }++    FOLLY_CORO_AWAIT_SUSPEND_NONTRIVIAL_ATTRIBUTES bool await_suspend(+        coroutine_handle<> continuation) noexcept {+      auto lock = mutex_->state_.lock();++      assert(lock->lockedFlagAndReaderCount_ & kUpgradeLockFlag);+      if (lock->lockedFlagAndReaderCount_ == kUpgradeLockFlag) {+        lock->lockedFlagAndReaderCount_ = kExclusiveLockFlag;+        return false;+      }++      continuation_ = continuation;+      assert(lock->upgrader_ == nullptr);+      lock->upgrader_ = this;+      return true;+    }++    void await_resume() noexcept {}+  };++  class ScopedLockAwaiter : public LockAwaiter {+   public:+    using LockAwaiter::LockAwaiter;++    [[nodiscard]] std::unique_lock<SharedMutexFair> await_resume() noexcept {+      LockAwaiter::await_resume();+      return std::unique_lock<SharedMutexFair>{*mutex_, std::adopt_lock};+    }+  };++  class ScopedLockSharedAwaiter : public LockSharedAwaiter {+   public:+    using LockSharedAwaiter::LockSharedAwaiter;++    [[nodiscard]] SharedLock<SharedMutexFair> await_resume() noexcept {+      LockSharedAwaiter::await_resume();+      return SharedLock<SharedMutexFair>{*mutex_, std::adopt_lock};+    }+  };++  class ScopedLockUpgradeAwaiter : public LockUpgradeAwaiter {+   public:+    using LockUpgradeAwaiter::LockUpgradeAwaiter;++    [[nodiscard]] UpgradeLock<SharedMutexFair> await_resume() noexcept {+      LockUpgradeAwaiter::await_resume();+      return UpgradeLock<SharedMutexFair>{*mutex_, std::adopt_lock};+    }+  };++  class ScopedUnlockUpgradeAndLockAwaiter : public UnlockUpgradeAndLockAwaiter {+   public:+    using UnlockUpgradeAndLockAwaiter::UnlockUpgradeAndLockAwaiter;++    [[nodiscard]] std::unique_lock<SharedMutexFair> await_resume() noexcept {+      UnlockUpgradeAndLockAwaiter::await_resume();+      return std::unique_lock<SharedMutexFair>{*mutex_, std::adopt_lock};+    }+  };++  friend class LockAwaiter;++  template <typename Awaiter>+  class LockOperation {+   public:+    explicit LockOperation(SharedMutexFair& mutex) noexcept : mutex_(mutex) {}++    auto viaIfAsync(folly::Executor::KeepAlive<> executor) const {+      return folly::coro::co_viaIfAsync(std::move(executor), Awaiter{mutex_});+    }++   private:+    SharedMutexFair& mutex_;+  };++  // There is an invariant that if the mutex state is unlocked, there must be no+  // waiters; the converse is obviously not always true. This is guaranteed by+  // the `getWaitersToResume` function. If there are waiters after an unlock_*+  // operation, the mutex state will transition to a non-unlocked state.+  // This helps avoid a redundant check on the waiters list when the mutex is+  // unlocked.+  struct State {+    State() noexcept+        : lockedFlagAndReaderCount_(kUnlocked),+          waitingWriterCount_(0),+          waitersHead_(nullptr),+          upgrader_(nullptr),+          waitersTailNext_(&waitersHead_) {}++    // bit 0 - exclusive lock is held+    // bit 1 - upgrade lock is held+    // bits 2-[31/63] - count of held shared locks+    std::size_t lockedFlagAndReaderCount_;+    std::size_t waitingWriterCount_;+    LockAwaiterBase* waitersHead_;+    // active upgrade lock holder who's waiting to upgrade to exclusive+    // at most one waiter can be in such state+    LockAwaiterBase* upgrader_;+    LockAwaiterBase** waitersTailNext_;+  };++  static LockAwaiterBase* getWaitersToResume(+      State& state, LockType prevLockType) noexcept;+  static LockAwaiterBase* scanReadersAndUpgrader(+      LockAwaiterBase* head,+      State& lockedState,+      LockType prevLockType) noexcept;++  static void resumeWaiters(LockAwaiterBase* awaiters) noexcept;+  static bool canLockShared(const State& state) noexcept {+    // a shared lock can be acquired if there are no exclusive locks held,+    // exclusive lock pending or lock transition pending+    // an exclusive lock is pending if there are queued waiters for+    // it; there is a pending lock transition if there is active upgrade lock+    // waiting to be upgraded to exclusive+    return state.lockedFlagAndReaderCount_ == kUnlocked ||+        (state.lockedFlagAndReaderCount_ != kExclusiveLockFlag &&+         state.waitingWriterCount_ == 0 && state.upgrader_ == nullptr);+  }+  static bool canLockUpgrade(const State& state) noexcept {+    return state.lockedFlagAndReaderCount_ == kUnlocked ||+        ((state.lockedFlagAndReaderCount_ &+          (kExclusiveLockFlag | kUpgradeLockFlag)) == 0 &&+         state.waitingWriterCount_ == 0);+  }++  static constexpr std::size_t kUnlocked = 0;+  static constexpr std::size_t kExclusiveLockFlag = 1;+  static constexpr std::size_t kUpgradeLockFlag = 2;+  static constexpr std::size_t kSharedLockCountIncrement = 4;++  folly::Synchronized<State, folly::SpinLock> state_;+};++inline SharedMutexFair::LockOperation<SharedMutexFair::LockAwaiter>+SharedMutexFair::co_lock() noexcept {+  return LockOperation<LockAwaiter>{*this};+}++inline SharedMutexFair::LockOperation<SharedMutexFair::LockSharedAwaiter>+SharedMutexFair::co_lock_shared() noexcept {+  return LockOperation<LockSharedAwaiter>{*this};+}++inline SharedMutexFair::LockOperation<SharedMutexFair::ScopedLockAwaiter>+SharedMutexFair::co_scoped_lock() noexcept {+  return LockOperation<ScopedLockAwaiter>{*this};+}++inline SharedMutexFair::LockOperation<SharedMutexFair::ScopedLockSharedAwaiter>+SharedMutexFair::co_scoped_lock_shared() noexcept {+  return LockOperation<ScopedLockSharedAwaiter>{*this};+}++inline SharedMutexFair::LockOperation<SharedMutexFair::LockUpgradeAwaiter>+SharedMutexFair::co_lock_upgrade() noexcept {+  return LockOperation<LockUpgradeAwaiter>{*this};+}++inline SharedMutexFair::LockOperation<SharedMutexFair::ScopedLockUpgradeAwaiter>+SharedMutexFair::co_scoped_lock_upgrade() noexcept {+  return LockOperation<ScopedLockUpgradeAwaiter>{*this};+}++inline SharedMutexFair::LockOperation<+    SharedMutexFair::UnlockUpgradeAndLockAwaiter>+SharedMutexFair::co_unlock_upgrade_and_lock() noexcept {+  return LockOperation<UnlockUpgradeAndLockAwaiter>{*this};+}++inline SharedMutexFair::LockOperation<+    SharedMutexFair::ScopedUnlockUpgradeAndLockAwaiter>+SharedMutexFair::co_scoped_unlock_upgrade_and_lock() noexcept {+  return LockOperation<ScopedUnlockUpgradeAndLockAwaiter>{*this};+}++// The default SharedMutex is SharedMutexFair.+using SharedMutex = SharedMutexFair;++} // namespace coro+} // namespace folly++#endif // FOLLY_HAS_COROUTINES
+ folly/folly/coro/SharedPromise.h view
@@ -0,0 +1,201 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <cstddef>+#include <type_traits>+#include <utility>++#include <folly/Likely.h>+#include <folly/Synchronized.h>+#include <folly/Utility.h>+#include <folly/coro/Promise.h>+#include <folly/futures/Promise.h>+#include <folly/small_vector.h>++#if FOLLY_HAS_COROUTINES++namespace folly::coro {++/**+ * SharedPromise is a simple wrapper around folly::coro::Promise and+ * folly::coro::Future that allows for fetching cancellable and awaitable+ * futures from a single promise.+ *+ * It has the same behavior as folly::SharedPromise<>.  This includes the+ * difference in behavior of folly::SharedPromise and folly::Promise with+ * regards to invalid promise exceptions -- when SharedPromise<> is+ * moved from, calling setValue(), setTry(), or setException() don't result in+ * a PromiseInvalid exception.+ */+template <typename T>+class SharedPromise {+  using TryType = Try<lift_unit_t<T>>;++ public:+  /**+   * Constructors have behavior identical to folly::SharedPromise.+   */+  SharedPromise() = default;+  SharedPromise(SharedPromise&&) noexcept;+  SharedPromise& operator=(SharedPromise&&) noexcept;+  SharedPromise(const SharedPromise&) = delete;+  SharedPromise& operator=(const SharedPromise&) = delete;++  /**+   * Returns a future that is fulfilled when the user sets a value on the+   * promise.  Because this is a coro::Future, it supports cancellation.+   */+  folly::coro::Future<T> getFuture() const;++  /**+   * Returns the number of futures associated with the SharedPromise.+   */+  std::size_t size() const;++  /**+   * Returns true if the promise has either a value or an exception set.+   */+  bool isFulfilled() const;++  /**+   * Sets an exception in the promise.+   */+  void setException(folly::exception_wrapper&&);++  /**+   * Sets a value in the promise.+   */+  template <typename U = T>+  void setValue(U&&);+  template <typename U = T, typename = std::enable_if_t<std::is_void_v<U>>>+  void setValue();++  /**+   * Sets a folly::Try object in the promise.+   */+  void setTry(TryType&&);++ private:+  struct State {+    TryType result;+    folly::small_vector<folly::coro::Promise<T>> promises;+  };++  static bool isFulfilled(const State&);+  static void setTry(State&, TryType&&);++  mutable folly::Synchronized<State> state_;+};++template <typename T>+SharedPromise<T>::SharedPromise(SharedPromise&& other) noexcept+    : state_{std::exchange(*other.state_.wlock(), {})} {}++template <typename T>+SharedPromise<T>& SharedPromise<T>::operator=(SharedPromise&& other) noexcept {+  if (FOLLY_LIKELY(this != &other)) {+    synchronized(+        [](auto self, auto other) { *self = std::exchange(*other, {}); },+        wlock(state_),+        wlock(other.state_));+  }+  return *this;+}++template <typename T>+folly::coro::Future<T> SharedPromise<T>::getFuture() const {+  return state_.withWLock([&](auto& state) {+    // if the promise already has a value, then we just return a ready future+    if (isFulfilled(state)) {+      if constexpr (std::is_void_v<T>) {+        return state.result.hasValue()+            ? folly::coro::makeFuture()+            : folly::coro::makeFuture<void>(+                  folly::copy(state.result.exception()));+      } else {+        return state.result.hasValue()+            ? folly::coro::makeFuture<T>(folly::copy(state.result.value()))+            : folly::coro::makeFuture<T>(folly::copy(state.result.exception()));+      }+    }++    auto [promise, future] = folly::coro::makePromiseContract<T>();+    state.promises.push_back(std::move(promise));+    return std::move(future);+  });+}++template <typename T>+std::size_t SharedPromise<T>::size() const {+  return state_.withRLock([](auto& state) { return state.promises.size(); });+}++template <typename T>+bool SharedPromise<T>::isFulfilled() const {+  return state_.withRLock([](auto& state) { return isFulfilled(state); });+}++template <typename T>+void SharedPromise<T>::setException(folly::exception_wrapper&& exception) {+  state_.withWLock([&](auto& state) {+    setTry(state, TryType{std::move(exception)});+  });+}++template <typename T>+template <typename U>+void SharedPromise<T>::setValue(U&& input) {+  state_.withWLock([&](auto& state) {+    setTry(state, TryType{std::in_place, std::forward<U>(input)});+  });+}++template <typename T>+template <typename U, typename>+void SharedPromise<T>::setValue() {+  setTry(TryType{unit});+}++template <typename T>+void SharedPromise<T>::setTry(TryType&& result) {+  state_.withWLock([&](auto& state) { setTry(state, std::move(result)); });+}++template <typename T>+bool SharedPromise<T>::isFulfilled(const SharedPromise<T>::State& state) {+  return state.result.hasException() || state.result.hasValue();+}++template <typename T>+void SharedPromise<T>::setTry(+    SharedPromise<T>::State& state, TryType&& result) {+  if (isFulfilled(state)) {+    throw_exception<PromiseAlreadySatisfied>();+  }++  auto promises = std::exchange(state.promises, {});+  for (auto& promise : promises) {+    promise.setResult(folly::copy(result));+  }++  state.result = std::move(result);+}++} // namespace folly::coro++#endif
+ folly/folly/coro/Sleep-inl.h view
@@ -0,0 +1,46 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <folly/coro/FutureUtil.h>++#if FOLLY_HAS_COROUTINES++namespace folly {+namespace coro {++inline Task<void> sleep(HighResDuration d, Timekeeper* tk) {+  //  using via with the current executor is observed to deadlock in some cases,+  //  so convert to future without via and thereby bypass conversion using via+  //  in the overload of toTaskInterruptOnCancel taking semi-future; woroks only+  //  since sleep() returns a semi-future without any deferred work attached+  auto f = folly::futures::sleep(d, tk).toUnsafeFuture();+  co_await co_nothrow(toTaskInterruptOnCancel(std::move(f)));+}++inline Task<void> sleepReturnEarlyOnCancel(HighResDuration d, Timekeeper* tk) {+  auto result = co_await co_awaitTry(sleep(d, tk));+  if (result.hasException<OperationCancelled>()) {+    co_return;+  }+  co_yield co_result(std::move(result));+}++} // namespace coro+} // namespace folly++#endif // FOLLY_HAS_COROUTINES
+ folly/folly/coro/Sleep.h view
@@ -0,0 +1,46 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <folly/coro/Coroutine.h>+#include <folly/coro/Task.h>+#include <folly/futures/Future.h>++#if FOLLY_HAS_COROUTINES++namespace folly {+namespace coro {++/// Return a task that, when awaited, will sleep for the specified duration.+///+/// Throws folly::OperationCancelled if cancellation is requested on the+/// awaiting coroutine's associated CancellationToken.+Task<void> sleep(HighResDuration d, Timekeeper* tk = nullptr);++/// Return a task that, when awaited, will sleep for the specified duration.+///+/// May complete sooner that the specified duration if cancellation is requested+/// on the awaiting coroutine's associated CancellationToken.+Task<void> sleepReturnEarlyOnCancel(+    HighResDuration d, Timekeeper* tk = nullptr);++} // namespace coro+} // namespace folly++#endif // FOLLY_HAS_COROUTINES++#include <folly/coro/Sleep-inl.h>
+ folly/folly/coro/SmallUnboundedQueue.h view
@@ -0,0 +1,96 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <folly/coro/Baton.h>+#include <folly/coro/Coroutine.h>+#include <folly/coro/Mutex.h>+#include <folly/coro/Task.h>+#include <folly/experimental/channels/detail/AtomicQueue.h>++#if FOLLY_HAS_COROUTINES++namespace folly {+namespace coro {+namespace detail {+template <bool UseMutex>+struct SmallUnboundedQueueBase {+  auto co_scoped_lock() { return ready_awaitable(true); }+};+template <>+struct SmallUnboundedQueueBase<true> {+  auto co_scoped_lock() { return mutex_.co_scoped_lock(); }+  folly::coro::Mutex mutex_;+};+} // namespace detail++// Alternative to coro::UnboundedQueue with much smaller memory size when empty+// but lower throughput.+// Substantially worse in multi-consumer case.+// Only supports enqueue(T) and dequeue().++template <typename T, bool SingleProducer = false, bool SingleConsumer = false>+class SmallUnboundedQueue : detail::SmallUnboundedQueueBase<!SingleConsumer> {+  struct Consumer {+    void consume() { baton.post(); }+    void canceled() { std::terminate(); }+    folly::coro::Baton baton;+  };++ public:+  ~SmallUnboundedQueue() { queue_.close(); }++  template <typename U = T>+  void enqueue(U&& val) {+    queue_.push(T(std::forward<U>(val)));+  }++  folly::coro::Task<T> dequeue() {+    [[maybe_unused]] auto maybeLock = co_await this->co_scoped_lock();+    if (buffer_.empty()) {+      Consumer c;+      if (queue_.wait(&c)) {+        bool cancelled = false;+        CancellationCallback cb(co_await co_current_cancellation_token, [&] {+          if (queue_.cancelCallback()) {+            cancelled = true;+            c.baton.post();+          }+        });+        co_await c.baton;+        if (cancelled) {+          co_yield co_cancelled;+        }+      }+      buffer_ = queue_.getMessages();+      DCHECK(!buffer_.empty());+    }+    SCOPE_EXIT {+      buffer_.pop();+    };+    co_return std::move(buffer_.front());+  }++ private:+  folly::channels::detail::AtomicQueue<Consumer, T> queue_;+  folly::channels::detail::Queue<T> buffer_;+};++} // namespace coro+} // namespace folly++#endif // FOLLY_HAS_COROUTINES
+ folly/folly/coro/Synchronized.h view
@@ -0,0 +1,277 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <functional>+#include <mutex>+#include <utility>++#include <folly/Utility.h>+#include <folly/coro/SharedLock.h>+#include <folly/coro/SharedMutex.h>+#include <folly/coro/Task.h>+#include <folly/coro/Traits.h>++namespace folly::coro {++namespace detail {++template <typename CoroMutexType>+struct SynchronizedMutexTraits;++template <>+struct SynchronizedMutexTraits<SharedMutexFair> {+  using CoroMutex = SharedMutexFair;+  using ReadLock = SharedLock<CoroMutex>;+  using WriteLock = std::unique_lock<CoroMutex>;++  static inline auto co_readLock(CoroMutex& mutex) {+    return mutex.co_scoped_lock_shared();+  }++  static inline ReadLock tryReadLock(CoroMutex& mutex) noexcept(+      noexcept(ReadLock(mutex, std::try_to_lock))) {+    return ReadLock(mutex, std::try_to_lock);+  }++  static inline auto co_writeLock(CoroMutex& mutex) {+    return mutex.co_scoped_lock();+  }++  static inline auto tryWriteLock(CoroMutex& mutex) noexcept(+      noexcept(WriteLock(mutex, std::try_to_lock))) {+    return WriteLock(mutex, std::try_to_lock);+  }++  static inline void unlock(ReadLock& lock) noexcept(noexcept(lock.unlock())) {+    lock.unlock();+  }++  static inline void unlock(WriteLock& lock) noexcept(noexcept(lock.unlock())) {+    lock.unlock();+  }++  static inline auto ownsLock(const ReadLock& lock) noexcept(+      noexcept(lock.owns_lock())) {+    return lock.owns_lock();+  }++  static inline auto ownsLock(const WriteLock& lock) noexcept(+      noexcept(lock.owns_lock())) {+    return lock.owns_lock();+  }+};++} // namespace detail++/**+ * This class is an adaptation of the folly::Synchronized class but is designed+ * to work with coro-compatible mutexes like coro::SharedMutexFair instead.+ *+ * In practice what this means is+ * that we can co_await gaining the read/write lock rather than blocking whilst+ * acquiring it.+ *+ * The API is not a complete clone of everything that folly::Synchronized+ * supports but is instead the minimum of what we need. Ultimately this classes+ * main job is to abstract away gaining the locks.+ */+template <+    typename Inner,+    typename CoroMutexType = SharedMutexFair,+    typename CoroMutexTraits = detail::SynchronizedMutexTraits<CoroMutexType>>+class Synchronized : public NonCopyableNonMovable {+ public:+  using Traits = CoroMutexTraits;+  using CoroMutex = typename Traits::CoroMutex;+  using ReadLock = typename Traits::ReadLock;+  using WriteLock = typename Traits::WriteLock;++  Synchronized() noexcept(noexcept(CoroMutex{}) && noexcept(Inner{})) = default;++  explicit Synchronized(const Inner& rhs) noexcept(noexcept(Inner(rhs)))+      : inner_(rhs) {}++  explicit Synchronized(Inner&& rhs) noexcept(noexcept(Inner(std::move(rhs))))+      : inner_(std::move(rhs)) {}++  template <typename... Args>+  explicit Synchronized(std::in_place_t, Args&&... args)+      : inner_(std::forward<Args>(args)...) {}++  /**+   * A RAII wrapper around a pointer to the underlying object together with+   * a lock on the underlying mutex.+   *+   * If acquired with a try-lock style method, you must check the boolean+   * value of the locked pointer before dereferencing it.+   */+  template <typename ValueType, typename LockType>+  class GenericLockedPtr : public MoveOnly {+   public:+    GenericLockedPtr(GenericLockedPtr&& other) noexcept(+        noexcept(LockType(std::move(other.lock_))))+        : lock_(std::move(other.lock_)),+          ptr_(std::exchange(other.ptr_, nullptr)) {}++    GenericLockedPtr& operator=(GenericLockedPtr&& other) noexcept(+        noexcept(lock_ = std::move(other.lock_))) {+      if (this != &other) {+        lock_ = std::move(other.lock_);+        ptr_ = std::exchange(other.ptr_, nullptr);+      }+      return *this;+    }++    ValueType* operator->() const noexcept {+      DCHECK_NE(ptr_, nullptr);+      return ptr_;+    }++    ValueType& operator*() const noexcept {+      DCHECK_NE(ptr_, nullptr);+      return *ptr_;+    }++    void unlock() {+      DCHECK_NE(ptr_, nullptr);+      ptr_ = nullptr;+      Traits::unlock(lock_);+    }++    explicit operator bool() const noexcept { return Traits::ownsLock(lock_); }++   private:+    friend class Synchronized;+    explicit GenericLockedPtr(LockType&& lock, ValueType* ptr)+        : lock_(std::move(lock)), ptr_(ptr) {}++    LockType lock_;+    ValueType* ptr_ = nullptr;+  };++  using ReadLockedPtr = GenericLockedPtr<const Inner, ReadLock>;+  using WriteLockedPtr = GenericLockedPtr<Inner, WriteLock>;++  Task<WriteLockedPtr> wLock() {+    auto lock = co_await Traits::co_writeLock(mutex_);+    co_return WriteLockedPtr{std::move(lock), &inner_};+  }++  Task<ReadLockedPtr> rLock() const {+    auto lock = co_await Traits::co_readLock(mutex_);+    co_return ReadLockedPtr{std::move(lock), &inner_};+  }++  ReadLockedPtr tryRLock() const {+    auto lock = Traits::tryReadLock(mutex_);+    auto* ptr = Traits::ownsLock(lock) ? &inner_ : nullptr;+    return ReadLockedPtr{std::move(lock), ptr};+  }++  WriteLockedPtr tryWLock() {+    auto lock = WriteLock{mutex_, std::try_to_lock};+    auto* ptr = Traits::ownsLock(lock) ? &inner_ : nullptr;+    return WriteLockedPtr{std::move(lock), ptr};+  }++  template <typename FuncT>+  using rlock_result_t = std::invoke_result_t<FuncT, ReadLockedPtr>;++  template <typename FuncT>+  using wlock_result_t = std::invoke_result_t<FuncT, WriteLockedPtr>;++  template <typename FuncT, typename ReturnT = rlock_result_t<FuncT>>+  typename std::enable_if<!is_semi_awaitable_v<ReturnT>, Task<ReturnT>>::type+  withRLock(FuncT func) const {+    auto lock = co_await Traits::co_readLock(mutex_);+    co_return func(ReadLockedPtr{std::move(lock), &inner_});+  }++  template <typename FuncT, typename ReturnT = rlock_result_t<FuncT>>+  typename std::enable_if<+      is_semi_awaitable_v<ReturnT>,+      Task<semi_await_result_t<ReturnT>>>::type+  withRLock(FuncT func) const {+    auto lock = co_await Traits::co_readLock(mutex_);+    co_return co_await func(ReadLockedPtr{std::move(lock), &inner_});+  }++  template <typename FuncT, typename ReturnT = wlock_result_t<FuncT>>+  typename std::enable_if<!is_semi_awaitable_v<ReturnT>, Task<ReturnT>>::type+  withWLock(FuncT func) {+    auto lock = co_await Traits::co_writeLock(mutex_);+    co_return func(WriteLockedPtr{std::move(lock), &inner_});+  }++  template <typename FuncT, typename ReturnT = wlock_result_t<FuncT>>+  typename std::enable_if<+      is_semi_awaitable_v<ReturnT>,+      Task<semi_await_result_t<ReturnT>>>::type+  withWLock(FuncT func) {+    auto lock = co_await Traits::co_writeLock(mutex_);+    co_return co_await func(WriteLockedPtr{std::move(lock), &inner_});+  }++  /**+   * Temporarlily locks both objects and swaps their underlying data.+   *+   * Mimics the behaviour of folly::Synchronized in that we return early if you+   * try to swap with itself and gains locks in ascending memory order to+   * prevent deadlocks.+   */+  Task<void> swap(Synchronized& rhs) {+    if (this == &rhs) {+      co_return;+    }++    // Can't compare pointers for inequality with operator> because it's+    // unspecified behavior unless they share provenance, see:+    // - https://en.wikipedia.org/wiki/Unspecified_behavior,+    // - https://en.cppreference.com/w/cpp/language/operator_comparison.+    if (std::greater<>()(this, &rhs)) {+      co_return co_await rhs.swap(*this);+    }++    auto guard1 = co_await wLock();+    auto guard2 = co_await rhs.wLock();++    using std::swap;+    swap(inner_, rhs.inner_);++    co_return;+  }++  Task<Inner> copy() const {+    auto lock = co_await Traits::co_readLock(mutex_);+    Inner res = folly::copy(inner_);+    co_return res;+  }++  Task<void> swap(Inner& newInner) {+    auto lock = co_await Traits::co_writeLock(mutex_);++    using std::swap;+    swap(inner_, newInner);+  }++ private:+  mutable CoroMutex mutex_;+  Inner inner_;+};++} // namespace folly::coro
+ folly/folly/coro/Task.h view
@@ -0,0 +1,1013 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++//+// Docs: https://fburl.com/fbcref_coro_task+//++#pragma once++#include <exception>+#include <type_traits>++#include <glog/logging.h>++#include <folly/CancellationToken.h>+#include <folly/DefaultKeepAliveExecutor.h>+#include <folly/Executor.h>+#include <folly/GLog.h>+#include <folly/Portability.h>+#include <folly/ScopeGuard.h>+#include <folly/Traits.h>+#include <folly/Try.h>+#include <folly/coro/AwaitImmediately.h>+#include <folly/coro/Coroutine.h>+#include <folly/coro/CurrentExecutor.h>+#include <folly/coro/Invoke.h>+#include <folly/coro/Result.h>+#include <folly/coro/ScopeExit.h>+#include <folly/coro/Traits.h>+#include <folly/coro/ViaIfAsync.h>+#include <folly/coro/WithAsyncStack.h>+#include <folly/coro/WithCancellation.h>+#include <folly/coro/detail/InlineTask.h>+#include <folly/coro/detail/Malloc.h>+#include <folly/coro/detail/Traits.h>+#include <folly/futures/Future.h>+#include <folly/io/async/Request.h>+#include <folly/lang/Assume.h>+#include <folly/lang/SafeAlias-fwd.h>+#include <folly/result/result.h>+#include <folly/result/try.h>+#include <folly/tracing/AsyncStack.h>++#if FOLLY_HAS_COROUTINES++namespace folly::coro {++template <typename T = void>+class Task;++template <typename T = void>+class TaskWithExecutor;++namespace detail {++class TaskPromiseBase;++class TaskPromisePrivate {+ private:+  friend TaskPromiseBase;+  TaskPromisePrivate() = default;+};++class TaskPromiseBase {+  static TaskPromisePrivate privateTag() { return TaskPromisePrivate{}; }++  class FinalAwaiter {+   public:+    bool await_ready() noexcept { return false; }++    template <typename Promise>+    FOLLY_CORO_AWAIT_SUSPEND_NONTRIVIAL_ATTRIBUTES coroutine_handle<>+    await_suspend(coroutine_handle<Promise> coro) noexcept {+      auto& promise = coro.promise();+      // If ScopeExitTask has been attached, then we expect that the+      // ScopeExitTask will handle the lifetime of the async stack. See+      // ScopeExitTaskPromise's FinalAwaiter for more details.+      //+      // This is a bit untidy, and hopefully something we can replace with+      // a virtual wrapper over coroutine_handle that handles the pop for us.+      if (promise.scopeExitRef(privateTag())) {+        promise.scopeExitRef(privateTag())+            .promise()+            .setContext(+                promise.continuationRef(privateTag()),+                &promise.getAsyncFrame(),+                promise.executorRef(privateTag()).get_alias(),+                promise.result().hasException()+                    ? promise.result().exception()+                    : exception_wrapper{});+        return promise.scopeExitRef(privateTag());+      }++      folly::popAsyncStackFrameCallee(promise.getAsyncFrame());+      if (promise.result().hasException()) {+        auto [handle, frame] =+            promise.continuationRef(privateTag())+                .getErrorHandle(promise.result().exception());+        return handle.getHandle();+      }+      return promise.continuationRef(privateTag()).getHandle();+    }++    [[noreturn]] void await_resume() noexcept { folly::assume_unreachable(); }+  };++  friend class FinalAwaiter;++ protected:+  TaskPromiseBase() noexcept = default;+  ~TaskPromiseBase() = default;++  template <typename Promise>+  variant_awaitable<FinalAwaiter, ready_awaitable<>> do_safe_point(+      Promise& promise) noexcept {+    if (cancelToken_.isCancellationRequested()) {+      return promise.yield_value(co_cancelled);+    }+    return ready_awaitable<>{};+  }++ public:+  static void* operator new(std::size_t size) {+    return ::folly_coro_async_malloc(size);+  }++  static void operator delete(void* ptr, std::size_t size) {+    ::folly_coro_async_free(ptr, size);+  }++  suspend_always initial_suspend() noexcept { return {}; }++  FinalAwaiter final_suspend() noexcept { return {}; }++  template <+      typename Awaitable,+      std::enable_if_t<!must_await_immediately_v<Awaitable>, int> = 0>+  auto await_transform(Awaitable&& awaitable) {+    bypassExceptionThrowing_ =+        bypassExceptionThrowing_ == BypassExceptionThrowing::REQUESTED+        ? BypassExceptionThrowing::ACTIVE+        : BypassExceptionThrowing::INACTIVE;++    return folly::coro::co_withAsyncStack(folly::coro::co_viaIfAsync(+        executor_.get_alias(),+        folly::coro::co_withCancellation(+            cancelToken_, static_cast<Awaitable&&>(awaitable))));+  }+  template <+      typename Awaitable,+      std::enable_if_t<must_await_immediately_v<Awaitable>, int> = 0>+  auto await_transform(Awaitable awaitable) {+    bypassExceptionThrowing_ =+        bypassExceptionThrowing_ == BypassExceptionThrowing::REQUESTED+        ? BypassExceptionThrowing::ACTIVE+        : BypassExceptionThrowing::INACTIVE;++    return folly::coro::co_withAsyncStack(folly::coro::co_viaIfAsync(+        executor_.get_alias(),+        folly::coro::co_withCancellation(+            cancelToken_,+            mustAwaitImmediatelyUnsafeMover(std::move(awaitable))())));+  }++  template <typename Awaitable>+  auto await_transform(NothrowAwaitable<Awaitable> awaitable) {+    bypassExceptionThrowing_ = BypassExceptionThrowing::REQUESTED;+    return await_transform(+        mustAwaitImmediatelyUnsafeMover(awaitable.unwrap())());+  }++  auto await_transform(co_current_executor_t) noexcept {+    return ready_awaitable<folly::Executor*>{executor_.get()};+  }++  auto await_transform(co_current_cancellation_token_t) noexcept {+    return ready_awaitable<const folly::CancellationToken&>{cancelToken_};+  }++  void setCancelToken(folly::CancellationToken&& cancelToken) noexcept {+    if (!hasCancelTokenOverride_) {+      cancelToken_ = std::move(cancelToken);+      hasCancelTokenOverride_ = true;+    }+  }++  folly::AsyncStackFrame& getAsyncFrame() noexcept { return asyncFrame_; }++  folly::Executor::KeepAlive<> getExecutor() const noexcept {+    return executor_;+  }++  // These getters exist so that `FinalAwaiter` can interact with wrapped+  // `TaskPromise`s, and not just `TaskPromiseBase` descendants.  We use a+  // private tag to let `TaskWrapper` call them without becoming a `friend`.+  auto& scopeExitRef(TaskPromisePrivate) { return scopeExit_; }+  auto& continuationRef(TaskPromisePrivate) { return continuation_; }+  // Unlike `getExecutor()`, does not copy an atomic.+  auto& executorRef(TaskPromisePrivate) { return executor_; }++ private:+  template <typename>+  friend class folly::coro::TaskWithExecutor;++  template <typename>+  friend class folly::coro::Task;++  friend coroutine_handle<ScopeExitTaskPromiseBase> tag_invoke(+      cpo_t<co_attachScopeExit>,+      TaskPromiseBase& p,+      coroutine_handle<ScopeExitTaskPromiseBase> scopeExit) noexcept {+    return std::exchange(p.scopeExit_, scopeExit);+  }++  ExtendedCoroutineHandle continuation_;+  folly::AsyncStackFrame asyncFrame_;+  folly::Executor::KeepAlive<> executor_;+  folly::CancellationToken cancelToken_;+  coroutine_handle<ScopeExitTaskPromiseBase> scopeExit_;+  bool hasCancelTokenOverride_ = false;++ protected:+  enum class BypassExceptionThrowing : uint8_t {+    INACTIVE,+    ACTIVE,+    REQUESTED,+  } bypassExceptionThrowing_{BypassExceptionThrowing::INACTIVE};+};++// Separate from `TaskPromiseBase` so the compiler has less to specialize.+template <typename Promise, typename T>+class TaskPromiseCrtpBase+    : public TaskPromiseBase,+      public ExtendedCoroutinePromise {+ public:+  using StorageType = detail::lift_lvalue_reference_t<T>;++  Task<T> get_return_object() noexcept;++  void unhandled_exception() noexcept {+    result_.emplaceException(exception_wrapper{current_exception()});+  }++  Try<StorageType>& result() { return result_; }++  auto yield_value(co_error ex) {+    result_.emplaceException(std::move(ex.exception()));+    return final_suspend();+  }++  auto yield_value(co_result<StorageType>&& result) {+    result_ = std::move(result.result());+    return final_suspend();+  }++  using TaskPromiseBase::await_transform;++  auto await_transform(co_safe_point_t) noexcept {+    return do_safe_point(*this);+  }++ protected:+  TaskPromiseCrtpBase() noexcept = default;+  ~TaskPromiseCrtpBase() = default;++  std::pair<ExtendedCoroutineHandle, AsyncStackFrame*> getErrorHandle(+      exception_wrapper& ex) final {+    auto& me = *static_cast<Promise*>(this);+    if (bypassExceptionThrowing_ == BypassExceptionThrowing::ACTIVE) {+      auto finalAwaiter = yield_value(co_error(std::move(ex)));+      DCHECK(!finalAwaiter.await_ready());+      return {+          finalAwaiter.await_suspend(+              coroutine_handle<Promise>::from_promise(me)),+          // finalAwaiter.await_suspend pops a frame+          getAsyncFrame().getParentFrame()};+    }+    return {coroutine_handle<Promise>::from_promise(me), nullptr};+  }++  Try<StorageType> result_;+};++template <typename T>+class TaskPromise final : public TaskPromiseCrtpBase<TaskPromise<T>, T> {+ public:+  static_assert(+      !std::is_rvalue_reference_v<T>,+      "Task<T&&> is not supported. "+      "Consider using Task<T> or Task<std::unique_ptr<T>> instead.");+  friend class TaskPromiseBase;++  using StorageType =+      typename TaskPromiseCrtpBase<TaskPromise<T>, T>::StorageType;++  TaskPromise() noexcept = default;++  template <typename U = T>+  void return_value(U&& value) {+    if constexpr (std::is_same_v<remove_cvref_t<U>, Try<StorageType>>) {+      DCHECK(value.hasValue() || (value.hasException() && value.exception()));+      this->result_ = static_cast<U&&>(value);+    } else if constexpr (+        std::is_same_v<remove_cvref_t<U>, Try<void>> &&+        std::is_same_v<remove_cvref_t<T>, Unit>) {+      // special-case to make task -> semifuture -> task preserve void type+      DCHECK(value.hasValue() || (value.hasException() && value.exception()));+      this->result_ = static_cast<Try<Unit>>(static_cast<U&&>(value));+    } else {+      static_assert(+          std::is_convertible<U&&, StorageType>::value,+          "cannot convert return value to type T");+      this->result_.emplace(static_cast<U&&>(value));+    }+  }+};++template <>+class TaskPromise<void> final+    : public TaskPromiseCrtpBase<TaskPromise<void>, void> {+ public:+  friend class TaskPromiseBase;++  using StorageType = void;++  TaskPromise() noexcept = default;++  void return_void() noexcept { this->result_.emplace(); }++  using TaskPromiseCrtpBase<TaskPromise<void>, void>::yield_value;++  auto yield_value(co_result<Unit>&& result) {+    this->result_ = std::move(result.result());+    return final_suspend();+  }+};++namespace adl {+// ADL should prefer your `friend co_withExecutor` over this dummy overload.+void co_withExecutor();+// This CPO deliberately does NOT use `tag_invoke`, but rather reuses the+// `co_withExecutor` name as the ADL implementation, just like `co_viaIfAsync`.+// The reason is that `tag_invoke()` would plumb through `Awaitable&&` instead+// of `Awaitable`, but `must_await_immediately_v` types require by-value.+struct WithExecutorFunction {+  template <typename Awaitable>+  // Pass `awaitable` by-value, since `&&` would break immediate types+  auto operator()(Executor::KeepAlive<> executor, Awaitable awaitable) const+      FOLLY_DETAIL_FORWARD_BODY(co_withExecutor(+          std::move(executor),+          mustAwaitImmediatelyUnsafeMover(std::move(awaitable))()))+};+} // namespace adl++} // namespace detail++// Semi-awaitables like `Task` should use this CPO to attach executors:+//   auto taskWithExec = co_withExecutor(std::move(exec), std::move(task));+FOLLY_DEFINE_CPO(detail::adl::WithExecutorFunction, co_withExecutor)++/// Represents an allocated but not yet started coroutine that has already+/// been bound to an executor.+///+/// This task, when co_awaited, will launch the task on the bound executor+/// and will resume the awaiting coroutine on the bound executor when it+/// completes.+///+/// More information on how to use this is available at folly::coro::Task.+template <typename T>+class FOLLY_NODISCARD TaskWithExecutor {+  using handle_t = coroutine_handle<detail::TaskPromise<T>>;+  using StorageType = typename detail::TaskPromise<T>::StorageType;++ public:+  /// @private+  ~TaskWithExecutor() {+    if (coro_) {+      coro_.destroy();+    }+  }++  TaskWithExecutor(TaskWithExecutor&& t) noexcept+      : coro_(std::exchange(t.coro_, {})) {}++  TaskWithExecutor& operator=(TaskWithExecutor t) noexcept {+    swap(t);+    return *this;+  }+  /// Returns the executor that the task is bound to+  folly::Executor* executor() const noexcept {+    return coro_.promise().executor_.get();+  }++  void swap(TaskWithExecutor& t) noexcept { std::swap(coro_, t.coro_); }++  /// Start eager execution of this task.+  ///+  /// This starts execution of the Task on the bound executor.+  /// @returns folly::SemiFuture<T> that will complete with the result.+  FOLLY_NOINLINE SemiFuture<lift_unit_t<StorageType>> start() && {+    folly::Promise<lift_unit_t<StorageType>> p;++    auto sf = p.getSemiFuture();++    std::move(*this).startImpl(+        [promise = std::move(p)](Try<StorageType>&& result) mutable {+          promise.setTry(std::move(result));+        },+        folly::CancellationToken{},+        FOLLY_ASYNC_STACK_RETURN_ADDRESS());++    return sf;+  }++  /// Start eager execution of the task and call the passed callback on+  /// completion+  ///+  /// This starts execution of the Task on the bound executor, and call the+  /// passed callback upon completion. The callback takes a Try<T> which+  /// represents either th value returned by the Task on success or an+  /// exception thrown by the Task+  /// @param tryCallback a function that takes in a Try<T>+  /// @param cancelToken a CancelationToken object+  template <typename F>+  FOLLY_NOINLINE void start(+      F&& tryCallback, folly::CancellationToken cancelToken = {}) && {+    std::move(*this).startImpl(+        static_cast<F&&>(tryCallback),+        std::move(cancelToken),+        FOLLY_ASYNC_STACK_RETURN_ADDRESS());+  }++  /// Start eager execution of this task on this thread.+  ///+  /// Assumes the current thread is already on the executor associated with the+  /// Task. Refer to TaskWithExecuter::start(F&& tryCallback,+  /// folly::CancellationToken cancelToken = {}) for more information.+  template <typename F>+  FOLLY_NOINLINE void startInlineUnsafe(+      F&& tryCallback, folly::CancellationToken cancelToken = {}) && {+    std::move(*this).startInlineImpl(+        static_cast<F&&>(tryCallback),+        std::move(cancelToken),+        FOLLY_ASYNC_STACK_RETURN_ADDRESS());+  }++  /// Start eager execution of this task on this thread.+  ///+  /// Assumes the current thread is already on the executor associated with the+  /// Task. Refer to TaskWithExecuter::start() for more information.+  FOLLY_NOINLINE SemiFuture<lift_unit_t<StorageType>> startInlineUnsafe() && {+    folly::Promise<lift_unit_t<StorageType>> p;++    auto sf = p.getSemiFuture();++    std::move(*this).startInlineImpl(+        [promise = std::move(p)](Try<StorageType>&& result) mutable {+          promise.setTry(std::move(result));+        },+        folly::CancellationToken{},+        FOLLY_ASYNC_STACK_RETURN_ADDRESS());++    return sf;+  }++ private:+  template <typename F>+  void startImpl(+      F&& tryCallback,+      folly::CancellationToken cancelToken,+      void* returnAddress) && {+    coro_.promise().setCancelToken(std::move(cancelToken));+    startImpl(std::move(*this), static_cast<F&&>(tryCallback))+        .start(returnAddress);+  }++  template <typename F>+  void startInlineImpl(+      F&& tryCallback,+      folly::CancellationToken cancelToken,+      void* returnAddress) && {+    coro_.promise().setCancelToken(std::move(cancelToken));+    // If the task replaces the request context and reaches a suspension point,+    // it will not have a chance to restore the previous context before we+    // return, so we need to ensure it is restored. This simulates starting the+    // coroutine in an actual executor, which would wrap the task with a guard.+    RequestContextScopeGuard contextScope{RequestContext::saveContext()};+    startInlineImpl(std::move(*this), static_cast<F&&>(tryCallback))+        .start(returnAddress);+  }++  template <typename F>+  detail::InlineTaskDetached startImpl(TaskWithExecutor task, F cb) {+    try {+      cb(co_await folly::coro::co_awaitTry(std::move(task)));+    } catch (...) {+      cb(Try<StorageType>(exception_wrapper(current_exception())));+    }+  }++  template <typename F>+  detail::InlineTaskDetached startInlineImpl(TaskWithExecutor task, F cb) {+    try {+      cb(co_await InlineTryAwaitable{std::exchange(task.coro_, {})});+    } catch (...) {+      cb(Try<StorageType>(exception_wrapper(current_exception())));+    }+  }++ public:+  class Awaiter {+   public:+    explicit Awaiter(handle_t coro) noexcept : coro_(coro) {}++    Awaiter(Awaiter&& other) noexcept : coro_(std::exchange(other.coro_, {})) {}++    ~Awaiter() {+      if (coro_) {+        coro_.destroy();+      }+    }++    bool await_ready() const noexcept { return false; }++    template <typename Promise>+    FOLLY_NOINLINE void await_suspend(+        coroutine_handle<Promise> continuation) noexcept {+      DCHECK(coro_);+      auto& promise = coro_.promise();+      DCHECK(!promise.continuation_);+      DCHECK(promise.executor_);+      DCHECK(!dynamic_cast<folly::InlineExecutor*>(promise.executor_.get()))+          << "InlineExecutor is not safe and is not supported for coro::Task. "+          << "If you need to run a task inline in a unit-test, you should use "+          << "coro::blockingWait instead.";+      DCHECK(!dynamic_cast<folly::QueuedImmediateExecutor*>(+          promise.executor_.get()))+          << "QueuedImmediateExecutor is not safe and is not supported for coro::Task. "+          << "If you need to run a task inline in a unit-test, you should use "+          << "coro::blockingWait instead.";+      if constexpr (kIsDebug) {+        if (dynamic_cast<InlineLikeExecutor*>(promise.executor_.get())) {+          FB_LOG_ONCE(ERROR)+              << "InlineLikeExecutor is not safe and is not supported for coro::Task. "+              << "If you need to run a task inline in a unit-test, you should use "+              << "coro::blockingWait or write your test using the CO_TEST* macros instead."+              << "If you are using folly::getCPUExecutor, switch to getGlobalCPUExecutor "+              << "or be sure to call setCPUExecutor first.";+        }+        if (dynamic_cast<folly::DefaultKeepAliveExecutor::WeakRefExecutor*>(+                promise.executor_.get())) {+          FB_LOG_ONCE(ERROR)+              << "You are scheduling a coro::Task on a weak executor. "+              << "It is not supported, and can lead to memory leaks. "+              << "Consider using CancellationToken instead.";+        }+      }++      auto& calleeFrame = promise.getAsyncFrame();+      calleeFrame.setReturnAddress();++      if constexpr (detail::promiseHasAsyncFrame_v<Promise>) {+        auto& callerFrame = continuation.promise().getAsyncFrame();+        calleeFrame.setParentFrame(callerFrame);+        folly::deactivateAsyncStackFrame(callerFrame);+      }++      promise.continuation_ = continuation;+      promise.executor_->add(+          [coro = coro_, ctx = RequestContext::saveContext()]() mutable {+            RequestContextScopeGuard contextScope{std::move(ctx)};+            folly::resumeCoroutineWithNewAsyncStackRoot(coro);+          });+    }++    T await_resume() {+      DCHECK(coro_);+      // Eagerly destroy the coroutine-frame once we have retrieved the result.+      SCOPE_EXIT {+        std::exchange(coro_, {}).destroy();+      };+      return std::move(coro_.promise().result()).value();+    }++    folly::Try<StorageType> await_resume_try() noexcept(+        std::is_nothrow_move_constructible_v<StorageType>) {+      SCOPE_EXIT {+        std::exchange(coro_, {}).destroy();+      };+      return std::move(coro_.promise().result());+    }++#if FOLLY_HAS_RESULT+    result<T> await_resume_result() noexcept(+        std::is_nothrow_move_constructible_v<StorageType>) {+      SCOPE_EXIT {+        std::exchange(coro_, {}).destroy();+      };+      return try_to_result(std::move(coro_.promise().result()));+    }+#endif++   private:+    handle_t coro_;+  };++  class InlineTryAwaitable {+   public:+    InlineTryAwaitable(handle_t coro) noexcept : coro_(coro) {}++    InlineTryAwaitable(InlineTryAwaitable&& other) noexcept+        : coro_(std::exchange(other.coro_, {})) {}++    ~InlineTryAwaitable() {+      if (coro_) {+        coro_.destroy();+      }+    }++    bool await_ready() noexcept { return false; }++    template <typename Promise>+    FOLLY_NOINLINE coroutine_handle<> await_suspend(+        coroutine_handle<Promise> continuation) {+      DCHECK(coro_);+      auto& promise = coro_.promise();+      DCHECK(!promise.continuation_);+      DCHECK(promise.executor_);++      promise.continuation_ = continuation;++      auto& calleeFrame = promise.getAsyncFrame();+      calleeFrame.setReturnAddress();++      // This awaitable is only ever awaited from a DetachedInlineTask+      // which is an async-stack-aware coroutine.+      //+      // Assume it has a .getAsyncFrame() and that this frame is currently+      // active.+      auto& callerFrame = continuation.promise().getAsyncFrame();+      folly::pushAsyncStackFrameCallerCallee(callerFrame, calleeFrame);+      return coro_;+    }++    folly::Try<StorageType> await_resume() {+      DCHECK(coro_);+      // Eagerly destroy the coroutine-frame once we have retrieved the result.+      SCOPE_EXIT {+        std::exchange(coro_, {}).destroy();+      };+      return std::move(coro_.promise().result());+    }++   private:+    friend InlineTryAwaitable tag_invoke(+        cpo_t<co_withAsyncStack>, InlineTryAwaitable&& awaitable) noexcept {+      return std::move(awaitable);+    }++    handle_t coro_;+  };++ public:+  Awaiter operator co_await() && noexcept {+    DCHECK(coro_);+    return Awaiter{std::exchange(coro_, {})};+  }++  std::pair<Task<T>, Executor::KeepAlive<>> unwrap() && {+    auto executor = std::move(coro_.promise().executor_);+    Task<T> task{std::exchange(coro_, {})};+    return {std::move(task), std::move(executor)};+  }++  friend ViaIfAsyncAwaitable<TaskWithExecutor> co_viaIfAsync(+      Executor::KeepAlive<> executor,+      TaskWithExecutor&& taskWithExecutor) noexcept {+    auto [task, taskExecutor] = std::move(taskWithExecutor).unwrap();+    return ViaIfAsyncAwaitable<TaskWithExecutor>(+        std::move(executor),+        co_withExecutor(std::move(taskExecutor), [](Task<T> t) -> Task<T> {+          co_yield co_result(co_await co_awaitTry(std::move(t)));+        }(std::move(task))));+  }++  friend TaskWithExecutor co_withCancellation(+      folly::CancellationToken cancelToken, TaskWithExecutor&& task) noexcept {+    DCHECK(task.coro_);+    task.coro_.promise().setCancelToken(std::move(cancelToken));+    return std::move(task);+  }++  friend TaskWithExecutor tag_invoke(+      cpo_t<co_withAsyncStack>, TaskWithExecutor&& task) noexcept {+    return std::move(task);+  }++  NoOpMover<TaskWithExecutor> getUnsafeMover(+      ForMustAwaitImmediately) && noexcept {+    return NoOpMover{std::move(*this)}; // Asserts `this` is nothrow-movable+  }++  using folly_private_task_without_executor_t = Task<T>;+  // See comment in `Task`, or use `SafeTaskWithExecutor` instead.+  using folly_private_safe_alias_t = safe_alias_constant<safe_alias::unsafe>;++ private:+  friend class Task<T>;++  explicit TaskWithExecutor(handle_t coro) noexcept : coro_(coro) {}++  handle_t coro_;+};++// This macro makes it easier for `TaskWrapper.h` users to apply the correct+// attributes for the wrapped `Task`s.+#define FOLLY_CORO_TASK_ATTRS \+  FOLLY_NODISCARD [[FOLLY_ATTR_CLANG_CORO_AWAIT_ELIDABLE]]++/// Represents an allocated, but not-started coroutine, which is not yet+/// been bound to an executor.+///+/// You can only co_await a Task from within another Task, in which case it+/// is implicitly bound to the same executor as the parent Task.+///+/// Alternatively, you can explicitly provide an executor by calling+/// `co_withExecutor(executor, task())`, which will return a not-yet-started+/// `TaskWithExecutor` that can be `co_await`ed anywhere and that will+/// automatically schedule the coroutine to start executing on the bound+/// executor when it is `co_await`ed.+///+/// Within the body of a Task's coroutine, executor binding to the parent+/// executor is maintained by implicitly transforming all 'co_await expr'+/// expressions into `co_await co_viaIfAsync(parentExecutor, expr)' to ensure+/// that the coroutine always resumes on the parent's executor.+///+/// The Task coroutine is RequestContext-aware+/// and will capture the current RequestContext at the time the coroutine+/// function is either awaited or explicitly started and will save/restore the+/// current RequestContext whenever the coroutine suspends and resumes at a+/// co_await expression.+///+/// More documentation on how to use coroutines is available at+/// https://github.com/facebook/folly/blob/main/folly/coro/README.md+///+/// @refcode folly/docs/examples/folly/coro/Task.cpp+template <typename T>+class FOLLY_CORO_TASK_ATTRS Task {+ public:+  using promise_type = detail::TaskPromise<T>;+  using StorageType = typename promise_type::StorageType;++ private:+  class Awaiter;+  using handle_t = coroutine_handle<promise_type>;++  void setExecutor(folly::Executor::KeepAlive<>&& e) noexcept {+    DCHECK(coro_);+    DCHECK(e);+    coro_.promise().executor_ = std::move(e);+  }++  // `co_withExecutor` implementation detail -- this works around the fact that+  // not all compilers consider the hidden friend `co_withExecutor` to be a+  // friend of `TaskWithExecutor`, and I found no uniform way to add the+  // friendship without making it non-hidden.  Try folding back into+  // `co_withExecutor` in 2027 or so, to see if the old compiler issue is gone.+  TaskWithExecutor<T> asTaskWithExecutor() && {+    return TaskWithExecutor<T>{std::exchange(coro_, {})};+  }++ public:+  Task(const Task& t) = delete;++  /// Create a Task, invalidating the original Task in the process.+  Task(Task&& t) noexcept : coro_(std::exchange(t.coro_, {})) {}++  /// @private+  ~Task() {+    if (coro_) {+      coro_.destroy();+    }+  }++  Task& operator=(Task t) noexcept {+    swap(t);+    return *this;+  }++  void swap(Task& t) noexcept { std::swap(coro_, t.coro_); }++  /// Specify the executor that this task should execute on:+  ///   co_withExecutor(executor, std::move(task))+  //+  /// @param executor An Executor::KeepAlive object, which can be implicity+  /// constructed from Executor*+  /// @returns a new TaskWithExecutor object, which represents the existing Task+  /// bound to an executor+  friend TaskWithExecutor<T> co_withExecutor(+      Executor::KeepAlive<> executor, Task task) noexcept {+    task.setExecutor(std::move(executor));+    DCHECK(task.coro_);+    return std::move(task).asTaskWithExecutor();+  }+  [[deprecated("Legacy form, prefer `co_withExecutor(exec, yourTask())`.")]]+  TaskWithExecutor<T> scheduleOn(Executor::KeepAlive<> executor) && noexcept {+    return co_withExecutor(std::move(executor), std::move(*this));+  }++  /// Converts a Task into a SemiFuture object.+  ///+  /// The SemiFuture object is implicitly of type Semifuture<Try<T>>, where the+  /// Try represents whether the execution of the converted Task succeeded and T+  /// is the original task's result type.+  /// @returns a SemiFuture object+  FOLLY_NOINLINE+  SemiFuture<folly::lift_unit_t<StorageType>> semi() && {+    return makeSemiFuture().deferExTry(+        [task = std::move(*this),+         returnAddress = FOLLY_ASYNC_STACK_RETURN_ADDRESS()](+            const Executor::KeepAlive<>& executor, Try<Unit>&&) mutable {+          folly::Promise<lift_unit_t<StorageType>> p;++          auto sf = p.getSemiFuture();++          co_withExecutor(executor, std::move(task))+              .startInlineImpl(+                  [promise = std::move(p)](Try<StorageType>&& result) mutable {+                    promise.setTry(std::move(result));+                  },+                  folly::CancellationToken{},+                  returnAddress);++          return sf;+        });+  }++  friend auto co_viaIfAsync(+      Executor::KeepAlive<> executor, Task<T>&& t) noexcept {+    DCHECK(t.coro_);+    // Child task inherits the awaiting task's executor+    t.setExecutor(std::move(executor));+    return Awaiter{std::exchange(t.coro_, {})};+  }++  friend Task co_withCancellation(+      folly::CancellationToken cancelToken, Task&& task) noexcept {+    DCHECK(task.coro_);+    task.coro_.promise().setCancelToken(std::move(cancelToken));+    return std::move(task);+  }++  template <typename F, typename... A, typename F_, typename... A_>+  friend Task tag_invoke(+      tag_t<co_invoke_fn>, tag_t<Task, F, A...>, F_ f, A_... a) {+    co_yield co_result(co_await co_awaitTry(+        invoke(static_cast<F&&>(f), static_cast<A&&>(a)...)));+  }++  NoOpMover<Task> getUnsafeMover(ForMustAwaitImmediately) && noexcept {+    return NoOpMover{std::move(*this)}; // Asserts `this` is nothrow-movable+  }++  using PrivateAwaiterTypeForTests = Awaiter;+  // Use `SafeTask` instead of `Task` to move tasks into other safe coro APIs.+  //+  // User-facing stuff from `Task.h` can trivially include unsafe aliasing, the+  // `folly::coro` docs include hundreds of words of pitfalls.  The intent here+  // is to catch people accidentally passing `Task`s into safer primitives, and+  // breaking their memory-safety guarantees.+  using folly_private_safe_alias_t = safe_alias_constant<safe_alias::unsafe>;++ private:+  friend class detail::TaskPromiseBase;+  friend class detail::TaskPromiseCrtpBase<detail::TaskPromise<T>, T>;+  friend class TaskWithExecutor<T>;++  class Awaiter {+   public:+    explicit Awaiter(handle_t coro) noexcept : coro_(coro) {}++    Awaiter(Awaiter&& other) noexcept : coro_(std::exchange(other.coro_, {})) {}++    Awaiter(const Awaiter&) = delete;++    ~Awaiter() {+      if (coro_) {+        coro_.destroy();+      }+    }++    bool await_ready() noexcept { return false; }++    template <typename Promise>+    FOLLY_NOINLINE auto await_suspend(+        coroutine_handle<Promise> continuation) noexcept {+      DCHECK(coro_);+      auto& promise = coro_.promise();++      promise.continuation_ = continuation;++      auto& calleeFrame = promise.getAsyncFrame();+      calleeFrame.setReturnAddress();++      if constexpr (detail::promiseHasAsyncFrame_v<Promise>) {+        auto& callerFrame = continuation.promise().getAsyncFrame();+        folly::pushAsyncStackFrameCallerCallee(callerFrame, calleeFrame);+        return coro_;+      } else {+        folly::resumeCoroutineWithNewAsyncStackRoot(coro_);+        return;+      }+    }++    T await_resume() {+      DCHECK(coro_);+      SCOPE_EXIT {+        std::exchange(coro_, {}).destroy();+      };+      return std::move(coro_.promise().result()).value();+    }++    folly::Try<StorageType> await_resume_try() noexcept(+        std::is_nothrow_move_constructible_v<StorageType>) {+      DCHECK(coro_);+      SCOPE_EXIT {+        std::exchange(coro_, {}).destroy();+      };+      return std::move(coro_.promise().result());+    }++#if FOLLY_HAS_RESULT+    result<T> await_resume_result() noexcept(+        std::is_nothrow_move_constructible_v<StorageType>) {+      DCHECK(coro_);+      SCOPE_EXIT {+        std::exchange(coro_, {}).destroy();+      };+      return try_to_result(std::move(coro_.promise().result()));+    }+#endif++   private:+    // This overload needed as Awaiter is returned from co_viaIfAsync() which is+    // then passed into co_withAsyncStack().+    friend Awaiter tag_invoke(+        cpo_t<co_withAsyncStack>, Awaiter&& awaiter) noexcept {+      return std::move(awaiter);+    }++    handle_t coro_;+  };++  Task(handle_t coro) noexcept : coro_(coro) {}++  handle_t coro_;+};++/// Make a task that trivially returns a value.+/// @param t value to be returned by the Task+template <class T>+Task<T> makeTask(T t) {+  co_return t;+}++/// Make a Task that trivially returns with no return value.+inline Task<void> makeTask() {+  co_return;+}+/// Same as makeTask(). See Unit+inline Task<void> makeTask(Unit) {+  co_return;+}++/// Make a Task that will trivially yield an Exception.+/// @param ew an exception_wrapper object+template <class T>+Task<T> makeErrorTask(exception_wrapper ew) {+  co_yield co_error(std::move(ew));+}++/// Make a Task out of a Try.+/// @tparam T the type of the value wrapped by the Try+/// @param t the Try to convert into a Task+/// @returns a Task that will yield the Try's value or exception.+template <class T>+Task<drop_unit_t<T>> makeResultTask(Try<T> t) {+  co_yield co_result(std::move(t));+}++template <typename Promise, typename T>+inline Task<T>+detail::TaskPromiseCrtpBase<Promise, T>::get_return_object() noexcept {+  return Task<T>{+      coroutine_handle<Promise>::from_promise(*static_cast<Promise*>(this))};+}++} // namespace folly::coro++#endif // FOLLY_HAS_COROUTINES
+ folly/folly/coro/TaskWrapper.h view
@@ -0,0 +1,421 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <folly/coro/AwaitImmediately.h>+#include <folly/coro/Task.h>++/// `TaskWrapper.h` provides base classes for wrapping `folly::coro::Task` with+/// custom functionality.  These work by composition, which avoids the pitfalls+/// of inheritance -- your custom wrapper will not be "is-a-Task", and will not+/// implicitly "object slice" to a `Task`.+///+/// The point of this header is to uniformly forward the large API surface of+/// `Task`, `TaskWithExecutor`, and `TaskPromise`, leaving just the "new logic"+/// in each wrapper's implementation.+///+///   - `TaskWrapperCrtp` makes your type (1)  a coroutine (`promise_type`)+///     that can `co_await` other `folly::coro` objects.  (2) semi-awaitable by+///     other `folly::coro` coroutines.  It has the following features:+///        * `co_await`ability (using `co_viaIfAsync`)+///        * Interoperates with `folly::coro` awaitable wrappers like+///          `co_awaitTry` and `co_nothrow`.+///        * `co_withCancellation` to add a cancellation token+///        * `co_withExecutor` to add a cancellation token+///        * Basic reflection via `folly/coro/Traits.h`+///        * Empty base optimization for zero runtime overhead+///+///   - `TaskWithExecutorWrapperCrtp` is awaitable, but not a coroutine.  It+///     has the same features, except for `co_withExecutor`.+///+/// ### WARNING: Do not blindly forward more APIs in `TaskWrapper.h`!+///+/// Several existing wrappers are immediately-awaitable (`AwaitImmediately.h`).+/// For those tasks (e.g. `NowTask`), API forwarding is risky:+///   - Do NOT forward `semi()`, `start*()`, `unwrap()`, or other methods, or+///     CPOs that take the awaitable by-reference.  All of those make it+///     trivial to accidentally break the immediately-awaitable invariant, and+///     cause lifetime bugs.+///   - When forwarding an API, use either a static method or CPO.  Then,+///     either ONLY take the awaitable by-value, or bifurcate the API on+///     `must_await_immediately_v<Awaitable>`, grep for examples.+///+/// If you **have** to forward an unsafe API, here are some suggestions:+///   - Only add them in your wrapper.+///   - Add them via `UnsafeTaskWrapperCrtp` deriving from `TaskWrapperCrtp`.+///   - Add boolean flags to the configuration struct, and gate the methods via+///     `enable_if`.  NB: You probably cannot gate these on `Derived` **not**+///     being `MustAwaitImmediately`, since CRTP bases see an incomplete type.+///+/// ### WARNING: Beware of object slicing in "unwrapping" APIs+///+/// Start by reading "A note on object slicing" in `AwaitImmediately.h`.+///+/// If your wrapper is adding new members, or customizing object lifecycle+/// (dtor / copy / move / assignment), then you must:+///   - Write a custom `getUnsafeMover()`.+///   - Overload the protected `unsafeTask()` and `unsafeTaskWithExecutor()` to+///     reduce slicing risk.+///   - Take care not to slice down to the `Crtp` bases.+///+/// ### How to implement a wrapper+///+/// First, read the WARNINGs above. Then, follow one of the "Tiny" examples+/// in `TaskWrapperTest.cpp`. The important things are:+///   - Actually read the "object slicing" warning above!+///   - In most cases, you'll need to both implement a task, and customize its+///     `TaskWithExecutorT`.  If you leave that as `coro::TaskWithExecutor`,+///     some users will accidentally avoid your wrapper's effects.+///   - Tag `YourTaskWithExecutor` with `FOLLY_NODISCARD`.+///   - Tag `YourTask` with the `FOLLY_CORO_TASK_ATTRS` attribute.  Caveat:+///     This assumes that the coro's caller will outlive it.  That is true for+///     `Task`, and almost certainly true of all sensible wrapper types.+///   - Mark your wrappers `final` to discourage inheritance and object-slicing+///     bugs.  They can still be wrapped recursively.+///+/// Future: Once this has a benchmark, see if `FOLLY_ALWAYS_INLINE` makes+/// any difference on the wrapped functions (it shouldn't).++#if FOLLY_HAS_IMMOVABLE_COROUTINES++namespace folly::coro {++namespace detail {++template <typename Wrapper>+using task_wrapper_inner_semiawaitable_t =+    typename Wrapper::folly_private_task_wrapper_inner_t;++template <typename SemiAwaitable, typename T>+inline constexpr bool is_task_or_wrapper_v =+    (!std::is_same_v<nonesuch, SemiAwaitable> && // Does not wrap Task+     (std::is_same_v<SemiAwaitable, Task<T>> || // Wraps Task+      is_task_or_wrapper_v<+          detected_t<task_wrapper_inner_semiawaitable_t, SemiAwaitable>,+          T>));++template <typename Wrapper>+using task_wrapper_inner_promise_t = typename Wrapper::TaskWrapperInnerPromise;++template <typename Promise, typename T>+inline constexpr bool is_task_promise_or_wrapper_v =+    (!std::is_same_v<nonesuch, Promise> && // Does not wrap TaskPromise+     (std::is_same_v<Promise, TaskPromise<T>> || // Wraps TaskPromise+      is_task_promise_or_wrapper_v<+          detected_t<task_wrapper_inner_promise_t, Promise>,+          T>));++template <typename T, typename WrapperTask, typename Promise>+class TaskPromiseWrapperBase {+ protected:+  static_assert(+      is_task_or_wrapper_v<WrapperTask, T>,+      "SemiAwaitable must be a sequence of wrappers ending in Task<T>");+  static_assert(+      is_task_promise_or_wrapper_v<Promise, T>,+      "Promise must be a sequence of wrappers ending in TaskPromise<T>");++  Promise promise_;++  TaskPromiseWrapperBase() noexcept = default;+  ~TaskPromiseWrapperBase() = default;++ public:+  using TaskWrapperInnerPromise = Promise;++  WrapperTask get_return_object() noexcept {+    return WrapperTask{promise_.get_return_object()};+  }++  static void* operator new(std::size_t size) {+    return ::folly_coro_async_malloc(size);+  }+  static void operator delete(void* ptr, std::size_t size) {+    ::folly_coro_async_free(ptr, size);+  }++  auto initial_suspend() noexcept { return promise_.initial_suspend(); }+  auto final_suspend() noexcept { return promise_.final_suspend(); }++  template <+      typename Awaitable,+      std::enable_if_t<!must_await_immediately_v<Awaitable>, int> = 0>+  auto await_transform(Awaitable&& what) {+    return promise_.await_transform(std::forward<Awaitable>(what));+  }+  template <+      typename Awaitable,+      std::enable_if_t<must_await_immediately_v<Awaitable>, int> = 0>+  auto await_transform(Awaitable what) {+    return promise_.await_transform(+        mustAwaitImmediatelyUnsafeMover(std::move(what))());+  }++  auto yield_value(auto&& v)+    requires requires { promise_.yield_value(std::forward<decltype(v)>(v)); }+  {+    return promise_.yield_value(std::forward<decltype(v)>(v));+  }++  void unhandled_exception() noexcept { promise_.unhandled_exception(); }++  // These getters are all interposed for `TaskPromiseBase::FinalAwaiter`+  decltype(auto) result() { return promise_.result(); }+  decltype(auto) getAsyncFrame() { return promise_.getAsyncFrame(); }+  auto& scopeExitRef(TaskPromisePrivate tag) {+    return promise_.scopeExitRef(tag);+  }+  auto& continuationRef(TaskPromisePrivate tag) {+    return promise_.continuationRef(tag);+  }+  auto& executorRef(TaskPromisePrivate tag) {+    return promise_.executorRef(tag);+  }+};++template <typename T, typename WrapperTask, typename Promise>+class TaskPromiseWrapper+    : public TaskPromiseWrapperBase<T, WrapperTask, Promise> {+ protected:+  TaskPromiseWrapper() noexcept = default;+  ~TaskPromiseWrapper() = default;++ public:+  template <typename U = T> // see "`co_return` with implicit ctor" test+  auto return_value(U&& value) {+    return this->promise_.return_value(std::forward<U>(value));+  }+};++template <typename WrapperTask, typename Promise>+class TaskPromiseWrapper<void, WrapperTask, Promise>+    : public TaskPromiseWrapperBase<void, WrapperTask, Promise> {+ protected:+  TaskPromiseWrapper() noexcept = default;+  ~TaskPromiseWrapper() = default;++ public:+  void return_void() noexcept { this->promise_.return_void(); }+};++// Mixin for TaskWrapper.h configs for `Task` & `TaskWithExecutor` types+struct DoesNotWrapAwaitable {+  template <typename Awaitable>+  static inline constexpr Awaitable&& wrapAwaitable(Awaitable&& awaitable) {+    return static_cast<Awaitable&&>(awaitable);+  }+};++} // namespace detail++// IMPORTANT: Read "Do not blindly forward more APIs" in the file docblock.  In+// a nutshell, adding methods, or by-ref CPOs, can compromise the safety of+// immediately-awaitable wrappers, so DON'T DO THAT.+template <typename Derived, typename Cfg>+class TaskWrapperCrtp {+ public:+  using promise_type = typename Cfg::PromiseT;++  // Pass `tw` by-value, since `&&` would break immediately-awaitable types+  friend typename Cfg::TaskWithExecutorT co_withExecutor(+      Executor::KeepAlive<> executor, Derived tw) noexcept {+    return typename Cfg::TaskWithExecutorT{+        co_withExecutor(std::move(executor), std::move(tw).unwrapTask())};+  }++  // Pass `tw` by-value, since `&&` would break immediately-awaitable types+  friend Derived co_withCancellation(+      CancellationToken cancelToken, Derived tw) noexcept {+    return Derived{co_withCancellation(+        std::move(cancelToken), std::move(tw).unwrapTask())};+  }++  // Pass `tw` by-value, since `&&` would break immediately-awaitable types+  // Has copy-pasta below in `TaskWithExecutorWrapperCrtp`.+  friend auto co_viaIfAsync(+      Executor::KeepAlive<> executor, Derived tw) noexcept {+    return Cfg::wrapAwaitable(co_viaIfAsync(+        std::move(executor),+        mustAwaitImmediatelyUnsafeMover(std::move(tw).unwrapTask())()));+  }++  // No `cpo_t<co_withAsyncStack>` since a "Task" is not an awaitable.++  auto getUnsafeMover(ForMustAwaitImmediately p) && noexcept {+    // See "A note on object slicing" above `mustAwaitImmediatelyUnsafeMover`+    static_assert(sizeof(Derived) == sizeof(typename Cfg::InnerTaskT));+    static_assert( // More `noexcept` tests in `MustAwaitImmediatelyUnsafeMover`+        noexcept(std::move(*this).unwrapTask().getUnsafeMover(p)));+    return MustAwaitImmediatelyUnsafeMover{+        (Derived*)nullptr, std::move(*this).unwrapTask().getUnsafeMover(p)};+  }++  using folly_private_task_wrapper_inner_t = typename Cfg::InnerTaskT;+  using folly_private_task_wrapper_crtp_base = TaskWrapperCrtp;++  // Wrappers can override these as-needed+  using folly_private_must_await_immediately_t =+      must_await_immediately_t<typename Cfg::InnerTaskT>;+  using folly_private_noexcept_awaitable_t =+      noexcept_awaitable_t<typename Cfg::InnerTaskT>;+  using folly_private_safe_alias_t =+      safe_alias_of<folly_private_task_wrapper_inner_t>;++ private:+  using Inner = folly_private_task_wrapper_inner_t;+  static_assert(+      detail::is_task_or_wrapper_v<Inner, typename Cfg::ValueT>,+      "*TaskWrapper must wrap a sequence of wrappers ending in Task<T>");++  Inner task_;++ protected:+  template <typename, typename, typename> // can construct+  friend class ::folly::coro::detail::TaskPromiseWrapperBase;+  friend class MustAwaitImmediatelyUnsafeMover< // can construct+      Derived,+      detail::unsafe_mover_for_must_await_immediately_t<Inner>>;++  explicit TaskWrapperCrtp(Inner t)+      // `mustAwaitImmediatelyUnsafeMover` has more `noexcept` assertions.+      noexcept(noexcept(Inner{FOLLY_DECLVAL(Inner)}))+      : task_(mustAwaitImmediatelyUnsafeMover(std::move(t))()) {+    static_assert(+        must_await_immediately_v<Derived> ||+            !must_await_immediately_v<typename Cfg::TaskWithExecutorT>,+        "`TaskWithExecutorT` must `AddMustAwaitImmediately` because the inner "+        "task did");+  }++  // See "A note on object slicing" above `mustAwaitImmediatelyUnsafeMover`+  Inner unwrapTask() && noexcept {+    static_assert(sizeof(Inner) == sizeof(Derived));+    return mustAwaitImmediatelyUnsafeMover(std::move(task_))();+  }+};++// IMPORTANT: Read "Do not blindly forward more APIs" in the file docblock.  In+// a nutshell, adding methods, or by-ref CPOs, can compromise the safety of+// immediately-awaitable wrappers, so DON'T DO THAT.+template <typename Derived, typename Cfg>+class TaskWithExecutorWrapperCrtp {+ private:+  using Inner = typename Cfg::InnerTaskWithExecutorT;+  Inner inner_;++ protected:+  friend class MustAwaitImmediatelyUnsafeMover< // can construct+      Derived,+      detail::unsafe_mover_for_must_await_immediately_t<Inner>>;++  // See "A note on object slicing" above `mustAwaitImmediatelyUnsafeMover`+  Inner unwrapTaskWithExecutor() && noexcept {+    static_assert(sizeof(Inner) == sizeof(Derived));+    return mustAwaitImmediatelyUnsafeMover(std::move(inner_))();+  }++  // Our task can construct us, and that logic lives in the CRTP base+  friend typename Cfg::WrapperTaskT::folly_private_task_wrapper_crtp_base;++  explicit TaskWithExecutorWrapperCrtp(Inner t)+      // `mustAwaitImmediatelyUnsafeMover` has more `noexcept` assertions.+      noexcept(noexcept(Inner{FOLLY_DECLVAL(Inner)}))+      : inner_(mustAwaitImmediatelyUnsafeMover(std::move(t))()) {}++ public:+  // This is a **deliberately undefined** declaration. It is provided so that+  // `await_result_t` can work, e.g. `AsyncScope` checks that for all tasks.+  //+  // We do NOT want a definition here, for two reasons:+  //   - As a destructive member function, this can easily violate the+  //     "immediately awaitable" invariant -- all you have to do is+  //     `twe.operator co_await()`.+  //   - A definition would have to handle `Cfg::wrapAwaitable`, but also avoid+  //     double-wrapping the awaitable (*if* that can occur?).  No definition+  //     means I don't have to think through this :)+  //+  // If, in the future, something requires `get_awaiter()` to handle a wrapped+  // task-with-executor in an **evaluated** context, we can then provide the+  // definition, being mindful of the above concerns.+  //+  // NB: Adding a definition should not let this naively wrong code compile --+  // that goes through `await_transform()`.  `NowTaskTest.cpp` checks this.+  //   auto t = co_withExecutor(ex, someNowTask());+  //   co_await std::move(t);+  auto operator co_await() && noexcept+      -> decltype(Cfg::wrapAwaitable(std::move(inner_)).operator co_await());++  // Pass `twe` by-value, since `&&` would break immediately-awaitable types+  friend Derived co_withCancellation(+      CancellationToken cancelToken, Derived twe) noexcept {+    return Derived{co_withCancellation(+        std::move(cancelToken),+        mustAwaitImmediatelyUnsafeMover(std::move(twe.inner_))())};+  }++  // Pass `twe` by-value, since `&&` would break immediately-awaitable types+  // Has copy-pasta above in `TaskWrapperCrtp`.+  friend auto co_viaIfAsync(+      Executor::KeepAlive<> executor, Derived twe) noexcept {+    return Cfg::wrapAwaitable(co_viaIfAsync(+        std::move(executor),+        mustAwaitImmediatelyUnsafeMover(std::move(twe.inner_))()));+  }++  // `AsyncScope` requires an awaitable with an executor already attached, and+  // thus directly calls `co_withAsyncStack` instead of `co_viaIfAsync`.  But,+  // we still need to wrap the awaitable on that code path.+  //+  // NB: Passing by-&& here looks like it could compromise the safety of+  // immediately-awaitable coros (`NowTask`, `NowTaskWithExecutor`).  With+  // by-value, `BlockingWaitTest.AwaitNowTaskWithExecutor` would not build.+  //+  // Supporting pass-by-value would require fixing a LOT of plumbing.+  //   - `WithAsyncStack.h` calls `is_tag_invocable_v`, which would fail on+  //     `NowTaskWithExecutor` if this is by-value, since the implementation of+  //     `is_tag_invocable_v` presents all args by-&&.+  //   - `CommutativeWrapperAwaitable` and `StackAwareViaIfAsyncAwaiter`,+  //     among others, also assume that `co_withAsyncStack` takes by-ref.+  //+  // Fortunately, I'm not aware of any practical reduction in+  // immediately-awaitable safety from this issue.  `co_withAsyncStack` should+  // never be called in user code.  Internal usage in `folly/coro` looks+  // overall immediately-awaitable-safe -- and the best safeguard for any+  // particular scenario is to test, see e.g. `NowTaskTest.blockingWait`.+  friend auto tag_invoke(cpo_t<co_withAsyncStack>, Derived&& twe) noexcept(+      noexcept(co_withAsyncStack(FOLLY_DECLVAL(Inner)))) {+    return Cfg::wrapAwaitable(co_withAsyncStack(std::move(twe.inner_)));+  }++  auto getUnsafeMover(ForMustAwaitImmediately p) && noexcept {+    // See "A note on object slicing" above `mustAwaitImmediatelyUnsafeMover`+    static_assert(sizeof(Derived) == sizeof(Inner));+    static_assert( // More `noexcept` tests in `MustAwaitImmediatelyUnsafeMover`+        noexcept(std::move(inner_).getUnsafeMover(p)));+    return MustAwaitImmediatelyUnsafeMover{+        (Derived*)nullptr, std::move(inner_).getUnsafeMover(p)};+  }++  using folly_private_must_await_immediately_t =+      must_await_immediately_t<Inner>;+  using folly_private_task_without_executor_t = typename Cfg::WrapperTaskT;+  using folly_private_safe_alias_t = safe_alias_of<Inner>;+};++} // namespace folly::coro++#endif
+ folly/folly/coro/TimedWait.h view
@@ -0,0 +1,89 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <type_traits>++#include <folly/Optional.h>+#include <folly/coro/Baton.h>+#include <folly/coro/Coroutine.h>+#include <folly/coro/Invoke.h>+#include <folly/coro/Task.h>+#include <folly/coro/Traits.h>+#include <folly/coro/detail/Helpers.h>+#include <folly/futures/Future.h>++#if FOLLY_HAS_COROUTINES++namespace folly {+namespace coro {+template <typename Awaitable>+Task<Optional<lift_unit_t<detail::decay_rvalue_reference_t<+    detail::lift_lvalue_reference_t<semi_await_result_t<Awaitable>>>>>>+timed_wait(Awaitable awaitable, Duration duration) {+  Baton baton;+  Try<lift_unit_t<detail::decay_rvalue_reference_t<+      detail::lift_lvalue_reference_t<semi_await_result_t<Awaitable>>>>>+      result;++  Executor* executor = co_await co_current_executor;+  auto sleepFuture = futures::sleep(duration).toUnsafeFuture();+  auto posted = new std::atomic<bool>(false);+  sleepFuture.setCallback_(+      [posted, &baton, executor = Executor::KeepAlive<>{executor}](+          auto&&, auto&&) {+        if (!posted->exchange(true, std::memory_order_acq_rel)) {+          executor->add([&baton] { baton.post(); });+        } else {+          delete posted;+        }+      },+      // No user logic runs in the callback, we can avoid the cost of switching+      // the context.+      /* context */ nullptr);++  {+    auto t = co_invoke(+        [awaitable = std::move(+             awaitable)]() mutable -> Task<semi_await_result_t<Awaitable>> {+          co_return co_await std::move(awaitable);+        });+    co_withExecutor(executor, std::move(t))+        .start([posted, &baton, &result, sleepFuture = std::move(sleepFuture)](+                   auto&& r) mutable {+          if (!posted->exchange(true, std::memory_order_acq_rel)) {+            result = std::move(r);+            baton.post();+            sleepFuture.cancel();+          } else {+            delete posted;+          }+        });+  }++  co_await detail::UnsafeResumeInlineSemiAwaitable{get_awaiter(baton)};++  if (!result.hasValue() && !result.hasException()) {+    co_return folly::none;+  }+  co_return std::move(*result);+}++} // namespace coro+} // namespace folly++#endif // FOLLY_HAS_COROUTINES
+ folly/folly/coro/Timeout-inl.h view
@@ -0,0 +1,151 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#include <folly/CancellationToken.h>+#include <folly/coro/Baton.h>+#include <folly/coro/WithCancellation.h>++#if FOLLY_HAS_COROUTINES++namespace folly::coro {++namespace detail {++template <bool>+struct DiscardImpl {+  folly::coro::Baton baton;+  exception_wrapper timeoutResult;+  bool parentCancelled = false;+  bool checkedTimeout = false;+};++template <>+struct DiscardImpl<false> {};++template <+    typename SemiAwaitable,+    typename Duration,+    bool discard,+    typename Fn,+    typename TimekeeperPtr>+typename detail::TimeoutTask<SemiAwaitable, TimekeeperPtr> timeoutImpl(+    Fn semiFn, Duration timeoutDuration, TimekeeperPtr tk) {+  CancellationSource cancelSource;+  DiscardImpl<discard> impl;+  auto sleepFuture =+      folly::futures::sleep(timeoutDuration, tk).toUnsafeFuture();+  sleepFuture.setCallback_(+      [&, cancelSource](Executor::KeepAlive<>&&, Try<Unit>&& result) noexcept {+        if constexpr (discard) {+          if (result.hasException()) {+            impl.timeoutResult = std::move(result.exception());+          } else {+            impl.timeoutResult = folly::make_exception_wrapper<FutureTimeout>();+          }+          impl.baton.post();+        }+        cancelSource.requestCancellation();+      });++  bool isSleepCancelled = false;+  auto tryCancelSleep = [&]() noexcept {+    if (!isSleepCancelled) {+      isSleepCancelled = true;+      sleepFuture.cancel();+    }+  };++  std::optional<CancellationCallback> cancelCallback{+      std::in_place, co_await co_current_cancellation_token, [&]() {+        cancelSource.requestCancellation();+        tryCancelSleep();+        if constexpr (discard) {+          impl.parentCancelled = true;+        }+      }};++  exception_wrapper error;+  try {+    auto resultTry =+        co_await folly::coro::co_awaitTry(folly::coro::co_withCancellation(+            cancelSource.getToken(), std::move(semiFn)()));++    cancelCallback.reset();++    if constexpr (discard) {+      if (!impl.parentCancelled && impl.baton.ready()) {+        // Timer already fired+        co_yield folly::coro::co_error(std::move(impl.timeoutResult));+      }+      impl.checkedTimeout = true;+    }++    tryCancelSleep();+    if constexpr (discard) {+      co_await impl.baton;+    }++    if (resultTry.hasException()) {+      co_yield folly::coro::co_error(std::move(resultTry).exception());+    }++    co_return std::move(resultTry).value();+  } catch (...) {+    error = exception_wrapper{current_exception()};+  }++  assert(error);++  cancelCallback.reset();++  if constexpr (discard) {+    if (!impl.checkedTimeout && !impl.parentCancelled && impl.baton.ready()) {+      // Timer already fired+      co_yield folly::coro::co_error(std::move(impl.timeoutResult));+    }+  }++  tryCancelSleep();+  if constexpr (discard) {+    co_await impl.baton;+  }++  co_yield folly::coro::co_error(std::move(error));+}++} // namespace detail++template <typename SemiAwaitable, typename Duration, typename TimekeeperPtr>+typename detail::TimeoutTask<SemiAwaitable, TimekeeperPtr> timeout(+    SemiAwaitable semiAwaitable, Duration timeoutDuration, TimekeeperPtr tk) {+  return detail::timeoutImpl<SemiAwaitable, Duration, /*discard=*/true>(+      mustAwaitImmediatelyUnsafeMover(std::move(semiAwaitable)),+      timeoutDuration,+      std::move(tk));+}++template <typename SemiAwaitable, typename Duration, typename TimekeeperPtr>+typename detail::TimeoutTask<SemiAwaitable, TimekeeperPtr> timeoutNoDiscard(+    SemiAwaitable semiAwaitable, Duration timeoutDuration, TimekeeperPtr tk) {+  return detail::timeoutImpl<SemiAwaitable, Duration, /*discard=*/false>(+      mustAwaitImmediatelyUnsafeMover(std::move(semiAwaitable)),+      timeoutDuration,+      std::move(tk));+}++} // namespace folly::coro++#endif // FOLLY_HAS_COROUTINES
+ folly/folly/coro/Timeout.h view
@@ -0,0 +1,110 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <folly/coro/Coroutine.h>+#include <folly/coro/Task.h>+#include <folly/coro/Traits.h>+#include <folly/coro/detail/PickTaskWrapper.h>+#include <folly/futures/Future.h>+// `timeout(coroFutureInt())` makes a `SafeTask`+#include <folly/coro/safe/SafeTask.h>+// `timeout(memberTask())` makes a `NowTask`+#include <folly/coro/safe/NowTask.h>++#if FOLLY_HAS_COROUTINES++namespace folly::coro {++namespace detail {+// This doesn't try to apply `AsNoexcept` to the output, since `timeout` is+// expected to throw, and `timeoutNoDiscard()` may either complete with a+// stopped state, or with an error.+template <typename SemiAwaitable, typename TimekeeperPtr>+using TimeoutTask = PickTaskWrapper<+    typename semi_await_try_result_t<SemiAwaitable>::element_type,+    std::min(safe_alias_of_v<TimekeeperPtr>, safe_alias_of_v<SemiAwaitable>),+    must_await_immediately_v<SemiAwaitable>>;+} // namespace detail++/// Returns a Task that, when started, starts a timer of duration+/// 'timeoutDuration' and awaits the passed SemiAwaitable.+///+/// If the timeoutDuration elapses before the 'co_await semiAwaitable'+/// operation completes then requests cancellation of the child operation+/// and completes with an error of type folly::FutureTimeout.+/// Otherwise, if the 'co_await semiAwaitable' operation completes before+/// the timeoutDuration elapses then cancels the timer and completes with+/// the result of the semiAwaitable.+///+/// IMPORTANT: The operation passed as the first argument must be able+/// to respond to a request for cancellation on the CancellationToken+/// injected to it via folly::coro::co_withCancellation in a timely manner for+/// the timeout to work as expected.+///+/// If a timekeeper is provided then uses that timekeeper to start the timer,+/// otherwise uses the process' default TimeKeeper if 'tk' is null.+///+/// \throws folly::FutureTimeout+/// \refcode folly/docs/examples/folly/coro/DetachOnCancel.cpp+template <+    typename SemiAwaitable,+    typename Duration,+    // Templated so we can take safe pointers like `capture<Timekeeper&>` from+    // `folly/coro/safe`, and return a `SafeTask`.+    typename TimekeeperPtr = std::nullptr_t>+typename detail::TimeoutTask<SemiAwaitable, TimekeeperPtr> timeout(+    SemiAwaitable semiAwaitable,+    Duration timeoutDuration,+    TimekeeperPtr tk = nullptr);++/// Returns a Task that, when started, starts a timer of duration+/// 'timeoutDuration' and awaits the passed SemiAwaitable (operation).+///+/// The returned result is *always* that of the operation. In other words the+/// result is never discarded, in contrast with `timeout`.+///+/// If the timeout duration elapses before the operation completes, the result+/// should and typically will reflect cancellation (e.g. `OperationCancelled`)+/// but this depends on how the operation responds (as cancellation is+/// cooperative).+///+/// To disambiguate between cancellation and timeout, callers can inspect their+/// own cancellation token.+///+/// IMPORTANT: This function has no effect if the passed operation does not+/// respond to cancellation. The operation passed as the first argument must be+/// able to respond to a request for cancellation on the CancellationToken+/// injected to it via folly::coro::co_withCancellation in a timely manner for+/// the timeout to work as expected.+///+/// If a timekeeper is provided then uses that timekeeper to start the timer,+/// otherwise uses the process' default TimeKeeper if 'tk' is null.+template <+    typename SemiAwaitable,+    typename Duration,+    typename TimekeeperPtr = std::nullptr_t> // templated for reason above+typename detail::TimeoutTask<SemiAwaitable, TimekeeperPtr> timeoutNoDiscard(+    SemiAwaitable semiAwaitable,+    Duration timeoutDuration,+    TimekeeperPtr tk = nullptr);++} // namespace folly::coro++#endif // FOLLY_HAS_COROUTINES++#include <folly/coro/Timeout-inl.h>
+ folly/folly/coro/Traits.h view
@@ -0,0 +1,231 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <folly/Traits.h>+#include <folly/coro/Coroutine.h>++#include <type_traits>++#if FOLLY_HAS_COROUTINES++namespace folly {+namespace coro {++/**+ * A type trait to unwrap a std::reference_wrapper<T> to a type T+ */+template <typename T>+struct remove_reference_wrapper {+  using type = T;+};+template <typename T>+struct remove_reference_wrapper<std::reference_wrapper<T>> {+  using type = T;+};+template <typename T>+using remove_reference_wrapper_t = typename remove_reference_wrapper<T>::type;++namespace detail {++template <typename T>+inline constexpr bool is_coroutine_handle_v = folly::is_instantiation_of_v< //+    coroutine_handle,+    T>;++} // namespace detail++/// is_awaiter<T>::value+/// is_awaiter_v<T>+///+/// Template metafunction for querying whether the specified type implements+/// the 'Awaiter' concept.+///+/// An 'Awaiter' must have the following three methods.+/// - awaiter.await_ready() -> bool+/// - awaiter.await_suspend(coroutine_handle<void>()) ->+///     void OR+///     bool OR+///     coroutine_handle<T> for some T+/// - awaiter.await_resume()+///+/// Note that we don't check for a valid await_suspend() method here since+/// we don't yet know the promise type to use and some await_suspend()+/// implementations have particular requirements on the promise (eg. the+/// stack-aware awaiters may require the .getAsyncFrame() method)+template <typename T, typename = void>+struct is_awaiter : std::bool_constant<!require_sizeof<T>> {};++template <typename T>+struct is_awaiter<T, std::enable_if_t<std::is_void_v<T>>> : std::false_type {};++template <typename T>+struct is_awaiter<+    T,+    folly::void_t<+        decltype(std::declval<T&>().await_ready()),+        decltype(std::declval<T&>().await_resume())>>+    : std::is_same<bool, decltype(std::declval<T&>().await_ready())> {};++template <typename T>+constexpr bool is_awaiter_v = is_awaiter<T>::value;++namespace detail {++template <typename Awaitable, typename = void>+struct _has_member_operator_co_await+    : std::bool_constant<!require_sizeof<Awaitable>> {};++template <typename T>+struct _has_member_operator_co_await<T, std::enable_if_t<std::is_void_v<T>>>+    : std::false_type {};++template <typename Awaitable>+struct _has_member_operator_co_await<+    Awaitable,+    folly::void_t<decltype(std::declval<Awaitable>().operator co_await())>>+    : is_awaiter<decltype(std::declval<Awaitable>().operator co_await())> {};++template <typename Awaitable, typename = void>+struct _has_free_operator_co_await+    : std::bool_constant<!require_sizeof<Awaitable>> {};++template <typename T>+struct _has_free_operator_co_await<T, std::enable_if_t<std::is_void_v<T>>>+    : std::false_type {};++template <typename Awaitable>+struct _has_free_operator_co_await<+    Awaitable,+    folly::void_t<decltype(operator co_await(std::declval<Awaitable>()))>>+    : is_awaiter<decltype(operator co_await(std::declval<Awaitable>()))> {};++} // namespace detail++/// is_awaitable<T>::value+/// is_awaitable_v<T>+///+/// Query if a type, T, is awaitable within the context of any coroutine whose+/// promise_type does not have an await_transform() that modifies what is+/// normally awaitable.+///+/// A type, T, is awaitable if it is an Awaiter, or if it has either a+/// member operator co_await() or a free-function operator co_await() that+/// returns an Awaiter.+template <typename T>+struct is_awaitable+    : folly::Disjunction<+          detail::_has_member_operator_co_await<T>,+          detail::_has_free_operator_co_await<T>,+          is_awaiter<T>> {};++template <typename T>+constexpr bool is_awaitable_v = is_awaitable<T>::value;++/// get_awaiter(Awaitable&&) -> awaiter_type_t<Awaitable>+///+/// The get_awaiter() function takes an Awaitable type and returns a value+/// that contains the await_ready(), await_suspend() and await_resume() methods+/// for that type.+///+/// This encapsulates calling 'operator co_await()' if it exists.+struct get_awaiter_fn {+  template <+      typename Awaitable,+      std::enable_if_t<+          folly::Conjunction<+              is_awaiter<Awaitable>,+              folly::Negation<detail::_has_free_operator_co_await<Awaitable>>,+              folly::Negation<+                  detail::_has_member_operator_co_await<Awaitable>>>::value,+          int> = 0>+  Awaitable& operator()(Awaitable&& awaitable) const {+    return static_cast<Awaitable&>(awaitable);+  }++  template <+      typename Awaitable,+      std::enable_if_t<+          detail::_has_member_operator_co_await<Awaitable>::value,+          int> = 0>+  decltype(auto) operator()(Awaitable&& awaitable) const {+    return static_cast<Awaitable&&>(awaitable).operator co_await();+  }++  template <+      typename Awaitable,+      std::enable_if_t<+          folly::Conjunction<+              detail::_has_free_operator_co_await<Awaitable>,+              folly::Negation<+                  detail::_has_member_operator_co_await<Awaitable>>>::value,+          int> = 0>+  decltype(auto) operator()(Awaitable&& awaitable) const {+    return operator co_await(static_cast<Awaitable&&>(awaitable));+  }+};+constexpr inline get_awaiter_fn get_awaiter{};++/// awaiter_type<Awaitable>+///+/// A template-metafunction that lets you query the type that will be used+/// as the Awaiter object when you co_await a value of type Awaitable.+/// This is the return-type of get_awaiter() when passed a value of type+/// Awaitable.+template <typename Awaitable, typename = void>+struct awaiter_type {};++template <typename Awaitable>+struct awaiter_type<Awaitable, std::enable_if_t<is_awaitable_v<Awaitable>>> {+  using type = decltype(get_awaiter(std::declval<Awaitable>()));+};++/// await_result<Awaitable>+///+/// A template metafunction that allows you to query the type that will result+/// from co_awaiting a value of that type in the context of a coroutine that+/// does not modify the normal behaviour with promise_type::await_transform().+template <typename Awaitable>+using awaiter_type_t = typename awaiter_type<Awaitable>::type;++template <typename Awaitable, typename = void>+struct await_result {};++template <typename Awaitable>+struct await_result<Awaitable, std::enable_if_t<is_awaitable_v<Awaitable>>> {+  using type = decltype(get_awaiter(std::declval<Awaitable>()).await_resume());+};++template <typename Awaitable>+using await_result_t = typename await_result<Awaitable>::type;++namespace detail {++template <typename Promise, typename = void>+constexpr bool promiseHasAsyncFrame_v = !require_sizeof<Promise>;++template <typename Promise>+constexpr bool promiseHasAsyncFrame_v<+    Promise,+    void_t<decltype(std::declval<Promise&>().getAsyncFrame())>> = true;++} // namespace detail++} // namespace coro+} // namespace folly++#endif // FOLLY_HAS_COROUTINES
+ folly/folly/coro/Transform-inl.h view
@@ -0,0 +1,48 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#include <folly/Traits.h>+#include <folly/coro/Transform.h>++#if FOLLY_HAS_COROUTINES++namespace folly {+namespace coro {++template <+    typename ReturnType,+    typename TransformFn,+    typename Reference,+    typename Value,+    typename ReturnReference>+AsyncGenerator<ReturnReference> transform(+    AsyncGenerator<Reference, Value> source, TransformFn transformFn) {+  while (auto item = co_await source.next()) {+    using InvokeResult = decltype(invoke(transformFn, std::move(item).value()));+    if constexpr (std::is_constructible_v<ReturnReference&&, InvokeResult>) {+      co_yield invoke(transformFn, std::move(item).value());+    } else {+      remove_cvref_t<ReturnReference> result =+          invoke(transformFn, std::move(item).value());+      co_yield std::forward<ReturnReference>(result);+    }+  }+}++} // namespace coro+} // namespace folly++#endif // FOLLY_HAS_COROUTINES
+ folly/folly/coro/Transform.h view
@@ -0,0 +1,83 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <type_traits>++#include <folly/coro/AsyncGenerator.h>+#include <folly/coro/Coroutine.h>++#if FOLLY_HAS_COROUTINES++namespace folly {+namespace coro {++namespace detail {+struct computed_from_input;+}++// Transform the Values from an input stream into a stream of the+// Trandformed Values.+//+// The input is a stream of Values.+//+// The output is a stream of Transformed Value.+//+// Example:+//   AsyncGenerator<int> stream();+//+//   Task<void> consumer() {+//     auto to_float = [](int i){ return i * 1.0f; };+//     AsyncGenerator<float&> events = transform(stream(), to_float);+//     try {+//       while (auto item = co_await events.next()) {+//         // Value+//         float& value = *item;+//         std::cout << "value " << value << "\n";+//       }+//       // End Of Stream+//       std::cout << "end\n";+//     } catch (const std::exception& error) {+//       // Exception+//       std::cout << "error " << error.what() << "\n";+//     }+//   }+//+// By default the AsyncGenerator returns a reference to the computed value.+// Specify the first template argument to override the return type of the+// generator.+//+// Example:+//   AsyncGenerator<double> events = transform<double>(stream(), to_float);+template <+    typename ReturnType = detail::computed_from_input,+    typename TransformFn,+    typename Reference,+    typename Value,+    typename ReturnReference = std::conditional_t<+        std::is_same_v<ReturnType, detail::computed_from_input>,+        invoke_result_t<TransformFn&, Reference&&>&&,+        ReturnType>>+AsyncGenerator<ReturnReference> transform(+    AsyncGenerator<Reference, Value> source, TransformFn transformFn);++} // namespace coro+} // namespace folly++#endif // FOLLY_HAS_COROUTINES++#include <folly/coro/Transform-inl.h>
+ folly/folly/coro/UnboundedQueue.h view
@@ -0,0 +1,97 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <folly/concurrency/UnboundedQueue.h>+#include <folly/coro/Coroutine.h>+#include <folly/coro/Task.h>+#include <folly/fibers/Semaphore.h>++#if FOLLY_HAS_COROUTINES++namespace folly {+namespace coro {++// Wrapper around folly::UnboundedQueue with async wait++template <typename T, bool SingleProducer = false, bool SingleConsumer = false>+class UnboundedQueue {+ public:+  template <typename U = T>+  void enqueue(U&& val) {+    queue_.enqueue(std::forward<U>(val));+    sem_.signal();+  }++  // Dequeue a value from the queue.+  // Note that this operation can be safely cancelled by requesting cancellation+  // on the awaiting coroutine's associated CancellationToken.+  // If the operation is successfully cancelled then it will complete with+  // an error of type folly::OperationCancelled.+  // WARNING: It is not safe to wrap this with folly::coro::timeout(). Wrap with+  // folly::coro::timeoutNoDiscard(), or use co_try_dequeue_for() instead.+  folly::coro::Task<T> dequeue() {+    folly::Try<void> result = co_await folly::coro::co_awaitTry(sem_.co_wait());+    if (result.hasException()) {+      co_yield co_error(std::move(result).exception());+    }++    co_return queue_.dequeue();+  }++  // Try to dequeue a value from the queue with a timeout. The operation will+  // either successfully dequeue an item from the queue, or else be cancelled+  // and complete with an error of type folly::OperationCancelled.+  template <typename Duration>+  folly::coro::Task<T> co_try_dequeue_for(Duration timeout) {+    folly::Try<void> result =+        co_await folly::coro::co_awaitTry(sem_.co_try_wait_for(timeout));+    if (result.hasException()) {+      co_yield co_error(std::move(result).exception());+    }++    co_return queue_.dequeue();+  }++  folly::coro::Task<void> dequeue(T& out) {+    co_await sem_.co_wait();+    queue_.dequeue(out);+  }++  folly::Optional<T> try_dequeue() {+    return sem_.try_wait() ? queue_.try_dequeue() : folly::none;+  }++  bool try_dequeue(T& out) {+    return sem_.try_wait() ? queue_.try_dequeue(out) : false;+  }++  bool empty() const { return queue_.empty(); }++  const T* try_peek() noexcept { return queue_.try_peek(); }++  size_t size() const { return queue_.size(); }++ private:+  folly::UnboundedQueue<T, SingleProducer, SingleConsumer, false> queue_;+  folly::fibers::Semaphore sem_{0};+};++} // namespace coro+} // namespace folly++#endif // FOLLY_HAS_COROUTINES
+ folly/folly/coro/ViaIfAsync.h view
@@ -0,0 +1,879 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <memory>++#include <folly/Executor.h>+#include <folly/Traits.h>+#include <folly/coro/AwaitImmediately.h>+#include <folly/coro/Coroutine.h>+#include <folly/coro/Traits.h>+#include <folly/coro/WithAsyncStack.h>+#include <folly/coro/WithCancellation.h>+#include <folly/coro/detail/Malloc.h>+#include <folly/io/async/Request.h>+#include <folly/lang/CustomizationPoint.h>+#include <folly/lang/SafeAlias-fwd.h>+#include <folly/tracing/AsyncStack.h>++#include <glog/logging.h>++#if FOLLY_HAS_COROUTINES++namespace folly::coro {++namespace detail {++class ViaCoroutinePromiseBase {+ public:+  static void* operator new(std::size_t size) {+    return ::folly_coro_async_malloc(size);+  }++  static void operator delete(void* ptr, std::size_t size) {+    ::folly_coro_async_free(ptr, size);+  }++  suspend_always initial_suspend() noexcept { return {}; }++  void return_void() noexcept {}++  [[noreturn]] void unhandled_exception() noexcept {+    folly::assume_unreachable();+  }++  void setExecutor(folly::Executor::KeepAlive<> executor) noexcept {+    executor_ = std::move(executor);+  }++  void setContinuation(ExtendedCoroutineHandle continuation) noexcept {+    continuation_ = continuation;+  }++  void setParentFrame(folly::AsyncStackFrame& parentFrame) noexcept {+    leafFrame_.setParentFrame(parentFrame);+  }++  void setReturnAddress(void* returnAddress) noexcept {+    leafFrame_.setReturnAddress(returnAddress);+  }++  folly::AsyncStackFrame& getLeafFrame() noexcept { return leafFrame_; }++  void setRequestContext(+      std::shared_ptr<folly::RequestContext> context) noexcept {+    context_ = std::move(context);+  }++ protected:+  void scheduleContinuation() noexcept {+    executor_->add([this]() noexcept { this->executeContinuation(); });+  }++ private:+  void executeContinuation() noexcept {+    RequestContextScopeGuard contextScope{std::move(context_)};+    if (folly::isSuspendedLeafActive(leafFrame_)) {+      folly::deactivateSuspendedLeaf(leafFrame_);+    }+    if (leafFrame_.getParentFrame()) {+      folly::resumeCoroutineWithNewAsyncStackRoot(+          continuation_.getHandle(), *leafFrame_.getParentFrame());+    } else {+      continuation_.resume();+    }+  }++ protected:+  virtual ~ViaCoroutinePromiseBase() = default;++  folly::Executor::KeepAlive<> executor_;+  ExtendedCoroutineHandle continuation_;+  folly::AsyncStackFrame leafFrame_;+  std::shared_ptr<RequestContext> context_;+};++template <bool IsStackAware>+class ViaCoroutine {+ public:+  class promise_type final+      : public ViaCoroutinePromiseBase,+        public ExtendedCoroutinePromise {+    struct FinalAwaiter {+      bool await_ready() noexcept { return false; }++      // This code runs immediately after the inner awaitable resumes its fake+      // continuation, and it schedules the real continuation on the awaiter's+      // executor+      FOLLY_CORO_AWAIT_SUSPEND_NONTRIVIAL_ATTRIBUTES void await_suspend(+          coroutine_handle<promise_type> h) noexcept {+        auto& promise = h.promise();+        if (!promise.context_) {+          promise.setRequestContext(RequestContext::saveContext());+        }++        if constexpr (IsStackAware) {+          folly::deactivateAsyncStackFrame(promise.getAsyncFrame());+        }++        promise.scheduleContinuation();+      }++      [[noreturn]] void await_resume() noexcept { folly::assume_unreachable(); }+    };++   public:+    ViaCoroutine get_return_object() noexcept {+      return ViaCoroutine{coroutine_handle<promise_type>::from_promise(*this)};+    }++    FinalAwaiter final_suspend() noexcept { return {}; }++    template <+        bool IsStackAware2 = IsStackAware,+        std::enable_if_t<IsStackAware2, int> = 0>+    folly::AsyncStackFrame& getAsyncFrame() noexcept {+      DCHECK(this->leafFrame_.getParentFrame() != nullptr);+      return *this->leafFrame_.getParentFrame();+    }++    folly::AsyncStackFrame& getLeafFrame() noexcept { return leafFrame_; }++    std::pair<ExtendedCoroutineHandle, AsyncStackFrame*> getErrorHandle(+        exception_wrapper& ex) final {+      auto [handle, frame] = continuation_.getErrorHandle(ex);+      setContinuation(handle);+      if (frame && IsStackAware) {+        leafFrame_.setParentFrame(*frame);+      }+      return {coroutine_handle<promise_type>::from_promise(*this), nullptr};+    }+  };++  ViaCoroutine(ViaCoroutine&& other) noexcept+      : coro_(std::exchange(other.coro_, {})) {}++  ~ViaCoroutine() {+    if (coro_) {+      coro_.destroy();+    }+  }++  static ViaCoroutine create(folly::Executor::KeepAlive<> executor) {+    ViaCoroutine coroutine = createImpl();+    coroutine.setExecutor(std::move(executor));+    return coroutine;+  }++  void setExecutor(folly::Executor::KeepAlive<> executor) noexcept {+    coro_.promise().setExecutor(std::move(executor));+  }++  void setContinuation(ExtendedCoroutineHandle continuation) noexcept {+    coro_.promise().setContinuation(continuation);+  }++  void setParentFrame(folly::AsyncStackFrame& frame) noexcept {+    coro_.promise().setParentFrame(frame);+  }++  void setReturnAddress(void* returnAddress) noexcept {+    coro_.promise().setReturnAddress(returnAddress);+  }++  folly::AsyncStackFrame& getLeafFrame() noexcept {+    return coro_.promise().getLeafFrame();+  }++  void destroy() noexcept {+    if (coro_) {+      std::exchange(coro_, {}).destroy();+    }+  }++  void saveContext() noexcept {+    coro_.promise().setRequestContext(folly::RequestContext::saveContext());+  }++  coroutine_handle<promise_type> getHandle() noexcept { return coro_; }++ private:+  explicit ViaCoroutine(coroutine_handle<promise_type> coro) noexcept+      : coro_(coro) {}++  static ViaCoroutine createImpl() { co_return; }++  coroutine_handle<promise_type> coro_;+};++} // namespace detail++template <typename Awaitable>+class StackAwareViaIfAsyncAwaiter {+  using WithAsyncStackAwaitable =+      decltype(folly::coro::co_withAsyncStack(std::declval<Awaitable>()));+  using Awaiter = folly::coro::awaiter_type_t<WithAsyncStackAwaitable>;+  using CoroutineType = detail::ViaCoroutine<true>;+  using CoroutinePromise = typename CoroutineType::promise_type;+  using WrapperHandle = coroutine_handle<CoroutinePromise>;++  using await_suspend_result_t =+      decltype(std::declval<Awaiter&>().await_suspend(+          std::declval<WrapperHandle>()));++ public:+  explicit StackAwareViaIfAsyncAwaiter(+      folly::Executor::KeepAlive<> executor, Awaitable&& awaitable)+      : viaCoroutine_(CoroutineType::create(std::move(executor))),+        awaitable_(folly::coro::co_withAsyncStack(+            static_cast<Awaitable&&>(awaitable))),+        awaiter_(+            get_awaiter(static_cast<WithAsyncStackAwaitable&&>(awaitable_))) {}++  decltype(auto) await_ready() noexcept(noexcept(awaiter_.await_ready())) {+    return awaiter_.await_ready();+  }++  template <typename Promise>+  auto await_suspend(coroutine_handle<Promise> h) noexcept(noexcept(+      std::declval<Awaiter&>().await_suspend(std::declval<WrapperHandle>())))+      -> await_suspend_result_t {+    auto& promise = h.promise();+    auto& asyncFrame = promise.getAsyncFrame();++    viaCoroutine_.setContinuation(h);+    viaCoroutine_.setParentFrame(asyncFrame);++    if constexpr (!detail::is_coroutine_handle_v<await_suspend_result_t>) {+      viaCoroutine_.saveContext();+    }++    return awaiter_.await_suspend(viaCoroutine_.getHandle());+  }++  decltype(auto) await_resume() noexcept(noexcept(awaiter_.await_resume())) {+    viaCoroutine_.destroy();+    return awaiter_.await_resume();+  }++  template <+      typename Awaiter2 = Awaiter,+      typename Result = decltype(std::declval<Awaiter2&>().await_resume_try())>+  Result await_resume_try() noexcept(+      noexcept(std::declval<Awaiter&>().await_resume_try())) {+    viaCoroutine_.destroy();+    return awaiter_.await_resume_try();+  }++#if FOLLY_HAS_RESULT+  template <+      typename Awaiter2 = Awaiter,+      typename Result =+          decltype(FOLLY_DECLVAL(Awaiter2&).await_resume_result())>+  Result await_resume_result() noexcept(+      noexcept(FOLLY_DECLVAL(Awaiter2&).await_resume_result())) {+    viaCoroutine_.destroy();+    return awaiter_.await_resume_result();+  }+#endif++ private:+  CoroutineType viaCoroutine_;+  WithAsyncStackAwaitable awaitable_;+  Awaiter awaiter_;+};++template <bool IsCallerAsyncStackAware, typename Awaitable>+class ViaIfAsyncAwaiter {+  using Awaiter = folly::coro::awaiter_type_t<Awaitable>;+  using CoroutineType = detail::ViaCoroutine<false>;+  using CoroutinePromise = typename CoroutineType::promise_type;+  using WrapperHandle = coroutine_handle<CoroutinePromise>;++  using await_suspend_result_t =+      decltype(std::declval<Awaiter&>().await_suspend(+          std::declval<WrapperHandle>()));++ public:+  explicit ViaIfAsyncAwaiter(+      folly::Executor::KeepAlive<> executor, Awaitable&& awaitable)+      : viaCoroutine_(CoroutineType::create(std::move(executor))),+        awaiter_(get_awaiter(static_cast<Awaitable&&>(awaitable))) {}++  decltype(auto) await_ready() noexcept(noexcept(awaiter_.await_ready())) {+    return awaiter_.await_ready();+  }++  // NOTE: We are using a heuristic here to determine when is the correct+  // time to capture the RequestContext. We want to capture the context just+  // before the coroutine suspends and execution is returned to the executor.+  //+  // In cases where we are awaiting another coroutine and symmetrically+  // transferring execution to another coroutine we are not yet returning+  // execution to the executor so we want to defer capturing the context until+  // the ViaCoroutine is resumed and suspends in final_suspend() before+  // scheduling the resumption on the executor.+  //+  // In cases where the awaitable may suspend without transferring execution+  // to another coroutine and will therefore return back to the executor we+  // want to capture the execution context before calling into the wrapped+  // awaitable's await_suspend() method (since it's await_suspend() method+  // might schedule resumption on another thread and could resume and destroy+  // the ViaCoroutine before the await_suspend() method returns).+  //+  // The heuristic is that if await_suspend() returns a coroutine_handle+  // then we assume it's the first case. Otherwise if await_suspend() returns+  // void/bool then we assume it's the second case.+  //+  // This heuristic isn't perfect since a coroutine_handle-returning+  // await_suspend() method could return noop_coroutine() in which case we+  // could fail to capture the current context. Awaitable types that do this+  // would need to provide a custom implementation of co_viaIfAsync() that+  // correctly captures the RequestContext to get correct behaviour in this+  // case.++  // NO_INLINE is required here because we capture the return address of the+  // calling coroutine+  template <typename Promise>+  FOLLY_NOINLINE auto+  await_suspend(coroutine_handle<Promise> continuation) noexcept(noexcept(+      std::declval<Awaiter&>().await_suspend(std::declval<WrapperHandle>())))+      -> await_suspend_result_t {+    viaCoroutine_.setContinuation(continuation);++    if constexpr (!detail::is_coroutine_handle_v<await_suspend_result_t>) {+      viaCoroutine_.saveContext();+    }++    if constexpr (IsCallerAsyncStackAware) {+      auto& asyncFrame = continuation.promise().getAsyncFrame();+      auto& stackRoot = *asyncFrame.getStackRoot();++      viaCoroutine_.setParentFrame(asyncFrame);+      viaCoroutine_.setReturnAddress(FOLLY_ASYNC_STACK_RETURN_ADDRESS());++      folly::deactivateAsyncStackFrame(asyncFrame);+      folly::activateSuspendedLeaf(viaCoroutine_.getLeafFrame());++      // Reactivate the stack-frame before we resume.+      auto rollback = makeGuard([&] {+        folly::activateAsyncStackFrame(stackRoot, asyncFrame);+        folly::deactivateSuspendedLeaf(viaCoroutine_.getLeafFrame());+      });+      if constexpr (std::is_same_v<await_suspend_result_t, bool>) {+        if (!awaiter_.await_suspend(viaCoroutine_.getHandle())) {+          return false;+        }+        rollback.dismiss();+        return true;+      } else if constexpr (std::is_same_v<await_suspend_result_t, void>) {+        awaiter_.await_suspend(viaCoroutine_.getHandle());+        rollback.dismiss();+        return;+      } else {+        auto ret = awaiter_.await_suspend(viaCoroutine_.getHandle());+        rollback.dismiss();+        return ret;+      }+    } else {+      return awaiter_.await_suspend(viaCoroutine_.getHandle());+    }+  }++  auto await_resume() noexcept(+      noexcept(std::declval<Awaiter&>().await_resume()))+      -> decltype(std::declval<Awaiter&>().await_resume()) {+    viaCoroutine_.destroy();+    return awaiter_.await_resume();+  }++  template <+      typename Awaiter2 = Awaiter,+      typename Result = decltype(std::declval<Awaiter2&>().await_resume_try())>+  Result await_resume_try() noexcept(+      noexcept(std::declval<Awaiter&>().await_resume_try())) {+    viaCoroutine_.destroy();+    return awaiter_.await_resume_try();+  }++#if FOLLY_HAS_RESULT+  template <+      typename Awaiter2 = Awaiter,+      typename Result =+          decltype(FOLLY_DECLVAL(Awaiter2&).await_resume_result())>+  Result await_resume_result() noexcept(+      noexcept(FOLLY_DECLVAL(Awaiter2&).await_resume_result())) {+    viaCoroutine_.destroy();+    return awaiter_.await_resume_result();+  }+#endif++ private:+  CoroutineType viaCoroutine_;+  Awaiter awaiter_;+};++template <typename Awaitable>+class StackAwareViaIfAsyncAwaitable {+ public:+  explicit StackAwareViaIfAsyncAwaitable(+      folly::Executor::KeepAlive<> executor,+      Awaitable&&+          awaitable) noexcept(std::is_nothrow_move_constructible<Awaitable>::+                                  value)+      : executor_(std::move(executor)),+        awaitable_(static_cast<Awaitable&&>(awaitable)) {}++  auto operator co_await() && {+    if constexpr (is_awaitable_async_stack_aware_v<Awaitable>) {+      return StackAwareViaIfAsyncAwaiter<Awaitable>{+          std::move(executor_), static_cast<Awaitable&&>(awaitable_)};+    } else {+      return ViaIfAsyncAwaiter<true, Awaitable>{+          std::move(executor_), static_cast<Awaitable&&>(awaitable_)};+    }+  }++ private:+  folly::Executor::KeepAlive<> executor_;+  Awaitable awaitable_;+};++template <typename Awaitable>+class ViaIfAsyncAwaitable {+ public:+  explicit ViaIfAsyncAwaitable(+      folly::Executor::KeepAlive<> executor,+      Awaitable&&+          awaitable) noexcept(std::is_nothrow_move_constructible<Awaitable>::+                                  value)+      : executor_(std::move(executor)),+        awaitable_(static_cast<Awaitable&&>(awaitable)) {}++  ViaIfAsyncAwaiter<false, Awaitable> operator co_await() && {+    return ViaIfAsyncAwaiter<false, Awaitable>{+        std::move(executor_), static_cast<Awaitable&&>(awaitable_)};+  }++  friend StackAwareViaIfAsyncAwaitable<Awaitable> tag_invoke(+      cpo_t<co_withAsyncStack>, ViaIfAsyncAwaitable&& self) {+    return StackAwareViaIfAsyncAwaitable<Awaitable>{+        std::move(self.executor_), static_cast<Awaitable&&>(self.awaitable_)};+  }++ private:+  folly::Executor::KeepAlive<> executor_;+  Awaitable awaitable_;+};++namespace detail {++template <typename SemiAwaitable, typename = void>+struct HasViaIfAsyncMethod+    : std::bool_constant<!require_sizeof<SemiAwaitable>> {};++template <typename SemiAwaitable>+struct HasViaIfAsyncMethod<+    SemiAwaitable,+    std::enable_if_t<std::is_void_v<SemiAwaitable>>> : std::false_type {};++template <typename SemiAwaitable>+struct HasViaIfAsyncMethod<+    SemiAwaitable,+    void_t<decltype(std::declval<SemiAwaitable>().viaIfAsync(+        std::declval<folly::Executor::KeepAlive<>>()))>> : std::true_type {};++namespace adl {++template <typename SemiAwaitable>+auto co_viaIfAsync(+    folly::Executor::KeepAlive<> executor,+    SemiAwaitable&&+        awaitable) noexcept(noexcept(static_cast<SemiAwaitable&&>(awaitable)+                                         .viaIfAsync(std::move(executor))))+    -> decltype(static_cast<SemiAwaitable&&>(awaitable).viaIfAsync(+        std::move(executor))) {+  return static_cast<SemiAwaitable&&>(awaitable).viaIfAsync(+      std::move(executor));+}++template <+    typename Awaitable,+    std::enable_if_t<+        is_awaitable_v<Awaitable> && !HasViaIfAsyncMethod<Awaitable>::value,+        int> = 0,+    std::enable_if_t<!must_await_immediately_v<Awaitable>, int> = 0>+auto co_viaIfAsync(folly::Executor::KeepAlive<> executor, Awaitable&& awaitable)+    -> ViaIfAsyncAwaitable<Awaitable> {+  return ViaIfAsyncAwaitable<Awaitable>{+      std::move(executor), static_cast<Awaitable&&>(awaitable)};+}+template <+    typename Awaitable,+    std::enable_if_t<+        is_awaitable_v<Awaitable> && !HasViaIfAsyncMethod<Awaitable>::value,+        int> = 0,+    std::enable_if_t<must_await_immediately_v<Awaitable>, int> = 0>+auto co_viaIfAsync(folly::Executor::KeepAlive<> executor, Awaitable awaitable)+    -> ViaIfAsyncAwaitable<Awaitable> {+  return ViaIfAsyncAwaitable<Awaitable>{+      std::move(executor), std::move(awaitable)};+}++struct ViaIfAsyncFunction {+  template <+      typename Awaitable,+      std::enable_if_t<!must_await_immediately_v<Awaitable>, int> = 0>+  auto operator()(folly::Executor::KeepAlive<> executor, Awaitable&& awaitable)+      const noexcept(noexcept(co_viaIfAsync(+          std::move(executor), static_cast<Awaitable&&>(awaitable))))+          -> decltype(co_viaIfAsync(+              std::move(executor), static_cast<Awaitable&&>(awaitable))) {+    return co_viaIfAsync(+        std::move(executor), static_cast<Awaitable&&>(awaitable));+  }+  template <+      typename Awaitable,+      std::enable_if_t<must_await_immediately_v<Awaitable>, int> = 0>+  auto operator()(folly::Executor::KeepAlive<> executor, Awaitable awaitable)+      const noexcept(noexcept(co_viaIfAsync(+          std::move(executor),+          mustAwaitImmediatelyUnsafeMover(std::move(awaitable))())))+          -> decltype(co_viaIfAsync(+              std::move(executor),+              mustAwaitImmediatelyUnsafeMover(std::move(awaitable))())) {+    return co_viaIfAsync(+        std::move(executor),+        mustAwaitImmediatelyUnsafeMover(std::move(awaitable))());+  }+};++} // namespace adl+} // namespace detail++/// Returns a new awaitable that will resume execution of the awaiting coroutine+/// on a specified executor in the case that the operation does not complete+/// synchronously.+///+/// If the operation completes synchronously then the awaiting coroutine+/// will continue execution on the current thread without transitioning+/// execution to the specified executor.+FOLLY_DEFINE_CPO(detail::adl::ViaIfAsyncFunction, co_viaIfAsync)++template <typename T>+using semi_await_awaitable_t = decltype(folly::coro::co_viaIfAsync(+    FOLLY_DECLVAL(folly::Executor::KeepAlive<>), FOLLY_DECLVAL(T)));++template <typename T, typename = void>+struct is_semi_awaitable : std::bool_constant<!require_sizeof<T>> {};++template <typename T>+struct is_semi_awaitable<T, std::enable_if_t<std::is_void_v<T>>>+    : std::false_type {};++template <typename T>+struct is_semi_awaitable<T, void_t<semi_await_awaitable_t<T>>>+    : std::true_type {};++template <typename T>+constexpr bool is_semi_awaitable_v = is_semi_awaitable<T>::value;++template <typename T>+using semi_await_result_t = await_result_t<semi_await_awaitable_t<T>>;++namespace detail {++template <typename T>+using noexcept_awaitable_of_ = typename T::folly_private_noexcept_awaitable_t;++template <typename Void, typename T>+struct noexcept_awaitable_ {+  static_assert(require_sizeof<T>, "`noexcept_awaitable_t` on incomplete type");+  using type = std::false_type;+};++template <>+struct noexcept_awaitable_<void, void> {+  using type = std::false_type;+};++template <typename T>+struct noexcept_awaitable_<void_t<noexcept_awaitable_of_<T>>, T> {+  using type = noexcept_awaitable_of_<T>;+};++} // namespace detail++// This trait is in `ViaIfAsync.h` so that we don't have include `Noexcept.h`+// If there's ever a use-case that doesn't depend on `ViaIfAsync.h`, this can+// be moved up to `Traits.h`+template <typename T>+using noexcept_awaitable_t =+    typename detail::noexcept_awaitable_<void, T>::type;+template <typename T>+inline constexpr bool noexcept_awaitable_v = noexcept_awaitable_t<T>::value;++namespace detail {++template <typename Awaiter>+using detect_await_resume_try =+    decltype(FOLLY_DECLVAL(Awaiter).await_resume_try());++template <typename Awaiter>+constexpr bool is_awaiter_try = is_detected_v<detect_await_resume_try, Awaiter>;++template <typename Awaitable>+constexpr bool is_awaitable_try = is_awaiter_try<awaiter_type_t<Awaitable>>;++template <typename Awaitable>+class TryAwaiter {+  static_assert(is_awaitable_try<Awaitable&&>);++  using Awaiter = awaiter_type_t<Awaitable>;++ public:+  explicit TryAwaiter(Awaitable&& awaiter)+      : awaiter_(get_awaiter(static_cast<Awaitable&&>(awaiter))) {}++  auto await_ready() noexcept(noexcept(std::declval<Awaiter&>().await_ready()))+      -> decltype(std::declval<Awaiter&>().await_ready()) {+    return awaiter_.await_ready();+  }++  template <typename Promise>+  auto await_suspend(coroutine_handle<Promise> coro) noexcept(+      noexcept(std::declval<Awaiter&>().await_suspend(coro)))+      -> decltype(std::declval<Awaiter&>().await_suspend(coro)) {+    return awaiter_.await_suspend(coro);+  }++  auto await_resume() noexcept(+      noexcept(std::declval<Awaiter&>().await_resume_try()))+      -> decltype(std::declval<Awaiter&>().await_resume_try()) {+    return awaiter_.await_resume_try();+  }++ private:+  Awaiter awaiter_;+};++/**+ * Common machinery for building wrappers like co_awaitTry+ * Allows the wrapper to commute with the universal wrappers like+ * co_withCancellation while keeping the corresponding awaitable on the outside+ */+template <template <typename T> typename Derived, typename T>+class CommutativeWrapperAwaitable {+ public:+  template <+      typename T2,+      std::enable_if_t<!must_await_immediately_v<T2>, int> = 0>+  explicit CommutativeWrapperAwaitable(T2&& awaitable) noexcept(+      std::is_nothrow_constructible_v<T, T2>)+      : inner_(static_cast<T2&&>(awaitable)) {}+  template <+      typename T2,+      std::enable_if_t<must_await_immediately_v<T2>, int> = 0>+  explicit CommutativeWrapperAwaitable(T2 awaitable)+      // `mustAwaitImmediatelyUnsafeMover` has more `noexcept` assertions.+      noexcept(noexcept(T{FOLLY_DECLVAL(T2)}))+      : inner_(mustAwaitImmediatelyUnsafeMover(std::move(awaitable))()) {}++  template <typename Factory>+  explicit CommutativeWrapperAwaitable(std::in_place_t, Factory&& factory)+      : inner_(factory()) {}++  template <+      typename T2 = T,+      typename Result = decltype(folly::coro::co_withCancellation(+          FOLLY_DECLVAL(const folly::CancellationToken&), FOLLY_DECLVAL(T2&&)))>+  friend Derived<Result> co_withCancellation(+      const folly::CancellationToken& cancelToken, Derived<T>&& awaitable) {+    return Derived<Result>{+        std::in_place, [&]() -> decltype(auto) {+          return folly::coro::co_withCancellation(+              cancelToken, static_cast<T&&>(awaitable.inner_));+        }};+  }+  // This overload exists to avoid unnecessarily copying `cancelToken`, which+  // has atomic refcount costs.+  //  - Taking it by-value would force unnecessary token copies for underlying+  //    awaitables that ignore the token.+  //  - If we merged the overloads into a single template, overload resolution+  //    rules would consider it ambiguous wrt the default implementation in+  //    `WithCancellation.h`.+  template <+      typename T2 = T,+      typename Result = decltype(folly::coro::co_withCancellation(+          FOLLY_DECLVAL(folly::CancellationToken&&), FOLLY_DECLVAL(T2&&)))>+  friend Derived<Result> co_withCancellation(+      folly::CancellationToken&& cancelToken, Derived<T>&& awaitable) {+    return Derived<Result>{+        std::in_place, [&]() -> decltype(auto) {+          return folly::coro::co_withCancellation(+              std::move(cancelToken), static_cast<T&&>(awaitable.inner_));+        }};+  }++  template <+      typename T2 = T,+      typename Result =+          decltype(folly::coro::co_withAsyncStack(std::declval<T2>()))>+  friend Derived<Result>+  tag_invoke(cpo_t<co_withAsyncStack>, Derived<T>&& awaitable) noexcept(+      noexcept(folly::coro::co_withAsyncStack(std::declval<T2>()))) {+    return Derived<Result>{+        std::in_place, [&]() -> decltype(auto) {+          return folly::coro::co_withAsyncStack(+              static_cast<T&&>(awaitable.inner_));+        }};+  }++  template <+      typename T2 = T,+      std::enable_if_t<!must_await_immediately_v<T2>, int> = 0,+      typename Result = semi_await_awaitable_t<T2>>+  friend Derived<Result> co_viaIfAsync(+      folly::Executor::KeepAlive<> executor,+      Derived<T>&& awaitable) //+      noexcept(noexcept(folly::coro::co_viaIfAsync(+          FOLLY_DECLVAL(folly::Executor::KeepAlive<>), FOLLY_DECLVAL(T2)))) {+    return Derived<Result>{+        std::in_place, [&]() -> decltype(auto) {+          return folly::coro::co_viaIfAsync(+              std::move(executor), static_cast<T&&>(awaitable.inner_));+        }};+  }+  template <+      typename T2 = T,+      std::enable_if_t<must_await_immediately_v<T2>, int> = 0,+      typename Result = semi_await_awaitable_t<T2>>+  friend Derived<Result> co_viaIfAsync(+      folly::Executor::KeepAlive<> executor,+      Derived<T> awaitable) //+      noexcept(noexcept(folly::coro::co_viaIfAsync(+          FOLLY_DECLVAL(folly::Executor::KeepAlive<>), FOLLY_DECLVAL(T2)))) {+    return Derived<Result>{+        std::in_place, [&]() {+          return folly::coro::co_viaIfAsync(+              std::move(executor),+              mustAwaitImmediatelyUnsafeMover(std::move(awaitable.inner_))());+        }};+  }++  template <+      typename T2 = T,+      typename = decltype(FOLLY_DECLVAL(T2&&).getUnsafeMover(+          FOLLY_DECLVAL(ForMustAwaitImmediately)))>+  auto getUnsafeMover(ForMustAwaitImmediately p) && noexcept {+    // See "A note on object slicing" above `mustAwaitImmediatelyUnsafeMover`+    static_assert(sizeof(Derived<T>) == sizeof(T));+    static_assert( // More `noexcept` tests in `MustAwaitImmediatelyUnsafeMover`+        noexcept(std::move(inner_).getUnsafeMover(p)));+    return MustAwaitImmediatelyUnsafeMover{+        (Derived<T>*)nullptr, std::move(inner_).getUnsafeMover(p)};+  }++  // IMPORTANT: If a commutative wrapper changes safety, immediate- or+  // noexcept-awaitability, it must remember to override these:+  using folly_private_must_await_immediately_t = must_await_immediately_t<T>;+  using folly_private_noexcept_awaitable_t = noexcept_awaitable_t<T>;+  using folly_private_safe_alias_t = safe_alias_of<T>;++ protected:+  T inner_;+};++template <typename T>+class [[FOLLY_ATTR_CLANG_CORO_AWAIT_ELIDABLE]] TryAwaitable+    : public CommutativeWrapperAwaitable<TryAwaitable, T> {+ public:+  using CommutativeWrapperAwaitable<TryAwaitable, T>::+      CommutativeWrapperAwaitable;++  template <+      typename Self,+      std::enable_if_t<+          std::is_same_v<remove_cvref_t<Self>, TryAwaitable>,+          int> = 0,+      typename T2 = like_t<Self, T>,+      std::enable_if_t<is_awaitable_v<T2>, int> = 0>+  friend TryAwaiter<T2> operator co_await(Self && self) {+    return TryAwaiter<T2>{static_cast<Self&&>(self).inner_};+  }++  using folly_private_noexcept_awaitable_t = std::true_type;+};++} // namespace detail++template <+    typename Awaitable,+    std::enable_if_t<!must_await_immediately_v<Awaitable>, int> = 0>+detail::TryAwaitable<remove_cvref_t<Awaitable>> co_awaitTry(+    [[FOLLY_ATTR_CLANG_CORO_AWAIT_ELIDABLE_ARGUMENT]] Awaitable&& awaitable) {+  return detail::TryAwaitable<remove_cvref_t<Awaitable>>{+      static_cast<Awaitable&&>(awaitable)};+}+template <+    typename Awaitable,+    std::enable_if_t<must_await_immediately_v<Awaitable>, int> = 0>+detail::TryAwaitable<Awaitable> co_awaitTry(+    [[FOLLY_ATTR_CLANG_CORO_AWAIT_ELIDABLE_ARGUMENT]] Awaitable awaitable) {+  return detail::TryAwaitable<Awaitable>{+      mustAwaitImmediatelyUnsafeMover(std::move(awaitable))()};+}++template <typename T>+using semi_await_try_result_t = await_result_t<semi_await_awaitable_t<+    decltype(folly::coro::co_awaitTry(FOLLY_DECLVAL(T)))>>;++namespace detail {++template <typename T>+class [[FOLLY_ATTR_CLANG_CORO_AWAIT_ELIDABLE]] NothrowAwaitable+    : public CommutativeWrapperAwaitable<NothrowAwaitable, T> {+ public:+  using CommutativeWrapperAwaitable<NothrowAwaitable, T>::+      CommutativeWrapperAwaitable;++  T&& unwrap() { return std::move(this->inner_); }+};++} // namespace detail++template <+    typename Awaitable,+    std::enable_if_t<!must_await_immediately_v<Awaitable>, int> = 0>+detail::NothrowAwaitable<remove_cvref_t<Awaitable>> co_nothrow(+    [[FOLLY_ATTR_CLANG_CORO_AWAIT_ELIDABLE_ARGUMENT]] Awaitable&& awaitable) {+  return detail::NothrowAwaitable<remove_cvref_t<Awaitable>>{+      static_cast<Awaitable&&>(awaitable)};+}+template <+    typename Awaitable,+    std::enable_if_t<must_await_immediately_v<Awaitable>, int> = 0>+detail::NothrowAwaitable<remove_cvref_t<Awaitable>> co_nothrow(+    [[FOLLY_ATTR_CLANG_CORO_AWAIT_ELIDABLE_ARGUMENT]] Awaitable awaitable) {+  return detail::NothrowAwaitable<remove_cvref_t<Awaitable>>{+      mustAwaitImmediatelyUnsafeMover(std::move(awaitable))()};+}++} // namespace folly::coro++#endif // FOLLY_HAS_COROUTINES
+ folly/folly/coro/WithAsyncStack.h view
@@ -0,0 +1,285 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <folly/coro/Coroutine.h>+#include <folly/coro/Traits.h>+#include <folly/functional/Invoke.h>+#include <folly/lang/Assume.h>+#include <folly/lang/CustomizationPoint.h>+#include <folly/tracing/AsyncStack.h>++#include <cassert>+#include <type_traits>+#include <utility>++#if FOLLY_HAS_COROUTINES++namespace folly::coro {++namespace detail {++class WithAsyncStackCoroutine {+ public:+  class promise_type {+   public:+    WithAsyncStackCoroutine get_return_object() noexcept {+      return WithAsyncStackCoroutine{+          coroutine_handle<promise_type>::from_promise(*this)};+    }++    suspend_always initial_suspend() noexcept { return {}; }++    struct FinalAwaiter {+      bool await_ready() noexcept { return false; }+      void await_suspend(coroutine_handle<promise_type> h) noexcept {+        auto& promise = h.promise();+        folly::deactivateSuspendedLeaf(promise.getLeafFrame());+        folly::resumeCoroutineWithNewAsyncStackRoot(+            promise.continuation_, *promise.getLeafFrame().getParentFrame());+      }++      [[noreturn]] void await_resume() noexcept { folly::assume_unreachable(); }+    };++    FinalAwaiter final_suspend() noexcept { return {}; }++    void return_void() noexcept {}++    [[noreturn]] void unhandled_exception() noexcept {+      folly::assume_unreachable();+    }++    folly::AsyncStackFrame& getLeafFrame() noexcept { return leafFrame; }++   private:+    friend WithAsyncStackCoroutine;++    coroutine_handle<> continuation_;+    folly::AsyncStackFrame leafFrame;+  };++  WithAsyncStackCoroutine() noexcept : coro_() {}++  WithAsyncStackCoroutine(WithAsyncStackCoroutine&& other) noexcept+      : coro_(std::exchange(other.coro_, {})) {}++  ~WithAsyncStackCoroutine() {+    if (coro_) {+      coro_.destroy();+    }+  }++  WithAsyncStackCoroutine& operator=(WithAsyncStackCoroutine other) noexcept {+    std::swap(coro_, other.coro_);+    return *this;+  }++  static WithAsyncStackCoroutine create() { co_return; }++  template <typename Promise>+  coroutine_handle<promise_type> getWrapperHandleFor(+      coroutine_handle<Promise> h, void* returnAddress) noexcept {+    auto& promise = coro_.promise();+    promise.continuation_ = h;+    promise.getLeafFrame().setParentFrame(h.promise().getAsyncFrame());+    promise.getLeafFrame().setReturnAddress(returnAddress);+    return coro_;+  }++  folly::AsyncStackFrame& getLeafFrame() noexcept {+    return coro_.promise().getLeafFrame();+  }++ private:+  explicit WithAsyncStackCoroutine(coroutine_handle<promise_type> h) noexcept+      : coro_(h) {}++  coroutine_handle<promise_type> coro_;+};++template <typename Awaitable>+class WithAsyncStackAwaiter {+  using Awaiter = awaiter_type_t<Awaitable>;++ public:+  explicit WithAsyncStackAwaiter(Awaitable&& awaitable)+      : awaiter_(get_awaiter(static_cast<Awaitable&&>(awaitable))),+        coroWrapper_(WithAsyncStackCoroutine::create()) {}++  auto await_ready() noexcept(noexcept(std::declval<Awaiter&>().await_ready()))+      -> decltype(std::declval<Awaiter&>().await_ready()) {+    return awaiter_.await_ready();+  }++  // needs to be no-inline as return address is being captured for async stack+  // tracing+  template <typename Promise>+  FOLLY_NOINLINE auto await_suspend(coroutine_handle<Promise> h) {+    AsyncStackFrame& callerFrame = h.promise().getAsyncFrame();+    AsyncStackRoot* stackRoot = callerFrame.getStackRoot();+    assert(stackRoot != nullptr);++    auto wrapperHandle =+        coroWrapper_.getWrapperHandleFor(h, FOLLY_ASYNC_STACK_RETURN_ADDRESS());++    folly::deactivateAsyncStackFrame(callerFrame);+    folly::activateSuspendedLeaf(coroWrapper_.getLeafFrame());++    using await_suspend_result_t =+        decltype(awaiter_.await_suspend(wrapperHandle));++    try {+      if constexpr (std::is_same_v<await_suspend_result_t, bool>) {+        if (!awaiter_.await_suspend(wrapperHandle)) {+          folly::activateAsyncStackFrame(*stackRoot, callerFrame);+          folly::deactivateSuspendedLeaf(coroWrapper_.getLeafFrame());+          return false;+        }+        return true;+      } else {+        return awaiter_.await_suspend(wrapperHandle);+      }+    } catch (...) {+      folly::activateAsyncStackFrame(*stackRoot, callerFrame);+      folly::deactivateSuspendedLeaf(coroWrapper_.getLeafFrame());+      throw;+    }+  }++  auto await_resume() noexcept(+      noexcept(std::declval<Awaiter&>().await_resume()))+      -> decltype(std::declval<Awaiter&>().await_resume()) {+    coroWrapper_ = WithAsyncStackCoroutine();+    return awaiter_.await_resume();+  }++  template <typename Awaiter2 = Awaiter>+  auto await_resume_try() noexcept(+      noexcept(std::declval<Awaiter2&>().await_resume_try()))+      -> decltype(std::declval<Awaiter2&>().await_resume_try()) {+    coroWrapper_ = WithAsyncStackCoroutine();+    return awaiter_.await_resume_try();+  }++#if FOLLY_HAS_RESULT+  template <typename Awaiter2 = Awaiter>+  auto await_resume_result() noexcept(+      noexcept(FOLLY_DECLVAL(Awaiter2&).await_resume_result()))+      -> decltype(FOLLY_DECLVAL(Awaiter2&).await_resume_result()) {+    coroWrapper_ = WithAsyncStackCoroutine();+    return awaiter_.await_resume_result();+  }+#endif++ private:+  awaiter_type_t<Awaitable> awaiter_;+  WithAsyncStackCoroutine coroWrapper_;+};++template <typename Awaitable>+class WithAsyncStackAwaitable {+ public:+  explicit WithAsyncStackAwaitable(Awaitable&& awaitable)+      : awaitable_(static_cast<Awaitable&&>(awaitable)) {}++  WithAsyncStackAwaiter<Awaitable&> operator co_await() & {+    return WithAsyncStackAwaiter<Awaitable&>{awaitable_};+  }++  WithAsyncStackAwaiter<Awaitable> operator co_await() && {+    return WithAsyncStackAwaiter<Awaitable>{+        static_cast<Awaitable&&>(awaitable_)};+  }++ private:+  Awaitable awaitable_;+};++struct WithAsyncStackFunction {+  // Dispatches to a custom implementation using tag_invoke()+  template <+      typename Awaitable,+      std::enable_if_t<+          folly::is_tag_invocable_v<WithAsyncStackFunction, Awaitable>,+          int> = 0>+  auto operator()(Awaitable&& awaitable) const noexcept(+      folly::is_nothrow_tag_invocable_v<WithAsyncStackFunction, Awaitable>)+      -> folly::tag_invoke_result_t<WithAsyncStackFunction, Awaitable> {+    return folly::tag_invoke(+        WithAsyncStackFunction{}, static_cast<Awaitable&&>(awaitable));+  }++  // Fallback implementation. Wraps the awaitable in the+  // WithAsyncStackAwaitable which just saves/restores the+  // awaiting coroutine's AsyncStackFrame.+  template <+      typename Awaitable,+      std::enable_if_t<+          !folly::is_tag_invocable_v<WithAsyncStackFunction, Awaitable>,+          int> = 0,+      std::enable_if_t<folly::coro::is_awaitable_v<Awaitable>, int> = 0>+  WithAsyncStackAwaitable<Awaitable> operator()(Awaitable&& awaitable) const+      noexcept(std::is_nothrow_move_constructible_v<Awaitable>) {+    return WithAsyncStackAwaitable<Awaitable>{+        static_cast<Awaitable&&>(awaitable)};+  }+};++} // namespace detail++template <typename Awaitable>+inline constexpr bool is_awaitable_async_stack_aware_v =+    folly::is_tag_invocable_v<detail::WithAsyncStackFunction, Awaitable>;++// Coroutines that support the AsyncStack protocol will apply the+// co_withAsyncStack() customisation-point to an awaitable inside its+// await_transform() to ensure that the current coroutine's AsyncStackFrame+// is saved and later restored when the coroutine resumes.+//+// The default implementation is used for awaitables that don't know+// about the AsyncStackFrame and just wraps the awaitable to ensure+// that the stack-frame is saved/restored if the coroutine suspends.+//+// Awaitables that know about the AsyncStackFrame protocol can customise+// this CPO by defining an overload of tag_invoke() for this CPO+// for their type.+//+// For example:+//   class MyAwaitable {+//     friend MyAwaitable&& tag_invoke(+//         cpo_t<folly::coro::co_withAsyncStack>, MyAwaitable&& awaitable) {+//       return std::move(awaitable);+//     }+//+//     ...+//   };+//+// If you customise this CPO then it is your responsibility to ensure that+// if the awaiting coroutine suspends then before the coroutine is resumed+// that its original AsyncStackFrame is activated on the current thread.+// e.g. using folly::activateAsyncStackFrame()+//+// The awaiting coroutine's AsyncStackFrame can be obtained from its+// promise, which is assumed to have a 'AsyncStackFrame& getAsyncFrame()'+// method that returns a reference to the parent coroutine's async frame.++FOLLY_DEFINE_CPO(detail::WithAsyncStackFunction, co_withAsyncStack)++} // namespace folly::coro++#endif // FOLLY_HAS_COROUTINES
+ folly/folly/coro/WithCancellation.h view
@@ -0,0 +1,118 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <folly/CancellationToken.h>+#include <folly/coro/AwaitImmediately.h>+#include <folly/coro/Coroutine.h>+#include <folly/lang/CustomizationPoint.h>++#if FOLLY_HAS_COROUTINES++/**+ * \file coro/WithCancellation.h+ * co_withCancellation allows caller to pass in a cancellation token to a+ * awaitable+ *+ * \refcode folly/docs/examples/folly/coro/WithCancellation.cpp+ */++namespace folly {+namespace coro {++namespace detail {+namespace adl {++/// Default implementation that does not hook the cancellation token.+/// Types must opt-in to hooking cancellation by customising this function.+template <+    typename Awaitable,+    std::enable_if_t<!must_await_immediately_v<Awaitable>, int> = 0>+Awaitable&& co_withCancellation(+    const folly::CancellationToken&, Awaitable&& awaitable) noexcept {+  return static_cast<Awaitable&&>(awaitable);+}+template <+    typename Awaitable,+    std::enable_if_t<must_await_immediately_v<Awaitable>, int> = 0>+Awaitable co_withCancellation(+    const folly::CancellationToken&, Awaitable awaitable) noexcept {+  return mustAwaitImmediatelyUnsafeMover(std::move(awaitable))();+}++struct WithCancellationFunction {+  template <+      typename Awaitable,+      std::enable_if_t<!must_await_immediately_v<Awaitable>, int> = 0>+  auto operator()(+      const folly::CancellationToken& cancelToken, Awaitable&& awaitable) const+      noexcept(noexcept(co_withCancellation(+          cancelToken, static_cast<Awaitable&&>(awaitable))))+          -> decltype(co_withCancellation(+              cancelToken, static_cast<Awaitable&&>(awaitable))) {+    return co_withCancellation(+        cancelToken, static_cast<Awaitable&&>(awaitable));+  }+  template <+      typename Awaitable,+      std::enable_if_t<must_await_immediately_v<Awaitable>, int> = 0>+  auto operator()(+      const folly::CancellationToken& cancelToken, Awaitable awaitable) const+      noexcept(noexcept(co_withCancellation(+          cancelToken,+          mustAwaitImmediatelyUnsafeMover(std::move(awaitable))())))+          -> decltype(co_withCancellation(+              cancelToken,+              mustAwaitImmediatelyUnsafeMover(std::move(awaitable))())) {+    return co_withCancellation(+        cancelToken, mustAwaitImmediatelyUnsafeMover(std::move(awaitable))());+  }+  template <+      typename Awaitable,+      std::enable_if_t<!must_await_immediately_v<Awaitable>, int> = 0>+  auto operator()(folly::CancellationToken&& cancelToken, Awaitable&& awaitable)+      const noexcept(noexcept(co_withCancellation(+          std::move(cancelToken), static_cast<Awaitable&&>(awaitable))))+          -> decltype(co_withCancellation(+              std::move(cancelToken), static_cast<Awaitable&&>(awaitable))) {+    return co_withCancellation(+        std::move(cancelToken), static_cast<Awaitable&&>(awaitable));+  }+  template <+      typename Awaitable,+      std::enable_if_t<must_await_immediately_v<Awaitable>, int> = 0>+  auto operator()(folly::CancellationToken&& cancelToken, Awaitable awaitable)+      const noexcept(noexcept(co_withCancellation(+          std::move(cancelToken),+          mustAwaitImmediatelyUnsafeMover(std::move(awaitable))())))+          -> decltype(co_withCancellation(+              std::move(cancelToken),+              mustAwaitImmediatelyUnsafeMover(std::move(awaitable))())) {+    return co_withCancellation(+        std::move(cancelToken),+        mustAwaitImmediatelyUnsafeMover(std::move(awaitable))());+  }+};+} // namespace adl+} // namespace detail++FOLLY_DEFINE_CPO(detail::adl::WithCancellationFunction, co_withCancellation)++} // namespace coro+} // namespace folly++#endif // FOLLY_HAS_COROUTINES
+ folly/folly/coro/detail/Barrier.h view
@@ -0,0 +1,159 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <folly/coro/Coroutine.h>+#include <folly/coro/Traits.h>+#include <folly/coro/WithAsyncStack.h>+#include <folly/tracing/AsyncStack.h>++#include <atomic>+#include <cassert>+#include <cstddef>+#include <cstdint>+#include <utility>++#if FOLLY_HAS_COROUTINES++namespace folly {+namespace coro {+namespace detail {++// A Barrier is a synchronisation building block that can be used to+// implement higher-level coroutine-based synchronisation primitives.+//+// It allows a single coroutine to wait until a counter reaches zero.+// The counter typically represents the amount of outstanding work.+// When a coroutine completes some work it should call arrive() which+// will return a continuation.+class Barrier {+ public:+  explicit Barrier(std::size_t initialCount = 0) noexcept+      : count_(initialCount) {}++  void add(std::size_t count = 1) noexcept {+    [[maybe_unused]] std::size_t oldCount =+        count_.fetch_add(count, std::memory_order_relaxed);+    // Check we didn't overflow the count.+    assert(SIZE_MAX - oldCount >= count);+  }++  // Query the number of remaining tasks that the barrier is waiting+  // for. This indicates the number of arrive() calls that must be+  // made before the Barrier will be released.+  //+  // Note that this should just be used as an approximate guide+  // for the number of outstanding tasks. This value may be out+  // of date immediately upon being returned.+  std::size_t remaining() const noexcept {+    return count_.load(std::memory_order_acquire);+  }++  [[nodiscard]] coroutine_handle<> arrive(+      folly::AsyncStackFrame& currentFrame) noexcept {+    auto& stackRoot = *currentFrame.getStackRoot();+    folly::deactivateAsyncStackFrame(currentFrame);++    const std::size_t oldCount = count_.fetch_sub(1, std::memory_order_acq_rel);++    // Invalid to call arrive() if you haven't previously incremented the+    // counter using .add().+    assert(oldCount >= 1);++    if (oldCount == 1) {+      if (asyncFrame_ != nullptr) {+        folly::activateAsyncStackFrame(stackRoot, *asyncFrame_);+      }+      return std::exchange(continuation_, {});+    } else {+      return coro::noop_coroutine();+    }+  }++  [[nodiscard]] coroutine_handle<> arrive() noexcept {+    const std::size_t oldCount = count_.fetch_sub(1, std::memory_order_acq_rel);++    // Invalid to call arrive() if you haven't previously incremented the+    // counter using .add().+    assert(oldCount >= 1);++    if (oldCount == 1) {+      auto coro = std::exchange(continuation_, {});+      if (asyncFrame_ != nullptr) {+        folly::resumeCoroutineWithNewAsyncStackRoot(coro, *asyncFrame_);+        return coro::noop_coroutine();+      } else {+        return coro;+      }+    } else {+      return coro::noop_coroutine();+    }+  }++ private:+  class Awaiter {+   public:+    explicit Awaiter(Barrier& barrier) noexcept : barrier_(barrier) {}++    bool await_ready() noexcept { return false; }++    template <typename Promise>+    coroutine_handle<> await_suspend(+        coroutine_handle<Promise> continuation) noexcept {+      if constexpr (detail::promiseHasAsyncFrame_v<Promise>) {+        barrier_.setContinuation(+            continuation, &continuation.promise().getAsyncFrame());+        return barrier_.arrive(continuation.promise().getAsyncFrame());+      } else {+        barrier_.setContinuation(continuation, nullptr);+        return barrier_.arrive();+      }+    }++    void await_resume() noexcept {}++   private:+    friend Awaiter tag_invoke(+        cpo_t<co_withAsyncStack>, Awaiter&& awaiter) noexcept {+      return Awaiter{awaiter.barrier_};+    }++    Barrier& barrier_;+  };++ public:+  auto arriveAndWait() noexcept { return Awaiter{*this}; }++  void setContinuation(+      coroutine_handle<> continuation,+      folly::AsyncStackFrame* parentFrame) noexcept {+    assert(!continuation_);+    continuation_ = continuation;+    asyncFrame_ = parentFrame;+  }++ private:+  std::atomic<std::size_t> count_;+  coroutine_handle<> continuation_;+  folly::AsyncStackFrame* asyncFrame_ = nullptr;+};++} // namespace detail+} // namespace coro+} // namespace folly++#endif // FOLLY_HAS_COROUTINES
+ folly/folly/coro/detail/BarrierTask.h view
@@ -0,0 +1,231 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <folly/coro/Coroutine.h>+#include <folly/coro/WithAsyncStack.h>+#include <folly/coro/detail/Barrier.h>+#include <folly/coro/detail/Malloc.h>++#include <cassert>+#include <utility>++#if FOLLY_HAS_COROUTINES++namespace folly {+namespace coro {+namespace detail {++class BarrierTask {+ public:+  class promise_type {+    struct FinalAwaiter {+      bool await_ready() noexcept { return false; }++      coroutine_handle<> await_suspend(+          coroutine_handle<promise_type> h) noexcept {+        auto& promise = h.promise();+        assert(promise.barrier_ != nullptr);+        return promise.barrier_->arrive(promise.asyncFrame_);+      }++      void await_resume() noexcept {}+    };++   public:+    static void* operator new(std::size_t size) {+      return ::folly_coro_async_malloc(size);+    }++    static void operator delete(void* ptr, std::size_t size) {+      ::folly_coro_async_free(ptr, size);+    }++    BarrierTask get_return_object() noexcept {+      return BarrierTask{coroutine_handle<promise_type>::from_promise(*this)};+    }++    suspend_always initial_suspend() noexcept { return {}; }++    FinalAwaiter final_suspend() noexcept { return {}; }++    template <typename Awaitable>+    auto await_transform(Awaitable&& awaitable) {+      return folly::coro::co_withAsyncStack(+          static_cast<Awaitable&&>(awaitable));+    }++    void return_void() noexcept {}++    [[noreturn]] void unhandled_exception() noexcept { std::terminate(); }++    void setBarrier(Barrier* barrier) noexcept {+      assert(barrier_ == nullptr);+      barrier_ = barrier;+    }++    folly::AsyncStackFrame& getAsyncFrame() noexcept { return asyncFrame_; }++   private:+    folly::AsyncStackFrame asyncFrame_;+    Barrier* barrier_ = nullptr;+  };++ private:+  using handle_t = coroutine_handle<promise_type>;++  explicit BarrierTask(handle_t coro) noexcept : coro_(coro) {}++ public:+  BarrierTask(BarrierTask&& other) noexcept+      : coro_(std::exchange(other.coro_, {})) {}++  ~BarrierTask() {+    if (coro_) {+      coro_.destroy();+    }+  }++  BarrierTask& operator=(BarrierTask other) noexcept {+    swap(other);+    return *this;+  }++  void swap(BarrierTask& b) noexcept { std::swap(coro_, b.coro_); }++  FOLLY_NOINLINE void start(Barrier* barrier) noexcept {+    start(barrier, folly::getDetachedRootAsyncStackFrame());+  }++  FOLLY_NOINLINE void start(+      Barrier* barrier, folly::AsyncStackFrame& parentFrame) noexcept {+    assert(coro_);+    auto& calleeFrame = coro_.promise().getAsyncFrame();+    calleeFrame.setParentFrame(parentFrame);+    calleeFrame.setReturnAddress();+    coro_.promise().setBarrier(barrier);++    folly::resumeCoroutineWithNewAsyncStackRoot(coro_);+  }++ private:+  handle_t coro_;+};++class DetachedBarrierTask {+ public:+  class promise_type {+   public:+    promise_type() noexcept {+      asyncFrame_.setParentFrame(folly::getDetachedRootAsyncStackFrame());+    }++    DetachedBarrierTask get_return_object() noexcept {+      return DetachedBarrierTask{+          coroutine_handle<promise_type>::from_promise(*this)};+    }++    suspend_always initial_suspend() noexcept { return {}; }++    auto final_suspend() noexcept {+      struct awaiter {+        bool await_ready() noexcept { return false; }+        auto await_suspend(coroutine_handle<promise_type> h) noexcept {+          assert(h.promise().barrier_ != nullptr);+          auto continuation =+              h.promise().barrier_->arrive(h.promise().getAsyncFrame());++          // Due to a bug in MSVC versions up to and including 19.39, we observe+          // an extra call to the destructor of the task with an explicit call+          // to coroutine_handle::destroy. Furthermore, with versions+          // above 19.30, this causes a crash when named return value+          // optimization is enabled.+#if !(!defined(__clang__) && defined(_MSC_VER) && _MSC_VER <= 1939)+          h.destroy();+#endif++          return continuation;+        }+        void await_resume() noexcept {}+      };+      return awaiter{};+    }++    [[noreturn]] void unhandled_exception() noexcept { std::terminate(); }++    void return_void() noexcept {}++    template <typename Awaitable>+    auto await_transform(Awaitable&& awaitable) {+      return folly::coro::co_withAsyncStack(+          static_cast<Awaitable&&>(awaitable));+    }++    void setBarrier(Barrier* barrier) noexcept { barrier_ = barrier; }++    AsyncStackFrame& getAsyncFrame() noexcept { return asyncFrame_; }++   private:+    AsyncStackFrame asyncFrame_;+    Barrier* barrier_;+  };++ private:+  using handle_t = coroutine_handle<promise_type>;++  explicit DetachedBarrierTask(handle_t coro) : coro_(coro) {}++ public:+  DetachedBarrierTask(DetachedBarrierTask&& other) noexcept+      : coro_(std::exchange(other.coro_, {})) {}++  ~DetachedBarrierTask() {+    if (coro_) {+      coro_.destroy();+    }+  }++  FOLLY_NOINLINE void start(Barrier* barrier) && noexcept {+    std::move(*this).start(barrier, FOLLY_ASYNC_STACK_RETURN_ADDRESS());+  }++  FOLLY_NOINLINE void start(+      Barrier* barrier, folly::AsyncStackFrame& parentFrame) && noexcept {+    assert(coro_);+    coro_.promise().getAsyncFrame().setParentFrame(parentFrame);+    std::move(*this).start(barrier, FOLLY_ASYNC_STACK_RETURN_ADDRESS());+  }++  void start(Barrier* barrier, void* returnAddress) && noexcept {+    assert(coro_);+    assert(barrier != nullptr);+    barrier->add(1);+    auto coro = std::exchange(coro_, {});+    coro.promise().setBarrier(barrier);+    coro.promise().getAsyncFrame().setReturnAddress(returnAddress);+    folly::resumeCoroutineWithNewAsyncStackRoot(coro);+  }++ private:+  handle_t coro_;+};++} // namespace detail+} // namespace coro+} // namespace folly++#endif // FOLLY_HAS_COROUTINES
+ folly/folly/coro/detail/CurrentAsyncFrame.h view
@@ -0,0 +1,76 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++/*+ * This macro enables FbSystrace usage in production for fb4a. When+ * FOLLY_SCOPED_TRACE_SECTION_HEADER is defined then a trace section is started+ * and later automatically terminated at the close of the scope it is called in.+ * In all other cases no action is taken.+ */++#pragma once++#include <folly/Executor.h>+#include <folly/coro/Coroutine.h>+#include <folly/coro/WithAsyncStack.h>+#include <folly/tracing/AsyncStack.h>++#if FOLLY_HAS_COROUTINES++namespace folly {+namespace coro {+namespace detail {++// Helper struct for getting access to the current coroutine's AsyncStackFrame+class CurrentAsyncStackFrameAwaitable {+  class Awaiter {+   public:+    bool await_ready() noexcept { return false; }++    template <typename Promise>+    bool await_suspend(coroutine_handle<Promise> h) noexcept {+      asyncFrame_ = &h.promise().getAsyncFrame();+      return false;+    }++    folly::AsyncStackFrame& await_resume() noexcept { return *asyncFrame_; }++   private:+    folly::AsyncStackFrame* asyncFrame_ = nullptr;+  };++ public:+  CurrentAsyncStackFrameAwaitable viaIfAsync(+      const folly::Executor::KeepAlive<>&) const noexcept {+    return {};+  }++  friend Awaiter tag_invoke(+      cpo_t<co_withAsyncStack>, CurrentAsyncStackFrameAwaitable) noexcept {+    return Awaiter{};+  }+};++// Await this object within a coroutine to obtain a reference to the current+// coroutine's AsyncStackFrame. This will only work within a coroutine whose+// promise_type implements the getAsyncFrame() method.+inline constexpr CurrentAsyncStackFrameAwaitable co_current_async_stack_frame{};++} // namespace detail+} // namespace coro+} // namespace folly++#endif // FOLLY_HAS_COROUTINES
+ folly/folly/coro/detail/Helpers.h view
@@ -0,0 +1,53 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <folly/Executor.h>+#include <folly/SingletonThreadLocal.h>+#include <folly/coro/Coroutine.h>+#include <folly/io/async/Request.h>+#include <folly/tracing/AsyncStack.h>++#if FOLLY_HAS_COROUTINES++namespace folly {+namespace coro {+namespace detail {+// Helper class that can be used to annotate Awaitable objects that will+// guarantee that they will be resumed on the correct executor so that+// when the object is awaited within a Task<T> it doesn't automatically+// wrap the Awaitable in something that forces a reschedule onto the+// executor.+template <typename Awaitable>+class UnsafeResumeInlineSemiAwaitable {+ public:+  explicit UnsafeResumeInlineSemiAwaitable(Awaitable&& awaitable) noexcept+      : awaitable_(awaitable) {}++  Awaitable&& viaIfAsync(folly::Executor::KeepAlive<>) && noexcept {+    return static_cast<Awaitable&&>(awaitable_);+  }++ private:+  Awaitable awaitable_;+};++} // namespace detail+} // namespace coro+} // namespace folly++#endif // FOLLY_HAS_COROUTINES
+ folly/folly/coro/detail/InlineTask.h view
@@ -0,0 +1,312 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <folly/ScopeGuard.h>+#include <folly/Try.h>+#include <folly/coro/Coroutine.h>+#include <folly/coro/WithAsyncStack.h>+#include <folly/coro/detail/Malloc.h>+#include <folly/lang/Assume.h>+#include <folly/tracing/AsyncStack.h>++#include <cassert>+#include <utility>++#if FOLLY_HAS_COROUTINES++namespace folly {+namespace coro {+namespace detail {++/// InlineTask<T> is a coroutine-return type where the coroutine is launched+/// inline in the current execution context when it is co_awaited and the+/// task's continuation is launched inline in the execution context that the+/// task completed on.+///+/// This task type is primarily intended as a building block for certain+/// coroutine operators. It is not intended for general use in application+/// code or in library interfaces exposed to library code as it can easily be+/// abused to accidentally run logic on the wrong execution context.+///+/// For this reason, the InlineTask<T> type has been placed inside the+/// folly::coro::detail namespace to discourage general usage.+template <typename T>+class InlineTask;++class InlineTaskPromiseBase {+  struct FinalAwaiter {+    bool await_ready() noexcept { return false; }++    template <typename Promise>+    coroutine_handle<> await_suspend(coroutine_handle<Promise> h) noexcept {+      InlineTaskPromiseBase& promise = h.promise();+      return promise.continuation_;+    }++    void await_resume() noexcept {}+  };++ protected:+  InlineTaskPromiseBase() noexcept = default;++  InlineTaskPromiseBase(const InlineTaskPromiseBase&) = delete;+  InlineTaskPromiseBase(InlineTaskPromiseBase&&) = delete;+  InlineTaskPromiseBase& operator=(const InlineTaskPromiseBase&) = delete;+  InlineTaskPromiseBase& operator=(InlineTaskPromiseBase&&) = delete;++ public:+  static void* operator new(std::size_t size) {+    return ::folly_coro_async_malloc(size);+  }++  static void operator delete(void* ptr, std::size_t size) {+    ::folly_coro_async_free(ptr, size);+  }++  suspend_always initial_suspend() noexcept { return {}; }++  auto final_suspend() noexcept { return FinalAwaiter{}; }++  void set_continuation(coroutine_handle<> continuation) noexcept {+    assert(!continuation_);+    continuation_ = continuation;+  }++ private:+  coroutine_handle<> continuation_;+};++template <typename T>+class InlineTaskPromise : public InlineTaskPromiseBase {+ public:+  static_assert(+      std::is_move_constructible<T>::value,+      "InlineTask<T> only supports types that are move-constructible.");+  static_assert(+      !std::is_rvalue_reference<T>::value, "InlineTask<T&&> is not supported");++  InlineTaskPromise() noexcept = default;++  ~InlineTaskPromise() = default;++  InlineTask<T> get_return_object() noexcept;++  template <+      typename Value = T,+      std::enable_if_t<std::is_convertible<Value&&, T>::value, int> = 0>+  void return_value(Value&& value) noexcept(+      std::is_nothrow_constructible<T, Value&&>::value) {+    result_.emplace(static_cast<Value&&>(value));+  }++  void unhandled_exception() noexcept {+    result_.emplaceException(folly::exception_wrapper{current_exception()});+  }++  T result() { return std::move(result_).value(); }++ private:+  // folly::Try<T> doesn't support storing reference types so we store a+  // std::reference_wrapper instead.+  using StorageType = std::conditional_t<+      std::is_lvalue_reference<T>::value,+      std::reference_wrapper<std::remove_reference_t<T>>,+      T>;++  folly::Try<StorageType> result_;+};++template <>+class InlineTaskPromise<void> : public InlineTaskPromiseBase {+ public:+  InlineTaskPromise() noexcept = default;++  InlineTask<void> get_return_object() noexcept;++  void return_void() noexcept {}++  void unhandled_exception() noexcept {+    result_.emplaceException(folly::exception_wrapper{current_exception()});+  }++  void result() { return result_.value(); }++ private:+  folly::Try<void> result_;+};++template <typename T>+class InlineTask {+ public:+  using promise_type = detail::InlineTaskPromise<T>;++ private:+  using handle_t = coroutine_handle<promise_type>;++ public:+  InlineTask(InlineTask&& other) noexcept+      : coro_(std::exchange(other.coro_, {})) {}++  ~InlineTask() {+    if (coro_) {+      coro_.destroy();+    }+  }++  class Awaiter {+   public:+    ~Awaiter() {+      if (coro_) {+        coro_.destroy();+      }+    }++    bool await_ready() noexcept { return false; }++    handle_t await_suspend(coroutine_handle<> awaitingCoroutine) noexcept {+      assert(coro_ && !coro_.done());+      coro_.promise().set_continuation(awaitingCoroutine);+      return coro_;+    }++    T await_resume() {+      auto destroyOnExit = folly::makeGuard([this] {+        std::exchange(coro_, {}).destroy();+      });+      return coro_.promise().result();+    }++   private:+    friend class InlineTask<T>;+    explicit Awaiter(handle_t coro) noexcept : coro_(coro) {}+    handle_t coro_;+  };++  Awaiter operator co_await() && {+    assert(coro_ && !coro_.done());+    return Awaiter{std::exchange(coro_, {})};+  }++ private:+  friend class InlineTaskPromise<T>;+  explicit InlineTask(handle_t coro) noexcept : coro_(coro) {}+  handle_t coro_;+};++template <typename T>+inline InlineTask<T> InlineTaskPromise<T>::get_return_object() noexcept {+  return InlineTask<T>{+      coroutine_handle<InlineTaskPromise<T>>::from_promise(*this)};+}++inline InlineTask<void> InlineTaskPromise<void>::get_return_object() noexcept {+  return InlineTask<void>{+      coroutine_handle<InlineTaskPromise<void>>::from_promise(*this)};+}++/// InlineTaskDetached is a coroutine-return type where the coroutine is+/// launched in the current execution context when it is created and the+/// task's continuation is launched inline in the execution context that the+/// task completed on.+///+/// This task type is primarily intended as a building block for certain+/// coroutine operators. It is not intended for general use in application+/// code or in library interfaces exposed to library code as it can easily be+/// abused to accidentally run logic on the wrong execution context.+///+/// For this reason, the InlineTaskDetached type has been placed inside the+/// folly::coro::detail namespace to discourage general usage.+struct InlineTaskDetached {+  class promise_type {+    struct FinalAwaiter {+      bool await_ready() noexcept { return false; }+      void await_suspend(coroutine_handle<promise_type> h) noexcept {+        folly::deactivateAsyncStackFrame(h.promise().getAsyncFrame());+        h.destroy();+      }+      [[noreturn]] void await_resume() noexcept { folly::assume_unreachable(); }+    };++   public:+    static void* operator new(std::size_t size) {+      return ::folly_coro_async_malloc(size);+    }++    static void operator delete(void* ptr, std::size_t size) {+      ::folly_coro_async_free(ptr, size);+    }++    promise_type() noexcept {+      asyncFrame_.setParentFrame(folly::getDetachedRootAsyncStackFrame());+    }++    InlineTaskDetached get_return_object() noexcept {+      return InlineTaskDetached{+          coroutine_handle<promise_type>::from_promise(*this)};+    }++    suspend_always initial_suspend() noexcept { return {}; }++    FinalAwaiter final_suspend() noexcept { return {}; }++    void return_void() noexcept {}++    [[noreturn]] void unhandled_exception() noexcept { std::terminate(); }++    template <typename Awaitable>+    decltype(auto) await_transform(Awaitable&& awaitable) {+      return folly::coro::co_withAsyncStack(+          static_cast<Awaitable&&>(awaitable));+    }++    folly::AsyncStackFrame& getAsyncFrame() noexcept { return asyncFrame_; }++   private:+    folly::AsyncStackFrame asyncFrame_;+  };++  InlineTaskDetached(InlineTaskDetached&& other) noexcept+      : coro_(std::exchange(other.coro_, {})) {}++  ~InlineTaskDetached() {+    if (coro_) {+      coro_.destroy();+    }+  }++  FOLLY_NOINLINE void start() noexcept {+    start(FOLLY_ASYNC_STACK_RETURN_ADDRESS());+  }++  void start(void* returnAddress) noexcept {+    coro_.promise().getAsyncFrame().setReturnAddress(returnAddress);+    folly::resumeCoroutineWithNewAsyncStackRoot(std::exchange(coro_, {}));+  }++ private:+  explicit InlineTaskDetached(coroutine_handle<promise_type> h) noexcept+      : coro_(h) {}++  coroutine_handle<promise_type> coro_;+};++} // namespace detail+} // namespace coro+} // namespace folly++#endif // FOLLY_HAS_COROUTINES
+ folly/folly/coro/detail/Malloc.cpp view
@@ -0,0 +1,43 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#include <folly/coro/detail/Malloc.h>++#include <folly/lang/Hint.h>+#include <folly/lang/New.h>++extern "C" {++FOLLY_NOINLINE+void* folly_coro_async_malloc(std::size_t size) {+  auto p = folly::operator_new(size);++  // Add this after the call to prevent the compiler from+  // turning the call to operator new() into a tailcall.+  folly::compiler_must_not_elide(p);++  return p;+}++FOLLY_NOINLINE+void folly_coro_async_free(void* ptr, std::size_t size) {+  folly::operator_delete(ptr, size);++  // Add this after the call to prevent the compiler from+  // turning the call to operator delete() into a tailcall.+  folly::compiler_must_not_elide(size);+}+} // extern "C"
+ folly/folly/coro/detail/Malloc.h view
@@ -0,0 +1,33 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <folly/CPortability.h>++#include <cstddef>++extern "C" {++// Heap allocations for coroutine-frames for all async coroutines+// (Task, AsyncGenerator, etc.) should be funneled through these+// functions to allow better tracing/profiling of coroutine allocations.+FOLLY_NOINLINE+void* folly_coro_async_malloc(std::size_t size);++FOLLY_NOINLINE+void folly_coro_async_free(void* ptr, std::size_t size);+} // extern "C"
+ folly/folly/coro/detail/ManualLifetime.h view
@@ -0,0 +1,134 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <memory>+#include <new>+#include <type_traits>++#include <folly/ScopeGuard.h>++namespace folly {+namespace coro {+namespace detail {++// Helper class for a variable with manually-controlled lifetime.+//+// You must explicitly call .construct() to construct/initialise the value.+//+// If it has been initialised then you must explicitly call .destruct() before+// the ManualLifetime object is destroyed to ensure the destructor is run.+template <typename T>+class ManualLifetime {+ public:+  ManualLifetime() noexcept {}+  ~ManualLifetime() {}++  template <+      typename... Args,+      std::enable_if_t<std::is_constructible<T, Args...>::value, int> = 0>+  void construct(Args&&... args) noexcept(+      noexcept(std::is_nothrow_constructible<T, Args...>::value)) {+    ::new (static_cast<void*>(std::addressof(value_)))+        T(static_cast<Args&&>(args)...);+  }++  void destruct() noexcept { value_.~T(); }++  const T& get() const& { return value_; }+  T& get() & { return value_; }+  const T&& get() const&& { return static_cast<const T&&>(value_); }+  T&& get() && { return static_cast<T&&>(value_); }++ private:+  union {+    std::remove_const_t<T> value_;+  };+};++template <typename T>+class ManualLifetime<T&> {+ public:+  ManualLifetime() noexcept : ptr_(nullptr) {}+  ~ManualLifetime() {}++  void construct(T& value) noexcept { ptr_ = std::addressof(value); }++  void destruct() noexcept { ptr_ = nullptr; }++  T& get() const noexcept { return *ptr_; }++ private:+  T* ptr_;+};++template <typename T>+class ManualLifetime<T&&> {+ public:+  ManualLifetime() noexcept : ptr_(nullptr) {}+  ~ManualLifetime() {}++  void construct(T&& value) noexcept { ptr_ = std::addressof(value); }++  void destruct() noexcept { ptr_ = nullptr; }++  T&& get() const noexcept { return static_cast<T&&>(*ptr_); }++ private:+  T* ptr_;+};++template <>+class ManualLifetime<void> {+ public:+  void construct() noexcept {}++  void destruct() noexcept {}++  void get() const noexcept {}+};++// For use when the ManualLifetime is a member of a union. First,+// it in-place constructs the ManualLifetime, making it the active+// member of the union. Then it calls 'construct' on it to construct+// the value inside it.+template <+    typename T,+    typename... Args,+    std::enable_if_t<std::is_constructible<T, Args...>::value, int> = 0>+void activate(ManualLifetime<T>& box, Args&&... args) noexcept(+    std::is_nothrow_constructible<T, Args...>::value) {+  auto* p = ::new (&box) ManualLifetime<T>{};+  // Use ScopeGuard to destruct the ManualLifetime if the 'construct' throws.+  auto guard = makeGuard([p]() noexcept { p->~ManualLifetime(); });+  p->construct(static_cast<Args&&>(args)...);+  guard.dismiss();+}++// For use when the ManualLifetime is a member of a union. First,+// it calls 'destruct' on the ManualLifetime to destroy the value+// inside it. Then it calls the destructor of the ManualLifetime+// object itself.+template <typename T>+void deactivate(ManualLifetime<T>& box) noexcept {+  box.destruct();+  box.~ManualLifetime();+}++} // namespace detail+} // namespace coro+} // namespace folly
+ folly/folly/coro/detail/PickTaskWrapper.h view
@@ -0,0 +1,151 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <folly/Portability.h>+#include <folly/lang/SafeAlias-fwd.h>++/// For functions-of-coros, like `timeout()` or `collectAll()`, we want the+/// outer coro to be able to pass through these attributes of the inner coro:+///  - `must_await_immediately_v`+///  - `noexcept_awaitable_v`+///  - `safe_alias_of_v`+///+/// Variation along these dimensions is currently implemented as a zoo of coro+/// templates and wrappers -- `Task` aka `UnsafeMovableTask`, `NowTask`,+/// `SafeTask`, `AsNoexcept<InnerTask>`.  The type function `PickTaskWrapper`+/// provides common logic for picking a task type with the given attributes.++#if FOLLY_HAS_COROUTINES++namespace folly::coro {++template <typename T>+class Task;+template <typename T>+class TaskWithExecutor;++template <safe_alias, typename>+class SafeTask;+template <safe_alias, typename>+class SafeTaskWithExecutor;++template <typename T>+class NowTask;+template <typename T>+class NowTaskWithExecutor;++template <typename, auto>+class AsNoexcept;++namespace detail {++struct identity_metafunction {+  template <typename T>+  using apply = T;+};++template <safe_alias, bool /*must await immediately (now)*/>+struct PickTaskWrapperImpl;++#if FOLLY_HAS_IMMOVABLE_COROUTINES++template <>+struct PickTaskWrapperImpl<safe_alias::unsafe, /*await now*/ false> {+  template <typename T>+  using Task = Task<T>;+  template <typename T>+  using TaskWithExecutor = TaskWithExecutor<T>;+};++template <>+struct PickTaskWrapperImpl<safe_alias::unsafe, /*await now*/ true> {+  template <typename T>+  using Task = NowTask<T>;+  template <typename T>+  using TaskWithExecutor = NowTaskWithExecutor<T>;+};++// These `SafeTask` types are immovable, so "await now" doesn't matter.+template <safe_alias Safety, bool AwaitNow>+  requires(Safety < safe_alias::closure_min_arg_safety)+struct PickTaskWrapperImpl<Safety, AwaitNow> {+  template <typename T>+  using Task = SafeTask<Safety, T>;+  template <typename T>+  using TaskWithExecutor = SafeTaskWithExecutor<Safety, T>;+};++template <safe_alias Safety>+  requires(Safety >= safe_alias::closure_min_arg_safety)+// Future: There is no principled reason we can't have must-await-immediately+// `SafeTask`s with these higher safety levels, but supporting that cleanly+// would require reorganizing the `folly/coro` task-wrapper implementations. Two+// possible approaches are:+//  - `NowTask<T> = AwaitNow<Task<T>>`+//  - Roll up `NowTask` and `SafeTask` into something like `BasicTask<T, Cfg>`,+//    where `Cfg` captures both safety & immediate-awaitability.+struct PickTaskWrapperImpl<Safety, /*await now*/ false> {+  template <typename T>+  using Task = SafeTask<Safety, T>;+  template <typename T>+  using TaskWithExecutor = SafeTaskWithExecutor<Safety, T>;+};++#else // no FOLLY_HAS_IMMOVABLE_COROUTINES++// This fallback is required because `coro::Future<SafeType>` is safe and is+// available on earlier build systems.  We have no choice but to emit `Task`.+template <safe_alias Safety>+struct PickTaskWrapperImpl<Safety, /*await now*/ false> {+  template <typename T>+  using Task = Task<T>;+  template <typename T>+  using TaskWithExecutor = TaskWithExecutor<T>;+};++#endif // FOLLY_HAS_IMMOVABLE_COROUTINES++// Pass this as `AddWrapperMetaFn` to `PickTaskWrapper` to add `AsNoexcept`.+template <auto CancelCfg>+struct AsNoexceptWithCancelCfg {+  template <typename T>+  using apply = AsNoexcept<T, CancelCfg>;+};++template <+    typename T,+    safe_alias Safety,+    bool MustAwaitImmediately,+    typename AddWrapperMetaFn = identity_metafunction>+using PickTaskWrapper = typename AddWrapperMetaFn::template apply<+    typename PickTaskWrapperImpl<Safety, MustAwaitImmediately>::template Task<+        T>>;++template <+    typename T,+    safe_alias Safety,+    bool MustAwaitImmediately,+    typename AddWrapperMetaFn = identity_metafunction>+using PickTaskWithExecutorWrapper = typename AddWrapperMetaFn::template apply<+    typename PickTaskWrapperImpl<Safety, MustAwaitImmediately>::+        template TaskWithExecutor<T>>;++} // namespace detail+} // namespace folly::coro++#endif
+ folly/folly/coro/detail/Traits.h view
@@ -0,0 +1,58 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <folly/Traits.h>++namespace folly {+namespace coro {+namespace detail {++/**+ * A type trait that lifts lvalue references into std::reference_wrapper<T>+ * eg. so the value can be stored in std::optional or folly::Try.+ */+template <typename T>+struct lift_lvalue_reference {+  using type = T;+};++template <typename T>+struct lift_lvalue_reference<T&> {+  using type = std::reference_wrapper<T>;+};++template <typename T>+using lift_lvalue_reference_t = typename lift_lvalue_reference<T>::type;++/**+ * A type trait to decay rvalue-reference types to a prvalue.+ */+template <typename T>+struct decay_rvalue_reference {+  using type = T;+};++template <typename T>+struct decay_rvalue_reference<T&&> : remove_cvref<T> {};++template <typename T>+using decay_rvalue_reference_t = typename decay_rvalue_reference<T>::type;++} // namespace detail+} // namespace coro+} // namespace folly
+ folly/folly/coro/safe/AsyncClosure-fwd.h view
@@ -0,0 +1,39 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++namespace folly::coro {++class AsyncObjectTag;++namespace detail {+template <auto>+auto bind_captures_to_closure(auto&&, auto);+} // namespace detail++// Tag type used by `async_closure` to trigger cleanup of `capture`s that+// have an `co_cleanup(async_closure_private_t)` overload.+class async_closure_private_t {+ protected:+  friend class AsyncObjectTag;+  template <auto>+  friend auto detail::bind_captures_to_closure(auto&&, auto);++  async_closure_private_t() = default;+};++} // namespace folly::coro
+ folly/folly/coro/safe/AsyncClosure.h view
@@ -0,0 +1,169 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <folly/coro/safe/detail/AsyncClosure.h>++#if FOLLY_HAS_IMMOVABLE_COROUTINES++namespace folly::coro {++/// Learn more about `coro/safe` tools by browsing `docs/`.  Start with+/// `README.md` and `AsyncClosure.md`.  Here's a tl;dr for `AsyncClosure.h`.+///+/// Use `async_closure()` / `async_now_closure()` only when `NowTask` is not+/// enough.  For your effort, you get (1) guaranteed, exception-safe async+/// RAII, and (2) the resulting coro is an automatically-measured movable+/// `SafeTask`, which improves lifetime safety (`LifetimeSafetyBenefits.md`).+///+/// Control flow matches a regular `NowTask` lazy-start coro:+///   - All "argument-binding" and "coro creation" work is eager.+///   - Your inner task & subsequent cleanup can only run when awaited.+///+/// `async_closure(bound_args{...}, taskFn)` wraps an outer task around yours,+/// unless elided via an automatic optimization.  The outer task owns special+/// "capture" args passed to the closure, ensuring they outlive the inner task.+///+/// Lifecycle contract:+///   - `bound_args{}` evaluate left-to-right (L2R) due to `{}`.+///   - Construction of `capture_in_place` / `make_in_place` args is also L2R.+///   - When args are passed to the inner coro, copy/move order is unspecified.+///   - Upon awaiting the inner coro, `setParentCancelToken()` is called on the+///     capture args in L2R order.+///   - After the inner coro exits, arg `co_cleanup()` is executed in R2L order.+///   - When the outer coro exits, captures are destroyed in R2L order.+///+/// The `co_cleanup()` and `setParentCancelToken()` protocols support capture+/// types like `SafeAsyncScope` and `BackgroundTask`, which give the user+/// guaranteed, exception-safe async cleanup. Before building custom async+/// RAII, carefully read `CoCleanupAsyncRAII.md`.+///+/// The `async_closure_make_outer_coro` machinery is reused by `AsyncObject`,+/// which implements a similar "async RAII" contract for object scopes.+///+/// The difference between `async_closure()` and `async_now_closure()` is that+/// the former measures argument & inner coro safety, and makes a `SafeTask`,+/// while the latter has no safety checks, and makes a `NowTask`.  Both make it+/// easy to write lifetime-safe code.++struct async_closure_config {+  /// POWER USERS ONLY: For efficiency, `async_closure` will elide the outer+  /// coro if there are no `co_cleanup` captures.  In particular,+  /// `setParentCancelToken` isn't currently part of this detection, since we+  /// don't expect it to be used without `co_cleanup`.+  ///+  /// This optimization has some observable effects on the types seen by the+  /// closure, e.g.  `capture<int&>` becomes `capture<int>` since the inner+  /// coro now owns the `int` instead of just holding a reference.  However, to+  /// the extent possible, the before/after types "quack" the same.+  ///+  /// There are some edge-case scenarios where you may want to disable this+  /// optimization. An incomplete list:+  ///   - If you're passing many in-place, non-movable captures, the current+  ///     implementation will allocate each one on the heap, separately.+  ///     Setting `.force_outer_coro = true` will consolidate them into one+  ///     `unique_ptr<tuple<>>` owned by the outer coro.  If this scenario+  ///     proves perf-sensitive, we may add an automatic heuristic.+  ///   - This can be required to make a member function coro own its object.+  ///+  /// NB: Currently, if you set `.force_outer_coro = true`, but there are no+  /// captures to store, the outer coro will still be elided.+  bool force_outer_coro = false;+};++// Implementation note: None of the below functions can take `make_inner_coro`+// by-value, because stateful callables (lambdas with captures) are allowed+// here -- even in `async_closure()` if it's a coroutine wrapper.  A callable+// passed by-value would be destroyed before it can be awaited, causing a+// stack-use-after-return error.++namespace detail {+template <bool ForceOuterCoro, bool EmitNowTask>+// OK to take `bound_args` by-ref since the porcelain functions take it by-value+auto async_closure_impl(auto&& bargs, auto&& make_inner_coro) {+  constexpr detail::async_closure_bindings_cfg Cfg{+      .force_outer_coro = ForceOuterCoro,+      // `NowTask`s closures have no safety controls, and thus -- like+      // "shared cleanup" closures -- don't get to upgrade `capture` refs.+      .force_shared_cleanup = EmitNowTask,+      .is_invoke_member = is_instantiation_of_v<+          invoke_member_wrapper_fn,+          std::remove_reference_t<decltype(make_inner_coro)>>};+  return detail::bind_captures_to_closure<Cfg>(+      static_cast<decltype(make_inner_coro)>(make_inner_coro),+      detail::async_closure_safeties_and_bindings<Cfg>(+          static_cast<decltype(bargs)>(bargs)));+}+} // namespace detail++// Makes a `SafeTask` whose safety is determined by the supplied arguments.+// `SafeTask` requires that (1) the inner coroutine must not take arguments+// by-reference, and (2) must have a `maybe_value`-safe return type.+//+// Caveat: When `make_inner_coro` is a coroutine wrapper, that part is+// evaluated synchronously, and is not subject to either (1) or (2).+//+// Coro creation, argument storage, and in-place construction are also+// synchronous, as is the movement of the args into the task coroutine.+//+// The first argument should be `bound_args{...}`.  For single-argument+// closures, you can omit the `bound_args` if you're passing `as_capture()`,+// `capture_in_place<>()`, or another `like_bound_args` item.+//+// Async RAII: Awaiting the task ensures `co_cleanup(async_closure_private_t)`+// is awaited for each of the `capture` arguments that defines it.+//+// Awaiting the task also forwards its ambient cancellation token to the+// captures that have a `setParentCancelToken()` member.  WARNING: If you want+// a type to define that, WITHOUT implementing `co_cleanup()`, then read the+// `force_outer_coro` doc above -- you'll have to add a bit of logic to+// `capture_needs_outer_coro()`.+template <async_closure_config Cfg = async_closure_config{}>+auto async_closure(auto bargs, auto&& make_inner_coro) {+  return folly::coro::detail::+      async_closure_impl<Cfg.force_outer_coro, /*EmitNowTask*/ false>(+             std::move(bargs),+             static_cast<decltype(make_inner_coro)>(make_inner_coro))+          .release_outer_coro();+}++// Like `async_closure` -- same argument binding semantics, same `co_cleanup`+// async RAII, and cancellation support, but returns a non-movable `NowTask`+// without the lifetime safety enforcement:+//   - `make_inner_coro` may return a `NowTask`, plain `Task`, or `SafeTask`.+//   - It can take arguments by ref, you can pass raw pointers, etc.+//   - There are no checks on the `co_return` type.+//+// Requiring the task to be immediately awaited prevents a lot of common+// lifetime bugs.  If you cannot immediately await the task, then you should+// review `LifetimSafetyBenefits.md` and use the `SafeTask`-enabled+// `async_closure()`, which is movable and schedulable on `SafeAsyncScope`.+//+// BEWARE: Returning `NowTask` doesn't prevent egregious bugs like returning+// a pointer to a local.  Instead, make sure to configure your compiler to+// error on simple, non-async lifetime bugs (e.g. `-Wdangling -Werror`).+template <async_closure_config Cfg = async_closure_config{}>+auto async_now_closure(auto bargs, auto&& make_inner_coro) {+  return folly::coro::detail::+      async_closure_impl<Cfg.force_outer_coro, /*EmitNowTask*/ true>(+          std::move(bargs),+          static_cast<decltype(make_inner_coro)>(make_inner_coro));+}++} // namespace folly::coro++#endif
+ folly/folly/coro/safe/Captures.h view
@@ -0,0 +1,1039 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <folly/Traits.h>+#include <folly/Utility.h>+#include <folly/coro/safe/AsyncClosure-fwd.h>+#include <folly/lang/SafeAlias-fwd.h>+// `#undef`ed at end-of-file not to leak this macro.+#include <folly/coro/safe/detail/DefineMovableDeepConstLrefCopyable.h>+#include <folly/detail/tuple.h>+#include <folly/lang/Assume.h>+#include <folly/lang/Bindings.h>+#include <folly/lang/named/Bindings.h>++///+/// Please read the user- and developer-facing docs in `Capture.md`.+///++#if FOLLY_HAS_IMMOVABLE_COROUTINES++namespace folly {+class CancellationToken;+class exception_wrapper;+} // namespace folly++namespace folly::coro {++// Re-export `bound_args` since it's required to use async closures & objects.+using ::folly::bindings::bound_args;++class AsyncObjectTag;++template <safe_alias, typename>+class SafeTask;++namespace detail {++namespace lite_tuple {+using namespace ::folly::detail::lite_tuple;+}++template <typename>+struct AsyncObjectRefForSlot;++template <typename T>+concept has_async_closure_co_cleanup_error_oblivious =+    requires(T t, async_closure_private_t p) { std::move(t).co_cleanup(p); };+template <typename T>+concept has_async_closure_co_cleanup_with_error = requires(+    T t, async_closure_private_t p, const exception_wrapper* e) {+  std::move(t).co_cleanup(p, e);+};+template <typename T> // DO NOT USE: for AsyncObject only+concept has_async_object_private_hack_co_cleanup = requires(+    T t, async_closure_private_t p, const exception_wrapper* e) {+  t.privateHack_co_cleanup(std::move(t), p, e);+};+template <typename T>+concept has_async_closure_co_cleanup =+    has_async_closure_co_cleanup_error_oblivious<T> ||+    has_async_closure_co_cleanup_with_error<T> ||+    has_async_object_private_hack_co_cleanup<T>;+// `T` must be immovable to go in `co_cleanup_capture<T>` and similar places.+// The aim here is to prevent bugs. A safe "move-like" operation for `T` must:+//   - Ensure that the destination of the move is "managed", i.e. is another+//     `co_cleanup_capture<>` or a similar object from `folly/coro/safe`+//     privileged to access `capture_private_t`. Otherwise, cleanup is no longer+//     guaranteed.+//   - Leave the moved-out object in a state where its `co_cleanup()`, which+//     will still be awaited, is a safe no-op.+//+// A regular move ctor cannot adequately vet the destination. That is because+// per `CoCleanupAsyncRAII.md`, a just-constructed object must NEVER require+// cleanup (required since closure setup is fallible, e.g. due to `bad_alloc`).+//+// So, where necessary (I don't have such a use-case yet) -- types should+// provide a specialized move operation instead.+template <typename T>+concept immovable_async_closure_co_cleanup =+    has_async_closure_co_cleanup<T> && !std::is_copy_constructible_v<T> &&+    !std::is_copy_assignable_v<T> && !std::is_move_constructible_v<T> &&+    !std::is_move_assignable_v<T> && !std::swappable<T>;++// Any binding with this key is meant to be owned by the async closure+enum class capture_kind {+  plain = 0,+  // Syntax sugar: Passing `as_capture_indirect()` with a pointer-like (e.g.+  // `unique_ptr<T>`), this emits a `capture_indirect<>`, giving access to+  // the underlying `T` with just one dereference `*` / `->`, instead of 2.+  indirect,+};++struct capture_bind_info_t : folly::bindings::ext::bind_info_t {+  capture_kind captureKind_;++  constexpr explicit capture_bind_info_t(+      // Using a constraint prevents object slicing+      std::same_as<folly::bindings::ext::bind_info_t> auto bi,+      capture_kind ap)+      : folly::bindings::ext::bind_info_t(std::move(bi)), captureKind_(ap) {}+};++template <capture_kind Kind, typename UpdateBI = std::identity>+struct as_capture_bind_info {+  // Using `auto` prevents object slicing+  constexpr auto operator()(auto bi) {+    return capture_bind_info_t{UpdateBI{}(std::move(bi)), Kind};+  }+};++template <typename, template <typename> class, typename>+class capture_crtp_base;++} // namespace detail++///+/// `as_capture()` and `as_capture_indirect()` work much like other+/// `folly::bindings` modifiers.  However, since they're primarily intended+/// for `async_closure` arguments, you will practically only use them:+///   - alone, for non-`co_cleanup` arguments;+///   - with `make_in_place()` or `make_in_place_with()`, for `co_cleanup`+///     arguments;+///   - with `constant()`, for either.+///+/// `capture_in_place<T>()` is short for `as_capture(make_in_place<T>())`.+///+/// See `Captures.md` and `folly/lang/Bindings.md`.+///++template <typename... Ts>+struct as_capture+    : ::folly::bindings::ext::merge_update_bound_args<+          detail::as_capture_bind_info<detail::capture_kind::plain>,+          Ts...> {+  using ::folly::bindings::ext::merge_update_bound_args<+      detail::as_capture_bind_info<detail::capture_kind::plain>,+      Ts...>::merge_update_bound_args;+};+template <typename... Ts>+as_capture(Ts&&...)+    -> as_capture<folly::bindings::ext::deduce_bound_args_t<Ts>...>;++template <typename... Ts>+struct as_capture_indirect+    : ::folly::bindings::ext::merge_update_bound_args<+          detail::as_capture_bind_info<detail::capture_kind::indirect>,+          Ts...> {+  using ::folly::bindings::ext::merge_update_bound_args<+      detail::as_capture_bind_info<detail::capture_kind::indirect>,+      Ts...>::merge_update_bound_args;+};+template <typename... Ts>+as_capture_indirect(Ts&&...)+    -> as_capture_indirect<folly::bindings::ext::deduce_bound_args_t<Ts>...>;++// Sugar for `as_capture{const_ref{...}}`+template <typename... Ts>+struct capture_const_ref+    : ::folly::bindings::ext::merge_update_bound_args<+          detail::as_capture_bind_info<+              detail::capture_kind::plain,+              ::folly::bindings::detail::const_ref_bind_info>,+          Ts...> {+  using ::folly::bindings::ext::merge_update_bound_args<+      detail::as_capture_bind_info<+          detail::capture_kind::plain,+          ::folly::bindings::detail::const_ref_bind_info>,+      Ts...>::merge_update_bound_args;+};+template <typename... Ts>+capture_const_ref(Ts&&...)+    -> capture_const_ref<folly::bindings::ext::deduce_bound_args_t<Ts>...>;+// Sugar for `as_capture{mut_ref{...}}`+template <typename... Ts>+struct capture_mut_ref+    : ::folly::bindings::ext::merge_update_bound_args<+          detail::as_capture_bind_info<+              detail::capture_kind::plain,+              ::folly::bindings::detail::mut_ref_bind_info>,+          Ts...> {+  using ::folly::bindings::ext::merge_update_bound_args<+      detail::as_capture_bind_info<+          detail::capture_kind::plain,+          ::folly::bindings::detail::mut_ref_bind_info>,+      Ts...>::merge_update_bound_args;+};+template <typename... Ts>+capture_mut_ref(Ts&&...)+    -> capture_mut_ref<folly::bindings::ext::deduce_bound_args_t<Ts>...>;++// Sugar for `as_capture{make_in_place<T>(...)}`+template <typename T>+auto capture_in_place(auto&&... as [[clang::lifetimebound]]) {+  return as_capture(+      ::folly::bindings::make_in_place<T>(static_cast<decltype(as)>(as)...));+}+// Sugar for `as_capture{make_in_place_with(fn, ...)}`+auto capture_in_place_with(+    auto make_fn, auto&&... as [[clang::lifetimebound]]) {+  return as_capture(::folly::bindings::make_in_place_with(+      std::move(make_fn), static_cast<decltype(as)>(as)...));+}++template <typename T>+  requires(!detail::has_async_closure_co_cleanup<T>)+class capture;+template <typename T>+  requires(!detail::has_async_closure_co_cleanup<T>)+class after_cleanup_capture;+template <typename T>+class capture_indirect;+template <typename T>+class after_cleanup_capture_indirect;++// Given a cvref-qualified `capture` type, what `capture` reference type is it+// convertible to?  The input value category affects the output reference type+// exactly as you'd expect for types NOT wrapped by `capture`.  But,+// additionally, this knows to pick the correct wrapper:+//   - `co_cleanup_capture` inputs become `co_cleanup_capture<SomeRef>`+//   - `after_cleanup_ref_*` inputs become `after_cleanup_capture<SomeRef>`+//   - everything else becomes just `capture<SomeRef>`+template <typename Captures>+using capture_ref_conversion_t =+    std::remove_cvref_t<Captures>::template ref_like_t<Captures>;++// This namespace has tools for library authors who're building new+// `co_cleanup` types. See the guide in `Captures.md`.+namespace ext {++// Used with `capture_proxy(capture_proxy_tag<KIND>, ...)`.  We don't+// need to track `const` state here, since prvalue semantics do apply any+// `const` qualifier on the return type of `capture_proxy()`.+enum class capture_proxy_kind {+  lval_ref,+  lval_ptr,+  rval_ref,+  rval_ptr,+};++// Passkey used with `capture_proxy` methods.+template <capture_proxy_kind Kind>+class capture_proxy_tag {+ private:+  template <typename, template <typename> class, typename>+  friend class ::folly::coro::detail::capture_crtp_base;+  explicit capture_proxy_tag() = default;+};++// When implementing the `capture_proxy()` ADL customization point, it is+// important for the second argument to match both `T&` and `const T&`:+//   template <capture_proxy_kind Kind, const_or_not<YourType> Me>+//   friend auto capture_proxy(capture_proxy_tag<Kind>, Me&);+template <typename T, typename U>+concept const_or_not = (std::same_as<T, U> || std::same_as<T, const U>);++} // namespace ext++namespace detail {++class capture_private_t {+ protected:+  friend struct CapturesTest;+  template <typename, template <typename> class, typename>+  friend class capture_crtp_base;+  template <typename, auto, size_t>+  friend class capture_binding_helper;+  template <auto>+  friend auto bind_captures_to_closure(auto&&, auto);+  friend constexpr capture_private_t coro_safe_detail_bindings_test_private();+  friend class ::folly::coro::AsyncObjectTag;+  explicit capture_private_t() = default;+};++struct capture_restricted_tag {}; // detail of `restricted_co_cleanup_capture`++template <typename T>+struct bind_wrapper_t {+  T t_;+  constexpr decltype(auto) what_to_bind() && { return static_cast<T&&>(t_); }+};++// Makes a `bind_wrapper_t` with a forwarding ref of the argument.+constexpr auto forward_bind_wrapper(auto&& v [[clang::lifetimebound]]) {+  static_assert(std::is_reference_v<decltype(v)>);+  return bind_wrapper_t<decltype(v)>{static_cast<decltype(v)>(v)};+}++constexpr auto unsafe_tuple_to_bind_wrapper(auto tup) {+  static_assert(1 == std::tuple_size_v<decltype(tup)>);+  return bind_wrapper_t<std::tuple_element_t<0, decltype(tup)>>{+      .t_ = lite_tuple::get<0>(std::move(tup))};+}++template <typename Derived, template <typename> class RefArgT, typename T>+class capture_crtp_base {+ private:+  static constexpr decltype(auto) assert_result_is_non_copyable_non_movable(+      auto&& fn) {+    using U = decltype(fn());+    // Tests `U&` instead of `is_copy_*` to catch non-regular classes that+    // declare a U(U&) ctor.  This implementation is for class types only.+    static_assert(+        // E.g. `AsyncObjectPtr::capture_proxy()` just returns a reference+        // or pointer, in effect emulating `capture_indirect`.+        std::is_reference_v<U> || std::is_pointer_v<U> ||+            !(std::is_constructible_v<U, U&> ||+              std::is_constructible_v<U, U&&> || std::is_assignable_v<U&, U&> ||+              std::is_assignable_v<U&, U&&>),+        "When a class provides custom dereferencing via `capture_proxy`, "+        "it must be `NonCopyableNonMovable` to ensure that it can only passed "+        "via `capture<Ref>`, not via your temporary proxy object. The goals "+        "are (1) ensure correct `safe_alias_of` markings, (2) keep the "+        "forwarding object as a hidden implementation detail.");+    return fn();+  }++  // Object intended for use with `capture`  (like `SafeAsyncScope`) may+  // provide overloads of the helper function `capture_proxy` to provide+  // proxy types for `capture` operators `*` and `->`.+  //+  // IMPORTANT: Be sure to cover the options in `capture_proxy_kind`.  Also,+  // if you provide a `const`-qualified `capture_proxy` it should model+  // `const` access.+  //+  // The reason for this indirection is as follows:+  //   - "Restricted" references to scopes must enforce stricter+  //     `safe_alias_of` constraints on their awaitables.+  //     `restricted_co_cleanup_capture` explains the usage.+  //   - A `restricted_co_cleanup_capture<Ref>` may be obtained from an+  //     `co_cleanup_capture<...AsyncScope...>` that was originally NOT+  //     restricted -- so, "restricted" is a property of the reference, not+  //     of the underlying scope object.+  //   - Therefore, the public API of `SafeAsyncScope` must sit in a+  //     "reference" object that knows if it's restricted, not in the storage+  //     object (which does not).+  //   - It would break encapsulation to put `AsyncScope`-specific logic like+  //     `add` / `schedule` / `schedule*Closure` into `Captures.h`.+  //+  // A type will not be accessible via `restricted_co_cleanup_capture`+  // unless it provides overloads for `capture_restricted_proxy`.  There's+  // no default behavior for restricted refs, because the underlying class+  // needs to implement strong enough safety constraints that the ref can be+  // `after_cleanup_ref`.+  template <ext::capture_proxy_kind Kind>+  static constexpr decltype(auto) get_proxy(+      ext::capture_proxy_tag<Kind> proxy_tag, auto& self) {+    auto& lref = self.get_lref();+    if constexpr (std::is_base_of_v<capture_restricted_tag, Derived>) {+      return assert_result_is_non_copyable_non_movable([&]() -> decltype(auto) {+        return capture_restricted_proxy(proxy_tag, lref);+      });+    } else if constexpr ( // Custom dereference+        requires { capture_proxy(proxy_tag, lref); }) {+      return assert_result_is_non_copyable_non_movable([&]() -> decltype(auto) {+        return capture_proxy(proxy_tag, lref);+      });+    } else if constexpr (Kind == ext::capture_proxy_kind::lval_ref) {+      return lref; // Unproxied l-value reference+    } else if constexpr (Kind == ext::capture_proxy_kind::rval_ref) {+      return std::move(lref); // Unproxied r-value reference+    } else if constexpr (+        Kind == ext::capture_proxy_kind::lval_ptr ||+        Kind == ext::capture_proxy_kind::rval_ptr) {+      return &lref; // Unproxied pointer+    } else {+      static_assert(false, "Unhandled capture_proxy_kind");+    }+  }++  // Invokes a callable, ensuring its return value is of type `Expected`,+  // while retaining prvalue semantics.+  template <typename Expected>+  static constexpr auto assert_return_type(auto fn) {+    static_assert(std::is_same_v<decltype(fn()), Expected>);+    return fn();+  }++ public:+  using capture_type = T;++  // Implement operators `*` and `->` for lvalue `capture` types.+  //+  // This rvalue specialization has an intentional & important deviation in+  // semantics:+  //   - All the getters require a `&&`-qualified object, i.e.  their intended+  //     use is destructive -- you can `*std::move(arg_ref)` once.  Thereafter,+  //     use-after-move linters will complain if you reuse the `capture<V&&>`.+  //   - Correspondingly, `operator*` returns `V&&` instead of `V&`.+  [[nodiscard]] constexpr decltype(auto) operator*() & noexcept {+    static_assert(+        !std::is_rvalue_reference_v<T>,+        "With `capture<T&&> a`, use `*std::move(a)`");+    return get_proxy(+        ext::capture_proxy_tag<ext::capture_proxy_kind::lval_ref>{},+        *static_cast<Derived*>(this));+  }+  [[nodiscard]] constexpr decltype(auto) operator*() && noexcept {+    return get_proxy(+        ext::capture_proxy_tag<ext::capture_proxy_kind::rval_ref>{},+        *static_cast<Derived*>(this));+  }+  [[nodiscard]] constexpr decltype(auto) operator->() & noexcept {+    static_assert(+        !std::is_rvalue_reference_v<T>,+        "With `capture<T&&> a`, use `std::move(a)->`");+    return get_proxy(+        ext::capture_proxy_tag<ext::capture_proxy_kind::lval_ptr>{},+        *static_cast<Derived*>(this));+  }+  [[nodiscard]] constexpr decltype(auto) operator->() && noexcept {+    return get_proxy(+        ext::capture_proxy_tag<ext::capture_proxy_kind::rval_ptr>{},+        *static_cast<Derived*>(this));+  }+  [[nodiscard]] constexpr decltype(auto) operator*() const& noexcept {+    static_assert(+        !std::is_rvalue_reference_v<T>,+        "With `capture<T&&> a`, use `*std::move(a)`");+    return get_proxy(+        ext::capture_proxy_tag<ext::capture_proxy_kind::lval_ref>{},+        *static_cast<const Derived*>(this));+  }+  [[nodiscard]] constexpr decltype(auto) operator*() const&& noexcept {+    return get_proxy(+        ext::capture_proxy_tag<ext::capture_proxy_kind::rval_ref>{},+        *static_cast<const Derived*>(this));+  }+  [[nodiscard]] constexpr decltype(auto) operator->() const& noexcept {+    static_assert(+        !std::is_rvalue_reference_v<T>,+        "With `capture<T&&> a`, use `std::move(a)->`");+    return get_proxy(+        ext::capture_proxy_tag<ext::capture_proxy_kind::lval_ptr>{},+        *static_cast<const Derived*>(this));+  }+  [[nodiscard]] constexpr decltype(auto) operator->() const&& noexcept {+    return get_proxy(+        ext::capture_proxy_tag<ext::capture_proxy_kind::rval_ptr>{},+        *static_cast<const Derived*>(this));+  }++  // Private implementation detail -- public users should instead use the below+  // conversions.  This is how `async_closure` (and similar) create a matching+  // `capture<Ref>` from a `Derived` instance.  The resulting type is+  // `RefArgT`, except for the narrow case when a non-`shared_cleanup` closure+  // is converting a `after_cleanup_ref_` input.+  //   - `Derived::capture_type` may be a value or a reference+  //   - `T` may be a value or reference+  // The main reason `to_capture_ref` is locked down is that when+  // `SharedCleanupClosure == false`, we upgrade `after_cleanup_ref_` refs.+  // This is unsafe to do unless we know that the ref is going into+  // an independent, nested async scope.+  template <bool SharedCleanupClosure>+  auto to_capture_ref(capture_private_t) & {+    return to_capture_ref_impl<SharedCleanupClosure>(+        static_cast<Derived&>(*this).get_lref());+  }+  template <bool SharedCleanupClosure>+  auto to_capture_ref(capture_private_t) const& {+    return to_capture_ref_impl<SharedCleanupClosure>(+        static_cast<const Derived&>(*this).get_lref());+  }+  template <bool SharedCleanupClosure>+  auto to_capture_ref(capture_private_t) && {+    return to_capture_ref_impl<SharedCleanupClosure>(+        std::move(static_cast<Derived&>(*this).get_lref()));+  }+  template <bool SharedCleanupClosure>+  auto to_capture_ref(capture_private_t) const&& {+    return to_capture_ref_impl<SharedCleanupClosure>(+        std::move(static_cast<const Derived&>(*this).get_lref()));+  }++  // Prefer `capture_ref_conversion_t`, which is easier to use.  Given an+  // instance of this `capture` of cvref category `LikeMe`, which+  // `capture<Ref>` can it be converted to?+  template <typename LikeMe>+  using ref_like_t = RefArgT<like_t<LikeMe&&, T>>;++  // Convert a capture instance to a capture reference of a matching cvref+  // category.+  //+  // Two implicit conversions are provided because we want capture-wrapped+  // types to act much like the underlying unwrapped types.  You can think of+  // this conversion as allowing cvref qualifiers on the wrapper to be moved+  // **inside** the wrapper.  The test shows full coverage, but in essence,+  // the outer reference category replaces the inner one, any `const` moves+  // inside the wrapper, and we never remove a `const` qualifier already+  // present in the wrapper.  Examples:+  //   capture<int>& -> capture<int&>+  //   const capture<int>& -> capture<const int&>+  //+  // The rvalue qualified analog is explicit, to avoid some bad side effects:+  //   capture<const int&>&& -> capture<const int&&> (explicit!)+  //+  // For those 3 conversions, find the destination `capture` type via the+  // function `capture_ref_conversion_t`.+  //+  // We also support an explicit rref to lref conversion:+  //   capture<int&&>&& -> capture<int&>+  // The idea here is that you're passing `capture<V&&>` down into a child+  // of your closure.  That deliberately has stricter single-use semantics+  // than `V&&` in vanilla C++ -- for example, without single-use, an rref+  // could be used to move out a value that is still referenced in+  // SafeAsyncScope task.  Having the explicit && -> & conversion permits+  // the child change its mind about moving out the value.+  //+  // Future ideas & implementation notes:+  //   - We may want to support implicitly adding `const`. Today's solution+  //     is to take `const capture`, which should be fine for most usage?+  //   - This (and `to_capture_ref` should technically have a `const&&`+  //     overload, but that's "impact for another day", whenever someone+  //     actually needs it.+  //   - All 3 of these conversions can be `operator auto`, but I suspect+  //     this would hurt compile-time.  Benchmark before changing.+  /*implicit*/ operator ref_like_t<int&>() & {+    return assert_return_type<ref_like_t<int&>>([&] {+      return static_cast<Derived&>(*this)+          .template to_capture_ref</*SharedCleanup*/ true>(capture_private_t{});+    });+  }+  /*implicit*/ operator ref_like_t<const int&>() const& {+    return assert_return_type<ref_like_t<const int&>>([&] {+      return static_cast<const Derived&>(*this)+          .template to_capture_ref</*SharedCleanup*/ true>(capture_private_t{});+    });+  }+  // This is explicit, because if it were implicit, then prvalues of type+  // `capture<int&>` would bind to arguments of type `capture<int&&>` which is+  // an unexpected / unsafe behavior.+  explicit operator auto() && { // Actually, `operator ref_like_t<int&&>`+    // This has to be `operator auto`, with a "stub" branch for cleanup+    // args, because an `co_cleanup_capture` constraint bans r-value+    // references, preventing us from unconditionally instantiating+    // `ref_like_t<int&&>` for all `capture` types.  It would be possible to+    // delay the "no rvalue reference" test by making it a `static_assert`+    // in a constructor (or another guaranteed-to-be-instantiated) function,+    // but this wouldn't be shorter, and it would be more fragile.+    if constexpr (has_async_closure_co_cleanup<std::remove_cvref_t<T>>) {+      return;+    } else {+      return assert_return_type<ref_like_t<int&&>>([&] {+        return static_cast<Derived&&>(*this)+            .template to_capture_ref</*SharedCleanup*/ true>(+                capture_private_t{});+      });+    }+  }+  // Allow explicitly moving `capture<V&&>` into `capture<V&>`. Example:+  //   auto lcap = capture<int&>{std::move(rcap)};+  explicit operator ref_like_t<int&>() &&+    requires(std::is_rvalue_reference_v<T>)+  {+    return assert_return_type<ref_like_t<int&>>([&] {+      return to_capture_ref_impl</*SharedCleanup*/ true>(+          static_cast<Derived&&>(*this).get_lref());+    });+  }++ private:+  template <bool SharedCleanupClosure, typename V>+  static auto to_capture_ref_impl(V&& v) {+    // If the receiving closure takes no `shared_cleanup` args, then it+    // cannot* pass any of its `capture` refs to an external, longer-lived+    // cleanup callback.  That implies we can safely upgrade any incoming+    // `after_cleanup_ref_` refs to regular post-cleanup `capture` refs --+    // anything received from the parent is `co_cleanup_safe_ref` from the point+    // of view of **this** closure's cleanup args, and it cannot access others.+    //+    // * As always, subject to the `SafeAlias.h` caveats.+    if constexpr (has_async_closure_co_cleanup<V>) {+      // Identical to the default `else` branch.  Required, since we cannot+      // instantiate `after_cleanup_capture<V>` when `V` has `co_cleanup`.+      return RefArgT<V&&>{+          capture_private_t{}, forward_bind_wrapper(static_cast<V&&>(v))};+    } else if constexpr (+        !SharedCleanupClosure &&+        std::is_same_v<RefArgT<V>, after_cleanup_capture<V>>) {+      return capture<V&&>{+          capture_private_t{}, forward_bind_wrapper(static_cast<V&&>(v))};+    } else if constexpr (+        !SharedCleanupClosure &&+        std::is_same_v<RefArgT<V>, after_cleanup_capture_indirect<V>>) {+      return capture_indirect<V&&>{+          capture_private_t{}, forward_bind_wrapper(static_cast<V&&>(v))};+    } else {+      return RefArgT<V&&>{+          capture_private_t{}, forward_bind_wrapper(static_cast<V&&>(v))};+    }+  }+};++// The primary template is for values, with a specialization for references.+// Value and lval refs should quack the same, exposing a pointer-like API,+// which (unlike regular pointers or ref wrappers) is deep-const.+//+// The rvalue reference specialization has a nonstandard semantics.  For+// `capture`s, rvalue refs are **single-use**.  Users should only create+// `capture<V&&>` if they intend to move the value, or perform another+// destructive operation.+//+// Why specialize for references instead of storing `T t_;` in a single+// class, and dispatch via SFINAE?  The main reason is that `T t_` wouldn't+// support assignment, since `T = V&` or `T = V&&` could not be rebound.+template <typename Derived, template <typename> class RefArgT, typename V>+class capture_storage : public capture_crtp_base<Derived, RefArgT, V> {+  static_assert(!std::is_reference_v<V>); // Specialized for refs below+ public:+  constexpr capture_storage(capture_private_t, auto bind_wrapper)+      : v_(std::move(bind_wrapper).what_to_bind()) {}++ protected:+  template <typename, template <typename> class, typename>+  friend class capture_crtp_base;+  friend void async_closure_set_cancel_token(+      async_closure_private_t, auto&&, const CancellationToken&);+  friend auto async_closure_make_cleanup_tuple(+      async_closure_private_t, auto&&, const exception_wrapper*);+  template <typename> // For the `capture` specializations only!+  friend struct AsyncObjectRefForSlot;+  template <typename ArgMap, size_t ArgI, typename Arg>+  friend decltype(auto) async_closure_resolve_backref(+      capture_private_t, auto&, Arg&);++  constexpr auto& get_lref() noexcept { return v_; }+  constexpr const auto& get_lref() const noexcept { return v_; }++  V v_;+};+// Future: When `R` is an rvalue reference, it might be good to support a+// runtime check against reuse, in the style of `RValueReferenceWrapper`.+// Unlike that class, I would make it DFATAL to avoid opt-build cost.+template <typename Derived, template <typename> class RefArgT, typename R>+  requires std::is_reference_v<R>+class capture_storage<Derived, RefArgT, R>+    : public capture_crtp_base<Derived, RefArgT, R> {+ public:+  // This double-cast is an ugly workaround to go from `V&&` to `V*`.  We+  // need the outer `const_cast` because C++ doesn’t allow address-of-rvalue+  // refs, and only allows them to be cast to `const` lvalue refs.  It is+  // safe, since the final destination type has the same const-qualification+  // as the original `what_to_bind()` result.+  constexpr capture_storage(capture_private_t, auto bind_wrapper)+      : p_(&const_cast<std::remove_reference_t<R>&>(+            static_cast<std::add_const_t<std::remove_reference_t<R>>&>(+                std::move(bind_wrapper).what_to_bind()))) {}++ protected:+  template <typename, template <typename> class, typename>+  friend class capture_crtp_base;+  constexpr auto& get_lref() noexcept { return *p_; }+  constexpr const auto& get_lref() const noexcept { return *p_; }++  std::remove_reference_t<R>* p_;+};++// There are no "heap reference" variants since a reference doesn't need to+// know how it's stored, and "heap" vs "plain" is meant to be a low-visibility+// implementation detail.+template <typename Derived, template <typename> class RefArgT, typename T>+  requires(!std::is_reference_v<T> && !has_async_closure_co_cleanup<T>)+// Since `capture_heap` is owned directly by the inner task, it has to be+// movable to be passed to the coroutine.  But, to stay API-compatible per+// above, it'd be preferable if users did NOT move it.  To help prevent such+// moves, a linter is proposed in `FutureLinters.md`.+//+// We deliberately do NOT support moving out the underlying `unique_ptr`+// because heap storage is meant to be an implementation detail, and is not+// intended to be nullable.  A user needing nullability should pass a+// `unique_ptr` either as `capture_indirect` (1 dereference) or `capture` (2).+class capture_heap_storage : public capture_crtp_base<Derived, RefArgT, T> {+ public:+  capture_heap_storage(capture_private_t, auto bind_wrapper)+      : p_(std::make_unique<T>(std::move(bind_wrapper).what_to_bind())) {}++ protected:+  template <typename, template <typename> class, typename>+  friend class capture_crtp_base;+  constexpr auto& get_lref() noexcept { return *p_; }+  constexpr const auto& get_lref() const noexcept { return *p_; }++  std::unique_ptr<T> p_;+};++// This is a direct counterpart to `capture_storage` that collapses two+// dereference operations into one for better UX.  There is no need for a+// `capture_heap_indirect_storage`, since this "indirect" syntax sugar only+// applies to pointer types, which are always cheaply movable, and thus+// don't benefit from `make_in_place`.+//+// Similarly, no support for `co_cleanup()` captures since those generally+// aren't pointer-like, and won't suffer from double-dereferences.+template <typename Derived, template <typename> class RefArgT, typename T>+  requires(!has_async_closure_co_cleanup<T>)+class capture_indirect_storage : public capture_storage<Derived, RefArgT, T> {+ public:+  using capture_storage<Derived, RefArgT, T>::capture_storage;++  // These are all intended to be equivalent to dereferencing the+  // corresponding `capture<T>` twice.+  [[nodiscard]] constexpr decltype(auto) operator*() & noexcept {+    return *(capture_storage<Derived, RefArgT, T>::operator*());+  }+  [[nodiscard]] constexpr decltype(auto) operator*() const& noexcept {+    return *(capture_storage<Derived, RefArgT, T>::operator*());+  }+  [[nodiscard]] constexpr decltype(auto) operator*() && noexcept {+    return *(+        std::move(*this).capture_storage<Derived, RefArgT, T>::operator*());+  }+  [[nodiscard]] constexpr decltype(auto) operator*() const&& noexcept {+    return *(+        std::move(*this).capture_storage<Derived, RefArgT, T>::operator*());+  }+  [[nodiscard]] constexpr decltype(auto) operator->() & noexcept {+    return (capture_storage<Derived, RefArgT, T>::operator->())->operator->();+  }+  [[nodiscard]] constexpr decltype(auto) operator->() const& noexcept {+    return (capture_storage<Derived, RefArgT, T>::operator->())->operator->();+  }+  [[nodiscard]] constexpr decltype(auto) operator->() && noexcept {+    return (std::move(*this).capture_storage<Derived, RefArgT, T>::operator->())+        ->operator->();+  }+  [[nodiscard]] constexpr decltype(auto) operator->() const&& noexcept {+    return (std::move(*this).capture_storage<Derived, RefArgT, T>::operator->())+        ->operator->();+  }++  // Unlike other captures, `capture_indirect` is nullable since the+  // underlying pointer type is, too.+  explicit constexpr operator bool() const+      noexcept(noexcept(this->get_lref().operator bool())) {+    return this->get_lref().operator bool();+  }++  // Use these to access the underlying `T`, instead of dereferencing twice.+  //+  // RISKS: Clearing or reallocating a pointer (e.g. `reset()`) in async+  // code can cause faults for other code that holds a `capture` reference.+  // Ideally, you should only use this if you can prove that there are no+  // other outstanding references, or that they all expect the change.+  decltype(auto) get_underlying_unsafe() & {+    return capture_storage<Derived, RefArgT, T>::operator*();+  }+  decltype(auto) get_underlying_unsafe() const& {+    return capture_storage<Derived, RefArgT, T>::operator*();+  }+  decltype(auto) get_underlying_unsafe() && {+    return std::move(capture_storage<Derived, RefArgT, T>::operator*());+  }+  decltype(auto) get_underlying_unsafe() const&& {+    return std::move(capture_storage<Derived, RefArgT, T>::operator*());+  }+};++} // namespace detail++// Please read the file docblock.+//+// Rationale for the move/copy policy of `A = capture<T>`:+//   - When `T` is a ref, `A` must be passed-by-value into coroutines, and+//     so must be at least movable.+//   - Ideally, for value `T`, the args would be permanently attached to the+//     originating closure, but we have to let them be movable so that+//     `async_closure`s without the outer task can own them.  To help+//     prevent this, a linter is proposed in `FutureLinters.md`.+//   - Forbid copying for rvalue ref `T` to make use-after-move linters useful.+//     We don't follow `folly::rvalue_reference_wrapper` in adding a runtime+//     `nullptr` check for moved-out refs, but this could be done later.+//   - Allowing copies of lvalue refs is optional, but helpful.  For example,+//     it lets users naturally pass arg refs into bare sub-tasks.  This seems+//     like a reasonable & low-risk thing to do -- our operators already expose+//     refs to the underlying data, so we can't prevent the user from passing+//     `T&` to non-`safe_alias` callables, anyhow.+template <typename T> // may be a value or reference+  requires(!detail::has_async_closure_co_cleanup<T>)+class capture : public detail::capture_storage<capture<T>, capture, T> {+ public:+  FOLLY_MOVABLE_AND_DEEP_CONST_LREF_COPYABLE(capture, T);+  using detail::capture_storage<capture<T>, capture, T>::capture_storage;+};+template <typename T> // may be a value or reference+  requires(!detail::has_async_closure_co_cleanup<T>)+class after_cleanup_capture+    : public detail::+          capture_storage<after_cleanup_capture<T>, after_cleanup_capture, T> {+ public:+  FOLLY_MOVABLE_AND_DEEP_CONST_LREF_COPYABLE(after_cleanup_capture, T);+  using detail::capture_storage<+      after_cleanup_capture<T>,+      after_cleanup_capture,+      T>::capture_storage;+};++// The use-case for `capture_heap` is to allow a closure without cleanup+// args to avoid an inner/outer task split, while still taking+// `make_in_place` arguments.  This is meant to be an implementation detail+// that's almost fully API-compatible with `capture`.  At a future+// point we *could* remove this:+//  - Then, any use of `make_in_place` would auto-create an outer task.+//  - Any user code that explicitly specifies `capture_heap` in signatures+//    would need to be updated to `capture`.+//  - Any places that rely on moving `capture_heap<V>` would need to migrate+//    to `capture_indirect<std::unique_ptr<V>>{}` (which, in contrast, is+//    nullable).  This should be rare, since we mark all value `capture`s as+//    `unsafe` to encourage leaving the value `capture` wrappers in-closure.+template <typename T>+class capture_heap+    : public detail::capture_heap_storage<capture_heap<T>, capture, T> {+ public:+  using detail::capture_heap_storage<capture_heap<T>, capture, T>::+      capture_heap_storage;+};+template <typename T>+class after_cleanup_capture_heap+    : public detail::capture_heap_storage<+          after_cleanup_capture_heap<T>,+          after_cleanup_capture,+          T> {+ public:+  using detail::capture_heap_storage<+      after_cleanup_capture_heap<T>,+      after_cleanup_capture,+      T>::capture_heap_storage;+};++// `capture_indirect<SomePtr<T>>` is like `capture<SomePtr<T>>` with syntax+// sugar to avoid dereferencing twice.  Use `get_underlying_unsafe()` instead+// of `*` / `->` to access the pointer object itself (see its doc for RISKS).+template <typename T>+class capture_indirect+    : public detail::+          capture_indirect_storage<capture_indirect<T>, capture_indirect, T> {+ public:+  using detail::capture_indirect_storage<+      capture_indirect<T>,+      capture_indirect,+      T>::capture_indirect_storage;+};+template <typename T>+class after_cleanup_capture_indirect+    : public detail::capture_indirect_storage<+          after_cleanup_capture_indirect<T>,+          after_cleanup_capture_indirect,+          T> {+ public:+  using detail::capture_indirect_storage<+      after_cleanup_capture_indirect<T>,+      after_cleanup_capture_indirect,+      T>::capture_indirect_storage;+};++// A closure that takes a cleanup arg is required to mark its directly-owned+// `capture`s with the `after_cleanup_` prefix, to prevent refs to these+// short-lived args from being passed into longer-lived callbacks. Similarly,+// it may not upgrade incoming `after_cleanup_capture`s to just `capture`s.+//+// Don't allow r-value refs to cleanup args, since moving those out of the+// owning closure is unexpected, and probably wrong.+template <typename T> // may be a value or lvalue reference+  requires(!std::is_rvalue_reference_v<T> &&+           detail::immovable_async_closure_co_cleanup<std::remove_cvref_t<T>>)+class co_cleanup_capture+    : public detail::+          capture_storage<co_cleanup_capture<T>, co_cleanup_capture, T>,+      std::conditional_t<+          !std::is_reference_v<T>,+          folly::NonCopyableNonMovable,+          tag_t<>> {+ public:+  FOLLY_MOVABLE_AND_DEEP_CONST_LREF_COPYABLE(co_cleanup_capture, T);+  using detail::capture_storage<co_cleanup_capture<T>, co_cleanup_capture, T>::+      capture_storage;+};++// What this accomplishes, in brief -- details in `Captures.md`:+//   - A closure that takes a `co_cleanup_capture<X&> x` from a parent will+//     see some of its arguments downgraded to `after_cleanup_capture`.+//   - To avoid the safety downgrade, the closure can instead take the ref+//     as `restricted_co_cleanup_capture<X&> xr`, whose APIs will mirror+//     those of `x`, but will be restricted to ONLY accept args with+//     `maybe_value` safety.+//+// This only takes `T = V&`, because "restricted" is always a view on+// an underlying `co_cleanup_capture`.+//+// `V` needs to ADL-customize `capture_restricted_proxy()`.+template <typename T>+  requires(std::is_lvalue_reference_v<T> &&+           detail::immovable_async_closure_co_cleanup<std::remove_cvref_t<T>>)+class restricted_co_cleanup_capture+    : public detail::capture_storage<+          restricted_co_cleanup_capture<T>,+          restricted_co_cleanup_capture,+          T>,+      private detail::capture_restricted_tag {+ public:+  FOLLY_MOVABLE_AND_DEEP_CONST_LREF_COPYABLE(restricted_co_cleanup_capture, T);+  using detail::capture_storage<+      restricted_co_cleanup_capture<T>,+      restricted_co_cleanup_capture,+      T>::capture_storage;+};++namespace detail {+template <typename T>+concept is_any_co_cleanup_capture =+    (is_instantiation_of_v<co_cleanup_capture, T> ||+     is_instantiation_of_v<restricted_co_cleanup_capture, T>);+template <typename T>+concept is_any_capture =+    (is_instantiation_of_v<capture, T> ||+     is_instantiation_of_v<capture_heap, T> ||+     is_instantiation_of_v<capture_indirect, T> ||+     is_instantiation_of_v<after_cleanup_capture, T> ||+     is_instantiation_of_v<after_cleanup_capture_heap, T> ||+     is_instantiation_of_v<after_cleanup_capture_indirect, T> ||+     is_instantiation_of_v<co_cleanup_capture, T> ||+     is_instantiation_of_v<restricted_co_cleanup_capture, T>);+template <typename T>+concept is_any_capture_ref =+    is_any_capture<T> && std::is_reference_v<typename T::capture_type>;+template <typename T>+concept is_any_capture_val =+    is_any_capture<T> && !std::is_reference_v<typename T::capture_type>;++// `capture_safety_impl_v` is separate for `AsyncObject.h` to specialize+template <typename T>+inline constexpr auto capture_safety_impl_v = safe_alias_of_v<T>;+// If the underlying type is `<= shared_cleanup`, that leaks through to+// all `capture`s containing it.  See e.g. `AsyncObjectPtr`.+//   * Note: A `shared_cleanup` type `T` gives a closure a way of passing refs+//     onto parent `SafeAsyncScope`s (generically: cleanup phases), so+//     `capture<T>` must never be safer than `T` (unless we're dealing with a+//     restricted capture ref),+//+// Otherwise, the safety measurement of `T` is "outer" to the current+// closure, and is one of `after_cleanup_ref`, `co_cleanup_safe_ref`, or+// `maybe_value`.  Those should all behave the same inside the closure,+// so `MaxRefSafety` is all that matters.+//   * Note: `capture<V>` is convertible to `capture<V&>` etc, so the ref+//     version should never be safer.+template <typename T, safe_alias MaxRefSafety>+struct capture_safety+    : safe_alias_constant<+          (capture_safety_impl_v<std::remove_reference_t<T>> <=+           safe_alias::shared_cleanup)+              ? std::min(+                    MaxRefSafety,+                    capture_safety_impl_v<std::remove_reference_t<T>>)+              : MaxRefSafety> {};++} // namespace detail++} // namespace folly::coro++namespace folly {++// Set `safe_alias` values for all the `capture` types.+//+// `capture` refs are only valid as long as their on-closure storage.  They+// can be copied/moved, so their `safe_alias` marking is the only thing+// preventing the use of invalid references.  The docs in `enum class+// safe_alias` discuss how safety levels are assigned for closure+// `capture`s.  `async_closure` invokes `to_capture_ref()` to emit refs with+// the appropriate safety.++template <typename T>+struct safe_alias_of<::folly::coro::capture<T>>+    : folly::coro::detail::capture_safety<T, safe_alias::co_cleanup_safe_ref> {+};+template <typename T>+struct safe_alias_of<::folly::coro::capture_heap<T>>+    : folly::coro::detail::capture_safety<T, safe_alias::co_cleanup_safe_ref> {+};+template <typename T>+struct safe_alias_of<::folly::coro::capture_indirect<T>>+    : folly::coro::detail::capture_safety<T, safe_alias::co_cleanup_safe_ref> {+};++template <typename T>+struct safe_alias_of<::folly::coro::after_cleanup_capture<T>>+    : folly::coro::detail::capture_safety<T, safe_alias::after_cleanup_ref> {};+template <typename T>+struct safe_alias_of<::folly::coro::after_cleanup_capture_heap<T>>+    : folly::coro::detail::capture_safety<T, safe_alias::after_cleanup_ref> {};+template <typename T>+struct safe_alias_of<::folly::coro::after_cleanup_capture_indirect<T>>+    : folly::coro::detail::capture_safety<T, safe_alias::after_cleanup_ref> {};++template <typename T>+struct safe_alias_of<::folly::coro::co_cleanup_capture<T>>+    : folly::coro::detail::capture_safety<T, safe_alias::shared_cleanup> {};+// FIXME: `capture_safety` will still measure this as `shared_cleanup` due+// to `T` being that safety.  So, when implementing restricted refs, we'll+// have to add a new case to `capture_safety` to handle this.+template <typename T>+struct safe_alias_of<::folly::coro::restricted_co_cleanup_capture<T>>+    : folly::coro::detail::capture_safety<T, safe_alias::after_cleanup_ref> {};++} // namespace folly++// We extended `folly::bindings` with `capture_kind`, so we must explicitly+// specialize `binding_policy`.  We reuse the standard rules.  Custom+// `capture` binding logic is in `async_closure_bindings()`.+namespace folly::bindings::ext {+template <auto BI, typename BindingType>+  requires std::same_as< // Written as a constraint to prevent object slicing+      decltype(BI),+      ::folly::coro::detail::capture_bind_info_t>+class binding_policy<ext::binding_t<BI, BindingType>> {+ private:+  using standard = binding_policy<ext::binding_t<bind_info_t{BI}, BindingType>>;++ public:+  using storage_type = typename standard::storage_type;+  using signature_type = typename standard::signature_type;+};+} // namespace folly::bindings::ext++#endif++#undef FOLLY_MOVABLE_AND_DEEP_CONST_LREF_COPYABLE
+ folly/folly/coro/safe/NowTask.h view
@@ -0,0 +1,155 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <folly/coro/TaskWrapper.h>+#include <folly/lang/SafeAlias-fwd.h>++#if FOLLY_HAS_IMMOVABLE_COROUTINES++/// `NowTask<T>` quacks like `Task<T>` but is immovable, and must be+/// `co_await`ed in the same expression that created it.+///+/// Using `NowTask` by default brings considerable safety benefits.  With+/// `Task`, the following would be anti-patterns that cause dangling reference+/// bugs, but with `NowTask`, C++ lifetime extension rules ensure that they+/// simply work.+///   - Pass-by-reference into coroutines.+///   - Ephemeral coro lambdas with captures.+///   - Coro lambdas with capture-by-reference.+///+/// Notes:+///   - (subject to change) Unlike `SafeTask`, `NowTask` does NOT check+///     `safe_alias_of` for the return type `T`.  `NowTask` is essentially an+///     immediate async function -- it satisfies the structured concurrency+///     maxim of "lexical scope drives both control flow & lifetime".  That+///     lowers the odds that returned pointers/references are unexpectedly+///     invalid.  The one failure mode I can think of is that the+///     pointed-to-data gets invalidated by a concurrent thread of execution,+///     but in that case the program almost certainly has a data race --+///     regardless of the lifetime bug -- and that requires runtime+///     instrumentation (like TSAN) to detect in present-day C++.++namespace folly::coro {++template <safe_alias, typename>+class BackgroundTask;++template <typename T = void>+class NowTask;++template <typename T = void>+class NowTaskWithExecutor;++namespace detail {+template <typename T>+struct NowTaskWithExecutorCfg : DoesNotWrapAwaitable {+  using InnerTaskWithExecutorT = TaskWithExecutor<T>;+  using WrapperTaskT = NowTask<T>;+};+template <typename T>+using NowTaskWithExecutorBase =+    AddMustAwaitImmediately<TaskWithExecutorWrapperCrtp<+        NowTaskWithExecutor<T>,+        detail::NowTaskWithExecutorCfg<T>>>;+} // namespace detail++template <typename T>+class FOLLY_NODISCARD NowTaskWithExecutor final+    : public detail::NowTaskWithExecutorBase<T> {+ protected:+  using detail::NowTaskWithExecutorBase<T>::NowTaskWithExecutorBase;++  template <safe_alias, typename>+  friend class BackgroundTask; // for `unwrapTaskWithExecutor`, remove later+};++namespace detail {+template <typename T>+class NowTaskPromise final+    : public TaskPromiseWrapper<T, NowTask<T>, TaskPromise<T>> {};+template <typename T>+struct NowTaskCfg : DoesNotWrapAwaitable {+  using ValueT = T;+  using InnerTaskT = Task<T>;+  using TaskWithExecutorT = NowTaskWithExecutor<T>;+  using PromiseT = NowTaskPromise<T>;+};+template <typename T>+using NowTaskBase =+    AddMustAwaitImmediately<TaskWrapperCrtp<NowTask<T>, detail::NowTaskCfg<T>>>;+} // namespace detail++template <safe_alias, typename>+class SafeTask;++template <safe_alias S, typename U>+auto toNowTask(SafeTask<S, U>);++template <typename T>+class FOLLY_CORO_TASK_ATTRS NowTask final : public detail::NowTaskBase<T> {+ protected:+  using detail::NowTaskBase<T>::NowTaskBase;++  template <typename U> // can construct+  friend auto toNowTask(Task<U>);+  template <safe_alias S, typename U> // can construct+  friend auto toNowTask(SafeTask<S, U>);+  template <typename U> // can construct & `unwrapTask`+  friend auto toNowTask(NowTask<U>);+};++// NB: `toNowTask(SafeTask)` is in `SafeTask.h` to avoid circular deps.+template <typename T>+auto toNowTask(Task<T> t) {+  return NowTask<T>{std::move(t)};+}+template <typename T>+auto toNowTask(NowTask<T> t) {+  return NowTask<T>{std::move(t).unwrapTask()};+}++// Apparently, Clang 15 has a bug in prvalue semantics support, so it cannot+// return immovable coroutines.+#if !defined(__clang__) || __clang_major__ > 15++/// Make a `NowTask` that trivially returns a value.+template <class T>+NowTask<T> makeNowTask(T t) {+  co_return t;+}++/// Make a `NowTask` that trivially returns no value+inline NowTask<> makeNowTask() {+  co_return;+}+/// Same as makeNowTask(). See Unit+inline NowTask<> makeNowTask(Unit) {+  co_return;+}++/// Make a `NowTask` that will trivially yield an exception.+template <class T>+NowTask<T> makeErrorNowTask(exception_wrapper ew) {+  co_yield co_error(std::move(ew));+}++#endif // no `makeNowTask` on old/buggy clang++} // namespace folly::coro++#endif
+ folly/folly/coro/safe/SafeAlias.h view
@@ -0,0 +1,203 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <folly/Traits.h>+#include <folly/lang/SafeAlias-fwd.h>++#include <type_traits>++namespace folly {+template <typename> // Forward-decl to keep `RValueReferenceWrapper.h` dep-free+class rvalue_reference_wrapper;+} // namespace folly++/*+"Aliasing" is indirect access to memory via pointers or references.  It is+the major cause of memory-safety bugs in C++, but is also essential for+writing correct & performant C++ programs.  Fortunately,+  - Much business logic can be written in a pure-functional style, where+    only value semantics are allowed.  Such code is easier to understand,+    and has much better memory-safety.+  - When references ARE used, the most common scenario is passing a+    reference from a parent lexical scope to descendant scopes.++`safe_alias_of_v` is a _heuristic_ to check whether a type is likely to be+memory-safe in the above settings.  The `safe_alias` enum shows a hierarchy+of memory safety, but you only need to know about two:+  - `unsafe` -- e.g. raw pointers or references, and+  - `maybe_value` -- `int`, `std::pair<int, char>`, or `std::unique_ptr<Foo>`.++A user can easily bypass the heuristic -- since C++ lacks full reflection,+it is impossible to make this bulletproof.  Our goals are much more modest:+  - Make unsafe aliasing **more** visible in code review, and+  - Encourage programmers to use safe semantics by default.++The BIG CAVEATS are:++ - The "composition hole" -- i.e. aliasing hidden in structures.  We can't see+   unsafe class members, so `UnsafeStruct` below will be deduced to have+   `maybe_value` safety unless you specialize `safe_alias_of<UnsafeStruct>`.+     struct UnsafeStruct { int* rawPtr; };+   Future: Perhaps with C++26 reflection, this could be fixed.++   The "lambda hole" is a particularly easy instance of the "composition hole".+   With lambda captures, a parent needs just one `&` to let a child pass a+   soon-to-be-dangling reference up the stack.  E.g. this compiles:+       int* badPtr;+       auto t = async_closure(+         // LAMBDA HOLE: We can't tell this callable object is unsafe!+         bound_args{[&](int p) { *badPtr = p; }},+         [](auto fn) -> ClosureTask<void> {+           int i = 5;+           fn(i); // FAILURE: Dereferencing uninitialized `badPtr`.+           co_return;+         });++ - Nullability & pointer stability: These hazards are not very specific to+   coroutines, and the current design of `folly/coro/safe` largely avoids+   unstable containers.  Nonetheless, you must beware container mutation is+   an easy way to invalidate `safe_alias` memory-safety measurements.  For+   example `unique_ptr<int>` and `vector<int>` have `maybe_value` safety.+   However, if you mutate them (`reset()`, `clear()`, etc), that would+   invalidate any async references (e.g.  `Captures.h`) pointing inside.+   Luckily, there's no implicit way of getting a safe reference to inside+   regular containers.  However, it is recommended to reduce accidental+   nullability where possible.  For example, `capture<unique_ptr<T>>`+   exposes `reset()`, but `capture_indirect<unique_ptr<T>>` hides it behind+   `get_underlying_unsafe()`.  Better yet, `capture<AsyncObjectPtr<T>>`+   blocks the underlying `clear()` method entirely.++If you need to bypass this control, prefer the `manual_safe_*` wrappers+below, instead of writing a custom workaround.  Always explain why it's safe.++To teach `safe_alias_of` about your type, include `SafeAlias-fwd.h` and either:+ 1) Add a member type alias to your class:+    using using folly_private_safe_alias_t = safe_alias_constant<...>;+ 2) Specialize `folly::safe_alias_of<YourT>`.++When adding `safe_alias` annotations to types, stick to these principles:+  - Always mark the `safe_alias` level in the header that declares your type.+    For `std` types you cannot change, add the specialization here, in+    `SafeAlias.h`.  Since we cannot forward-declare from `std`, this+    unfortunately imposes a tradeoff between build cost and safety.  Commonly+    used containers are worth the cost.  For less-commonly used containers, we+    could develop a multi-header setup, plus some linter coverage to ensure the+    right headers ultimately do get included.+  - Only use `maybe_value` if your type ACTUALLY follows value semantics.+  - Unless you're implementing an `async_closure`-integrated type, it is VERY+    unlikely that you should use anything besides `unsafe` or `maybe_value`.+  - Use `safe_alias_of_pack` to aggregate safety for a multi-part type.+*/+namespace folly {++// Types are `maybe_value` unless otherwise specified.  Note that+// `SafeAlias-fwd.h` already marks raw pointers & refs as `unsafe`, and peels+// off CV qualifiers from the type being tested.+//+// See also: `safe_alias_of_v`.+//+// As explained in `SafeAlias-fwd.h`, do NOT move this to the `fwd` header.  To+// guarantee safety, this permissive primary template must be colocated with+// the other specializations below.+template <typename T, typename /*SFINAE*/>+struct safe_alias_of : safe_alias_constant<safe_alias::maybe_value> {};++// Reference wrappers are unsafe.+template <typename T>+struct safe_alias_of<std::reference_wrapper<T>>+    : safe_alias_constant<safe_alias::unsafe> {};+template <typename T>+struct safe_alias_of<folly::rvalue_reference_wrapper<T>>+    : safe_alias_constant<safe_alias::unsafe> {};++// Let `safe_alias_of_v` recursively inspect `std` containers that are likely+// to be involved in bugs.  If you encounter a memory-safety issue that+// would've been caught by this, feel free to extend this.+template <typename... As>+struct safe_alias_of<std::tuple<As...>> : safe_alias_of_pack<As...> {};+template <typename... As>+struct safe_alias_of<std::pair<As...>> : safe_alias_of_pack<As...> {};+template <typename... As>+struct safe_alias_of<std::vector<As...>> : safe_alias_of_pack<As...> {};++// Recursing into `tag_t<>` type lists is nice for metaprogramming+template <typename... As>+struct safe_alias_of<::folly::tag_t<As...>> : safe_alias_of_pack<As...> {};++// IMPORTANT: If you use the `manual_safe_` escape-hatch wrappers, you MUST+// comment with clear proof of WHY your usage is safe.  The goal is to+// ensure careful review of such code.+//+// Careful: With the default `Safety`, the contained value or reference can be+// passed anywhere -- the wrapper pretends to be a value type.+//+// If you know a more restrictive safety level for your ref, annotate it to+// improve safety:+//  - `after_cleanup_ref` for things owned by co_cleanup args of this closure,+//  - `co_cleanup_safe_ref` for refs to non-cleanup args owned by this closure,+//    or any ancestor closure.+//+// The types are public since they may occur in user-facing signatures.++template <safe_alias, typename T>+struct manual_safe_ref_t : std::reference_wrapper<T> {+  using typename std::reference_wrapper<T>::type;+  using std::reference_wrapper<T>::reference_wrapper;+};++template <safe_alias, typename T>+struct manual_safe_val_t {+  using type = T;++  template <typename... Args>+  manual_safe_val_t(Args&&... args) : t_(static_cast<Args&&>(args)...) {}+  template <typename Fn>+  manual_safe_val_t(std::in_place_type_t<T>, Fn fn) : t_(fn()) {}++  T& get() & noexcept { return t_; }+  operator T&() & noexcept { return t_; }+  const T& get() const& noexcept { return t_; }+  operator const T&() const& noexcept { return t_; }+  T&& get() && noexcept { return std::move(t_); }+  operator T&&() && noexcept { return std::move(t_); }++ private:+  T t_;+};++template <safe_alias Safety = safe_alias::maybe_value, typename T = void>+auto manual_safe_ref(T& t) {+  return manual_safe_ref_t<Safety, T>{t};+}+template <safe_alias Safety = safe_alias::maybe_value, typename T>+auto manual_safe_val(T t) {+  return manual_safe_val_t<Safety, T>{std::move(t)};+}+template <safe_alias Safety = safe_alias::maybe_value, typename Fn>+auto manual_safe_with(Fn&& fn) {+  using FnRet = decltype(static_cast<Fn&&>(fn)());+  return manual_safe_val_t<Safety, FnRet>{+      std::in_place_type<FnRet>, static_cast<Fn&&>(fn)};+}++template <safe_alias S, typename T>+struct safe_alias_of<manual_safe_ref_t<S, T>> : safe_alias_constant<S> {};+template <safe_alias S, typename T>+struct safe_alias_of<manual_safe_val_t<S, T>> : safe_alias_constant<S> {};++} // namespace folly
+ folly/folly/coro/safe/SafeTask.h view
@@ -0,0 +1,428 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <folly/coro/TaskWrapper.h>+#include <folly/coro/safe/NowTask.h>+#include <folly/coro/safe/SafeAlias.h>++#if FOLLY_HAS_IMMOVABLE_COROUTINES++namespace folly::coro {++/// Why is `SafeTask.h` useful? See `SafeTask.md`.+///+/// Typically, you will not use `SafeTask` directly.  Instead, choose one of+/// the type-aliases below, following `APIBestPractices.md` guidance.  Briefly:+///   - `ValueTask`: Use if your coro only takes value-semantic args.+///   - `MemberTask`: Use for all non-static member functions.  Can be+///     awaited immediately (like `NowTask`), or wrapped in an+///     `async_closure` to support less-structured concurrency -- including+///     scheduling on a background scope belonging to the object.+///   - `ClosureTask`: Use if your coro is called via `async_closure`.+///   - `CoCleanupSafeTask`: Use for tasks that can be directly scheduled on a+///     `SafeAsyncScope`.+///   - (not in `SafeTask.h`) `NowTask`: All other coros.  This requires the+///     task to always be awaited in the expression that created it,+///     eliminating a variety of common dangling reference bugs.+///   - `AutoSafeTask`: Generic coros where you want the argument & return+///     types to automatically branch between a `NowTask` and a `SafeTask`.+///+/// `SafeTask` is a thin wrapper around `folly::coro::Task` that uses+/// `safe_alias_of` to enforce some compile-time guarantees:+///   - The `SafeTask` has `safe_alias_of` memory safety at least as high as+///     the coro's arguments.  In particular, no args are taken by reference.+///   - Regardless of the task's declared safety, the coro's return must+///     have safety `maybe_value` (explained in `SafeTaskRetAndArgs`).+///   - The coroutine is NOT a stateful callable -- this prohibits lambda+///     captures, since those are a very common cause of coro memory bugs.+template <safe_alias, typename = void>+class SafeTask;+template <safe_alias, typename = void>+class SafeTaskWithExecutor;++// A `SafeTask` whose args and return type follow value semantics.+template <typename T = void>+using ValueTask = SafeTask<safe_alias::maybe_value, T>;++// A `SafeTask` that can be added to `SafeAsyncScope`, and may run during+// closure cleanup.  Its content must therefore be `co_cleanup_safe_ref`-safe.+template <typename T = void>+using CoCleanupSafeTask = SafeTask<safe_alias::co_cleanup_safe_ref, T>;++// Use `ClosureTask` as the inner coro type for tasks meant to ALWAYS be+// wrapped in an `async_closure`.+//+// Outside of a closure, a `ClosureTask` is immovable.  If you are wanting to+// move a `ClosureTask`, construct it via an async closure, and you'll get back+// a `SafeTask` with safety measurements reflecting the safety of its args.+//+// If your use-case calls for a `SafeTask` that is sometimes wrapped in a+// closure, and sometimes is constructed without a closure, you might add a+// `MinClosureSafeTask` type alias for `closure_min_arg_safety`.+//+// Immovability rationale: `ClosureTask` is implemented as a `SafeTask` for+// reasons explained in the next paragraph.  But, its safety contract is weaker+// than that of the usual closure (it can take `capture<Val>`, which should+// never be moved) -- immovability is meant to reduce the odds of misuse.+// Making it truly opaque / not semi-awaitable would be a stronger safeguard,+// but that requires extra complexity even just so that+// `AsNoexcept<ClosureTask<>> foo()` would compile.+//+// "ClosureTask is a SafeTask" rationale: `async_closure` cannot emit a+// `SafeTask` without the inner coro being a `SafeTask` -- otherwise it could+// not guarantee that none of the args are taken by reference.  Conveniently,+// `SafeTask` also checks the return type is safe, and the coro's callable is+// stateless, so `async_closure` can skip those checks.+//+// This `ClosureTask` implementation uses a `safe_alias` level safer than+// `unsafe` to get all of the above `SafeAlias` checks.  The level also has+// to be less safe than `shared_cleanup` so we can treat these differently:+//  - `capture<Value>` (safety `unsafe_closure_internal`) should stay in the+//    original closure.  Users can move the content, but shouldn't move the+//    wrapper, since that messes with the safety system.+//  - `co_cleanup_capture<Value&>` refs (safety `shared_cleanup`) can safely+//    be moved or copied into other closures.+//  - By the way, `co_cleanup_capture<Value>` should never be moved from the+//    owning closure that's responsible for its cleanup.+template <typename T = void>+using ClosureTask = SafeTask<safe_alias::unsafe_closure_internal, T>;++// A `MemberTask` is a hybrid of `SafeTask` and `NowTask`, intended to make+// non-static member coroutines safer.+//   - It **is** a `SafeTask`, thereby forbidding `safe_alias::unsafe`+//     arguments, and unsafe return types.  However, since the callable of+//     member coros is inherently stateful, it is special-cased to omit the+//     safety checks on the implicit object parameter.+//   - It is immovable like `NowTask`, which makes typical "structured+//     concurrency" usage of coroutines quite safe (see `NowTask.h`).+//     `MemberTask` needs this, since members take `this`, whose lifetime is+//     unknown -- i.e.  outside of async closure usage, a `MemberTask` is just+//     a `NowTask`.+//+// For more complex usage (background tasks, async RAII), `MemberTask` has a+// special calling convention in `AsyncClosure.h`:+//+//   async_closure(bound_args{obj, args...}, FOLLY_INVOKE_MEMBER(memberFnName))+//+// Like any async closure, this safety-checks the now-explicit object param,+// and produces a movable `SafeTask` of the safety level determined from the+// arguments.  This integration lets us safely schedule member coros on+// `SafeAsyncScope`, pass `co_cleanup` args into such coros, etc.+template <typename T>+using MemberTask = SafeTask<safe_alias::unsafe_member_internal, T>;++// NB: There are some `async_closure`-specific values of `safe_alias` that+// do not yet have a `SafeTask` alias.  That's because they haven't come up+// in user-facing type signatures.++namespace detail {+template <typename T, safe_alias Safety>+using AutoSafeTaskImpl = std::conditional_t<+    // This checks both args & the return value because we want to avoid this+    // resolving to a `SafeTask` that won't actually compile.+    (Safety >= safe_alias::closure_min_arg_safety &&+     safe_alias_of_v<T> >= safe_alias::maybe_value),+    SafeTask<Safety, T>,+    NowTask<T>>;+}++/// Coros declared as `SafeTask<Safety, T>` will satisfy the strong+/// constraints above, or fail with a compile error.+///+/// The safety of a coroutine template may vary depending on the args or+/// return type, meaning that the user can't actually pick a fixed+/// safety level for their generic coro.+///+/// Instead, the generic coro can return `AutoSafeTask<ReturnT,+/// SafetyArgs...>`, where `SafetyArgs` is (typically) the subset of the+/// coroutine's argument types that may affect safety.+///+/// `AutoSafeTask` has a Significant Caveat -- you can't use it with+/// non-`static` member functions -- the implicit object parameter is unsafe+/// (as it should be).  And if you do use it, you will get a compile-time+/// error instead of a `NowTask`, simply because this type-function has no+/// access to the callable.  See `APIBestPractices.md` for workarounds.+template <typename T, typename... SafetyArgs>+using AutoSafeTask =+    detail::AutoSafeTaskImpl<T, safe_alias_of_pack<SafetyArgs...>::value>;++namespace detail {++struct SafeTaskTest;++template <safe_alias ArgSafety, typename RetT, typename... Args>+concept SafeTaskRetAndArgs = ((safe_alias_of_v<Args> >= ArgSafety) && ...) &&+    // In the event that you need a child scope to return a reference to+    // something owned by a still-valid ancestor scope, we don't have a good+    // way to detect this automatically.  To work around, use a `manual_safe_*`+    // wrapper in `SafeAlias.h`, and comment why it is safe.+    (safe_alias_of_v<RetT> >= safe_alias::maybe_value);++template <typename T>+concept is_stateless_class_or_func =+    (std::is_class_v<T> && std::is_empty_v<T>) ||+    (std::is_pointer_v<T> && std::is_function_v<std::remove_pointer_t<T>>);++template <safe_alias, typename...>+inline constexpr bool IsSafeTaskValid = false;+// Coros taking 0 args can't be methods (no implicit object parameter),+// so their safety is determined by the return type.+template <safe_alias ArgSafety, typename RetT>+inline constexpr bool IsSafeTaskValid<ArgSafety, RetT> =+    SafeTaskRetAndArgs<ArgSafety, RetT>;+// Inspect the first argument, which can be an implicit object parameter, to+// allow stateless callables (like lambdas), but to prohibit stateful+// callables (these can contain unsafe aliasing in their state, which we+// can't inspect).  If you need to make `SafeTask`s from a stateful object,+// pass `capture<Ref>` to a static func, and check out `AsyncObject.h`.+//+// How this works: With >= 1 args in the pack, the `First` argument+// **could** be an implicit object parameter.  We don't know if it is, but+// we do know that any such parameter has type lvalue reference, which means+// that it would fail `SafeTaskRetAndArgs<RetT, First, Args...>`.+//+// This test accepts any `First` that is an lref to a stateless class+// or function -- that is, it returns `true` if the first arg is either:+//   - not an lref, and has no unsafe aliasing (not an implicit object param)+//   - an lref to something stateless (MAY be an implicit object param)+// As a side effect, this allows coros without an implicit object param to+// pass a stateless class by reference, if it's the first param.  This+// should be harmless in practice.+//+// FIXME: It should (?) be fine to simplify this scenario by having+// `SafeAlias` mark as "safe" all references-to-empty-classes, and all+// function pointers.  Then, only a shortened comment would survive.+//+// For `MemberTask`, `First` is assumed to be the implicit object parameter.+// This cannot be safe, so we don't check it, and instead rely on+// `MemberTask`'s usage restrictions (see also `SafeTaskBaseTrait`).+template <safe_alias ArgSafety, typename RetT, typename First, typename... Args>+inline constexpr bool IsSafeTaskValid<ArgSafety, RetT, First, Args...> =+    ((ArgSafety == safe_alias::unsafe_member_internal) ||+     (std::is_lvalue_reference_v<First> &&+      is_stateless_class_or_func<std::remove_reference_t<First>>))+    ? SafeTaskRetAndArgs<ArgSafety, RetT, Args...>+    : SafeTaskRetAndArgs<ArgSafety, RetT, First, Args...>;++template <safe_alias ArgSafety, typename T, typename... Args>+class SafeTaskPromise final+    : public TaskPromiseWrapper<+          T,+          SafeTask<ArgSafety, T>,+          detail::TaskPromise<T>> {+  // "Unsafe" is not a "safe" task any more.  In the future, we could have+  // `SafeTask<unsafe, T>` act as `NowTask<T>`, but there's no present use+  // for this uniformity, but there are benefits to explicitness.+  static_assert(+      ArgSafety > safe_alias::unsafe,+      "Instead of making an unsafe `SafeTask`, use a `NowTask`, or "+      "`async_now_closure()`");++ public:+  // IMPORTANT: If you alter this arrangement, do the "Manual test" inside+  // `returnsVoid` in `SafeTaskTest.cpp`.+  //+  // This is a no-op wrapper.  It needs to exist because `IsSafeTaskValid`+  // requires the coroutine function to be a complete type before checking+  // if it's a stateless callable, and the easiest place to do that is in a+  // class function that's guaranteed to be instantiated, such as this.+  SafeTask<ArgSafety, T> get_return_object() noexcept {+    // If your build failed here, your `SafeTask<>` coro declaration is+    // invalid.  Specific causes for this failure:+    //   - One of the arguments, or the return value, contains "unsafe+    //     aliasing" -- see `SafeAlias.h` for the details.  Typical+    //     causes include raw pointers, references, reference wrappers, etc.+    //   - A stateful callable: lambda with captures, class with members, etc.+    static_assert(+        detail::IsSafeTaskValid<ArgSafety, T, Args...>,+        "Bad SafeTask: check for unsafe aliasing in arguments or return "+        "type; also ensure your callable is stateless.");+    return TaskPromiseWrapper<+        T,+        SafeTask<ArgSafety, T>,+        detail::TaskPromise<T>>::get_return_object();+  }+};++template <auto>+auto bind_captures_to_closure(auto&&, auto);++template <safe_alias ArgSafety, typename T>+struct SafeTaskWithExecutorCfg : DoesNotWrapAwaitable {+  using InnerTaskWithExecutorT = TaskWithExecutor<T>;+  using WrapperTaskT = SafeTask<ArgSafety, T>;+};++template <safe_alias, typename>+struct SafeTaskWithExecutorBaseTraits;++template <safe_alias ArgSafety, typename T>+  requires(ArgSafety >= safe_alias::closure_min_arg_safety)+struct SafeTaskWithExecutorBaseTraits<ArgSafety, T> {+  using type = TaskWithExecutorWrapperCrtp<+      SafeTaskWithExecutor<ArgSafety, T>,+      SafeTaskWithExecutorCfg<ArgSafety, T>>;+};++// `MemberTask` and `ClosureTask` are immovable.+template <safe_alias ArgSafety, typename T>+  requires(ArgSafety < safe_alias::closure_min_arg_safety)+struct SafeTaskWithExecutorBaseTraits<ArgSafety, T> {+  using type = AddMustAwaitImmediately<TaskWithExecutorWrapperCrtp<+      SafeTaskWithExecutor<ArgSafety, T>,+      SafeTaskWithExecutorCfg<ArgSafety, T>>>;+};++template <safe_alias ArgSafety, typename T>+struct SafeTaskCfg : DoesNotWrapAwaitable {+  using ValueT = T;+  using InnerTaskT = Task<T>;+  using TaskWithExecutorT = SafeTaskWithExecutor<ArgSafety, T>;+  // There is no `promise_type` here because it's added by `coroutine_traits`+  // below.  This is the mechanism that enables `SafeTaskPromise` to inspect+  // the specific arguments of the coroutine (including the implicit object+  // parameter), and fail the compilation if anything looks unsafe.+  using PromiseT = void;+};++template <safe_alias ArgSafety, typename T>+struct SafeTaskBaseTraits {+  using type =+      TaskWrapperCrtp<SafeTask<ArgSafety, T>, SafeTaskCfg<ArgSafety, T>>;+};++// `MemberTask` and `ClosureTask` are immovable.+template <safe_alias ArgSafety, typename T>+  requires(ArgSafety < safe_alias::closure_min_arg_safety)+struct SafeTaskBaseTraits<ArgSafety, T> {+  using type = AddMustAwaitImmediately<+      TaskWrapperCrtp<SafeTask<ArgSafety, T>, SafeTaskCfg<ArgSafety, T>>>;+};++} // namespace detail++template <safe_alias, typename>+class BackgroundTask;++// IMPORTANT: This omits `start()` because backgrounded tasks can easily+// outlive the references they took, defeating the purpose of `SafeTask`.+// See `BackgroundTask` instead.+template <safe_alias ArgSafety, typename T>+class FOLLY_NODISCARD SafeTaskWithExecutor final+    : public detail::SafeTaskWithExecutorBaseTraits<ArgSafety, T>::type {+ protected:+  using detail::SafeTaskWithExecutorBaseTraits<ArgSafety, T>::type::type;++  template <safe_alias, typename>+  friend class BackgroundTask; // for `unwrapTaskWithExecutor()`, remove later++ public:+  using folly_private_safe_alias_t = safe_alias_constant<ArgSafety>;++  [[deprecated(+      "`asUnsafe()` is provided as an escape hatch for interoperating with "+      "older futures-based code, or other places not yet compatible with "+      "true structured concurrency patterns. Beware, the full `Task` API "+      "abounds with footguns like `start()` and `semi()` -- including UB, "+      "leaks, and lost errors.")]]+  TaskWithExecutor<T> asUnsafe() && {+    return std::move(*this).unwrapTaskWithExecutor();+  }+};++template <safe_alias ArgSafety, typename T>+class FOLLY_CORO_TASK_ATTRS SafeTask final+    : public detail::SafeTaskBaseTraits<ArgSafety, T>::type {+ protected:+  friend struct folly::coro::detail::SafeTaskTest; // to test `withNewSafety`+  template <safe_alias, typename>+  friend class SafeTask; // `withNewSafety` makes a different `SafeTask`+  template <auto> // uses `withNewSafety`+  friend auto detail::bind_captures_to_closure(auto&&, auto);+  template <safe_alias Safety, typename U>+  friend auto toNowTask(SafeTask<Safety, U>);++  // The `async_closure` implementation is allowed to override the+  // argument-deduced `safe_alias_of_v` for a `SafeTask` because+  // `capture_safety` marks some coro-stored `*capture*`s as `unsafe` even+  // though they're safe -- to discourage users from moving them.+  template <safe_alias NewSafety>+  SafeTask<NewSafety, T> withNewSafety() && {+    return SafeTask<NewSafety, T>{std::move(*this).unwrapTask()};+  }++ public:+  using detail::SafeTaskBaseTraits<ArgSafety, T>::type::type;+  using folly_private_safe_alias_t = safe_alias_constant<ArgSafety>;++  [[deprecated(+      "`asUnsafe()` is provided as an escape hatch for interoperating with "+      "older futures-based code, or other places not yet compatible with "+      "true structured concurrency patterns. Beware, the full `Task` API "+      "abounds with footguns like `start()` and `semi()` -- including UB, "+      "leaks, and lost errors.")]]+  Task<T> asUnsafe() && {+    return std::move(*this).unwrapTask();+  }+};++template <safe_alias Safety, typename T>+auto toNowTask(SafeTask<Safety, T> t) {+  return NowTask<T>{std::move(t).unwrapTask()};+}++namespace detail {++template <typename>+struct safe_task_traits;++template <typename T>+struct safe_task_traits<Task<T>> {+  static constexpr safe_alias arg_safety = safe_alias::unsafe;+  using return_type = T;+};+template <typename T>+struct safe_task_traits<TaskWithExecutor<T>> : safe_task_traits<Task<T>> {};+template <typename T>+struct safe_task_traits<NowTask<T>> : safe_task_traits<Task<T>> {};+template <typename T>+struct safe_task_traits<NowTaskWithExecutor<T>> : safe_task_traits<Task<T>> {};++template <safe_alias ArgSafety, typename T>+struct safe_task_traits<SafeTask<ArgSafety, T>> {+  static constexpr safe_alias arg_safety = ArgSafety;+  using return_type = T;+};+template <safe_alias ArgSafety, typename T>+struct safe_task_traits<SafeTaskWithExecutor<ArgSafety, T>>+    : safe_task_traits<SafeTask<ArgSafety, T>> {};++} // namespace detail++} // namespace folly::coro++template <folly::safe_alias ArgSafety, typename T, typename... Args>+struct folly::coro::+    coroutine_traits<folly::coro::SafeTask<ArgSafety, T>, Args...> {+  // UGH: Pass `Args...` into `SafeTaskPromise` because at this point, the+  // coroutine function is still an incomplete type, and can't be validated.+  using promise_type =+      folly::coro::detail::SafeTaskPromise<ArgSafety, T, Args...>;+};++#endif
+ folly/folly/coro/safe/detail/AsyncClosure.h view
@@ -0,0 +1,675 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <folly/coro/Noexcept.h>+#include <folly/coro/safe/SafeTask.h>+#include <folly/coro/safe/detail/AsyncClosureBindings.h>+#include <folly/detail/tuple.h>++#if FOLLY_HAS_IMMOVABLE_COROUTINES+FOLLY_PUSH_WARNING+FOLLY_DETAIL_LITE_TUPLE_ADJUST_WARNINGS++// DANGER: Do NOT touch this implementation without understanding the contract,+// at least at the level of the tl;dr in `safe/AsyncClosure.h`, and in full+// depth if you're changing `safe_alias` measurements.++namespace folly::coro::detail {++void async_closure_set_cancel_token(+    async_closure_private_t priv, auto&& arg, const CancellationToken& ctok) {+  if constexpr ( // DO NOT USE: for AsyncObject only+      requires { arg.privateHackSetParentCancelToken(arg, priv, ctok); }) {+    arg.privateHackSetParentCancelToken(arg, priv, ctok);+  } else if constexpr ( //+      requires {+        {+          arg.get_lref().setParentCancelToken(priv, ctok)+        } -> std::same_as<void>;+      }) {+    arg.get_lref().setParentCancelToken(priv, ctok);+  }+}++auto async_closure_make_cleanup_tuple(+    async_closure_private_t priv, auto&& arg, const exception_wrapper* err) {+  // `co_cleanup` is allowed to return `Task<void>` or a tuple of them.+  auto to_lite_tuple = []<typename T>(T task) {+    static_assert(+        noexcept_awaitable_v<T> && std::is_void_v<semi_await_result_t<T>>,+        "`co_cleanup()` must return a `noexcept`-awaitable `void` coro. "+        "Change your return type to `AsNoexcept<Task<>>` and don't throw.");+    return lite_tuple::tuple{std::move(task)};+  };+  if constexpr (has_async_object_private_hack_co_cleanup<decltype(arg)>) {+    return arg.privateHack_co_cleanup(std::move(arg), priv, err);+  } else {+    using ArgT = typename std::remove_reference_t<decltype(arg)>::capture_type;+    if constexpr (has_async_closure_co_cleanup_with_error<ArgT>) {+      return to_lite_tuple(std::move(arg.get_lref()).co_cleanup(priv, err));+    } else if constexpr (has_async_closure_co_cleanup_error_oblivious<ArgT>) {+      return to_lite_tuple(std::move(arg.get_lref()).co_cleanup(priv));+    } else {+      return lite_tuple::tuple{};+    }+  }+}++template <typename T>+concept has_result_after_cleanup = requires(+    lift_unit_t<T> t, async_closure_private_t priv) {+  std::move(t).result_after_cleanup(priv);+};++template <bool AssertNoexcept, typename T>+  requires(!std::is_reference_v<T>)+auto async_closure_outer_coro_result(async_closure_private_t priv, T r) {+  if constexpr (has_result_after_cleanup<T>) {+    static_assert(+        !AssertNoexcept || noexcept(std::move(r).result_after_cleanup(priv)));+    return std::move(r).result_after_cleanup(priv);+  } else {+    static_assert(!AssertNoexcept || std::is_nothrow_constructible_v<T, T&&>);+    (void)priv;+    return r;+  }+}++template <+    bool SetCancelTok,+    typename ResultT,+    safe_alias OuterSafety,+    bool AssertNoexcept>+auto async_closure_make_outer_coro(+    async_closure_private_t priv, auto inner_mover, auto storage_ptr) {+  return lite_tuple::apply(+      [&](auto... reversed_noexcept_cleanups) {+        return async_closure_outer_coro<+            SetCancelTok,+            ResultT,+            OuterSafety,+            AssertNoexcept>(+            priv,+            // Doesn't downgrade safety, since movers are library-internal+            // "unsafe" types that don't expose the inner type's `safe_alias`.+            std::move(inner_mover),+            std::move(storage_ptr),+            // We don't require a `SafeTask` for `co_cleanup` because the coro+            // cannot outlive the object (or `exception_ptr*`) it references.+            manual_safe_val(std::move(reversed_noexcept_cleanups))...);+      },+      // Contract: `co_cleanup()`s are awaited sequentially right-to-left, in+      // the reverse of the construction order.  All cleanups finish before any+      // of the destructors; those also run right-to-left.+      //+      // Implementation notes:+      //   - `bad_alloc` safety: make the tasks before awaiting the inner coro.+      //   - This "apply" is outside of `async_closure_outer_coro` because+      //     that saves us a coro frame allocation.+      lite_tuple::reverse_apply( // Merge `co_cleanup` tuples from all the args+          [&](auto&... args) {+            return lite_tuple::tuple_cat(async_closure_make_cleanup_tuple(+                priv, args, storage_ptr->inner_err_ptr())...);+          },+          storage_ptr->storage_tuple_like()));+}++// IMPORTANT: This must not allow unhandled exceptions to escape, since for+// noexcept-awaitable inner coros, the outer one is marked noexcept-awaitable.+template <+    bool SetCancelTok,+    typename ResultT,+    safe_alias OuterSafety,+    // This coro is noexcept-awaitable iff `async_closure_outer_coro_result` is+    // `noexcept`.  But we don't want to restrict it for coros that are not+    // marked `AsNoexcept` -- this boolean toggles its "is noexcept" asserts.+    bool AssertNoexcept,+    typename OuterResT =+        drop_unit_t<decltype(async_closure_outer_coro_result<AssertNoexcept>(+            std::declval<async_closure_private_t>(),+            std::declval<lift_unit_t<ResultT>&&>()))>>+std::conditional_t<+    OuterSafety >= safe_alias::closure_min_arg_safety,+    SafeTask<OuterSafety, OuterResT>,+    NowTask<OuterResT>>+async_closure_outer_coro(+    async_closure_private_t priv,+    auto inner_mover,+    auto storage_ptr,+    auto... reversed_noexcept_cleanups) {+  auto& inner_err = *storage_ptr->inner_err_ptr();+  if constexpr (kIsDebug) {+    inner_err.reset(); // Clear `BUG_co_cleanup_must_not_copy_error`+  }++  // Pass our cancellation token to args that want it for cleanup.  The user+  // code can throw -- e.g. `CancellationToken::merge()` may allocate.+  if constexpr (SetCancelTok) {+    const auto& ctok = co_await co_current_cancellation_token;+    inner_err = try_and_catch([&]() {+      lite_tuple::apply(+          [&](auto&&... args) {+            (async_closure_set_cancel_token(priv, args, ctok), ...);+          },+          storage_ptr->storage_tuple_like());+    });+  }++  // Await the inner task (unless some `setParentCancelToken` failed)+  Try<ResultT> res;+  if (!inner_err) {+    // NOTE: Here and below, assume that the semi-awaitable `co_viaIfAsync`+    // machinery for `Task` (or other `inner` type) is non-throwing.+    // I would love a `static_assert(noexcept(...))` to prove this, but that+    // requires plumbing `noexcept(noexcept(...))` annotations through more+    // of `ViaIfAsync.h`.+    res = co_await co_awaitTry(std::move(inner_mover)());+    if (res.hasException()) {+      inner_err = std::move(res.exception());+    }+  }++  // We took the cleanup tasks as a pack to let us await them without making an+  // extra coro frame.+  (co_await std::move(reversed_noexcept_cleanups.get()), ...);++  if (FOLLY_LIKELY(res.hasValue())) {+    if constexpr (std::is_void_v<ResultT>) {+      co_return;+    } else {+      co_return async_closure_outer_coro_result<AssertNoexcept>(+          priv, std::move(res).value());+    }+  } else if (FOLLY_LIKELY(res.hasException())) {+    co_yield co_error(std::move(inner_err));+  } else { // should never happen+    co_yield co_error(UsingUninitializedTry{});+  }+  (void)storage_ptr; // This param keeps the stored args alive+}++// E.g. maps <0, 2, 1, 0, 2> to <0, 2, 3, 3> -- see Test.cpp+template <auto Sum, auto...>+inline constexpr auto cumsum_except_last = vtag<>;+template <auto Sum, auto Head, auto... Tail>+inline constexpr auto cumsum_except_last<Sum, Head, Tail...> =+    []<auto... Vs>(vtag_t<Vs...>) {+      return vtag<Sum, Vs...>;+    }(cumsum_except_last<Sum + Head, Tail...>);++// When returned from `bind_captures_to_closure`, this wraps a coroutine+// instance.  This reconciles two goals:+//  - Let tests cover the `is_safe()` logic.+//  - `static_assert()` the closure's safety before releasing it.+//+// Closure safety checks follow the model of `SafeTask.h` -- and actually+// reuse most of that implementation by requiring the inner coro to be a+// `SafeTask`.+//+// Note that we don't check whether the callable passed into `async_closure`+// is stateless, and we don't need to -- it is executed eagerly, and may be+// a coroutine wrapper.  The coro callable underlying the inner `SafeTask`+// will have been verified to be stateless.+//+// Future: An `AsyncGenerator` closure flavor is possible, just think about+// safety assertions on the yielded type, and review+// https://fburl.com/asyncgenerator_delegation+template < // inner coro safety is measured BEFORE re-wrapping it!+    safe_alias OuterSafety,+    safe_alias InnerSafety,+    typename NoexceptWrap,+    typename OuterMover>+class async_closure_wrap_coro {+ private:+  OuterMover outer_mover_;++ protected:+  template <auto>+  friend auto bind_captures_to_closure(auto&&, auto);+  explicit async_closure_wrap_coro(OuterMover outer_mover)+      : outer_mover_(std::move(outer_mover)) {}++ public:+  // Don't allow closures with `unsafe*` args.+  static constexpr bool has_safe_args =+      (OuterSafety >= safe_alias::closure_min_arg_safety);++  // The reason we need `SafeTask` here is that it have already detected any+  // by-reference arguments (impossible to detect otherwise), stateful+  // coros, and unsafe return types.+  static constexpr bool is_inner_coro_safe =+      (InnerSafety >= safe_alias::unsafe_closure_internal);++  // KEEP IN SYNC with `release_outer_coro`. Separate for testing.+  static consteval bool is_safe() {+    return has_safe_args && is_inner_coro_safe;+  }++  // Delay the `static_assert`s so we can test `bind_captures_to_closure`+  // on unsafe inputs.+  auto release_outer_coro() && {+    // KEEP IN SYNC with `is_safe`.+    static_assert(+        has_safe_args,+        "Args passed into `async_closure()` must have `safe_alias_of` of at "+        "least `shared_cleanup`. `NowTask` and `async_now_closure()` do not "+        "have this constraint. To force a movable closure, use `manual_safe_*`,"+        " and comment with a proof of why your usage is memory-safe.");+    static_assert(+        is_inner_coro_safe,+        "`async_closure` currently only supports `SafeTask` as the inner coro.");+    return NoexceptWrap::wrap_with([&]() { return std::move(outer_mover_)(); });+  }+};++// The compiler cannot deduce that `async_closure_outer_stored_arg` cannot+// occur when `storage_ptr` is `nullopt_t`.  This helper function just+// delays instantiation of `storage_ptr->`.+template <size_t Idx>+decltype(auto) get_from_storage_ptr(auto& p) {+  return lite_tuple::get<Idx>(p->storage_tuple_like());+}++template <bool Debug = kIsDebug> // ODR safeguard+inline auto async_closure_default_inner_err() {+  if constexpr (Debug) {+    // If you see this diagnostic, check that your `co_cleanup` does not+    // inadvertently copy the `exception_wrapper` parameter before creating the+    // coro frame.  Store the provided pointer instead.+    struct BUG_co_cleanup_must_not_copy_error : std::exception {};+    return make_exception_wrapper<BUG_co_cleanup_must_not_copy_error>();+  } else {+    return exception_wrapper{};+  }+}++template <auto Tag, size_t ArgI, size_t StoredI>+struct async_closure_backref_entry {+  static inline constexpr auto tag = Tag;+  static inline constexpr size_t arg_idx = ArgI;+  static inline constexpr size_t stored_idx = StoredI;+};++template <typename... Entries>+struct async_closure_backref_map : Entries... {};++template <auto Tag, size_t ArgI, size_t StoredI>+async_closure_backref_entry<Tag, ArgI, StoredI> async_closure_backref_get(+    async_closure_backref_entry<Tag, ArgI, StoredI>);++template <typename, size_t, typename T>+  requires(!std::is_lvalue_reference_v<T>)+struct async_closure_backref_populator {+  T&& operator()(capture_private_t, auto&, T&& t) const {+    return static_cast<T&&>(t);+  }+};++template <typename ArgMap, size_t ArgI, typename Arg>+decltype(auto) async_closure_resolve_backref(+    capture_private_t priv, auto& tup, Arg&) {+  constexpr auto Tag = Arg::folly_bindings_identifier_tag;+  // `AsyncClosureBindings.h` populates tags via `named_bind_info_tag_v`, which+  // uses `no_tag_t` to mean "no tag was set" -- so you can't look it up.+  static_assert(!std::is_same_v<folly::bindings::ext::no_tag_t, decltype(Tag)>);+  // This will fail on missing, or ambiguous tags.+  using Entry = decltype(async_closure_backref_get<Tag>(FOLLY_DECLVAL(ArgMap)));+  static_assert(+      Entry::arg_idx < ArgI,+      "Can only take backrefs to capture storage to the left of the current "+      "capture, since in-place captures are constructed left-to-right.");+  auto& target = lite_tuple::get<Entry::stored_idx>(tup);+  using SourceCapture = std::remove_reference_t<decltype(target)>;+  static_assert(is_any_capture<SourceCapture>);+  using Source = typename SourceCapture::capture_type;+  // At present, it's not even possible to add an `"x"_id` tag to a non-stored+  // argument.  We would also never want to allow backrefs to rvalue reference+  // captures, since those are meant to be single-use.+  static_assert(!std::is_reference_v<Source>);+  return capture<Source&>(priv, forward_bind_wrapper(target.get_lref()));+}++// Replace `"x"_id` backreferences in the args of `capture_in_place` and+// `capture_in_place_with` with `capture<T&>` references to the corresponding+// capture storage.+//+// Backrefs may ONLY point to capture storage -- any args moved into the inner+// coro are subject to unspecified destruction order, and so could not safely+// reference each other.  In principle, we could allow backrefs to+// `capture<T&>` refs being passed from the parent, but that adds complexity,+// and isn't very useful.+//+// We don't need an explicit "closure has outer coro" test, since the+// backref-population logic ONLY runs against stored args.+template <typename ArgMap, size_t ArgI, typename T, typename... Args>+  requires(requires(Args a) { a.folly_bindings_identifier_tag; } || ...)+struct async_closure_backref_populator<+    ArgMap,+    ArgI,+    bind_wrapper_t<folly::bindings::detail::in_place_args_maker<T, Args...>>> {+  using BindWrap =+      bind_wrapper_t<folly::bindings::detail::in_place_args_maker<T, Args...>>;+  auto operator()(capture_private_t priv, auto& tup, BindWrap&& bw) const {+    return lite_tuple::apply(+        [&](Args&&... args) {+          return unsafe_tuple_to_bind_wrapper(+              folly::bindings::make_in_place_with([&]() {+                return T{[&]() -> decltype(auto) {+                  if constexpr (requires(Args a) {+                                  a.folly_bindings_identifier_tag;+                                }) {+                    // Pass (and take) `args` by lvalue ref because moving+                    // backref tokens doesn't make sense.+                    return async_closure_resolve_backref<ArgMap, ArgI>(+                        priv, tup, args);+                  } else {+                    return static_cast<Args&&>(args);+                  }+                }()...};+              }).unsafe_tuple_to_bind());+        },+        static_cast<BindWrap&&>(bw).what_to_bind().release_arg_tuple());+  }+};++template <typename ArgMap, typename... Ts>+struct async_closure_storage {+  template <typename... StoredArgs> // no forwarding refs+  explicit async_closure_storage(capture_private_t priv, StoredArgs&&... sas)+      : inner_err_(async_closure_default_inner_err()),+        // Curly braces guarantee that in-place construction is left-to-right+        storage_tuple_{Ts{+            priv,+            async_closure_backref_populator<+                ArgMap,+                StoredArgs::arg_idx,+                decltype(sas.bindWrapper_)>{}(+                priv,+                // Here, we access `storage_tuple_` before it is constructed,+                // which emits an "uninitialized access" warning.  However,+                // this one is safe because:+                //   - lite_tuple constructs elements left-to-right+                //   - we check above that backrefs only point right-to-left+                //+                // clang-format off+                FOLLY_PUSH_WARNING+                FOLLY_GNU_DISABLE_WARNING("-Wuninitialized")+                storage_tuple_,+                FOLLY_POP_WARNING+                // clang-format on+                static_cast<StoredArgs&&>(sas)+                    .bindWrapper_)}...} {}++  // We go through getters so that `AsyncObject` can reuse closure machinery.+  // Note that we only need lvalue refs to the storage tuple, meaning that+  // returning a ref-to-a-tuple is as good as a tuple-of-refs here.+  // We return an rvalue ref for compatibility with the latter scenario.+  auto&& storage_tuple_like() { return storage_tuple_; }+  auto* inner_err_ptr() { return &inner_err_; }++  // For `bad_alloc` safety, we must create the cleanup coros before awaiting+  // the inner coro.  This preallocated exception (which is passed to the+  // cleanup coros by-reference) further enables us to create the cleanup coros+  // before we even create the outer coro.  That avoids an extra coro frame+  // that would otherwise be need to await a cleanup tuple.+  exception_wrapper inner_err_;+  lite_tuple::tuple<Ts...> storage_tuple_;+};++template <size_t StorageI, typename Bs>+decltype(auto) async_closure_bind_inner_coro_arg(+    capture_private_t priv, Bs& bs, auto& storage_ptr) {+  if constexpr (is_async_closure_outer_stored_arg<Bs>) {+    // "own": arg was already moved into `storage_ptr`.+    auto& storage_ref = get_from_storage_ptr<StorageI>(storage_ptr);+    static_assert(+        std::is_same_v<+            typename Bs::storage_type,+            std::remove_reference_t<decltype(storage_ref)>>);+    // `SharedCleanupClosure=true` preserves the `after_cleanup_ref_` prefix of+    // the storage type.+    return storage_ref.template to_capture_ref</*shared*/ true>(priv);+  } else if constexpr (+      // "own": Move stored `as_capture()` into inner coro.+      is_instantiation_of_v<async_closure_inner_stored_arg, Bs> ||+      // `scheduleSelfClosure` / `scheduleScopeClosure` self-references.+      is_instantiation_of_v<async_closure_scope_self_ref_hack, Bs>) {+    return typename Bs::storage_type{priv, std::move(bs.bindWrapper_)};+  } else if constexpr (is_any_capture<Bs>) {+    // "pass": Move `capture<Ref>` into the inner coro.+    static_assert(std::is_reference_v<typename Bs::capture_type>);+    return std::move(bs);+  } else { // "regular": Non-`capture` binding.+    static_assert(is_instantiation_of_v<async_closure_regular_arg, Bs>);+    // We don't inspect `storage_type` here -- `detail/AsyncClosureBindings.h`+    // should have ensured that `bind_info_t` was in a default, no-op state.+    return std::move(bs).bindWrapper_.what_to_bind();+  }+}++template <typename, typename T>+struct with_tag {+  T value;+};++// Eagerly construct -- but do not await -- an `async_closure`:+//   - Resolve bindings.+//   - Construct & store args for the user-supplied inner coro.+//   - For ensuring cleanup in the face of `bad_alloc`, pre-allocate the+//     outer task & `co_cleanup` tasks, if needed.+//   - Create the inner coro, passing it `capture` references, or -- if+//     there are no `co_cleanup` args and no outer coro -- quack-alike+//     owning wrappers.+//   - Marks the final user-facing task with the `safe_alias` that+//     describes the memory-safety of the closure's arguments.+//   - Returns the task inside a wrapper that statically checks the memory+//     safety of the return & `make_inner_coro` types when+//     `release_outer_coro()` is called.+//+// NB: Due to the "omit outer coro" optimization, `release_outer_coro()`+// will in some cases return a no-overhead wrapper around the coro returned+// by `make_inner_coro()`.+//+// Rationale: "Eager" is the only option matching user expectations, since+// regular coroutine args are bound eagerly too.  Implementation-wise, all+// `lang/Bindings.h` logic has to be resolved within the current statement,+// since the auxiliary reference-bearing objects aren't valid beyond that.+template <auto Cfg>+auto bind_captures_to_closure(auto&& make_inner_coro, auto safeties_and_binds) {+  auto& [arg_safeties, b_tup] = safeties_and_binds;++  using BTupIs = std::make_index_sequence<std::tuple_size_v<decltype(b_tup)>>;+  // For stored arg  @ `i`, `VtagStorageIs[i]` is a `*storage_ptr` index.+  using VtagStorageIs = decltype(lite_tuple::apply(+      [&]<typename... Bs>(Bs&...) {+        return cumsum_except_last<+            (size_t)0,+            is_async_closure_outer_stored_arg<Bs>...>;+      },+      b_tup));++  // If some arguments require outer-coro storage, construct them in-place+  // on a `unique_ptr<tuple<>>`.  Without an outer coro, this stores `nullopt`.+  //+  // Rationale: Storing on-heap allows the outer coro own the arguments,+  // while simultaneously providing stable pointers to be passed into the+  // inner coro.+  //+  // Future: With a custom coro class, it should be possible to store the+  // argument tuple ON the coro frame, saving one allocation.+  auto storage_ptr = lite_tuple::apply(+      []<typename... Entries, typename... SAs>(with_tag<Entries, SAs>... as) {+        if constexpr (sizeof...(SAs) == 0) {+          return std::nullopt; // Signals "no outer closure" to the caller+        } else {+          // (2) Construct all the storage args in-place in one tuple.+          return std::make_unique<async_closure_storage<+              async_closure_backref_map<Entries...>,+              typename SAs::storage_type...>>(+              capture_private_t{}, std::move(as).value...);+        }+      },+      // (1) Collect the args that need storage on the outer coro.+      []<size_t... ArgIs, size_t... StorageIs>(+          auto& tup, std::index_sequence<ArgIs...>, vtag_t<StorageIs...>) {+        // Future: Could support using the `self_id` backref to get a capture+        // ref to the `async_closure_scope_self_ref_hack` arg.+        return lite_tuple::tuple_cat([]<typename B>(B& b) {+          if constexpr (is_async_closure_outer_stored_arg<B>) {+            static_assert(ArgIs == B::arg_idx);+            return lite_tuple::tuple{with_tag<+                async_closure_backref_entry<B::tag, ArgIs, StorageIs>,+                B>{std::move(b)}};+          } else {+            return lite_tuple::tuple{};+          }+        }(lite_tuple::get<ArgIs>(tup))...);+      }(b_tup, BTupIs{}, VtagStorageIs{}));++  auto raw_inner_coro = lite_tuple::apply(+      [&]<typename... Bs>(Bs&... bs) {+        return [&]<size_t... ArgIs, size_t... StorageIs>(+                   std::index_sequence<ArgIs...>, vtag_t<StorageIs...>) {+          return make_inner_coro(+              // Unpack `Bs`, `ArgIs`, and `StorageIs` jointly+              [&]() -> decltype(auto) {+                if constexpr (Cfg.is_invoke_member && ArgIs == 0) {+                  // We have a `FOLLY_INVOKE_MEMBER`.  It accesses the+                  // member function via `.`, but this arg is expected to be+                  // `co_cleanup_capture<>` or `AsyncObjectPtr<>`, so we+                  // "magically" dereference it here.+                  //+                  // On safety: Below, we assert that it it made a+                  // `MemberTask<T>`, which `inner_rewrapped` will+                  // implicitly unwrap & mark with a higher safety level.+                  // `MemberTask` provides only a minimal safety+                  // attestation, namely (besides arg 1, the implicit object+                  // param), none of its args are taken by-reference.  This+                  // is fine, since for `OuterSafety`, we will have+                  // accounted for all the args' safety levels.+                  return *async_closure_bind_inner_coro_arg<StorageIs, Bs>(+                      capture_private_t{}, bs, storage_ptr);+                } else {+                  return async_closure_bind_inner_coro_arg<StorageIs, Bs>(+                      capture_private_t{}, bs, storage_ptr);+                }+              }()...);+        }(BTupIs{}, VtagStorageIs{}); // `StorageIs` indexes into `storage_ptr`+      },+      b_tup);++  // First, unwrap `AsNoexcept` so that `safe_task_traits` below can work.+  // We only allow `AsNoexcept` as the outer wrapper.+  using NoexceptWrap = as_noexcept_rewrapper<decltype(raw_inner_coro)>;+  auto unwrapped_inner = []<typename T>(T&& t) {+    if constexpr (NoexceptWrap::as_noexcept_wrapped) {+      return NoexceptWrap::unwrapTask(std::move(t));+    } else {+      return mustAwaitImmediatelyUnsafeMover(std::move(t))();+    }+  }(std::move(raw_inner_coro));++  // Compute the safety of the arguments being passed by the caller.+  constexpr safe_alias OuterSafety = Cfg.force_shared_cleanup // making NowTask+      ? safe_alias::unsafe+      : vtag_least_safe_alias(decltype(arg_safeties){});+  // Also check that the coroutine function's signature looks safe.+  constexpr safe_alias InnerSafety =+      safe_task_traits<decltype(unwrapped_inner)>::arg_safety;++  // This converts `raw_inner_task` into a "task mover" that can be plumbed+  // down to, and used by, `async_closure_outer_coro()`.  We do 3 tricks here:+  //   - Wrap all tasks into a "mover" to handle immovables like `NowTask`.+  //   - For `ClosureTask`, we'll internally LIE about its safety to let it be+  //     `co_await`ed. Per below, that's OK thanks to `async_closure_wrap_coro`.+  //   - For `SafeTask` closures with the "no outer coro" optimization, we set+  //     the inner coro's safety to `OuterSafety`, for reasons explained below.+  auto inner_mover = [&]() {+    // The first branch is always taken for safe/movable `async_closure()`+    // invocations.  For `async_now_closure()`, this branch is taken iff the+    // inner coro is a `ClosureTask` or other `SafeTask`.+    if constexpr (InnerSafety >= safe_alias::unsafe_closure_internal) {+      // In the presence of stored `capture`s, `InnerSafety` (as measured by+      // `safe_alias_of` on the inner coro) is not what we want.  That's+      // because `Captures.h` marks owned captures as `unsafe_closure_internal`+      // to discourage them being moved out of the closure.  Instead, we set+      // safety based on `vtag_safety_of_async_closure_args` (`OuterSafety`).+      //+      // `ClosureTask` cannot be `co_await`ed, so clip to `>= min_arg_safety`.+      // This is OK since `async_closure_wrap_coro` will later enforce:+      //   OuterSafety >= closure_min_arg_safety+      constexpr auto newSafety =+          std::max(OuterSafety, safe_alias::closure_min_arg_safety);+      return mustAwaitImmediatelyUnsafeMover(+          std::move(unwrapped_inner).template withNewSafety<newSafety>());+    } else { // The "new safety" rewrite doesn't apply to unsafe tasks!+      return mustAwaitImmediatelyUnsafeMover(std::move(unwrapped_inner));+    }+  }();++  using ResultT = semi_await_result_t<decltype(std::move(inner_mover)())>;++  // We require this calling convention because the `is_invoke_member`+  // branch above dereferences the 1st arg.  That is only sensible if+  // we KNOW that the arg is the implicit object parameter, which+  // would not be true e.g.  if the user passed something like this:+  //   [](int num, auto me) { return me->addNumber(num); }+  static_assert(+      std::is_same_v<MemberTask<ResultT>, decltype(unwrapped_inner)> ==+          Cfg.is_invoke_member,+      "To use `MemberTask<>` coros with `async_closure`, you must pass "+      "the callable as `FOLLY_INVOKE_MEMBER(memberName)`, and pass the "+      "instance's `capture`/`AsyncObjectPtr`/... as the first argument.");++  auto outer_mover = [&] {+    if constexpr (std::is_same_v<decltype(storage_ptr), std::nullopt_t>) {+      // No outer coro is needed, so we can return the inner one.+      static_assert(+          !has_result_after_cleanup<ResultT>,+          "Cannot `co_return *after_cleanup()` without a cleanup arg");+      return std::move(inner_mover);+    } else {+      return mustAwaitImmediatelyUnsafeMover(+          async_closure_make_outer_coro<+              /*cancelTok*/ true,+              ResultT,+              OuterSafety,+              NoexceptWrap::as_noexcept_wrapped>(+              async_closure_private_t{},+              std::move(inner_mover),+              std::move(storage_ptr)));+    }+  }();++  if constexpr (Cfg.force_shared_cleanup) {+    return NoexceptWrap::wrap_with([&]() {+      return toNowTask(std::move(outer_mover)());+    });+  } else {+    return async_closure_wrap_coro<+        OuterSafety,+        InnerSafety,+        NoexceptWrap,+        decltype(outer_mover)>{std::move(outer_mover)};+  }+}++} // namespace folly::coro::detail++FOLLY_POP_WARNING+#endif
+ folly/folly/coro/safe/detail/AsyncClosureBindings.h view
@@ -0,0 +1,773 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <compare>++#include <folly/coro/safe/Captures.h>+#include <folly/coro/safe/SafeAlias.h>++/// This header's `async_closure_safeties_and_bindings` implements the+/// argument-binding logic for `async_closure`.+///+/// Before reading further, make sure to get familiar with:+///   - `folly/lang/Bindings.md` for `bound_args` & friends.+///   - `docs/Captures.md` to understand the `capture` type wrappers, and the+///     safety upgrade/downgrade rules for passing them into closures.+/// In particular, know this distinction:+///   * "owned captures" look like `capture<V>`.  These are wrappers tied+///     to the closure whose `as_capture()` created it. Note that a closure+///     with an outer coro will pass these as `capture<V&>` to the inner task.+///   * "capture references" are `capture<V&>` or `<V&&>, implicitly created+///     for the closure from any caller-provided `capture` (value or ref).+///+/// `async_closure` takes user-specified closure arguments as `bound_args`, an+/// immovable object with 1-expression lifetime.  This header plumbs them+/// through some transformations.  In opt builds, it is intended to be elided+/// by compiler's alias analysis (NB: this needs benchmarks & perhaps tweaks).+///+/// Every variant of `async_closure` invokes `..._safeties_and_bindings()`,+/// which does several jobs.  Here's a summary of the data flow:+///   * Figure out if the closure needs an outer coro, or if it can be elided.+///   * Measure the safety of the closure, as seen by its caller.  This uses+///     the safeties of the arguments **before** any transformations.+///   * Transform the arg tuple:+///       - Convert each entry of the `bound_args` into a `capture` ref (when+///         the caller gave us a `capture`), or one of 4 tag types+///         (`async_closure*_arg` or `async_closure*_self_ref_hack`).  The tag+///         types tells the `async_closure` implementation whether to store the+///         arg, and how to bind it to the inner closure.+///       - Figure out the storage type for each `as_capture` binding using+///         `folly::bindings::binding_policy`, to support `make_in_place*`.+///       - Transform non-owned `capture`s via `to_capture_ref`.  Parents'+///         owned captures are implicitly passed by-ref; `after_cleanup_` refs+///         are "upgraded" if possible.  Docs in `Captures.md`.+///       - Apply special handling to the first argument when the closure runs+///         `FOLLY_INVOKE_MEMBER`.+///       - Other args are perfectly forwarded.+///   * Validate the user inputs and try to issue readable error messages.+///     Also check internal invariants.+///+/// After the validation & transformation, `detail/AsyncClosure.h` is+/// responsible for actually storing the args, and creating the coroutine.+///+/// ## Implementation glossary+///+/// This header classifies the bound arguments into a few categories:+///   * "owned capture" or just "own": Make a new `capture` whose storage (and+///     cleanup) belongs to this closure.+///       - These correspond to `async_closure_{inner,outer}_stored_arg`.+///       - If the closure is "shared cleanup", the safety of the new capture+///         is downgraded to `after_cleanup_ref`.+///       - The inner task sees a `capture` ref for closures with an outer+///         coro, and a `capture` value otherwise.+///   * "pass capture ref" or just "pass": A `capture` from the caller.+///       - These are passed as `capture<Ref>`, even if the input is a value.+///       - The inner coro may see upgraded safety relative to the caller.+///   * "regular arg": The easy / normal case -- simply bind a forwarding+///     reference from the caller to the inner coro.  The reference is dressed+///     in `async_closure_regular_arg`.+///   * "self-reference hack": See `async_closure_scope_self_ref_hack`.++#if FOLLY_HAS_IMMOVABLE_COROUTINES+FOLLY_PUSH_WARNING+FOLLY_DETAIL_LITE_TUPLE_ADJUST_WARNINGS++namespace folly::coro {+class AsyncObject;+class AsyncScopeSlotObject;+template <typename, size_t>+class SafeAsyncScopeContextProxy;+template <typename>+class AsyncObjectNonSlotPtr;+} // namespace folly::coro++namespace folly::coro::detail {++template <safe_alias... Vs>+constexpr safe_alias vtag_least_safe_alias(vtag_t<Vs...>) {+  return std::min({safe_alias::maybe_value, Vs...});+}++//+// There are 4 tag types here, which all quack the same interface:+//   * `storage_type`: For storing "owned captures", but also for measuring+//     "caller's point-of-view" safety of regular args.+//   * `bindWrapper_`: A `bind_wrapper_t<T>`, which is literally just `T` but+//     preserving the value category even for references.  It has either the+//     forwarding reference or the value being bound.  Note that `T` is not+//     necessarily related to `storage_type` -- for in-place construction, it+//     is a "maker", which is implicitly-convertible to the `storage_type`.+//+// There's also 5th case without a tag type -- when passing capture refs,+// `async_closure_safeties_and_bindings()` simply emits an unwrapped `capture`,+// and `AsyncClosure.h` detects it via `is_any_capture<Bs>`.+//+// The goal is that none of the 5 cases instantiate any storage, or call+// copy/move constructors until the final moment, when we either:+//   - pass the arg to the inner coro, or+//   - store it in the outer coro's `unique_ptr<tuple<>>` of owned captures.+//++// This is just a fancy forwarding reference, never a value.+template <typename Storage, typename BindWrapper>+struct async_closure_regular_arg {+  using storage_type = Storage;+  BindWrapper bindWrapper_;+};++// Use `is_base_of` since `instantiation_of` cannot handle no-type templates+class async_closure_outer_stored_arg_base {};+template <typename T>+concept is_async_closure_outer_stored_arg =+    std::is_base_of_v<async_closure_outer_stored_arg_base, T>;++// For a given closure, all `_stored_arg` tags are going to be of one flavor.+// With an outer coro, we capture info required to resolve backrefs.  Also, the+// "has outer coro" decision isn't exported in any way besides the "inner" vs+// "outer" stored arg type.  If useful, we could easily refactor this to be one+// `_owned_capture` tag type, and branch on `has_outer_coro` in+// `AsyncClosure.h`.  Reworking things this way would make it possible to have+// an outer coro without storage, which might be needed if someone has a+// legitimate use-case for captures that define `setParentCancelToken()` or+// `setParentExecutor()` without defining `co_cleanup()` (dubious!).+template <is_any_capture Storage, typename BindWrapper, size_t ArgI, auto Tag>+struct async_closure_outer_stored_arg : async_closure_outer_stored_arg_base {+  using storage_type = Storage;+  constexpr static inline size_t arg_idx = ArgI;+  constexpr static inline auto tag = Tag;+  BindWrapper bindWrapper_;+};+template <is_any_capture Storage, typename BindWrapper>+struct async_closure_inner_stored_arg {+  using storage_type = Storage;+  BindWrapper bindWrapper_;+};++// ## Why does this `self_ref_hack` type even exist?+//+// To avoid synchronization costs, `folly::coro` async scopes disallow `add()`+// after `joinAsync()` has completed (violating this is UB).  However, it is+// always safe to call `add()` on an async scope from a task running on that+// **same** scope, even if `joinAsync()` has already started. This is because+// any active scope awaitable prevents its `joinAsync()` from completing.+//+// Unfortunately, though it would be safe, you cannot directly pass an+// `async_arg_cleanup<Scope&>` ref (`shared_cleanup` safety) into a closure+// being scheduled on the SAME async scope (needs `>= co_cleanup_safe_ref`).+// That's because at compile-time, we cannot tell if the "scope being scheduled+// on" is the same as "the scope being referenced".+//+// Moreover, "pass a ref to the current scope" is the ONLY case that is always+// safe [1].  Otherwise, the closure with the scope ref could run `add()` when+// the (other) scope that it references had already been cleaned up.+//+// [1] Future: Thanks to `co_cleanup` ordering, it may later be feasible to+// allow scopes that are passed later (cleaned up earlier) to reference those+// that are passed earlier (cleaned up later).+//+// ## How does `self_ref_hack` help allow safe self-references for scopes?+//+// The solution is to introduce `scheduleScopeClosure()`, which acts just like+// `schedule(async_closure())`, but prepends an "implicit scope parameter" to+// the user-supplied `bound_args{}`.+//+// This implicit param is `self_ref_hack`, which is handled specially inside+// `detail/AsyncClosure*`.  As a result, the user's inner task gets a+// `co_cleanup_capture<Scope&>` as its first argument.+//+// Analogously, `AsyncObject::scheduleSelfClosure()` uses this mechanism to+// allow sub-closures to safely reference the `Slot` of the object, which+// contains the "current" async scope.+//+// ## Why are all aspects of `self_ref_hack` protected?+//+// This is NOT safe to use in other settings, because:+// (1) We instantiates a new `capture` ref from a bare reference, bypassing the+//     usual lifetime safety checks.+// (2) This `ref_hack` object is deliberately excluded from the final+//     `async_closure`'s safety accounting -- it only affects the+//     `shard_cleanup` downgrade.+// Both of these are ONLY okay because the sub-closure's own scope outlives it.+template <typename Storage, typename BindWrapper>+struct async_closure_scope_self_ref_hack {+  using storage_type = Storage;++ protected:+  template <size_t, typename Bs>+  friend decltype(auto) async_closure_bind_inner_coro_arg(+      capture_private_t, Bs&, auto&);+  template <typename, size_t>+  friend class folly::coro::SafeAsyncScopeContextProxy;+  friend class folly::coro::AsyncScopeSlotObject;+  // The innards are `protected`, since `transform_binding` lets this be+  // supplied from outside the closure machinery.  Any new client being+  // added here must think THOROUGHLY about the risks in the class docblock!+  explicit async_closure_scope_self_ref_hack(BindWrapper b)+      : bindWrapper_(std::move(b)) {}+  BindWrapper bindWrapper_;+};++struct binding_helper_cfg {+  bool is_shared_cleanup_closure;+  bool has_outer_coro;+  bool in_safety_measurement_pass;+  constexpr auto operator<=>(const binding_helper_cfg&) const = default;+};++template <typename Binding, auto Cfg, size_t ArgI>+class capture_binding_helper;++struct capture_ref_measurement_stub {};++// This helper class only has static members.  It exists only so that the+// various functions can share some type aliases.+template <+    std::derived_from<folly::bindings::ext::bind_info_t> auto BI,+    typename BindingType,+    auto Cfg,+    size_t ArgI>+class capture_binding_helper<+    folly::bindings::ext::binding_t<BI, BindingType>,+    Cfg,+    ArgI> {+ private:+  // A constraint on the template would make forward-declarations messy.+  static_assert(std::is_same_v<decltype(Cfg), binding_helper_cfg>);++  using category_t = folly::bindings::ext::category_t;+  using ST = typename folly::bindings::ext::binding_policy<+      folly::bindings::ext::binding_t<BI, BindingType>>::storage_type;+  using UncvrefST = std::remove_cvref_t<ST>;++  // "Pass capture ref" validation.  Here, `ST` is either a value or a+  // reference `capture`.  `RetST` is the `capture<Ref>` for the inner task.+  //+  // There is no "business logic" here.  This only documents the possible data+  // flows, `static_assert`s a few common user errors for better compiler+  // messages, and refers to the relevant unit tests.+  template <typename RetST>+  static constexpr void static_assert_passing_capture() {+    using ArgT = typename UncvrefST::capture_type;+    static_assert(std::is_reference_v<ST> == (BI.category == category_t::ref));+    if constexpr (std::is_reference_v<ArgT>) { // Is `capture<Ref>`?+      // Design note: Why do we automatically pass all `capture`s by-reference?+      // As an alternative, recall that `folly::bindings` has `const_ref` /+      // `mut_ref`.  These modifiers could be hijacked as a mandatory marking+      // for `captures` that get passed by-reference.  That might seem more+      // explicit, but also more confusing and harder to use:+      //   - `const_ref` / `mut_ref CANNOT be used for "regular" args -- they're+      //     by-reference iff the caller writes `T&` in the signature.+      //   - `capture`s are intended to belong to the parent closure, it rarely+      //     makes sense to copy or move them.+      //   - Syntactically, `capture`s behave like pointers.+      //   - If we needed an explicit `const_ref` / `mut_ref` only to pass+      //     `capture<Val>` as a `capture<Ref>`, then migrating a closure from+      //     "has outer coro" to "lacks outer coro" would require adding such a+      //     modifier at every callsite.+      //   - You can still move out the contents of a capture into a child by+      //     passing an rvalue ref as `std::move(cap)` to the child, or by+      //     passing the actual value via `*std::move(cap)`.  Similarly, `*cap`+      //     would copy the value.+      // N.B.  We DO use `as_capture{const_ref{}}` etc in order to convert+      // plain references from a parent coro into `capture` refs in a child+      // closure, see "capture-by-reference" in `Captures.md`.+      static_assert(+          !std::is_reference_v<ST>,+          "Pass `capture<Ref>` by value, do not use `const_ref` / `mut_ref`");+      // Passing the caller's `capture<Ref>` makes a new `capture` ref object.+      if constexpr (std::is_lvalue_reference_v<BindingType>) {+        // Improve errors over just "deleted copy ctor".+        static_assert(+            !std::is_rvalue_reference_v<ArgT>,+            "capture<V&&> is move-only. Try std::move(yourRef).");++        // `check_capture_lref_to_lref` tests this branch -- passing a+        // `capture<Ref>` that the caller bound as an lvalue.  We'll copy it,+        // potentially upgrading `after_cleanup_capture` -> `capture.+      } else {+        // Cleanup args don't support rval refs.  No user-facing message, since+        // `co_cleanup_capture` would first have a constraint failure.+        static_assert(!is_any_co_cleanup_capture<UncvrefST>);++        // Tests: `check_capture_lref_to_rref` & `check_capture_rref_to_rref`.+        // An input `capture<Ref>` bound as an rvalue gives the child a+        // `capture<Val&&>`.+        //+        // It is in some sense optional to support this, since users can+        // pass around lval refs and `std::move(*argRef)` at the last+        // minute.  However, I wanted to encourage the best practice of+        // `std::move(argRef)` at the outermost callsite that knows about+        // the move.  Reasons:+        //   - The initial `std::move(arg)` enables use-after-move linting+        //     in the outermost scope.+        //   - `capture<T&&>` is move-only, meaning subsequent scopes also+        //     get use-after-move linting.+        //   - Future: we could build a debug-only use-after-move runtime+        //     checker by adding some state on `capture`s.+      }+    } else { // Is `capture<Val>`?+      // Cleanup args require an outer coro, so it should never be the case+      // that `*co_cleanup_capture<Val>` is being passed to a child..+      static_assert(!is_any_co_cleanup_capture<UncvrefST>);++      // Tested in `check_capture_val_to_ref`: `capture<Val>` is implicitly+      // passed as `capture<Ref>`.+    }+  }++  template <typename T>+  static constexpr auto store_as(auto bind_wrapper) {+    if constexpr (Cfg.has_outer_coro) {+      return async_closure_outer_stored_arg<+          T,+          decltype(bind_wrapper),+          ArgI,+          folly::bindings::ext::named_bind_info_tag_v<decltype(BI)>>{+          .bindWrapper_ = std::move(bind_wrapper)};+    } else {+      return async_closure_inner_stored_arg<T, decltype(bind_wrapper)>{+          .bindWrapper_ = std::move(bind_wrapper)};+    }+  }++  // "owned capture": The closure creates storage for `as_capture()` bindings+  static constexpr auto store_capture_binding(auto bind_wrapper) {+    static_assert(+        !is_any_capture<ST>,+        "Given a capture `c`, do not write `as_capture(c)` to pass it to a "+        "closure. Just write `c` as the argument, and it'll automatically "+        "be passed as a capture reference.");+    if constexpr (has_async_closure_co_cleanup<ST>) {+      static_assert(Cfg.has_outer_coro);+      // Future: Add a toggle to emit `restricted_co_cleanup_capture`+      return store_as<co_cleanup_capture<ST>>(std::move(bind_wrapper));+    } else if constexpr (BI.captureKind_ == capture_kind::indirect) {+      if constexpr (Cfg.is_shared_cleanup_closure) {+        return store_as<after_cleanup_capture_indirect<ST>>(+            std::move(bind_wrapper));+      } else {+        return store_as<capture_indirect<ST>>(std::move(bind_wrapper));+      }+    } else if constexpr (+        !Cfg.has_outer_coro &&+        // `make_in_place*` is often used for immovable types, so without an+        // outer coro, they must be on-heap to pass ownership to the inner coro.+        folly::bindings::ext::is_binding_t_type_in_place<BindingType> &&+        // Heuristic: Moving a type is usually cheaper than putting it on+        // the heap.  If not, people can always use `capture_indirect` with+        // `unique_ptr`...  Or, we could later add new capture kinds, like+        // `plain_auto_storage = 0`, `plain_heap`, and `plain_non_heap`.+        !std::is_move_constructible_v<BindingType>) {+      if constexpr (Cfg.is_shared_cleanup_closure) {+        return store_as<after_cleanup_capture_heap<ST>>(+            std::move(bind_wrapper));+      } else {+        return store_as<capture_heap<ST>>(std::move(bind_wrapper));+      }+    } else {+      if constexpr (Cfg.is_shared_cleanup_closure) {+        return store_as<after_cleanup_capture<ST>>(std::move(bind_wrapper));+      } else {+        return store_as<capture<ST>>(std::move(bind_wrapper));+      }+    }+  }++  template <typename>+  static inline constexpr bool is_supported_capture_bind_info_v = false;++  template <>+  static inline constexpr bool+      is_supported_capture_bind_info_v<capture_bind_info_t> = true;++  // Future: Right now, we only check that `"x"_id = ` tags are unique at time+  // of use, and this only applies for stored captures.  But, from a pure "code+  // quality" point of view, it would be reasonable to demand that all tags are+  // unique, and that they are all used.  This could be done either as a linter+  // or in this file, at some compile-time cost.+  template <auto Tag>+  static inline constexpr bool is_supported_capture_bind_info_v<+      folly::bindings::ext::named_bind_info_t<Tag, capture_bind_info_t>> = true;++ public:+  // Transforms the binding as per the file docblock, returns a new binding.+  // (either one of the 4 tag types above, or `capture<Ref>`)+  static constexpr auto transform_binding(auto bind_wrapper) {+    if constexpr (is_supported_capture_bind_info_v<decltype(BI)>) {+      // Implement "capture-by-reference", docs in `Captures.md`+      if constexpr (BI.category == category_t::ref) {+        // Test in `check_parent_capture_ref`+        static_assert(std::is_reference_v<ST>);+        static_assert(+            !is_any_capture<UncvrefST>,+            "Do not use `const_ref` / `mut_ref` verbs to pass a `capture` to "+            "a child closure -- just pass it directly.");+        // It should be hard to get a ref to a co_cleanup type+        static_assert(!has_async_closure_co_cleanup<UncvrefST>);+        if constexpr (Cfg.in_safety_measurement_pass) {+          return capture_ref_measurement_stub{};+        } else if constexpr (Cfg.is_shared_cleanup_closure) {+          return after_cleanup_capture<ST>{+              capture_private_t{}, std::move(bind_wrapper)};+        } else {+          return capture<ST>{capture_private_t{}, std::move(bind_wrapper)};+        }+      } else { // Tests in `check_stored_*`.+        static_assert(!std::is_reference_v<ST>);+        return store_capture_binding(std::move(bind_wrapper));+      }+    } else { // Bindings for arguments the closure does NOT store.+      static_assert(+          std::is_same_v<+              vtag_t<BI>,+              vtag_t<folly::bindings::ext::bind_info_t{}>>,+          "`folly::bindings::` modifiers like `constant` (or `\"x\"_id = `) "+          "only make sense with `as_capture()` bindings -- for example, to "+          "move a mutable value into `const` capture storage. For regular "+          "args, use `const` in the signature of your inner coro, and/or "+          "`std::as_const` when passing the arg.");+      // If we allowed `make_in_place` without `as_capture`, the argument would+      // require a copy or a move to be passed to the inner task (which the+      // type may not support).  If `as_capture` isn't appropriate, the user+      // can also work around that via `std::make_unique<TheirType>` and/or+      // `as_capture_indirect`.+      static_assert(+          !folly::bindings::ext::is_binding_t_type_in_place<BindingType>,+          "Did you mean `capture_in_place<T>(...)`?");+      if constexpr (is_any_capture<UncvrefST>) { // Tests in `check_capture_*`+        // Pass preexisting `capture`s (NOT owned by this closure).+        // Future: Add a toggle to make `restricted_co_cleanup_capture` refs.+        auto arg_ref =+            std::move(bind_wrapper)+                .what_to_bind()+                .template to_capture_ref<Cfg.is_shared_cleanup_closure>(+                    capture_private_t{});+        static_assert_passing_capture<decltype(arg_ref)>();+        return std::move(arg_ref);+      } else if constexpr (+          is_instantiation_of_v<async_closure_scope_self_ref_hack, UncvrefST>) {+        // This `ref_hack` type quacks like the `stored_arg` types, but we need+        // to unwrap it for it to be handled correctly downstream.+        return std::move(bind_wrapper).what_to_bind();+      } else { // Test in `check_regular_args`+        // "regular" args -- neither an owned capture (`as_capture()` et al),+        // nor a parent's `capture`. Passed via forwarding reference.++        // This may be redundant, since `co_cleanup_capture` enforces that+        // cleanup types are immovable.  If we did allow passing bare+        // `co_cleanup` types, it could violate memory safety protections for+        // `async_closure`s.  For `async_now_closure`, there is also no obvious+        // use-case for passing `*captureVar` by-reference into the child.+        static_assert(+            !has_async_closure_co_cleanup<UncvrefST>,+            "This argument implements `async_closure` cleanup, so you should "+            "almost certainly pass it `as_capture()` -- or, if you already "+            "have as a reference `capture`, by-value.");++        return async_closure_regular_arg<ST, decltype(bind_wrapper)>{+            .bindWrapper_ = std::move(bind_wrapper)};+      }+    }+  }+};++// See `vtag_safety_of_async_closure_args` for the docs.+// NB: As a nested lambda, this breaks on clang-17 due to compiler bugs.+template <bool ParentViewOfSafety, typename T>+auto vtag_safety_of_async_closure_arg() {+  // "owned capture": `store_as` outputs `async_closure_*_stored_arg`.+  if constexpr (+      is_async_closure_outer_stored_arg<T> ||+      is_instantiation_of_v<async_closure_inner_stored_arg, T>) {+    using CT = typename T::storage_type::capture_type;+    static_assert(!std::is_reference_v<CT>);+    // Stored captures are as safe as the type being stored.  For example, when+    // a closure stores a `BackgroundTask<Safety, T>`, it cannot be safer than+    // `Safety`.  We don't use `safe_alias_of_v` here because `AsyncObject.h`+    // specializes `capture_safety_impl_v`.+    return vtag<capture_safety_impl_v<CT>>;+  } else if constexpr ( //+      is_instantiation_of_v<async_closure_scope_self_ref_hack, T>) {+    // This is a closure made by `spawn_self_closure()` et al. It must:+    //  - Avoid marking the closure's outer task `shared_cleanup`, so it can+    //    still be added to the scope that made it (`if` branch).+    //  - Downgrade [*] the safety of its own captures (`else` branch).+    //+    // [*] It would be memory-unsafe to reference such captures from+    // recursively scheduled closures on the same scope!+    if constexpr (ParentViewOfSafety) {+      return vtag<>;+    } else {+      constexpr auto storage_safety = safe_alias_of_v<typename T::storage_type>;+      // In current usage, ref_hack can only contain `co_cleanup_capture<V&>`.+      static_assert(storage_safety == safe_alias::shared_cleanup);+      return vtag<storage_safety>;+    }+  } else if constexpr (is_any_capture<T>) {+    // "pass capture ref": Output of the `to_capture_ref` branch.+    static_assert(std::is_reference_v<typename T::capture_type>);+    return vtag<safe_alias_of_v<T>>;+  } else if constexpr (std::is_same_v<capture_ref_measurement_stub, T>) {+    if constexpr (ParentViewOfSafety) {+      // Only allow capture-by-reference in `async_now_closure`s+      return vtag<safe_alias::unsafe>;+    } else {+      // But, don't do closure-internal downgrades, since `transform_bindings`+      // tries to prevent it from taking in refs to co_cleanup types this way.+      return vtag<>;+    }+  } else {+    // "regular arg": A non-`capture` passed via forwarding reference.+    static_assert(is_instantiation_of_v<async_closure_regular_arg, T>);+    return vtag<safe_alias_of_v<typename T::storage_type>>;+  }+}++// Returns a vtag of `safe_alias_v` for the storage type of the args that+// did not come from `store_capture_binding`.+//+// We have to special-case the stored ones because `Captures.h` marks the+// `capture` wrappers for on-closure stored values `unsafe` to discourage users+// from moving them from the original closure.  And, the wrappers themselves+// check the safety of the underlying type (via `capture_safety`).+//+// The doc in `scheduleScopeClosure()` justifies why our first call to this+// function includes `ref_hack` args in the measurement (we want the+// closure's own args downgraded to `after_cleanup_ref` safety), but not in+// the second (we don't want the emitted `SafeTask` to be knocked down to+// `shared_cleanup` safety, since that would make it unschedulable).+template <bool ParentViewOfSafety, typename TransformedBindingList>+constexpr auto vtag_safety_of_async_closure_args() {+  return []<typename... T>(tag_t<T...>) {+    return value_list_concat_t<+        vtag_t,+        decltype(vtag_safety_of_async_closure_arg<+                 ParentViewOfSafety,+                 T>())...>{};+  }(TransformedBindingList{});+}++template <typename BindingT>+constexpr bool capture_needs_outer_coro() {+  using BP = folly::bindings::ext::binding_policy<BindingT>;+  using ST = typename BP::storage_type;+  return has_async_closure_co_cleanup<ST>;+}++struct async_closure_bindings_cfg {+  bool force_outer_coro;+  bool force_shared_cleanup;+  bool is_invoke_member;+};++// For `is_invoke_member` closures, we must run an additional lifetime-safety+// check.  For convenience, we also implicitly wrap the first argument with+// `as_capture` when that's the obviously right choice.+template <async_closure_bindings_cfg Cfg>+struct async_closure_invoke_member_bindings {+  constexpr auto operator()(tag_t<>) { return tag<>; }+  template <auto BI0, typename BT0, auto... BI, typename... BT>+  constexpr auto operator()(+      tag_t<+          folly::bindings::ext::binding_t<BI0, BT0>,+          folly::bindings::ext::binding_t<BI, BT>...>) {+    using T = std::remove_cvref_t<BT0>;+    constexpr bool arg0_is_non_owning_ptr =+        // `transform_binding()` passes captures as non-owning refs+        is_any_capture<T> ||+        // Raw pointers are allowed in `async_now_closure()`+        std::is_pointer_v<T> ||+        is_instantiation_of_v<folly::coro::AsyncObjectNonSlotPtr, T> ||+        // `scheduleScopeClosure` & `scheduleSelfClosure` give non-owning+        // pointers.  NB: This covers `SlotLimitedObjectPtr`.+        is_instantiation_of_v<async_closure_scope_self_ref_hack, T>;+    // Invoking a `MemberTask` requires `force_outer_coro` iff the first arg+    // is an owning capture.+    //+    // NB: Both implicit & explicit `as_capture()`s are assumed to be owning,+    // and thus also `force_outer_coro`.+    //+    // The reason that `force_outer_coro` is NOT done automatically is that+    // it adds perf overhead, which would be easily avoided if the user made+    // their member function `static` instead.+    static_assert(+        Cfg.force_outer_coro || !Cfg.is_invoke_member || arg0_is_non_owning_ptr,+        "It looks like you want the `MemberTask` closure to own the object "+        "instance. Use `async_now_closure(bound_args{&obj}, fn)` if that "+        "applies. The next best approach is to make your task `static`, "+        "with its first arg `auto self`. If that's not viable, then use "+        "`async_closure_config{.force_outer_coro = true}` to allocate a "+        "coro frame to own your object.");+    // Syntax sugar: `as_capture()` may be left as implicit for the arg0+    // "object parameter" of `FOLLY_INVOKE_MEMBER`.+    if constexpr (+        Cfg.is_invoke_member &&+        // If arg0 is `as_capture()` or similar, don't double-wrap it.+        !std::derived_from<decltype(BI0), capture_bind_info_t> &&+        // Non-owning pointer-like things don't need to be captured.+        !arg0_is_non_owning_ptr) {+      static_assert(+          // BT0 is a value for `make_in_place`, rval ref otherwise.+          !std::is_lvalue_reference_v<BT0>,+          "If you call `async_closure` with `FOLLY_INVOKE_MEMBER` and "+          "a non-`capture` argument, then it has to be an r-value, so "+          "that the closure can take ownership of the object instance. "+          "Consider `folly::copy()` or `std::move()`.");+      return tag<+          folly::bindings::ext::+              binding_t<as_capture_bind_info<capture_kind::plain>{}(BI0), BT0>,+          folly::bindings::ext::binding_t<BI, BT>...>;+    } else {+      return tag<+          folly::bindings::ext::binding_t<BI0, BT0>,+          folly::bindings::ext::binding_t<BI, BT>...>;+    }+  }+};++// Converts forwarded arguments to bindings, figures out the storage policy+// (outer coro?, shared cleanup?), and applies `transform_bindings` to compute+// the final storage & binding outcome for each argument.  The caller should+// create an outer coro iff the resulting `tuple` contains at least one+// `async_closure_outer_stored_arg`.+//+// Returns a pair:+//   - vtag<safe_alias> computed as in `vtag_safety_of_async_closure_arg()`+//   - transformed bindings: binding | async_closure_{inner,outer}_stored_arg+//+// NB: It's fine for this implementation detail to take `BoundArgs` by-ref+// because `async_closure` & friends took them by value.+template <async_closure_bindings_cfg Cfg, typename BoundArgs>+constexpr auto async_closure_safeties_and_bindings(BoundArgs&& bargs) {+  using Bindings = decltype(async_closure_invoke_member_bindings<Cfg>{}(+      typename BoundArgs::binding_list_t{}));++  auto tup = static_cast<BoundArgs&&>(bargs).unsafe_tuple_to_bind();+  auto make_result_tuple =+      [&]<binding_helper_cfg HelperCfg>(vtag_t<HelperCfg>) {+        return [&]<size_t... Is>(std::index_sequence<Is...>) {+          return lite_tuple::tuple{[&]() {+            using Binding = type_list_element_t<Is, Bindings>;+            using T = std::tuple_element_t<Is, decltype(tup)>;+            return capture_binding_helper<Binding, HelperCfg, Is>::+                transform_binding(bind_wrapper_t<T>{+                    .t_ = static_cast<T&&>(lite_tuple::get<Is>(tup))});+          }()...};+        }(std::make_index_sequence<type_list_size_v<Bindings>>{});+      };++  // Future: If there are many `make_in_place` arguments (which require+  // `capture_heap`), it may be more efficient to auto-select an outer coro,+  // for just 2 heap allocations.  Beware: this changes user-facing types+  // (`capture_heap` to `capture`), but most users shouldn't depend on that.+  constexpr bool has_outer_coro =+      Cfg.force_outer_coro || []<typename... Bs>(tag_t<Bs...>) {+        return (capture_needs_outer_coro<Bs>() || ...);+      }(Bindings{});+  // Figure out `IsSharedCleanupClosure` for `binding_cfg` for the real+  // `transform_binding` call.+  //+  // Our choice of `is_shared_cleanup_closure = true` is important since we+  // reuse this type list for the returned+  // `vtag_safety_of_async_closure_args`.  That vtag is used by+  // `async_closure` to compute the safety level for the resulting+  // `SafeTask`.  This safety must NOT be increased by reference upgrades --+  // a reference's safety is only upgraded inside the child closure, but the+  // original safety applies in the parent closure, which is where the+  // returned `vtag` is consumed.+  //+  // Choosing `true` here does not affect the `is_shared_cleanup` choice below+  // It merely toggles between `after_cleanup_ref_capture` and `capture`, with+  // either `after_cleanup_ref` or `co_cleanup_safe_ref` safety.+  using shared_cleanup_transformed_binding_types = type_list_concat_t<+      tag_t,+      decltype(make_result_tuple(+          vtag<binding_helper_cfg{+              .is_shared_cleanup_closure = true,+              .has_outer_coro = has_outer_coro,+              .in_safety_measurement_pass = true}>))>;++  // Why do we evaluate arg safety with `ParentViewOfSafety == true` here,+  // and with `false` in the returned `vtag_safety_of_async_closure_args`?+  //+  // This toggle supports two usage scenarios:+  //+  // (1) Capture-by-reference behaviors, like `capture_const_ref()` /+  // `as_capture(const_ref())` et al.+  //    - `unsafe` for parent --  Since these are raw references from the+  //      parent's scope, ensure they're only allowed in `async_now_closure`s.+  //    - Ignored by child -- Simultaneously, we don't want the internal coro+  //      to be subject to "shared cleanup" downgrades.  Doing that would,+  //      e.g., break the useful pattern of an on-closure scope collecting+  //      results on a parent collector passed via capture-by-reference.+  //+  // (2) Closures created by `spawn_self_closure()` et al.  Also see+  // `async_closure_scope_self_ref_hack`.+  //+  //   - In the returned `vtag` that measures the parent's view of the safety+  //     of the closure, `ParentViewOfSafety == true` will exclude the+  //     closure's first arg (the scope or object ref) from the vtag -- it+  //     would otherwise be `shared_cleanup`.  That is, of course, the entire+  //     point of `spawn_self_closure()` -- we happen to know that the scope+  //     ref is safe because of the circumstances of the closure's creation.+  //+  //   - Using `ParentViewOfSafety = false` here makes `spawn_self_closure`s+  //     **internally** consider themselves to be `shared_cleanup` closures.+  //     I.e. `after_cleanup_` inputs are not upgraded, and owned captures are+  //     downgraded to `after_cleanup_`.+  //+  //     To see that these downgrades are the correct behavior, imagine a chain+  //     of closures, each calling `spawn_self_closure()` to make the next.+  //     `SafeAsyncScope` awaits these concurrently, so they must not take+  //     dependencies on each other's owned captures.+  constexpr auto internal_arg_min_safety = vtag_least_safe_alias(+      vtag_safety_of_async_closure_args<+          /*ParentViewOfSafety*/ false,+          shared_cleanup_transformed_binding_types>());++  // Compute the `after_cleanup_` downgrade/upgrade behavior for the closure.+  // Two possible scenarios:+  //  - An `async_closure` takes a `SafeTask` and emits a `SafeTask`.  Then+  //    we'll have `==` iff we got a `co_cleanup_capture` ref from a parent.+  //  - An `async_closure` taking an unconstrained task (may have by-ref+  //    args, ref captures), and emitting a `NowTask`.  In this case, the arg+  //    safety doesn't actually matter -- the caller must always+  //    `force_shared_cleanup` simply because the lambda callable might+  //    capture a `co_cleanup` ref inside it.+  static_assert(+      safe_alias::closure_min_arg_safety == safe_alias::shared_cleanup);+  constexpr bool is_shared_cleanup = Cfg.force_shared_cleanup ||+      (safe_alias::shared_cleanup >= internal_arg_min_safety);++  return lite_tuple::tuple{+      // Safety of the closure's arguments from the parent's perspective+      vtag_safety_of_async_closure_args<+          /*ParentViewOfSafety*/ true,+          shared_cleanup_transformed_binding_types>(),+      // How the child closure should store and/or bind its arguments+      make_result_tuple(+          vtag<binding_helper_cfg{+              .is_shared_cleanup_closure = is_shared_cleanup,+              .has_outer_coro = has_outer_coro,+              .in_safety_measurement_pass = false}>)};+}++} // namespace folly::coro::detail++FOLLY_POP_WARNING+#endif
+ folly/folly/coro/safe/detail/DefineMovableDeepConstLrefCopyable.h view
@@ -0,0 +1,95 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++// DELIBERATELY omits `#pragma once`++#include <type_traits>++#ifdef FOLLY_MOVABLE_AND_DEEP_CONST_LREF_COPYABLE+#error "FOLLY_MOVABLE_AND_DEEP_CONST_LREF_COPYABLE already defined"+#endif++// IMPORTANT:+//   - `#include` just before use.+//   - `#undef` after using, don't leak the macro from your header!+//+// ## Purpose+//+// This macro helps implement `capture<T>`, so `inner_type` refers to `T`.+//+// "Deep const" means that `const capture<V&>` prohibits write access to the+// underlying V.  This macro customizes constructors to close the following+// hole -- the default copy ctor discards the outer const qualifier:+//+//   const capture<V&> constRef = someCapture;+//   capture<V&> nonConstRef = constRef; // SHOULD NOT COMPILE!+//+// This is like `folly::MoveOnly`, plus some copyability if `T` is an lval ref:+//   - `YourClass<const V&>` is fully copyable+//   - For non-const `V`, you can copy from `YourClass<V&>&` but NOT from+//     `const YourClass<V&>&`.+//+// In other words, `YourClass<const int&>` is fully copyable, but+// `YourClass<int&>` is only copyable from a non-const ref.+//+// ## Usage+//+// In the below, `YourClass` should be a leaf of the inheritance hierarchy,+// in that none of its child classes should define special constructors.+//+//   #include "DefineMovableDeepConstLrefCopyable.h"+//   template <typename T> // either ref or value+//   class YourClass {+//    public:+//     FOLLY_MOVABLE_AND_DEEP_CONST_LREF_COPYABLE(YourClass, T);+//   };+//   // Don't leak the macro from your header!+//   #undef FOLLY_MOVABLE_AND_DEEP_CONST_LREF_COPYABLE+//+// ## Why is this a macro?+//+// I wish this could be a private base like `MoveOnly`, but it HAS to be a+// macro, applied on the leaf class in your inheritance hierarchy.  This+// requirement comes about because if you derive from a class with this+// macro, the C++ default-constructor machinery treats the base class as+// non-copyable, instead of the more nuanced behavior we need.  To+// understand this in detail, check out this Compiler Explorer demo (also+// see the commit message): https://godbolt.org/z/9bh4Wcso7+//+// ## Other design notes+//+//   - This lacks restrictions on a defaulted move from `const classname&&`+//     because this would only apply when copyable, and our copy constructor+//     already enforces the "deep const" behavior.+//   - The destructor is provided just to shut up an over-simple linter.+#define FOLLY_MOVABLE_AND_DEEP_CONST_LREF_COPYABLE(class_name, inner_type) \+  class_name(class_name&&) = default;                                      \+  class_name& operator=(class_name&&) = default;                           \+  class_name(class_name&)                                                  \+    requires std::is_lvalue_reference_v<inner_type>                        \+  = default;                                                               \+  class_name& operator=(class_name&)                                       \+    requires std::is_lvalue_reference_v<inner_type>                        \+  = default;                                                               \+  class_name(const class_name&)                                            \+    requires(std::is_lvalue_reference_v<inner_type> &&                     \+             std::is_const_v<std::remove_reference_t<inner_type>>)         \+  = default;                                                               \+  class_name& operator=(const class_name&)                                 \+    requires(std::is_lvalue_reference_v<inner_type> &&                     \+             std::is_const_v<std::remove_reference_t<inner_type>>)         \+  = default;                                                               \+  ~class_name() = default
+ folly/folly/crypto/Blake2xb.cpp view
@@ -0,0 +1,207 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#include <array>++#include <folly/crypto/Blake2xb.h>+#include <folly/lang/Bits.h>++namespace folly {+namespace crypto {++namespace {++// In libsodium 1.0.17, the crypto_generichash_blake2b_state struct was made+// opaque. We have to copy the internal definition of the real struct here+// so we can properly initialize it.+#if SODIUM_LIBRARY_VERSION_MAJOR > 10 || \+    (SODIUM_LIBRARY_VERSION_MAJOR == 10 && SODIUM_LIBRARY_VERSION_MINOR >= 2)+struct _blake2b_state {+  uint64_t h[8];+  uint64_t t[2];+  uint64_t f[2];+  uint8_t buf[256];+  size_t buflen;+  uint8_t last_node;+};+#define __LIBSODIUM_BLAKE2B_OPAQUE__ 1+#endif++constexpr std::array<uint64_t, 8> kBlake2bIV = {{+    0x6a09e667f3bcc908ULL,+    0xbb67ae8584caa73bULL,+    0x3c6ef372fe94f82bULL,+    0xa54ff53a5f1d36f1ULL,+    0x510e527fade682d1ULL,+    0x9b05688c2b3e6c1fULL,+    0x1f83d9abfb41bd6bULL,+    0x5be0cd19137e2179ULL,+}};++void initStateFromParams(+    crypto_generichash_blake2b_state* _state,+    const detail::Blake2xbParam& param,+    ByteRange key) {+#ifdef __LIBSODIUM_BLAKE2B_OPAQUE__+  auto state = reinterpret_cast<_blake2b_state*>(_state);+#else+  crypto_generichash_blake2b_state* state = _state;+#endif+  auto p = reinterpret_cast<const uint64_t*>(&param);+  for (int i = 0; i < 8; ++i) {+    state->h[i] = kBlake2bIV.data()[i] ^ Endian::little(p[i]);+  }+  std::memset(+      reinterpret_cast<uint8_t*>(state) + sizeof(state->h),+      0,+      sizeof(*state) - sizeof(state->h));+  if (!key.empty()) {+    if (key.size() < crypto_generichash_blake2b_KEYBYTES_MIN ||+        key.size() > crypto_generichash_blake2b_KEYBYTES_MAX) {+      throw std::runtime_error("invalid key size");+    }+    std::array<uint8_t, 128> block;+    memcpy(block.data(), key.data(), key.size());+    memset(block.data() + key.size(), 0, block.size() - key.size());+    crypto_generichash_blake2b_update(+#ifdef __LIBSODIUM_BLAKE2B_OPAQUE__+        reinterpret_cast<decltype(_state)>(state),+#else+        state,+#endif+        block.data(),+        block.size());+    sodium_memzero(block.data(), block.size()); // erase key from stack+  }+}+} // namespace++Blake2xb::Blake2xb()+    : param_{},+      state_{},+      outputLengthKnown_{false},+      initialized_{false},+      finished_{false} {+  static const int sodiumInitResult = sodium_init();+  if (sodiumInitResult == -1) {+    throw std::runtime_error("sodium_init() failed");+  }+}++Blake2xb::~Blake2xb() = default;++void Blake2xb::init(+    size_t outputLength,+    ByteRange key /* = {} */,+    ByteRange salt /* = {} */,+    ByteRange personalization /* = {}*/) {+  if (outputLength == kUnknownOutputLength) {+    outputLengthKnown_ = false;+    outputLength = kUnknownOutputLengthMagic;+  } else if (outputLength > kMaxOutputLength) {+    throw std::runtime_error("Output length too large");+  } else {+    outputLengthKnown_ = true;+  }+  std::memset(&param_, 0, sizeof(param_));+  param_.digestLength = crypto_generichash_blake2b_BYTES_MAX;+  param_.keyLength = static_cast<uint8_t>(key.size());+  param_.fanout = 1;+  param_.depth = 1;+  param_.xofLength = Endian::little(static_cast<uint32_t>(outputLength));+  if (!salt.empty()) {+    if (salt.size() != crypto_generichash_blake2b_SALTBYTES) {+      throw std::runtime_error("Invalid salt length, must be 16 bytes");+    }+    std::memcpy(param_.salt, salt.data(), sizeof(param_.salt));+  }+  if (!personalization.empty()) {+    if (personalization.size() != crypto_generichash_blake2b_PERSONALBYTES) {+      throw std::runtime_error(+          "Invalid personalization length, must be 16 bytes");+    }+    std::memcpy(+        param_.personal, personalization.data(), sizeof(param_.personal));+  }+  initStateFromParams(&state_, param_, key);+  initialized_ = true;+  finished_ = false;+}++void Blake2xb::update(ByteRange data) {+  if (!initialized_) {+    throw std::runtime_error("Must call init() before calling update()");+  } else if (finished_) {+    throw std::runtime_error("Can't call update() after finish()");+  }+  int res =+      crypto_generichash_blake2b_update(&state_, data.data(), data.size());+  if (res != 0) {+    throw std::runtime_error("crypto_generichash_blake2b_update() failed");+  }+}++void Blake2xb::finish(MutableByteRange out) {+  if (!initialized_) {+    throw std::runtime_error("Must call init() before calling finish()");+  } else if (finished_) {+    throw std::runtime_error("finish() already called");+  }++  if (outputLengthKnown_) {+    auto outLength = static_cast<uint32_t>(out.size());+    if (outLength != Endian::little(param_.xofLength)) {+      throw std::runtime_error("out.size() must equal output length");+    }+  }++  std::array<uint8_t, crypto_generichash_blake2b_BYTES_MAX> h0;+  int res = crypto_generichash_blake2b_final(&state_, h0.data(), h0.size());+  if (res != 0) {+    throw std::runtime_error("crypto_generichash_blake2b_final() failed");+  }++  param_.keyLength = 0;+  param_.fanout = 0;+  param_.depth = 0;+  param_.leafLength = Endian::little(+      static_cast<uint32_t>(crypto_generichash_blake2b_BYTES_MAX));+  param_.innerLength = crypto_generichash_blake2b_BYTES_MAX;+  size_t pos = 0;+  size_t remaining = out.size();+  while (remaining > 0) {+    param_.nodeOffset = Endian::little(+        static_cast<uint32_t>(pos / crypto_generichash_blake2b_BYTES_MAX));+    size_t len = std::min(+        static_cast<size_t>(crypto_generichash_blake2b_BYTES_MAX), remaining);+    param_.digestLength = static_cast<uint8_t>(len);+    initStateFromParams(&state_, param_, {} /* key */);+    res = crypto_generichash_blake2b_update(&state_, h0.data(), h0.size());+    if (res != 0) {+      throw std::runtime_error("crypto_generichash_blake2b_update() failed");+    }+    res = crypto_generichash_blake2b_final(&state_, out.data() + pos, len);+    if (res != 0) {+      throw std::runtime_error("crypto_generichash_blake2b_final() failed");+    }+    pos += len;+    remaining -= len;+  }+  finished_ = true;+}++} // namespace crypto+} // namespace folly
+ folly/folly/crypto/Blake2xb.h view
@@ -0,0 +1,157 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <sodium.h>++#include <folly/Range.h>++namespace folly {+namespace crypto {++namespace detail {++struct Blake2xbParam {+  uint8_t digestLength; /*  1 */+  uint8_t keyLength; /*  2 */+  uint8_t fanout; /*  3 */+  uint8_t depth; /*  4 */+  uint32_t leafLength; /*  8 */+  uint32_t nodeOffset; /* 12 */+  uint32_t xofLength; /* 16 */+  uint8_t nodeDepth; /* 17 */+  uint8_t innerLength; /* 18 */+  uint8_t reserved[14]; /* 32 */+  uint8_t salt[16]; /* 48 */+  uint8_t personal[16]; /* 64 */+};++static_assert(sizeof(Blake2xbParam) == 64, "wrong sizeof(Blake2xbParam)");++} // namespace detail++/**+ * An implementation of the BLAKE2x XOF (extendable output function)+ * hash function using BLAKE2b as the underlying hash. This hash function+ * can produce cryptographic hashes of arbitrary length (between 1 and 2^32 - 2+ * bytes) from inputs of arbitrary size. Like BLAKE2b, it can be keyed, and can+ * accept optional salt and personlization parameters.+ *+ * Note that if you need to compute hashes between 16 and 64 bytes in length,+ * you should use Blake2b instead - it's more efficient and you will have an+ * easier time interoperating with other languages, since implementations of+ * Blake2b are more common than implementations of Blake2xb. You can generate+ * a blake2b hash using the following functions from libsodium:+ * - crypto_generichash_blake2b()+ * - crypto_generichash_blake2b_salt_personal()+ */+class Blake2xb {+ public:+  /**+   * Minimum output hash size, if it is known in advance.+   */+  static constexpr size_t kMinOutputLength = 1;+  /**+   * Maximum output hash size, if it is known in advance.+   */+  static constexpr size_t kMaxOutputLength = 0xfffffffeULL;+  /**+   * If the amount of output data desired is not known in advance, use this+   * constant as the outputLength parameter to init().+   */+  static constexpr size_t kUnknownOutputLength = 0;++  /**+   * Creates a new uninitialized Blake2xb instance. The init() method must+   * be called before it can be used.+   */+  Blake2xb();++  /**+   * Shorthand for calling the no-argument constructor followed by+   * newInstance.init(outputLength, key, salt, personlization).+   */+  explicit Blake2xb(+      size_t outputLength,+      ByteRange key = {},+      ByteRange salt = {},+      ByteRange personalization = {})+      : Blake2xb() {+    init(outputLength, key, salt, personalization);+  }++  ~Blake2xb();++  /**+   * Initializes the digest object. This must be called after a new instance+   * is constructed and before update() is called. It can also be called on+   * a previously-used instance to reset its internal state and reuse it for+   * a new hash computation.+   */+  void init(+      size_t outputLength,+      ByteRange key = {},+      ByteRange salt = {},+      ByteRange personalization = {});++  /**+   * Hashes some more input data.+   */+  void update(ByteRange data);++  /**+   * Computes the final hash and stores it in the given output. The value of+   * out.size() MUST equal the outputLength parameter that was given to the+   * last init() call, except when the outputLength parameter was+   * kUnknownOutputLength.+   *+   * WARNING: never compare the results of two Blake2xb.finish() calls+   * using non-constant time comparison. The recommended way to compare+   * cryptographic hashes is with sodium_memcmp() (or some other constant-time+   * memory comparison function).+   */+  void finish(MutableByteRange out);++  /**+   * Convenience function, use this if you are hashing a single input buffer,+   * the output length is known in advance, and the output data is allocated+   * and ready to accept the hash value.+   */+  static void hash(+      MutableByteRange out,+      ByteRange data,+      ByteRange key = {},+      ByteRange salt = {},+      ByteRange personalization = {}) {+    Blake2xb d;+    d.init(out.size(), key, salt, personalization);+    d.update(data);+    d.finish(out);+  }++ private:+  static constexpr size_t kUnknownOutputLengthMagic = 0xffffffffULL;++  detail::Blake2xbParam param_;+  crypto_generichash_blake2b_state state_;+  bool outputLengthKnown_;+  bool initialized_;+  bool finished_;+};++} // namespace crypto+} // namespace folly
+ folly/folly/crypto/LtHash-inl.h view
@@ -0,0 +1,461 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#include <cstring>+#include <stdexcept>++#include <sodium.h>++#include <folly/crypto/detail/LtHashInternal.h>+#include <folly/lang/Bits.h>++namespace folly {+namespace crypto {++namespace detail {++/**+ * Implements bit twiddling operations for elements of size B bits.+ * Currently there are specializations for B = 16, B = 20, and B = 32.+ * All operations are performed on groups of elements packed into uint64_t+ * operands.+ *+ * When B == 16, each uint64_t contains 4 elements without any padding bits.+ * Both SSE2 and AVX2 have native support for adding vectors of 16-bit ints+ * so we can use those directly. When not using SSE2 or AVX2, there is some+ * minor inefficiency because the odd and even elements of each 64-bit block+ * need to be added separately, then XORed together.+ * The packed int looks like:+ *  <16 bits of data> <16 bits of data> <16 bits of data> <16 bits of data>.+ *+ * When B == 20, each uint64_t contains 3 elements with 0 padding bits at+ *   0-based positions 63, 62, 41, and 20. The packed int looks like:+ *   00 <20 bits of data> 0 <20 bits of data> 0 <20 bits of data>.+ *+ * When B == 32, each uint64_t contains 2 elements without any padding bits.+ * Both SSE2 and AVX2 have native support for adding vectors of 32-bit ints+ * so we can use those directly. When not using SSE2 or AVX2, there is some+ * minor inefficiency because the high and low elements of each 64-bit block+ * need to be added separately, then XORed together.+ * The packed int looks like:+ *  <32 bits of data> <32 bits of data>.+ */+template <std::size_t B>+struct Bits {+  static inline constexpr uint64_t kDataMask();+  static inline constexpr bool needsPadding();+};++////// Template specialization for B = 16++// static+template <>+inline constexpr uint64_t Bits<16>::kDataMask() {+  return 0xffffffffffffffffULL;+}++// static+template <>+inline constexpr bool Bits<16>::needsPadding() {+  return false;+}++////// Template specialization for B = 20++// static+template <>+inline constexpr uint64_t Bits<20>::kDataMask() {+  // In binary this mask looks like:+  // 00 <1 repeated 20 times> 0 <1 repeated 20 times> 0 <1 repeated 20 times>+  return ~0xC000020000100000ULL;+}++// static+template <>+inline constexpr bool Bits<20>::needsPadding() {+  return true;+}++////// Template specialization for B = 32++// static+template <>+inline constexpr uint64_t Bits<32>::kDataMask() {+  return 0xffffffffffffffffULL;+}++// static+template <>+inline constexpr bool Bits<32>::needsPadding() {+  return false;+}++/* static */+template <std::size_t B>+constexpr size_t getElementsPerUint64() {+  // how many elements fit into a 64-bit int? If padding is needed, assumes that+  // there is 1 padding bit between elements and any partial space is not used.+  // If padding is not needed, the computation is a trivial division.+  return detail::Bits<B>::needsPadding()+      ? ((sizeof(uint64_t) * 8) / (B + 1))+      : ((sizeof(uint64_t) * 8) / B);+}++// Compile-time computation of the checksum size for a hash with given B and N.+template <std::size_t B, std::size_t N>+constexpr size_t getChecksumSizeBytes() {+  constexpr size_t elemsPerUint64 = getElementsPerUint64<B>();+  static_assert(+      N % elemsPerUint64 == 0,+      "Invalid parameters: N %% elemsPerUint64 must be 0");+  return (N / elemsPerUint64) * sizeof(uint64_t);+}++} // namespace detail++template <std::size_t B, std::size_t N>+LtHash<B, N>::LtHash(const folly::IOBuf& initialChecksum)+    : checksum_{}, key_{folly::none} {+  static_assert(N > 999, "element count must be at least 1000");+  static_assert(+      B == 16 || B == 20 || B == 32,+      "invalid element size in bits, must be one of: [ 16, 20, 32 ]");++  // Make sure libsodium is initialized, but only do it once.+  static const int sodiumInitResult = []() { return sodium_init(); }();++  if (sodiumInitResult == -1) {+    throw std::runtime_error("sodium_init() failed");+  }++  if (initialChecksum.length() == 0) {+    checksum_ = detail::allocateCacheAlignedIOBuf(getChecksumSizeBytes());+    checksum_.append(getChecksumSizeBytes());+    reset();+  } else {+    setChecksum(initialChecksum);+  }+}++template <std::size_t B, std::size_t N>+LtHash<B, N>::LtHash(std::unique_ptr<folly::IOBuf> initialChecksum)+    : checksum_{}, key_{folly::none} {+  // Make sure libsodium is initialized, but only do it once.+  static const int sodiumInitResult = []() { return sodium_init(); }();++  if (sodiumInitResult == -1) {+    throw std::runtime_error("sodium_init() failed");+  }++  setChecksum(std::move(initialChecksum));+}++template <std::size_t B, std::size_t N>+LtHash<B, N>::LtHash(const LtHash<B, N>& that)+    : checksum_{}, key_{folly::none} {+  // Note: we don't need to initialize libsodium in the copy constructor, since+  // before a copy constructor is called, at least one object of this type must+  // be constructed without using a copy constructor, so we know that libsodium+  // must have been initialized already.+  setChecksum(that.checksum_);+  key_ = that.key_;+}++template <std::size_t B, std::size_t N>+LtHash<B, N>& LtHash<B, N>::operator=(const LtHash<B, N>& that) {+  if (checksum_.length() == that.checksum_.length()) {+    std::memcpy(+        checksum_.writableData(), that.checksum_.data(), checksum_.length());+  } else {+    // this probably means that this object was moved away from and+    // checksum_.length() is 0, so we need to allocate a new checksum_ and+    // copy the contents.+    setChecksum(that.checksum_);+  }+  key_ = that.key_;+  return *this;+}++template <std::size_t B, std::size_t N>+LtHash<B, N>::~LtHash() {+  clearKey(); // securely erase the old key if there is one+}++template <std::size_t B, std::size_t N>+void LtHash<B, N>::setKey(folly::ByteRange key) {+  if (key.size() < crypto_generichash_blake2b_KEYBYTES_MIN ||+      key.size() > crypto_generichash_blake2b_KEYBYTES_MAX) {+    throw std::runtime_error("invalid key size");+  }+  clearKey(); // securely erase the old key if there is one+  key_ = std::vector<uint8_t>{key.begin(), key.end()};+}++template <std::size_t B, std::size_t N>+void LtHash<B, N>::clearKey() {+  if (key_.has_value()) {+    sodium_memzero(key_->data(), key_->size());+    key_ = folly::none;+  }+}++template <std::size_t B, std::size_t N>+LtHash<B, N>& LtHash<B, N>::operator+=(const LtHash<B, N>& rhs) {+  if (!keysEqual(*this, rhs)) {+    throw std::runtime_error("Cannot add 2 LtHashes with different keys");+  }+  detail::MathOperation<detail::MathEngine::AUTO>::add(+      detail::Bits<B>::kDataMask(),+      B,+      {checksum_.data(), checksum_.length()},+      {rhs.checksum_.data(), rhs.checksum_.length()},+      {checksum_.writableData(), checksum_.length()});+  return *this;+}++template <std::size_t B, std::size_t N>+LtHash<B, N>& LtHash<B, N>::operator-=(const LtHash<B, N>& rhs) {+  if (!keysEqual(*this, rhs)) {+    throw std::runtime_error("Cannot subtract 2 LtHashes with different keys");+  }+  detail::MathOperation<detail::MathEngine::AUTO>::sub(+      detail::Bits<B>::kDataMask(),+      B,+      {checksum_.data(), checksum_.length()},+      {rhs.checksum_.data(), rhs.checksum_.length()},+      {checksum_.writableData(), checksum_.length()});+  return *this;+}++template <std::size_t B, std::size_t N>+bool LtHash<B, N>::operator==(const LtHash<B, N>& that) const {+  if (this == &that) { // same memory location means it's the same object+    return true;+  } else if (this->checksum_.length() != that.checksum_.length()) {+    return false;+  } else if (this->checksum_.length() == 0) {+    // both objects must have been moved away from+    return true;+  } else {+    int cmp = sodium_memcmp(+        this->checksum_.data(),+        that.checksum_.data(),+        this->checksum_.length());+    return cmp == 0;+  }+}++template <std::size_t B, std::size_t N>+bool LtHash<B, N>::checksumEquals(folly::ByteRange otherChecksum) const {+  if (otherChecksum.size() != getChecksumSizeBytes()) {+    throw std::runtime_error("Invalid checksum size");+  } else if (this->checksum_.length() != otherChecksum.size()) {+    return false;+  } else {+    int cmp = sodium_memcmp(+        this->checksum_.data(), otherChecksum.data(), this->checksum_.length());+    return cmp == 0;+  }+}++template <std::size_t B, std::size_t N>+bool LtHash<B, N>::operator!=(const LtHash<B, N>& that) const {+  return !(*this == that);+}++template <std::size_t B, std::size_t N>+void LtHash<B, N>::reset() {+  std::memset(checksum_.writableData(), 0, checksum_.length());+}++template <std::size_t B, std::size_t N>+void LtHash<B, N>::setChecksum(const folly::IOBuf& checksum) {+  if (checksum.computeChainDataLength() != getChecksumSizeBytes()) {+    throw std::runtime_error("Invalid checksum size");+  }+  folly::IOBuf checksumCopy =+      detail::allocateCacheAlignedIOBuf(getChecksumSizeBytes());+  for (auto range : checksum) {+    std::memcpy(checksumCopy.writableTail(), range.data(), range.size());+    checksumCopy.append(range.size());+  }+  if constexpr (detail::Bits<B>::needsPadding()) {+    bool isPaddedCorrectly =+        detail::MathOperation<detail::MathEngine::AUTO>::checkPaddingBits(+            detail::Bits<B>::kDataMask(),+            {checksumCopy.data(), checksumCopy.length()});+    if (!isPaddedCorrectly) {+      throw std::runtime_error("Invalid checksum has non-0 padding bits");+    }+  }+  checksum_ = std::move(checksumCopy);+}++template <std::size_t B, std::size_t N>+void LtHash<B, N>::setChecksum(std::unique_ptr<folly::IOBuf> checksum) {+  if (checksum == nullptr) {+    throw std::runtime_error("null checksum");+  }+  // If the checksum is not eligible for move, call the copy version+  if (checksum->isChained() || checksum->isShared() ||+      !detail::isCacheAlignedAddress(checksum->data())) {+    setChecksum(*checksum);+    return;+  }++  if (checksum->computeChainDataLength() != getChecksumSizeBytes()) {+    throw std::runtime_error("Invalid checksum size");+  }++  // If we get here, we know that the input is not null, shared, or chained,+  // is the proper size, and is aligned on a cache line boundary.+  // Just need to check the padding bits before taking ownership of the buffer.+  if constexpr (detail::Bits<B>::needsPadding()) {+    bool isPaddedCorrectly =+        detail::MathOperation<detail::MathEngine::AUTO>::checkPaddingBits(+            detail::Bits<B>::kDataMask(),+            {checksum->data(), checksum->length()});+    if (!isPaddedCorrectly) {+      throw std::runtime_error("Invalid checksum has non-0 padding bits");+    }+  }+  checksum_ = std::move(*checksum);+}++template <std::size_t B, std::size_t N>+template <typename... Args>+void LtHash<B, N>::hashObject(+    folly::MutableByteRange out,+    folly::ByteRange firstRange,+    Args&&... moreRanges) {+  CHECK_EQ(getChecksumSizeBytes(), out.size());+  Blake2xb digest;+  if (key_.has_value()) {+    digest.init(out.size(), folly::range(*key_));+  } else {+    digest.init(out.size());+  }+  updateDigest(digest, firstRange, std::forward<Args>(moreRanges)...);+  digest.finish(out);+  if constexpr (detail::Bits<B>::needsPadding()) {+    detail::MathOperation<detail::MathEngine::AUTO>::clearPaddingBits(+        detail::Bits<B>::kDataMask(), out);+  }+}++template <std::size_t B, std::size_t N>+template <typename... Args>+void LtHash<B, N>::updateDigest(+    Blake2xb& digest, folly::ByteRange firstRange, Args&&... moreRanges) {+  digest.update(firstRange);+  updateDigest(digest, std::forward<Args>(moreRanges)...);+}++template <std::size_t B, std::size_t N>+void LtHash<B, N>::updateDigest(Blake2xb& /* digest */) {}++template <std::size_t B, std::size_t N>+template <typename... Args>+LtHash<B, N>& LtHash<B, N>::addObject(+    folly::ByteRange firstRange, Args&&... moreRanges) {+  // hash obj and add to elements of checksum+  using H = std::array<unsigned char, getChecksumSizeBytes()>;+  alignas(detail::kCacheLineSize) H h;+  hashObject(+      {h.data(), h.size()}, firstRange, std::forward<Args>(moreRanges)...);+  detail::MathOperation<detail::MathEngine::AUTO>::add(+      detail::Bits<B>::kDataMask(),+      B,+      {checksum_.data(), checksum_.length()},+      {h.data(), h.size()},+      {checksum_.writableData(), checksum_.length()});+  return *this;+}++template <std::size_t B, std::size_t N>+template <typename... Args>+LtHash<B, N>& LtHash<B, N>::removeObject(+    folly::ByteRange firstRange, Args&&... moreRanges) {+  // hash obj and subtract from elements of checksum+  using H = std::array<unsigned char, getChecksumSizeBytes()>;+  alignas(detail::kCacheLineSize) H h;+  hashObject(+      {h.data(), h.size()}, firstRange, std::forward<Args>(moreRanges)...);+  detail::MathOperation<detail::MathEngine::AUTO>::sub(+      detail::Bits<B>::kDataMask(),+      B,+      {checksum_.data(), checksum_.length()},+      {h.data(), h.size()},+      {checksum_.writableData(), checksum_.length()});+  return *this;+}++/* static */+template <std::size_t B, std::size_t N>+constexpr size_t LtHash<B, N>::getChecksumSizeBytes() {+  return detail::getChecksumSizeBytes<B, N>();+}++/* static */+template <std::size_t B, std::size_t N>+constexpr size_t LtHash<B, N>::getElementSizeInBits() {+  return B;+}++/* static */+template <std::size_t B, std::size_t N>+constexpr size_t LtHash<B, N>::getElementsPerUint64() {+  return detail::getElementsPerUint64<B>();+}++/* static */+template <std::size_t B, std::size_t N>+constexpr size_t LtHash<B, N>::getElementCount() {+  return N;+}++/* static */+template <std::size_t B, std::size_t N>+constexpr bool LtHash<B, N>::hasPaddingBits() {+  return detail::Bits<B>::needsPadding();+}++template <std::size_t B, std::size_t N>+std::unique_ptr<folly::IOBuf> LtHash<B, N>::getChecksum() const {+  auto result = std::make_unique<folly::IOBuf>(+      detail::allocateCacheAlignedIOBuf(checksum_.length()));+  result->append(checksum_.length());+  std::memcpy(result->writableData(), checksum_.data(), checksum_.length());+  return result;+}++// static+template <std::size_t B, std::size_t N>+bool LtHash<B, N>::keysEqual(const LtHash<B, N>& h1, const LtHash<B, N>& h2) {+  if (h1.key_.has_value() != h2.key_.has_value()) {+    return false;+  }+  if (!h1.key_.has_value()) {+    return true; // both LtHashes have empty keys+  }+  if (h1.key_->size() != h2.key_->size()) {+    return false;+  }+  return sodium_memcmp(h1.key_->data(), h2.key_->data(), h1.key_->size()) == 0;+}++} // namespace crypto+} // namespace folly
+ folly/folly/crypto/LtHash.cpp view
@@ -0,0 +1,185 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#include <folly/crypto/LtHash.h>++#include <folly/CpuId.h>++#include <folly/Memory.h>++namespace folly {+namespace crypto {+namespace detail {++folly::IOBuf allocateCacheAlignedIOBuf(size_t size) {+  void* ptr = folly::aligned_malloc(size, kCacheLineSize);+  if (ptr == nullptr) {+    throw std::bad_alloc();+  }+  return folly::IOBuf(+      folly::IOBuf::TAKE_OWNERSHIP,+      ptr,+      static_cast<uint64_t>(size), // capacity+      0ULL, // initial size+      [](void* addr, void* /* userData*/) { folly::aligned_free(addr); });+}++std::unique_ptr<folly::IOBuf> allocateCacheAlignedIOBufUnique(size_t size) {+  return std::make_unique<folly::IOBuf>(allocateCacheAlignedIOBuf(size));+}++bool isCacheAlignedAddress(const void* addr) {+  auto addrValue = reinterpret_cast<size_t>(addr);+  return (addrValue & (kCacheLineSize - 1)) == 0;+}++// static+template <>+bool MathOperation<MathEngine::SIMPLE>::isAvailable() {+  return true;+}++// static+template <>+bool MathOperation<MathEngine::SSE2>::isAvailable() {+  static const bool kIsAvailable =+      CpuId().sse2() && MathOperation<MathEngine::SSE2>::isImplemented();+  return kIsAvailable;+}++// static+template <>+bool MathOperation<MathEngine::AVX2>::isAvailable() {+  static const bool kIsAvailable =+      CpuId().avx2() && MathOperation<MathEngine::AVX2>::isImplemented();+  return kIsAvailable;+}++// static+template <>+bool MathOperation<MathEngine::AUTO>::isAvailable() {+  return true;+}++// static+template <>+bool MathOperation<MathEngine::AUTO>::isImplemented() {+  return true;+}++// static+template <>+void MathOperation<MathEngine::AUTO>::add(+    uint64_t dataMask,+    size_t bitsPerElement,+    folly::ByteRange b1,+    folly::ByteRange b2,+    folly::MutableByteRange out) {+  // Note: implementation is a function pointer that is initialized to point+  // at the fastest available implementation the first time this function is+  // called.+  static auto implementation = []() {+    if (MathOperation<MathEngine::AVX2>::isAvailable()) {+      LOG(INFO) << "Selected AVX2 MathEngine for add() operation";+      return MathOperation<MathEngine::AVX2>::add;+    } else if (MathOperation<MathEngine::SSE2>::isAvailable()) {+      LOG(INFO) << "Selected SSE2 MathEngine for add() operation";+      return MathOperation<MathEngine::SSE2>::add;+    } else {+      LOG(INFO) << "Selected SIMPLE MathEngine for add() operation";+      return MathOperation<MathEngine::SIMPLE>::add;+    }+  }();+  implementation(dataMask, bitsPerElement, b1, b2, out);+}++// static+template <>+void MathOperation<MathEngine::AUTO>::sub(+    uint64_t dataMask,+    size_t bitsPerElement,+    folly::ByteRange b1,+    folly::ByteRange b2,+    folly::MutableByteRange out) {+  // Note: implementation is a function pointer that is initialized to point+  // at the fastest available implementation the first time this function is+  // called.+  static auto implementation = []() {+    if (MathOperation<MathEngine::AVX2>::isAvailable()) {+      LOG(INFO) << "Selected AVX2 MathEngine for sub() operation";+      return MathOperation<MathEngine::AVX2>::sub;+    } else if (MathOperation<MathEngine::SSE2>::isAvailable()) {+      LOG(INFO) << "Selected SSE2 MathEngine for sub() operation";+      return MathOperation<MathEngine::SSE2>::sub;+    } else {+      LOG(INFO) << "Selected SIMPLE MathEngine for sub() operation";+      return MathOperation<MathEngine::SIMPLE>::sub;+    }+  }();+  implementation(dataMask, bitsPerElement, b1, b2, out);+}++// static+template <>+void MathOperation<MathEngine::AUTO>::clearPaddingBits(+    uint64_t dataMask, folly::MutableByteRange buf) {+  // Note: implementation is a function pointer that is initialized to point+  // at the fastest available implementation the first time this function is+  // called.+  static auto implementation = []() {+    if (MathOperation<MathEngine::AVX2>::isAvailable()) {+      LOG(INFO) << "Selected AVX2 MathEngine for clearPaddingBits() operation";+      return MathOperation<MathEngine::AVX2>::clearPaddingBits;+    } else if (MathOperation<MathEngine::SSE2>::isAvailable()) {+      LOG(INFO) << "Selected SSE2 MathEngine for clearPaddingBits() operation";+      return MathOperation<MathEngine::SSE2>::clearPaddingBits;+    } else {+      LOG(INFO)+          << "Selected SIMPLE MathEngine for clearPaddingBits() operation";+      return MathOperation<MathEngine::SIMPLE>::clearPaddingBits;+    }+  }();+  implementation(dataMask, buf);+}++// static+template <>+bool MathOperation<MathEngine::AUTO>::checkPaddingBits(+    uint64_t dataMask, folly::ByteRange buf) {+  // Note: implementation is a function pointer that is initialized to point+  // at the fastest available implementation the first time this function is+  // called.+  static auto implementation = []() {+    if (MathOperation<MathEngine::AVX2>::isAvailable()) {+      LOG(INFO) << "Selected AVX2 MathEngine for checkPaddingBits() operation";+      return MathOperation<MathEngine::AVX2>::checkPaddingBits;+    } else if (MathOperation<MathEngine::SSE2>::isAvailable()) {+      LOG(INFO) << "Selected SSE2 MathEngine for checkPaddingBits() operation";+      return MathOperation<MathEngine::SSE2>::checkPaddingBits;+    } else {+      LOG(INFO)+          << "Selected SIMPLE MathEngine for checkPaddingBits() operation";+      return MathOperation<MathEngine::SIMPLE>::checkPaddingBits;+    }+  }();+  return implementation(dataMask, buf);+}++template struct MathOperation<MathEngine::AUTO>;++} // namespace detail+} // namespace crypto+} // namespace folly
+ folly/folly/crypto/LtHash.h view
@@ -0,0 +1,315 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <cstddef>+#include <memory>+#include <vector>++#include <folly/Optional.h>+#include <folly/Range.h>+#include <folly/crypto/Blake2xb.h>+#include <folly/io/IOBuf.h>++namespace folly {+namespace crypto {++namespace detail {+/**+ * Allocates an IOBuf of the given size, aligned on a cache line boundary.+ * Similar to folly::IOBuf::create(), the returned IOBuf has an initial+ * capacity == size and an initial length == 0.+ */+folly::IOBuf allocateCacheAlignedIOBuf(size_t size);++/**+ * Similar to allocateCacheAlignedIOBuf(), but returns a unique_ptr to an IOBuf+ * instead of an IOBuf.+ */+std::unique_ptr<folly::IOBuf> allocateCacheAlignedIOBufUnique(size_t size);++/**+ * Returns true if the given memory address is aligned on a cache line boundary+ * and false if it isn't.+ */+bool isCacheAlignedAddress(const void* addr);++} // namespace detail++/**+ * Templated homomorphic hash, using LtHash (lattice-based crypto).+ * Template parameters: B = element size in bits, N = number of elements.+ *+ * Current constraints (checked at compile time with static asserts):+ * (1) B must be 16, 20 or 32.+ * (2) N must be > 999.+ * (3) when B is 16, N must be divisible by 32.+ * (4) when B is 20, N must be divisible by 24.+ * (5) when B is 32, N must be divisible by 16.+ */+template <std::size_t B, std::size_t N>+class LtHash {+ public:+  explicit LtHash(const folly::IOBuf& initialChecksum = {});++  /**+   * Like the above constructor but takes ownership of the checksum buffer,+   * avoiding a copy if these conditions about the input buffer are met:+   * - initialChecksum->isChained() is false+   * - initialChecksum->isShared() is false+   * - detail::isCacheAlignedAddress(initialChecksum.data()) is true+   *+   * If you want to take advantage of this and need to make sure your IOBuf+   * address is aligned on a cache line boundary, you can use the+   * function detail::allocateCacheAlignedIOBufUnique() to do it.+   */+  explicit LtHash(std::unique_ptr<folly::IOBuf> initialChecksum);++  // Note: we explicitly implement copy constructor and copy assignment+  // operator to make sure the checksum_ IOBuf is deep-copied.+  LtHash(const LtHash<B, N>& that);+  LtHash<B, N>& operator=(const LtHash<B, N>& that);++  LtHash(LtHash<B, N>&& that) noexcept = default;+  LtHash<B, N>& operator=(LtHash<B, N>&& that) noexcept = default;+  ~LtHash();++  /**+   * Sets the secret Blake2xb key. The key will be used to hash every element+   * added with addObject() / removed with removeObject(). This can be used+   * to compute a keyed LtHash value for a set of elements, if desired.+   *+   * The key must be between 16 and 64 bytes long (inclusive) and should be+   * a cryptographic key (e.g. a random value generated by a CPRNG).+   *+   * Note that if the LtHash value is transmitted from one user to another, the+   * two users will have to securely share the secret key before the receiver+   * can verify the integrity of the LtHash value they got from the sender.+   */+  void setKey(folly::ByteRange key);++  /**+   * Unsets the secret Blake2xb key and erases the key contents from memory.+   */+  void clearKey();++  /**+   * Resets the checksum in this LtHash. This puts the hash into the same+   * state as if it was just constructed with the zero-argument constructor.+   */+  void reset();++  /**+   * IMPORTANT: Unlike regular hash, the incremental hash functions operate on+   * individual objects, not a stream of data. For example, the following+   * example codes will lead to different checksum values.+   * (1) addObject("Hello"); addObject(" World");+   * (2) addObject("Hello World");+   * because addObject() calculates hashes for the two words separately, and+   * aggregate them to update checksum.+   *+   * addObject() is commutative. LtHash generates the same checksum over a+   * given set of objects regardless of the order they were added.+   * Example: H(a + b + c) = H(b + c + a)+   *+   * addObject() can be called with multiple ByteRange parameters, in which+   * case it will behave as if it was called with a single ByteRange which+   * contained the concatenation of all the input ByteRanges. This allows+   * adding an object whose hash is computed from several non-contiguous+   * ranges of data, without having to copy the data to a contiguous+   * piece of memory.+   *+   * Example: addObject(r1, r2, r3) is equivalent to+   * addObject(r4) where r4 contains the concatenation of r1 + r2 + r3.+   */+  template <typename... Args>+  LtHash<B, N>& addObject(folly::ByteRange firstRange, Args&&... moreRanges);++  /**+   * removeObject() is the inverse function of addObject(). Note that it does+   * NOT check whether the object has been actually added to LtHash. The caller+   * should ensure that the object is valid.+   *+   * Example: H(a - a + b - b + c - c) = H(a + b + c - a - b - c) = H()+   *+   * Similar to addObject(), removeObject() can be called with more than one+   * ByteRange parameter.+   */+  template <typename... Args>+  LtHash<B, N>& removeObject(folly::ByteRange firstRange, Args&&... moreRanges);++  /**+   * Because the addObject() operation in LtHash is commutative and transitive,+   * it's possible to break down a large LtHash computation (i.e. adding 100k+   * objects) into several parallel steps each of which computes a LtHash of a+   * subset of the objects, and then add the LtHash objects together.+   * Pseudocode:+   *+   *   std::vector<std::string> objects = ...;+   *   Future<LtHash<20, 1008>> h1 = computeInBackgroundThread(+   *       &objects[0], &objects[10000]);+   *   Future<LtHash<20, 1008>> h2 = computeInBackgroundThread(+   *       &objects[10001], &objects[20000]);+   *   LtHash<20, 1008> result = h1.get() + h2.get();+   */+  LtHash<B, N>& operator+=(const LtHash<B, N>& rhs);+  friend LtHash<B, N> operator+(+      const LtHash<B, N>& lhs, const LtHash<B, N>& rhs) {+    LtHash<B, N> result = lhs;+    result += rhs;+    return result;+  }+  friend LtHash<B, N> operator+(LtHash<B, N>&& lhs, const LtHash<B, N>& rhs) {+    LtHash<B, N> result = std::move(lhs);+    result += rhs;+    return result;+  }+  friend LtHash<B, N> operator+(const LtHash<B, N>& lhs, LtHash<B, N>&& rhs) {+    // addition is commutative so we can just swap the two arguments+    return std::move(rhs) + lhs;+  }+  friend LtHash<B, N> operator+(LtHash<B, N>&& lhs, LtHash<B, N>&& rhs) {+    LtHash<B, N> result = std::move(lhs);+    result += rhs;+    return result;+  }++  /**+   * The subtraction operator is provided for symmetry, but I'm not sure if+   * anyone will ever actually use it outside of tests.+   */+  LtHash<B, N>& operator-=(const LtHash<B, N>& rhs);+  friend LtHash<B, N> operator-(+      const LtHash<B, N>& lhs, const LtHash<B, N>& rhs) {+    LtHash<B, N> result = lhs;+    result -= rhs;+    return result;+  }+  friend LtHash<B, N> operator-(LtHash<B, N>&& lhs, const LtHash<B, N>& rhs) {+    LtHash<B, N> result = std::move(lhs);+    result -= rhs;+    return result;+  }++  /**+   * Equality comparison operator, implemented in a data-independent way to+   * guard against timing attacks. Always use this to check if two LtHash+   * values are equal instead of manually comparing checksum buffers.+   */+  bool operator==(const LtHash<B, N>& that) const;++  /**+   * Equality comparison operator for checksum in ByteRange, implemented in a+   * data-independent way to guard against timing attacks.+   */+  bool checksumEquals(folly::ByteRange otherChecksum) const;++  /**+   * Inequality comparison operator.+   */+  bool operator!=(const LtHash<B, N>& that) const;++  /**+   * Sets the initial checksum value to use for processing objects in the+   * xxxObject() calls.+   */+  void setChecksum(const folly::IOBuf& checksum);++  /**+   * Like the above method but takes ownership of the checksum buffer,+   * avoiding a copy if these conditions about the input buffer are met:+   * - checksum->isChained() is false+   * - checksum->isShared() is false+   * - detail::isCacheAlignedAddress(checksum.data()) is true+   *+   * If you want to take advantage of this and need to make sure your IOBuf+   * address is aligned on a cache line boundary, you can use the+   * function detail::allocateCacheAlignedIOBufUnique() to do it.+   */+  void setChecksum(std::unique_ptr<folly::IOBuf> checksum);++  /**+   * Returns the total length of the checksum (element_count * element_length)+   */+  static constexpr size_t getChecksumSizeBytes();++  /**+   * Returns the template parameter B.+   */+  static constexpr size_t getElementSizeInBits();++  /**+   * Returns the number of elements that get packed into a single uint64_t.+   */+  static constexpr size_t getElementsPerUint64();++  /**+   * Returns the template parameter N.+   */+  static constexpr size_t getElementCount();++  /**+   * Retruns true if the internal checksum uses padding bits between elements.+   */+  static constexpr bool hasPaddingBits();++  /**+   * Returns a copy of the current checksum value+   */+  std::unique_ptr<folly::IOBuf> getChecksum() const;++ private:+  template <typename... Args>+  void hashObject(+      folly::MutableByteRange out,+      folly::ByteRange firstRange,+      Args&&... moreRanges);++  template <typename... Args>+  void updateDigest(+      Blake2xb& digest, folly::ByteRange range, Args&&... moreRanges);++  void updateDigest(Blake2xb& digest);++  static bool keysEqual(const LtHash<B, N>& h1, const LtHash<B, N>& h2);++  // current checksum+  folly::IOBuf checksum_;+  folly::Optional<std::vector<uint8_t>> key_;+};++} // namespace crypto+} // namespace folly++#include <folly/crypto/LtHash-inl.h>++namespace folly {+namespace crypto {++// This is the fastest and smallest specialization and should be+// preferred in most cases. It provides over 200 bits of security+// which should be good enough for most cases.+using LtHash16_1024 = LtHash<16, 1024>;++// These specializations are available to users who want a higher+// level of cryptographic security. They are slower and larger than+// the one above.+using LtHash20_1008 = LtHash<20, 1008>;+using LtHash32_1024 = LtHash<32, 1024>;++} // namespace crypto+} // namespace folly
+ folly/folly/crypto/detail/LtHashInternal.h view
@@ -0,0 +1,149 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <folly/Range.h>++namespace folly {+namespace crypto {+namespace detail {++// As of 2019, most (or all?) modern Intel CPUs have 64-byte L1 cache lines,+// and aligning data buffers on cache line boundaries on such CPUs+// noticeably benefits performance (up to 10% difference).+//+// If you change this, code that depends on it in MathOperation_*.cpp may+// break and could need fixing.+constexpr size_t kCacheLineSize = 64;++// Invariants about kCacheLineSize that other logic depends on: it must be+// a power of 2 and cannot be zero.+static_assert(kCacheLineSize > 0, "kCacheLineSize cannot be 0");+static_assert(+    (kCacheLineSize & (kCacheLineSize - 1)) == 0,+    "kCacheLineSize must be a power of 2");++/**+ * Defines available math engines that we can use to perform element-wise+ * modular addition or subtraction of element vectors.+ * - AUTO: pick the best available, from best to worst: AVX2, SSE2, SIMPLE+ * - SIMPLE: perform addition/subtraction using uint64_t values+ * - SSE2: perform addition/subtraction using 128-bit __m128i values.+ *   Intel only, requires SSE2 instruction support.+ * - AVX2: perform addition/subtraction using 256-bit __m256i values.+ *   Intel only, requires AVX2 instruction support.+ */+enum class MathEngine { AUTO, SIMPLE, SSE2, AVX2 };++/**+ * This actually implements the bulk addition/subtraction operations.+ */+template <MathEngine E>+struct MathOperation {+  /**+   * Returns true if the math engine E is supported by the CPU and OS and is+   * implemented.+   */+  static bool isAvailable();++  /**+   * Returns true if the math engine E is implemented.+   */+  static bool isImplemented();++  /**+   * Performs element-wise modular addition of 2 vectors of elements packed+   * into the buffers b1 and b2. Writes the output into the buffer out. The+   * output buffer may be the same as one of the input buffers. The dataMask+   * parameter should be Bits<B>::kDataMask() where B is the element size+   * in bits.+   */+  static void add(+      uint64_t dataMask,+      size_t bitsPerElement,+      ByteRange b1,+      ByteRange b2,+      MutableByteRange out);++  /**+   * Performs element-wise modular subtraction of 2 groups of elements packed+   * into the buffers b1 and b2. Note that (a - b) % M == (a + (M - b)) % M,+   *  which is how we actually implement it to avoid underflow issues. The+   * dataMask parameter should be Bits<B>::kDataMask() where B is the element+   * size in bits.+   */+  static void sub(+      uint64_t dataMask,+      size_t bitsPerElement,+      ByteRange b1,+      ByteRange b2,+      MutableByteRange out);++  /**+   * Clears the padding bits of the given buffer according to the given+   * data mask: for each uint64_t in the input buffer, all 0 bits in the+   * data mask are cleared, and all 1 bits in the data mask are preserved.+   */+  static void clearPaddingBits(uint64_t dataMask, MutableByteRange buf);++  /**+   * Returns true if the given checksum buffer contains 0 bits at the padding+   * bit positions, according to the given data mask.+   */+  static bool checkPaddingBits(uint64_t dataMask, ByteRange buf);+};++// These forward declarations of explicit template instantiations seem to be+// required to get things to compile. I tried to get things to work without it,+// but the compiler complained when I had any AVX2 types in this header, so I+// think they need to be hidden in the .cpp file for some reason.+#define FORWARD_DECLARE_EXTERN_TEMPLATE(E)                                   \+  template <>                                                                \+  bool MathOperation<E>::isAvailable();                                      \+  template <>                                                                \+  bool MathOperation<E>::isImplemented();                                    \+  template <>                                                                \+  void MathOperation<E>::add(                                                \+      uint64_t dataMask,                                                     \+      size_t bitsPerElement,                                                 \+      ByteRange b1,                                                          \+      ByteRange b2,                                                          \+      MutableByteRange out);                                                 \+  template <>                                                                \+  void MathOperation<E>::sub(                                                \+      uint64_t dataMask,                                                     \+      size_t bitsPerElement,                                                 \+      ByteRange b1,                                                          \+      ByteRange b2,                                                          \+      MutableByteRange out);                                                 \+  template <>                                                                \+  void MathOperation<E>::clearPaddingBits(                                   \+      uint64_t dataMask, MutableByteRange buf);                              \+  template <>                                                                \+  bool MathOperation<E>::checkPaddingBits(uint64_t dataMask, ByteRange buf); \+  extern template struct MathOperation<E>++FORWARD_DECLARE_EXTERN_TEMPLATE(MathEngine::AUTO);+FORWARD_DECLARE_EXTERN_TEMPLATE(MathEngine::SIMPLE);+FORWARD_DECLARE_EXTERN_TEMPLATE(MathEngine::SSE2);+FORWARD_DECLARE_EXTERN_TEMPLATE(MathEngine::AVX2);++#undef FORWARD_DECLARE_EXTERN_TEMPLATE++} // namespace detail+} // namespace crypto+} // namespace folly
+ folly/folly/crypto/detail/MathOperation_AVX2.cpp view
@@ -0,0 +1,271 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++// Implementation of the MathOperation<MathEngine::AVX2> template+// specializations.+#include <folly/crypto/detail/LtHashInternal.h>++#include <glog/logging.h>++#ifdef __AVX2__+#include <immintrin.h>+#include <sodium.h>++#include <folly/lang/Bits.h>+#endif // __AVX2__++#include <folly/Memory.h>++namespace folly {+namespace crypto {+namespace detail {++#ifdef __AVX2__++// static+template <>+bool MathOperation<MathEngine::AVX2>::isImplemented() {+  return true;+}++// static+template <>+void MathOperation<MathEngine::AVX2>::add(+    uint64_t dataMask,+    size_t bitsPerElement,+    ByteRange b1,+    ByteRange b2,+    MutableByteRange out) {+  DCHECK_EQ(b1.size(), b2.size());+  DCHECK_EQ(b1.size(), out.size());+  DCHECK_EQ(0, b1.size() % kCacheLineSize);+  static_assert(+      kCacheLineSize % sizeof(__m256i) == 0,+      "kCacheLineSize must be a multiple of sizeof(__m256i)");+  static constexpr size_t kValsPerCacheLine = kCacheLineSize / sizeof(__m256i);+  static_assert(+      kValsPerCacheLine > 0, "kCacheLineSize must be >= sizeof(__m256i)");++  // gcc issues 'ignoring attributes on template argument' warning if+  // __m256i is used below, so have to type explicitly+  alignas(kCacheLineSize) std::array<+      long long __attribute__((__vector_size__(sizeof(__m256i)))),+      kValsPerCacheLine>+      results;++  // Note: AVX2 is Intel x86_64 only which is little-endian, so we don't need+  // the Endian::little() conversions when loading or storing data.+  if (bitsPerElement == 16 || bitsPerElement == 32) {+    for (size_t pos = 0; pos < b1.size(); pos += kCacheLineSize) {+      const __m256i* v1p = reinterpret_cast<const __m256i*>(b1.data() + pos);+      const __m256i* v2p = reinterpret_cast<const __m256i*>(b2.data() + pos);+      for (size_t i = 0; i < kValsPerCacheLine; ++i) {+        __m256i v1 = _mm256_load_si256(v1p + i);+        __m256i v2 = _mm256_load_si256(v2p + i);+        if (bitsPerElement == 16) {+          results[i] = _mm256_add_epi16(v1, v2);+        } else { // bitsPerElement == 32+          results[i] = _mm256_add_epi32(v1, v2);+        }+      }+      std::memcpy(out.data() + pos, results.data(), sizeof(results));+    }+  } else {+    __m256i mask = _mm256_set1_epi64x(dataMask);+    for (size_t pos = 0; pos < b1.size(); pos += kCacheLineSize) {+      const __m256i* v1p = reinterpret_cast<const __m256i*>(b1.data() + pos);+      const __m256i* v2p = reinterpret_cast<const __m256i*>(b2.data() + pos);+      for (size_t i = 0; i < kValsPerCacheLine; ++i) {+        __m256i v1 = _mm256_load_si256(v1p + i);+        __m256i v2 = _mm256_load_si256(v2p + i);+        results[i] = _mm256_and_si256(_mm256_add_epi64(v1, v2), mask);+      }+      std::memcpy(out.data() + pos, results.data(), sizeof(results));+    }+  }+}++// static+template <>+void MathOperation<MathEngine::AVX2>::sub(+    uint64_t dataMask,+    size_t bitsPerElement,+    ByteRange b1,+    ByteRange b2,+    MutableByteRange out) {+  DCHECK_EQ(b1.size(), b2.size());+  DCHECK_EQ(b1.size(), out.size());+  DCHECK_EQ(0, b1.size() % kCacheLineSize);+  static_assert(+      kCacheLineSize % sizeof(__m256i) == 0,+      "kCacheLineSize must be a multiple of sizeof(__m256i)");+  static constexpr size_t kValsPerCacheLine = kCacheLineSize / sizeof(__m256i);+  static_assert(+      kValsPerCacheLine > 0, "kCacheLineSize must be >= sizeof(__m256i)");++  // gcc issues 'ignoring attributes on template argument' warning if+  // __m256i is used below, so have to type explicitly+  alignas(kCacheLineSize) std::array<+      long long __attribute__((__vector_size__(sizeof(__m256i)))),+      kValsPerCacheLine>+      results;++  // Note: AVX2 is Intel x86_64 only which is little-endian, so we don't need+  // the Endian::little() conversions when loading or storing data.+  if (bitsPerElement == 16 || bitsPerElement == 32) {+    for (size_t pos = 0; pos < b1.size(); pos += kCacheLineSize) {+      const __m256i* v1p = reinterpret_cast<const __m256i*>(b1.data() + pos);+      const __m256i* v2p = reinterpret_cast<const __m256i*>(b2.data() + pos);+      for (size_t i = 0; i < kValsPerCacheLine; ++i) {+        __m256i v1 = _mm256_load_si256(v1p + i);+        __m256i v2 = _mm256_load_si256(v2p + i);+        if (bitsPerElement == 16) {+          results[i] = _mm256_sub_epi16(v1, v2);+        } else { // bitsPerElement == 32+          results[i] = _mm256_sub_epi32(v1, v2);+        }+      }+      std::memcpy(out.data() + pos, results.data(), sizeof(results));+    }+  } else {+    __m256i mask = _mm256_set1_epi64x(dataMask);+    __m256i paddingMask = _mm256_set1_epi64x(~dataMask);+    for (size_t pos = 0; pos < b1.size(); pos += kCacheLineSize) {+      const __m256i* v1p = reinterpret_cast<const __m256i*>(b1.data() + pos);+      const __m256i* v2p = reinterpret_cast<const __m256i*>(b2.data() + pos);+      for (size_t i = 0; i < kValsPerCacheLine; ++i) {+        __m256i v1 = _mm256_load_si256(v1p + i);+        __m256i v2 = _mm256_load_si256(v2p + i);+        __m256i negV2 =+            _mm256_and_si256(_mm256_sub_epi64(paddingMask, v2), mask);+        results[i] = _mm256_and_si256(_mm256_add_epi64(v1, negV2), mask);+      }+      std::memcpy(out.data() + pos, results.data(), sizeof(results));+    }+  }+}++template <>+void MathOperation<MathEngine::AVX2>::clearPaddingBits(+    uint64_t dataMask, MutableByteRange buf) {+  if (dataMask == 0xffffffffffffffffULL) {+    return;+  }+  DCHECK_EQ(0, buf.size() % kCacheLineSize);+  static_assert(+      kCacheLineSize % sizeof(__m256i) == 0,+      "kCacheLineSize must be a multiple of sizeof(__m256i)");+  static constexpr size_t kValsPerCacheLine = kCacheLineSize / sizeof(__m256i);+  static_assert(+      kValsPerCacheLine > 0, "kCacheLineSize must be >= sizeof(__m256i)");+  // gcc issues 'ignoring attributes on template argument' warning if+  // __m256i is used below, so have to type explicitly+  alignas(kCacheLineSize) std::array<+      long long __attribute__((__vector_size__(sizeof(__m256i)))),+      kValsPerCacheLine>+      results;+  __m256i mask = _mm256_set1_epi64x(dataMask);+  for (size_t pos = 0; pos < buf.size(); pos += kCacheLineSize) {+    const __m256i* p = reinterpret_cast<const __m256i*>(buf.data() + pos);+    for (size_t i = 0; i < kValsPerCacheLine; ++i) {+      results[i] = _mm256_and_si256(_mm256_load_si256(p + i), mask);+    }+    std::memcpy(buf.data() + pos, results.data(), sizeof(results));+  }+}++template <>+bool MathOperation<MathEngine::AVX2>::checkPaddingBits(+    uint64_t dataMask, ByteRange buf) {+  if (dataMask == 0xffffffffffffffffULL) {+    return true;+  }+  DCHECK_EQ(0, buf.size() % sizeof(__m256i));+  __m256i paddingMask = _mm256_set1_epi64x(~dataMask);+  static const __m256i kZero = _mm256_setzero_si256();+  for (size_t pos = 0; pos < buf.size(); pos += sizeof(__m256i)) {+    __m256i val =+        _mm256_load_si256(reinterpret_cast<const __m256i*>(buf.data() + pos));+    __m256i paddingBits = _mm256_and_si256(val, paddingMask);+    if (sodium_memcmp(&paddingBits, &kZero, sizeof(kZero)) != 0) {+      return false;+    }+  }+  return true;+}++#else // !__AVX2__++// static+template <>+bool MathOperation<MathEngine::AVX2>::isImplemented() {+  return false;+}++// static+template <>+void MathOperation<MathEngine::AVX2>::add(+    uint64_t /* dataMask */,+    size_t bitsPerElement,+    ByteRange /* b1 */,+    ByteRange /* b2 */,+    MutableByteRange /* out */) {+  if (bitsPerElement != 0) { // hack to defeat [[noreturn]] compiler warning+    LOG(FATAL) << "Unimplemented function MathOperation<MathEngine::AVX2>::"+               << "add() called";+  }+}++// static+template <>+void MathOperation<MathEngine::AVX2>::sub(+    uint64_t /* dataMask */,+    size_t bitsPerElement,+    ByteRange /* b1 */,+    ByteRange /* b2 */,+    MutableByteRange /* out */) {+  if (bitsPerElement != 0) { // hack to defeat [[noreturn]] compiler warning+    LOG(FATAL) << "Unimplemented function MathOperation<MathEngine::AVX2>::"+               << "sub() called";+  }+}++template <>+void MathOperation<MathEngine::AVX2>::clearPaddingBits(+    uint64_t /* dataMask */, MutableByteRange buf) {+  if (buf.data() != nullptr) { // hack to defeat [[noreturn]] compiler warning+    LOG(FATAL) << "Unimplemented function MathOperation<MathEngine::AVX2>::"+               << "clearPaddingBits() called";+  }+}++template <>+bool MathOperation<MathEngine::AVX2>::checkPaddingBits(+    uint64_t /* dataMask */, ByteRange buf) {+  if (buf.data() != nullptr) { // hack to defeat [[noreturn]] compiler warning+    LOG(FATAL) << "Unimplemented function MathOperation<MathEngine::AVX2>::"+               << "checkPaddingBits() called";+  }+  return false;+}++#endif // __AVX2__++template struct MathOperation<MathEngine::AVX2>;++} // namespace detail+} // namespace crypto+} // namespace folly
+ folly/folly/crypto/detail/MathOperation_SSE2.cpp view
@@ -0,0 +1,271 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++// Implementation of the MathOperation<MathEngine::SSE2> template+// specializations.+#include <folly/crypto/detail/LtHashInternal.h>++#include <glog/logging.h>++#ifdef __SSE2__+#include <emmintrin.h>+#include <sodium.h>++#include <folly/lang/Bits.h>+#endif // __SSE2__++#include <folly/Memory.h>++namespace folly {+namespace crypto {+namespace detail {++#ifdef __SSE2__+// static+template <>+bool MathOperation<MathEngine::SSE2>::isImplemented() {+  return true;+}++// static+template <>+void MathOperation<MathEngine::SSE2>::add(+    uint64_t dataMask,+    size_t bitsPerElement,+    ByteRange b1,+    ByteRange b2,+    MutableByteRange out) {+  DCHECK_EQ(b1.size(), b2.size());+  DCHECK_EQ(b1.size(), out.size());+  DCHECK_EQ(0, b1.size() % kCacheLineSize);+  static_assert(+      kCacheLineSize % sizeof(__m128i) == 0,+      "kCacheLineSize must be a multiple of sizeof(__m128i)");+  static constexpr size_t kValsPerCacheLine = kCacheLineSize / sizeof(__m128i);+  static_assert(+      kValsPerCacheLine > 0, "kCacheLineSize must be >= sizeof(__m128i)");++  // gcc issues 'ignoring attributes on template argument' warning if+  // __m128i is used below, so have to type explicitly+  alignas(kCacheLineSize) std::array<+      long long __attribute__((__vector_size__(sizeof(__m128i)))),+      kValsPerCacheLine>+      results;++  // Note: SSE2 is Intel x86(_64) only which is little-endian, so we don't need+  // the Endian::little() conversions when loading or storing data.+  if (bitsPerElement == 16 || bitsPerElement == 32) {+    for (size_t pos = 0; pos < b1.size(); pos += kCacheLineSize) {+      auto v1p = reinterpret_cast<const __m128i*>(b1.data() + pos);+      auto v2p = reinterpret_cast<const __m128i*>(b2.data() + pos);+      for (size_t i = 0; i < kValsPerCacheLine; ++i) {+        __m128i v1 = _mm_load_si128(v1p + i);+        __m128i v2 = _mm_load_si128(v2p + i);+        if (bitsPerElement == 16) {+          results[i] = _mm_add_epi16(v1, v2);+        } else { // bitsPerElement == 32+          results[i] = _mm_add_epi32(v1, v2);+        }+      }+      std::memcpy(out.data() + pos, results.data(), sizeof(results));+    }+  } else {+    __m128i mask = _mm_set_epi64x(dataMask, dataMask);+    for (size_t pos = 0; pos < b1.size(); pos += kCacheLineSize) {+      auto v1p = reinterpret_cast<const __m128i*>(b1.data() + pos);+      auto v2p = reinterpret_cast<const __m128i*>(b2.data() + pos);+      for (size_t i = 0; i < kValsPerCacheLine; ++i) {+        __m128i v1 = _mm_load_si128(v1p + i);+        __m128i v2 = _mm_load_si128(v2p + i);+        results[i] = _mm_and_si128(_mm_add_epi64(v1, v2), mask);+      }+      std::memcpy(out.data() + pos, results.data(), sizeof(results));+    }+  }+}++// static+template <>+void MathOperation<MathEngine::SSE2>::sub(+    uint64_t dataMask,+    size_t bitsPerElement,+    ByteRange b1,+    ByteRange b2,+    MutableByteRange out) {+  DCHECK_EQ(b1.size(), b2.size());+  DCHECK_EQ(b1.size(), out.size());+  DCHECK_EQ(0, b1.size() % kCacheLineSize);+  static_assert(+      kCacheLineSize % sizeof(__m128i) == 0,+      "kCacheLineSize must be a multiple of sizeof(__m128i)");+  static constexpr size_t kValsPerCacheLine = kCacheLineSize / sizeof(__m128i);+  static_assert(+      kValsPerCacheLine > 0, "kCacheLineSize must be >= sizeof(__m128i)");+  // gcc issues 'ignoring attributes on template argument' warning if+  // __m128i is used below, so have to type explicitly+  alignas(kCacheLineSize) std::array<+      long long __attribute__((__vector_size__(sizeof(__m128i)))),+      kValsPerCacheLine>+      results;++  // Note: SSE2 is Intel x86(_64) only which is little-endian, so we don't need+  // the Endian::little() conversions when loading or storing data.+  if (bitsPerElement == 16 || bitsPerElement == 32) {+    for (size_t pos = 0; pos < b1.size(); pos += kCacheLineSize) {+      auto v1p = reinterpret_cast<const __m128i*>(b1.data() + pos);+      auto v2p = reinterpret_cast<const __m128i*>(b2.data() + pos);+      for (size_t i = 0; i < kValsPerCacheLine; ++i) {+        __m128i v1 = _mm_load_si128(v1p + i);+        __m128i v2 = _mm_load_si128(v2p + i);+        if (bitsPerElement == 16) {+          results[i] = _mm_sub_epi16(v1, v2);+        } else { // bitsPerElement == 32+          results[i] = _mm_sub_epi32(v1, v2);+        }+      }+      std::memcpy(out.data() + pos, results.data(), sizeof(results));+    }+  } else {+    __m128i mask = _mm_set_epi64x(dataMask, dataMask);+    __m128i paddingMask = _mm_set_epi64x(~dataMask, ~dataMask);+    for (size_t pos = 0; pos < b1.size(); pos += kCacheLineSize) {+      auto v1p = reinterpret_cast<const __m128i*>(b1.data() + pos);+      auto v2p = reinterpret_cast<const __m128i*>(b2.data() + pos);+      for (size_t i = 0; i < kValsPerCacheLine; ++i) {+        __m128i v1 = _mm_load_si128(v1p + i);+        __m128i v2 = _mm_load_si128(v2p + i);+        __m128i negV2 = _mm_and_si128(_mm_sub_epi64(paddingMask, v2), mask);+        results[i] = _mm_and_si128(_mm_add_epi64(v1, negV2), mask);+      }+      std::memcpy(out.data() + pos, results.data(), sizeof(results));+    }+  }+}++template <>+void MathOperation<MathEngine::SSE2>::clearPaddingBits(+    uint64_t dataMask, MutableByteRange buf) {+  if (dataMask == 0xffffffffffffffffULL) {+    return;+  }+  DCHECK_EQ(0, buf.size() % kCacheLineSize);+  static_assert(+      kCacheLineSize % sizeof(__m128i) == 0,+      "kCacheLineSize must be a multiple of sizeof(__m128i)");+  static constexpr size_t kValsPerCacheLine = kCacheLineSize / sizeof(__m128i);+  static_assert(+      kValsPerCacheLine > 0, "kCacheLineSize must be >= sizeof(__m128i)");++  // gcc issues 'ignoring attributes on template argument' warning if+  // __m128i is used below, so have to type explicitly+  alignas(kCacheLineSize) std::array<+      long long __attribute__((__vector_size__(sizeof(__m128i)))),+      kValsPerCacheLine>+      results;++  __m128i mask = _mm_set_epi64x(dataMask, dataMask);+  for (size_t pos = 0; pos < buf.size(); pos += kCacheLineSize) {+    auto p = reinterpret_cast<const __m128i*>(buf.data() + pos);+    for (size_t i = 0; i < kValsPerCacheLine; ++i) {+      results[i] = _mm_and_si128(_mm_load_si128(p + i), mask);+    }+    std::memcpy(buf.data() + pos, results.data(), sizeof(results));+  }+}++template <>+bool MathOperation<MathEngine::SSE2>::checkPaddingBits(+    uint64_t dataMask, ByteRange buf) {+  if (dataMask == 0xffffffffffffffffULL) {+    return true;+  }+  DCHECK_EQ(0, buf.size() % sizeof(__m128i));+  __m128i paddingMask = _mm_set_epi64x(~dataMask, ~dataMask);+  static const __m128i kZero = _mm_setzero_si128();+  for (size_t pos = 0; pos < buf.size(); pos += sizeof(__m128i)) {+    __m128i val =+        _mm_load_si128(reinterpret_cast<const __m128i*>(buf.data() + pos));+    __m128i paddingBits = _mm_and_si128(val, paddingMask);+    if (sodium_memcmp(&paddingBits, &kZero, sizeof(kZero)) != 0) {+      return false;+    }+  }+  return true;+}++#else // !__SSE2__++// static+template <>+bool MathOperation<MathEngine::SSE2>::isImplemented() {+  return false;+}++// static+template <>+void MathOperation<MathEngine::SSE2>::add(+    uint64_t /* dataMask */,+    size_t bitsPerElement,+    ByteRange /* b1 */,+    ByteRange /* b2 */,+    MutableByteRange /* out */) {+  if (bitsPerElement != 0) { // hack to defeat [[noreturn]] compiler warning+    LOG(FATAL) << "Unimplemented function MathOperation<MathEngine::SSE2>::"+               << "add() called";+  }+}++// static+template <>+void MathOperation<MathEngine::SSE2>::sub(+    uint64_t /* dataMask */,+    size_t bitsPerElement,+    ByteRange /* b1 */,+    ByteRange /* b2 */,+    MutableByteRange /* out */) {+  if (bitsPerElement != 0) { // hack to defeat [[noreturn]] compiler warning+    LOG(FATAL) << "Unimplemented function MathOperation<MathEngine::SSE2>::"+               << "sub() called";+  }+}++template <>+void MathOperation<MathEngine::SSE2>::clearPaddingBits(+    uint64_t /* dataMask */, MutableByteRange buf) {+  if (buf.data() != nullptr) { // hack to defeat [[noreturn]] compiler warning+    LOG(FATAL) << "Unimplemented function MathOperation<MathEngine::SSE2>::"+               << "clearPaddingBits() called";+  }+  return; // not reached+}++template <>+bool MathOperation<MathEngine::SSE2>::checkPaddingBits(+    uint64_t /* dataMask */, ByteRange buf) {+  if (buf.data() != nullptr) { // hack to defeat [[noreturn]] compiler warning+    LOG(FATAL) << "Unimplemented function MathOperation<MathEngine::SSE2>::"+               << "checkPaddingBits() called";+  }+  return false;+}++#endif // __SSE2__++template struct MathOperation<MathEngine::SSE2>;++} // namespace detail+} // namespace crypto+} // namespace folly
+ folly/folly/crypto/detail/MathOperation_Simple.cpp view
@@ -0,0 +1,213 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++// Implementation of the MathOperation<MathEngine::SIMPLE> template+// specializations.+#include <folly/crypto/detail/LtHashInternal.h>++#include <glog/logging.h>++#include <folly/Memory.h>+#include <folly/lang/Bits.h>++namespace folly {+namespace crypto {+namespace detail {++// static+template <>+bool MathOperation<MathEngine::SIMPLE>::isImplemented() {+  return true;+}++// static+template <>+void MathOperation<MathEngine::SIMPLE>::add(+    uint64_t dataMask,+    size_t bitsPerElement,+    ByteRange b1,+    ByteRange b2,+    MutableByteRange out) {+  DCHECK_EQ(b1.size(), b2.size());+  DCHECK_EQ(b1.size(), out.size());+  DCHECK_EQ(0, b1.size() % kCacheLineSize);+  static_assert(+      kCacheLineSize % sizeof(uint64_t) == 0,+      "kCacheLineSize must be a multiple of sizeof(uint64_t)");+  static constexpr size_t kValsPerCacheLine = kCacheLineSize / sizeof(uint64_t);+  static_assert(+      kValsPerCacheLine > 0, "kCacheLineSize must be >= sizeof(uint64_t)");+  alignas(kCacheLineSize) std::array<uint64_t, kValsPerCacheLine> results;++  if (bitsPerElement == 16 || bitsPerElement == 32) {+    // When bitsPerElement is 16:+    // There are no padding bits, 4x 16-bit values fit exactly into a uint64_t:+    // uint64_t U = [ uint16_t W, uint16_t X, uint16_t Y, uint16_t Z ].+    // We break them up into A and B groups, with each group containing+    // alternating elements, such that A | B = the original number:+    // uint64_t A = [ uint16_t W,          0, uint16_t Y,          0 ]+    // uint64_t B = [          0, uint16_t X,          0, uint16_t Z ]+    // Then we add the A group and B group independently, and bitwise-OR+    // the results.+    // When bitsPerElement is 32:+    // There are no padding bits, 2x 32-bit values fit exactly into a uint64_t.+    // We independently add the high and low halves and then XOR them together.+    const uint64_t kMaskA =+        bitsPerElement == 16 ? 0xffff0000ffff0000ULL : 0xffffffff00000000ULL;+    const uint64_t kMaskB = ~kMaskA;+    for (size_t pos = 0; pos < b1.size(); pos += kCacheLineSize) {+      auto v1p = reinterpret_cast<const uint64_t*>(b1.data() + pos);+      auto v2p = reinterpret_cast<const uint64_t*>(b2.data() + pos);+      for (size_t i = 0; i < kValsPerCacheLine; ++i) {+        uint64_t v1 = Endian::little(*(v1p + i));+        uint64_t v2 = Endian::little(*(v2p + i));+        uint64_t v1a = v1 & kMaskA;+        uint64_t v1b = v1 & kMaskB;+        uint64_t v2a = v2 & kMaskA;+        uint64_t v2b = v2 & kMaskB;+        uint64_t v3a = (v1a + v2a) & kMaskA;+        uint64_t v3b = (v1b + v2b) & kMaskB;+        results[i] = Endian::little(v3a | v3b);+      }+      std::memcpy(out.data() + pos, results.data(), sizeof(results));+    }+  } else {+    for (size_t pos = 0; pos < b1.size(); pos += kCacheLineSize) {+      auto v1p = reinterpret_cast<const uint64_t*>(b1.data() + pos);+      auto v2p = reinterpret_cast<const uint64_t*>(b2.data() + pos);+      for (size_t i = 0; i < kValsPerCacheLine; ++i) {+        uint64_t v1 = Endian::little(*(v1p + i));+        uint64_t v2 = Endian::little(*(v2p + i));+        results[i] = Endian::little((v1 + v2) & dataMask);+      }+      std::memcpy(out.data() + pos, results.data(), sizeof(results));+    }+  }+}++// static+template <>+void MathOperation<MathEngine::SIMPLE>::sub(+    uint64_t dataMask,+    size_t bitsPerElement,+    ByteRange b1,+    ByteRange b2,+    MutableByteRange out) {+  DCHECK_EQ(b1.size(), b2.size());+  DCHECK_EQ(b1.size(), out.size());+  DCHECK_EQ(0, b1.size() % kCacheLineSize);+  static_assert(+      kCacheLineSize % sizeof(uint64_t) == 0,+      "kCacheLineSize must be a multiple of sizeof(uint64_t)");+  static constexpr size_t kValsPerCacheLine = kCacheLineSize / sizeof(uint64_t);+  static_assert(+      kValsPerCacheLine > 0, "kCacheLineSize must be >= sizeof(uint64_t)");+  alignas(kCacheLineSize) std::array<uint64_t, kValsPerCacheLine> results;++  if (bitsPerElement == 16 || bitsPerElement == 32) {+    // When bitsPerElement is 16:+    // There are no padding bits, 4x 16-bit values fit exactly into a uint64_t:+    // uint64_t U = [ uint16_t W, uint16_t X, uint16_t Y, uint16_t Z ].+    // We break them up into A and B groups, with each group containing+    // alternating elements, such that A | B = the original number:+    // uint64_t A = [ uint16_t W,          0, uint16_t Y,          0 ]+    // uint64_t B = [          0, uint16_t X,          0, uint16_t Z ]+    // Then we add the A group and B group independently, and bitwise-OR+    // the results.+    // When bitsPerElement is 32:+    // There are no padding bits, 2x 32-bit values fit exactly into a uint64_t.+    // We independently add the high and low halves and then XOR them together.+    const uint64_t kMaskA =+        bitsPerElement == 16 ? 0xffff0000ffff0000ULL : 0xffffffff00000000ULL;+    const uint64_t kMaskB = ~kMaskA;+    for (size_t pos = 0; pos < b1.size(); pos += kCacheLineSize) {+      auto v1p = reinterpret_cast<const uint64_t*>(b1.data() + pos);+      auto v2p = reinterpret_cast<const uint64_t*>(b2.data() + pos);+      for (size_t i = 0; i < kValsPerCacheLine; ++i) {+        uint64_t v1 = Endian::little(*(v1p + i));+        uint64_t v2 = Endian::little(*(v2p + i));+        uint64_t v1a = v1 & kMaskA;+        uint64_t v1b = v1 & kMaskB;+        uint64_t v2a = v2 & kMaskA;+        uint64_t v2b = v2 & kMaskB;+        uint64_t v3a = (v1a + (kMaskB - v2a)) & kMaskA;+        uint64_t v3b = (v1b + (kMaskA - v2b)) & kMaskB;+        results[i] = Endian::little(v3a | v3b);+      }+      std::memcpy(out.data() + pos, results.data(), sizeof(results));+    }+  } else {+    for (size_t pos = 0; pos < b1.size(); pos += kCacheLineSize) {+      auto v1p = reinterpret_cast<const uint64_t*>(b1.data() + pos);+      auto v2p = reinterpret_cast<const uint64_t*>(b2.data() + pos);+      for (size_t i = 0; i < kValsPerCacheLine; ++i) {+        uint64_t v1 = Endian::little(*(v1p + i));+        uint64_t v2 = Endian::little(*(v2p + i));+        results[i] =+            Endian::little((v1 + ((~dataMask - v2) & dataMask)) & dataMask);+      }+      std::memcpy(out.data() + pos, results.data(), sizeof(results));+    }+  }+}++template <>+void MathOperation<MathEngine::SIMPLE>::clearPaddingBits(+    uint64_t dataMask, MutableByteRange buf) {+  if (dataMask == 0xffffffffffffffffULL) {+    return;+  }++  DCHECK_EQ(0, buf.size() % kCacheLineSize);+  static_assert(+      kCacheLineSize % sizeof(uint64_t) == 0,+      "kCacheLineSize must be a multiple of sizeof(uint64_t)");+  static constexpr size_t kValsPerCacheLine = kCacheLineSize / sizeof(uint64_t);+  static_assert(+      kValsPerCacheLine > 0, "kCacheLineSize must be >= sizeof(uint64_t)");+  alignas(kCacheLineSize) std::array<uint64_t, kValsPerCacheLine> results;+  for (size_t pos = 0; pos < buf.size(); pos += kCacheLineSize) {+    auto p = reinterpret_cast<const uint64_t*>(buf.data() + pos);+    for (size_t i = 0; i < kValsPerCacheLine; ++i) {+      results[i] = Endian::little(Endian::little(*(p + i)) & dataMask);+    }+    std::memcpy(buf.data() + pos, results.data(), sizeof(results));+  }+}++template <>+bool MathOperation<MathEngine::SIMPLE>::checkPaddingBits(+    uint64_t dataMask, ByteRange buf) {+  if (dataMask == 0xffffffffffffffffULL) {+    return true;+  }++  DCHECK_EQ(0, buf.size() % sizeof(uint64_t));+  for (size_t pos = 0; pos < buf.size(); pos += sizeof(uint64_t)) {+    uint64_t val =+        Endian::little(*reinterpret_cast<const uint64_t*>(buf.data() + pos));+    if ((val & ~dataMask) != 0ULL) {+      return false;+    }+  }+  return true;+}++template struct MathOperation<MathEngine::SIMPLE>;++} // namespace detail+} // namespace crypto+} // namespace folly
+ folly/folly/debugging/exception_tracer/Compatibility.h view
@@ -0,0 +1,38 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++#include <version>++#if (defined(__GLIBCXX__) || defined(_LIBCPP_VERSION))+#define FOLLY_DETAIL_EXN_TRACER_CXX_STDLIB_COMPATIBLE 1+#else+#define FOLLY_DETAIL_EXN_TRACER_CXX_STDLIB_COMPATIBLE 0+#endif // (defined(__GLIBCXX__) || defined(_LIBCPP_VERSION))++#if (!defined(__FreeBSD__) && !_WIN32)+#define FOLLY_DETAIL_EXN_TRACER_OS_CXX_ABI_COMPATIBLE 1+#else+#define FOLLY_DETAIL_EXN_TRACER_OS_CXX_ABI_COMPATIBLE 0+#endif // (!defined(__FreeBSD__) && ! _WIN32)++#if FOLLY_DETAIL_EXN_TRACER_CXX_STDLIB_COMPATIBLE && \+    FOLLY_DETAIL_EXN_TRACER_OS_CXX_ABI_COMPATIBLE+#define FOLLY_HAS_EXCEPTION_TRACER 1+#else+#define FOLLY_HAS_EXCEPTION_TRACER 0+#endif // CXX_STDLIB_COMPATIBLE && OS_CXX_ABI_COMPATIBLE
+ folly/folly/debugging/exception_tracer/ExceptionAbi.h view
@@ -0,0 +1,80 @@+/*+ * Copyright (c) Meta Platforms, Inc. and affiliates.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ *     http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#pragma once++// A clone of the relevant parts of unwind-cxx.h from libstdc+++// The layout of these structures is defined by the ABI.++#include <exception>+#include <typeinfo>++#include <folly/debugging/exception_tracer/Compatibility.h>++#if FOLLY_HAS_EXCEPTION_TRACER++#include <unwind.h>++namespace __cxxabiv1 {++struct __cxa_exception {+// Unlike other implementations, GCC's libsupc++ doesn't have this code.+// See gcc/libstdc++-v3/libsupc++/unwind-cxx.h+#if !defined(__GLIBCXX__)+#if defined(__LP64__) || defined(_WIN64) || defined(_LIBCXXABI_ARM_EHABI)+  // Now _Unwind_Exception is marked with __attribute__((aligned)),+  // which implies __cxa_exception is also aligned. Insert padding+  // in the beginning of the struct, rather than before unwindHeader.+  void* reserve;++  // This is a new field to support C++11 exception_ptr.+  // For binary compatibility it is at the start of this+  // struct which is prepended to the object thrown in+  // __cxa_allocate_exception.+  size_t referenceCount;+#endif // defined(__LP64__) || defined(_WIN64) || defined(_LIBCXXABI_ARM_EHABI)+#endif // !defined(__GLIBCXX__)++  std::type_info* exceptionType;+  void (*exceptionDestructor)(void*);+  void (*unexpectedHandler)(); // std::unexpected_handler has been removed from+                               // C++17.+  std::terminate_handler terminateHandler;+  __cxa_exception* nextException;++  int handlerCount;+  int handlerSwitchValue;+  const char* actionRecord;+  const char* languageSpecificData;+  void* catchTemp;+  void* adjustedPtr;++  _Unwind_Exception unwindHeader;+};++struct __cxa_eh_globals {+  __cxa_exception* caughtExceptions;+  unsigned int uncaughtExceptions;+};++extern "C" {+__cxa_eh_globals* __cxa_get_globals(void) noexcept;+__cxa_eh_globals* __cxa_get_globals_fast(void) noexcept;+}++} // namespace __cxxabiv1++#endif // FOLLY_HAS_EXCEPTION_TRACER
+ folly/folly/debugging/exception_tracer/ExceptionCounterLib.cpp view

file too large to diff

+ folly/folly/debugging/exception_tracer/ExceptionCounterLib.h view

file too large to diff

+ folly/folly/debugging/exception_tracer/ExceptionStackTraceLib.cpp view

file too large to diff

+ folly/folly/debugging/exception_tracer/ExceptionTracer.cpp view

file too large to diff

+ folly/folly/debugging/exception_tracer/ExceptionTracer.h view

file too large to diff

+ folly/folly/debugging/exception_tracer/ExceptionTracerLib.cpp view

file too large to diff

+ folly/folly/debugging/exception_tracer/ExceptionTracerLib.h view

file too large to diff

+ folly/folly/debugging/exception_tracer/SmartExceptionStackTraceHooks.cpp view

file too large to diff

+ folly/folly/debugging/exception_tracer/SmartExceptionTracer.cpp view

file too large to diff

+ folly/folly/debugging/exception_tracer/SmartExceptionTracer.h view

file too large to diff

+ folly/folly/debugging/exception_tracer/SmartExceptionTracerSingleton.cpp view

file too large to diff

+ folly/folly/debugging/exception_tracer/SmartExceptionTracerSingleton.h view

file too large to diff

+ folly/folly/debugging/exception_tracer/StackTrace.cpp view

file too large to diff

+ folly/folly/debugging/exception_tracer/StackTrace.h view

file too large to diff

+ folly/folly/debugging/symbolizer/Dwarf.cpp view

file too large to diff

+ folly/folly/debugging/symbolizer/Dwarf.h view

file too large to diff

+ folly/folly/debugging/symbolizer/DwarfImpl.cpp view

file too large to diff

+ folly/folly/debugging/symbolizer/DwarfImpl.h view

file too large to diff

+ folly/folly/debugging/symbolizer/DwarfLineNumberVM.cpp view

file too large to diff

+ folly/folly/debugging/symbolizer/DwarfLineNumberVM.h view

file too large to diff

+ folly/folly/debugging/symbolizer/DwarfSection.cpp view

file too large to diff

+ folly/folly/debugging/symbolizer/DwarfSection.h view

file too large to diff

+ folly/folly/debugging/symbolizer/DwarfUtil.cpp view

file too large to diff

+ folly/folly/debugging/symbolizer/DwarfUtil.h view

file too large to diff

+ folly/folly/debugging/symbolizer/Elf-inl.h view

file too large to diff

+ folly/folly/debugging/symbolizer/Elf.cpp view

file too large to diff

+ folly/folly/debugging/symbolizer/Elf.h view

file too large to diff

+ folly/folly/debugging/symbolizer/ElfCache.cpp view

file too large to diff

+ folly/folly/debugging/symbolizer/ElfCache.h view

file too large to diff

+ folly/folly/debugging/symbolizer/LineReader.cpp view

file too large to diff

+ folly/folly/debugging/symbolizer/LineReader.h view

file too large to diff

+ folly/folly/debugging/symbolizer/SignalHandler.cpp view

file too large to diff

+ folly/folly/debugging/symbolizer/SignalHandler.h view

file too large to diff

+ folly/folly/debugging/symbolizer/StackTrace.cpp view

file too large to diff

+ folly/folly/debugging/symbolizer/StackTrace.h view

file too large to diff

+ folly/folly/debugging/symbolizer/SymbolizePrinter.cpp view

file too large to diff

+ folly/folly/debugging/symbolizer/SymbolizePrinter.h view

file too large to diff

+ folly/folly/debugging/symbolizer/SymbolizedFrame.cpp view

file too large to diff

+ folly/folly/debugging/symbolizer/SymbolizedFrame.h view

file too large to diff

+ folly/folly/debugging/symbolizer/Symbolizer.cpp view

file too large to diff

+ folly/folly/debugging/symbolizer/Symbolizer.h view

file too large to diff

+ folly/folly/debugging/symbolizer/detail/Debug.cpp view

file too large to diff

+ folly/folly/debugging/symbolizer/detail/Debug.h view

file too large to diff

+ folly/folly/detail/AsyncTrace.cpp view

file too large to diff

+ folly/folly/detail/AsyncTrace.h view

file too large to diff

+ folly/folly/detail/AtomicHashUtils.h view

file too large to diff

+ folly/folly/detail/AtomicUnorderedMapUtils.h view

file too large to diff

+ folly/folly/detail/DiscriminatedPtrDetail.h view

file too large to diff

+ folly/folly/detail/FileUtilDetail.cpp view

file too large to diff

+ folly/folly/detail/FileUtilDetail.h view

file too large to diff

+ folly/folly/detail/FileUtilVectorDetail.h view

file too large to diff

+ folly/folly/detail/FingerprintPolynomial.h view

file too large to diff

+ folly/folly/detail/Futex-inl.h view

file too large to diff

+ folly/folly/detail/Futex.cpp view

file too large to diff

+ folly/folly/detail/Futex.h view

file too large to diff

+ folly/folly/detail/GroupVarintDetail.h view

file too large to diff

+ folly/folly/detail/IPAddress.cpp view

file too large to diff

+ folly/folly/detail/IPAddress.h view

file too large to diff

+ folly/folly/detail/IPAddressSource.h view

file too large to diff

+ folly/folly/detail/Iterators.h view

file too large to diff

+ folly/folly/detail/MPMCPipelineDetail.h view

file too large to diff

+ folly/folly/detail/MemoryIdler.cpp view

file too large to diff

+ folly/folly/detail/MemoryIdler.h view

file too large to diff

+ folly/folly/detail/PerfScoped.cpp view

file too large to diff

+ folly/folly/detail/PerfScoped.h view

file too large to diff

+ folly/folly/detail/PolyDetail.h view

file too large to diff

+ folly/folly/detail/RangeCommon.cpp view

file too large to diff

+ folly/folly/detail/RangeCommon.h view

file too large to diff

+ folly/folly/detail/RangeSimd.cpp view

file too large to diff

+ folly/folly/detail/RangeSimd.h view

file too large to diff

+ folly/folly/detail/RangeSse42.cpp view

file too large to diff

+ folly/folly/detail/RangeSse42.h view

file too large to diff

+ folly/folly/detail/SimpleSimdStringUtils.cpp view

file too large to diff

+ folly/folly/detail/SimpleSimdStringUtils.h view

file too large to diff

+ folly/folly/detail/SimpleSimdStringUtilsImpl.h view

file too large to diff

+ folly/folly/detail/Singleton.h view

file too large to diff

+ folly/folly/detail/SlowFingerprint.h view

file too large to diff

+ folly/folly/detail/SocketFastOpen.cpp view

file too large to diff

+ folly/folly/detail/SocketFastOpen.h view

file too large to diff

+ folly/folly/detail/SplitStringSimd.cpp view

file too large to diff

+ folly/folly/detail/SplitStringSimd.h view

file too large to diff

+ folly/folly/detail/SplitStringSimdImpl.h view

file too large to diff

+ folly/folly/detail/Sse.cpp view

file too large to diff

+ folly/folly/detail/Sse.h view

file too large to diff

+ folly/folly/detail/StaticSingletonManager.cpp view

file too large to diff

+ folly/folly/detail/StaticSingletonManager.h view

file too large to diff

+ folly/folly/detail/ThreadLocalDetail.cpp view

file too large to diff

+ folly/folly/detail/ThreadLocalDetail.h view

file too large to diff

+ folly/folly/detail/TrapOnAvx512.cpp view

file too large to diff

+ folly/folly/detail/TrapOnAvx512.h view

file too large to diff

+ folly/folly/detail/TurnSequencer.h view

file too large to diff

+ folly/folly/detail/TypeList.h view

file too large to diff

+ folly/folly/detail/UniqueInstance.cpp view

file too large to diff

+ folly/folly/detail/UniqueInstance.h view

file too large to diff

+ folly/folly/detail/base64_detail/Base64Api.cpp view

file too large to diff

+ folly/folly/detail/base64_detail/Base64Api.h view

file too large to diff

+ folly/folly/detail/base64_detail/Base64Common.h view

file too large to diff

+ folly/folly/detail/base64_detail/Base64Constants.h view

file too large to diff

+ folly/folly/detail/base64_detail/Base64HiddenConstants.h view

file too large to diff

+ folly/folly/detail/base64_detail/Base64SWAR.cpp view

file too large to diff

+ folly/folly/detail/base64_detail/Base64SWAR.h view

file too large to diff

+ folly/folly/detail/base64_detail/Base64Scalar.h view

file too large to diff

+ folly/folly/detail/base64_detail/Base64Simd.h view

file too large to diff

+ folly/folly/detail/base64_detail/Base64_SSE4_2.cpp view

file too large to diff

+ folly/folly/detail/base64_detail/Base64_SSE4_2.h view

file too large to diff

+ folly/folly/detail/base64_detail/Base64_SSE4_2_Platform.h view

file too large to diff

+ folly/folly/detail/thread_local_globals.cpp view

file too large to diff

+ folly/folly/detail/thread_local_globals.h view

file too large to diff

+ folly/folly/detail/tuple.h view

file too large to diff

+ folly/folly/dynamic-inl.h view

file too large to diff

+ folly/folly/dynamic.h view

file too large to diff

+ folly/folly/executors/Async.h view

file too large to diff

+ folly/folly/executors/CPUThreadPoolExecutor.cpp view

file too large to diff

+ folly/folly/executors/CPUThreadPoolExecutor.h view

file too large to diff

+ folly/folly/executors/Codel.cpp view

file too large to diff

+ folly/folly/executors/Codel.h view

file too large to diff

+ folly/folly/executors/DrivableExecutor.h view

file too large to diff

+ folly/folly/executors/EDFThreadPoolExecutor.cpp view

file too large to diff

+ folly/folly/executors/EDFThreadPoolExecutor.h view

file too large to diff

+ folly/folly/executors/ExecutionObserver.cpp view

file too large to diff

+ folly/folly/executors/ExecutionObserver.h view

file too large to diff

+ folly/folly/executors/ExecutorWithPriority-inl.h view

file too large to diff

+ folly/folly/executors/ExecutorWithPriority.cpp view

file too large to diff

+ folly/folly/executors/ExecutorWithPriority.h view

file too large to diff

+ folly/folly/executors/FiberIOExecutor.h view

file too large to diff

+ folly/folly/executors/FunctionScheduler.cpp view

file too large to diff

+ folly/folly/executors/FunctionScheduler.h view

file too large to diff

+ folly/folly/executors/FutureExecutor.h view

file too large to diff

+ folly/folly/executors/GlobalExecutor.cpp view

file too large to diff

+ folly/folly/executors/GlobalExecutor.h view

file too large to diff

+ folly/folly/executors/GlobalThreadPoolList.cpp view

file too large to diff

+ folly/folly/executors/GlobalThreadPoolList.h view

file too large to diff

+ folly/folly/executors/IOExecutor.h view

file too large to diff

+ folly/folly/executors/IOObjectCache.h view

file too large to diff

+ folly/folly/executors/IOThreadPoolDeadlockDetectorObserver.cpp view

file too large to diff

+ folly/folly/executors/IOThreadPoolDeadlockDetectorObserver.h view

file too large to diff

+ folly/folly/executors/IOThreadPoolExecutor.cpp view

file too large to diff

+ folly/folly/executors/IOThreadPoolExecutor.h view

file too large to diff

+ folly/folly/executors/InlineExecutor.cpp view

file too large to diff

+ folly/folly/executors/InlineExecutor.h view

file too large to diff

+ folly/folly/executors/ManualExecutor.cpp view

file too large to diff

+ folly/folly/executors/ManualExecutor.h view

file too large to diff

+ folly/folly/executors/MeteredExecutor-inl.h view

file too large to diff

+ folly/folly/executors/MeteredExecutor.h view

file too large to diff

+ folly/folly/executors/QueueObserver.cpp view

file too large to diff

+ folly/folly/executors/QueueObserver.h view

file too large to diff

+ folly/folly/executors/QueuedImmediateExecutor.cpp view

file too large to diff

+ folly/folly/executors/QueuedImmediateExecutor.h view

file too large to diff

+ folly/folly/executors/ScheduledExecutor.h view

file too large to diff

+ folly/folly/executors/SequencedExecutor.h view

file too large to diff

+ folly/folly/executors/SerialExecutor-inl.h view

file too large to diff

+ folly/folly/executors/SerialExecutor.h view

file too large to diff

+ folly/folly/executors/SerializedExecutor.h view

file too large to diff

+ folly/folly/executors/SoftRealTimeExecutor.cpp view

file too large to diff

+ folly/folly/executors/SoftRealTimeExecutor.h view

file too large to diff

+ folly/folly/executors/StrandExecutor.cpp view

file too large to diff

+ folly/folly/executors/StrandExecutor.h view

file too large to diff

+ folly/folly/executors/ThreadPoolExecutor.cpp view

file too large to diff

+ folly/folly/executors/ThreadPoolExecutor.h view

file too large to diff

+ folly/folly/executors/ThreadedExecutor.cpp view

file too large to diff

+ folly/folly/executors/ThreadedExecutor.h view

file too large to diff

+ folly/folly/executors/ThreadedRepeatingFunctionRunner.cpp view

file too large to diff

+ folly/folly/executors/ThreadedRepeatingFunctionRunner.h view

file too large to diff

+ folly/folly/executors/TimedDrivableExecutor.cpp view

file too large to diff

+ folly/folly/executors/TimedDrivableExecutor.h view

file too large to diff

+ folly/folly/executors/TimekeeperScheduledExecutor.cpp view

file too large to diff

+ folly/folly/executors/TimekeeperScheduledExecutor.h view

file too large to diff

+ folly/folly/executors/VirtualExecutor.h view

file too large to diff

+ folly/folly/executors/task_queue/BlockingQueue.h view

file too large to diff

+ folly/folly/executors/task_queue/LifoSemMPMCQueue.h view

file too large to diff

+ folly/folly/executors/task_queue/PriorityLifoSemMPMCQueue.h view

file too large to diff

+ folly/folly/executors/task_queue/PriorityUnboundedBlockingQueue.h view

file too large to diff

+ folly/folly/executors/task_queue/UnboundedBlockingQueue.h view

file too large to diff

+ folly/folly/executors/thread_factory/InitThreadFactory.h view

file too large to diff

+ folly/folly/executors/thread_factory/NamedThreadFactory.h view

file too large to diff

+ folly/folly/executors/thread_factory/PriorityThreadFactory.cpp view

file too large to diff

+ folly/folly/executors/thread_factory/PriorityThreadFactory.h view

file too large to diff

+ folly/folly/executors/thread_factory/ThreadFactory.h view

file too large to diff

+ folly/folly/experimental/EventCount.h view

file too large to diff

+ folly/folly/experimental/FlatCombiningPriorityQueue.h view

file too large to diff

+ folly/folly/experimental/FunctionScheduler.h view

file too large to diff

+ folly/folly/experimental/TestUtil.h view

file too large to diff

+ folly/folly/experimental/ThreadedRepeatingFunctionRunner.h view

file too large to diff

+ folly/folly/experimental/channels/Channel-fwd.h view

file too large to diff

+ folly/folly/experimental/channels/Channel-inl.h view

file too large to diff

+ folly/folly/experimental/channels/Channel.h view

file too large to diff

+ folly/folly/experimental/channels/ChannelCallbackHandle.h view

file too large to diff

+ folly/folly/experimental/channels/ChannelProcessor-inl.h view

file too large to diff

+ folly/folly/experimental/channels/ChannelProcessor.h view

file too large to diff

+ folly/folly/experimental/channels/ConsumeChannel-inl.h view

file too large to diff

+ folly/folly/experimental/channels/ConsumeChannel.h view

file too large to diff

+ folly/folly/experimental/channels/FanoutChannel-inl.h view

file too large to diff

+ folly/folly/experimental/channels/FanoutChannel.h view

file too large to diff

+ folly/folly/experimental/channels/FanoutSender-inl.h view

file too large to diff

+ folly/folly/experimental/channels/FanoutSender.h view

file too large to diff

+ folly/folly/experimental/channels/MaxConcurrentRateLimiter.h view

file too large to diff

+ folly/folly/experimental/channels/Merge-inl.h view

file too large to diff

+ folly/folly/experimental/channels/Merge.h view

file too large to diff

+ folly/folly/experimental/channels/MergeChannel-inl.h view

file too large to diff

+ folly/folly/experimental/channels/MergeChannel.h view

file too large to diff

+ folly/folly/experimental/channels/MultiplexChannel-inl.h view

file too large to diff

+ folly/folly/experimental/channels/MultiplexChannel.h view

file too large to diff

+ folly/folly/experimental/channels/OnClosedException.h view

file too large to diff

+ folly/folly/experimental/channels/Producer-inl.h view

file too large to diff

+ folly/folly/experimental/channels/Producer.h view

file too large to diff

+ folly/folly/experimental/channels/ProxyChannel-inl.h view

file too large to diff

+ folly/folly/experimental/channels/ProxyChannel.h view

file too large to diff

+ folly/folly/experimental/channels/RateLimiter.h view

file too large to diff

+ folly/folly/experimental/channels/Transform-inl.h view

file too large to diff

+ folly/folly/experimental/channels/Transform.h view

file too large to diff

+ folly/folly/experimental/channels/detail/AtomicQueue.h view

file too large to diff

+ folly/folly/experimental/channels/detail/ChannelBridge.h view

file too large to diff

+ folly/folly/experimental/channels/detail/FunctionTraits.h view

file too large to diff

+ folly/folly/experimental/channels/detail/IntrusivePtr.h view

file too large to diff

+ folly/folly/experimental/channels/detail/MultiplexerTraits.h view

file too large to diff

+ folly/folly/experimental/channels/detail/PointerVariant.h view

file too large to diff

+ folly/folly/experimental/channels/detail/Utility.h view

file too large to diff

+ folly/folly/experimental/coro/AsyncGenerator.h view

file too large to diff

+ folly/folly/experimental/coro/AsyncPipe.h view

file too large to diff

+ folly/folly/experimental/coro/AsyncScope.h view

file too large to diff

+ folly/folly/experimental/coro/AsyncStack.h view

file too large to diff

+ folly/folly/experimental/coro/AutoCleanup-fwd.h view

file too large to diff

+ folly/folly/experimental/coro/AutoCleanup.h view

file too large to diff

+ folly/folly/experimental/coro/Baton.h view

file too large to diff

+ folly/folly/experimental/coro/BlockingWait.h view

file too large to diff

+ folly/folly/experimental/coro/BoundedQueue.h view

file too large to diff

+ folly/folly/experimental/coro/Cleanup.h view

file too large to diff

+ folly/folly/experimental/coro/Collect-inl.h view

file too large to diff

+ folly/folly/experimental/coro/Collect.h view

file too large to diff

+ folly/folly/experimental/coro/Concat-inl.h view

file too large to diff

+ folly/folly/experimental/coro/Concat.h view

file too large to diff

+ folly/folly/experimental/coro/Coroutine.h view

file too large to diff

+ folly/folly/experimental/coro/CurrentExecutor.h view

file too large to diff

+ folly/folly/experimental/coro/DetachOnCancel.h view

file too large to diff

+ folly/folly/experimental/coro/Filter-inl.h view

file too large to diff

+ folly/folly/experimental/coro/Filter.h view

file too large to diff

+ folly/folly/experimental/coro/FutureUtil.h view

file too large to diff

+ folly/folly/experimental/coro/Generator.h view

file too large to diff

+ folly/folly/experimental/coro/GmockHelpers.h view

file too large to diff

+ folly/folly/experimental/coro/GtestHelpers.h view

file too large to diff

+ folly/folly/experimental/coro/Invoke.h view

file too large to diff

+ folly/folly/experimental/coro/Merge-inl.h view

file too large to diff

+ folly/folly/experimental/coro/Merge.h view

file too large to diff

+ folly/folly/experimental/coro/Mutex.h view

file too large to diff

+ folly/folly/experimental/coro/Promise.h view

file too large to diff

+ folly/folly/experimental/coro/Result.h view

file too large to diff

+ folly/folly/experimental/coro/Retry.h view

file too large to diff

+ folly/folly/experimental/coro/RustAdaptors.h view

file too large to diff

+ folly/folly/experimental/coro/ScopeExit.h view

file too large to diff

+ folly/folly/experimental/coro/SharedLock.h view

file too large to diff

+ folly/folly/experimental/coro/SharedMutex.h view

file too large to diff

+ folly/folly/experimental/coro/SharedPromise.h view

file too large to diff

+ folly/folly/experimental/coro/Sleep-inl.h view

file too large to diff

+ folly/folly/experimental/coro/Sleep.h view

file too large to diff

+ folly/folly/experimental/coro/SmallUnboundedQueue.h view

file too large to diff

+ folly/folly/experimental/coro/Task.h view

file too large to diff

+ folly/folly/experimental/coro/TimedWait.h view

file too large to diff

+ folly/folly/experimental/coro/Timeout-inl.h view

file too large to diff

+ folly/folly/experimental/coro/Timeout.h view

file too large to diff

+ folly/folly/experimental/coro/Traits.h view

file too large to diff

+ folly/folly/experimental/coro/Transform-inl.h view

file too large to diff

+ folly/folly/experimental/coro/Transform.h view

file too large to diff

+ folly/folly/experimental/coro/UnboundedQueue.h view

file too large to diff

+ folly/folly/experimental/coro/ViaIfAsync.h view

file too large to diff

+ folly/folly/experimental/coro/WithAsyncStack.h view

file too large to diff

+ folly/folly/experimental/coro/WithCancellation.h view

file too large to diff

+ folly/folly/experimental/coro/detail/Barrier.h view

file too large to diff

+ folly/folly/experimental/coro/detail/BarrierTask.h view

file too large to diff

+ folly/folly/experimental/coro/detail/CurrentAsyncFrame.h view

file too large to diff

+ folly/folly/experimental/coro/detail/Helpers.h view

file too large to diff

+ folly/folly/experimental/coro/detail/InlineTask.h view

file too large to diff

+ folly/folly/experimental/coro/detail/Malloc.h view

file too large to diff

+ folly/folly/experimental/coro/detail/ManualLifetime.h view

file too large to diff

+ folly/folly/experimental/coro/detail/Traits.h view

file too large to diff

+ folly/folly/experimental/crypto/Blake2xb.h view

file too large to diff

+ folly/folly/experimental/crypto/LtHash.h view

file too large to diff

+ folly/folly/experimental/exception_tracer/ExceptionAbi.h view

file too large to diff

+ folly/folly/experimental/exception_tracer/ExceptionCounterLib.h view

file too large to diff

+ folly/folly/experimental/exception_tracer/ExceptionTracer.h view

file too large to diff

+ folly/folly/experimental/exception_tracer/ExceptionTracerLib.h view

file too large to diff

+ folly/folly/experimental/exception_tracer/SmartExceptionTracer.h view

file too large to diff

+ folly/folly/experimental/exception_tracer/SmartExceptionTracerSingleton.h view

file too large to diff

+ folly/folly/experimental/exception_tracer/StackTrace.h view

file too large to diff

+ folly/folly/experimental/flat_combining/FlatCombining.h view

file too large to diff

+ folly/folly/experimental/io/AsyncBase.h view

file too large to diff

+ folly/folly/experimental/io/AsyncIO.h view

file too large to diff

+ folly/folly/experimental/io/AsyncIoUringSocket.h view

file too large to diff

+ folly/folly/experimental/io/AsyncIoUringSocketFactory.h view

file too large to diff

+ folly/folly/experimental/io/Epoll.h view

file too large to diff

+ folly/folly/experimental/io/EpollBackend.h view

file too large to diff

+ folly/folly/experimental/io/EventBasePoller.h view

file too large to diff

+ folly/folly/experimental/io/FsUtil.h view

file too large to diff

+ folly/folly/experimental/io/HugePages.h view

file too large to diff

+ folly/folly/experimental/io/IoUring.h view

file too large to diff

+ folly/folly/experimental/io/IoUringBackend.h view

file too large to diff

+ folly/folly/experimental/io/IoUringBase.h view

file too large to diff

+ folly/folly/experimental/io/IoUringEvent.h view

file too large to diff

+ folly/folly/experimental/io/IoUringEventBaseLocal.h view

file too large to diff

+ folly/folly/experimental/io/IoUringProvidedBufferRing.h view

file too large to diff

+ folly/folly/experimental/io/Liburing.h view

file too large to diff

+ folly/folly/experimental/io/MuxIOThreadPoolExecutor.h view

file too large to diff

+ folly/folly/experimental/io/SimpleAsyncIO.h view

file too large to diff

+ folly/folly/experimental/observer/CoreCachedObserver.h view

file too large to diff

+ folly/folly/experimental/observer/HazptrObserver.h view

file too large to diff

+ folly/folly/experimental/observer/Observable-inl.h view

file too large to diff

+ folly/folly/experimental/observer/Observable.h view

file too large to diff

+ folly/folly/experimental/observer/Observer-inl.h view

file too large to diff

+ folly/folly/experimental/observer/Observer-pre.h view

file too large to diff

+ folly/folly/experimental/observer/Observer.h view

file too large to diff

+ folly/folly/experimental/observer/ReadMostlyTLObserver.h view

file too large to diff

+ folly/folly/experimental/observer/SimpleObservable-inl.h view

file too large to diff

+ folly/folly/experimental/observer/SimpleObservable.h view

file too large to diff

+ folly/folly/experimental/observer/WithJitter-inl.h view

file too large to diff

+ folly/folly/experimental/observer/WithJitter.h view

file too large to diff

+ folly/folly/experimental/observer/detail/Core.h view

file too large to diff

+ folly/folly/experimental/observer/detail/GraphCycleDetector.h view

file too large to diff

+ folly/folly/experimental/observer/detail/ObserverManager.h view

file too large to diff

+ folly/folly/experimental/settings/Immutables.h view

file too large to diff

+ folly/folly/experimental/settings/Settings.h view

file too large to diff

+ folly/folly/experimental/settings/Types.h view

file too large to diff

+ folly/folly/experimental/settings/detail/SettingsImpl.h view

file too large to diff

+ folly/folly/experimental/symbolizer/Dwarf.h view

file too large to diff

+ folly/folly/experimental/symbolizer/DwarfImpl.h view

file too large to diff

+ folly/folly/experimental/symbolizer/DwarfLineNumberVM.h view

file too large to diff

+ folly/folly/experimental/symbolizer/DwarfSection.h view

file too large to diff

+ folly/folly/experimental/symbolizer/DwarfUtil.h view

file too large to diff

+ folly/folly/experimental/symbolizer/Elf-inl.h view

file too large to diff

+ folly/folly/experimental/symbolizer/Elf.h view

file too large to diff

+ folly/folly/experimental/symbolizer/ElfCache.h view

file too large to diff

+ folly/folly/experimental/symbolizer/LineReader.h view

file too large to diff

+ folly/folly/experimental/symbolizer/SignalHandler.h view

file too large to diff

+ folly/folly/experimental/symbolizer/StackTrace.h view

file too large to diff

+ folly/folly/experimental/symbolizer/SymbolizePrinter.h view

file too large to diff

+ folly/folly/experimental/symbolizer/SymbolizedFrame.h view

file too large to diff

+ folly/folly/experimental/symbolizer/Symbolizer.h view

file too large to diff

+ folly/folly/experimental/symbolizer/detail/Debug.h view

file too large to diff

+ folly/folly/ext/buck2/test_ext.cpp view

file too large to diff

+ folly/folly/ext/test_ext.cpp view

file too large to diff

+ folly/folly/ext/test_ext.h view

file too large to diff

+ folly/folly/external/aor/asmdefs.h view

file too large to diff

+ folly/folly/external/farmhash/farmhash.cpp view

file too large to diff

+ folly/folly/external/farmhash/farmhash.h view

file too large to diff

+ folly/folly/external/fast-crc32/avx512_crc32c_v8s3x4.cpp view

file too large to diff

+ folly/folly/external/fast-crc32/avx512_crc32c_v8s3x4.h view

file too large to diff

+ folly/folly/external/fast-crc32/neon_crc32c_v3s4x2e_v2.cpp view

file too large to diff

+ folly/folly/external/fast-crc32/neon_crc32c_v3s4x2e_v2.h view

file too large to diff

+ folly/folly/external/fast-crc32/neon_eor3_crc32_v9s3x2e_s3.cpp view

file too large to diff

+ folly/folly/external/fast-crc32/neon_eor3_crc32_v9s3x2e_s3.h view

file too large to diff

+ folly/folly/external/fast-crc32/neon_eor3_crc32c_v8s2x4_s3.cpp view

file too large to diff

+ folly/folly/external/fast-crc32/neon_eor3_crc32c_v8s2x4_s3.h view

file too large to diff

+ folly/folly/external/fast-crc32/sse_crc32c_v8s3x3.cpp view

file too large to diff

+ folly/folly/external/fast-crc32/sse_crc32c_v8s3x3.h view

file too large to diff

+ folly/folly/external/nvidia/detail/RangeSve2.cpp view

file too large to diff

+ folly/folly/external/nvidia/detail/RangeSve2.h view

file too large to diff

+ folly/folly/external/nvidia/hash/Checksum.cpp view

file too large to diff

+ folly/folly/external/nvidia/hash/detail/Crc32cCombineDetail.h view

file too large to diff

+ folly/folly/external/nvidia/hash/detail/Crc32cDetail.cpp view

file too large to diff

+ folly/folly/external/rapidhash/rapidhash.h view

file too large to diff

+ folly/folly/fibers/AddTasks-inl.h view

file too large to diff

+ folly/folly/fibers/AddTasks.h view

file too large to diff

+ folly/folly/fibers/AtomicBatchDispatcher-inl.h view

file too large to diff

+ folly/folly/fibers/AtomicBatchDispatcher.h view

file too large to diff

+ folly/folly/fibers/BatchDispatcher.h view

file too large to diff

+ folly/folly/fibers/BatchSemaphore.cpp view

file too large to diff

+ folly/folly/fibers/BatchSemaphore.h view

file too large to diff

+ folly/folly/fibers/Baton-inl.h view

file too large to diff

+ folly/folly/fibers/Baton.cpp view

file too large to diff

+ folly/folly/fibers/Baton.h view

file too large to diff

+ folly/folly/fibers/BoostContextCompatibility.h view

file too large to diff

+ folly/folly/fibers/CallOnce.h view

file too large to diff

+ folly/folly/fibers/EventBaseLoopController-inl.h view

file too large to diff

+ folly/folly/fibers/EventBaseLoopController.h view

file too large to diff

+ folly/folly/fibers/ExecutorBasedLoopController.h view

file too large to diff

+ folly/folly/fibers/ExecutorLoopController-inl.h view

file too large to diff

+ folly/folly/fibers/ExecutorLoopController.h view

file too large to diff

+ folly/folly/fibers/Fiber-inl.h view

file too large to diff

+ folly/folly/fibers/Fiber.cpp view

file too large to diff

+ folly/folly/fibers/Fiber.h view

file too large to diff

+ folly/folly/fibers/FiberManager-inl.h view

file too large to diff

+ folly/folly/fibers/FiberManager.cpp view

file too large to diff

+ folly/folly/fibers/FiberManager.h view

file too large to diff

+ folly/folly/fibers/FiberManagerInternal-inl.h view

file too large to diff

+ folly/folly/fibers/FiberManagerInternal.h view

file too large to diff

+ folly/folly/fibers/FiberManagerMap-inl.h view

file too large to diff

+ folly/folly/fibers/FiberManagerMap.h view

file too large to diff

+ folly/folly/fibers/ForEach-inl.h view

file too large to diff

+ folly/folly/fibers/ForEach.h view

file too large to diff

+ folly/folly/fibers/GenericBaton.h view

file too large to diff

+ folly/folly/fibers/GuardPageAllocator.cpp view

file too large to diff

+ folly/folly/fibers/GuardPageAllocator.h view

file too large to diff

+ folly/folly/fibers/LoopController.h view

file too large to diff

+ folly/folly/fibers/Promise-inl.h view

file too large to diff

+ folly/folly/fibers/Promise.h view

file too large to diff

+ folly/folly/fibers/Semaphore.cpp view

file too large to diff

+ folly/folly/fibers/Semaphore.h view

file too large to diff

+ folly/folly/fibers/SemaphoreBase.cpp view

file too large to diff

+ folly/folly/fibers/SemaphoreBase.h view

file too large to diff

+ folly/folly/fibers/SimpleLoopController.cpp view

file too large to diff

+ folly/folly/fibers/SimpleLoopController.h view

file too large to diff

+ folly/folly/fibers/TimedMutex-inl.h view

file too large to diff

+ folly/folly/fibers/TimedMutex.h view

file too large to diff

+ folly/folly/fibers/WhenN-inl.h view

file too large to diff

+ folly/folly/fibers/WhenN.h view

file too large to diff

+ folly/folly/fibers/async/Async.cpp view

file too large to diff

+ folly/folly/fibers/async/Async.h view

file too large to diff

+ folly/folly/fibers/async/AsyncStack.h view

file too large to diff

+ folly/folly/fibers/async/Baton.h view

file too large to diff

+ folly/folly/fibers/async/Collect-inl.h view

file too large to diff

+ folly/folly/fibers/async/Collect.h view

file too large to diff

+ folly/folly/fibers/async/FiberManager.h view

file too large to diff

+ folly/folly/fibers/async/Future.h view

file too large to diff

+ folly/folly/fibers/async/Promise.h view

file too large to diff

+ folly/folly/fibers/async/Task.h view

file too large to diff

+ folly/folly/fibers/async/WaitUtils.h view

file too large to diff

+ folly/folly/fibers/detail/AtomicBatchDispatcher.cpp view

file too large to diff

+ folly/folly/fibers/detail/AtomicBatchDispatcher.h view

file too large to diff

+ folly/folly/fibers/traits.h view

file too large to diff

+ folly/folly/functional/ApplyTuple.h view

file too large to diff

+ folly/folly/functional/Invoke.h view

file too large to diff

+ folly/folly/functional/Partial.h view

file too large to diff

+ folly/folly/functional/protocol.h view

file too large to diff

+ folly/folly/functional/traits.h view

file too large to diff

+ folly/folly/futures/Barrier.cpp view

file too large to diff

+ folly/folly/futures/Barrier.h view

file too large to diff

+ folly/folly/futures/Cleanup.h view

file too large to diff

+ folly/folly/futures/Future-inl.h view

file too large to diff

+ folly/folly/futures/Future-pre.h view

file too large to diff

+ folly/folly/futures/Future.cpp view

file too large to diff

+ folly/folly/futures/Future.h view

file too large to diff

+ folly/folly/futures/FutureSplitter.h view

file too large to diff

+ folly/folly/futures/HeapTimekeeper.cpp view

file too large to diff

+ folly/folly/futures/HeapTimekeeper.h view

file too large to diff

+ folly/folly/futures/ManualTimekeeper.cpp view

file too large to diff

+ folly/folly/futures/ManualTimekeeper.h view

file too large to diff

+ folly/folly/futures/Portability.h view

file too large to diff

+ folly/folly/futures/Promise-inl.h view

file too large to diff

+ folly/folly/futures/Promise.cpp view

file too large to diff

+ folly/folly/futures/Promise.h view

file too large to diff

+ folly/folly/futures/Retrying.h view

file too large to diff

+ folly/folly/futures/SharedPromise-inl.h view

file too large to diff

+ folly/folly/futures/SharedPromise.cpp view

file too large to diff

+ folly/folly/futures/SharedPromise.h view

file too large to diff

+ folly/folly/futures/ThreadWheelTimekeeper.cpp view

file too large to diff

+ folly/folly/futures/ThreadWheelTimekeeper.h view

file too large to diff

+ folly/folly/futures/WTCallback.h view

file too large to diff

+ folly/folly/futures/detail/Core.cpp view

file too large to diff

+ folly/folly/futures/detail/Core.h view

file too large to diff

+ folly/folly/futures/detail/Types.h view

file too large to diff

+ folly/folly/gen/Base-inl.h view

file too large to diff

+ folly/folly/gen/Base.h view

file too large to diff

+ folly/folly/gen/Combine-inl.h view

file too large to diff

+ folly/folly/gen/Combine.h view

file too large to diff

+ folly/folly/gen/Core-inl.h view

file too large to diff

+ folly/folly/gen/Core.h view

file too large to diff

+ folly/folly/gen/File-inl.h view

file too large to diff

+ folly/folly/gen/File.h view

file too large to diff

+ folly/folly/gen/IStream.h view

file too large to diff

+ folly/folly/gen/Parallel-inl.h view

file too large to diff

+ folly/folly/gen/Parallel.h view

file too large to diff

+ folly/folly/gen/ParallelMap-inl.h view

file too large to diff

+ folly/folly/gen/ParallelMap.h view

file too large to diff

+ folly/folly/gen/String-inl.h view

file too large to diff

+ folly/folly/gen/String.h view

file too large to diff

+ folly/folly/hash/Checksum.cpp view

file too large to diff

+ folly/folly/hash/Checksum.h view

file too large to diff

+ folly/folly/hash/FarmHash.h view

file too large to diff

+ folly/folly/hash/Hash.h view

file too large to diff

+ folly/folly/hash/MurmurHash.h view

file too large to diff

+ folly/folly/hash/SpookyHashV1.cpp view

file too large to diff

+ folly/folly/hash/SpookyHashV1.h view

file too large to diff

+ folly/folly/hash/SpookyHashV2.cpp view

file too large to diff

+ folly/folly/hash/SpookyHashV2.h view

file too large to diff

+ folly/folly/hash/detail/ChecksumDetail.cpp view

file too large to diff

+ folly/folly/hash/detail/ChecksumDetail.h view

file too large to diff

+ folly/folly/hash/detail/Crc32CombineDetail.cpp view

file too large to diff

+ folly/folly/hash/detail/Crc32cDetail.cpp view

file too large to diff

+ folly/folly/hash/rapidhash.h view

file too large to diff

+ folly/folly/hash/traits.h view

file too large to diff

+ folly/folly/init/Init.cpp view

file too large to diff

+ folly/folly/init/Init.h view

file too large to diff

+ folly/folly/init/Phase.cpp view

file too large to diff

+ folly/folly/init/Phase.h view

file too large to diff

+ folly/folly/io/Cursor-inl.h view

file too large to diff

+ folly/folly/io/Cursor.cpp view

file too large to diff

+ folly/folly/io/Cursor.h view

file too large to diff

+ folly/folly/io/FsUtil.cpp view

file too large to diff

+ folly/folly/io/FsUtil.h view

file too large to diff

+ folly/folly/io/GlobalShutdownSocketSet.cpp view

file too large to diff

+ folly/folly/io/GlobalShutdownSocketSet.h view

file too large to diff

+ folly/folly/io/HugePages.cpp view

file too large to diff

+ folly/folly/io/HugePages.h view

file too large to diff

+ folly/folly/io/IOBuf.cpp view

file too large to diff

+ folly/folly/io/IOBuf.h view

file too large to diff

+ folly/folly/io/IOBufIovecBuilder.cpp view

file too large to diff

+ folly/folly/io/IOBufIovecBuilder.h view

file too large to diff

+ folly/folly/io/IOBufQueue.cpp view

file too large to diff

+ folly/folly/io/IOBufQueue.h view

file too large to diff

+ folly/folly/io/RecordIO-inl.h view

file too large to diff

+ folly/folly/io/RecordIO.cpp view

file too large to diff

+ folly/folly/io/RecordIO.h view

file too large to diff

+ folly/folly/io/ShutdownSocketSet.cpp view

file too large to diff

+ folly/folly/io/ShutdownSocketSet.h view

file too large to diff

+ folly/folly/io/SocketOptionMap.cpp view

file too large to diff

+ folly/folly/io/SocketOptionMap.h view

file too large to diff

+ folly/folly/io/SocketOptionValue.cpp view

file too large to diff

+ folly/folly/io/SocketOptionValue.h view

file too large to diff

+ folly/folly/io/TypedIOBuf.h view

file too large to diff

+ folly/folly/io/async/AsyncBase.cpp view

file too large to diff

+ folly/folly/io/async/AsyncBase.h view

file too large to diff

+ folly/folly/io/async/AsyncIO.cpp view

file too large to diff

+ folly/folly/io/async/AsyncIO.h view

file too large to diff

+ folly/folly/io/async/AsyncIoUringSocket.cpp view

file too large to diff

+ folly/folly/io/async/AsyncIoUringSocket.h view

file too large to diff

+ folly/folly/io/async/AsyncIoUringSocketFactory.h view

file too large to diff

+ folly/folly/io/async/AsyncPipe.cpp view

file too large to diff

+ folly/folly/io/async/AsyncPipe.h view

file too large to diff

+ folly/folly/io/async/AsyncSSLSocket.cpp view

file too large to diff

+ folly/folly/io/async/AsyncSSLSocket.h view

file too large to diff

+ folly/folly/io/async/AsyncServerSocket.cpp view

file too large to diff

+ folly/folly/io/async/AsyncServerSocket.h view

file too large to diff

+ folly/folly/io/async/AsyncSignalHandler.cpp view

file too large to diff

+ folly/folly/io/async/AsyncSignalHandler.h view

file too large to diff

+ folly/folly/io/async/AsyncSocket.cpp view

file too large to diff

+ folly/folly/io/async/AsyncSocket.h view

file too large to diff

+ folly/folly/io/async/AsyncSocketBase.h view

file too large to diff

+ folly/folly/io/async/AsyncSocketException.cpp view

file too large to diff

+ folly/folly/io/async/AsyncSocketException.h view

file too large to diff

+ folly/folly/io/async/AsyncSocketTransport.cpp view

file too large to diff

+ folly/folly/io/async/AsyncSocketTransport.h view

file too large to diff

+ folly/folly/io/async/AsyncTimeout.cpp view

file too large to diff

+ folly/folly/io/async/AsyncTimeout.h view

file too large to diff

+ folly/folly/io/async/AsyncTransport.h view

file too large to diff

+ folly/folly/io/async/AsyncTransportCertificate.h view

file too large to diff

+ folly/folly/io/async/AsyncUDPServerSocket.h view

file too large to diff

+ folly/folly/io/async/AsyncUDPSocket.cpp view

file too large to diff

+ folly/folly/io/async/AsyncUDPSocket.h view

file too large to diff

+ folly/folly/io/async/AtomicNotificationQueue-inl.h view

file too large to diff

+ folly/folly/io/async/AtomicNotificationQueue.h view

file too large to diff

+ folly/folly/io/async/CertificateIdentityVerifier.h view

file too large to diff

+ folly/folly/io/async/DecoratedAsyncTransportWrapper.h view

file too large to diff

+ folly/folly/io/async/DelayedDestruction.cpp view

file too large to diff

+ folly/folly/io/async/DelayedDestruction.h view

file too large to diff

+ folly/folly/io/async/DelayedDestructionBase.h view

file too large to diff

+ folly/folly/io/async/DestructorCheck.h view

file too large to diff

+ folly/folly/io/async/Epoll.h view

file too large to diff

+ folly/folly/io/async/EpollBackend.cpp view

file too large to diff

+ folly/folly/io/async/EpollBackend.h view

file too large to diff

+ folly/folly/io/async/EventBase.cpp view

file too large to diff

+ folly/folly/io/async/EventBase.h view

file too large to diff

+ folly/folly/io/async/EventBaseAtomicNotificationQueue-inl.h view

file too large to diff

+ folly/folly/io/async/EventBaseAtomicNotificationQueue.h view

file too large to diff

+ folly/folly/io/async/EventBaseBackendBase.cpp view

file too large to diff

+ folly/folly/io/async/EventBaseBackendBase.h view

file too large to diff

+ folly/folly/io/async/EventBaseLocal.cpp view

file too large to diff

+ folly/folly/io/async/EventBaseLocal.h view

file too large to diff

+ folly/folly/io/async/EventBaseManager.cpp view

file too large to diff

+ folly/folly/io/async/EventBaseManager.h view

file too large to diff

+ folly/folly/io/async/EventBasePoller.cpp view

file too large to diff

+ folly/folly/io/async/EventBasePoller.h view

file too large to diff

+ folly/folly/io/async/EventBaseThread.cpp view

file too large to diff

+ folly/folly/io/async/EventBaseThread.h view

file too large to diff

+ folly/folly/io/async/EventHandler.cpp view

file too large to diff

+ folly/folly/io/async/EventHandler.h view

file too large to diff

+ folly/folly/io/async/EventUtil.h view

file too large to diff

+ folly/folly/io/async/HHWheelTimer-fwd.h view

file too large to diff

+ folly/folly/io/async/HHWheelTimer.cpp view

file too large to diff

+ folly/folly/io/async/HHWheelTimer.h view

file too large to diff

+ folly/folly/io/async/IoUring.cpp view

file too large to diff

+ folly/folly/io/async/IoUring.h view

file too large to diff

+ folly/folly/io/async/IoUringBackend.cpp view

file too large to diff

+ folly/folly/io/async/IoUringBackend.h view

file too large to diff

+ folly/folly/io/async/IoUringBase.h view

file too large to diff

+ folly/folly/io/async/IoUringEvent.cpp view

file too large to diff

+ folly/folly/io/async/IoUringEvent.h view

file too large to diff

+ folly/folly/io/async/IoUringEventBaseLocal.cpp view

file too large to diff

+ folly/folly/io/async/IoUringEventBaseLocal.h view

file too large to diff

+ folly/folly/io/async/IoUringProvidedBufferRing.cpp view

file too large to diff

+ folly/folly/io/async/IoUringProvidedBufferRing.h view

file too large to diff

+ folly/folly/io/async/IoUringZeroCopyBufferPool.cpp view

file too large to diff

+ folly/folly/io/async/IoUringZeroCopyBufferPool.h view

file too large to diff

+ folly/folly/io/async/Liburing.h view

file too large to diff

+ folly/folly/io/async/MuxIOThreadPoolExecutor.cpp view

file too large to diff

+ folly/folly/io/async/MuxIOThreadPoolExecutor.h view

file too large to diff

+ folly/folly/io/async/NotificationQueue.h view

file too large to diff

+ folly/folly/io/async/PasswordInFile.cpp view

file too large to diff

+ folly/folly/io/async/PasswordInFile.h view

file too large to diff

+ folly/folly/io/async/Request.cpp view

file too large to diff

+ folly/folly/io/async/Request.h view

file too large to diff

+ folly/folly/io/async/SSLContext.cpp view

file too large to diff

+ folly/folly/io/async/SSLContext.h view

file too large to diff

+ folly/folly/io/async/SSLOptions.cpp view

file too large to diff

+ folly/folly/io/async/SSLOptions.h view

file too large to diff

+ folly/folly/io/async/STTimerFDTimeoutManager.cpp view

file too large to diff

+ folly/folly/io/async/STTimerFDTimeoutManager.h view

file too large to diff

+ folly/folly/io/async/ScopedEventBaseThread.cpp view

file too large to diff

+ folly/folly/io/async/ScopedEventBaseThread.h view

file too large to diff

+ folly/folly/io/async/SimpleAsyncIO.cpp view

file too large to diff

+ folly/folly/io/async/SimpleAsyncIO.h view

file too large to diff

+ folly/folly/io/async/TerminateCancellationToken.cpp view

file too large to diff

+ folly/folly/io/async/TerminateCancellationToken.h view

file too large to diff

+ folly/folly/io/async/TimeoutManager.cpp view

file too large to diff

+ folly/folly/io/async/TimeoutManager.h view

file too large to diff

+ folly/folly/io/async/TimerFD.cpp view

file too large to diff

+ folly/folly/io/async/TimerFD.h view

file too large to diff

+ folly/folly/io/async/TimerFDTimeoutManager.cpp view

file too large to diff

+ folly/folly/io/async/TimerFDTimeoutManager.h view

file too large to diff

+ folly/folly/io/async/VirtualEventBase.cpp view

file too large to diff

+ folly/folly/io/async/VirtualEventBase.h view

file too large to diff

+ folly/folly/io/async/WriteChainAsyncTransportWrapper.h view

file too large to diff

+ folly/folly/io/async/WriteFlags.h view

file too large to diff

+ folly/folly/io/async/fdsock/AsyncFdSocket.cpp view

file too large to diff

+ folly/folly/io/async/fdsock/AsyncFdSocket.h view

file too large to diff

+ folly/folly/io/async/fdsock/SocketFds.cpp view

file too large to diff

+ folly/folly/io/async/fdsock/SocketFds.h view

file too large to diff

+ folly/folly/io/async/observer/AsyncSocketObserverContainer.h view

file too large to diff

+ folly/folly/io/async/observer/AsyncSocketObserverInterface.h view

file too large to diff

+ folly/folly/io/async/ssl/BasicTransportCertificate.h view

file too large to diff

+ folly/folly/io/async/ssl/OpenSSLTransportCertificate.h view

file too large to diff

+ folly/folly/io/async/ssl/OpenSSLUtils.cpp view

file too large to diff

+ folly/folly/io/async/ssl/OpenSSLUtils.h view

file too large to diff

+ folly/folly/io/async/ssl/SSLErrors.cpp view

file too large to diff

+ folly/folly/io/async/ssl/SSLErrors.h view

file too large to diff

+ folly/folly/io/async/ssl/TLSDefinitions.h view

file too large to diff

+ folly/folly/io/async/test/AsyncSSLSocketTest.h view

file too large to diff

+ folly/folly/io/async/test/AsyncSocketTest.h view

file too large to diff

+ folly/folly/io/async/test/AsyncSocketTest2.h view

file too large to diff

+ folly/folly/io/async/test/BlockingSocket.h view

file too large to diff

+ folly/folly/io/async/test/CallbackStateEnum.h view

file too large to diff

+ folly/folly/io/async/test/ConnCallback.h view

file too large to diff

+ folly/folly/io/async/test/MockAsyncSSLSocket.h view

file too large to diff

+ folly/folly/io/async/test/MockAsyncServerSocket.h view

file too large to diff

+ folly/folly/io/async/test/MockAsyncSocket.h view

file too large to diff

+ folly/folly/io/async/test/MockAsyncTransport.h view

file too large to diff

+ folly/folly/io/async/test/MockAsyncUDPSocket.h view

file too large to diff

+ folly/folly/io/async/test/MockTimeoutManager.h view

file too large to diff

+ folly/folly/io/async/test/ScopedBoundPort.cpp view

file too large to diff

+ folly/folly/io/async/test/ScopedBoundPort.h view

file too large to diff

+ folly/folly/io/async/test/SocketPair.cpp view

file too large to diff

+ folly/folly/io/async/test/SocketPair.h view

file too large to diff

+ folly/folly/io/async/test/TFOUtil.h view

file too large to diff

+ folly/folly/io/async/test/TestSSLServer.h view

file too large to diff

+ folly/folly/io/async/test/TimeUtil.cpp view

file too large to diff

+ folly/folly/io/async/test/TimeUtil.h view

file too large to diff

+ folly/folly/io/async/test/UndelayedDestruction.h view

file too large to diff

+ folly/folly/io/async/test/Util.h view

file too large to diff

+ folly/folly/io/coro/ServerSocket.cpp view

file too large to diff

+ folly/folly/io/coro/ServerSocket.h view

file too large to diff

+ folly/folly/io/coro/Transport.cpp view

file too large to diff

+ folly/folly/io/coro/Transport.h view

file too large to diff

+ folly/folly/io/coro/TransportCallbackBase.h view

file too large to diff

+ folly/folly/io/coro/TransportCallbacks.h view

file too large to diff

+ folly/folly/json.h view

file too large to diff

+ folly/folly/json/DynamicConverter.h view

file too large to diff

+ folly/folly/json/DynamicParser-inl.h view

file too large to diff

+ folly/folly/json/DynamicParser.cpp view

file too large to diff

+ folly/folly/json/DynamicParser.h view

file too large to diff

+ folly/folly/json/JSONSchema.cpp view

file too large to diff

+ folly/folly/json/JSONSchema.h view

file too large to diff

+ folly/folly/json/JsonMockUtil.h view

file too large to diff

+ folly/folly/json/JsonTestUtil.cpp view

file too large to diff

+ folly/folly/json/JsonTestUtil.h view

file too large to diff

+ folly/folly/json/bser/Bser.h view

file too large to diff

+ folly/folly/json/bser/Dump.cpp view

file too large to diff

+ folly/folly/json/bser/Load.cpp view

file too large to diff

+ folly/folly/json/dynamic-inl.h view

file too large to diff

+ folly/folly/json/dynamic.cpp view

file too large to diff

+ folly/folly/json/dynamic.h view

file too large to diff

+ folly/folly/json/json.cpp view

file too large to diff

+ folly/folly/json/json.h view

file too large to diff

+ folly/folly/json/json_patch.cpp view

file too large to diff

+ folly/folly/json/json_patch.h view

file too large to diff

+ folly/folly/json/json_pointer.cpp view

file too large to diff

+ folly/folly/json/json_pointer.h view

file too large to diff

+ folly/folly/json_patch.h view

file too large to diff

+ folly/folly/json_pointer.h view

file too large to diff

+ folly/folly/lang/Access.h view

file too large to diff

+ folly/folly/lang/Align.h view

file too large to diff

+ folly/folly/lang/Aligned.h view

file too large to diff

+ folly/folly/lang/Assume.h view

file too large to diff

+ folly/folly/lang/Badge.h view

file too large to diff

+ folly/folly/lang/Bindings.h view

file too large to diff

+ folly/folly/lang/Bits.h view

file too large to diff

+ folly/folly/lang/BitsClass.h view

file too large to diff

+ folly/folly/lang/Builtin.h view

file too large to diff

+ folly/folly/lang/CArray.h view

file too large to diff

+ folly/folly/lang/CString.cpp view

file too large to diff

+ folly/folly/lang/CString.h view

file too large to diff

+ folly/folly/lang/Cast.h view

file too large to diff

+ folly/folly/lang/CheckedMath.h view

file too large to diff

+ folly/folly/lang/CustomizationPoint.h view

file too large to diff

+ folly/folly/lang/Exception.cpp view

file too large to diff

+ folly/folly/lang/Exception.h view

file too large to diff

+ folly/folly/lang/Extern.h view

file too large to diff

+ folly/folly/lang/Hint-inl.h view

file too large to diff

+ folly/folly/lang/Hint.h view

file too large to diff

+ folly/folly/lang/Keep.h view

file too large to diff

+ folly/folly/lang/New.h view

file too large to diff

+ folly/folly/lang/Ordering.h view

file too large to diff

+ folly/folly/lang/Pretty.h view

file too large to diff

+ folly/folly/lang/PropagateConst.h view

file too large to diff

+ folly/folly/lang/RValueReferenceWrapper.h view

file too large to diff

+ folly/folly/lang/SafeAlias-fwd.h view

file too large to diff

+ folly/folly/lang/SafeAssert.cpp view

file too large to diff

+ folly/folly/lang/SafeAssert.h view

file too large to diff

+ folly/folly/lang/StaticConst.h view

file too large to diff

+ folly/folly/lang/Switch.h view

file too large to diff

+ folly/folly/lang/Thunk.h view

file too large to diff

+ folly/folly/lang/ToAscii.cpp view

file too large to diff

+ folly/folly/lang/ToAscii.h view

file too large to diff

+ folly/folly/lang/TypeInfo.h view

file too large to diff

+ folly/folly/lang/UncaughtExceptions.cpp view

file too large to diff

+ folly/folly/lang/UncaughtExceptions.h view

file too large to diff

+ folly/folly/lang/VectorTraits.h view

file too large to diff

+ folly/folly/lang/named/Bindings.h view

file too large to diff

+ folly/folly/logging/AsyncFileWriter.cpp view

file too large to diff

+ folly/folly/logging/AsyncFileWriter.h view

file too large to diff

+ folly/folly/logging/AsyncLogWriter.cpp view

file too large to diff

+ folly/folly/logging/AsyncLogWriter.h view

file too large to diff

+ folly/folly/logging/AutoTimer.h view

file too large to diff

+ folly/folly/logging/BridgeFromGoogleLogging.cpp view

file too large to diff

+ folly/folly/logging/BridgeFromGoogleLogging.h view

file too large to diff

+ folly/folly/logging/CustomLogFormatter.cpp view

file too large to diff

+ folly/folly/logging/CustomLogFormatter.h view

file too large to diff

+ folly/folly/logging/FileHandlerFactory.cpp view

file too large to diff

+ folly/folly/logging/FileHandlerFactory.h view

file too large to diff

+ folly/folly/logging/FileWriterFactory.cpp view

file too large to diff

+ folly/folly/logging/FileWriterFactory.h view

file too large to diff

+ folly/folly/logging/GlogStyleFormatter.cpp view

file too large to diff

+ folly/folly/logging/GlogStyleFormatter.h view

file too large to diff

+ folly/folly/logging/ImmediateFileWriter.cpp view

file too large to diff

+ folly/folly/logging/ImmediateFileWriter.h view

file too large to diff

+ folly/folly/logging/Init.cpp view

file too large to diff

+ folly/folly/logging/Init.h view

file too large to diff

+ folly/folly/logging/InitWeak.cpp view

file too large to diff

+ folly/folly/logging/LogCategory.cpp view

file too large to diff

+ folly/folly/logging/LogCategory.h view

file too large to diff

+ folly/folly/logging/LogCategoryConfig.cpp view

file too large to diff

+ folly/folly/logging/LogCategoryConfig.h view

file too large to diff

+ folly/folly/logging/LogConfig.cpp view

file too large to diff

+ folly/folly/logging/LogConfig.h view

file too large to diff

+ folly/folly/logging/LogConfigParser.cpp view

file too large to diff

+ folly/folly/logging/LogConfigParser.h view

file too large to diff

+ folly/folly/logging/LogFormatter.h view

file too large to diff

+ folly/folly/logging/LogHandler.h view

file too large to diff

+ folly/folly/logging/LogHandlerConfig.cpp view

file too large to diff

+ folly/folly/logging/LogHandlerConfig.h view

file too large to diff

+ folly/folly/logging/LogHandlerFactory.h view

file too large to diff

+ folly/folly/logging/LogLevel.cpp view

file too large to diff

+ folly/folly/logging/LogLevel.h view

file too large to diff

+ folly/folly/logging/LogMessage.cpp view

file too large to diff

+ folly/folly/logging/LogMessage.h view

file too large to diff

+ folly/folly/logging/LogName.cpp view

file too large to diff

+ folly/folly/logging/LogName.h view

file too large to diff

+ folly/folly/logging/LogStream.cpp view

file too large to diff

+ folly/folly/logging/LogStream.h view

file too large to diff

+ folly/folly/logging/LogStreamProcessor.cpp view

file too large to diff

+ folly/folly/logging/LogStreamProcessor.h view

file too large to diff

+ folly/folly/logging/LogWriter.h view

file too large to diff

+ folly/folly/logging/Logger.cpp view

file too large to diff

+ folly/folly/logging/Logger.h view

file too large to diff

+ folly/folly/logging/LoggerDB.cpp view

file too large to diff

+ folly/folly/logging/LoggerDB.h view

file too large to diff

+ folly/folly/logging/ObjectToString.cpp view

file too large to diff

+ folly/folly/logging/ObjectToString.h view

file too large to diff

+ folly/folly/logging/RateLimiter.cpp view

file too large to diff

+ folly/folly/logging/RateLimiter.h view

file too large to diff

+ folly/folly/logging/StandardLogHandler.cpp view

file too large to diff

+ folly/folly/logging/StandardLogHandler.h view

file too large to diff

+ folly/folly/logging/StandardLogHandlerFactory.cpp view

file too large to diff

+ folly/folly/logging/StandardLogHandlerFactory.h view

file too large to diff

+ folly/folly/logging/StreamHandlerFactory.cpp view

file too large to diff

+ folly/folly/logging/StreamHandlerFactory.h view

file too large to diff

+ folly/folly/logging/xlog.cpp view

file too large to diff

+ folly/folly/logging/xlog.h view

file too large to diff

+ folly/folly/memcpy_select_aarch64.cpp view

file too large to diff

+ folly/folly/memory/Arena-inl.h view

file too large to diff

+ folly/folly/memory/Arena.h view

file too large to diff

+ folly/folly/memory/JemallocHugePageAllocator.cpp view

file too large to diff

+ folly/folly/memory/JemallocHugePageAllocator.h view

file too large to diff

+ folly/folly/memory/JemallocNodumpAllocator.cpp view

file too large to diff

+ folly/folly/memory/JemallocNodumpAllocator.h view

file too large to diff

+ folly/folly/memory/MallctlHelper.cpp view

file too large to diff

+ folly/folly/memory/MallctlHelper.h view

file too large to diff

+ folly/folly/memory/Malloc.h view

file too large to diff

+ folly/folly/memory/MemoryResource.h view

file too large to diff

+ folly/folly/memory/ReentrantAllocator.cpp view

file too large to diff

+ folly/folly/memory/ReentrantAllocator.h view

file too large to diff

+ folly/folly/memory/SanitizeAddress.cpp view

file too large to diff

+ folly/folly/memory/SanitizeAddress.h view

file too large to diff

+ folly/folly/memory/SanitizeLeak.cpp view

file too large to diff

+ folly/folly/memory/SanitizeLeak.h view

file too large to diff

+ folly/folly/memory/ThreadCachedArena.cpp view

file too large to diff

+ folly/folly/memory/ThreadCachedArena.h view

file too large to diff

+ folly/folly/memory/UninitializedMemoryHacks.h view

file too large to diff

+ folly/folly/memory/detail/MallocImpl.cpp view

file too large to diff

+ folly/folly/memory/detail/MallocImpl.h view

file too large to diff

+ folly/folly/memory/not_null-inl.h view

file too large to diff

+ folly/folly/memory/not_null.h view

file too large to diff

+ folly/folly/memory/shared_from_this_ptr.h view

file too large to diff

+ folly/folly/memset_select_aarch64.cpp view

file too large to diff

+ folly/folly/net/NetOps.cpp view

file too large to diff

+ folly/folly/net/NetOps.h view

file too large to diff

+ folly/folly/net/NetOpsDispatcher.cpp view

file too large to diff

+ folly/folly/net/NetOpsDispatcher.h view

file too large to diff

+ folly/folly/net/NetworkSocket.h view

file too large to diff

+ folly/folly/net/TcpInfo.cpp view

file too large to diff

+ folly/folly/net/TcpInfo.h view

file too large to diff

+ folly/folly/net/TcpInfoDispatcher.cpp view

file too large to diff

+ folly/folly/net/TcpInfoDispatcher.h view

file too large to diff

+ folly/folly/net/TcpInfoTypes.h view

file too large to diff

+ folly/folly/net/detail/SocketFileDescriptorMap.cpp view

file too large to diff

+ folly/folly/net/detail/SocketFileDescriptorMap.h view

file too large to diff

+ folly/folly/observer/CoreCachedObserver.h view

file too large to diff

+ folly/folly/observer/HazptrObserver.h view

file too large to diff

+ folly/folly/observer/Observable-inl.h view

file too large to diff

+ folly/folly/observer/Observable.h view

file too large to diff

+ folly/folly/observer/Observer-inl.h view

file too large to diff

+ folly/folly/observer/Observer-pre.h view

file too large to diff

+ folly/folly/observer/Observer.h view

file too large to diff

+ folly/folly/observer/ReadMostlyTLObserver.h view

file too large to diff

+ folly/folly/observer/SimpleObservable-inl.h view

file too large to diff

+ folly/folly/observer/SimpleObservable.h view

file too large to diff

+ folly/folly/observer/WithJitter-inl.h view

file too large to diff

+ folly/folly/observer/WithJitter.h view

file too large to diff

+ folly/folly/observer/detail/Core.cpp view

file too large to diff

+ folly/folly/observer/detail/Core.h view

file too large to diff

+ folly/folly/observer/detail/GraphCycleDetector.h view

file too large to diff

+ folly/folly/observer/detail/ObserverManager.cpp view

file too large to diff

+ folly/folly/observer/detail/ObserverManager.h view

file too large to diff

+ folly/folly/poly/Nullable.h view

file too large to diff

+ folly/folly/poly/Regular.h view

file too large to diff

+ folly/folly/portability/Asm.h view

file too large to diff

+ folly/folly/portability/Atomic.h view

file too large to diff

+ folly/folly/portability/Builtins.cpp view

file too large to diff

+ folly/folly/portability/Builtins.h view

file too large to diff

+ folly/folly/portability/Config.h view

file too large to diff

+ folly/folly/portability/Constexpr.h view

file too large to diff

+ folly/folly/portability/Dirent.cpp view

file too large to diff

+ folly/folly/portability/Dirent.h view

file too large to diff

+ folly/folly/portability/Event.h view

file too large to diff

+ folly/folly/portability/Fcntl.cpp view

file too large to diff

+ folly/folly/portability/Fcntl.h view

file too large to diff

+ folly/folly/portability/Filesystem.cpp view

file too large to diff

+ folly/folly/portability/Filesystem.h view

file too large to diff

+ folly/folly/portability/FmtCompile.h view

file too large to diff

+ folly/folly/portability/GFlags.cpp view

file too large to diff

+ folly/folly/portability/GFlags.h view

file too large to diff

+ folly/folly/portability/GMock.h view

file too large to diff

+ folly/folly/portability/GTest.h view

file too large to diff

+ folly/folly/portability/GTestProd.h view

file too large to diff

+ folly/folly/portability/IOVec.h view

file too large to diff

+ folly/folly/portability/Libgen.cpp view

file too large to diff

+ folly/folly/portability/Libgen.h view

file too large to diff

+ folly/folly/portability/Libunwind.h view

file too large to diff

+ folly/folly/portability/Malloc.cpp view

file too large to diff

+ folly/folly/portability/Malloc.h view

file too large to diff

+ folly/folly/portability/Math.h view

file too large to diff

+ folly/folly/portability/Memory.h view

file too large to diff

+ folly/folly/portability/OpenSSL.cpp view

file too large to diff

+ folly/folly/portability/OpenSSL.h view

file too large to diff

+ folly/folly/portability/PThread.cpp view

file too large to diff

+ folly/folly/portability/PThread.h view

file too large to diff

+ folly/folly/portability/Sched.cpp view

file too large to diff

+ folly/folly/portability/Sched.h view

file too large to diff

+ folly/folly/portability/Sockets.cpp view

file too large to diff

+ folly/folly/portability/Sockets.h view

file too large to diff

+ folly/folly/portability/SourceLocation.h view

file too large to diff

+ folly/folly/portability/Stdio.cpp view

file too large to diff

+ folly/folly/portability/Stdio.h view

file too large to diff

+ folly/folly/portability/Stdlib.cpp view

file too large to diff

+ folly/folly/portability/Stdlib.h view

file too large to diff

+ folly/folly/portability/String.cpp view

file too large to diff

+ folly/folly/portability/String.h view

file too large to diff

+ folly/folly/portability/SysFile.cpp view

file too large to diff

+ folly/folly/portability/SysFile.h view

file too large to diff

+ folly/folly/portability/SysMembarrier.cpp view

file too large to diff

+ folly/folly/portability/SysMembarrier.h view

file too large to diff

+ folly/folly/portability/SysMman.cpp view

file too large to diff

+ folly/folly/portability/SysMman.h view

file too large to diff

+ folly/folly/portability/SysResource.cpp view

file too large to diff

+ folly/folly/portability/SysResource.h view

file too large to diff

+ folly/folly/portability/SysStat.cpp view

file too large to diff

+ folly/folly/portability/SysStat.h view

file too large to diff

+ folly/folly/portability/SysSyscall.h view

file too large to diff

+ folly/folly/portability/SysTime.cpp view

file too large to diff

+ folly/folly/portability/SysTime.h view

file too large to diff

+ folly/folly/portability/SysTypes.h view

file too large to diff

+ folly/folly/portability/SysUio.cpp view

file too large to diff

+ folly/folly/portability/SysUio.h view

file too large to diff

+ folly/folly/portability/Syslog.h view

file too large to diff

+ folly/folly/portability/Time.cpp view

file too large to diff

+ folly/folly/portability/Time.h view

file too large to diff

+ folly/folly/portability/Unistd.cpp view

file too large to diff

+ folly/folly/portability/Unistd.h view

file too large to diff

+ folly/folly/portability/Windows.h view

file too large to diff

+ folly/folly/portability/openat2.h view

file too large to diff

+ folly/folly/python/AsyncioExecutor.h view

file too large to diff

+ folly/folly/python/Weak.h view

file too large to diff

+ folly/folly/python/async_generator.h view

file too large to diff

+ folly/folly/python/coro.h view

file too large to diff

+ folly/folly/python/error.h view

file too large to diff

+ folly/folly/python/executor.h view

file too large to diff

+ folly/folly/python/futures.h view

file too large to diff

+ folly/folly/python/import.h view

file too large to diff

+ folly/folly/python/iobuf.h view

file too large to diff

+ folly/folly/random/xoshiro256pp.h view

file too large to diff

+ folly/folly/result/gtest_helpers.h view

file too large to diff

+ folly/folly/result/result.cpp view

file too large to diff

+ folly/folly/result/result.h view

file too large to diff

+ folly/folly/result/try.h view

file too large to diff

+ folly/folly/settings/CommandLineParser.cpp view

file too large to diff

+ folly/folly/settings/CommandLineParser.h view

file too large to diff

+ folly/folly/settings/Immutables.cpp view

file too large to diff

+ folly/folly/settings/Immutables.h view

file too large to diff

+ folly/folly/settings/Observer.h view

file too large to diff

+ folly/folly/settings/Settings.cpp view

file too large to diff

+ folly/folly/settings/Settings.h view

file too large to diff

+ folly/folly/settings/SettingsAccessorProxy.cpp view

file too large to diff

+ folly/folly/settings/SettingsAccessorProxy.h view

file too large to diff

+ folly/folly/settings/Types.cpp view

file too large to diff

+ folly/folly/settings/Types.h view

file too large to diff

+ folly/folly/settings/detail/SettingsImpl.h view

file too large to diff

+ folly/folly/small_vector.h view

file too large to diff

+ folly/folly/sorted_vector_types.h view

file too large to diff

+ folly/folly/ssl/OpenSSLCertUtils.cpp view

file too large to diff

+ folly/folly/ssl/OpenSSLCertUtils.h view

file too large to diff

+ folly/folly/ssl/OpenSSLHash.cpp view

file too large to diff

+ folly/folly/ssl/OpenSSLHash.h view

file too large to diff

+ folly/folly/ssl/OpenSSLKeyUtils.cpp view

file too large to diff

+ folly/folly/ssl/OpenSSLKeyUtils.h view

file too large to diff

+ folly/folly/ssl/OpenSSLPtrTypes.h view

file too large to diff

+ folly/folly/ssl/OpenSSLTicketHandler.h view

file too large to diff

+ folly/folly/ssl/OpenSSLVersionFinder.h view

file too large to diff

+ folly/folly/ssl/PasswordCollector.cpp view

file too large to diff

+ folly/folly/ssl/PasswordCollector.h view

file too large to diff

+ folly/folly/ssl/SSLSession.h view

file too large to diff

+ folly/folly/ssl/SSLSessionManager.cpp view

file too large to diff

+ folly/folly/ssl/SSLSessionManager.h view

file too large to diff

+ folly/folly/ssl/detail/OpenSSLSession.cpp view

file too large to diff

+ folly/folly/ssl/detail/OpenSSLSession.h view

file too large to diff

+ folly/folly/stats/BucketedTimeSeries-inl.h view

file too large to diff

+ folly/folly/stats/BucketedTimeSeries.h view

file too large to diff

+ folly/folly/stats/DigestBuilder-inl.h view

file too large to diff

+ folly/folly/stats/DigestBuilder.h view

file too large to diff

+ folly/folly/stats/Histogram-inl.h view

file too large to diff

+ folly/folly/stats/Histogram.h view

file too large to diff

+ folly/folly/stats/MultiLevelTimeSeries-inl.h view

file too large to diff

+ folly/folly/stats/MultiLevelTimeSeries.h view

file too large to diff

+ folly/folly/stats/QuantileEstimator-inl.h view

file too large to diff

+ folly/folly/stats/QuantileEstimator.cpp view

file too large to diff

+ folly/folly/stats/QuantileEstimator.h view

file too large to diff

+ folly/folly/stats/QuantileHistogram-inl.h view

file too large to diff

+ folly/folly/stats/QuantileHistogram.h view

file too large to diff

+ folly/folly/stats/StreamingStats.h view

file too large to diff

+ folly/folly/stats/TDigest.cpp view

file too large to diff

+ folly/folly/stats/TDigest.h view

file too large to diff

+ folly/folly/stats/TimeseriesHistogram-inl.h view

file too large to diff

+ folly/folly/stats/TimeseriesHistogram.h view

file too large to diff

+ folly/folly/stats/detail/Bucket.h view

file too large to diff

+ folly/folly/stats/detail/BufferedStat-inl.h view

file too large to diff

+ folly/folly/stats/detail/BufferedStat.h view

file too large to diff

+ folly/folly/stats/detail/DoubleRadixSort.cpp view

file too large to diff

+ folly/folly/stats/detail/DoubleRadixSort.h view

file too large to diff

+ folly/folly/stats/detail/SlidingWindow-inl.h view

file too large to diff

+ folly/folly/stats/detail/SlidingWindow.h view

file too large to diff

+ folly/folly/stop_watch.h view

file too large to diff

+ folly/folly/synchronization/AsymmetricThreadFence.cpp view

file too large to diff

+ folly/folly/synchronization/AsymmetricThreadFence.h view

file too large to diff

+ folly/folly/synchronization/AtomicNotification-inl.h view

file too large to diff

+ folly/folly/synchronization/AtomicNotification.cpp view

file too large to diff

+ folly/folly/synchronization/AtomicNotification.h view

file too large to diff

+ folly/folly/synchronization/AtomicRef.h view

file too large to diff

+ folly/folly/synchronization/AtomicStruct.h view

file too large to diff

+ folly/folly/synchronization/AtomicUtil-inl.h view

file too large to diff

+ folly/folly/synchronization/AtomicUtil.h view

file too large to diff

+ folly/folly/synchronization/Baton.h view

file too large to diff

+ folly/folly/synchronization/CallOnce.h view

file too large to diff

+ folly/folly/synchronization/DelayedInit.h view

file too large to diff

+ folly/folly/synchronization/DistributedMutex-inl.h view

file too large to diff

+ folly/folly/synchronization/DistributedMutex.cpp view

file too large to diff

+ folly/folly/synchronization/DistributedMutex.h view

file too large to diff

+ folly/folly/synchronization/EventCount.h view

file too large to diff

+ folly/folly/synchronization/FlatCombining.h view

file too large to diff

+ folly/folly/synchronization/Hazptr-fwd.h view

file too large to diff

+ folly/folly/synchronization/Hazptr.cpp view

file too large to diff

+ folly/folly/synchronization/Hazptr.h view

file too large to diff

+ folly/folly/synchronization/HazptrDomain.cpp view

file too large to diff

+ folly/folly/synchronization/HazptrDomain.h view

file too large to diff

+ folly/folly/synchronization/HazptrHolder.h view

file too large to diff

+ folly/folly/synchronization/HazptrObj.h view

file too large to diff

+ folly/folly/synchronization/HazptrObjLinked.h view

file too large to diff

+ folly/folly/synchronization/HazptrRec.h view

file too large to diff

+ folly/folly/synchronization/HazptrThrLocal.h view

file too large to diff

+ folly/folly/synchronization/HazptrThreadPoolExecutor.cpp view

file too large to diff

+ folly/folly/synchronization/HazptrThreadPoolExecutor.h view

file too large to diff

+ folly/folly/synchronization/Latch.h view

file too large to diff

+ folly/folly/synchronization/LifoSem.h view

file too large to diff

+ folly/folly/synchronization/Lock.h view

file too large to diff

+ folly/folly/synchronization/MicroSpinLock.h view

file too large to diff

+ folly/folly/synchronization/NativeSemaphore.h view

file too large to diff

+ folly/folly/synchronization/ParkingLot.cpp view

file too large to diff

+ folly/folly/synchronization/ParkingLot.h view

file too large to diff

+ folly/folly/synchronization/PicoSpinLock.h view

file too large to diff

+ folly/folly/synchronization/RWSpinLock.h view

file too large to diff

+ folly/folly/synchronization/Rcu.cpp view

file too large to diff

+ folly/folly/synchronization/Rcu.h view

file too large to diff

+ folly/folly/synchronization/RelaxedAtomic.h view

file too large to diff

+ folly/folly/synchronization/SanitizeThread.cpp view

file too large to diff

+ folly/folly/synchronization/SanitizeThread.h view

file too large to diff

+ folly/folly/synchronization/SaturatingSemaphore.h view

file too large to diff

+ folly/folly/synchronization/SmallLocks.h view

file too large to diff

+ folly/folly/synchronization/ThrottledLifoSem.h view

file too large to diff

+ folly/folly/synchronization/WaitOptions.cpp view

file too large to diff

+ folly/folly/synchronization/WaitOptions.h view

file too large to diff

+ folly/folly/synchronization/detail/AtomicUtils.h view

file too large to diff

+ folly/folly/synchronization/detail/Hardware.cpp view

file too large to diff

+ folly/folly/synchronization/detail/Hardware.h view

file too large to diff

+ folly/folly/synchronization/detail/HazptrUtils.h view

file too large to diff

+ folly/folly/synchronization/detail/InlineFunctionRef.h view

file too large to diff

+ folly/folly/synchronization/detail/Sleeper.cpp view

file too large to diff

+ folly/folly/synchronization/detail/Sleeper.h view

file too large to diff

+ folly/folly/synchronization/detail/Spin.h view

file too large to diff

+ folly/folly/synchronization/detail/ThreadCachedLists.h view

file too large to diff

+ folly/folly/synchronization/detail/ThreadCachedReaders.h view

file too large to diff

+ folly/folly/synchronization/detail/ThreadCachedTag.h view

file too large to diff

+ folly/folly/synchronization/example/HazptrLockFreeLIFO.h view

file too large to diff

+ folly/folly/synchronization/example/HazptrSWMRSet.h view

file too large to diff

+ folly/folly/synchronization/example/HazptrWideCAS.h view

file too large to diff

+ folly/folly/synchronization/test/Semaphore.h view

file too large to diff

+ folly/folly/system/AtFork.cpp view

file too large to diff

+ folly/folly/system/AtFork.h view

file too large to diff

+ folly/folly/system/AuxVector.h view

file too large to diff

+ folly/folly/system/EnvUtil.cpp view

file too large to diff

+ folly/folly/system/EnvUtil.h view

file too large to diff

+ folly/folly/system/HardwareConcurrency.cpp view

file too large to diff

+ folly/folly/system/HardwareConcurrency.h view

file too large to diff

+ folly/folly/system/MemoryMapping.cpp view

file too large to diff

+ folly/folly/system/MemoryMapping.h view

file too large to diff

+ folly/folly/system/Pid.cpp view

file too large to diff

+ folly/folly/system/Pid.h view

file too large to diff

+ folly/folly/system/Shell.cpp view

file too large to diff

+ folly/folly/system/Shell.h view

file too large to diff

+ folly/folly/system/ThreadId.cpp view

file too large to diff

+ folly/folly/system/ThreadId.h view

file too large to diff

+ folly/folly/system/ThreadName.cpp view

file too large to diff

+ folly/folly/system/ThreadName.h view

file too large to diff

+ folly/folly/test/DeterministicSchedule.h view

file too large to diff

+ folly/folly/test/TestUtils.h view

file too large to diff

+ folly/folly/testing/TestUtil.cpp view

file too large to diff

+ folly/folly/testing/TestUtil.h view

file too large to diff

+ folly/folly/tracing/AsyncStack-inl.h view

file too large to diff

+ folly/folly/tracing/AsyncStack.cpp view

file too large to diff

+ folly/folly/tracing/AsyncStack.h view

file too large to diff

+ folly/folly/tracing/ScopedTraceSection.h view

file too large to diff

+ folly/folly/tracing/StaticTracepoint-ELF.h view

file too large to diff

+ folly/folly/tracing/StaticTracepoint.h view

file too large to diff