diff --git a/BUILD.bazel b/BUILD.bazel index 264ebd51..cc6f32e4 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -10,6 +10,8 @@ sse_copts = ['-msse', '-msse2', '-msse4.1', '-msse4.2','-mpclmul'] arm_copts = ['-march=armv8-a'] apple_silicon_copts = [ '-mcpu=apple-m1' ] sve2_copts = ['-march=armv8-a+sve2', '-msve-vector-bits=128'] +riscv64_copts = ['-march=rv64gc'] +riscv64_zbb_copts = ['-march=rv64gc_zbb'] default_copts = ['-march=native'] benchmark_copts = ['-O3', '-DNDEBUG', '-std=c++17'] static_dispatch_copts = [] @@ -29,6 +31,7 @@ filesystem_linkopts = select({ string_flag( name = "sonic_arch", build_setting_default = "default", + values = ["default", "haswell", "westmere", "arm", "sve2", "riscv64", "riscv64_zbb"], ) string_flag( @@ -60,6 +63,16 @@ config_setting( flag_values = {":sonic_arch": "sve2"}, ) +config_setting( + name = "riscv64_build", + flag_values = {":sonic_arch": "riscv64"}, +) + +config_setting( + name = "riscv64_zbb_build", + flag_values = {":sonic_arch": "riscv64_zbb"}, +) + config_setting( name = "apple_silicon_build", values = {"cpu": "darwin_arm64"}, @@ -136,6 +149,8 @@ cc_binary( "sse_build": sse_copts + benchmark_copts, "avx2_build": avx2_copts + benchmark_copts, "apple_silicon_build": apple_silicon_copts + benchmark_copts, + "riscv64_build": riscv64_copts + benchmark_copts, + "riscv64_zbb_build": riscv64_zbb_copts + benchmark_copts, "//conditions:default": default_copts + benchmark_copts, }) +\ macos_version_copts +\ @@ -183,6 +198,8 @@ cc_test( "sse_build": sse_copts + benchmark_copts, "avx2_build": avx2_copts + benchmark_copts, "apple_silicon_build": apple_silicon_copts + benchmark_copts, + "riscv64_build": riscv64_copts + benchmark_copts, + "riscv64_zbb_build": riscv64_zbb_copts + benchmark_copts, "//conditions:default": default_copts + benchmark_copts, }) +\ macos_version_copts +\ diff --git a/CMakeLists.txt b/CMakeLists.txt index 020b9d2b..69bd55c3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -11,6 +11,7 @@ option(BUILD_UNITTEST "Build unittest." ON) option(BUILD_FUZZ "Build fuzz." OFF) option(BUILD_BENCH "Build benchmark." OFF) option(ENABLE_SVE2_128 "Build for Arm SVE2 with 128 bit vector size" OFF) +option(ENABLE_RISCV_ZBB "Enable RISC-V Zbb extension for bit manipulation optimizations" OFF) set(CMAKE_CXX_EXTENSIONS OFF) if(BUILD_UNITTEST) diff --git a/cmake/set_arch_flags.cmake b/cmake/set_arch_flags.cmake index 43948401..8c679989 100644 --- a/cmake/set_arch_flags.cmake +++ b/cmake/set_arch_flags.cmake @@ -12,6 +12,12 @@ function(set_arch_flags target arch) target_compile_options(${target} PRIVATE -march=armv8-a) endif() endif() + elseif(arch MATCHES "riscv64|riscv") + if(ENABLE_RISCV_ZBB) + target_compile_options(${target} PRIVATE -march=rv64gc_zbb) + else() + target_compile_options(${target} PRIVATE -march=rv64gc) + endif() else() message(FATAL_ERROR "Unsupported architecture: ${arch}") endif() diff --git a/include/sonic/allocator.h b/include/sonic/allocator.h index 52958acc..15e8de41 100644 --- a/include/sonic/allocator.h +++ b/include/sonic/allocator.h @@ -93,8 +93,9 @@ class SpinLock { __builtin_ia32_pause(); #elif defined(__aarch64__) || defined(_M_ARM64) asm volatile("yield"); +#elif defined(__riscv) + __asm__ volatile("pause"); #endif - #endif } } diff --git a/include/sonic/dom/flags.h b/include/sonic/dom/flags.h index c6849864..e4afdde2 100644 --- a/include/sonic/dom/flags.h +++ b/include/sonic/dom/flags.h @@ -28,6 +28,22 @@ enum ParseFlag { kParseOverflowNumAsNumStr = 1 << 3, }; +// Scoped spelling used by newer downstream code while retaining the legacy +// ParseFlag template API on this branch. +struct ParseFlags { + static constexpr ParseFlag kParseDefault = ::kParseDefault; + static constexpr ParseFlag kParseAllowUnescapedControlChars = + ::kParseAllowUnescapedControlChars; + static constexpr ParseFlag kParseIntegerAsRaw = ::kParseIntegerAsRaw; + static constexpr ParseFlag kParseOverflowNumAsNumStr = + ::kParseOverflowNumAsNumStr; +}; + +constexpr ParseFlag operator|(ParseFlag lhs, ParseFlag rhs) { + return static_cast(static_cast(lhs) | + static_cast(rhs)); +} + // SerializeFlags is one-hot encoded for different serializing option. // User can define customed flags through combinations. enum SerializeFlags { diff --git a/include/sonic/internal/arch/riscv/base.h b/include/sonic/internal/arch/riscv/base.h new file mode 100644 index 00000000..f7d37692 --- /dev/null +++ b/include/sonic/internal/arch/riscv/base.h @@ -0,0 +1,78 @@ +// Copyright 2018-2019 The simdjson authors + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 + +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// This file may have been modified by ByteDance authors. All ByteDance +// Modifications are Copyright 2022 ByteDance Authors. + +#pragma once + +#include + +#include +#include + +namespace sonic_json { +namespace internal { +namespace riscv { + +// We sometimes call trailing_zero on inputs that are zero, +// but the algorithms do not end up using the returned value. +// Sadly, sanitizers are not smart enough to figure it out. + +sonic_force_inline __attribute__((no_sanitize("undefined"))) int TrailingZeroes( + uint64_t input_num) { + return __builtin_ctzll(input_num); +} + +/* result might be undefined when input_num is zero */ +sonic_force_inline uint64_t ClearLowestBit(uint64_t input_num) { + return input_num & (input_num - 1); +} + +/* result might be undefined when input_num is zero */ +sonic_force_inline int LeadingZeroes(uint64_t input_num) { + return __builtin_clzll(input_num); +} + +sonic_force_inline long long int CountOnes(uint64_t input_num) { + return __builtin_popcountll(input_num); +} + +sonic_force_inline uint64_t PrefixXor(uint64_t bitmask) { + bitmask ^= bitmask << 1; + bitmask ^= bitmask << 2; + bitmask ^= bitmask << 4; + bitmask ^= bitmask << 8; + bitmask ^= bitmask << 16; + bitmask ^= bitmask << 32; + return bitmask; +} + +template +sonic_force_inline void Xmemcpy(void* dst_, const void* src_, size_t chunks) { + std::memcpy(dst_, src_, chunks * ChunkSize); +} + +sonic_force_inline bool InlinedMemcmpEq(const void* _a, const void* _b, + size_t s) { + return std::memcmp(_a, _b, s) == 0; +} + +sonic_force_inline int InlinedMemcmp(const void* _l, const void* _r, size_t s) { + return std::memcmp(_l, _r, s); +} + +} // namespace riscv +} // namespace internal +} // namespace sonic_json diff --git a/include/sonic/internal/arch/riscv/itoa.h b/include/sonic/internal/arch/riscv/itoa.h new file mode 100644 index 00000000..d8036ad0 --- /dev/null +++ b/include/sonic/internal/arch/riscv/itoa.h @@ -0,0 +1,85 @@ +/* + * Copyright 2022 ByteDance Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include + +#include +#include + +namespace sonic_json { +namespace internal { +namespace riscv { + +// Scalar implementation of Utoa_4_helper: convert num {abcd} to digits +static sonic_force_inline void Utoa_4_helper_scalar(uint16_t num, + uint8_t digits[4]) { + digits[0] = static_cast((num / 1000) % 10); + digits[1] = static_cast((num / 100) % 10); + digits[2] = static_cast((num / 10) % 10); + digits[3] = static_cast(num % 10); +} + +// Convert num's each digit to bytes. +// num's digits as abcdefgh (high bits is 0 if not enough) +// The converted digits are { a, b, c, d, e, f, g, h } +static sonic_force_inline void Utoa_Scalar(uint32_t num, uint8_t digits[8]) { + uint16_t hi = static_cast(num % 10000); // {efgh} + uint16_t lo = static_cast(num / 10000); // {abcd} + + uint8_t lo_digits[4]; + uint8_t hi_digits[4]; + Utoa_4_helper_scalar(lo, lo_digits); + Utoa_4_helper_scalar(hi, hi_digits); + + digits[0] = lo_digits[0]; + digits[1] = lo_digits[1]; + digits[2] = lo_digits[2]; + digits[3] = lo_digits[3]; + digits[4] = hi_digits[0]; + digits[5] = hi_digits[1]; + digits[6] = hi_digits[2]; + digits[7] = hi_digits[3]; +} + +static sonic_force_inline char *Utoa_8(uint32_t val, char *out) { + uint8_t digits[8]; + Utoa_Scalar(val, digits); + for (int i = 0; i < 8; i++) { + out[i] = static_cast('0' + digits[i]); + } + return out + 8; +} + +static sonic_force_inline char *Utoa_16(uint64_t val, char *out) { + uint8_t digits_hi[8]; + uint8_t digits_lo[8]; + Utoa_Scalar(static_cast(val / 100000000), digits_hi); + Utoa_Scalar(static_cast(val % 100000000), digits_lo); + + for (int i = 0; i < 8; i++) { + out[i] = static_cast('0' + digits_hi[i]); + } + for (int i = 0; i < 8; i++) { + out[i + 8] = static_cast('0' + digits_lo[i]); + } + return out + 16; +} + +} // namespace riscv +} // namespace internal +} // namespace sonic_json diff --git a/include/sonic/internal/arch/riscv/quote.h b/include/sonic/internal/arch/riscv/quote.h new file mode 100644 index 00000000..b8e77e84 --- /dev/null +++ b/include/sonic/internal/arch/riscv/quote.h @@ -0,0 +1,242 @@ +/* + * Copyright 2022 ByteDance Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#define VEC_LEN 16 + +#include +#include + +#include +#include + +#include "../common/quote_common.h" +#include "../common/quote_tables.h" +#include "base.h" +#include "simd.h" +#include "unicode.h" + +#ifndef PAGE_SIZE +#define PAGE_SIZE 4096 +#endif + +#ifdef __GNUC__ +#if defined(__SANITIZE_THREAD__) || defined(__SANITIZE_ADDRESS__) || \ + defined(__SANITIZE_LEAK__) || defined(__SANITIZE_UNDEFINED__) +#ifndef SONIC_USE_SANITIZE +#define SONIC_USE_SANITIZE +#endif +#endif +#endif + +#if defined(__clang__) +#if defined(__has_feature) +#if __has_feature(address_sanitizer) || __has_feature(thread_sanitizer) || \ + __has_feature(memory_sanitizer) || \ + __has_feature(undefined_behavior_sanitizer) || \ + __has_feature(leak_sanitizer) +#ifndef SONIC_USE_SANITIZE +#define SONIC_USE_SANITIZE +#endif +#endif +#endif +#endif + +#define MOVE_N_CHARS(src, N) \ + { \ + (src) += (N); \ + nb -= (N); \ + dst += (N); \ + } + +namespace sonic_json { +namespace internal { +namespace riscv { + +using sonic_json::internal::common::handle_unicode_codepoint; + +static sonic_force_inline uint64_t CopyAndGetEscapMask128(const char* src, + char* dst, + bool escape_emoji) { + simd8 v = + simd8::load(reinterpret_cast(src)); + v.store(reinterpret_cast(dst)); + + simd8 bs = simd8::splat('\\'); + simd8 quote = simd8::splat('"'); + simd8 ctrl = simd8::splat('\x20'); + + simd8 m1 = (v == bs); + simd8 m2 = (v == quote); + simd8 m3 = (v < ctrl); + + simd8 mask = m1 | m2; + mask = mask | m3; + if (escape_emoji) { + simd8 emoji = simd8::splat(0xF0); + simd8 m_emoji = (v >= emoji); + mask = mask | m_emoji; + } + + return mask.to_bitmask(); +} + +sonic_static_inline char* Quote(const char* src, size_t nb, char* dst, + bool escape_emoji) { + *dst++ = '"'; + sonic_assert(nb < (1ULL << 32)); + uint64_t mm; + int cn; + + /* VEC_LEN byte loop */ + while (nb >= VEC_LEN) { + /* check for matches */ + if ((mm = CopyAndGetEscapMask128(src, dst, escape_emoji)) != 0) { + cn = TrailingZeroes(mm) >> 2; + MOVE_N_CHARS(src, cn); + DoEscape(src, dst, nb, escape_emoji); + } else { + /* move to next block */ + MOVE_N_CHARS(src, VEC_LEN); + } + } + + if (nb > 0) { + char tmp_src[64]; + const char* src_r; +#ifdef SONIC_USE_SANITIZE + if (0) { +#else + /* This code would cause address sanitizer report heap-buffer-overflow. */ + if (((size_t)(src) & (PAGE_SIZE - 1)) <= (PAGE_SIZE - 64)) { + src_r = src; +#endif + } else { + std::memcpy(tmp_src, src, nb); + src_r = tmp_src; + } + while (nb > 0) { + mm = CopyAndGetEscapMask128(src_r, dst, escape_emoji) & + (0xFFFFFFFFFFFFFFFF >> ((VEC_LEN - nb) << 2)); + if (mm) { + cn = TrailingZeroes(mm) >> 2; + MOVE_N_CHARS(src_r, cn); + DoEscape(src_r, dst, nb, escape_emoji); + } else { + dst += nb; + nb = 0; + } + } + } + + *dst++ = '"'; + return dst; +} + +sonic_force_inline size_t +parseStringInplace(uint8_t*& src, SonicError& err, + bool allow_unescaped_control_chars = false) { +#define SONIC_REPEAT8(v) {v v v v v v v v} + uint8_t* dst = src; + uint8_t* sdst = src; + while (1) { + find: + auto block = StringBlock::Find(src); + if (block.HasQuoteFirst(allow_unescaped_control_chars)) { + int idx = block.QuoteIndex(); + src += idx; + *src++ = '\0'; + return src - sdst - 1; + } + if (!allow_unescaped_control_chars && block.HasUnescaped()) { + err = kParseErrorUnEscaped; + return 0; + } + if (!block.HasBackslash()) { + src += VEC_LEN; + goto find; + } + + /* find out where the backspace is */ + auto bs_dist = block.BsIndex(); + src += bs_dist; + dst = src; + cont: + uint8_t escape_char = src[1]; + if (sonic_unlikely(escape_char == 'u')) { + if (!handle_unicode_codepoint(const_cast(&src), &dst)) { + err = kParseErrorEscapedUnicode; + return 0; + } + } else { + *dst = kEscapedMap[escape_char]; + if (sonic_unlikely(*dst == 0u)) { + err = kParseErrorEscapedFormat; + return 0; + } + src += 2; + dst += 1; + } + // fast path for continuous escaped chars + if (*src == '\\') { + bs_dist = 0; + goto cont; + } + + find_and_move: + // Copy the next n bytes, and find the backslash and quote in them. + simd8 v = simd8::load(src); + block = StringBlock::Find(v); + // If the next thing is the end quote, copy and return + if (block.HasQuoteFirst(allow_unescaped_control_chars)) { + // we encountered quotes first. Move dst to point to quotes and exit + while (1) { + SONIC_REPEAT8(if (sonic_unlikely(*src == '"')) break; + else { *dst++ = *src++; }); + } + *dst = '\0'; + src++; + return dst - sdst; + } + if (!allow_unescaped_control_chars && block.HasUnescaped()) { + err = kParseErrorUnEscaped; + return 0; + } + if (!block.HasBackslash()) { + /* they are the same. Since they can't co-occur, it means we + * encountered neither. */ + v.store(dst); + src += VEC_LEN; + dst += VEC_LEN; + goto find_and_move; + } + while (1) { + SONIC_REPEAT8(if (sonic_unlikely(*src == '\\')) break; + else { *dst++ = *src++; }); + } + goto cont; + } + sonic_assert(false); +#undef SONIC_REPEAT8 +} + +} // namespace riscv +} // namespace internal +} // namespace sonic_json + +#undef VEC_LEN +#undef MOVE_N_CHARS diff --git a/include/sonic/internal/arch/riscv/simd.h b/include/sonic/internal/arch/riscv/simd.h new file mode 100644 index 00000000..c7673131 --- /dev/null +++ b/include/sonic/internal/arch/riscv/simd.h @@ -0,0 +1,760 @@ +// Copyright 2018-2019 The simdjson authors + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 + +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// This file may have been modified by ByteDance authors. All ByteDance +// Modifications are Copyright 2022 ByteDance Authors. + +// RISC-V scalar simd facade with optional Zbb byte-search helpers. + +#pragma once + +#include + +#include +#include + +namespace sonic_json { +namespace internal { +namespace riscv { + +#if defined(__riscv) && defined(__riscv_xlen) && (__riscv_xlen == 64) && \ + defined(__riscv_zbb) +#define SONIC_RISCV_SIMD_HAS_ZBB 1 + +namespace riscv_zbb_detail { + +static constexpr uint64_t kByteMsb = 0x8080808080808080ULL; +static constexpr uint64_t kByteLsb = 0x0101010101010101ULL; +static constexpr uint64_t kMovemaskMultiplier = 0x0002040810204081ULL; + +sonic_force_inline uint64_t load_u64(const uint8_t* ptr) { + uint64_t value; + std::memcpy(&value, ptr, sizeof(value)); + return value; +} + +sonic_force_inline uint64_t orc_b(uint64_t value) { + uint64_t result; + __asm__ volatile("orc.b %0, %1" : "=r"(result) : "r"(value)); + return result; +} + +sonic_force_inline uint32_t byte_msb_to_mask(uint64_t value) { + return static_cast(((value & kByteMsb) * kMovemaskMultiplier) >> + 56); +} + +sonic_force_inline uint32_t nonzero_byte_mask(uint64_t value) { + return byte_msb_to_mask(orc_b(value)); +} + +sonic_force_inline uint32_t zero_byte_mask(uint64_t value) { + return byte_msb_to_mask(~orc_b(value)); +} + +sonic_force_inline uint32_t bool_mask16(const uint8_t* ptr) { + uint32_t low = nonzero_byte_mask(load_u64(ptr)); + uint32_t high = nonzero_byte_mask(load_u64(ptr + 8)); + return low | (high << 8); +} + +sonic_force_inline uint32_t eq_mask16(const uint8_t* ptr, uint8_t value) { + uint64_t repeated = kByteLsb * value; + uint32_t low = zero_byte_mask(load_u64(ptr) ^ repeated); + uint32_t high = zero_byte_mask(load_u64(ptr + 8) ^ repeated); + return low | (high << 8); +} + +sonic_force_inline uint32_t lteq_mask16(const uint8_t* ptr, uint8_t value) { + if (value == 0xFFu) { + return 0xFFFFu; + } + if (value == 0x1Fu) { + constexpr uint64_t high_three_bits = 0xE0E0E0E0E0E0E0E0ULL; + uint32_t low = zero_byte_mask(load_u64(ptr) & high_three_bits); + uint32_t high = zero_byte_mask(load_u64(ptr + 8) & high_three_bits); + return low | (high << 8); + } + if (value == 0x7Fu) { + uint32_t low = byte_msb_to_mask(~load_u64(ptr)); + uint32_t high = byte_msb_to_mask(~load_u64(ptr + 8)); + return low | (high << 8); + } + + uint32_t mask = 0; + for (int i = 0; i < 16; i++) { + if (ptr[i] <= value) mask |= (1u << i); + } + return mask; +} + +} // namespace riscv_zbb_detail + +#else +#define SONIC_RISCV_SIMD_HAS_ZBB 0 +#endif + +template +struct simd8; + +// SIMD byte mask type (returned by things like eq and gt) +template <> +struct simd8 { + typedef uint16_t bitmask_t; + typedef uint32_t bitmask2_t; + + uint8_t data[16]; + + static sonic_force_inline simd8 splat(bool _value) { + simd8 r; + std::memset(r.data, _value ? 0xFF : 0x00, 16); + return r; + } + + sonic_force_inline simd8() { std::memset(data, 0, 16); } + sonic_force_inline simd8(bool _value) { + std::memset(data, _value ? 0xFF : 0x00, 16); + } + + // Construct from raw array + sonic_force_inline explicit simd8(const uint8_t d[16]) { + std::memcpy(data, d, 16); + } + + // We return uint64_t with 4 bits per byte (same as NEON movemask format) + // so that TrailingZeroes(x) >> 2 gives the correct byte index. + sonic_force_inline uint64_t to_bitmask() const { +#if SONIC_RISCV_SIMD_HAS_ZBB + uint32_t bits = riscv_zbb_detail::bool_mask16(data); + uint64_t mask = 0; + for (int i = 0; i < 16; i++) { + if (bits & (1u << i)) { + mask |= (0xFULL << (i * 4)); + } + } + return mask; +#else + uint64_t mask = 0; + for (int i = 0; i < 16; i++) { + if (data[i]) { + mask |= (0xFULL << (i * 4)); + } + } + return mask; +#endif + } + + sonic_force_inline bool any() const { +#if SONIC_RISCV_SIMD_HAS_ZBB + return (riscv_zbb_detail::load_u64(data) | + riscv_zbb_detail::load_u64(data + 8)) != 0; +#else + for (int i = 0; i < 16; i++) { + if (data[i]) return true; + } + return false; +#endif + } + + sonic_force_inline simd8 operator~() const { + simd8 r; + for (int i = 0; i < 16; i++) r.data[i] = ~data[i]; + return r; + } + + sonic_force_inline simd8 operator|(const simd8 other) const { + simd8 r; + for (int i = 0; i < 16; i++) r.data[i] = data[i] | other.data[i]; + return r; + } + sonic_force_inline simd8 operator&(const simd8 other) const { + simd8 r; + for (int i = 0; i < 16; i++) r.data[i] = data[i] & other.data[i]; + return r; + } + sonic_force_inline simd8 operator^(const simd8 other) const { + simd8 r; + for (int i = 0; i < 16; i++) r.data[i] = data[i] ^ other.data[i]; + return r; + } + sonic_force_inline simd8 bit_andnot(const simd8 other) const { + simd8 r; + for (int i = 0; i < 16; i++) r.data[i] = data[i] & ~other.data[i]; + return r; + } + sonic_force_inline simd8& operator|=(const simd8 other) { + for (int i = 0; i < 16; i++) data[i] |= other.data[i]; + return *this; + } + sonic_force_inline simd8& operator&=(const simd8 other) { + for (int i = 0; i < 16; i++) data[i] &= other.data[i]; + return *this; + } + sonic_force_inline simd8& operator^=(const simd8 other) { + for (int i = 0; i < 16; i++) data[i] ^= other.data[i]; + return *this; + } +}; + +// Free function for skip.inc.h compatibility (same as NEON's to_bitmask) +sonic_force_inline uint64_t to_bitmask(const simd8& v) { + return v.to_bitmask(); +} + +// Unsigned bytes +template <> +struct simd8 { + static const int SIZE = 16; + uint8_t data[16]; + + static sonic_force_inline simd8 splat(uint8_t _value) { + simd8 r; + std::memset(r.data, _value, 16); + return r; + } + static sonic_force_inline simd8 zero() { + simd8 r; + std::memset(r.data, 0, 16); + return r; + } + static sonic_force_inline simd8 load(const uint8_t* values) { + simd8 r; + std::memcpy(r.data, values, 16); + return r; + } + + sonic_force_inline simd8() { std::memset(data, 0, 16); } + sonic_force_inline simd8(const uint8_t values[16]) { + std::memcpy(data, values, 16); + } + sonic_force_inline simd8(uint8_t _value) { std::memset(data, _value, 16); } + // Conversion from bool mask + sonic_force_inline simd8(const simd8& other) { + std::memcpy(data, other.data, 16); + } + sonic_force_inline simd8(uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, + uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, + uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, + uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15) { + data[0] = v0; + data[1] = v1; + data[2] = v2; + data[3] = v3; + data[4] = v4; + data[5] = v5; + data[6] = v6; + data[7] = v7; + data[8] = v8; + data[9] = v9; + data[10] = v10; + data[11] = v11; + data[12] = v12; + data[13] = v13; + data[14] = v14; + data[15] = v15; + } + + // Repeat 16 values as many times as necessary (usually for lookup tables) + static sonic_force_inline simd8 repeat_16( + uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, + uint8_t v6, uint8_t v7, uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, + uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15) { + return simd8(v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, + v13, v14, v15); + } + + sonic_force_inline void store(uint8_t dst[16]) const { + std::memcpy(dst, data, 16); + } + + // Saturated math + sonic_force_inline simd8 saturating_add( + const simd8 other) const { + simd8 r; + for (int i = 0; i < 16; i++) { + uint16_t sum = (uint16_t)data[i] + other.data[i]; + r.data[i] = sum > 255 ? 255 : (uint8_t)sum; + } + return r; + } + sonic_force_inline simd8 saturating_sub( + const simd8 other) const { + simd8 r; + for (int i = 0; i < 16; i++) { + r.data[i] = data[i] > other.data[i] ? data[i] - other.data[i] : 0; + } + return r; + } + + // Arithmetic + sonic_force_inline simd8 operator+( + const simd8 other) const { + simd8 r; + for (int i = 0; i < 16; i++) r.data[i] = data[i] + other.data[i]; + return r; + } + sonic_force_inline simd8 operator-( + const simd8 other) const { + simd8 r; + for (int i = 0; i < 16; i++) r.data[i] = data[i] - other.data[i]; + return r; + } + sonic_force_inline simd8& operator+=(const simd8 other) { + for (int i = 0; i < 16; i++) data[i] += other.data[i]; + return *this; + } + sonic_force_inline simd8& operator-=(const simd8 other) { + for (int i = 0; i < 16; i++) data[i] -= other.data[i]; + return *this; + } + + // Bitwise + sonic_force_inline simd8 operator|( + const simd8 other) const { + simd8 r; + for (int i = 0; i < 16; i++) r.data[i] = data[i] | other.data[i]; + return r; + } + sonic_force_inline simd8 operator&( + const simd8 other) const { + simd8 r; + for (int i = 0; i < 16; i++) r.data[i] = data[i] & other.data[i]; + return r; + } + sonic_force_inline simd8 operator^( + const simd8 other) const { + simd8 r; + for (int i = 0; i < 16; i++) r.data[i] = data[i] ^ other.data[i]; + return r; + } + sonic_force_inline simd8 bit_andnot( + const simd8 other) const { + simd8 r; + for (int i = 0; i < 16; i++) r.data[i] = data[i] & ~other.data[i]; + return r; + } + sonic_force_inline simd8 operator~() const { + simd8 r; + for (int i = 0; i < 16; i++) r.data[i] = ~data[i]; + return r; + } + sonic_force_inline simd8& operator|=(const simd8 other) { + for (int i = 0; i < 16; i++) data[i] |= other.data[i]; + return *this; + } + sonic_force_inline simd8& operator&=(const simd8 other) { + for (int i = 0; i < 16; i++) data[i] &= other.data[i]; + return *this; + } + sonic_force_inline simd8& operator^=(const simd8 other) { + for (int i = 0; i < 16; i++) data[i] ^= other.data[i]; + return *this; + } + + // Comparison + friend sonic_force_inline simd8 operator==(const simd8 lhs, + const simd8 rhs) { + simd8 r; + for (int i = 0; i < 16; i++) + r.data[i] = (lhs.data[i] == rhs.data[i]) ? 0xFF : 0x00; + return r; + } + sonic_force_inline simd8 operator<=(const simd8 other) const { + simd8 r; + for (int i = 0; i < 16; i++) + r.data[i] = (data[i] <= other.data[i]) ? 0xFF : 0x00; + return r; + } + sonic_force_inline simd8 operator>=(const simd8 other) const { + simd8 r; + for (int i = 0; i < 16; i++) + r.data[i] = (data[i] >= other.data[i]) ? 0xFF : 0x00; + return r; + } + sonic_force_inline simd8 operator>(const simd8 other) const { + simd8 r; + for (int i = 0; i < 16; i++) + r.data[i] = (data[i] > other.data[i]) ? 0xFF : 0x00; + return r; + } + sonic_force_inline simd8 operator<(const simd8 other) const { + simd8 r; + for (int i = 0; i < 16; i++) + r.data[i] = (data[i] < other.data[i]) ? 0xFF : 0x00; + return r; + } + + // gt_bits / lt_bits (non-zero instead of all-1s) + sonic_force_inline simd8 gt_bits(const simd8 other) const { + return this->saturating_sub(other); + } + sonic_force_inline simd8 lt_bits(const simd8 other) const { + return other.saturating_sub(*this); + } + + // Max / min + sonic_force_inline uint8_t max_val() const { + uint8_t m = 0; + for (int i = 0; i < 16; i++) + if (data[i] > m) m = data[i]; + return m; + } + sonic_force_inline uint8_t min_val() const { + uint8_t m = 255; + for (int i = 0; i < 16; i++) + if (data[i] < m) m = data[i]; + return m; + } + sonic_force_inline simd8 max_val(const simd8 other) const { + simd8 r; + for (int i = 0; i < 16; i++) + r.data[i] = data[i] > other.data[i] ? data[i] : other.data[i]; + return r; + } + sonic_force_inline simd8 min_val(const simd8 other) const { + simd8 r; + for (int i = 0; i < 16; i++) + r.data[i] = data[i] < other.data[i] ? data[i] : other.data[i]; + return r; + } + + // Bit-specific operations + sonic_force_inline simd8 any_bits_set(simd8 bits) const { + simd8 r; + for (int i = 0; i < 16; i++) + r.data[i] = (data[i] & bits.data[i]) ? 0xFF : 0x00; + return r; + } + sonic_force_inline bool any_bits_set_anywhere() const { + return max_val() != 0; + } + sonic_force_inline bool any_bits_set_anywhere(simd8 bits) const { + return (*this & bits).any_bits_set_anywhere(); + } + + template + sonic_force_inline simd8 shr() const { + simd8 r; + for (int i = 0; i < 16; i++) r.data[i] = data[i] >> N; + return r; + } + template + sonic_force_inline simd8 shl() const { + simd8 r; + for (int i = 0; i < 16; i++) r.data[i] = data[i] << N; + return r; + } + + // prev: shift in bytes from prev_chunk + template + sonic_force_inline simd8 prev( + const simd8 prev_chunk) const { + simd8 r; + for (int i = 0; i < N; i++) r.data[i] = prev_chunk.data[16 - N + i]; + for (int i = N; i < 16; i++) r.data[i] = data[i - N]; + return r; + } + + // lookup_16: table lookup, byte by byte + template + sonic_force_inline simd8 lookup_16(simd8 lookup_table) const { + simd8 r; + for (int i = 0; i < 16; i++) { + uint8_t idx = data[i] & 0x0F; + r.data[i] = lookup_table.data[idx]; + } + return r; + } + + template + sonic_force_inline simd8 lookup_16(L r0, L r1, L r2, L r3, L r4, L r5, + L r6, L r7, L r8, L r9, L r10, L r11, + L r12, L r13, L r14, L r15) const { + return lookup_16(simd8::repeat_16(r0, r1, r2, r3, r4, r5, r6, r7, r8, r9, + r10, r11, r12, r13, r14, r15)); + } + + template + sonic_force_inline simd8 apply_lookup_16_to( + const simd8 original) { + simd8 r; + for (int i = 0; i < 16; i++) { + uint8_t idx = (uint8_t)original.data[i]; + r.data[i] = idx < 16 ? data[idx] : 0; + } + return r; + } +}; + +// Signed bytes +template <> +struct simd8 { + int8_t data[16]; + + static sonic_force_inline simd8 splat(int8_t _value) { + simd8 r; + std::memset(r.data, (uint8_t)_value, 16); + return r; + } + static sonic_force_inline simd8 zero() { + simd8 r; + std::memset(r.data, 0, 16); + return r; + } + static sonic_force_inline simd8 load(const int8_t values[16]) { + simd8 r; + std::memcpy(r.data, values, 16); + return r; + } + + sonic_force_inline simd8() { std::memset(data, 0, 16); } + sonic_force_inline simd8(int8_t _value) { + std::memset(data, (uint8_t)_value, 16); + } + sonic_force_inline simd8(const int8_t* values) { + std::memcpy(data, values, 16); + } + sonic_force_inline simd8(int8_t v0, int8_t v1, int8_t v2, int8_t v3, + int8_t v4, int8_t v5, int8_t v6, int8_t v7, + int8_t v8, int8_t v9, int8_t v10, int8_t v11, + int8_t v12, int8_t v13, int8_t v14, int8_t v15) { + data[0] = v0; + data[1] = v1; + data[2] = v2; + data[3] = v3; + data[4] = v4; + data[5] = v5; + data[6] = v6; + data[7] = v7; + data[8] = v8; + data[9] = v9; + data[10] = v10; + data[11] = v11; + data[12] = v12; + data[13] = v13; + data[14] = v14; + data[15] = v15; + } + + static sonic_force_inline simd8 repeat_16( + int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, + int8_t v6, int8_t v7, int8_t v8, int8_t v9, int8_t v10, int8_t v11, + int8_t v12, int8_t v13, int8_t v14, int8_t v15) { + return simd8(v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, + v13, v14, v15); + } + + sonic_force_inline void store(int8_t dst[16]) const { + std::memcpy(dst, data, 16); + } + + // Explicit conversion to uint8_t + sonic_force_inline explicit simd8(const simd8& other) { + std::memcpy(data, other.data, 16); + } + sonic_force_inline explicit operator simd8() const { + simd8 r; + std::memcpy(r.data, data, 16); + return r; + } + + // Bitwise + sonic_force_inline simd8 operator|(const simd8 other) const { + simd8 r; + for (int i = 0; i < 16; i++) r.data[i] = data[i] | other.data[i]; + return r; + } + + // Arithmetic + sonic_force_inline simd8 operator+(const simd8 other) const { + simd8 r; + for (int i = 0; i < 16; i++) r.data[i] = data[i] + other.data[i]; + return r; + } + sonic_force_inline simd8 operator-(const simd8 other) const { + simd8 r; + for (int i = 0; i < 16; i++) r.data[i] = data[i] - other.data[i]; + return r; + } + sonic_force_inline simd8& operator+=(const simd8 other) { + for (int i = 0; i < 16; i++) data[i] += other.data[i]; + return *this; + } + sonic_force_inline simd8& operator-=(const simd8 other) { + for (int i = 0; i < 16; i++) data[i] -= other.data[i]; + return *this; + } + + // Comparison + sonic_force_inline simd8 max_val(const simd8 other) const { + simd8 r; + for (int i = 0; i < 16; i++) + r.data[i] = data[i] > other.data[i] ? data[i] : other.data[i]; + return r; + } + sonic_force_inline simd8 min_val(const simd8 other) const { + simd8 r; + for (int i = 0; i < 16; i++) + r.data[i] = data[i] < other.data[i] ? data[i] : other.data[i]; + return r; + } + sonic_force_inline simd8 operator>(const simd8 other) const { + simd8 r; + for (int i = 0; i < 16; i++) + r.data[i] = (data[i] > other.data[i]) ? 0xFF : 0x00; + return r; + } + sonic_force_inline simd8 operator<(const simd8 other) const { + simd8 r; + for (int i = 0; i < 16; i++) + r.data[i] = (data[i] < other.data[i]) ? 0xFF : 0x00; + return r; + } + friend sonic_force_inline simd8 operator==(const simd8 lhs, + const simd8 rhs) { + simd8 r; + for (int i = 0; i < 16; i++) + r.data[i] = (lhs.data[i] == rhs.data[i]) ? 0xFF : 0x00; + return r; + } + + template + sonic_force_inline simd8 prev(const simd8 prev_chunk) const { + simd8 r; + for (int i = 0; i < N; i++) r.data[i] = prev_chunk.data[16 - N + i]; + for (int i = N; i < 16; i++) r.data[i] = data[i - N]; + return r; + } + + template + sonic_force_inline simd8 lookup_16(simd8 lookup_table) const { + return lookup_table.apply_lookup_16_to(*this); + } + template + sonic_force_inline simd8 lookup_16(L r0, L r1, L r2, L r3, L r4, L r5, + L r6, L r7, L r8, L r9, L r10, L r11, + L r12, L r13, L r14, L r15) const { + return lookup_16(simd8::repeat_16(r0, r1, r2, r3, r4, r5, r6, r7, r8, r9, + r10, r11, r12, r13, r14, r15)); + } + + template + sonic_force_inline simd8 apply_lookup_16_to(const simd8 original) { + simd8 r; + for (int i = 0; i < 16; i++) { + uint8_t idx = (uint8_t)original.data[i]; + r.data[i] = idx < 16 ? data[idx] : 0; + } + return r; + } +}; + +template +struct simd8x64 { + static constexpr int NUM_CHUNKS = 64 / sizeof(simd8); + static_assert(NUM_CHUNKS == 4, + "RISC-V kernel should use four registers per 64-byte block."); + const simd8 chunks[NUM_CHUNKS]; + + simd8x64(const simd8x64& o) = delete; // no copy allowed + simd8x64& operator=(const simd8& other) = + delete; // no assignment allowed + simd8x64() = delete; // no default constructor allowed + + sonic_force_inline simd8x64(const simd8 chunk0, const simd8 chunk1, + const simd8 chunk2, const simd8 chunk3) + : chunks{chunk0, chunk1, chunk2, chunk3} {} + sonic_force_inline simd8x64(const T ptr[64]) + : chunks{simd8::load((const uint8_t*)ptr), + simd8::load((const uint8_t*)(ptr + 16)), + simd8::load((const uint8_t*)(ptr + 32)), + simd8::load((const uint8_t*)(ptr + 48))} {} + + sonic_force_inline void store(T ptr[64]) const { + this->chunks[0].store((uint8_t*)(ptr)); + this->chunks[1].store((uint8_t*)(ptr + 16)); + this->chunks[2].store((uint8_t*)(ptr + 32)); + this->chunks[3].store((uint8_t*)(ptr + 48)); + } + + sonic_force_inline simd8 reduce_or() const { + return (this->chunks[0] | this->chunks[1]) | + (this->chunks[2] | this->chunks[3]); + } + + sonic_force_inline uint64_t to_bitmask() const { + // Each chunk's to_bitmask() returns 64 bits with 4 bits per byte. + // Compress to 1 bit per byte (16 bits per chunk) before combining. + auto compress = [](uint64_t mask) -> uint64_t { + uint64_t result = 0; + for (int i = 0; i < 16; i++) { + if (mask & (0xFULL << (i * 4))) { + result |= (1ULL << i); + } + } + return result; + }; + uint64_t r0 = compress(this->chunks[0].to_bitmask()); + uint64_t r1 = compress(this->chunks[1].to_bitmask()); + uint64_t r2 = compress(this->chunks[2].to_bitmask()); + uint64_t r3 = compress(this->chunks[3].to_bitmask()); + return r0 | (r1 << 16) | (r2 << 32) | (r3 << 48); + } + + sonic_force_inline uint64_t eq(const T m) const { +#if SONIC_RISCV_SIMD_HAS_ZBB + const uint8_t byte = static_cast(m); + uint64_t r0 = riscv_zbb_detail::eq_mask16( + reinterpret_cast(this->chunks[0].data), byte); + uint64_t r1 = riscv_zbb_detail::eq_mask16( + reinterpret_cast(this->chunks[1].data), byte); + uint64_t r2 = riscv_zbb_detail::eq_mask16( + reinterpret_cast(this->chunks[2].data), byte); + uint64_t r3 = riscv_zbb_detail::eq_mask16( + reinterpret_cast(this->chunks[3].data), byte); + return r0 | (r1 << 16) | (r2 << 32) | (r3 << 48); +#else + const simd8 mask = simd8::splat(m); + return simd8x64(this->chunks[0] == mask, this->chunks[1] == mask, + this->chunks[2] == mask, this->chunks[3] == mask) + .to_bitmask(); +#endif + } + + sonic_force_inline uint64_t lteq(const T m) const { +#if SONIC_RISCV_SIMD_HAS_ZBB + const uint8_t byte = static_cast(m); + uint64_t r0 = riscv_zbb_detail::lteq_mask16( + reinterpret_cast(this->chunks[0].data), byte); + uint64_t r1 = riscv_zbb_detail::lteq_mask16( + reinterpret_cast(this->chunks[1].data), byte); + uint64_t r2 = riscv_zbb_detail::lteq_mask16( + reinterpret_cast(this->chunks[2].data), byte); + uint64_t r3 = riscv_zbb_detail::lteq_mask16( + reinterpret_cast(this->chunks[3].data), byte); + return r0 | (r1 << 16) | (r2 << 32) | (r3 << 48); +#else + const simd8 mask = simd8::splat(m); + return simd8x64(this->chunks[0] <= mask, this->chunks[1] <= mask, + this->chunks[2] <= mask, this->chunks[3] <= mask) + .to_bitmask(); +#endif + } +}; // struct simd8x64 + +} // namespace riscv +} // namespace internal +} // namespace sonic_json + +#undef SONIC_RISCV_SIMD_HAS_ZBB diff --git a/include/sonic/internal/arch/riscv/skip.h b/include/sonic/internal/arch/riscv/skip.h new file mode 100644 index 00000000..26fd2de3 --- /dev/null +++ b/include/sonic/internal/arch/riscv/skip.h @@ -0,0 +1,65 @@ +/* + * Copyright 2022 ByteDance Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#define VEC_LEN 16 + +#include +#include + +#include "base.h" +#include "simd.h" + +namespace sonic_json { +namespace internal { +namespace riscv { + +using sonic_json::internal::common::EqBytes4; +using sonic_json::internal::common::SkipLiteral; + +#include "skip.inc.h" + +sonic_force_inline bool SkipContainer(const uint8_t *data, size_t &pos, + size_t len, uint8_t left, uint8_t right) { + return skip_container>(data, pos, len, left, right); +} + +// TODO: optimize by removing bound checking. +sonic_force_inline uint8_t skip_space(const uint8_t *data, size_t &pos, + size_t &, uint64_t &) { + // fast path for single space + if (!IsSpace(data[pos++])) return data[pos - 1]; + if (!IsSpace(data[pos++])) return data[pos - 1]; + + // current pos is out of block + while (1) { + uint64_t nonspace = GetNonSpaceBits(data + pos); + if (nonspace) { + pos += TrailingZeroes(nonspace) >> 2; + return data[pos++]; + } else { + pos += 16; + } + } + sonic_assert(false && "!should not happen"); +} + +} // namespace riscv +} // namespace internal +} // namespace sonic_json + +#undef VEC_LEN diff --git a/include/sonic/internal/arch/riscv/skip.inc.h b/include/sonic/internal/arch/riscv/skip.inc.h new file mode 100644 index 00000000..4d3a4113 --- /dev/null +++ b/include/sonic/internal/arch/riscv/skip.inc.h @@ -0,0 +1,168 @@ +/* + * Copyright 2022 ByteDance Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef VEC_LEN +#error "Define vector length firstly!" +#endif + +template +sonic_force_inline uint64_t GetStringBits(const uint8_t *data, + uint64_t &prev_instring, + uint64_t &prev_escaped) { + const T v(data); + uint64_t escaped = 0; + uint64_t bs_bits = v.eq('\\'); + if (bs_bits) { + escaped = common::GetEscaped<64>(prev_escaped, bs_bits); + } else { + escaped = prev_escaped; + prev_escaped = 0; + } + uint64_t quote_bits = v.eq('"') & ~escaped; + uint64_t in_string = PrefixXor(quote_bits) ^ prev_instring; + prev_instring = uint64_t(static_cast(in_string) >> 63); + return in_string; +} + +// GetNextToken find the next characters in tokens and update the position to +// it. +template +sonic_force_inline uint8_t GetNextToken(const uint8_t *data, size_t &pos, + size_t len, const char (&tokens)[N]) { + while (pos + VEC_LEN <= len) { + simd8 v = simd8::load(data + pos); + simd8 vor = simd8::splat(false); + for (size_t i = 0; i < N - 1; i++) { + simd8 vtmp = (v == simd8::splat((uint8_t)(tokens[i]))); + vor = vor | vtmp; + } + + uint64_t next = vor.to_bitmask(); + if (next) { + pos += (TrailingZeroes(next) >> 2); + return data[pos]; + } + pos += VEC_LEN; + } + while (pos < len) { + for (size_t i = 0; i < N - 1; i++) { + if (data[pos] == tokens[i]) { + return tokens[i]; + } + } + pos++; + } + return '\0'; +} + +// pos is the after the ending quote +sonic_force_inline int SkipString(const uint8_t *data, size_t &pos, + size_t len) { + const static int kEscaped = 2; + const static int kNormal = 1; + const static int kUnclosed = 0; + uint64_t quote_bits = 0; + uint64_t bs_bits = 0; + int ret = kNormal; + while (pos + VEC_LEN <= len) { + simd8 v = simd8::load(data + pos); + simd8 bs_cmp = (v == simd8::splat('\\')); + simd8 quote_cmp = (v == simd8::splat('"')); + bs_bits = bs_cmp.to_bitmask(); + quote_bits = quote_cmp.to_bitmask(); + if (((bs_bits - 1) & quote_bits) != 0) { + pos += (TrailingZeroes(quote_bits) >> 2) + 1; + return ret; + } + if (bs_bits) { + ret = kEscaped; + pos += ((TrailingZeroes(bs_bits) >> 2) + 2); + while (pos < len) { + if (data[pos] == '\\') { + pos += 2; + } else { + break; + } + } + } else { + pos += VEC_LEN; + } + } + while (pos < len) { + if (data[pos] == '\\') { + if (pos + 1 >= len) { + return kUnclosed; + } + ret = kEscaped; + pos += 2; + continue; + } + if (data[pos++] == '"') { + return ret; + } + }; + return kUnclosed; +} + +// return true if container is closed. +template +sonic_force_inline bool skip_container(const uint8_t *data, size_t &pos, + size_t len, uint8_t left, + uint8_t right) { + uint64_t prev_instring = 0, prev_escaped = 0, instring; + int rbrace_num = 0, lbrace_num = 0, last_lbrace_num; + const uint8_t *p; + while (pos + 64 <= len) { + p = data + pos; +#define SKIP_LOOP() \ + { \ + instring = GetStringBits(p, prev_instring, prev_escaped); \ + T v(p); \ + last_lbrace_num = lbrace_num; \ + uint64_t rbrace = v.eq(right) & ~instring; \ + uint64_t lbrace = v.eq(left) & ~instring; \ + /* traverse each '}' */ \ + while (rbrace > 0) { \ + rbrace_num++; \ + lbrace_num = last_lbrace_num + CountOnes((rbrace - 1) & lbrace); \ + bool is_closed = lbrace_num < rbrace_num; \ + if (is_closed) { \ + sonic_assert(rbrace_num == lbrace_num + 1); \ + pos += TrailingZeroes(rbrace) + 1; \ + return true; \ + } \ + rbrace &= (rbrace - 1); \ + } \ + lbrace_num = last_lbrace_num + CountOnes(lbrace); \ + } + SKIP_LOOP(); + pos += 64; + } + uint8_t buf[64] = {0}; + std::memcpy(buf, data + pos, len - pos); + p = buf; + SKIP_LOOP(); +#undef SKIP_LOOP + return false; +} + +sonic_force_inline uint8_t skip_space_safe(const uint8_t *data, size_t &pos, + size_t len, size_t &, uint64_t &) { + while (pos < len && IsSpace(data[pos++])) + ; + // if not found, still return the space chars + return data[pos - 1]; +} diff --git a/include/sonic/internal/arch/riscv/str2int.h b/include/sonic/internal/arch/riscv/str2int.h new file mode 100644 index 00000000..c9ddeb5c --- /dev/null +++ b/include/sonic/internal/arch/riscv/str2int.h @@ -0,0 +1,29 @@ +/* + * Copyright 2022 ByteDance Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "../common/arm_common/str2int.h" + +namespace sonic_json { +namespace internal { +namespace riscv { + +using sonic_json::internal::arm_common::simd_str2int; + +} // namespace riscv +} // namespace internal +} // namespace sonic_json diff --git a/include/sonic/internal/arch/riscv/unicode.h b/include/sonic/internal/arch/riscv/unicode.h new file mode 100644 index 00000000..fe5ee33a --- /dev/null +++ b/include/sonic/internal/arch/riscv/unicode.h @@ -0,0 +1,116 @@ +// Copyright 2018-2019 The simdjson authors + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 + +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// This file may have been modified by ByteDance authors. All ByteDance +// Modifications are Copyright 2022 ByteDance Authors. + +#pragma once + +#include + +#include +#include + +#include "../common/unicode_common.h" +#include "base.h" +#include "simd.h" + +namespace sonic_json { +namespace internal { +namespace riscv { + +using sonic_json::internal::common::handle_unicode_codepoint; + +struct StringBlock { + public: + sonic_force_inline static StringBlock Find(const uint8_t *src); + sonic_force_inline static StringBlock Find(simd8 &v); + sonic_force_inline bool HasQuoteFirst(bool allow_unesc) const { + return (((bs_bits - 1) & quote_bits) != 0) && + (allow_unesc || !HasUnescaped()); + } + sonic_force_inline bool HasBackslash() const { + return ((quote_bits - 1) & bs_bits) != 0; + } + sonic_force_inline bool HasUnescaped() const { + return ((quote_bits - 1) & unescaped_bits) != 0; + } + sonic_force_inline int QuoteIndex() const { + return TrailingZeroes(quote_bits) >> 2; + } + sonic_force_inline int BsIndex() const { + return TrailingZeroes(bs_bits) >> 2; + } + + uint64_t bs_bits; + uint64_t quote_bits; + uint64_t unescaped_bits; +}; + +sonic_force_inline StringBlock StringBlock::Find(const uint8_t *src) { + simd8 v = simd8::load(src); + return StringBlock::Find(v); +} + +sonic_force_inline StringBlock StringBlock::Find(simd8 &v) { + simd8 bs = simd8::splat('\\'); + simd8 quote = simd8::splat('"'); + simd8 ctrl = simd8::splat('\x1f'); + + simd8 bs_cmp = (v == bs); + simd8 quote_cmp = (v == quote); + simd8 ctrl_cmp = (v <= ctrl); + + return { + bs_cmp.to_bitmask(), + quote_cmp.to_bitmask(), + ctrl_cmp.to_bitmask(), + }; +} + +sonic_force_inline uint64_t GetNonSpaceBits(const uint8_t *data) { + simd8 v = simd8::load(data); + simd8 space = simd8::splat(' '); + simd8 tab = simd8::splat('\t'); + simd8 nl = simd8::splat('\n'); + simd8 cr = simd8::splat('\r'); + + simd8 m1 = (v == space); + simd8 m2 = (v == tab); + simd8 m3 = (v == nl); + simd8 m4 = (v == cr); + + simd8 m5 = m1 | m2; + simd8 m6 = m3 | m4; + simd8 m7 = m5 | m6; + simd8 m8 = ~m7; + + return m8.to_bitmask(); +} + +sonic_force_inline bool IsAscii(const simd8x64 &input) { + simd8 bits = input.reduce_or(); + return bits.max_val() < 0x80u; +} + +sonic_force_inline simd8 MustBe2_3Continuation( + const simd8 prev2, const simd8 prev3) { + simd8 is_third_byte = prev2 >= uint8_t(0xe0u); + simd8 is_fourth_byte = prev3 >= uint8_t(0xf0u); + return is_third_byte ^ is_fourth_byte; +} + +} // namespace riscv +} // namespace internal +} // namespace sonic_json diff --git a/include/sonic/internal/arch/simd_dispatch.h b/include/sonic/internal/arch/simd_dispatch.h index 0c474ce0..a8ee07f7 100644 --- a/include/sonic/internal/arch/simd_dispatch.h +++ b/include/sonic/internal/arch/simd_dispatch.h @@ -39,6 +39,9 @@ #elif defined(SONIC_HAVE_NEON) #define SONIC_USING_ARCH_FUNC(func) using neon::func #define INCLUDE_ARCH_FILE(file) SONIC_STRINGIFY(neon/file) +#elif defined(SONIC_HAVE_RISCV) +#define SONIC_USING_ARCH_FUNC(func) using riscv::func +#define INCLUDE_ARCH_FILE(file) SONIC_STRINGIFY(riscv/file) #endif #elif defined(SONIC_DYNAMIC_DISPATCH) @@ -50,6 +53,9 @@ #elif defined(SONIC_HAVE_NEON) #define SONIC_USING_ARCH_FUNC(func) using neon::func #define INCLUDE_ARCH_FILE(file) SONIC_STRINGIFY(neon/file) +#elif defined(SONIC_HAVE_RISCV) +#define SONIC_USING_ARCH_FUNC(func) using riscv::func +#define INCLUDE_ARCH_FILE(file) SONIC_STRINGIFY(riscv/file) #endif #endif diff --git a/include/sonic/internal/arch/sonic_cpu_feature.h b/include/sonic/internal/arch/sonic_cpu_feature.h index 21d2707f..17222fb9 100644 --- a/include/sonic/internal/arch/sonic_cpu_feature.h +++ b/include/sonic/internal/arch/sonic_cpu_feature.h @@ -37,11 +37,18 @@ #if defined(__AVX2__) #define SONIC_HAVE_AVX2 #endif -#else +#elif defined(__ARM_NEON) || defined(__ARM_NEON__) || \ + defined(__ARM_FEATURE_SVE2) #if defined(__ARM_NEON) || defined(__ARM_NEON__) #define SONIC_HAVE_NEON #endif #if defined(__ARM_FEATURE_SVE2) && (__ARM_FEATURE_SVE_BITS == 128) #define SONIC_HAVE_SVE2_128 #endif +#elif defined(__riscv) && defined(__riscv_xlen) && (__riscv_xlen == 64) +#define SONIC_IS_RISCV 1 +#if defined(__riscv_zbb) +#define SONIC_HAVE_RISCV_ZBB 1 +#endif +#define SONIC_HAVE_RISCV 1 #endif