diff --git a/base64x_error_native_amd64_test.go b/base64x_error_native_amd64_test.go index 8b62c03..16026f2 100644 --- a/base64x_error_native_amd64_test.go +++ b/base64x_error_native_amd64_test.go @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -//go:build amd64 && !noasm && !appengine -// +build amd64,!noasm,!appengine +//go:build !goexperiment.simd && amd64 && !noasm && !appengine +// +build !goexperiment.simd,amd64,!noasm,!appengine package base64x diff --git a/base64x_fallback_test.go b/base64x_fallback_test.go index 1447270..7a445ff 100644 --- a/base64x_fallback_test.go +++ b/base64x_fallback_test.go @@ -1,5 +1,5 @@ -//go:build !amd64 || noasm || appengine -// +build !amd64 noasm appengine +//go:build goexperiment.simd || !amd64 || noasm || appengine +// +build goexperiment.simd !amd64 noasm appengine /* * Copyright 2024 CloudWeGo Authors diff --git a/base64x_native.go b/base64x_native.go index 5643729..dd5c4f3 100644 --- a/base64x_native.go +++ b/base64x_native.go @@ -1,5 +1,5 @@ -//go:build amd64 && !noasm && !appengine -// +build amd64,!noasm,!appengine +//go:build !goexperiment.simd && amd64 && !noasm && !appengine +// +build !goexperiment.simd,amd64,!noasm,!appengine /* * Copyright 2024 CloudWeGo Authors diff --git a/base64x_simd_amd64.go b/base64x_simd_amd64.go new file mode 100644 index 0000000..9d73099 --- /dev/null +++ b/base64x_simd_amd64.go @@ -0,0 +1,158 @@ +//go:build goexperiment.simd && amd64 && !noasm && !appengine + +/* + * Copyright 2024 CloudWeGo Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package base64x + +import ( + "encoding/base64" + "encoding/json" + "simd/archsimd" + + "github.com/cloudwego/base64x/internal/simd" +) + +func encode(out *[]byte, src []byte, mode int) { + if !archsimd.X86.AVX2() { + encodeStdlib(out, src, mode) + return + } + url := 0 + if mode&_MODE_URL != 0 { + url = 1 + } + pad := 1 + if mode&_MODE_RAW != 0 { + pad = 0 + } + enc := encodingForModeSimd(mode) + dst := *out + start := len(dst) + need := enc.EncodedLen(len(src)) + dst = dst[:start+need] + simd.B64_encode(src, uint64(len(src)), dst[start:start+need], uint64(need), url, pad) + *out = dst[:start+need] +} + +func decode(out *[]byte, src []byte, mode int) (int, error) { + input := src + if mode&_MODE_JSON != 0 { + unquoted, err := unquoteJSONBase64(src) + if err != nil { + return 0, err + } + input = unquoted + } + if !archsimd.X86.AVX2() { + return decodeStdlib(out, input, mode) + } + + url := 0 + if mode&_MODE_URL != 0 { + url = 1 + } + pad := 1 + if mode&_MODE_RAW != 0 { + pad = 0 + } + enc := encodingForModeSimd(mode) + dst := *out + start := len(dst) + need := enc.DecodedLen(len(input)) + + // The vector decode loop writes in 32-byte chunks. + // If the destination slice capacity cannot safely hold 32 bytes from start, + // decode into a stack scratch buffer and copy back to avoid out-of-bounds writes. + if cap(dst)-start < 32 || cap(dst)-start < need+8 { + var tmp [256]byte + var buf []byte + allocCap := need + 32 + if allocCap <= len(tmp) { + buf = tmp[:allocCap] + } else { + buf = make([]byte, allocCap) + } + var posErr uint64 + w := simd.B64_decode(input, uint64(len(input)), buf, uint64(len(buf)), url, pad, &posErr) + if posErr != ^uint64(0) { + return 0, base64.CorruptInputError(posErr) + } + dst = dst[:start+int(w)] + copy(dst[start:], buf[:w]) + *out = dst + return int(w), nil + } + + dst = dst[:start+need] + var posErr uint64 + availCap := uint64(cap(dst) - start) + w := simd.B64_decode(input, uint64(len(input)), dst[start:cap(dst)], availCap, url, pad, &posErr) + if posErr != ^uint64(0) { + return 0, base64.CorruptInputError(posErr) + } + *out = dst[:start+int(w)] + return int(w), nil +} + +func encodingForModeSimd(mode int) *base64.Encoding { + switch mode & (_MODE_URL | _MODE_RAW) { + case _MODE_URL: + return base64.URLEncoding + case _MODE_RAW: + return base64.RawStdEncoding + case _MODE_URL | _MODE_RAW: + return base64.RawURLEncoding + default: + return base64.StdEncoding + } +} + +func encodeStdlib(out *[]byte, src []byte, mode int) { + enc := encodingForModeSimd(mode) + dst := *out + start := len(dst) + need := enc.EncodedLen(len(src)) + dst = dst[:start+need] + enc.Encode(dst[start:], src) + *out = dst +} + +func decodeStdlib(out *[]byte, input []byte, mode int) (int, error) { + enc := encodingForModeSimd(mode) + dst := *out + start := len(dst) + buf := dst[start:cap(dst)] + n, err := enc.Decode(buf, input) + if err != nil { + return 0, err + } + *out = dst[:start+n] + return n, nil +} + +func unquoteJSONBase64(src []byte) ([]byte, error) { + quoted := make([]byte, 0, len(src)+2) + quoted = append(quoted, '"') + quoted = append(quoted, src...) + quoted = append(quoted, '"') + + var decoded string + if err := json.Unmarshal(quoted, &decoded); err != nil { + return nil, err + } + return []byte(decoded), nil +} diff --git a/base64x_subr_native_amd64.go b/base64x_subr_native_amd64.go index 094ad84..b092604 100644 --- a/base64x_subr_native_amd64.go +++ b/base64x_subr_native_amd64.go @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -//go:build amd64 && !noasm && !appengine -// +build amd64,!noasm,!appengine +//go:build !goexperiment.simd && amd64 && !noasm && !appengine +// +build !goexperiment.simd,amd64,!noasm,!appengine package base64x diff --git a/internal/simd/b64_gen.go b/internal/simd/b64_gen.go new file mode 100644 index 0000000..2c87e37 --- /dev/null +++ b/internal/simd/b64_gen.go @@ -0,0 +1,311 @@ +//go:build goexperiment.simd && amd64 + +// Code generated by sgoiter emit go127; DO NOT EDIT. +// sgoiter-stamp: sha256=8c4229c7b8ba80231b22b3f6f7e3bf620480aace1357e583e0881d0026d0bfa4 +package simd + +import ( + "math/bits" + "simd/archsimd" +) + +var ( + TabEncodeCharsetStd = [64]byte{'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', '/'} + TabEncodeCharsetURL = [64]byte{'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-', '_'} + VecEncodeShuffles = [32]byte{1, 0, 2, 1, 4, 3, 5, 4, 7, 6, 8, 7, 10, 9, 11, 10, 1, 0, 2, 1, 4, 3, 5, 4, 7, 6, 8, 7, 10, 9, 11, 10} + VecEncodeCharsetStd = [32]byte{71, 252, 252, 252, 252, 252, 252, 252, 252, 252, 252, 237, 240, 'A', 0, 0, 71, 252, 252, 252, 252, 252, 252, 252, 252, 252, 252, 237, 240, 'A', 0, 0} + VecEncodeCharsetURL = [32]byte{71, 252, 252, 252, 252, 252, 252, 252, 252, 252, 252, 239, 32, 'A', 0, 0, 71, 252, 252, 252, 252, 252, 252, 252, 252, 252, 252, 239, 32, 'A', 0, 0} + VecPacking = [32]byte{2, 1, 0, 6, 5, 4, 10, 9, 8, 14, 13, 12, 128, 128, 128, 128, 2, 1, 0, 6, 5, 4, 10, 9, 8, 14, 13, 12, 128, 128, 128, 128} + VecDecodeBits = [32]byte{0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00} + VecDecodeTableStd = [128]byte{0x00, 0x00, 0x13, 0x04, 0xbf, 0xbf, 0xb9, 0xb9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x13, 0x04, 0xbf, 0xbf, 0xb9, 0xb9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf0, 0x54, 0x50, 0x50, 0x50, 0x54, 0xa8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf0, 0x54, 0x50, 0x50, 0x50, 0x54, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10} + VecDecodeTableURL = [128]byte{0x00, 0x00, 0x11, 0x04, 0xbf, 0xbf, 0xb9, 0xb9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x11, 0x04, 0xbf, 0xbf, 0xb9, 0xb9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf0, 0x50, 0x50, 0x54, 0x50, 0x70, 0xa8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf0, 0x50, 0x50, 0x54, 0x50, 0x70, 0x5f, 0x5f, 0x5f, 0x5f, 0x5f, 0x5f, 0x5f, 0x5f, 0x5f, 0x5f, 0x5f, 0x5f, 0x5f, 0x5f, 0x5f, 0x5f, 0x5f, 0x5f, 0x5f, 0x5f, 0x5f, 0x5f, 0x5f, 0x5f, 0x5f, 0x5f, 0x5f, 0x5f, 0x5f, 0x5f, 0x5f, 0x5f, 0xe0, 0xe0, 0xe0, 0xe0, 0xe0, 0xe0, 0xe0, 0xe0, 0xe0, 0xe0, 0xe0, 0xe0, 0xe0, 0xe0, 0xe0, 0xe0, 0xe0, 0xe0, 0xe0, 0xe0, 0xe0, 0xe0, 0xe0, 0xe0, 0xe0, 0xe0, 0xe0, 0xe0, 0xe0, 0xe0, 0xe0, 0xe0} + TabDecodeCharsetStd = [256]byte{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 62, 0xff, 0xff, 0xff, 63, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff} + TabDecodeCharsetURL = [256]byte{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 62, 0xff, 0xff, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 0xff, 0xff, 0xff, 0xff, 63, 0xff, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff} +) + +func __avx2() int { + v4 := func() int { + if archsimd.X86.AVX2() { + return 1 + } + return 0 + }() + return v4 +} + +func b64_encode_tail(src []byte, src_len uint64, i uint64, dst []byte, o uint64, url_mode int, pad_mode int) uint64 { + var v9 uint32 + var v11 []byte + var v91 uint64 + if uint64(len(src)) < src_len { + src_len = uint64(len(src)) + } + v11 = TabEncodeCharsetStd[:] + if url_mode != 0 { + v11 = TabEncodeCharsetURL[:] + } + for (src_len - i) >= 3 { + v9 = uint32((uint64(uint64(uint32(src[i]))<<16) | uint64(uint64(uint32(src[i+1]))<<8)) | uint64(uint32(src[i+2]))) + v34 := uint64(uint64(v9)>>18) & 63 + dst[o] = v11[v34] + v39 := uint64(uint64(v9)>>12) & 63 + dst[o+1] = v11[v39] + v46 := uint64(uint64(v9)>>6) & 63 + dst[o+2] = v11[v46] + v51 := v9 & 63 + dst[o+3] = v11[v51] + i += 3 + o += 4 + } + v59 := src_len - i + if v59 == 0 { + return o + } + v9 = uint32(uint64(uint64(uint32(src[i])) << 16)) + if v59 == 2 { + v9 = uint32(uint64(v9) | uint64(uint64(uint32(src[i+1]))<<8)) + } + v76 := uint64(uint64(v9)>>18) & 63 + dst[o] = v11[v76] + v81 := uint64(uint64(v9)>>12) & 63 + dst[o+1] = v11[v81] + o += 2 + if v59 == 2 { + v91 = uint64(uint64(v9)>>6) & 63 + dst[o] = v11[v91] + o++ + if pad_mode != 0 { + dst[o] = '=' + o++ + } + } else { + if pad_mode != 0 { + dst[o] = '=' + dst[o+1] = '=' + o += 2 + } + } + return o +} + +func B64_encode_scalar(src []byte, src_len uint64, dst []byte, dst_cap uint64, url_mode int, pad_mode int) uint64 { + if uint64(len(src)) < src_len { + src_len = uint64(len(src)) + } + return uint64(b64_encode_tail(src[:], src_len, uint64(0), dst[:], uint64(0), url_mode, pad_mode)) +} + +func B64_encode_avx2(src []byte, src_len uint64, dst []byte, dst_cap uint64, url_mode int, pad_mode int) uint64 { + var v9 uint64 + var v11 uint64 + var v13 []byte + if uint64(len(src)) < src_len { + src_len = uint64(len(src)) + } + v9 = 0 + v11 = v9 + v13 = VecEncodeCharsetStd[:] + if url_mode != 0 { + v13 = VecEncodeCharsetURL[:] + } + for { + v21 := ((src_len - v9) >= 28) && ((dst_cap - v11) >= 32) + if !v21 { + break + } + v41 := archsimd.BroadcastUint8x32(0).SetLo(archsimd.LoadUint8x16(src[v9:][:16])).SetHi(archsimd.LoadUint8x16(src[int(v9+12):][:16])).PermuteOrZeroGrouped(archsimd.LoadUint8x32(VecEncodeShuffles[0:32]).BitsToInt8()) + v59 := v41.And(archsimd.BroadcastUint32x8(uint32(264305664)).ReshapeToUint8s()).ReshapeToUint16s().MulHigh(archsimd.BroadcastUint32x8(uint32(67108928)).ReshapeToUint8s().ReshapeToUint16s()).ReshapeToUint8s().Or(v41.And(archsimd.BroadcastUint32x8(uint32(4129776)).ReshapeToUint8s()).ReshapeToUint16s().BitsToInt16().Mul(archsimd.BroadcastUint32x8(uint32(16777232)).ReshapeToUint8s().ReshapeToUint16s().BitsToInt16()).ToBits().ReshapeToUint8s()) + v59.Add(archsimd.LoadUint8x32(v13[0:32]).PermuteOrZeroGrouped(v59.SubSaturated(archsimd.BroadcastUint8x32(byte(51))).Or(archsimd.BroadcastUint8x32(byte(13)).And(archsimd.BroadcastUint8x32(byte(26)).BitsToInt8().Greater(v59.BitsToInt8()).ToInt8x32().ToBits())).BitsToInt8())).Store(dst[v11:][:32]) + v9 += 24 + v11 += 32 + } + return uint64(b64_encode_tail(src[:], src_len, v9, dst[:], v11, url_mode, pad_mode)) +} + +func B64_encode(src []byte, src_len uint64, dst []byte, dst_cap uint64, url_mode int, pad_mode int) uint64 { + if uint64(len(src)) < src_len { + src_len = uint64(len(src)) + } + if archsimd.X86.AVX2() { + return uint64(B64_encode_avx2(src[:], src_len, dst[:], dst_cap, url_mode, pad_mode)) + } + return uint64(B64_encode_scalar(src[:], src_len, dst[:], dst_cap, url_mode, pad_mode)) +} + +func b64_decode_tail(src []byte, src_len uint64, si uint64, dst []byte, o uint64, url_mode int, pad_mode int, pos_err *uint64) uint64 { + var v10 []byte + var v11 uint32 + var v14 int + var v15 int + var v16 uint64 + var v24 int + if uint64(len(src)) < src_len { + src_len = uint64(len(src)) + } + v10 = TabDecodeCharsetStd[:] + if url_mode != 0 { + v10 = TabDecodeCharsetURL[:] + } + for si < src_len { + v11 = 0 + v15 = 4 + v16 = 0xffffffffffffffff + v14 = 0 + for v14 < 4 { + if si == src_len { + if v14 == 0 { + *pos_err = 0xffffffffffffffff + return o + } + if (v14 == 1) || (pad_mode != 0) { + *pos_err = si - uint64(v14) + return o + } + v15 = v14 + break + } + v34 := src[si] + si++ + v37 := v10[v34] + if v37 != 0xff { + v11 = uint32(uint64(v11) | uint64(uint64(uint32(v37))<> 16)) + if v15 >= 3 { + dst[o+1] = byte(uint64(uint64(v11) >> 8)) + } + if v15 == 4 { + dst[o+2] = byte(v11) + } + o += uint64(v15 - 1) + if v16 != 0xffffffffffffffff { + *pos_err = v16 + return o + } + } + *pos_err = 0xffffffffffffffff + return o +} + +func B64_decode_scalar(src []byte, src_len uint64, dst []byte, dst_cap uint64, url_mode int, pad_mode int, pos_err *uint64) uint64 { + if uint64(len(src)) < src_len { + src_len = uint64(len(src)) + } + return uint64(b64_decode_tail(src[:], src_len, uint64(0), dst[:], uint64(0), url_mode, pad_mode, pos_err)) +} + +func B64_decode_avx2(src []byte, src_len uint64, dst []byte, dst_cap uint64, url_mode int, pad_mode int, pos_err *uint64) uint64 { + var v11 uint64 + var v13 uint64 + var v17 []byte + if uint64(len(src)) < src_len { + src_len = uint64(len(src)) + } + v11 = 0 + v13 = v11 + v17 = VecDecodeTableStd[:] + if url_mode != 0 { + v17 = VecDecodeTableURL[:] + } + for { + v25 := ((src_len - v11) >= 32) && ((dst_cap - v13) >= 32) + if !v25 { + break + } + v30 := archsimd.LoadUint8x32(src[v11:][:32]) + v36 := archsimd.BroadcastUint8x32(byte(15)) + v41 := v30.ReshapeToUint32s().ShiftAllRight(4).ReshapeToUint8s().And(v36) + v109 := archsimd.LoadUint32x8Array(&[8]uint32{uint32(0), uint32(1), uint32(2), uint32(4), uint32(5), uint32(6), uint32(3), uint32(7)}).ReshapeToUint8s() + v110 := v30.Add(archsimd.LoadUint8x32(v17[96:128]).IfElse(v30.Equal(archsimd.LoadUint8x32(v17[64:96])).ToInt8x32().ToBits().BitsToInt8().Less(archsimd.BroadcastInt8x32(0)), archsimd.LoadUint8x32(v17[0:32]).PermuteOrZeroGrouped(v41.BitsToInt8()))).And(archsimd.BroadcastUint8x32(byte(63))).DotProductPairsSaturated(archsimd.BroadcastUint32x8(uint32(20971840)).ReshapeToUint8s().BitsToInt8()).ToBits().ReshapeToUint8s().ReshapeToUint16s().BitsToInt16().DotProductPairs(archsimd.BroadcastUint32x8(uint32(69632)).ReshapeToUint8s().ReshapeToUint16s().BitsToInt16()).ToBits().ReshapeToUint8s().PermuteOrZeroGrouped(archsimd.LoadUint8x32(VecPacking[0:32]).BitsToInt8()).ReshapeToUint32s().Permute(v109.ReshapeToUint32s()).ReshapeToUint8s() + if int32(bits.TrailingZeros64(uint64(int64(int(int32(archsimd.LoadUint8x32(v17[32:64]).PermuteOrZeroGrouped(v30.And(v36).BitsToInt8()).And(archsimd.LoadUint8x32(VecDecodeBits[0:32]).PermuteOrZeroGrouped(v41.BitsToInt8())).Equal(archsimd.BroadcastUint8x32(byte(0))).ToBits())))|int64(-4294967296)))) < 32 { + break + } + v110.Store(dst[v13:][:32]) + v11 += 32 + v13 += 24 + } + return uint64(b64_decode_tail(src[:], src_len, v11, dst[:], v13, url_mode, pad_mode, pos_err)) +} + +func B64_decode(src []byte, src_len uint64, dst []byte, dst_cap uint64, url_mode int, pad_mode int, pos_err *uint64) uint64 { + if uint64(len(src)) < src_len { + src_len = uint64(len(src)) + } + if archsimd.X86.AVX2() { + return uint64(B64_decode_avx2(src[:], src_len, dst[:], dst_cap, url_mode, pad_mode, pos_err)) + } + return uint64(B64_decode_scalar(src[:], src_len, dst[:], dst_cap, url_mode, pad_mode, pos_err)) +} + +// IRCosts counted from IR opcodes via rodata.IROpCost; not a kernel-name table. +const ( + __avx2_IRCost = uint64(6) + B64_encode_tail_IRCost = uint64(122) + B64_encode_scalar_IRCost = uint64(4) + B64_encode_avx2_IRCost = uint64(88) + B64_encode_IRCost = uint64(5) + B64_decode_tail_IRCost = uint64(144) + B64_decode_scalar_IRCost = uint64(4) + B64_decode_avx2_IRCost = uint64(125) + B64_decode_IRCost = uint64(5) +) diff --git a/internal/simd/b64_nosimd_gen.go b/internal/simd/b64_nosimd_gen.go new file mode 100644 index 0000000..acd21ac --- /dev/null +++ b/internal/simd/b64_nosimd_gen.go @@ -0,0 +1,227 @@ +//go:build !goexperiment.simd || !amd64 + +// Code generated by sgoiter emit go127; DO NOT EDIT. +// sgoiter-stamp: sha256=047ae148fa4614d63ab852dbd74f52171de727ab3e8e0fd127dfe8d2b6611cfb +package simd + +var ( + TabEncodeCharsetStd = [64]byte{'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', '/'} + TabEncodeCharsetURL = [64]byte{'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-', '_'} + VecEncodeShuffles = [32]byte{1, 0, 2, 1, 4, 3, 5, 4, 7, 6, 8, 7, 10, 9, 11, 10, 1, 0, 2, 1, 4, 3, 5, 4, 7, 6, 8, 7, 10, 9, 11, 10} + VecEncodeCharsetStd = [32]byte{71, 252, 252, 252, 252, 252, 252, 252, 252, 252, 252, 237, 240, 'A', 0, 0, 71, 252, 252, 252, 252, 252, 252, 252, 252, 252, 252, 237, 240, 'A', 0, 0} + VecEncodeCharsetURL = [32]byte{71, 252, 252, 252, 252, 252, 252, 252, 252, 252, 252, 239, 32, 'A', 0, 0, 71, 252, 252, 252, 252, 252, 252, 252, 252, 252, 252, 239, 32, 'A', 0, 0} + VecPacking = [32]byte{2, 1, 0, 6, 5, 4, 10, 9, 8, 14, 13, 12, 128, 128, 128, 128, 2, 1, 0, 6, 5, 4, 10, 9, 8, 14, 13, 12, 128, 128, 128, 128} + VecDecodeBits = [32]byte{0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00} + VecDecodeTableStd = [128]byte{0x00, 0x00, 0x13, 0x04, 0xbf, 0xbf, 0xb9, 0xb9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x13, 0x04, 0xbf, 0xbf, 0xb9, 0xb9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf0, 0x54, 0x50, 0x50, 0x50, 0x54, 0xa8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf0, 0x54, 0x50, 0x50, 0x50, 0x54, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10} + VecDecodeTableURL = [128]byte{0x00, 0x00, 0x11, 0x04, 0xbf, 0xbf, 0xb9, 0xb9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x11, 0x04, 0xbf, 0xbf, 0xb9, 0xb9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf0, 0x50, 0x50, 0x54, 0x50, 0x70, 0xa8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf0, 0x50, 0x50, 0x54, 0x50, 0x70, 0x5f, 0x5f, 0x5f, 0x5f, 0x5f, 0x5f, 0x5f, 0x5f, 0x5f, 0x5f, 0x5f, 0x5f, 0x5f, 0x5f, 0x5f, 0x5f, 0x5f, 0x5f, 0x5f, 0x5f, 0x5f, 0x5f, 0x5f, 0x5f, 0x5f, 0x5f, 0x5f, 0x5f, 0x5f, 0x5f, 0x5f, 0x5f, 0xe0, 0xe0, 0xe0, 0xe0, 0xe0, 0xe0, 0xe0, 0xe0, 0xe0, 0xe0, 0xe0, 0xe0, 0xe0, 0xe0, 0xe0, 0xe0, 0xe0, 0xe0, 0xe0, 0xe0, 0xe0, 0xe0, 0xe0, 0xe0, 0xe0, 0xe0, 0xe0, 0xe0, 0xe0, 0xe0, 0xe0, 0xe0} + TabDecodeCharsetStd = [256]byte{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 62, 0xff, 0xff, 0xff, 63, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff} + TabDecodeCharsetURL = [256]byte{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 62, 0xff, 0xff, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 0xff, 0xff, 0xff, 0xff, 63, 0xff, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff} +) + +func b64_encode_tail(src []byte, src_len uint64, i uint64, dst []byte, o uint64, url_mode int, pad_mode int) uint64 { + var v9 uint32 + var v11 []byte + var v91 uint64 + if uint64(len(src)) < src_len { + src_len = uint64(len(src)) + } + v11 = TabEncodeCharsetStd[:] + if url_mode != 0 { + v11 = TabEncodeCharsetURL[:] + } + for (src_len - i) >= 3 { + v9 = uint32((uint64(uint64(uint32(src[i]))<<16) | uint64(uint64(uint32(src[i+1]))<<8)) | uint64(uint32(src[i+2]))) + v34 := uint64(uint64(v9)>>18) & 63 + dst[o] = v11[v34] + v39 := uint64(uint64(v9)>>12) & 63 + dst[o+1] = v11[v39] + v46 := uint64(uint64(v9)>>6) & 63 + dst[o+2] = v11[v46] + v51 := v9 & 63 + dst[o+3] = v11[v51] + i += 3 + o += 4 + } + v59 := src_len - i + if v59 == 0 { + return o + } + v9 = uint32(uint64(uint64(uint32(src[i])) << 16)) + if v59 == 2 { + v9 = uint32(uint64(v9) | uint64(uint64(uint32(src[i+1]))<<8)) + } + v76 := uint64(uint64(v9)>>18) & 63 + dst[o] = v11[v76] + v81 := uint64(uint64(v9)>>12) & 63 + dst[o+1] = v11[v81] + o += 2 + if v59 == 2 { + v91 = uint64(uint64(v9)>>6) & 63 + dst[o] = v11[v91] + o++ + if pad_mode != 0 { + dst[o] = '=' + o++ + } + } else { + if pad_mode != 0 { + dst[o] = '=' + dst[o+1] = '=' + o += 2 + } + } + return o +} + +func B64_encode_scalar(src []byte, src_len uint64, dst []byte, dst_cap uint64, url_mode int, pad_mode int) uint64 { + if uint64(len(src)) < src_len { + src_len = uint64(len(src)) + } + return uint64(b64_encode_tail(src[:], src_len, uint64(0), dst[:], uint64(0), url_mode, pad_mode)) +} + +func B64_encode(src []byte, src_len uint64, dst []byte, dst_cap uint64, url_mode int, pad_mode int) uint64 { + if uint64(len(src)) < src_len { + src_len = uint64(len(src)) + } + return uint64(B64_encode_scalar(src[:], src_len, dst[:], dst_cap, url_mode, pad_mode)) +} + +func b64_decode_tail(src []byte, src_len uint64, si uint64, dst []byte, o uint64, url_mode int, pad_mode int, pos_err *uint64) uint64 { + var v10 []byte + var v11 uint32 + var v14 int + var v15 int + var v16 uint64 + var v24 int + if uint64(len(src)) < src_len { + src_len = uint64(len(src)) + } + v10 = TabDecodeCharsetStd[:] + if url_mode != 0 { + v10 = TabDecodeCharsetURL[:] + } + for si < src_len { + v11 = 0 + v15 = 4 + v16 = 0xffffffffffffffff + v14 = 0 + for v14 < 4 { + if si == src_len { + if v14 == 0 { + *pos_err = 0xffffffffffffffff + return o + } + if (v14 == 1) || (pad_mode != 0) { + *pos_err = si - uint64(v14) + return o + } + v15 = v14 + break + } + v34 := src[si] + si++ + v37 := v10[v34] + if v37 != 0xff { + v11 = uint32(uint64(v11) | uint64(uint64(uint32(v37))<> 16)) + if v15 >= 3 { + dst[o+1] = byte(uint64(uint64(v11) >> 8)) + } + if v15 == 4 { + dst[o+2] = byte(v11) + } + o += uint64(v15 - 1) + if v16 != 0xffffffffffffffff { + *pos_err = v16 + return o + } + } + *pos_err = 0xffffffffffffffff + return o +} + +func B64_decode_scalar(src []byte, src_len uint64, dst []byte, dst_cap uint64, url_mode int, pad_mode int, pos_err *uint64) uint64 { + if uint64(len(src)) < src_len { + src_len = uint64(len(src)) + } + return uint64(b64_decode_tail(src[:], src_len, uint64(0), dst[:], uint64(0), url_mode, pad_mode, pos_err)) +} + +func B64_decode(src []byte, src_len uint64, dst []byte, dst_cap uint64, url_mode int, pad_mode int, pos_err *uint64) uint64 { + if uint64(len(src)) < src_len { + src_len = uint64(len(src)) + } + return uint64(B64_decode_scalar(src[:], src_len, dst[:], dst_cap, url_mode, pad_mode, pos_err)) +} + +// IRCosts counted from IR opcodes via rodata.IROpCost; not a kernel-name table. +const ( + B64_encode_tail_IRCost = uint64(122) + B64_encode_scalar_IRCost = uint64(4) + B64_encode_IRCost = uint64(2) + B64_decode_tail_IRCost = uint64(144) + B64_decode_scalar_IRCost = uint64(4) + B64_decode_IRCost = uint64(2) +)