chore: migrate to gitea
Some checks failed
golangci-lint / lint (push) Failing after 1m30s
Test / test (push) Failing after 2m17s

This commit is contained in:
2026-01-27 00:40:46 +01:00
parent 1e05874160
commit f8df24c29d
3169 changed files with 1216434 additions and 1587 deletions

View File

@@ -0,0 +1,285 @@
/*
* Copyright 2021 ByteDance Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package alg
import (
"encoding"
"reflect"
"strconv"
"sync"
"unsafe"
"github.com/bytedance/sonic/internal/encoder/vars"
"github.com/bytedance/sonic/internal/rt"
)
type _MapPair struct {
k string // when the map key is integer, k is pointed to m
v unsafe.Pointer
m [32]byte
}
type MapIterator struct {
It rt.GoMapIterator // must be the first field
kv rt.GoSlice // slice of _MapPair
ki int
}
var (
iteratorPool = sync.Pool{}
iteratorPair = rt.UnpackType(reflect.TypeOf(_MapPair{}))
)
func init() {
if unsafe.Offsetof(MapIterator{}.It) != 0 {
panic("_MapIterator.it is not the first field")
}
}
func newIterator() *MapIterator {
if v := iteratorPool.Get(); v == nil {
return new(MapIterator)
} else {
return resetIterator(v.(*MapIterator))
}
}
func resetIterator(p *MapIterator) *MapIterator {
p.ki = 0
p.It = rt.GoMapIterator{}
p.kv.Len = 0
return p
}
func (self *MapIterator) at(i int) *_MapPair {
return (*_MapPair)(unsafe.Pointer(uintptr(self.kv.Ptr) + uintptr(i) * unsafe.Sizeof(_MapPair{})))
}
func (self *MapIterator) add() (p *_MapPair) {
p = self.at(self.kv.Len)
self.kv.Len++
return
}
func (self *MapIterator) data() (p []_MapPair) {
*(*rt.GoSlice)(unsafe.Pointer(&p)) = self.kv
return
}
func (self *MapIterator) append(t *rt.GoType, k unsafe.Pointer, v unsafe.Pointer) (err error) {
p := self.add()
p.v = v
tk := t.Kind()
// followed as `encoding/json/emcode.go:resolveKeyName
if tk == reflect.String {
p.k = *(*string)(k)
return nil
}
// check if the key implements the encoding.TextMarshaler interface
if t.Pack().Implements(vars.EncodingTextMarshalerType) {
if tk != reflect.Interface {
return self.appendConcrete(p, t, k)
} else {
return self.appendInterface(p, t, k)
}
}
return self.appendGeneric(p, t, tk, k)
}
func (self *MapIterator) appendGeneric(p *_MapPair, t *rt.GoType, v reflect.Kind, k unsafe.Pointer) error {
switch v {
case reflect.Int : p.k = rt.Mem2Str(strconv.AppendInt(p.m[:0], int64(*(*int)(k)), 10)) ; return nil
case reflect.Int8 : p.k = rt.Mem2Str(strconv.AppendInt(p.m[:0], int64(*(*int8)(k)), 10)) ; return nil
case reflect.Int16 : p.k = rt.Mem2Str(strconv.AppendInt(p.m[:0], int64(*(*int16)(k)), 10)) ; return nil
case reflect.Int32 : p.k = rt.Mem2Str(strconv.AppendInt(p.m[:0], int64(*(*int32)(k)), 10)) ; return nil
case reflect.Int64 : p.k = rt.Mem2Str(strconv.AppendInt(p.m[:0], int64(*(*int64)(k)), 10)) ; return nil
case reflect.Uint : p.k = rt.Mem2Str(strconv.AppendUint(p.m[:0], uint64(*(*uint)(k)), 10)) ; return nil
case reflect.Uint8 : p.k = rt.Mem2Str(strconv.AppendUint(p.m[:0], uint64(*(*uint8)(k)), 10)) ; return nil
case reflect.Uint16 : p.k = rt.Mem2Str(strconv.AppendUint(p.m[:0], uint64(*(*uint16)(k)), 10)) ; return nil
case reflect.Uint32 : p.k = rt.Mem2Str(strconv.AppendUint(p.m[:0], uint64(*(*uint32)(k)), 10)) ; return nil
case reflect.Uint64 : p.k = rt.Mem2Str(strconv.AppendUint(p.m[:0], uint64(*(*uint64)(k)), 10)) ; return nil
case reflect.Uintptr : p.k = rt.Mem2Str(strconv.AppendUint(p.m[:0], uint64(*(*uintptr)(k)), 10)) ; return nil
case reflect.Bool : if *(*bool)(k) { p.k = "true" } else { p.k = "false" }; return nil
default : return vars.Error_type(t.Pack())
}
}
func (self *MapIterator) appendConcrete(p *_MapPair, t *rt.GoType, k unsafe.Pointer) error {
// compiler has already checked that the type implements the encoding.MarshalText interface
if !t.Indirect() {
k = *(*unsafe.Pointer)(k)
}
// check the TextMarshaler interface
eface := rt.GoEface{Value: k, Type: t}.Pack()
e, ok := eface.(encoding.TextMarshaler)
if !ok {
return vars.Error_type(t.Pack())
}
// check for nil pointer
if t.Kind() == reflect.Ptr && k == nil {
p.k = ""
return nil
}
out, err := e.MarshalText()
if err != nil {
return err
}
p.k = rt.Mem2Str(out)
return nil
}
func (self *MapIterator) appendInterface(p *_MapPair, t *rt.GoType, k unsafe.Pointer) (err error) {
if len(rt.IfaceType(t).Methods) == 0 {
panic("unexpected map key type")
}
p.k, err = asText(k)
return
}
func IteratorStop(p *MapIterator) {
iteratorPool.Put(p)
}
func IteratorNext(p *MapIterator) {
i := p.ki
t := &p.It
/* check for unordered iteration */
if i < 0 {
rt.Mapiternext(t)
return
}
/* check for end of iteration */
if p.ki >= p.kv.Len {
t.K = nil
t.V = nil
return
}
/* update the key-value pair, and increase the pointer */
t.K = unsafe.Pointer(&p.at(p.ki).k)
t.V = p.at(p.ki).v
p.ki++
}
func IteratorStart(t *rt.GoMapType, m unsafe.Pointer, fv uint64) (*MapIterator, error) {
it := newIterator()
rt.Mapiterinit(t, m, &it.It)
count := rt.Maplen(m)
/* check for key-sorting, empty map don't need sorting */
if count == 0 || (fv & (1<<BitSortMapKeys)) == 0 {
it.ki = -1
return it, nil
}
/* pre-allocate space if needed */
if count > it.kv.Cap {
it.kv = rt.GrowSlice(iteratorPair, it.kv, count)
}
/* dump all the key-value pairs */
for ; it.It.K != nil; rt.Mapiternext(&it.It) {
if err := it.append(t.Key, it.It.K, it.It.V); err != nil {
IteratorStop(it)
return nil, err
}
}
/* sort the keys, map with only 1 item don't need sorting */
if it.ki = 1; count > 1 {
radixQsort(it.data(), 0, maxDepth(it.kv.Len))
}
/* load the first pair into iterator */
it.It.V = it.at(0).v
it.It.K = unsafe.Pointer(&it.at(0).k)
return it, nil
}
func asText(v unsafe.Pointer) (string, error) {
text := rt.AssertI2I(rt.UnpackType(vars.EncodingTextMarshalerType), *(*rt.GoIface)(v))
r, e := (*(*encoding.TextMarshaler)(unsafe.Pointer(&text))).MarshalText()
return rt.Mem2Str(r), e
}
func IsValidNumber(s string) bool {
// This function implements the JSON numbers grammar.
// See https://tools.ietf.org/html/rfc7159#section-6
// and https://www.json.org/img/number.png
if s == "" {
return false
}
// Optional -
if s[0] == '-' {
s = s[1:]
if s == "" {
return false
}
}
// Digits
switch {
default:
return false
case s[0] == '0':
s = s[1:]
case '1' <= s[0] && s[0] <= '9':
s = s[1:]
for len(s) > 0 && '0' <= s[0] && s[0] <= '9' {
s = s[1:]
}
}
// . followed by 1 or more digits.
if len(s) >= 2 && s[0] == '.' && '0' <= s[1] && s[1] <= '9' {
s = s[2:]
for len(s) > 0 && '0' <= s[0] && s[0] <= '9' {
s = s[1:]
}
}
// e or E followed by an optional - or + and
// 1 or more digits.
if len(s) >= 2 && (s[0] == 'e' || s[0] == 'E') {
s = s[1:]
if s[0] == '+' || s[0] == '-' {
s = s[1:]
if s == "" {
return false
}
}
for len(s) > 0 && '0' <= s[0] && s[0] <= '9' {
s = s[1:]
}
}
// Make sure we are at the end.
return s == ""
}

View File

@@ -0,0 +1,31 @@
/**
* Copyright 2024 ByteDance Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package alg
const (
BitSortMapKeys = iota
BitEscapeHTML
BitCompactMarshaler
BitNoQuoteTextMarshaler
BitNoNullSliceOrMap
BitValidateString
BitNoValidateJSONMarshaler
BitNoEncoderNewline
BitEncodeNullForInfOrNan
BitPointerValue = 63
)

View File

@@ -0,0 +1,206 @@
/*
* Copyright 2021 ByteDance Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package alg
// Algorithm 3-way Radix Quicksort, d means the radix.
// Reference: https://algs4.cs.princeton.edu/51radix/Quick3string.java.html
func radixQsort(kvs []_MapPair, d, maxDepth int) {
for len(kvs) > 11 {
// To avoid the worst case of quickSort (time: O(n^2)), use introsort here.
// Reference: https://en.wikipedia.org/wiki/Introsort and
// https://github.com/golang/go/issues/467
if maxDepth == 0 {
heapSort(kvs, 0, len(kvs))
return
}
maxDepth--
p := pivot(kvs, d)
lt, i, gt := 0, 0, len(kvs)
for i < gt {
c := byteAt(kvs[i].k, d)
if c < p {
swap(kvs, lt, i)
i++
lt++
} else if c > p {
gt--
swap(kvs, i, gt)
} else {
i++
}
}
// kvs[0:lt] < v = kvs[lt:gt] < kvs[gt:len(kvs)]
// Native implementation:
// radixQsort(kvs[:lt], d, maxDepth)
// if p > -1 {
// radixQsort(kvs[lt:gt], d+1, maxDepth)
// }
// radixQsort(kvs[gt:], d, maxDepth)
// Optimize as follows: make recursive calls only for the smaller parts.
// Reference: https://www.geeksforgeeks.org/quicksort-tail-call-optimization-reducing-worst-case-space-log-n/
if p == -1 {
if lt > len(kvs) - gt {
radixQsort(kvs[gt:], d, maxDepth)
kvs = kvs[:lt]
} else {
radixQsort(kvs[:lt], d, maxDepth)
kvs = kvs[gt:]
}
} else {
ml := maxThree(lt, gt-lt, len(kvs)-gt)
if ml == lt {
radixQsort(kvs[lt:gt], d+1, maxDepth)
radixQsort(kvs[gt:], d, maxDepth)
kvs = kvs[:lt]
} else if ml == gt-lt {
radixQsort(kvs[:lt], d, maxDepth)
radixQsort(kvs[gt:], d, maxDepth)
kvs = kvs[lt:gt]
d += 1
} else {
radixQsort(kvs[:lt], d, maxDepth)
radixQsort(kvs[lt:gt], d+1, maxDepth)
kvs = kvs[gt:]
}
}
}
insertRadixSort(kvs, d)
}
func insertRadixSort(kvs []_MapPair, d int) {
for i := 1; i < len(kvs); i++ {
for j := i; j > 0 && lessFrom(kvs[j].k, kvs[j-1].k, d); j-- {
swap(kvs, j, j-1)
}
}
}
func pivot(kvs []_MapPair, d int) int {
m := len(kvs) >> 1
if len(kvs) > 40 {
// Tukey's ``Ninther,'' median of three mediankvs of three.
t := len(kvs) / 8
return medianThree(
medianThree(byteAt(kvs[0].k, d), byteAt(kvs[t].k, d), byteAt(kvs[2*t].k, d)),
medianThree(byteAt(kvs[m].k, d), byteAt(kvs[m-t].k, d), byteAt(kvs[m+t].k, d)),
medianThree(byteAt(kvs[len(kvs)-1].k, d),
byteAt(kvs[len(kvs)-1-t].k, d),
byteAt(kvs[len(kvs)-1-2*t].k, d)))
}
return medianThree(byteAt(kvs[0].k, d), byteAt(kvs[m].k, d), byteAt(kvs[len(kvs)-1].k, d))
}
func medianThree(i, j, k int) int {
if i > j {
i, j = j, i
} // i < j
if k < i {
return i
}
if k > j {
return j
}
return k
}
func maxThree(i, j, k int) int {
max := i
if max < j {
max = j
}
if max < k {
max = k
}
return max
}
// maxDepth returns a threshold at which quicksort should switch
// to heapsort. It returnkvs 2*ceil(lg(n+1)).
func maxDepth(n int) int {
var depth int
for i := n; i > 0; i >>= 1 {
depth++
}
return depth * 2
}
// siftDown implements the heap property on kvs[lo:hi].
// first is an offset into the array where the root of the heap lies.
func siftDown(kvs []_MapPair, lo, hi, first int) {
root := lo
for {
child := 2*root + 1
if child >= hi {
break
}
if child+1 < hi && kvs[first+child].k < kvs[first+child+1].k {
child++
}
if kvs[first+root].k >= kvs[first+child].k {
return
}
swap(kvs, first+root, first+child)
root = child
}
}
func heapSort(kvs []_MapPair, a, b int) {
first := a
lo := 0
hi := b - a
// Build heap with the greatest element at top.
for i := (hi - 1) / 2; i >= 0; i-- {
siftDown(kvs, i, hi, first)
}
// Pop elements, the largest first, into end of kvs.
for i := hi - 1; i >= 0; i-- {
swap(kvs, first, first+i)
siftDown(kvs, lo, i, first)
}
}
// Note that _MapPair.k is NOT pointed to _MapPair.m when map key is integer after swap
func swap(kvs []_MapPair, a, b int) {
kvs[a].k, kvs[b].k = kvs[b].k, kvs[a].k
kvs[a].v, kvs[b].v = kvs[b].v, kvs[a].v
}
// Compare two strings from the pos d.
func lessFrom(a, b string, d int) bool {
l := len(a)
if l > len(b) {
l = len(b)
}
for i := d; i < l; i++ {
if a[i] == b[i] {
continue
}
return a[i] < b[i]
}
return len(a) < len(b)
}
func byteAt(b string, p int) int {
if p < len(b) {
return int(b[p])
}
return -1
}

View File

@@ -0,0 +1,189 @@
//go:build (amd64 && go1.16 && !go1.27) || (arm64 && go1.20 && !go1.27)
// +build amd64,go1.16,!go1.27 arm64,go1.20,!go1.27
/**
* Copyright 2024 ByteDance Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package alg
import (
"runtime"
"strconv"
"unsafe"
"github.com/bytedance/sonic/internal/native"
"github.com/bytedance/sonic/internal/native/types"
"github.com/bytedance/sonic/internal/rt"
)
// Valid validates json and returns first non-blank character position,
// if it is only one valid json value.
// Otherwise returns invalid character position using start.
//
// Note: it does not check for the invalid UTF-8 characters.
func Valid(data []byte) (ok bool, start int) {
n := len(data)
if n == 0 {
return false, -1
}
s := rt.Mem2Str(data)
p := 0
m := types.NewStateMachine()
ret := native.ValidateOne(&s, &p, m, 0)
types.FreeStateMachine(m)
if ret < 0 {
return false, p-1
}
/* check for trailing spaces */
for ;p < n; p++ {
if (types.SPACE_MASK & (1 << data[p])) == 0 {
return false, p
}
}
return true, ret
}
var typeByte = rt.UnpackEface(byte(0)).Type
func Quote(buf []byte, val string, double bool) []byte {
if len(val) == 0 {
if double {
return append(buf, `"\"\""`...)
}
return append(buf, `""`...)
}
if double {
buf = append(buf, `"\"`...)
} else {
buf = append(buf, `"`...)
}
sp := rt.IndexChar(val, 0)
nb := len(val)
buf = rt.GuardSlice2(buf, nb+1)
b := (*rt.GoSlice)(unsafe.Pointer(&buf))
// input buffer
for nb > 0 {
// output buffer
dp := unsafe.Pointer(uintptr(b.Ptr) + uintptr(b.Len))
dn := b.Cap - b.Len
// call native.Quote, dn is byte count it outputs
opts := uint64(0)
if double {
opts = types.F_DOUBLE_UNQUOTE
}
ret := native.Quote(sp, nb, dp, &dn, opts)
// update *buf length
b.Len += dn
// no need more output
if ret >= 0 {
break
}
// double buf size
*b = rt.GrowSlice(typeByte, *b, b.Cap*2)
// ret is the complement of consumed input
ret = ^ret
// update input buffer
nb -= ret
if nb > 0 {
sp = unsafe.Pointer(uintptr(sp) + uintptr(ret))
}
}
runtime.KeepAlive(buf)
runtime.KeepAlive(sp)
if double {
buf = append(buf, `\""`...)
} else {
buf = append(buf, `"`...)
}
return buf
}
func HtmlEscape(dst []byte, src []byte) []byte {
var sidx int
dst = append(dst, src[:0]...) // avoid check nil dst
sbuf := (*rt.GoSlice)(unsafe.Pointer(&src))
dbuf := (*rt.GoSlice)(unsafe.Pointer(&dst))
/* grow dst if it is shorter */
if cap(dst)-len(dst) < len(src)+types.BufPaddingSize {
cap := len(src)*3/2 + types.BufPaddingSize
*dbuf = rt.GrowSlice(typeByte, *dbuf, cap)
}
for sidx < sbuf.Len {
sp := rt.Add(sbuf.Ptr, uintptr(sidx))
dp := rt.Add(dbuf.Ptr, uintptr(dbuf.Len))
sn := sbuf.Len - sidx
dn := dbuf.Cap - dbuf.Len
nb := native.HTMLEscape(sp, sn, dp, &dn)
/* check for errors */
if dbuf.Len += dn; nb >= 0 {
break
}
/* not enough space, grow the slice and try again */
sidx += ^nb
*dbuf = rt.GrowSlice(typeByte, *dbuf, dbuf.Cap*2)
}
return dst
}
func F64toa(buf []byte, v float64) ([]byte) {
if v == 0 {
return append(buf, '0')
}
buf = rt.GuardSlice2(buf, 64)
ret := native.F64toa((*byte)(rt.IndexByte(buf, len(buf))), v)
if ret > 0 {
return buf[:len(buf)+ret]
} else {
return buf
}
}
func F32toa(buf []byte, v float32) ([]byte) {
if v == 0 {
return append(buf, '0')
}
buf = rt.GuardSlice2(buf, 64)
ret := native.F32toa((*byte)(rt.IndexByte(buf, len(buf))), v)
if ret > 0 {
return buf[:len(buf)+ret]
} else {
return buf
}
}
func I64toa(buf []byte, v int64) ([]byte) {
return strconv.AppendInt(buf, v, 10)
}
func U64toa(buf []byte, v uint64) ([]byte) {
return strconv.AppendUint(buf, v, 10)
}

View File

@@ -0,0 +1,148 @@
// +build !amd64,!arm64 go1.27 !go1.16 arm64,!go1.20
/**
* Copyright 2024 ByteDance Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package alg
import (
_ "unsafe"
"unicode/utf8"
"strconv"
"bytes"
"encoding/json"
"github.com/bytedance/sonic/internal/rt"
)
// Valid validates json and returns first non-blank character position,
// if it is only one valid json value.
// Otherwise returns invalid character position using start.
//
// Note: it does not check for the invalid UTF-8 characters.
func Valid(data []byte) (ok bool, start int) {
ok = json.Valid(data)
return ok, 0
}
var typeByte = rt.UnpackEface(byte(0)).Type
func Quote(e []byte, s string, double bool) []byte {
if len(s) == 0 {
if double {
return append(e, `"\"\""`...)
}
return append(e, `""`...)
}
b := e
ss := len(e)
e = append(e, '"')
start := 0
for i := 0; i < len(s); {
if b := s[i]; b < utf8.RuneSelf {
if rt.SafeSet[b] {
i++
continue
}
if start < i {
e = append(e, s[start:i]...)
}
e = append(e, '\\')
switch b {
case '\\', '"':
e = append(e, b)
case '\n':
e = append(e, 'n')
case '\r':
e = append(e, 'r')
case '\t':
e = append(e, 't')
default:
// This encodes bytes < 0x20 except for \t, \n and \r.
// If escapeHTML is set, it also escapes <, >, and &
// because they can lead to security holes when
// user-controlled strings are rendered into JSON
// and served to some browsers.
e = append(e, `u00`...)
e = append(e, rt.Hex[b>>4])
e = append(e, rt.Hex[b&0xF])
}
i++
start = i
continue
}
c, size := utf8.DecodeRuneInString(s[i:])
// if correct && c == utf8.RuneError && size == 1 {
// if start < i {
// e = append(e, s[start:i]...)
// }
// e = append(e, `\ufffd`...)
// i += size
// start = i
// continue
// }
if c == '\u2028' || c == '\u2029' {
if start < i {
e = append(e, s[start:i]...)
}
e = append(e, `\u202`...)
e = append(e, rt.Hex[c&0xF])
i += size
start = i
continue
}
i += size
}
if start < len(s) {
e = append(e, s[start:]...)
}
e = append(e, '"')
if double {
return strconv.AppendQuote(b, string(e[ss:]))
} else {
return e
}
}
func HtmlEscape(dst []byte, src []byte) []byte {
buf := bytes.NewBuffer(dst)
json.HTMLEscape(buf, src)
return buf.Bytes()
}
func F64toa(buf []byte, v float64) ([]byte) {
bs := bytes.NewBuffer(buf)
_ = json.NewEncoder(bs).Encode(v)
return bs.Bytes()
}
func F32toa(buf []byte, v float32) ([]byte) {
bs := bytes.NewBuffer(buf)
_ = json.NewEncoder(bs).Encode(v)
return bs.Bytes()
}
func I64toa(buf []byte, v int64) ([]byte) {
return strconv.AppendInt(buf, int64(v), 10)
}
func U64toa(buf []byte, v uint64) ([]byte) {
return strconv.AppendUint(buf, v, 10)
}