chore: migrate to gitea
Some checks failed
golangci-lint / lint (push) Failing after 21s
Test / test (push) Failing after 2m17s

This commit is contained in:
2026-01-27 01:40:31 +01:00
parent a9bca767a9
commit 1a27ed5274
3163 changed files with 1216358 additions and 1529 deletions

View File

@@ -0,0 +1,92 @@
// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package gensupport
import (
"bytes"
"hash/crc32"
"io"
"google.golang.org/api/googleapi"
)
// MediaBuffer buffers data from an io.Reader to support uploading media in
// retryable chunks. It should be created with NewMediaBuffer.
type MediaBuffer struct {
media io.Reader
chunk []byte // The current chunk which is pending upload. The capacity is the chunk size.
err error // Any error generated when populating chunk by reading media.
// The absolute position of chunk in the underlying media.
off int64
// fullObjectChecksum holds the running checksum of streamed media chunks when automatic checksum
// calculation is enabled via enableAutoChecksum.
fullObjectChecksum uint32
enableAutoChecksum bool
}
var (
crc32cTable = crc32.MakeTable(crc32.Castagnoli)
)
// NewMediaBuffer initializes a MediaBuffer.
func NewMediaBuffer(media io.Reader, chunkSize int) *MediaBuffer {
return &MediaBuffer{media: media, chunk: make([]byte, 0, chunkSize)}
}
// Chunk returns the current buffered chunk, the offset in the underlying media
// from which the chunk is drawn, and the size of the chunk.
// Successive calls to Chunk return the same chunk between calls to Next.
func (mb *MediaBuffer) Chunk() (chunk io.Reader, off int64, size int, err error) {
// There may already be data in chunk if Next has not been called since the previous call to Chunk.
if mb.err == nil && len(mb.chunk) == 0 {
mb.err = mb.loadChunk()
}
return bytes.NewReader(mb.chunk), mb.off, len(mb.chunk), mb.err
}
// loadChunk will read from media into chunk, up to the capacity of chunk.
func (mb *MediaBuffer) loadChunk() error {
bufSize := cap(mb.chunk)
mb.chunk = mb.chunk[:bufSize]
read := 0
var err error
for err == nil && read < bufSize {
var n int
n, err = mb.media.Read(mb.chunk[read:])
read += n
}
mb.chunk = mb.chunk[:read]
if mb.enableAutoChecksum {
mb.fullObjectChecksum = crc32.Update(mb.fullObjectChecksum, crc32cTable, mb.chunk)
}
return err
}
// Next advances to the next chunk, which will be returned by the next call to Chunk.
// Calls to Next without a corresponding prior call to Chunk will have no effect.
func (mb *MediaBuffer) Next() {
mb.off += int64(len(mb.chunk))
mb.chunk = mb.chunk[0:0]
}
type readerTyper struct {
io.Reader
googleapi.ContentTyper
}
// ReaderAtToReader adapts a ReaderAt to be used as a Reader.
// If ra implements googleapi.ContentTyper, then the returned reader
// will also implement googleapi.ContentTyper, delegating to ra.
func ReaderAtToReader(ra io.ReaderAt, size int64) io.Reader {
r := io.NewSectionReader(ra, 0, size)
if typer, ok := ra.(googleapi.ContentTyper); ok {
return readerTyper{r, typer}
}
return r
}

View File

@@ -0,0 +1,10 @@
// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Package gensupport is an internal implementation detail used by code
// generated by the google-api-go-generator tool.
//
// This package may be modified at any time without regard for backwards
// compatibility. It should not be used directly by API users.
package gensupport

View File

@@ -0,0 +1,24 @@
// Copyright 2022 Google LLC. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package gensupport
import (
"errors"
"github.com/googleapis/gax-go/v2/apierror"
"google.golang.org/api/googleapi"
)
// WrapError creates an [apierror.APIError] from err, wraps it in err, and
// returns err. If err is not a [googleapi.Error] (or a
// [google.golang.org/grpc/status.Status]), it returns err without modification.
func WrapError(err error) error {
var herr *googleapi.Error
apiError, ok := apierror.ParseError(err, false)
if ok && errors.As(err, &herr) {
herr.Wrap(apiError)
}
return err
}

View File

@@ -0,0 +1,236 @@
// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package gensupport
import (
"encoding/json"
"fmt"
"reflect"
"strings"
)
// MarshalJSON returns a JSON encoding of schema containing only selected fields.
// A field is selected if any of the following is true:
// - it has a non-empty value
// - its field name is present in forceSendFields and it is not a nil pointer or nil interface
// - its field name is present in nullFields.
//
// The JSON key for each selected field is taken from the field's json: struct tag.
func MarshalJSON(schema interface{}, forceSendFields, nullFields []string) ([]byte, error) {
if len(forceSendFields) == 0 && len(nullFields) == 0 {
return json.Marshal(schema)
}
mustInclude := make(map[string]bool)
for _, f := range forceSendFields {
mustInclude[f] = true
}
useNull := make(map[string]bool)
useNullMaps := make(map[string]map[string]bool)
for _, nf := range nullFields {
parts := strings.SplitN(nf, ".", 2)
field := parts[0]
if len(parts) == 1 {
useNull[field] = true
} else {
if useNullMaps[field] == nil {
useNullMaps[field] = map[string]bool{}
}
useNullMaps[field][parts[1]] = true
}
}
dataMap, err := schemaToMap(schema, mustInclude, useNull, useNullMaps)
if err != nil {
return nil, err
}
return json.Marshal(dataMap)
}
func schemaToMap(schema interface{}, mustInclude, useNull map[string]bool, useNullMaps map[string]map[string]bool) (map[string]interface{}, error) {
m := make(map[string]interface{})
s := reflect.ValueOf(schema)
st := s.Type()
for i := 0; i < s.NumField(); i++ {
jsonTag := st.Field(i).Tag.Get("json")
if jsonTag == "" {
continue
}
tag, err := parseJSONTag(jsonTag)
if err != nil {
return nil, err
}
if tag.ignore {
continue
}
v := s.Field(i)
f := st.Field(i)
if useNull[f.Name] {
if !isEmptyValue(v) {
return nil, fmt.Errorf("field %q in NullFields has non-empty value", f.Name)
}
m[tag.apiName] = nil
continue
}
if !includeField(v, f, mustInclude) {
continue
}
// If map fields are explicitly set to null, use a map[string]interface{}.
if f.Type.Kind() == reflect.Map && useNullMaps[f.Name] != nil {
ms, ok := v.Interface().(map[string]string)
if !ok {
mi, err := initMapSlow(v, f.Name, useNullMaps)
if err != nil {
return nil, err
}
m[tag.apiName] = mi
continue
}
mi := map[string]interface{}{}
for k, v := range ms {
mi[k] = v
}
for k := range useNullMaps[f.Name] {
mi[k] = nil
}
m[tag.apiName] = mi
continue
}
// nil maps are treated as empty maps.
if f.Type.Kind() == reflect.Map && v.IsNil() {
m[tag.apiName] = map[string]string{}
continue
}
// nil slices are treated as empty slices.
if f.Type.Kind() == reflect.Slice && v.IsNil() {
m[tag.apiName] = []bool{}
continue
}
if tag.stringFormat {
m[tag.apiName] = formatAsString(v, f.Type.Kind())
} else {
m[tag.apiName] = v.Interface()
}
}
return m, nil
}
// initMapSlow uses reflection to build up a map object. This is slower than
// the default behavior so it should be used only as a fallback.
func initMapSlow(rv reflect.Value, fieldName string, useNullMaps map[string]map[string]bool) (map[string]interface{}, error) {
mi := map[string]interface{}{}
iter := rv.MapRange()
for iter.Next() {
k, ok := iter.Key().Interface().(string)
if !ok {
return nil, fmt.Errorf("field %q has keys in NullFields but is not a map[string]any", fieldName)
}
v := iter.Value().Interface()
mi[k] = v
}
for k := range useNullMaps[fieldName] {
mi[k] = nil
}
return mi, nil
}
// formatAsString returns a string representation of v, dereferencing it first if possible.
func formatAsString(v reflect.Value, kind reflect.Kind) string {
if kind == reflect.Ptr && !v.IsNil() {
v = v.Elem()
}
return fmt.Sprintf("%v", v.Interface())
}
// jsonTag represents a restricted version of the struct tag format used by encoding/json.
// It is used to describe the JSON encoding of fields in a Schema struct.
type jsonTag struct {
apiName string
stringFormat bool
ignore bool
}
// parseJSONTag parses a restricted version of the struct tag format used by encoding/json.
// The format of the tag must match that generated by the Schema.writeSchemaStruct method
// in the api generator.
func parseJSONTag(val string) (jsonTag, error) {
if val == "-" {
return jsonTag{ignore: true}, nil
}
var tag jsonTag
i := strings.Index(val, ",")
if i == -1 || val[:i] == "" {
return tag, fmt.Errorf("malformed json tag: %s", val)
}
tag = jsonTag{
apiName: val[:i],
}
switch val[i+1:] {
case "omitempty":
case "omitempty,string":
tag.stringFormat = true
default:
return tag, fmt.Errorf("malformed json tag: %s", val)
}
return tag, nil
}
// Reports whether the struct field "f" with value "v" should be included in JSON output.
func includeField(v reflect.Value, f reflect.StructField, mustInclude map[string]bool) bool {
// The regular JSON encoding of a nil pointer is "null", which means "delete this field".
// Therefore, we could enable field deletion by honoring pointer fields' presence in the mustInclude set.
// However, many fields are not pointers, so there would be no way to delete these fields.
// Rather than partially supporting field deletion, we ignore mustInclude for nil pointer fields.
// Deletion will be handled by a separate mechanism.
if f.Type.Kind() == reflect.Ptr && v.IsNil() {
return false
}
// The "any" type is represented as an interface{}. If this interface
// is nil, there is no reasonable representation to send. We ignore
// these fields, for the same reasons as given above for pointers.
if f.Type.Kind() == reflect.Interface && v.IsNil() {
return false
}
return mustInclude[f.Name] || !isEmptyValue(v)
}
// isEmptyValue reports whether v is the empty value for its type. This
// implementation is based on that of the encoding/json package, but its
// correctness does not depend on it being identical. What's important is that
// this function return false in situations where v should not be sent as part
// of a PATCH operation.
func isEmptyValue(v reflect.Value) bool {
switch v.Kind() {
case reflect.Array, reflect.Map, reflect.Slice, reflect.String:
return v.Len() == 0
case reflect.Bool:
return !v.Bool()
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
return v.Int() == 0
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
return v.Uint() == 0
case reflect.Float32, reflect.Float64:
return v.Float() == 0
case reflect.Interface, reflect.Ptr:
return v.IsNil()
}
return false
}

View File

@@ -0,0 +1,47 @@
// Copyright 2016 Google LLC.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package gensupport
import (
"encoding/json"
"errors"
"fmt"
"math"
)
// JSONFloat64 is a float64 that supports proper unmarshaling of special float
// values in JSON, according to
// https://developers.google.com/protocol-buffers/docs/proto3#json. Although
// that is a proto-to-JSON spec, it applies to all Google APIs.
//
// The jsonpb package
// (https://github.com/golang/protobuf/blob/master/jsonpb/jsonpb.go) has
// similar functionality, but only for direct translation from proto messages
// to JSON.
type JSONFloat64 float64
func (f *JSONFloat64) UnmarshalJSON(data []byte) error {
var ff float64
if err := json.Unmarshal(data, &ff); err == nil {
*f = JSONFloat64(ff)
return nil
}
var s string
if err := json.Unmarshal(data, &s); err == nil {
switch s {
case "NaN":
ff = math.NaN()
case "Infinity":
ff = math.Inf(1)
case "-Infinity":
ff = math.Inf(-1)
default:
return fmt.Errorf("google.golang.org/api/internal: bad float string %q", s)
}
*f = JSONFloat64(ff)
return nil
}
return errors.New("google.golang.org/api/internal: data not float or string")
}

View File

@@ -0,0 +1,318 @@
// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package gensupport
import (
"bytes"
"fmt"
"io"
"mime"
"mime/multipart"
"net/http"
"net/textproto"
"strings"
"sync"
"time"
gax "github.com/googleapis/gax-go/v2"
"google.golang.org/api/googleapi"
)
type typeReader struct {
io.Reader
typ string
}
// multipartReader combines the contents of multiple readers to create a multipart/related HTTP body.
// Close must be called if reads from the multipartReader are abandoned before reaching EOF.
type multipartReader struct {
pr *io.PipeReader
ctype string
mu sync.Mutex
pipeOpen bool
}
// boundary optionally specifies the MIME boundary
func newMultipartReader(parts []typeReader, boundary string) *multipartReader {
mp := &multipartReader{pipeOpen: true}
var pw *io.PipeWriter
mp.pr, pw = io.Pipe()
mpw := multipart.NewWriter(pw)
if boundary != "" {
mpw.SetBoundary(boundary)
}
mp.ctype = "multipart/related; boundary=" + mpw.Boundary()
go func() {
for _, part := range parts {
w, err := mpw.CreatePart(typeHeader(part.typ))
if err != nil {
mpw.Close()
pw.CloseWithError(fmt.Errorf("googleapi: CreatePart failed: %v", err))
return
}
_, err = io.Copy(w, part.Reader)
if err != nil {
mpw.Close()
pw.CloseWithError(fmt.Errorf("googleapi: Copy failed: %v", err))
return
}
}
mpw.Close()
pw.Close()
}()
return mp
}
func (mp *multipartReader) Read(data []byte) (n int, err error) {
return mp.pr.Read(data)
}
func (mp *multipartReader) Close() error {
mp.mu.Lock()
if !mp.pipeOpen {
mp.mu.Unlock()
return nil
}
mp.pipeOpen = false
mp.mu.Unlock()
return mp.pr.Close()
}
// CombineBodyMedia combines a json body with media content to create a multipart/related HTTP body.
// It returns a ReadCloser containing the combined body, and the overall "multipart/related" content type, with random boundary.
//
// The caller must call Close on the returned ReadCloser if reads are abandoned before reaching EOF.
func CombineBodyMedia(body io.Reader, bodyContentType string, media io.Reader, mediaContentType string) (io.ReadCloser, string) {
return combineBodyMedia(body, bodyContentType, media, mediaContentType, "")
}
// combineBodyMedia is CombineBodyMedia but with an optional mimeBoundary field.
func combineBodyMedia(body io.Reader, bodyContentType string, media io.Reader, mediaContentType, mimeBoundary string) (io.ReadCloser, string) {
mp := newMultipartReader([]typeReader{
{body, bodyContentType},
{media, mediaContentType},
}, mimeBoundary)
return mp, mp.ctype
}
func typeHeader(contentType string) textproto.MIMEHeader {
h := make(textproto.MIMEHeader)
if contentType != "" {
h.Set("Content-Type", contentType)
}
return h
}
// PrepareUpload determines whether the data in the supplied reader should be
// uploaded in a single request, or in sequential chunks.
// chunkSize is the size of the chunk that media should be split into.
//
// If chunkSize is zero, media is returned as the first value, and the other
// two return values are nil, true.
//
// Otherwise, a MediaBuffer is returned, along with a bool indicating whether the
// contents of media fit in a single chunk.
//
// After PrepareUpload has been called, media should no longer be used: the
// media content should be accessed via one of the return values.
func PrepareUpload(media io.Reader, chunkSize int, enableAutoChecksum bool) (r io.Reader, mb *MediaBuffer, singleChunk bool) {
if chunkSize == 0 { // do not chunk
return media, nil, true
}
mb = NewMediaBuffer(media, chunkSize)
mb.enableAutoChecksum = enableAutoChecksum
_, _, _, err := mb.Chunk()
// If err is io.EOF, we can upload this in a single request. Otherwise, err is
// either nil or a non-EOF error. If it is the latter, then the next call to
// mb.Chunk will return the same error. Returning a MediaBuffer ensures that this
// error will be handled at some point.
return nil, mb, err == io.EOF
}
// MediaInfo holds information for media uploads. It is intended for use by generated
// code only.
type MediaInfo struct {
// At most one of Media and MediaBuffer will be set.
media io.Reader
buffer *MediaBuffer
singleChunk bool
mType string
size int64 // mediaSize, if known. Used only for calls to progressUpdater_.
progressUpdater googleapi.ProgressUpdater
chunkRetryDeadline time.Duration
chunkTransferTimeout time.Duration
}
// NewInfoFromMedia should be invoked from the Media method of a call. It returns a
// MediaInfo populated with chunk size and content type, and a reader or MediaBuffer
// if needed.
func NewInfoFromMedia(r io.Reader, options []googleapi.MediaOption) *MediaInfo {
mi := &MediaInfo{}
opts := googleapi.ProcessMediaOptions(options)
if !opts.ForceEmptyContentType {
mi.mType = opts.ContentType
if mi.mType == "" {
r, mi.mType = gax.DetermineContentType(r)
}
}
mi.chunkRetryDeadline = opts.ChunkRetryDeadline
mi.chunkTransferTimeout = opts.ChunkTransferTimeout
mi.media, mi.buffer, mi.singleChunk = PrepareUpload(r, opts.ChunkSize, opts.EnableAutoChecksum)
return mi
}
// NewInfoFromResumableMedia should be invoked from the ResumableMedia method of a
// call. It returns a MediaInfo using the given reader, size and media type.
func NewInfoFromResumableMedia(r io.ReaderAt, size int64, mediaType string) *MediaInfo {
rdr := ReaderAtToReader(r, size)
mType := mediaType
if mType == "" {
rdr, mType = gax.DetermineContentType(rdr)
}
return &MediaInfo{
size: size,
mType: mType,
buffer: NewMediaBuffer(rdr, googleapi.DefaultUploadChunkSize),
media: nil,
singleChunk: false,
}
}
// SetProgressUpdater sets the progress updater for the media info.
func (mi *MediaInfo) SetProgressUpdater(pu googleapi.ProgressUpdater) {
if mi != nil {
mi.progressUpdater = pu
}
}
// UploadType determines the type of upload: a single request, or a resumable
// series of requests.
func (mi *MediaInfo) UploadType() string {
if mi.singleChunk {
return "multipart"
}
return "resumable"
}
// UploadRequest sets up an HTTP request for media upload. It adds headers
// as necessary, and returns a replacement for the body and a function for http.Request.GetBody.
func (mi *MediaInfo) UploadRequest(reqHeaders http.Header, body io.Reader) (newBody io.Reader, getBody func() (io.ReadCloser, error), cleanup func()) {
if body == nil {
body = new(bytes.Buffer)
}
cleanup = func() {}
if mi == nil {
return body, nil, cleanup
}
var media io.Reader
if mi.media != nil {
// This only happens when the caller has turned off chunking. In that
// case, we write all of media in a single non-retryable request.
media = mi.media
} else if mi.singleChunk {
// The data fits in a single chunk, which has now been read into the MediaBuffer.
// We obtain that chunk so we can write it in a single request. The request can
// be retried because the data is stored in the MediaBuffer.
media, _, _, _ = mi.buffer.Chunk()
}
toCleanup := []io.Closer{}
if media != nil {
fb := readerFunc(body)
fm := readerFunc(media)
combined, ctype := CombineBodyMedia(body, "application/json", media, mi.mType)
toCleanup = append(toCleanup, combined)
if fb != nil && fm != nil {
getBody = func() (io.ReadCloser, error) {
rb := io.NopCloser(fb())
rm := io.NopCloser(fm())
var mimeBoundary string
if _, params, err := mime.ParseMediaType(ctype); err == nil {
mimeBoundary = params["boundary"]
}
r, _ := combineBodyMedia(rb, "application/json", rm, mi.mType, mimeBoundary)
toCleanup = append(toCleanup, r)
return r, nil
}
}
reqHeaders.Set("Content-Type", ctype)
body = combined
}
if mi.buffer != nil && mi.mType != "" && !mi.singleChunk {
// This happens when initiating a resumable upload session.
// The initial request contains a JSON body rather than media.
// It can be retried with a getBody function that re-creates the request body.
fb := readerFunc(body)
if fb != nil {
getBody = func() (io.ReadCloser, error) {
rb := io.NopCloser(fb())
toCleanup = append(toCleanup, rb)
return rb, nil
}
}
reqHeaders.Set("X-Upload-Content-Type", mi.mType)
}
// Ensure that any bodies created in getBody are cleaned up.
cleanup = func() {
for _, closer := range toCleanup {
_ = closer.Close()
}
}
return body, getBody, cleanup
}
// readerFunc returns a function that always returns an io.Reader that has the same
// contents as r, provided that can be done without consuming r. Otherwise, it
// returns nil.
// See http.NewRequest (in net/http/request.go).
func readerFunc(r io.Reader) func() io.Reader {
switch r := r.(type) {
case *bytes.Buffer:
buf := r.Bytes()
return func() io.Reader { return bytes.NewReader(buf) }
case *bytes.Reader:
snapshot := *r
return func() io.Reader { r := snapshot; return &r }
case *strings.Reader:
snapshot := *r
return func() io.Reader { r := snapshot; return &r }
default:
return nil
}
}
// ResumableUpload returns an appropriately configured ResumableUpload value if the
// upload is resumable, or nil otherwise.
func (mi *MediaInfo) ResumableUpload(locURI string) *ResumableUpload {
if mi == nil || mi.singleChunk {
return nil
}
return &ResumableUpload{
URI: locURI,
Media: mi.buffer,
MediaType: mi.mType,
Callback: func(curr int64) {
if mi.progressUpdater != nil {
mi.progressUpdater(curr, mi.size)
}
},
ChunkRetryDeadline: mi.chunkRetryDeadline,
ChunkTransferTimeout: mi.chunkTransferTimeout,
}
}
// SetGetBody sets the GetBody field of req to f. This was once needed
// to gracefully support Go 1.7 and earlier which didn't have that
// field.
//
// Deprecated: the code generator no longer uses this as of
// 2019-02-19. Nothing else should be calling this anyway, but we
// won't delete this immediately; it will be deleted in as early as 6
// months.
func SetGetBody(req *http.Request, f func() (io.ReadCloser, error)) {
req.GetBody = f
}

View File

@@ -0,0 +1,78 @@
// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package gensupport
import (
"net/http"
"net/url"
"google.golang.org/api/googleapi"
"google.golang.org/api/internal"
)
// URLParams is a simplified replacement for url.Values
// that safely builds up URL parameters for encoding.
type URLParams map[string][]string
// Get returns the first value for the given key, or "".
func (u URLParams) Get(key string) string {
vs := u[key]
if len(vs) == 0 {
return ""
}
return vs[0]
}
// Set sets the key to value.
// It replaces any existing values.
func (u URLParams) Set(key, value string) {
u[key] = []string{value}
}
// SetMulti sets the key to an array of values.
// It replaces any existing values.
// Note that values must not be modified after calling SetMulti
// so the caller is responsible for making a copy if necessary.
func (u URLParams) SetMulti(key string, values []string) {
u[key] = values
}
// Encode encodes the values into “URL encoded” form
// ("bar=baz&foo=quux") sorted by key.
func (u URLParams) Encode() string {
return url.Values(u).Encode()
}
// SetOptions sets the URL params and any additional `CallOption` or
// `MultiCallOption` passed in.
func SetOptions(u URLParams, opts ...googleapi.CallOption) {
for _, o := range opts {
m, ok := o.(googleapi.MultiCallOption)
if ok {
u.SetMulti(m.GetMulti())
continue
}
u.Set(o.Get())
}
}
// SetHeaders sets common headers for all requests. The keyvals header pairs
// should have a corresponding value for every key provided. If there is an odd
// number of keyvals this method will panic.
func SetHeaders(userAgent, contentType string, userHeaders http.Header, keyvals ...string) http.Header {
reqHeaders := make(http.Header)
reqHeaders.Set("x-goog-api-client", "gl-go/"+GoVersion()+" gdcl/"+internal.Version)
for i := 0; i < len(keyvals); i = i + 2 {
reqHeaders.Set(keyvals[i], keyvals[i+1])
}
reqHeaders.Set("User-Agent", userAgent)
if contentType != "" {
reqHeaders.Set("Content-Type", contentType)
}
for k, v := range userHeaders {
reqHeaders[k] = v
}
return reqHeaders
}

View File

@@ -0,0 +1,356 @@
// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package gensupport
import (
"context"
"encoding/base64"
"encoding/binary"
"errors"
"fmt"
"io"
"net/http"
"strings"
"sync"
"time"
"github.com/google/uuid"
"google.golang.org/api/internal"
)
const (
crc32cPrefix = "crc32c"
hashHeaderKey = "X-Goog-Hash"
)
// ResumableUpload is used by the generated APIs to provide resumable uploads.
// It is not used by developers directly.
type ResumableUpload struct {
Client *http.Client
// URI is the resumable resource destination provided by the server after specifying "&uploadType=resumable".
URI string
UserAgent string // User-Agent for header of the request
// Media is the object being uploaded.
Media *MediaBuffer
// MediaType defines the media type, e.g. "image/jpeg".
MediaType string
mu sync.Mutex // guards progress
progress int64 // number of bytes uploaded so far
// Callback is an optional function that will be periodically called with the cumulative number of bytes uploaded.
Callback func(int64)
// Retry optionally configures retries for requests made against the upload.
Retry *RetryConfig
// ChunkRetryDeadline configures the per-chunk deadline after which no further
// retries should happen.
ChunkRetryDeadline time.Duration
// ChunkTransferTimeout configures the per-chunk transfer timeout. If a chunk upload stalls for longer than
// this duration, the upload will be retried.
ChunkTransferTimeout time.Duration
// Track current request invocation ID and attempt count for retry metrics
// and idempotency headers.
invocationID string
attempts int
}
// Progress returns the number of bytes uploaded at this point.
func (rx *ResumableUpload) Progress() int64 {
rx.mu.Lock()
defer rx.mu.Unlock()
return rx.progress
}
// doUploadRequest performs a single HTTP request to upload data.
// off specifies the offset in rx.Media from which data is drawn.
// size is the number of bytes in data.
// final specifies whether data is the final chunk to be uploaded.
func (rx *ResumableUpload) doUploadRequest(ctx context.Context, data io.Reader, off, size int64, final bool) (*http.Response, error) {
req, err := http.NewRequest("POST", rx.URI, data)
if err != nil {
return nil, err
}
req.ContentLength = size
var contentRange string
if final {
if size == 0 {
contentRange = fmt.Sprintf("bytes */%v", off)
} else {
contentRange = fmt.Sprintf("bytes %v-%v/%v", off, off+size-1, off+size)
}
} else {
contentRange = fmt.Sprintf("bytes %v-%v/*", off, off+size-1)
}
req.Header.Set("Content-Range", contentRange)
req.Header.Set("Content-Type", rx.MediaType)
req.Header.Set("User-Agent", rx.UserAgent)
// TODO(b/274504690): Consider dropping gccl-invocation-id key since it
// duplicates the X-Goog-Gcs-Idempotency-Token header (added in v0.115.0).
baseXGoogHeader := "gl-go/" + GoVersion() + " gdcl/" + internal.Version
invocationHeader := fmt.Sprintf("gccl-invocation-id/%s gccl-attempt-count/%d", rx.invocationID, rx.attempts)
req.Header.Set("X-Goog-Api-Client", strings.Join([]string{baseXGoogHeader, invocationHeader}, " "))
// Set idempotency token header which is used by GCS uploads.
req.Header.Set("X-Goog-Gcs-Idempotency-Token", rx.invocationID)
// Google's upload endpoint uses status code 308 for a
// different purpose than the "308 Permanent Redirect"
// since-standardized in RFC 7238. Because of the conflict in
// semantics, Google added this new request header which
// causes it to not use "308" and instead reply with 200 OK
// and sets the upload-specific "X-HTTP-Status-Code-Override:
// 308" response header.
req.Header.Set("X-GUploader-No-308", "yes")
// Server accepts checksum only on final request through header.
if final && rx.Media.enableAutoChecksum {
req.Header.Set(hashHeaderKey, fmt.Sprintf("%v=%v", crc32cPrefix, encodeUint32(rx.Media.fullObjectChecksum)))
}
return SendRequest(ctx, rx.Client, req)
}
func statusResumeIncomplete(resp *http.Response) bool {
// This is how the server signals "status resume incomplete"
// when X-GUploader-No-308 is set to "yes":
return resp != nil && resp.Header.Get("X-Http-Status-Code-Override") == "308"
}
// reportProgress calls a user-supplied callback to report upload progress.
// If old==updated, the callback is not called.
func (rx *ResumableUpload) reportProgress(old, updated int64) {
if updated-old == 0 {
return
}
rx.mu.Lock()
rx.progress = updated
rx.mu.Unlock()
if rx.Callback != nil {
rx.Callback(updated)
}
}
// transferChunk performs a single HTTP request to upload a single chunk.
// It uses a goroutine to perform the upload and a timer to enforce ChunkTransferTimeout.
func (rx *ResumableUpload) transferChunk(ctx context.Context, chunk io.Reader, off, size int64, done bool) (*http.Response, error) {
// If no timeout is specified, perform the request synchronously without a timer.
if rx.ChunkTransferTimeout == 0 {
res, err := rx.doUploadRequest(ctx, chunk, off, size, done)
if err != nil {
return res, err
}
return res, nil
}
// Start a timer for the ChunkTransferTimeout duration.
timer := time.NewTimer(rx.ChunkTransferTimeout)
// A struct to hold the result from the goroutine.
type uploadResult struct {
res *http.Response
err error
}
// A buffered channel to receive the result of the upload.
resultCh := make(chan uploadResult, 1)
// Create a cancellable context for the upload request. This allows us to
// abort the request if the timer fires first.
rCtx, cancel := context.WithCancel(ctx)
// NOTE: We do NOT use `defer cancel()` here. The context must remain valid
// for the caller to read the response body of a successful request.
// Cancellation is handled manually on timeout paths.
// Starting the chunk upload in parallel.
go func() {
res, err := rx.doUploadRequest(rCtx, chunk, off, size, done)
resultCh <- uploadResult{res: res, err: err}
}()
// Wait for timer to fire or result channel to have the uploadResult or ctx to be cancelled.
select {
// Note: Calling cancel() will guarantee that the goroutine finishes,
// so these two cases will never block forever on draining the resultCh.
case <-ctx.Done():
// Context is cancelled for the overall upload.
cancel()
// Drain resultCh.
<-resultCh
return nil, ctx.Err()
case <-timer.C:
// Chunk Transfer timer fired before resultCh so we return context.DeadlineExceeded.
cancel()
// Drain resultCh.
<-resultCh
return nil, context.DeadlineExceeded
case result := <-resultCh:
// Handle the result from the upload.
if result.err != nil {
return result.res, result.err
}
return result.res, nil
}
}
// uploadChunkWithRetries attempts to upload a single chunk, with retries
// within ChunkRetryDeadline if ChunkTransferTimeout is non-zero.
func (rx *ResumableUpload) uploadChunkWithRetries(ctx context.Context, chunk io.Reader, off, size int64, done bool) (*http.Response, error) {
// Configure error retryable criteria.
shouldRetry := rx.Retry.errorFunc()
// Configure single chunk retry deadline.
chunkRetryDeadline := defaultRetryDeadline
if rx.ChunkRetryDeadline != 0 {
chunkRetryDeadline = rx.ChunkRetryDeadline
}
// Each chunk gets its own initialized-at-zero backoff and invocation ID.
bo := rx.Retry.backoff()
quitAfterTimer := time.NewTimer(chunkRetryDeadline)
defer quitAfterTimer.Stop()
rx.attempts = 1
rx.invocationID = uuid.New().String()
var pause time.Duration
var resp *http.Response
var err error
// Retry loop for a single chunk.
for {
// Wait for the backoff period, unless the context is canceled or the
// retry deadline is hit.
backoffPauseTimer := time.NewTimer(pause)
select {
case <-ctx.Done():
backoffPauseTimer.Stop()
if err == nil {
err = ctx.Err()
}
return resp, err
case <-backoffPauseTimer.C:
case <-quitAfterTimer.C:
backoffPauseTimer.Stop()
return resp, err
}
backoffPauseTimer.Stop()
// Check for context cancellation or timeout once more. If more than one
// case in the select statement above was satisfied at the same time, Go
// will choose one arbitrarily.
// That can cause an operation to go through even if the context was
// canceled before or the timeout was reached.
select {
case <-ctx.Done():
if err == nil {
err = ctx.Err()
}
return resp, err
case <-quitAfterTimer.C:
return resp, err
default:
}
// We close the response's body here, since we definitely will not
// return `resp` now. If we close it before the select case above, a
// timer may fire and cause us to return a response with a closed body
// (in which case, the caller will not get the error message in the body).
if resp != nil && resp.Body != nil {
// Read the body to EOF - if the Body is not both read to EOF and closed,
// the Client's underlying RoundTripper may not be able to re-use the
// persistent TCP connection to the server for a subsequent "keep-alive" request.
// See https://pkg.go.dev/net/http#Client.Do
io.Copy(io.Discard, resp.Body)
resp.Body.Close()
}
resp, err = rx.transferChunk(ctx, chunk, off, size, done)
status := 0
if resp != nil {
status = resp.StatusCode
}
// We sent "X-GUploader-No-308: yes" (see comment elsewhere in
// this file), so we don't expect to get a 308.
if status == 308 {
return nil, errors.New("unexpected 308 response status code")
}
// Chunk upload should be retried if the ChunkTransferTimeout is non-zero and err is context deadline exceeded
// or we encounter a retryable error.
if (rx.ChunkTransferTimeout != 0 && errors.Is(err, context.DeadlineExceeded)) || shouldRetry(status, err) {
rx.attempts++
pause = bo.Pause()
chunk, _, _, _ = rx.Media.Chunk()
continue
}
return resp, err
}
}
// Upload starts the process of a resumable upload with a cancellable context.
// It is called from the auto-generated API code and is not visible to the user.
// Before sending an HTTP request, Upload calls any registered hook functions,
// and calls the returned functions after the request returns (see send.go).
// rx is private to the auto-generated API code.
// Exactly one of resp or err will be nil. If resp is non-nil, the caller must call resp.Body.Close.
// Upload does not parse the response into the error on a non 200 response;
// it is the caller's responsibility to call resp.Body.Close.
func (rx *ResumableUpload) Upload(ctx context.Context) (*http.Response, error) {
for {
chunk, off, size, err := rx.Media.Chunk()
done := err == io.EOF
if !done && err != nil {
return nil, err
}
resp, err := rx.uploadChunkWithRetries(ctx, chunk, off, int64(size), done)
// There are a couple of cases where it's possible for err and resp to both
// be non-nil. However, we expose a simpler contract to our callers: exactly
// one of resp and err will be non-nil. This means that any response body
// must be closed here before returning a non-nil error.
if err != nil {
if resp != nil && resp.Body != nil {
resp.Body.Close()
}
// If there were retries, indicate this in the error message and wrap the final error.
if rx.attempts > 1 {
return nil, fmt.Errorf("chunk upload failed after %d attempts, final error: %w", rx.attempts, err)
}
return nil, err
}
// This case is very unlikely but possible only if rx.ChunkRetryDeadline is
// set to a very small value, in which case no requests will be sent before
// the deadline. Return an error to avoid causing a panic.
if resp == nil {
return nil, fmt.Errorf("upload request to %v not sent, choose larger value for ChunkRetryDeadline", rx.URI)
}
if resp.StatusCode == http.StatusOK {
rx.reportProgress(off, off+int64(size))
}
if statusResumeIncomplete(resp) {
// The upload is not yet complete, but the server has acknowledged this chunk.
// We don't have anything to do with the response body.
if resp.Body != nil {
io.Copy(io.Discard, resp.Body)
resp.Body.Close()
}
rx.Media.Next()
continue
}
return resp, nil
}
}
// Encode a uint32 as Base64 in big-endian byte order.
func encodeUint32(u uint32) string {
b := make([]byte, 4)
binary.BigEndian.PutUint32(b, u)
return base64.StdEncoding.EncodeToString(b)
}

View File

@@ -0,0 +1,123 @@
// Copyright 2021 Google LLC.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package gensupport
import (
"errors"
"io"
"net"
"net/url"
"strings"
"time"
"github.com/googleapis/gax-go/v2"
"google.golang.org/api/googleapi"
)
// Backoff is an interface around gax.Backoff's Pause method, allowing tests to provide their
// own implementation.
type Backoff interface {
Pause() time.Duration
}
// These are declared as global variables so that tests can overwrite them.
var (
// Default per-chunk deadline for resumable uploads.
defaultRetryDeadline = 32 * time.Second
// Default backoff timer.
backoff = func() Backoff {
return &gax.Backoff{Initial: 100 * time.Millisecond}
}
)
const (
// statusTooManyRequests is returned by the storage API if the
// per-project limits have been temporarily exceeded. The request
// should be retried.
// https://cloud.google.com/storage/docs/json_api/v1/status-codes#standardcodes
statusTooManyRequests = 429
// statusRequestTimeout is returned by the storage API if the
// upload connection was broken. The request should be retried.
statusRequestTimeout = 408
)
// shouldRetry indicates whether an error is retryable for the purposes of this
// package, unless a ShouldRetry func is specified by the RetryConfig instead.
// It follows guidance from
// https://cloud.google.com/storage/docs/exponential-backoff .
func shouldRetry(status int, err error) bool {
if 500 <= status && status <= 599 {
return true
}
if status == statusTooManyRequests || status == statusRequestTimeout {
return true
}
if errors.Is(err, io.ErrUnexpectedEOF) {
return true
}
if errors.Is(err, net.ErrClosed) {
return true
}
switch e := err.(type) {
case *net.OpError, *url.Error:
// Retry socket-level errors ECONNREFUSED and ECONNRESET (from syscall).
// Unfortunately the error type is unexported, so we resort to string
// matching.
retriable := []string{"connection refused", "connection reset", "broken pipe"}
for _, s := range retriable {
if strings.Contains(e.Error(), s) {
return true
}
}
case interface{ Temporary() bool }:
if e.Temporary() {
return true
}
}
// If error unwrapping is available, use this to examine wrapped
// errors.
if e, ok := err.(interface{ Unwrap() error }); ok {
return shouldRetry(status, e.Unwrap())
}
return false
}
// RetryConfig allows configuration of backoff timing and retryable errors.
type RetryConfig struct {
Backoff *gax.Backoff
ShouldRetry func(err error) bool
}
// Get a new backoff object based on the configured values.
func (r *RetryConfig) backoff() Backoff {
if r == nil || r.Backoff == nil {
return backoff()
}
return &gax.Backoff{
Initial: r.Backoff.Initial,
Max: r.Backoff.Max,
Multiplier: r.Backoff.Multiplier,
}
}
// This is kind of hacky; it is necessary because ShouldRetry expects to
// handle HTTP errors via googleapi.Error, but the error has not yet been
// wrapped with a googleapi.Error at this layer, and the ErrorFunc type
// in the manual layer does not pass in a status explicitly as it does
// here. So, we must wrap error status codes in a googleapi.Error so that
// ShouldRetry can parse this correctly.
func (r *RetryConfig) errorFunc() func(status int, err error) bool {
if r == nil || r.ShouldRetry == nil {
return shouldRetry
}
return func(status int, err error) bool {
if status >= 400 {
return r.ShouldRetry(&googleapi.Error{Code: status})
}
return r.ShouldRetry(err)
}
}

View File

@@ -0,0 +1,241 @@
// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package gensupport
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"strings"
"time"
"github.com/google/uuid"
"github.com/googleapis/gax-go/v2"
"github.com/googleapis/gax-go/v2/callctx"
)
// Use this error type to return an error which allows introspection of both
// the context error and the error from the service.
type wrappedCallErr struct {
ctxErr error
wrappedErr error
}
func (e wrappedCallErr) Error() string {
return fmt.Sprintf("retry failed with %v; last error: %v", e.ctxErr, e.wrappedErr)
}
func (e wrappedCallErr) Unwrap() error {
return e.wrappedErr
}
// Is allows errors.Is to match the error from the call as well as context
// sentinel errors.
func (e wrappedCallErr) Is(target error) bool {
return errors.Is(e.ctxErr, target) || errors.Is(e.wrappedErr, target)
}
// SendRequest sends a single HTTP request using the given client.
// If ctx is non-nil, it calls all hooks, then sends the request with
// req.WithContext, then calls any functions returned by the hooks in
// reverse order.
func SendRequest(ctx context.Context, client *http.Client, req *http.Request) (*http.Response, error) {
// Add headers set in context metadata.
if ctx != nil {
headers := callctx.HeadersFromContext(ctx)
for k, vals := range headers {
if k == "x-goog-api-client" {
// Merge all values into a single "x-goog-api-client" header.
var mergedVal strings.Builder
baseXGoogHeader := req.Header.Get("X-Goog-Api-Client")
if baseXGoogHeader != "" {
mergedVal.WriteString(baseXGoogHeader)
mergedVal.WriteRune(' ')
}
for _, v := range vals {
mergedVal.WriteString(v)
mergedVal.WriteRune(' ')
}
// Remove the last space and replace the header on the request.
req.Header.Set(k, mergedVal.String()[:mergedVal.Len()-1])
} else {
for _, v := range vals {
req.Header.Add(k, v)
}
}
}
}
// Disallow Accept-Encoding because it interferes with the automatic gzip handling
// done by the default http.Transport. See https://github.com/google/google-api-go-client/issues/219.
if _, ok := req.Header["Accept-Encoding"]; ok {
return nil, errors.New("google api: custom Accept-Encoding headers not allowed")
}
if ctx == nil {
return client.Do(req)
}
return send(ctx, client, req)
}
func send(ctx context.Context, client *http.Client, req *http.Request) (*http.Response, error) {
if client == nil {
client = http.DefaultClient
}
resp, err := client.Do(req.WithContext(ctx))
// If we got an error, and the context has been canceled,
// the context's error is probably more useful.
if err != nil {
select {
case <-ctx.Done():
err = ctx.Err()
default:
}
}
return resp, err
}
// SendRequestWithRetry sends a single HTTP request using the given client,
// with retries if a retryable error is returned.
// If ctx is non-nil, it calls all hooks, then sends the request with
// req.WithContext, then calls any functions returned by the hooks in
// reverse order.
func SendRequestWithRetry(ctx context.Context, client *http.Client, req *http.Request, retry *RetryConfig) (*http.Response, error) {
// Add headers set in context metadata.
if ctx != nil {
headers := callctx.HeadersFromContext(ctx)
for k, vals := range headers {
for _, v := range vals {
req.Header.Add(k, v)
}
}
}
// Disallow Accept-Encoding because it interferes with the automatic gzip handling
// done by the default http.Transport. See https://github.com/google/google-api-go-client/issues/219.
if _, ok := req.Header["Accept-Encoding"]; ok {
return nil, errors.New("google api: custom Accept-Encoding headers not allowed")
}
if ctx == nil {
return client.Do(req)
}
return sendAndRetry(ctx, client, req, retry)
}
func sendAndRetry(ctx context.Context, client *http.Client, req *http.Request, retry *RetryConfig) (*http.Response, error) {
if client == nil {
client = http.DefaultClient
}
var resp *http.Response
var err error
attempts := 1
invocationID := uuid.New().String()
xGoogHeaderVals := req.Header.Values("X-Goog-Api-Client")
baseXGoogHeader := strings.Join(xGoogHeaderVals, " ")
// Loop to retry the request, up to the context deadline.
var pause time.Duration
var bo Backoff
if retry != nil && retry.Backoff != nil {
bo = &gax.Backoff{
Initial: retry.Backoff.Initial,
Max: retry.Backoff.Max,
Multiplier: retry.Backoff.Multiplier,
}
} else {
bo = backoff()
}
var errorFunc = retry.errorFunc()
for {
t := time.NewTimer(pause)
select {
case <-ctx.Done():
t.Stop()
// If we got an error and the context has been canceled, return an error acknowledging
// both the context cancelation and the service error.
if err != nil {
return resp, wrappedCallErr{ctx.Err(), err}
}
return resp, ctx.Err()
case <-t.C:
}
if ctx.Err() != nil {
// Check for context cancellation once more. If more than one case in a
// select is satisfied at the same time, Go will choose one arbitrarily.
// That can cause an operation to go through even if the context was
// canceled before.
if err != nil {
return resp, wrappedCallErr{ctx.Err(), err}
}
return resp, ctx.Err()
}
// Set retry metrics and idempotency headers for GCS.
// TODO(b/274504690): Consider dropping gccl-invocation-id key since it
// duplicates the X-Goog-Gcs-Idempotency-Token header (added in v0.115.0).
invocationHeader := fmt.Sprintf("gccl-invocation-id/%s gccl-attempt-count/%d", invocationID, attempts)
xGoogHeader := strings.Join([]string{invocationHeader, baseXGoogHeader}, " ")
req.Header.Set("X-Goog-Api-Client", xGoogHeader)
req.Header.Set("X-Goog-Gcs-Idempotency-Token", invocationID)
resp, err = client.Do(req.WithContext(ctx))
var status int
if resp != nil {
status = resp.StatusCode
}
// Check if we can retry the request. A retry can only be done if the error
// is retryable and the request body can be re-created using GetBody (this
// will not be possible if the body was unbuffered).
if req.GetBody == nil || !errorFunc(status, err) {
break
}
attempts++
var errBody error
req.Body, errBody = req.GetBody()
if errBody != nil {
break
}
pause = bo.Pause()
if resp != nil && resp.Body != nil {
resp.Body.Close()
}
}
return resp, err
}
// DecodeResponse decodes the body of res into target. If there is no body,
// target is unchanged.
func DecodeResponse(target interface{}, res *http.Response) error {
if res.StatusCode == http.StatusNoContent {
return nil
}
return json.NewDecoder(res.Body).Decode(target)
}
// DecodeResponseBytes decodes the body of res into target and returns bytes read
// from the body. If there is no body, target is unchanged.
func DecodeResponseBytes(target interface{}, res *http.Response) ([]byte, error) {
if res.StatusCode == http.StatusNoContent {
return nil, nil
}
b, err := io.ReadAll(res.Body)
if err != nil {
return nil, err
}
if err := json.Unmarshal(b, target); err != nil {
return nil, err
}
return b, nil
}

View File

@@ -0,0 +1,53 @@
// Copyright 2020 Google LLC. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package gensupport
import (
"runtime"
"strings"
"unicode"
)
// GoVersion returns the Go runtime version. The returned string
// has no whitespace.
func GoVersion() string {
return goVersion
}
var goVersion = goVer(runtime.Version())
const develPrefix = "devel +"
func goVer(s string) string {
if strings.HasPrefix(s, develPrefix) {
s = s[len(develPrefix):]
if p := strings.IndexFunc(s, unicode.IsSpace); p >= 0 {
s = s[:p]
}
return s
}
if strings.HasPrefix(s, "go1") {
s = s[2:]
var prerelease string
if p := strings.IndexFunc(s, notSemverRune); p >= 0 {
s, prerelease = s[:p], s[p:]
}
if strings.HasSuffix(s, ".") {
s += "0"
} else if strings.Count(s, ".") < 2 {
s += ".0"
}
if prerelease != "" {
s += "-" + prerelease
}
return s
}
return ""
}
func notSemverRune(r rune) bool {
return !strings.ContainsRune("0123456789.", r)
}