tidb parse 源码
tidb parse 代码
文件路径：/br/pkg/storage/parse.go
// Copyright 2020 PingCAP, Inc. Licensed under Apache-2.0.
package storage
import (
"net/url"
"path/filepath"
"reflect"
"strconv"
"strings"
"github.com/pingcap/errors"
backuppb "github.com/pingcap/kvproto/pkg/brpb"
berrors "github.com/pingcap/tidb/br/pkg/errors"
)
// BackendOptions further configures the storage backend not expressed by the
// storage URL.
type BackendOptions struct {
S3 S3BackendOptions `json:"s3" toml:"s3"`
GCS GCSBackendOptions `json:"gcs" toml:"gcs"`
Azblob AzblobBackendOptions `json:"azblob" toml:"azblob"`
}
// ParseRawURL parse raw url to url object.
func ParseRawURL(rawURL string) (*url.URL, error) {
// https://github.com/pingcap/br/issues/603
// In aws the secret key may contain '/+=' and '+' has a special meaning in URL.
// Replace "+" by "%2B" here to avoid this problem.
rawURL = strings.ReplaceAll(rawURL, "+", "%2B")
u, err := url.Parse(rawURL)
if err != nil {
return nil, errors.Trace(err)
}
return u, nil
}
// ParseBackend constructs a structured backend description from the
// storage URL.
func ParseBackend(rawURL string, options *BackendOptions) (*backuppb.StorageBackend, error) {
if len(rawURL) == 0 {
return nil, errors.Annotate(berrors.ErrStorageInvalidConfig, "empty store is not allowed")
}
u, err := ParseRawURL(rawURL)
if err != nil {
return nil, errors.Trace(err)
}
switch u.Scheme {
case "":
absPath, err := filepath.Abs(rawURL)
if err != nil {
return nil, errors.Annotatef(berrors.ErrStorageInvalidConfig, "covert data-source-dir '%s' to absolute path failed", rawURL)
}
local := &backuppb.Local{Path: absPath}
return &backuppb.StorageBackend{Backend: &backuppb.StorageBackend_Local{Local: local}}, nil
case "local", "file":
local := &backuppb.Local{Path: u.Path}
return &backuppb.StorageBackend{Backend: &backuppb.StorageBackend_Local{Local: local}}, nil
case "hdfs":
hdfs := &backuppb.HDFS{Remote: rawURL}
return &backuppb.StorageBackend{Backend: &backuppb.StorageBackend_Hdfs{Hdfs: hdfs}}, nil
case "noop":
noop := &backuppb.Noop{}
return &backuppb.StorageBackend{Backend: &backuppb.StorageBackend_Noop{Noop: noop}}, nil
case "s3":
if u.Host == "" {
return nil, errors.Annotatef(berrors.ErrStorageInvalidConfig, "please specify the bucket for s3 in %s", rawURL)
}
prefix := strings.Trim(u.Path, "/")
s3 := &backuppb.S3{Bucket: u.Host, Prefix: prefix}
if options == nil {
options = &BackendOptions{S3: S3BackendOptions{ForcePathStyle: true}}
}
ExtractQueryParameters(u, &options.S3)
if err := options.S3.Apply(s3); err != nil {
return nil, errors.Trace(err)
}
return &backuppb.StorageBackend{Backend: &backuppb.StorageBackend_S3{S3: s3}}, nil
case "gs", "gcs":
if u.Host == "" {
return nil, errors.Annotatef(berrors.ErrStorageInvalidConfig, "please specify the bucket for gcs in %s", rawURL)
}
prefix := strings.Trim(u.Path, "/")
gcs := &backuppb.GCS{Bucket: u.Host, Prefix: prefix}
if options == nil {
options = &BackendOptions{}
}
ExtractQueryParameters(u, &options.GCS)
if err := options.GCS.apply(gcs); err != nil {
return nil, errors.Trace(err)
}
return &backuppb.StorageBackend{Backend: &backuppb.StorageBackend_Gcs{Gcs: gcs}}, nil
case "azure", "azblob":
if u.Host == "" {
return nil, errors.Annotatef(berrors.ErrStorageInvalidConfig, "please specify the bucket for azblob in %s", rawURL)
}
prefix := strings.Trim(u.Path, "/")
azblob := &backuppb.AzureBlobStorage{Bucket: u.Host, Prefix: prefix}
if options == nil {
options = &BackendOptions{}
}
ExtractQueryParameters(u, &options.Azblob)
if err := options.Azblob.apply(azblob); err != nil {
return nil, errors.Trace(err)
}
return &backuppb.StorageBackend{Backend: &backuppb.StorageBackend_AzureBlobStorage{AzureBlobStorage: azblob}}, nil
default:
return nil, errors.Annotatef(berrors.ErrStorageInvalidConfig, "storage %s not support yet", u.Scheme)
}
}
// ExtractQueryParameters moves the query parameters of the URL into the options
// using reflection.
//
// The options must be a pointer to a struct which contains only string or bool
// fields (more types will be supported in the future), and tagged for JSON
// serialization.
//
// All of the URL's query parameters will be removed after calling this method.
func ExtractQueryParameters(u *url.URL, options interface{}) {
type field struct {
index int
kind reflect.Kind
}
// First, find all JSON fields in the options struct type.
o := reflect.Indirect(reflect.ValueOf(options))
ty := o.Type()
numFields := ty.NumField()
tagToField := make(map[string]field, numFields)
for i := 0; i < numFields; i++ {
f := ty.Field(i)
tag := f.Tag.Get("json")
tagToField[tag] = field{index: i, kind: f.Type.Kind()}
}
// Then, read content from the URL into the options.
for key, params := range u.Query() {
if len(params) == 0 {
continue
}
param := params[0]
normalizedKey := strings.ToLower(strings.ReplaceAll(key, "_", "-"))
if f, ok := tagToField[normalizedKey]; ok {
field := o.Field(f.index)
switch f.kind {
case reflect.Bool:
if v, e := strconv.ParseBool(param); e == nil {
field.SetBool(v)
}
case reflect.String:
field.SetString(param)
default:
panic("BackendOption introduced an unsupported kind, please handle it! " + f.kind.String())
}
}
}
// Clean up the URL finally.
u.RawQuery = ""
}
// FormatBackendURL obtains the raw URL which can be used the reconstruct the
// backend. The returned URL does not contain options for further configurating
// the backend. This is to avoid exposing secret tokens.
func FormatBackendURL(backend *backuppb.StorageBackend) (u url.URL) {
switch b := backend.Backend.(type) {
case *backuppb.StorageBackend_Local:
u.Scheme = "local"
u.Path = b.Local.Path
case *backuppb.StorageBackend_Noop:
u.Scheme = "noop"
u.Path = "/"
case *backuppb.StorageBackend_S3:
u.Scheme = "s3"
u.Host = b.S3.Bucket
u.Path = b.S3.Prefix
case *backuppb.StorageBackend_Gcs:
u.Scheme = "gcs"
u.Host = b.Gcs.Bucket
u.Path = b.Gcs.Prefix
case *backuppb.StorageBackend_AzureBlobStorage:
u.Scheme = "azure"
u.Host = b.AzureBlobStorage.Bucket
u.Path = b.AzureBlobStorage.Prefix
}
return
}
相关信息
相关文章
0
赞
热门推荐
-
2、 - 优质文章
-
3、 gate.io
-
6、 golang
-
7、 openharmony