Warning: mysqli_query(): (HY000/1030): Got error 28 from storage engine in /www/wwwroot/www.sreguide.com/wp-includes/wp-db.php on line 2007

Warning: mysqli_query(): (HY000/1030): Got error 28 from storage engine in /www/wwwroot/www.sreguide.com/wp-includes/wp-db.php on line 2007
kubernetes storage cacher 系统设计与底层源码剖析 | 专注go语言|微服务|分布式|kubernetes系统研发与源码分析
Warning: mysqli_query(): (HY000/1030): Got error 28 from storage engine in /www/wwwroot/www.sreguide.com/wp-includes/wp-db.php on line 2007

Warning: mysqli_query(): (HY000/1030): Got error 28 from storage engine in /www/wwwroot/www.sreguide.com/wp-includes/wp-db.php on line 2007

kubernetes storage cacher 系统设计与底层源码剖析

storage cacher

背景

kubernetes storage cacher 系统设计与底层源码剖析

主要功能

K8s里面所有的数据增、删、改、查、WATCH都通过apiserver进行, 为了避免对Etcd的访问压力, k8s里面抽象了一个Cacher的struct,所有etcd事件的分发和访问, 都通过该对象进行, 该对象包装了etcd的client为一个storage在多层store里面共享

功能拆解

  • store: 提供对数据的操作接口,比如增删改查和watch
  • watcher: 监听store里面数据的变化, 同时做一些处理

注意, watcher和store是一个通用的数据操作抽象, 比如对etcd的操作有一套store和watcher, 上面的缓存也有一套store和watcher, rest那层还有一套store和watch

模式理解

kubernetes storage cacher 系统设计与底层源码剖析
  • store: 提供统一接口,屏蔽对底层 数据存储 的操作
  • watcher: watcher用于从store获取事件变化,以便进行一些自定义操作的处理
  • client: client如果要进行新功能开发,只要起一个新的watcher管住自己关心的数据变化, 完成自定义逻辑处理就可以

如果再说大一点可能就是 数据存储层 、 框架(store) 、 业务逻辑 桑层的解耦了

详细的调用图

kubernetes storage cacher 系统设计与底层源码剖析

代码实现

关键数据结构

  • Cacher 负责与后端etcd store进行交互, 分发事件给所有的watcher
type Cacher struct {

    // 当前incoming队列中的长度性能指标
    incomingHWM HighWateMark
    // incoming 事件管道, 回被分发给所有的watchers
    incoming chan watchCacheEvent

    sync.RWMutex

    // 当前cache ready状态必须为ok才可以被访问
    ready *ready

    // 后端存储数据接口
    storage Store

    // 对象类型
    objectType reflect.Type

    watchCache watchCache
    reflector  *Reflector

    versioner Versioner

    triggerFunc TriggerPublisherFunc
    watcherIdx  int
    watchers    indexedWatchers

    dispatchTimeoutBudge *timeBudget

    stopLock sync.RWMutex
    stopped  bool
    stopCh   chan struct{}
    stopWg   sync.WaitGroup
}
  • cacherWatcher 接收上面Cacher发送事件同时分发给rest websocket接口
// cacheWatcher 实现了watch接口
type cacheWatcher struct {
    sync.Mutex
    input     chan *watchCacheEvent
    result    chan Event
    done      chan struct{}
    filter    filterWithAttrsFunc
    stopped   bool
    forget     func(bool)
    versioner Versioner
}
  • Reflector watch后端数据变换同时把事件发送到watchCache(watch是以store的身份传递到Reflector中)
// Reflector 反射
type Reflector struct {
    name string

    expectedType reflect.Type

    store Store

    listerWatcher ListerWatcher
}

关键方法实现

func NewCacherFromConfig(config Config) *Cacher {
    // 首先生成一个watchCache
    watchCache := newWatchCache(config.CacheCapacity, config.KeyFunc, config.GetAttrsFunc, config.Versioner)
    listerWatcher := newCacherListerWatcher(config.Storage, config.ResourcePrefix, config.NewListFunc)
    reflectorName := "storage/cacher.go:" + config.ResourcePrefix

    stopCh := make(chan struct{})
    cacher := &Cacher{
        ready:       newReady(),
        storage:     config.Storage,
        objectType:  reflect.TypeOf(config.Type),
        watchCache:  watchCache,
            // watchCache 会被党委store传递到后端的reflector中, 用于reflector获取数据后将etcd中的数据转换成event事件
        reflector:   cache.NewNamedReflector(reflectorName, listerWatcher, config.Type, watchCache, 0),
        versioner:   config.Versioner,

    }
        // watchCache会设置SetOnEvent方法为cacher的processEvent, 在watchCache的所有事件类型处理的时候都会被改方法进行处理
    watchCache.SetOnEvent(cacher.processEvent)
    go cacher.dispatchEvents()

    cacher.stopWg.Add(1)
    go func() {
        defer cacher.stopWg.Done()
        wait.Until(
            func() {
                if !cacher.isStopped() {
                    cacher.startCaching(stopCh)
                }
            }, time.Second, stopCh,
        )
    }()
    return cacher
}
  • processEvent 将事件存放到自己的incoming队列中
func (c *Cacher) processEvent(event *watchCacheEvent) {
    if curLen := int64(len(c.incoming)); c.incomingHWM.Update(curLen) {
        fmt.Println("cacher %v: %v objects queued in comming channel", c.objectType.String(), curLen)
    }
    c.incoming <- *event
}
  • 事件传递给前端watcher
func (c *Cacher) dispatchEvents() {
    for {
        select {
        case event, ok := <-c.incoming:
            if !ok {
                return
            }
            c.dispatchEvent(&event)
        case <-c.stopCh:
            return
        }
    }
}

func (c *Cacher) dispatchEvent(event *watchCacheEvent) {
    triggerValues, supported := c.triggerValues(event)

    c.Lock()
    defer c.Unlock()

    for _, watcher := range c.watchers.allWatchers {
        watcher.add(event, d.dispatchTimeoutBudge)
    }
    if supported {
        for _, triggerValue := range triggerValues {
            for _, watcher := range c.watchers.valueWatchers[triggerValue] {
                watcher.add(event, d.dispatchTimeoutBudge)
            }
        }
    } else {
        for _, watchers := range c.watchers.valueWatchers {
            for _, watcher := range watchers {
                watcher.add(event, c.dispatchTimeoutBudge)
            }
        }
    }
}

完整代码

package cacher

import (
    "context"
    "fmt"
    "reflect"
    "sync"
    "sync/atomic"
    "time"

    "k8s.io/apimachinery/pkg/api/meta"
    "k8s.io/apimachinery/pkg/conversion"
)

// HighWateMark 性能
type HighWateMark int64

// Update 原子更新
func (hwm *HighWateMark) Update(current int64) bool {
    for {
        old := atomic.LoadInt64((*int64)(hwm))
        if current <= old {
            return false
        }
        if atomic.CompareAndSwapInt64((*int64)(hwm), old, current) {
            return true
        }
    }
}

// ready
type ready struct {
    ok bool
    c  *sync.Cond
}

// newReady 新建
func newReady() *ready {
    return &ready{c: sync.NewCond(&sync.Mutex{})}
}

// wait
func (r *ready) wait() {
    r.c.L.Lock()
    for !r.ok {
        r.c.Wait()
    }
    r.c.L.Unlock()
}

// check 返回当前的状态
func (r *ready) check() bool {
    r.c.L.Lock()
    defer r.c.L.Unlock()
    return r.ok
}

// set 修改状态
func (r *ready) set(ok bool) {
    r.c.L.Lock()
    defer r.c.L.Unlock()
    r.ok = ok
    r.c.Broadcast()
}

// TypeMeta API请求元数据
type TypeMeta struct {
    Kind       string `json:"kind,omitempty" protobuf:"bytes,1,opt,name=kind"`
    APIVersion string `json:"apiVersion,omitempty" protobuf:"bytes,2,opt,name=apiVersion"`
}

// ListOption 请求的参数
type ListOption struct {
    TypeMeta `json: ",inline"`

    LabelSelector string
    FieldSelector string
    // 是否包含初始化的资源
    IncludeUninitialized bool
    // 使用websocket反馈资源的Add、update、and remove事件通知
    Watch           bool
    ResourceVersion string
    TimeoutSecond   *int64
    Limit           int64
}

// ListerWatcher 抽象接口
type ListerWatcher interface {
    List(option ListOption) (Object, error)
    Watch(option ListOption) (Interface, error)
}

// Reflector 反射
type Reflector struct {
    name string

    expectedType reflect.Type

    store Store

    listerWatcher ListerWatcher
}

// ListAndWatch 获取最新版本并且watch数据变化
func (r *Reflector) ListAndWatch(stopCh <-chan struct{}) error {
    return nil
}

// MatchValue 批判
type MatchValue struct {
    IndexName string
    Value     string
}

// TriggerPublisherFunc 获取匹配的数据
type TriggerPublisherFunc func(obj Object) []MatchValue

//
type filterWithAttrsFunc func(key string, l Set, f Set, uninitializer bool) bool

// cacheWatcher 实现了watch接口
type cacheWatcher struct {
    sync.Mutex
    input     chan *watchCacheEvent
    result    chan Event
    done      chan struct{}
    filter    filterWithAttrsFunc
    stopped   bool
    forget     func(bool)
    versioner Versioner
}

func (c *cacheWatcher) Stop() {
    c.forget(true)
    c.stop()
}

func (c *cacheWatcher) stop() {
    c.Lock()
    defer c.Unlock()
    if c.stopped {
        c.stopped = true
        close(c.done)
        close(c.input)
    }
}

func newCacheWatcher(resourceVersion uint64, chanSize int, initEvents []*watchCacheEvent, filter filterWithAttrsFunc, forget func(bool), versioner Versioner) *cacheWatcher {
    watcher := &cacheWatcher{
        input:      make(chan *watchCacheEvent, chanSize),
        result:     make(chan Event, chanSize),
        done:       make(chan struct{}),
        filter:     filter,
        stopped:    false,
        forget:     forget
    }
    go watcher.process(initEvents, resourceVersion)
    return watcher
}

type watchersMap map[int]*cacheWatcher

func (wm watchersMap) terminateAll() {
    for key, watcher := range wm {
        delete(wm, key)
        watcher.Stop()
    }
}

type indexedWatchers struct {
    allWatchers   watchersMap
    valueWatchers map[string]watchersMap
}

func (i *indexedWatchers) terminateAll(objectType reflect.Type) {
    if len(i.allWatchers) > 0 || len(i.valueWatchers) > 0 {
        fmt.Println("Terminating all watchers from cacher %v", objectType)
    }
    i.allWatchers.terminateAll()
    for index, watchers := range i.valueWatchers {
        watchers.terminateAll()
        delete(i.valueWatchers, index)
    }
}

func (i *indexedWatchers) deleteWatcher(number int, value string, supported bool) {
    if supported {
        i.valueWatchers[value].deleteWatcher(number)
        if len(i.valueWatchers[value]) == 0 {
            delete(i.valueWatchers, value)
        }
    } else {
        i.allWatchers.deleteWatcher(number)
    }
}

func (i *indexedWatchers) addWatcher(w *cacheWatcher, number int, value string, supported bool) {
    if supported {
        if _, ok := i.valueWatchers[value]; !{
            i.valueWatchers[value] = watchersMap{}
        }
        i.valueWatchers[value].addWatcher(w, number)
    } else {
        i.allWatchers.addWatcher(w, number)
    }
}

type timeBudget struct {
    sync.Mutex
    budget time.Duration

    refresh   time.Duration
    maxBudget time.Duration
}

type Labels interface {
    Hash(label string) (exists bool)
    Get(label string) (value string)
}

type Selector interface {
    Matchs(Labels) bool
    Empty() bool
    String() string
    RequiresExactMatch(field string) (value string, found bool)
    DeepCopySelector() Selector
}

// AttrFunc 获取对象的Label和Field集合
type AttrFunc func(obj Object) (Set, Set, bool, error)

// SelectPredicate  对象的表现形式
type SelectionPredicate struct {
    Label                Selector
    Fielld               Selector
    IncludeUninitialized bool
    GetAttrs             AttrFunc
    InedxFields          []string
    Limit                int64
    Continue             string
}

func (s *SelectionPredicate) MatcherIndex() []MatchValue {
    var result []MatchValue
    for , field := range s.InedxFields {
        if value, ok := s.Fielld.RequiresExactMatch(field); ok {
            result = append(result, MatchValue{IndexName: field, Value: value})
        }
    }
    return result
}

type Feature string

type FeatureGate interface {
    Enabled(key Feature) bool
}

type UID string

type Preconditions struct {
    UID *UID
}

type StatusError struct {
    ErrStatus metav1.Status
}

type errWatcher struct {
    result chan Event
}

func newErrWatcher(err error) *errWatcher {
    errEvent := Event{Type: Error}
    switch err := err.(type) {
    case Object:
        errEvent.Object = err
    case StatusError:
        errEvent.Object = &err.ErrStatus
    default:
        errEvent.Object = &metav1.Status{
            Status:  metav1.StatusFailure,
            Message: err.Error(),
            Reason:  metav1.StatusReasonInternalError,
            Code:    http.StatusInternalServerError,
        }
    }
}

type UpdateFunc func(input runtime.Object, res ResponseMeta) (output runtime.Object, ttl *uint64, err error)


// Cacher 缓存
type Cacher struct {

    // 当前incoming队列中的长度性能指标
    incomingHWM HighWateMark
    // incoming 事件管道, 回被分发给所有的watchers
    incoming chan watchCacheEvent

    sync.RWMutex

    // 当前cache ready状态必须为ok才可以被访问
    ready *ready

    // 后端存储数据接口
    storage Store

    // 对象类型
    objectType reflect.Type

    watchCache watchCache
    reflector  *Reflector

    versioner Versioner

    triggerFunc TriggerPublisherFunc
    watcherIdx  int
    watchers    indexedWatchers

    dispatchTimeoutBudge *timeBudget

    stopLock sync.RWMutex
    stopped  bool
    stopCh   chan struct{}
    stopWg   sync.WaitGroup
}

func (c *Cacher) Versioner() Versioner {
    return c.storage.Versioner()
}

func (c *Cacher) Create(ctx context.Context, ket string, out Object preconditions *Preconditions) {
    c.storage.Create(ctx, key, out, preconditions)
}

func (c *Cacher) Delete(ctx context.Context, key string, out Object, preconditions *Preconditions) error {
    c.storage.Delete(ctx, key, out, preconditions)
}

func (c *Cacher) Watch(ctx context.Context, key string, resourceVersion string, pred SelectionPredicate) (Interface, error) {
    watchRV,, err := c.versioner.ParseResourceVersion(resourceVersion)
    if err != nil {
        return nil, err
    }
    
    c.ready.wait()

    c.watchCache.RLock()
    defer c.watchCache.RUnlock()
    initEvents, err := c.watchCache.GetAllEventsSinceThreadUnsafe(watchRV)
    if err != nil {
        return newErrWatcher(err), nil
    }

    triggerValue, triggerSupported := "", false
    if matchValues := pred.MatchIndex(); len(matchValues) > 0 {
        triggerValue, triggerSupported = matchValues[0].Value, true
    }

    chanSize := 10
    if c.triggerFunc != nil && !triggerSupported {
        chanSize = 100
    }

    c.Lock()
    defer c.Unlock()
    forget := forgetWatcher(c, c.watcherIdx, triggerValue, triggerSupported)
    watcher := newCacheWatcher(watchRv, chanSize, initEvents, filterWithAttrsFunc(key, pred), forget, c.versioner)

    c.watchers.addWatcher(watcher, c.watcherIdx, triggerValue, triggerSupported)
    c.watcherIdx++
    return watcher, nil
}

func (c *Cacher) WatchList(ctx context.Context, key, string, resourceVersion string, pred SelectionPredicate) (Interface, error) {
    return c.Watch(ctx, key, resourceVersion, pred)
}

func (c *Cacher) Get(ctx context.Context, key string, resourceVersion string, objPtr Object, ignoreNotFound bool) error {
    if resourceVersion == "" {
        return c.storage.Get(ctx, key, resourceVersion, objPtr, ignoreNotFound)
    }

    getRv, err := c.versioner.ParseResourceVersion(resourceVersion)
    if err != nil {
        return nil, err
    }

    if getRv == 0 && !c.ready.check() {
        return c.storage.Get(ctx, key, resourceVersion, objPtr, ignoreNotFound)
    }

    c.ready.wait()

    objValue, err := conversion.EnforcePtr(objPtr)
    if err != nil {
        return nil, err
    }

    obj, exists, resourceVersion, err := c.watchCache.WaitUntilFreshAndGet(getRv, key, nil)
    if err != nil {
        return nil, err
    }

    if exists {
        elem, ok := obj.(*storeElement)
        if !ok {
            return fmt.Errorf("non *storeElement returned form storage : %v", obj)
        }
        objValue.Set(reflect.ValueOf(elem.Object).Elem())
    } else {
        objValue.Set(reflect.Zero(objValue.Type()))
        if !ignoreNotFound {
            return fmt.Errorf("key: %v resourversion: %v", objValue, getRv)
        }
    }
    return nil
}

func (c *Cacher) List(ctx context.Context, ket string, resourceVersion string, pred SelectionPredicate, listObj Object) error {
    if resourceVersion == "" {
        // 实际上这个地方嗨会有判断分页等信息目前胜率 
        return c.storage.list(ctx, key, resourceVersion, pred, listObj)
    }

    listRV, err := c.versioner.ParseResourceVersion(resourceVersion)
    if err != nil {
        return err
    }

    if listRV ==0 && !c.ready.check() {
        return c.storage.List(ctx, key, resourceVersion, pred, listObj)
    } 

    c.ready.wait()

    listPtr, err := conversion.EnforcePtr(listPtr)
    if err != nil {
        return err
    }
    listVal, err := conversion.EnforcePtr(listPtr)
    if err != nil || listVal.Kind() != reflect.Slice {
        return fmt.Errorf("need a pointer to slice got %v", listVal.Kind())
    }

    filter := filterWithAttrsFunc(key, pred)

    objs, resourceVersion, err := c.watchCache.WaitUntilFreshAndGet(listRV)
    if err != nil {
        return err
    }

    if len(objs) > listVal.Cap() && pred.Label.Empty() && pred.Fielld.Empty() {
        // 如果发现对象超过切片的容量就重新生成一个
        listVal.Set(reflect.MakeSlice(reflect.SliceOf(c.objectType.Elem()), 0, len(objs)))
    }
    for _, obj := range objs {
        elem, ok := obj.(*storeElement)
        if !ok {
            return fmt.Errorf("non *storeElement returned from storage: %v", obj)
        }
        if filter(elem.Key, elem.Fields, elem.Labels, elem.Uninitialized) {
            // 反射后续需要学习
            listVal.Set(reflect.Append(listVal, reflect.ValueOf(elem.Object).Elem()))
        }
    }
    if c.versioner != nil {
        if err := c.versioner.UpdateList(listObj, resourceVersion, ""); err != nil {
            return err
        }
    }
    return nil
}

func (c *Cacher) GuaranteedUpdate(
    ctx context.Context, key string, ptrToType Object, ignoreNotFound bool,
    preconditions * Preconditions, tryUpdate UpdateFunc, _...Object) error {
        if elem, exists, err := c.watchCache.GetByKey(key); err != nil {
            fmt.Printf("GetByKey returned error: %v", err)
        } else if exists {
            currObj := elem.(*storeElement).Object.DeepCopyObject()
            return c.storage.GuaranteedUpdate(ctx, key, ptrToType, ignoreNotFound, preconditions, tryUpdate, currObj)
        }
        return c.storage.GuaranteedUpdate(ctx, key, ptrToType, ignoreNotFound, preconditions, tryUpdate)
}

func (c *Cacher) Count(pathPrefix string) (int64, error) {
    return c.storage.Count(pathPrefix)
}

func (c *Cacher) triggerValues(event *watchCacheEvent))([]string, bool) {
    if c.triggerFunc == nil {
        return nil, false
    }

    result := make([]string, 2)
    matchValues := c.triggerFunc(event.Object)
    if len(matchValues) > 0 {
        result = append(result, matchValues[0].Value)
    }
    if event.PrevObject == nil {
        return result, len(result) > 0
    }
    prevMatchValues := c.triggerFunc(event.PrevObject)
    if len(prevMatchValues) > 0 {
        if len(result) == 0 || result[0] != prevMatchValues[0].Value {
            result = append(result, prevMatchValues[0].Value)
        }
    }
    return result, len(result) > 0
}

func (c *Cacher) processEvent(event *watchCacheEvent) {
    if curLen := int64(len(c.incoming)); c.incomingHWM.Update(curLen) {
        fmt.Println("cacher %v: %v objects queued in comming channel", c.objectType.String(), curLen)
    }
    c.incoming <- *event
}

func (c *Cacher) dispatchEvents() {
    for {
        select {
        case event, ok := <-c.incoming:
            if !ok {
                return
            }
            c.dispatchEvent(&event)
        case <-c.stopCh:
            return
        }
    }
}

func (c *Cacher) dispatchEvent(event *watchCacheEvent) {
    triggerValues, supported := c.triggerValues(event)

    c.Lock()
    defer c.Unlock()

    for _, watcher := range c.watchers.allWatchers {
        watcher.add(event, d.dispatchTimeoutBudge)
    }
    if supported {
        for _, triggerValue := range triggerValues {
            for _, watcher := range c.watchers.valueWatchers[triggerValue] {
                watcher.add(event, d.dispatchTimeoutBudge)
            }
        }
    } else {
        for _, watchers := range c.watchers.valueWatchers {
            for _, watcher := range watchers {
                watcher.add(event, c.dispatchTimeoutBudge)
            }
        }
    }
}


func (c *Cacher) GetToList(ctx context.Context, key string, resourceVersion string, pred SelectionPredicate, listObj Object) error {

    if resourceVersion == "" || (len(pred.Continue) > 0 || pred.Limit > 0) {
        // 如果resourceVersion是空就直接从storage里面获取对应的数据
        return c.storage.GetToList(ctx, key, resourceVersion, pred, listObj)
    }

    // 调用versioner解析出list版本
    listRv, err := c.versioner.ParseResourceVersion(resourceVersion)
    if err != nil {
        return err
    }

    // listRv为0并且未完成更新缓存, 从后端直接获取
    if listRv == 0 && !c.ready.check() {
        return c.storage.GetToList(ctx, key, resourceVersion, pred, listObj)
    }

    c.ready.wait()

    listPtr, err := meta.GetItemsPtr(listObj)
    if err != nil {
        return err
    }
    listVal, err := conversion.EnforcePtr(listObj)
    if err != nil || listVal.Kind() != reflect.Slice {
        return fmt.Errorf("need a prointer to slice got %v", listVal.Kind())
    }
    filter := filterWithAttrsFunc(key, pred)

    // 根据上面转换的resource version从后端获取对应的额objs
    obj, exists, resourceVersion, err := c.watchCache.WaitUntilFreshAndGet(listRv, key)
    if err != nil {
        return err
    }

    if exits {
        elem, ok := obj.(*storeElement)
        if !ok {
            return fmt.Errorf("non *storeElement returned from storage: %v", obj)
        }
        if filter(elem.Key, elem.Labels, elem.Fields, elem.Uninitialized) {
            listVal.Set(reflect.Append(listVal, reflect.ValueOf(elem.Object).Elem())
        }
        if c.versioner != nil {
            if err := c.versioner.UpdateList(listObj, resourceVersion, ""); err != nil {
                return err
            }
        }
    }
    return nil
}


func (c *Cacher) startCaching(stopChannel <-chan struct{}) {
    successfulList := false
    c.watchCache.SetOnReplace(func() {
        successfulList = true
        c.ready.set(true)
    })
    defer func() {
        if successfulList {
            c.ready.set(false)
        }
    }()
    c.terminateAllWatchers()
    if err := c.reflector.ListAndWatch(stopChannel); err != nil {
        fmt.Errorf("unexpected listAndWatch error: %v", err)
    }
}

func (c *Cacher) terminateAllWatchers() {
    c.Lock()
    defer c.Unlock()
    c.watchers.terminateAll(c.objectType)
}

func forgetWatcher(c *Cacher, index int, triggerValue string, triggerSupported bool) func(bool) {
    return func(lock bool) {
        if lock {
            c.Lock()
            defer c.Unlock()
        } else {
            fmt.Errorf("Forcing watcher close due to unresponsiveness: %v", c.objectType.String())
        }
        c.watchers.deleteWatcher(index, triggerValue, triggerSupported)
    }
}

感悟

其实看了很多, 一层层, 但要完全说明每一层,每一个函数都是怎么转换的帧的太多了, 其实核心就两个Storage和Watcher, Storage的核心是数据变化通知的上层实现,Watcher用于关注数据变化传递响应, 接下来,可能会暂停看apiserver端,先看一下controller和client-go量部分,先这样吧,Good Night

原创文章,作者:baxiaoshi,如若转载,请注明出处:http://www.sreguide.com/uncategorized/kubernetes-storage-cacher-%e7%b3%bb%e7%bb%9f%e8%ae%be%e8%ae%a1%e4%b8%8e%e5%ba%95%e5%b1%82%e6%ba%90%e7%a0%81%e5%89%96%e6%9e%90.html

发表评论

电子邮件地址不会被公开。 必填项已用*标注

联系我们

QQ: 52866169