Не удалось перезапустить отдельный узел и т.д. с помощью data_dir, сопоставленного с постоянным объемом в kubernetes

Question

Привет! Я развертываю контейнер etcd в модуле k8s с data_dir, сопоставленным с заявкой на постоянный том, созданной для модуля. При первом создании pvc и pod служба etcd работает и все работает как положено.

Как только я удаляю развертывание k8s и создаю его снова, он во время начальной загрузки распознает перезапуск существующего члена (возможно, из-за непустого data_dir), но не может запустить службу etcd с неожиданной ошибкой адреса ошибки.

В настоящее время мы используем конфигурацию кластера etcd с одним узлом, которая будет достаточна для того, чтобы у нас была и служба, и база данных etcd в одном модуле. Еще одна вещь, цель использования persistentvolume состоит в том, чтобы гарантировать отсутствие потери данных между перезапусками модуля.

etcd Версия: 3.3.11

PVC.yml

apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: sample-etcd-db-pvc
  annotations:
    volume.beta.kubernetes.io/storage-class: glusterfs-storage
spec:
  accessModes:
  - ReadWriteMany
  resources:
    requests:
      storage: 1Gi

deployment.yml

apiVersion: apps/v1
kind: Deployment
metadata:
  name: sample-etcd-db-service
spec:
  replicas: 1
  strategy:
    type: RollingUpdate
    rollingUpdate:
      maxUnavailable: 1
  selector:
    matchLabels:
      app: sample-etcd-db-service
      version: 0.1.0-rc.29
  template:
    metadata:
      labels:
        app: sample-etcd-db-service
        version: 0.1.0-rc.29
    spec:
      containers:
      - name: sample-etcd-db
        image: quay.io/coreos/etcd:v3.3.11
        imagePullPolicy: IfNotPresent
        command:
        - etcd
        - --name=sample-etcd-db-new
        - --listen-client-urls=http://0.0.0.0:2379
        - --advertise-client-urls=http://0.0.0.0:2379
        - --data-dir=/var/etcd/data
        volumeMounts:
        - mountPath: /var/etcd/data
          name: sample-etcd-db-pvc
        ports:
        - containerPort: 2379
      volumes:
      - name: sample-etcd-db-pvc
        persistentVolumeClaim:
          claimName: sample-etcd-db-pvc

Первое создание PVC и развертывание в кластере k8s

[root@centos-vm etcd_bug]# kubectl create -f pvc.yml
[root@centos-vm etcd_bug]# kubectl create -f deployment.yml

--- Попытка перезапустить etcd путем удаления и повторного создания развертывания k8s ---

[root@centos-vm etcd_bug]# kubectl delete deployment sample-etcd-db-service
[root@centos-vm etcd_bug]# kubectl create -f deployment.yml

Attaching logs from the pod.
[root@centos-vm etcd_bug]# kubectl logs sample-etcd-db-service-98f4f9459-4s27f -c sample-etcd-db
2019-02-19 20:25:22.344488 I | etcdmain: etcd Version: 3.3.11
2019-02-19 20:25:22.344636 I | etcdmain: Git SHA: 2cf9e51
2019-02-19 20:25:22.344645 I | etcdmain: Go Version: go1.10.7
2019-02-19 20:25:22.344651 I | etcdmain: Go OS/Arch: linux/amd64
2019-02-19 20:25:22.344659 I | etcdmain: setting maximum number of CPUs to 8, total number of available CPUs is 8
2019-02-19 20:25:22.348118 N | etcdmain: the server is already initialized as member before, starting as etcd member...
2019-02-19 20:25:22.348516 I | embed: listening for peers on http://0.0.0.0:2380
2019-02-19 20:25:22.349140 I | embed: listening for client requests on 0.0.0.0:2379
2019-02-19 20:25:22.391321 I | etcdserver: name = sample-etcd-db-new
2019-02-19 20:25:22.391362 I | etcdserver: data dir = /var/etcd/data
2019-02-19 20:25:22.391379 I | etcdserver: member dir = /var/etcd/data/member
2019-02-19 20:25:22.391387 I | etcdserver: heartbeat = 100ms
2019-02-19 20:25:22.391394 I | etcdserver: election = 1000ms
2019-02-19 20:25:22.391401 I | etcdserver: snapshot count = 100000
2019-02-19 20:25:22.391423 I | etcdserver: advertise client URLs = http://0.0.0.0:2379
2019-02-19 20:25:22.407858 I | etcdserver: restarting member 1c70f9bbb41018f in cluster a0d2de0531db7884 at commit index 4
2019-02-19 20:25:22.407995 I | raft: 1c70f9bbb41018f became follower at term 2
2019-02-19 20:25:22.408039 I | raft: newRaft 1c70f9bbb41018f [peers: [], term: 2, commit: 4, applied: 0, lastindex: 4, lastterm: 2]
unexpected fault address 0x7f43819ee000
fatal error: fault
[signal SIGBUS: bus error code=0x2 addr=0x7f43819ee000 pc=0x8808fd]

goroutine 1 [running]:
runtime.throw(0xfc556e, 0x5)
/usr/local/go/src/runtime/panic.go:616 +0x81 fp=0xc420222ed0 sp=0xc420222eb0 pc=0x42ade1
runtime.sigpanic()
/usr/local/go/src/runtime/signal_unix.go:385 +0x273 fp=0xc420222f20 sp=0xc420222ed0 pc=0x4405b3
github.com/coreos/etcd/cmd/vendor/github.com/coreos/bbolt.(*DB).page(...)
/tmp/etcd-release-3.3.11/etcd/release/etcd/gopath/src/github.com/coreos/etcd/cmd/vendor/github.com/coreos/bbolt/db.go:859
github.com/coreos/etcd/cmd/vendor/github.com/coreos/bbolt.(*Tx).page(...)
/tmp/etcd-release-3.3.11/etcd/release/etcd/gopath/src/github.com/coreos/etcd/cmd/vendor/github.com/coreos/bbolt/tx.go:599
github.com/coreos/etcd/cmd/vendor/github.com/coreos/bbolt.(*Bucket).pageNode(0xc4202b00f8, 0x2, 0x18, 0xc4201b78e0)
/tmp/etcd-release-3.3.11/etcd/release/etcd/gopath/src/github.com/coreos/etcd/cmd/vendor/github.com/coreos/bbolt/bucket.go:724 +0xad fp=0xc420222f98 sp=0xc420222f20 pc=0x8808fd
github.com/coreos/etcd/cmd/vendor/github.com/coreos/bbolt.(*Cursor).search(0xc420223138, 0x16062d0, 0x7, 0x7, 0x2)
/tmp/etcd-release-3.3.11/etcd/release/etcd/gopath/src/github.com/coreos/etcd/cmd/vendor/github.com/coreos/bbolt/cursor.go:254 +0x50 fp=0xc420223050 sp=0xc420222f98 pc=0x881d90
github.com/coreos/etcd/cmd/vendor/github.com/coreos/bbolt.(*Cursor).seek(0xc420223138, 0x16062d0, 0x7, 0x7, 0x0, 0x0, 0x4, 0xc4201bc8c0, 0x1, 0x1, ...)
/tmp/etcd-release-3.3.11/etcd/release/etcd/gopath/src/github.com/coreos/etcd/cmd/vendor/github.com/coreos/bbolt/cursor.go:159 +0xa5 fp=0xc4202230a0 sp=0xc420223050 pc=0x881695
github.com/coreos/etcd/cmd/vendor/github.com/coreos/bbolt.(*Bucket).CreateBucket(0xc4202b00f8, 0x16062d0, 0x7, 0x7, 0x0, 0x0, 0x0)
/tmp/etcd-release-3.3.11/etcd/release/etcd/gopath/src/github.com/coreos/etcd/cmd/vendor/github.com/coreos/bbolt/bucket.go:165 +0xfa fp=0xc4202231a0 sp=0xc4202230a0 pc=0x87dc6a
github.com/coreos/etcd/cmd/vendor/github.com/coreos/bbolt.(*Tx).CreateBucket(0xc4202b00e0, 0x16062d0, 0x7, 0x7, 0x2, 0x1c70f9bbb41018f, 0x4)
/tmp/etcd-release-3.3.11/etcd/release/etcd/gopath/src/github.com/coreos/etcd/cmd/vendor/github.com/coreos/bbolt/tx.go:108 +0x4f fp=0xc4202231e8 sp=0xc4202231a0 pc=0x88cbcf
github.com/coreos/etcd/cmd/vendor/github.com/coreos/etcd/mvcc/backend.(*batchTx).UnsafeCreateBucket(0xc4201be6f0, 0x16062d0, 0x7, 0x7)
/tmp/etcd-release-3.3.11/etcd/release/etcd/gopath/src/github.com/coreos/etcd/cmd/vendor/github.com/coreos/etcd/mvcc/backend/batch_tx.go:48 +0x6b fp=0xc420223280 sp=0xc4202231e8 pc=0x8de51b
github.com/coreos/etcd/cmd/vendor/github.com/coreos/etcd/etcdserver/membership.mustCreateBackendBuckets(0x10b3700, 0xc4201bb9d0)
/tmp/etcd-release-3.3.11/etcd/release/etcd/gopath/src/github.com/coreos/etcd/cmd/vendor/github.com/coreos/etcd/etcdserver/membership/store.go:166 +0xb7 fp=0xc4202232c0 sp=0xc420223280 pc=0x957827
github.com/coreos/etcd/cmd/vendor/github.com/coreos/etcd/etcdserver/membership.(*RaftCluster).SetBackend(0xc4201ce720, 0x10b3700, 0xc4201bb9d0)
/tmp/etcd-release-3.3.11/etcd/release/etcd/gopath/src/github.com/coreos/etcd/cmd/vendor/github.com/coreos/etcd/etcdserver/membership/cluster.go:203 +0x54 fp=0xc4202232e0 sp=0xc4202232c0 pc=0x9525f4
github.com/coreos/etcd/cmd/vendor/github.com/coreos/etcd/etcdserver.NewServer(0x7ffee163af6a, 0x12, 0x0, 0x0, 0x0, 0x0, 0xc4201a1200, 0x1, 0x1, 0xc4201a1100, ...)
/tmp/etcd-release-3.3.11/etcd/release/etcd/gopath/src/github.com/coreos/etcd/cmd/vendor/github.com/coreos/etcd/etcdserver/server.go:396 +0x921 fp=0xc420223ab0 sp=0xc4202232e0 pc=0xb762e1
github.com/coreos/etcd/cmd/vendor/github.com/coreos/etcd/embed.StartEtcd(0xc420294000, 0xc420294480, 0x0, 0x0)
.....

[root@centos-vm etcd_bug]# kubectl logs sample-etcd-db-service-98f4f9459-4s27f -c sample-etcd-db
2019-02-19 20:36:20.383231 I | etcdmain: etcd Version: 3.3.11
2019-02-19 20:36:20.383404 I | etcdmain: Git SHA: 2cf9e51
2019-02-19 20:36:20.383413 I | etcdmain: Go Version: go1.10.7
2019-02-19 20:36:20.383419 I | etcdmain: Go OS/Arch: linux/amd64
2019-02-19 20:36:20.383434 I | etcdmain: setting maximum number of CPUs to 8, total number of available CPUs is 8
2019-02-19 20:36:20.386048 N | etcdmain: the server is already initialized as member before, starting as etcd member...
2019-02-19 20:36:20.386987 I | embed: listening for peers on http://localhost:2380
2019-02-19 20:36:20.388330 I | embed: listening for client requests on 0.0.0.0:2379
2019-02-19 20:36:20.437097 I | etcdserver: name = sample-etcd-db-new
2019-02-19 20:36:20.437177 I | etcdserver: data dir = /var/etcd/data
2019-02-19 20:36:20.437198 I | etcdserver: member dir = /var/etcd/data/member
2019-02-19 20:36:20.437211 I | etcdserver: heartbeat = 100ms
2019-02-19 20:36:20.437222 I | etcdserver: election = 1000ms
2019-02-19 20:36:20.437233 I | etcdserver: snapshot count = 100000
2019-02-19 20:36:20.437284 I | etcdserver: advertise client URLs = http://0.0.0.0:2379
2019-02-19 20:36:20.456385 I | etcdserver: restarting member 1c70f9bbb41018f in cluster a0d2de0531db7884 at commit index 4
2019-02-19 20:36:20.456489 I | raft: 1c70f9bbb41018f became follower at term 2
2019-02-19 20:36:20.456520 I | raft: newRaft 1c70f9bbb41018f [peers: [], term: 2, commit: 4, applied: 0, lastindex: 4, lastterm: 2]
unexpected fault address 0x7efabfc3e000
fatal error: fault
[signal SIGBUS: bus error code=0x2 addr=0x7efabfc3e000 pc=0x8808fd]

goroutine 1 [running]:
runtime.throw(0xfc556e, 0x5)
/usr/local/go/src/runtime/panic.go:616 +0x81 fp=0xc420258ed0 sp=0xc420258eb0 pc=0x42ade1
runtime.sigpanic()
/usr/local/go/src/runtime/signal_unix.go:385 +0x273 fp=0xc420258f20 sp=0xc420258ed0 pc=0x4405b3
github.com/coreos/etcd/cmd/vendor/github.com/coreos/bbolt.(*DB).page(...)
/tmp/etcd-release-3.3.11/etcd/release/etcd/gopath/src/github.com/coreos/etcd/cmd/vendor/github.com/coreos/bbolt/db.go:859
github.com/coreos/etcd/cmd/vendor/github.com/coreos/bbolt.(*Tx).page(...)
/tmp/etcd-release-3.3.11/etcd/release/etcd/gopath/src/github.com/coreos/etcd/cmd/vendor/github.com/coreos/bbolt/tx.go:599
github.com/coreos/etcd/cmd/vendor/github.com/coreos/bbolt.(*Bucket).pageNode(0xc4203220f8, 0x2, 0x18, 0xc420227580)
/tmp/etcd-release-3.3.11/etcd/release/etcd/gopath/src/github.com/coreos/etcd/cmd/vendor/github.com/coreos/bbolt/bucket.go:724 +0xad fp=0xc420258f98 sp=0xc420258f20 pc=0x8808fd
github.com/coreos/etcd/cmd/vendor/github.com/coreos/bbolt.(*Cursor).search(0xc420259138, 0x16062d0, 0x7, 0x7, 0x2)
/tmp/etcd-release-3.3.11/etcd/release/etcd/gopath/src/github.com/coreos/etcd/cmd/vendor/github.com/coreos/bbolt/cursor.go:254 +0x50 fp=0xc420259050 sp=0xc420258f98 pc=0x881d90
github.com/coreos/etcd/cmd/vendor/github.com/coreos/bbolt.(*Cursor).seek(0xc420259138, 0x16062d0, 0x7, 0x7, 0x0, 0x0, 0x4, 0xc4202287f0, 0x1, 0x1, ...)
/tmp/etcd-release-3.3.11/etcd/release/etcd/gopath/src/github.com/coreos/etcd/cmd/vendor/github.com/coreos/bbolt/cursor.go:159 +0xa5 fp=0xc4202590a0 sp=0xc420259050 pc=0x881695
github.com/coreos/etcd/cmd/vendor/github.com/coreos/bbolt.(*Bucket).CreateBucket(0xc4203220f8, 0x16062d0, 0x7, 0x7, 0x0, 0x0, 0x0)
/tmp/etcd-release-3.3.11/etcd/release/etcd/gopath/src/github.com/coreos/etcd/cmd/vendor/github.com/coreos/bbolt/bucket.go:165 +0xfa fp=0xc4202591a0 sp=0xc4202590a0 pc=0x87dc6a
github.com/coreos/etcd/cmd/vendor/github.com/coreos/bbolt.(*Tx).CreateBucket(0xc4203220e0, 0x16062d0, 0x7, 0x7, 0x2, 0x1c70f9bbb41018f, 0x4)
/tmp/etcd-release-3.3.11/etcd/release/etcd/gopath/src/github.com/coreos/etcd/cmd/vendor/github.com/coreos/bbolt/tx.go:108 +0x4f fp=0xc4202591e8 sp=0xc4202591a0 pc=0x88cbcf
github.com/coreos/etcd/cmd/vendor/github.com/coreos/etcd/mvcc/backend.(*batchTx).UnsafeCreateBucket(0xc420240b40, 0x16062d0, 0x7, 0x7)
/tmp/etcd-release-3.3.11/etcd/release/etcd/gopath/src/github.com/coreos/etcd/cmd/vendor/github.com/coreos/etcd/mvcc/backend/batch_tx.go:48 +0x6b fp=0xc420259280 sp=0xc4202591e8 pc=0x8de51b
github.com/coreos/etcd/cmd/vendor/github.com/coreos/etcd/etcdserver/membership.mustCreateBackendBuckets(0x10b3700, 0xc4202c8620)
/tmp/etcd-release-3.3.11/etcd/release/etcd/gopath/src/github.com/coreos/etcd/cmd/vendor/github.com/coreos/etcd/etcdserver/membership/store.go:166 +0xb7 fp=0xc4202592c0 sp=0xc420259280 pc=0x957827
github.com/coreos/etcd/cmd/vendor/github.com/coreos/etcd/etcdserver/membership.(*RaftCluster).SetBackend(0xc42022a960, 0x10b3700, 0xc4202c8620)
/tmp/etcd-release-3.3.11/etcd/release/etcd/gopath/src/github.com/coreos/etcd/cmd/vendor/github.com/coreos/etcd/etcdserver/membership/cluster.go:203 +0x54 fp=0xc4202592e0 sp=0xc4202592c0 pc=0x9525f4
github.com/coreos/etcd/cmd/vendor/github.com/coreos/etcd/etcdserver.NewServer(0x7fff206f1fa8, 0x12, 0x0, 0x0, 0x0, 0x0, 0xc420280a00, 0x1, 0x1, 0xc420280700, ...)
/tmp/etcd-release-3.3.11/etcd/release/etcd/gopath/src/github.com/coreos/etcd/cmd/vendor/github.com/coreos/etcd/etcdserver/server.go:396 +0x921 fp=0xc420259ab0 sp=0xc4202592e0 pc=0xb762e1
github.com/coreos/etcd/cmd/vendor/github.com/coreos/etcd/embed.StartEtcd(0xc4202da000, 0xc4202da480, 0x0, 0x0)
/tmp/etcd-release-3.3.11/etcd/release/etcd/gopath/src/github.com/coreos/etcd/cmd/vendor/github.com/coreos/etcd/embed/etcd.go:179 +0x811 fp=0xc42025a6e8 sp=0xc420259ab0 pc=0xcb6361
github.com/coreos/etcd/cmd/vendor/github.com/coreos/etcd/etcdmain.startEtcd(0xc4202da000, 0xfc6677, 0x6, 0xc42025ad01, 0x2)
/tmp/etcd-release-3.3.11/etcd/release/etcd/gopath/src/github.com/coreos/etcd/cmd/vendor/github.com/coreos/etcd/etcdmain/etcd.go:181 +0x40 fp=0xc42025a7b0 sp=0xc42025a6e8 pc=0xd263c0
github.com/coreos/etcd/cmd/vendor/github.com/coreos/etcd/etcdmain.startEtcdOrProxyV2()
/tmp/etcd-release-3.3.11/etcd/release/etcd/gopath/src/github.com/coreos/etcd/cmd/vendor/github.com/coreos/etcd/etcdmain/etcd.go:102 +0x1369 fp=0xc42025bf08 sp=0xc42025a7b0 pc=0xd25d79
github.com/coreos/etcd/cmd/vendor/github.com/coreos/etcd/etcdmain.Main()
/tmp/etcd-release-3.3.11/etcd/release/etcd/gopath/src/github.com/coreos/etcd/cmd/vendor/github.com/coreos/etcd/etcdmain/main.go:46 +0x3f fp=0xc42025bf78 sp=0xc42025bf08 pc=0xd2c81f
main.main()
/tmp/etcd-release-3.3.11/etcd/release/etcd/gopath/src/github.com/coreos/etcd/cmd/etcd/main.go:28 +0x20 fp=0xc42025bf88 sp=0xc42025bf78 pc=0xd2e910
runtime.main()
/usr/local/go/src/runtime/proc.go:198 +0x212 fp=0xc42025bfe0 sp=0xc42025bf88 pc=0x42c652
runtime.goexit()
/usr/local/go/src/runtime/asm_amd64.s:2361 +0x1 fp=0xc42025bfe8 sp=0xc42025bfe0 pc=0x459f81

goroutine 51 [chan receive]:
github.com/coreos/etcd/cmd/vendor/github.com/coreos/etcd/pkg/logutil.(*MergeLogger).outputLoop(0xc4201c3720)
/tmp/etcd-release-3.3.11/etcd/release/etcd/gopath/src/github.com/coreos/etcd/cmd/vendor/github.com/coreos/etcd/pkg/logutil/merge_logger.go:174 +0x40d
created by github.com/coreos/etcd/cmd/vendor/github.com/coreos/etcd/pkg/logutil.NewMergeLogger
/tmp/etcd-release-3.3.11/etcd/release/etcd/gopath/src/github.com/coreos/etcd/cmd/vendor/github.com/coreos/etcd/pkg/logutil/merge_logger.go:92 +0x85

goroutine 104 [chan receive]:
github.com/coreos/etcd/cmd/vendor/github.com/coreos/etcd/pkg/logutil.(*MergeLogger).outputLoop(0xc420262680)
/tmp/etcd-release-3.3.11/etcd/release/etcd/gopath/src/github.com/coreos/etcd/cmd/vendor/github.com/coreos/etcd/pkg/logutil/merge_logger.go:174 +0x40d
created by github.com/coreos/etcd/cmd/vendor/github.com/coreos/etcd/pkg/logutil.NewMergeLogger
/tmp/etcd-release-3.3.11/etcd/release/etcd/gopath/src/github.com/coreos/etcd/cmd/vendor/github.com/coreos/etcd/pkg/logutil/merge_logger.go:92 +0x85

goroutine 72 [chan receive]:
github.com/coreos/etcd/cmd/vendor/github.com/coreos/etcd/pkg/logutil.(*MergeLogger).outputLoop(0xc420224860)
/tmp/etcd-release-3.3.11/etcd/release/etcd/gopath/src/github.com/coreos/etcd/cmd/vendor/github.com/coreos/etcd/pkg/logutil/merge_logger.go:174 +0x40d
created by github.com/coreos/etcd/cmd/vendor/github.com/coreos/etcd/pkg/logutil.NewMergeLogger
/tmp/etcd-release-3.3.11/etcd/release/etcd/gopath/src/github.com/coreos/etcd/cmd/vendor/github.com/coreos/etcd/pkg/logutil/merge_logger.go:92 +0x85

goroutine 116 [syscall]:
os/signal.signal_recv(0x0)
/usr/local/go/src/runtime/sigqueue.go:139 +0xa6
os/signal.loop()
/usr/local/go/src/os/signal/signal_unix.go:22 +0x22
created by os/signal.init.0
/usr/local/go/src/os/signal/signal_unix.go:28 +0x41

goroutine 91 [select]:
github.com/coreos/etcd/cmd/vendor/github.com/coreos/etcd/wal.(*filePipeline).run(0xc42023fb00)
/tmp/etcd-release-3.3.11/etcd/release/etcd/gopath/src/github.com/coreos/etcd/cmd/vendor/github.com/coreos/etcd/wal/file_pipeline.go:89 +0x139
created by github.com/coreos/etcd/cmd/vendor/github.com/coreos/etcd/wal.newFilePipeline
/tmp/etcd-release-3.3.11/etcd/release/etcd/gopath/src/github.com/coreos/etcd/cmd/vendor/github.com/coreos/etcd/wal/file_pipeline.go:47 +0x11a

goroutine 90 [select]:
github.com/coreos/etcd/cmd/vendor/github.com/coreos/etcd/mvcc/backend.(*backend).run(0xc4202c8620)
/tmp/etcd-release-3.3.11/etcd/release/etcd/gopath/src/github.com/coreos/etcd/cmd/vendor/github.com/coreos/etcd/mvcc/backend/backend.go:267 +0x180
created by github.com/coreos/etcd/cmd/vendor/github.com/coreos/etcd/mvcc/backend.newBackend
/tmp/etcd-release-3.3.11/etcd/release/etcd/gopath/src/github.com/coreos/etcd/cmd/vendor/github.com/coreos/etcd/mvcc/backend/backend.go:161 +0x2ea

goroutine 92 [select]:
github.com/coreos/etcd/cmd/vendor/github.com/coreos/etcd/raft.(*node).run(0xc42022a9c0, 0xc4202d4100)
/tmp/etcd-release-3.3.11/etcd/release/etcd/gopath/src/github.com/coreos/etcd/cmd/vendor/github.com/coreos/etcd/raft/node.go:313 +0x5f8
created by github.com/coreos/etcd/cmd/vendor/github.com/coreos/etcd/raft.RestartNode
/tmp/etcd-release-3.3.11/etcd/release/etcd/gopath/src/github.com/coreos/etcd/cmd/vendor/github.com/coreos/etcd/raft/node.go:223 +0x321