Я развернулasticsearch в моем кластере kubernetes, используя эту команду, выполните следующие действия: do c:
kubectl apply -f https://download.elastic.co/downloads/eck/1.0.1/all-in-one.yaml
create service:
apiVersion: elasticsearch.k8s.elastic.co/v1
kind: Elasticsearch
metadata:
name: quickstart
spec:
version: 7.6.0
nodeSets:
- name: default
count: 1
config:
node.master: true
node.data: true
node.ingest: true
node.store.allow_mmap: false
теперь все запуск службы успешно, за исключением пакетов quickstart-es-default-0. Это показывает Back-off restarting failed container. Проверьте вывод журнала:
~/Library/Mobile Documents/com~apple~CloudDocs/Document/k8s/work/skywalking ⌚ 10:02:01
$ kubectl logs quickstart-es-default-0
Error from server (BadRequest): container "elasticsearch" in pod "quickstart-es-default-0" is waiting to start: PodInitializing
Я мог бы получить любое полезное сообщение из вывода журнала. Теперь я хочу знать, почему это могло произойти, как это исправить?
Я пытался:
Я обнаружил некоторое сбойное состояние от kubernetes приборной панели, но до сих пор не знаю, как это исправить:
PS: это мои Stateful Sets yaml:
{
"kind": "StatefulSet",
"apiVersion": "apps/v1beta2",
"metadata": {
"name": "quickstart-es-default",
"namespace": "dabai-fat",
"selfLink": "/apis/apps/v1beta2/namespaces/dabai-fat/statefulsets/quickstart-es-default",
"uid": "430b0f0d-a771-4ad5-8eaa-963e5646d508",
"resourceVersion": "16783276",
"generation": 9,
"creationTimestamp": "2020-02-27T04:39:27Z",
"labels": {
"common.k8s.elastic.co/template-hash": "1894377832",
"common.k8s.elastic.co/type": "elasticsearch",
"elasticsearch.k8s.elastic.co/cluster-name": "quickstart",
"elasticsearch.k8s.elastic.co/statefulset-name": "quickstart-es-default"
},
"ownerReferences": [
{
"apiVersion": "elasticsearch.k8s.elastic.co/v1",
"kind": "Elasticsearch",
"name": "quickstart",
"uid": "3b2483cb-6da2-418f-8e9d-5a3059720d7a",
"controller": true,
"blockOwnerDeletion": true
}
]
},
"spec": {
"replicas": 1,
"selector": {
"matchLabels": {
"common.k8s.elastic.co/type": "elasticsearch",
"elasticsearch.k8s.elastic.co/cluster-name": "quickstart",
"elasticsearch.k8s.elastic.co/statefulset-name": "quickstart-es-default"
}
},
"template": {
"metadata": {
"creationTimestamp": null,
"labels": {
"common.k8s.elastic.co/type": "elasticsearch",
"elasticsearch.k8s.elastic.co/cluster-name": "quickstart",
"elasticsearch.k8s.elastic.co/config-hash": "2034778696",
"elasticsearch.k8s.elastic.co/http-scheme": "https",
"elasticsearch.k8s.elastic.co/node-data": "true",
"elasticsearch.k8s.elastic.co/node-ingest": "true",
"elasticsearch.k8s.elastic.co/node-master": "true",
"elasticsearch.k8s.elastic.co/node-ml": "true",
"elasticsearch.k8s.elastic.co/statefulset-name": "quickstart-es-default",
"elasticsearch.k8s.elastic.co/version": "7.6.0"
}
},
"spec": {
"volumes": [
{
"name": "downward-api",
"downwardAPI": {
"items": [
{
"path": "labels",
"fieldRef": {
"apiVersion": "v1",
"fieldPath": "metadata.labels"
}
}
],
"defaultMode": 420
}
},
{
"name": "elastic-internal-elasticsearch-bin-local",
"emptyDir": {}
},
{
"name": "elastic-internal-elasticsearch-config",
"secret": {
"secretName": "quickstart-es-default-es-config",
"defaultMode": 420,
"optional": false
}
},
{
"name": "elastic-internal-elasticsearch-config-local",
"emptyDir": {}
},
{
"name": "elastic-internal-elasticsearch-plugins-local",
"emptyDir": {}
},
{
"name": "elastic-internal-http-certificates",
"secret": {
"secretName": "quickstart-es-http-certs-internal",
"defaultMode": 420,
"optional": false
}
},
{
"name": "elastic-internal-probe-user",
"secret": {
"secretName": "quickstart-es-internal-users",
"items": [
{
"key": "elastic-internal-probe",
"path": "elastic-internal-probe"
}
],
"defaultMode": 420,
"optional": false
}
},
{
"name": "elastic-internal-scripts",
"configMap": {
"name": "quickstart-es-scripts",
"defaultMode": 493,
"optional": false
}
},
{
"name": "elastic-internal-transport-certificates",
"secret": {
"secretName": "quickstart-es-transport-certificates",
"defaultMode": 420,
"optional": false
}
},
{
"name": "elastic-internal-unicast-hosts",
"configMap": {
"name": "quickstart-es-unicast-hosts",
"defaultMode": 420,
"optional": false
}
},
{
"name": "elastic-internal-xpack-file-realm",
"secret": {
"secretName": "quickstart-es-xpack-file-realm",
"defaultMode": 420,
"optional": false
}
},
{
"name": "elasticsearch-data",
"persistentVolumeClaim": {
"claimName": "claim-name-placeholder"
}
},
{
"name": "elasticsearch-logs",
"emptyDir": {}
}
],
"initContainers": [
{
"name": "elastic-internal-init-filesystem",
"image": "docker.elastic.co/elasticsearch/elasticsearch:7.6.0",
"command": [
"bash",
"-c",
"/mnt/elastic-internal/scripts/prepare-fs.sh"
],
"env": [
{
"name": "POD_IP",
"valueFrom": {
"fieldRef": {
"apiVersion": "v1",
"fieldPath": "status.podIP"
}
}
},
{
"name": "POD_NAME",
"valueFrom": {
"fieldRef": {
"apiVersion": "v1",
"fieldPath": "metadata.name"
}
}
},
{
"name": "POD_IP",
"valueFrom": {
"fieldRef": {
"apiVersion": "v1",
"fieldPath": "status.podIP"
}
}
},
{
"name": "POD_NAME",
"valueFrom": {
"fieldRef": {
"apiVersion": "v1",
"fieldPath": "metadata.name"
}
}
}
],
"resources": {
"limits": {
"cpu": "100m",
"memory": "50Mi"
},
"requests": {
"cpu": "100m",
"memory": "50Mi"
}
},
"volumeMounts": [
{
"name": "elastic-internal-elasticsearch-config-local",
"mountPath": "/mnt/elastic-internal/elasticsearch-config-local"
},
{
"name": "elastic-internal-elasticsearch-plugins-local",
"mountPath": "/mnt/elastic-internal/elasticsearch-plugins-local"
},
{
"name": "elastic-internal-elasticsearch-bin-local",
"mountPath": "/mnt/elastic-internal/elasticsearch-bin-local"
},
{
"name": "elastic-internal-transport-certificates",
"readOnly": true,
"mountPath": "/mnt/elastic-internal/transport-certificates"
},
{
"name": "elastic-internal-scripts",
"readOnly": true,
"mountPath": "/mnt/elastic-internal/scripts"
},
{
"name": "elasticsearch-data",
"mountPath": "/usr/share/elasticsearch/data"
},
{
"name": "elasticsearch-logs",
"mountPath": "/usr/share/elasticsearch/logs"
},
{
"name": "downward-api",
"readOnly": true,
"mountPath": "/mnt/elastic-internal/downward-api"
},
{
"name": "elastic-internal-elasticsearch-config",
"readOnly": true,
"mountPath": "/mnt/elastic-internal/elasticsearch-config"
},
{
"name": "elastic-internal-http-certificates",
"readOnly": true,
"mountPath": "/usr/share/elasticsearch/config/http-certs"
},
{
"name": "elastic-internal-probe-user",
"readOnly": true,
"mountPath": "/mnt/elastic-internal/probe-user"
},
{
"name": "elastic-internal-unicast-hosts",
"readOnly": true,
"mountPath": "/mnt/elastic-internal/unicast-hosts"
},
{
"name": "elastic-internal-xpack-file-realm",
"readOnly": true,
"mountPath": "/mnt/elastic-internal/xpack-file-realm"
}
],
"terminationMessagePath": "/dev/termination-log",
"terminationMessagePolicy": "File",
"imagePullPolicy": "IfNotPresent",
"securityContext": {
"privileged": false
}
}
],
"containers": [
{
"name": "elasticsearch",
"image": "docker.elastic.co/elasticsearch/elasticsearch:7.6.0",
"ports": [
{
"name": "http",
"containerPort": 9200,
"protocol": "TCP"
},
{
"name": "transport",
"containerPort": 9300,
"protocol": "TCP"
}
],
"env": [
{
"name": "HEADLESS_SERVICE_NAME",
"value": "quickstart-es-default"
},
{
"name": "NSS_SDB_USE_CACHE",
"value": "no"
},
{
"name": "POD_IP",
"valueFrom": {
"fieldRef": {
"apiVersion": "v1",
"fieldPath": "status.podIP"
}
}
},
{
"name": "POD_NAME",
"valueFrom": {
"fieldRef": {
"apiVersion": "v1",
"fieldPath": "metadata.name"
}
}
},
{
"name": "PROBE_PASSWORD_PATH",
"value": "/mnt/elastic-internal/probe-user/elastic-internal-probe"
},
{
"name": "PROBE_USERNAME",
"value": "elastic-internal-probe"
},
{
"name": "READINESS_PROBE_PROTOCOL",
"value": "https"
}
],
"resources": {
"limits": {
"memory": "2Gi"
},
"requests": {
"memory": "2Gi"
}
},
"volumeMounts": [
{
"name": "downward-api",
"readOnly": true,
"mountPath": "/mnt/elastic-internal/downward-api"
},
{
"name": "elastic-internal-elasticsearch-bin-local",
"mountPath": "/usr/share/elasticsearch/bin"
},
{
"name": "elastic-internal-elasticsearch-config",
"readOnly": true,
"mountPath": "/mnt/elastic-internal/elasticsearch-config"
},
{
"name": "elastic-internal-elasticsearch-config-local",
"mountPath": "/usr/share/elasticsearch/config"
},
{
"name": "elastic-internal-elasticsearch-plugins-local",
"mountPath": "/usr/share/elasticsearch/plugins"
},
{
"name": "elastic-internal-http-certificates",
"readOnly": true,
"mountPath": "/usr/share/elasticsearch/config/http-certs"
},
{
"name": "elastic-internal-probe-user",
"readOnly": true,
"mountPath": "/mnt/elastic-internal/probe-user"
},
{
"name": "elastic-internal-scripts",
"readOnly": true,
"mountPath": "/mnt/elastic-internal/scripts"
},
{
"name": "elastic-internal-transport-certificates",
"readOnly": true,
"mountPath": "/usr/share/elasticsearch/config/transport-certs"
},
{
"name": "elastic-internal-unicast-hosts",
"readOnly": true,
"mountPath": "/mnt/elastic-internal/unicast-hosts"
},
{
"name": "elastic-internal-xpack-file-realm",
"readOnly": true,
"mountPath": "/mnt/elastic-internal/xpack-file-realm"
},
{
"name": "elasticsearch-data",
"mountPath": "/usr/share/elasticsearch/data"
},
{
"name": "elasticsearch-logs",
"mountPath": "/usr/share/elasticsearch/logs"
}
],
"readinessProbe": {
"exec": {
"command": [
"bash",
"-c",
"/mnt/elastic-internal/scripts/readiness-probe-script.sh"
]
},
"initialDelaySeconds": 10,
"timeoutSeconds": 5,
"periodSeconds": 5,
"successThreshold": 1,
"failureThreshold": 3
},
"lifecycle": {
"preStop": {
"exec": {
"command": [
"bash",
"-c",
"/mnt/elastic-internal/scripts/pre-stop-hook-script.sh"
]
}
}
},
"terminationMessagePath": "/dev/termination-log",
"terminationMessagePolicy": "File",
"imagePullPolicy": "IfNotPresent"
}
],
"restartPolicy": "Always",
"terminationGracePeriodSeconds": 180,
"dnsPolicy": "ClusterFirst",
"automountServiceAccountToken": false,
"securityContext": {},
"affinity": {
"podAntiAffinity": {
"preferredDuringSchedulingIgnoredDuringExecution": [
{
"weight": 100,
"podAffinityTerm": {
"labelSelector": {
"matchLabels": {
"elasticsearch.k8s.elastic.co/cluster-name": "quickstart"
}
},
"topologyKey": "kubernetes.io/hostname"
}
}
]
}
},
"schedulerName": "default-scheduler"
}
},
"volumeClaimTemplates": [
{
"metadata": {
"name": "elasticsearch-data",
"creationTimestamp": null,
"ownerReferences": [
{
"apiVersion": "elasticsearch.k8s.elastic.co/v1",
"kind": "Elasticsearch",
"name": "quickstart",
"uid": "3b2483cb-6da2-418f-8e9d-5a3059720d7a",
"controller": true,
"blockOwnerDeletion": false
}
]
},
"spec": {
"accessModes": [
"ReadWriteOnce"
],
"resources": {
"requests": {
"storage": "1Gi"
}
},
"volumeMode": "Filesystem"
},
"status": {
"phase": "Pending"
}
}
],
"serviceName": "quickstart-es-default",
"podManagementPolicy": "Parallel",
"updateStrategy": {
"type": "OnDelete"
},
"revisionHistoryLimit": 10
},
"status": {
"observedGeneration": 9,
"replicas": 1,
"currentReplicas": 1,
"updatedReplicas": 1,
"currentRevision": "quickstart-es-default-7c6864f4c",
"updateRevision": "quickstart-es-default-7c6864f4c",
"collisionCount": 0
}
}
Это вывод событий (кстати, теперь я переместил весь сервис в elastic-system пространство имен):
Это вывод с описанием состояния с описанием состояния :
$ kubectl describe statefulset quickstart-es-default
Name: quickstart-es-default
Namespace: elastic-system
CreationTimestamp: Thu, 27 Feb 2020 13:57:05 +0800
Selector: common.k8s.elastic.co/type=elasticsearch,elasticsearch.k8s.elastic.co/cluster-name=quickstart,elasticsearch.k8s.elastic.co/statefulset-name=quickstart-es-default
Labels: common.k8s.elastic.co/template-hash=617547124
common.k8s.elastic.co/type=elasticsearch
elasticsearch.k8s.elastic.co/cluster-name=quickstart
elasticsearch.k8s.elastic.co/statefulset-name=quickstart-es-default
Annotations: <none>
Replicas: 1 desired | 1 total
Update Strategy: OnDelete
Pods Status: 0 Running / 1 Waiting / 0 Succeeded / 0 Failed
Pod Template:
Labels: common.k8s.elastic.co/type=elasticsearch
elasticsearch.k8s.elastic.co/cluster-name=quickstart
elasticsearch.k8s.elastic.co/config-hash=2034778696
elasticsearch.k8s.elastic.co/http-scheme=https
elasticsearch.k8s.elastic.co/node-data=true
elasticsearch.k8s.elastic.co/node-ingest=true
elasticsearch.k8s.elastic.co/node-master=true
elasticsearch.k8s.elastic.co/node-ml=true
elasticsearch.k8s.elastic.co/statefulset-name=quickstart-es-default
elasticsearch.k8s.elastic.co/version=7.6.0
Init Containers:
elastic-internal-init-filesystem:
Image: docker.elastic.co/elasticsearch/elasticsearch:7.6.0
Port: <none>
Host Port: <none>
Command:
bash
-c
/mnt/elastic-internal/scripts/prepare-fs.sh
Limits:
cpu: 100m
memory: 50Mi
Requests:
cpu: 100m
memory: 50Mi
Environment:
POD_IP: (v1:status.podIP)
POD_NAME: (v1:metadata.name)
POD_IP: (v1:status.podIP)
POD_NAME: (v1:metadata.name)
Mounts:
/mnt/elastic-internal/downward-api from downward-api (ro)
/mnt/elastic-internal/elasticsearch-bin-local from elastic-internal-elasticsearch-bin-local (rw)
/mnt/elastic-internal/elasticsearch-config from elastic-internal-elasticsearch-config (ro)
/mnt/elastic-internal/elasticsearch-config-local from elastic-internal-elasticsearch-config-local (rw)
/mnt/elastic-internal/elasticsearch-plugins-local from elastic-internal-elasticsearch-plugins-local (rw)
/mnt/elastic-internal/probe-user from elastic-internal-probe-user (ro)
/mnt/elastic-internal/scripts from elastic-internal-scripts (ro)
/mnt/elastic-internal/transport-certificates from elastic-internal-transport-certificates (ro)
/mnt/elastic-internal/unicast-hosts from elastic-internal-unicast-hosts (ro)
/mnt/elastic-internal/xpack-file-realm from elastic-internal-xpack-file-realm (ro)
/usr/share/elasticsearch/config/http-certs from elastic-internal-http-certificates (ro)
/usr/share/elasticsearch/data from elasticsearch-data (rw)
/usr/share/elasticsearch/logs from elasticsearch-logs (rw)
Containers:
elasticsearch:
Image: docker.elastic.co/elasticsearch/elasticsearch:7.6.0
Ports: 9200/TCP, 9300/TCP
Host Ports: 0/TCP, 0/TCP
Limits:
memory: 2Gi
Requests:
memory: 2Gi
Readiness: exec [bash -c /mnt/elastic-internal/scripts/readiness-probe-script.sh] delay=10s timeout=5s period=5s #success=1 #failure=3
Environment:
HEADLESS_SERVICE_NAME: quickstart-es-default
NSS_SDB_USE_CACHE: no
POD_IP: (v1:status.podIP)
POD_NAME: (v1:metadata.name)
PROBE_PASSWORD_PATH: /mnt/elastic-internal/probe-user/elastic-internal-probe
PROBE_USERNAME: elastic-internal-probe
READINESS_PROBE_PROTOCOL: https
Mounts:
/mnt/elastic-internal/downward-api from downward-api (ro)
/mnt/elastic-internal/elasticsearch-config from elastic-internal-elasticsearch-config (ro)
/mnt/elastic-internal/probe-user from elastic-internal-probe-user (ro)
/mnt/elastic-internal/scripts from elastic-internal-scripts (ro)
/mnt/elastic-internal/unicast-hosts from elastic-internal-unicast-hosts (ro)
/mnt/elastic-internal/xpack-file-realm from elastic-internal-xpack-file-realm (ro)
/usr/share/elasticsearch/bin from elastic-internal-elasticsearch-bin-local (rw)
/usr/share/elasticsearch/config from elastic-internal-elasticsearch-config-local (rw)
/usr/share/elasticsearch/config/http-certs from elastic-internal-http-certificates (ro)
/usr/share/elasticsearch/config/transport-certs from elastic-internal-transport-certificates (ro)
/usr/share/elasticsearch/data from elasticsearch-data (rw)
/usr/share/elasticsearch/logs from elasticsearch-logs (rw)
/usr/share/elasticsearch/plugins from elastic-internal-elasticsearch-plugins-local (rw)
Volumes:
downward-api:
Type: DownwardAPI (a volume populated by information about the pod)
Items:
metadata.labels -> labels
elastic-internal-elasticsearch-bin-local:
Type: EmptyDir (a temporary directory that shares a pod's lifetime)
Medium:
SizeLimit: <unset>
elastic-internal-elasticsearch-config:
Type: Secret (a volume populated by a Secret)
SecretName: quickstart-es-default-es-config
Optional: false
elastic-internal-elasticsearch-config-local:
Type: EmptyDir (a temporary directory that shares a pod's lifetime)
Medium:
SizeLimit: <unset>
elastic-internal-elasticsearch-plugins-local:
Type: EmptyDir (a temporary directory that shares a pod's lifetime)
Medium:
SizeLimit: <unset>
elastic-internal-http-certificates:
Type: Secret (a volume populated by a Secret)
SecretName: quickstart-es-http-certs-internal
Optional: false
elastic-internal-probe-user:
Type: Secret (a volume populated by a Secret)
SecretName: quickstart-es-internal-users
Optional: false
elastic-internal-scripts:
Type: ConfigMap (a volume populated by a ConfigMap)
Name: quickstart-es-scripts
Optional: false
elastic-internal-transport-certificates:
Type: Secret (a volume populated by a Secret)
SecretName: quickstart-es-transport-certificates
Optional: false
elastic-internal-unicast-hosts:
Type: ConfigMap (a volume populated by a ConfigMap)
Name: quickstart-es-unicast-hosts
Optional: false
elastic-internal-xpack-file-realm:
Type: Secret (a volume populated by a Secret)
SecretName: quickstart-es-xpack-file-realm
Optional: false
elasticsearch-data:
Type: PersistentVolumeClaim (a reference to a PersistentVolumeClaim in the same namespace)
ClaimName: claim-name-placeholder
ReadOnly: false
elasticsearch-logs:
Type: EmptyDir (a temporary directory that shares a pod's lifetime)
Medium:
SizeLimit: <unset>
Volume Claims:
Name: elasticsearch-data
StorageClass:
Labels: <none>
Annotations: <none>
Capacity: 1Gi
Access Modes: [ReadWriteOnce]
Events:
Type Reason Age From Message
---- ------ ---- ---- -------
Normal SuccessfulCreate 58m statefulset-controller create Claim elasticsearch-data-quickstart-es-default-0 Pod quickstart-es-default-0 in StatefulSet quickstart-es-default success
Normal SuccessfulCreate 35m statefulset-controller create Claim elasticsearch-data-quickstart-es-default-1 Pod quickstart-es-default-1 in StatefulSet quickstart-es-default success
Normal SuccessfulCreate 35m statefulset-controller create Pod quickstart-es-default-1 in StatefulSet quickstart-es-default successful
Normal SuccessfulDelete 30m (x3 over 47m) statefulset-controller delete Pod quickstart-es-default-0 in StatefulSet quickstart-es-default successful
Normal SuccessfulDelete 30m statefulset-controller delete Pod quickstart-es-default-1 in StatefulSet quickstart-es-default successful
Normal SuccessfulCreate 30m (x4 over 58m) statefulset-controller create Pod quickstart-es-default-0 in StatefulSet quickstart-es-default successful