I have setup loki-grafana using helm to check logs. I have created it few days back and everything was working fine, but now suddenly started facing the error saying CrashLoopBackOff with error-code 2 in statefulset pod called loki-stack.Can anyone please help solving this issue?
This is the last few logs of that pod
goroutine 183 [select]:
github.com/grafana/loki/pkg/storage/chunk/cache.(*FifoCache).runPruneJob(0xc00066a8f0, 0x0, 0xc0004eb620)
/src/loki/pkg/storage/chunk/cache/fifo_cache.go:213 +0xdf
created by github.com/grafana/loki/pkg/storage/chunk/cache.NewFifoCache
/src/loki/pkg/storage/chunk/cache/fifo_cache.go:202 +0x10f6
goroutine 189 [select]:
github.com/grafana/loki/pkg/storage/chunk/local.(*BoltIndexClient).loop(0xc00013f9f0)
/src/loki/pkg/storage/chunk/local/boltdb_index_client.go:82 +0x105
this is deployment file
apiVersion: apps/v1
kind: Deployment
metadata:
name: loki-stack-grafana
namespace: grafana-loki
uid: bf25a492-a0b2-40c2-9848-08931722e254
resourceVersion: '145239013'
generation: 4
creationTimestamp: '2022-06-08T06:45:57Z'
labels:
app.kubernetes.io/instance: loki-stack
app.kubernetes.io/managed-by: Helm
app.kubernetes.io/name: grafana
app.kubernetes.io/version: 8.3.5
helm.sh/chart: grafana-6.24.1
annotations:
deployment.kubernetes.io/revision: '4'
meta.helm.sh/release-name: loki-stack
meta.helm.sh/release-namespace: grafana-loki
managedFields:
- manager: helm
operation: Update
apiVersion: apps/v1
time: '2022-06-08T06:45:57Z'
fieldsType: FieldsV1
fieldsV1:
f:metadata:
f:annotations:
.: {}
f:meta.helm.sh/release-name: {}
f:meta.helm.sh/release-namespace: {}
f:labels:
.: {}
f:app.kubernetes.io/instance: {}
f:app.kubernetes.io/managed-by: {}
f:app.kubernetes.io/name: {}
f:app.kubernetes.io/version: {}
f:helm.sh/chart: {}
f:spec:
f:progressDeadlineSeconds: {}
f:replicas: {}
f:revisionHistoryLimit: {}
f:selector: {}
f:strategy:
f:rollingUpdate:
.: {}
f:maxSurge: {}
f:maxUnavailable: {}
f:type: {}
f:template:
f:metadata:
f:annotations:
.: {}
f:checksum/config: {}
f:checksum/dashboards-json-config: {}
f:checksum/sc-dashboard-provider-config: {}
f:checksum/secret: {}
f:labels:
.: {}
f:app.kubernetes.io/instance: {}
f:app.kubernetes.io/name: {}
f:spec:
f:automountServiceAccountToken: {}
f:containers:
k:{"name":"grafana"}:
.: {}
f:env:
.: {}
k:{"name":"GF_PATHS_DATA"}:
.: {}
f:name: {}
f:value: {}
k:{"name":"GF_PATHS_LOGS"}:
.: {}
f:name: {}
f:value: {}
k:{"name":"GF_PATHS_PLUGINS"}:
.: {}
f:name: {}
f:value: {}
k:{"name":"GF_PATHS_PROVISIONING"}:
.: {}
f:name: {}
f:value: {}
k:{"name":"GF_SECURITY_ADMIN_PASSWORD"}:
.: {}
f:name: {}
f:valueFrom:
.: {}
f:secretKeyRef:
.: {}
f:key: {}
f:name: {}
k:{"name":"GF_SECURITY_ADMIN_USER"}:
.: {}
f:name: {}
f:valueFrom:
.: {}
f:secretKeyRef:
.: {}
f:key: {}
f:name: {}
f:image: {}
f:imagePullPolicy: {}
f:livenessProbe:
.: {}
f:failureThreshold: {}
f:httpGet:
.: {}
f:path: {}
f:port: {}
f:scheme: {}
f:initialDelaySeconds: {}
f:periodSeconds: {}
f:successThreshold: {}
f:timeoutSeconds: {}
f:name: {}
f:ports:
.: {}
k:{"containerPort":80,"protocol":"TCP"}:
.: {}
f:containerPort: {}
f:name: {}
f:protocol: {}
k:{"containerPort":3000,"protocol":"TCP"}:
.: {}
f:containerPort: {}
f:name: {}
f:protocol: {}
f:readinessProbe:
.: {}
f:failureThreshold: {}
f:httpGet:
.: {}
f:path: {}
f:port: {}
f:scheme: {}
f:periodSeconds: {}
f:successThreshold: {}
f:timeoutSeconds: {}
f:resources: {}
f:terminationMessagePath: {}
f:terminationMessagePolicy: {}
f:volumeMounts:
.: {}
k:{"mountPath":"/etc/grafana/grafana.ini"}:
.: {}
f:mountPath: {}
f:name: {}
f:subPath: {}
k:{"mountPath":"/etc/grafana/provisioning/datasources"}:
.: {}
f:mountPath: {}
f:name: {}
k:{"mountPath":"/var/lib/grafana"}:
.: {}
f:mountPath: {}
f:name: {}
k:{"name":"grafana-sc-datasources"}:
.: {}
f:env:
.: {}
k:{"name":"FOLDER"}:
.: {}
f:name: {}
f:value: {}
k:{"name":"LABEL"}:
.: {}
f:name: {}
f:value: {}
k:{"name":"METHOD"}:
.: {}
f:name: {}
f:value: {}
k:{"name":"REQ_METHOD"}:
.: {}
f:name: {}
f:value: {}
k:{"name":"REQ_PASSWORD"}:
.: {}
f:name: {}
f:valueFrom:
.: {}
f:secretKeyRef:
.: {}
f:key: {}
f:name: {}
k:{"name":"REQ_URL"}:
.: {}
f:name: {}
f:value: {}
k:{"name":"REQ_USERNAME"}:
.: {}
f:name: {}
f:valueFrom:
.: {}
f:secretKeyRef:
.: {}
f:key: {}
f:name: {}
k:{"name":"RESOURCE"}:
.: {}
f:name: {}
f:value: {}
f:image: {}
f:imagePullPolicy: {}
f:name: {}
f:resources: {}
f:terminationMessagePath: {}
f:terminationMessagePolicy: {}
f:volumeMounts:
.: {}
k:{"mountPath":"/etc/grafana/provisioning/datasources"}:
.: {}
f:mountPath: {}
f:name: {}
f:dnsPolicy: {}
f:enableServiceLinks: {}
f:restartPolicy: {}
f:schedulerName: {}
f:securityContext:
.: {}
f:fsGroup: {}
f:runAsGroup: {}
f:runAsUser: {}
f:serviceAccount: {}
f:serviceAccountName: {}
f:terminationGracePeriodSeconds: {}
f:volumes:
.: {}
k:{"name":"config"}:
.: {}
f:configMap:
.: {}
f:defaultMode: {}
f:name: {}
f:name: {}
k:{"name":"sc-datasources-volume"}:
.: {}
f:emptyDir: {}
f:name: {}
k:{"name":"storage"}:
.: {}
f:emptyDir: {}
f:name: {}
- manager: node-fetch
operation: Update
apiVersion: apps/v1
time: '2022-06-20T09:09:18Z'
fieldsType: FieldsV1
fieldsV1:
f:spec:
f:template:
f:metadata:
f:annotations:
f:kubectl.kubernetes.io/restartedAt: {}
- manager: kube-controller-manager
operation: Update
apiVersion: apps/v1
time: '2022-06-20T10:02:09Z'
fieldsType: FieldsV1
fieldsV1:
f:metadata:
f:annotations:
f:deployment.kubernetes.io/revision: {}
f:status:
f:availableReplicas: {}
f:conditions:
.: {}
k:{"type":"Available"}:
.: {}
f:lastTransitionTime: {}
f:lastUpdateTime: {}
f:message: {}
f:reason: {}
f:status: {}
f:type: {}
k:{"type":"Progressing"}:
.: {}
f:lastTransitionTime: {}
f:lastUpdateTime: {}
f:message: {}
f:reason: {}
f:status: {}
f:type: {}
f:observedGeneration: {}
f:readyReplicas: {}
f:replicas: {}
f:updatedReplicas: {}
selfLink: /apis/apps/v1/namespaces/grafana-loki/deployments/loki-stack-grafana
status:
observedGeneration: 4
replicas: 1
updatedReplicas: 1
readyReplicas: 1
availableReplicas: 1
conditions:
- type: Available
status: 'True'
lastUpdateTime: '2022-06-08T06:46:27Z'
lastTransitionTime: '2022-06-08T06:46:27Z'
reason: MinimumReplicasAvailable
message: Deployment has minimum availability.
- type: Progressing
status: 'True'
lastUpdateTime: '2022-06-20T10:02:09Z'
lastTransitionTime: '2022-06-08T06:45:57Z'
reason: NewReplicaSetAvailable
message: ReplicaSet "loki-stack-grafana-7c5d9b4bd5" has successfully progressed.
spec:
replicas: 1
selector:
matchLabels:
app.kubernetes.io/instance: loki-stack
app.kubernetes.io/name: grafana
template:
metadata:
creationTimestamp: null
labels:
app.kubernetes.io/instance: loki-stack
app.kubernetes.io/name: grafana
annotations:
checksum/config: c9555bdb452894c521c85985f3b8a62b655d831921884d4a87c2159575c246d3
checksum/dashboards-json-config: 01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b
checksum/sc-dashboard-provider-config: 01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b
checksum/secret: cf6cd98e17ab9ab31c08e2d08258834e3a939d64366f351d1089fa4b9599c3ba
kubectl.kubernetes.io/restartedAt: '2022-06-20T10:01:43Z'
spec:
volumes:
- name: config
configMap:
name: loki-stack-grafana
defaultMode: 420
- name: storage
emptyDir: {}
- name: sc-datasources-volume
emptyDir: {}
containers:
- name: grafana-sc-datasources
image: quay.io/kiwigrid/k8s-sidecar:1.15.6
env:
- name: METHOD
value: WATCH
- name: LABEL
value: grafana_datasource
- name: FOLDER
value: /etc/grafana/provisioning/datasources
- name: RESOURCE
value: both
- name: REQ_USERNAME
valueFrom:
secretKeyRef:
name: loki-stack-grafana
key: admin-user
- name: REQ_PASSWORD
valueFrom:
secretKeyRef:
name: loki-stack-grafana
key: admin-password
- name: REQ_URL
value: http://localhost:3000/api/admin/provisioning/datasources/reload
- name: REQ_METHOD
value: POST
resources: {}
volumeMounts:
- name: sc-datasources-volume
mountPath: /etc/grafana/provisioning/datasources
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
imagePullPolicy: IfNotPresent
- name: grafana
image: grafana/grafana:8.3.5
ports:
- name: service
containerPort: 80
protocol: TCP
- name: grafana
containerPort: 3000
protocol: TCP
env:
- name: GF_SECURITY_ADMIN_USER
valueFrom:
secretKeyRef:
name: loki-stack-grafana
key: admin-user
- name: GF_SECURITY_ADMIN_PASSWORD
valueFrom:
secretKeyRef:
name: loki-stack-grafana
key: admin-password
- name: GF_PATHS_DATA
value: /var/lib/grafana/
- name: GF_PATHS_LOGS
value: /var/log/grafana
- name: GF_PATHS_PLUGINS
value: /var/lib/grafana/plugins
- name: GF_PATHS_PROVISIONING
value: /etc/grafana/provisioning
resources: {}
volumeMounts:
- name: config
mountPath: /etc/grafana/grafana.ini
subPath: grafana.ini
- name: storage
mountPath: /var/lib/grafana
- name: sc-datasources-volume
mountPath: /etc/grafana/provisioning/datasources
livenessProbe:
httpGet:
path: /api/health
port: 3000
scheme: HTTP
initialDelaySeconds: 60
timeoutSeconds: 30
periodSeconds: 10
successThreshold: 1
failureThreshold: 10
readinessProbe:
httpGet:
path: /api/health
port: 3000
scheme: HTTP
timeoutSeconds: 1
periodSeconds: 10
successThreshold: 1
failureThreshold: 3
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
imagePullPolicy: IfNotPresent
restartPolicy: Always
terminationGracePeriodSeconds: 30
dnsPolicy: ClusterFirst
serviceAccountName: loki-stack-grafana
serviceAccount: loki-stack-grafana
automountServiceAccountToken: true
securityContext:
runAsUser: 472
runAsGroup: 472
fsGroup: 472
schedulerName: default-scheduler
enableServiceLinks: true
strategy:
type: RollingUpdate
rollingUpdate:
maxUnavailable: 25%
maxSurge: 25%
revisionHistoryLimit: 10
progressDeadlineSeconds: 600
This the PVC
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: storage-loki-stack-0
namespace: grafana-loki
uid: efbcac70-785e-4ee5-bf40-2a869349606f
resourceVersion: '142888329'
creationTimestamp: '2022-06-08T06:30:21Z'
labels:
app: loki
release: loki-stack
annotations:
pv.kubernetes.io/bind-completed: 'yes'
pv.kubernetes.io/bound-by-controller: 'yes'
volume.beta.kubernetes.io/storage-provisioner: dobs.csi.digitalocean.com
finalizers:
- kubernetes.io/pvc-protection
managedFields:
- manager: kube-controller-manager
operation: Update
apiVersion: v1
time: '2022-06-08T06:30:25Z'
fieldsType: FieldsV1
fieldsV1:
f:metadata:
f:annotations:
.: {}
f:pv.kubernetes.io/bind-completed: {}
f:pv.kubernetes.io/bound-by-controller: {}
f:volume.beta.kubernetes.io/storage-provisioner: {}
f:labels:
.: {}
f:app: {}
f:release: {}
f:spec:
f:accessModes: {}
f:resources:
f:requests:
.: {}
f:storage: {}
f:volumeMode: {}
f:volumeName: {}
f:status:
f:accessModes: {}
f:capacity:
.: {}
f:storage: {}
f:phase: {}
selfLink: /api/v1/namespaces/grafana-loki/persistentvolumeclaims/storage-loki-stack-0
status:
phase: Bound
accessModes:
- ReadWriteOnce
capacity:
storage: 10Gi
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 10Gi
volumeName: pvc-efbcac70-785e-4ee5-bf40-2a869349606f
storageClassName: do-block-storage
volumeMode: Filesystem
This is PV
apiVersion: v1
kind: PersistentVolume
metadata:
name: pvc-efbcac70-785e-4ee5-bf40-2a869349606f
uid: c91f45dc-b0c2-4077-a76c-16c83256587c
resourceVersion: '142888376'
creationTimestamp: '2022-06-08T06:30:25Z'
annotations:
pv.kubernetes.io/provisioned-by: dobs.csi.digitalocean.com
finalizers:
- kubernetes.io/pv-protection
- external-attacher/dobs-csi-digitalocean-com
managedFields:
- manager: csi-provisioner
operation: Update
apiVersion: v1
time: '2022-06-08T06:30:25Z'
fieldsType: FieldsV1
fieldsV1:
f:metadata:
f:annotations:
.: {}
f:pv.kubernetes.io/provisioned-by: {}
f:spec:
f:accessModes: {}
f:capacity:
.: {}
f:storage: {}
f:claimRef:
.: {}
f:apiVersion: {}
f:kind: {}
f:name: {}
f:namespace: {}
f:resourceVersion: {}
f:uid: {}
f:csi:
.: {}
f:driver: {}
f:fsType: {}
f:volumeAttributes:
.: {}
f:storage.kubernetes.io/csiProvisionerIdentity: {}
f:volumeHandle: {}
f:persistentVolumeReclaimPolicy: {}
f:storageClassName: {}
f:volumeMode: {}
- manager: kube-controller-manager
operation: Update
apiVersion: v1
time: '2022-06-08T06:30:25Z'
fieldsType: FieldsV1
fieldsV1:
f:status:
f:phase: {}
- manager: csi-attacher
operation: Update
apiVersion: v1
time: '2022-06-08T06:30:41Z'
fieldsType: FieldsV1
fieldsV1:
f:metadata:
f:finalizers:
v:"external-attacher/dobs-csi-digitalocean-com": {}
selfLink: /api/v1/persistentvolumes/pvc-efbcac70-785e-4ee5-bf40-2a869349606f
status:
phase: Bound
spec:
capacity:
storage: 10Gi
csi:
driver: dobs.csi.digitalocean.com
volumeHandle: 7b1253ab-e6f4-11ec-9c18-0a58ac14b6a4
fsType: ext4
volumeAttributes:
storage.kubernetes.io/csiProvisionerIdentity: 1651923394559-8081-dobs.csi.digitalocean.com
accessModes:
- ReadWriteOnce
claimRef:
kind: PersistentVolumeClaim
namespace: grafana-loki
name: storage-loki-stack-0
uid: efbcac70-785e-4ee5-bf40-2a869349606f
apiVersion: v1
resourceVersion: '142888304'
persistentVolumeReclaimPolicy: Delete
storageClassName: do-block-storage
volumeMode: Filesystem
Let me know if anything else is needed. thank you
