We have a pod with an app container and istio-proxy container running. Due to the failure of liveliness and readiness probes on the app container, a SIGTERM signal was sent to the app container, and the app container went to a terminating state. We have a prestop hook written on the app container to check for active connections and if active connections are 0 it will stop the server and stop the container.
During this terminating phase of the app container, technically speaking istio-proxy container should stop sending new traffic to the terminating container and redirect new traffic to other pods. But in this scenario even when the app container is in a terminating state istio-proxy is sending traffic to this terminating container. This is leading to a state where terminationGracePeriodSeconds is reached and executing requests are sent 503.
we tried reducing the timeperiod in prestop hook, didnt het any clue on how to let istio-proxy know that app container is terminating and dont send new traffic to it untill it starts and becomes active again.
below is the pos yaml for reference.
apiVersion: v1
kind: Pod
metadata:
labels:
app: netsuite-webservices-adaptor
name: netsuite-webservices-adaptor-free-67cfc4985d-w524p
namespace: io
spec:
containers:
- image: netsuite-webservices-adaptor:ml-0.1.1.92.0
imagePullPolicy: IfNotPresent
lifecycle:
preStop:
exec:
command:
- sh
- '-c'
- >-
expected="0"; count=""; while ! [ "$count" == "$expected" ]; do
count=$(curl -XGET 'http://localhost:5000/openConnections');
sleep 1; done; curl -XGET 'http://localhost:5000/stopServer';
echo "Done";
livenessProbe:
failureThreshold: 1
httpGet:
path: /app-health/netsuite-webservices-adaptor/livez
port: 15020
scheme: HTTP
initialDelaySeconds: 20
periodSeconds: 1
successThreshold: 1
timeoutSeconds: 1
name: netsuite-webservices-adaptor
ports:
- containerPort: 5000
name: http
protocol: TCP
readinessProbe:
failureThreshold: 1
httpGet:
path: /app-health/netsuite-webservices-adaptor/readyz
port: 15020
scheme: HTTP
initialDelaySeconds: 20
periodSeconds: 1
successThreshold: 1
timeoutSeconds: 1
resources:
limits:
cpu: '1'
memory: 1Gi
requests:
cpu: 200m
memory: 512Mi
- args:
- proxy
- sidecar
- '--domain'
- $(POD_NAMESPACE).svc.cluster.local
- '--proxyLogLevel=warning'
- '--proxyComponentLogLevel=misc:error'
- '--log_output_level=default:info'
- '--concurrency'
- '2'
env:
- name: EXIT_ON_ZERO_ACTIVE_CONNECTIONS
value: 'true'
- name: JWT_POLICY
value: third-party-jwt
- name: PILOT_CERT_PROVIDER
value: istiod
- name: CA_ADDR
value: 'istiod.istio-system.svc:15012'
- name: POD_NAME
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: metadata.name
- name: POD_NAMESPACE
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: metadata.namespace
- name: INSTANCE_IP
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: status.podIP
- name: SERVICE_ACCOUNT
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: spec.serviceAccountName
- name: HOST_IP
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: status.hostIP
- name: PROXY_CONFIG
value: |
{}
- name: ISTIO_META_POD_PORTS
value: |-
[
{"name":"http","containerPort":5000,"protocol":"TCP"}
]
- name: ISTIO_META_APP_CONTAINERS
value: netsuite-webservices-adaptor
- name: ISTIO_META_CLUSTER_ID
value: Kubernetes
- name: ISTIO_META_INTERCEPTION_MODE
value: REDIRECT
- name: ISTIO_META_WORKLOAD_NAME
value: netsuite-webservices-adaptor-free
- name: ISTIO_META_OWNER
value: >-
kubernetes://apis/apps/v1/namespaces/io/deployments/netsuite-webservices-adaptor-free
- name: ISTIO_META_MESH_ID
value: cluster.local
- name: TRUST_DOMAIN
value: cluster.local
- name: ISTIO_KUBE_APP_PROBERS
value: >-
{"/app-health/netsuite-webservices-adaptor/livez":{"httpGet":{"path":"/livez","port":5000,"scheme":"HTTP"},"timeoutSeconds":1},"/app-health/netsuite-webservices-adaptor/readyz":{"httpGet":{"path":"/readyz","port":5000,"scheme":"HTTP"},"timeoutSeconds":1}}
image: 'docker.io/istio/proxyv2:1.15.3'
imagePullPolicy: IfNotPresent
name: istio-proxy
ports:
- containerPort: 15090
name: http-envoy-prom
protocol: TCP
readinessProbe:
failureThreshold: 30
httpGet:
path: /healthz/ready
port: 15021
scheme: HTTP
initialDelaySeconds: 1
periodSeconds: 2
successThreshold: 1
timeoutSeconds: 3
resources:
limits:
cpu: '2'
memory: 1Gi
requests:
cpu: 100m
memory: 128Mi
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
privileged: false
readOnlyRootFilesystem: true
runAsGroup: 1337
runAsNonRoot: true
runAsUser: 1337
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
volumeMounts:
- mountPath: /var/run/secrets/workload-spiffe-uds
name: workload-socket
- mountPath: /var/run/secrets/credential-uds
name: credential-socket
- mountPath: /var/run/secrets/workload-spiffe-credentials
name: workload-certs
- mountPath: /var/run/secrets/istio
name: istiod-ca-cert
- mountPath: /var/lib/istio/data
name: istio-data
- mountPath: /etc/istio/proxy
name: istio-envoy
- mountPath: /var/run/secrets/tokens
name: istio-token
- mountPath: /etc/istio/pod
name: istio-podinfo
- mountPath: /var/run/secrets/eks.amazonaws.com/serviceaccount
name: aws-iam-token
readOnly: true
- mountPath: /var/run/secrets/kubernetes.io/serviceaccount
name: kube-api-access-cbthq
readOnly: true
dnsPolicy: ClusterFirst
enableServiceLinks: true
initContainers:
- args:
- istio-iptables
- '-p'
- '15001'
- '-z'
- '15006'
- '-u'
- '1337'
- '-m'
- REDIRECT
- '-i'
- '*'
- '-x'
- ''
- '-b'
- '*'
- '-d'
- '15090,15021,15020'
- '--log_output_level=default:info'
image: 'docker.io/istio/proxyv2:1.15.3'
imagePullPolicy: IfNotPresent
name: istio-init
resources:
limits:
cpu: '2'
memory: 1Gi
requests:
cpu: 100m
memory: 128Mi
securityContext:
allowPrivilegeEscalation: false
capabilities:
add:
- NET_ADMIN
- NET_RAW
drop:
- ALL
privileged: false
readOnlyRootFilesystem: false
runAsGroup: 0
runAsNonRoot: false
runAsUser: 0
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
volumeMounts:
- mountPath: /var/run/secrets/eks.amazonaws.com/serviceaccount
name: aws-iam-token
readOnly: true
- mountPath: /var/run/secrets/kubernetes.io/serviceaccount
name: kube-api-access-cbthq
readOnly: true
preemptionPolicy: PreemptLowerPriority
priority: 0
restartPolicy: Always
schedulerName: default-scheduler
securityContext: {}
serviceAccount: netsuite-webservices-adaptor
serviceAccountName: netsuite-webservices-adaptor
terminationGracePeriodSeconds: 86400
tolerations:
- effect: NoExecute
key: node.kubernetes.io/not-ready
operator: Exists
tolerationSeconds: 300
- effect: NoExecute
key: node.kubernetes.io/unreachable
operator: Exists
tolerationSeconds: 300
- emptyDir: {}
name: workload-socket
- emptyDir: {}
name: credential-socket
- emptyDir: {}
name: workload-certs
- emptyDir:
medium: Memory
name: istio-envoy
- emptyDir: {}
name: istio-data
- downwardAPI:
defaultMode: 420
items:
- fieldRef:
apiVersion: v1
fieldPath: metadata.labels
path: labels
- fieldRef:
apiVersion: v1
fieldPath: metadata.annotations
path: annotations
name: istio-podinfo
- name: istio-token
projected:
defaultMode: 420
sources:
- serviceAccountToken:
audience: istio-ca
expirationSeconds: 43200
path: istio-token
- configMap:
defaultMode: 420
name: istio-ca-root-cert
name: istiod-ca-cert
- name: kube-api-access-cbthq
projected:
defaultMode: 420
sources:
- serviceAccountToken:
expirationSeconds: 3607
path: token
- configMap:
items:
- key: ca.crt
path: ca.crt
name: kube-root-ca.crt
- downwardAPI:
items:
- fieldRef:
apiVersion: v1
fieldPath: metadata.namespace
path: namespace```
