curl -XPOST 'http://node1:8998/sessions' -H "Content-Type:application/json" --data '{"kind":"spark"}'
A fully distributed cluster of three nodes. spark3.0.0.
livy.conf:
livy.spark.master = yarn
livy.spark.deployMode = cluster
livy.environment = production
livy.impersonation.enabled = true
livy.server.csrf_protection.enabled false
livy.server.port = 8998
livy.server.session.timeout = 3600000
livy.server.recovery.mode = recovery
livy.server.recovery.state-store=filesystem
livy.server.recovery.state-store.url=/tmp/livy
livy-snv.sh:
export JAVA_HOME=/export/server/jdk1.8.0_231
export HADOOP_HOME=/export/server/hadoop
export HADOOP_CONF_DIR=/export/server/hadoop/etc/hadoop
export SPARK_CONF_DIR=/export/server/spark/conf
export SPARK_HOME=/export/server/spark
export LIVY_LOG_DIR=/export/server/livy/log
export LIVY_PID_DIR=/export/server/livy/pid-dir
export LIVY_SERVER_JAVA_OPTS="-Xmx2g"
learn how to use livy, and slove this problem.