I have created network
docker network create app-tier --driver bridge
and used this docker compose file
networks:
default:
external:
name: app-tier
services:
minio:
image: 'bitnami/minio:latest'
container_name: my-minio-server
environment:
- MINIO_ROOT_USER=theroot
- MINIO_ROOT_PASSWORD=theroot123
ports:
- '9000:9000'
- '9001:9001'
volumes:
- ${HOME}/minio/data:/data
spark:
image: docker.io/bitnami/spark:3
environment:
- SPARK_MODE=master
- SPARK_RPC_AUTHENTICATION_ENABLED=no
- SPARK_RPC_ENCRYPTION_ENABLED=no
- SPARK_LOCAL_STORAGE_ENCRYPTION_ENABLED=no
- SPARK_SSL_ENABLED=no
ports:
- '8080:8080'
- '7077:7077'
volumes:
- ./conf/spark-defaults.conf:/opt/bitnami/spark/conf/spark-defaults.conf
spark-worker1:
image: docker.io/bitnami/spark:3
links:
- "spark:spark"
environment:
- SPARK_MODE=worker
- SPARK_MASTER_URL=spark://spark:7077
- SPARK_WORKER_MEMORY=1G
- SPARK_WORKER_CORES=1
- SPARK_RPC_AUTHENTICATION_ENABLED=no
- SPARK_RPC_ENCRYPTION_ENABLED=no
- SPARK_LOCAL_STORAGE_ENCRYPTION_ENABLED=no
- SPARK_SSL_ENABLED=no
ports:
- '7181:8081'
volumes:
- ./work1:/opt/bitnami/spark/work
- ./conf/spark-defaults.conf:/opt/bitnami/spark/conf/spark-defaults.conf
spark-worker2:
image: docker.io/bitnami/spark:3
links:
- "spark:spark"
environment:
- SPARK_MODE=worker
- SPARK_MASTER_URL=spark://spark:7077
- SPARK_WORKER_MEMORY=1G
- SPARK_WORKER_CORES=1
- SPARK_RPC_AUTHENTICATION_ENABLED=no
- SPARK_RPC_ENCRYPTION_ENABLED=no
- SPARK_LOCAL_STORAGE_ENCRYPTION_ENABLED=no
- SPARK_SSL_ENABLED=no
ports:
- '7182:8082'
volumes:
- ./work2:/opt/bitnami/spark/work
- ./conf/spark-defaults.conf:/opt/bitnami/spark/conf/spark-defaults.conf
I connected to minio at http://127.0.0.1:9001 with the above credentials and I created a service account and an "asiatrip" bucket.
It has the following
s3accessKeyAws = "n1Z8USynE2uOBJmc"
s3secretKeyAws = "RjK4uL35tFNTROo2WsPVZhA77AJ5qJEx"
I can successfully connect to it via the minio client
docker run -it --rm --name minio-client \
--env MINIO_SERVER_HOST="my-minio-server" \
--env MINIO_SERVER_ACCESS_KEY="theroot" \
--env MINIO_SERVER_SECRET_KEY="theroot123" \
--network app-tier --volume $HOME/mcconf:/.mc \
bitnami/minio-client alias set minio http://my-minio-server:9000 n1Z8USynE2uOBJmc RjK4uL35tFNTROo2WsPVZhA77AJ5qJEx --api S3v4
and
docker run -it --rm --name minio-client \
--env MINIO_SERVER_HOST="my-minio-server" \
--env MINIO_SERVER_ACCESS_KEY="theroot" \
--env MINIO_SERVER_SECRET_KEY="theroot123" \
--network app-tier --volume $HOME/mcconf:/.mc \
bitnami/minio-client ls minio
I also can use minio via a docker jupyter in that network
docker run -it --network app-tier -p 8888:8888 jupyter/scipy-notebook:latest
after installing minio package with
!pip install minio
and execute python script
from minio import Minio
from minio.error import S3Error
client = Minio(
"my-minio-server:9000",
access_key="n1Z8USynE2uOBJmc",
secret_key="RjK4uL35tFNTROo2WsPVZhA77AJ5qJEx",
secure=False,
)
# Make 'asiatrip' bucket if not exist.
found = client.bucket_exists("asiatrip")
if not found:
client.make_bucket("asiatrip")
else:
print("Bucket 'asiatrip' already exists")
list(client.list_objects("asiatrip"))
So everything seems set
I installed hadoop-3.3.2 and spark-3.2.1-bin-without-hadoop
I setup my env as follows
export HADOOP_HOME=$HOME/Downloads/hadoop-3.3.2
export SPARK_HOME=$HOME/Downloads/spark-3.2.1-bin-without-hadoop
export PATH=$SPARK_HOME/bin:$HADOOP_HOME/bin:$PATH
export HADOOP_OPTIONAL_TOOLS="hadoop-aws"
export SPARK_DIST_CLASSPATH=$(hadoop classpath)
when I run this python file as
from pyspark.sql import SparkSession
spark = SparkSession\
.builder\
.appName("Test json")\
.getOrCreate()
s3accessKeyAws = "n1Z8USynE2uOBJmc"
s3secretKeyAws = "RjK4uL35tFNTROo2WsPVZhA77AJ5qJEx"
connectionTimeOut = "1000"
s3endPointLoc = "http://127.0.0.1:9000"
sourceBucket = "asiatrip"
spark.sparkContext._jsc.hadoopConfiguration().set("fs.s3a.endpoint", s3endPointLoc)
spark.sparkContext._jsc.hadoopConfiguration().set("fs.s3a.access.key", s3accessKeyAws)
spark.sparkContext._jsc.hadoopConfiguration().set("fs.s3a.secret.key", s3secretKeyAws)
spark.sparkContext._jsc.hadoopConfiguration().set("fs.s3a.connection.timeout", connectionTimeOut)
spark.sparkContext._jsc.hadoopConfiguration().set("spark.sql.debug.maxToStringFields", "100")
spark.sparkContext._jsc.hadoopConfiguration().set("fs.s3a.path.style.access", "true")
spark.sparkContext._jsc.hadoopConfiguration().set("fs.s3a.impl", "org.apache.hadoop.fs.s3a.S3AFileSystem")
spark.sparkContext._jsc.hadoopConfiguration().set("fs.s3a.connection.ssl.enabled", "false")
inputPath = f"s3a://{sourceBucket}/addresses.csv"
outputPath = f"s3a://{sourceBucket}/output_survey.csv"
df = spark.read.option("header", "true").format("s3selectCSV").csv(inputPath)
df.write.mode("overwrite").parquet(outputPath)
spark.stop()
as
spark-submit miniospark.py
it works fine for the addresses.csv file
a,b
1,2
3,4
6,7
8,9
in asiatrip bucket.
When I submit as
spark-submit --master spark://127.0.0.1:7077 miniospark.py
with
s3endPointLoc = "http://my-minio-server:9000"
It gives up after some time because it cannot resolve my-minio-server.
2022-05-18 15:12:32,246 WARN streaming.FileStreamSink: Assume no metadata directory. Error while looking for metadata directory in the path: s3a://asiatrip/addresses.csv.
org.apache.hadoop.fs.s3a.AWSClientIOException: getFileStatus on s3a://asiatrip/addresses.csv: com.amazonaws.SdkClientException: Unable to execute HTTP request: my-minio-server: nodename nor servname provided, or not known: Unable to execute HTTP request: my-minio-server: nodename nor servname provided, or not known
I am on a Mac x64 with Docker Desktop
