My goal is to build an AWS Lambda Layer to supply the mariadb module for AWS Lambdas running python 3.8.
Following these instructions to do so, I need to install the mariadb module on an instance compatible with the AWS Lambda platform. Installing the module like so ...
python3 -m pip install mariadb --target lambda-layer/python/lib/python3.8/site-packages
... fails with error message ...
mariadb_config not found.
Please make sure, that MariaDB Connector/C is installed on your system, edit the
configuration file 'site.cfg' and set the 'mariadb_config'
noption, which should point to the mariadb_config utility.
What do I need to fix this?
I assume that installing 'MariaDB Connector/C' is a prequisite? But how do do this? and how to do it in a way that it can be packaged up into a zip that can be used to define an AWS Lambda Layer?
Some context
I ran this on an Amazon instance, which I believe is the same as the Lamba hosts, to wit "Amazon Linux AMI 2018.03.0 (HVM), SSD Volume Type - ami-02769748522663066". cat /etc/os-release returns ...
NAME="Amazon Linux AMI"
VERSION="2018.03"
ID="amzn"
ID_LIKE="rhel fedora"
VERSION_ID="2018.03"
PRETTY_NAME="Amazon Linux AMI 2018.03"
ANSI_COLOR="0;33"
CPE_NAME="cpe:/o:amazon:linux:2018.03:ga"
HOME_URL="http://aws.amazon.com/amazon-linux-ami/"
Would a different instance AMI be better for packaging up a python3.8 Lambda Layer?
And I setup python3 with this (The box does not have python 3.8 in its gallery):
sudo yum install python36 -y
Update and self-answer
So the short answer is that, at the present, this is not possible. The MariaDb Connector does not support the Amazon flavour of linux. However, there is an alternate python library, PyMySQL, will connect to mariadb servers.
Here is the content of a python3.8 AWS Lambda function which manufactures AWS Function Layers for Lambda.
The Factory (AWS Lambda) Function
import json, boto3
from pathlib import Path
import os, sys, subprocess, shutil
from zipfile import ZipFile
def run( commandLine, captures, workingDir):
options = {}
if workingDir is not None:
options['cwd'] = workingDir
if 'standard-output' in captures:
options['stdout'] = subprocess.PIPE
if 'error-output' in captures:
options['stderr'] = subprocess.PIPE
proc = subprocess.run( commandLine, **options)
outputLines = None
errorLines = None
if 'standard-output' in captures:
outputLines = proc.stdout.decode('UTF-8').strip().split( os.linesep)
if 'error-output' in captures:
errorLines = proc.stderr.decode('UTF-8').strip().split( os.linesep)
if outputLines == ['']:
outputLines = None
if errorLines == ['']:
errorLines = None
return proc.returncode, outputLines, errorLines
def lambda_handler(event, context):
request = json.loads( event['body'])
sitePackages = '/tmp/aws-lambda-layer/lambda-layer/python/lib/python3.8/site-packages'
Path( sitePackages).mkdir(parents=True, exist_ok=True)
lines = request['requirements']
print( lines)
bucket = request['bucket']
packageKey = request['key']
ExtraArgs = request.get('ExtraArgs')
layerName = request['layer-name']
licenceInfo = request.get('license-info')
requiresFN = '/tmp/aws-lambda-layer/requirements.txt'
origin = os.environ.get( 'ORIGIN', '*')
with open( requiresFN, 'w') as file:
file.writelines( lines)
returncode, outputLines, errorLines = run(
[sys.executable, '-m', 'pip', 'install', '-r', requiresFN, '--target', sitePackages],
['standard-output','error-output'], '/tmp/aws-lambda-layer')
base = '/tmp/aws-lambda-layer/lambda-layer'
zipFN = '/tmp/aws-lambda-layer/package.zip'
l = len( base) + 1
with ZipFile( zipFN, 'w') as package:
for folderName, subfolders, filenames in os.walk( base):
for filename in filenames:
filePath = os.path.join( folderName, filename)
path = filePath[l:]
if os.path.basename( filePath) != 'yum.log':
package.write( filePath, path)
if ExtraArgs is None:
ExtraArgs = {}
ExtraArgs['ContentType'] = 'application/zip'
runtime_region = os.environ.get('AWS_REGION')
parms = {}
if runtime_region is not None:
parms['region_name'] = runtime_region
s3Client = boto3.client( 's3' , **parms)
lambdaClient = boto3.client( 'lambda', **parms)
s3Client.upload_file( zipFN, bucket, packageKey, ExtraArgs)
try:
shutil.rmtree( '/tmp/aws-lambda-layer', ignore_errors=True)
except:
print( 'Errors in cleaning up /tmp/aws-lambda-layer')
description = 'python 3.8 requirements:' + '\n'.join( lines)
parms = {'LayerName' : layerName,
'Description': description,
'Content': { 'S3Bucket': bucket, 'S3Key': packageKey},
'CompatibleRuntimes': ['python3.8']}
if licenceInfo is not None:
parms['LicenseInfo'] = licenceInfo
try:
response = lambdaClient.publish_layer_version( **parms)
response = response['layer']['LayerVersionArn']
except:
print('Error in publishing.')
response = None
try:
lambdaClient.delete_object( Bucket=bucket, Key=packageKey)
except:
print( 'Failure to delete s3 staging package.')
statusCode = 200
response = {'pip-return-code': returncode, 'out': outputLines,
'err': errorLines, 'layer': response}
ret = {'statusCode': statusCode,
'body': json.dumps( response),
'headers': {
'Content-Type': 'application/json',
'Access-Control-Allow-Credentials': 'true',
'Access-Control-Allow-Origin': origin}}
return ret
On the client side, this can be called (python script again) like so ... The factory function function takes the python pip freeze requirements and manufactures a Layer than serves the specifies modules. In my case, requirements.txt will include PyMySQL .
import json
import boto3
# Config section. Adapt as required.
lambdaRegion = 'ap-southeast-2'
bucket = 'some-bucket'
key = 'provisioning-workspace/layer-package.zip'
layerName = 'someLayerName'
filenm = 'some-path\\requirements.txt'
functionName = 'layer-factory-p38'
# End config section.
parms = {}
if lambdaRegion is not None:
parms['region_name'] = lambdaRegion
client = boto3.client( 'lambda', **parms)
def invoke( functionName, qualifier, inData):
parms = {'FunctionName' : functionName,
'InvocationType': 'RequestResponse',
'LogType' : 'None',
'Payload' : str.encode( json.dumps( {'body': json.dumps( inData)}))}
if qualifier is not None:
parms['Qualifier'] = str( qualifier)
response = client.invoke( **parms)
response = json.loads( response['Payload'].read().decode('utf-8'))
try:
statusCode = int( response['statusCode'])
except:
statusCode = 0
try:
response = json.loads( response['body'])
except:
print( response)
response = {}
return statusCode, response
def buildLayer( factoryFunctionName, requirementsFN, layerName, bucket, key):
with open( requirementsFN, 'r') as file:
lines = file.readlines()
statusCode, response = invoke( factoryFunctionName, None, {
'requirements': lines, 'bucket': bucket, 'key': key, 'layer-name': layerName})
return response['layer']
layerArn = buildLayer( functionName, filenm, layerName, bucket, key)
print( layerArn)
