AWS AppFlow create_flow for GA4

Question

I want to programmatically create a new AppFlow flow to transfer data from Google Analytics 4 to an S3 bucket. Getting an error:

"An error occurred (ValidationException) when calling the CreateFlow operation: ConnectorRuntimeSettings object are not required but found."

This is the code

response = client.create_flow(
    flowName='GA4_flow_test1',
    description='GA4 flow',
    triggerConfig={
        'triggerType': 'OnDemand'
    },
    sourceFlowConfig={
        'connectorType': 'CustomConnector',
        'apiVersion': 'v1beta',
        'connectorProfileName': 'my_connector',
        'sourceConnectorProperties': {
            'CustomConnector': {
                'entityName': 'Googleanalytics4',
                'customProperties': {
                     'object': 'core-reports'
                }
            }
           
        },
        'incrementalPullConfig': {
            'datetimeTypeFieldName': 'startDate'
        }
    },
    destinationFlowConfigList=[
        {
            'connectorType': 'S3',
            'destinationConnectorProperties': {
               
                'S3': {
                    'bucketName': 'my-bucket',
                    'bucketPrefix': 'ga4',
                    's3OutputFormatConfig': {
                        'fileType': 'JSON',
                        'aggregationConfig': {
                            'aggregationType': 'None'
                        }
                    }
                }
               
            }
        }
    ],
    tasks=[
        {
            "taskType": "Map",
           "sourceFields": ["Dimension:customUser:GroupId"],
           "destinationField": "customUser:GroupId"
        }
    ]
)

Looks like there is something wrong with tasks[] param. I tried different values for the taskType but nothing seems to work. Any ideas?

score 0 · Answer 1 · answered Jun 29 '23 at 17:03

You have to specify the full object id as EntityName. Attaching a working template for your reference:

    {
"AWSTemplateFormatVersion":"2010-09-09",
"Parameters" : {
    "FlowNameParameter" : {
        "Type" : "String"           
    },
    "ObjectParameter" : {
        "Type" : "String"
    },
    "BucketParameter" : {
        "Type" : "String"
    }
},
"Resources": {
    "TestFlow": {
        "Type": "AWS::AppFlow::Flow",
        "Properties": {
            "FlowName": { 
                "Ref" : "FlowNameParameter" 
            },
            "TriggerConfig": {
                "TriggerType": "Scheduled",
                "TriggerProperties": {
                    "ScheduleExpression": "rate(1days)",
                    "DataPullMode": "Incremental",
                    "ScheduleStartTime": 1688187601,
                    "TimeZone": "America/New_York",
                    "ScheduleOffset": 0
                }
            },
            "SourceFlowConfig": {
                "ConnectorType": "CustomConnector",
                "ApiVersion":"v1beta",
                "ConnectorProfileName": "%YourConnectorProfileName%",
                "SourceConnectorProperties": {
                    "CustomConnector": {
                        "EntityName": { "Ref" : "ObjectParameter" }
                    }
                }
            },
            "DestinationFlowConfigList": [
                {
                    "ConnectorType": "S3",
                    "DestinationConnectorProperties": {
                        "S3": {
                            "BucketName": { "Ref" : "BucketParameter" },
                            "BucketPrefix": "%YourPrefixIfAny%",
                            "S3OutputFormatConfig": {                                 
                                "PrefixConfig": {},
                                "AggregationConfig": {
                                    "AggregationType": "None"
                                },
                                "FileType": "PARQUET",
                                "PreserveSourceDataTyping": true
                            }
                        }
                    }
                }
            ],
            "Tasks": [{
                "SourceFields": ["country", "deviceCategory"],
                "ConnectorOperator": {
                    "CustomConnector": "PROJECTION"
                },
                "TaskType": "Filter"
            }, 
            {
                "SourceFields": ["country"],
                "ConnectorOperator": {
                    "CustomConnector": "NO_OP"
                },
                "DestinationField": "country",
                "TaskType": "Map"
            }, 
            {
                "SourceFields": ["deviceCategory"],
                "ConnectorOperator": {
                    "CustomConnector": "NO_OP"
                },
                "DestinationField": "deviceCategory",
                "TaskType": "Map"
            }
        }
    }
}

}

score 0 · Answer 2 · answered Jul 06 '23 at 17:14

I got it working, here is the full code



    import boto3
    import collections as ct
    
    client = boto3.client('appflow')
    
    dimensions = [
      # my dimensions
    ]
    
    # GA4 limits 9 dimenions per request
    dims = ct.defaultdict(list)
    for i, x in enumerate(dimensions):
        dims[i//9].append(x)
    
    for i in range(len(dims)):
      ga_tasks = [ 
            {
                'sourceFields': dims[i],
                'connectorOperator':{'CustomConnector': 'PROJECTION'},
                'taskType': 'Filter'
            }
        ]
      curr_dim = dims[i]
      for k in range(len(curr_dim)):
        ga_tasks.append({
            'sourceFields': [curr_dim[k]],
            'destinationField': curr_dim[k],
            'taskType': 'Map'
        })
          
      response = client.create_flow(
        flowName='GA4_flow_test' + str(i),
        description='GA4 flow',
        triggerConfig={
            'triggerType': 'OnDemand'
        },
        sourceFlowConfig={
            'connectorType': 'CustomConnector',
            'apiVersion': 'v1beta',
            'connectorProfileName': 'MY_CONNECTOR',
            'sourceConnectorProperties': {
                'CustomConnector': {
                    'entityName': 'core-reports/accounts/111/properties/111',
                }
               
            }
        },
        destinationFlowConfigList=[
            {
                'connectorType': 'S3',
                'destinationConnectorProperties': {
                   
                    'S3': {
                        'bucketName': 'my-bucket',
                        'bucketPrefix': 'ga4',
                        's3OutputFormatConfig': {
                            'fileType': 'JSON',
                            'aggregationConfig': {
                                'aggregationType': 'None'
                            }
                        }
                    }
                   
                }
            }
        ],
        tasks= ga_tasks
      )

AWS AppFlow create_flow for GA4

2 Answers2