如果您遇到了“AWS Glue Job - 无法将转换后的Glue数据传递给Lambda函数”的问题,以下是一个可能的解决方法,包含代码示例:
import boto3
import json
def create_lambda_permission(lambda_function_name, glue_role_arn):
client = boto3.client('lambda')
response = client.add_permission(
FunctionName=lambda_function_name,
StatementId='glue-lambda-permission',
Action='lambda:InvokeFunction',
Principal='glue.amazonaws.com',
SourceArn=glue_role_arn
)
return response
def invoke_lambda_function(lambda_function_name, payload):
client = boto3.client('lambda')
response = client.invoke(
FunctionName=lambda_function_name,
InvocationType='Event',
Payload=json.dumps(payload)
)
return response
import sys
from awsglue.transforms import *
from awsglue.utils import getResolvedOptions
from pyspark.context import SparkContext
from awsglue.context import GlueContext
from awsglue.job import Job
def main():
args = getResolvedOptions(sys.argv, ['JOB_NAME', 'LAMBDA_FUNCTION_NAME'])
job_name = args['JOB_NAME']
lambda_function_name = args['LAMBDA_FUNCTION_NAME']
sc = SparkContext()
glueContext = GlueContext(sc)
spark = glueContext.spark_session
job = Job(glueContext)
job.init(job_name, args)
# Glue Job的转换逻辑
# ...
# 设置Lambda函数的权限
glue_role_arn = job.get_job_run().get_role()
create_lambda_permission(lambda_function_name, glue_role_arn)
# 调用Lambda函数
payload = { 'key': 'value' } # 根据实际需求传递的数据
invoke_lambda_function(lambda_function_name, payload)
job.commit()
if __name__ == '__main__':
main()
以上是一个示例解决方法,它涵盖了设置Glue Job和Lambda函数的权限,并在Glue Job中调用Lambda函数的过程。您可以根据自己的需求进行调整和扩展。