Add id template, avoid ECS override limits #364
GitHub Actions / Black
failed
Jul 27, 2023 in 0s
1 error
Black found 1 error
Annotations
github-actions / Black
/home/runner/work/veda-data-airflow/veda-data-airflow/dags/veda_data_pipeline/utils/s3_discovery.py#L73-L84
regex = asset_definition["regex"]
if re.match(regex, filename):
asset_type = asset_name
break
grouped_files.append(
- {"prefix": prefix, "filename": filename, "asset_type": asset_type, "item_id": item_id}
+ {
+ "prefix": prefix,
+ "filename": filename,
+ "asset_type": asset_type,
+ "item_id": item_id,
+ }
)
else:
print(f"Warning: skipping file. No id match found: {filename}")
# At this point, files are labeled with type and id. Now, group them by id
sorted_list = sorted(grouped_files, key=lambda x: x["item_id"])
Loading