Here is my code: import boto3 import pandas as pd import json s3 = boto3.resource('s3') my_bucket = s3.Bucket('bucket-name-here') def verify_unique_col(): with open(r'unique_columns.json', 'r') as f: config = json.load(f) for my_bucket_object in my_bucket.objects.filter(Prefix='decrypted/'): if my_bucket_object.key.endswith('.csv'): filename = my_bucket_object.key.split('/')[-1] for table in config['Unique_Column_Combination']['Table_Name']: unique_col_comb = config['Unique_Column_Combination']['Table_Name'][f'{table}'] df = pd.read_csv(f's3://path/to/{filename}', sep='|') df_unique = df.set_index(unique_col_comb.split(", ")).index.is_unique print(df_unique) verify_unique_col() I am trying to iterate through each CSV file in my bucket and read...
A site where you can share knowledge