# AUTOGENERATED! DO NOT EDIT! File to edit: 01_s3_data_utils.ipynb (unless otherwise specified).

__all__ = ['read_csv_from_s3']

# Cell
import io
import boto3
import pandas as pd
def read_csv_from_s3(source_data_bucket_name, source_data_path):
    prefix_objs = list(boto3.resource('s3').Bucket(source_data_bucket_name).objects.filter(Prefix=''.join([source_data_path])))
    prefix_df = []

    for obj in prefix_objs:
        key = obj.key
        print('key: {}'.format(key))
        body = obj.get()['Body'].read()
        df = pd.read_csv(io.BytesIO(body), low_memory=False, chunksize=100000)

        li = []
        counter = 1

        # Process the chunk (DataFrame)
        for chunk in df:
            print('chunk no. {}'.format(counter))
            try:
                li.append(chunk)
            except Exception as e:
                print('Exception:', e)
            counter += 1

        data1 = pd.concat(li, axis=0, ignore_index=True, sort=True)

        prefix_df.append(data1)

        del data1  # Free-up memory.

    if len(prefix_objs) > 1:
        data = pd.concat(prefix_df, axis=0, ignore_index=True, sort=True)
    else:
        data = prefix_df[0]

    return data