# -------------------------------------------------------------------------
# Copyright (c) Switch Automation Pty Ltd. All rights reserved.
# Licensed under the MIT License. See License.txt in the project root for
# license information.
# --------------------------------------------------------------------------
"""
A module for integrating asset creation, asset updates, data ingestion, etc into the Switch Automation platform.
"""
import sys
import pandas
import pandera
import requests
import datetime
import logging
import uuid
from .._utils._platform import _get_structure, Blob
from .._utils._constants import api_prefix
from .._utils._utils import ApiInputs, _with_func_attrs, _column_name_cap, _work_order_schema
from ..integration._utils import _adx_support, _timezone_offsets

logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)
consoleHandler = logging.StreamHandler(stream=sys.stdout)
consoleHandler.setLevel(logging.INFO)

logger.addHandler(consoleHandler)
formatter = logging.Formatter('%(asctime)s  switch_api.%(module)s.%(funcName)s  %(levelname)s: %(message)s',
                              datefmt='%Y-%m-%dT%H:%M:%S')
consoleHandler.setFormatter(formatter)


@_with_func_attrs(df_required_columns=['InstallationCode', 'DeviceCode', 'DeviceName', 'SensorName', 'SensorTemplate',
                                       'SensorUnitOfMeasure', 'EquipmentClass', 'EquipmentLabel'])
def upsert_device_sensors(df: pandas.DataFrame, api_inputs: ApiInputs, tag_columns: list = None,
                          metadata_columns: list = None, save_additional_columns_as_slices: bool = False):
    """Upsert device(s) and sensor(s)

    Required fields are:

    - InstallationCode
    - DeviceCode
    - DeviceName
    - SensorName
    - SensorTemplate
    - SensorUnitOfMeasure
    - EquipmentClass
    - EquipmentLabel

    Parameters
    ----------
    df: pandas.DataFrame
        The asset register created by the driver including the minimum required set of columns.
    api_inputs : ApiInputs
        Object returned by initialize() function.
    tag_columns : list, default = None
        Columns of dataframe that contain tags.
    metadata_columns : list, default = None
        Column(s) of dataframe that contain device-level metadata.
    save_additional_columns_as_slices : bool, default = False
        Whether additional columns should be saved as slices (Default value = False).

    Returns
    -------
    tuple[str, pandas.DataFrame]
        (response, df) - Returns the response status and the dataframe containing the parsed response text.

    """
    data_frame = df.copy()

    if api_inputs.datacentre == '' or api_inputs.api_key == '':
        logger.error("You must call initialize() before using API.")
        return pandas.DataFrame()

    required_columns = ['InstallationCode', 'DeviceCode', 'DeviceName', 'SensorName', 'SensorTemplate',
                        'SensorUnitOfMeasure', 'EquipmentClass', 'EquipmentLabel']
    proposed_columns = data_frame.columns.tolist()

    if not set(required_columns).issubset(data_frame.columns):
        logger.exception('Missing required column(s): %s', set(required_columns).difference(proposed_columns))
        return 'Integration.upsert_device_sensors(): data_frame must contain the following columns: ' + ', '.join(
            required_columns)

    if tag_columns is not None and not set(tag_columns).issubset(data_frame.columns):
        logger.exception('Missing expected tag column(s): %s', set(tag_columns).difference(proposed_columns))
        return 'Integration.upsert_device_sensors(): data_frame expected to contain the following tag column(s): ' + \
               ', '.join(tag_columns)
    elif tag_columns is None:
        tag_columns = []

    if metadata_columns is not None and not set(metadata_columns).issubset(data_frame.columns):
        logger.exception('Missing expected metadata column(s): %s', set(metadata_columns).difference(proposed_columns))
        return 'Integration.upsert_device_sensors(): data_frame expected to contain the following metadata ' \
               'column(s): ' + ', '.join(metadata_columns)
    elif metadata_columns is None:
        metadata_columns = []

    slice_columns = set(proposed_columns).difference(set(required_columns)) - set(tag_columns) - set(metadata_columns)
    slices_data_frame = pandas.DataFrame()

    if len(slice_columns) > 0 or len(tag_columns) > 0 or len(metadata_columns) > 0:
        def update_values(row, mode):
            if mode == 'A':
                j_row = row[slice_columns].to_json()
                return str(j_row)
            elif mode == 'B':
                j_row = row[tag_columns].to_json()
                return str(j_row)
            else:
                j_row = row[metadata_columns].to_json()
                return str(j_row)

        data_frame['Slices'] = data_frame.apply(update_values, args="A", axis=1)
        data_frame['TagsJson'] = data_frame.apply(update_values, args="B", axis=1)
        data_frame['MetadataJson'] = data_frame.apply(update_values, args="C", axis=1)
        data_frame = data_frame.drop(columns=slice_columns)
        slices_data_frame = data_frame[['DeviceCode', 'Slices']]

    headers = {
        'x-functions-key': api_inputs.api_key,
        'Content-Type': 'application/json; charset=utf-8',
        'user-key': api_inputs.user_id
    }

    url = api_prefix + api_inputs.datacentre + "/" + api_inputs.api_project_id + "/Integration/DeviceSensors/Upsert"

    data_frame_grpd = data_frame.groupby(['InstallationCode', 'DeviceCode'])
    chunk_list = []
    for name, group in data_frame_grpd:
        logger.info('Upserting data for InstallationCode = %s and DeviceCode = %s', str(name[0]), str(name[1]))
        # logger.info('Sensor count to upsert: %s', str(group.shape[0]))

        response = requests.post(url, data=group.to_json(orient='records'), headers=headers)
        response_status = '{} {}'.format(response.status_code, response.reason)
        logger.info("Response status: %s", response_status)
        if response.status_code != 200:
            logger.error("API Call was not successful. Response Status: %s. Reason: %s.", response.status_code,
                         response.reason)
            chunk_list += [{'Chunk': name, 'DeviceCountToUpsert': 1, 'SensorCountToUpsert': str(group.shape[0]),
                            'response_status': response_status, 'response_df': pandas.DataFrame(),
                            'invalid_rows': pandas.DataFrame()}]
        elif len(response.text) == 0:
            logger.error('No data returned for this API call. %s', response.request.__dict__)
            chunk_list += [{'Chunk': name, 'DeviceCountToUpsert': 1, 'SensorCountToUpsert': str(group.shape[0]),
                            'response_status': response_status, 'response_df': pandas.DataFrame(),
                            'invalid_rows': pandas.DataFrame()}]
        elif response.status_code == 200 and len(response.text) > 0:
            response_data_frame = pandas.read_json(response.text)
            logger.info('Dataframe response row count = %s', str(response_data_frame.shape[0]))
            if response_data_frame.shape[1] > 0:
                response_data_frame = response_data_frame.assign(InstallationCode=str(name[0]),
                                                                 SensorCount=group.shape[0])
                invalid_rows = response_data_frame[response_data_frame['status'] != 'Ok']
                if invalid_rows.shape[0] > 0:
                    logger.error("The following rows contain invalid data: %s", invalid_rows)
                    chunk_list += [
                        {'Chunk': name, 'DeviceCountToUpsert': 1,
                         'SensorCountToUpsert': str(group.shape[0]),
                         'response_status': response_status,
                         'response_df': response_data_frame[response_data_frame['status'] == 'Ok'],
                         'invalid_rows': invalid_rows}]
                else:
                    chunk_list += [{'Chunk': name, 'DeviceCountToUpsert': 1,
                                    'SensorCountToUpsert': str(group.shape[0]),
                                    'response_status': response_status, 'response_df': response_data_frame,
                                    'invalid_rows': invalid_rows}]

    upsert_response_df = pandas.DataFrame()
    upsert_invalid_rows_df = pandas.DataFrame()
    upsert_response_status_list = []
    for i in range(len(chunk_list)):
        upsert_response_df = upsert_response_df.append(chunk_list[i]['response_df'])
        upsert_invalid_rows_df = upsert_invalid_rows_df.append(chunk_list[i]['invalid_rows'])
        upsert_response_status_list += [chunk_list[i]['response_status']]

    if save_additional_columns_as_slices and slices_data_frame.shape[0] > 0:
        slices_merged = pandas.merge(left=upsert_response_df, right=slices_data_frame, left_on='deviceCode',
                                     right_on='DeviceCode')
        slices_data_frame = slices_merged[['deviceId', 'Slices']]
        slices_data_frame = slices_data_frame.rename(columns={'deviceId': 'DeviceId'})
        upsert_data(slices_data_frame, api_inputs=api_inputs, key_columns=['DeviceId'], table_name='DeviceSensorSlices',
                    is_slices_table=True)

    _adx_support(api_inputs=api_inputs, build_sensors=True)
    upsert_response_df.columns = _column_name_cap(columns=upsert_response_df.columns)

    logger.info("Ingestion Complete. ")
    return upsert_response_status_list, upsert_response_df


@_with_func_attrs(df_required_columns=['InstallationName', 'InstallationCode', 'Address', 'Country', 'Suburb', 'State',
                                       'StateName', 'FloorAreaM2', 'ZipPostCode'],
                  df_optional_columns=['Latitude', 'Longitude', 'Timezone', 'InstallationId'])
def upsert_sites(df: pandas.DataFrame, api_inputs: ApiInputs, tag_columns: list = None,
                 save_additional_columns_as_slices: bool = False):
    """Upsert site(s).

    The `df` input must contain the following columns:
        - InstallationName
        - InstallationCode
        - Address
        - Suburb
        - State
        - StateName
        - Country
        - FloorAreaM2
        - ZipPostCode

    The following additional columns are optional:
        - Latitude
        - Longitude
        - Timezone
        - InstallationId
            - The UUID of the existing site within the Switch Automation platform.

    Parameters
    ----------
    df: pandas.DataFrame :
        The dataframe containing the sites to be created/updated in the Switch platform. All required columns must be
        present with no null values.
    api_inputs : ApiInputs
        Object returned by initialize() function.
    tag_columns : list, default=[]
        The columns containing site-level tags. The column header will be the tag group name. (Default value = True)
    save_additional_columns_as_slices : bool, default = False
        Whether any additional columns should be saved as slices. (Default value = False)

    Returns
    -------

    """
    data_frame = df.copy()
    # payload = {}
    headers = {
        'x-functions-key': api_inputs.api_key,
        'Content-Type': 'application/json; charset=utf-8',
        'user-key': api_inputs.user_id
    }

    if api_inputs.datacentre == '' or api_inputs.api_key == '':
        logger.error("You must call initialize() before using API.")
        return pandas.DataFrame()

    required_columns = ['InstallationName', 'InstallationCode', 'Address', 'Country', 'Suburb', 'State', 'StateName',
                        'FloorAreaM2', 'ZipPostCode']
    optional_columns = ['Latitude', 'Longitude', 'Timezone', 'InstallationId']
    proposed_columns = data_frame.columns.tolist()

    if not set(required_columns).issubset(proposed_columns):
        logger.exception('Missing required column(s): %s', set(required_columns).difference(proposed_columns))
        return 'Integration.upsert_sites() - data_frame must contain the following columns: ' + ', '.join(
            required_columns) + '. Optional Columns include: ' + ', '.join(optional_columns)

    if tag_columns is not None and not set(tag_columns).issubset(data_frame.columns):
        logger.exception('Missing expected tag column(s): %s', set(tag_columns).difference(proposed_columns))
        return 'Integration.upsert_device_sensors(): data_frame must contain the following tag columns: ' + ', '.join(
            tag_columns)
    elif tag_columns is None:
        tag_columns = []

    slice_columns = set(proposed_columns).difference(set(required_columns + optional_columns)) - set(tag_columns)
    slices_data_frame = pandas.DataFrame()

    if len(slice_columns) > 0 or len(tag_columns) > 0:
        def update_values(row, mode):
            if mode == 'A':
                j_row = row[slice_columns].to_json()
                return str(j_row)
            else:
                j_row = row[tag_columns].to_json()
                return str(j_row)

        data_frame['Slices'] = data_frame.apply(update_values, args=('A',), axis=1)
        data_frame['TagsJson'] = data_frame.apply(update_values, args=('B',), axis=1)

        data_frame = data_frame.drop(columns=slice_columns)
        slices_data_frame = data_frame[['InstallationCode', 'Slices']]

    url = api_prefix + api_inputs.datacentre + "/" + api_inputs.api_project_id + "/Integration/Sites/Upsert"
    response = requests.post(url, data=data_frame.to_json(orient='records'), headers=headers)

    response_data_frame = pandas.read_json(response.text)
    response_status = '{} {}'.format(response.status_code, response.reason)
    if response.status_code != 200:
        logger.error("API Call was not successful. Response Status: %s. Reason: %s.", response.status_code,
                     response.reason)
        return response_status, pandas.DataFrame()
    elif len(response.text) == 0:
        logger.error('No data returned for this API call. %s', response.request.__dict__)
        return response_status, pandas.DataFrame()

    if save_additional_columns_as_slices and slices_data_frame.shape[0] > 0:
        # print('Doing Slices: ' + str(save_additional_columns_as_slices) + ' - ' + str(slices_data_frame.shape[0]))
        slices_merged = pandas.merge(left=response_data_frame, right=slices_data_frame, left_on='installationCode',
                                     right_on='InstallationCode')
        slices_data_frame = slices_merged[['installationId', 'Slices']]
        slices_data_frame = slices_data_frame.rename(columns={'installationId': 'InstallationId'})
        upsert_data(slices_data_frame, api_inputs=api_inputs, key_columns=['InstallationId'],
                    table_name='InstallationSlices', is_slices_table=True)

    _adx_support(api_inputs=api_inputs, build_sites=True)
    logger.info("Ingestion complete. ")

    return 'Ok'


@_with_func_attrs(df_required_columns=['WorkOrderId', 'InstallationId', 'WorkOrderSiteIdentifier', 'Status',
                                       'RawStatus', 'Priority', 'RawPriority', 'WorkOrderCategory',
                                       'RawWorkOrderCategory', 'Type', 'Description', 'CreatedDate',
                                       'LastModifiedDate', 'WorkStartedDate', 'WorkCompletedDate', 'ClosedDate'],
                  df_optional_columns=['SubType', 'Vendor', 'VendorId', 'EquipmentClass', 'RawEquipmentClass',
                                       'EquipmentLabel', 'RawEquipmentId', 'TenantId', 'TenantName', 'NotToExceedCost',
                                       'TotalCost', 'BillableCost', 'NonBillableCost', 'Location', 'RawLocation',
                                       'ScheduledStartDate', 'ScheduledCompletionDate'])
def upsert_workorders(df: pandas.DataFrame, api_inputs: ApiInputs, save_additional_columns_as_slices: bool = False):
    """Upsert data to the Workorder table.

    The following columns are required to be present in the df:

    - ``WorkOrderId``: unique identifier for the work order instance
    - ``InstallationId``: the InstallationId (guid) used to uniquely identify a given site within the Switch platform
    - ``WorkOrderSiteIdentifier``: the work order provider's raw/native site identifier field
    - ``Status``: the status mapped to the Switch standard values defined by literal: `WORK_ORDER_STATUS`
    - ``RawStatus``: the work order provider's raw/native status
    - ``Priority``: the priority mapped to the Switch standard values defined by literal: `WORK_ORDER_PRIORITY`
    - ``RawPriority``: the work order provider's raw/native priority
    - ``WorkOrderCategory``: the category mapped to the Switch standard values defined by literal: `WORK_ORDER_CATEGORY`
    - ``RawWorkOrderCategory``: the work order provider's raw/native category
    - ``Type`` - work order type (as defined by provider) - e.g. HVAC - Too Hot, etc.
    - ``Description``: description of the work order.
    - ``CreatedDate``: the date the work order was created (Submitted status)
    - ``LastModifiedDate``: datetime the workorder was last modified
    - ``WorkStartedDate``: datetime work started on the work order (In Progress status)
    - ``WorkCompletedDate``: datetime work was completed for the work order (Resolved status)
    - ``ClosedDate``: datetime the workorder was closed (Closed status)

    The following columns are optional:

    - ``SubType``: the sub-type of the work order
    - ``Vendor``: the name of the vendor
    - ``VendorId``: the vendor id
    - ``EquipmentClass``: the Switch defined Equipment Class mapped from the work order provider's definition
    - ``RawEquipmentClass``: the work order provider's raw/native equipment class
    - ``EquipmentLabel``: the EquipmentLabel as defined within the Switch platform
    - ``RawEquipmentId``: the work order provider's raw/native equipment identifier/label
    - ``TenantId``: the tenant id
    - ``TenantName``: the name of the tenant
    - ``NotToExceedCost``: the cost not to be exceeded for the given work order
    - ``TotalCost``: total cost of the work order
    - ``BillableCost``: the billable portion of the work order cost
    - ``NonBillableCost``: the non-billable portion of the work order cost.
    - ``Location``: the Location as defined within the Switch platform
    - ``RawLocation``: the work order provider's raw/native location definition
    - ``ScheduledStartDate``: datetime work was scheduled to start on the given work order
    - ``ScheduledCompletionDate``" datetime work was scheduled to be completed for the given work order


    Parameters
    ----------
    df: pandas.DataFrame
        Dataframe containing the work order data to be upserted.
    api_inputs: ApiInputs
        Object returned by initialize() function.
    save_additional_columns_as_slices : bool, default = False
         (Default value = False)

    Returns
    -------

    """

    data_frame = df.copy()

    if api_inputs.datacentre == '' or api_inputs.api_key == '':
        logger.error("You must call initialize() before using API.")
        return pandas.DataFrame()

    required_columns = ['WorkOrderId', 'InstallationId', 'WorkOrderSiteIdentifier', 'Status',
                        'RawStatus', 'Priority', 'RawPriority', 'WorkOrderCategory', 'RawWorkOrderCategory', 'Type',
                        'Description', 'CreatedDate', 'LastModifiedDate', 'WorkStartedDate', 'WorkCompletedDate',
                        'ClosedDate']
    optional_columns = ['SubType', 'Vendor', 'VendorId', 'EquipmentClass', 'RawEquipmentClass', 'EquipmentLabel',
                        'RawEquipmentId', 'TenantId', 'TenantName', 'NotToExceedCost', 'TotalCost', 'BillableCost',
                        'NonBillableCost', 'Location', 'RawLocation', 'ScheduledStartDate', 'ScheduledCompletionDate']

    req_cols = ', '.join(required_columns)
    proposed_columns = data_frame.columns.tolist()

    if not set(required_columns).issubset(proposed_columns):
        logger.exception('Missing required column(s): %s', set(required_columns).difference(proposed_columns))
        return 'Integration.upsert_workorder() - data_frame must contain the following columns: ' + req_cols

    try:
        _work_order_schema.validate(data_frame, lazy=True)
    except pandera.errors.SchemaErrors as err:
        logger.error('Errors present with columns in df provided.')
        logger.error(err.failure_cases)
        schema_error = err.failure_cases
        return schema_error

    slice_columns = set(proposed_columns).difference(set(required_columns + optional_columns))

    missing_optional_columns = set(optional_columns) - set(proposed_columns)
    for missing_column in missing_optional_columns:
        data_frame[missing_column] = ''

    if len(slice_columns) > 0 and save_additional_columns_as_slices is True:
        def update_values(row):
            j_row = row[slice_columns].to_json()
            return str(j_row)

        data_frame['Meta'] = data_frame.apply(update_values, axis=1)
        data_frame = data_frame.drop(columns=slice_columns)
    elif len(slice_columns) > 0 and save_additional_columns_as_slices is not True:
        data_frame = data_frame.drop(columns=slice_columns)
        data_frame['Meta'] = ''
    else:
        data_frame['Meta'] = ''

    # payload = {}
    headers = {
        'x-functions-key': api_inputs.api_key,
        'Content-Type': 'application/json; charset=utf-8',
        'user-key': api_inputs.user_id
    }

    logger.info("Upserting data to Workorders.")

    data_frame = data_frame.loc[:, ['WorkOrderId', 'InstallationId', 'WorkOrderSiteIdentifier', 'Status', 'Priority',
                                    'WorkOrderCategory', 'Type', 'Description', 'CreatedDate', 'LastModifiedDate',
                                    'WorkStartedDate', 'WorkCompletedDate', 'ClosedDate', 'RawPriority',
                                    'RawWorkOrderCategory', 'RawStatus', 'SubType', 'Vendor', 'VendorId',
                                    'EquipmentClass', 'RawEquipmentClass', 'EquipmentLabel', 'RawEquipmentId',
                                    'TenantId', 'TenantName', 'NotToExceedCost', 'TotalCost', 'BillableCost',
                                    'NonBillableCost', 'Location', 'RawLocation', 'ScheduledStartDate',
                                    'ScheduledCompletionDate', 'Meta']]

    upload_result = Blob.upload(api_inputs=api_inputs, data_frame=data_frame, name='WorkOrder',
                                api_project_id=api_inputs.api_project_id, is_timeseries=True)
    json_payload = {"path": upload_result[0], "fileCount": upload_result[1], "operation": "append",
                    "tableDef": _get_structure(data_frame),
                    "keyColumns": ["WorkOrderId"]
                    }

    url = api_prefix + api_inputs.datacentre + "/" + api_inputs.db_name + "/" + api_inputs.api_project_id + \
        "/WorkOrderOperation/"

    logger.info("Sending request to ingest %s files from %s", str(upload_result[1]), upload_result[0])
    response = requests.post(url, json=json_payload, headers=headers)
    response_status = '{} {}'.format(response.status_code, response.reason)
    if response.status_code != 200:
        logger.error("API Call was not successful. Response Status: %s. Reason: %s.", response.status_code,
                     response.reason)
        return response_status
    elif len(response.text) == 0:
        logger.error('No data returned for this API call. %s', response.request.__dict__)
        return response_status

    logger.info("Response status: %s", response_status)
    logger.info("Ingestion complete. ")
    return response.text


@_with_func_attrs(df_required_columns=['ObjectPropertyId', 'InstallationId', 'Timestamp', 'Value'])
def upsert_timeseries_ds(df: pandas.DataFrame, api_inputs: ApiInputs, is_local_time: bool = True,
                         save_additional_columns_as_slices: bool = False, data_feed_file_status_id: uuid.UUID = None):
    """Upserts to Timeseries_Ds table.

    The following columns are required to be present in the data_frame:
    - InstallationId
    - ObjectPropertyId
    - Timestamp
    - Value

    Parameters
    ----------
    df: pandas.DataFrame
        Dataframe containing the data to be appended to timeseries.
    api_inputs: ApiInputs
        Object returned by initialize() function.
    is_local_time : bool, default = True
         Whether the datetime values are in local time or UTC. If false, then UTC (Default value = True).
    save_additional_columns_as_slices : bool, default = False
         (Default value = False)
    data_feed_file_status_id : uuid.UUID, default = None
         Enables developer to identify upserted rows using during development. This data is posted to the
         DataFeedFileStatusId in the Timeseries_Ds table.

         Once deployed, the DataFeedFileStatusId field will contain a unique Guid which will assist in
         tracking upload results and logging.

    Returns
    -------

    """

    data_frame = df.copy()

    if api_inputs.datacentre == '' or api_inputs.api_key == '':
        logger.error("You must call initialize() before using API.")
        return pandas.DataFrame()

    required_columns = ['ObjectPropertyId', 'InstallationId', 'Timestamp', 'Value']
    req_cols = ', '.join(required_columns)
    proposed_columns = data_frame.columns.tolist()

    if not set(required_columns).issubset(proposed_columns):
        logger.exception('Missing required column(s): %s', set(required_columns).difference(proposed_columns))
        return 'integration.upsert_timeseries_ds() - data_frame must contain the following columns: ' + req_cols

    slice_columns = set(proposed_columns).difference(set(required_columns))

    if len(slice_columns) > 0 and save_additional_columns_as_slices is True:
        def update_values(row):
            j_row = row[slice_columns].to_json()
            return str(j_row)

        data_frame['Meta'] = data_frame.apply(update_values, axis=1)
        data_frame = data_frame.drop(columns=slice_columns)
    elif len(slice_columns) > 0 and save_additional_columns_as_slices is not True:
        data_frame = data_frame.drop(columns=slice_columns)
        data_frame['Meta'] = ''
    else:
        data_frame['Meta'] = ''

    if api_inputs.data_feed_file_status_id is not None and api_inputs.data_feed_file_status_id != '00000000-0000-0000' \
                                                                                                  '-0000-000000000000':
        data_frame['DataFeedFileStatusId'] = api_inputs.data_feed_file_status_id
    elif data_feed_file_status_id is not None:
        data_frame['DataFeedFileStatusId'] = data_feed_file_status_id
    else:
        data_frame['DataFeedFileStatusId'] = '00000000-0000-0000-0000-000000000000'

    site_list = data_frame['InstallationId'].unique().tolist()
    start_date = data_frame['Timestamp'].min(axis=0, skipna=True)
    end_date = data_frame['Timestamp'].max(axis=0, skipna=True)

    timezones = _timezone_offsets(api_inputs=api_inputs, date_from=start_date.date(), date_to=end_date.date(),
                                  installation_id_list=site_list)
    timezones['dateFrom'] = timezones['dateFrom'].apply(lambda x: pandas.to_datetime(x))
    timezones['dateTo'] = timezones['dateTo'].apply(lambda x: pandas.to_datetime(x))

    data_frame = data_frame.merge(timezones, left_on='InstallationId', right_on='installationId', how='inner')

    def in_range(row):
        j_row = (row['Timestamp'] >= row['dateFrom']) & (
                row['Timestamp'] < (row['dateTo'] + datetime.timedelta(days=1)))
        return str(j_row)

    data_frame['InDateRange'] = data_frame.apply(in_range, axis=1)
    data_frame = data_frame[data_frame['InDateRange'] == 'True']

    if is_local_time:
        def to_utc(row):
            if row['offsetToUtcMinutes'] >= 0:
                j_row = row['TimestampLocal'] - datetime.timedelta(minutes=row['offsetToUtcMinutes'])
            else:
                j_row = row['TimestampLocal'] + datetime.timedelta(minutes=abs(row['offsetToUtcMinutes']))
            return j_row

        data_frame = data_frame.assign(TimestampLocal=data_frame['Timestamp'])
        data_frame['Timestamp'] = data_frame.apply(to_utc, axis=1)
    elif not is_local_time:
        def from_utc(row):
            if row['offsetToUtcMinutes'] >= 0:
                j_row = row['Timestamp'] + datetime.timedelta(minutes=row['offsetToUtcMinutes'])
            else:
                j_row = row['Timestamp'] - datetime.timedelta(minutes=abs(row['offsetToUtcMinutes']))
            return j_row

        data_frame['TimestampLocal'] = data_frame.apply(from_utc, axis=1)

    def bin_to_15_minute_interval(row, date_col):
        if row[date_col].minute < 15:
            j_row = row[date_col].replace(minute=0)
            return j_row
        elif row[date_col].minute >= 15 & row[date_col].minute < 29:
            j_row = row[date_col].replace(minute=15)
            return j_row
        elif row[date_col].minute >= 30 & row[date_col].minute < 44:
            j_row = row[date_col].replace(minute=30)
            return j_row
        else:
            j_row = row[date_col].replace(minute=45)
            return j_row

    data_frame['TimestampId'] = data_frame.apply(bin_to_15_minute_interval, args=('Timestamp',), axis=1)
    data_frame['TimestampLocalId'] = data_frame.apply(bin_to_15_minute_interval, args=('TimestampLocal',), axis=1)
    data_frame = data_frame.drop(columns=['InDateRange', 'dateFrom', 'dateTo', 'installationId', 'offsetToUtcMinutes'])

    # payload = {}
    headers = {
        'x-functions-key': api_inputs.api_key,
        'Content-Type': 'application/json; charset=utf-8',
        'user-key': api_inputs.user_id
    }

    logger.info("Upserting data to Timeseries_Ds.")

    data_frame = data_frame.loc[:, ['ObjectPropertyId', 'Timestamp', 'TimestampId', 'TimestampLocal',
                                    'TimestampLocalId', 'Value', 'DataFeedFileStatusId', 'InstallationId', 'Meta']]

    name = 'Timeseries_Ds'

    upload_result = Blob.upload(api_inputs=api_inputs, data_frame=data_frame, name=name,
                                api_project_id=api_inputs.api_project_id, is_timeseries=True,
                                batch_id=data_feed_file_status_id)

    json_payload = {"path": upload_result[0], "fileCount": upload_result[1], "operation": "upsert",
                    "isLocalTime": is_local_time,
                    "tableDef": _get_structure(data_frame),
                    "keyColumns": ["ObjectPropertyId", "Timestamp"]
                    }

    # {'ObjectPropertyId': 'object', 'Timestamp': 'datetime64[ns]', 'Value': 'float64', 'InstallationId': 'object'}

    # ObjectPropertyId: string, Timestamp: datetime, TimestampId: datetime, TimestampLocal: datetime,
    # TimestampLocalId: datetime, Value: real, DataFeedFileStatusId: string, InstallationId: string, Meta: dynamic

    url = api_prefix + api_inputs.datacentre + "/" + api_inputs.db_name + "/" + api_inputs.api_project_id + \
        "/AdxTimeSeriesOperation/" + name

    logger.info("Sending request to ingest %s files from %s", str(upload_result[1]), upload_result[0])
    response = requests.post(url, json=json_payload, headers=headers)
    response_status = '{} {}'.format(response.status_code, response.reason)
    if response.status_code != 200:
        logger.error("API Call was not successful. Response Status: %s. Reason: %s.", response.status_code,
                     response.reason)
        return response_status
    elif len(response.text) == 0:
        logger.error('No data returned for this API call. %s', response.request.__dict__)
        return response_status

    logger.info("Response status: %s", response_status)
    logger.info("Ingestion complete. ")
    return response.text


def upsert_data(data_frame, api_inputs: ApiInputs, table_name: str, key_columns: list,
                is_slices_table=False):
    """Upsert data

    Upserts data to the `table_name` provided to the function call and uses the `key_columns` provided to determine the
    unique records.

    Parameters
    ----------
    data_frame : pandas.DataFrame
        Dataframe containing the data to be upserted.
    api_inputs : ApiInputs
        Object returned by initialize() function.
    table_name : str
        The name of the table where data will be upserted.
    key_columns : list
        The columns that determine a unique instance of a record. These are used to update records if new data is
        provided.
    is_slices_table : bool
         (Default value = False)

    Returns
    -------
    str
        Returns the text of the response from API.

    """
    # payload = {}
    headers = {
        'x-functions-key': api_inputs.api_key,
        'Content-Type': 'application/json; charset=utf-8',
        'user-key': api_inputs.user_id
    }

    if api_inputs.datacentre == '' or api_inputs.api_key == '':
        logger.error("You must call initialize() before using API.")
        return pandas.DataFrame()

    if len(key_columns) == 0 or key_columns is None:
        logger.error(
            "You must provide key_columns. This allows the Switch Automation platform to identify the rows to update.")
        return False

    logger.info("Data is being upserted for %s", table_name)

    table_structure = _get_structure(data_frame)
    if is_slices_table is True and 'Slices' in table_structure:
        table_structure['Slices'] = 'dynamic'

    # upload Blobs to folder
    upload_result = Blob.upload(api_inputs=api_inputs, data_frame=data_frame, name=table_name,
                                api_project_id=api_inputs.api_project_id, is_timeseries=False)
    json_payload = {"path": upload_result[0], "fileCount": upload_result[1], "operation": "upsert",
                    "tableDef": table_structure, "keyColumns": key_columns}

    url = api_prefix + api_inputs.datacentre + "/" + api_inputs.db_name + "/" + api_inputs.api_project_id + \
        "/AdxDataOperation/" + table_name
    logger.info("Sending request to ingest %s files from %s", str(upload_result[1]), upload_result[0])

    response = requests.post(url, json=json_payload, headers=headers)
    response_status = '{} {}'.format(response.status_code, response.reason)
    if response.status_code != 200:
        logger.error("API Call was not successful. Response Status: %s. Reason: %s.", response.status_code,
                     response.reason)
        return response_status
    elif len(response.text) == 0:
        logger.error('No data returned for this API call. %s', response.request.__dict__)
        return response_status

    logger.info("Response status: %s", response_status)
    logger.info("Ingestion complete. ")

    return response.text


def replace_data(data_frame, api_inputs: ApiInputs, table_name: str):
    """Replace data

    Replaces the data in the ``table_name`` provided to the function call.

    Parameters
    ----------
    data_frame : pandas.DataFrame
        Data frame to be used to replace the data in the `table_name` table.
    api_inputs : ApiInputs
        Object returned by initialize() function.
    table_name :
        The name of the table where data will be replaced.

    Returns
    -------
    str
        Returns the text of the response from API.

    """
    # payload = {}
    headers = {
        'x-functions-key': api_inputs.api_key,
        'Content-Type': 'application/json; charset=utf-8',
        'user-key': api_inputs.user_id
    }

    if api_inputs.datacentre == '' or api_inputs.api_key == '':
        logger.error("You must call initialize() before using API.")
        return pandas.DataFrame()

    logger.info("Replacing all data for %s", table_name)

    # upload Blobs to folder
    upload_result = Blob.upload(api_inputs=api_inputs, data_frame=data_frame, name=table_name,
                                api_project_id=api_inputs.api_project_id, is_timeseries=False)
    json_payload = {"path": upload_result[0], "fileCount": upload_result[1], "operation": "replace",
                    "tableDef": _get_structure(data_frame)}
    logger.info(json_payload)
    url = api_prefix + api_inputs.datacentre + "/" + api_inputs.db_name + "/" + api_inputs.api_project_id + \
        "/AdxDataOperation/" + table_name
    logger.info("Sending request to ingest %s files from %s", str(upload_result[1]), upload_result[0])

    response = requests.post(url, json=json_payload, headers=headers)
    response_status = '{} {}'.format(response.status_code, response.reason)
    if response.status_code != 200:
        logger.error("API Call was not successful. Response Status: %s. Reason: %s.", response.status_code,
                     response.reason)
        return response_status
    elif len(response.text) == 0:
        logger.error('No data returned for this API call. %s', response.request.__dict__)
        return response_status

    logger.info("Ingestion complete. ")

    return response.text


def append_data(data_frame, api_inputs: ApiInputs, table_name: str):
    """Append data.

    Appends data to the ``table_name`` provided to the function call.

    Parameters
    ----------
    data_frame : pandas.DataFrame
        Data to be appended.
    api_inputs : ApiInputs
        Object returned by initialize() function.
    table_name : str
        The name of the table where data will be appended.

    Returns
    -------
    str
        Returns the text of the response from API.

    """
    # payload = {}
    headers = {
        'x-functions-key': api_inputs.api_key,
        'Content-Type': 'application/json; charset=utf-8',
        'user-key': api_inputs.user_id
    }

    if api_inputs.datacentre == '' or api_inputs.api_key == '':
        logger.error("You must call initialize() before using API.")
        return pandas.DataFrame()

    logger.info("Appending data for %s", table_name)

    # upload Blobs to folder
    upload_result = Blob.upload(api_inputs=api_inputs, data_frame=data_frame, name=table_name,
                                api_project_id=api_inputs.api_project_id, is_timeseries=False)
    json_payload = {"path": upload_result[0], "fileCount": upload_result[1], "operation": "append",
                    "tableDef": _get_structure(data_frame)}

    url = api_prefix + api_inputs.datacentre + "/" + api_inputs.db_name + "/" + api_inputs.api_project_id + \
        "/AdxDataOperation/" + table_name

    logger.info("Sending request to ingest %s files from %s", str(upload_result[1]), upload_result[0])

    response = requests.post(url, json=json_payload, headers=headers)
    response_status = '{} {}'.format(response.status_code, response.reason)
    if response.status_code != 200:
        logger.error("API Call was not successful. Response Status: %s. Reason: %s.", response.status_code,
                     response.reason)
        return response_status
    elif len(response.text) == 0:
        logger.error('No data returned for this API call. %s', response.request.__dict__)
        return response_status

    logger.info("API Response: %s", response_status)
    logger.info("Ingestion complete. ")

    return response.text
