import json
import logging
import re
import urllib.parse
from abc import ABC, abstractmethod
from contextlib import contextmanager
from datetime import datetime
from pathlib import Path
from typing import Union, Optional, TypeVar, Any

import pandas as pd
import plotly.graph_objects as go
from powerbot_client import Signal, Trade, InternalTrade, OwnOrder
from pydantic import BaseModel, Field, validate_arguments, root_validator, validator
from sqlalchemy import create_engine
from sqlalchemy.exc import NoSuchModuleError, InterfaceError, ProgrammingError, OperationalError, TimeoutError
from sqlalchemy.orm import Session

import powerbot_backtesting as pb
from powerbot_backtesting.exceptions import SQLExporterError
from powerbot_backtesting.utils.constants import *

# Implement custom type for validation
pandas_DataFrame = TypeVar('pandas.core.frame.DataFrame')
ApiClient = TypeVar('powerbot_client.api_client.ApiClient')
HistoryApiClient = TypeVar('powerbot_backtesting.models.history_api_models.HistoryApiClient')

class BaseExporter(BaseModel, ABC):
    """
    Base class to all of PowerBot's exporter classes.
    """

    @abstractmethod
    def get_contract_ids(self, **kwargs) -> dict[str, list[str]]:
        """Acquire contracts from source as defined by specific exporter class."""

    @abstractmethod
    def get_public_trades(self, **kwargs) -> dict[str, pd.DataFrame]:
        """Acquire public trades from source as defined by specific exporter class."""

    @abstractmethod
    def get_contract_history(self, **kwargs) -> dict[str, pd.DataFrame]:
        """Acquire contract history from source as defined by specific exporter class."""

    @validate_arguments
    def get_ohlc_data(self,
                      trade_data: dict[str, pandas_DataFrame],
                      timesteps: int,
                      time_unit: str,
                      delivery_area: str = None,
                      use_cached_data: bool = True,
                      caching: bool = True,
                      gzip_files: bool = True,
                      one_file: bool = False) -> dict[str, pd.DataFrame]:
        """
        Converts trade data into Open-High-Low-Close format in the specified timesteps.

        Args:
            trade_data (dict{key: DataFrame}): Dictionary of Dataframes containing Contract Trade Data
            timesteps (int): Timesteps to group Trades by
            time_unit (str): Time units for timesteps (either hours, minutes or seconds)
            delivery_area (str): Area Code for Delivery Area (not needed when loading from historic cache)
            use_cached_data (bool): If True, function tries to load data from cache wherever possible
            caching (bool): True if data should be cached
            gzip_files (bool): True if cached files should be gzipped
            one_file (bool): True if data should be cached in a single JSON file

        Returns:
            dict{key: DataFrame}: Dictionary of DataFrames
        """
        return pb.get_ohlc_data(trade_data=trade_data,
                                timesteps=timesteps,
                                time_unit=time_unit,
                                delivery_area=delivery_area,
                                api_client=self.client if hasattr(self, 'client') else None,
                                use_cached_data=use_cached_data,
                                caching=caching,
                                gzip_files=gzip_files,
                                one_file=one_file)

    @validate_arguments
    def get_orderbooks(self,
                       contract_hist_data: dict[str, pandas_DataFrame],
                       delivery_area: str = None,
                       timesteps: int = 15,
                       time_unit: str = "minutes",
                       timestamp: list[datetime] = None,
                       from_timestamp: bool = False,
                       use_cached_data: bool = True,
                       caching: bool = True,
                       as_json: bool = False) -> dict[str, pd.DataFrame]:
        """
        Converts contract history data into order books in the specified timesteps. If no API client is passed, the function
        will automatically assume that the data is production data.

        Please be aware that optimally only data from one exchange at a time should be used (e.g. only EPEX).

        To generate specific order books for a position closing algorithm, the timestamp and from_timestamp parameters can
        be used.

        Args:
            contract_hist_data (dict{key: DataFrame}): Dictionary of Dataframes containing Contract History Data
            delivery_area (str): Area Code for Delivery Area (not needed when loading from historic cache)
            timesteps (int): Timesteps to group order books by
            time_unit (str): Time units for timesteps (either hours, minutes or seconds)
            timestamp (list[datetime]): List of timestamps to generate order books at/ from
            from_timestamp (bool): True if timestamp serves as starting point for order book generation
            use_cached_data (bool): If True, function tries to load data from cache wherever possible
            caching (bool): True if single order books should be cached as JSON
            as_json (bool): True if complete order book should be cached as JSON

        Returns:
            dict{key: DataFrame}: Dictionary of DataFrames
        """
        return pb.get_orderbooks(contract_hist_data=contract_hist_data,
                                 delivery_area=delivery_area,
                                 timesteps=timesteps,
                                 time_unit=time_unit,
                                 timestamp=timestamp,
                                 from_timestamp=from_timestamp,
                                 api_client=self.client if hasattr(self, 'client') else None,
                                 use_cached_data=use_cached_data,
                                 caching=caching,
                                 as_json=as_json)

    @staticmethod
    @validate_arguments
    def get_orders(contract_hist_data: dict[str, pandas_DataFrame]) -> dict[str, list[dict]]:
        """
        Extracts all order data from contract history as is, without performing any quality control.

        Args:
            contract_hist_data (dict): Dictionary of Dataframes containing Contract History Data

        Returns:
            dict{key: [orders]}: Dictionary of Lists of Orders
        """
        return pb.get_orders(contract_hist_data=contract_hist_data)

    @staticmethod
    @validate_arguments
    def calc_single_vwap(objects: dict[str, pandas_DataFrame],
                         desired_depth: float,
                         min_depth: float = None) -> dict[str, float]:
        """
        This method can be used to calculate the weighted average price for a dictionary of dataframes (e.g. orders, trades)
        at a desired depth. The output is a singular value for each dataframe.
        This function does not load any data, therefore the already existing data object has to be passed as an argument.

        Args:
            objects (dict[str, DataFrame or list[str, dict]): A dictionary of dataframes, each of which needs to have a
            'quantity' and a 'price' field. Alternatively, a list of dictionaries can be passed as well.
            desired_depth (float): The depth (in MW) specifying how many of the objects should be taken into consideration.
            min_depth (float): The required minimum depth (in percent of the desired depth). If this requirement is not met,
            return value is 0.

        Returns:
            dict[str, float]: The weighted average price for the desired depth for each key in the dictionary.
        """
        return pb.calc_single_vwap(objects=objects,
                                   desired_depth=desired_depth,
                                   min_depth=min_depth)

    @staticmethod
    @validate_arguments
    def plot_ohlc(ohlc_data: dict[str, pandas_DataFrame],
                  ohlc_key: Union[int, str] = 0) -> Union[go.Figure, None]:
        """
        Creates a plotly plot of all ohlc data to be displayed by browser or Dash server. Set ohlc_key to change
        displayed dataframe.

        Args:
            ohlc_data (dict{key: DataFrame}): OHLC Data
            ohlc_key (int/ str): Dictionary key

        Returns:
            Plotly plot
        """
        return pb.plot_ohlc(ohlc_data=ohlc_data,
                            ohlc_key=ohlc_key)

    @staticmethod
    @validate_arguments
    def ohlc_table(ohlc_data: dict[str, pandas_DataFrame],
                   ohlc_key: int = 0) -> pd.DataFrame:
        """
        Creates a custom DataFrame to be displayed by Dash server.

        Args:
            ohlc_data (dict[key: DataFrame]): OHLC Data
            ohlc_key (int): Dictionary key

        Returns:
            DataFrame
        """
        return pb.ohlc_table(ohlc_data=ohlc_data,
                             ohlc_key=ohlc_key)

    @staticmethod
    @validate_arguments
    def plot_orderbook(orderbooks: dict[str, dict[str, pandas_DataFrame]],
                       orderbook_key: Union[int, str] = 0,
                       timestamp: Union[int, str] = -1) -> Union[go.Figure, None]:
        """
        Creates a plotly plot of a single order book to be displayed by browser or Dash server. Use order book_key
        to specify an order book and timestamp to specify the timeframe to display.

        Args:
            orderbooks (dict{key: DataFrame}): Order books
            orderbook_key (int): Dictionary key
            timestamp (int): Order book Key

        Returns:
            Plotly plot
        """
        return pb.plot_orderbook(orderbooks=orderbooks,
                                 orderbook_key=orderbook_key,
                                 timestamp=timestamp)

    @staticmethod
    @validate_arguments
    def plot_volume_history(trade_data: dict[str, pandas_DataFrame],
                            trade_key: Union[int, str] = 0) -> Union[go.Figure, None]:
        """
        Creates a plotly plot of the trade volume for a single contract to be displayed by browser or Dash server.

        Args:
            trade_data (dict{key: DataFrame}):  Trade Data
            trade_key (int/str): Dictionary key

        Returns:
            Plotly plot
        """
        return pb.plot_volume_history(trade_data=trade_data,
                                      trade_key=trade_key)


class ApiExporter(BaseExporter):
    """
    Exporter class for interaction with the PowerBot API.

    This class can/ should be used when:
        - the requested data is recent enough to still be stored in the PowerBot instance (see data retention policy)
        - the requested data is fairly small in size (multiple hours, not multiple day -> extensive loading time &
          constant strain on the API rate limit)
        - the requested data is already stored in the local __pb_cache__ and has been loaded via API.

    ATTENTION: if you try to load data from your cache and the original data is already purged from your instance,
    you are no longer able to create an index of contract IDs to load the local data with. Should this occur, please
    load the data in question via the HistoryExporter.
    """
    api_key: str = Field(description="A Standard API Key",
                         example="XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXX")
    host: str = Field(description="URL of the PowerBot instance to connect to",
                      example="https://backup.powerbot-trading.com/{COMPANY NAME}/{EXCHANGE}/v2/api")

    client: ApiClient = Field(description="Placeholder value, do not overwrite", default=None)

    def __init__(self, **data: Any):
        super().__init__(**data)

        # Init client
        self.client = pb.init_client(api_key=self.api_key, host=self.host)

    @root_validator
    def check_credentials(cls, values):
        api_key, host = values.get('api_key'), values.get('host')

        pattern_key = "\w{8}-\w{4}-\w{4}-\w{4}-\w{12}"
        pattern_host = "https://\w{4,7}.powerbot-trading.com/\w+/\w+/v\d{1}/api"

        if not re.match(pattern_key, api_key) or not re.match(pattern_host, host):
            raise ValueError("Your credentials do not conform to the necessary formats")

        return values

    @validate_arguments
    def get_contract_ids(self,
                         time_from: datetime = Field(description="Datetime in format yyyy-mm-dd hh:mm:ss", default=None),
                         time_till: datetime = Field(description="Datetime in format yyyy-mm-dd hh:mm:ss", default=None),
                         contract_ids: list[str] = None,
                         contract_time: str = "all",
                         products: list[str] = None,
                         allow_udc: bool = False,
                         delivery_areas: list[str] = None,
                         return_contract_objects: bool = False) -> Union[dict[str, list[str]], tuple[dict, dict]]:
        """
        Loads all contract IDs for specified timeframe, either by hours, single day or multiple days. Alternatively, a list
        of contract IDs can be passed instead of a timeframe, returning a dictionary of contract IDs compatible with all other
        functions of the Backtesting package. If return_contract_objects is True, a dictionary of contract objects will be
        returned as well.

        If time_from includes hh:mm:ss
            loads data between both times in timesteps according to contract_time

        If time_from doesn't include hh:mm:ss
            loads data between both dates in timesteps according to contract_time

        Args:
            time_from (datetime): yyyy-mm-dd hh:mm:ss
            time_till (datetime): yyyy-mm-dd hh:mm:ss
            contract_ids (list[str]): Optionally, a list of specific contract IDs can be passed to return contract objects
            contract_time (str): all, hourly, half-hourly or quarter-hourly (all includes UDC)
            products (list): Optional list of specific products to return
            allow_udc (bool): True if user-defined contracts (block products) should also be returned on contract_time = all
            delivery_areas (list): List of EIC-codes
            return_contract_objects (bool): If True, returns complete Contract object

        Returns:
            dict{key: (list[str])}: Dictionary of Contract IDs
            (Optional) Contract: Contract Object
        """
        return pb.get_contract_ids(api_client=self.client,
                                   time_from=time_from,
                                   time_till=time_till,
                                   contract_ids=contract_ids,
                                   contract_time=contract_time,
                                   products=products,
                                   allow_udc=allow_udc,
                                   delivery_areas=delivery_areas,
                                   return_contract_objects=return_contract_objects)

    @validate_arguments
    def get_public_trades(self,
                          contract_ids: dict[str, list[str]],
                          delivery_area: str,
                          contract_time: str,
                          iteration_delay: float = 0.4,
                          serialize_data: bool = True,
                          add_vwap: bool = False,
                          use_cached_data: bool = True,
                          caching: bool = True,
                          gzip_files: bool = True,
                          as_csv: bool = False) -> dict[str, pd.DataFrame]:
        """
        Load trade data for given contract IDs. If add_vwap is True, VWAP will be calculated for each trade, incorporating
        all previous trades.

        Args:
            contract_ids (dict): Dictionary of Contract IDs
            delivery_area (str): EIC Area Code for Delivery Area (not needed when loading from historic cache)
            contract_time (str): all, hourly, half-hourly or quarter-hourly
            iteration_delay (float): Optional delay between iterations to prevent hitting API rate limits
            serialize_data (bool): If False, request is received without serialization. Recommended for large data collections
            add_vwap (bool): If True, additional VWAP parameters will be added to each dataframe
            use_cached_data (bool): If True, function tries to load data from cache wherever possible
            caching (bool): True if data should be cached
            gzip_files (bool): True if cached files should be gzipped
            as_csv (bool): if True, will save files as CSV, additionally to JSON

        Returns:
            dict{key: DataFrame}: Dictionary of DataFrames
        """
        return pb.get_public_trades(api_client=self.client,
                                    contract_ids=contract_ids,
                                    contract_time=contract_time,
                                    delivery_area=delivery_area,
                                    iteration_delay=iteration_delay,
                                    serialize_data=serialize_data,
                                    add_vwap=add_vwap,
                                    use_cached_data=use_cached_data,
                                    caching=caching,
                                    gzip_files=gzip_files,
                                    as_csv=as_csv)

    @validate_arguments
    def get_public_trades_by_days(self,
                                  previous_days: int,
                                  delivery_area: str,
                                  time_from: datetime = Field(description="Datetime in format yyyy-mm-dd hh:mm:ss", default=None),
                                  contract_time: str = None,
                                  contract_id: str = None) -> dict[str, pd.DataFrame]:
        """
        Gets the contract ID specified by a timeframe or directly by ID and load all trade data for this contract and all
        contracts in the same timeframe for X previous days.

        Args:
            time_from (datetime): YYYY-MM-DD hh:mm:ss
            previous_days (int): Amount of previous days to load data for
            delivery_area (str): EIC Area Code for Delivery Area
            contract_time (str): hourly, half-hourly or quarter-hourly
            contract_id (str): specific contract ID

        Returns:
            dict{key: DataFrame}: Dictionary of DataFrames
        """
        return pb.get_public_trades_by_days(api_client=self.client,
                                            previous_days=previous_days,
                                            delivery_area=delivery_area,
                                            time_from=time_from,
                                            contract_time=contract_time,
                                            contract_id=contract_id)

    @validate_arguments
    def get_contract_history(self,
                             contract_ids: dict[str, list[str]],
                             delivery_area: str,
                             iteration_delay: float = 0.4,
                             serialize_data: bool = True,
                             use_cached_data: bool = True,
                             caching: bool = True,
                             gzip_files: bool = True,
                             as_csv: bool = False) -> dict[str, pd.DataFrame]:
        """
        Load contract history for given contract IDs.

        Args:
            contract_ids (dict): Dictionary of Contract IDs
            delivery_area (str): EIC Area Code for Delivery Area (not needed when loading from historic cache)
            iteration_delay (float): Optional delay between iterations to prevent hitting API rate limits
            serialize_data (bool): If False, request is received without serialization. Recommended for large data collections
            use_cached_data (bool): If True, function tries to load data from cache wherever possible
            caching (bool): True if data should be cached
            gzip_files (bool): True if cached files should be gzipped
            as_csv (bool): if True, will save files as CSV, additionally to JSON

        Returns:
            dict{key: DataFrame}: Dictionary of DataFrames
        """
        return pb.get_contract_history(api_client=self.client,
                                       contract_ids=contract_ids,
                                       delivery_area=delivery_area,
                                       iteration_delay=iteration_delay,
                                       serialize_data=serialize_data,
                                       use_cached_data=use_cached_data,
                                       caching=caching,
                                       gzip_files=gzip_files,
                                       as_csv=as_csv)

    @validate_arguments
    def get_signals(self,
                    time_from: datetime = Field(description="Datetime in format yyyy-mm-dd hh:mm:ss"),
                    time_till: datetime = Field(description="Datetime in format yyyy-mm-dd hh:mm:ss"),
                    delivery_area: str = None,
                    portfolio_id: list[str] = None) -> list[Signal]:
        """
        Function gathers all Signals received by the API in the specified time period and gathers them in a list.

        Args:
            time_from (datetime): YYYY-MM-DD or YYYY-MM-DD hh:mm:ss
            time_till (datetime): YYYY-MM-DD or YYYY-MM-DD hh:mm:ss
            delivery_area (str): EIC Area Code for Delivery Area
            portfolio_id (str): List of all portfolios that signals should be loaded from

        Returns:
            list[Signal]
        """
        return pb.get_signals(api_client=self.client,
                              time_from=time_from,
                              time_till=time_till,
                              delivery_area=delivery_area,
                              portfolio_id=portfolio_id)

    @validate_arguments
    def get_own_trades(self,
                       time_from: datetime = Field(description="Datetime in format yyyy-mm-dd hh:mm:ss"),
                       time_till: datetime = Field(description="Datetime in format yyyy-mm-dd hh:mm:ss"),
                       delivery_area: str = None,
                       portfolio_id: list[str] = None) -> list[Trade]:
        """
        Function to collect all Own Trades for the defined time period, either specific to portfolio and/or delivery area
        or all portfolios and delivery areas used API key has access to.

        Args:
            time_from (datetime): YYYY-MM-DD or YYYY-MM-DD hh:mm:ss
            time_till (datetime): YYYY-MM-DD or YYYY-MM-DD hh:mm:ss
            delivery_area (str): EIC Area Code for Delivery Area
            portfolio_id (list[str]): List of specific portfolio IDs to load trades from. If left out, will load from all IDs.

        Returns:
            list[Trade]
        """
        return pb.get_own_trades(api_client=self.client,
                                 time_from=time_from,
                                 time_till=time_till,
                                 delivery_area=delivery_area,
                                 portfolio_id=portfolio_id)

    @validate_arguments
    def get_internal_trades(self,
                            time_from: datetime = Field(description="Datetime in format yyyy-mm-dd hh:mm:ss"),
                            time_till: datetime = Field(description="Datetime in format yyyy-mm-dd hh:mm:ss"),
                            delivery_area: str = None,
                            portfolio_id: list[str] = None) -> list[InternalTrade]:
        """
        Function to collect all Internal Trades for the defined time period, either specific to portfolio and/or delivery
        area or all portfolios and delivery areas used API key has access to.

        Args:
            time_from (datetime): YYYY-MM-DD or YYYY-MM-DD hh:mm:ss
            time_till (datetime): YYYY-MM-DD or YYYY-MM-DD hh:mm:ss
            delivery_area (str): EIC Area Code for Delivery Area
            portfolio_id (list[str]): List of specific portfolio IDs to load trades from. If left out, will load from all IDs.

        Returns:
            list[InternalTrade]
        """
        return pb.get_internal_trades(api_client=self.client,
                                      time_from=time_from,
                                      time_till=time_till,
                                      delivery_area=delivery_area,
                                      portfolio_id=portfolio_id)

    @validate_arguments
    def get_own_orders(self,
                       delivery_area: str = None,
                       portfolio_id: list[str] = None,
                       active_only: bool = False) -> list[OwnOrder]:
        """
        Function to collect all available Own Orders, either specific to portfolio and/or delivery area or all portfolios
        and delivery areas used API key has access to.

        Args:
            delivery_area (str): EIC Area Code for Delivery Area
            portfolio_id (list[str]): List of specific portfolio IDs to load trades from. If left out, will load from all IDs.
            active_only (bool):  True if only active orders should be loaded. If False, loads also hibernate and inactive.

        Returns:
            list[OwnOrder]
        """
        return pb.get_own_orders(api_client=self.client,
                                 delivery_area=delivery_area,
                                 portfolio_id=portfolio_id,
                                 active_only=active_only)

    @validate_arguments
    def calc_trade_vwap(self,
                        contract_time: str,
                        delivery_area: str,
                        trade_data: dict[str, pandas_DataFrame] = None,
                        time_from: datetime = Field(description="Datetime in format yyyy-mm-dd hh:mm:ss", default=None),
                        previous_days: int = 10,
                        contract_id: str = None,
                        index: str = "ID3") -> pd.DataFrame:
        """
        Function gets trades for a certain contract for X previous days in the same delivery period and calculates their
        VWAP for ID3 or ID1 or all. Generates a new list of trades for these contracts.
        Can take either a time period or a specific contract ID to load data for.

        If previous days is 0, only the trades for the original time period/ contract will be loaded.

        Can also be called directly from get_public_trades with parameter 'add_vwap' to add VWAP to loaded trades.

        Args:
            contract_time (str): hourly, half-hourly or quarter-hourly
            delivery_area (str): Area Code for Delivery Area
            trade_data (dict[str, pd.DataFrame]: Dictionary of Dataframes containing Contract Trade Data
            time_from (datetime): yyyy-mm-dd hh:mm:ss
            previous_days (int): Amount of previous days to load data
            contract_id (str): ID of specific Contract
            index (str): all, ID3, ID1

        Returns:
            DataFrame: Trade Data with added calculated fields
        """
        return pb.calc_trade_vwap(api_client=self.client,
                                  contract_time=contract_time,
                                  delivery_area=delivery_area,
                                  trade_data=trade_data,
                                  time_from=time_from,
                                  previous_days=previous_days,
                                  contract_id=contract_id,
                                  index=index)


class HistoryExporter(BaseExporter):
    """
    Exporter class for interaction with the PowerBot History API and the subsequently created local __pb_cache__.

    This class can/ should be used when:
        - the requested data has is older than at least 2-3 days and has already been made available via History API
        - the requested data is already stored in the local __pb_cache__ and has been loaded via History API.

    ATTENTION: loading historic data from the History API will create a json file containing all contract information
    for the respective day. If this file should be deleted, the HistoryExporter can no longer create an index of
    contract IDs and therefore not load anything from the local cache.
    """
    exchange: str = Field(description="The exchange data should be loaded for")
    delivery_area: str = Field(description="EIC-code of the delivery area data should be loaded for")

    client: HistoryApiClient = Field(description="Placeholder value, do not overwrite", default=None)

    def __init__(self, **data: Any):
        super().__init__(**data)

        # Init client
        self.client = pb.init_historic_client(self.exchange, self.delivery_area)

    @root_validator
    def check_credentials(cls, values):
        exchange, delivery_area = values.get('exchange'), values.get('delivery_area')

        assert exchange in EXCHANGES, "Exchange is not in allowed exchanges"
        assert delivery_area in EIC_CODES[exchange], "Delivery area is not in allowed delivery areas for this exchange"

        return values

    @validate_arguments
    def get_historic_data(self,
                          api_key: str,
                          day_from: Union[str, datetime],
                          day_to: Union[str, datetime] = None,
                          delivery_areas: list[str] = None,
                          cache_path: Path = None,
                          extract_files: bool = False,
                          process_data: bool = False,
                          skip_on_error: bool = False,
                          keep_zip_files: bool = False) -> Union[list, dict]:
        """
        Function loads all public data for specified days in the specified delivery area. Output is a zipped directory
        containing all files in JSON format. Optionally, zip file can be extracted automatically and processed to be
        compatible with other functions in the powerbot_backtesting package.

        Args:
            api_key (str): Specific history instance API key
            day_from (str): Datetime/ String in format YYYY-MM-DD
            day_to (str): Datetime/ String in format YYYY-MM-DD
            delivery_areas (list): List of EIC Area Codes for Delivery Areas
            cache_path (Path): Optional path for caching files
            extract_files (bool): True if zipped files should be extracted automatically (Warning: immense size increase)
            process_data (bool): True if extracted files should be processed to resemble files loaded via API
            skip_on_error (bool): True if all dates that cannot possibly be loaded (e.g. due to lack of access rights) are
            skipped if the difference between day_from and day_to is at least 2 days
            keep_zip_files (bool): True if zip-files should be kept after download

        Returns:
            list of loaded file paths | dict
        """
        return pb.get_historic_data(api_key=api_key,
                                    exchange=self.client.exchange,
                                    delivery_areas=delivery_areas if delivery_areas else [self.client.delivery_area],
                                    day_from=day_from,
                                    day_to=day_to,
                                    cache_path=cache_path,
                                    extract_files=extract_files,
                                    process_data=process_data,
                                    skip_on_error=skip_on_error,
                                    keep_zip_files=keep_zip_files)

    @validate_arguments
    def get_contract_ids(self,
                         time_from: datetime = Field(description="Datetime in format yyyy-mm-dd hh:mm:ss"),
                         time_till: datetime = Field(description="Datetime in format yyyy-mm-dd hh:mm:ss"),
                         contract_time: Optional[str] = "all",
                         products: Optional[list[str]] = None,
                         allow_udc: Optional[bool] = False) -> dict[str, list[str]]:
        """
        Loads all contract IDs for specified timeframe, either by hours, single day or multiple days from the local cache.
        The cached data has to have been loaded via get_historic_data, since this function utilizes the contract file
        as an index.

        If time_from includes hh:mm:ss
            loads data between both times in timesteps according to contract_time

        If time_from doesn't include hh:mm:ss
            loads data between both dates in timesteps according to contract_time

        Args:
            time_from (datetime): yyyy-mm-dd hh:mm:ss
            time_till (datetime): yyyy-mm-dd hh:mm:ss
            contract_time (str): all, hourly, half-hourly or quarter-hourly (all includes UDC)
            products (list): Optional list of specific products to return
            allow_udc (bool): True if user-defined contracts (block products) should also be returned on contract_time = all

        Returns:
            dict{key: (list[str])}: Dictionary of Contract IDs
        """
        return pb.get_historic_contract_ids(client=self.client,
                                            time_from=time_from,
                                            time_till=time_till,
                                            contract_time=contract_time,
                                            products=products if products else [],
                                            allow_udc=allow_udc)

    @validate_arguments
    def get_public_trades(self,
                          contract_ids: dict[str, list[str]],
                          contract_time: str,
                          add_vwap: bool = False,
                          as_csv: bool = False) -> dict[str, pd.DataFrame]:
        """
        Load trade data for given contract IDs from local cache. If add_vwap is True, VWAP will be calculated for each
        trade, incorporating all previous trades.

        Args:
            contract_ids (dict): Dictionary of Contract IDs
            contract_time (str): all, hourly, half-hourly or quarter-hourly
            add_vwap (bool): If True, additional VWAP parameters will be added to each dataframe
            as_csv (bool): if True, will save files as CSV, additionally to JSON

        Returns:
            dict{key: DataFrame}: Dictionary of DataFrames
        """
        return pb.get_public_trades(api_client=self.client,
                                    contract_ids=contract_ids,
                                    contract_time=contract_time,
                                    add_vwap=add_vwap,
                                    caching=True if as_csv else False,
                                    as_csv=as_csv)

    @validate_arguments
    def get_contract_history(self,
                             contract_ids: dict[str, list[str]],
                             as_csv: bool = False) -> dict[str, pd.DataFrame]:
        """
        Load contract history for given contract IDs from the local cache.

        Args:
            contract_ids (dict): Dictionary of Contract IDs
            as_csv (bool): if True, will save files as CSV, additionally to JSON

        Returns:
            dict{key: DataFrame}: Dictionary of DataFrames
        """
        return pb.get_contract_history(api_client=self.client,
                                       contract_ids=contract_ids,
                                       as_csv=as_csv)


class SQLExporter(BaseExporter):
    """
    Exporter class for interaction with a SQL database containing PowerBot data.

    This class can/ should be used when:
        - a database containing the structure as defined per PowerBot SQLImporter exists and contains data.

    Instructions for passing arguments to exporter functions:
        - String
            If keyword argument is a string, it will be simply compared with '='. Optionally, a mathematical/ SQL
            operator (LIKE, <, <=, >, >=, <>) can be passed within the string. This operator will be used instead of '='.

            Example:
                best_bid='> 0.00' -> best_bid > 0.00 / as_of="> 2020-09-20 00:00:00" -> as_of > '2020-09-20 00:00:00'

        - Tuple
            If keyword argument is a tuple, it will be checked, if parameter is one of the elements of the tuple.

            Example:
                exchange=("Epex","NordPool") -> exchange IN ('Epex','NordPool')

        - List
            If keyword argument is a list, each element will be checked if it is in the parameter.

            Example:
                portfolio_ids=["TP1","TP2"] -> (portfolio_id LIKE '%TP1%' OR portfolio_id LIKE '%TP2%')

        - Dictionary
            If keyword argument is a dictionary, all values will be extracted and put into a tuple. Afterwards, the
            behaviour is the same as with tuples.

            Example:
                exchange={1:"Epex",2:"NordPool"} -> exchange IN ("Epex","NordPool")

        - Datetime
            If keyword argument is a datetime, parameter will be searched for the exact time of the datetime argument.
            This will in most cases not provide a satisfying result, therefore it is recommended to pass a datetime as
            a string with an operator in front.

            Example:
                as_of=datetime.datetime(2020, 9, 30, 10, 0, 0) -> as_of = '2020-09-30 10:00:00'
    """
    db_type: str = Field(description="Database type")
    user: str = Field(description="Database user")
    password: str = Field(description="Database password")
    host: str = Field(description="Database host address")
    database: str = Field(description="Database name")
    port: int = Field(description="Database port")

    logger: None = Field(description="Placeholder value, do not overwrite", default=None)
    engine: None = Field(description="Placeholder value, do not overwrite", default=None)

    SQL_ERRORS = (
        InterfaceError,
        OperationalError,
        ProgrammingError,
        TimeoutError,
    )

    def __init__(self, **data: Any):
        super().__init__(**data)

        # Logging Setup
        logging.basicConfig(format="PowerBot_SQL_Exporter %(asctime)s %(levelname)-8s %(message)s",
                            level=logging.INFO)
        self.logger = logging.getLogger()

        # Initialize Connection
        self.engine = self.__create_sql_engine()

    @validator("db_type")
    def validate_db_type(cls, value):
        allowed_db_types = ['mysql', 'mariadb', 'postgresql', 'oracle', 'mssql', 'amazon_redshift', 'apache_drill',
                            'apache_druid', 'apache_hive', 'apache_solr', 'cockroachdb', 'cratedb', 'exasolution',
                            'firebird', 'ibm_db2', 'monetdb', 'snowflake', 'teradata_vantage']

        assert value in allowed_db_types, f"Database {value} is not in allowed database"

        return value

    def __install_packages(self):
        """
        Tests if required packages for chosen SQL database type are available and installs them if necessary.
        """
        db_packages = {"mysql": ["mysql-connector-python"],
                       "mariadb": ["PyMySQL"],
                       "postgresql": ["psycopg2"],
                       "oracle": ["cx-Oracle"],
                       "mssql": ["pyodbc"],
                       "amazon_redshift": ["sqlalchemy-redshift", "psycopg2"],
                       "apache_drill": ["sqlalchemy-drill"],
                       "apache_druid": ["pydruid"],
                       "apache_hive": ["PyHive"],
                       "apache_solr": ["sqlalchemy-solr"],
                       "cockroachdb": ["sqlalchemy-cockroachdb", "psycopg2"],
                       "cratedb": ["crate-python"],
                       "exasolution": ["sqlalchemy_exasol", "pyodbc"],
                       "firebird": ["sqlalchemy-firebird"],
                       "ibm_db2": ["ibm_db_sa"],
                       "monetdb": ["sqlalchemy_monetdb"],
                       "snowflake": ["snowflake-sqlalchemy"],
                       "teradata_vantage": ["teradatasqlalchemy"]}

        self.logger.info("Now installing the following necessary package(s):\n"
                        f"{db_packages[self.db_type]}")

        import subprocess
        import sys
        for pkg in db_packages[self.db_type]:
            subprocess.check_call([sys.executable, "-m", "pip", "install", pkg])

        return self.__create_sql_engine()

    def __create_sql_engine(self):
        """
        Initializes connection to SQL database
        """
        db_types = {"mysql": "mysql+mysqlconnector",
                    "mariadb": "mariadb+pymysql",
                    "postgresql": "postgresql+psycopg2",
                    "oracle": "oracle+cx_oracle",
                    "mssql": "mssql+pyodbc",
                    "amazon_redshift": "redshift+psycopg2",
                    "apache_drill": "drill+sadrill",
                    "apache_druid": "druid",
                    "apache_hive": "hive",
                    "apache_solr": "solr",
                    "cockroachdb": "cockroachdb",
                    "cratedb": "crate",
                    "exasolution": "exa+pyodbc",
                    "firebird": "firebird",
                    "ibm_db2": "db2+ibm_db",
                    "monetdb": "monetdb",
                    "snowflake": "snowflake",
                    "teradata_vantage": "teradatasql"}

        if self.port:
            try:
                engine = create_engine(
                    f'{db_types[self.db_type]}://{self.user}:{self.password}@{self.host}:{self.port}/{self.database}')
            except:
                pass
        try:
            engine = create_engine(
                f'{db_types[self.db_type]}://{self.user}:{self.password}@{self.host}/{self.database}')

            # Test connection
            engine.connect()

        except (NoSuchModuleError, ModuleNotFoundError):
            self.logger.info("You currently do not have all the necessary packages installed to access a database of"
                            f" type {self.db_type}.")
            return self.__install_packages()

        except ProgrammingError:
            self.logger.error("Could not establish connection to database. Please recheck your credentials!")

        except InterfaceError:
            self.logger.error("Database is not available at the moment!")

        except Exception as e:
            raise SQLExporterError(f"Could not establish connection to database. Reason: \n{e}")

        self.logger.info(f"Connection to database '{self.database}' with user '{self.user}' established")
        self.logger.info("Connection ready to export data")

        return engine

    @contextmanager
    def __get_session(self):
        """
        Context manager to handle sessions connecting to the database.
        """
        try:
            session = Session(bind=self.engine)
        except self.SQL_ERRORS:
            session = Session(bind=self.__create_sql_engine())
        try:
            yield session
        finally:
            session.close()

    @validate_arguments
    def get_contracts(self,
                      as_dataframe: bool = True,
                      **kwargs) -> pd.DataFrame:
        """
        Exports contracts from SQL database. To use different mathematical/SQL operators, pass keyworded arguments
        as strings and include the desired operator followed by a space (e.g. revisions='<> 0').

        Following operators can be passed:
        LIKE, <, <=, >, >=, <>

        Following parameters can be passed as optional keyworded arguments:
        exchange, contract_id, product, type, undrlng_contracts, name, delivery_start, delivery_end, delivery_areas,
        predefined, duration, delivery_units

        Args:
            as_dataframe (bool): If False -> returns list
            **kwargs: any additional fields of SQL table

        Returns:
            list/ DataFrame: SQL query
        """
        allowed_kwargs = ["name", "delivery_areas", "delivery_start", "delivery_end", "delivery_areas", "type",
                          "predefined", "duration", "delivery_units", "contract_id", "exchange", "product",
                          "undrlng_contracts"]

        sql_params = self.__handle_sql_args(kwargs, allowed_kwargs)

        with self.__get_session() as session:
            result = session.execute(f"SELECT * FROM contracts{sql_params}").fetchall()

        if as_dataframe:
            output = pd.DataFrame(result)
            output = output.rename(
                columns={0: 'exchange', 1: 'contract_id', 2: 'product', 3: 'type', 4: 'undrlng_contracts',
                         5: 'name', 6: 'delivery_start', 7: 'delivery_end', 8: 'delivery_areas',
                         9: 'predefined', 10: 'duration', 11: 'delivery_units', 12: 'details'})
            return output

        return result

    @validate_arguments
    def get_contract_ids(self,
                         time_from: datetime = Field(description="Datetime in format yyyy-mm-dd hh:mm:ss"),
                         time_till: datetime = Field(description="Datetime in format yyyy-mm-dd hh:mm:ss"),
                         delivery_area: str = Field(description="EIC-code of delivery area"),
                         contract_time: str = "all",
                         exchange: str = "epex",
                         as_list: bool = False) -> dict[str, list[str]]:
        """
            Returns dictionary of contract IDs in a format compatible with backtesting pipeline.

        Args:
            time_from (datetime): yyyy-mm-dd hh:mm:ss
            time_till (datetime): yyyy-mm-dd hh:mm:ss
            delivery_area (str): EIC-Code
            contract_time (str): all, hourly, half-hourly or quarter-hourly
            exchange (str): Name of exchange in lowercase
            as_list (bool): True if output should be list of contract IDs

        Returns:
            dict{key: (list[str])}: Dictionary of Contract IDs
        """
        if not isinstance(time_from, datetime) or not isinstance(time_till, datetime):
            raise SQLExporterError("Please use datetime format for time_from & time_till!")

        with self.__get_session() as session:
            result = session.execute(f"SELECT delivery_start, delivery_end, contract_id FROM contracts "
                                     f"WHERE delivery_start >= '{time_from}' "
                                     f"AND delivery_end <= '{time_till}' "
                                     f"AND delivery_areas LIKE '%{delivery_area}%' "
                                     f"AND product IN {PRODUCTS[contract_time]} "
                                     f"AND exchange = '{exchange}'").fetchall()
            if not result:
                result = session.execute(f"SELECT delivery_start, delivery_end, contract_id FROM contracts "
                                         f"WHERE delivery_start >= '{time_from}' "
                                         f"AND delivery_end <= '{time_till}' "
                                         f"AND product IN {PRODUCTS[contract_time]} "
                                         f"AND exchange = '{exchange}'").fetchall()

        if not as_list:
            contract_ids = {f"{i[0].strftime(DATE_YMD_TIME_HM)} - {i[1].strftime(TIME_HM)}": [] for i in result}
            for i in result:
                contract_ids[f"{i[0].strftime(DATE_YMD_TIME_HM)} - {i[1].strftime(TIME_HM)}"].append(i[2])

            # Quality Check
            if not all(i for i in contract_ids.values()):
                raise SQLExporterError("There is no contract data for the specified timeframe!")
        else:
            contract_ids = [i for i in result]

        self.logger.info("Successfully exported contract ids")
        return contract_ids

    @validate_arguments
    def get_public_trades(self,
                          as_dataframe: bool = True,
                          delivery_area: list[str] = None,
                          **kwargs) -> Union[pd.DataFrame, dict[str, pd.DataFrame]]:
        """
        Exports trades from SQL database. To use different mathematical/SQL operators, pass keyworded arguments
        as strings and include the desired operator followed by a space (e.g. price='<> 0.00').

        Following operators can be passed:
        LIKE, <, <=, >, >=, <>

        Following parameters can be passed as optional keyworded arguments (kwargs):
        price, quantity, prc_x_qty, exchange, contract_id, trade_id, exec_time, api_timestamp, self_trade

        Args:
            as_dataframe (bool): If False -> returns list
            delivery_area (tuple[str]): Multiple delivery areas inside a tuple. Single del. area can be passed as a string
            **kwargs: any additional fields of SQL table

        Returns:
            list/ DataFrame: SQL query
        """
        allowed_kwargs = ["price", "quantity", "prc_x_qty", "exchange", "contract_id", "trade_id", "exec_time",
                          "api_timestamp", "self_trade"]

        sql_params = self.__handle_sql_args(kwargs, allowed_kwargs)
        if delivery_area:
            for i in delivery_area:
                sql_params += f" AND (buy_delivery_area = '{i}' OR sell_delivery_area = '{i}')"

        with self.__get_session() as session:
            result = session.execute(f"SELECT * FROM public_trades{sql_params}").fetchall()

        if as_dataframe:
            output = pd.DataFrame(result)
            if not output.empty:
                output = output.rename(columns={0: 'exchange', 1: 'contract_id', 2: 'trade_id', 3: 'api_timestamp',
                                                4: 'exec_time', 5: 'buy_delivery_area', 6: 'sell_delivery_area',
                                                7: 'price', 8: 'quantity', 9: 'prc_x_qty', 10: "currency",
                                                11: 'self_trade'})
                if not delivery_area:
                    raise SQLExporterError("Please specify at least one delivery area!")

                output['api_timestamp'] = pd.to_datetime(output['api_timestamp'], utc=True)
                output['exec_time'] = pd.to_datetime(output['exec_time'], utc=True)

                self.logger.info("Successfully exported trades")
                return self.__convert_dataframe("trades", output)
            else:
                raise SQLExporterError("There is no trade data for the specified timeframe!")

        self.logger.info("Successfully exported trades")

        return result

    @validate_arguments
    def get_contract_history(self,
                             as_dataframe: bool = True,
                             **kwargs) -> Union[pd.DataFrame, dict[str, pd.DataFrame]]:
        """
        Exports contract revisions from SQL database. To use different mathematical/SQL operators, pass keyworded arguments
        as strings and include the desired operator followed by a space (e.g. best_bid='<> 0.00').

        Following operators can be passed:
        LIKE, <, <=, >, >=, <>

        Following parameters can be passed as optional keyworded arguments:
        exchange, contract_id, exchange, delivery_area, revision_no, as_of, best_bid, best_bid_qty, best_ask,
        best_ask_qty, vwap, high, low, last_price, last_qty, last_trade_time, volume, delta

        Args:
            as_dataframe (bool): If False -> returns list
            **kwargs: any additional fields of SQL table

        Returns:
            list/ DataFrame: SQL query
        """
        allowed_kwargs = ["exchange", "contract_id", "delivery_area", "revision_no", "as_of", "best_bid",
                          "best_bid_qty", "best_ask", "best_ask_qty", "vwap", "high", "low", "last_price",
                          "last_qty", "last_trade_time", "volume", "delta"]

        sql_params = self.__handle_sql_args(kwargs, allowed_kwargs)

        with self.__get_session() as session:
            result = session.execute(f"SELECT * FROM contract_revisions{sql_params}").fetchall()

        if as_dataframe:
            output = pd.DataFrame(result)
            if not output.empty:
                output = output.rename(columns={0: 'exchange', 1: 'contract_id', 2: 'delivery_area', 3: 'revision_no',
                                                4: 'as_of', 5: 'best_bid', 6: 'best_bid_qty', 7: 'best_ask',
                                                8: 'best_ask_qty', 9: 'vwap', 10: 'high', 11: 'low', 12: 'last_price',
                                                13: 'last_qty', 14: "last_trade_time", 15: 'volume', 16: 'delta',
                                                17: 'bids', 18: 'asks'})
                if "delivery_area" not in kwargs:
                    raise SQLExporterError("Please specify one specific delivery area!")
                self.logger.info("Successfully exported contract history")
                return self.__convert_dataframe("orders", output)
            else:
                raise SQLExporterError("There is no order data for the specified timeframe!")

        self.logger.info("Successfully exported contract history")

        return result

    @validate_arguments
    def get_own_trades(self,
                       delivery_area: list[str] = None,
                       as_dataframe: bool = True,
                       as_objects: bool = False,
                       **kwargs) -> Union[pd.DataFrame, list[Trade]]:
        """
        Exports Own Trades from SQL database. To use different mathematical/SQL operators, pass keyworded arguments
        as strings and include the desired operator followed by a space (e.g. position_short='<> 0.00').

        Following operators can be passed:
        LIKE, <, <=, >, >=, <>

        Following parameters can be passed as optional keyworded arguments (kwargs):
        exchange, contract_id, contract_name, prod, delivery_start, delivery_end, trade_id, api_timestamp, exec_time,
        buy, sell, price, quantity, state, buy_delivery_area, sell_delivery_area, buy_order_id, buy_clOrderId, buy_txt,
        buy_user_code, buy_member_id, buy_aggressor_indicator, buy_portfolio_id, sell_order_id, sell_clOrderId,
        sell_txt, sell_user_code, sell_member_id, sell_aggressor_indicator, sell_portfolio_id, self_trade, pre_arranged,
        pre_arrange_type

        Args:
            delivery_area (tuple[str]): Multiple delivery areas inside a tuple. Single del. area can be passed as a string
            as_dataframe (bool): True if output should be DataFrame
            as_objects (bool): True if output should be list of OwnTrades
            **kwargs: any additional fields of SQL table

        Returns:
            list: SQL query
        """

        allowed_kwargs = ["exchange", "contract_id", "contract_name", "prod", "delivery_start", "delivery_end",
                          "trade_id",
                          "api_timestamp", "exec_time", "buy", "sell", "price", "quantity", "state",
                          "buy_delivery_area",
                          "sell_delivery_area", "buy_order_id", "buy_clOrderId", "buy_txt", "buy_user_code",
                          "buy_member_id",
                          "buy_aggressor_indicator", "buy_portfolio_id", "sell_order_id", "sell_clOrderId", "sell_txt",
                          "sell_user_code", "sell_member_id", "sell_aggressor_indicator", "sell_portfolio_id",
                          "self_trade",
                          "pre_arranged", "pre_arrange_type"]

        sql_params = self.__handle_sql_args(kwargs, allowed_kwargs)

        if delivery_area:
            for i in delivery_area:
                sql_params += f" AND (buy_delivery_area = '{i}' OR sell_delivery_area = '{i}')"

        with self.__get_session() as session:
            result = session.execute(f"SELECT * FROM own_trades{sql_params}").fetchall()

        self.logger.info("Successfully exported own trades")

        # Convert data back to Trade objects
        if result and as_objects:
            own_trades = [Trade(exchange=i[0],
                                contract_id=i[1],
                                contract_name=i[2],
                                prod=i[3],
                                delivery_start=i[4],
                                delivery_end=i[5],
                                trade_id=i[6],
                                api_timestamp=i[7],
                                exec_time=i[8],
                                buy=i[9],
                                sell=i[10],
                                price=i[11],
                                quantity=i[12],
                                delivery_area=i[13],
                                state=i[14],
                                buy_delivery_area=i[15],
                                sell_delivery_area=i[16],
                                buy_order_id=i[17],
                                buy_cl_order_id=i[18],
                                buy_txt=i[19],
                                buy_user_code=i[20],
                                buy_member_id=i[21],
                                buy_aggressor_indicator=i[22],
                                buy_portfolio_id=i[23],
                                sell_order_id=i[24],
                                sell_cl_order_id=i[25],
                                sell_txt=i[26],
                                sell_user_code=i[27],
                                sell_member_id=i[28],
                                sell_aggressor_indicator=i[29],
                                sell_portfolio_id=i[30],
                                self_trade=i[31],
                                pre_arranged=i[32],
                                pre_arrange_type=i[33])
                          for i in result]
            return own_trades

        if result and as_dataframe:
            return pd.DataFrame(result)

        return result

    @validate_arguments
    def get_internal_trades(self,
                            delivery_area: tuple[str] = None,
                            as_dataframe: bool = True,
                            as_objects: bool = False,
                            **kwargs) -> Union[pd.DataFrame, list[InternalTrade]]:
        """
        Exports Internal Trades from SQL database. To use different mathematical/SQL operators, pass keyworded arguments
        as strings and include the desired operator followed by a space (e.g. position_short='<> 0.00').

        Following operators can be passed:
        LIKE, <, <=, >, >=, <>

        Following parameters can be passed as optional keyworded arguments (kwargs):
        exchange, contract_id, contract_name, prod, delivery_start, delivery_end, internal_trade_id, api_timestamp,
        exec_time, price, quantity, state, buy_delivery_area, sell_delivery_area, buy_order_id, buy_clOrderId, buy_txt,
        buy_aggressor_indicator, buy_portfolio_id, sell_order_id, sell_clOrderId, sell_txt, sell_aggressor_indicator,
        sell_portfolio_id

        Args:
            delivery_area (tuple[str]): Multiple delivery areas inside a tuple. Single del. area can be passed as a string
            as_dataframe (bool): True if output should be DataFrame
            as_objects (bool): True if output should be list of InternalTrades
            **kwargs: any additional fields of SQL table

        Returns:
            list: SQL query
        """
        allowed_kwargs = ["exchange", "contract_id", "contract_name", "prod", "delivery_start", "delivery_end",
                          "internal_trade_id", "api_timestamp", "exec_time", "price", "quantity", "state",
                          "buy_delivery_area", "sell_delivery_area", "buy_order_id", "buy_clOrderId", "buy_txt",
                          "buy_aggressor_indicator", "buy_portfolio_id", "sell_order_id", "sell_clOrderId", "sell_txt",
                          "sell_aggressor_indicator", "sell_portfolio_id"]

        sql_params = self.__handle_sql_args(kwargs, allowed_kwargs)

        if delivery_area:
            for i in delivery_area:
                sql_params += f" AND (buy_delivery_area = '{i}' OR sell_delivery_area = '{i}')"

        with self.__get_session() as session:
            result = session.execute(f"SELECT * FROM internal_trades{sql_params}").fetchall()

        self.logger.info("Successfully exported internal trades")

        # Convert data back to InternalTrade objects
        if result and as_objects:
            internal_trades = [InternalTrade(exchange=i[0],
                                             contract_id=i[1],
                                             contract_name=i[2],
                                             prod=i[3],
                                             delivery_start=i[4],
                                             delivery_end=i[5],
                                             internal_trade_id=i[6],
                                             api_timestamp=i[7],
                                             exec_time=i[8],
                                             price=i[9],
                                             quantity=i[10],
                                             buy_delivery_area=i[11],
                                             sell_delivery_area=i[12],
                                             buy_order_id=i[13],
                                             buy_cl_order_id=i[14],
                                             buy_txt=i[15],
                                             buy_aggressor_indicator=i[16],
                                             buy_portfolio_id=i[17],
                                             sell_order_id=i[18],
                                             sell_cl_order_id=i[19],
                                             sell_txt=i[20],
                                             sell_aggressor_indicator=i[21],
                                             sell_portfolio_id=i[22])
                               for i in result]
            return internal_trades

        if result and as_dataframe:
            return pd.DataFrame(result)

        return result

    @validate_arguments
    def get_signals(self,
                    time_from: datetime = Field(description="Datetime in format yyyy-mm-dd hh:mm:ss"),
                    time_till: datetime = Field(description="Datetime in format yyyy-mm-dd hh:mm:ss"),
                    as_dataframe: bool = True,
                    as_objects: bool = False,
                    **kwargs) -> Union[pd.DataFrame, list[Signal]]:
        """
        Exports signals from SQL database. To use different mathematical/SQL operators, pass keyworded arguments
        as strings and include the desired operator followed by a space (e.g. position_short='<> 0.00').

        Following operators can be passed:
        LIKE, <, <=, >, >=, <>

        Following parameters can be passed as optional keyworded arguments:
        id, source, received_at, revision, delivery_areas, portfolio_ids, tenant_id, position_short,
        position_long, value

        Args:
            time_from (datetime): yyyy-mm-dd hh:mm:ss
            time_till (datetime): yyyy-mm-dd hh:mm:ss
            as_dataframe (bool): True if output should be DataFrame
            as_objects (bool): True if output should be list of Signals
            **kwargs: any additional fields of SQL table

        Returns:
            list: SQL query
        """
        allowed_kwargs = ["id", "source", "received_at", "revision", "delivery_areas", "portfolio_ids", "tenant_id",
                          "position_short", "position_long", "value"]

        sql_params = self.__handle_sql_args(kwargs, allowed_kwargs)
        sql_op = "AND" if sql_params else "WHERE"

        with self.__get_session() as session:
            result = session.execute(f"SELECT * FROM signals{sql_params} "
                                     f"{sql_op} delivery_start >= '{time_from}' "
                                     f"AND delivery_end <= '{time_till}'").fetchall()

        self.logger.info("Successfully exported signals")

        # Convert data back to InternalTrade objects
        if result and as_objects:
            signals = [Signal(id=i[0],
                              source=i[1],
                              received_at=i[2],
                              revision=i[3],
                              delivery_start=i[4],
                              delivery_end=i[5],
                              portfolio_ids=i[6],
                              tenant_id=i[7],
                              position_short=i[8],
                              position_long=i[9],
                              value=i[10])
                       for i in result]

            return signals

        if result and as_dataframe:
            return pd.DataFrame(result)

        return result

    @validate_arguments
    def send_raw_sql(self,
                     sql_statement: str):
        """
        Function allows for raw SQL queries to be sent to the database.

        Args:
            sql_statement (str): SQL query

        Returns:

        """
        with self.__get_session() as session:
            try:
                result = session.execute(sql_statement).fetchall()
            except self.SQL_ERRORS as e:
                return self.logger.error(e)
        return result

    @staticmethod
    def __handle_sql_args(kwargs,
                          allowed_kwargs: list[str]) -> str:
        """
        Handles incoming arguments by adjusting them to be compatible with SQL.

        Args:
            kwargs: **kwargs of export functions
            allowed_kwargs (list[str]): list of allowed kwargs

        Returns:
            str: SQL request
        """
        if not all(arg for arg in kwargs.values()):
            raise SQLExporterError("Some of your input values are invalid or empty!")
        sql_params = ""
        operators = ["LIKE", "<", "<=", ">", ">=", "<>"]

        for keyword, argument in kwargs.items():
            op = "="
            sql_statement = "WHERE" if sql_params == "" else "AND"

            if keyword not in allowed_kwargs:
                raise SQLExporterError(f"{keyword} not in allowed keywords. Allowed keywords: {allowed_kwargs}")
            else:
                if isinstance(argument, str):
                    # Check For SQL Commands Or Mathematical Operators
                    if any(x in argument for x in operators):
                        if len(argument.split(" ")) > 2:
                            op = argument.split(" ")[0]
                            argument = argument.replace(f"{op} ", "")
                        else:
                            op, argument = argument.split(" ")
                        if op == "LIKE":
                            argument = f"%{argument}%"
                        try:
                            datetime.strptime(argument, DATE_YMD_TIME_HMS)
                        except:
                            pass
                elif isinstance(argument, tuple):
                    if len(argument) == 1:
                        argument = argument[0]
                    else:
                        op = "IN"
                elif isinstance(argument, list):
                    for nr, element in enumerate(argument):
                        if not nr:
                            if element == argument[-1]:
                                sql_params += f" {sql_statement} ({keyword} LIKE '%{element}%')"
                            else:
                                sql_params += f" {sql_statement} ({keyword} LIKE '%{element}%'"
                        elif element == argument[-1]:
                            sql_params += f" OR {keyword} LIKE '%{element}%')"
                        else:
                            sql_params += f" OR {keyword} LIKE '%{element}%'"
                    continue

                elif isinstance(argument, dict):
                    op = "IN"
                    temp_list = []
                    for value in argument.values():
                        for item in value:
                            temp_list.append(item)
                    argument = tuple(temp_list)
                try:
                    if not isinstance(argument, tuple) and keyword != "contract_id":
                        argument = float(argument)
                    sql_params += f" {sql_statement} {keyword} {op} {argument}"
                except:
                    sql_params += f" {sql_statement} {keyword} {op} '{argument}'"

        return sql_params

    def __convert_dataframe(self,
                            df_type: str,
                            dataframe: pandas_DataFrame) -> dict[str, pd.DataFrame]:
        """
        Function to convert dataframe to required format to be processed by backtesting data pipeline.

        Args:
            df_type (str): orders/trades/orderbooks
            dataframe (DataFrame): DataFrame containing exported Data

        Returns:
            dict{key: DataFrame}: Dictionary of DataFrames
        """
        output = {}

        contract_ids = dataframe.contract_id.unique().tolist()
        contracts = self.get_contracts(contract_id=contract_ids)

        if df_type == "trades":
            dataframe = dataframe.astype({'price': 'float', 'quantity': 'float'})

        elif df_type == "orders":
            dataframe["bids"] = [json.loads(i) if i else None for i in dataframe.bids.tolist()]
            dataframe["asks"] = [json.loads(i) if i else None for i in dataframe.asks.tolist()]

        # 	dataframe.drop(columns=["bids", "asks"])
        # 	orders = []
        # 	for nr, val in enumerate(bids):
        # 		orders.append({"bid": val, "ask": asks[nr]})
        # 	dataframe["orders"] = orders

        for row_nr, row_id in enumerate(contracts.contract_id):
            key = f"{contracts.iloc[row_nr].delivery_start.strftime(DATE_YMD_TIME_HM)} - " \
                  f"{contracts.iloc[row_nr].delivery_end.strftime(TIME_HM)}"

            if key not in [*output]:
                output[key] = dataframe[dataframe["contract_id"] == row_id]
            else:
                output[key].append(dataframe[dataframe["contract_id"] == row_id])

        return output
