from datetime import datetime
import numpy as np
from numba import jit, njit
import numba
from numba.typed import List

from exetera.core import validation as val
from exetera.core.abstract_types import Field
from exetera.core import fields, utils

DEFAULT_CHUNKSIZE = 1 << 20
INVALID_INDEX = 1 << 62
MAX_DATETIME = datetime(year=3000, month=1, day=1) #.timestamp()


def chunks(length, chunksize=1 << 20):
    cur = 0
    while cur < length:
        next_ = min(length, cur + chunksize)
        yield cur, next_
        cur = next_


def safe_map(field, map_field, map_filter, empty_value=None):
    if isinstance(field, Field):
        if field.indexed:
            return safe_map_indexed_values(
                field.indices[:], field.values[:], map_field, map_filter, empty_value)
        else:
            return safe_map_values(field.data[:], map_field, map_filter, empty_value)
    elif isinstance(field, np.ndarray):
        return safe_map_values(field, map_field, map_filter, empty_value)


@njit
def safe_map_indexed_values(data_indices, data_values, map_field, map_filter, empty_value=None):
    empty_value_len = 0 if empty_value is None else len(empty_value)
    value_length = 0
    for i in range(len(map_field)):
        if map_filter[i]:
            value_length += data_indices[map_field[i]+1] - data_indices[map_field[i]]
        else:
            value_length += empty_value_len

    i_result = np.zeros(len(map_field) + 1, dtype=data_indices.dtype)
    v_result = np.zeros(value_length, dtype=data_values.dtype)

    offset = 0
    i_result[0] = 0
    for i in range(len(map_field)):
        if map_filter[i]:
            sst = data_indices[map_field[i]]
            sse = data_indices[map_field[i]+1]
            dst = offset
            delta = sse - sst
            dse = offset + delta
            i_result[i+1] = dse
            v_result[dst:dse] = data_values[sst:sse]
            offset += delta
        else:
            dst = offset
            dse = offset + empty_value_len
            i_result[i+1] = dse
            if empty_value is not None:
                v_result[dst:dse] = empty_value
            offset += dse - dst

    return i_result, v_result


@njit
def safe_map_values(data_field, map_field, map_filter, empty_value=None):
    result = np.zeros_like(map_field, dtype=data_field.dtype)
    empty_val = result[0] if empty_value is None else empty_value
    for i in range(len(map_field)):
        if map_filter[i]:
            result[i] = data_field[map_field[i]]
        else:
            result[i] = empty_val
    return result


@njit
def map_valid(data_field, map_field, result=None):
    if result is None:
        result = np.zeros_like(map_field, dtype=data_field.dtype)
    for i in range(len(map_field)):
        if map_field[i] < INVALID_INDEX:
            result[i] = data_field[map_field[i]]
    return result


def ordered_map_valid_stream(data_field, map_field, result_field, chunksize=DEFAULT_CHUNKSIZE):
    df_it = iter(chunks(len(data_field.data), chunksize=chunksize))
    mf_it = iter(chunks(len(map_field.data), chunksize=chunksize))
    df_range = next(df_it)
    mf_range = next(mf_it)
    dfc = data_field.data[df_range[0]:df_range[1]]
    mfc = map_field.data[mf_range[0]:mf_range[1]]

    is_field_parameter = val.is_field_parameter(result_field)
    result_dtype = result_field.data.dtype if is_field_parameter else result_field.dtype
    rslt = np.zeros(chunksize, dtype=result_dtype)

    m = 0
    d = 0
    while m < len(map_field.data):
        mm, dd = ordered_map_valid_partial(df_range[0], dfc, mfc, rslt)
        if mm > 0:
            if is_field_parameter:
                result_field.data.write(rslt[:mm])
            else:
                result_field[m:m + mm] = rslt[:mm]
            rslt[:] = 0
        m += mm
        if m == mf_range[1] and m < len(map_field.data):
            mf_range = next(mf_it)
            mfc = map_field.data[mf_range[0]:mf_range[1]]
        else:
            mfc = mfc[mm:]
        if dd >= df_range[1] and dd < len(data_field.data):
            df_range = next(df_it)
            dfc = data_field.data[df_range[0]:df_range[1]]
        # else:
        #     dfc = dfc[dd:]


# 0 2 3 4 5 7 8 9 11 12 14 15 17 18 19
# 0 0 0 0 1 1 1 1  2  2  2  2  3  3  3
@njit
def ordered_map_valid_partial(d, data_field, map_field, result):
    i = 0
    while True:
        val = map_field[i]
        if val < INVALID_INDEX:
            if val >= d + len(data_field):
                # need a new data_field chunk
                return i, val
            result[i] = data_field[val - d]
        i += 1
        if i >= len(map_field):
            return i, val


@njit
def data_iterator(data_field, chunksize=1 << 20):
    cur = np.int64(0)
    chunks_ = chunks(len(data_field.data), chunksize)
    for c in chunks_:
        start = c[0]
        data = data_field.data[start:start + chunksize * 2]
        for v in range(c[0], c[1]):
            yield data[v]

@njit
def apply_filter_to_index_values(index_filter, indices, values):
    # pass 1 - determine the destination lengths
    cur_ = indices[:-1]
    next_ = indices[1:]
    count = 0
    total = 0
    for i in range(len(index_filter)):
        if index_filter[i] == True:
            count += 1
            total += next_[i] - cur_[i]
    dest_indices = np.zeros(count+1, indices.dtype)
    dest_values = np.zeros(total, values.dtype)
    dest_indices[0] = 0
    count = 1
    total = 0
    for i in range(len(index_filter)):
        if index_filter[i] == True:
            n = next_[i]
            c = cur_[i]
            delta = n - c
            dest_values[total:total + delta] = values[c:n]
            total += delta
            dest_indices[count] = total
            count += 1
    return dest_indices, dest_values


@njit
def apply_indices_to_index_values(indices_to_apply, indices, values):
    # pass 1 - determine the destination lengths
    cur_ = indices[:-1]
    next_ = indices[1:]
    count = 0
    total = 0
    for i in indices_to_apply:
        count += 1
        total += next_[i] - cur_[i]
    dest_indices = np.zeros(count+1, indices.dtype)
    dest_values = np.zeros(total, values.dtype)
    dest_indices[0] = 0
    count = 1
    total = 0
    for i in indices_to_apply:
        n = next_[i]
        c = cur_[i]
        delta = n - c
        dest_values[total:total + delta] = values[c:n]
        total += delta
        dest_indices[count] = total
        count += 1
    return dest_indices, dest_values


def get_spans_for_field(ndarray):
    results = np.zeros(len(ndarray) + 1, dtype=bool)
    if np.issubdtype(ndarray.dtype, np.number):
        fn = np.not_equal
    else:
        fn = np.char.not_equal
    results[1:-1] = fn(ndarray[:-1], ndarray[1:])

    results[0] = True
    results[-1] = True
    return np.nonzero(results)[0]


@njit
def _get_spans_for_2_fields_by_spans(span0, span1):
    spans = []
    j=0
    for i in range(len(span0)):
        if j<len(span1):
            while span1[j] < span0[i]:
                spans.append(span1[j])
                j += 1
            if span1[j] == span0[i]:
                j += 1
        spans.append(span0[i])
    if j<len(span1): #if two ndarray are not equally sized
        spans.extend(span1[j:])
    return spans


@njit
def _get_spans_for_2_fields(ndarray0, ndarray1):
    count = 0
    spans = np.zeros(len(ndarray0)+1, dtype=np.uint32)
    spans[0] = 0
    for i in np.arange(1, len(ndarray0)):
        if ndarray0[i] != ndarray0[i-1] or ndarray1[i] != ndarray1[i-1]:
            count += 1
            spans[count] = i
    spans[count+1] = len(ndarray0)
    return spans[:count+2]


@njit
def _get_spans_for_index_string_field(indices,values):
    result = []
    result.append(0)
    for i in range(1, len(indices) - 1):
        last = indices[i - 1]
        current = indices[i]
        next = indices[i + 1]
        if next - current != current - last:  # compare size first
            result.append(i)
            continue
        if not np.array_equal(values[last:current], values[current:next]):
            result.append(i)
    result.append(len(indices) - 1)  # total number of elements
    return result


@njit
def apply_spans_index_of_min(spans, src_array, dest_array):
    for i in range(len(spans)-1):
        cur = spans[i]
        next = spans[i+1]

        if next - cur == 1:
            dest_array[i] = cur
        else:
            dest_array[i] = cur + src_array[cur:next].argmin()

    return dest_array


@njit
def apply_spans_index_of_min_indexed(spans, src_indices, src_values, dest_array):
    for i in range(len(spans)-1):
        cur = spans[i]
        next = spans[i+1]

        if next - cur == 1:
            dest_array[i] = cur
        else:
            minind = cur
            minstart = src_indices[cur]
            minend = src_indices[cur+1]
            minlen = minend - minstart
            for j in range(cur+1, next):
                curstart = src_indices[j]
                curend = src_indices[j+1]
                curlen = curend - curstart
                shortlen = min(curlen, minlen)
                found = False
                for k in range(shortlen):
                    if src_values[curstart+k] < src_values[minstart+k]:
                        minind = j
                        minstart = curstart
                        minend = curend
                        found = True
                        break
                    elif src_values[curstart+k] > src_values[minstart+k]:
                        found = True
                        break
                if not found and curlen < minlen:
                    minind = j
                    minstart = curstart
                    minend = curend

            dest_array[i] = minind

    return dest_array


@njit
def apply_spans_index_of_max_indexed(spans, src_indices, src_values, dest_array):
    for i in range(len(spans)-1):
        cur = spans[i]
        next = spans[i+1]

        if next - cur == 1:
            dest_array[i] = cur
        else:
            minind = cur
            minstart = src_indices[cur]
            minend = src_indices[cur+1]
            minlen = minend - minstart
            for j in range(cur+1, next):
                curstart = src_indices[j]
                curend = src_indices[j+1]
                curlen = curend - curstart
                shortlen = min(curlen, minlen)
                found = False
                for k in range(shortlen):
                    if src_values[curstart+k] > src_values[minstart+k]:
                        minind = j
                        minstart = curstart
                        minlen = curend - curstart
                        found = True
                        break
                    elif src_values[curstart+k] < src_values[minstart+k]:
                        found = True
                        break
                if not found and curlen > minlen:
                    minind = j
                    minstart = curstart
                    minlen = curend - curstart

            dest_array[i] = minind

    return dest_array


@njit
def apply_spans_index_of_max(spans, src_array, dest_array):
    for i in range(len(spans)-1):
        cur = spans[i]
        next = spans[i+1]

        if next - cur == 1:
            dest_array[i] = cur
        else:
            dest_array[i] = cur + src_array[cur:next].argmax()

    return dest_array


@njit
def apply_spans_index_of_first(spans, dest_array):
    dest_array[:] = spans[:-1]


@njit
def apply_spans_index_of_last(spans, dest_array):
    dest_array[:] = spans[1:] - 1


@njit
def apply_spans_index_of_min_filter(spans, src_array, dest_array, filter_array):
    for i in range(len(spans) - 1):
        cur = spans[i]
        next = spans[i + 1]
        if next - cur == 0:
            filter_array[i] = False
        elif next - cur == 1:
            filter_array[i] = True
            dest_array[i] = cur
        else:
            filter_array[i] = True
            dest_array[i] = cur + src_array[cur:next].argmin()

    return dest_array, filter_array


@njit
def apply_spans_index_of_max_filter(spans, src_array, dest_array, filter_array):
    for i in range(len(spans) - 1):
        cur = spans[i]
        next = spans[i + 1]
        if next - cur == 0:
            filter_array[i] = False
        elif next - cur == 1:
            filter_array[i] = True
            dest_array[i] = cur
        else:
            filter_array[i] = True
            dest_array[i] = cur + src_array[cur:next].argmax()

    return dest_array, filter_array


@njit
def apply_spans_index_of_first_filter(spans, dest_array, filter_array):
    for i in range(len(spans) - 1):
        cur = spans[i]
        next = spans[i + 1]
        if next - cur == 0:
            filter_array[i] = False
        else:
            filter_array[i] = True
            dest_array[i] = spans[i]

    return dest_array, filter_array


@njit
def apply_spans_index_of_last_filter(spans, dest_array, filter_array):
    for i in range(len(spans) - 1):
        cur = spans[i]
        next = spans[i + 1]
        if next - cur == 0:
            filter_array[i] = False
        else:
            filter_array[i] = True
            dest_array[i] = spans[i+1]-1

    return dest_array, filter_array


@njit
def apply_spans_count(spans, dest_array):
    for i in range(len(spans)-1):
        dest_array[i] = np.int64(spans[i+1] - spans[i])


@njit
def apply_spans_first(spans, src_array, dest_array):
    dest_array[:] = src_array[spans[:-1]]


@njit
def apply_spans_last(spans, src_array, dest_array):
    spans = spans[1:]-1
    dest_array[:] = src_array[spans]


@njit
def apply_spans_max(spans, src_array, dest_array):

    for i in range(len(spans)-1):
        cur = spans[i]
        next = spans[i+1]
        if next - cur == 1:
            dest_array[i] = src_array[cur]
        else:
            dest_array[i] = src_array[cur:next].max()


@njit
def apply_spans_min(spans, src_array, dest_array):

    for i in range(len(spans)-1):
        cur = spans[i]
        next = spans[i+1]
        if next - cur == 1:
            dest_array[i] = src_array[cur]
        else:
            dest_array[i] = src_array[cur:next].min()


# def _apply_spans_concat(spans, src_field):
#     dest_values = [None] * (len(spans)-1)
#     for i in range(len(spans)-1):
#         cur = spans[i]
#         next = spans[i+1]
#         if next - cur == 1:
#             dest_values[i] = src_field[cur]
#         else:
#             src = [s for s in src_field[cur:next] if len(s) > 0]
#             if len(src) > 0:
#                 dest_values[i] = ','.join(utils.to_escaped(src))
#             else:
#                 dest_values[i] = ''
#             # if len(dest_values[i]) > 0:
#             #     print(dest_values[i])
#     return dest_values


@njit
def apply_spans_concat(spans, src_index, src_values, dest_index, dest_values,
                       max_index_i, max_value_i, s_start):
    separator = np.frombuffer(b',', dtype=np.uint8)[0]
    delimiter = np.frombuffer(b'"', dtype=np.uint8)[0]
    if s_start == 0:
        index_i = np.uint32(1)
        index_v = np.int64(0)
        dest_index[0] = spans[0]
    else:
        index_i = np.uint32(0)
        index_v = np.int64(0)

    s_end = len(spans)-1
    for s in range(s_start, s_end):
        cur = spans[s]
        next = spans[s+1]
        cur_src_i = src_index[cur]
        next_src_i = src_index[next]

        dest_index[index_i] = next_src_i
        index_i += 1

        if next_src_i - cur_src_i > 1:
            if next - cur == 1:
                # only one entry to be copied, so commas not required
                next_index_v = next_src_i - cur_src_i + np.int64(index_v)
                dest_values[index_v:next_index_v] = src_values[cur_src_i:next_src_i]
                index_v = next_index_v
            else:
                # check to see how many non-zero-length entries there are; >1 means we must
                # separate them by commas
                non_empties = 0
                for e in range(cur, next):
                   if src_index[e] < src_index[e+1]:
                       non_empties += 1
                if non_empties == 1:
                    # only one non-empty entry to be copied, so commas not required
                    next_index_v = next_src_i - cur_src_i + np.int64(index_v)
                    dest_values[index_v:next_index_v] = src_values[cur_src_i:next_src_i]
                    index_v = next_index_v
                else:
                    # the outer conditional already determines that we have a non-empty entry
                    # so there must be multiple non-empty entries and commas are required
                    for e in range(cur, next):
                        src_start = src_index[e]
                        src_end = src_index[e+1]
                        comma = False
                        quotes = False
                        for i_c in range(src_start, src_end):
                            if src_values[i_c] == separator:
                                comma = True
                            elif src_values[i_c] == delimiter:
                                quotes = True

                        d_index = np.int64(0)
                        if comma or quotes:
                            dest_values[d_index] = delimiter
                            d_index += 1
                            for i_c in range(src_start, src_end):
                                if src_values[i_c] == delimiter:
                                    dest_values[d_index] = src_values[i_c]
                                    d_index += 1
                                dest_values[d_index] = src_values[i_c]
                                d_index += 1
                            dest_values[d_index] = delimiter
                            d_index += 1
                        else:
                            s_len = np.int64(src_end - src_start)
                            dest_values[index_v:index_v + s_len] = src_values[src_start:src_end]
                            d_index += s_len
                        index_v += np.int64(d_index)

        # if either the index or values are past the threshold, write them
        if index_i >= max_index_i or index_v >= max_value_i:
            break
    return s+1, index_i, index_v


# def ordered_map_to_right_left_unique_streamed(left, right, left_to_right, chunksize=1 << 20):
#     i = 0
#     j = 0
#     lc_it = iter(chunks(len(left.data), chunksize))
#     lc_range = next(lc_it)
#     rc_it = iter(chunks(len(right.data), chunksize))
#     rc_range = next(rc_it)
#     lc = left.data[lc_range[0]:lc_range[1]]
#     rc = right.data[rc_range[0]:rc_range[1]]
#     acc_written = 0
#
#     ltri = np.zeros(chunksize, dtype=left_to_right.data.dtype)
#     unmapped = 0
#     while i < len(left.data) and j < len(right.data):
#         ii, jj, u = ordered_map_to_right_left_unique_partial(i, lc, rc, ltri)
#         unmapped += u
#         if jj > 0:
#             if val.is_field_parameter(left_to_right):
#                 left_to_right.data.write(ltri[:jj])
#             else:
#                 left_to_right[acc_written:acc_written + jj] = ltri[:jj]
#                 acc_written += jj
#         i += ii
#         j += jj
#         if i > lc_range[1]:
#             raise ValueError("'i' has got ahead of current chunk; this shouldn't happen")
#         if j > rc_range[1]:
#             raise ValueError("'j' has got ahead of current chunk; this shouldn't happen")
#         if i == lc_range[1] and i < len(left.data):
#             lc_range = next(lc_it)
#             lc = left.data[lc_range[0]:lc_range[1]]
#         else:
#             lc = lc[ii:]
#         if j == rc_range[1] and j < len(right.data):
#             rc_range = next(rc_it)
#             rc = right.data[rc_range[0]:rc_range[1]]
#         else:
#             rc = rc[jj:]
#     return unmapped > 0


# @njit
# def ordered_map_to_right_left_unique_partial(d_i, left, right, left_to_right):
#     """
#     Returns:
#     [0]: how many positions forward i moved
#     [1]: how many positions forward j moved
#     [2]: how many elements were written
#     """
#     i = 0
#     j = 0
#     unmapped = 0
#     while i < len(left) and j < len(right):
#         if left[i] < right[j]:
#             i += 1
#         elif left[i] > right[j]:
#             left_to_right[j] = INVALID_INDEX
#             j += 1
#             unmapped += 1
#         else:
#             left_to_right[j] = i + d_i
#             if j+1 < len(right) and right[j+1] != right[j]:
#                 i += 1
#             j += 1
#     return i, j, unmapped


def ordered_map_to_right_right_unique_streamed(left, right, left_to_right, chunksize=1 << 20):
    i = 0
    j = 0
    lc_it = iter(chunks(len(left.data), chunksize))
    lc_range = next(lc_it)
    rc_it = iter(chunks(len(right.data), chunksize))
    rc_range = next(rc_it)
    lc = left.data[lc_range[0]:lc_range[1]]
    rc = right.data[rc_range[0]:rc_range[1]]
    acc_written = 0

    is_field_parameter = val.is_field_parameter(left_to_right)
    result_dtype = left_to_right.data.dtype if is_field_parameter else left_to_right.dtype
    ltri = np.zeros(chunksize, dtype=result_dtype)
    unmapped = 0
    while i < len(left.data) and j < len(right.data):
        ii, jj, u = ordered_map_to_right_right_unique_partial(j, lc, rc, ltri)
        unmapped += u
        if ii > 0:
            if is_field_parameter:
                left_to_right.data.write(ltri[:ii])
            else:
                left_to_right[acc_written:acc_written + ii] = ltri[:ii]
                acc_written += ii
        i += ii
        j += jj
        if i > lc_range[1]:
            raise ValueError("'i' has got ahead of current chunk; this shouldn't happen")
        if j > rc_range[1]:
            raise ValueError("'j' has got ahead of current chunk; this shouldn't happen")
        if i == lc_range[1] and i < len(left.data):
            lc_range = next(lc_it)
            lc = left.data[lc_range[0]:lc_range[1]]
        else:
            lc = lc[ii:]
        if j == rc_range[1] and j < len(right.data):
            rc_range = next(rc_it)
            rc = right.data[rc_range[0]:rc_range[1]]
        else:
            rc = rc[jj:]
    return unmapped > 0


@njit
def ordered_map_to_right_right_unique_partial(d_j, left, right, left_to_right):
    """
    Returns:
    [0]: how many positions forward i moved
    [1]: how many positions forward j moved
    [2]: how many elements were written
    """
    i = 0
    j = 0
    unmapped = 0
    while i < len(left) and j < len(right):
        if left[i] < right[j]:
            left_to_right[i] = INVALID_INDEX
            i += 1
            unmapped += 1
        elif left[i] > right[j]:
            j += 1
        else:
            left_to_right[i] = j + d_j
            if i+1 >= len(left) or left[i+1] != left[i]:
                j += 1
            i += 1
            # if j+1 < len(right) and right[j+1] != right[j]:
            #     i += 1
            # j += 1
    return i, j, unmapped


@njit
def ordered_map_to_right_right_unique(first, second, result):
    if len(first) != len(result):
        msg = "'first' and 'result' must be the same length"
        raise ValueError(msg)
    i = 0
    j = 0
    unmapped = 0
    while i < len(first) and j < len(second):
        if first[i] < second[j]:
            result[i] = INVALID_INDEX
            i += 1
            unmapped += 1
        elif first[i] > second[j]:
            j += 1
        else:
            result[i] = j
            if i+1 >= len(first) or first[i+1] != first[i]:
                j += 1
            i += 1

    while j < len(second):
        result[j] = INVALID_INDEX
        j += 1

    return unmapped > 0


@njit
def ordered_map_to_right_both_unique(first, second, result):
    if len(first) != len(result):
        msg = "'second' and 'result' must be the same length"
        raise ValueError(msg)
    i = 0
    j = 0

    unmapped = 0
    while i < len(first) and j < len(second):
        if first[i] < second[j]:
            result[i] = INVALID_INDEX
            i += 1
            unmapped += 1
        elif first[i] > second[j]:
            j += 1
        else:
            result[i] = j
            i += 1
            j += 1

    while i < len(first):
        result[i] = INVALID_INDEX
        i += 1

    return unmapped > 0


@njit
def ordered_inner_map_result_size(left, right):
    i = 0
    j = 0
    result_size = 0

    while i < len(left) and j < len(right):
        if left[i] < right[j]:
            i += 1
        elif left[i] > right[j]:
            j += 1
        else:
            cur_i_count = 1
            while i+1 < len(left) and left[i + 1] == left[i]:
                cur_i_count += 1
                i += 1
            cur_j_count = 1
            while j+1 < len(right) and right[j + 1] == right[j]:
                cur_j_count += 1
                j += 1
            result_size += cur_i_count * cur_j_count
            i += 1
            j += 1
    return result_size


@njit
def ordered_outer_map_result_size_both_unique(left, right):
    i = 0
    j = 0
    result_size = 0
    while i < len(left) and j < len(right):
        if left[i] < right[j]:
            i += 1
        elif left[i] > right[j]:
            j += 1
        else:
            i += 1
            j += 1
        result_size += 1
    while i < len(left):
        i += 1
        result_size += 1
    while j < len(right):
        j += 1
        result_size += 1
    return result_size


@njit
def ordered_inner_map_both_unique(left, right, left_to_inner, right_to_inner):
    i = 0
    j = 0
    cur_m = 0
    while i < len(left) and j < len(right):
        if left[i] < right[j]:
            i += 1
        elif left[i] > right[j]:
            j += 1
        else:
            left_to_inner[cur_m] = i
            right_to_inner[cur_m] = j
            cur_m += 1
            i += 1
            j += 1


def ordered_inner_map_left_unique_streamed(left, right, left_to_inner, right_to_inner,
                                           chunksize=1 << 20):
    i = 0
    j = 0
    left_chunks = chunks(len(left.data), 4)
    right_chunks = chunks(len(right.data), 4)

    lc_it = iter(left_chunks)
    lc_range = next(lc_it)
    rc_it = iter(right_chunks)
    rc_range = next(rc_it)
    lc = left.data[lc_range[0]:lc_range[1]]
    rc = right.data[rc_range[0]:rc_range[1]]

    lti = np.zeros(4, dtype=left_to_inner.data.dtype)
    rti = np.zeros(4, dtype=right_to_inner.data.dtype)
    while i < len(left.data) and j < len(right.data):
        ii, jj, m = ordered_inner_map_left_unique_partial(i, j, lc, rc, lti, rti)
        if m > 0:
            left_to_inner.data.write(lti[:m])
            right_to_inner.data.write(rti[:m])
        i += ii
        j += jj
        if i > lc_range[1]:
            raise ValueError("'i' has got ahead of current chunk; this shouldn't happen")
        if j > rc_range[1]:
            raise ValueError("'j' has got ahead of current chunk; this shouldn't happen")
        if i == lc_range[1] and i < len(left.data):
            lc_range = next(lc_it)
            lc = left.data[lc_range[0]:lc_range[1]]
        else:
            lc = lc[ii:]
        if j == rc_range[1] and j < len(right.data):
            rc_range = next(rc_it)
            rc = right.data[rc_range[0]:rc_range[1]]
        else:
            rc = rc[jj:]

@njit
def ordered_inner_map_left_unique_partial(d_i, d_j, left, right,
                                          left_to_inner, right_to_inner):
    """
    Returns:
    [0]: how many positions forward i moved
    [1]: how many positions forward j moved
    [2]: how many elements were written
    """
    i = 0
    j = 0
    m = 0
    while i < len(left) and j < len(right) and m < len(left_to_inner):
        if left[i] < right[j]:
            i += 1

        elif left[i] > right[j]:
            j += 1
        else:
            left_to_inner[m] = i + d_i
            right_to_inner[m] = j + d_j
            m += 1
            if j+1 >= len(right) or right[j+1] != right[j]:
                i += 1
            j += 1
    return i, j, m


@njit
def ordered_inner_map_left_unique(left, right, left_to_inner, right_to_inner):
    i = 0
    j = 0
    cur_m = 0
    while i < len(left) and j < len(right):
        if left[i] < right[j]:
            i += 1
        elif left[i] > right[j]:
            j += 1
        else:
            cur_j = j
            while cur_j + 1 < len(right) and right[cur_j + 1] == right[cur_j]:
                cur_j += 1
            for jj in range(j, cur_j+1):
                left_to_inner[cur_m] = i
                right_to_inner[cur_m] = jj
                cur_m += 1
            i += 1
            j = cur_j + 1


@njit
def ordered_inner_map(left, right, left_to_inner, right_to_inner):
    i = 0
    j = 0
    cur_m = 0
    while i < len(left) and j < len(right):
        if left[i] < right[j]:
            i += 1
        elif left[i] > right[j]:
            j += 1
        else:
            cur_i = i
            while cur_i + 1 < len(left) and left[cur_i + 1] == left[cur_i]:
                cur_i += 1
            cur_j = j
            while cur_j + 1 < len(right) and right[cur_j + 1] == right[cur_j]:
                cur_j += 1
            for ii in range(i, cur_i+1):
                for jj in range(j, cur_j+1):
                    left_to_inner[cur_m] = ii
                    right_to_inner[cur_m] = jj
                    cur_m += 1
            i = cur_i + 1
            j = cur_j + 1


@njit
def ordered_get_last_as_filter(field):
    result = np.zeros(len(field), dtype=numba.types.boolean)
    for i in range(len(field)-1):
        result[i] = field[i] != field[i+1]
    result[-1] = True
    return result


@njit
def ordered_generate_journalling_indices(old, new):
    i = 0
    j = 0
    total = 0
    while i < len(old) and j < len(new):
        if old[i] < new[j]:
            while i+1 < len(old) and old[i+1] == old[i]:
                i += 1
            i += 1
            total += 1
        elif old[i] > new[j]:
            j += 1
            total += 1
        else:
            while i+1 < len(old) and old[i+1] == old[i]:
                i += 1
            i += 1
            j += 1
            total += 1
    while i < len(old):
        while i+1 < len(old) and old[i+1] == old[i]:
            i += 1
        i += 1
        total += 1
    while j < len(new):
        j += 1
        total += 1

    old_inds = np.full(total, -1, dtype=np.int64)
    new_inds = np.full(total, -1, dtype=np.int64)

    i = 0
    j = 0
    joint = 0
    while i < len(old) and j < len(new):
        if old[i] < new[j]:
            while i+1 < len(old) and old[i+1] == old[i]:
                i += 1
            old_inds[joint] = i
            new_inds[joint] = -1
            i += 1
            joint += 1
        elif old[i] > new[j]:
            old_inds[joint] = -1
            new_inds[joint] = j
            j += 1
            joint += 1
        else:
            while i+1 < len(old) and old[i+1] == old[i]:
                i += 1
            old_inds[joint] = i
            new_inds[joint] = j
            i += 1
            j += 1
            joint += 1

    while i < len(old):
        while i+1 < len(old) and old[i+1] == old[i]:
            i += 1
        old_inds[joint] = i
        new_inds[joint] = -1
        i += 1
        joint += 1
    while j < len(new):
        old_inds[joint] = -1
        new_inds[joint] = j
        j += 1
        joint += 1

    return old_inds, new_inds


@njit
def compare_rows_for_journalling(old_map, new_map, old_field, new_field, to_keep):
    for i in range(len(old_map)):
        if to_keep[i] == False:
            if old_map[i] == -1:
                # row is new so must be kept
                to_keep[i] = True
            elif new_map[i] == -1:
                # row has been removed so don't count as kept
                to_keep[i] = False
            else:
                to_keep[i] = old_field[old_map[i]] != new_field[new_map[i]]


@njit
def compare_indexed_rows_for_journalling(old_map, new_map,
                                         old_indices, old_values, new_indices, new_values,
                                         to_keep):
    assert len(old_map) == len(new_map)
    assert old_indices[-1] == len(old_values)
    assert new_indices[-1] == len(new_values)
    for i in range(len(old_map)):
        if to_keep[i] == False:
            if old_map[i] == -1:
                # row is new so must be kept
                to_keep[i] = True
            elif new_map[i] == -1:
                # row has been removed so don't count as kept
                to_keep[i] = False
            else:
                old_value = old_values[old_indices[old_map[i]]:old_indices[old_map[i]+1]]
                new_value = new_values[new_indices[new_map[i]]:new_indices[new_map[i]+1]]
                to_keep[i] = not np.array_equal(old_value, new_value)


@njit
def merge_journalled_entries(old_map, new_map, to_keep, old_src, new_src, dest):
    cur_old = 0
    cur_dest = 0
    for i in range(len(old_map)):
        # copy all rows up to the entry, if there are any
        while cur_old <= old_map[i]:
            dest[cur_dest] = old_src[cur_old]
            cur_old += 1
            cur_dest += 1
        # copy the new row if there is one
        if to_keep[i] == True:
            dest[cur_dest] = new_src[new_map[i]]
            cur_dest += 1

# def merge_journalled_entries(old_map, new_map, to_keep, old_src, new_src, dest):
#     for om, im, tk in zip(old_map, new_map, to_keep):
#         for omi in old_map:
#             dest.add_to(next(old_src))
#         if tk:
#             dest.add_to(next(new_src))


@njit
def merge_indexed_journalled_entries_count(old_map, new_map, to_keep, old_src_inds, new_src_inds):
    cur_old = 0
    acc_val = 0
    for i in range(len(old_map)):
        while cur_old <= old_map[i]:
            ind_delta = old_src_inds[cur_old+1] - old_src_inds[cur_old]
            acc_val += ind_delta
            cur_old += 1
        if to_keep[i] == True:
            ind_delta = new_src_inds[new_map[i]+1] - new_src_inds[new_map[i]]
            acc_val += ind_delta
    return acc_val


@njit
def merge_indexed_journalled_entries(old_map, new_map, to_keep,
                                    old_src_inds, old_src_vals,
                                    new_src_inds, new_src_vals,
                                    dest_inds, dest_vals):
    cur_old = 0
    cur_dest = 1
    ind_acc = 0
    dest_inds[0] = 0
    for i in range(len(old_map)):
        # copy all rows up to the entry, if there are any
        while cur_old <= old_map[i]:
            ind_delta = old_src_inds[cur_old + 1] - old_src_inds[cur_old]
            ind_acc += ind_delta
            dest_inds[cur_dest] = ind_acc
            if ind_delta > 0:
                dest_vals[ind_acc-ind_delta:ind_acc] = \
                    old_src_vals[old_src_inds[cur_old]:old_src_inds[cur_old + 1]]
            cur_old += 1
            cur_dest += 1
        # copy the new row if there is one
        if to_keep[i] == True:
            ind_delta = new_src_inds[new_map[i] + 1] - new_src_inds[new_map[i]]
            ind_acc += ind_delta
            dest_inds[cur_dest] = ind_acc
            if ind_delta > 0:
                dest_vals[ind_acc-ind_delta:ind_acc] = \
                    new_src_vals[new_src_inds[new_map[i]]:new_src_inds[new_map[i] + 1]]
            cur_dest += 1


def merge_entries_segment(i_start, cur_old_start,
                          old_map, new_map, to_keep, old_src, new_src, dest):
    """
    :param i_start: the initial value to apply to 'i'
    :param cur_old_start: the initial value to apply to 'cur_old
    :param old_map: the map (in i-space) for the existing records
    :param new_map: the map (in i-space) for the new records
    :param to_keep: the flags (in i-space) indicating whether the new record should be kept
    :param old_src: the source for the existing records
    :param new_src: the source for the new records
    :param dest: the sink for the merged sources
    :return:
    """
    cur_old = cur_old_start
    cur_dest = 0
    for i in range(i_start, len(old_map)):
        # copy all rows up to the entry, if there are any
        while cur_old <= old_map[i]:
            dest[cur_dest] = old_src[cur_old]
            cur_old += 1
            cur_dest += 1
            if cur_dest == len(dest):
                return i, cur_old
        # copy the new row if there is one
        if to_keep[i] == True:
            dest[cur_dest] = new_src[new_map[i]]
            cur_dest += 1
            if cur_dest == len(dest):
                return i, cur_old


def streaming_sort_merge(src_index_f, src_value_f, tgt_index_f, tgt_value_f,
                         segment_length, chunk_length):

    # get the number of segments
    segment_count = len(src_index_f.data) // segment_length
    if len(src_index_f.data) % segment_length != 0:
        segment_count += 1

    segment_starts = np.zeros(segment_count, dtype=np.int64)
    segment_lengths = np.zeros(segment_count, dtype=np.int64)
    for i, s in enumerate(utils.chunks(len(src_index_f.data), segment_length)):
        segment_starts[i] = s[0]
        segment_lengths[i] = s[1] - s[0]

    # original segment indices for debugging
    segment_indices = np.zeros(segment_count, dtype=np.int32)

    # the index of the chunk within a given segment
    chunk_indices = np.zeros(segment_count, dtype=np.int64)

    # the (chunk_local) index for each segment
    in_chunk_indices = np.zeros(segment_count, dtype=np.int64)

    # the (chunk_local) length for each segment
    in_chunk_lengths = np.zeros(segment_count, dtype=np.int64)

    src_value_chunks = List()
    src_index_chunks = List()

    # get the first chunk for each segment
    for i in range(segment_count):
        index_start = segment_starts[i] + chunk_indices[i] * chunk_length
        src_value_chunks.append(src_value_f.data[index_start:index_start+chunk_length])
        src_index_chunks.append(src_index_f.data[index_start:index_start+chunk_length])
        in_chunk_lengths[i] = len(src_value_chunks[i])

    dest_indices = np.zeros(segment_count * chunk_length, dtype=src_index_f.data.dtype)
    dest_values = np.zeros(segment_count * chunk_length, dtype=src_value_f.data.dtype)

    target_index = 0
    while target_index < len(src_index_f.data):
        index_delta = streaming_sort_partial(in_chunk_indices, in_chunk_lengths,
                                             src_value_chunks, src_index_chunks,
                                             dest_values, dest_indices)
        tgt_index_f.data.write(dest_indices[:index_delta])
        tgt_value_f.data.write(dest_values[:index_delta])
        target_index += index_delta

        chunk_filter = np.ones(segment_count, dtype=bool)
        for i in range(segment_count):
            if in_chunk_indices[i] == in_chunk_lengths[i]:
                chunk_indices[i] += 1
                index_start = segment_starts[i] + chunk_indices[i] * chunk_length
                remaining = segment_starts[i] + segment_lengths[i] - index_start
                remaining = min(remaining, chunk_length)
                if remaining > 0:
                    src_value_chunks[i] = src_value_f.data[index_start:index_start+remaining]
                    src_index_chunks[i] = src_index_f.data[index_start:index_start+remaining]
                    in_chunk_lengths[i] = len(src_value_chunks[i])
                    in_chunk_indices[i] = 0
                else:
                    # can't clear list contents because we are using numba list, but they
                    # get filtered out in the following section
                    chunk_filter[i] = 0

        if chunk_filter.sum() < len(chunk_filter):
            segment_count = chunk_filter.sum()
            segment_indices = segment_indices[chunk_filter]
            segment_starts = segment_starts[chunk_filter]
            segment_lengths = segment_lengths[chunk_filter]
            chunk_indices = chunk_indices[chunk_filter]
            in_chunk_indices = in_chunk_indices[chunk_filter]
            in_chunk_lengths = in_chunk_lengths[chunk_filter]
            filtered_value_chunks = List()
            filtered_index_chunks = List()
            for i in range(len(src_value_chunks)):
                if chunk_filter[i]:
                    filtered_value_chunks.append(src_value_chunks[i])
                    filtered_index_chunks.append(src_index_chunks[i])
            src_value_chunks = filtered_value_chunks
            src_index_chunks = filtered_index_chunks


@njit
def streaming_sort_partial(in_chunk_indices, in_chunk_lengths,
                           src_value_chunks, src_index_chunks, dest_value_chunk, dest_index_chunk):
    dest_index = 0
    max_possible = in_chunk_lengths.sum()
    while(dest_index < max_possible):
        if in_chunk_indices[0] == in_chunk_lengths[0]:
            return dest_index
        min_value = src_value_chunks[0][in_chunk_indices[0]]
        min_value_index = 0
        for i in range(1, len(in_chunk_indices)):
            if in_chunk_indices[i] == in_chunk_lengths[i]:
                return dest_index
            cur_value = src_value_chunks[i][in_chunk_indices[i]]
            if cur_value < min_value:
                min_value = cur_value
                min_value_index = i

        min_index = src_index_chunks[min_value_index][in_chunk_indices[min_value_index]]
        dest_index_chunk[dest_index] = min_index
        dest_value_chunk[dest_index] = min_value
        dest_index += 1
        in_chunk_indices[min_value_index] += 1

    return dest_index


def is_ordered(field):
    if len(field) == 1:
        return True

    if np.issubdtype(field.dtype, np.number):
        fn = np.greater
    else:
        fn = np.char.greater
    return not np.any(fn(field[:-1], field[1:]))
