/* * * * *
 *  AzSortedFeat.cpp 
 *  Copyright (C) 2011, 2012 Rie Johnson
 *
 * This software may be modified and distributed under the terms
 * of the MIT license.  See the COPYING file for details.
 * * * * */

#include "AzSortedFeat.hpp"
#include "AzTools.hpp"
#include "AzPrint.hpp"

/*------------------------------------------------------*/
/*------------------------------------------------------*/
void AzSortedFeat_Dense::reset(const AzDvect *v_data_transpose, 
                                   const AzIntArr *ia_dx) 
{
  v_dx2v = v_data_transpose; 
  const double *dx2value = v_dx2v->point(); 

  const int *dxs = ia_dx->point(); 
  AzIFarr ifa_dx_val; 
  ifa_dx_val.prepare(ia_dx->size()); 
  for (int ix = 0; ix < ia_dx->size(); ++ix) {
    int dx = dxs[ix]; 
    ifa_dx_val.put(dx, dx2value[dx]); 
  }
  ifa_dx_val.sort_FloatInt(true); /* ascending order */

  ia_index.reset(); 
  ia_index.prepare(ifa_dx_val.size()); 
  for (int ix = 0; ix < ifa_dx_val.size(); ++ix) {
    int dx; 
    ifa_dx_val.get(ix, &dx); 
    ia_index.put(dx); 
  }

  index = ia_index.point(&index_num); 
  offset = 0; 
  isOriginal = true; /* This is the original one.  Don't change. */
}

/*------------------------------------------------------*/
void AzSortedFeat_Dense::filter(const AzSortedFeat_Dense *inp, 
                          const AzIntArr *ia_isYes, 
                          int yes_num)
{
  ia_index.prepare(yes_num); 
  v_dx2v = inp->v_dx2v; 

  int max_dx = ia_isYes->size() - 1; 
  const int *isYes = ia_isYes->point(); 

  int inp_index_num; 
  const int *inp_index = inp->ia_index.point(&inp_index_num); 

  for (int ix = 0; ix < inp_index_num; ++ix) {
    int dx = inp_index[ix]; 
    if (dx <= max_dx && isYes[dx]) {
      ia_index.put(dx); 
    }
  }

#if 0 
  offset = -1; /* not pointing to the original indexes */
#else
  offset = 0; /* 04/06/2012 */
#endif 
  index = ia_index.point(&index_num); 
}

/*------------------------------------------------------*/
/* place indexes so that yes's first and no's last and  */
/* the order with yes's and no's does not change.       */
/*------------------------------------------------------*/
/* static */
void AzSortedFeat_Dense::separate_indexes(int *index, 
                           int index_num, 
                           const int *isYes, 
                           int yes_num, 
                           int max_dx)
{
  AzIntArr ia_no;  
  ia_no.prepare(index_num-yes_num); 
  int yes_ix = 0; 
  for (int ix = 0; ix < index_num; ++ix) {
    int dx = index[ix]; 
    if (dx <= max_dx && isYes[dx]) {
      if (yes_ix != ix) {
        index[yes_ix] = dx; 
      }
      ++yes_ix; 
    }
    else {
      ia_no.put(dx); 
    }
  }

  if (ia_no.size() > 0) {
    memcpy(index+yes_ix, ia_no.point(), sizeof(int)*ia_no.size()); 
  }
  if (yes_ix != yes_num) {
    throw new AzException("AzSortedFeat_Dense::separate_indexes", 
                          "conflict in # of yes's"); 
  }
}

/*------------------------------------------------------*/
void AzSortedFeat_Dense::separate(
                          AzSortedFeat_Dense *base,  
                          const AzSortedFeat_Dense *inp, 
                          const AzIntArr *ia_isYes, 
                          int yes_num, 
                          AzSortedFeat_Dense *yes, 
                          AzSortedFeat_Dense *no)
{
  const char *eyec = "AzSortedFeat_Dense::separate"; 

  yes->v_dx2v = inp->v_dx2v; 
  no->v_dx2v = inp->v_dx2v; 

  int max_dx = ia_isYes->size() - 1; 
  const int *isYes = ia_isYes->point(); 

  int base_index_num; 
  int *base_index = base->base_index_for_update(&base_index_num); 

  int *sub_index = base_index + inp->offset; 

  if (inp->index != sub_index || 
      inp->offset+inp->index_num > base_index_num) {
    throw new AzException(eyec, "index conflict"); 
  }

  separate_indexes(sub_index, inp->index_num, 
                   isYes, yes_num, max_dx); 

  yes->index = sub_index; 
  yes->index_num = yes_num; 
  yes->offset = inp->offset; 
  no->index = sub_index + yes_num; 
  no->index_num = inp->index_num - yes_num; 
  no->offset = inp->offset + yes_num; 

  if (base_index+yes->offset != yes->index || 
      base_index+no->offset != no->index) {
    throw new AzException(eyec, "conflict in pointers"); 
  }
}

/*------------------------------------------------------*/
void AzSortedFeat_Dense::copy_base(const AzSortedFeat_Dense *inp)
{
  if (inp->ia_index.size() <= 0 || 
      inp->index != inp->ia_index.point() || 
      inp->index_num != inp->ia_index.size() || 
      inp->offset != 0) {
    throw new AzException("AzSortedFeat_Dense::copy_base", 
                          "Expected the base as input"); 
  }

  ia_index.reset(&inp->ia_index); 
  v_dx2v = inp->v_dx2v; 
  index = ia_index.point(&index_num); 
  offset = 0; 

  isOriginal = false; /* This is a copy. */
}

/* called when and only when data points are sampled */
/*--------------------------------------------------------*/
void AzSortedFeatArr::filter_base(const AzSortedFeatArr *inp, 
                                  const int *dxs, 
                                  int dxs_num) 
{
  const char *eyec = "AzSortedFeatArr::filter_base"; 
  beTight = inp->beTight; 
  f_num = inp->featNum(); 
  a_sparse.free(&arrs); 
  a_dense.free(&arrd); 

  ia_isActive.reset(); 
  ia_isActive.toOnOff(dxs, dxs_num); 
  active_num = dxs_num; 

  if (beTight) {
    return; 
  }

  if (inp->doingSparse()) {
    a_sparse.alloc(&arrs, f_num, eyec, "arrs"); 
    for (int fx = 0; fx < f_num; ++fx) {
      if (inp->arrs == NULL || inp->arrs[fx] == NULL) {
        throw new AzException(eyec, "No sorted sparse features?!");
      }
      arrs[fx] = new AzSortedFeat_Sparse(inp->arrs[fx], &ia_isActive, active_num); 
    }
  }
  else {
    a_dense.alloc(&arrd, f_num, eyec, "arrd"); 
    for (int fx = 0; fx < f_num; ++fx) {
      if (inp->arrd == NULL || inp->arrd[fx] == NULL) {
        throw new AzException(eyec, "No sorted dense features?!");
      }
      arrd[fx] = new AzSortedFeat_Dense(inp->arrd[fx], &ia_isActive, active_num); 
    }
  }
}

/*--------------------------------------------------------*/
const int *AzSortedFeat_Dense::next(AzCursor &cur, 
                              double *out_val, int *out_num) /* output */
const 
{
  const char *eyec = "AzSortedFeat_Dense::next"; 

  int cursor = cur.get(); 
  if (cursor >= index_num) {
    return NULL;  /* end of data */
  }

  const double *dx2value = v_dx2v->point(); 

  int dx = index[cursor]; 
  double curr_val = dx2value[dx]; 

  int begin = cursor; 
  cursor = cur.inc(); 
  if (cursor >= index_num) {
    return  NULL; /* this will produce all vs none anyway */
  }

  double avg_val = curr_val + 0.00000001; 
  for ( ; cursor < index_num; cursor=cur.inc()) {
    int dx = index[cursor]; 
    double next_val = dx2value[dx]; 
    if (next_val != curr_val) {
      avg_val = (curr_val + next_val) / 2; 
      break; 
    }
  }
  if (cursor >= index_num) {
    return NULL; /* this will produce all vs none anyway */
  }

  *out_val = avg_val; 
  *out_num = cursor - begin; 
  return index + begin;   
}

/*------------------------------------------------------*/
void AzSortedFeat_Dense::getIndexes(const int *inp_dxs, /* not used */
                              int inp_dxs_num, 
                              double border_val, 
                              /*---  output  ---*/
                              AzIntArr *ia_le_dx, 
                              AzIntArr *ia_gt_dx)
const
{
  if (inp_dxs_num != index_num) {
    throw new AzException("AzSortedFeat_Dense::getIndexes", 
                          "Conflict in # of data points"); 
  }

  const double *dx2value = v_dx2v->point(); 
  int ix;
  for (ix = 0; ix < index_num; ++ix) {
    int dx = index[ix]; 
    if (dx2value[dx] > border_val) break; 
  }
  int le_size = ix; 
  ia_le_dx->reset(index, le_size); 
  ia_gt_dx->reset(index+le_size, index_num-le_size); 
}


/*-------------------------------------------------------*/
/*-------------------------------------------------------*/
/* 
 * For sparse data with many zeroes                      
 *   Avoid going through /carrying around data indexes for  
 *   zero entries as much as possible.                     
 * (NOTE) The order of addition of floating-point values becomes 
 *        different from AzSortedFeat_Dense, which may cause slight 
 *        differences in the results.  
 */
void AzSortedFeat_Sparse::reset(const AzSvect *v_data_transpose, 
                         const AzIntArr *ia_dx) /* must be sorted */
{
  data_num = ia_dx->size(); 

  AzIFarr ifa_dx_val; 
  v_data_transpose->filter(ia_dx, /* must be sorted */
                           &ifa_dx_val, NULL); 

  int zero_num = ia_dx->size() - ifa_dx_val.size(); 
  ia_zero.reset(); 

  double min_val = ifa_dx_val.findMin(); 
  double max_val = ifa_dx_val.findMax(); 

  if (min_val == max_val) { /* one value, no need to sort */
    if (zero_num > 0) { /* some values are zero */
      if (min_val > 0) {
        ifa_dx_val.insert(0, AzNone, (double)0); /* zero is smaller */
      }
      else {
        ifa_dx_val.put(AzNone, (double)0); /* zero is bigger */
      }     
    }
  }
  else {
    if (zero_num > 0) {
      ifa_dx_val.put(AzNone, (double)0); /* place holder for zero */
    }
    ifa_dx_val.sort_FloatInt(true); /* ascending order */
  }

  /*---  make it flat for faster access later on  ---*/
  AzTools::flatten(&ifa_dx_val, &ia_index, &v_value); 

  /*---  ---*/
  _shouldDoBackward = false;
  if (zero_num > 0) {
    if (max_val < 0) { /* zero is at the end */
      _shouldDoBackward = false; 
    }
    else if (min_val > 0) { /* zero is at the beginning */
      _shouldDoBackward = true; 
    }
    else {
      /*---  we can't avoid getting indexes for zero values  ---*/
      v_data_transpose->filter(ia_dx, NULL, &ia_zero); 
    }
  }
}

/*------------------------------------------------------*/
void AzSortedFeat_Sparse::filter(const AzSortedFeat_Sparse *inp, 
                          const AzIntArr *ia_isYes, 
                          int yes_num)
{
  sub_initialize(inp, yes_num, this); 

  int max_dx = ia_isYes->size() - 1; 
  const int *isYes = ia_isYes->point(); 
  int inp_zero_num; 
  const int *inp_zero = inp->ia_zero.point(&inp_zero_num); 
  for (int ix = 0; ix < inp_zero_num; ++ix) {
    int dx = inp_zero[ix]; 
    if (dx <= max_dx && isYes[dx]) {
      ia_zero.put(dx); 
    }
  }

  int inp_index_num; 
  const int *inp_index = inp->ia_index.point(&inp_index_num); 
  const double *inp_value = inp->v_value.point(); 
  int where_is_zero = -1; 
  for (int ix = 0; ix < inp_index_num; ++ix) {
    int dx = inp_index[ix]; 
    if (dx == AzNone) {  /* place holder for zero */
      ia_index.put(AzNone); 
      where_is_zero = ia_index.size()-1; 
    }
    else if (dx <= max_dx && isYes[dx]) {
      ia_index.put(dx); 
      v_value.set(ia_index.size()-1, inp_value[ix]); 
    }
  }

  sub_terminate(this, yes_num, where_is_zero); 
}

/*------------------------------------------------------*/
void AzSortedFeat_Sparse::separate(const AzSortedFeat_Sparse *inp, 
                          const AzIntArr *ia_isYes, 
                          int yes_num, 
                          AzSortedFeat_Sparse *yes, 
                          AzSortedFeat_Sparse *no)
{
  sub_initialize(inp, yes_num, yes); 
  int no_num = inp->data_num - yes_num; 
  sub_initialize(inp, no_num, no); 

  int max_dx = ia_isYes->size() - 1; 
  const int *isYes = ia_isYes->point(); 
  int inp_zero_num; 
  const int *inp_zero = inp->ia_zero.point(&inp_zero_num); 
  for (int ix = 0; ix < inp_zero_num; ++ix) {
    int dx = inp_zero[ix]; 
    if (dx <= max_dx && isYes[dx]) {
      yes->ia_zero.put(dx); 
    }
    else {
      no->ia_zero.put(dx); 
    }
  }

  int inp_index_num; 
  const int *inp_index = inp->ia_index.point(&inp_index_num); 
  const double *inp_value = inp->v_value.point(); 
  int yes_where_is_zero = -1, no_where_is_zero = -1; 
  for (int ix = 0; ix < inp_index_num; ++ix) {
    int dx = inp_index[ix]; 
    if (dx == AzNone) {  /* place holder for zero */
      yes->ia_index.put(AzNone); 
      no->ia_index.put(AzNone); 
      yes_where_is_zero = yes->ia_index.size()-1; 
      no_where_is_zero = no->ia_index.size()-1; 
    }
    else if (dx <= max_dx && isYes[dx]) {
      yes->ia_index.put(dx); 
      yes->v_value.set(yes->ia_index.size()-1, inp_value[ix]); 
    }
    else {
      no->ia_index.put(dx); 
      no->v_value.set(no->ia_index.size()-1, inp_value[ix]); 
    }
  }

  sub_terminate(yes, yes_num, yes_where_is_zero); 
  sub_terminate(no, no_num, no_where_is_zero); 
}

/*------------------------------------------------------*/
void AzSortedFeat_Sparse::sub_initialize(const AzSortedFeat_Sparse *inp, 
                                 int num, 
                                 AzSortedFeat_Sparse *ptr)
{
  ptr->ia_zero.reset(); 
  ptr->ia_zero.prepare(MIN(num, inp->ia_zero.size())); 
  ptr->ia_index.reset(); 
  int i_max = MIN(inp->ia_index.size(), num+1); /* plus one for zero */
  ptr->ia_index.prepare(i_max); 
  ptr->v_value.reform(i_max); 
  ptr->_shouldDoBackward = inp->_shouldDoBackward; 
}

/*------------------------------------------------------*/
void AzSortedFeat_Sparse::sub_terminate(AzSortedFeat_Sparse *ptr, 
                                      int num, 
                                      int where_is_zero)
{
  const char *eyec = "AzSortedFeat_Sparse::sub_terminate"; 
  if (ptr->v_value.rowNum() > ptr->ia_index.size()) {
    ptr->v_value.resize(ptr->ia_index.size()); 
  }
  ptr->data_num = num; 
  int zero_num = 0; 
  if (where_is_zero >= 0) {
    zero_num = num - (ptr->ia_index.size() - 1); /* -1 for dummy entry for zero */
    if (zero_num < 0) {
      throw new AzException(eyec, "numbers don't match around zero"); 
    }
    if (zero_num == 0) {
      /*---  remove dummy entry  ---*/
      ptr->ia_index.remove(where_is_zero); 
      double *value = ptr->v_value.point_u(); 
      int ix; 
      for (ix = where_is_zero+1; ix < ptr->v_value.rowNum(); ++ix) {
        value[ix-1] = value[ix]; 
      }
    }
  }
  else {
    if (ptr->ia_index.size() != num) {
      throw new AzException(eyec, "numbers don't match around zero"); 
    }
  }

  ptr->_shouldDoBackward = false; 
  if (zero_num == 0) {
    if (ptr->ia_zero.size() > 0) {
      throw new AzException(eyec, "conflict in #zero"); 
    }
  }
  else {
    if (where_is_zero == 0) {
      ptr->_shouldDoBackward = true; 
    }
  }
}

/*------------------------------------------------------*/
void AzSortedFeat_Sparse::copy(const AzSortedFeat_Sparse *inp) 
{
  /*---  copy  ---*/    
  ia_zero.reset(&inp->ia_zero); 
  ia_index.reset(&inp->ia_index); 
  v_value.set(&inp->v_value); 
  data_num = inp->data_num; 
  _shouldDoBackward = inp->_shouldDoBackward; 
}

/*--------------------------------------------------------*/
const int *AzSortedFeat_Sparse::forward(AzCursor &cur, 
                                        double *out_val, int *out_num) const
{
  const char *eyec = "AzSortedFeat_Sparse::forward"; 
  if (_shouldDoBackward) {
    throw new AzException(eyec, "_shouldDoBackward is on"); 
  }

  int num; 
  const int *index = ia_index.point(&num); 
  int cursor = cur.get(); 
  if (cursor >= num) {
    return NULL;  /* end of data */
  }

  const double *value = v_value.point(); 

  int dx = index[cursor]; 
  double curr_val = value[cursor]; 
  if (dx == AzNone) { /* value=0 */
    cursor = cur.inc(); 
    if (cursor >= num) {
      return NULL; /* since this will cause all vs empty anyway */
    }
    if (ia_zero.size() <= 0) {
      throw new AzException(eyec, "empty zero in the middle.  something is wrong"); 
    }
    double next_val = value[cursor]; 
    *out_val = (curr_val + next_val) / 2; 
    *out_num = ia_zero.size(); 
    return ia_zero.point(); 
  }

  int begin = cursor; 
  cursor = cur.inc();
  double avg_val = curr_val + 0.00000001; 
  for ( ; cursor < num; cursor=cur.inc()) {
    double next_val = value[cursor]; 
    if (next_val != curr_val) {
      avg_val = (curr_val + next_val) / 2; 
      break; 
    }
  }

  if (cursor >= num) {
    return NULL;  /* end of data */
  }

  *out_val = avg_val; 
  *out_num = cursor - begin; 
  return index + begin;   
}

/*--------------------------------------------------------*/
const int *AzSortedFeat_Sparse::backward(AzCursor &cur, 
                              double *out_val, int *out_num) /* output */
const
{
  const char *eyec = "AzSortedFeat_Sparse::backward"; 
  if (!_shouldDoBackward) {
    throw new AzException(eyec, "_shouldDoBackward is off"); 
  }

  int num; 
  const int *index = ia_index.point(&num); 
  int cursor = cur.get(); 
  if (cursor < 1) {
    return NULL;  /* end of data */
  }

  const double *value = v_value.point(); 

  int dx = index[cursor-1]; 
  double curr_val = value[cursor-1]; 
  if (dx == AzNone) { /* value=0 */
    cursor = cur.dec(); 
    if (cursor < 1) {
      return NULL; /* b/c this will cause empty vs all anyway */
    }
    if (ia_zero.size() <= 0) {
      throw new AzException(eyec, "empty zero in the middle.  something is wrong"); 
    }
    double prev_val = value[cursor-1]; 
    *out_val = (curr_val + prev_val)/2; 
    *out_num = ia_zero.size(); 
    return ia_zero.point(); 
  }

  int end = cursor; 
  cursor = cur.dec(); 
  double avg_val = curr_val - 0.00000001; 
  for ( ; cursor >= 1; cursor=cur.dec()) {
    double prev_val = value[cursor-1]; 
    if (prev_val != curr_val) {
      avg_val = (curr_val + prev_val) / 2; 
      break; 
    }
  }

  if (cursor < 1) {
    return NULL;  /* end of data */
  }

  *out_val = avg_val; 
  *out_num = end - cursor; 
  return index + cursor;   
}

/*------------------------------------------------------*/
void AzSortedFeat_Sparse::getIndexes(const int *inp_dxs, 
                              int inp_dxs_num, 
                              double border_val, 
                              /*---  output  ---*/
                              AzIntArr *ia_le_dx, 
                              AzIntArr *ia_gt_dx) const
{
  ia_le_dx->reset(); 
  ia_gt_dx->reset(); 
  ia_le_dx->prepare(inp_dxs_num); 
  ia_gt_dx->prepare(inp_dxs_num); 

  AzIntArr ia_temp; 
  const AzIntArr *ia_zero_index = getIndexes_Zero(inp_dxs, inp_dxs_num, &ia_temp); 

  int num; 
  const int *index = ia_index.point(&num); 
  const double *value = v_value.point(); 

  for (int ix = 0; ix < num; ++ix) {
    int dx = index[ix]; 
    if (value[ix] == 0) {
      if (ia_zero_index->size() > 0) {
        if (0 <= border_val) {
          ia_le_dx->concat(ia_zero_index);  
        }    
        else {
          ia_gt_dx->concat(ia_zero_index); 
        }
      }
    }
    else if (value[ix] <= border_val) {
      ia_le_dx->put(dx); 
    }
    else {
      ia_gt_dx->put(dx); 
    }
  }
  if (ia_le_dx->size() + ia_gt_dx->size() != inp_dxs_num) {
    throw new AzException("AzSortedFeat_Sparse::getIndexes", "num conflict"); 
  }
}

/*------------------------------------------------------*/
const AzIntArr *AzSortedFeat_Sparse::getIndexes_Zero(
                          const int *inp_dxs, 
                          int inp_dxs_num, 
                          AzIntArr *out_ia_zero)
const
{
  if (ia_zero.size() > 0) {
    return &ia_zero; 
  }

  AzIntArr ia_isNonZero; 
  bool isThereZero = ia_index.toOnOff(&ia_isNonZero); 
  if (!isThereZero) return &ia_zero; 

  out_ia_zero->reset(); 
  out_ia_zero->prepare(inp_dxs_num - ia_index.size() + 1); 
  int nz_dx_max = ia_isNonZero.size()-1; 
  const int *isNonZero = ia_isNonZero.point(); 
  for (int jx = 0; jx < inp_dxs_num; ++jx) {
    int dx = inp_dxs[jx]; 
    if (dx > nz_dx_max || !isNonZero[dx]) {
      out_ia_zero->put(dx); 
    }
  }
  return out_ia_zero; 
}


/*--------------------------------------------------------*/
/*--------------------------------------------------------*/
void AzSortedFeatArr::reset_sparse(const AzSmat *m_tran, 
                            bool inp_beTight)
{
  const char *eyec = "AzSortedFeatArr::reset_sparse"; 
  beTight = inp_beTight; 
  f_num = m_tran->colNum(); 
  int data_num = m_tran->rowNum(); 
  ia_isActive.reset(); 
  active_num = 0; 

  a_sparse.free(&arrs); 
  a_dense.free(&arrd); 
  a_sparse.alloc(&arrs, f_num, eyec, "arrs"); 

  int nz_num = 0; 
  AzIntArr ia_all_dx; 
  ia_all_dx.range(0, data_num); 
  int fx; 
  for (fx = 0; fx < f_num; ++fx) {
    arrs[fx] = new AzSortedFeat_Sparse(m_tran->col(fx), &ia_all_dx); 
  }
}

/*--------------------------------------------------------*/
void AzSortedFeatArr::reset_dense(const AzDmat *m_tran_dense,   /* set */
                                  bool inp_beTight)
{
  const char *eyec = "AzSortedFeatArr::reset (dense)"; 

  beTight = inp_beTight; 
  f_num = m_tran_dense->colNum(); 
  int data_num = m_tran_dense->rowNum(); 
  ia_isActive.reset(); 
  active_num = 0; 

  a_sparse.free(&arrs); 
  a_dense.free(&arrd); 
  a_dense.alloc(&arrd, f_num, eyec, "arrd"); 

  int nz_num = 0; 
  AzIntArr ia_all_dx; 
  ia_all_dx.range(0, data_num); 
  for (int fx = 0; fx < f_num; ++fx) {
    arrd[fx] = new AzSortedFeat_Dense(m_tran_dense->col(fx), &ia_all_dx); 
  }
}

/*--------------------------------------------------------*/
void AzSortedFeatArr::copy_base(const AzSortedFeatArr *inp)
{
  const char *eyec = "AzSortedFeatArr::copy_base"; 
  beTight = inp->beTight; 
  f_num = inp->featNum(); 
  a_sparse.free(&arrs); 
  a_dense.free(&arrd); 

  ia_isActive.reset(); 
  active_num = 0; 

  if (beTight) {
    return; 
  }

  if (inp->doingSparse()) {
    a_sparse.alloc(&arrs, f_num, eyec, "arrs"); 
    int fx; 
    for (fx = 0; fx < f_num; ++fx) {
      if (inp->arrs == NULL || inp->arrs[fx] == NULL) {
        throw new AzException(eyec, "No sorted sparse features?!");
      }
      arrs[fx] = new AzSortedFeat_Sparse(inp->arrs[fx]); 
    }
  }
  else {
    a_dense.alloc(&arrd, f_num, eyec, "arrd"); 
    int fx; 
    for (fx = 0; fx < f_num; ++fx) {
      if (inp->arrd == NULL || inp->arrd[fx] == NULL) {
        throw new AzException(eyec, "No sorted dense features?!");
      }
      arrd[fx] = new AzSortedFeat_Dense(inp->arrd[fx]); 
    }
  }
}

/*--------------------------------------------------------*/
/*
 *  called when beTight is on. 
 */
const AzSortedFeat *AzSortedFeatArr::sorted(const AzSortedFeatArr *inp, 
                             int fx, 
                             AzSortedFeatWork *out) const 
{
  const char *eyec = "AzSortedFeatArr::sorted(inp,fx,work0)"; 
  if (f_num != inp->featNum() || 
      ia_isActive.size() <= 0 || active_num <= 0) {
    throw new AzException(eyec, "not ready?!"); 
  }
  if (fx < 0 || fx >= f_num) {
    throw new AzException(eyec, "out of range"); 
  }
  if (inp->doingSparse()) {
    if (inp->arrs == NULL || inp->arrs[fx] == NULL) {
      throw new AzException(eyec, "Expected sparse sorted features as input"); 
    }
    out->tmps.filter(inp->arrs[fx], &ia_isActive, active_num); 
    return &out->tmps; 
  }
  else {
    if (inp->arrd == NULL || inp->arrd[fx] == NULL) {
      throw new AzException(eyec, "Expected dense sorted features as input"); 
    }
    out->tmpd.filter(inp->arrd[fx], &ia_isActive, active_num); 
    return &out->tmpd; 
  }
}

/*--------------------------------------------------------*/
void AzSortedFeatArr::sub_initialize(const AzSortedFeatArr *inp, 
                      AzSortedFeatArr *ptr)
{
  ptr->ia_isActive.reset(); 
  ptr->active_num = 0; 
  ptr->beTight = inp->beTight; 
  ptr->f_num = inp->featNum(); 
  ptr->a_sparse.free(&ptr->arrs);
  ptr->a_dense.free(&ptr->arrd); 

  if (!inp->beTight) {
    const char *eyec = "AzSortedFeatArr::sub_initialize"; 
    if (inp->doingSparse()) {
      ptr->a_sparse.alloc(&ptr->arrs, ptr->f_num, eyec, "arrs"); 
    }
    else {
      ptr->a_dense.alloc(&ptr->arrd, ptr->f_num, eyec, "arrd");
    }
  }
}

/*--------------------------------------------------------*/
/* static */
void AzSortedFeatArr::separate(AzSortedFeatArr *base, /* used only by Dense */
             const AzSortedFeatArr *inp, 
             const int *yes_dxs, int yes_dxs_num, 
             const int *no_dxs, int no_dxs_num, 
             AzSortedFeatArr *yes, AzSortedFeatArr *no) 
{
  const char *eyec = "AzSortedFeatArr::separate"; 
  sub_initialize(inp, yes); 
  sub_initialize(inp, no); 

  AzIntArr ia_isActive; 
  ia_isActive.toOnOff(yes_dxs, yes_dxs_num); 
  int active_num = yes_dxs_num; 

  if (inp->beTight) {
    yes->ia_isActive.reset(&ia_isActive); 
    yes->active_num = active_num; 
    no->ia_isActive.toOnOff(no_dxs, no_dxs_num); 
    no->active_num = no_dxs_num; 
    return; 
  }

  if (inp->doingSparse()) {
    for (int fx = 0; fx < inp->featNum(); ++fx) {
      if (inp->arrs == NULL || inp->arrs[fx] == NULL) {
        throw new AzException(eyec, "No sparse sorted featuers given as input"); 
      }
      yes->arrs[fx] = new AzSortedFeat_Sparse(); 
      no->arrs[fx] = new AzSortedFeat_Sparse(); 
      AzSortedFeat_Sparse::separate(inp->arrs[fx], &ia_isActive, active_num,  
                             yes->arrs[fx], no->arrs[fx]); 
      if (yes->arrs[fx]->dataNum() != yes_dxs_num || 
          no->arrs[fx]->dataNum() != no_dxs_num) {
        throw new AzException(eyec, "conflict in pop (sparse)"); 
      }
    }
  }
  else {
    if (base == NULL) {
      throw new AzException(eyec, "base is null.  something is wrong"); 
    }
    for (int fx = 0; fx < inp->featNum(); ++fx) {
      if (base->arrd == NULL || base->arrd[fx] == NULL) {
        throw new AzException(eyec, "No dense sorted featuers given as base"); 
      }
      if (inp->arrd == NULL || inp->arrd[fx] == NULL) {
        throw new AzException(eyec, "No dense sorted featuers given as input"); 
      }
      yes->arrd[fx] = new AzSortedFeat_Dense(); 
      no->arrd[fx] = new AzSortedFeat_Dense(); 
      AzSortedFeat_Dense::separate(base->arrd[fx], 
                               inp->arrd[fx], &ia_isActive, active_num,  
                               yes->arrd[fx], no->arrd[fx]); 
      if (yes->arrd[fx]->dataNum() != yes_dxs_num || 
          no->arrd[fx]->dataNum() != no_dxs_num) {
        throw new AzException(eyec, "conflict in pop (dense)"); 
      }
    }
  }
}
