#pragma once

#include "Defs.h"

#include <vector>

/*-=====  Pre-defined compression levels  =====-*/
#ifndef ZSTD_CLEVEL_DEFAULT
    #define ZSTD_CLEVEL_DEFAULT 3
#endif

#ifndef ZSTD_MAX_CLEVEL
    #define ZSTD_MAX_CLEVEL 22
#endif

#ifndef ZSTD_MIN_CLEVEL
    #define ZSTD_MIN_CLEVEL -5
#endif

// note: should an enum
#define COMPRESSION_TYPE_NONE 0
#define COMPRESSION_TYPE_ZSTD 1

// note: should an enum
#define COMPRESSION_MODE_COMPRESS 0
#define COMPRESSION_MODE_DECOMPRESS 1
#define COMPRESSION_MODE_COMPRESS_FILE 2
#define COMPRESSION_MODE_DECOMPRESS_FILE 3
#define COMPRESSION_MODE_SHAREDMEMORY 4
#define COMPRESSION_MODE_INFO 5
#define COMPRESSION_MODE_COMPRESS_APPEND_FILE 6

// note: should an enum
#define SDS_MULTI_MODE_ERROR 0
#define SDS_MULTI_MODE_READ_MANY 1
#define SDS_MULTI_MODE_READ_MANY_INFO 2
#define SDS_MULTI_MODE_STACK_MANY 3
#define SDS_MULTI_MODE_STACK_MANY_INFO 4
#define SDS_MULTI_MODE_CONCAT_MANY 5
// when not sure if stacking or reading horizontal
#define SDS_MULTI_MODE_STACK_OR_READMANY 6

#define COMPRESSION_MAGIC 253

#define SDS_VERSION_HIGH 4

// TJD added version low = 2 May  2019 for "folders"
// TJD added version low = 3 July 2019 for "bandsize"
// TJD added version low = 4 August 2019 for "section"
// TJD added version low = 5 March 2020 for timestamps and better section,
// boolean bitmasks
#define SDS_VERSION_LOW 5

#define SDS_VALUE_ERROR 1

#define SDS_MAGIC 0x20534453
#define SDS_PADSIZE 512
#define SDS_PAD_NUMBER(_NUM_) ((_NUM_ + 511) & ~511)

#if defined(_WIN32) && ! defined(__GNUC__)

    #define SDS_FILE_HANDLE void *
    #define SDS_EVENT_HANDLE void *
    #define BAD_SDS_HANDLE NULL

#else
    #define SDS_FILE_HANDLE int
    #define SDS_EVENT_HANDLE void *
    #define BAD_SDS_HANDLE 0

#endif

extern std::vector<std::string> g_gatewaylist;
extern char g_errorbuffer[512];
extern int g_lastexception;

typedef std::vector<const char *> SDS_STRING_LIST;

#define SDS_MAX_FILENAME 300
#define SDS_MAX_SECTIONNAME 64

//----------------------------------------------------
// NOTE:
// Filename rule "!" may be used to separate nested structs
// array name rule "!" may be used for sub arrays
// sam!col_0/col_1  <- categoricals
//
// 8 bits in the enum -- the LSB 2 bits are defined
#define SDS_FLAGS_ORIGINAL_CONTAINER 1 // Represents a column name in Dataset
#define SDS_FLAGS_STACKABLE \
    2 // Often used to mark a categorical side car data (can stack but not part of
      // orig container)
#define SDS_FLAGS_SCALAR 4
#define SDS_FLAGS_NESTED 8 // This is set when there is a hierarchy but NO DATA
#define SDS_FLAGS_META 16  // This array is string or unicode that contains meta data (from onefile)

// IN HEADER
#define SDS_FILE_TYPE_UNKNOWN 0
#define SDS_FILE_TYPE_STRUCT 1
#define SDS_FILE_TYPE_DATASET 2
#define SDS_FILE_TYPE_TABLE 3
#define SDS_FILE_TYPE_ARRAY 4
#define SDS_FILE_TYPE_ONEFILE 5

#define SDS_STACK_TYPE_NONE 0
#define SDS_STACK_TYPE_CONCAT 1 // When sds_concat is called and reader wants to stack

#define SDS_AUTHOR_ID_UNKNOWN 0
#define SDS_AUTHOR_ID_PYTHON 1
#define SDS_AUTHOR_ID_MATLAB 2
#define SDS_AUTHOR_ID_CSHARP 3

//===========================================
// taken from numpy
enum SDS_TYPES
{
    SDS_BOOL = 0,
    SDS_BYTE,
    SDS_UBYTE,
    SDS_SHORT,
    SDS_USHORT,
    SDS_INT,
    SDS_UINT,
    SDS_LONG,
    SDS_ULONG,
    SDS_LONGLONG,
    SDS_ULONGLONG,
    SDS_FLOAT,
    SDS_DOUBLE,
    SDS_LONGDOUBLE,
    SDS_CFLOAT,
    SDS_CDOUBLE,
    SDS_CLONGDOUBLE,
    SDS_OBJECT = 17,
    SDS_STRING,
    SDS_UNICODE,
    SDS_VOID
};

#define SDS_INT8 SDS_BYTE
#define SDS_UINT8 SDS_UBYTE
#define SDS_INT16 SDS_SHORT
#define SDS_UINT16 SDS_USHORT
#define SDS_INT32 SDS_INT
#define SDS_UINT32 SDS_UINT
#define SDS_INT64 SDS_LONGLONG
#define SDS_UINT64 SDS_ULONGLONG

/*
 * Means c-style contiguous (last index varies the fastest). The data
 * elements right after each other.
 *
 * This flag may be requested in constructor functions.
 * This flag may be tested for in PyArray_FLAGS(arr).
 */
#define SDS_ARRAY_C_CONTIGUOUS 0x0001

/*
 * Set if array is a contiguous Fortran array: the first index varies
 * the fastest in memory (strides array is reverse of C-contiguous
 * array)
 *
 * This flag may be requested in constructor functions.
 * This flag may be tested for in PyArray_FLAGS(arr).
 */
#define SDS_ARRAY_F_CONTIGUOUS 0x0002

/*
 * Note: all 0-d arrays are C_CONTIGUOUS and F_CONTIGUOUS. If a
 * 1-d array is C_CONTIGUOUS it is also F_CONTIGUOUS. Arrays with
 * more then one dimension can be C_CONTIGUOUS and F_CONTIGUOUS
 * at the same time if they have either zero or one element.
 * If SDS_RELAXED_STRIDES_CHECKING is set, a higher dimensional
 * array is always C_CONTIGUOUS and F_CONTIGUOUS if it has zero elements
 * and the array is contiguous if ndarray.squeeze() is contiguous.
 * I.e. dimensions for which `ndarray.shape[dimension] == 1` are
 * ignored.
 */

/*
 * If set, the array owns the data: it will be free'd when the array
 * is deleted.
 *
 * This flag may be tested for in PyArray_FLAGS(arr).
 */
#define SDS_ARRAY_OWNDATA 0x0004

/*
 * An array never has the next four set; they're only used as parameter
 * flags to the various FromAny functions
 *
 * This flag may be requested in constructor functions.
 */

/* Cause a cast to occur regardless of whether or not it is safe. */
#define SDS_ARRAY_FORCECAST 0x0010

/*
 * Always copy the array. Returned arrays are always CONTIGUOUS,
 * ALIGNED, and WRITEABLE.
 *
 * This flag may be requested in constructor functions.
 */
#define SDS_ARRAY_ENSURECOPY 0x0020

/*
 * Make sure the returned array is a base-class ndarray
 *
 * This flag may be requested in constructor functions.
 */
#define SDS_ARRAY_ENSUREARRAY 0x0040

/*
 * Make sure that the strides are in units of the element size Needed
 * for some operations with record-arrays.
 *
 * This flag may be requested in constructor functions.
 */
#define SDS_ARRAY_ELEMENTSTRIDES 0x0080

/*
 * Array data is aligned on the appropriate memory address for the type
 * stored according to how the compiler would align things (e.g., an
 * array of integers (4 bytes each) starts on a memory address that's
 * a multiple of 4)
 *
 * This flag may be requested in constructor functions.
 * This flag may be tested for in PyArray_FLAGS(arr).
 */
#define SDS_ARRAY_ALIGNED 0x0100

/*
 * Array data has the native endianness
 *
 * This flag may be requested in constructor functions.
 */
#define SDS_ARRAY_NOTSWAPPED 0x0200

/*
 * Array data is writeable
 *
 * This flag may be requested in constructor functions.
 * This flag may be tested for in PyArray_FLAGS(arr).
 */
#define SDS_ARRAY_WRITEABLE 0x0400

/*
 * If this flag is set, then base contains a pointer to an array of
 * the same size that should be updated with the current contents of
 * this array when this array is deallocated
 *
 * This flag may be requested in constructor functions.
 * This flag may be tested for in PyArray_FLAGS(arr).
 */
#define SDS_ARRAY_UPDATEIFCOPY 0x1000

// TJD New for ver 4.3
// If this flag is set, the allocation of the array has been filtered
#define SDS_ARRAY_FILTERED 0x8000

// The max dimensions we can store, if > 5, the remaining dims are multiplied
// together
#define SDS_MAX_DIMS 5

// Max number of worker threads allowed
#define SDS_MAX_THREADS 64

// File format layout for one array block header
struct SDS_ARRAY_BLOCK
{
    //----- offset 0 -----
    int16_t HeaderTag;
    int16_t HeaderLength;

    uint8_t Magic; // See compression magic
    int8_t CompressionType;
    int8_t DType;
    int8_t NDim;

    //----- offset 8 ----- // BUGBUG All these offsets are wrong, this struct has
    // default member alignments.
    int32_t ItemSize;
    int32_t Flags;

    //----- offset 16 -----
    // no more than 5 dims
    int64_t Dimensions[SDS_MAX_DIMS];
    int64_t Strides[SDS_MAX_DIMS];

    int64_t ArrayDataOffset; // Start of data in file
                             // If banding, then (UncompressedSize + (BandSize-1))
                             // / BandSize => # of bands
    int64_t ArrayCompressedSize;
    int64_t ArrayUncompressedSize;
    int32_t ArrayBandCount; // 0=no banding
    int32_t ArrayBandSize;  // 0=no banding
};

//----------------------------------
// At offset 0 of the file is this header
struct SDS_FILE_HEADER
{
    uint32_t SDSHeaderMagic;
    int16_t VersionHigh;
    int16_t VersionLow;

    int16_t CompMode;
    int16_t CompType;
    int16_t CompLevel;

    //----- offset 16 -----
    int64_t NameBlockSize;
    int64_t NameBlockOffset;
    int64_t NameBlockCount;

    int16_t FileType;  // see SDS_FILE_TYPE
    int16_t StackType; // see SDS_STACK_TYPE
    int32_t AuthorId;  // see SDS_AUTHOR_ID

    //----- offset 48 -----
    int64_t MetaBlockSize;
    int64_t MetaBlockOffset;

    //----- offset 64 -----
    int64_t TotalMetaCompressedSize;
    int64_t TotalMetaUncompressedSize;

    //----- offset 80 -----
    int64_t ArrayBlockSize;
    int64_t ArrayBlockOffset;

    //----- offset 96 -----
    int64_t ArraysWritten;
    int64_t ArrayFirstOffset;

    //----- offset 112 -----
    int64_t TotalArrayCompressedSize; // Includes the SDS_PADDING, next relative
                                      // offset to write to
    int64_t TotalArrayUncompressedSize;

    //----- offset 128 -----
    //------------- End of Version 4.1 ---------
    //------------- Version 4.3 starts here ----
    int64_t BandBlockSize;   // 00 if nothing
    int64_t BandBlockOffset; // 00
    int64_t BandBlockCount;  // Number of names
    int64_t BandSize;        // How many elements before creating another band

    //----- offset 160 -----
    //------------- End of Version 4.3 ---------
    //------------- Version 4.4 starts here ----
    int64_t SectionBlockSize;         // how many bytes valid data
    int64_t SectionBlockOffset;       // points to section directory if it exists (often
                                      // NULL if file was never appended) if exists was
                                      // often the LastFileOffset when an append
                                      // operation was done Then can read that offset to
                                      // get a list of NAMES\0\NAME2\0
    int64_t SectionBlockCount;        // number of names (number of sections total)
                                      // (often 0 if never appended)
    int64_t SectionBlockReservedSize; // total size of what we reserved for this
                                      // block (so we can append names)

    //----- offset 192 -----
    int64_t FileOffset;         // this file offset within the file
    uint64_t TimeStampUTCNanos; // time stamp when file was last written (0s
                                // indicate no time stamp)

    //----- offset 208 -----
    char Reserved[512 - 208]; // BUGBUG This should be done using align statements,
                              // not magic number math with incorrect arguments.

    //-----------------------------
    // function to multithreaded safe calculate the next array offset to write to
    // returns the relative file offset (has added pFileHeader->ArrayFirstOffset)
    int64_t AddArrayCompressedSize(int64_t cSize)
    {
        int64_t padSize = SDS_PAD_NUMBER(cSize);
        int64_t fileOffset = InterlockedAdd64(&TotalArrayCompressedSize, padSize) - padSize;

        // Calculate file offset location where arrays are stored
        fileOffset += ArrayFirstOffset;

        return fileOffset;
    }

    // only valid after file header has been filled in
    int64_t GetEndOfFileOffset()
    {
        return TotalArrayCompressedSize + ArrayFirstOffset;
    }
};

//---------------------------------------
class SDSSectionName
{
public:
    // Section data created when user appends to a file with section name
    char * pSectionData = NULL;
    const char ** pSectionNames = NULL;
    int64_t * pSectionOffsets = NULL;
    int64_t SectionCount = 0;
    int64_t SectionOffset = 0;

    // For creating one when it was missing (first time appended)
    const char * g_firstsectioname = "0";
    int64_t g_firstsectionoffset = 0;
    char g_firstsectiondata[10] = { '0', 0, 0, 0, 0, 0, 0, 0, 0, 0 };

    //-----------------------------------------------------------
    // Add one more name and offset
    // Returns sizeof new section
    // Returns pointer in *pListNames
    // NOTE: caller must WORKSPACE_FREE *pListNames
    int64_t BuildSectionNamesAndOffsets(char ** pListNames, // Returned
                                        const char * pNewSectionName,
                                        int64_t newSectionOffset // 0 Allowed
    );
    void AllocateSectionData(int64_t sectionBlockCount, int64_t sectionSize);

    void DeleteSectionData();
    void MakeListSections(const int64_t sectionBlockCount, const int64_t sectionByteSize);
    char * MakeFirstSectionName();
    char * ReadListSections(SDS_FILE_HANDLE SDSFile, SDS_FILE_HEADER * pFileHeader);
    ~SDSSectionName();
};

//------------ PAD TO 512 ----------------------
// 4 of these (4*128) can fit in 512
// TJD: NOT USED YET
// BUGBUG: EST: Needs alignment / corrected padding work
struct SDS_FOLDER_HEADER
{
    int64_t FolderHeaderOffset; // points to SDS_FILE_HEADER
    int64_t FolderCreateTimeUTC;
    int8_t FolderType;       // 0 for now SDS_FILE_TYPE_UNKNOWN, etc.
    int8_t FolderNameLength; // up to 110 chars
    char FolderName[128 - 16 - 2];
};

//---------------------------------------------------------------------
// NOTE: See ArrayInfo and keep similar
struct SDSArrayInfo
{
    // Numpy/Matlab/3rd party object  ??? is this needed anymore since pData
    // exists?
    void * pArrayObject;

    // First bytes of array data
    char * pData;

    // total number of items
    int64_t ArrayLength;

    // total number of items * itemsize
    int64_t NumBytes;

    // see SDS_TYPES which follow numpy dtypes
    int32_t NumpyDType;

    // Number of dimensions (not to exceed 3 currently)
    int32_t NDim;

    // Width in bytes of one row
    int32_t ItemSize;

    // See SDS_FLAGS which are same as numpy flags
    int32_t Flags;

    // no more than SDS_MAX_DIMS dims
    int64_t Dimensions[SDS_MAX_DIMS];
    int64_t Strides[SDS_MAX_DIMS];
};

//---------------------------------------------------------------------
// NOTE: filter or mask must be passed
struct SDSFilterInfo
{
    int64_t TrueCount; // If zero, dont bother reading in data
    int64_t RowOffset; // sum of all previous True Count
};

struct SDS_FILTER
{
    // int32_t*                           pFancyMask;
    // int64_t                            FancyLength;         // length of the
    // index array as well as final array

    // if BoolMaskLength == BoolMaskTrueCount and BoolMaskLength < 100
    // we assume this is looking at the header (first 5, 10, 100 rows)
    bool * pBoolMask;
    int64_t BoolMaskLength;      // length of bool mask (but not final array)
    int64_t BoolMaskTrueCount;   // length of final array
    SDSFilterInfo * pFilterInfo; // allocated on the fly when stacking
};

//==================================================
// Internal structure used to decompress or read SDS files
//
struct SDS_WRITE_COMPRESS_ARRAYS
{
    SDSArrayInfo * pArrayInfo;
    SDS_ARRAY_BLOCK * pBlockInfo;

    // used when decompressing
    SDS_FILE_HANDLE fileHandle;
    SDS_FILE_HEADER * pFileHeader;

    // Used when going to shared memory
    class SharedMemory * pMemoryIO;

    int64_t totalHeaders;
    int16_t compMode;
    int16_t compType;
    int32_t compLevel;

    // Per core allocations
    void * pCoreMemory[SDS_MAX_THREADS];
    int64_t pCoreMemorySize[SDS_MAX_THREADS];
    SDS_EVENT_HANDLE eventHandles[SDS_MAX_THREADS];

    // See: compMode value -- COMPRESSIOM_MODE_SHAREDMEMORY
    // used when compressing to file

    // This is the callback from SDS_ALLOCATE_ARRAYS
    // The bottom end of this structure is allocated based on how many arrays
    //??SDSArrayInfo          ArrayInfo[1];
};

//==================================================
// Internal structure used to decompress or read SDS files
//
struct SDS_READ_DECOMPRESS_ARRAYS
{
    struct SDS_READ_CALLBACKS * pReadCallbacks;
    // SDSArrayInfo*        pArrayInfo;
    SDS_ARRAY_BLOCK * pBlockInfo;

    // used when decompressing
    SDS_FILE_HANDLE fileHandle;
    SDS_FILE_HEADER * pFileHeader;

    // New filtering
    SDS_FILTER * pFilter;

    // Used when going to shared memory
    class SharedMemory * pMemoryIO;

    int64_t totalHeaders;
    int16_t compMode;
    int16_t compType;
    int32_t compLevel;

    // Per core allocations
    void * pCoreMemory[SDS_MAX_THREADS];
    int64_t pCoreMemorySize[SDS_MAX_THREADS];
    SDS_EVENT_HANDLE eventHandles[SDS_MAX_THREADS];

    // See: compMode value -- COMPRESSIOM_MODE_SHAREDMEMORY
    // used when compressing to file

    // This is the callback from SDS_ALLOCATE_ARRAYS
    // The bottom end of this structure is allocated based on how many arrays
    SDSArrayInfo ArrayInfo[1];
};

struct SDS_STACK_CALLBACK_FILES
{
    const char * Filename;
    const char * MetaData;
    int64_t MetaDataSize;
    SDS_FILE_HEADER * pFileHeader;
};

//-------------------------------------------------
// For reading...
struct SDS_STACK_CALLBACK
{
    const char * ArrayName;
    void * pArrayObject;
    int64_t * pArrayOffsets;
    SDS_ARRAY_BLOCK * pArrayBlock;
    int32_t ArrayEnum;
};

//-------------------------------------------------
// For reading...
struct SDS_FINAL_CALLBACK
{
    SDS_FILE_HEADER * pFileHeader;

    int32_t mode; // info or decompress
    int32_t reserved1;

    int64_t arraysWritten;
    SDS_ARRAY_BLOCK * pArrayBlocks;
    SDSArrayInfo * pArrayInfo;

    // SDS_READ_DECOMPRESS_ARRAYS*  pstCompressArrays;

    char * metaData;
    int64_t metaSize;

    char * nameData;               // array name data
    SDSSectionName * pSectionName; // if the file has section, else NULL
    const char * strFileName;      // the filename read in
};

//-------------------------------------------------
struct SDS_SHARED_MEMORY_CALLBACK
{
    SDS_FILE_HEADER * pFileHeader;
    char * baseOffset;

    int32_t mode;
    int32_t reserved1;
    void * pSDSDecompressFile;

    // used to close memory/file handles
    void * pMapStruct;
};

//---------------------------------------
// Callee must fill in pDestInfo
struct SDS_ALLOCATE_ARRAY
{
    SDSArrayInfo * pDestInfo;
    // void*          pInclusionList;
    // void*          pExclusionList;

    // Tells caller how to allocate
    int32_t ndim;
    int64_t * dims;
    int32_t numpyType;
    int64_t itemsize;
    char * data;
    int32_t numpyFlags;
    int64_t * strides;
    const char * pArrayName;
    int32_t sdsFlags; //  see SDS_FLAGS_ORIGINAL_CONTAINER, etc.
};

//-------------------------------------------------
// Callbacks

// Called at the end of reading a file
typedef void * (*SDS_READ_FINAL_CALLBACK)(SDS_FINAL_CALLBACK * pSDSFinalCallback, int64_t finalCount);

// Called at the end of reading a stacked file
typedef void * (*SDS_STACK_FINAL_CALLBACK)(SDS_STACK_CALLBACK * pSDSFinalCallback, int64_t finalCount,
                                           SDS_STACK_CALLBACK_FILES * pSDSFileInfo, SDS_FILTER * pSDSFilter, int64_t fileCount);

// Only called when reading from shared memory
typedef void * (*SDS_READ_SHARED_MEMORY_CALLBACK)(SDS_SHARED_MEMORY_CALLBACK * pSDSMemoryCallback);

// examples PyArrayObject* AllocateNumpyArray(int ndim, npy_intp* dims, int32_t
// numpyType, int64_t itemsize, char* data, int32_t numpyFlags, npy_intp*
// strides) {
// Copy all information into pDestInfo
// Called at the beginning of reading a file
// if pData return as NULL, data is assume to be EXCLUDED
typedef void (*SDS_ALLOCATE_ARRAY_CALLBACK)(SDS_ALLOCATE_ARRAY * pSDSAllocateArray);

// Pass arrayobject returned from SDS_ALLOCATE_ARRAY_CALLBACK and return pointer
// to first location of array example: return PyArray_BYTES(pArrayObject, 0);
// typedef void*(*SDS_GET_ARRAY_POINTER_CALLBACK)(void * pArrayObject);

// example: void BeginAllowThreads() {
// return PyEval_SaveThread();
//   }
typedef void * (*SDS_BEGIN_ALLOW_THREADS)();

// example: void EndAllowThreads() {
//   PyEval_RestoreThread(_save);
//}
typedef void (*SDS_END_ALLOW_THREADS)(void *);

typedef void (*SDS_WARNING)(const char * format, ...);

struct SDS_READ_CALLBACKS
{
    SDS_READ_FINAL_CALLBACK ReadFinalCallback;
    SDS_STACK_FINAL_CALLBACK StackFinalCallback;
    SDS_READ_SHARED_MEMORY_CALLBACK
    ReadMemoryCallback; // Only called for shared memory
    SDS_ALLOCATE_ARRAY_CALLBACK
    AllocateArrayCallback; // NULL now allowed?  needs testing

    SDS_BEGIN_ALLOW_THREADS BeginAllowThreads; // Must be set even if does nothing
    SDS_END_ALLOW_THREADS EndAllowThreads;     // Must be set even if does nothing

    SDS_WARNING WarningCallback; // To send warnings, may be null

    SDS_STRING_LIST * pInclusionList;       // Set to NULL if nothing included
    SDS_STRING_LIST * pExclusionList;       // Set to NULL if nothing excluded
    SDS_STRING_LIST * pFolderInclusionList; // Set to NULL if nothing excluded

    double ReserveSpace; // From 0.0 to 1.0 -- percent to reserve

    bool MustExist;
    bool Dummy1;
    int16_t Dummy2;
    // new for filtering
    SDS_FILTER Filter;

    // the output when concat
    const char * strOutputFilename;
};

struct SDS_READ_INFO
{
    int32_t mode; // = COMPRESSION_MODE_COMPRESS_FILE,
};

struct SDS_WRITE_CALLBACKS
{
    SDS_BEGIN_ALLOW_THREADS BeginAllowThreads;
    SDS_END_ALLOW_THREADS EndAllowThreads;
};

struct SDS_WRITE_INFO
{
    // arrays to save information
    SDSArrayInfo * aInfo;
    int64_t arrayCount;

    // meta information
    const char * metaData;
    uint32_t metaDataSize;

    // names of arrays information
    char * pListNames;
    int64_t listNameSize;  // total byte size (store in memory)
    int64_t listNameCount; // number of names

    // compressed or uncompressed
    int32_t mode;     // = COMPRESSION_MODE_COMPRESS_FILE,
    int32_t compType; // = COMPRESSION_TYPE_ZSTD,
    int32_t level;    // = ZSTD_CLEVEL_DEFAULT;

    int32_t sdsFileType; // = SDS_FILE_TYPE_STRUCT
    int32_t sdsAuthorId; // = SDS_AUTHOR_IS

    bool appendFileHeadersMode; // True if appending to existing file
    bool appendRowsMode;        // Appending rows only
    bool appendColumnsMode;     // Appending columns only (code not written yet)
    bool appendReserved1;

    int64_t bandSize; // banding divides a column into chunks (set to 0 for no
                      // banding)

    const char * sectionName;
    int64_t sectionNameSize;
};

struct SDS_MULTI_READ
{
    const char * pFileName;
    SDS_STRING_LIST * pFolderName;
    int64_t Index;
    SDS_FINAL_CALLBACK FinalCallback;
};

extern "C"
{
    //-------------------------------------------------
    // Main API to write SDS file
    // File only
    DllExport int32_t SDSWriteFile(const char * fileName,
                                   const char * shareName, // can be NULL
                                   SDS_STRING_LIST * folderName,

                                   // arrays to save information
                                   SDS_WRITE_INFO * pWriteInfo,

                                   SDS_WRITE_CALLBACKS * pWriteCallbacks);

    //---------------------------------------------
    // Main API to read SDS file
    // Can read or get infomration
    DllExport void * SDSReadFile(const char * fileName, const char * shareName, SDS_STRING_LIST * pFolderName,
                                 SDS_STRING_LIST * pSectionsName, // can be NULL
                                 SDS_READ_INFO * pReadInfo,       // has mode= COMPRESSION_MODE_DECOMPRESS_FILE OR INFO MODE
                                 SDS_READ_CALLBACKS * pReadCallbacks);

    //---------------------------------------------
    DllExport void * SDSReadManyFiles(SDS_MULTI_READ * pMultiRead,
                                      SDS_STRING_LIST * pInclusionList, // may be set to NULL
                                      SDS_STRING_LIST * pFolderList,    // may be set to NULL
                                      SDS_STRING_LIST * pSectionsName,  // can be NULL
                                      int64_t fileCount,
                                      int32_t multiMode, // see SDS_MULTI
                                      SDS_READ_CALLBACKS * pReadCallbacks);

    DllExport char * SDSGetLastError();

    DllExport int32_t CloseSharedMemory(void * pMapStruct);

    DllExport int32_t CloseDecompressFile(void * pSDSDecompressFile);

    // no longer supported
    // DllExport void SDSClearBuffers();

    typedef bool (*SDS_WRITE_FILE)(const char *, const char *, SDS_WRITE_INFO *, SDS_WRITE_CALLBACKS *);
    typedef void * (*SDS_READ_FILE)(const char *, const char *, SDS_READ_INFO *, SDS_READ_CALLBACKS *);
    typedef void * (*SDS_READ_MANY_FILES)(SDS_MULTI_READ *, SDS_STRING_LIST *, int64_t, int, SDS_READ_CALLBACKS *);
    typedef char * (*SDS_GET_LAST_ERROR)();
    typedef void (*SDS_CLEAR_BUFFERS)();
}

#ifdef SDS_SPECIAL_INCLUDE

//====================================================================
// Windows only special include
// TODO: Make this a function table that is loaded once
SDS_READ_FILE g_fpReadFile = NULL;
SDS_READ_MANY_FILES g_fpReadManyFiles = NULL;
SDS_WRITE_FILE g_fpWriteFile = NULL;
SDS_GET_LAST_ERROR g_fpGetLastError = NULL;
SDS_CLEAR_BUFFERS g_fpClearBuffers = NULL;
HMODULE g_hModule = NULL;

void _LazyLoad(const char * dllLoadPath)
{
    // If they pass in a path, try to load that first
    if (g_hModule == NULL && dllLoadPath != NULL)
    {
        g_hModule = LoadLibraryA(dllLoadPath);
    }
    if (g_hModule == NULL)
    {
        g_hModule = LoadLibraryW(L"SDSFile.dll");
    }
    if (g_hModule == NULL)
    {
        printf("Failed to load DLL file SDSFile.dll\n");
    }
}

//----------------------------------------------------
// For matlab which does not statically link to the DLL
// Load DLL on demand
// dllLoadPath may be null, it will try to load from normal paths
int32_t _SDSWriteFile(const char * dllLoadPath, const char * fileName,
                      const char * shareName, // can be NULL

                      // arrays to save information
                      SDS_WRITE_INFO * pWriteInfo, SDS_WRITE_CALLBACKS * pWriteCallbacks)
{
    // Lazy DLL load
    _LazyLoad(dllLoadPath);

    if (g_hModule != NULL)
    {
        if (g_fpWriteFile == NULL)
        {
            g_fpWriteFile = (SDS_WRITE_FILE)GetProcAddress(g_hModule, "SDSWriteFile");
        }

        if (g_fpWriteFile != NULL)
        {
            return g_fpWriteFile(fileName, shareName, pWriteInfo, pWriteCallbacks);
        }
    }
    return false;
}

//===================================================================
void * _SDSReadFile(const char * dllLoadPath, const char * fileName, const char * shareName,
                    SDS_READ_INFO * pReadInfo, // = COMPRESSION_MODE_DECOMPRESS_FILE OR INFO MODE
                    SDS_READ_CALLBACKS * pReadCallbacks)
{
    // Lazy DLL load
    _LazyLoad(dllLoadPath);

    if (g_hModule != NULL)
    {
        if (g_fpReadFile == NULL)
        {
            g_fpReadFile = (SDS_READ_FILE)GetProcAddress(g_hModule, "SDSReadFile");
        }

        if (g_fpReadFile != NULL)
        {
            return g_fpReadFile(fileName, shareName, pReadInfo, pReadCallbacks);
        }
    }
    return NULL;
}

//===================================================================
void * _SDSReadManyFiles(const char * dllLoadPath, SDS_MULTI_READ * pMultiRead,
                         SDS_STRING_LIST * pInclusionList, // may be set to NULL
                         int64_t fileCount,
                         int32_t multiMode, // see SDS_MULTI
                         SDS_READ_CALLBACKS * pReadCallbacks)
{
    // Lazy DLL load
    _LazyLoad(dllLoadPath);

    if (g_hModule != NULL)
    {
        if (g_fpReadManyFiles == NULL)
        {
            g_fpReadManyFiles = (SDS_READ_MANY_FILES)GetProcAddress(g_hModule, "SDSReadManyFiles");
        }

        if (g_fpReadManyFiles != NULL)
        {
            return g_fpReadManyFiles(pMultiRead, pInclusionList, fileCount, multiMode, pReadCallbacks);
        }
    }
    return NULL;
}

char * _SDSGetLastError(const char * dllLoadPath)
{
    // Lazy DLL load
    _LazyLoad(dllLoadPath);

    if (g_hModule != NULL)
    {
        if (g_fpGetLastError == NULL)
        {
            g_fpGetLastError = (SDS_GET_LAST_ERROR)GetProcAddress(g_hModule, "SDSGetLastError");
        }

        if (g_fpGetLastError != NULL)
        {
            return g_fpGetLastError();
        }
    }
    return NULL;
}

void _SDSClearBuffers(const char * dllLoadPath)
{
    // Lazy DLL load
    _LazyLoad(dllLoadPath);

    if (g_hModule != NULL)
    {
        if (g_fpClearBuffers == NULL)
        {
            g_fpClearBuffers = (SDS_CLEAR_BUFFERS)GetProcAddress(g_hModule, "SDSClearBuffers");
        }

        if (g_fpClearBuffers != NULL)
        {
            return g_fpClearBuffers();
        }
    }
}
#endif
