drwnPersistentStorage.h

/*****************************************************************************
** DARWIN: A FRAMEWORK FOR MACHINE LEARNING RESEARCH AND DEVELOPMENT
** Distributed under the terms of the BSD license (see the LICENSE file)
** Copyright (c) 2007-2015, Stephen Gould
** All rights reserved.
**
******************************************************************************
** FILENAME:    drwnPersistentStorage.h
** AUTHOR(S):   Stephen Gould <stephen.gould@anu.edu.au>
**
*****************************************************************************/

#pragma once

#include <cstdlib>
#include <map>
#include <list>
#include <vector>

using namespace std;

// drwnPersistentRecord ------------------------------------------------------

class drwnPersistentRecord {
 public:
    drwnPersistentRecord() { /* do nothing */ };
    virtual ~drwnPersistentRecord() { /* do nothing */ };

    virtual size_t numBytesOnDisk() const = 0;
    virtual bool write(ostream& os) const = 0;
    virtual bool read(istream& is) = 0;
};

// drwnPersistentVectorRecord ------------------------------------------------

template <class T>
class drwnPersistentVectorRecord : public drwnPersistentRecord {
 public:
    std::vector<T> data; //<! the data

 public:
    drwnPersistentVectorRecord() { /* do nothing */ }
    ~drwnPersistentVectorRecord() { /* do nothing */ }

    // i/o (from drwnPersistentRecord)
    size_t numBytesOnDisk() const {
        return sizeof(size_t) + data.size() * sizeof(T);
    }

    bool write(ostream& os) const {
        size_t n = data.size();
        os.write((char *)&n, sizeof(size_t));
        if (n != 0) {
            os.write((char *)&data[0], n * sizeof(T));
        }
        return true;
    }

    bool read(istream& is) {
        size_t n = 0;
        is.read((char *)&n, sizeof(size_t));
        data.resize(n);
        if (n > 0) {
            is.read((char *)&data[0], n * sizeof(T));
        }
        return true;
    }
};

// drwnPersistentVectorVectorRecord ------------------------------------------

template <class T>
class drwnPersistentVectorVectorRecord : public drwnPersistentRecord {
 public:
    std::vector<std::vector<T> > data; //<! the data

 public:
    drwnPersistentVectorVectorRecord() { /* do nothing */ }
    ~drwnPersistentVectorVectorRecord() { /* do nothing */ }

    // i/o (from drwnPersistentRecord)
    size_t numBytesOnDisk() const {
        size_t n = sizeof(size_t) * (data.size() + 1);
        for (unsigned i = 0; i < data.size(); i++) {
            n += data[i].size() * sizeof(T);
        }
        return n;
    }

    bool write(ostream& os) const {
        size_t n = data.size();
        os.write((char *)&n, sizeof(size_t));
        for (unsigned i = 0; i < data.size(); i++) {
            n = data[i].size();
            os.write((char *)&n, sizeof(size_t));
            if (n != 0) {
                os.write((char *)&data[i][0], n * sizeof(T));
            }
        }
        return true;
    }

    bool read(istream& is) {
        size_t n = 0;
        is.read((char *)&n, sizeof(size_t));
        data.resize(n);
        for (unsigned i = 0; i < data.size(); i++) {
            is.read((char *)&n, sizeof(size_t));
            data[i].resize(n);
            if (n > 0) {
                is.read((char *)&data[i][0], n * sizeof(T));
            }
        }
        return true;
    }
};

// drwnPersistentBlock -------------------------------------------------------

class drwnPersistentBlock {
 public:
    size_t start;   
    size_t length;  

 public:
    drwnPersistentBlock() { /* do nothing */ };
    drwnPersistentBlock(size_t strt, size_t len) : start(strt), length(len) { /* do nothing */ };
    ~drwnPersistentBlock() { /* do nothing */ };
};

// drwnPersistentStorage -----------------------------------------------------

class drwnPersistentStorage {
 public:
    static int MAX_OPEN;                          
    static string DEFAULT_INDEX_EXT;              
    static string DEFAULT_DATA_EXT;               

 protected:
    static list<drwnPersistentStorage *> _openList; 

    string _indexFilename;                        
    string _dataFilename;                         
    map<string, drwnPersistentBlock> _recordMapping; 
    list<drwnPersistentBlock> _freeSpace;         

    bool _bCompressed;                            
    fstream *_fsdata;                             
    bool _bSuspended;                             
    bool _bDirty;                                 

#ifdef DRWN_USE_PTHREADS
    static pthread_mutex_t _gmutex;               
    pthread_mutex_t _mutex;                       
#endif

 public:
    drwnPersistentStorage(bool bCompressed = false);
    drwnPersistentStorage(const drwnPersistentStorage& storage);
    virtual ~drwnPersistentStorage();

    bool open(const char *indexFile, const char *dataFile);
    bool open(const char *fileStem);
    bool reopen();
    bool close();

    bool isOpen() const { return (_fsdata != NULL) || _bSuspended; }
    bool canReopen() const { return (!_indexFilename.empty() && !_dataFilename.empty() && (_fsdata == NULL)); }

    int numRecords() const { return (int)_recordMapping.size(); }
    size_t numTotalBytes() const;
    size_t numUsedBytes() const;
    size_t numFreeBytes() const;

    bool hasKey(const char *key) const { return _recordMapping.find(string(key)) != _recordMapping.end(); }
    set<string> getKeys() const;

    bool erase(const char *key);
    bool read(const char *key, drwnPersistentRecord *record);
    bool write(const char *key, const drwnPersistentRecord *record);
    size_t bytes(const char *key) const;

    bool clear();
    bool defragment();

 protected:
    void suspend();  
    void resume();   

    bool atomic_reopen(bool locked);  
    bool atomic_close(bool locked);   
    void atomic_suspend(bool locked); 
};