00001
00002
00003
00004
00005
00006
00007
00008
00009
00010
00011 #ifndef __DATABASE_H__
00012 #define __DATABASE_H__
00013
00014 #include "class.h"
00015 #include "reference.h"
00016 #include "file.h"
00017 #include "pagepool.h"
00018
00019 BEGIN_GIGABASE_NAMESPACE
00020
00021 #ifdef _WINCE
00022
00025 const size_t dbDefaultInitIndexSize = 10*1024;
00026
00030 const size_t dbDefaultExtensionQuantum = 1*512*1024;
00031 #else
00032
00035 const size_t dbDefaultInitIndexSize = 512*1024;
00036
00040 const size_t dbDefaultExtensionQuantum = 4*1024*1024;
00041 #endif
00042
00046 const unsigned dbMaxParallelSearchThreads = 64;
00047
00051 enum dbHandleFlags {
00052 dbPageObjectFlag = 0x1,
00053 dbModifiedFlag = 0x2,
00054 dbFreeHandleFlag = 0x4,
00055 dbFlagsMask = 0x7,
00056 dbFlagsBits = 3
00057 };
00058
00059 const size_t dbAllocationQuantumBits = 6;
00060 const size_t dbAllocationQuantum = 1 << dbAllocationQuantumBits;
00061 const size_t dbPageBits = 13;
00062 const size_t dbPageSize = 1 << dbPageBits;
00063 const size_t dbIdsPerPage = dbPageSize / sizeof(oid_t);
00064 const size_t dbHandlesPerPage = dbPageSize / sizeof(offs_t);
00065 const size_t dbHandleBits = 1 + sizeof(offs_t)/4;
00066 const size_t dbBitmapSegmentBits = dbPageBits + 3 + dbAllocationQuantumBits;
00067 const size_t dbBitmapSegmentSize = 1 << dbBitmapSegmentBits;
00068 const size_t dbBitmapPages = 1 << (dbDatabaseOffsetBits-dbBitmapSegmentBits);
00069 const size_t dbDirtyPageBitmapSize = 1 << (dbDatabaseOidBits-dbPageBits+dbHandleBits-3);
00070 const size_t dbDefaultSelectionLimit = 2000000000;
00075 const int dbBMsearchThreshold = 512;
00079 const size_t dbIndexedMergeThreshold = 100;
00080
00081
00082 const char_t dbMatchAnyOneChar = '_';
00083 const char_t dbMatchAnySubstring = '%';
00084
00085 const int dbMaxFileSegments = 64;
00086
00090 enum dbPredefinedIds {
00091 dbInvalidId,
00092 dbMetaTableId,
00093 dbBitmapId,
00094 dbFirstUserId = dbBitmapId + dbBitmapPages
00095 };
00096
00100 enum dbLockType {
00101 dbNoLock,
00102 dbSharedLock,
00103 dbUpdateLock,
00104 dbExclusiveLock
00105 };
00106
00110 class dbHeader {
00111 public:
00112 int4 curr;
00113 int4 dirty;
00114 int4 initialized;
00115 struct {
00116 offs_t size;
00117 offs_t index;
00118 offs_t shadowIndex;
00119 oid_t indexSize;
00120 oid_t shadowIndexSize;
00121 oid_t indexUsed;
00122 oid_t freeList;
00123 oid_t bitmapEnd;
00124 } root[2];
00125
00126 int4 versionMagor;
00127 int4 versionMinor;
00128
00129 bool isInitialized() {
00130 return initialized == 1
00131 && (dirty == 1 || dirty == 0)
00132 && (curr == 1 || curr == 0)
00133 && root[curr].size > root[curr].index
00134 && root[curr].size > root[curr].shadowIndex
00135 && root[curr].size > root[curr].indexSize*sizeof(offs_t)
00136 + root[curr].shadowIndexSize*sizeof(offs_t)
00137 && root[curr].indexSize >= root[curr].indexUsed
00138 && root[curr].indexUsed >= dbFirstUserId
00139 && root[curr].bitmapEnd > dbBitmapId;
00140 }
00141 };
00142
00143 class dbSynthesizedAttribute;
00144 class dbInheritedAttribute;
00145 class dbDatabaseThreadContext;
00146
00147 class dbMonitor {
00148 public:
00149 dbLockType accLock;
00150
00151 dbDatabaseThreadContext* firstPending;
00152 dbDatabaseThreadContext* lastPending;
00153
00154 int nLockUpgrades;
00155
00156 int nReaders;
00157 int nWriters;
00158 int backupInProgress;
00159
00160 void wait(dbLockType type, dbMutex& mutex, dbDatabaseThreadContext* ctx);
00161
00162 dbMonitor() {
00163 firstPending = lastPending = NULL;
00164 accLock = dbNoLock;
00165 backupInProgress = 0;
00166 nReaders = nWriters = 0;
00167 nLockUpgrades = 0;
00168 }
00169 };
00170
00171
00172
00173 class dbAnyCursor;
00174 class dbQuery;
00175 class dbExprNode;
00176 class dbSearchContext;
00177
00178
00179 class dbVisitedObject {
00180 public:
00181 dbVisitedObject* next;
00182 oid_t oid;
00183
00184 dbVisitedObject(oid_t oid, dbVisitedObject* chain) {
00185 this->oid = oid;
00186 next = chain;
00187 }
00188 };
00189
00193 class GIGABASE_DLL_ENTRY dbDatabase {
00194 friend class dbSelection;
00195 friend class dbAnyCursor;
00196 friend class dbHashTable;
00197 friend class dbQuery;
00198 friend class dbRtree;
00199 friend class dbRtreePage;
00200 friend class dbBtree;
00201 friend class dbBtreePage;
00202 friend class dbThickBtreePage;
00203 friend class dbInheritedAttribute;
00204 friend class dbParallelQueryContext;
00205 friend class dbServer;
00206 friend class dbPagePool;
00207
00208 friend class dbBlob;
00209 friend class dbBlobIterator;
00210 friend class dbBlobReadIterator;
00211 friend class dbBlobWriteIterator;
00212 friend class dbAnyContainer;
00213
00214 friend class dbGetTie;
00215 friend class dbPutTie;
00216
00217 friend class dbUserFunctionArgument;
00218
00219 friend class dbCLI;
00220 friend class GiSTdb;
00221 public:
00229 bool open(char_t const* databaseName, time_t transactionCommitDelay = 0, int openAttr = dbFile::no_buffering);
00230
00238 bool open(dbFile* file, time_t transactionCommitDelay = 0, bool deleteFileOnClose = false);
00239
00243 virtual void close();
00244
00248 void commit();
00249
00253 void executeBatch();
00254
00259 void precommit();
00260
00264 void rollback();
00265
00270 void attach();
00271
00272 enum DetachFlags {
00273 COMMIT = 1,
00274 DESTROY_CONTEXT = 2
00275 };
00280 void detach(int flags = COMMIT|DESTROY_CONTEXT);
00281
00286 void lock(dbLockType type = dbExclusiveLock) { beginTransaction(type); }
00287
00296 bool backup(char_t const* backupFileName, bool compactify);
00297
00304 bool restore(char_t const* backupFileName, char_t const* databaseFileName);
00305
00309 int getVersion();
00310
00315 void assign(dbTableDescriptor& desc) {
00316 assert(((void)"Table is not yet assigned to the database",
00317 desc.tableId == 0));
00318 desc.db = this;
00319 desc.fixedDatabase = true;
00320 }
00321
00328 dbTableDescriptor* lookupTable(dbTableDescriptor* desc);
00329
00337 void setConcurrency(unsigned nThreads);
00338
00343 long getAllocatedSize() { return allocatedSize; }
00344
00352 void allowColumnsDeletion(bool enabled = true) {
00353 confirmDeleteColumns = enabled;
00354 }
00355
00359 enum dbErrorClass {
00360 NoError,
00361 QueryError,
00362 ArithmeticError,
00363 IndexOutOfRangeError,
00364 DatabaseOpenError,
00365 FileError,
00366 OutOfMemoryError,
00367 Deadlock,
00368 NullReferenceError,
00369 FileLimitExeeded,
00370 DatabaseReadOnly
00371 };
00372 typedef void (*dbErrorHandler)(int error, char const* msg, int msgarg);
00373
00379 dbErrorHandler setErrorHandler(dbErrorHandler newHandler);
00380
00381
00388 virtual void scheduleBackup(char_t const* fileName, time_t periodSec);
00389
00390
00398 virtual void handleError(dbErrorClass error, char const* msg = NULL,
00399 int arg = 0);
00400
00401 enum dbAccessType {
00402 dbReadOnly = 0,
00403 dbAllAccess = 1
00404 };
00405 const dbAccessType accessType;
00406 const size_t extensionQuantum;
00407 const size_t initIndexSize;
00408
00409 static unsigned dbParallelScanThreshold;
00410
00419 void insertRecord(dbTableDescriptor* table, dbAnyReference* ref,
00420 void const* record, bool batch);
00425 offs_t used();
00426
00430 bool isOpen() const { return opened; }
00431
00438 offs_t getDatabaseSize() {
00439 return header->root[1-curr].size;
00440 }
00441
00448 void setFileExtensionQuantum(offs_t quantum) {
00449 dbFileExtensionQuantum = quantum;
00450 }
00451
00456 void setFileSizeLimit(offs_t limit) {
00457 dbFileSizeLimit = limit;
00458 }
00459
00460 #ifndef NO_MEMBER_TEMPLATES
00461
00466 template<class T>
00467 dbReference<T> insert(T const& record) {
00468 dbReference<T> ref;
00469 insertRecord(lookupTable(&T::dbDescriptor), &ref, &record, false);
00470 return ref;
00471 }
00478 template<class T>
00479 dbReference<T> batchInsert(T const& record) {
00480 dbReference<T> ref;
00481 insertRecord(lookupTable(&T::dbDescriptor), &ref, &record, true);
00482 return ref;
00483 }
00484 #endif
00485
00498 dbDatabase(dbAccessType type = dbAllAccess,
00499 size_t poolSize = 0,
00500 size_t dbExtensionQuantum = dbDefaultExtensionQuantum,
00501 size_t dbInitIndexSize = dbDefaultInitIndexSize,
00502 int nThreads = 1
00503
00504
00505
00506
00507 #ifdef NO_PTHREADS
00508 , bool usePthreads = false
00509 #endif
00510 );
00511
00515 virtual ~dbDatabase();
00516
00517 protected:
00518 dbThreadContext<dbDatabaseThreadContext> threadContext;
00519
00520 dbThreadPool threadPool;
00521
00522 dbHeader* header;
00523 int4* dirtyPagesMap;
00524 unsigned parThreads;
00525 bool modified;
00526
00527 int curr;
00528
00529
00530 bool uncommittedChanges;
00531
00532 offs_t dbFileExtensionQuantum;
00533 offs_t dbFileSizeLimit;
00534
00535
00536 volatile int commitInProgress;
00537 volatile int concurrentTransId;
00538
00539 size_t currRBitmapPage;
00540 size_t currRBitmapOffs;
00541
00542 size_t currPBitmapPage;
00543 size_t currPBitmapOffs;
00544
00545
00546 struct dbLocation {
00547 offs_t pos;
00548 size_t size;
00549 dbLocation* next;
00550 };
00551 dbLocation* reservedChain;
00552
00553 size_t committedIndexSize;
00554 size_t currIndexSize;
00555
00556 oid_t updatedRecordId;
00557
00558 dbFile* file;
00559 dbMutex mutex;
00560 dbSemaphore writeSem;
00561 dbSemaphore readSem;
00562 dbSemaphore upgradeSem;
00563 dbEvent backupCompletedEvent;
00564 dbMonitor monitor;
00565 dbPagePool pool;
00566 dbTableDescriptor* tables;
00567
00568 int* bitmapPageAvailableSpace;
00569 bool opened;
00570
00571 long allocatedSize;
00572
00573 int forceCommitCount;
00574 time_t commitDelay;
00575 time_t commitTimeout;
00576 time_t commitTimerStarted;
00577
00578 dbMutex commitThreadSyncMutex;
00579 dbMutex delayedCommitStartTimerMutex;
00580 dbMutex delayedCommitStopTimerMutex;
00581 dbEvent commitThreadSyncEvent;
00582
00583 dbEvent delayedCommitStartTimerEvent;
00584
00585 dbEvent delayedCommitStopTimerEvent;
00586 dbDatabaseThreadContext* delayedCommitContext;
00587
00588 dbMutex backupMutex;
00589 dbEvent backupInitEvent;
00590 char_t* backupFileName;
00591 time_t backupPeriod;
00592
00593 dbThread backupThread;
00594 dbThread commitThread;
00595
00596 dbTableDescriptor* batchList;
00597
00598 int accessCount;
00599
00600 dbL2List threadContextList;
00601 dbMutex threadContextListMutex;
00602
00603 dbErrorHandler errorHandler;
00604
00605 bool confirmDeleteColumns;
00606 int schemeVersion;
00607 dbVisitedObject* visitedChain;
00608
00609 bool deleteFile;
00610
00611 void releaseFile() {
00612 file->close();
00613 if (deleteFile) {
00614 delete file;
00615 }
00616 }
00617
00621 virtual void replicatePage(offs_t pageOffs, void* pageData);
00622
00626 void delayedCommit();
00627
00631 void backupScheduler();
00632
00633 static void thread_proc delayedCommitProc(void* arg) {
00634 ((dbDatabase*)arg)->delayedCommit();
00635 }
00636
00637 static void thread_proc backupSchedulerProc(void* arg) {
00638 ((dbDatabase*)arg)->backupScheduler();
00639 }
00640
00645 void commit(dbDatabaseThreadContext* ctx);
00646
00652 offs_t getPos(oid_t oid) {
00653 byte* p = pool.get(header->root[1-curr].index
00654 + oid / dbHandlesPerPage * dbPageSize);
00655 offs_t pos = *((offs_t*)p + oid % dbHandlesPerPage);
00656 pool.unfix(p);
00657 return pos;
00658 }
00659
00665 void setPos(oid_t oid, offs_t pos) {
00666 byte* p = pool.put(header->root[1-curr].index
00667 + oid / dbHandlesPerPage * dbPageSize);
00668 *((offs_t*)p + oid % dbHandlesPerPage) = pos;
00669 pool.unfix(p);
00670 }
00671
00678 dbRecord* getRow(dbGetTie& tie, oid_t oid) {
00679 offs_t pos = getPos(oid);
00680 assert(!(pos & (dbFreeHandleFlag|dbPageObjectFlag)));
00681 tie.set(pool, pos & ~dbFlagsMask);
00682 return (dbRecord*)tie.get();
00683 }
00684
00690 void getHeader(dbRecord& rec, oid_t oid) {
00691 offs_t pos = getPos(oid);
00692 int offs = (int)pos & (dbPageSize-1);
00693 byte* p = pool.get(pos - offs);
00694 rec = *(dbRecord*)(p + (offs & ~dbFlagsMask));
00695 pool.unfix(p);
00696 }
00697
00703 byte* put(oid_t oid) {
00704 offs_t pos = getPos(oid);
00705 int offs = (int)pos & (dbPageSize-1);
00706 return pool.put(pos-offs) + (offs & ~dbFlagsMask);
00707 }
00708
00714 byte* get(oid_t oid) {
00715 offs_t pos = getPos(oid);
00716 int offs = (int)pos & (dbPageSize-1);
00717 return pool.get(pos-offs) + (offs & ~dbFlagsMask);
00718 }
00719
00727 dbRecord* putRow(dbPutTie& tie, oid_t oid, size_t newSize);
00728
00735 dbRecord* putRow(dbPutTie& tie, oid_t oid);
00736
00743 byte* put(dbPutTie& tie, oid_t oid);
00744
00749 void restoreTablesConsistency();
00750
00756 void applyIndex(dbFieldDescriptor* field, dbSearchContext& sc);
00757
00774 bool isIndexApplicable(dbAnyCursor* cursor, dbExprNode* expr, dbQuery& query,
00775 dbFieldDescriptor* &indexedField, bool& truncate, bool ascent, bool forAll);
00776
00784 bool isIndexApplicableToExpr(dbSearchContext& sc, dbExprNode* expr);
00785
00789 bool followInverseReference(dbExprNode* expr, dbExprNode* andExpr,
00790 dbAnyCursor* cursor, oid_t iref);
00791
00799 bool existsInverseReference(dbExprNode* expr, int nExistsClauses);
00800
00807 static void _fastcall execute(dbExprNode* expr,
00808 dbInheritedAttribute& iattr,
00809 dbSynthesizedAttribute& sattr);
00818 bool evaluateBoolean(dbExprNode* expr, oid_t oid, dbTableDescriptor* table, dbAnyCursor* cursor);
00819
00829 size_t evaluateString(dbExprNode* expr, oid_t oid, dbTableDescriptor* table, char_t* buf, size_t bufSize);
00830
00838 void evaluate(dbExprNode* expr, oid_t oid, dbTableDescriptor* table, dbSynthesizedAttribute& result);
00839
00844 void select(dbAnyCursor* cursor);
00845
00851 void select(dbAnyCursor* cursor, dbQuery& query);
00852
00858 void traverse(dbAnyCursor* cursor, dbQuery& query);
00859
00866 void update(oid_t oid, dbTableDescriptor* table, void const* record);
00867
00873 void remove(dbTableDescriptor* table, oid_t oid);
00874
00882 offs_t allocate(size_t size, oid_t oid = 0);
00883
00889 void free(offs_t pos, size_t size);
00890
00895 void extend(offs_t size);
00896
00903 void cloneBitmap(offs_t pos, size_t size);
00904
00909 oid_t allocateId();
00910
00915 void freeId(oid_t oid);
00916
00923 void updateCursors(oid_t oid, bool removed = false);
00924
00929 oid_t allocatePage() {
00930 oid_t oid = allocateId();
00931 setPos(oid, allocate(dbPageSize) | dbPageObjectFlag | dbModifiedFlag);
00932 return oid;
00933 }
00934
00939 void freePage(oid_t oid);
00940
00948 oid_t allocateRow(oid_t tableId, size_t size,
00949 dbTableDescriptor* desc = NULL)
00950 {
00951 oid_t oid = allocateId();
00952 allocateRow(tableId, oid, size, desc);
00953 return oid;
00954 }
00955
00964 void allocateRow(oid_t tableId, oid_t oid, size_t size, dbTableDescriptor* desc);
00965
00972 void freeRow(oid_t tableId, oid_t oid, dbTableDescriptor* desc = NULL);
00973
00978 static void deleteCompiledQuery(dbExprNode* tree);
00979
00984 void beginTransaction(dbLockType type);
00989 void endTransaction(dbDatabaseThreadContext* ctx);
00990
00994 void initializeMetaTable();
00995
01002 bool loadScheme();
01003
01009 bool completeDescriptorsInitialization();
01010
01016 void reformatTable(oid_t tableId, dbTableDescriptor* desc);
01017
01023 void addIndices(dbTableDescriptor* desc);
01024
01030 oid_t addNewTable(dbTableDescriptor* desc);
01031
01038 void updateTableDescriptor(dbTableDescriptor* desc,
01039 oid_t tableId, dbTable* table);
01040
01041
01047 void removeInverseReferences(dbTableDescriptor* desc, oid_t oid);
01048
01049
01058 void insertInverseReference(dbFieldDescriptor* desc, oid_t reverseId,
01059 oid_t targetId);
01060
01069 void removeInverseReference(dbFieldDescriptor* desc,
01070 oid_t reverseId, oid_t targetId);
01071
01072
01077 void deleteTable(dbTableDescriptor* desc);
01078
01083 void dropTable(dbTableDescriptor* desc);
01084
01089 void createIndex(dbFieldDescriptor* fd);
01090
01095 void createHashTable(dbFieldDescriptor* fd);
01096
01101 void dropIndex(dbFieldDescriptor* fd);
01102
01107 void dropHashTable(dbFieldDescriptor* fd);
01108
01114 void linkTable(dbTableDescriptor* table, oid_t tableId);
01115
01120 void unlinkTable(dbTableDescriptor* table);
01121
01122
01129 bool wasReserved(offs_t pos, size_t size);
01130
01139 void reserveLocation(dbLocation& location, offs_t pos, size_t size);
01140
01145 void commitLocation();
01146
01152 dbTableDescriptor* findTable(char_t const* name);
01153
01160 dbTableDescriptor* findTableByName(char_t const* name);
01161
01162
01167 dbTableDescriptor* getTables();
01168
01169
01174 void dbDatabase::cleanupOnOpenError();
01175
01179 void setDirty();
01180 };
01181
01182 template<class T>
01183 dbReference<T> insert(T const& record) {
01184 dbReference<T> ref;
01185 T::dbDescriptor.getDatabase()->insertRecord(&T::dbDescriptor, &ref, &record, false);
01186 return ref;
01187 }
01188
01189 template<class T>
01190 dbReference<T> batchInsert(T const& record) {
01191 dbReference<T> ref;
01192 T::dbDescriptor.getDatabase()->insertRecord(&T::dbDescriptor, &ref, &record, true);
01193 return ref;
01194 }
01195
01196 #ifdef NO_MEMBER_TEMPLATES
01197 template<class T>
01198 dbReference<T> insert(dbDatabase& db, T const& record) {
01199 dbReference<T> ref;
01200 db.insertRecord(db.lookupTable(&T::dbDescriptor), &ref, &record, false);
01201 return ref;
01202 }
01203 template<class T>
01204 dbReference<T> batchInsert(dbDatabase& db, T const& record) {
01205 dbReference<T> ref;
01206 db.insertRecord(db.lookupTable(&T::dbDescriptor), &ref, &record, true);
01207 return ref;
01208 }
01209 #endif
01210
01211 class dbSearchContext {
01212 public:
01213 dbDatabase* db;
01214 dbExprNode* condition;
01215 dbAnyCursor* cursor;
01216 bool spatialSearch;
01217 char_t* firstKey;
01218 int firstKeyInclusion;
01219 char_t* lastKey;
01220 int lastKeyInclusion;
01221 int offs;
01222 int probes;
01223 bool ascent;
01224 bool tmpKeys;
01225 union {
01226 bool b;
01227 int1 i1;
01228 int2 i2;
01229 int4 i4;
01230 db_int8 i8;
01231 real4 f4;
01232 real8 f8;
01233 oid_t oid;
01234 void* raw;
01235 rectangle* rect;
01236 char_t* s;
01237 } literal[2];
01238 };
01239
01240 END_GIGABASE_NAMESPACE
01241
01242 #endif
01243