"use strict"; const { existsSync, mkdirSync, readFileSync, writeFileSync } = require("graceful-fs"); const { join, resolve } = require("path"); const { aggregateStoreData, aggregateStoreDataSync, distributeStoreData, distributeStoreDataSync, deleteStoreData, deleteStoreDataSync, dropEverything, dropEverythingSync, getStoreNames, getStoreNamesSync, insertStoreData, insertStoreDataSync, insertFileData, selectStoreData, selectStoreDataSync, statsStoreData, statsStoreDataSync, updateStoreData, updateStoreDataSync } = require("./njodb"); const { Randomizer, Reducer, Result } = require("./objects"); const { validateArray, validateFunction, validateName, validateObject, validatePath, validateSize } = require("./validators"); const defaults = { "datadir": "data", "dataname": "data", "datastores": 5, "tempdir": "tmp", "lockoptions": { "stale": 5000, "update": 1000, "retries": { "retries": 5000, "minTimeout": 250, "maxTimeout": 5000, "factor": 0.15, "randomize": false } } }; const mergeProperties = (defaults, userProperties) => { var target = Object.assign({}, defaults); for (let key of Object.keys(userProperties)) { if (Object.prototype.hasOwnProperty.call(target, key)) { if (typeof userProperties[key] !== 'object' && !Array.isArray(userProperties[key])) { Object.assign(target, { [key]: userProperties[key] }); } else { target[key] = mergeProperties(target[key], userProperties[key]); } } } return target; } const saveProperties = (root, properties) => { properties = { "datadir": properties.datadir, "dataname": properties.dataname, "datastores": properties.datastores, "tempdir": properties.tempdir, "lockoptions": properties.lockoptions }; const propertiesFile = join(root, "njodb.properties"); writeFileSync(propertiesFile, JSON.stringify(properties, null, 4)); return properties; } process.on("uncaughtException", error => { if (error.code === "ECOMPROMISED") { console.error(Object.assign(new Error("Stale lock or attempt to update it after release"), { code: error.code })); } else { throw error; } }); class Database { constructor(root, properties = {}) { validateObject(properties); this.properties = {}; if (root !== undefined && root !== null) { validateName(root); this.properties.root = root; } else { this.properties.root = process.cwd(); } if (!existsSync(this.properties.root)) mkdirSync(this.properties.root); else { console.log('Db already exists', root) } const propertiesFile = join(this.properties.root, "njodb.properties"); if (existsSync(propertiesFile)) { this.setProperties(JSON.parse(readFileSync(propertiesFile))); } else { this.setProperties(mergeProperties(defaults, properties)); } if (!existsSync(this.properties.datapath)) mkdirSync(this.properties.datapath); if (!existsSync(this.properties.temppath)) mkdirSync(this.properties.temppath); this.properties.storenames = getStoreNamesSync(this.properties.datapath, this.properties.dataname); return this; } // Database management methods getProperties() { return this.properties; } setProperties(properties) { validateObject(properties); this.properties.datadir = (validateName(properties.datadir)) ? properties.datadir : defaults.datadir; this.properties.dataname = (validateName(properties.dataname)) ? properties.dataname : defaults.dataname; this.properties.datastores = (validateSize(properties.datastores)) ? properties.datastores : defaults.datastores; this.properties.tempdir = (validateName(properties.tempdir)) ? properties.tempdir : defaults.tempdir; this.properties.lockoptions = (validateObject(properties.lockoptions)) ? properties.lockoptions : defaults.lockoptions; this.properties.datapath = join(this.properties.root, this.properties.datadir); this.properties.temppath = join(this.properties.root, this.properties.tempdir); saveProperties(this.properties.root, this.properties); return this.properties; } async stats() { var stats = { root: resolve(this.properties.root), data: resolve(this.properties.datapath), temp: resolve(this.properties.temppath) }; var promises = []; for (const storename of this.properties.storenames) { const storepath = join(this.properties.datapath, storename); promises.push(statsStoreData(storepath, this.properties.lockoptions)); } const results = await Promise.all(promises); return Object.assign(stats, Reducer("stats", results)); } statsSync() { var stats = { root: resolve(this.properties.root), data: resolve(this.properties.datapath), temp: resolve(this.properties.temppath) }; var results = []; for (const storename of this.properties.storenames) { const storepath = join(this.properties.datapath, storename); results.push(statsStoreDataSync(storepath)); } return Object.assign(stats, Reducer("stats", results)); } async grow() { this.properties.datastores++; const results = await distributeStoreData(this.properties); this.properties.storenames = await getStoreNames(this.properties.datapath, this.properties.dataname); saveProperties(this.properties.root, this.properties); return results; } growSync() { this.properties.datastores++; const results = distributeStoreDataSync(this.properties); this.properties.storenames = getStoreNamesSync(this.properties.datapath, this.properties.dataname); saveProperties(this.properties.root, this.properties); return results; } async shrink() { if (this.properties.datastores > 1) { this.properties.datastores--; const results = await distributeStoreData(this.properties); this.properties.storenames = await getStoreNames(this.properties.datapath, this.properties.dataname); saveProperties(this.properties.root, this.properties); return results; } else { throw new Error("Database cannot shrink any further"); } } shrinkSync() { if (this.properties.datastores > 1) { this.properties.datastores--; const results = distributeStoreDataSync(this.properties); this.properties.storenames = getStoreNamesSync(this.properties.datapath, this.properties.dataname); saveProperties(this.properties.root, this.properties); return results; } else { throw new Error("Database cannot shrink any further"); } } async resize(size) { validateSize(size); this.properties.datastores = size; const results = await distributeStoreData(this.properties); this.properties.storenames = await getStoreNames(this.properties.datapath, this.properties.dataname); saveProperties(this.properties.root, this.properties); return results; } resizeSync(size) { validateSize(size); this.properties.datastores = size; const results = distributeStoreDataSync(this.properties); this.properties.storenames = getStoreNamesSync(this.properties.datapath, this.properties.dataname); saveProperties(this.properties.root, this.properties); return results; } async drop() { const results = await dropEverything(this.properties); return Reducer("drop", results); } dropSync() { const results = dropEverythingSync(this.properties); return Reducer("drop", results); } // Data manipulation methods async insert(data) { validateArray(data); var promises = []; var records = []; for (let i = 0; i < this.properties.datastores; i++) { records[i] = ""; } for (let i = 0; i < data.length; i++) { records[i % this.properties.datastores] += JSON.stringify(data[i]) + "\n"; } const randomizer = Randomizer(Array.from(Array(this.properties.datastores).keys()), false); for (var j = 0; j < records.length; j++) { if (records[j] !== "") { const storenumber = randomizer.next(); const storename = [this.properties.dataname, storenumber, "json"].join("."); const storepath = join(this.properties.datapath, storename) promises.push(insertStoreData(storepath, records[j], this.properties.lockoptions)); } } const results = await Promise.all(promises); this.properties.storenames = await getStoreNames(this.properties.datapath, this.properties.dataname); return Reducer("insert", results); } insertSync(data) { validateArray(data); var results = []; var records = []; for (let i = 0; i < this.properties.datastores; i++) { records[i] = ""; } for (let i = 0; i < data.length; i++) { records[i % this.properties.datastores] += JSON.stringify(data[i]) + "\n"; } const randomizer = Randomizer(Array.from(Array(this.properties.datastores).keys()), false); for (var j = 0; j < records.length; j++) { if (records[j] !== "") { const storenumber = randomizer.next(); const storename = [this.properties.dataname, storenumber, "json"].join("."); const storepath = join(this.properties.datapath, storename) results.push(insertStoreDataSync(storepath, records[j], this.properties.lockoptions)); } } this.properties.storenames = getStoreNamesSync(this.properties.datapath, this.properties.dataname); return Reducer("insert", results); } async insertFile(file) { validatePath(file); const results = await insertFileData(file, this.properties.datapath, this.properties.storenames, this.properties.lockoptions); return results; } insertFileSync(file) { validatePath(file); const data = readFileSync(file, "utf8").split("\n"); var records = []; var results = Result("insertFile"); for (var record of data) { record = record.trim() results.lines++; if (record.length > 0) { try { records.push(JSON.parse(record)); } catch (error) { results.errors.push({ error: error.message, line: results.lines, data: record }); } } else { results.blanks++; } } return Object.assign(results, this.insertSync(records)); } async select(match, project) { validateFunction(match); if (project) validateFunction(project); var promises = []; for (const storename of this.properties.storenames) { const storepath = join(this.properties.datapath, storename); promises.push(selectStoreData(storepath, match, project, this.properties.lockoptions)); } const results = await Promise.all(promises); return Reducer("select", results); } selectSync(match, project) { validateFunction(match); if (project) validateFunction(project); var results = []; for (const storename of this.properties.storenames) { const storepath = join(this.properties.datapath, storename); results.push(selectStoreDataSync(storepath, match, project)); } return Reducer("select", results); } async update(match, update) { validateFunction(match); validateFunction(update); var promises = []; for (const storename of this.properties.storenames) { const storepath = join(this.properties.datapath, storename); const tempstorename = [storename, Date.now(), "tmp"].join("."); const tempstorepath = join(this.properties.temppath, tempstorename); promises.push(updateStoreData(storepath, match, update, tempstorepath, this.properties.lockoptions)); } const results = await Promise.all(promises); return Reducer("update", results); } updateSync(match, update) { validateFunction(match); validateFunction(update); var results = []; for (const storename of this.properties.storenames) { const storepath = join(this.properties.datapath, storename); const tempstorename = [storename, Date.now(), "tmp"].join("."); const tempstorepath = join(this.properties.temppath, tempstorename); results.push(updateStoreDataSync(storepath, match, update, tempstorepath)); } return Reducer("update", results); } async delete(match) { validateFunction(match); var promises = []; for (const storename of this.properties.storenames) { const storepath = join(this.properties.datapath, storename); const tempstorename = [storename, Date.now(), "tmp"].join("."); const tempstorepath = join(this.properties.temppath, tempstorename); promises.push(deleteStoreData(storepath, match, tempstorepath, this.properties.lockoptions)); } const results = await Promise.all(promises); return Reducer("delete", results); } deleteSync(match) { validateFunction(match); var results = []; for (const storename of this.properties.storenames) { const storepath = join(this.properties.datapath, storename); const tempstorename = [storename, Date.now(), "tmp"].join("."); const tempstorepath = join(this.properties.temppath, tempstorename); results.push(deleteStoreDataSync(storepath, match, tempstorepath)); } return Reducer("delete", results); } async aggregate(match, index, project) { validateFunction(match); validateFunction(index); if (project) validateFunction(project); var promises = []; for (const storename of this.properties.storenames) { const storepath = join(this.properties.datapath, storename); promises.push(aggregateStoreData(storepath, match, index, project, this.properties.lockoptions)); } const results = await Promise.all(promises); return Reducer("aggregate", results); } aggregateSync(match, index, project) { validateFunction(match); validateFunction(index); if (project) validateFunction(project); var results = []; for (const storename of this.properties.storenames) { const storepath = join(this.properties.datapath, storename); results.push(aggregateStoreDataSync(storepath, match, index, project)); } return Reducer("aggregate", results); } } exports.Database = Database;