CodeMirror/budibase
1
0
Fork 0
mirror of synced 2024-07-04 05:50:57 +12:00
Code Issues Projects Releases Wiki Activity

#9 All ids - replace with folder list items

Browse source
This commit is contained in:
Michael Shanks 2019-12-22 07:12:21 +00:00
parent fa683385b6
commit bc00afcf36
44 changed files with 764 additions and 451 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

40
packages/core/dist/budibase-core.cjs.js vendored
View file

@ -726,7 +726,7 @@ const getNodesInPath = appHierarchy => key => $(appHierarchy, [
fp.filter(n => new RegExp(`${n.pathRegx()}`).test(key)),
]);
const getExactNodeForPath = appHierarchy => key => $(appHierarchy, [
const getExactNodeForKey = appHierarchy => key => $(appHierarchy, [
getFlattenedHierarchy,
fp.find(n => new RegExp(`${n.pathRegx()}$`).test(key)),
]);
@ -764,7 +764,7 @@ const getCollectionNode = (appHierarchy, nodeKey) => $(appHierarchy, [
]);
const getNodeByKeyOrNodeKey = (appHierarchy, keyOrNodeKey) => {
const nodeByKey = getExactNodeForPath(appHierarchy)(keyOrNodeKey);
const nodeByKey = getExactNodeForKey(appHierarchy)(keyOrNodeKey);
return isNothing(nodeByKey)
? getNode(appHierarchy, keyOrNodeKey)
: nodeByKey;
@ -777,7 +777,7 @@ const getCollectionNodeByKeyOrNodeKey = (appHierarchy, keyOrNodeKey) => {
: nodeByKey;
};
const isNode = (appHierarchy, key) => isSomething(getExactNodeForPath(appHierarchy)(key));
const isNode = (appHierarchy, key) => isSomething(getExactNodeForKey(appHierarchy)(key));
const getActualKeyOfParent = (parentNodeKey, actualChildKey) => $(actualChildKey, [
splitKey,
@ -876,7 +876,7 @@ const fieldReversesReferenceToIndex = indexNode => field => field.type === 'refe
var hierarchy = {
getLastPartInKey,
getNodesInPath,
getExactNodeForPath,
getExactNodeForKey,
hasMatchingAncestor,
getNode,
getNodeByKeyOrNodeKey,
@ -1612,7 +1612,7 @@ const load = app => async key => {
const _load = async (app, key, keyStack = []) => {
key = safeKey(key);
const recordNode = getExactNodeForPath(app.hierarchy)(key);
const recordNode = getExactNodeForKey(app.hierarchy)(key);
const storedData = await app.datastore.loadJson(
getRecordFileName(key),
);
@ -1758,7 +1758,7 @@ const getIndexedDataKey = (indexNode, indexKey, record) => {
};
const getShardKeysInRange = async (app, indexKey, startRecord = null, endRecord = null) => {
const indexNode = getExactNodeForPath(app.hierarchy)(indexKey);
const indexNode = getExactNodeForKey(app.hierarchy)(indexKey);
const startShardName = !startRecord
? null
@ -4403,7 +4403,7 @@ const _listItems = async (app, indexKey, options = defaultOptions) => {
));
indexKey = safeKey(indexKey);
const indexNode = getExactNodeForPath(app.hierarchy)(indexKey);
const indexNode = getExactNodeForKey(app.hierarchy)(indexKey);
if (!isIndex(indexNode)) { throw new Error('supplied key is not an index'); }
@ -4435,7 +4435,7 @@ const getContext = app => recordKey => {
const _getContext = (app, recordKey) => {
recordKey = safeKey(recordKey);
const recordNode = getExactNodeForPath(app.hierarchy)(recordKey);
const recordNode = getExactNodeForKey(app.hierarchy)(recordKey);
const cachedReferenceIndexes = {};
@ -4543,7 +4543,7 @@ const validate = app => async (record, context) => {
? _getContext(app, record.key)
: context;
const recordNode = getExactNodeForPath(app.hierarchy)(record.key);
const recordNode = getExactNodeForKey(app.hierarchy)(record.key);
const fieldParseFails = validateAllFieldParse(record, recordNode);
// non parsing would cause further issues - exit here
@ -4603,7 +4603,7 @@ const initialiseRootCollections = async (datastore, hierarchy) => {
const initialiseChildCollections = async (app, recordKey) => {
const childCollectionRecords = $(recordKey, [
getExactNodeForPath(app.hierarchy),
getExactNodeForKey(app.hierarchy),
n => n.children,
fp.filter(isCollectionRecord),
]);
@ -4974,7 +4974,7 @@ const _save = async (app, record, context, skipValidation = false) => {
}
if (recordClone.isNew) {
const recordNode = getExactNodeForPath(app.hierarchy)(record.key);
const recordNode = getExactNodeForKey(app.hierarchy)(record.key);
if(!recordNode)
throw new Error("Cannot find node for " + record.key);
@ -5023,7 +5023,7 @@ const _save = async (app, record, context, skipValidation = false) => {
};
const initialiseAncestorIndexes = async (app, record) => {
const recordNode = getExactNodeForPath(app.hierarchy)(record.key);
const recordNode = getExactNodeForKey(app.hierarchy)(record.key);
for (const index of recordNode.indexes) {
const indexKey = joinKey(record.key, index.name);
@ -5032,7 +5032,7 @@ const initialiseAncestorIndexes = async (app, record) => {
};
const initialiseReverseReferenceIndexes = async (app, record) => {
const recordNode = getExactNodeForPath(app.hierarchy)(record.key);
const recordNode = getExactNodeForKey(app.hierarchy)(record.key);
const indexNodes = $(fieldsThatReferenceThisRecord(app, recordNode), [
fp.map(f => $(f.typeOptions.reverseIndexNodeKeys, [
@ -5131,7 +5131,7 @@ const deleteRecords = async (app, key) => {
const _deleteIndex = async (app, indexKey, includeFolder) => {
indexKey = safeKey(indexKey);
const indexNode = getExactNodeForPath(app.hierarchy)(indexKey);
const indexNode = getExactNodeForKey(app.hierarchy)(indexKey);
if (!isIndex(indexNode)) { throw new Error('Supplied key is not an index'); }
@ -5176,7 +5176,7 @@ const deleteRecord$1 = (app, disableCleanup = false) => async key => {
// called deleteRecord because delete is a keyword
const _deleteRecord = async (app, key, disableCleanup) => {
key = safeKey(key);
const node = getExactNodeForPath(app.hierarchy)(key);
const node = getExactNodeForKey(app.hierarchy)(key);
const record = await _load(app, key);
await transactionForDeleteRecord(app, record);
@ -5203,7 +5203,7 @@ const _deleteRecord = async (app, key, disableCleanup) => {
};
const deleteIndexes = async (app, key) => {
const node = getExactNodeForPath(app.hierarchy)(key);
const node = getExactNodeForKey(app.hierarchy)(key);
/* const reverseIndexKeys = $(app.hierarchy, [
getFlattenedHierarchy,
map(n => n.fields),
@ -5310,7 +5310,7 @@ const _uploadFile = async (app, recordKey, readableStream, relativeFilePath) =>
};
const checkFileSizeAgainstFields = (app, record, relativeFilePath, expectedSize) => {
const recordNode = getExactNodeForPath(app.hierarchy)(record.key);
const recordNode = getExactNodeForKey(app.hierarchy)(record.key);
const incorrectFileFields = $(recordNode.fields, [
fp.filter(f => f.type === 'file'
@ -5557,7 +5557,7 @@ const aggregates = app => async (indexKey, rangeStartParams = null, rangeEndPara
const _aggregates = async (app, indexKey, rangeStartParams, rangeEndParams) => {
indexKey = safeKey(indexKey);
const indexNode = getExactNodeForPath(app.hierarchy)(indexKey);
const indexNode = getExactNodeForKey(app.hierarchy)(indexKey);
if (!isIndex(indexNode)) { throw new BadRequestError('supplied key is not an index'); }
@ -7431,7 +7431,7 @@ const getRelevantAncestorIndexes = (appHierarchy, record) => {
};
const getRelevantReverseReferenceIndexes = (appHierarchy, record) => $(record.key, [
getExactNodeForPath(appHierarchy),
getExactNodeForKey(appHierarchy),
n => n.fields,
fp.filter(f => f.type === 'reference'
&& isSomething(record[f.name])
@ -7841,7 +7841,7 @@ const getBuildIndexTransactionsByShard = (hierarchy, transactions) => {
}
if (isReferenceIndex(indexNode)) {
const recordNode = getExactNodeForPath(hierarchy)(t.record.key);
const recordNode = getExactNodeForKey(hierarchy)(t.record.key);
const refFields = $(recordNode.fields, [
fp.filter(fieldReversesReferenceToIndex(indexNode)),
]);

40
packages/core/dist/budibase-core.esm.mjs vendored
View file

@ -719,7 +719,7 @@ const getNodesInPath = appHierarchy => key => $(appHierarchy, [
filter(n => new RegExp(`${n.pathRegx()}`).test(key)),
]);
const getExactNodeForPath = appHierarchy => key => $(appHierarchy, [
const getExactNodeForKey = appHierarchy => key => $(appHierarchy, [
getFlattenedHierarchy,
find(n => new RegExp(`${n.pathRegx()}$`).test(key)),
]);
@ -757,7 +757,7 @@ const getCollectionNode = (appHierarchy, nodeKey) => $(appHierarchy, [
]);
const getNodeByKeyOrNodeKey = (appHierarchy, keyOrNodeKey) => {
const nodeByKey = getExactNodeForPath(appHierarchy)(keyOrNodeKey);
const nodeByKey = getExactNodeForKey(appHierarchy)(keyOrNodeKey);
return isNothing(nodeByKey)
? getNode(appHierarchy, keyOrNodeKey)
: nodeByKey;
@ -770,7 +770,7 @@ const getCollectionNodeByKeyOrNodeKey = (appHierarchy, keyOrNodeKey) => {
: nodeByKey;
};
const isNode = (appHierarchy, key) => isSomething(getExactNodeForPath(appHierarchy)(key));
const isNode = (appHierarchy, key) => isSomething(getExactNodeForKey(appHierarchy)(key));
const getActualKeyOfParent = (parentNodeKey, actualChildKey) => $(actualChildKey, [
splitKey,
@ -869,7 +869,7 @@ const fieldReversesReferenceToIndex = indexNode => field => field.type === 'refe
var hierarchy = {
getLastPartInKey,
getNodesInPath,
getExactNodeForPath,
getExactNodeForKey,
hasMatchingAncestor,
getNode,
getNodeByKeyOrNodeKey,
@ -1605,7 +1605,7 @@ const load = app => async key => {
const _load = async (app, key, keyStack = []) => {
key = safeKey(key);
const recordNode = getExactNodeForPath(app.hierarchy)(key);
const recordNode = getExactNodeForKey(app.hierarchy)(key);
const storedData = await app.datastore.loadJson(
getRecordFileName(key),
);
@ -1751,7 +1751,7 @@ const getIndexedDataKey = (indexNode, indexKey, record) => {
};
const getShardKeysInRange = async (app, indexKey, startRecord = null, endRecord = null) => {
const indexNode = getExactNodeForPath(app.hierarchy)(indexKey);
const indexNode = getExactNodeForKey(app.hierarchy)(indexKey);
const startShardName = !startRecord
? null
@ -4396,7 +4396,7 @@ const _listItems = async (app, indexKey, options = defaultOptions) => {
));
indexKey = safeKey(indexKey);
const indexNode = getExactNodeForPath(app.hierarchy)(indexKey);
const indexNode = getExactNodeForKey(app.hierarchy)(indexKey);
if (!isIndex(indexNode)) { throw new Error('supplied key is not an index'); }
@ -4428,7 +4428,7 @@ const getContext = app => recordKey => {
const _getContext = (app, recordKey) => {
recordKey = safeKey(recordKey);
const recordNode = getExactNodeForPath(app.hierarchy)(recordKey);
const recordNode = getExactNodeForKey(app.hierarchy)(recordKey);
const cachedReferenceIndexes = {};
@ -4536,7 +4536,7 @@ const validate = app => async (record, context) => {
? _getContext(app, record.key)
: context;
const recordNode = getExactNodeForPath(app.hierarchy)(record.key);
const recordNode = getExactNodeForKey(app.hierarchy)(record.key);
const fieldParseFails = validateAllFieldParse(record, recordNode);
// non parsing would cause further issues - exit here
@ -4596,7 +4596,7 @@ const initialiseRootCollections = async (datastore, hierarchy) => {
const initialiseChildCollections = async (app, recordKey) => {
const childCollectionRecords = $(recordKey, [
getExactNodeForPath(app.hierarchy),
getExactNodeForKey(app.hierarchy),
n => n.children,
filter(isCollectionRecord),
]);
@ -4967,7 +4967,7 @@ const _save = async (app, record, context, skipValidation = false) => {
}
if (recordClone.isNew) {
const recordNode = getExactNodeForPath(app.hierarchy)(record.key);
const recordNode = getExactNodeForKey(app.hierarchy)(record.key);
if(!recordNode)
throw new Error("Cannot find node for " + record.key);
@ -5016,7 +5016,7 @@ const _save = async (app, record, context, skipValidation = false) => {
};
const initialiseAncestorIndexes = async (app, record) => {
const recordNode = getExactNodeForPath(app.hierarchy)(record.key);
const recordNode = getExactNodeForKey(app.hierarchy)(record.key);
for (const index of recordNode.indexes) {
const indexKey = joinKey(record.key, index.name);
@ -5025,7 +5025,7 @@ const initialiseAncestorIndexes = async (app, record) => {
};
const initialiseReverseReferenceIndexes = async (app, record) => {
const recordNode = getExactNodeForPath(app.hierarchy)(record.key);
const recordNode = getExactNodeForKey(app.hierarchy)(record.key);
const indexNodes = $(fieldsThatReferenceThisRecord(app, recordNode), [
map(f => $(f.typeOptions.reverseIndexNodeKeys, [
@ -5124,7 +5124,7 @@ const deleteRecords = async (app, key) => {
const _deleteIndex = async (app, indexKey, includeFolder) => {
indexKey = safeKey(indexKey);
const indexNode = getExactNodeForPath(app.hierarchy)(indexKey);
const indexNode = getExactNodeForKey(app.hierarchy)(indexKey);
if (!isIndex(indexNode)) { throw new Error('Supplied key is not an index'); }
@ -5169,7 +5169,7 @@ const deleteRecord$1 = (app, disableCleanup = false) => async key => {
// called deleteRecord because delete is a keyword
const _deleteRecord = async (app, key, disableCleanup) => {
key = safeKey(key);
const node = getExactNodeForPath(app.hierarchy)(key);
const node = getExactNodeForKey(app.hierarchy)(key);
const record = await _load(app, key);
await transactionForDeleteRecord(app, record);
@ -5196,7 +5196,7 @@ const _deleteRecord = async (app, key, disableCleanup) => {
};
const deleteIndexes = async (app, key) => {
const node = getExactNodeForPath(app.hierarchy)(key);
const node = getExactNodeForKey(app.hierarchy)(key);
/* const reverseIndexKeys = $(app.hierarchy, [
getFlattenedHierarchy,
map(n => n.fields),
@ -5303,7 +5303,7 @@ const _uploadFile = async (app, recordKey, readableStream, relativeFilePath) =>
};
const checkFileSizeAgainstFields = (app, record, relativeFilePath, expectedSize) => {
const recordNode = getExactNodeForPath(app.hierarchy)(record.key);
const recordNode = getExactNodeForKey(app.hierarchy)(record.key);
const incorrectFileFields = $(recordNode.fields, [
filter(f => f.type === 'file'
@ -5550,7 +5550,7 @@ const aggregates = app => async (indexKey, rangeStartParams = null, rangeEndPara
const _aggregates = async (app, indexKey, rangeStartParams, rangeEndParams) => {
indexKey = safeKey(indexKey);
const indexNode = getExactNodeForPath(app.hierarchy)(indexKey);
const indexNode = getExactNodeForKey(app.hierarchy)(indexKey);
if (!isIndex(indexNode)) { throw new BadRequestError('supplied key is not an index'); }
@ -7424,7 +7424,7 @@ const getRelevantAncestorIndexes = (appHierarchy, record) => {
};
const getRelevantReverseReferenceIndexes = (appHierarchy, record) => $(record.key, [
getExactNodeForPath(appHierarchy),
getExactNodeForKey(appHierarchy),
n => n.fields,
filter(f => f.type === 'reference'
&& isSomething(record[f.name])
@ -7834,7 +7834,7 @@ const getBuildIndexTransactionsByShard = (hierarchy, transactions) => {
}
if (isReferenceIndex(indexNode)) {
const recordNode = getExactNodeForPath(hierarchy)(t.record.key);
const recordNode = getExactNodeForKey(hierarchy)(t.record.key);
const refFields = $(recordNode.fields, [
filter(fieldReversesReferenceToIndex(indexNode)),
]);

40
packages/core/dist/budibase-core.umd.js vendored
View file

@ -721,7 +721,7 @@
fp.filter(n => new RegExp(`${n.pathRegx()}`).test(key)),
]);
const getExactNodeForPath = appHierarchy => key => $(appHierarchy, [
const getExactNodeForKey = appHierarchy => key => $(appHierarchy, [
getFlattenedHierarchy,
fp.find(n => new RegExp(`${n.pathRegx()}$`).test(key)),
]);
@ -759,7 +759,7 @@
]);
const getNodeByKeyOrNodeKey = (appHierarchy, keyOrNodeKey) => {
const nodeByKey = getExactNodeForPath(appHierarchy)(keyOrNodeKey);
const nodeByKey = getExactNodeForKey(appHierarchy)(keyOrNodeKey);
return isNothing(nodeByKey)
? getNode(appHierarchy, keyOrNodeKey)
: nodeByKey;
@ -772,7 +772,7 @@
: nodeByKey;
};
const isNode = (appHierarchy, key) => isSomething(getExactNodeForPath(appHierarchy)(key));
const isNode = (appHierarchy, key) => isSomething(getExactNodeForKey(appHierarchy)(key));
const getActualKeyOfParent = (parentNodeKey, actualChildKey) => $(actualChildKey, [
splitKey,
@ -871,7 +871,7 @@
var hierarchy = {
getLastPartInKey,
getNodesInPath,
getExactNodeForPath,
getExactNodeForKey,
hasMatchingAncestor,
getNode,
getNodeByKeyOrNodeKey,
@ -1607,7 +1607,7 @@
const _load = async (app, key, keyStack = []) => {
key = safeKey(key);
const recordNode = getExactNodeForPath(app.hierarchy)(key);
const recordNode = getExactNodeForKey(app.hierarchy)(key);
const storedData = await app.datastore.loadJson(
getRecordFileName(key),
);
@ -1753,7 +1753,7 @@
};
const getShardKeysInRange = async (app, indexKey, startRecord = null, endRecord = null) => {
const indexNode = getExactNodeForPath(app.hierarchy)(indexKey);
const indexNode = getExactNodeForKey(app.hierarchy)(indexKey);
const startShardName = !startRecord
? null
@ -4398,7 +4398,7 @@
));
indexKey = safeKey(indexKey);
const indexNode = getExactNodeForPath(app.hierarchy)(indexKey);
const indexNode = getExactNodeForKey(app.hierarchy)(indexKey);
if (!isIndex(indexNode)) { throw new Error('supplied key is not an index'); }
@ -4430,7 +4430,7 @@
const _getContext = (app, recordKey) => {
recordKey = safeKey(recordKey);
const recordNode = getExactNodeForPath(app.hierarchy)(recordKey);
const recordNode = getExactNodeForKey(app.hierarchy)(recordKey);
const cachedReferenceIndexes = {};
@ -4538,7 +4538,7 @@
? _getContext(app, record.key)
: context;
const recordNode = getExactNodeForPath(app.hierarchy)(record.key);
const recordNode = getExactNodeForKey(app.hierarchy)(record.key);
const fieldParseFails = validateAllFieldParse(record, recordNode);
// non parsing would cause further issues - exit here
@ -4598,7 +4598,7 @@
const initialiseChildCollections = async (app, recordKey) => {
const childCollectionRecords = $(recordKey, [
getExactNodeForPath(app.hierarchy),
getExactNodeForKey(app.hierarchy),
n => n.children,
fp.filter(isCollectionRecord),
]);
@ -4969,7 +4969,7 @@
}
if (recordClone.isNew) {
const recordNode = getExactNodeForPath(app.hierarchy)(record.key);
const recordNode = getExactNodeForKey(app.hierarchy)(record.key);
if(!recordNode)
throw new Error("Cannot find node for " + record.key);
@ -5018,7 +5018,7 @@
};
const initialiseAncestorIndexes = async (app, record) => {
const recordNode = getExactNodeForPath(app.hierarchy)(record.key);
const recordNode = getExactNodeForKey(app.hierarchy)(record.key);
for (const index of recordNode.indexes) {
const indexKey = joinKey(record.key, index.name);
@ -5027,7 +5027,7 @@
};
const initialiseReverseReferenceIndexes = async (app, record) => {
const recordNode = getExactNodeForPath(app.hierarchy)(record.key);
const recordNode = getExactNodeForKey(app.hierarchy)(record.key);
const indexNodes = $(fieldsThatReferenceThisRecord(app, recordNode), [
fp.map(f => $(f.typeOptions.reverseIndexNodeKeys, [
@ -5126,7 +5126,7 @@
const _deleteIndex = async (app, indexKey, includeFolder) => {
indexKey = safeKey(indexKey);
const indexNode = getExactNodeForPath(app.hierarchy)(indexKey);
const indexNode = getExactNodeForKey(app.hierarchy)(indexKey);
if (!isIndex(indexNode)) { throw new Error('Supplied key is not an index'); }
@ -5171,7 +5171,7 @@
// called deleteRecord because delete is a keyword
const _deleteRecord = async (app, key, disableCleanup) => {
key = safeKey(key);
const node = getExactNodeForPath(app.hierarchy)(key);
const node = getExactNodeForKey(app.hierarchy)(key);
const record = await _load(app, key);
await transactionForDeleteRecord(app, record);
@ -5198,7 +5198,7 @@
};
const deleteIndexes = async (app, key) => {
const node = getExactNodeForPath(app.hierarchy)(key);
const node = getExactNodeForKey(app.hierarchy)(key);
/* const reverseIndexKeys = $(app.hierarchy, [
getFlattenedHierarchy,
map(n => n.fields),
@ -5305,7 +5305,7 @@
};
const checkFileSizeAgainstFields = (app, record, relativeFilePath, expectedSize) => {
const recordNode = getExactNodeForPath(app.hierarchy)(record.key);
const recordNode = getExactNodeForKey(app.hierarchy)(record.key);
const incorrectFileFields = $(recordNode.fields, [
fp.filter(f => f.type === 'file'
@ -5552,7 +5552,7 @@
const _aggregates = async (app, indexKey, rangeStartParams, rangeEndParams) => {
indexKey = safeKey(indexKey);
const indexNode = getExactNodeForPath(app.hierarchy)(indexKey);
const indexNode = getExactNodeForKey(app.hierarchy)(indexKey);
if (!isIndex(indexNode)) { throw new BadRequestError('supplied key is not an index'); }
@ -7426,7 +7426,7 @@
};
const getRelevantReverseReferenceIndexes = (appHierarchy, record) => $(record.key, [
getExactNodeForPath(appHierarchy),
getExactNodeForKey(appHierarchy),
n => n.fields,
fp.filter(f => f.type === 'reference'
&& isSomething(record[f.name])
@ -7836,7 +7836,7 @@
}
if (isReferenceIndex(indexNode)) {
const recordNode = getExactNodeForPath(hierarchy)(t.record.key);
const recordNode = getExactNodeForKey(hierarchy)(t.record.key);
const refFields = $(recordNode.fields, [
fp.filter(fieldReversesReferenceToIndex(indexNode)),
]);

0
packages/core/src/appInitialise/inProcIndexer.js
View file

16
packages/core/src/appInitialise/index.js
View file

@ -1,23 +1,27 @@
import { retry } from '../common/index';
import { NotFoundError } from '../common/errors';
const createJson = originalCreateFile => async (key, obj, retries = 5, delay = 500) => await retry(originalCreateFile, retries, delay, key, JSON.stringify(obj));
const createJson = originalCreateFile => async (key, obj, retries = 2, delay = 100) => await retry(originalCreateFile, retries, delay, key, JSON.stringify(obj));
const createNewFile = originalCreateFile => async (path, content, retries = 5, delay = 500) => await retry(originalCreateFile, retries, delay, path, content);
const createNewFile = originalCreateFile => async (path, content, retries = 2, delay = 100) => await retry(originalCreateFile, retries, delay, path, content);
const loadJson = datastore => async (key, retries = 5, delay = 500) => {
const loadJson = datastore => async (key, retries = 3, delay = 100) => {
try {
return await retry(JSON.parse, retries, delay, await datastore.loadFile(key));
} catch (err) {
throw new NotFoundError(err.message);
const newErr = new NotFoundError(err.message);
newErr.stack = err.stack;
throw(newErr);
}
}
const updateJson = datastore => async (key, obj, retries = 5, delay = 500) => {
const updateJson = datastore => async (key, obj, retries = 3, delay = 100) => {
try {
return await retry(datastore.updateFile, retries, delay, key, JSON.stringify(obj));
} catch (err) {
throw new NotFoundError(err.message);
const newErr = new NotFoundError(err.message);
newErr.stack = err.stack;
throw(newErr);
}
}

5
packages/core/src/appInitialise/initialiseData.js
View file

@ -35,7 +35,9 @@ const initialiseRootIndexes = async (datastore, hierarchy) => {
]);
for (const index of globalIndexes) {
if (!await datastore.exists(index.nodeKey())) { await initialiseIndex(datastore, '', index); }
if (!await datastore.exists(index.nodeKey())) {
await initialiseIndex(datastore, '', index);
}
}
};
@ -52,6 +54,7 @@ const initialiseRootSingleRecords = async (datastore, hierarchy) => {
]);
for (let record of singleRecords) {
await datastore.createFolder(record.nodeKey());
const result = _getNew(record, "");
await _save(app,result);
}

48
packages/core/src/collectionApi/delete.js
View file

@ -1,13 +1,11 @@
import { includes } from 'lodash/fp';
import { getNodeForCollectionPath } from '../templateApi/hierarchy';
import {
safeKey, apiWrapper,
events, joinKey,
} from '../common';
import { _deleteRecord } from '../recordApi/delete';
import { getAllIdsIterator, getAllIdsShardKey, folderStructureArray } from '../indexing/allIds';
import { getAllIdsIterator } from '../indexing/allIds';
import { permission } from '../authApi/permissions';
import { getRecordInfo } from "../recordApi/recordInfo";
import { getCollectionDir } from "../recordApi/recordInfo";
export const deleteCollection = (app, disableCleanup = false) => async key => apiWrapper(
app,
@ -17,28 +15,38 @@ export const deleteCollection = (app, disableCleanup = false) => async key => ap
_deleteCollection, app, key, disableCleanup,
);
/*
const recordNode = getCollectionNode(app.hierarchy, key);
*/
export const _deleteCollection = async (app, key, disableCleanup) => {
const recordInfo = getRecordInfo(key);
await app.datastore.deleteFolder(recordInfo)
key = safeKey(key);
const collectionDir = getCollectionDir(app.hierarchy, key);
await deleteRecords(app, key);
await deleteCollectionFolder(app, collectionDir);
if (!disableCleanup) { await app.cleanupTransactions(); }
};
const deleteCollectionFolder = async (app, key) => await app.datastore.deleteFolder(key);
const deleteCollectionFolder = async (app, dir) =>
await app.datastore.deleteFolder(dir);
const deleteRecords = async (app, key) => {
const iterate = await getAllIdsIterator(app)(key);
const deleteShardFolders = async (app, node, key) => {
let ids = await iterate();
while (!ids.done) {
if (ids.result.collectionKey === key) {
for (const id of ids.result.ids) {
await _deleteRecord(
app,
joinKey(key, id),
true,
);
}
}
await app.datastore.deleteFolder(
joinKey(
key, 'allids',
node.nodeId,
),
);
await app.datastore.deleteFolder(
joinKey(key, 'allids'),
);
ids = await iterate();
}
};

4
packages/core/src/collectionApi/initialise.js
View file

@ -3,13 +3,13 @@ import {
getFlattenedHierarchy,
isCollectionRecord,
isRoot,
getExactNodeForPath,
} from '../templateApi/hierarchy';
import { $, allTrue, joinKey } from '../common';
const ensureCollectionIsInitialised = async (datastore, node, dir) => {
if (!await datastore.exists(dir)) {
await datastore.createFolder(dir);
await datastore.createFolder(joinKey(dir, node.nodeId));
}
};
@ -29,7 +29,7 @@ export const initialiseRootCollections = async (datastore, hierarchy) => {
await ensureCollectionIsInitialised(
datastore,
col,
col.collectionPathRegx(),
col.collectionPathRegx()
);
}
};

16
packages/core/src/common/index.js
View file

@ -4,13 +4,12 @@ import {
tail, findIndex, startsWith,
dropRight, flow, takeRight, trim,
replace
} from 'lodash';
import {
some, reduce, isEmpty, isArray, join,
isString, isInteger, isDate, toNumber,
isUndefined, isNaN, isNull, constant,
split, includes
split, includes, filter
} from 'lodash/fp';
import { events, eventsList } from './events';
import { apiWrapper } from './apiWrapper';
@ -32,7 +31,13 @@ export const safeKey = key => replace(`${keySep}${trimKeySep(key)}`, `${keySep}$
export const joinKey = (...strs) => {
const paramsOrArray = strs.length === 1 & isArray(strs[0])
? strs[0] : strs;
return safeKey(join(keySep)(paramsOrArray));
return $(paramsOrArray, [
filter(s => !isUndefined(s)
&& !isNull(s)
&& s.toString().length > 0),
join(keySep),
safeKey
]);
};
export const splitKey = $$(trimKeySep, splitByKeySep);
export const getDirFomKey = $$(splitKey, dropRight, p => joinKey(...p));
@ -183,6 +188,10 @@ export const toNumberOrNull = s => (isNull(s) ? null
export const isArrayOfString = opts => isArray(opts) && all(isString)(opts);
export const pushAll = (target, items) => {
for(let i of items) target.push(i);
}
export const pause = async duration => new Promise(res => setTimeout(res, duration));
export const retry = async (fn, retries, delay, ...args) => {
@ -267,4 +276,5 @@ export default {
insensitiveEquals,
pause,
retry,
pushAll
};

10
packages/core/src/indexApi/aggregates.js
View file

@ -10,12 +10,13 @@ import {
getShardKeysInRange,
} from '../indexing/sharding';
import {
getExactNodeForPath, isIndex,
getExactNodeForKey, isIndex,
isShardedIndex,
} from '../templateApi/hierarchy';
import { CONTINUE_READING_RECORDS } from '../indexing/serializer';
import { permission } from '../authApi/permissions';
import { BadRequestError } from '../common/errors';
import { getIndexDir } from "./getIndexDir";
export const aggregates = app => async (indexKey, rangeStartParams = null, rangeEndParams = null) => apiWrapper(
app,
@ -27,13 +28,14 @@ export const aggregates = app => async (indexKey, rangeStartParams = null, range
const _aggregates = async (app, indexKey, rangeStartParams, rangeEndParams) => {
indexKey = safeKey(indexKey);
const indexNode = getExactNodeForPath(app.hierarchy)(indexKey);
const indexNode = getExactNodeForKey(app.hierarchy)(indexKey);
const indexDir = getIndexDir(app.hierarchy, indexKey);
if (!isIndex(indexNode)) { throw new BadRequestError('supplied key is not an index'); }
if (isShardedIndex(indexNode)) {
const shardKeys = await getShardKeysInRange(
app, indexKey, rangeStartParams, rangeEndParams,
app, indexNode, indexDir, rangeStartParams, rangeEndParams,
);
let aggregateResult = null;
for (const k of shardKeys) {
@ -53,7 +55,7 @@ const _aggregates = async (app, indexKey, rangeStartParams, rangeEndParams) => {
app.hierarchy,
app.datastore,
indexNode,
getUnshardedIndexDataKey(indexKey),
getUnshardedIndexDataKey(indexDir),
);
};

16
packages/core/src/indexApi/buildIndex.js
View file

@ -1,16 +1,16 @@
import {
find, filter,
filter,
includes, some,
} from 'lodash/fp';
import { getAllIdsIterator } from '../indexing/allIds';
import {
getFlattenedHierarchy, getRecordNodeById,
getCollectionNodeByKeyOrNodeKey, getNode, isIndex,
isRecord, isDecendant, getAllowedRecordNodesForIndex,
getNode, isIndex,
isRecord, getAllowedRecordNodesForIndex,
fieldReversesReferenceToIndex,
} from '../templateApi/hierarchy';
import {
joinKey, apiWrapper, events, $, allTrue,
joinKey, apiWrapper, events, $
} from '../common';
import {
createBuildIndexFolder,
@ -82,9 +82,11 @@ const buildReverseReferenceIndex = async (app, indexNode) => {
}
};
/*
const getAllowedParentCollectionNodes = (hierarchy, indexNode) => $(getAllowedRecordNodesForIndex(hierarchy, indexNode), [
map(n => n.parent()),
]);
*/
const buildHeirarchalIndex = async (app, indexNode) => {
let recordCount = 0;
@ -127,10 +129,11 @@ const buildHeirarchalIndex = async (app, indexNode) => {
return recordCount;
};
const chooseChildRecordNodeByKey = (collectionNode, recordId) => find(c => recordId.startsWith(c.nodeId))(collectionNode.children);
// const chooseChildRecordNodeByKey = (collectionNode, recordId) => find(c => recordId.startsWith(c.nodeId))(collectionNode.children);
const recordNodeApplies = indexNode => recordNode => includes(recordNode.nodeId)(indexNode.allowedRecordNodeIds);
/*
const hasApplicableDecendant = (hierarchy, ancestorNode, indexNode) => $(hierarchy, [
getFlattenedHierarchy,
filter(
@ -141,7 +144,9 @@ const hasApplicableDecendant = (hierarchy, ancestorNode, indexNode) => $(hierarc
),
),
]);
*/
/*
const applyAllDecendantRecords = async (app, collection_Key_or_NodeKey,
indexNode, indexKey, currentIndexedData,
currentIndexedDataKey, recordCount = 0) => {
@ -194,5 +199,6 @@ const applyAllDecendantRecords = async (app, collection_Key_or_NodeKey,
return recordCount;
};
*/
export default buildIndex;

14
packages/core/src/indexApi/delete.js
View file

@ -4,21 +4,23 @@ import {
} from '../common';
import {
isIndex, isShardedIndex,
getExactNodeForPath,
getExactNodeForKey,
} from '../templateApi/hierarchy';
import {
getAllShardKeys, getShardMapKey,
getUnshardedIndexDataKey,
} from '../indexing/sharding';
import { getIndexDir } from "./getIndexDir";
export const _deleteIndex = async (app, indexKey, includeFolder) => {
indexKey = safeKey(indexKey);
const indexNode = getExactNodeForPath(app.hierarchy)(indexKey);
const indexNode = getExactNodeForKey(app.hierarchy)(indexKey);
const indexDir = getIndexDir(app.hierarchy, indexKey);
if (!isIndex(indexNode)) { throw new Error('Supplied key is not an index'); }
if (isShardedIndex(indexNode)) {
const shardKeys = await getAllShardKeys(app, indexKey);
const shardKeys = await getAllShardKeys(app, indexNode, indexDir);
for (const k of shardKeys) {
await tryAwaitOrIgnore(
app.datastore.deleteFile(k),
@ -26,20 +28,20 @@ export const _deleteIndex = async (app, indexKey, includeFolder) => {
}
tryAwaitOrIgnore(
await app.datastore.deleteFile(
getShardMapKey(indexKey),
getShardMapKey(indexDir),
),
);
} else {
await tryAwaitOrIgnore(
app.datastore.deleteFile(
getUnshardedIndexDataKey(indexKey),
getUnshardedIndexDataKey(indexDir),
),
);
}
if (includeFolder) {
tryAwaitOrIgnore(
await app.datastore.deleteFolder(indexKey),
await app.datastore.deleteFolder(indexDir),
);
}
};

20
packages/core/src/indexApi/getIndexDir.js Normal file
View file

@ -0,0 +1,20 @@
import { getRecordInfo } from "../recordApi/recordInfo";
import {
getParentKey, getLastPartInKey
} from "../templateApi/hierarchy";
import { keySep } from "../common";
export const getIndexDir = (hierarchy, indexKey) => {
const parentKey = getParentKey(indexKey);
if(parentKey === "") return indexKey;
if(parentKey === keySep) return indexKey;
const recordInfo = getRecordInfo(
hierarchy,
parentKey);
return recordInfo.child(
getLastPartInKey(indexKey));
}

16
packages/core/src/indexApi/listItems.js
View file

@ -9,10 +9,11 @@ import {
getShardKeysInRange,
} from '../indexing/sharding';
import {
getExactNodeForPath, isIndex,
getExactNodeForKey, isIndex,
isShardedIndex,
} from '../templateApi/hierarchy';
import { permission } from '../authApi/permissions';
import { getIndexDir } from "./getIndexDir";
export const listItems = app => async (indexKey, options) => {
indexKey = safeKey(indexKey);
@ -33,29 +34,30 @@ const _listItems = async (app, indexKey, options = defaultOptions) => {
merge(defaultOptions),
]);
const getItems = async key => (isNonEmptyString(searchPhrase)
const getItems = async indexedDataKey => (isNonEmptyString(searchPhrase)
? await searchIndex(
app.hierarchy,
app.datastore,
indexNode,
key,
indexedDataKey,
searchPhrase,
)
: await readIndex(
app.hierarchy,
app.datastore,
indexNode,
key,
indexedDataKey,
));
indexKey = safeKey(indexKey);
const indexNode = getExactNodeForPath(app.hierarchy)(indexKey);
const indexNode = getExactNodeForKey(app.hierarchy)(indexKey);
const indexDir = getIndexDir(app.hierarchy, indexKey);
if (!isIndex(indexNode)) { throw new Error('supplied key is not an index'); }
if (isShardedIndex(indexNode)) {
const shardKeys = await getShardKeysInRange(
app, indexKey, rangeStartParams, rangeEndParams,
app, indexNode, indexDir, rangeStartParams, rangeEndParams,
);
const items = [];
for (const k of shardKeys) {
@ -64,6 +66,6 @@ const _listItems = async (app, indexKey, options = defaultOptions) => {
return flatten(items);
}
return await getItems(
getUnshardedIndexDataKey(indexKey),
getUnshardedIndexDataKey(indexDir),
);
};

203
packages/core/src/indexing/allIds.js
View file

@ -1,13 +1,14 @@
import {
join, flatten, orderBy,
filter
flatten, orderBy,
filter, isUndefined
} from 'lodash/fp';
import {
import hierarchy, {
getFlattenedHierarchy,
getCollectionNodeByKeyOrNodeKey,
isCollectionRecord, isAncestor,
} from '../templateApi/hierarchy';
import { joinKey, safeKey, $ } from '../common';
import { getCollectionDir } from "../recordApi/recordInfo";
export const RECORDS_PER_FOLDER = 1000;
export const allIdChars = '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ_-';
@ -23,19 +24,39 @@ export const allIdChars = '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQ
* - [64, 64, 10] = all records fit into 64 * 64 * 10 folder
* (there are 64 possible chars in allIsChars)
*/
export const folderStructureArray = (recordNode, currentArray=[], currentFolderPosition=0) => {
export const folderStructureArray = (recordNode) => {
const totalFolders = Math.ceil(recordNode.estimatedRecordCount / 1000);
const folderArray = [];
let levelCount = 1;
while(64**levelCount < totalFolders) {
levelCount += 1;
folderArray.push(64);
}
const parentFactor = (64**folderArray.length);
if(parentFactor < totalFolders) {
folderArray.push(
Math.ceil(totalFolders / parentFactor)
);
}
return folderArray;
/*
const maxRecords = currentFolderPosition === 0
? RECORDS_PER_FOLDER
: currentFolderPosition * 64 * RECORDS_PER_FOLDER;
if(maxRecords < recordNode.estimatedRecordCount) {
return folderStructureArray(
recordNode,
[...currentArray, 64],
currentFolderPosition + 1);
} else {
const childFolderCount = Math.ceil(maxRecords / recordNode.estimatedRecordCount);
const childFolderCount = Math.ceil(recordNode.estimatedRecordCount / maxRecords );
return [...currentArray, childFolderCount]
}
}*/
}
@ -51,88 +72,150 @@ export const getAllIdsIterator = app => async (collection_Key_or_NodeKey) => {
const folderStructure = folderStructureArray(recordNode)
let currentFolderContents = [];
let currentFolderIndexes = [];
let currentSubPath = [];
let currentPosition = [];
const collectionDir = getCollectionDir(app.hierarchy, collectionKey);
const basePath = joinKey(
collectionKey, recordNode.nodeId.toString());
collectionDir, recordNode.nodeId.toString());
let folderLevel = 0;
const levels = folderStructure.length;
// "folderStructure" determines the top, sharding folders
// we need to add one, for the collection root folder, which
// always exists
const levels = folderStructure.length + 1;
const topLevel = levels -1;
const lastPathHasContent = () =>
folderLevel === 0
|| currentFolderContents[folderLevel - 1].length > 0;
/* populate initial directory structure in form:
[
{path: "/a", contents: ["b", "c", "d"]},
{path: "/a/b", contents: ["e","f","g"]},
{path: "/a/b/e", contents: ["1-abcd","2-cdef","3-efgh"]},
]
// stores contents on each parent level
// top level has ID folders
*/
const firstFolder = async () => {
while (folderLevel < folderStructure.length && lastPathHasContent()) {
if(folderLevel < topLevel) {
const contentsThisLevel =
await app.datastore.getFolderContents(
join(basePath, ...currentSubPath));
let folderLevel = 0;
currentFolderContents.push(contentsThisLevel);
currentFolderIndexes.push(0);
currentSubPath.push(currentFolderContents[0])
} else {
// placesholders only for the top level (which will be populated by nextFolder())
currentFolderContents.push([])
currentFolderIndexes.push(-1);
currentSubPath.push("");
}
const lastPathHasContent = () =>
folderLevel === 0
|| currentFolderContents[folderLevel - 1].contents.length > 0;
folderLevel+=1;
}
while (folderLevel <= topLevel && lastPathHasContent()) {
const nextFolder = async (lev=-1) => {
lev = (lev === -1) ? topLevel : lev;
if(currentFolderIndexes[lev] !== currentFolderContents[lev].length - 1){
const folderIndexThisLevel = currentFolderIndexes[lev] + 1;
currentFolderIndexes[lev] = folderIndexThisLevel;
currentSubPath[lev] = currentFolderContents[folderIndexThisLevel]
if(lev < topLevel) {
let loopLev = lev + 1;
while(loopLev <= topLevel) {
currentFolderContents[loopLev] =
await app.datastore.getFolderContents(join(basePath, ...currentSubPath));
loopLev+=1;
}
let thisPath = basePath;
for(let lev = 0; lev < currentPosition.length; lev++) {
thisPath = joinKey(
thisPath, currentFolderContents[lev].contents[0]);
}
return false; // not complete
const contentsThisLevel =
await app.datastore.getFolderContents(thisPath);
currentFolderContents.push({
contents:contentsThisLevel,
path: thisPath
});
} else {
if(lev === 0) return true; // complete
return await nextFolder(lev - 1);
// should start as something like [0,0]
if(folderLevel < topLevel)
currentPosition.push(0);
folderLevel+=1;
}
return (currentPosition.length === levels - 1);
}
const isOnLastFolder = level => {
const result = currentPosition[level] === currentFolderContents[level].contents.length - 1;
return result;
}
const getNextFolder = async (lev=undefined) => {
lev = isUndefined(lev) ? topLevel : lev;
const parentLev = lev - 1;
if(parentLev < 0) return false;
if(isOnLastFolder(parentLev)) {
return await getNextFolder(parentLev);
}
const newPosition = currentPosition[parentLev] + 1;
currentPosition[parentLev] = newPosition;
const nextFolder = joinKey(
currentFolderContents[parentLev].path,
currentFolderContents[parentLev].contents[newPosition]);
currentFolderContents[lev].contents = await app.datastore.getFolderContents(
nextFolder
);
currentFolderContents[lev].path = nextFolder;
if(lev !== topLevel) {
// we just advanced a parent folder, so now need to
// do the same to the next levels
let loopLevel = lev + 1;
while(loopLevel <= topLevel) {
const loopParentLevel = loopLevel-1;
currentPosition[loopParentLevel] = 0;
const nextLoopFolder = joinKey(
currentFolderContents[loopParentLevel].path,
currentFolderContents[loopParentLevel].contents[0]);
currentFolderContents[loopLevel].contents = await app.datastore.getFolderContents(
nextLoopFolder
);
currentFolderContents[loopLevel].path = nextLoopFolder;
loopLevel+=1;
}
}
// true ==has more ids... (just loaded more)
return true;
}
const idsCurrentFolder = () =>
currentFolderContents[currentFolderContents.length - 1].contents;
const fininshedResult = ({ done: true, result: { ids: [], collectionKey } });
let hasStarted = false;
let hasMore = true;
const getIdsFromCurrentfolder = async () => {
if(currentFolderIndexes.length < folderStructure)
if(!hasMore) {
return fininshedResult;
}
const hasMore = await nextFolder();
if(!hasStarted) {
hasMore = await firstFolder();
hasStarted = true;
return ({
result: {
ids: idsCurrentFolder(),
collectionKey
},
done: false
})
}
if(!hasMore) return fininshedResult;
hasMore = await getNextFolder();
const result = ({
return ({
result: {
ids: await app.datastore.getFolderContents(
joinKey(basePath, currentSubPath)),
ids: hasMore ? idsCurrentFolder() : [],
collectionKey
},
done:false
})
return result;
done: !hasMore
});
}
return getIdsFromCurrentfolder;

28
packages/core/src/indexing/apply.js
View file

@ -1,13 +1,13 @@
import { ensureShardNameIsInShardMap } from './sharding';
import { getIndexWriter } from './serializer';
import { isShardedIndex } from '../templateApi/hierarchy';
import { isShardedIndex, getParentKey } from '../templateApi/hierarchy';
import {promiseWriteableStream} from "./promiseWritableStream";
import {promiseReadableStream} from "./promiseReadableStream";
export const applyToShard = async (hierarchy, store, indexKey,
export const applyToShard = async (hierarchy, store, indexDir,
indexNode, indexShardKey, recordsToWrite, keysToRemove) => {
const createIfNotExists = recordsToWrite.length > 0;
const writer = await getWriter(hierarchy, store, indexKey, indexShardKey, indexNode, createIfNotExists);
const writer = await getWriter(hierarchy, store, indexDir, indexShardKey, indexNode, createIfNotExists);
if (writer === SHARD_DELETED) return;
await writer.updateIndex(recordsToWrite, keysToRemove);
@ -15,13 +15,17 @@ export const applyToShard = async (hierarchy, store, indexKey,
};
const SHARD_DELETED = 'SHARD_DELETED';
const getWriter = async (hierarchy, store, indexKey, indexedDataKey, indexNode, createIfNotExists) => {
const getWriter = async (hierarchy, store, indexDir, indexedDataKey, indexNode, createIfNotExists) => {
let readableStream = null;
if (isShardedIndex(indexNode)) {
await ensureShardNameIsInShardMap(store, indexKey, indexedDataKey);
await ensureShardNameIsInShardMap(store, indexDir, indexedDataKey);
if(!await store.exists(indexedDataKey)) {
await store.createFile(indexedDataKey, "");
if (await store.exists(getParentKey(indexedDataKey))) {
await store.createFile(indexedDataKey, "");
} else {
return SHARD_DELETED;
}
}
}
@ -37,7 +41,11 @@ const getWriter = async (hierarchy, store, indexKey, indexedDataKey, indexNode,
throw e;
} else {
if (createIfNotExists) {
await store.createFile(indexedDataKey, '');
if(await store.exists(getParentKey(indexedDataKey))) {
await store.createFile(indexedDataKey, '');
} else {
return SHARD_DELETED;
}
} else {
return SHARD_DELETED;
}
@ -65,6 +73,12 @@ const swapTempFileIn = async (store, indexedDataKey, isRetry = false) => {
await store.deleteFile(indexedDataKey);
} catch (e) {
// ignore failure, incase it has not been created yet
// if parent folder does not exist, assume that this index
// should not be there
if(!await store.exists(getParentKey(indexedDataKey))) {
return;
}
}
try {
await store.renameFile(tempFile, indexedDataKey);

8
packages/core/src/indexing/initialiseIndex.js
View file

@ -3,19 +3,19 @@ import { joinKey } from '../common';
import { getShardMapKey, getUnshardedIndexDataKey, createIndexFile } from './sharding';
export const initialiseIndex = async (datastore, dir, index) => {
const indexKey = joinKey(dir, index.name);
const indexDir = joinKey(dir, index.name);
await datastore.createFolder(indexKey);
await datastore.createFolder(indexDir);
if (isShardedIndex(index)) {
await datastore.createFile(
getShardMapKey(indexKey),
getShardMapKey(indexDir),
'[]',
);
} else {
await createIndexFile(
datastore,
getUnshardedIndexDataKey(indexKey),
getUnshardedIndexDataKey(indexDir),
index,
);
}

33
packages/core/src/indexing/read.js
View file

@ -1,12 +1,4 @@
import lunr from 'lunr';
import {
getHashCode,
joinKey
} from '../common';
import {
getActualKeyOfParent,
isGlobalIndex,
} from '../templateApi/hierarchy';
import {promiseReadableStream} from "./promiseReadableStream";
import { createIndexFile } from './sharding';
import { generateSchema } from './indexSchemaCreator';
@ -50,31 +42,6 @@ export const searchIndex = async (hierarchy, datastore, index, indexedDataKey, s
return await doRead(hierarchy, datastore, index, indexedDataKey);
};
export const getIndexedDataKey_fromIndexKey = (indexKey) =>
`${indexKey}${indexKey.endsWith('.csv') ? '' : '.csv'}`;
export const uniqueIndexName = index => `idx_${
getHashCode(`${index.filter}${index.map}`)
}.csv`;
export const getIndexedDataKey = (decendantKey, indexNode) => {
if (isGlobalIndex(indexNode)) { return `${indexNode.nodeKey()}.csv`; }
const indexedDataParentKey = getActualKeyOfParent(
indexNode.parent().nodeKey(),
decendantKey,
);
const indexName = indexNode.name
? `${indexNode.name}.csv`
: uniqueIndexName(indexNode);
return joinKey(
indexedDataParentKey,
indexName,
);
};
export const iterateIndex = (onGetItem, getFinalResult) => async (hierarchy, datastore, index, indexedDataKey) => {
try {
const readableStream = promiseReadableStream(

32
packages/core/src/indexing/relevant.js
View file

@ -9,21 +9,23 @@ import {
} from '../common';
import {
getFlattenedHierarchy, getNode, getRecordNodeId,
getExactNodeForPath, recordNodeIdIsAllowed,
getExactNodeForKey, recordNodeIdIsAllowed,
isRecord, isGlobalIndex,
} from '../templateApi/hierarchy';
import { indexTypes } from '../templateApi/indexes';
import { getIndexDir } from "../indexApi/getIndexDir";
import { getRecordInfo} from "../recordApi/recordInfo";
export const getRelevantAncestorIndexes = (appHierarchy, record) => {
export const getRelevantAncestorIndexes = (hierarchy, record) => {
const key = record.key;
const keyParts = splitKey(key);
const nodeId = getRecordNodeId(key);
const flatHierarchy = orderBy(getFlattenedHierarchy(appHierarchy),
const flatHierarchy = orderBy(getFlattenedHierarchy(hierarchy),
[node => node.pathRegx().length],
['desc']);
const makeindexNodeAndKey_ForAncestorIndex = (indexNode, indexKey) => makeIndexNodeAndKey(indexNode, joinKey(indexKey, indexNode.name));
const makeindexNodeAndDir_ForAncestorIndex = (indexNode, parentRecordDir) => makeIndexNodeAndDir(indexNode, joinKey(parentRecordDir, indexNode.name));
const traverseAncestorIndexesInPath = () => reduce((acc, part) => {
const currentIndexKey = joinKey(acc.lastIndexKey, part);
@ -42,8 +44,10 @@ export const getRelevantAncestorIndexes = (appHierarchy, record) => {
|| includes(nodeId)(i.allowedRecordNodeIds))),
]);
const currentRecordDir = getRecordInfo(hierarchy, currentIndexKey).dir;
each(v => acc.nodesAndKeys.push(
makeindexNodeAndKey_ForAncestorIndex(v, currentIndexKey),
makeindexNodeAndDir_ForAncestorIndex(v, currentRecordDir),
))(indexes);
return acc;
@ -51,31 +55,35 @@ export const getRelevantAncestorIndexes = (appHierarchy, record) => {
const rootIndexes = $(flatHierarchy, [
filter(n => isGlobalIndex(n) && recordNodeIdIsAllowed(n)(nodeId)),
map(i => makeIndexNodeAndKey(i, i.nodeKey())),
map(i => makeIndexNodeAndDir(
i,
getIndexDir(hierarchy, i.nodeKey()))),
]);
return union(traverseAncestorIndexesInPath())(rootIndexes);
};
export const getRelevantReverseReferenceIndexes = (appHierarchy, record) => $(record.key, [
getExactNodeForPath(appHierarchy),
export const getRelevantReverseReferenceIndexes = (hierarchy, record) => $(record.key, [
getExactNodeForKey(hierarchy),
n => n.fields,
filter(f => f.type === 'reference'
&& isSomething(record[f.name])
&& isNonEmptyString(record[f.name].key)),
map(f => $(f.typeOptions.reverseIndexNodeKeys, [
map(n => ({
recordNode: getNode(appHierarchy, n),
recordNode: getNode(hierarchy, n),
field: f,
})),
])),
flatten,
map(n => makeIndexNodeAndKey(
map(n => makeIndexNodeAndDir(
n.recordNode,
joinKey(record[n.field.name].key, n.recordNode.name),
joinKey(
getRecordInfo(hierarchy, record[n.field.name].key).dir,
n.recordNode.name),
)),
]);
const makeIndexNodeAndKey = (indexNode, indexKey) => ({ indexNode, indexKey });
const makeIndexNodeAndDir = (indexNode, indexDir) => ({ indexNode, indexDir });
export default getRelevantAncestorIndexes;

48
packages/core/src/indexing/sharding.js
View file

@ -5,13 +5,14 @@ import {
import {
getActualKeyOfParent, isGlobalIndex,
getParentKey, isShardedIndex,
getExactNodeForPath,
getExactNodeForKey,
} from '../templateApi/hierarchy';
import {
joinKey, isNonEmptyString, splitKey, $,
} from '../common';
export const getIndexedDataKey = (indexNode, indexKey, record) => {
export const getIndexedDataKey = (indexNode, indexDir, record) => {
const getShardName = (indexNode, record) => {
const shardNameFunc = compileCode(indexNode.getShardName);
try {
@ -27,18 +28,16 @@ export const getIndexedDataKey = (indexNode, indexKey, record) => {
? `${getShardName(indexNode, record)}.csv`
: 'index.csv';
return joinKey(indexKey, shardName);
return joinKey(indexDir, shardName);
};
export const getShardKeysInRange = async (app, indexKey, startRecord = null, endRecord = null) => {
const indexNode = getExactNodeForPath(app.hierarchy)(indexKey);
export const getShardKeysInRange = async (app, indexNode, indexDir, startRecord = null, endRecord = null) => {
const startShardName = !startRecord
? null
: shardNameFromKey(
getIndexedDataKey(
indexNode,
indexKey,
indexDir,
startRecord,
),
);
@ -48,29 +47,29 @@ export const getShardKeysInRange = async (app, indexKey, startRecord = null, end
: shardNameFromKey(
getIndexedDataKey(
indexNode,
indexKey,
indexDir,
endRecord,
),
);
return $(await getShardMap(app.datastore, indexKey), [
return $(await getShardMap(app.datastore, indexDir), [
filter(k => (startRecord === null || k >= startShardName)
&& (endRecord === null || k <= endShardName)),
map(k => joinKey(indexKey, `${k}.csv`)),
map(k => joinKey(indexDir, `${k}.csv`)),
]);
};
export const ensureShardNameIsInShardMap = async (store, indexKey, indexedDataKey) => {
const map = await getShardMap(store, indexKey);
export const ensureShardNameIsInShardMap = async (store, indexDir, indexedDataKey) => {
const map = await getShardMap(store, indexDir);
const shardName = shardNameFromKey(indexedDataKey);
if (!includes(shardName)(map)) {
map.push(shardName);
await writeShardMap(store, indexKey, map);
await writeShardMap(store, indexDir, map);
}
};
export const getShardMap = async (datastore, indexKey) => {
const shardMapKey = getShardMapKey(indexKey);
export const getShardMap = async (datastore, indexDir) => {
const shardMapKey = getShardMapKey(indexDir);
try {
return await datastore.loadJson(shardMapKey);
} catch (_) {
@ -79,27 +78,26 @@ export const getShardMap = async (datastore, indexKey) => {
}
};
export const writeShardMap = async (datastore, indexKey, shardMap) => await datastore.updateJson(
getShardMapKey(indexKey),
export const writeShardMap = async (datastore, indexDir, shardMap) => await datastore.updateJson(
getShardMapKey(indexDir),
shardMap,
);
export const getAllShardKeys = async (app, indexKey) => await getShardKeysInRange(app, indexKey);
export const getAllShardKeys = async (app, indexNode, indexDir) =>
await getShardKeysInRange(app, indexNode, indexDir);
export const getShardMapKey = indexKey => joinKey(indexKey, 'shardMap.json');
export const getShardMapKey = indexDir => joinKey(indexDir, 'shardMap.json');
export const getUnshardedIndexDataKey = indexKey => joinKey(indexKey, 'index.csv');
export const getIndexFolderKey = indexKey => indexKey;
export const getUnshardedIndexDataKey = indexDir => joinKey(indexDir, 'index.csv');
export const createIndexFile = async (datastore, indexedDataKey, index) => {
if (isShardedIndex(index)) {
const indexKey = getParentKey(indexedDataKey);
const shardMap = await getShardMap(datastore, indexKey);
const indexDir = getParentKey(indexedDataKey);
const shardMap = await getShardMap(datastore, indexDir);
shardMap.push(
shardNameFromKey(indexedDataKey),
);
await writeShardMap(datastore, indexKey, shardMap);
await writeShardMap(datastore, indexDir, shardMap);
}
await datastore.createFile(indexedDataKey, '');
};

0
packages/core/src/indexing/sweeper.js
View file

25
packages/core/src/recordApi/delete.js
View file

@ -1,11 +1,15 @@
import {
safeKey, apiWrapper,
events,
events, joinKey,
} from '../common';
import { _loadFromInfo } from './load';
import { _load } from './load';
import { _deleteCollection } from '../collectionApi/delete';
import {
getExactNodeForKey
} from '../templateApi/hierarchy';
import { transactionForDeleteRecord } from '../transactions/create';
import { permission } from '../authApi/permissions';
import { getRecordInfo } from "./recordInfo";
import { getRecordInfo } from './recordInfo';
export const deleteRecord = (app, disableCleanup = false) => async key => {
key = safeKey(key);
@ -20,17 +24,22 @@ export const deleteRecord = (app, disableCleanup = false) => async key => {
// called deleteRecord because delete is a keyword
export const _deleteRecord = async (app, key, disableCleanup) => {
const recordInfo = getRecordInfo(app, key);
const recordInfo = getRecordInfo(app.hierarchy, key);
key = recordInfo.key;
const node = getExactNodeForKey(app.hierarchy)(key);
const record = await _loadFromInfo(app, recordInfo);
const record = await _load(app, key);
await transactionForDeleteRecord(app, record);
for (const collectionRecord of node.children) {
const collectionKey = joinKey(
key, collectionRecord.collectionName,
);
await _deleteCollection(app, collectionKey, true);
}
await app.datastore.deleteFolder(recordInfo.dir);
if (!disableCleanup) { await app.cleanupTransactions(); }
};

4
packages/core/src/recordApi/downloadFile.js
View file

@ -2,6 +2,7 @@ import { apiWrapper, events, isNothing } from '../common';
import { permission } from '../authApi/permissions';
import { safeGetFullFilePath } from './uploadFile';
import { BadRequestError } from '../common/errors';
import { getRecordInfo } from "./recordInfo";
export const downloadFile = app => async (recordKey, relativePath) => apiWrapper(
app,
@ -16,9 +17,10 @@ const _downloadFile = async (app, recordKey, relativePath) => {
if (isNothing(recordKey)) { throw new BadRequestError('Record Key not supplied'); }
if (isNothing(relativePath)) { throw new BadRequestError('file path not supplied'); }
const {dir} = getRecordInfo(app.hierarchy, recordKey);
return await app.datastore.readableFileStream(
safeGetFullFilePath(
recordKey, relativePath,
dir, relativePath,
),
);
};

4
packages/core/src/recordApi/getContext.js
View file

@ -1,6 +1,6 @@
import { map, isString, has, some } from 'lodash/fp';
import {
getExactNodeForPath,
getExactNodeForKey,
findField, getNode, isGlobalIndex,
} from '../templateApi/hierarchy';
import { listItems } from '../indexApi/listItems';
@ -23,7 +23,7 @@ export const getContext = app => recordKey => {
export const _getContext = (app, recordKey) => {
recordKey = safeKey(recordKey);
const recordNode = getExactNodeForPath(app.hierarchy)(recordKey);
const recordNode = getExactNodeForKey(app.hierarchy)(recordKey);
const cachedReferenceIndexes = {};

4
packages/core/src/recordApi/load.js
View file

@ -2,7 +2,7 @@ import {
keyBy, mapValues, filter,
map, includes, last,
} from 'lodash/fp';
import { getExactNodeForPath, getNode } from '../templateApi/hierarchy';
import { getExactNodeForKey, getNode } from '../templateApi/hierarchy';
import { safeParseField } from '../types';
import {
$, splitKey, safeKey, isNonEmptyString,
@ -72,7 +72,7 @@ export const _loadFromInfo = async (app, recordInfo, keyStack = []) => {
export const _load = async (app, key, keyStack = []) =>
_loadFromInfo(
app,
getRecordInfo(app, key),
getRecordInfo(app.hierarchy, key),
keyStack);

64
packages/core/src/recordApi/recordInfo.js
View file

@ -1,18 +1,19 @@
import {
getExactNodeForPath, getActualKeyOfParent, isRoot
getExactNodeForKey, getActualKeyOfParent,
isRoot, isSingleRecord, getNodeForCollectionPath
} from '../templateApi/hierarchy';
import {
map, reduce
reduce, find, filter, take
} from 'lodash/fp';
import {
$, getDirFomKey, getFileFromKey, joinKey, safeKey
$, getFileFromKey, joinKey, safeKey, keySep
} from '../common';
import {
folderStructureArray, allIdChars
} from "../indexing/allIds";
export const getRecordInfo = (app, key) => {
const recordNode = getExactNodeForPath(app.hierarchy)(key);
export const getRecordInfo = (hierarchy, key) => {
const recordNode = getExactNodeForKey(hierarchy)(key);
const pathInfo = getRecordDirectory(recordNode, key);
const dir = joinKey(pathInfo.base, ...pathInfo.subdirs);
@ -25,6 +26,13 @@ export const getRecordInfo = (app, key) => {
};
}
export const getCollectionDir = (hierarchy, collectionKey) => {
const recordNode = getNodeForCollectionPath(hierarchy)(collectionKey);
const dummyRecordKey = joinKey(collectionKey, "1-abcd");
const pathInfo = getRecordDirectory(recordNode, dummyRecordKey);
return pathInfo.base;
}
const recordJson = (dir) =>
joinKey(dir, "record.json")
@ -34,31 +42,53 @@ const files = (dir) =>
const getRecordDirectory = (recordNode, key) => {
const id = getFileFromKey(key);
const traverseParentKeys = (n, keys=[]) => {
if(isRoot(n)) return keys;
const k = getActualKeyOfParent(n, key);
const traverseParentKeys = (n, parents=[]) => {
if(isRoot(n)) return parents;
const k = getActualKeyOfParent(n.nodeKey(), key);
const thisNodeDir = {
node:n,
relativeDir: joinKey(
recordRelativeDirectory(n, getFileFromKey(k)))
};
return traverseParentKeys(
n.parent(),
[thisNodeDir, ...parents]);
}
const parentDirs = $(recordNode.parent(), [
traverseParentKeys,
reduce((key, item) => {
return joinKey(key, item.node.collectionName, item.relativeDir)
}, keySep)
]);
const subdirs = isSingleRecord(recordNode)
? []
: recordRelativeDirectory(recordNode, id);
const base = isSingleRecord(recordNode)
? joinKey(parentDirs, recordNode.name)
: joinKey(parentDirs, recordNode.collectionName);
return ({
base:getDirFomKey(key),
subdirs: [recordNode.nodeId.toString(), ...subfolders]
subdirs, base
});
}
const recordRelativeDirectory = (recordNode, id) => {
const folderStructure = folderStructureArray(recordNode);
const strippedId = id.substring(recordNode.nodeId.toString().length + 1);
const subfolders = $(folderStructure, [
reduce((result, currentCount) => {
result.folders.push(
folderForChar(id[result.level], currentCount)
);
return {level:result.level+1, folders:result.folders}
result.folders.push(
folderForChar(strippedId[result.level], currentCount)
);
return {level:result.level+1, folders:result.folders};
}, {level:0, folders:[]}),
map(f => f.folders),
f => f.folders,
filter(f => !!f)
]);
return [recordNode.nodeId.toString(), ...subfolders]
return [recordNode.nodeId.toString(), ...subfolders, id]
}
const folderForChar = (char, folderCount) =>

22
packages/core/src/recordApi/save.js
View file

@ -4,13 +4,13 @@ import {
} from 'lodash/fp';
import { initialiseChildCollections } from '../collectionApi/initialise';
import { validate } from './validate';
import { _loadFromInfo, getRecordFileName } from './load';
import { _loadFromInfo } from './load';
import {
apiWrapper, events, $, joinKey,
} from '../common';
import {
getFlattenedHierarchy, getExactNodeForPath,
isRecord, getNode, fieldReversesReferenceToNode,
getFlattenedHierarchy, isRecord, getNode,
fieldReversesReferenceToNode,
} from '../templateApi/hierarchy';
import {
transactionForCreateRecord,
@ -42,7 +42,7 @@ export const _save = async (app, record, context, skipValidation = false) => {
}
}
const recordInfo = getRecordInfo(app.record.key);
const recordInfo = getRecordInfo(app.hierarchy, record.key);
const {
recordNode, pathInfo,
recordJson, files,
@ -92,7 +92,9 @@ export const _save = async (app, record, context, skipValidation = false) => {
const initialiseAncestorIndexes = async (app, recordInfo) => {
for (const index of recordInfo.recordNode.indexes) {
const indexKey = recordInfo.child(index.name);
if (!await app.datastore.exists(indexKey)) { await initialiseIndex(app.datastore, recordInfo.dir, index); }
if (!await app.datastore.exists(indexKey)) {
await initialiseIndex(app.datastore, recordInfo.dir, index);
}
}
};
@ -125,7 +127,7 @@ const fieldsThatReferenceThisRecord = (app, recordNode) => $(app.hierarchy, [
const createRecordFolderPath = async (datastore, pathInfo) => {
const recursiveCreateFolder = async (subdirs, dirsThatNeedCreated=[]) => {
const recursiveCreateFolder = async (subdirs, dirsThatNeedCreated=undefined) => {
// iterate backwards through directory hierachy
// until we get to a folder that exists, then create the rest
@ -138,12 +140,16 @@ const createRecordFolderPath = async (datastore, pathInfo) => {
if(await datastore.exists(thisFolder)) {
let creationFolder = thisFolder;
for(let nextDir of dirsThatNeedCreated) {
for(let nextDir of (dirsThatNeedCreated || []) ) {
creationFolder = joinKey(creationFolder, nextDir);
await datastore.createFolder(creationFolder);
}
} else if(dirsThatNeedCreated.length > 0) {
} else if(!dirsThatNeedCreated || dirsThatNeedCreated.length > 0) {
dirsThatNeedCreated = !dirsThatNeedCreated
? []
:dirsThatNeedCreated;
await recursiveCreateFolder(
take(subdirs.length - 1)(subdirs),

36
packages/core/src/recordApi/uploadFile.js
View file

@ -3,15 +3,16 @@ import {
map, some,
} from 'lodash/fp';
import { generate } from 'shortid';
import { _load } from './load';
import { _loadFromInfo } from './load';
import {
apiWrapper, events, splitKey,
$, joinKey, isNothing, tryAwaitOrIgnore,
} from '../common';
import { getExactNodeForPath } from '../templateApi/hierarchy';
import { getExactNodeForKey } from '../templateApi/hierarchy';
import { permission } from '../authApi/permissions';
import { isLegalFilename } from '../types/file';
import { BadRequestError, ForbiddenError } from '../common/errors';
import { getRecordInfo } from "./recordInfo";
export const uploadFile = app => async (recordKey, readableStream, relativeFilePath) => apiWrapper(
app,
@ -26,10 +27,11 @@ const _uploadFile = async (app, recordKey, readableStream, relativeFilePath) =>
if (isNothing(relativeFilePath)) { throw new BadRequestError('file path not supplied'); }
if (!isLegalFilename(relativeFilePath)) { throw new BadRequestError('Illegal filename'); }
const record = await _load(app, recordKey);
const recordInfo = getRecordInfo(app.hierarchy, recordKey);
const record = await _loadFromInfo(app, recordInfo);
const fullFilePath = safeGetFullFilePath(
recordKey, relativeFilePath,
recordInfo.dir, relativeFilePath,
);
const tempFilePath = `${fullFilePath}_${generate()}.temp`;
@ -54,30 +56,10 @@ const _uploadFile = async (app, recordKey, readableStream, relativeFilePath) =>
.then(() => tryAwaitOrIgnore(app.datastore.deleteFile, fullFilePath))
.then(() => app.datastore.renameFile(tempFilePath, fullFilePath));
/*
readableStream.pipe(outputStream);
await new Promise(fulfill => outputStream.on('finish', fulfill));
const isExpectedFileSize = checkFileSizeAgainstFields(
app,
record, relativeFilePath,
await app.datastore.getFileSize(tempFilePath),
);
if (!isExpectedFileSize) {
throw new Error(
`Fields for ${relativeFilePath} do not have expected size`);
}
await tryAwaitOrIgnore(app.datastore.deleteFile, fullFilePath);
await app.datastore.renameFile(tempFilePath, fullFilePath);
*/
};
const checkFileSizeAgainstFields = (app, record, relativeFilePath, expectedSize) => {
const recordNode = getExactNodeForPath(app.hierarchy)(record.key);
const recordNode = getExactNodeForKey(app.hierarchy)(record.key);
const incorrectFileFields = $(recordNode.fields, [
filter(f => f.type === 'file'
@ -107,7 +89,7 @@ const checkFileSizeAgainstFields = (app, record, relativeFilePath, expectedSize)
return true;
};
export const safeGetFullFilePath = (recordKey, relativeFilePath) => {
export const safeGetFullFilePath = (recordDir, relativeFilePath) => {
const naughtyUser = () => { throw new ForbiddenError('naughty naughty'); };
if (relativeFilePath.startsWith('..')) naughtyUser();
@ -116,7 +98,7 @@ export const safeGetFullFilePath = (recordKey, relativeFilePath) => {
if (includes('..')(pathParts)) naughtyUser();
const recordKeyParts = splitKey(recordKey);
const recordKeyParts = splitKey(recordDir);
const fullPathParts = [
...recordKeyParts,

4
packages/core/src/recordApi/validate.js
View file

@ -4,7 +4,7 @@ import {
} from 'lodash/fp';
import { compileExpression } from '@nx-js/compiler-util';
import _ from 'lodash';
import { getExactNodeForPath } from '../templateApi/hierarchy';
import { getExactNodeForKey } from '../templateApi/hierarchy';
import { validateFieldParse, validateTypeConstraints } from '../types';
import { $, isNothing, isNonEmptyString } from '../common';
import { _getContext } from './getContext';
@ -63,7 +63,7 @@ export const validate = app => async (record, context) => {
? _getContext(app, record.key)
: context;
const recordNode = getExactNodeForPath(app.hierarchy)(record.key);
const recordNode = getExactNodeForKey(app.hierarchy)(record.key);
const fieldParseFails = validateAllFieldParse(record, recordNode);
// non parsing would cause further issues - exit here

2
packages/core/src/templateApi/createNodes.js
View file

@ -171,7 +171,7 @@ const _getNewRecordTemplate = (parent, name, createDefaultIndex, isSingle) => {
validationRules: [],
nodeId: getNodeId(parent),
indexes: [],
allidsShardFactor: isRecord(parent) ? 1 : 64,
estimatedRecordCount: isRecord(parent) ? 500 : 1000000,
collectionName: '',
isSingle,
});

19
packages/core/src/templateApi/hierarchy.js
View file

@ -49,7 +49,7 @@ export const getNodesInPath = appHierarchy => key => $(appHierarchy, [
filter(n => new RegExp(`${n.pathRegx()}`).test(key)),
]);
export const getExactNodeForPath = appHierarchy => key => $(appHierarchy, [
export const getExactNodeForKey = appHierarchy => key => $(appHierarchy, [
getFlattenedHierarchy,
find(n => new RegExp(`${n.pathRegx()}$`).test(key)),
]);
@ -87,7 +87,7 @@ export const getCollectionNode = (appHierarchy, nodeKey) => $(appHierarchy, [
]);
export const getNodeByKeyOrNodeKey = (appHierarchy, keyOrNodeKey) => {
const nodeByKey = getExactNodeForPath(appHierarchy)(keyOrNodeKey);
const nodeByKey = getExactNodeForKey(appHierarchy)(keyOrNodeKey);
return isNothing(nodeByKey)
? getNode(appHierarchy, keyOrNodeKey)
: nodeByKey;
@ -100,13 +100,14 @@ export const getCollectionNodeByKeyOrNodeKey = (appHierarchy, keyOrNodeKey) => {
: nodeByKey;
};
export const isNode = (appHierarchy, key) => isSomething(getExactNodeForPath(appHierarchy)(key));
export const isNode = (appHierarchy, key) => isSomething(getExactNodeForKey(appHierarchy)(key));
export const getActualKeyOfParent = (parentNodeKey, actualChildKey) => $(actualChildKey, [
splitKey,
take(splitKey(parentNodeKey).length),
ks => joinKey(...ks),
]);
export const getActualKeyOfParent = (parentNodeKey, actualChildKey) =>
$(actualChildKey, [
splitKey,
take(splitKey(parentNodeKey).length),
ks => joinKey(...ks),
]);
export const getParentKey = (key) => {
return $(key, [
@ -199,7 +200,7 @@ export const fieldReversesReferenceToIndex = indexNode => field => field.type ==
export default {
getLastPartInKey,
getNodesInPath,
getExactNodeForPath,
getExactNodeForKey,
hasMatchingAncestor,
getNode,
getNodeByKeyOrNodeKey,

54
packages/core/src/transactions/execute.js
View file

@ -24,8 +24,10 @@ import { applyToShard } from '../indexing/apply';
import {
getActualKeyOfParent,
isGlobalIndex, fieldReversesReferenceToIndex, isReferenceIndex,
getExactNodeForPath,
getExactNodeForKey,
} from '../templateApi/hierarchy';
import { getRecordInfo } from "../recordApi/recordInfo";
import { getIndexDir } from '../indexApi/getIndexDir';
export const executeTransactions = app => async (transactions) => {
const recordsByShard = mappedRecordsByIndexShard(app.hierarchy, transactions);
@ -33,7 +35,7 @@ export const executeTransactions = app => async (transactions) => {
for (const shard of keys(recordsByShard)) {
await applyToShard(
app.hierarchy, app.datastore,
recordsByShard[shard].indexKey,
recordsByShard[shard].indexDir,
recordsByShard[shard].indexNode,
shard,
recordsByShard[shard].writes,
@ -77,8 +79,8 @@ const mappedRecordsByIndexShard = (hierarchy, transactions) => {
transByShard[t.indexShardKey] = {
writes: [],
removes: [],
indexKey: t.indexKey,
indexNodeKey: t.indexNodeKey,
indexDir: t.indexDir,
indexNodeKey: t.indexNode.nodeKey(),
indexNode: t.indexNode,
};
}
@ -109,10 +111,10 @@ const getUpdateTransactionsByShard = (hierarchy, transactions) => {
return ({
mappedRecord,
indexNode: indexNodeAndPath.indexNode,
indexKey: indexNodeAndPath.indexKey,
indexDir: indexNodeAndPath.indexDir,
indexShardKey: getIndexedDataKey(
indexNodeAndPath.indexNode,
indexNodeAndPath.indexKey,
indexNodeAndPath.indexDir,
mappedRecord.result,
),
});
@ -219,54 +221,56 @@ const getBuildIndexTransactionsByShard = (hierarchy, transactions) => {
if (!isNonEmptyArray(buildTransactions)) return [];
const indexNode = transactions.indexNode;
const getIndexKeys = (t) => {
const getIndexDirs = (t) => {
if (isGlobalIndex(indexNode)) {
return [indexNode.nodeKey()];
}
if (isReferenceIndex(indexNode)) {
const recordNode = getExactNodeForPath(hierarchy)(t.record.key);
const recordNode = getExactNodeForKey(hierarchy)(t.record.key);
const refFields = $(recordNode.fields, [
filter(fieldReversesReferenceToIndex(indexNode)),
]);
const indexKeys = [];
const indexDirs = [];
for (const refField of refFields) {
const refValue = t.record[refField.name];
if (isSomething(refValue)
&& isNonEmptyString(refValue.key)) {
const indexKey = joinKey(
refValue.key,
const indexDir = joinKey(
getRecordInfo(hierarchy, refValue.key).dir,
indexNode.name,
);
if (!includes(indexKey)(indexKeys)) { indexKeys.push(indexKey); }
if (!includes(indexDir)(indexDirs)) { indexDirs.push(indexDir); }
}
}
return indexKeys;
return indexDirs;
}
return [joinKey(
const indexKey = joinKey(
getActualKeyOfParent(
indexNode.parent().nodeKey(),
t.record.key,
),
indexNode.name,
)];
);
return [getIndexDir(hierarchy, indexKey)];
};
return $(buildTransactions, [
map((t) => {
const mappedRecord = evaluate(t.record)(indexNode);
if (!mappedRecord.passedFilter) return null;
const indexKeys = getIndexKeys(t);
return $(indexKeys, [
map(indexKey => ({
const indexDirs = getIndexDirs(t);
return $(indexDirs, [
map(indexDir => ({
mappedRecord,
indexNode,
indexKey,
indexDir,
indexShardKey: getIndexedDataKey(
indexNode,
indexKey,
indexDir,
mappedRecord.result,
),
})),
@ -286,10 +290,10 @@ const get_Create_Delete_TransactionsByShard = pred => (hierarchy, transactions)
return ({
mappedRecord,
indexNode: n.indexNode,
indexKey: n.indexKey,
indexDir: n.indexDir,
indexShardKey: getIndexedDataKey(
n.indexNode,
n.indexKey,
n.indexDir,
mappedRecord.result,
),
});
@ -327,17 +331,17 @@ const diffReverseRefForUpdate = (appHierarchy, oldRecord, newRecord) => {
);
const unReferenced = differenceBy(
i => i.indexKey,
i => i.indexDir,
oldIndexes, newIndexes,
);
const newlyReferenced = differenceBy(
i => i.indexKey,
i => i.indexDir,
newIndexes, oldIndexes,
);
const notChanged = intersectionBy(
i => i.indexKey,
i => i.indexDir,
newIndexes, oldIndexes,
);

3
packages/core/test/collectionApi.spec.js
View file

@ -1,5 +1,4 @@
import {setupApphierarchy, basicAppHierarchyCreator_WithFields,
basicAppHierarchyCreator_WithFields_AndIndexes} from "./specHelpers";
import {setupApphierarchy, basicAppHierarchyCreator_WithFields} from "./specHelpers";
import {includes, union} from "lodash";
import {joinKey} from "../src/common";

6
packages/core/test/indexApi.buildIndex.spec.js
View file

@ -4,7 +4,7 @@ import { joinKey } from "../src/common";
import {some} from "lodash";
import {_deleteIndex} from "../src/indexApi/delete";
import {permission} from "../src/authApi/permissions";
import { getExactNodeForPath } from "../src/templateApi/hierarchy";
import { getExactNodeForKey } from "../src/templateApi/hierarchy";
describe("buildIndex > Global index", () => {
@ -213,7 +213,7 @@ describe("buildIndex > nested collection", () => {
const indexKey = joinKey(customer.key, "invoice_index");
await _deleteIndex(app, indexKey, false);
const indexNode = getExactNodeForPath(appHierarchy.root)(indexKey);
const indexNode = getExactNodeForKey(appHierarchy.root)(indexKey);
await indexApi.buildIndex(indexNode.nodeKey());
const indexItems = await indexApi.listItems(indexKey);
@ -269,7 +269,7 @@ describe("buildIndex > nested collection", () => {
const indexKey = joinKey(customer.key, "invoice_index");
await _deleteIndex(app, indexKey, false);
const indexNode = getExactNodeForPath(appHierarchy.root)(indexKey);
const indexNode = getExactNodeForKey(appHierarchy.root)(indexKey);
await indexApi.buildIndex(
indexNode.nodeKey());
const indexItems = await indexApi.listItems(indexKey);

18
packages/core/test/indexing.concurrency.spec.js
View file

@ -5,7 +5,7 @@ import {getLockFileContent} from "../src/common/lock";
import {some, isArray} from "lodash";
import {cleanup} from "../src/transactions/cleanup";
import {LOCK_FILE_KEY} from "../src/transactions/transactionsCommon";
import { getRecordInfo } from "../src/recordApi/recordInfo";
describe("cleanup transactions", () => {
@ -96,7 +96,7 @@ describe("cleanup transactions", () => {
it("should not reindex when transactionId does not match that of the record", async () => {
const {recordApi, app,
const {recordApi, app,
indexApi} = await setupApphierarchy(basicAppHierarchyCreator_WithFields_AndIndexes, true);
const record = recordApi.getNew("/customers", "customer");
record.surname = "Ledog";
@ -109,8 +109,10 @@ describe("cleanup transactions", () => {
await recordApi.save(savedRecord);
savedRecord.transactionId = "something else";
const recordInfo = getRecordInfo(app.hierarchy, savedRecord.key);
await recordApi._storeHandle.updateJson(
joinKey(savedRecord.key, "record.json"),
recordInfo.child("record.json"),
savedRecord);
await cleanup(app);
@ -123,7 +125,7 @@ describe("cleanup transactions", () => {
it("should not reindex when transactionId does not match that of the record, and has multiple transactions", async () => {
const {recordApi, app,
const {recordApi, app,
indexApi} = await setupApphierarchy(basicAppHierarchyCreator_WithFields_AndIndexes, true);
const record = recordApi.getNew("/customers", "customer");
record.surname = "Ledog";
@ -139,8 +141,10 @@ describe("cleanup transactions", () => {
await recordApi.save(savedRecord);
savedRecord.transactionId = "something else";
const recordInfo = getRecordInfo(app.hierarchy, savedRecord.key);
await recordApi._storeHandle.updateJson(
joinKey(savedRecord.key, "record.json"),
recordInfo.child("record.json"),
savedRecord);
await cleanup(app);
@ -228,7 +232,7 @@ describe("cleanup transactions", () => {
indexApi} = await setupApphierarchy(basicAppHierarchyCreator_WithFields_AndIndexes, true);
const record = recordApi.getNew("/customers", "customer");
record.surname = "Ledog";
const savedRecord = await recordApi.save(record);
await recordApi.save(record);
const currentTime = await app.getEpochTime();
await recordApi._storeHandle.createFile(
LOCK_FILE_KEY,
@ -252,7 +256,7 @@ describe("cleanup transactions", () => {
indexApi} = await setupApphierarchy(basicAppHierarchyCreator_WithFields_AndIndexes, true);
const record = recordApi.getNew("/customers", "customer");
record.surname = "Ledog";
const savedRecord = await recordApi.save(record);
await recordApi.save(record);
await recordApi._storeHandle.createFile(
LOCK_FILE_KEY,
getLockFileContent(30000, (new Date(1990,1,1,0,0,0,0).getTime()))

41
packages/core/test/indexing.getRelevantIndexes.spec.js
View file

@ -1,11 +1,14 @@
import {getMemoryTemplateApi,
basicAppHierarchyCreator_WithFields,
setupApphierarchy,
basicAppHierarchyCreator_WithFields_AndIndexes} from "./specHelpers";
import {getRelevantReverseReferenceIndexes,
getRelevantAncestorIndexes} from "../src/indexing/relevant";
import {
setupApphierarchy,
basicAppHierarchyCreator_WithFields_AndIndexes
} from "./specHelpers";
import {
getRelevantReverseReferenceIndexes,
getRelevantAncestorIndexes
} from "../src/indexing/relevant";
import {some} from "lodash";
import {joinKey} from "../src/common";
import { getRecordInfo } from "../src/recordApi/recordInfo";
describe("getRelevantIndexes", () => {
@ -45,7 +48,7 @@ describe("getRelevantIndexes", () => {
expect(indexes.length).toBe(4);
const indexExists = key =>
some(indexes, c => c.indexKey === key);
some(indexes, c => c.indexDir === key);
expect(indexExists("/customer_index")).toBeTruthy();
expect(indexExists("/deceased")).toBeTruthy();
@ -64,7 +67,7 @@ describe("getRelevantIndexes", () => {
appHierarchy.root, invoice);
const indexExists = key =>
some(indexes, c => c.indexKey === key);
some(indexes, c => c.indexDir === key);
expect(indexExists("/customersBySurname")).toBeFalsy();
});
@ -82,7 +85,7 @@ describe("getRelevantIndexes", () => {
expect(indexes.length).toBe(4);
const indexExists = key =>
some(indexes, c => c.indexKey === key);
some(indexes, c => c.indexDir === key);
expect(indexExists("/customersBySurname")).toBeTruthy();
});
@ -96,14 +99,14 @@ describe("getRelevantIndexes", () => {
const indexes = getRelevantAncestorIndexes(
appHierarchy.root, invoice);
const {dir} = getRecordInfo(appHierarchy.root, `/customers/${nodeid}-1234`);
expect(indexes.length).toBe(4);
expect(some(indexes, i => i.indexKey === `/customer_invoices`)).toBeTruthy();
expect(some(indexes, i => i.indexKey === `/customers/${nodeid}-1234/invoice_index`)).toBeTruthy();
expect(some(indexes, i => i.indexDir === `/customer_invoices`)).toBeTruthy();
expect(some(indexes, i => i.indexDir === `${dir}/invoice_index`)).toBeTruthy();
});
it("should get reverseReferenceIndex accross hierarchy branches", async () => {
const {appHierarchy,
const {appHierarchy,
recordApi} = await setupApphierarchy(basicAppHierarchyCreator_WithFields_AndIndexes);
const partner = recordApi.getNew("/partners", "partner");
@ -118,8 +121,9 @@ describe("getRelevantIndexes", () => {
const indexes = getRelevantReverseReferenceIndexes(
appHierarchy.root, customer);
expect(indexes.length).toBe(1);
expect(indexes[0].indexKey)
.toBe(joinKey(partner.key, appHierarchy.partnerCustomersReverseIndex.name));
const partnerdir = getRecordInfo(appHierarchy.root, partner.key).dir;
expect(indexes[0].indexDir)
.toBe(joinKey(partnerdir, appHierarchy.partnerCustomersReverseIndex.name));
});
@ -136,8 +140,11 @@ describe("getRelevantIndexes", () => {
const indexes = getRelevantReverseReferenceIndexes(
appHierarchy.root, referredToCustomer);
const referredByCustomerDir = getRecordInfo(appHierarchy.root, referredByCustomer.key).dir;
expect(indexes.length).toBe(1);
expect(indexes[0].indexKey)
.toBe(joinKey(referredByCustomer.key, appHierarchy.referredToCustomersReverseIndex.name));
expect(indexes[0].indexDir)
.toBe(joinKey(referredByCustomerDir, appHierarchy.referredToCustomersReverseIndex.name));
});
});

2
packages/core/test/indexing.schema.spec.js
View file

@ -1,5 +1,5 @@
import {generateSchema} from "../src/indexing/indexSchemaCreator";
import {setupApphierarchy, findCollectionDefaultIndex} from "./specHelpers";
import {setupApphierarchy} from "./specHelpers";
import {find} from "lodash";
import {indexTypes} from "../src/templateApi/indexes";

11
packages/core/test/initialiseData.spec.js
View file

@ -21,17 +21,6 @@ describe("initialiseData", () => {
expect(await datastore.exists(`/customers`)).toBeTruthy();
});
it("should create allids folders", async () => {
const {appDef, datastore, h} = getApplicationDefinition();
await initialiseData(datastore, appDef);
const allIdsTypeFolder = "/customers/allids/" + h.customerRecord.nodeId;
const allIdsFolder = "/customers/allids";
expect(await datastore.exists(allIdsTypeFolder)).toBeTruthy();
expect(await datastore.exists(allIdsFolder)).toBeTruthy();
});
it("should create transactions folder", async () => {
const {appDef, datastore} = getApplicationDefinition();
await initialiseData(datastore, appDef);

57
packages/core/test/memory.js
View file

@ -2,11 +2,16 @@ import {isUndefined, has} from "lodash";
import {take} from "lodash/fp";
import {Readable, Writable} from "readable-stream";
import { Buffer } from "safe-buffer";
import {splitKey, joinKey, $} from "../src/common";
import {splitKey, joinKey, $, keySep, getFileFromKey} from "../src/common";
import {getLastPartInKey} from "../src/templateApi/hierarchy";
const folderMarker = "OH-YES-ITSA-FOLDER-";
const isFolder = val => val.includes(folderMarker);
const isFolder = val => {
if(isUndefined(val)) {
throw new Error("Passed undefined value for folder");
}
return val.includes(folderMarker);
}
const getParentFolderKey = key =>
$(key, [
@ -16,7 +21,9 @@ const getParentFolderKey = key =>
]);
const getParentFolder = (data,key) => {
if(key === keySep) return null;
const parentKey = getParentFolderKey(key);
if(parentKey === keySep) return null;
if(data[parentKey] === undefined)
throw new Error("Parent folder for " + key + " does not exist (" + parentKey + ")");
return JSON.parse(data[parentKey]);
@ -39,12 +46,18 @@ export const createFile = data => async (path, content) => {
};
export const updateFile = data => async (path, content) => {
// putting this check in to force use of create
if(!await exists(data)(path)) throw new Error("cannot update " + path + " - does not exist");
if(!await exists(data)(path)) {
throw new Error("cannot update " + path + " - does not exist");
}
data[path] = content;
}
export const writableFileStream = data => async (path) => {
//if(!await exists(data)(path)) throw new Error("cannot write stream to " + path + " - does not exist");
if(!getParentFolder(data, path)) {
throw new Error("Parent folder for " + path + " does not exist");
}
const stream = Writable();
stream._write = (chunk, encoding, done) => {
data[path] = data[path] === undefined
@ -52,6 +65,9 @@ export const writableFileStream = data => async (path) => {
data[path] = [...data[path], ...chunk];
done();
};
addItemToParentFolder(data, path);
return stream;
};
@ -77,11 +93,19 @@ export const renameFile = data => async (oldKey, newKey) => {
if(await exists(data)(newKey)) throw new Error("cannot rename path: " + newKey + " ... already exists");
data[newKey] = data[oldKey];
delete data[oldKey];
const parent = getParentFolder(data, newKey);
const oldFileName = getFileFromKey(oldKey);
const newFileName = getFileFromKey(newKey);
parent.items = [...parent.items.filter(i => i !== oldFileName), newFileName];
data[getParentFolderKey(newKey)] = JSON.stringify(parent);
};
export const loadFile = data => async (path) => {
const result = data[path];
if(isUndefined(result)) throw new Error("Load failed - path " + path + " does not exist");
if(isUndefined(result)) {
throw new Error("Load failed - path " + path + " does not exist");
}
return result;
};
export const exists = data => async (path) => has(data, path);
@ -95,7 +119,8 @@ export const deleteFile = data => async (path) => {
delete data[path];
}
export const createFolder = data => async (path) => {
if(await exists(data)(path)) throw new Error("Cannot create folder, path " + path + " already exists");
if(await exists(data)(path))
throw new Error("Cannot create folder, path " + path + " already exists");
addItemToParentFolder(data, path);
data[path] = JSON.stringify({folderMarker, items:[]});
}
@ -103,14 +128,30 @@ export const deleteFolder = data => async (path) => {
if(!await exists(data)(path)) throw new Error("Cannot delete folder, path " + path + " does not exist");
if(!isFolder(data[path]))
throw new Error("DeleteFolder: Path " + path + " is not a folder");
for(let item of JSON.parse(data[path]).items) {
const fullItemPath = `${path}/${item}`;
if(isFolder(data[fullItemPath])) {
await deleteFolder(data)(fullItemPath);
} else {
await deleteFile(data)(fullItemPath);
}
}
const parent = getParentFolder(data, path);
if(parent) {
parent.items = parent.items.filter(f => f !== getLastPartInKey(path));
data[getParentFolderKey(path)] = JSON.stringify(parent);
}
delete data[path];
}
export const getFolderContents = data => async (folderPath) => {
if(!isFolder(data[folderPath]))
throw new Error("Not a folder: " + folderPath);
export const getFolderContents = data => async (folderPath) => {
if(!await exists(data)(folderPath))
throw new Error("Folder does not exist: " + folderPath);
if(!isFolder(data[folderPath]))
throw new Error("Not a folder: " + folderPath);
return JSON.parse(data[folderPath]).items;
};

3
packages/core/test/recordApi.files.spec.js
View file

@ -1,8 +1,5 @@
import {setupApphierarchy,
basicAppHierarchyCreator_WithFields} from "./specHelpers";
import {keys, filter} from "lodash/fp";
import {$} from "../src/common";
import {permission} from "../src/authApi/permissions";
import {Readable} from "readable-stream";

124
packages/core/test/recordApi.getRecordInfo.spec.js Normal file
View file

@ -0,0 +1,124 @@
import { folderStructureArray } from "../src/indexing/allIds";
import { getRecordInfo } from "../src/recordApi/recordInfo";
import {setupApphierarchy} from "./specHelpers";
describe("getRecordInfo", () => {
it("dir should not be sharded when record count = 1000", async () => {
const {root} = (await setup({parentCount: 1000})).appHierarchy;
const {dir} = getRecordInfo(root, "/parents/1-abcd");
expect(dir).toBe("/parents/1/1-abcd");
});
it("dir should be sharded when record count = 1001", async () => {
const {root} = (await setup({parentCount: 1001})).appHierarchy;
const {dir} = getRecordInfo(root, "/parents/1-abcd");
expect(dir).toBe("/parents/1/0123456789abcdefghijklmnopqrstuv/1-abcd");
});
it("dir should be sharded to one char per folder when record count = 63,000 (64*1000)", async () => {
const {root} = (await setup({parentCount: 64000})).appHierarchy;
const {dir} = getRecordInfo(root, "/parents/1-abcd");
expect(dir).toBe("/parents/1/a/1-abcd");
});
it("dir should be sharded to one char per folder, on 2 levels when record count = 4096000 (64*64*1000)", async () => {
const {root} = (await setup({parentCount: 4096000})).appHierarchy;
const {dir} = getRecordInfo(root, "/parents/1-abcd");
expect(dir).toBe("/parents/1/a/b/1-abcd");
});
it("child dir should not be sharded when record count = 1000", async () => {
const {root, child} = (await setup({parentCount: 4096000, childCount: 1000})).appHierarchy;
const {dir} = getRecordInfo(root, `/parents/1-abcd/children/${child.nodeId}-defg`);
expect(dir).toBe(`/parents/1/a/b/1-abcd/children/${child.nodeId}/${child.nodeId}-defg`);
});
it("grandchild dir should not be sharded when record count = 1000", async () => {
const {root, child, grandchild} = (await setup({parentCount: 4096000, childCount: 4096000})).appHierarchy;
const {dir} = getRecordInfo(root, `/parents/1-abcd/children/${child.nodeId}-defg/grandchildren/${grandchild.nodeId}-hijk`);
expect(dir).toBe(`/parents/1/a/b/1-abcd/children/${child.nodeId}/d/e/${child.nodeId}-defg/grandchildren/${grandchild.nodeId}/${grandchild.nodeId}-hijk`);
});
it("grandchild dir should be sharded when record count = 4096000", async () => {
const {root, child, grandchild} = (await setup({parentCount: 4096000, childCount: 4096000, grandChildCount: 4096000})).appHierarchy;
const {dir} = getRecordInfo(root, `/parents/1-abcd/children/${child.nodeId}-defg/grandchildren/${grandchild.nodeId}-hijk`);
expect(dir).toBe(`/parents/1/a/b/1-abcd/children/${child.nodeId}/d/e/${child.nodeId}-defg/grandchildren/${grandchild.nodeId}/h/i/${grandchild.nodeId}-hijk`);
});
it("child levels can be sharded, with parent not", async () => {
const {root, child, grandchild} = (await setup({parentCount: 1000, childCount: 4096000, grandChildCount: 4096000})).appHierarchy;
const {dir} = getRecordInfo(root, `/parents/1-abcd/children/${child.nodeId}-defg/grandchildren/${grandchild.nodeId}-hijk`);
expect(dir).toBe(`/parents/1/1-abcd/children/${child.nodeId}/d/e/${child.nodeId}-defg/grandchildren/${grandchild.nodeId}/h/i/${grandchild.nodeId}-hijk`);
});
});
describe("folderStructureArray", () => {
const recordNode = (count) => ({estimatedRecordCount: count});
it("should return [] when folder count < 1000", () => {
const result = folderStructureArray(recordNode(999));
expect(result).toEqual([]);
});
it("should return [4] when folder count between 3000 - 4000", () => {
const result = folderStructureArray(recordNode(3456));
expect(result).toEqual([4]);
})
it("should return [64, 2] when folder count between 64000 - 65000", () => {
const result = folderStructureArray(recordNode(64001));
expect(result).toEqual([64, 2]);
})
it("should return [64, 64] when folder = 4095999", () => {
const result = folderStructureArray(recordNode(4095999));
expect(result).toEqual([64, 64]);
});
it("should return [64, 64] when folder = 4096000", () => {
const result = folderStructureArray(recordNode(4096000));
expect(result).toEqual([64, 64]);
});
it("should return [64, 64, 2] when folder = 4096001", () => {
const result = folderStructureArray(recordNode(4096001));
expect(result).toEqual([64, 64, 2]);
});
});
const setup = ({parentCount, childCount, grandChildCount}) =>
setupApphierarchy((templateApi) => {
const root = templateApi.getNewRootLevel();
const addField = (recordNode) => {
const field = templateApi.getNewField("string");
field.name = "test";
templateApi.addField(recordNode, field);
return field;
};
const parent = templateApi.getNewRecordTemplate(root, "parent");
parent.estimatedRecordCount = parentCount || 1000;
parent.collectionName = "parents";
addField(parent);
const child = templateApi.getNewRecordTemplate(parent, "child");
child.estimatedRecordCount = childCount || 1000;
child.collectionName = "children";
addField(child);
const grandchild = templateApi.getNewRecordTemplate(child, "grandchild");
grandchild.estimatedRecordCount = grandChildCount || 1000;
grandchild.collectionName = "grandchildren";
addField(grandchild);
return ({
parent, child, grandchild, root
});
});

65
packages/core/test/recordApi.save.spec.js
View file

@ -2,7 +2,7 @@ import {setupApphierarchy, basicAppHierarchyCreator_WithFields,
stubEventHandler} from "./specHelpers";
import {events, isNonEmptyString} from "../src/common";
import {permission} from "../src/authApi/permissions";
import { getRecordInfo } from "../src/recordApi/recordInfo";
describe('recordApi > save then load', () => {
@ -202,48 +202,41 @@ describe("save", () => {
});
it("should create folder and index for subcollection", async () => {
const {recordApi} = await setupApphierarchy(basicAppHierarchyCreator_WithFields);
const record = recordApi.getNew("/customers", "customer");
record.surname = "Ledog";
await recordApi.save(record);
expect(await recordApi._storeHandle.exists(`${record.key}/invoice_index/index.csv`)).toBeTruthy()
expect(await recordApi._storeHandle.exists(`${record.key}/invoice_index`)).toBeTruthy()
expect(await recordApi._storeHandle.exists(`${record.key}/invoices`)).toBeTruthy()
});
it("should create index folder and shardMap for sharded reverse reference index", async () => {
const {recordApi} = await setupApphierarchy(basicAppHierarchyCreator_WithFields);
const record = recordApi.getNew("/customers", "customer");
record.surname = "Ledog";
await recordApi.save(record);
expect(await recordApi._storeHandle.exists(`${record.key}/referredToCustomers/shardMap.json`)).toBeTruthy();
expect(await recordApi._storeHandle.exists(`${record.key}/referredToCustomers`)).toBeTruthy();
});
it("should create folder for record", async () => {
const {recordApi} = await setupApphierarchy(basicAppHierarchyCreator_WithFields);
const record = recordApi.getNew("/customers", "customer");
record.surname = "Ledog";
await recordApi.save(record);
expect(await recordApi._storeHandle.exists(`${record.key}`)).toBeTruthy();
expect(await recordApi._storeHandle.exists(`${record.key}/record.json`)).toBeTruthy();
});
it("should create allids file", async () => {
const {recordApi, appHierarchy} = await setupApphierarchy(basicAppHierarchyCreator_WithFields);
const record = recordApi.getNew("/customers", "customer");
record.surname = "Ledog";
await recordApi.save(record);
const allIdsPath = `/customers/allids/${appHierarchy.customerRecord.nodeId}/${record.id[2]}`;
expect(await recordApi._storeHandle.exists(allIdsPath)).toBeTruthy();
const recordDir = getRecordInfo(appHierarchy.root, record.key).dir;
expect(await recordApi._storeHandle.exists(`${recordDir}/invoice_index/index.csv`)).toBeTruthy()
expect(await recordApi._storeHandle.exists(`${recordDir}/invoice_index`)).toBeTruthy()
expect(await recordApi._storeHandle.exists(`${recordDir}/invoices`)).toBeTruthy()
});
it("should create index folder and shardMap for sharded reverse reference index", async () => {
const {recordApi, appHierarchy} = await setupApphierarchy(basicAppHierarchyCreator_WithFields);
const record = recordApi.getNew("/customers", "customer");
record.surname = "Ledog";
await recordApi.save(record);
const recordDir = getRecordInfo(appHierarchy.root, record.key).dir;
expect(await recordApi._storeHandle.exists(`${recordDir}/referredToCustomers/shardMap.json`)).toBeTruthy();
expect(await recordApi._storeHandle.exists(`${recordDir}/referredToCustomers`)).toBeTruthy();
});
it("should create folder for record", async () => {
const {recordApi, appHierarchy} = await setupApphierarchy(basicAppHierarchyCreator_WithFields);
const record = recordApi.getNew("/customers", "customer");
record.surname = "Ledog";
await recordApi.save(record);
const recordDir = getRecordInfo(appHierarchy.root, record.key).dir;
expect(await recordApi._storeHandle.exists(`${recordDir}`)).toBeTruthy();
expect(await recordApi._storeHandle.exists(`${recordDir}/record.json`)).toBeTruthy();
});
it("create should throw error, user user does not have permission", async () => {
const {recordApi, app, appHierarchy} = await setupApphierarchy(basicAppHierarchyCreator_WithFields);
const record = recordApi.getNew("/customers", "customer");

8
packages/core/test/specHelpers.js
View file

@ -22,12 +22,10 @@ import {permission} from "../src/authApi/permissions";
import {generateFullPermissions} from "../src/authApi/generateFullPermissions"
import {initialiseData} from "../src/appInitialise/initialiseData";
const exp = module.exports;
export const testFileArea = (testNameArea) => path.join("test", "fs_test_area", testNameArea);
export const testConfigFolder = (testAreaName) => path.join(exp.testFileArea(testAreaName), configFolder);
export const testFieldDefinitionsPath = (testAreaName) => path.join(exp.testFileArea(testAreaName), fieldDefinitions);
export const testTemplatesPath = (testAreaName) => path.join(exp.testFileArea(testAreaName), templateDefinitions);
export const testConfigFolder = (testAreaName) => path.join(testFileArea(testAreaName), configFolder);
export const testFieldDefinitionsPath = (testAreaName) => path.join(testFileArea(testAreaName), fieldDefinitions);
export const testTemplatesPath = (testAreaName) => path.join(testFileArea(testAreaName), templateDefinitions);
export const getMemoryStore = () => setupDatastore(memory({}));
export const getMemoryTemplateApi = () => {

2
packages/core/test/templateApi.constructHeirarchy.spec.js
View file

@ -27,7 +27,7 @@ describe("hierarchy node creation", () => {
expect(record.indexes).toEqual([]);
expect(record.parent()).toBe(root);
expect(record.collectionName).toBe("");
expect(record.allidsShardFactor).toBe(64);
expect(record.estimatedRecordCount).toBe(1000000);
expect(record.isSingle).toBe(false);
record.collectionName = "records";