CodeMirror/budibase
1
0
Fork 0
mirror of synced 2024-10-05 12:34:50 +13:00
Code Issues Projects Releases Wiki Activity

Use inflight max instead of batch wait

Browse source
This commit is contained in:
Adria Navarro 2023-12-21 19:06:26 +01:00
parent 494a2ff91f
commit abf025b3f7
1 changed files with 20 additions and 10 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

30
packages/server/scripts/load/create-many-relationships.js
View file

@ -13,11 +13,10 @@ if (!process.argv[2]) {
}
const start = Date.now()
async function batchCreate(apiKey, appId, table, items, batchSize = 100) {
async function batchCreate(apiKey, appId, table, items, batchSize = 10) {
let i = 0
let errors = 0
async function createSingleRow(item) {
try {
const row = await createRow(apiKey, appId, table, item)
@ -33,14 +32,25 @@ async function batchCreate(apiKey, appId, table, items, batchSize = 100) {
}
const rows = []
for (let j = 0; j < items.length; j += batchSize) {
const batchPromises = items.slice(j, j + batchSize).map(createSingleRow)
const batchRows = await Promise.all(batchPromises)
rows.push(...batchRows)
}
if (errors) {
console.error(`Error creating ${errors} row`)
const maxConcurrency = Math.min(batchSize, items.length)
const inFlight = {}
for (let itemIndex = 0; itemIndex < items.length; itemIndex++) {
const item = items[itemIndex]
const promise = createSingleRow(item).then(result => {
rows.push(result)
delete inFlight[itemIndex]
})
inFlight[itemIndex] = promise
if (Object.keys(inFlight).length >= maxConcurrency) {
await Promise.race(Object.values(inFlight))
}
}
await Promise.all(Object.values(inFlight))
return rows
}