fix(scheduled-tasks): prevent dual-server double dispatch of daily tasks (#2713)
* fix(scheduled-tasks): claim schedule occurrences across server instances Two OpenChamber servers sharing project config each armed timers and both dispatched the same daily/weekly/cron/once slot (#2710). Claim the occurrence in shared config under a cross-process write lock before creating a session. Co-authored-by: Serhii Dziupin <makeittech@users.noreply.github.com> * fix(scheduled-tasks): harden occurrence claim failure and lock ownership Address PR review blockers: release running-slot bookkeeping when claim throws, avoid silently dropping an armed occurrence after a due-slack sync, verify lock-file ownership on release, and cover real on-disk lock behavior. Co-authored-by: Serhii Dziupin <makeittech@users.noreply.github.com> * fix(scheduled-tasks): always release running slot on state-write failures Wrap runTask bookkeeping in finally so claim, manual-start, and completion lock timeouts cannot stuck-run a task; drop the diskNext claim guard that suppressed later occurrences; recover unparseable locks via mtime age. Co-authored-by: Serhii Dziupin <makeittech@users.noreply.github.com> * fix(scheduled-tasks): stop re-arming past nextRunAt and clear stuck running Only schedule future nextRunAt values so once-task losers and claim-failed paths cannot spin delay-0 retries. Clear past once nextRunAt on claim, and on completion-write failure retry terminal status so manual runNow still returns the session instead of a hard 500 with lastStatus stuck running. Co-authored-by: Serhii Dziupin <makeittech@users.noreply.github.com> * fix(scheduled-tasks): release write chain on lock acquire timeout withProjectWriteLock left the in-process promise chain pending when acquireProjectFileLock timed out, wedging every later project write and stranding runTask before finally. Always release the chain; surface persistError on run; record once claim failures in task state. Co-authored-by: Serhii Dziupin <makeittech@users.noreply.github.com> --------- Co-authored-by: Serhii Dziupin <makeittech@users.noreply.github.com>
This commit is contained in:
Serhii Dziupin
committed by
GitHub
GitHub
co-authored by
Serhii Dziupin
Serhii Dziupin
parent
86e6a2ae76
commit
e99f6560be
@@ -327,7 +327,7 @@ export const createScheduledTasksRuntime = (deps) => {
|
||||
if (!task || !task.enabled) {
|
||||
return;
|
||||
}
|
||||
queueTaskRun(projectID, taskID, 'scheduled');
|
||||
queueTaskRun(projectID, taskID, 'scheduled', nextRunAt);
|
||||
pumpQueue();
|
||||
}, boundedDelay);
|
||||
|
||||
@@ -429,13 +429,18 @@ export const createScheduledTasksRuntime = (deps) => {
|
||||
}
|
||||
};
|
||||
|
||||
const queueTaskRun = (projectID, taskID, reason) => {
|
||||
const queueTaskRun = (projectID, taskID, reason, scheduledFor) => {
|
||||
const taskKey = buildTaskKey(projectID, taskID);
|
||||
if (queuedTaskKeys.has(taskKey) || runningTaskKeys.has(taskKey)) {
|
||||
return;
|
||||
}
|
||||
queuedTaskKeys.add(taskKey);
|
||||
queue.push({ projectID, taskID, reason });
|
||||
queue.push({
|
||||
projectID,
|
||||
taskID,
|
||||
reason,
|
||||
...(Number.isFinite(scheduledFor) ? { scheduledFor } : {}),
|
||||
});
|
||||
};
|
||||
|
||||
const canRunTask = (projectID) => {
|
||||
@@ -605,7 +610,51 @@ export const createScheduledTasksRuntime = (deps) => {
|
||||
};
|
||||
};
|
||||
|
||||
const runTask = async (projectID, taskID, reason) => {
|
||||
const releaseRunningSlot = (projectID, taskKey) => {
|
||||
runningTaskKeys.delete(taskKey);
|
||||
runningGlobalCount = Math.max(0, runningGlobalCount - 1);
|
||||
const nextProjectCount = Math.max(0, (runningCountByProject.get(projectID) || 1) - 1);
|
||||
if (nextProjectCount === 0) {
|
||||
runningCountByProject.delete(projectID);
|
||||
} else {
|
||||
runningCountByProject.set(projectID, nextProjectCount);
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* Arm a timer only for a future occurrence. Scheduling a past nextRunAt
|
||||
* (delay 0 + jitter) re-enters the claim path immediately and can spin —
|
||||
* especially for once tasks where the claim cannot advance nextRunAt.
|
||||
*/
|
||||
const scheduleFutureRun = (projectID, taskID, nextRunAt, fromMs = Date.now()) => {
|
||||
if (!Number.isFinite(nextRunAt)) {
|
||||
return false;
|
||||
}
|
||||
const base = Number.isFinite(fromMs) ? fromMs : Date.now();
|
||||
if (nextRunAt <= base) {
|
||||
return false;
|
||||
}
|
||||
scheduleTask(projectID, taskID, nextRunAt);
|
||||
return true;
|
||||
};
|
||||
|
||||
const rearmFromTaskOrCompute = (projectID, taskID, fallbackTask, fromMs) => {
|
||||
const latest = (tasksByProject.get(projectID)?.get(taskID)) || fallbackTask;
|
||||
if (!latest?.enabled) {
|
||||
return;
|
||||
}
|
||||
const base = Number.isFinite(fromMs) ? fromMs : Date.now();
|
||||
const persistedNext = latest.state?.nextRunAt;
|
||||
// Prefer a still-future persisted slot; never re-arm a past occurrence
|
||||
// (that created silent once-task loser loops and claim-failed retry spam).
|
||||
if (scheduleFutureRun(projectID, taskID, persistedNext, base)) {
|
||||
return;
|
||||
}
|
||||
const computedNext = computeNextRunAt(latest, base);
|
||||
scheduleFutureRun(projectID, taskID, computedNext, base);
|
||||
};
|
||||
|
||||
const runTask = async (projectID, taskID, reason, scheduledFor) => {
|
||||
const taskMap = tasksByProject.get(projectID);
|
||||
const task = taskMap?.get(taskID);
|
||||
if (!task || !task.enabled) {
|
||||
@@ -621,125 +670,328 @@ export const createScheduledTasksRuntime = (deps) => {
|
||||
runningGlobalCount += 1;
|
||||
runningCountByProject.set(projectID, (runningCountByProject.get(projectID) || 0) + 1);
|
||||
|
||||
const runStartedAt = Date.now();
|
||||
await projectConfigRuntime.updateScheduledTaskState(projectID, taskID, {
|
||||
lastRunAt: runStartedAt,
|
||||
lastStatus: 'running',
|
||||
lastError: undefined,
|
||||
updatedAt: runStartedAt,
|
||||
}).then((result) => {
|
||||
if (result.task) {
|
||||
updateInMemoryTask(projectID, result.task);
|
||||
}
|
||||
});
|
||||
|
||||
let status = 'success';
|
||||
let sessionID;
|
||||
let durationMs = 0;
|
||||
let errorMessage;
|
||||
|
||||
// Every path that holds the running slot must exit through this finally so
|
||||
// lock timeouts / fs errors on claim, manual-start, or completion writes
|
||||
// cannot permanently stuck-run the task in this process.
|
||||
try {
|
||||
const runPromise = runTaskWithWatchdog(projectID, task, reason);
|
||||
let timeoutID;
|
||||
const timeoutPromise = new Promise((_, reject) => {
|
||||
timeoutID = setTimeout(() => {
|
||||
reject(new Error('scheduled task run timed out'));
|
||||
}, maxRunDurationMs);
|
||||
});
|
||||
const runStartedAt = Date.now();
|
||||
|
||||
const result = await Promise.race([runPromise, timeoutPromise]).finally(() => {
|
||||
if (timeoutID) {
|
||||
clearTimeout(timeoutID);
|
||||
// Scheduled dispatches must claim the occurrence in shared project config
|
||||
// before creating a session. Two server instances (e.g. CLI serve + desktop)
|
||||
// each arm their own timer; without this claim both would run (#2710).
|
||||
if (reason === 'scheduled') {
|
||||
if (!Number.isFinite(scheduledFor)) {
|
||||
return { ok: false, skipped: true, reason: 'missing-scheduled-for' };
|
||||
}
|
||||
});
|
||||
sessionID = result.sessionID;
|
||||
durationMs = result.durationMs;
|
||||
status = 'success';
|
||||
logger.info?.(
|
||||
'[ScheduledTasks] run completed',
|
||||
{ projectID, taskID, status, reason, sessionID, durationMs }
|
||||
);
|
||||
} catch (error) {
|
||||
status = 'error';
|
||||
errorMessage = safeErrorMessage(error);
|
||||
logger.warn?.('[ScheduledTasks] run failed', {
|
||||
projectID,
|
||||
taskID,
|
||||
reason,
|
||||
status,
|
||||
error: errorMessage,
|
||||
});
|
||||
}
|
||||
|
||||
const finishedAt = Date.now();
|
||||
if (!durationMs) {
|
||||
durationMs = Math.max(0, finishedAt - runStartedAt);
|
||||
}
|
||||
let latestTask = (tasksByProject.get(projectID)?.get(taskID)) || task;
|
||||
const shouldConsumeOneTimeTask = latestTask?.schedule?.kind === 'once' && reason === 'scheduled';
|
||||
if (shouldConsumeOneTimeTask && latestTask?.enabled) {
|
||||
const nextAfterClaim = computeNextRunAt(task, Math.max(runStartedAt, scheduledFor + 1));
|
||||
const claimPatch = {
|
||||
lastScheduledFor: Math.round(scheduledFor),
|
||||
lastRunAt: runStartedAt,
|
||||
lastStatus: 'running',
|
||||
lastError: undefined,
|
||||
updatedAt: runStartedAt,
|
||||
// Always set nextRunAt so a past once-slot is cleared when there is
|
||||
// no following occurrence (omitting the key would leave the past value).
|
||||
nextRunAt: Number.isFinite(nextAfterClaim) ? nextAfterClaim : undefined,
|
||||
};
|
||||
|
||||
// Duplicate protection is solely lastScheduledFor within slack of this
|
||||
// occurrence. Do not reject on advanced disk nextRunAt: lastScheduledFor
|
||||
// persists across days, so a second-instance sync inside TASK_DUE_SLACK_MS
|
||||
// would otherwise suppress every armed occurrence after the first.
|
||||
const canClaimOccurrence = (candidate) => {
|
||||
if (!candidate?.enabled) {
|
||||
return false;
|
||||
}
|
||||
const lastScheduledFor = candidate.state?.lastScheduledFor;
|
||||
if (
|
||||
Number.isFinite(lastScheduledFor)
|
||||
&& Math.abs(lastScheduledFor - scheduledFor) <= TASK_DUE_SLACK_MS
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
};
|
||||
|
||||
let claimResult;
|
||||
try {
|
||||
if (typeof projectConfigRuntime.updateScheduledTaskStateIf === 'function') {
|
||||
claimResult = await projectConfigRuntime.updateScheduledTaskStateIf(
|
||||
projectID,
|
||||
taskID,
|
||||
canClaimOccurrence,
|
||||
claimPatch,
|
||||
);
|
||||
} else {
|
||||
// Fallback for older test doubles: unconditional update (single-instance only).
|
||||
claimResult = await projectConfigRuntime.updateScheduledTaskState(projectID, taskID, claimPatch);
|
||||
claimResult = { ...claimResult, updated: Boolean(claimResult?.task) };
|
||||
}
|
||||
} catch (claimError) {
|
||||
const message = safeErrorMessage(claimError);
|
||||
logger.warn?.('[ScheduledTasks] occurrence claim failed', {
|
||||
projectID,
|
||||
taskID,
|
||||
error: message,
|
||||
});
|
||||
rearmFromTaskOrCompute(projectID, taskID, task, Math.max(runStartedAt, scheduledFor + 1));
|
||||
|
||||
// Best-effort record so once tasks are not left enabled-but-inert with
|
||||
// no UI signal. Do not clobber a winner that claimed this occurrence.
|
||||
const claimFailurePatch = {
|
||||
lastStatus: 'error',
|
||||
lastError: `Scheduled claim failed: ${message}`,
|
||||
updatedAt: Date.now(),
|
||||
};
|
||||
try {
|
||||
if (typeof projectConfigRuntime.updateScheduledTaskStateIf === 'function') {
|
||||
const recorded = await projectConfigRuntime.updateScheduledTaskStateIf(
|
||||
projectID,
|
||||
taskID,
|
||||
(candidate) => {
|
||||
const lastScheduledFor = candidate.state?.lastScheduledFor;
|
||||
if (
|
||||
Number.isFinite(lastScheduledFor)
|
||||
&& Math.abs(lastScheduledFor - scheduledFor) <= TASK_DUE_SLACK_MS
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
},
|
||||
claimFailurePatch,
|
||||
);
|
||||
if (recorded.task) {
|
||||
updateInMemoryTask(projectID, recorded.task);
|
||||
}
|
||||
} else {
|
||||
const recorded = await projectConfigRuntime.updateScheduledTaskState(
|
||||
projectID,
|
||||
taskID,
|
||||
claimFailurePatch,
|
||||
);
|
||||
if (recorded.task) {
|
||||
updateInMemoryTask(projectID, recorded.task);
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
updateInMemoryTask(projectID, {
|
||||
...task,
|
||||
state: {
|
||||
...(task.state || {}),
|
||||
...claimFailurePatch,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
return { ok: false, skipped: true, reason: 'claim-failed', error: message };
|
||||
}
|
||||
|
||||
if (!claimResult?.updated) {
|
||||
if (claimResult?.task) {
|
||||
updateInMemoryTask(projectID, claimResult.task);
|
||||
// Loser must not schedule a past nextRunAt (once-task spin).
|
||||
rearmFromTaskOrCompute(
|
||||
projectID,
|
||||
taskID,
|
||||
claimResult.task,
|
||||
Math.max(Date.now(), scheduledFor + 1),
|
||||
);
|
||||
}
|
||||
return { ok: false, skipped: true, reason: 'occurrence-claimed' };
|
||||
}
|
||||
|
||||
if (claimResult.task) {
|
||||
updateInMemoryTask(projectID, claimResult.task);
|
||||
}
|
||||
} else {
|
||||
try {
|
||||
const startResult = await projectConfigRuntime.updateScheduledTaskState(projectID, taskID, {
|
||||
lastRunAt: runStartedAt,
|
||||
lastStatus: 'running',
|
||||
lastError: undefined,
|
||||
updatedAt: runStartedAt,
|
||||
});
|
||||
if (startResult.task) {
|
||||
updateInMemoryTask(projectID, startResult.task);
|
||||
}
|
||||
} catch (startError) {
|
||||
const message = safeErrorMessage(startError);
|
||||
logger.warn?.('[ScheduledTasks] manual start state write failed', {
|
||||
projectID,
|
||||
taskID,
|
||||
error: message,
|
||||
});
|
||||
return { ok: false, error: message, reason: 'start-state-failed' };
|
||||
}
|
||||
}
|
||||
|
||||
let status = 'success';
|
||||
let sessionID;
|
||||
let durationMs = 0;
|
||||
let errorMessage;
|
||||
|
||||
try {
|
||||
const consumed = await projectConfigRuntime.upsertScheduledTask(projectID, {
|
||||
...latestTask,
|
||||
enabled: false,
|
||||
const runPromise = runTaskWithWatchdog(projectID, task, reason);
|
||||
let timeoutID;
|
||||
const timeoutPromise = new Promise((_, reject) => {
|
||||
timeoutID = setTimeout(() => {
|
||||
reject(new Error('scheduled task run timed out'));
|
||||
}, maxRunDurationMs);
|
||||
});
|
||||
latestTask = consumed.task || latestTask;
|
||||
updateInMemoryTask(projectID, latestTask);
|
||||
} catch (consumeError) {
|
||||
logger.warn?.('[ScheduledTasks] failed to consume one-time task', {
|
||||
|
||||
const result = await Promise.race([runPromise, timeoutPromise]).finally(() => {
|
||||
if (timeoutID) {
|
||||
clearTimeout(timeoutID);
|
||||
}
|
||||
});
|
||||
sessionID = result.sessionID;
|
||||
durationMs = result.durationMs;
|
||||
status = 'success';
|
||||
logger.info?.(
|
||||
'[ScheduledTasks] run completed',
|
||||
{ projectID, taskID, status, reason, sessionID, durationMs }
|
||||
);
|
||||
} catch (error) {
|
||||
status = 'error';
|
||||
errorMessage = safeErrorMessage(error);
|
||||
logger.warn?.('[ScheduledTasks] run failed', {
|
||||
projectID,
|
||||
taskID,
|
||||
error: safeErrorMessage(consumeError),
|
||||
reason,
|
||||
status,
|
||||
error: errorMessage,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const nextRunAt = computeNextRunAt(latestTask, finishedAt);
|
||||
|
||||
const statePatch = {
|
||||
lastStatus: status,
|
||||
lastDurationMs: durationMs,
|
||||
lastError: status === 'error' ? errorMessage : undefined,
|
||||
lastSessionId: status === 'success' ? sessionID : undefined,
|
||||
nextRunAt: Number.isFinite(nextRunAt) ? nextRunAt : undefined,
|
||||
updatedAt: finishedAt,
|
||||
};
|
||||
|
||||
const stateResult = await projectConfigRuntime.updateScheduledTaskState(projectID, taskID, statePatch);
|
||||
if (stateResult.task) {
|
||||
updateInMemoryTask(projectID, stateResult.task);
|
||||
if (stateResult.task.enabled && Number.isFinite(stateResult.task.state?.nextRunAt)) {
|
||||
scheduleTask(projectID, taskID, stateResult.task.state.nextRunAt);
|
||||
const finishedAt = Date.now();
|
||||
if (!durationMs) {
|
||||
durationMs = Math.max(0, finishedAt - runStartedAt);
|
||||
}
|
||||
let latestTask = (tasksByProject.get(projectID)?.get(taskID)) || task;
|
||||
const shouldConsumeOneTimeTask = latestTask?.schedule?.kind === 'once' && reason === 'scheduled';
|
||||
if (shouldConsumeOneTimeTask && latestTask?.enabled) {
|
||||
try {
|
||||
const consumed = await projectConfigRuntime.upsertScheduledTask(projectID, {
|
||||
...latestTask,
|
||||
enabled: false,
|
||||
});
|
||||
latestTask = consumed.task || latestTask;
|
||||
updateInMemoryTask(projectID, latestTask);
|
||||
} catch (consumeError) {
|
||||
logger.warn?.('[ScheduledTasks] failed to consume one-time task', {
|
||||
projectID,
|
||||
taskID,
|
||||
error: safeErrorMessage(consumeError),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
emitTaskRunEvent?.({
|
||||
projectID,
|
||||
taskID,
|
||||
ranAt: finishedAt,
|
||||
const nextRunAt = computeNextRunAt(latestTask, finishedAt);
|
||||
|
||||
const statePatch = {
|
||||
lastStatus: status,
|
||||
lastDurationMs: durationMs,
|
||||
lastError: status === 'error' ? errorMessage : undefined,
|
||||
lastSessionId: status === 'success' ? sessionID : undefined,
|
||||
nextRunAt: Number.isFinite(nextRunAt) ? nextRunAt : undefined,
|
||||
updatedAt: finishedAt,
|
||||
};
|
||||
|
||||
let stateResult = { task: null };
|
||||
try {
|
||||
stateResult = await projectConfigRuntime.updateScheduledTaskState(projectID, taskID, statePatch);
|
||||
if (stateResult.task) {
|
||||
updateInMemoryTask(projectID, stateResult.task);
|
||||
if (stateResult.task.enabled) {
|
||||
scheduleFutureRun(
|
||||
projectID,
|
||||
taskID,
|
||||
stateResult.task.state?.nextRunAt,
|
||||
finishedAt,
|
||||
);
|
||||
}
|
||||
}
|
||||
} catch (persistError) {
|
||||
const message = safeErrorMessage(persistError);
|
||||
logger.warn?.('[ScheduledTasks] run completion state write failed', {
|
||||
projectID,
|
||||
taskID,
|
||||
reason,
|
||||
error: message,
|
||||
});
|
||||
|
||||
// Keep in-memory status terminal so this process does not advertise
|
||||
// a stuck "running" task after the session already finished.
|
||||
const recoveredTask = {
|
||||
...latestTask,
|
||||
state: {
|
||||
...(latestTask.state || {}),
|
||||
lastStatus: status,
|
||||
lastDurationMs: durationMs,
|
||||
lastError: status === 'error' ? errorMessage : undefined,
|
||||
lastSessionId: status === 'success' ? sessionID : undefined,
|
||||
nextRunAt: Number.isFinite(nextRunAt) ? nextRunAt : undefined,
|
||||
updatedAt: finishedAt,
|
||||
},
|
||||
};
|
||||
updateInMemoryTask(projectID, recoveredTask);
|
||||
|
||||
// Best-effort single retry so persisted lastStatus does not stay 'running'.
|
||||
try {
|
||||
const retry = await projectConfigRuntime.updateScheduledTaskState(projectID, taskID, statePatch);
|
||||
if (retry.task) {
|
||||
updateInMemoryTask(projectID, retry.task);
|
||||
stateResult = retry;
|
||||
if (retry.task.enabled) {
|
||||
scheduleFutureRun(projectID, taskID, retry.task.state?.nextRunAt, finishedAt);
|
||||
}
|
||||
}
|
||||
} catch (retryError) {
|
||||
logger.warn?.('[ScheduledTasks] run completion state retry failed', {
|
||||
projectID,
|
||||
taskID,
|
||||
reason,
|
||||
error: safeErrorMessage(retryError),
|
||||
});
|
||||
stateResult = { task: recoveredTask };
|
||||
rearmFromTaskOrCompute(projectID, taskID, recoveredTask, finishedAt);
|
||||
}
|
||||
|
||||
// The session already ran — surface persist failure without treating a
|
||||
// successful dispatch as a hard run failure (manual runNow would 500).
|
||||
return {
|
||||
ok: status === 'success',
|
||||
status,
|
||||
sessionID,
|
||||
task: stateResult.task || recoveredTask,
|
||||
error: status === 'error' ? errorMessage : undefined,
|
||||
persistError: message,
|
||||
reason: 'completion-state-failed',
|
||||
};
|
||||
}
|
||||
|
||||
try {
|
||||
emitTaskRunEvent?.({
|
||||
projectID,
|
||||
taskID,
|
||||
ranAt: finishedAt,
|
||||
status,
|
||||
...(sessionID ? { sessionID } : {}),
|
||||
});
|
||||
} catch {
|
||||
}
|
||||
|
||||
return {
|
||||
ok: status === 'success',
|
||||
status,
|
||||
...(sessionID ? { sessionID } : {}),
|
||||
});
|
||||
} catch {
|
||||
sessionID,
|
||||
task: stateResult.task || null,
|
||||
error: errorMessage,
|
||||
};
|
||||
} finally {
|
||||
releaseRunningSlot(projectID, taskKey);
|
||||
}
|
||||
|
||||
runningTaskKeys.delete(taskKey);
|
||||
runningGlobalCount = Math.max(0, runningGlobalCount - 1);
|
||||
const nextProjectCount = Math.max(0, (runningCountByProject.get(projectID) || 1) - 1);
|
||||
if (nextProjectCount === 0) {
|
||||
runningCountByProject.delete(projectID);
|
||||
} else {
|
||||
runningCountByProject.set(projectID, nextProjectCount);
|
||||
}
|
||||
|
||||
return {
|
||||
ok: status === 'success',
|
||||
status,
|
||||
sessionID,
|
||||
task: stateResult.task || null,
|
||||
error: errorMessage,
|
||||
};
|
||||
};
|
||||
|
||||
const pumpQueue = () => {
|
||||
@@ -761,9 +1013,18 @@ export const createScheduledTasksRuntime = (deps) => {
|
||||
queuedTaskKeys.delete(taskKey);
|
||||
consumed = true;
|
||||
|
||||
void runTask(item.projectID, item.taskID, item.reason).finally(() => {
|
||||
pumpQueue();
|
||||
});
|
||||
void runTask(item.projectID, item.taskID, item.reason, item.scheduledFor)
|
||||
.catch((error) => {
|
||||
logger.warn?.('[ScheduledTasks] queued run rejected', {
|
||||
projectID: item.projectID,
|
||||
taskID: item.taskID,
|
||||
reason: item.reason,
|
||||
error: safeErrorMessage(error),
|
||||
});
|
||||
})
|
||||
.finally(() => {
|
||||
pumpQueue();
|
||||
});
|
||||
}
|
||||
|
||||
if (!consumed && queue.length > 0) {
|
||||
|
||||
Reference in New Issue
Block a user