feat: add message stall detection and soft resync in event stream
This commit is contained in:
@@ -22,6 +22,16 @@ interface EventData {
|
|||||||
properties?: Record<string, unknown>;
|
properties?: Record<string, unknown>;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const readStringProp = (obj: unknown, keys: string[]): string | null => {
|
||||||
|
if (!obj || typeof obj !== 'object') return null;
|
||||||
|
const record = obj as Record<string, unknown>;
|
||||||
|
for (let i = 0; i < keys.length; i++) {
|
||||||
|
const value = record[keys[i]];
|
||||||
|
if (typeof value === 'string' && value.length > 0) return value;
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
};
|
||||||
|
|
||||||
type MessageTracker = (messageId: string, event?: string, extraData?: Record<string, unknown>) => void;
|
type MessageTracker = (messageId: string, event?: string, extraData?: Record<string, unknown>) => void;
|
||||||
|
|
||||||
declare global {
|
declare global {
|
||||||
@@ -347,6 +357,10 @@ export const useEventStream = () => {
|
|||||||
[resyncMessages]
|
[resyncMessages]
|
||||||
);
|
);
|
||||||
|
|
||||||
|
React.useEffect(() => {
|
||||||
|
scheduleSoftResyncRef.current = scheduleSoftResync;
|
||||||
|
}, [scheduleSoftResync]);
|
||||||
|
|
||||||
const trackMessage = React.useCallback((messageId: string, event?: string, extraData?: Record<string, unknown>) => {
|
const trackMessage = React.useCallback((messageId: string, event?: string, extraData?: Record<string, unknown>) => {
|
||||||
if (streamDebugEnabled()) {
|
if (streamDebugEnabled()) {
|
||||||
console.debug(`[MessageTracker] ${messageId}: ${event}`, extraData);
|
console.debug(`[MessageTracker] ${messageId}: ${event}`, extraData);
|
||||||
@@ -389,6 +403,14 @@ export const useEventStream = () => {
|
|||||||
const pauseTimeoutRef = React.useRef<NodeJS.Timeout | null>(null);
|
const pauseTimeoutRef = React.useRef<NodeJS.Timeout | null>(null);
|
||||||
const staleCheckIntervalRef = React.useRef<NodeJS.Timeout | null>(null);
|
const staleCheckIntervalRef = React.useRef<NodeJS.Timeout | null>(null);
|
||||||
const lastEventTimestampRef = React.useRef<number>(Date.now());
|
const lastEventTimestampRef = React.useRef<number>(Date.now());
|
||||||
|
const lastMessageEventBySessionRef = React.useRef<Map<string, number>>(new Map());
|
||||||
|
const pendingMessageStallTimersRef = React.useRef<Map<string, NodeJS.Timeout>>(new Map());
|
||||||
|
const lastMessageStallRecoveryBySessionRef = React.useRef<Map<string, number>>(new Map());
|
||||||
|
|
||||||
|
const scheduleSoftResyncRef = React.useRef<
|
||||||
|
(sessionId: string, reason: string, limit?: number) => Promise<void>
|
||||||
|
>(() => Promise.resolve());
|
||||||
|
const scheduleReconnectRef = React.useRef<(hint?: string) => void>(() => {});
|
||||||
const isDesktopRuntimeRef = React.useRef<boolean>(false);
|
const isDesktopRuntimeRef = React.useRef<boolean>(false);
|
||||||
|
|
||||||
const maybeBootstrapIfStale = React.useCallback(
|
const maybeBootstrapIfStale = React.useCallback(
|
||||||
@@ -493,6 +515,52 @@ export const useEventStream = () => {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const shouldArmMessageStallCheck = storePhase === 'idle' && (phase === 'busy' || phase === 'cooldown');
|
||||||
|
const shouldDisarmMessageStallCheck = phase === 'idle';
|
||||||
|
|
||||||
|
if (shouldDisarmMessageStallCheck) {
|
||||||
|
const pending = pendingMessageStallTimersRef.current.get(sessionId);
|
||||||
|
if (pending) {
|
||||||
|
clearTimeout(pending);
|
||||||
|
pendingMessageStallTimersRef.current.delete(sessionId);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (shouldArmMessageStallCheck) {
|
||||||
|
const pending = pendingMessageStallTimersRef.current.get(sessionId);
|
||||||
|
if (pending) {
|
||||||
|
clearTimeout(pending);
|
||||||
|
pendingMessageStallTimersRef.current.delete(sessionId);
|
||||||
|
}
|
||||||
|
|
||||||
|
const startAt = Date.now();
|
||||||
|
const timer = setTimeout(() => {
|
||||||
|
const currentPhase = useSessionStore.getState().sessionActivityPhase?.get(sessionId);
|
||||||
|
if (currentPhase !== 'busy' && currentPhase !== 'cooldown') {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const lastRecoveryAt = lastMessageStallRecoveryBySessionRef.current.get(sessionId) ?? 0;
|
||||||
|
if (Date.now() - lastRecoveryAt < 15000) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const lastMsgAt = lastMessageEventBySessionRef.current.get(sessionId) ?? 0;
|
||||||
|
if (lastMsgAt >= startAt) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
lastMessageStallRecoveryBySessionRef.current.set(sessionId, Date.now());
|
||||||
|
|
||||||
|
void scheduleSoftResyncRef.current(sessionId, 'activity_started_no_message', getActiveSessionWindow())
|
||||||
|
.finally(() => {
|
||||||
|
scheduleReconnectRef.current('No message events after activity start');
|
||||||
|
});
|
||||||
|
}, 2000);
|
||||||
|
|
||||||
|
pendingMessageStallTimersRef.current.set(sessionId, timer);
|
||||||
|
}
|
||||||
|
|
||||||
const existingTimer = sessionCooldownTimersRef.current.get(sessionId);
|
const existingTimer = sessionCooldownTimersRef.current.get(sessionId);
|
||||||
if (existingTimer) {
|
if (existingTimer) {
|
||||||
clearTimeout(existingTimer);
|
clearTimeout(existingTimer);
|
||||||
@@ -737,25 +805,19 @@ export const useEventStream = () => {
|
|||||||
const partExt = part as Record<string, unknown>;
|
const partExt = part as Record<string, unknown>;
|
||||||
const messageInfo = (typeof props.info === 'object' && props.info !== null) ? (props.info as Record<string, unknown>) : props;
|
const messageInfo = (typeof props.info === 'object' && props.info !== null) ? (props.info as Record<string, unknown>) : props;
|
||||||
|
|
||||||
const messageInfoSessionId = typeof (messageInfo as { sessionID?: unknown }).sessionID === 'string'
|
const messageInfoSessionId = readStringProp(messageInfo, ['sessionID', 'sessionId']);
|
||||||
? (messageInfo as { sessionID?: string }).sessionID
|
|
||||||
: null;
|
|
||||||
|
|
||||||
const resolvedSessionId =
|
const resolvedSessionId =
|
||||||
(typeof partExt.sessionID === 'string' && (partExt.sessionID as string).length > 0) ? (partExt.sessionID as string) :
|
readStringProp(partExt, ['sessionID', 'sessionId']) ||
|
||||||
(typeof messageInfoSessionId === 'string' && messageInfoSessionId.length > 0) ? messageInfoSessionId :
|
messageInfoSessionId ||
|
||||||
(typeof props.sessionID === 'string' && (props.sessionID as string).length > 0) ? (props.sessionID as string) :
|
readStringProp(props, ['sessionID', 'sessionId']);
|
||||||
null;
|
|
||||||
|
|
||||||
const messageInfoId = typeof (messageInfo as { id?: unknown }).id === 'string'
|
const messageInfoId = readStringProp(messageInfo, ['messageID', 'messageId', 'id']);
|
||||||
? (messageInfo as { id?: string }).id
|
|
||||||
: null;
|
|
||||||
|
|
||||||
const resolvedMessageId =
|
const resolvedMessageId =
|
||||||
(typeof partExt.messageID === 'string' && (partExt.messageID as string).length > 0) ? (partExt.messageID as string) :
|
readStringProp(partExt, ['messageID', 'messageId']) ||
|
||||||
(typeof messageInfoId === 'string' && messageInfoId.length > 0) ? messageInfoId :
|
messageInfoId ||
|
||||||
(typeof props.messageID === 'string' && (props.messageID as string).length > 0) ? (props.messageID as string) :
|
readStringProp(props, ['messageID', 'messageId']);
|
||||||
null;
|
|
||||||
|
|
||||||
if (!resolvedSessionId || !resolvedMessageId) {
|
if (!resolvedSessionId || !resolvedMessageId) {
|
||||||
if (streamDebugEnabled()) {
|
if (streamDebugEnabled()) {
|
||||||
@@ -770,6 +832,13 @@ export const useEventStream = () => {
|
|||||||
const sessionId = resolvedSessionId;
|
const sessionId = resolvedSessionId;
|
||||||
const messageId = resolvedMessageId;
|
const messageId = resolvedMessageId;
|
||||||
|
|
||||||
|
lastMessageEventBySessionRef.current.set(sessionId, Date.now());
|
||||||
|
const pendingTimer = pendingMessageStallTimersRef.current.get(sessionId);
|
||||||
|
if (pendingTimer) {
|
||||||
|
clearTimeout(pendingTimer);
|
||||||
|
pendingMessageStallTimersRef.current.delete(sessionId);
|
||||||
|
}
|
||||||
|
|
||||||
const trimmedHeadMaxId = useSessionStore.getState().sessionMemoryState.get(sessionId)?.trimmedHeadMaxId;
|
const trimmedHeadMaxId = useSessionStore.getState().sessionMemoryState.get(sessionId)?.trimmedHeadMaxId;
|
||||||
if (trimmedHeadMaxId && !isIdNewer(messageId, trimmedHeadMaxId)) {
|
if (trimmedHeadMaxId && !isIdNewer(messageId, trimmedHeadMaxId)) {
|
||||||
if (streamDebugEnabled()) {
|
if (streamDebugEnabled()) {
|
||||||
@@ -817,14 +886,12 @@ export const useEventStream = () => {
|
|||||||
const messageExt = message as Record<string, unknown>;
|
const messageExt = message as Record<string, unknown>;
|
||||||
|
|
||||||
const resolvedSessionId =
|
const resolvedSessionId =
|
||||||
(typeof messageExt.sessionID === 'string' && (messageExt.sessionID as string).length > 0) ? (messageExt.sessionID as string) :
|
readStringProp(messageExt, ['sessionID', 'sessionId']) ||
|
||||||
(typeof props.sessionID === 'string' && (props.sessionID as string).length > 0) ? (props.sessionID as string) :
|
readStringProp(props, ['sessionID', 'sessionId']);
|
||||||
null;
|
|
||||||
|
|
||||||
const resolvedMessageId =
|
const resolvedMessageId =
|
||||||
(typeof messageExt.id === 'string' && (messageExt.id as string).length > 0) ? (messageExt.id as string) :
|
readStringProp(messageExt, ['messageID', 'messageId', 'id']) ||
|
||||||
(typeof props.messageID === 'string' && (props.messageID as string).length > 0) ? (props.messageID as string) :
|
readStringProp(props, ['messageID', 'messageId']);
|
||||||
null;
|
|
||||||
|
|
||||||
if (!resolvedSessionId || !resolvedMessageId) {
|
if (!resolvedSessionId || !resolvedMessageId) {
|
||||||
if (streamDebugEnabled()) {
|
if (streamDebugEnabled()) {
|
||||||
@@ -839,6 +906,13 @@ export const useEventStream = () => {
|
|||||||
const sessionId = resolvedSessionId;
|
const sessionId = resolvedSessionId;
|
||||||
const messageId = resolvedMessageId;
|
const messageId = resolvedMessageId;
|
||||||
|
|
||||||
|
lastMessageEventBySessionRef.current.set(sessionId, Date.now());
|
||||||
|
const pendingTimer = pendingMessageStallTimersRef.current.get(sessionId);
|
||||||
|
if (pendingTimer) {
|
||||||
|
clearTimeout(pendingTimer);
|
||||||
|
pendingMessageStallTimersRef.current.delete(sessionId);
|
||||||
|
}
|
||||||
|
|
||||||
const trimmedHeadMaxId = useSessionStore.getState().sessionMemoryState.get(sessionId)?.trimmedHeadMaxId;
|
const trimmedHeadMaxId = useSessionStore.getState().sessionMemoryState.get(sessionId)?.trimmedHeadMaxId;
|
||||||
if (trimmedHeadMaxId && !isIdNewer(messageId, trimmedHeadMaxId)) {
|
if (trimmedHeadMaxId && !isIdNewer(messageId, trimmedHeadMaxId)) {
|
||||||
if (streamDebugEnabled()) {
|
if (streamDebugEnabled()) {
|
||||||
@@ -1776,6 +1850,10 @@ export const useEventStream = () => {
|
|||||||
}, delay);
|
}, delay);
|
||||||
}, [shouldHoldConnection, stopStream, publishStatus, startStream]);
|
}, [shouldHoldConnection, stopStream, publishStatus, startStream]);
|
||||||
|
|
||||||
|
React.useEffect(() => {
|
||||||
|
scheduleReconnectRef.current = scheduleReconnect;
|
||||||
|
}, [scheduleReconnect]);
|
||||||
|
|
||||||
React.useEffect(() => {
|
React.useEffect(() => {
|
||||||
const cooldownTimers = sessionCooldownTimersRef.current;
|
const cooldownTimers = sessionCooldownTimersRef.current;
|
||||||
|
|
||||||
@@ -1930,6 +2008,7 @@ export const useEventStream = () => {
|
|||||||
const hasBusySessions = Array.from(useSessionStore.getState().sessionActivityPhase?.values?.() ?? []).some(
|
const hasBusySessions = Array.from(useSessionStore.getState().sessionActivityPhase?.values?.() ?? []).some(
|
||||||
(phase) => phase === 'busy' || phase === 'cooldown'
|
(phase) => phase === 'busy' || phase === 'cooldown'
|
||||||
);
|
);
|
||||||
|
|
||||||
if (hasBusySessions) {
|
if (hasBusySessions) {
|
||||||
void refreshSessionActivityStatus();
|
void refreshSessionActivityStatus();
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user