diff --git a/frontend/src/ts/containers/mapDispatchToProps.ts b/frontend/src/ts/containers/mapDispatchToProps.ts index a6d4c52..e7de95f 100644 --- a/frontend/src/ts/containers/mapDispatchToProps.ts +++ b/frontend/src/ts/containers/mapDispatchToProps.ts @@ -49,12 +49,12 @@ const valueFromInput = (element: HTMLInputElement): string => { let syncedComments = 0; let totalSyncedComments = 0; -const syncComments = async (commentQueue: any[], dispatch: Redux.Dispatch) => { +const syncComments = async (commentQueue: any[], dispatch: Redux.Dispatch, commentType: String) => { + const parallelRequests: Promise [] = []; // We need to throttle the amount of parallel sync requests that we make // because large forums could be syncing thousands of comments - const maxParallelRequests = 1; - const parallelRequests: Promise [] = []; - + // Parent comments can be synced in parallel but child comments need to be synced 1 at a time + const maxParallelRequests = commentType === 'parentComments' ? 100 : 1; for (let comment of commentQueue) { // Make the sync request and add its promise to the queue, // and remove it from queue when complete @@ -85,12 +85,14 @@ const syncComments = async (commentQueue: any[], dispatch: Redux.Dispatch { - dispatch(setValueAction('isManualSyncRunning', false)); + // We sync parent comments first then child comments immediately after, so we only update the state of the sync after child comments + if (commentType === 'childComments') { + dispatch(setValueAction('isManualSyncRunning', false)); + } dispatch(updateSyncStatusAction({ is_manual: true, - progress_message: `Complete (${syncedComments} of ${totalSyncedComments})`, + progress_message: `Completed (${syncedComments} of ${totalSyncedComments})`, })); }); } @@ -151,11 +153,8 @@ const mapDispatchToProps = (dispatch: Redux.Dispatch) => { const endDate: Moment = getDateFromInput(rangeEndInput); // Create a queue of comments within the provided date-range from the Disqus API - let commentQueue: any[] = []; - // To prevent API errors, we need to make sure parent comments are synced before child comments. - // To sort this correctly, we use a list of parent comment objects and a dictionary of child comment objects organized by parent comment ID const parentCommentList: any[] = []; - const childCommentDict: any = {}; + const childCommentList: any[] = []; const getDisqusComments = async (cursor: string = '') => { return new Promise((resolve, reject) => { DisqusApi.instance.listPostsForForum(cursor, startDate, endDate, 100, async (xhr: Event) => { @@ -177,54 +176,37 @@ const mapDispatchToProps = (dispatch: Redux.Dispatch) => { } const pendingComments = disqusData.response; - totalSyncedComments += pendingComments.length - 1; + totalSyncedComments += pendingComments.length; pendingComments.forEach((comment: any) => { - if (!comment.parent) { - // if the comment is a parent comment, push it to the parent comment list. - parentCommentList.push(comment); - } else { - // if the comment is a child comment, add it to the dict organized by parent comment ID - childCommentDict[comment.parent] = comment; - } + comment.parent ? childCommentList.push(comment) : parentCommentList.push(comment); }); - - // now we populate the queue that starts with a parent comment and then is followed by all of its children in order. - for (let i = 0; i < parentCommentList.length; i ++ ) { - // add the parent comment to the end of the queue - const currentParent = parentCommentList[i]; - commentQueue.push(currentParent) - - // if the childCommentDict has the currentParent's ID in it, then we know the current parent's child comment. - let currentParentID = currentParent.id - while (currentParentID && childCommentDict[currentParentID]) { - let childComment = childCommentDict[currentParentID]; - // add the child comment to the end of the queue - commentQueue.push(childComment) - // make the currentParentID the child comment's ID to search for more child comments - currentParentID = childComment.id; - } - } - const nextCursor = disqusData.cursor; if (nextCursor && nextCursor.hasNext) { await getDisqusComments(nextCursor.next); } + const commentQueue = [parentCommentList, childCommentList]; resolve(commentQueue); }); }); }; dispatch(setValueAction('isManualSyncRunning', true)); - getDisqusComments().then((commentQueue: any[]) => { - syncComments(commentQueue, dispatch); - }).catch((err: any) => { - dispatch(setMessageAction({ - onDismiss: handleClearMessage, - text: (err && err.response) || 'Error connecting to the Disqus API', - type: 'error', - })); - }); + getDisqusComments() + .then((commentQueue: any[]) => { + syncComments(commentQueue[0], dispatch, 'parentComments'); + return commentQueue; + }) + .then((commentQueue: any[]) => { + syncComments(commentQueue[1], dispatch, 'childComments'); + }) + .catch((err: any) => { + dispatch(setMessageAction({ + onDismiss: handleClearMessage, + text: (err && err.response) || 'Error connecting to the Disqus API', + type: 'error', + })); + }); }, onSubmitSiteForm: (event: React.SyntheticEvent) => { event.preventDefault();