Make ReindexWorker resilient to ES connection issues (#29908)

This commit is contained in:
Josh Dover 2019-02-02 12:11:25 -06:00 committed by GitHub
parent 6888cd49ca
commit c0a2e8014e
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23

View file

@ -37,6 +37,7 @@ export class ReindexWorker {
private static workerSingleton?: ReindexWorker;
private continuePolling: boolean = false;
private updateOperationLoopRunning: boolean = false;
private timeout?: NodeJS.Timeout;
private inProgressOps: ReindexSavedObject[] = [];
private readonly reindexService: ReindexService;
@ -73,6 +74,10 @@ export class ReindexWorker {
*/
public stop = () => {
this.log(['debug', ...LOG_TAGS], `Stopping worker...`);
if (this.timeout) {
clearTimeout(this.timeout);
}
this.updateOperationLoopRunning = false;
this.continuePolling = false;
};
@ -114,12 +119,17 @@ export class ReindexWorker {
await this.refresh();
if (this.continuePolling) {
setTimeout(this.pollForOperations, POLL_INTERVAL);
this.timeout = setTimeout(this.pollForOperations, POLL_INTERVAL);
}
};
private refresh = async () => {
this.inProgressOps = await this.reindexService.findAllByStatus(ReindexStatus.inProgress);
try {
this.inProgressOps = await this.reindexService.findAllByStatus(ReindexStatus.inProgress);
} catch (e) {
this.log(['debug', ...LOG_TAGS], `Could not fetch riendex operations from Elasticsearch`);
this.inProgressOps = [];
}
// If there are operations in progress and we're not already updating operations, kick off the update loop
if (!this.updateOperationLoopRunning) {