protocol/apps-node/rfq-api/test/utils/rfqm_health_check_test.ts

import { BigNumber } from '@0x/utils';
import { Producer } from 'sqs-producer';
import { instance, mock, when } from 'ts-mockito';

import { ETH_DECIMALS } from '../../src/core/constants';
import { RfqmWorkerHeartbeatEntity } from '../../src/entities';
import {
    checkSqsQueueAsync,
    checkWorkerHeartbeatsAsync,
    getHttpIssues,
    HealthCheckStatus,
} from '../../src/utils/rfqm_health_check';

let producerMock: Producer;

const MS_IN_MINUTE = 60000;
const fullBalance = new BigNumber(1).shiftedBy(ETH_DECIMALS);

describe('RFQm Health Check', () => {
    describe('checkSqsQueueAsync', () => {
        beforeEach(() => {
            producerMock = mock(Producer);
        });

        describe('queue size check', () => {
            it('creates no issues if there are 10 or less jobs in the queue', async () => {
                when(producerMock.queueSize()).thenResolve(1);

                const issues = await checkSqsQueueAsync(instance(producerMock));

                expect(issues.length).toEqual(0);
            });

            it('creates a DEGRADED issue if there are more than 5 messages in the queue', async () => {
                when(producerMock.queueSize()).thenResolve(11);

                const issues = await checkSqsQueueAsync(instance(producerMock));

                expect(issues.length).toEqual(1);
                expect(issues[0].status).toEqual(HealthCheckStatus.Degraded);
            });

            it('creates a FAILED issue if there are more than 20 messages in the queue', async () => {
                when(producerMock.queueSize()).thenResolve(21);

                const issues = await checkSqsQueueAsync(instance(producerMock));

                expect(issues.length).toEqual(1);
                expect(issues[0].status).toEqual(HealthCheckStatus.Failed);
            });
        });
    });

    describe('checkWorkerHeartbeatsAsync', () => {
        it('creates a failed issue when no heartbeats are found', async () => {
            const issues = await checkWorkerHeartbeatsAsync([]);

            expect(issues.length).toEqual(1);
            expect(issues[0].status).toEqual(HealthCheckStatus.Failed);
        });

        describe('Heartbeat age', () => {
            it('creates a failed issue with no recent heartbeats', async () => {
                const now = new Date();
                const nowTime = now.getTime();
                const heartbeat = new RfqmWorkerHeartbeatEntity({
                    address: '0x00',
                    balance: fullBalance,
                    index: 0,
                    timestamp: new Date(nowTime - MS_IN_MINUTE * 6),
                    chainId: 1337,
                });

                const issues = await checkWorkerHeartbeatsAsync([heartbeat], now);
                const failedIssues = issues.filter(({ status }) => status === HealthCheckStatus.Failed);

                expect(failedIssues.length).toEqual(1);
            });

            it('creates degraded issues for stale heartbeats', async () => {
                const now = new Date();
                const nowTime = now.getTime();
                const heartbeat1 = new RfqmWorkerHeartbeatEntity({
                    address: '0x00',
                    balance: fullBalance,
                    index: 0,
                    timestamp: now,
                    chainId: 1337,
                });
                const heartbeat2 = new RfqmWorkerHeartbeatEntity({
                    address: '0x01',
                    balance: fullBalance,
                    index: 1,
                    timestamp: new Date(nowTime - MS_IN_MINUTE * 8),
                    chainId: 1337,
                });

                const issues = await checkWorkerHeartbeatsAsync([heartbeat1, heartbeat2], now);
                const failedIssues = issues.filter(({ status }) => status === HealthCheckStatus.Failed);
                expect(failedIssues.length).toEqual(0);
                const degradedIssues = issues.filter(({ status }) => status === HealthCheckStatus.Degraded);
                expect(degradedIssues.length).toEqual(1);
                expect(degradedIssues[0].description).toContain('0x01');
            });
        });

        describe('Worker balance', () => {
            it('creates a failed issue when no worker has a balance above the failed threshold', async () => {
                const now = new Date();
                const heartbeat = new RfqmWorkerHeartbeatEntity({
                    address: '0x00',
                    balance: new BigNumber(0.01),
                    index: 0,
                    timestamp: now,
                    chainId: 1337,
                });

                const issues = await checkWorkerHeartbeatsAsync([heartbeat], now);
                const failedIssues = issues.filter(({ status }) => status === HealthCheckStatus.Failed);

                expect(failedIssues.length).toEqual(1);
            });

            it('creates degraded issues for low worker balances', async () => {
                const now = new Date();
                const heartbeat1 = new RfqmWorkerHeartbeatEntity({
                    address: '0x00',
                    balance: new BigNumber(0.01),
                    index: 0,
                    timestamp: now,
                    chainId: 1337,
                });
                const heartbeat2 = new RfqmWorkerHeartbeatEntity({
                    address: '0x01',
                    balance: fullBalance,
                    index: 1,
                    timestamp: now,
                    chainId: 1337,
                });

                const issues = await checkWorkerHeartbeatsAsync([heartbeat1, heartbeat2], now);
                const failedIssues = issues.filter(({ status }) => status === HealthCheckStatus.Failed);
                expect(failedIssues.length).toEqual(0);

                const degradedIssues = issues.filter(({ status }) => status === HealthCheckStatus.Degraded);

                expect(degradedIssues.length).toEqual(1);
                expect(degradedIssues[0].description).toContain(
                    'Less than two workers have a balance above the degraded threshold',
                );
            });
        });
    });

    describe('getHttpIssues', () => {
        it('goes into maintainence mode', async () => {
            const issues = getHttpIssues(/* isMaintainenceMode */ true);

            expect(issues[0].status).toEqual(HealthCheckStatus.Maintenance);
        });
    });
});