public-pool/test/timescale-redis.integration-spec.ts

import { ConfigService } from '@nestjs/config';
import { DataSource } from 'typeorm';

import { AcceptedShareEntity } from '../src/ORM/accepted-share/accepted-share.entity';
import { UserAgentReportService } from '../src/ORM/_views/user-agent-report/user-agent-report.service';
import { UserAgentReportView } from '../src/ORM/_views/user-agent-report/user-agent-report.view';
import { ClientStatisticsService } from '../src/ORM/client-statistics/client-statistics.service';
import { ClientEntity } from '../src/ORM/client/client.entity';
import { ShareAccountingService } from '../src/ORM/share-accounting/share-accounting.service';
import { RedisMessagingService } from '../src/services/redis-messaging.service';

describe('TimescaleDB and Redis integration', () => {
  let dataSource: DataSource;
  let redisMessagingService: RedisMessagingService;

  beforeAll(async () => {
    process.env.DB_HOST ??= '127.0.0.1';
    process.env.DB_PORT ??= '15432';
    process.env.DB_USERNAME ??= 'public_pool';
    process.env.DB_PASSWORD ??= 'public_pool';
    process.env.DB_DATABASE ??= 'public_pool_test';
    process.env.REDIS_URL ??= 'redis://127.0.0.1:16379';

    const imported = await import('../src/data-source');
    dataSource = imported.AppDataSource;
    if (!dataSource.isInitialized) {
      await dataSource.initialize();
    }
    await dataSource.runMigrations();

    redisMessagingService = new RedisMessagingService({
      get: jest.fn((key: string) => key === 'REDIS_URL' ? process.env.REDIS_URL : null),
    } as unknown as ConfigService);
    await redisMessagingService.connect();
  });

  afterAll(async () => {
    await redisMessagingService?.onModuleDestroy();
    if (dataSource?.isInitialized) {
      await dataSource.destroy();
    }
  });

  beforeEach(async () => {
    await dataSource.query(`DELETE FROM share_rollup_batch_summary`);
    await dataSource.query(`DELETE FROM share_rollup_batch`);
    await dataSource.query(`DELETE FROM accepted_share_entity`);
    await dataSource.query(`DELETE FROM client_entity`);
    await dataSource.query(`REFRESH MATERIALIZED VIEW user_agent_report_view`);
    await redisMessagingService.clearClientPresence();
    await (redisMessagingService as any).publisher.del('json-cache:presence:user-agent-report');
  });

  it('should create Timescale extension, hypertable, continuous aggregates, and operational policies', async () => {
    const extensions = await dataSource.query(`SELECT extname FROM pg_extension WHERE extname = 'timescaledb'`);
    expect(extensions).toHaveLength(1);

    const hypertables = await dataSource.query(`
      SELECT hypertable_name, compression_enabled
      FROM timescaledb_information.hypertables
      WHERE hypertable_name = 'accepted_share_entity'
    `);
    expect(hypertables).toHaveLength(1);
    expect(hypertables[0].compression_enabled).toBe(true);

    const aggregates = await dataSource.query(`
      SELECT view_name
      FROM timescaledb_information.continuous_aggregates
      WHERE view_name IN ('accepted_share_10m', 'accepted_share_1h', 'accepted_share_1d', 'accepted_share_block_10m')
      ORDER BY view_name
    `);
    expect(aggregates.map(row => row.view_name)).toEqual([
      'accepted_share_10m',
      'accepted_share_1d',
      'accepted_share_1h',
      'accepted_share_block_10m',
    ]);

    const legacyTables = await dataSource.query(`
      SELECT table_name
      FROM information_schema.tables
      WHERE table_schema = 'public'
        AND table_name IN ('client_statistics_entity', 'home_graph_entity')
    `);
    expect(legacyTables).toHaveLength(0);

    const policies = await dataSource.query(`
      SELECT proc_name, schedule_interval::text AS schedule_interval
      FROM timescaledb_information.jobs
      WHERE hypertable_name = 'accepted_share_entity'
         OR application_name LIKE 'Refresh Continuous Aggregate Policy%'
      ORDER BY proc_name, schedule_interval
    `);
    expect(policies).toEqual(expect.arrayContaining([
      expect.objectContaining({ proc_name: 'policy_compression' }),
      expect.objectContaining({ proc_name: 'policy_retention' }),
      expect.objectContaining({ proc_name: 'policy_refresh_continuous_aggregate' }),
    ]));

    const shareOrderObjects = await dataSource.query(`
      SELECT to_regclass('public.accepted_share_index_seq') AS sequence_name,
        to_regclass('public."IDX_accepted_share_order"') AS index_name
    `);
    expect(shareOrderObjects[0]).toEqual({
      sequence_name: 'accepted_share_index_seq',
      index_name: '"IDX_accepted_share_order"',
    });

    const shareRollupObjects = await dataSource.query(`
      SELECT
        to_regclass('public.share_rollup_batch') AS batch_table,
        to_regclass('public.share_rollup_batch_summary') AS summary_table,
        to_regclass('public."IDX_share_rollup_batch_finalized_end"') AS finalized_index
    `);
    expect(shareRollupObjects[0]).toEqual({
      batch_table: 'share_rollup_batch',
      summary_table: 'share_rollup_batch_summary',
      finalized_index: '"IDX_share_rollup_batch_finalized_end"',
    });
  });

  it('should persist accepted shares and refresh the 10 minute aggregate', async () => {
    const client = await dataSource.getRepository(ClientEntity).save({
      address: 'tb1qumezefzdeqqwn5zfvgdrhxjzc5ylr39uhuxcz4',
      clientName: 'worker',
      sessionId: '57a6f098',
      userAgent: 'integration-test',
      startTime: new Date(),
      bestDifficulty: 0,
      hashRate: 0,
    });
    const service = new ShareAccountingService(dataSource.getRepository(AcceptedShareEntity));
    const acceptedAt = new Date('2026-06-07T12:10:00Z');

    await service.recordAcceptedShare({
      protocol: 'sv1',
      acceptedAt,
      address: client.address,
      clientName: client.clientName,
      sessionId: client.sessionId,
      clientId: client.id,
      jobId: '1',
      jobTemplateId: '1',
      blockHeight: 900000,
      creditedDifficulty: 64,
      submissionDifficulty: 128,
      networkDifficulty: 100000,
      nonce: 'ed460d91',
      ntime: '64b3f3ec',
      version: '20000000',
      extraNonce2: 'c708000000000000',
      isBlockCandidate: false,
      blockSubmissionResult: null,
    });
    await service.recordAcceptedShare({
      protocol: 'sv1',
      acceptedAt: new Date(acceptedAt.getTime() + 1),
      address: client.address,
      clientName: client.clientName,
      sessionId: client.sessionId,
      clientId: client.id,
      jobId: '2',
      jobTemplateId: '1',
      blockHeight: 900000,
      creditedDifficulty: 32,
      submissionDifficulty: 64,
      networkDifficulty: 100000,
      nonce: 'ed460d92',
      ntime: '64b3f3ec',
      version: '20000000',
      extraNonce2: 'c708000000000001',
      isBlockCandidate: false,
      blockSubmissionResult: null,
    });

    const rows = await dataSource.query(`
      SELECT COUNT(*)::int AS count, MIN("shareIndex")::bigint AS first, MAX("shareIndex")::bigint AS last
      FROM accepted_share_entity
    `);
    expect(rows[0].count).toBeGreaterThanOrEqual(1);
    expect(Number(rows[0].last)).toBeGreaterThan(Number(rows[0].first));

    await dataSource.query(`CALL refresh_continuous_aggregate('accepted_share_10m', NULL, NULL)`);
    await dataSource.query(`CALL refresh_continuous_aggregate('accepted_share_block_10m', NULL, NULL)`);
    const aggregateRows = await dataSource.query(`
      SELECT "shares"::float AS shares, "acceptedCount"::int AS "acceptedCount"
      FROM accepted_share_10m
      WHERE "address" = $1 AND "clientName" = $2
    `, [client.address, client.clientName]);

    expect(aggregateRows).toEqual(expect.arrayContaining([
      expect.objectContaining({ shares: 96, acceptedCount: 2 }),
    ]));

    const blockAggregateRows = await dataSource.query(`
      SELECT
        "shares"::float AS shares,
        "acceptedCount"::int AS "acceptedCount",
        "networkDifficulty"::float AS "networkDifficulty"
      FROM accepted_share_block_10m
      WHERE "blockHeight" = $1
    `, [900000]);

    expect(blockAggregateRows).toEqual(expect.arrayContaining([
      expect.objectContaining({ shares: 96, acceptedCount: 2, networkDifficulty: 100000 }),
    ]));
  });

  it('should finalize accepted shares into share rollup batches without mutating raw rows', async () => {
    const client = await dataSource.getRepository(ClientEntity).save({
      address: 'tb1qumezefzdeqqwn5zfvgdrhxjzc5ylr39uhuxcz4',
      clientName: 'rollup-worker',
      sessionId: '47a6f098',
      userAgent: 'integration-test',
      startTime: new Date(),
      bestDifficulty: 0,
      hashRate: 0,
    });
    const service = new ShareAccountingService(dataSource.getRepository(AcceptedShareEntity));
    const acceptedAt = new Date('2026-06-07T12:20:00Z');

    await service.recordAcceptedShare({
      protocol: 'sv1',
      acceptedAt,
      address: client.address,
      clientName: client.clientName,
      sessionId: client.sessionId,
      clientId: client.id,
      jobId: 'rollup-1',
      jobTemplateId: 'rollup-template',
      blockHeight: 900010,
      creditedDifficulty: 64,
      submissionDifficulty: 128,
      networkDifficulty: 100000,
      nonce: 'rollup-nonce-1',
      ntime: '64b3f3ec',
      version: '20000000',
      extraNonce2: 'c708000000000001',
      isBlockCandidate: false,
      blockSubmissionResult: null,
    });
    await service.recordAcceptedShare({
      protocol: 'sv2',
      acceptedAt: new Date(acceptedAt.getTime() + 1),
      address: client.address,
      clientName: client.clientName,
      sessionId: client.sessionId,
      clientId: client.id,
      jobId: 'rollup-2',
      jobTemplateId: 'rollup-template',
      blockHeight: 900010,
      creditedDifficulty: 32,
      submissionDifficulty: 64,
      networkDifficulty: 100000,
      nonce: 'rollup-nonce-2',
      ntime: '64b3f3ec',
      version: '20000000',
      extraNonce2: 'c708000000000002',
      isBlockCandidate: false,
      blockSubmissionResult: null,
    });

    await expect(service.processPendingShareRollupBatch()).resolves.toEqual(expect.objectContaining({
      processed: true,
      acceptedShareCount: 2,
      creditedDifficulty: 96,
    }));
    await expect(service.processPendingShareRollupBatch()).resolves.toEqual({
      processed: false,
      reason: 'no-shares',
    });

    const batches = await dataSource.query(`
      SELECT "acceptedShareCount"::int AS "acceptedShareCount", "creditedDifficulty"::float AS "creditedDifficulty"
      FROM share_rollup_batch
    `);
    expect(batches).toEqual([{
      acceptedShareCount: 2,
      creditedDifficulty: 96,
    }]);

    const summaries = await dataSource.query(`
      SELECT
        "protocol",
        "blockHeight",
        "acceptedShareCount"::int AS "acceptedShareCount",
        "creditedDifficulty"::float AS "creditedDifficulty",
        "bestSubmissionDifficulty"::float AS "bestSubmissionDifficulty"
      FROM share_rollup_batch_summary
      WHERE "address" = $1 AND "clientName" = $2
      ORDER BY "protocol"
    `, [client.address, client.clientName]);
    expect(summaries).toEqual([
      { protocol: 'sv1', blockHeight: 900010, acceptedShareCount: 1, creditedDifficulty: 64, bestSubmissionDifficulty: 128 },
      { protocol: 'sv2', blockHeight: 900010, acceptedShareCount: 1, creditedDifficulty: 32, bestSubmissionDifficulty: 64 },
    ]);

    const rawRows = await dataSource.query(`SELECT COUNT(*)::int AS count FROM accepted_share_entity`);
    expect(rawRows[0].count).toBe(2);
  });

  it('should exclude soft-deleted clients from the user-agent report', async () => {
    const userAgent = `integration-reconnect-${Date.now()}`;
    const repository = dataSource.getRepository(ClientEntity);

    const activeClient = await repository.save({
      address: 'tb1qumezefzdeqqwn5zfvgdrhxjzc5ylr39uhuxcz4',
      clientName: 'active-worker',
      sessionId: 'a1b2c3d4',
      userAgent,
      startTime: new Date(),
      bestDifficulty: 10,
      hashRate: 100,
    });
    const staleClient = await repository.save({
      address: 'tb1qumezefzdeqqwn5zfvgdrhxjzc5ylr39uhuxcz4',
      clientName: 'stale-worker',
      sessionId: 'd4c3b2a1',
      userAgent,
      startTime: new Date(),
      bestDifficulty: 20,
      hashRate: 200,
    });
    await repository.softDelete(staleClient.id);

    await dataSource.query(`REFRESH MATERIALIZED VIEW user_agent_report_view`);
    const rows = await dataSource.query(`
      SELECT "count"::int AS "count", "bestDifficulty"::float AS "bestDifficulty", "totalHashRate"::float AS "totalHashRate"
      FROM user_agent_report_view
      WHERE "userAgent" = $1
    `, [userAgent]);

    expect(rows).toEqual([{
      count: 1,
      bestDifficulty: Number(activeClient.bestDifficulty),
      totalHashRate: Number(activeClient.hashRate),
    }]);
  });

  it('should serve live active user-agent reports without waiting for materialized refresh', async () => {
    const userAgent = `integration-live-reconnect-${Date.now()}`;
    const repository = dataSource.getRepository(ClientEntity);
    const reportService = new UserAgentReportService(
      dataSource.getRepository(UserAgentReportView),
      repository,
      redisMessagingService,
    );

    const activeClient = await repository.save({
      address: 'tb1qumezefzdeqqwn5zfvgdrhxjzc5ylr39uhuxcz4',
      clientName: 'active-live-worker',
      sessionId: '91b2c3d4',
      userAgent,
      startTime: new Date(),
      bestDifficulty: 11,
      hashRate: 0,
    });
    await repository.save({
      address: 'tb1qumezefzdeqqwn5zfvgdrhxjzc5ylr39uhuxcz4',
      clientName: 'stale-live-worker',
      sessionId: '92b2c3d4',
      userAgent,
      startTime: new Date(),
      bestDifficulty: 22,
      hashRate: 200,
    });

    await redisMessagingService.setClientPresence({
      clientId: activeClient.id,
      address: activeClient.address,
      clientName: activeClient.clientName,
      sessionId: activeClient.sessionId,
      userAgent,
      startTime: activeClient.startTime.toISOString(),
      lastSeen: new Date().toISOString(),
      bestDifficulty: Number(activeClient.bestDifficulty),
      hashRate: 0,
    });

    const rows = await reportService.getReport();
    expect(rows).toEqual(expect.arrayContaining([{
      userAgent,
      count: '1',
      bestDifficulty: Number(activeClient.bestDifficulty),
      totalHashRate: '0',
    }]));
  });

  it('should serve completed chart buckets and omit the current in-progress bucket', async () => {
    const client = await dataSource.getRepository(ClientEntity).save({
      address: 'tb1qumezefzdeqqwn5zfvgdrhxjzc5ylr39uhuxcz4',
      clientName: 'worker',
      sessionId: '8f3c5d2a',
      userAgent: 'integration-test',
      startTime: new Date(),
      bestDifficulty: 0,
      hashRate: 0,
    });
    const accountingService = new ShareAccountingService(dataSource.getRepository(AcceptedShareEntity));
    const statisticsService = new ClientStatisticsService(dataSource);
    const currentBucketStart = Math.floor(Date.now() / (10 * 60 * 1000)) * 10 * 60 * 1000;
    const completedBucketStart = currentBucketStart - (10 * 60 * 1000);
    const completedBucketEnd = currentBucketStart;
    const acceptedAt = new Date(completedBucketStart + 1000);

    for (const [index, difficulty] of [64, 32].entries()) {
      await accountingService.recordAcceptedShare({
        protocol: 'sv1',
        acceptedAt: new Date(acceptedAt.getTime() + index),
        address: client.address,
        clientName: client.clientName,
        sessionId: client.sessionId,
        clientId: client.id,
        jobId: `realtime-${index}`,
        jobTemplateId: 'realtime-template',
        blockHeight: 900001,
        creditedDifficulty: difficulty,
        submissionDifficulty: difficulty * 2,
        networkDifficulty: 100000,
        nonce: `nonce-${index}`,
        ntime: '64b3f3ec',
        version: '20000000',
        extraNonce2: 'c708000000000000',
        isBlockCandidate: false,
        blockSubmissionResult: null,
      });
    }
    await accountingService.recordAcceptedShare({
      protocol: 'sv1',
      acceptedAt: new Date(currentBucketStart + 1000),
      address: client.address,
      clientName: client.clientName,
      sessionId: client.sessionId,
      clientId: client.id,
      jobId: 'current-bucket',
      jobTemplateId: 'realtime-template',
      blockHeight: 900001,
      creditedDifficulty: 1024,
      submissionDifficulty: 2048,
      networkDifficulty: 100000,
      nonce: 'nonce-current',
      ntime: '64b3f3ec',
      version: '20000000',
      extraNonce2: 'c708000000000000',
      isBlockCandidate: false,
      blockSubmissionResult: null,
    });
    await dataSource.query(
      `CALL refresh_continuous_aggregate('accepted_share_10m', $1::timestamptz, $2::timestamptz)`,
      [new Date(completedBucketStart), new Date(completedBucketEnd)],
    );

    const expectedHashRate = (96 * 4294967296) / 600;
    const expectedChartData = Math.round(expectedHashRate).toString();
    const expectedLiveHashRate = ((96 + 1024) * 4294967296) / 600;
    expect(await statisticsService.getHashRateForGroup(client.address, client.clientName))
      .toBeCloseTo(expectedLiveHashRate);

    const addressChart = await statisticsService.getChartDataForAddress(client.address);
    const groupChart = await statisticsService.getChartDataForGroup(client.address, client.clientName);
    const sessionChart = await statisticsService.getChartDataForSession(client.id);
    const siteChart = await statisticsService.getChartDataForSite();

    for (const chart of [addressChart, groupChart, sessionChart, siteChart]) {
      expect(chart).toEqual(expect.arrayContaining([
        expect.objectContaining({ data: expectedChartData }),
      ]));
      expect(chart).not.toEqual(expect.arrayContaining([
        expect.objectContaining({ label: new Date(currentBucketStart).toISOString() }),
      ]));
    }
  });

  it('should replay mining info and block templates through Redis', async () => {
    await redisMessagingService.setLatestMiningInfo({ blocks: 900001 } as any);
    await redisMessagingService.setBlockTemplate(900001, { height: 900001, transactions: [] } as any);

    expect(await redisMessagingService.getLatestMiningInfo()).toEqual({ blocks: 900001 });
    expect(await redisMessagingService.getBlockTemplate(900001)).toEqual({ height: 900001, transactions: [] });

    const received: any[] = [];
    await redisMessagingService.subscribeMiningInfoUpdates(async miningInfo => {
      received.push(miningInfo);
    });
    await redisMessagingService.publishMiningInfoUpdate({ blocks: 900002 } as any);
    await new Promise(resolve => setTimeout(resolve, 100));

    expect(received).toEqual([{ blocks: 900002 }]);
  });
});