Speed up template publication and share validation

src/ORM/payout-snapshot/payout-snapshot.entity.ts

@@ -17,7 +17,7 @@ export class PayoutSnapshotEntity {
     payoutMode: PayoutMode;
 
     @Column({ length: 16, type: 'varchar', default: 'finalized' })
-    status: 'finalized';
+    status: 'finalized' | 'bridge_seed';
 
     @Column({ type: 'bigint' })
     blockHeight: number;

src/ORM/payout-snapshot/payout-snapshot.service.spec.ts

@@ -1,5 +1,6 @@
 import { DataSource, EntityManager } from 'typeorm';
 
+import type { PayoutDistributionEntry } from './payout-distribution';
 import { PayoutSnapshotService } from './payout-snapshot.service';
 
 const ADDRESS_A = 'bc1qs29kyaqqc0fkvj897ke9e5xa9utljjey0y5jjn';
@@ -52,7 +53,6 @@ describe('PayoutSnapshotService', () => {
             .mockResolvedValueOnce([])
             .mockResolvedValueOnce([{ id: '55' }])
             .mockResolvedValueOnce([])
-            .mockResolvedValueOnce([])
             .mockResolvedValueOnce([{
                 id: '55',
                 method: 'pplns',
@@ -91,8 +91,106 @@ describe('PayoutSnapshotService', () => {
             { address: ADDRESS_A, amountSats: 600, percent: 60 },
             { address: ADDRESS_B, amountSats: 400, percent: 40 },
         ]);
-        expect(manager.query).toHaveBeenCalledTimes(9);
+        expect(manager.query).toHaveBeenCalledTimes(8);
         expect(manager.query.mock.calls[0][1]).toEqual([100, 'pplns']);
+
+        const entryInsertCalls = manager.query.mock.calls.filter(([sql]) => (
+            sql.includes('INSERT INTO "payout_snapshot_entry"')
+        ));
+        expect(entryInsertCalls).toHaveLength(1);
+        expect(entryInsertCalls[0][0]).toContain('FROM UNNEST(');
+        expect(entryInsertCalls[0][0]).toContain('WITH ORDINALITY');
+        expect(entryInsertCalls[0][0]).toContain('ORDER BY entry."ordinality"');
+        expect(entryInsertCalls[0][1]).toEqual([
+            '55',
+            'pplns',
+            [ADDRESS_A, ADDRESS_B],
+            [60, 40],
+            [3, 2],
+            [0.6, 0.4],
+            [600, 400],
+            [600, 400],
+            [0, 0],
+            [0, 0],
+            [true, true],
+            ['coinbase', 'coinbase'],
+            [1, 2],
+        ]);
+    });
+
+    it('should bound and chunk set-based snapshot entry inserts without changing values', async () => {
+        const entries: PayoutDistributionEntry[] = Array.from({ length: 1_001 }, (_, index) => {
+            const includedInCoinbase = index % 3 !== 0;
+            const balanceBeforeSats = 200 + index;
+            const grossPayoutSats = 10_000 + index;
+            return {
+                address: `test-address-${index}`,
+                creditedDifficulty: index + 0.5,
+                acceptedShareCount: index + 1,
+                payoutWeight: (index + 1) / 1_001,
+                grossPayoutSats,
+                payoutSats: includedInCoinbase ? 1_000 + index : 0,
+                balanceBeforeSats,
+                balanceAfterSats: includedInCoinbase
+                    ? balanceBeforeSats
+                    : balanceBeforeSats + grossPayoutSats,
+                includedInCoinbase,
+                rank: 1_001 - index,
+            };
+        });
+        const insertSnapshotEntries = Reflect.get(service, 'insertSnapshotEntries') as (
+            manager: EntityManager,
+            snapshotId: string,
+            entries: PayoutDistributionEntry[],
+        ) => Promise<void>;
+        let resolveFirstInsert: () => void = () => {
+            throw new Error('First insert did not start');
+        };
+        manager.query
+            .mockImplementationOnce(() => new Promise<void>(resolve => {
+                resolveFirstInsert = () => resolve();
+            }))
+            .mockResolvedValueOnce([]);
+
+        const insertion = insertSnapshotEntries.call(
+            service,
+            manager as unknown as EntityManager,
+            '77',
+            entries,
+        );
+
+        expect(manager.query).toHaveBeenCalledTimes(1);
+        resolveFirstInsert();
+        await insertion;
+        expect(manager.query).toHaveBeenCalledTimes(2);
+        const calls = manager.query.mock.calls;
+        expect(calls.map(([, parameters]) => parameters.length)).toEqual([13, 13]);
+        expect(calls.map(([, parameters]) => parameters[2].length)).toEqual([1_000, 1]);
+        for (const [sql, parameters] of calls) {
+            expect(sql).toContain('FROM UNNEST(');
+            expect(sql).toContain('ORDER BY entry."ordinality"');
+            for (const values of parameters.slice(2)) {
+                expect(values.length).toBeLessThanOrEqual(1_000);
+            }
+        }
+
+        const expectedParameters = (batch: PayoutDistributionEntry[]) => [
+            '77',
+            'pplns',
+            batch.map(entry => entry.address),
+            batch.map(entry => entry.creditedDifficulty),
+            batch.map(entry => entry.acceptedShareCount),
+            batch.map(entry => entry.payoutWeight),
+            batch.map(entry => entry.grossPayoutSats),
+            batch.map(entry => entry.payoutSats),
+            batch.map(entry => entry.balanceBeforeSats),
+            batch.map(entry => entry.balanceAfterSats),
+            batch.map(entry => entry.includedInCoinbase),
+            batch.map(entry => entry.includedInCoinbase ? 'coinbase' : 'pending'),
+            batch.map(entry => entry.rank),
+        ];
+        expect(calls[0][1]).toEqual(expectedParameters(entries.slice(0, 1_000)));
+        expect(calls[1][1]).toEqual(expectedParameters(entries.slice(1_000)));
     });
 
     it('should bootstrap the PPLNS window from paid block count up to the configured factor', async () => {
@@ -116,7 +214,6 @@ describe('PayoutSnapshotService', () => {
             .mockResolvedValueOnce([])
             .mockResolvedValueOnce([{ id: '56' }])
             .mockResolvedValueOnce([])
-            .mockResolvedValueOnce([])
             .mockResolvedValueOnce([{
                 id: '56',
                 method: 'pplns',
@@ -213,6 +310,56 @@ describe('PayoutSnapshotService', () => {
         expect(manager.query).toHaveBeenCalledTimes(4);
     });
 
+    it('stores bridge seeds with a non-active status excluded from latest payout queries', async () => {
+        manager.query
+            .mockResolvedValueOnce([{
+                startBatchId: '10',
+                endBatchId: '12',
+                windowStartShareIndex: '1000',
+                windowEndShareIndex: '2000',
+                totalCreditedDifficulty: '100',
+                totalAcceptedShareCount: '5',
+            }])
+            .mockResolvedValueOnce([])
+            .mockResolvedValueOnce([
+                { address: ADDRESS_A, creditedDifficulty: 100, acceptedShareCount: 5 },
+            ])
+            .mockResolvedValueOnce([])
+            .mockResolvedValueOnce([{ id: 'seed-57' }])
+            .mockResolvedValueOnce([])
+            .mockResolvedValueOnce([{
+                id: 'seed-57',
+                method: 'pplns',
+                payoutMode: 'pplns',
+                blockHeight: 900002,
+                coinbaseValueSats: '1000',
+                windowStartShareIndex: '1000',
+                windowEndShareIndex: '2000',
+                totalCreditedDifficulty: 100,
+                totalAcceptedShareCount: '5',
+                eligibleAddressCount: 1,
+                includedOutputCount: 1,
+                distributedSats: '1000',
+                unallocatedRemainderSats: '0',
+            }])
+            .mockResolvedValueOnce([{ address: ADDRESS_A, payoutSats: '1000' }]);
+
+        await expect(service.createSnapshotForTemplate({
+            blockHeight: 900002,
+            coinbaseValueSats: 1000,
+            networkDifficulty: 25,
+            visibility: 'bridge_seed',
+        })).resolves.toEqual(expect.objectContaining({ id: 'seed-57' }));
+
+        const existingLookupSql = manager.query.mock.calls[1][0] as string;
+        const insertSql = manager.query.mock.calls[4][0] as string;
+        expect(existingLookupSql).toContain(`"status" = 'bridge_seed'`);
+        expect(insertSql).toContain(`'bridge_seed'`);
+        dataSource.query.mockResolvedValueOnce([]);
+        await expect(service.getLatestSnapshot()).resolves.toBeNull();
+        expect(dataSource.query.mock.calls[0][0]).toContain(`"status" = 'finalized'`);
+    });
+
     it('should finalize snapshot balances and payout history for a found block', async () => {
         manager.query
             .mockResolvedValueOnce([{ id: '55', coinbaseValueSats: '1000' }])

src/ORM/payout-snapshot/payout-snapshot.service.ts

@@ -58,6 +58,9 @@ const DEFAULT_PAYOUT_WINDOW_FACTOR = 4;
 const DEFAULT_COINBASE_WEIGHT_BUDGET = 26_000;
 const DEFAULT_PAYOUT_BOOTSTRAP_WINDOW = true;
 const PPLNS_PAYOUT_MODE: PayoutMode = 'pplns';
+// UNNEST keeps each statement at 13 bind parameters; this cap bounds array payload size.
+const PAYOUT_SNAPSHOT_ENTRY_INSERT_BATCH_SIZE = 1_000;
+type PayoutSnapshotStatus = 'finalized' | 'bridge_seed';
 
 @Injectable()
 export class PayoutSnapshotService {
@@ -79,12 +82,17 @@ export class PayoutSnapshotService {
         blockHeight: number;
         coinbaseValueSats: number;
         networkDifficulty: number;
+        /** Bridge seeds are frozen/reconstructable but excluded from active payout APIs. */
+        visibility?: 'active' | 'bridge_seed';
     }): Promise<PayoutSnapshotForTemplate | null> {
         if (!this.snapshotsEnabled || input.coinbaseValueSats <= 0 || input.networkDifficulty <= 0) {
             return null;
         }
 
         return this.dataSource.transaction(async manager => {
+            const snapshotStatus: PayoutSnapshotStatus = input.visibility === 'bridge_seed'
+                ? 'bridge_seed'
+                : 'finalized';
             const effectiveWindowFactor = await this.getEffectiveWindowFactor(manager);
             const windowTargetDifficulty = input.networkDifficulty * effectiveWindowFactor;
             const window = await this.getPplnsWindow(manager, windowTargetDifficulty);
@@ -96,6 +104,7 @@ export class PayoutSnapshotService {
                 blockHeight: input.blockHeight,
                 coinbaseValueSats: input.coinbaseValueSats,
                 windowEndShareIndex: window.windowEndShareIndex,
+                status: snapshotStatus,
             });
             if (existing != null) {
                 return existing;
@@ -153,7 +162,7 @@ export class PayoutSnapshotService {
                 ) VALUES (
                     $1,
                     $2,
-                    'finalized',
+                    '${snapshotStatus}',
                     $3::bigint,
                     $4::bigint,
                     $5::numeric,
@@ -199,9 +208,7 @@ export class PayoutSnapshotService {
                 unallocatedRemainderSats.toString(),
             ]);
 
-            for (const entry of entries) {
-                await this.insertSnapshotEntry(manager, snapshotRow.id, entry);
-            }
+            await this.insertSnapshotEntries(manager, snapshotRow.id, entries);
 
             return this.getSnapshotById(manager, snapshotRow.id);
         });
@@ -554,14 +561,19 @@ export class PayoutSnapshotService {
 
     private async getExistingSnapshot(
         manager: EntityManager,
-        input: { blockHeight: number; coinbaseValueSats: number; windowEndShareIndex: string },
+        input: {
+            blockHeight: number;
+            coinbaseValueSats: number;
+            windowEndShareIndex: string;
+            status: PayoutSnapshotStatus;
+        },
     ): Promise<PayoutSnapshotForTemplate | null> {
         const [snapshot] = await manager.query(`
             SELECT "id"::text AS "id"
             FROM "payout_snapshot"
             WHERE "method" = $1
               AND "payoutMode" = $5
-              AND "status" = 'finalized'
+              AND "status" = '${input.status}'
               AND "blockHeight" = $2::bigint
               AND "coinbaseValueSats" = $3::bigint
               AND "windowEndShareIndex" = $4::bigint
@@ -641,52 +653,86 @@ export class PayoutSnapshotService {
         };
     }
 
-    private async insertSnapshotEntry(manager: EntityManager, snapshotId: string, entry: PayoutDistributionEntry): Promise<void> {
-        await manager.query(`
-            INSERT INTO "payout_snapshot_entry" (
-                "snapshotId",
-                "payoutMode",
-                "address",
-                "creditedDifficulty",
-                "acceptedShareCount",
-                "payoutWeight",
-                "grossPayoutSats",
-                "payoutSats",
-                "balanceBeforeSats",
-                "balanceAfterSats",
-                "includedInCoinbase",
-                "rowType",
-                "rank"
-            ) VALUES (
-                $1::bigint,
-                $2,
-                $3,
-                $4::numeric,
-                $5::bigint,
-                $6::numeric,
-                $7::bigint,
-                $8::bigint,
-                $9::bigint,
-                $10::bigint,
-                $11::boolean,
-                $12,
-                $13::int
-            )
-        `, [
-            snapshotId,
-            PPLNS_PAYOUT_MODE,
-            entry.address,
-            entry.creditedDifficulty,
-            entry.acceptedShareCount,
-            entry.payoutWeight,
-            entry.grossPayoutSats,
-            entry.payoutSats,
-            entry.balanceBeforeSats,
-            entry.balanceAfterSats,
-            entry.includedInCoinbase,
-            entry.includedInCoinbase ? 'coinbase' : 'pending',
-            entry.rank,
-        ]);
+    private async insertSnapshotEntries(
+        manager: EntityManager,
+        snapshotId: string,
+        entries: PayoutDistributionEntry[],
+    ): Promise<void> {
+        for (let offset = 0; offset < entries.length; offset += PAYOUT_SNAPSHOT_ENTRY_INSERT_BATCH_SIZE) {
+            const batch = entries.slice(offset, offset + PAYOUT_SNAPSHOT_ENTRY_INSERT_BATCH_SIZE);
+            await manager.query(`
+                INSERT INTO "payout_snapshot_entry" (
+                    "snapshotId",
+                    "payoutMode",
+                    "address",
+                    "creditedDifficulty",
+                    "acceptedShareCount",
+                    "payoutWeight",
+                    "grossPayoutSats",
+                    "payoutSats",
+                    "balanceBeforeSats",
+                    "balanceAfterSats",
+                    "includedInCoinbase",
+                    "rowType",
+                    "rank"
+                )
+                SELECT
+                    $1::bigint,
+                    $2,
+                    entry."address",
+                    entry."creditedDifficulty",
+                    entry."acceptedShareCount",
+                    entry."payoutWeight",
+                    entry."grossPayoutSats",
+                    entry."payoutSats",
+                    entry."balanceBeforeSats",
+                    entry."balanceAfterSats",
+                    entry."includedInCoinbase",
+                    entry."rowType",
+                    entry."rank"
+                FROM UNNEST(
+                    $3::varchar[],
+                    $4::numeric[],
+                    $5::bigint[],
+                    $6::numeric[],
+                    $7::bigint[],
+                    $8::bigint[],
+                    $9::bigint[],
+                    $10::bigint[],
+                    $11::boolean[],
+                    $12::varchar[],
+                    $13::int[]
+                ) WITH ORDINALITY AS entry(
+                    "address",
+                    "creditedDifficulty",
+                    "acceptedShareCount",
+                    "payoutWeight",
+                    "grossPayoutSats",
+                    "payoutSats",
+                    "balanceBeforeSats",
+                    "balanceAfterSats",
+                    "includedInCoinbase",
+                    "rowType",
+                    "rank",
+                    "ordinality"
+                )
+                ORDER BY entry."ordinality"
+            `, [
+                snapshotId,
+                PPLNS_PAYOUT_MODE,
+                batch.map(entry => entry.address),
+                batch.map(entry => entry.creditedDifficulty),
+                batch.map(entry => entry.acceptedShareCount),
+                batch.map(entry => entry.payoutWeight),
+                batch.map(entry => entry.grossPayoutSats),
+                batch.map(entry => entry.payoutSats),
+                batch.map(entry => entry.balanceBeforeSats),
+                batch.map(entry => entry.balanceAfterSats),
+                batch.map(entry => entry.includedInCoinbase),
+                batch.map(entry => entry.includedInCoinbase ? 'coinbase' : 'pending'),
+                batch.map(entry => entry.rank),
+            ]);
+        }
     }
 
     private limitCoinbaseOutputs(entries: PayoutDistributionEntry[]): PayoutDistributionEntry[] {