From a33c530268825115b80ebaf6e9dcf1b80696d4d7 Mon Sep 17 00:00:00 2001 From: Ben Date: Fri, 12 Jun 2026 23:54:50 -0400 Subject: [PATCH] Add SV2 DATUM payout hardening --- .env.example | 27 + .gitignore | 2 +- docker-compose.external-db.yml | 9 + full-setup/docker-compose-mainnet.yml | 9 + package-lock.json | 62 ++ package.json | 2 + ...eptedShareProtocolMetadata1781130600000.ts | 74 ++ .../BlocksSubmissionMetadata1781220000000.ts | 27 + .../InitialTimescaleSchema1780859300000.ts | 2 + .../PayoutSnapshots1780969800000.ts | 143 ++++ .../accepted-share/accepted-share.entity.ts | 10 +- src/ORM/blocks/blocks.entity.ts | 9 + src/ORM/blocks/blocks.service.spec.ts | 87 +++ src/ORM/blocks/blocks.service.ts | 96 ++- .../payout-snapshot/payout-balance.entity.ts | 24 + .../payout-distribution.spec.ts | 135 ++++ .../payout-snapshot/payout-distribution.ts | 276 +++++++ .../payout-snapshot/payout-history.entity.ts | 41 + .../payout-snapshot-entry.entity.ts | 54 ++ .../payout-snapshot/payout-snapshot.entity.ts | 80 ++ .../payout-snapshot/payout-snapshot.module.ts | 21 + .../payout-snapshot.service.spec.ts | 213 ++++++ .../payout-snapshot.service.ts | 617 +++++++++++++++ .../share-accounting.service.spec.ts | 20 + .../share-accounting.service.ts | 6 +- src/app.module.ts | 16 +- src/database.config.ts | 14 + src/models/MiningJob.spec.ts | 19 + src/models/MiningJob.ts | 9 +- src/models/StratumV1Client.spec.ts | 1 + src/models/StratumV1Client.ts | 36 +- src/models/StratumV2Client.spec.ts | 117 ++- src/models/StratumV2Client.ts | 202 ++++- src/models/bitcoin-rpc/IBlockTemplate.ts | 11 +- src/models/datum/datum-codec.spec.ts | 257 +++++++ src/models/datum/datum-codec.ts | 506 ++++++++++++ src/models/datum/datum-crypto.spec.ts | 94 +++ src/models/datum/datum-crypto.ts | 245 ++++++ src/models/stratum.constants.ts | 2 + src/models/sv2/sv2-extranonce-manager.spec.ts | 3 + src/models/sv2/sv2-extranonce-manager.ts | 18 +- src/services/bitcoin-rpc.service.ts | 74 +- src/services/custom-work.service.spec.ts | 94 +++ src/services/custom-work.service.ts | 161 ++++ src/services/datum.service.spec.ts | 156 ++++ src/services/datum.service.ts | 722 ++++++++++++++++++ src/services/stratum-v1-jobs.service.spec.ts | 18 + src/services/stratum-v1-jobs.service.ts | 33 +- src/services/stratum-v1.service.ts | 14 +- src/services/stratum-v2.service.ts | 13 + ...2-job-declaration-registry.service.spec.ts | 58 ++ .../sv2-job-declaration-registry.service.ts | 161 ++++ .../sv2-job-declaration.service.spec.ts | 503 ++++++++++++ src/services/sv2-job-declaration.service.ts | 473 ++++++++++++ .../sv2-template-distribution.service.spec.ts | 290 +++++++ .../sv2-template-distribution.service.ts | 444 +++++++++++ .../template-provider.service.spec.ts | 331 ++++++++ src/services/template-provider.service.ts | 656 ++++++++++++++++ test/timescale-redis.integration-spec.ts | 136 ++++ 59 files changed, 7852 insertions(+), 81 deletions(-) create mode 100644 src/ORM/_migrations/AcceptedShareProtocolMetadata1781130600000.ts create mode 100644 src/ORM/_migrations/BlocksSubmissionMetadata1781220000000.ts create mode 100644 src/ORM/_migrations/PayoutSnapshots1780969800000.ts create mode 100644 src/ORM/blocks/blocks.service.spec.ts create mode 100644 src/ORM/payout-snapshot/payout-balance.entity.ts create mode 100644 src/ORM/payout-snapshot/payout-distribution.spec.ts create mode 100644 src/ORM/payout-snapshot/payout-distribution.ts create mode 100644 src/ORM/payout-snapshot/payout-history.entity.ts create mode 100644 src/ORM/payout-snapshot/payout-snapshot-entry.entity.ts create mode 100644 src/ORM/payout-snapshot/payout-snapshot.entity.ts create mode 100644 src/ORM/payout-snapshot/payout-snapshot.module.ts create mode 100644 src/ORM/payout-snapshot/payout-snapshot.service.spec.ts create mode 100644 src/ORM/payout-snapshot/payout-snapshot.service.ts create mode 100644 src/models/datum/datum-codec.spec.ts create mode 100644 src/models/datum/datum-codec.ts create mode 100644 src/models/datum/datum-crypto.spec.ts create mode 100644 src/models/datum/datum-crypto.ts create mode 100644 src/services/custom-work.service.spec.ts create mode 100644 src/services/custom-work.service.ts create mode 100644 src/services/datum.service.spec.ts create mode 100644 src/services/datum.service.ts create mode 100644 src/services/sv2-job-declaration-registry.service.spec.ts create mode 100644 src/services/sv2-job-declaration-registry.service.ts create mode 100644 src/services/sv2-job-declaration.service.spec.ts create mode 100644 src/services/sv2-job-declaration.service.ts create mode 100644 src/services/sv2-template-distribution.service.spec.ts create mode 100644 src/services/sv2-template-distribution.service.ts create mode 100644 src/services/template-provider.service.spec.ts create mode 100644 src/services/template-provider.service.ts diff --git a/.env.example b/.env.example index 60a8abf..e12c751 100644 --- a/.env.example +++ b/.env.example @@ -40,6 +40,20 @@ STRATUM_TCP_KEEPALIVE_INITIAL_DELAY_MS=60000 #SV2_DIFFICULTY_CHECK_INTERVAL_MS=60000 #STRATUM_V2_SOCKET_TIMEOUT_MS=3600000 #SV2_AUTHORITY_PRIVKEY= +#SV2_JDP_PORTS= +#SV2_JDP_POOL_PAYOUT_ADDRESS= +#SV2_JDP_TOKEN_TTL_MS=600000 +#SV2_TDP_PORTS= +#SV2_TDP_POOL_PAYOUT_ADDRESS= + +# Optional DATUM Prime-compatible listener. DATUM_POOL_PAYOUT_ADDRESS is the +# required pool coinbase output DATUM gateways must include in custom templates. +#DATUM_PORTS= +#DATUM_POOL_PAYOUT_ADDRESS= +#DATUM_SHARE_DIFFICULTY=1 +# 32-byte hex seed for the DATUM server identity key. DATUM clients must be +# configured with the matching logged public key. +#DATUM_IDENTITY_SEED= STRATUM_SECURE=true SECURE_STRATUM_PORTS=4333,4332,4331,4330 @@ -82,6 +96,19 @@ SHARE_ROLLUP_INTERVAL_MS=60000 SHARE_ROLLUP_SAFETY_LAG_SECONDS=30 SHARE_ROLLUP_MAX_SHARES_PER_BATCH=5000000 +# payout snapshot accounting +PAYOUT_SNAPSHOT_ENABLED=true +PAYOUT_METHOD=pplns +PAYOUT_WINDOW_FACTOR=4 +# solo preserves current one-address coinbase payouts. Set to snapshot to pay +# the latest payout snapshot directly in coinbase outputs. +PAYOUT_COINBASE_MODE=solo +PAYOUT_MAX_COINBASE_OUTPUTS=10 +PAYOUT_MIN_OUTPUT_SATS=546 +PAYOUT_FEE_ADDRESS= +PAYOUT_FEE_PERCENT=0 +PAYOUT_COINBASE_WEIGHT_BUDGET=50000 + #redis REDIS_URL=redis://redis:6379 diff --git a/.gitignore b/.gitignore index 6985701..e47fed6 100644 --- a/.gitignore +++ b/.gitignore @@ -42,7 +42,7 @@ settings.json #DB **.sqlite** -./full-setup/data/* +/full-setup/data/ #https **.pem diff --git a/docker-compose.external-db.yml b/docker-compose.external-db.yml index d91d88a..cd0c038 100644 --- a/docker-compose.external-db.yml +++ b/docker-compose.external-db.yml @@ -80,6 +80,15 @@ services: SHARE_ROLLUP_INTERVAL_MS: ${SHARE_ROLLUP_INTERVAL_MS:-60000} SHARE_ROLLUP_SAFETY_LAG_SECONDS: ${SHARE_ROLLUP_SAFETY_LAG_SECONDS:-30} SHARE_ROLLUP_MAX_SHARES_PER_BATCH: ${SHARE_ROLLUP_MAX_SHARES_PER_BATCH:-5000000} + PAYOUT_SNAPSHOT_ENABLED: ${PAYOUT_SNAPSHOT_ENABLED:-true} + PAYOUT_METHOD: ${PAYOUT_METHOD:-pplns} + PAYOUT_WINDOW_FACTOR: ${PAYOUT_WINDOW_FACTOR:-4} + PAYOUT_COINBASE_MODE: ${PAYOUT_COINBASE_MODE:-solo} + PAYOUT_MAX_COINBASE_OUTPUTS: ${PAYOUT_MAX_COINBASE_OUTPUTS:-10} + PAYOUT_MIN_OUTPUT_SATS: ${PAYOUT_MIN_OUTPUT_SATS:-546} + PAYOUT_FEE_ADDRESS: ${PAYOUT_FEE_ADDRESS:-} + PAYOUT_FEE_PERCENT: ${PAYOUT_FEE_PERCENT:-0} + PAYOUT_COINBASE_WEIGHT_BUDGET: ${PAYOUT_COINBASE_WEIGHT_BUDGET:-50000} healthcheck: test: ["CMD-SHELL", "node -e \"const http=require('http'); const https=require('https'); const secure=process.env.API_SECURE==='true'; const client=secure?https:http; const req=client.get({hostname:'127.0.0.1',port:process.env.API_PORT||3334,path:'/api/network',rejectUnauthorized:false},res=>process.exit(res.statusCode<500?0:1)); req.on('error',()=>process.exit(1)); req.setTimeout(5000,()=>{req.destroy(); process.exit(1);});\""] interval: 30s diff --git a/full-setup/docker-compose-mainnet.yml b/full-setup/docker-compose-mainnet.yml index 1d4037b..5d94c26 100644 --- a/full-setup/docker-compose-mainnet.yml +++ b/full-setup/docker-compose-mainnet.yml @@ -117,6 +117,15 @@ services: SHARE_ROLLUP_INTERVAL_MS: ${SHARE_ROLLUP_INTERVAL_MS:-60000} SHARE_ROLLUP_SAFETY_LAG_SECONDS: ${SHARE_ROLLUP_SAFETY_LAG_SECONDS:-30} SHARE_ROLLUP_MAX_SHARES_PER_BATCH: ${SHARE_ROLLUP_MAX_SHARES_PER_BATCH:-5000000} + PAYOUT_SNAPSHOT_ENABLED: ${PAYOUT_SNAPSHOT_ENABLED:-true} + PAYOUT_METHOD: ${PAYOUT_METHOD:-pplns} + PAYOUT_WINDOW_FACTOR: ${PAYOUT_WINDOW_FACTOR:-4} + PAYOUT_COINBASE_MODE: ${PAYOUT_COINBASE_MODE:-solo} + PAYOUT_MAX_COINBASE_OUTPUTS: ${PAYOUT_MAX_COINBASE_OUTPUTS:-10} + PAYOUT_MIN_OUTPUT_SATS: ${PAYOUT_MIN_OUTPUT_SATS:-546} + PAYOUT_FEE_ADDRESS: ${PAYOUT_FEE_ADDRESS:-} + PAYOUT_FEE_PERCENT: ${PAYOUT_FEE_PERCENT:-0} + PAYOUT_COINBASE_WEIGHT_BUDGET: ${PAYOUT_COINBASE_WEIGHT_BUDGET:-50000} networks: bitcoin: diff --git a/package-lock.json b/package-lock.json index 4fddc84..7c690d4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -28,6 +28,7 @@ "class-validator": "^0.14.4", "discord.js": "^14.26.4", "dotenv": "^16.6.1", + "libsodium-wrappers-sumo": "^0.8.4", "merkle-lib": "^2.0.10", "pg": "^8.20.0", "pm2": "^7.0.1", @@ -45,6 +46,7 @@ "@types/cron": "^2.0.1", "@types/express": "^4.17.25", "@types/jest": "^29.5.14", + "@types/libsodium-wrappers-sumo": "^0.7.8", "@types/node": "^18.19.130", "@types/supertest": "^7.2.0", "@typescript-eslint/eslint-plugin": "^5.0.0", @@ -3194,6 +3196,23 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/libsodium-wrappers": { + "version": "0.7.14", + "resolved": "https://registry.npmjs.org/@types/libsodium-wrappers/-/libsodium-wrappers-0.7.14.tgz", + "integrity": "sha512-5Kv68fXuXK0iDuUir1WPGw2R9fOZUlYlSAa0ztMcL0s0BfIDTqg9GXz8K30VJpPP3sxWhbolnQma2x+/TfkzDQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/libsodium-wrappers-sumo": { + "version": "0.7.8", + "resolved": "https://registry.npmjs.org/@types/libsodium-wrappers-sumo/-/libsodium-wrappers-sumo-0.7.8.tgz", + "integrity": "sha512-N2+df4MB/A+W0RAcTw7A5oxKgzD+Vh6Ye7lfjWIi5SdTzVLfHPzxUjhwPqHLO5Ev9fv/+VHl+sUaUuTg4fUPqw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/libsodium-wrappers": "*" + } + }, "node_modules/@types/luxon": { "version": "3.7.1", "resolved": "https://registry.npmjs.org/@types/luxon/-/luxon-3.7.1.tgz", @@ -8443,6 +8462,21 @@ "integrity": "sha512-oKQFPTibqQwZZkChCDVMFVJXMZdyJNqDWZWYNn8BgyAaK/6yFJEowxCY0RVFirRyWP63hMRuKlkSEd9qlvbWXg==", "license": "MIT" }, + "node_modules/libsodium-sumo": { + "version": "0.8.4", + "resolved": "https://registry.npmjs.org/libsodium-sumo/-/libsodium-sumo-0.8.4.tgz", + "integrity": "sha512-TMtHShQfVVsaxDygyapvUC3o7YsPgXa/hRWeIgzyFz6w5k/1hirGptCxp1U7XwW3rCskaTTYKgV10v86UiGgNw==", + "license": "ISC" + }, + "node_modules/libsodium-wrappers-sumo": { + "version": "0.8.4", + "resolved": "https://registry.npmjs.org/libsodium-wrappers-sumo/-/libsodium-wrappers-sumo-0.8.4.tgz", + "integrity": "sha512-ql7hcgulKZ3ekfa2DGAogcCKsWU0diA/0nArz1CFzh93WQdb46/Kj18ka/Hifq6uA3Ush34Pc6vU/6HXeRwUkg==", + "license": "ISC", + "dependencies": { + "libsodium-sumo": "^0.8.0" + } + }, "node_modules/light-my-request": { "version": "6.6.0", "resolved": "https://registry.npmjs.org/light-my-request/-/light-my-request-6.6.0.tgz", @@ -14998,6 +15032,21 @@ "integrity": "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==", "dev": true }, + "@types/libsodium-wrappers": { + "version": "0.7.14", + "resolved": "https://registry.npmjs.org/@types/libsodium-wrappers/-/libsodium-wrappers-0.7.14.tgz", + "integrity": "sha512-5Kv68fXuXK0iDuUir1WPGw2R9fOZUlYlSAa0ztMcL0s0BfIDTqg9GXz8K30VJpPP3sxWhbolnQma2x+/TfkzDQ==", + "dev": true + }, + "@types/libsodium-wrappers-sumo": { + "version": "0.7.8", + "resolved": "https://registry.npmjs.org/@types/libsodium-wrappers-sumo/-/libsodium-wrappers-sumo-0.7.8.tgz", + "integrity": "sha512-N2+df4MB/A+W0RAcTw7A5oxKgzD+Vh6Ye7lfjWIi5SdTzVLfHPzxUjhwPqHLO5Ev9fv/+VHl+sUaUuTg4fUPqw==", + "dev": true, + "requires": { + "@types/libsodium-wrappers": "*" + } + }, "@types/luxon": { "version": "3.7.1", "resolved": "https://registry.npmjs.org/@types/luxon/-/luxon-3.7.1.tgz", @@ -18619,6 +18668,19 @@ "resolved": "https://registry.npmjs.org/libphonenumber-js/-/libphonenumber-js-1.12.42.tgz", "integrity": "sha512-oKQFPTibqQwZZkChCDVMFVJXMZdyJNqDWZWYNn8BgyAaK/6yFJEowxCY0RVFirRyWP63hMRuKlkSEd9qlvbWXg==" }, + "libsodium-sumo": { + "version": "0.8.4", + "resolved": "https://registry.npmjs.org/libsodium-sumo/-/libsodium-sumo-0.8.4.tgz", + "integrity": "sha512-TMtHShQfVVsaxDygyapvUC3o7YsPgXa/hRWeIgzyFz6w5k/1hirGptCxp1U7XwW3rCskaTTYKgV10v86UiGgNw==" + }, + "libsodium-wrappers-sumo": { + "version": "0.8.4", + "resolved": "https://registry.npmjs.org/libsodium-wrappers-sumo/-/libsodium-wrappers-sumo-0.8.4.tgz", + "integrity": "sha512-ql7hcgulKZ3ekfa2DGAogcCKsWU0diA/0nArz1CFzh93WQdb46/Kj18ka/Hifq6uA3Ush34Pc6vU/6HXeRwUkg==", + "requires": { + "libsodium-sumo": "^0.8.0" + } + }, "light-my-request": { "version": "6.6.0", "resolved": "https://registry.npmjs.org/light-my-request/-/light-my-request-6.6.0.tgz", diff --git a/package.json b/package.json index bcc48a5..2804241 100644 --- a/package.json +++ b/package.json @@ -47,6 +47,7 @@ "class-validator": "^0.14.4", "discord.js": "^14.26.4", "dotenv": "^16.6.1", + "libsodium-wrappers-sumo": "^0.8.4", "merkle-lib": "^2.0.10", "pg": "^8.20.0", "pm2": "^7.0.1", @@ -64,6 +65,7 @@ "@types/cron": "^2.0.1", "@types/express": "^4.17.25", "@types/jest": "^29.5.14", + "@types/libsodium-wrappers-sumo": "^0.7.8", "@types/node": "^18.19.130", "@types/supertest": "^7.2.0", "@typescript-eslint/eslint-plugin": "^5.0.0", diff --git a/src/ORM/_migrations/AcceptedShareProtocolMetadata1781130600000.ts b/src/ORM/_migrations/AcceptedShareProtocolMetadata1781130600000.ts new file mode 100644 index 0000000..0a844cf --- /dev/null +++ b/src/ORM/_migrations/AcceptedShareProtocolMetadata1781130600000.ts @@ -0,0 +1,74 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; + +export class AcceptedShareProtocolMetadata1781130600000 implements MigrationInterface { + public name = 'AcceptedShareProtocolMetadata1781130600000'; + public transaction = false; + + public async up(queryRunner: QueryRunner): Promise { + await this.decompressAcceptedShareChunks(queryRunner); + await queryRunner.query(` + DO $$ + BEGIN + IF EXISTS ( + SELECT 1 + FROM information_schema.columns + WHERE table_name = 'accepted_share_entity' + AND column_name = 'protocol' + AND COALESCE(character_maximum_length, 0) < 16 + ) THEN + ALTER TABLE "accepted_share_entity" ALTER COLUMN "protocol" TYPE varchar(16); + END IF; + END + $$ + `); + await queryRunner.query(`ALTER TABLE "accepted_share_entity" ADD COLUMN IF NOT EXISTS "workSource" varchar(16) NOT NULL DEFAULT 'pool_template'`); + await queryRunner.query(`ALTER TABLE "accepted_share_entity" ADD COLUMN IF NOT EXISTS "workProtocol" varchar(16) NOT NULL DEFAULT 'pool'`); + await this.restoreAcceptedShareCompression(queryRunner); + await queryRunner.query(` + CREATE INDEX IF NOT EXISTS "IDX_accepted_share_protocol_work" + ON "accepted_share_entity" ("protocol", "workSource", "workProtocol", "acceptedAt" DESC) + `); + } + + public async down(queryRunner: QueryRunner): Promise { + await this.decompressAcceptedShareChunks(queryRunner); + await queryRunner.query(`DROP INDEX IF EXISTS "IDX_accepted_share_protocol_work"`); + await queryRunner.query(`ALTER TABLE "accepted_share_entity" DROP COLUMN IF EXISTS "workProtocol"`); + await queryRunner.query(`ALTER TABLE "accepted_share_entity" DROP COLUMN IF EXISTS "workSource"`); + await queryRunner.query(`ALTER TABLE "accepted_share_entity" ALTER COLUMN "protocol" TYPE varchar(8)`); + await this.restoreAcceptedShareCompression(queryRunner); + } + + private async decompressAcceptedShareChunks(queryRunner: QueryRunner): Promise { + await queryRunner.query(`SELECT remove_compression_policy('accepted_share_entity', if_exists => TRUE)`); + await queryRunner.query(` + DO $$ + DECLARE + chunk_name regclass; + BEGIN + FOR chunk_name IN SELECT show_chunks('accepted_share_entity') + LOOP + PERFORM decompress_chunk(chunk_name, if_compressed => TRUE); + END LOOP; + END + $$ + `); + } + + private async restoreAcceptedShareCompression(queryRunner: QueryRunner): Promise { + await queryRunner.query(` + ALTER TABLE "accepted_share_entity" SET ( + timescaledb.compress, + timescaledb.compress_orderby = '"acceptedAt" DESC', + timescaledb.compress_segmentby = '"address","clientName"' + ) + `); + await queryRunner.query(` + SELECT add_compression_policy( + 'accepted_share_entity', + INTERVAL '24 hours', + if_not_exists => TRUE + ) + `); + } +} diff --git a/src/ORM/_migrations/BlocksSubmissionMetadata1781220000000.ts b/src/ORM/_migrations/BlocksSubmissionMetadata1781220000000.ts new file mode 100644 index 0000000..ebb984e --- /dev/null +++ b/src/ORM/_migrations/BlocksSubmissionMetadata1781220000000.ts @@ -0,0 +1,27 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; + +export class BlocksSubmissionMetadata1781220000000 implements MigrationInterface { + public name = 'BlocksSubmissionMetadata1781220000000'; + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TABLE "blocks_entity" ADD COLUMN IF NOT EXISTS "blockHash" varchar`); + await queryRunner.query(`ALTER TABLE "blocks_entity" ADD COLUMN IF NOT EXISTS "blockSubmissionResult" varchar`); + await queryRunner.query(` + CREATE INDEX IF NOT EXISTS "IDX_blocks_successful_found" + ON "blocks_entity" ("height" DESC, "createdAt" DESC) + WHERE "blockSubmissionResult" IS NULL OR "blockSubmissionResult" = 'SUCCESS!' + `); + await queryRunner.query(` + CREATE UNIQUE INDEX IF NOT EXISTS "UQ_blocks_block_hash" + ON "blocks_entity" ("blockHash") + WHERE "blockHash" IS NOT NULL + `); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`DROP INDEX IF EXISTS "UQ_blocks_block_hash"`); + await queryRunner.query(`DROP INDEX IF EXISTS "IDX_blocks_successful_found"`); + await queryRunner.query(`ALTER TABLE "blocks_entity" DROP COLUMN IF EXISTS "blockSubmissionResult"`); + await queryRunner.query(`ALTER TABLE "blocks_entity" DROP COLUMN IF EXISTS "blockHash"`); + } +} diff --git a/src/ORM/_migrations/InitialTimescaleSchema1780859300000.ts b/src/ORM/_migrations/InitialTimescaleSchema1780859300000.ts index 82d6b61..bf3dfe6 100644 --- a/src/ORM/_migrations/InitialTimescaleSchema1780859300000.ts +++ b/src/ORM/_migrations/InitialTimescaleSchema1780859300000.ts @@ -88,6 +88,8 @@ export class InitialTimescaleSchema1780859300000 implements MigrationInterface { "worker" varchar NOT NULL, "sessionId" varchar(8) NOT NULL, "blockData" varchar NOT NULL, + "blockHash" varchar, + "blockSubmissionResult" varchar, "deletedAt" timestamptz, "createdAt" timestamptz NOT NULL DEFAULT now(), "updatedAt" timestamptz NOT NULL DEFAULT now() diff --git a/src/ORM/_migrations/PayoutSnapshots1780969800000.ts b/src/ORM/_migrations/PayoutSnapshots1780969800000.ts new file mode 100644 index 0000000..5bdc546 --- /dev/null +++ b/src/ORM/_migrations/PayoutSnapshots1780969800000.ts @@ -0,0 +1,143 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; + +export class PayoutSnapshots1780969800000 implements MigrationInterface { + public name = 'PayoutSnapshots1780969800000'; + public transaction = false; + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(` + CREATE TABLE IF NOT EXISTS "payout_snapshot" ( + "id" bigserial PRIMARY KEY, + "method" varchar(32) NOT NULL, + "status" varchar(16) NOT NULL DEFAULT 'finalized', + "blockHeight" bigint NOT NULL, + "coinbaseValueSats" bigint NOT NULL, + "networkDifficulty" numeric NOT NULL, + "windowTargetDifficulty" numeric NOT NULL DEFAULT 0, + "windowFactor" numeric NOT NULL DEFAULT 4, + "feeAddress" varchar(62), + "feeSats" bigint NOT NULL DEFAULT 0, + "minPayoutSats" integer NOT NULL DEFAULT 546, + "coinbaseWeightBudget" integer NOT NULL DEFAULT 50000, + "startBatchId" bigint NOT NULL REFERENCES "share_rollup_batch" ("id"), + "endBatchId" bigint NOT NULL REFERENCES "share_rollup_batch" ("id"), + "windowStartShareIndex" bigint NOT NULL, + "windowEndShareIndex" bigint NOT NULL, + "totalCreditedDifficulty" numeric NOT NULL, + "totalAcceptedShareCount" bigint NOT NULL, + "eligibleAddressCount" integer NOT NULL, + "includedOutputCount" integer NOT NULL, + "distributedSats" bigint NOT NULL, + "unallocatedRemainderSats" bigint NOT NULL, + "createdAt" timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP + ) + `); + await queryRunner.query(` + CREATE INDEX IF NOT EXISTS "IDX_payout_snapshot_latest" + ON "payout_snapshot" ("status", "createdAt" DESC, "id" DESC) + `); + await queryRunner.query(` + CREATE INDEX IF NOT EXISTS "IDX_payout_snapshot_window" + ON "payout_snapshot" ("windowEndShareIndex" DESC) + `); + await queryRunner.query(` + CREATE UNIQUE INDEX IF NOT EXISTS "IDX_payout_snapshot_template_window" + ON "payout_snapshot" ("method", "blockHeight", "coinbaseValueSats", "windowEndShareIndex") + WHERE "status" = 'finalized' + `); + + await queryRunner.query(` + CREATE TABLE IF NOT EXISTS "payout_snapshot_entry" ( + "id" bigserial PRIMARY KEY, + "snapshotId" bigint NOT NULL REFERENCES "payout_snapshot" ("id") ON DELETE CASCADE, + "address" varchar(62) NOT NULL, + "creditedDifficulty" numeric NOT NULL, + "acceptedShareCount" bigint NOT NULL, + "payoutWeight" numeric NOT NULL, + "grossPayoutSats" bigint NOT NULL, + "payoutSats" bigint NOT NULL, + "balanceBeforeSats" bigint NOT NULL DEFAULT 0, + "balanceAfterSats" bigint NOT NULL DEFAULT 0, + "includedInCoinbase" boolean NOT NULL DEFAULT true, + "rowType" varchar(16) NOT NULL DEFAULT 'coinbase', + "rank" integer NOT NULL, + "createdAt" timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP + ) + `); + await queryRunner.query(` + CREATE UNIQUE INDEX IF NOT EXISTS "IDX_payout_snapshot_entry_snapshot_address" + ON "payout_snapshot_entry" ("snapshotId", "address") + `); + await queryRunner.query(` + CREATE INDEX IF NOT EXISTS "IDX_payout_snapshot_entry_snapshot_rank" + ON "payout_snapshot_entry" ("snapshotId", "rank") + `); + await queryRunner.query(` + CREATE INDEX IF NOT EXISTS "IDX_payout_snapshot_entry_address" + ON "payout_snapshot_entry" ("address") + `); + + await queryRunner.query(` + CREATE TABLE IF NOT EXISTS "payout_balance" ( + "address" varchar(62) PRIMARY KEY, + "balanceSats" bigint NOT NULL DEFAULT 0, + "totalPaidSats" bigint NOT NULL DEFAULT 0, + "lastAcceptedShareAt" timestamptz, + "updatedAt" timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP + ) + `); + await queryRunner.query(` + CREATE INDEX IF NOT EXISTS "IDX_payout_balance_nonzero" + ON "payout_balance" ("balanceSats") + WHERE "balanceSats" != 0 + `); + + await queryRunner.query(` + CREATE TABLE IF NOT EXISTS "payout_history" ( + "id" bigserial PRIMARY KEY, + "blockHeight" bigint NOT NULL, + "payoutSnapshotId" bigint NOT NULL REFERENCES "payout_snapshot" ("id") ON DELETE CASCADE, + "address" varchar(62) NOT NULL, + "paidSats" bigint NOT NULL DEFAULT 0, + "percent" numeric NOT NULL DEFAULT 0, + "balanceBeforeSats" bigint NOT NULL DEFAULT 0, + "balanceAfterSats" bigint NOT NULL DEFAULT 0, + "rowType" varchar(16) NOT NULL DEFAULT 'coinbase', + "createdAt" timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP + ) + `); + await queryRunner.query(` + CREATE UNIQUE INDEX IF NOT EXISTS "UQ_payout_history_block_address" + ON "payout_history" ("blockHeight", "address") + `); + await queryRunner.query(` + CREATE INDEX IF NOT EXISTS "IDX_payout_history_address_created" + ON "payout_history" ("address", "createdAt" DESC) + `); + + await queryRunner.query(`ALTER TABLE "blocks_entity" ADD COLUMN IF NOT EXISTS "payoutSnapshotId" bigint`); + await queryRunner.query(` + CREATE INDEX IF NOT EXISTS "IDX_blocks_payout_snapshot" + ON "blocks_entity" ("payoutSnapshotId") + WHERE "payoutSnapshotId" IS NOT NULL + `); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`DROP INDEX IF EXISTS "IDX_blocks_payout_snapshot"`); + await queryRunner.query(`ALTER TABLE "blocks_entity" DROP COLUMN IF EXISTS "payoutSnapshotId"`); + await queryRunner.query(`DROP INDEX IF EXISTS "IDX_payout_history_address_created"`); + await queryRunner.query(`DROP INDEX IF EXISTS "UQ_payout_history_block_address"`); + await queryRunner.query(`DROP TABLE IF EXISTS "payout_history"`); + await queryRunner.query(`DROP INDEX IF EXISTS "IDX_payout_balance_nonzero"`); + await queryRunner.query(`DROP TABLE IF EXISTS "payout_balance"`); + await queryRunner.query(`DROP INDEX IF EXISTS "IDX_payout_snapshot_entry_address"`); + await queryRunner.query(`DROP INDEX IF EXISTS "IDX_payout_snapshot_entry_snapshot_rank"`); + await queryRunner.query(`DROP INDEX IF EXISTS "IDX_payout_snapshot_entry_snapshot_address"`); + await queryRunner.query(`DROP TABLE IF EXISTS "payout_snapshot_entry"`); + await queryRunner.query(`DROP INDEX IF EXISTS "IDX_payout_snapshot_template_window"`); + await queryRunner.query(`DROP INDEX IF EXISTS "IDX_payout_snapshot_window"`); + await queryRunner.query(`DROP INDEX IF EXISTS "IDX_payout_snapshot_latest"`); + await queryRunner.query(`DROP TABLE IF EXISTS "payout_snapshot"`); + } +} diff --git a/src/ORM/accepted-share/accepted-share.entity.ts b/src/ORM/accepted-share/accepted-share.entity.ts index 1296f1a..d3106f0 100644 --- a/src/ORM/accepted-share/accepted-share.entity.ts +++ b/src/ORM/accepted-share/accepted-share.entity.ts @@ -15,8 +15,14 @@ export class AcceptedShareEntity { @Column({ type: 'bigint', default: () => `nextval('accepted_share_index_seq')` }) shareIndex: number; - @Column({ length: 8, type: 'varchar' }) - protocol: 'sv1' | 'sv2'; + @Column({ length: 16, type: 'varchar' }) + protocol: 'sv1' | 'sv1_tls' | 'sv2' | 'sv2_jdp' | 'datum'; + + @Column({ length: 16, type: 'varchar', default: 'pool_template' }) + workSource: 'pool_template' | 'miner_template'; + + @Column({ length: 16, type: 'varchar', default: 'pool' }) + workProtocol: 'pool' | 'sv2_jdp' | 'datum'; @Column({ length: 62, type: 'varchar' }) address: string; diff --git a/src/ORM/blocks/blocks.entity.ts b/src/ORM/blocks/blocks.entity.ts index b156b70..5f69f57 100644 --- a/src/ORM/blocks/blocks.entity.ts +++ b/src/ORM/blocks/blocks.entity.ts @@ -24,4 +24,13 @@ export class BlocksEntity extends TrackedEntity { @Column() blockData: string; + @Column({ nullable: true }) + blockHash?: string; + + @Column({ type: 'varchar', nullable: true }) + blockSubmissionResult?: string; + + @Column({ type: 'bigint', nullable: true }) + payoutSnapshotId?: string; + } diff --git a/src/ORM/blocks/blocks.service.spec.ts b/src/ORM/blocks/blocks.service.spec.ts new file mode 100644 index 0000000..3015f1f --- /dev/null +++ b/src/ORM/blocks/blocks.service.spec.ts @@ -0,0 +1,87 @@ +import * as bitcoinjs from 'bitcoinjs-lib'; +import { BlocksService } from './blocks.service'; + +describe('BlocksService', () => { + it('computes block hash metadata when saving found blocks', async () => { + const repository = createRepository(); + repository.findOne.mockResolvedValue(null); + const service = new BlocksService({} as any, repository as any); + const blockHex = createBlockHex(); + const blockHash = bitcoinjs.Block.fromHex(blockHex).getId(); + + await service.save({ + height: 123, + minerAddress: 'tb1qexample', + worker: 'worker', + sessionId: 'abcd1234', + blockData: blockHex, + blockSubmissionResult: 'SUCCESS!', + }); + + expect(repository.save).toHaveBeenCalledWith(expect.objectContaining({ + blockHash, + blockSubmissionResult: 'SUCCESS!', + })); + }); + + it('does not downgrade an existing successful block on duplicate rejected submissions', async () => { + const repository = createRepository(); + repository.findOne.mockResolvedValue({ + id: 9, + blockSubmissionResult: 'SUCCESS!', + payoutSnapshotId: '7', + }); + const service = new BlocksService({} as any, repository as any); + + await service.save({ + blockData: createBlockHex(), + blockSubmissionResult: 'duplicate', + payoutSnapshotId: '8', + }); + + expect(repository.update).not.toHaveBeenCalled(); + expect(repository.save).not.toHaveBeenCalled(); + }); + + it('returns one public found-block row per block hash or legacy height', async () => { + const repository = createRepository(); + repository.find.mockResolvedValue([ + { height: 124, minerAddress: 'addr2', worker: 'b', sessionId: '2222', blockHash: 'hash-b' }, + { height: 124, minerAddress: 'addr2', worker: 'b-dup', sessionId: '3333', blockHash: 'hash-b' }, + { height: 123, minerAddress: 'addr1', worker: 'a', sessionId: '1111', blockHash: null }, + { height: 123, minerAddress: 'addr1', worker: 'a-dup', sessionId: '4444', blockHash: null }, + ]); + const service = new BlocksService({} as any, repository as any); + + await expect(service.getFoundBlocks()).resolves.toEqual([ + { height: 124, minerAddress: 'addr2', worker: 'b', sessionId: '2222', blockHash: 'hash-b' }, + { height: 123, minerAddress: 'addr1', worker: 'a', sessionId: '1111', blockHash: null }, + ]); + }); + + function createRepository() { + return { + save: jest.fn().mockResolvedValue(undefined), + find: jest.fn().mockResolvedValue([]), + findOne: jest.fn().mockResolvedValue(null), + update: jest.fn().mockResolvedValue(undefined), + }; + } + + function createBlockHex(): string { + const tx = new bitcoinjs.Transaction(); + tx.version = 1; + tx.addInput(Buffer.alloc(32), 0xffffffff, 0xffffffff, Buffer.from([0x01])); + tx.addOutput(Buffer.from('6a', 'hex'), 0); + + const block = new bitcoinjs.Block(); + block.version = 1; + block.prevHash = Buffer.alloc(32); + block.timestamp = 1; + block.bits = 0x207fffff; + block.nonce = 1; + block.transactions = [tx]; + block.merkleRoot = bitcoinjs.Block.calculateMerkleRoot(block.transactions, false); + return block.toHex(false); + } +}); diff --git a/src/ORM/blocks/blocks.service.ts b/src/ORM/blocks/blocks.service.ts index bb9964b..70613ca 100644 --- a/src/ORM/blocks/blocks.service.ts +++ b/src/ORM/blocks/blocks.service.ts @@ -1,6 +1,7 @@ import { Injectable } from '@nestjs/common'; import { InjectRepository } from '@nestjs/typeorm'; -import { DataSource, Repository } from 'typeorm'; +import * as bitcoinjs from 'bitcoinjs-lib'; +import { DataSource, IsNull, Repository } from 'typeorm'; import { BlocksEntity } from './blocks.entity'; @@ -19,31 +20,106 @@ export class BlocksService { public async save(block: Partial) { - await this.blocksRepository.save(block); + const blockHash = block.blockHash ?? this.getBlockHash(block.blockData); + const existing = blockHash == null + ? null + : await this.blocksRepository.findOne({ + select: { id: true, blockSubmissionResult: true, payoutSnapshotId: true }, + where: { blockHash }, + }); + if (existing != null) { + if (this.isSuccessfulBlockSubmission(block.blockSubmissionResult)) { + await this.blocksRepository.update(existing.id, { + blockSubmissionResult: 'SUCCESS!', + payoutSnapshotId: block.payoutSnapshotId ?? existing.payoutSnapshotId, + }); + } + return; + } + + await this.blocksRepository.save({ + ...block, + blockHash, + }); } public async getFoundBlocks() { - return await this.blocksRepository.find({ + const rows = await this.blocksRepository.find({ select: { height: true, minerAddress: true, worker: true, - sessionId: true - } + sessionId: true, + blockHash: true, + blockSubmissionResult: true, + createdAt: true, + }, + where: [ + { blockSubmissionResult: 'SUCCESS!' }, + { blockSubmissionResult: IsNull() }, + ], + order: { + height: 'DESC', + createdAt: 'DESC', + }, }); + return this.uniqueFoundBlocks(rows); } public async getFoundBlocksByAddress(address: string) { - return await this.blocksRepository.find({ + const rows = await this.blocksRepository.find({ select: { height: true, minerAddress: true, worker: true, - sessionId: true + sessionId: true, + blockHash: true, + blockSubmissionResult: true, + createdAt: true, }, - where: { - minerAddress: address + where: [ + { minerAddress: address, blockSubmissionResult: 'SUCCESS!' }, + { minerAddress: address, blockSubmissionResult: IsNull() }, + ], + order: { + height: 'DESC', + createdAt: 'DESC', + }, + }); + return this.uniqueFoundBlocks(rows); + } + + private uniqueFoundBlocks(rows: Partial[]): Partial[] { + const seenBlocks = new Set(); + const seenHeights = new Set(); + return rows.filter(row => { + if (row.height != null && seenHeights.has(row.height)) { + return false; } + const key = row.blockHash || `height:${row.height}`; + if (seenBlocks.has(key)) { + return false; + } + seenBlocks.add(key); + if (row.height != null) { + seenHeights.add(row.height); + } + return true; }); } -} \ No newline at end of file + + private getBlockHash(blockData?: string): string | undefined { + if (blockData == null || blockData.length < 160) { + return undefined; + } + try { + return bitcoinjs.Block.fromHex(blockData).getId(); + } catch { + return undefined; + } + } + + private isSuccessfulBlockSubmission(result?: string | null): boolean { + return result == null || result === 'SUCCESS!'; + } +} diff --git a/src/ORM/payout-snapshot/payout-balance.entity.ts b/src/ORM/payout-snapshot/payout-balance.entity.ts new file mode 100644 index 0000000..7e34804 --- /dev/null +++ b/src/ORM/payout-snapshot/payout-balance.entity.ts @@ -0,0 +1,24 @@ +import { Column, Entity, PrimaryColumn, UpdateDateColumn } from 'typeorm'; + +const bigintTransformer = { + from: (value: string | number | null): number => Number(value ?? 0), + to: (value: number): number => Math.trunc(value ?? 0), +}; + +@Entity({ name: 'payout_balance' }) +export class PayoutBalanceEntity { + @PrimaryColumn({ length: 62, type: 'varchar' }) + address: string; + + @Column({ type: 'bigint', default: 0, transformer: bigintTransformer }) + balanceSats: number; + + @Column({ type: 'bigint', default: 0, transformer: bigintTransformer }) + totalPaidSats: number; + + @Column({ type: 'timestamptz', nullable: true }) + lastAcceptedShareAt: Date | null; + + @UpdateDateColumn() + updatedAt: Date; +} diff --git a/src/ORM/payout-snapshot/payout-distribution.spec.ts b/src/ORM/payout-snapshot/payout-distribution.spec.ts new file mode 100644 index 0000000..3fb46fa --- /dev/null +++ b/src/ORM/payout-snapshot/payout-distribution.spec.ts @@ -0,0 +1,135 @@ +import { buildPayoutDistribution } from './payout-distribution'; + +const ADDRESS_A = 'bc1qs29kyaqqc0fkvj897ke9e5xa9utljjey0y5jjn'; +const ADDRESS_B = 'bc1q99n3pu025yyu0jlywpmwzalyhm36tg5u37w20d'; +const ADDRESS_C = 'bc1q7yx4d3xspf4xvssmsm4prf4mg2slkv8jtkc5l6'; + +describe('buildPayoutDistribution', () => { + it('should conserve the full coinbase reward across outputs', () => { + const result = buildPayoutDistribution({ + addressWork: [ + { address: ADDRESS_A, creditedDifficulty: 2, acceptedShareCount: 2 }, + { address: ADDRESS_B, creditedDifficulty: 1, acceptedShareCount: 1 }, + ], + balances: [], + coinbaseValueSats: 1000, + feePercent: 0, + minOutputSats: 546, + coinbaseWeightBudget: 50_000, + }); + + expect(result.payoutOutputs.map(entry => ({ + address: entry.address, + payoutSats: entry.payoutSats, + balanceAfterSats: entry.balanceAfterSats, + }))).toEqual([ + { address: ADDRESS_A, payoutSats: 1000, balanceAfterSats: -334 }, + ]); + expect(result.entries.find(entry => entry.address === ADDRESS_B)) + .toEqual(expect.objectContaining({ includedInCoinbase: false, balanceAfterSats: 333 })); + expect(result.distributedSats).toBe(1000); + expect(result.unallocatedRemainderSats).toBe(0); + }); + + it('should carry sub-dust work forward as signed credit', () => { + const result = buildPayoutDistribution({ + addressWork: [ + { address: ADDRESS_A, creditedDifficulty: 999, acceptedShareCount: 999 }, + { address: ADDRESS_B, creditedDifficulty: 1, acceptedShareCount: 1 }, + ], + balances: [], + coinbaseValueSats: 1000, + feePercent: 0, + minOutputSats: 546, + coinbaseWeightBudget: 50_000, + }); + + const pending = result.entries.find(entry => entry.address === ADDRESS_B); + const paid = result.entries.find(entry => entry.address === ADDRESS_A); + + expect(pending).toEqual(expect.objectContaining({ + includedInCoinbase: false, + grossPayoutSats: 1, + payoutSats: 0, + balanceAfterSats: 1, + })); + expect(paid).toEqual(expect.objectContaining({ + includedInCoinbase: true, + payoutSats: 1000, + balanceAfterSats: -1, + })); + expect(result.distributedSats).toBe(1000); + }); + + it('should trim outputs that do not fit the coinbase weight budget', () => { + const result = buildPayoutDistribution({ + addressWork: [ + { address: ADDRESS_A, creditedDifficulty: 10, acceptedShareCount: 1 }, + { address: ADDRESS_B, creditedDifficulty: 9, acceptedShareCount: 1 }, + { address: ADDRESS_C, creditedDifficulty: 8, acceptedShareCount: 1 }, + ], + balances: [], + coinbaseValueSats: 3000, + feePercent: 0, + minOutputSats: 546, + coinbaseWeightBudget: 850, + }); + + expect(result.payoutOutputs).toHaveLength(1); + expect(result.payoutOutputs[0].address).toBe(ADDRESS_A); + expect(result.entries.find(entry => entry.address === ADDRESS_B)) + .toEqual(expect.objectContaining({ includedInCoinbase: false, balanceAfterSats: 1000 })); + expect(result.entries.find(entry => entry.address === ADDRESS_C)) + .toEqual(expect.objectContaining({ includedInCoinbase: false, balanceAfterSats: 888 })); + }); + + it('should apply positive and negative carried balances to the next distribution', () => { + const result = buildPayoutDistribution({ + addressWork: [ + { address: ADDRESS_A, creditedDifficulty: 10, acceptedShareCount: 1 }, + { address: ADDRESS_B, creditedDifficulty: 10, acceptedShareCount: 1 }, + ], + balances: [ + { address: ADDRESS_A, balanceSats: 100 }, + { address: ADDRESS_B, balanceSats: -100 }, + ], + coinbaseValueSats: 2000, + feePercent: 0, + minOutputSats: 546, + coinbaseWeightBudget: 50_000, + }); + + expect(result.payoutOutputs.map(entry => ({ + address: entry.address, + payoutSats: entry.payoutSats, + balanceAfterSats: entry.balanceAfterSats, + }))).toEqual([ + { address: ADDRESS_A, payoutSats: 1100, balanceAfterSats: 0 }, + { address: ADDRESS_B, payoutSats: 900, balanceAfterSats: 0 }, + ]); + expect(result.unallocatedRemainderSats).toBe(0); + }); + + it('should reserve a configured fee output before miner distribution', () => { + const result = buildPayoutDistribution({ + addressWork: [ + { address: ADDRESS_A, creditedDifficulty: 1, acceptedShareCount: 1 }, + ], + balances: [], + coinbaseValueSats: 100000, + feeAddress: ADDRESS_C, + feePercent: 2, + minOutputSats: 546, + coinbaseWeightBudget: 50_000, + }); + + expect(result.payoutOutputs[0]).toEqual(expect.objectContaining({ + address: ADDRESS_C, + payoutSats: 2000, + })); + expect(result.payoutOutputs[1]).toEqual(expect.objectContaining({ + address: ADDRESS_A, + payoutSats: 98000, + })); + }); +}); diff --git a/src/ORM/payout-snapshot/payout-distribution.ts b/src/ORM/payout-snapshot/payout-distribution.ts new file mode 100644 index 0000000..d5ded34 --- /dev/null +++ b/src/ORM/payout-snapshot/payout-distribution.ts @@ -0,0 +1,276 @@ +import { AddressType, getAddressInfo } from 'bitcoin-address-validation'; + +export interface PayoutAddressWork { + address: string; + creditedDifficulty: number; + acceptedShareCount: number; +} + +export interface PayoutBalanceInput { + address: string; + balanceSats: number; +} + +export interface PayoutDistributionEntry { + address: string; + creditedDifficulty: number; + acceptedShareCount: number; + payoutWeight: number; + grossPayoutSats: number; + payoutSats: number; + balanceBeforeSats: number; + balanceAfterSats: number; + includedInCoinbase: boolean; + rank: number; +} + +export interface PayoutDistributionResult { + entries: PayoutDistributionEntry[]; + payoutOutputs: PayoutDistributionEntry[]; + consideredAddressCount: number; + distributedSats: number; + unallocatedRemainderSats: number; + feeSats: number; +} + +export interface PayoutDistributionInput { + addressWork: PayoutAddressWork[]; + balances: PayoutBalanceInput[]; + coinbaseValueSats: number; + feeAddress?: string; + feePercent: number; + minOutputSats: number; + coinbaseWeightBudget: number; +} + +const DEFAULT_COINBASE_WEIGHT_BUDGET = 50_000; +const DUST_LIMIT_SATS = 546; +const COINBASE_BASE_WEIGHT = 328; +const COINBASE_WITNESS_COMMITMENT_WEIGHT = 188; +const COINBASE_OUTPUT_WEIGHT = 172; +const BUDGET_SAFETY_MARGIN_WEIGHT = 200; + +const OUTPUT_WEIGHT_BY_TYPE: Record = { + [AddressType.p2wpkh]: 124, + [AddressType.p2sh]: 128, + [AddressType.p2pkh]: 136, + [AddressType.p2wsh]: 172, + [AddressType.p2tr]: 172, +}; + +const outputWeightCache = new Map(); + +interface ComputedAddress { + address: string; + creditedDifficulty: number; + acceptedShareCount: number; + payoutWeight: number; + rawFairSats: number; + balanceBeforeSats: number; + targetSats: number; + payoutSats: number; + balanceAfterSats: number; + includedInCoinbase: boolean; +} + +export function buildPayoutDistribution(input: PayoutDistributionInput): PayoutDistributionResult { + const coinbaseValueSats = Math.max(0, Math.floor(input.coinbaseValueSats)); + const minOutputSats = Math.max(DUST_LIMIT_SATS, Math.floor(input.minOutputSats)); + const coinbaseWeightBudget = input.coinbaseWeightBudget > 0 + ? input.coinbaseWeightBudget + : DEFAULT_COINBASE_WEIGHT_BUDGET; + const feePercent = Number.isFinite(input.feePercent) && input.feePercent > 0 + ? input.feePercent + : 0; + const feeAddress = input.feeAddress?.trim() ?? ''; + const wantedFeeSats = Math.floor((feePercent / 100) * coinbaseValueSats); + const feeSats = feeAddress.length > 0 && wantedFeeSats >= minOutputSats ? wantedFeeSats : 0; + const minerRewardSats = coinbaseValueSats - feeSats; + + const workByAddress = new Map(); + for (const work of input.addressWork) { + if (!work.address || !Number.isFinite(work.creditedDifficulty) || work.creditedDifficulty <= 0) { + continue; + } + const existing = workByAddress.get(work.address) ?? { creditedDifficulty: 0, acceptedShareCount: 0 }; + existing.creditedDifficulty += work.creditedDifficulty; + existing.acceptedShareCount += work.acceptedShareCount; + workByAddress.set(work.address, existing); + } + + const balanceByAddress = new Map(); + for (const balance of input.balances) { + if (balance.address && Number.isFinite(balance.balanceSats) && balance.balanceSats !== 0) { + balanceByAddress.set(balance.address, Math.trunc(balance.balanceSats)); + } + } + + let totalCreditedDifficulty = 0; + for (const work of workByAddress.values()) { + totalCreditedDifficulty += work.creditedDifficulty; + } + if (coinbaseValueSats <= 0 || totalCreditedDifficulty <= 0) { + return { + entries: [], + payoutOutputs: [], + consideredAddressCount: 0, + distributedSats: 0, + unallocatedRemainderSats: coinbaseValueSats, + feeSats, + }; + } + + const computations = new Map(); + const considered = new Set([...workByAddress.keys(), ...balanceByAddress.keys()]); + + for (const address of considered) { + const work = workByAddress.get(address); + const creditedDifficulty = work?.creditedDifficulty ?? 0; + const acceptedShareCount = work?.acceptedShareCount ?? 0; + const payoutWeight = creditedDifficulty > 0 ? creditedDifficulty / totalCreditedDifficulty : 0; + const rawFairSats = Math.floor(payoutWeight * minerRewardSats); + const balanceBeforeSats = balanceByAddress.get(address) ?? 0; + const targetSats = rawFairSats + balanceBeforeSats; + computations.set(address, { + address, + creditedDifficulty, + acceptedShareCount, + payoutWeight, + rawFairSats, + balanceBeforeSats, + targetSats, + payoutSats: 0, + balanceAfterSats: targetSats, + includedInCoinbase: false, + }); + } + + const feeWeight = feeSats > 0 ? outputWeightForAddress(feeAddress) : 0; + let usedWeight = COINBASE_BASE_WEIGHT + COINBASE_WITNESS_COMMITMENT_WEIGHT + feeWeight; + const effectiveBudget = coinbaseWeightBudget - BUDGET_SAFETY_MARGIN_WEIGHT; + const eligible = [...computations.values()] + .filter(row => row.targetSats >= minOutputSats) + .sort((a, b) => b.targetSats - a.targetSats || a.address.localeCompare(b.address)); + + for (const row of eligible) { + const outputWeight = outputWeightForAddress(row.address); + if (usedWeight + outputWeight > effectiveBudget) { + continue; + } + row.includedInCoinbase = true; + row.payoutSats = row.targetSats; + row.balanceAfterSats = 0; + usedWeight += outputWeight; + } + + let distributedToMiners = 0; + for (const row of computations.values()) { + distributedToMiners += row.payoutSats; + } + + const remainder = minerRewardSats - distributedToMiners; + if (remainder > 0) { + const keptActive = [...computations.values()] + .filter(row => row.includedInCoinbase && row.creditedDifficulty > 0) + .sort((a, b) => b.creditedDifficulty - a.creditedDifficulty || a.address.localeCompare(b.address)); + if (keptActive.length > 0) { + let assigned = 0; + for (const row of keptActive) { + const bonus = Math.floor((remainder * row.creditedDifficulty) / totalCreditedDifficulty); + row.payoutSats += bonus; + row.balanceAfterSats -= bonus; + assigned += bonus; + } + const residual = remainder - assigned; + if (residual > 0) { + keptActive[0].payoutSats += residual; + keptActive[0].balanceAfterSats -= residual; + } + } + } + + const feeEntry = feeSats > 0 + ? { + address: feeAddress, + creditedDifficulty: 0, + acceptedShareCount: 0, + payoutWeight: 0, + grossPayoutSats: feeSats, + payoutSats: feeSats, + balanceBeforeSats: 0, + balanceAfterSats: 0, + includedInCoinbase: true, + rank: 1, + } + : null; + + const minerEntries = [...computations.values()] + .filter(row => row.includedInCoinbase || row.balanceBeforeSats !== row.balanceAfterSats || row.balanceAfterSats !== 0) + .sort((a, b) => { + if (a.includedInCoinbase !== b.includedInCoinbase) { + return a.includedInCoinbase ? -1 : 1; + } + return b.payoutSats - a.payoutSats || b.targetSats - a.targetSats || a.address.localeCompare(b.address); + }) + .map(row => ({ + address: row.address, + creditedDifficulty: row.creditedDifficulty, + acceptedShareCount: row.acceptedShareCount, + payoutWeight: row.payoutWeight, + grossPayoutSats: row.rawFairSats, + payoutSats: row.payoutSats, + balanceBeforeSats: row.balanceBeforeSats, + balanceAfterSats: row.balanceAfterSats, + includedInCoinbase: row.includedInCoinbase, + rank: 0, + })); + + const mergedEntries = feeEntry == null + ? minerEntries + : mergeFeeEntry(feeEntry, minerEntries); + const entries = mergedEntries + .map((entry, index) => ({ ...entry, rank: index + 1 })); + const payoutOutputs = entries.filter(entry => entry.includedInCoinbase); + const distributedSats = payoutOutputs.reduce((sum, entry) => sum + entry.payoutSats, 0); + + return { + entries, + payoutOutputs, + consideredAddressCount: considered.size, + distributedSats, + unallocatedRemainderSats: coinbaseValueSats - distributedSats, + feeSats, + }; +} + +function mergeFeeEntry( + feeEntry: PayoutDistributionEntry, + minerEntries: Omit[], +): Omit[] { + const existing = minerEntries.find(entry => entry.address === feeEntry.address && entry.includedInCoinbase); + if (existing == null) { + return [feeEntry, ...minerEntries]; + } + existing.payoutSats += feeEntry.payoutSats; + existing.grossPayoutSats += feeEntry.grossPayoutSats; + return minerEntries; +} + +export function outputWeightForAddress(address: string): number { + if (!address) { + return COINBASE_OUTPUT_WEIGHT; + } + const cached = outputWeightCache.get(address); + if (cached != null) { + return cached; + } + let weight = COINBASE_OUTPUT_WEIGHT; + try { + const info = getAddressInfo(address); + weight = OUTPUT_WEIGHT_BY_TYPE[info.type] ?? COINBASE_OUTPUT_WEIGHT; + } catch { + weight = COINBASE_OUTPUT_WEIGHT; + } + outputWeightCache.set(address, weight); + return weight; +} diff --git a/src/ORM/payout-snapshot/payout-history.entity.ts b/src/ORM/payout-snapshot/payout-history.entity.ts new file mode 100644 index 0000000..f5431b2 --- /dev/null +++ b/src/ORM/payout-snapshot/payout-history.entity.ts @@ -0,0 +1,41 @@ +import { Column, CreateDateColumn, Entity, Index, PrimaryGeneratedColumn } from 'typeorm'; + +const bigintTransformer = { + from: (value: string | number | null): number => Number(value ?? 0), + to: (value: number): number => Math.trunc(value ?? 0), +}; + +@Entity({ name: 'payout_history' }) +@Index('UQ_payout_history_block_address', ['blockHeight', 'address'], { unique: true }) +@Index('IDX_payout_history_address_created', ['address', 'createdAt']) +export class PayoutHistoryEntity { + @PrimaryGeneratedColumn({ type: 'bigint' }) + id: string; + + @Column({ type: 'bigint' }) + blockHeight: number; + + @Column({ type: 'bigint' }) + payoutSnapshotId: string; + + @Column({ length: 62, type: 'varchar' }) + address: string; + + @Column({ type: 'bigint', default: 0, transformer: bigintTransformer }) + paidSats: number; + + @Column({ type: 'decimal', default: 0 }) + percent: number; + + @Column({ type: 'bigint', default: 0, transformer: bigintTransformer }) + balanceBeforeSats: number; + + @Column({ type: 'bigint', default: 0, transformer: bigintTransformer }) + balanceAfterSats: number; + + @Column({ length: 16, type: 'varchar', default: 'coinbase' }) + rowType: 'coinbase' | 'pending'; + + @CreateDateColumn() + createdAt: Date; +} diff --git a/src/ORM/payout-snapshot/payout-snapshot-entry.entity.ts b/src/ORM/payout-snapshot/payout-snapshot-entry.entity.ts new file mode 100644 index 0000000..d429bc0 --- /dev/null +++ b/src/ORM/payout-snapshot/payout-snapshot-entry.entity.ts @@ -0,0 +1,54 @@ +import { Column, Entity, Index, JoinColumn, ManyToOne, PrimaryGeneratedColumn } from 'typeorm'; + +import { PayoutSnapshotEntity } from './payout-snapshot.entity'; + +@Entity({ name: 'payout_snapshot_entry' }) +@Index('IDX_payout_snapshot_entry_snapshot_rank', ['snapshotId', 'rank']) +@Index('IDX_payout_snapshot_entry_address', ['address']) +export class PayoutSnapshotEntryEntity { + @PrimaryGeneratedColumn({ type: 'bigint' }) + id: string; + + @Column({ type: 'bigint' }) + snapshotId: string; + + @ManyToOne(() => PayoutSnapshotEntity, snapshot => snapshot.entries, { onDelete: 'CASCADE' }) + @JoinColumn({ name: 'snapshotId' }) + snapshot: PayoutSnapshotEntity; + + @Column({ length: 62, type: 'varchar' }) + address: string; + + @Column({ type: 'decimal' }) + creditedDifficulty: number; + + @Column({ type: 'bigint' }) + acceptedShareCount: string; + + @Column({ type: 'decimal' }) + payoutWeight: number; + + @Column({ type: 'bigint' }) + grossPayoutSats: string; + + @Column({ type: 'bigint' }) + payoutSats: string; + + @Column({ type: 'bigint', default: 0 }) + balanceBeforeSats: string; + + @Column({ type: 'bigint', default: 0 }) + balanceAfterSats: string; + + @Column({ type: 'boolean', default: true }) + includedInCoinbase: boolean; + + @Column({ length: 16, type: 'varchar', default: 'coinbase' }) + rowType: 'coinbase' | 'pending'; + + @Column({ type: 'integer' }) + rank: number; + + @Column({ type: 'timestamptz', default: () => 'CURRENT_TIMESTAMP' }) + createdAt: Date; +} diff --git a/src/ORM/payout-snapshot/payout-snapshot.entity.ts b/src/ORM/payout-snapshot/payout-snapshot.entity.ts new file mode 100644 index 0000000..33f95f8 --- /dev/null +++ b/src/ORM/payout-snapshot/payout-snapshot.entity.ts @@ -0,0 +1,80 @@ +import { Column, Entity, Index, OneToMany, PrimaryGeneratedColumn } from 'typeorm'; + +import { PayoutSnapshotEntryEntity } from './payout-snapshot-entry.entity'; + +@Entity({ name: 'payout_snapshot' }) +@Index('IDX_payout_snapshot_latest', ['status', 'createdAt']) +@Index('IDX_payout_snapshot_window', ['windowEndShareIndex']) +export class PayoutSnapshotEntity { + @PrimaryGeneratedColumn({ type: 'bigint' }) + id: string; + + @Column({ length: 32, type: 'varchar' }) + method: string; + + @Column({ length: 16, type: 'varchar', default: 'finalized' }) + status: 'finalized'; + + @Column({ type: 'bigint' }) + blockHeight: number; + + @Column({ type: 'bigint' }) + coinbaseValueSats: string; + + @Column({ type: 'decimal' }) + networkDifficulty: number; + + @Column({ type: 'decimal', default: 0 }) + windowTargetDifficulty: number; + + @Column({ type: 'decimal', default: 4 }) + windowFactor: number; + + @Column({ length: 62, type: 'varchar', nullable: true }) + feeAddress?: string | null; + + @Column({ type: 'bigint', default: 0 }) + feeSats: string; + + @Column({ type: 'integer', default: 546 }) + minPayoutSats: number; + + @Column({ type: 'integer', default: 50000 }) + coinbaseWeightBudget: number; + + @Column({ type: 'bigint' }) + startBatchId: string; + + @Column({ type: 'bigint' }) + endBatchId: string; + + @Column({ type: 'bigint' }) + windowStartShareIndex: string; + + @Column({ type: 'bigint' }) + windowEndShareIndex: string; + + @Column({ type: 'decimal' }) + totalCreditedDifficulty: number; + + @Column({ type: 'bigint' }) + totalAcceptedShareCount: string; + + @Column({ type: 'integer' }) + eligibleAddressCount: number; + + @Column({ type: 'integer' }) + includedOutputCount: number; + + @Column({ type: 'bigint' }) + distributedSats: string; + + @Column({ type: 'bigint' }) + unallocatedRemainderSats: string; + + @Column({ type: 'timestamptz', default: () => 'CURRENT_TIMESTAMP' }) + createdAt: Date; + + @OneToMany(() => PayoutSnapshotEntryEntity, entry => entry.snapshot) + entries: PayoutSnapshotEntryEntity[]; +} diff --git a/src/ORM/payout-snapshot/payout-snapshot.module.ts b/src/ORM/payout-snapshot/payout-snapshot.module.ts new file mode 100644 index 0000000..b395c0d --- /dev/null +++ b/src/ORM/payout-snapshot/payout-snapshot.module.ts @@ -0,0 +1,21 @@ +import { Global, Module } from '@nestjs/common'; +import { TypeOrmModule } from '@nestjs/typeorm'; + +import { PayoutBalanceEntity } from './payout-balance.entity'; +import { PayoutHistoryEntity } from './payout-history.entity'; +import { PayoutSnapshotEntryEntity } from './payout-snapshot-entry.entity'; +import { PayoutSnapshotEntity } from './payout-snapshot.entity'; +import { PayoutSnapshotService } from './payout-snapshot.service'; + +@Global() +@Module({ + imports: [TypeOrmModule.forFeature([ + PayoutBalanceEntity, + PayoutHistoryEntity, + PayoutSnapshotEntity, + PayoutSnapshotEntryEntity, + ])], + providers: [PayoutSnapshotService], + exports: [TypeOrmModule, PayoutSnapshotService], +}) +export class PayoutSnapshotModule { } diff --git a/src/ORM/payout-snapshot/payout-snapshot.service.spec.ts b/src/ORM/payout-snapshot/payout-snapshot.service.spec.ts new file mode 100644 index 0000000..ae835e5 --- /dev/null +++ b/src/ORM/payout-snapshot/payout-snapshot.service.spec.ts @@ -0,0 +1,213 @@ +import { DataSource, EntityManager } from 'typeorm'; + +import { PayoutSnapshotService } from './payout-snapshot.service'; + +const ADDRESS_A = 'bc1qs29kyaqqc0fkvj897ke9e5xa9utljjey0y5jjn'; +const ADDRESS_B = 'bc1q99n3pu025yyu0jlywpmwzalyhm36tg5u37w20d'; + +describe('PayoutSnapshotService', () => { + let originalEnv: NodeJS.ProcessEnv; + let manager: { query: jest.Mock }; + let dataSource: { transaction: jest.Mock; query: jest.Mock }; + let service: PayoutSnapshotService; + + beforeEach(() => { + originalEnv = { ...process.env }; + process.env.PAYOUT_SNAPSHOT_ENABLED = 'true'; + process.env.PAYOUT_METHOD = 'pplns'; + process.env.PAYOUT_WINDOW_FACTOR = '4'; + process.env.PAYOUT_MAX_COINBASE_OUTPUTS = '10'; + process.env.PAYOUT_MIN_OUTPUT_SATS = '546'; + process.env.PAYOUT_FEE_PERCENT = '0'; + process.env.PAYOUT_FEE_ADDRESS = ''; + process.env.PAYOUT_COINBASE_WEIGHT_BUDGET = '50000'; + manager = { query: jest.fn() }; + dataSource = { + transaction: jest.fn(async callback => callback(manager as unknown as EntityManager)), + query: jest.fn(), + }; + service = new PayoutSnapshotService(dataSource as unknown as DataSource); + }); + + afterEach(() => { + process.env = originalEnv; + }); + + it('should create a PPLNS payout snapshot from finalized rollup batches', async () => { + manager.query + .mockResolvedValueOnce([{ + startBatchId: '10', + endBatchId: '12', + windowStartShareIndex: '1000', + windowEndShareIndex: '2000', + totalCreditedDifficulty: '100', + totalAcceptedShareCount: '5', + }]) + .mockResolvedValueOnce([]) + .mockResolvedValueOnce([ + { address: ADDRESS_A, creditedDifficulty: 60, acceptedShareCount: 3 }, + { address: ADDRESS_B, creditedDifficulty: 40, acceptedShareCount: 2 }, + ]) + .mockResolvedValueOnce([]) + .mockResolvedValueOnce([{ id: '55' }]) + .mockResolvedValueOnce([]) + .mockResolvedValueOnce([]) + .mockResolvedValueOnce([{ + id: '55', + method: 'pplns', + blockHeight: 900000, + coinbaseValueSats: '1000', + windowStartShareIndex: '1000', + windowEndShareIndex: '2000', + totalCreditedDifficulty: 100, + totalAcceptedShareCount: '5', + eligibleAddressCount: 2, + includedOutputCount: 1, + distributedSats: '1000', + unallocatedRemainderSats: '0', + }]) + .mockResolvedValueOnce([ + { address: ADDRESS_A, payoutSats: '1000' }, + ]); + + const snapshot = await service.createSnapshotForTemplate({ + blockHeight: 900000, + coinbaseValueSats: 1000, + networkDifficulty: 25, + }); + + expect(snapshot).toEqual(expect.objectContaining({ + id: '55', + method: 'pplns', + blockHeight: 900000, + totalCreditedDifficulty: 100, + totalAcceptedShareCount: 5, + distributedSats: '1000', + unallocatedRemainderSats: '0', + })); + expect(snapshot.payoutOutputs).toEqual([ + { address: ADDRESS_A, amountSats: 1000, percent: 100 }, + ]); + expect(manager.query).toHaveBeenCalledTimes(9); + expect(manager.query.mock.calls[0][1]).toEqual([100]); + }); + + it('should not create a snapshot when there is no finalized rollup work', async () => { + manager.query.mockResolvedValueOnce([{ + startBatchId: null, + endBatchId: null, + windowStartShareIndex: null, + windowEndShareIndex: null, + totalCreditedDifficulty: '0', + totalAcceptedShareCount: '0', + }]); + + await expect(service.createSnapshotForTemplate({ + blockHeight: 900000, + coinbaseValueSats: 1000, + networkDifficulty: 25, + })).resolves.toBeNull(); + expect(manager.query).toHaveBeenCalledTimes(1); + }); + + it('should reuse an existing snapshot for the same template and rollup window', async () => { + manager.query + .mockResolvedValueOnce([{ + startBatchId: '10', + endBatchId: '12', + windowStartShareIndex: '1000', + windowEndShareIndex: '2000', + totalCreditedDifficulty: '100', + totalAcceptedShareCount: '5', + }]) + .mockResolvedValueOnce([{ id: '55' }]) + .mockResolvedValueOnce([{ + id: '55', + method: 'pplns', + blockHeight: 900000, + coinbaseValueSats: '1000', + windowStartShareIndex: '1000', + windowEndShareIndex: '2000', + totalCreditedDifficulty: 100, + totalAcceptedShareCount: '5', + eligibleAddressCount: 2, + includedOutputCount: 1, + distributedSats: '1000', + unallocatedRemainderSats: '0', + }]) + .mockResolvedValueOnce([ + { address: ADDRESS_A, payoutSats: '1000' }, + ]); + + const snapshot = await service.createSnapshotForTemplate({ + blockHeight: 900000, + coinbaseValueSats: 1000, + networkDifficulty: 25, + }); + + expect(snapshot.id).toBe('55'); + expect(snapshot.payoutOutputs).toEqual([ + { address: ADDRESS_A, amountSats: 1000, percent: 100 }, + ]); + expect(manager.query).toHaveBeenCalledTimes(4); + }); + + it('should finalize snapshot balances and payout history for a found block', async () => { + manager.query + .mockResolvedValueOnce([{ id: '55', coinbaseValueSats: '1000' }]) + .mockResolvedValueOnce([]) + .mockResolvedValueOnce([ + { + address: ADDRESS_A, + payoutSats: '1000', + balanceBeforeSats: '0', + balanceAfterSats: '-400', + creditedDifficulty: 60, + includedInCoinbase: true, + rowType: 'coinbase', + }, + { + address: ADDRESS_B, + payoutSats: '0', + balanceBeforeSats: '0', + balanceAfterSats: '400', + creditedDifficulty: 40, + includedInCoinbase: false, + rowType: 'pending', + }, + ]) + .mockResolvedValueOnce([]) + .mockResolvedValueOnce([]) + .mockResolvedValueOnce([]) + .mockResolvedValueOnce([]); + + await expect(service.finalizeSnapshotForBlock({ + payoutSnapshotId: '55', + blockHeight: 900000, + blockSubmissionResult: 'SUCCESS!', + })).resolves.toEqual({ + finalized: true, + payoutSnapshotId: '55', + historyRows: 2, + balanceRows: 2, + }); + expect(manager.query).toHaveBeenCalledTimes(7); + }); + + it('should skip finalization when a block was already processed', async () => { + manager.query + .mockResolvedValueOnce([{ id: '55', coinbaseValueSats: '1000' }]) + .mockResolvedValueOnce([{ id: '99' }]); + + await expect(service.finalizeSnapshotForBlock({ + payoutSnapshotId: '55', + blockHeight: 900000, + blockSubmissionResult: 'SUCCESS!', + })).resolves.toEqual({ + finalized: false, + reason: 'already-finalized', + payoutSnapshotId: '55', + }); + expect(manager.query).toHaveBeenCalledTimes(2); + }); +}); diff --git a/src/ORM/payout-snapshot/payout-snapshot.service.ts b/src/ORM/payout-snapshot/payout-snapshot.service.ts new file mode 100644 index 0000000..7e9a119 --- /dev/null +++ b/src/ORM/payout-snapshot/payout-snapshot.service.ts @@ -0,0 +1,617 @@ +import { Injectable } from '@nestjs/common'; +import { DataSource, EntityManager } from 'typeorm'; + +import type { AddressObject } from '../../models/MiningJob'; +import { + buildPayoutDistribution, + PayoutAddressWork, + PayoutBalanceInput, + PayoutDistributionEntry, +} from './payout-distribution'; + +export interface PayoutSnapshotForTemplate { + id: string; + method: string; + blockHeight: number; + coinbaseValueSats: string; + windowStartShareIndex: string; + windowEndShareIndex: string; + totalCreditedDifficulty: number; + totalAcceptedShareCount: number; + eligibleAddressCount: number; + includedOutputCount: number; + distributedSats: string; + unallocatedRemainderSats: string; + payoutOutputs: AddressObject[]; +} + +export interface PayoutFinalizationResult { + finalized: boolean; + reason?: 'disabled' | 'missing-snapshot' | 'already-finalized' | 'duplicate'; + payoutSnapshotId?: string; + historyRows?: number; + balanceRows?: number; +} + +const DEFAULT_MAX_COINBASE_OUTPUTS = 10; +const DEFAULT_MIN_OUTPUT_SATS = 546; +const DEFAULT_PAYOUT_METHOD = 'pplns'; +const DEFAULT_PAYOUT_WINDOW_FACTOR = 4; +const DEFAULT_COINBASE_WEIGHT_BUDGET = 50_000; + +@Injectable() +export class PayoutSnapshotService { + private readonly snapshotsEnabled = this.readBoolean('PAYOUT_SNAPSHOT_ENABLED', true); + private readonly method = process.env.PAYOUT_METHOD ?? DEFAULT_PAYOUT_METHOD; + private readonly windowFactor = this.readPositiveNumber('PAYOUT_WINDOW_FACTOR', DEFAULT_PAYOUT_WINDOW_FACTOR); + private readonly maxCoinbaseOutputs = this.readPositiveInt('PAYOUT_MAX_COINBASE_OUTPUTS', DEFAULT_MAX_COINBASE_OUTPUTS); + private readonly minOutputSats = this.readNonNegativeInt('PAYOUT_MIN_OUTPUT_SATS', DEFAULT_MIN_OUTPUT_SATS); + private readonly feeAddress = process.env.PAYOUT_FEE_ADDRESS?.trim() ?? ''; + private readonly feePercent = this.readNonNegativeNumber('PAYOUT_FEE_PERCENT', 0); + private readonly coinbaseWeightBudget = this.readPositiveInt('PAYOUT_COINBASE_WEIGHT_BUDGET', DEFAULT_COINBASE_WEIGHT_BUDGET); + + constructor( + private readonly dataSource: DataSource, + ) { } + + public async createSnapshotForTemplate(input: { + blockHeight: number; + coinbaseValueSats: number; + networkDifficulty: number; + }): Promise { + if (!this.snapshotsEnabled || input.coinbaseValueSats <= 0 || input.networkDifficulty <= 0) { + return null; + } + + return this.dataSource.transaction(async manager => { + const windowTargetDifficulty = input.networkDifficulty * this.windowFactor; + const window = await this.getPplnsWindow(manager, windowTargetDifficulty); + if (window == null || this.toNumber(window.totalCreditedDifficulty) <= 0) { + return null; + } + + const existing = await this.getExistingSnapshot(manager, { + blockHeight: input.blockHeight, + coinbaseValueSats: input.coinbaseValueSats, + windowEndShareIndex: window.windowEndShareIndex, + }); + if (existing != null) { + return existing; + } + + const addressWork = await this.getAddressWork(manager, window); + const balances = await this.getOpenBalances(manager); + if (addressWork.length === 0) { + return null; + } + + const distribution = buildPayoutDistribution({ + addressWork, + balances, + coinbaseValueSats: input.coinbaseValueSats, + feeAddress: this.feeAddress, + feePercent: this.feePercent, + minOutputSats: this.minOutputSats, + coinbaseWeightBudget: this.coinbaseWeightBudget, + }); + const entries = this.limitCoinbaseOutputs(distribution.entries); + if (entries.every(entry => !entry.includedInCoinbase)) { + return null; + } + + const includedOutputCount = entries.filter(entry => entry.includedInCoinbase).length; + const distributedSats = entries + .filter(entry => entry.includedInCoinbase) + .reduce((sum, entry) => sum + entry.payoutSats, 0); + const unallocatedRemainderSats = input.coinbaseValueSats - distributedSats; + const [snapshotRow] = await manager.query(` + INSERT INTO "payout_snapshot" ( + "method", + "status", + "blockHeight", + "coinbaseValueSats", + "networkDifficulty", + "windowTargetDifficulty", + "windowFactor", + "feeAddress", + "feeSats", + "minPayoutSats", + "coinbaseWeightBudget", + "startBatchId", + "endBatchId", + "windowStartShareIndex", + "windowEndShareIndex", + "totalCreditedDifficulty", + "totalAcceptedShareCount", + "eligibleAddressCount", + "includedOutputCount", + "distributedSats", + "unallocatedRemainderSats" + ) VALUES ( + $1, + 'finalized', + $2::bigint, + $3::bigint, + $4::numeric, + $5::numeric, + $6::numeric, + NULLIF($7, ''), + $8::bigint, + $9::int, + $10::int, + $11::bigint, + $12::bigint, + $13::bigint, + $14::bigint, + $15::numeric, + $16::bigint, + $17::int, + $18::int, + $19::bigint, + $20::bigint + ) + RETURNING "id"::text AS "id" + `, [ + this.method, + input.blockHeight, + input.coinbaseValueSats.toString(), + input.networkDifficulty, + windowTargetDifficulty, + this.windowFactor, + this.feeAddress, + distribution.feeSats.toString(), + this.minOutputSats, + this.coinbaseWeightBudget, + window.startBatchId, + window.endBatchId, + window.windowStartShareIndex, + window.windowEndShareIndex, + window.totalCreditedDifficulty, + window.totalAcceptedShareCount, + distribution.consideredAddressCount, + includedOutputCount, + distributedSats.toString(), + unallocatedRemainderSats.toString(), + ]); + + for (const entry of entries) { + await this.insertSnapshotEntry(manager, snapshotRow.id, entry); + } + + return this.getSnapshotById(manager, snapshotRow.id); + }); + } + + public async finalizeSnapshotForBlock(input: { + payoutSnapshotId?: string | null; + blockHeight: number; + blockSubmissionResult?: string | null; + }): Promise { + if (!this.snapshotsEnabled) { + return { finalized: false, reason: 'disabled' }; + } + if (!this.isSuccessfulBlockSubmission(input.blockSubmissionResult)) { + return { finalized: false, reason: 'missing-snapshot' }; + } + if (input.payoutSnapshotId == null) { + return { finalized: false, reason: 'missing-snapshot' }; + } + + return this.dataSource.transaction(async manager => { + const [snapshot] = await manager.query(` + SELECT "id"::text AS "id", "coinbaseValueSats"::text AS "coinbaseValueSats" + FROM "payout_snapshot" + WHERE "id" = $1::bigint + LIMIT 1 + `, [input.payoutSnapshotId]); + if (snapshot == null) { + return { finalized: false, reason: 'missing-snapshot' }; + } + + const [existing] = await manager.query(` + SELECT "id"::text AS "id" + FROM "payout_history" + WHERE "blockHeight" = $1::bigint + LIMIT 1 + `, [input.blockHeight]); + if (existing != null) { + return { + finalized: false, + reason: 'already-finalized', + payoutSnapshotId: input.payoutSnapshotId, + }; + } + + const entries = await manager.query(` + SELECT + "address", + "payoutSats"::text AS "payoutSats", + "balanceBeforeSats"::text AS "balanceBeforeSats", + "balanceAfterSats"::text AS "balanceAfterSats", + "creditedDifficulty"::float AS "creditedDifficulty", + "includedInCoinbase", + "rowType" + FROM "payout_snapshot_entry" + WHERE "snapshotId" = $1::bigint + ORDER BY "rank" + `, [input.payoutSnapshotId]); + if (entries.length === 0) { + return { finalized: false, reason: 'missing-snapshot' }; + } + + let balanceRows = 0; + let historyRows = 0; + const coinbaseValue = this.toNumber(snapshot.coinbaseValueSats); + for (const entry of entries) { + const payoutSats = this.toNumber(entry.payoutSats); + const balanceBeforeSats = this.toNumber(entry.balanceBeforeSats); + const balanceAfterSats = this.toNumber(entry.balanceAfterSats); + const isCoinbase = entry.includedInCoinbase === true; + const wasActive = this.toNumber(entry.creditedDifficulty) > 0; + + await manager.query(` + INSERT INTO "payout_balance" ( + "address", + "balanceSats", + "totalPaidSats", + "lastAcceptedShareAt", + "updatedAt" + ) VALUES ( + $1, + $2::bigint, + $3::bigint, + CASE WHEN $4::boolean THEN NOW() ELSE NULL END, + NOW() + ) + ON CONFLICT ("address") DO UPDATE SET + "balanceSats" = EXCLUDED."balanceSats", + "totalPaidSats" = "payout_balance"."totalPaidSats" + EXCLUDED."totalPaidSats", + "lastAcceptedShareAt" = CASE + WHEN $4::boolean THEN NOW() + ELSE "payout_balance"."lastAcceptedShareAt" + END, + "updatedAt" = NOW() + `, [ + entry.address, + balanceAfterSats.toString(), + isCoinbase ? payoutSats.toString() : '0', + wasActive, + ]); + balanceRows++; + + await manager.query(` + INSERT INTO "payout_history" ( + "blockHeight", + "payoutSnapshotId", + "address", + "paidSats", + "percent", + "balanceBeforeSats", + "balanceAfterSats", + "rowType" + ) VALUES ( + $1::bigint, + $2::bigint, + $3, + $4::bigint, + $5::numeric, + $6::bigint, + $7::bigint, + $8 + ) + ON CONFLICT ("blockHeight", "address") DO NOTHING + `, [ + input.blockHeight, + input.payoutSnapshotId, + entry.address, + payoutSats.toString(), + coinbaseValue > 0 && isCoinbase ? (payoutSats / coinbaseValue) * 100 : 0, + balanceBeforeSats.toString(), + balanceAfterSats.toString(), + isCoinbase ? 'coinbase' : 'pending', + ]); + historyRows++; + } + + return { + finalized: true, + payoutSnapshotId: input.payoutSnapshotId, + historyRows, + balanceRows, + }; + }).catch(error => { + if (error?.code === '23505') { + return { + finalized: false, + reason: 'duplicate', + payoutSnapshotId: input.payoutSnapshotId ?? undefined, + }; + } + throw error; + }); + } + + public async getLatestSnapshot(): Promise { + const [snapshot] = await this.dataSource.query(` + SELECT "id"::text AS "id" + FROM "payout_snapshot" + WHERE "status" = 'finalized' + ORDER BY "createdAt" DESC, "id" DESC + LIMIT 1 + `); + if (snapshot?.id == null) { + return null; + } + return this.dataSource.transaction(manager => this.getSnapshotById(manager, snapshot.id)); + } + + private async getPplnsWindow(manager: EntityManager, windowTargetDifficulty: number): Promise<{ + startBatchId: string; + endBatchId: string; + windowStartShareIndex: string; + windowEndShareIndex: string; + totalCreditedDifficulty: string; + totalAcceptedShareCount: string; + } | null> { + const [window] = await manager.query(` + WITH ordered AS MATERIALIZED ( + SELECT + "id", + "startShareIndex", + "endShareIndex", + "acceptedShareCount", + "creditedDifficulty", + COALESCE( + SUM("creditedDifficulty") OVER ( + ORDER BY "endShareIndex" DESC + ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING + ), + 0 + ) AS "difficultyAfter" + FROM "share_rollup_batch" + WHERE "status" = 'finalized' + ), + selected AS MATERIALIZED ( + SELECT * + FROM ordered + WHERE "difficultyAfter" < $1::numeric + ) + SELECT + MIN("id")::text AS "startBatchId", + MAX("id")::text AS "endBatchId", + MIN("startShareIndex")::text AS "windowStartShareIndex", + MAX("endShareIndex")::text AS "windowEndShareIndex", + COALESCE(SUM("creditedDifficulty"), 0)::numeric AS "totalCreditedDifficulty", + COALESCE(SUM("acceptedShareCount"), 0)::bigint AS "totalAcceptedShareCount" + FROM selected + `, [windowTargetDifficulty]); + + if (window?.startBatchId == null || window?.windowEndShareIndex == null) { + return null; + } + return window; + } + + private async getAddressWork( + manager: EntityManager, + window: { windowStartShareIndex: string; windowEndShareIndex: string }, + ): Promise { + const rows = await manager.query(` + SELECT + s."address", + COALESCE(SUM(s."creditedDifficulty"), 0)::float AS "creditedDifficulty", + COALESCE(SUM(s."acceptedShareCount"), 0)::int AS "acceptedShareCount" + FROM "share_rollup_batch_summary" s + JOIN "share_rollup_batch" b ON b."id" = s."batchId" + WHERE b."status" = 'finalized' + AND b."endShareIndex" >= $1::bigint + AND b."endShareIndex" <= $2::bigint + GROUP BY s."address" + HAVING COALESCE(SUM(s."creditedDifficulty"), 0) > 0 + `, [ + window.windowStartShareIndex, + window.windowEndShareIndex, + ]); + return rows.map(row => ({ + address: row.address, + creditedDifficulty: this.toNumber(row.creditedDifficulty), + acceptedShareCount: this.toNumber(row.acceptedShareCount), + })); + } + + private async getOpenBalances(manager: EntityManager): Promise { + const rows = await manager.query(` + SELECT "address", "balanceSats"::text AS "balanceSats" + FROM "payout_balance" + WHERE "balanceSats" != 0 + `); + return rows.map(row => ({ + address: row.address, + balanceSats: this.toNumber(row.balanceSats), + })); + } + + private async getExistingSnapshot( + manager: EntityManager, + input: { blockHeight: number; coinbaseValueSats: number; windowEndShareIndex: string }, + ): Promise { + const [snapshot] = await manager.query(` + SELECT "id"::text AS "id" + FROM "payout_snapshot" + WHERE "method" = $1 + AND "status" = 'finalized' + AND "blockHeight" = $2::bigint + AND "coinbaseValueSats" = $3::bigint + AND "windowEndShareIndex" = $4::bigint + ORDER BY "id" DESC + LIMIT 1 + `, [ + this.method, + input.blockHeight, + input.coinbaseValueSats.toString(), + input.windowEndShareIndex, + ]); + + if (snapshot?.id == null) { + return null; + } + return this.getSnapshotById(manager, snapshot.id); + } + + private async getSnapshotById(manager: EntityManager, id: string): Promise { + const [snapshot] = await manager.query(` + SELECT + "id"::text AS "id", + "method", + "blockHeight"::int AS "blockHeight", + "coinbaseValueSats"::text AS "coinbaseValueSats", + "windowStartShareIndex"::text AS "windowStartShareIndex", + "windowEndShareIndex"::text AS "windowEndShareIndex", + "totalCreditedDifficulty"::float AS "totalCreditedDifficulty", + "totalAcceptedShareCount"::bigint AS "totalAcceptedShareCount", + "eligibleAddressCount", + "includedOutputCount", + "distributedSats"::text AS "distributedSats", + "unallocatedRemainderSats"::text AS "unallocatedRemainderSats" + FROM "payout_snapshot" + WHERE "id" = $1::bigint + `, [id]); + if (snapshot == null) { + return null; + } + + const entries = await manager.query(` + SELECT + "address", + "payoutSats"::text AS "payoutSats" + FROM "payout_snapshot_entry" + WHERE "snapshotId" = $1::bigint + AND "includedInCoinbase" = true + ORDER BY "rank" + `, [id]); + const coinbaseValue = Number(snapshot.coinbaseValueSats); + + return { + id: snapshot.id, + method: snapshot.method, + blockHeight: this.toNumber(snapshot.blockHeight), + coinbaseValueSats: snapshot.coinbaseValueSats, + windowStartShareIndex: snapshot.windowStartShareIndex, + windowEndShareIndex: snapshot.windowEndShareIndex, + totalCreditedDifficulty: this.toNumber(snapshot.totalCreditedDifficulty), + totalAcceptedShareCount: this.toNumber(snapshot.totalAcceptedShareCount), + eligibleAddressCount: this.toNumber(snapshot.eligibleAddressCount), + includedOutputCount: this.toNumber(snapshot.includedOutputCount), + distributedSats: snapshot.distributedSats, + unallocatedRemainderSats: snapshot.unallocatedRemainderSats, + payoutOutputs: entries.map(entry => ({ + address: entry.address, + amountSats: this.toNumber(entry.payoutSats), + percent: coinbaseValue > 0 + ? (this.toNumber(entry.payoutSats) / coinbaseValue) * 100 + : 0, + })), + }; + } + + private async insertSnapshotEntry(manager: EntityManager, snapshotId: string, entry: PayoutDistributionEntry): Promise { + await manager.query(` + INSERT INTO "payout_snapshot_entry" ( + "snapshotId", + "address", + "creditedDifficulty", + "acceptedShareCount", + "payoutWeight", + "grossPayoutSats", + "payoutSats", + "balanceBeforeSats", + "balanceAfterSats", + "includedInCoinbase", + "rowType", + "rank" + ) VALUES ( + $1::bigint, + $2, + $3::numeric, + $4::bigint, + $5::numeric, + $6::bigint, + $7::bigint, + $8::bigint, + $9::bigint, + $10::boolean, + $11, + $12::int + ) + `, [ + snapshotId, + entry.address, + entry.creditedDifficulty, + entry.acceptedShareCount, + entry.payoutWeight, + entry.grossPayoutSats, + entry.payoutSats, + entry.balanceBeforeSats, + entry.balanceAfterSats, + entry.includedInCoinbase, + entry.includedInCoinbase ? 'coinbase' : 'pending', + entry.rank, + ]); + } + + private limitCoinbaseOutputs(entries: PayoutDistributionEntry[]): PayoutDistributionEntry[] { + let included = 0; + return entries.map(entry => { + if (!entry.includedInCoinbase) { + return entry; + } + included++; + if (included <= this.maxCoinbaseOutputs) { + return entry; + } + return { + ...entry, + payoutSats: 0, + balanceAfterSats: entry.balanceAfterSats + entry.payoutSats, + includedInCoinbase: false, + }; + }); + } + + private isSuccessfulBlockSubmission(result?: string | null): boolean { + return result == null || result === 'SUCCESS!'; + } + + private toNumber(value: unknown): number { + const parsed = Number(value ?? 0); + return Number.isFinite(parsed) ? parsed : 0; + } + + private readPositiveInt(name: string, defaultValue: number): number { + const value = Number(process.env[name]); + return Number.isInteger(value) && value > 0 ? value : defaultValue; + } + + private readNonNegativeInt(name: string, defaultValue: number): number { + const value = Number(process.env[name]); + return Number.isInteger(value) && value >= 0 ? value : defaultValue; + } + + private readPositiveNumber(name: string, defaultValue: number): number { + const value = Number(process.env[name]); + return Number.isFinite(value) && value > 0 ? value : defaultValue; + } + + private readNonNegativeNumber(name: string, defaultValue: number): number { + const value = Number(process.env[name]); + return Number.isFinite(value) && value >= 0 ? value : defaultValue; + } + + private readBoolean(name: string, defaultValue: boolean): boolean { + const value = process.env[name]?.toLowerCase(); + if (value == null || value.length === 0) { + return defaultValue; + } + return value === 'true' || value === '1' || value === 'yes'; + } +} diff --git a/src/ORM/share-accounting/share-accounting.service.spec.ts b/src/ORM/share-accounting/share-accounting.service.spec.ts index f19a65d..0231d76 100644 --- a/src/ORM/share-accounting/share-accounting.service.spec.ts +++ b/src/ORM/share-accounting/share-accounting.service.spec.ts @@ -59,6 +59,26 @@ describe('ShareAccountingService', () => { })]); }); + it('should accept TLS SV1 protocol labels for accounting', async () => { + const repository = { + create: jest.fn(value => value), + insert: jest.fn().mockResolvedValue({}), + }; + process.env.SHARE_ACCOUNTING_BATCH_SIZE = '1'; + const service = new ShareAccountingService(repository as any); + + await expect(service.recordAcceptedShare({ + ...buildRecord('sv1-tls'), + protocol: 'sv1_tls', + })).resolves.toEqual(expect.objectContaining({ + protocol: 'sv1_tls', + })); + + expect(repository.insert).toHaveBeenCalledWith([expect.objectContaining({ + protocol: 'sv1_tls', + })]); + }); + it('should batch accepted share inserts until the batch size is reached', async () => { process.env.SHARE_ACCOUNTING_BATCH_SIZE = '2'; process.env.SHARE_ACCOUNTING_FLUSH_INTERVAL_MS = '1000'; diff --git a/src/ORM/share-accounting/share-accounting.service.ts b/src/ORM/share-accounting/share-accounting.service.ts index d3c836a..34d5cc0 100644 --- a/src/ORM/share-accounting/share-accounting.service.ts +++ b/src/ORM/share-accounting/share-accounting.service.ts @@ -6,7 +6,9 @@ import { AcceptedShareEntity } from '../accepted-share/accepted-share.entity'; import { RedisMessagingService } from '../../services/redis-messaging.service'; export interface AcceptedShareRecord { - protocol: 'sv1' | 'sv2'; + protocol: 'sv1' | 'sv1_tls' | 'sv2' | 'sv2_jdp' | 'datum'; + workSource?: 'pool_template' | 'miner_template'; + workProtocol?: 'pool' | 'sv2_jdp' | 'datum'; acceptedAt?: Date; address: string; clientName: string; @@ -121,6 +123,8 @@ export class ShareAccountingService implements OnModuleInit, OnModuleDestroy { public async recordAcceptedShare(record: AcceptedShareRecord): Promise { const acceptedShare = this.acceptedShareRepository.create({ ...record, + workSource: record.workSource ?? 'pool_template', + workProtocol: record.workProtocol ?? 'pool', acceptedAt: record.acceptedAt ?? new Date(), nonce: record.nonce.toString(), ntime: record.ntime.toString(), diff --git a/src/app.module.ts b/src/app.module.ts index 50f6e29..c65fbf7 100644 --- a/src/app.module.ts +++ b/src/app.module.ts @@ -14,6 +14,7 @@ import { AddressSettingsModule } from './ORM/address-settings/address-settings.m import { BlocksModule } from './ORM/blocks/blocks.module'; import { ClientStatisticsModule } from './ORM/client-statistics/client-statistics.module'; import { ClientModule } from './ORM/client/client.module'; +import { PayoutSnapshotModule } from './ORM/payout-snapshot/payout-snapshot.module'; import { RpcBlocksModule } from './ORM/rpc-block/rpc-block.module'; import { ShareAccountingModule } from './ORM/share-accounting/share-accounting.module'; import { TelegramSubscriptionsModule } from './ORM/telegram-subscriptions/telegram-subscriptions.module'; @@ -23,12 +24,18 @@ import { BitcoinRpcService } from './services/bitcoin-rpc.service'; import { BraiinsService } from './services/braiins.service'; import { BTCPayService } from './services/btc-pay.service'; import { DiscordService } from './services/discord.service'; +import { CustomWorkService } from './services/custom-work.service'; +import { DatumService } from './services/datum.service'; import { NotificationService } from './services/notification.service'; import { RedisMessagingModule } from './services/redis-messaging.module'; import { StratumV1JobsService } from './services/stratum-v1-jobs.service'; import { StratumV1Service } from './services/stratum-v1.service'; +import { Sv2JobDeclarationRegistryService } from './services/sv2-job-declaration-registry.service'; +import { Sv2JobDeclarationService } from './services/sv2-job-declaration.service'; +import { Sv2TemplateDistributionService } from './services/sv2-template-distribution.service'; import { StratumV2Service } from './services/stratum-v2.service'; import { TelegramService } from './services/telegram.service'; +import { TemplateProviderService } from './services/template-provider.service'; const ORMModules = [ @@ -39,7 +46,8 @@ const ORMModules = [ BlocksModule, RpcBlocksModule, UserAgentReportModule, - ShareAccountingModule + ShareAccountingModule, + PayoutSnapshotModule ] @Module({ @@ -82,6 +90,12 @@ const ORMModules = [ BitcoinAddressValidator, StratumV1JobsService, StratumV2Service, + TemplateProviderService, + Sv2JobDeclarationRegistryService, + Sv2JobDeclarationService, + Sv2TemplateDistributionService, + CustomWorkService, + DatumService, BTCPayService, BraiinsService ], diff --git a/src/database.config.ts b/src/database.config.ts index fd22222..bc68597 100644 --- a/src/database.config.ts +++ b/src/database.config.ts @@ -11,11 +11,18 @@ import { CurrentRoundBestShareIndex1780897600000 } from './ORM/_migrations/Curre import { CurrentRoundWorkRollup1780899000000 } from './ORM/_migrations/CurrentRoundWorkRollup1780899000000'; import { ShareRollupBatches1780962600000 } from './ORM/_migrations/ShareRollupBatches1780962600000'; import { AcceptedShareRetentionCompression1780966200000 } from './ORM/_migrations/AcceptedShareRetentionCompression1780966200000'; +import { PayoutSnapshots1780969800000 } from './ORM/_migrations/PayoutSnapshots1780969800000'; +import { AcceptedShareProtocolMetadata1781130600000 } from './ORM/_migrations/AcceptedShareProtocolMetadata1781130600000'; +import { BlocksSubmissionMetadata1781220000000 } from './ORM/_migrations/BlocksSubmissionMetadata1781220000000'; import { UserAgentReportView } from './ORM/_views/user-agent-report/user-agent-report.view'; import { AcceptedShareEntity } from './ORM/accepted-share/accepted-share.entity'; import { AddressSettingsEntity } from './ORM/address-settings/address-settings.entity'; import { BlocksEntity } from './ORM/blocks/blocks.entity'; import { ClientEntity } from './ORM/client/client.entity'; +import { PayoutBalanceEntity } from './ORM/payout-snapshot/payout-balance.entity'; +import { PayoutHistoryEntity } from './ORM/payout-snapshot/payout-history.entity'; +import { PayoutSnapshotEntity } from './ORM/payout-snapshot/payout-snapshot.entity'; +import { PayoutSnapshotEntryEntity } from './ORM/payout-snapshot/payout-snapshot-entry.entity'; import { RpcBlockEntity } from './ORM/rpc-block/rpc-block.entity'; import { TelegramSubscriptionsEntity } from './ORM/telegram-subscriptions/telegram-subscriptions.entity'; @@ -27,6 +34,10 @@ export const databaseEntities = [ TelegramSubscriptionsEntity, UserAgentReportView, AcceptedShareEntity, + PayoutBalanceEntity, + PayoutHistoryEntity, + PayoutSnapshotEntity, + PayoutSnapshotEntryEntity, ]; export const databaseMigrations = [ @@ -40,6 +51,9 @@ export const databaseMigrations = [ CurrentRoundWorkRollup1780899000000, ShareRollupBatches1780962600000, AcceptedShareRetentionCompression1780966200000, + PayoutSnapshots1780969800000, + AcceptedShareProtocolMetadata1781130600000, + BlocksSubmissionMetadata1781220000000, ]; export function createDatabaseOptions(env: NodeJS.ProcessEnv): TypeOrmModuleOptions & DataSourceOptions { diff --git a/src/models/MiningJob.spec.ts b/src/models/MiningJob.spec.ts index 4df1e87..73f906d 100644 --- a/src/models/MiningJob.spec.ts +++ b/src/models/MiningJob.spec.ts @@ -64,6 +64,25 @@ describe('MiningJob', () => { .toContain(`${extraNonce1}${extraNonce2}`); }); + it('should support exact satoshi payout outputs', () => { + const firstAmount = Math.floor(jobTemplate.blockData.coinbasevalue / 3); + const secondAmount = jobTemplate.blockData.coinbasevalue - firstAmount; + const exactPayoutJob = new MiningJob( + bitcoinjs.networks.testnet, + '2', + [ + { address: 'tb1qumezefzdeqqwn5zfvgdrhxjzc5ylr39uhuxcz4', amountSats: firstAmount }, + { address: 'tb1qumezefzdeqqwn5zfvgdrhxjzc5ylr39uhuxcz4', amountSats: secondAmount }, + ], + jobTemplate + ); + const coinbase = exactPayoutJob.cloneCoinbaseTransaction(); + + expect(coinbase.outs[0].value).toBe(firstAmount); + expect(coinbase.outs[1].value).toBe(secondAmount); + expect(coinbase.outs[2].value).toBe(0); + }); + it('should update block nonce, timestamp, version mask, and coinbase script', () => { const extraNonce1 = '57a6f098'; const extraNonce2 = 'c708000000000000'; diff --git a/src/models/MiningJob.ts b/src/models/MiningJob.ts index 05b3b95..1ecaa7c 100644 --- a/src/models/MiningJob.ts +++ b/src/models/MiningJob.ts @@ -8,9 +8,10 @@ import { IMiningNotify } from './stratum-messages/IMiningNotify'; import { TOTAL_EXTRANONCE_SIZE_BYTES } from './stratum.constants'; -interface AddressObject { +export interface AddressObject { address: string; - percent: number; + percent?: number; + amountSats?: number; } export class MiningJob { @@ -177,7 +178,9 @@ export class MiningJob { let rewardBalance = reward; addresses.forEach(recipientAddress => { - const amount = Math.floor((recipientAddress.percent / 100) * reward); + const amount = recipientAddress.amountSats == null + ? Math.floor(((recipientAddress.percent ?? 0) / 100) * reward) + : recipientAddress.amountSats; rewardBalance -= amount; coinbaseTransaction.addOutput(this.getPaymentScript(recipientAddress.address), amount); }) diff --git a/src/models/StratumV1Client.spec.ts b/src/models/StratumV1Client.spec.ts index 940b6c0..aeea309 100644 --- a/src/models/StratumV1Client.spec.ts +++ b/src/models/StratumV1Client.spec.ts @@ -646,6 +646,7 @@ describe('StratumV1Client', () => { }); it('should submit and persist found blocks', async () => { + (bitcoinRpcService.SUBMIT_BLOCK as jest.Mock).mockResolvedValue('SUCCESS!'); jest.spyOn(client as any, 'write').mockImplementation((data) => Promise.resolve(true)); jest.spyOn(client as any, 'calculateDifficulty').mockReturnValue({ submissionDifficulty: Number.MAX_SAFE_INTEGER, diff --git a/src/models/StratumV1Client.ts b/src/models/StratumV1Client.ts index aa2beb0..9d9304a 100644 --- a/src/models/StratumV1Client.ts +++ b/src/models/StratumV1Client.ts @@ -11,6 +11,7 @@ import { AddressSettingsService } from '../ORM/address-settings/address-settings import { BlocksService } from '../ORM/blocks/blocks.service'; import { ClientEntity } from '../ORM/client/client.entity'; import { ClientService } from '../ORM/client/client.service'; +import { PayoutSnapshotService } from '../ORM/payout-snapshot/payout-snapshot.service'; import { ShareAccountingService } from '../ORM/share-accounting/share-accounting.service'; import { BitcoinRpcService } from '../services/bitcoin-rpc.service'; import { NotificationService } from '../services/notification.service'; @@ -21,7 +22,7 @@ import { hash256 } from '../utils/hash.utils'; import { eRequestMethod } from './enums/eRequestMethod'; import { eResponseMethod } from './enums/eResponseMethod'; import { eStratumErrorCode } from './enums/eStratumErrorCode'; -import { MiningJob } from './MiningJob'; +import { AddressObject, MiningJob } from './MiningJob'; import { AuthorizationMessage } from './stratum-messages/AuthorizationMessage'; import { ConfigurationMessage } from './stratum-messages/ConfigurationMessage'; import { MiningSubmitMessage } from './stratum-messages/MiningSubmitMessage'; @@ -79,7 +80,9 @@ export class StratumV1Client { private readonly configService: ConfigService, private readonly addressSettingsService: AddressSettingsService, private readonly shareAccountingService?: ShareAccountingService, - private readonly redisMessagingService?: RedisMessagingService + private readonly redisMessagingService?: RedisMessagingService, + private readonly payoutSnapshotService?: PayoutSnapshotService, + private readonly accountingProtocol: 'sv1' | 'sv1_tls' = 'sv1', ) { this.socketDataHandler = (data: Buffer) => { @@ -445,9 +448,7 @@ export class StratumV1Client { private async sendNewMiningJob(jobTemplate: IJobTemplate) { - let payoutInformation= [ - { address: this.clientAuthorization.address, percent: 100 } - ]; + let payoutInformation = this.getPayoutInformation(jobTemplate, this.clientAuthorization.address); // const devFeeAddress = this.configService.get('DEV_FEE_ADDRESS'); // //50Th/s // this.noFee = false; @@ -648,19 +649,26 @@ export class StratumV1Client { minerAddress: this.clientAuthorization.address, worker: this.clientAuthorization.worker, sessionId: this.extraNonceAndSessionId, - blockData: blockHex + blockData: blockHex, + blockSubmissionResult, + payoutSnapshotId: jobTemplate.blockData.payoutSnapshotId ?? null, + }); + await this.payoutSnapshotService?.finalizeSnapshotForBlock({ + payoutSnapshotId: jobTemplate.blockData.payoutSnapshotId, + blockHeight: jobTemplate.blockData.height, + blockSubmissionResult, }); await this.notificationService.notifySubscribersBlockFound(this.clientAuthorization.address, jobTemplate.blockData.height, updatedJobBlock, blockSubmissionResult); //success - if (blockSubmissionResult == null) { + if (this.isSuccessfulBlockSubmission(blockSubmissionResult)) { await this.addressSettingsService.resetBestDifficultyAndShares(); } } await this.ensureClientEntity(); try { await this.shareAccountingService?.recordAcceptedShare({ - protocol: 'sv1', + protocol: this.accountingProtocol, address: this.clientAuthorization.address, clientName: this.clientAuthorization.worker, sessionId: this.extraNonceAndSessionId, @@ -877,6 +885,18 @@ export class StratumV1Client { return Number.isFinite(configured) && configured > 0 ? configured : null; } + private getPayoutInformation(jobTemplate: IJobTemplate, fallbackAddress: string): AddressObject[] { + if (this.configService.get('PAYOUT_COINBASE_MODE') === 'snapshot' && jobTemplate.blockData.payoutOutputs?.length > 0) { + return jobTemplate.blockData.payoutOutputs; + } + + return [{ address: fallbackAddress, percent: 100 }]; + } + + private isSuccessfulBlockSubmission(result?: string | null): boolean { + return result == null || result === 'SUCCESS!'; + } + private closeSocket() { this.connectionClosed = true; if (!this.socket.destroyed) { diff --git a/src/models/StratumV2Client.spec.ts b/src/models/StratumV2Client.spec.ts index 3585b78..79faae4 100644 --- a/src/models/StratumV2Client.spec.ts +++ b/src/models/StratumV2Client.spec.ts @@ -5,16 +5,22 @@ import { BehaviorSubject, firstValueFrom } from 'rxjs'; import { MockRecording1 } from '../../test/models/MockRecording1'; import { ClientEntity } from '../ORM/client/client.entity'; +import { CustomWorkService } from '../services/custom-work.service'; import { StratumV1JobsService } from '../services/stratum-v1-jobs.service'; import { BufferReader } from './sv2/sv2-binary-codec'; -import { SV2_CHANNEL_MSG_FLAG, Sv2MsgType } from './sv2/sv2-constants'; +import { SV2_CHANNEL_MSG_FLAG, Sv2MiningSetupFlags, Sv2MsgType, Sv2Protocol } from './sv2/sv2-constants'; import { deserializeOpenExtendedMiningChannelSuccess, deserializeNewExtendedMiningJob, serializeOpenExtendedMiningChannel, serializeSubmitSharesExtended, } from './sv2/sv2-extended-messages'; -import { deserializeSetNewPrevHash, deserializeSubmitSharesError } from './sv2/sv2-messages'; +import { deserializeSetNewPrevHash, deserializeSubmitSharesError, serializeSetupConnection } from './sv2/sv2-messages'; +import { + deserializeSetCustomMiningJobError, + deserializeSetCustomMiningJobSuccess, + serializeSetCustomMiningJob, +} from './sv2/sv2-jdp-messages'; import { MiningJob } from './MiningJob'; import { StratumV2Client } from './StratumV2Client'; @@ -60,7 +66,7 @@ describe('StratumV2Client extended channels', () => { const success = deserializeOpenExtendedMiningChannelSuccess(new BufferReader(successFrame.payload)); expect(success.requestId).toBe(7); expect(success.channelId).toBe(1); - expect(success.extranonceSize).toBe(8); + expect(success.extranonceSize).toBe(10); expect(success.extranoncePrefix).toEqual(Buffer.from('00000001', 'hex')); expect(jobFrame?.extensionType).toBe(SV2_CHANNEL_MSG_FLAG); @@ -109,6 +115,89 @@ describe('StratumV2Client extended channels', () => { expect(error.errorCode).toBe('invalid-extranonce-size'); }); + it('accepts SV2 work-selection setup and SetCustomMiningJob without sending pool work', async () => { + const { client, sentFrames } = await createClient(); + await (client as any).handleSetupConnection(serializeSetupConnection({ + protocol: Sv2Protocol.MINING, + minVersion: 2, + maxVersion: 2, + flags: Sv2MiningSetupFlags.REQUIRES_WORK_SELECTION | Sv2MiningSetupFlags.REQUIRES_VERSION_ROLLING, + endpoint_host: 'localhost', + endpoint_port: 3333, + vendor: 'jd-client', + hardwareVersion: '', + firmwareVersion: '', + deviceId: '', + })); + + await (client as any).handleOpenExtendedMiningChannel(serializeOpenExtendedMiningChannel({ + requestId: 1, + userIdentity: 'tb1qumezefzdeqqwn5zfvgdrhxjzc5ylr39uhuxcz4.worker', + nominalHashRate: 0, + maxTarget: Buffer.alloc(32, 0xff), + minExtranonceSize: 8, + })); + expect(sentFrames.some(frame => frame.msgType === Sv2MsgType.NEW_EXTENDED_MINING_JOB)).toBe(false); + + sentFrames.length = 0; + await (client as any).handleSetCustomMiningJob(serializeSetCustomMiningJob({ + channelId: 1, + requestId: 99, + token: Buffer.from('aa', 'hex'), + version: MockRecording1.BLOCK_TEMPLATE.version, + prevHash: Buffer.alloc(32, 1), + minNtime: parseInt(MockRecording1.TIME, 16), + nBits: parseInt(MockRecording1.BLOCK_TEMPLATE.bits, 16), + coinbaseTxVersion: 2, + coinbasePrefix: Buffer.from('51', 'hex'), + coinbaseTxInputNSequence: 0xffffffff, + coinbaseTxOutputs: Buffer.from('010000000000000000016a', 'hex'), + coinbaseTxLocktime: 0, + merklePath: [], + })); + + const successFrame = sentFrames.find(frame => frame.msgType === Sv2MsgType.SET_CUSTOM_MINING_JOB_SUCCESS); + const success = deserializeSetCustomMiningJobSuccess(new BufferReader(successFrame.payload)); + expect(success.channelId).toBe(1); + expect(success.requestId).toBe(99); + expect((client as any).channels.get(1).extendedJobs.get(success.jobId).workProtocol).toBe('sv2_jdp'); + }); + + it('rejects SetCustomMiningJob when work selection was not negotiated', async () => { + const { client, sentFrames } = await createClient(); + await (client as any).handleOpenExtendedMiningChannel(serializeOpenExtendedMiningChannel({ + requestId: 1, + userIdentity: 'tb1qumezefzdeqqwn5zfvgdrhxjzc5ylr39uhuxcz4.worker', + nominalHashRate: 0, + maxTarget: Buffer.alloc(32, 0xff), + minExtranonceSize: 8, + })); + + sentFrames.length = 0; + await (client as any).handleSetCustomMiningJob(serializeSetCustomMiningJob({ + channelId: 1, + requestId: 100, + token: Buffer.from('aa', 'hex'), + version: MockRecording1.BLOCK_TEMPLATE.version, + prevHash: Buffer.alloc(32, 1), + minNtime: parseInt(MockRecording1.TIME, 16), + nBits: parseInt(MockRecording1.BLOCK_TEMPLATE.bits, 16), + coinbaseTxVersion: 2, + coinbasePrefix: Buffer.from('51', 'hex'), + coinbaseTxInputNSequence: 0xffffffff, + coinbaseTxOutputs: Buffer.from('010000000000000000016a', 'hex'), + coinbaseTxLocktime: 0, + merklePath: [], + })); + + const errorFrame = sentFrames.find(frame => frame.msgType === Sv2MsgType.SET_CUSTOM_MINING_JOB_ERROR); + const error = deserializeSetCustomMiningJobError(new BufferReader(errorFrame.payload)); + expect(error.channelId).toBe(1); + expect(error.requestId).toBe(100); + expect(error.errorCode).toBe('work-selection-not-negotiated'); + expect((client as any).channels.get(1).extendedJobs.has(100)).toBe(false); + }); + it('records accepted SV2 shares before presence updates', async () => { const { client, shareAccountingService, redisMessagingService, jobTemplate } = await createClient(); (client as any).address = 'tb1qumezefzdeqqwn5zfvgdrhxjzc5ylr39uhuxcz4'; @@ -184,7 +273,8 @@ describe('StratumV2Client extended channels', () => { }); it('submits a block when the SV2 hash target exactly meets network target', async () => { - const { client, bitcoinRpcService, blocksService, notificationService, jobTemplate } = await createClient(); + const { client, bitcoinRpcService, blocksService, notificationService, addressSettingsService, jobTemplate } = await createClient(); + bitcoinRpcService.SUBMIT_BLOCK.mockResolvedValue('SUCCESS!'); (client as any).address = 'tb1qumezefzdeqqwn5zfvgdrhxjzc5ylr39uhuxcz4'; (client as any).workerName = 'worker'; (client as any).sessionId = 'sv2-session'; @@ -220,6 +310,7 @@ describe('StratumV2Client extended channels', () => { blockData: expect.any(String), })); expect(notificationService.notifySubscribersBlockFound).toHaveBeenCalled(); + expect(addressSettingsService.resetBestDifficultyAndShares).toHaveBeenCalled(); }); async function createClient(): Promise<{ @@ -228,6 +319,7 @@ describe('StratumV2Client extended channels', () => { bitcoinRpcService: { SUBMIT_BLOCK: jest.Mock }; blocksService: { save: jest.Mock }; notificationService: { notifySubscribersBlockFound: jest.Mock }; + addressSettingsService: { resetBestDifficultyAndShares: jest.Mock; updateBestDifficultyIfHigher: jest.Mock }; shareAccountingService: { recordAcceptedShare: jest.Mock }; redisMessagingService: { setClientPresence: jest.Mock; removeClientPresence: jest.Mock }; jobTemplate: any; @@ -286,6 +378,10 @@ describe('StratumV2Client extended channels', () => { setClientPresence: jest.fn().mockResolvedValue(undefined), removeClientPresence: jest.fn().mockResolvedValue(undefined), }; + const addressSettingsService = { + resetBestDifficultyAndShares: jest.fn().mockResolvedValue(undefined), + updateBestDifficultyIfHigher: jest.fn().mockResolvedValue(undefined), + }; const client = new StratumV2Client( socket, Buffer.alloc(0), @@ -306,7 +402,8 @@ describe('StratumV2Client extended channels', () => { generateExtranoncePrefix: () => Buffer.from('00000002', 'hex'), allocateExtendedExtranoncePrefix: jest.fn(() => Buffer.from('00000001', 'hex')), releaseExtendedExtranoncePrefix: jest.fn(), - getExtendedMinerExtranonceSize: () => 8, + getExtendedMinerExtranonceSize: () => 10, + getExtendedTotalExtranonceSize: () => 14, } as any, stratumV1JobsService, bitcoinRpcService as any, @@ -325,9 +422,14 @@ describe('StratumV2Client extended channels', () => { } }), } as unknown as ConfigService, + addressSettingsService as any, + new CustomWorkService(), { - resetBestDifficultyAndShares: jest.fn().mockResolvedValue(undefined), - updateBestDifficultyIfHigher: jest.fn().mockResolvedValue(undefined), + hasKnownToken: jest.fn().mockReturnValue(true), + getDeclaredJob: jest.fn().mockReturnValue({ + validationMode: 'full_template', + job: { version: MockRecording1.BLOCK_TEMPLATE.version }, + }), } as any, shareAccountingService as any, redisMessagingService as any, @@ -343,6 +445,7 @@ describe('StratumV2Client extended channels', () => { bitcoinRpcService, blocksService, notificationService, + addressSettingsService, shareAccountingService, redisMessagingService, jobTemplate, diff --git a/src/models/StratumV2Client.ts b/src/models/StratumV2Client.ts index ec62238..713e8f1 100644 --- a/src/models/StratumV2Client.ts +++ b/src/models/StratumV2Client.ts @@ -9,18 +9,20 @@ import { AddressSettingsService } from '../ORM/address-settings/address-settings import { BlocksService } from '../ORM/blocks/blocks.service'; import { ClientEntity } from '../ORM/client/client.entity'; import { ClientService } from '../ORM/client/client.service'; +import { PayoutSnapshotService } from '../ORM/payout-snapshot/payout-snapshot.service'; import { ShareAccountingService } from '../ORM/share-accounting/share-accounting.service'; import { BitcoinRpcService } from '../services/bitcoin-rpc.service'; +import { CustomWorkService } from '../services/custom-work.service'; import { NotificationService } from '../services/notification.service'; import { RedisMessagingService } from '../services/redis-messaging.service'; import { StratumV2Service } from '../services/stratum-v2.service'; import { IJobTemplate, StratumV1JobsService } from '../services/stratum-v1-jobs.service'; +import { Sv2JobDeclarationRegistryService } from '../services/sv2-job-declaration-registry.service'; import { patchCoinbasePrefixVarint } from '../utils/coinbase-prefix.utils'; import { DifficultyUtils } from '../utils/difficulty.utils'; import { hash256 } from '../utils/hash.utils'; -import { MiningJob } from './MiningJob'; +import { AddressObject, MiningJob } from './MiningJob'; import { StratumV1ClientStatistics } from './StratumV1ClientStatistics'; -import { TOTAL_EXTRANONCE_SIZE_BYTES } from './stratum.constants'; import { BufferReader } from './sv2/sv2-binary-codec'; import { SV2_CHANNEL_MSG_FLAG, @@ -57,6 +59,11 @@ import { Sv2SubmitSharesExtended, } from './sv2/sv2-extended-messages'; import { Sv2NoiseSession } from './sv2/sv2-noise'; +import { + deserializeSetCustomMiningJob, + serializeSetCustomMiningJobError, + serializeSetCustomMiningJobSuccess, +} from './sv2/sv2-jdp-messages'; const DEFAULT_START_DIFFICULTY = 100000; const DEFAULT_MIN_DIFFICULTY = 0.001; @@ -77,6 +84,7 @@ interface ExtendedJobData { miningJob: MiningJob; retiredAt?: number; creation: number; + workProtocol?: 'pool' | 'sv2_jdp'; } interface ChannelState { @@ -123,6 +131,7 @@ export class StratumV2Client { private clientEntity: ClientEntity = null; private creatingEntity: Promise = null; private readonly firstChunkSummary: string; + private workSelectionEnabled = false; constructor( private readonly socket: Socket, @@ -135,8 +144,11 @@ export class StratumV2Client { private readonly blocksService: BlocksService, private readonly configService: ConfigService, private readonly addressSettingsService: AddressSettingsService, + private readonly customWorkService: CustomWorkService, + private readonly jobDeclarationRegistry: Sv2JobDeclarationRegistryService, private readonly shareAccountingService?: ShareAccountingService, private readonly redisMessagingService?: RedisMessagingService, + private readonly payoutSnapshotService?: PayoutSnapshotService, ) { this.firstChunkSummary = this.describeChunk(firstChunk); this.noiseSession = new Sv2NoiseSession(this.stratumV2Service.getNoiseConfig()); @@ -258,6 +270,9 @@ export class StratumV2Client { case Sv2MsgType.SUBMIT_SHARES_EXTENDED: await this.handleSubmitSharesExtended(payload); break; + case Sv2MsgType.SET_CUSTOM_MINING_JOB: + await this.handleSetCustomMiningJob(payload); + break; case Sv2MsgType.UPDATE_CHANNEL: await this.handleUpdateChannel(payload); break; @@ -299,6 +314,7 @@ export class StratumV2Client { } const supportedFlags = Sv2MiningSetupFlags.REQUIRES_STANDARD_JOBS + | Sv2MiningSetupFlags.REQUIRES_WORK_SELECTION | Sv2MiningSetupFlags.REQUIRES_VERSION_ROLLING; const unsupportedFlags = message.flags & ~supportedFlags; if (unsupportedFlags !== 0) { @@ -314,6 +330,7 @@ export class StratumV2Client { } const versionRolling = (message.flags & Sv2MiningSetupFlags.REQUIRES_VERSION_ROLLING) !== 0; + this.workSelectionEnabled = (message.flags & Sv2MiningSetupFlags.REQUIRES_WORK_SELECTION) !== 0; const successFlags = versionRolling ? 0 : Sv2MiningSetupSuccessFlags.REQUIRES_FIXED_VERSION; await this.sendFrame( Sv2MsgType.SETUP_CONNECTION_SUCCESS, @@ -399,10 +416,12 @@ export class StratumV2Client { }), ); - const jobTemplate = await firstValueFrom(this.stratumV1JobsService.newMiningJob$); - await this.sendNewMiningJob(channel, jobTemplate, true); + if (!this.workSelectionEnabled) { + const jobTemplate = await firstValueFrom(this.stratumV1JobsService.newMiningJob$); + await this.sendNewMiningJob(channel, jobTemplate, true); + } - if (isFirstChannel) { + if (isFirstChannel && !this.workSelectionEnabled) { this.subscribeToJobs(); this.startDifficultyTimer(); } @@ -436,14 +455,18 @@ export class StratumV2Client { const channelId = this.stratumV2Service.getNextChannelId(); const extranoncePrefix = this.stratumV2Service.allocateExtendedExtranoncePrefix(channelId); - const maxMinerExtranonceSize = Math.max(0, TOTAL_EXTRANONCE_SIZE_BYTES - extranoncePrefix.length); + const maxMinerExtranonceSize = Math.max(0, this.stratumV2Service.getExtendedTotalExtranonceSize() - extranoncePrefix.length); const defaultMinerExtranonceSize = Math.min( this.stratumV2Service.getExtendedMinerExtranonceSize(), maxMinerExtranonceSize, ); - const extranonceSize = message.minExtranonceSize > 0 - ? Math.min(message.minExtranonceSize, maxMinerExtranonceSize) - : defaultMinerExtranonceSize; + const requestedMinerExtranonceSize = Math.max(0, message.minExtranonceSize); + const extranonceSize = Math.max(defaultMinerExtranonceSize, requestedMinerExtranonceSize); + if (extranonceSize > maxMinerExtranonceSize) { + this.stratumV2Service.releaseExtendedExtranoncePrefix(channelId); + await this.sendOpenChannelError(message.requestId, 'min-extranonce-size-too-large'); + return; + } let channelDifficulty = this.sessionDifficulty; if (Number.isFinite(message.nominalHashRate) && message.nominalHashRate > 0) { @@ -493,10 +516,12 @@ export class StratumV2Client { }), ); - const jobTemplate = await firstValueFrom(this.stratumV1JobsService.newMiningJob$); - await this.sendNewExtendedMiningJob(channel, jobTemplate, true); + if (!this.workSelectionEnabled) { + const jobTemplate = await firstValueFrom(this.stratumV1JobsService.newMiningJob$); + await this.sendNewExtendedMiningJob(channel, jobTemplate, true); + } - if (isFirstChannel) { + if (isFirstChannel && !this.workSelectionEnabled) { this.subscribeToJobs(); this.startDifficultyTimer(); } @@ -651,11 +676,16 @@ export class StratumV2Client { channel.acceptedShareCount++; let updatedJobBlock: bitcoinjs.Block = null; + let blockCandidateResult: string | null = null; if (DifficultyUtils.meetsTarget( hashBuffer, DifficultyUtils.difficultyToTarget(extendedJob.jobTemplate.blockData.networkDifficulty), )) { - updatedJobBlock = this.reconstructExtendedBlock(extendedJob, submission, merkleRoot, channel.extranoncePrefix); + if (extendedJob.workProtocol === 'sv2_jdp') { + blockCandidateResult = 'sv2-jdp-client-submit-expected'; + } else { + updatedJobBlock = this.reconstructExtendedBlock(extendedJob, submission, merkleRoot, channel.extranoncePrefix); + } } await this.recordAcceptedShare(submissionDifficulty, jobDifficulty, extendedJob.jobTemplate, updatedJobBlock, { jobId: submission.jobId.toString(16), @@ -663,9 +693,119 @@ export class StratumV2Client { ntime: submission.ntime, version: submission.version, extraNonce2: submission.extranonce.toString('hex'), + }, { + workProtocol: extendedJob.workProtocol ?? 'pool', + blockCandidateResult, }); } + private async handleSetCustomMiningJob(payload: Buffer): Promise { + const msg = deserializeSetCustomMiningJob(new BufferReader(payload)); + if (!this.workSelectionEnabled) { + await this.sendFrame( + Sv2MsgType.SET_CUSTOM_MINING_JOB_ERROR, + serializeSetCustomMiningJobError({ + channelId: msg.channelId, + requestId: msg.requestId, + errorCode: 'work-selection-not-negotiated', + }), + ); + return; + } + const channel = this.channels.get(msg.channelId); + if (channel == null) { + await this.sendFrame( + Sv2MsgType.SET_CUSTOM_MINING_JOB_ERROR, + serializeSetCustomMiningJobError({ + channelId: msg.channelId, + requestId: msg.requestId, + errorCode: 'invalid-channel-id', + }), + ); + return; + } + if (channel.channelType !== 'extended') { + await this.sendFrame( + Sv2MsgType.SET_CUSTOM_MINING_JOB_ERROR, + serializeSetCustomMiningJobError({ + channelId: msg.channelId, + requestId: msg.requestId, + errorCode: 'invalid-channel-type', + }), + ); + return; + } + const declaredJob = this.jobDeclarationRegistry?.getDeclaredJob(msg.token); + if (declaredJob == null || declaredJob.validationMode !== 'full_template') { + await this.sendFrame( + Sv2MsgType.SET_CUSTOM_MINING_JOB_ERROR, + serializeSetCustomMiningJobError({ + channelId: msg.channelId, + requestId: msg.requestId, + errorCode: 'invalid-mining-job-token', + }), + ); + return; + } + if (declaredJob.job.version !== msg.version) { + await this.sendFrame( + Sv2MsgType.SET_CUSTOM_MINING_JOB_ERROR, + serializeSetCustomMiningJobError({ + channelId: msg.channelId, + requestId: msg.requestId, + errorCode: 'declared-job-mismatch', + }), + ); + return; + } + + const latestTemplate = await firstValueFrom(this.stratumV1JobsService.newMiningJob$); + const jobIdHex = this.stratumV1JobsService.getNextId(); + const jobId = parseInt(jobIdHex, 16); + const split = this.customWorkService.buildSv2CoinbaseSplit( + msg, + channel.extranoncePrefix.length + channel.extranonceSize, + ); + const placeholderJob = new MiningJob( + this.network, + jobIdHex, + this.getPayoutInformation(latestTemplate, this.address), + latestTemplate, + ); + this.stratumV1JobsService.addJob(placeholderJob); + + channel.jobIdToDifficulty.set(jobId, channel.sessionDifficulty); + channel.extendedJobs.set(jobId, { + coinbasePrefix: split.coinbasePrefix, + coinbaseSuffix: split.coinbaseSuffix, + merklePath: msg.merklePath.map(branch => Buffer.from(branch)), + prevHash: Buffer.from(msg.prevHash), + nBits: msg.nBits, + minNtime: msg.minNtime, + jobTemplate: { + ...latestTemplate, + block: Object.assign(new bitcoinjs.Block(), latestTemplate.block, { + prevHash: Buffer.from(msg.prevHash), + bits: msg.nBits, + version: msg.version, + timestamp: msg.minNtime, + }), + }, + miningJob: placeholderJob, + creation: Date.now(), + workProtocol: 'sv2_jdp', + }); + + await this.sendFrame( + Sv2MsgType.SET_CUSTOM_MINING_JOB_SUCCESS, + serializeSetCustomMiningJobSuccess({ + channelId: msg.channelId, + requestId: msg.requestId, + jobId, + }), + ); + } + private async handleAcceptedShare( submission: { nonce: number; ntime: number; version: number }, channel: ChannelState, @@ -706,10 +846,11 @@ export class StratumV2Client { jobTemplate: IJobTemplate, updatedJobBlock: bitcoinjs.Block | null, share: { jobId: string; nonce: number; ntime: number; version: number; extraNonce2: string }, + metadata: { workProtocol?: 'pool' | 'sv2_jdp'; blockCandidateResult?: string | null } = {}, ): Promise { await this.ensureClientEntity(); - let blockSubmissionResult: string = null; + let blockSubmissionResult: string = metadata.blockCandidateResult ?? null; if (updatedJobBlock != null) { console.log(`[SV2 ${this.sessionId}] BLOCK FOUND at height ${jobTemplate.blockData.height}`); const blockHex = updatedJobBlock.toHex(false); @@ -720,6 +861,13 @@ export class StratumV2Client { worker: this.workerName, sessionId: this.sessionId, blockData: blockHex, + blockSubmissionResult, + payoutSnapshotId: jobTemplate.blockData.payoutSnapshotId ?? null, + }); + await this.payoutSnapshotService?.finalizeSnapshotForBlock({ + payoutSnapshotId: jobTemplate.blockData.payoutSnapshotId, + blockHeight: jobTemplate.blockData.height, + blockSubmissionResult, }); await this.notificationService.notifySubscribersBlockFound( this.address, @@ -727,13 +875,15 @@ export class StratumV2Client { updatedJobBlock, blockSubmissionResult, ); - if (blockSubmissionResult == null) { + if (this.isSuccessfulBlockSubmission(blockSubmissionResult)) { await this.addressSettingsService.resetBestDifficultyAndShares(); } } await this.shareAccountingService?.recordAcceptedShare({ - protocol: 'sv2', + protocol: metadata.workProtocol === 'sv2_jdp' ? 'sv2_jdp' : 'sv2', + workSource: metadata.workProtocol === 'sv2_jdp' ? 'miner_template' : 'pool_template', + workProtocol: metadata.workProtocol ?? 'pool', address: this.address, clientName: this.workerName, sessionId: this.sessionId, @@ -748,7 +898,7 @@ export class StratumV2Client { ntime: share.ntime, version: share.version, extraNonce2: share.extraNonce2, - isBlockCandidate: updatedJobBlock != null, + isBlockCandidate: updatedJobBlock != null || metadata.blockCandidateResult != null, blockSubmissionResult, }); await this.statistics.addShares(this.clientEntity, jobDifficulty); @@ -782,7 +932,7 @@ export class StratumV2Client { const coinbaseTx = extendedJob.miningJob.cloneCoinbaseTransaction(); const script = coinbaseTx.ins[0].script; coinbaseTx.ins[0].script = Buffer.concat([ - script.subarray(0, script.length - TOTAL_EXTRANONCE_SIZE_BYTES), + script.subarray(0, script.length - (extranoncePrefix.length + submission.extranonce.length)), extranoncePrefix, submission.extranonce, ]); @@ -928,7 +1078,7 @@ export class StratumV2Client { const job = new MiningJob( this.network, jobIdHex, - [{ address: this.address, percent: 100 }], + this.getPayoutInformation(jobTemplate, this.address), jobTemplate, ); this.stratumV1JobsService.addJob(job); @@ -982,7 +1132,7 @@ export class StratumV2Client { const job = new MiningJob( this.network, jobIdHex, - [{ address: this.address, percent: 100 }], + this.getPayoutInformation(jobTemplate, this.address), jobTemplate, ); this.stratumV1JobsService.addJob(job); @@ -1243,6 +1393,18 @@ export class StratumV2Client { return Number.isFinite(configured) && configured > 0 ? configured : DEFAULT_DIFFICULTY_CHECK_INTERVAL_MS; } + private getPayoutInformation(jobTemplate: IJobTemplate, fallbackAddress: string): AddressObject[] { + if (this.configService.get('PAYOUT_COINBASE_MODE') === 'snapshot' && jobTemplate.blockData.payoutOutputs?.length > 0) { + return jobTemplate.blockData.payoutOutputs; + } + + return [{ address: fallbackAddress, percent: 100 }]; + } + + private isSuccessfulBlockSubmission(result?: string | null): boolean { + return result == null || result === 'SUCCESS!'; + } + private getNetwork(): bitcoinjs.networks.Network { const networkConfig = this.configService.get('NETWORK'); if (networkConfig === 'mainnet') { diff --git a/src/models/bitcoin-rpc/IBlockTemplate.ts b/src/models/bitcoin-rpc/IBlockTemplate.ts index baed66d..f060177 100644 --- a/src/models/bitcoin-rpc/IBlockTemplate.ts +++ b/src/models/bitcoin-rpc/IBlockTemplate.ts @@ -7,6 +7,13 @@ export interface IBlockTemplateTx { sigops: number, // (numeric) total SigOps cost, as counted for purposes of block limits; if key is not present, sigop cost is unknown and clients MUST NOT assume it is zero weight: number // (numeric) total transaction weight, as counted for purposes of block limits } + +export interface IBlockTemplatePayoutOutput { + address: string; + percent?: number; + amountSats?: number; +} + export interface IBlockTemplate { // (json object) version: number; // (numeric) The preferred block version @@ -35,5 +42,7 @@ export interface IBlockTemplate { height: number, // (numeric) The height of the next block default_witness_commitment: string // (string, optional) a valid witness commitment for the unmodified block template capabilities: string[] + payoutSnapshotId?: string; + payoutOutputs?: IBlockTemplatePayoutOutput[]; -} \ No newline at end of file +} diff --git a/src/models/datum/datum-codec.spec.ts b/src/models/datum/datum-codec.spec.ts new file mode 100644 index 0000000..80e8aa8 --- /dev/null +++ b/src/models/datum/datum-codec.spec.ts @@ -0,0 +1,257 @@ +import { + DATUM_INITIAL_HEADER_KEY, + DATUM_EXTRANONCE_SIZE, + DatumFrameReader, + DatumMiningCommand, + DatumProtocolCommand, + DatumShareResponseStatus, + datumHeaderXorFeedback, + decodeDatumHeader, + deserializeDatumCoinbaserFetch, + deserializeDatumJobValidationResponse, + deserializeDatumMiningCommand, + deserializeDatumPowSubmit, + encodeDatumFrame, + encodeDatumHeader, + serializeDatumJobValidationFullTransactionBlobRequest, + serializeDatumJobValidationFullTransactionsRequest, + serializeDatumJobValidationShortTxIdsRequest, + serializeDatumCoinbaserFetchResponse, + serializeDatumMiningCommand, + serializeDatumShareResponse, +} from './datum-codec'; + +describe('datum-codec', () => { + it('round-trips DATUM packed headers with XOR key', () => { + const encoded = encodeDatumHeader({ + cmdLen: 1234, + reserved: 0, + isSigned: true, + isEncryptedPubkey: false, + isEncryptedChannel: true, + protoCmd: DatumProtocolCommand.MINING, + }, DATUM_INITIAL_HEADER_KEY); + + expect(decodeDatumHeader(encoded, DATUM_INITIAL_HEADER_KEY)).toEqual({ + cmdLen: 1234, + reserved: 0, + isSigned: true, + isEncryptedPubkey: false, + isEncryptedChannel: true, + protoCmd: DatumProtocolCommand.MINING, + }); + }); + + it('matches the DATUM header feedback function for stable inputs', () => { + expect(datumHeaderXorFeedback(0).toString(16)).toBe('74a55cf6'); + expect(datumHeaderXorFeedback(0xdc871829).toString(16)).toBe('88e1697d'); + }); + + it('splits framed DATUM payloads', () => { + const first = encodeDatumFrame({ + header: { + cmdLen: 0, + reserved: 0, + isSigned: false, + isEncryptedPubkey: false, + isEncryptedChannel: true, + protoCmd: DatumProtocolCommand.MINING, + }, + payload: serializeDatumMiningCommand(DatumMiningCommand.TEMPLATE_REFRESH), + }); + const second = encodeDatumFrame({ + header: { + cmdLen: 0, + reserved: 0, + isSigned: false, + isEncryptedPubkey: false, + isEncryptedChannel: true, + protoCmd: DatumProtocolCommand.MINING, + }, + payload: serializeDatumMiningCommand(DatumMiningCommand.FETCH_COINBASER, Buffer.alloc(8)), + }, datumHeaderXorFeedback(DATUM_INITIAL_HEADER_KEY)); + + const frames = new DatumFrameReader().feed(Buffer.concat([first, second])); + expect(frames).toHaveLength(2); + expect(deserializeDatumMiningCommand(frames[0].payload).command).toBe(DatumMiningCommand.TEMPLATE_REFRESH); + expect(deserializeDatumMiningCommand(frames[1].payload).command).toBe(DatumMiningCommand.FETCH_COINBASER); + }); + + it('serializes coinbaser fetch responses', () => { + const response = serializeDatumCoinbaserFetchResponse(50n, [ + { value: 50n, scriptPubKey: Buffer.from('6a', 'hex') }, + ]); + const mining = deserializeDatumMiningCommand(response); + expect(mining.command).toBe(DatumMiningCommand.FETCH_COINBASER_RESPONSE); + expect(mining.body.readBigUInt64LE(0)).toBe(50n); + expect(mining.body.readUInt32LE(8)).toBe(11); + }); + + it('deserializes coinbaser fetch requests', () => { + const payload = Buffer.alloc(8); + payload.writeBigUInt64LE(123n, 0); + expect(deserializeDatumCoinbaserFetch(payload)).toEqual({ rewardValue: 123n }); + }); + + it('deserializes fixed DATUM POW submissions', () => { + const username = Buffer.from('bc1ptest.worker\0', 'utf8'); + const extranonce = Buffer.from('000102030405060708090a0b', 'hex'); + const payload = Buffer.alloc(17 + DATUM_EXTRANONCE_SIZE); + payload[0] = 7; + payload[1] = 1; + payload[2] = 0x01; + payload[3] = 4; + payload.writeUInt32LE(100, 4); + payload.writeUInt32LE(200, 8); + payload.writeUInt32LE(0x20000000, 12); + payload[16] = DATUM_EXTRANONCE_SIZE; + extranonce.copy(payload, 17); + const parsed = deserializeDatumPowSubmit(Buffer.concat([ + payload, + username, + Buffer.alloc(4), + Buffer.from([0xfe]), + ])); + + expect(parsed.jobId).toBe(7); + expect(parsed.coinbaseId).toBe(1); + expect(parsed.isBlock).toBe(true); + expect(parsed.extranonce).toEqual(extranonce); + expect(parsed.username).toBe('bc1ptest.worker'); + }); + + it('deserializes cached DATUM POW context sections', () => { + const username = Buffer.from('bc1ptest.worker\0', 'utf8'); + const extranonce = Buffer.from('000102030405060708090a0b', 'hex'); + const base = Buffer.alloc(17 + DATUM_EXTRANONCE_SIZE); + base[0] = 3; + base[1] = 2; + base[2] = 0; + base[3] = 0x12; + base.writeUInt32LE(100, 4); + base.writeUInt32LE(200, 8); + base.writeUInt32LE(0x20000000, 12); + base[16] = DATUM_EXTRANONCE_SIZE; + extranonce.copy(base, 17); + + const templateSection = Buffer.concat([ + Buffer.from([0x01]), + Buffer.alloc(32, 0xaa), + Buffer.from([0x09, 0x00]), + Buffer.from('ffff7f20', 'hex'), + Buffer.from([7]), + Buffer.from('64000000', 'hex'), + Buffer.from('0100000000000000', 'hex'), + Buffer.from('02000000', 'hex'), + Buffer.from('03000000', 'hex'), + Buffer.from('04000000', 'hex'), + Buffer.from('05000000', 'hex'), + Buffer.from([1]), + Buffer.alloc(32, 0xbb), + ]); + const coinbaseSection = Buffer.concat([ + Buffer.from([0x02, 2]), + Buffer.from([0x02, 0x00]), + Buffer.from([0x03, 0x00]), + Buffer.from('abcd', 'hex'), + Buffer.from('010203', 'hex'), + ]); + + const parsed = deserializeDatumPowSubmit(Buffer.concat([ + base, + username, + Buffer.alloc(4), + templateSection, + coinbaseSection, + Buffer.from([0xfe]), + ])); + + expect(parsed.jobId).toBe(3); + expect(parsed.coinbaseId).toBe(2); + expect(parsed.targetByte).toBe(0x12); + expect(parsed.prevBlockHash?.equals(Buffer.alloc(32, 0xaa))).toBe(true); + expect(parsed.targetByteIndex).toBe(9); + expect(parsed.nBits?.toString('hex')).toBe('ffff7f20'); + expect(parsed.height).toBe(100); + expect(parsed.merkleBranches?.[0].equals(Buffer.alloc(32, 0xbb))).toBe(true); + expect(parsed.coinbasePairs.get(2)?.coinb1.toString('hex')).toBe('abcd'); + expect(parsed.coinbasePairs.get(2)?.coinb2.toString('hex')).toBe('010203'); + }); + + it('rejects DATUM POW submissions with non-standard extranonce sizes', () => { + const payload = Buffer.alloc(18); + payload[16] = 1; + payload[17] = 0xaa; + + expect(() => deserializeDatumPowSubmit(payload)).toThrow('Unsupported DATUM extranonce size 1'); + }); + + it('serializes share responses', () => { + const response = serializeDatumShareResponse({ + status: DatumShareResponseStatus.ACCEPTED, + reasonCode: 0, + nonce: 123, + targetByte: 4, + jobId: 7, + }); + expect(response.toString('hex')).toBe('8f5000007b0000000407'); + }); + + it('serializes DATUM job validation requests', () => { + expect(serializeDatumJobValidationShortTxIdsRequest(7).toString('hex')).toBe('501007'); + expect(serializeDatumJobValidationFullTransactionBlobRequest(7).toString('hex')).toBe('501207'); + expect(serializeDatumJobValidationFullTransactionsRequest(7, [1, 258]).toString('hex')).toBe('501107020001000201'); + }); + + it('deserializes DATUM short transaction ID validation responses', () => { + const payload = Buffer.concat([ + Buffer.from([0x90, 7, 0x01]), + Buffer.from('0200', 'hex'), + Buffer.from('010203040506', 'hex'), + Buffer.from('111213141516', 'hex'), + Buffer.alloc(32, 0xaa), + Buffer.from([0xfe, 0x00, 0x00]), + ]); + + const parsed = deserializeDatumJobValidationResponse(payload); + expect(parsed.kind).toBe('short-txids'); + expect(parsed.jobId).toBe(7); + expect(parsed.status).toBe(1); + expect(parsed.transactionCount).toBe(2); + if (parsed.kind !== 'short-txids') { + throw new Error(`Unexpected DATUM validation response kind ${parsed.kind}`); + } + expect(parsed.shortTxIds.map(id => id.toString('hex'))).toEqual(['010203040506', '111213141516']); + expect(parsed.crosscheck?.equals(Buffer.alloc(32, 0xaa))).toBe(true); + }); + + it('deserializes DATUM full transaction blob validation responses', () => { + const tx = Buffer.from( + '0100000001' + + '0000000000000000000000000000000000000000000000000000000000000000' + + 'ffffffff00ffffffff' + + '01000000000000000000' + + '00000000', + 'hex', + ); + const txSize = Buffer.alloc(3); + txSize.writeUIntLE(tx.length, 0, 3); + const payload = Buffer.concat([ + Buffer.from([0x92, 9, 0x01]), + Buffer.from('0100', 'hex'), + txSize, + tx, + Buffer.from([0xfe]), + ]); + + const parsed = deserializeDatumJobValidationResponse(payload); + expect(parsed.kind).toBe('full-transaction-blob'); + expect(parsed.jobId).toBe(9); + expect(parsed.status).toBe(1); + expect(parsed.transactionCount).toBe(1); + if (parsed.kind !== 'full-transaction-blob') { + throw new Error(`Unexpected DATUM validation response kind ${parsed.kind}`); + } + expect(parsed.transactions[0]).toEqual(tx); + }); +}); diff --git a/src/models/datum/datum-codec.ts b/src/models/datum/datum-codec.ts new file mode 100644 index 0000000..c93310c --- /dev/null +++ b/src/models/datum/datum-codec.ts @@ -0,0 +1,506 @@ +import { BufferReader, BufferWriter } from '../sv2/sv2-binary-codec'; + +export const DATUM_PROTOCOL_VERSION = 'v0.4.1-beta'; +export const DATUM_INITIAL_HEADER_KEY = 0xdc871829; +export const DATUM_MAX_PAYLOAD_SIZE = 4194304; +export const DATUM_MAX_JOBS = 8; +export const DATUM_EXTRANONCE_SIZE = 12; + +export enum DatumProtocolCommand { + PING = 0x01, + HANDSHAKE_INIT = 0x01, + HANDSHAKE_RESPONSE = 0x02, + MINING = 0x05, +} + +export enum DatumMiningCommand { + FETCH_COINBASER = 0x10, + FETCH_COINBASER_RESPONSE = 0x11, + SUBMIT_POW = 0x27, + JOB_VALIDATION = 0x50, + SHARE_RESPONSE = 0x8f, + CLIENT_CONFIGURE = 0x99, + TEMPLATE_REFRESH = 0xf9, +} + +export enum DatumJobValidationCommand { + REQUEST_SHORT_TX_IDS = 0x10, + REQUEST_FULL_TRANSACTIONS = 0x11, + REQUEST_FULL_TRANSACTION_BLOB = 0x12, + SHORT_TX_IDS_RESPONSE = 0x90, + FULL_TRANSACTIONS_RESPONSE = 0x91, + FULL_TRANSACTION_BLOB_RESPONSE = 0x92, +} + +export enum DatumJobValidationStatus { + SUCCESS = 0x01, +} + +export enum DatumShareResponseStatus { + ACCEPTED = 0x50, + ACCEPTED_TENTATIVE = 0x55, + REJECTED = 0x66, +} + +export enum DatumRejectReason { + BAD_JOB_ID = 10, + BAD_COINBASE_ID = 11, + BAD_EXTRANONCE_SIZE = 12, + BAD_TARGET = 13, + BAD_USERNAME = 14, + BAD_COINBASER_ID = 15, + BAD_MERKLE_COUNT = 16, + BAD_COINBASE_TOO_LARGE = 17, + COINBASE_MISSING = 18, + TARGET_MISMATCH = 19, + HIGH_HASH = 21, + BAD_NTIME = 23, + BAD_VERSION = 24, + STALE_BLOCK = 25, + DUPLICATE_WORK = 29, + OTHER = 30, +} + +export interface DatumHeader { + cmdLen: number; + reserved: number; + isSigned: boolean; + isEncryptedPubkey: boolean; + isEncryptedChannel: boolean; + protoCmd: number; +} + +export interface DatumFrame { + header: DatumHeader; + payload: Buffer; +} + +export interface DatumCoinbaserFetch { + rewardValue: bigint; +} + +export interface DatumPayoutOutput { + value: bigint; + scriptPubKey: Buffer; +} + +export interface DatumPowSubmit { + jobId: number; + coinbaseId: number; + isBlock: boolean; + subsidyOnly: boolean; + quickDiff: boolean; + targetByte: number; + ntime: number; + nonce: number; + version: number; + extranonce: Buffer; + username: string; + reserved: Buffer; + prevBlockHash?: Buffer; + targetByteIndex?: number; + nBits?: Buffer; + coinbaserId?: number; + height?: number; + coinbaseValue?: bigint; + transactionCount?: number; + totalWeight?: number; + totalSize?: number; + totalSigops?: number; + merkleBranches?: Buffer[]; + coinbasePairs: Map; + subsidyOnlyCoinbase?: { coinb1: Buffer; coinb2: Buffer }; +} + +export interface DatumShortTxIdsResponse { + kind: 'short-txids'; + jobId: number; + status: number; + transactionCount: number; + shortTxIds: Buffer[]; + crosscheck: Buffer | null; +} + +export interface DatumFullTransactionsResponse { + kind: 'full-transactions'; + jobId: number; + status: number; + transactionCount: number; + transactions: Buffer[]; +} + +export interface DatumFullTransactionBlobResponse { + kind: 'full-transaction-blob'; + jobId: number; + status: number; + transactionCount: number; + transactions: Buffer[]; +} + +export type DatumJobValidationResponse = + | DatumShortTxIdsResponse + | DatumFullTransactionsResponse + | DatumFullTransactionBlobResponse; + +export function encodeDatumHeader(header: DatumHeader, xorKey = 0): Buffer { + if (header.cmdLen < 0 || header.cmdLen > DATUM_MAX_PAYLOAD_SIZE - 1) { + throw new RangeError(`DATUM payload length ${header.cmdLen} exceeds protocol limit`); + } + + let raw = header.cmdLen & 0x3fffff; + raw |= (header.reserved & 0x03) << 22; + raw |= (header.isSigned ? 1 : 0) << 24; + raw |= (header.isEncryptedPubkey ? 1 : 0) << 25; + raw |= (header.isEncryptedChannel ? 1 : 0) << 26; + raw |= (header.protoCmd & 0x1f) << 27; + + const out = Buffer.alloc(4); + out.writeUInt32LE((raw ^ xorKey) >>> 0, 0); + return out; +} + +export function decodeDatumHeader(input: Buffer, xorKey = 0): DatumHeader { + if (input.length < 4) { + throw new RangeError('DATUM header requires 4 bytes'); + } + const raw = (input.readUInt32LE(0) ^ xorKey) >>> 0; + return { + cmdLen: raw & 0x3fffff, + reserved: (raw >>> 22) & 0x03, + isSigned: ((raw >>> 24) & 0x01) !== 0, + isEncryptedPubkey: ((raw >>> 25) & 0x01) !== 0, + isEncryptedChannel: ((raw >>> 26) & 0x01) !== 0, + protoCmd: (raw >>> 27) & 0x1f, + }; +} + +export function datumHeaderXorFeedback(input: number): number { + let h = 0xb10cfeed >>> 0; + let k = input >>> 0; + k = Math.imul(k, 0xcc9e2d51) >>> 0; + k = (((k << 15) >>> 0) | (k >>> 17)) >>> 0; + k = Math.imul(k, 0x1b873593) >>> 0; + h = (h ^ k) >>> 0; + h = (((h << 13) >>> 0) | (h >>> 19)) >>> 0; + h = (Math.imul(h, 5) + 0xe6546b64) >>> 0; + h = (h ^ 4) >>> 0; + h = (h ^ (h >>> 16)) >>> 0; + h = Math.imul(h, 0x85ebca6b) >>> 0; + h = (h ^ (h >>> 13)) >>> 0; + h = Math.imul(h, 0xc2b2ae35) >>> 0; + h = (h ^ (h >>> 16)) >>> 0; + return h >>> 0; +} + +export class DatumFrameReader { + private buffer = Buffer.alloc(0); + + constructor(private headerXorKey = DATUM_INITIAL_HEADER_KEY) {} + + public setHeaderXorKey(headerXorKey: number): void { + this.headerXorKey = headerXorKey >>> 0; + } + + public feed(data: Buffer): DatumFrame[] { + this.buffer = Buffer.concat([this.buffer, data]); + const frames: DatumFrame[] = []; + + while (this.buffer.length >= 4) { + const header = decodeDatumHeader(this.buffer.subarray(0, 4), this.headerXorKey); + if (header.cmdLen > DATUM_MAX_PAYLOAD_SIZE) { + throw new RangeError(`DATUM payload length ${header.cmdLen} exceeds protocol limit`); + } + if (this.buffer.length < 4 + header.cmdLen) { + break; + } + + frames.push({ + header, + payload: Buffer.from(this.buffer.subarray(4, 4 + header.cmdLen)), + }); + this.buffer = this.buffer.subarray(4 + header.cmdLen); + if (header.isEncryptedChannel) { + this.headerXorKey = datumHeaderXorFeedback(this.headerXorKey); + } + } + + return frames; + } +} + +export function encodeDatumFrame(frame: DatumFrame, xorKey = DATUM_INITIAL_HEADER_KEY): Buffer { + return Buffer.concat([ + encodeDatumHeader({ ...frame.header, cmdLen: frame.payload.length }, xorKey), + frame.payload, + ]); +} + +export function deserializeDatumMiningCommand(payload: Buffer): { command: DatumMiningCommand; body: Buffer } { + if (payload.length < 1) { + throw new RangeError('DATUM mining command requires a sub-command byte'); + } + return { + command: payload[0], + body: Buffer.from(payload.subarray(1)), + }; +} + +export function serializeDatumMiningCommand(command: DatumMiningCommand, body = Buffer.alloc(0)): Buffer { + return Buffer.concat([Buffer.from([command]), body]); +} + +export function serializeDatumJobValidationShortTxIdsRequest(jobId: number): Buffer { + return serializeDatumMiningCommand(DatumMiningCommand.JOB_VALIDATION, Buffer.from([ + DatumJobValidationCommand.REQUEST_SHORT_TX_IDS, + jobId & 0xff, + ])); +} + +export function serializeDatumJobValidationFullTransactionBlobRequest(jobId: number): Buffer { + return serializeDatumMiningCommand(DatumMiningCommand.JOB_VALIDATION, Buffer.from([ + DatumJobValidationCommand.REQUEST_FULL_TRANSACTION_BLOB, + jobId & 0xff, + ])); +} + +export function serializeDatumJobValidationFullTransactionsRequest(jobId: number, transactionIndexes: number[]): Buffer { + if (transactionIndexes.length > 0xffff) { + throw new RangeError('DATUM requested transaction index count must fit in u16'); + } + const w = new BufferWriter(); + w.writeU8(DatumJobValidationCommand.REQUEST_FULL_TRANSACTIONS); + w.writeU8(jobId); + w.writeU16(transactionIndexes.length); + for (const index of transactionIndexes) { + if (!Number.isInteger(index) || index < 0 || index > 0xffff) { + throw new RangeError(`Invalid DATUM transaction index ${index}`); + } + w.writeU16(index); + } + return serializeDatumMiningCommand(DatumMiningCommand.JOB_VALIDATION, Buffer.from(w.toBuffer())); +} + +export function deserializeDatumJobValidationResponse(payload: Buffer): DatumJobValidationResponse { + const reader = new BufferReader(payload); + const responseCommand = reader.readU8(); + const jobId = reader.readU8(); + const status = reader.readU8(); + + if (responseCommand === DatumJobValidationCommand.SHORT_TX_IDS_RESPONSE) { + if (status !== DatumJobValidationStatus.SUCCESS) { + return { + kind: 'short-txids', + jobId, + status, + transactionCount: 0, + shortTxIds: [], + crosscheck: null, + }; + } + const transactionCount = reader.readU16(); + const shortTxIds: Buffer[] = []; + for (let i = 0; i < transactionCount; i++) { + shortTxIds.push(reader.readBytes(6)); + } + const crosscheck = reader.remaining >= 32 ? reader.readBytes(32) : null; + consumeDatumTerminatorAndPadding(reader); + return { + kind: 'short-txids', + jobId, + status, + transactionCount, + shortTxIds, + crosscheck, + }; + } + + if (responseCommand === DatumJobValidationCommand.FULL_TRANSACTIONS_RESPONSE) { + return deserializeDatumTransactionListResponse('full-transactions', reader, jobId, status); + } + + if (responseCommand === DatumJobValidationCommand.FULL_TRANSACTION_BLOB_RESPONSE) { + return deserializeDatumTransactionListResponse('full-transaction-blob', reader, jobId, status); + } + + throw new RangeError(`Unsupported DATUM job validation response 0x${responseCommand.toString(16)}`); +} + +export function deserializeDatumCoinbaserFetch(payload: Buffer): DatumCoinbaserFetch { + if (payload.length < 8) { + throw new RangeError('DATUM coinbaser fetch requires reward value'); + } + return { rewardValue: payload.readBigUInt64LE(0) }; +} + +export function serializeDatumCoinbaserFetchResponse(rewardValue: bigint, outputs: DatumPayoutOutput[]): Buffer { + const w = new BufferWriter(); + w.writeU64(rewardValue); + const outputsBuffer = serializeDatumPayoutOutputs(outputs); + w.writeU32(outputsBuffer.length); + w.writeBytes(outputsBuffer); + return serializeDatumMiningCommand(DatumMiningCommand.FETCH_COINBASER_RESPONSE, Buffer.from(w.toBuffer())); +} + +export function serializeDatumPayoutOutputs(outputs: DatumPayoutOutput[]): Buffer { + if (outputs.length > 255) { + throw new RangeError('DATUM payout output count must fit in one byte'); + } + const w = new BufferWriter(); + w.writeU8(outputs.length); + for (const output of outputs) { + if (output.scriptPubKey.length > 255) { + throw new RangeError('DATUM scriptPubKey length must fit in one byte'); + } + w.writeU64(output.value); + w.writeU8(output.scriptPubKey.length); + w.writeBytes(output.scriptPubKey); + } + return w.toBuffer(); +} + +export function deserializeDatumPowSubmit(payload: Buffer): DatumPowSubmit { + const reader = new BufferReader(payload); + const jobId = reader.readU8(); + const coinbaseId = reader.readU8(); + const flags = reader.readU8(); + const targetByte = reader.readU8(); + const ntime = reader.readU32(); + const nonce = reader.readU32(); + const version = reader.readU32(); + const extranonceSize = reader.readU8(); + if (extranonceSize !== DATUM_EXTRANONCE_SIZE) { + throw new RangeError(`Unsupported DATUM extranonce size ${extranonceSize}`); + } + const extranonce = reader.readBytes(extranonceSize); + const username = readNullTerminatedString(reader); + const reserved = reader.readBytes(Math.min(4, reader.remaining)); + const coinbasePairs = new Map(); + const result: DatumPowSubmit = { + jobId, + coinbaseId, + isBlock: (flags & 0x01) !== 0, + subsidyOnly: (flags & 0x02) !== 0, + quickDiff: (flags & 0x04) !== 0, + targetByte, + ntime, + nonce, + version, + extranonce, + username, + reserved, + coinbasePairs, + }; + + while (reader.remaining > 0) { + const section = reader.readU8(); + if (section === 0xfe) { + break; + } + if (section === 0x01) { + result.prevBlockHash = reader.readBytes(32); + result.targetByteIndex = reader.readU16(); + result.nBits = reader.readBytes(4); + result.coinbaserId = reader.readU8(); + result.height = reader.readU32(); + result.coinbaseValue = reader.readU64(); + result.transactionCount = reader.readU32(); + result.totalWeight = reader.readU32(); + result.totalSize = reader.readU32(); + result.totalSigops = reader.readU32(); + const merkleBranchCount = reader.readU8(); + result.merkleBranches = []; + for (let i = 0; i < merkleBranchCount; i++) { + result.merkleBranches.push(reader.readBytes(32)); + } + continue; + } + if (section === 0x02) { + const coinbaseType = reader.readU8(); + const coinb1Len = reader.readU16(); + const coinb2Len = reader.readU16(); + const coinbase = { + coinb1: reader.readBytes(coinb1Len), + coinb2: reader.readBytes(coinb2Len), + }; + if (coinbaseType === 255 || coinbaseId === 255) { + result.subsidyOnlyCoinbase = coinbase; + } else { + coinbasePairs.set(coinbaseType, coinbase); + } + continue; + } + throw new RangeError(`Unsupported DATUM POW section 0x${section.toString(16)}`); + } + + return result; +} + +export function serializeDatumShareResponse(input: { + status: DatumShareResponseStatus; + reasonCode: number; + nonce: number; + targetByte: number; + jobId: number; +}): Buffer { + const w = new BufferWriter(); + w.writeU8(input.status); + w.writeU16(input.reasonCode); + w.writeU32(input.nonce); + w.writeU8(input.targetByte); + w.writeU8(input.jobId); + return serializeDatumMiningCommand(DatumMiningCommand.SHARE_RESPONSE, Buffer.from(w.toBuffer())); +} + +function deserializeDatumTransactionListResponse( + kind: 'full-transactions' | 'full-transaction-blob', + reader: BufferReader, + jobId: number, + status: number, +): DatumFullTransactionsResponse | DatumFullTransactionBlobResponse { + if (status !== DatumJobValidationStatus.SUCCESS) { + return { + kind, + jobId, + status, + transactionCount: 0, + transactions: [], + }; + } + + const transactionCount = reader.readU16(); + const transactions: Buffer[] = []; + for (let i = 0; i < transactionCount; i++) { + const transactionSize = reader.readU24(); + transactions.push(reader.readBytes(transactionSize)); + } + consumeDatumTerminatorAndPadding(reader); + return { + kind, + jobId, + status, + transactionCount, + transactions, + }; +} + +function consumeDatumTerminatorAndPadding(reader: BufferReader): void { + if (reader.remaining <= 0) { + return; + } + const terminator = reader.readU8(); + if (terminator !== 0xfe) { + throw new RangeError(`DATUM job validation response missing terminator; got 0x${terminator.toString(16)}`); + } +} + +function readNullTerminatedString(reader: BufferReader): string { + const bytes: number[] = []; + while (reader.remaining > 0) { + const byte = reader.readU8(); + if (byte === 0) { + break; + } + bytes.push(byte); + } + return Buffer.from(bytes).toString('utf8'); +} diff --git a/src/models/datum/datum-crypto.spec.ts b/src/models/datum/datum-crypto.spec.ts new file mode 100644 index 0000000..5a363ba --- /dev/null +++ b/src/models/datum/datum-crypto.spec.ts @@ -0,0 +1,94 @@ +import * as sodium from 'libsodium-wrappers-sumo'; +import { + DATUM_INITIAL_HEADER_KEY, + DatumProtocolCommand, + datumHeaderXorFeedback, + decodeDatumHeader, + encodeDatumFrame, +} from './datum-codec'; +import { DatumCryptoSession } from './datum-crypto'; + +describe('DatumCryptoSession', () => { + it('derives stable DATUM public keys from a configured seed', async () => { + await sodium.ready; + const seed = Buffer.from('000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f', 'hex'); + + const first = DatumCryptoSession.generateKeyPairFromSeed(seed); + const second = DatumCryptoSession.generateKeyPairFromSeed(seed); + + expect(Buffer.concat([first.edPublicKey, first.xPublicKey]).toString('hex')) + .toBe(Buffer.concat([second.edPublicKey, second.xPublicKey]).toString('hex')); + expect(first.edSecretKey).toEqual(second.edSecretKey); + }); + + it('opens signed sealed handshakes and builds encrypted responses', async () => { + await sodium.ready; + const serverIdentity = DatumCryptoSession.generateKeyPair(); + const serverSessionKeys = DatumCryptoSession.generateKeyPair(); + const clientIdentity = DatumCryptoSession.generateKeyPair(); + const clientSessionKeys = DatumCryptoSession.generateKeyPair(); + const server = await DatumCryptoSession.create(serverIdentity, serverSessionKeys); + const nextHeaderKey = 0x12345678; + + const signedBody = Buffer.concat([ + clientIdentity.edPublicKey, + clientIdentity.xPublicKey, + clientSessionKeys.edPublicKey, + clientSessionKeys.xPublicKey, + Buffer.from('datum-test/0\0', 'utf8'), + Buffer.from([0xfe]), + uint32(nextHeaderKey), + Buffer.from([1, 2, 3]), + ]); + const signature = Buffer.from(sodium.crypto_sign_detached(signedBody, clientIdentity.edSecretKey)); + const encrypted = Buffer.from(sodium.crypto_box_seal( + Buffer.concat([signedBody, signature]), + serverIdentity.xPublicKey, + )); + const frame = encodeDatumFrame({ + header: { + cmdLen: encrypted.length, + reserved: 0, + isSigned: true, + isEncryptedPubkey: true, + isEncryptedChannel: false, + protoCmd: DatumProtocolCommand.HANDSHAKE_INIT, + }, + payload: encrypted, + }, DATUM_INITIAL_HEADER_KEY); + const header = decodeDatumHeader(frame.subarray(0, 4), DATUM_INITIAL_HEADER_KEY); + + const hello = server.openHandshake(frame.subarray(4, 4 + header.cmdLen)); + expect(hello.userAgent).toBe('datum-test/0'); + expect(hello.nextHeaderKey).toBe(nextHeaderKey); + expect(hello.sessionXPublicKey).toEqual(clientSessionKeys.xPublicKey); + + const response = server.buildHandshakeResponse(hello, 'motd'); + const responseHeader = decodeDatumHeader(response.subarray(0, 4), datumHeaderXorFeedback((~nextHeaderKey) >>> 0)); + expect(responseHeader.protoCmd).toBe(DatumProtocolCommand.HANDSHAKE_RESPONSE); + expect(responseHeader.isSigned).toBe(true); + expect(responseHeader.isEncryptedPubkey).toBe(true); + const opened = Buffer.from(sodium.crypto_box_seal_open( + response.subarray(4), + clientSessionKeys.xPublicKey, + clientSessionKeys.xSecretKey, + )); + expect(opened.subarray(0, 32)).toEqual(clientIdentity.edPublicKey); + expect(opened.subarray(128, 160)).toEqual(serverSessionKeys.edPublicKey); + + const ping = server.encryptChannelFrame(DatumProtocolCommand.PING, Buffer.alloc(0)); + const pingHeader = decodeDatumHeader( + ping.subarray(0, 4), + datumHeaderXorFeedback(datumHeaderXorFeedback((~nextHeaderKey) >>> 0)), + ); + expect(pingHeader.protoCmd).toBe(DatumProtocolCommand.PING); + expect(pingHeader.isEncryptedChannel).toBe(true); + expect(pingHeader.cmdLen).toBe(sodium.crypto_box_MACBYTES); + }); +}); + +function uint32(value: number): Buffer { + const result = Buffer.alloc(4); + result.writeUInt32LE(value >>> 0, 0); + return result; +} diff --git a/src/models/datum/datum-crypto.ts b/src/models/datum/datum-crypto.ts new file mode 100644 index 0000000..39f92ef --- /dev/null +++ b/src/models/datum/datum-crypto.ts @@ -0,0 +1,245 @@ +import * as crypto from 'crypto'; +import * as sodium from 'libsodium-wrappers-sumo'; +import { + DATUM_INITIAL_HEADER_KEY, + DatumFrame, + DatumProtocolCommand, + decodeDatumHeader, + encodeDatumFrame, + datumHeaderXorFeedback, +} from './datum-codec'; + +export interface DatumKeyPair { + edPublicKey: Buffer; + edSecretKey: Buffer; + xPublicKey: Buffer; + xSecretKey: Buffer; +} + +export interface DatumHello { + identityEdPublicKey: Buffer; + identityXPublicKey: Buffer; + sessionEdPublicKey: Buffer; + sessionXPublicKey: Buffer; + userAgent: string; + nextHeaderKey: number; +} + +export class DatumCryptoSession { + private initialized = false; + private sharedSecret: Uint8Array | null = null; + private receiveNonce: Buffer | null = null; + private sendNonce: Buffer | null = null; + private receiveHeaderKey = DATUM_INITIAL_HEADER_KEY; + private sendHeaderKey = DATUM_INITIAL_HEADER_KEY; + private clientIdentityXPublicKey: Buffer | null = null; + + private constructor( + private readonly identityKeys: DatumKeyPair, + private readonly sessionKeys: DatumKeyPair, + ) {} + + public static async create(identityKeys?: DatumKeyPair, sessionKeys?: DatumKeyPair): Promise { + await sodium.ready; + return new DatumCryptoSession( + identityKeys ?? DatumCryptoSession.generateKeyPair(), + sessionKeys ?? DatumCryptoSession.generateKeyPair(), + ); + } + + public get publicKeyHex(): string { + return Buffer.concat([this.identityKeys.edPublicKey, this.identityKeys.xPublicKey]).toString('hex'); + } + + public get currentReceiveHeaderKey(): number { + return this.receiveHeaderKey; + } + + public get currentSendHeaderKey(): number { + return this.sendHeaderKey; + } + + public openHandshake(framePayload: Buffer): DatumHello { + const opened = Buffer.from(sodium.crypto_box_seal_open( + framePayload, + this.identityKeys.xPublicKey, + this.identityKeys.xSecretKey, + )); + if (opened.length < 32 * 4 + 1 + 4 + sodium.crypto_sign_BYTES) { + throw new Error('DATUM handshake payload is too short'); + } + + const signature = opened.subarray(opened.length - sodium.crypto_sign_BYTES); + const signedPayload = opened.subarray(0, opened.length - sodium.crypto_sign_BYTES); + const clientIdentityEdPublicKey = signedPayload.subarray(0, 32); + if (!sodium.crypto_sign_verify_detached(signature, signedPayload, clientIdentityEdPublicKey)) { + throw new Error('DATUM handshake signature verification failed'); + } + + let offset = 0; + const hello: DatumHello = { + identityEdPublicKey: Buffer.from(signedPayload.subarray(offset, offset += 32)), + identityXPublicKey: Buffer.from(signedPayload.subarray(offset, offset += 32)), + sessionEdPublicKey: Buffer.from(signedPayload.subarray(offset, offset += 32)), + sessionXPublicKey: Buffer.from(signedPayload.subarray(offset, offset += 32)), + userAgent: '', + nextHeaderKey: 0, + }; + + const userAgentStart = offset; + while (offset < signedPayload.length && signedPayload[offset] !== 0) { + offset++; + } + hello.userAgent = signedPayload.subarray(userAgentStart, offset).toString('utf8'); + offset++; + + if (signedPayload[offset] !== 0xfe) { + throw new Error('DATUM handshake missing key delimiter'); + } + offset++; + hello.nextHeaderKey = signedPayload.readUInt32LE(offset); + + this.clientIdentityXPublicKey = hello.identityXPublicKey; + this.sharedSecret = sodium.crypto_box_beforenm(hello.sessionXPublicKey, this.sessionKeys.xSecretKey); + this.receiveHeaderKey = datumHeaderXorFeedback(hello.nextHeaderKey); + this.sendHeaderKey = datumHeaderXorFeedback((~hello.nextHeaderKey) >>> 0); + this.initializeNonces(hello.nextHeaderKey, hello.sessionEdPublicKey); + this.initialized = true; + + return hello; + } + + public buildHandshakeResponse(hello: DatumHello, motd: string): Buffer { + if (this.clientIdentityXPublicKey == null) { + throw new Error('Cannot build DATUM handshake response before opening client hello'); + } + + const body = Buffer.concat([ + hello.identityEdPublicKey, + hello.identityXPublicKey, + hello.sessionEdPublicKey, + hello.sessionXPublicKey, + this.sessionKeys.edPublicKey, + this.sessionKeys.xPublicKey, + Buffer.from(`${motd}\0`, 'utf8'), + ]); + const signature = Buffer.from(sodium.crypto_sign_detached(body, this.identityKeys.edSecretKey)); + const encrypted = Buffer.from(sodium.crypto_box_seal( + Buffer.concat([body, signature]), + hello.sessionXPublicKey, + )); + + const response = encodeDatumFrame({ + header: { + cmdLen: encrypted.length, + reserved: 0, + isSigned: true, + isEncryptedPubkey: true, + isEncryptedChannel: false, + protoCmd: DatumProtocolCommand.HANDSHAKE_RESPONSE, + }, + payload: encrypted, + }, this.sendHeaderKey); + this.sendHeaderKey = datumHeaderXorFeedback(this.sendHeaderKey); + return response; + } + + public decryptChannelPayload(payload: Buffer): Buffer { + if (!this.initialized || this.sharedSecret == null || this.receiveNonce == null) { + throw new Error('DATUM channel is not initialized'); + } + const opened = sodium.crypto_box_open_easy_afternm(payload, this.receiveNonce, this.sharedSecret); + this.incrementNonce(this.receiveNonce); + this.receiveHeaderKey = datumHeaderXorFeedback(this.receiveHeaderKey); + return Buffer.from(opened); + } + + public encryptChannelFrame(protoCmd: DatumProtocolCommand, payload: Buffer, signed = false): Buffer { + if (!this.initialized || this.sharedSecret == null || this.sendNonce == null) { + throw new Error('DATUM channel is not initialized'); + } + const signedPayload = signed + ? Buffer.concat([payload, Buffer.from(sodium.crypto_sign_detached(payload, this.sessionKeys.edSecretKey))]) + : payload; + const encrypted = Buffer.from(sodium.crypto_box_easy_afternm(signedPayload, this.sendNonce, this.sharedSecret)); + const frame = encodeDatumFrame({ + header: { + cmdLen: encrypted.length, + reserved: 0, + isSigned: signed, + isEncryptedPubkey: false, + isEncryptedChannel: true, + protoCmd, + }, + payload: encrypted, + }, this.sendHeaderKey); + this.incrementNonce(this.sendNonce); + this.sendHeaderKey = datumHeaderXorFeedback(this.sendHeaderKey); + return frame; + } + + public decodeHeader(headerBytes: Buffer): ReturnType { + return decodeDatumHeader(headerBytes, this.receiveHeaderKey); + } + + private initializeNonces(nextHeaderKey: number, clientSessionEdPublicKey: Buffer): void { + const receiver = Buffer.alloc(sodium.crypto_box_NONCEBYTES); + let key = ((nextHeaderKey - 42) ^ clientSessionEdPublicKey.readUInt32LE(7)) >>> 0; + for (let i = 0; i < receiver.length; i += 4) { + receiver.writeUInt32LE(datumHeaderXorFeedback((key - 42) >>> 0), i); + key = (~receiver.readUInt32LE(i)) >>> 0; + } + + const sender = Buffer.alloc(receiver.length); + for (let i = 0; i < receiver.length; i += 4) { + sender.writeUInt32LE((receiver.readUInt32LE(i) ^ 0x57575757) >>> 0, i); + } + + this.sendNonce = receiver; + this.receiveNonce = sender; + } + + private incrementNonce(nonce: Buffer): void { + for (let offset = 0; offset < nonce.length; offset += 4) { + const next = (nonce.readUInt32LE(offset) + 1) >>> 0; + nonce.writeUInt32LE(next, offset); + if (next !== 0) { + return; + } + } + } + + public static generateKeyPair(): DatumKeyPair { + const seed = crypto.randomBytes(32); + return DatumCryptoSession.generateKeyPairFromSeed(seed); + } + + public static generateKeyPairFromSeed(seed: Buffer): DatumKeyPair { + if (seed.length !== 32) { + throw new RangeError(`DATUM key seed must be 32 bytes, got ${seed.length}`); + } + const ed = sodium.crypto_sign_seed_keypair(seed); + const xPublicKey = sodium.crypto_sign_ed25519_pk_to_curve25519(ed.publicKey); + const xSecretKey = sodium.crypto_sign_ed25519_sk_to_curve25519(ed.privateKey); + return { + edPublicKey: Buffer.from(ed.publicKey), + edSecretKey: Buffer.from(ed.privateKey), + xPublicKey: Buffer.from(xPublicKey), + xSecretKey: Buffer.from(xSecretKey), + }; + } +} + +export function buildDatumPlainFrame(protoCmd: DatumProtocolCommand, payload: Buffer): DatumFrame { + return { + header: { + cmdLen: payload.length, + reserved: 0, + isSigned: false, + isEncryptedPubkey: false, + isEncryptedChannel: false, + protoCmd, + }, + payload, + }; +} diff --git a/src/models/stratum.constants.ts b/src/models/stratum.constants.ts index 28cc459..92eb84b 100644 --- a/src/models/stratum.constants.ts +++ b/src/models/stratum.constants.ts @@ -1,3 +1,5 @@ export const EXTRANONCE1_SIZE_BYTES = 4; export const EXTRANONCE2_SIZE_BYTES = 8; export const TOTAL_EXTRANONCE_SIZE_BYTES = EXTRANONCE1_SIZE_BYTES + EXTRANONCE2_SIZE_BYTES; +export const SV2_EXTENDED_EXTRANONCE2_SIZE_BYTES = 10; +export const SV2_EXTENDED_TOTAL_EXTRANONCE_SIZE_BYTES = EXTRANONCE1_SIZE_BYTES + SV2_EXTENDED_EXTRANONCE2_SIZE_BYTES; diff --git a/src/models/sv2/sv2-extranonce-manager.spec.ts b/src/models/sv2/sv2-extranonce-manager.spec.ts index c920718..10eb6ba 100644 --- a/src/models/sv2/sv2-extranonce-manager.spec.ts +++ b/src/models/sv2/sv2-extranonce-manager.spec.ts @@ -27,6 +27,9 @@ describe('Sv2ExtranonceManager', () => { }); it('minerExtranonceSize is total minus prefix', () => { + const defaultMgr = new Sv2ExtranonceManager(); + expect(defaultMgr.minerExtranonceSize).toBe(10); + const mgr = new Sv2ExtranonceManager(4, 8); expect(mgr.minerExtranonceSize).toBe(4); diff --git a/src/models/sv2/sv2-extranonce-manager.ts b/src/models/sv2/sv2-extranonce-manager.ts index 22e9179..f7ede76 100644 --- a/src/models/sv2/sv2-extranonce-manager.ts +++ b/src/models/sv2/sv2-extranonce-manager.ts @@ -1,4 +1,6 @@ -// ── SV2 Extranonce Manager ────────────────────────────────────────── +import { SV2_EXTENDED_TOTAL_EXTRANONCE_SIZE_BYTES } from '../stratum.constants'; + +// -- SV2 Extranonce Manager ------------------------------------------ // Manages extranonce prefix allocation for extended mining channels. // Ensures no two channels share the same prefix to prevent hash collisions. @@ -13,13 +15,11 @@ export class Sv2ExtranonceManager { /** * @param prefixSize Bytes used for pool-assigned prefix (default 4) - * @param totalExtranonceSize Total extranonce bytes (default 12: 4 prefix + 8 miner-controlled). - * Must match MiningJob.ts's coinbase slot size (12 bytes) — both - * protocols share the same coinbase template. Bumped from the - * previous 8-byte total because the Braiins Hashpower marketplace - * requires extranonce2_size >= 7 (compatibility spec). + * @param totalExtranonceSize Total extranonce bytes (default 14: 4 prefix + 10 miner/JDC-controlled). + * SV2 extended jobs patch the coinbase script length when the + * negotiated total differs from the SV1-compatible template slot. */ - constructor(prefixSize = 4, totalExtranonceSize = 12) { + constructor(prefixSize = 4, totalExtranonceSize = SV2_EXTENDED_TOTAL_EXTRANONCE_SIZE_BYTES) { this.prefixSize = prefixSize; this.totalExtranonceSize = totalExtranonceSize; @@ -34,6 +34,10 @@ export class Sv2ExtranonceManager { return this.totalExtranonceSize - this.prefixSize; } + get totalSize(): number { + return this.totalExtranonceSize; + } + /** * Allocate a unique extranonce prefix for a channel. * Returns a Buffer of `prefixSize` bytes. diff --git a/src/services/bitcoin-rpc.service.ts b/src/services/bitcoin-rpc.service.ts index d0d9171..9f7cb2b 100644 --- a/src/services/bitcoin-rpc.service.ts +++ b/src/services/bitcoin-rpc.service.ts @@ -1,7 +1,8 @@ -import { Injectable, OnModuleInit } from '@nestjs/common'; +import { Injectable, OnModuleInit, Optional } from '@nestjs/common'; import { ConfigService } from '@nestjs/config'; import axios, { AxiosInstance } from 'axios'; import { asyncScheduler, BehaviorSubject, delay, filter, from, interval, scheduled, shareReplay, startWith, Subject, switchMap } from 'rxjs'; +import { PayoutSnapshotService } from '../ORM/payout-snapshot/payout-snapshot.service'; import { RpcBlockService } from '../ORM/rpc-block/rpc-block.service'; import * as zmq from 'zeromq'; @@ -24,7 +25,9 @@ export class BitcoinRpcService implements OnModuleInit { constructor( private readonly configService: ConfigService, private rpcBlockService: RpcBlockService, - private readonly redisMessagingService: RedisMessagingService + private readonly redisMessagingService: RedisMessagingService, + @Optional() + private readonly payoutSnapshotService?: PayoutSnapshotService ) { } @@ -118,7 +121,17 @@ export class BitcoinRpcService implements OnModuleInit { } public async getAndBroadcastLatestTemplate() { + if (this.miningInfo?.blocks == null) { + console.warn('Skipping block template broadcast because mining info is not available'); + return; + } + const blockTemplate = await this.loadBlockTemplate(this.miningInfo.blocks); + if (blockTemplate == null) { + console.warn(`Skipping block template broadcast for height ${this.miningInfo.blocks}; block template is not available`); + return; + } + this._newBlockTemplate$.next(blockTemplate); await this.redisMessagingService.setLatestMiningInfo(this.miningInfo); await this.redisMessagingService.setBlockTemplate(this.miningInfo.blocks, blockTemplate); @@ -158,13 +171,32 @@ export class BitcoinRpcService implements OnModuleInit { let blockTemplate: IBlockTemplate; while (blockTemplate == null) { - blockTemplate = await this.callRpc('getblocktemplate', [ - { - rules: ['segwit'], - mode: 'template', - capabilities: ['serverlist', 'proposal'] - } - ]); + try { + blockTemplate = await this.callRpc('getblocktemplate', [ + { + rules: ['segwit'], + mode: 'template', + capabilities: ['serverlist', 'proposal'] + } + ]); + } catch (e) { + console.warn(`Block template is not available yet: ${e.message ?? e}`); + await new Promise(resolve => setTimeout(resolve, 10_000)); + } + } + + try { + const payoutSnapshot = await this.payoutSnapshotService?.createSnapshotForTemplate({ + blockHeight, + coinbaseValueSats: blockTemplate.coinbasevalue, + networkDifficulty: this.calculateNetworkDifficulty(parseInt(blockTemplate.bits, 16)), + }); + if (payoutSnapshot != null) { + blockTemplate.payoutSnapshotId = payoutSnapshot.id; + blockTemplate.payoutOutputs = payoutSnapshot.payoutOutputs; + } + } catch (e) { + console.error('Error creating payout snapshot', e); } try { @@ -199,13 +231,25 @@ export class BitcoinRpcService implements OnModuleInit { console.log(hexdata); console.log(JSON.stringify(response)); } catch (e) { - response = e; - console.log(`BLOCK SUBMISSION RESPONSE ERROR: ${e}`); + response = e instanceof Error ? e.message : String(e); + console.log(`BLOCK SUBMISSION RESPONSE ERROR: ${response}`); } return response; } + public async TEST_MEMPOOL_ACCEPT(rawTransactions: Buffer[]): Promise> { + return this.callRpc('testmempoolaccept', [ + rawTransactions.map(tx => tx.toString('hex')), + ]); + } + private async callRpc(method: string, params: unknown[] = []): Promise { const response = await this.client.post('', { jsonrpc: '1.0', @@ -221,6 +265,14 @@ export class BitcoinRpcService implements OnModuleInit { return response.data.result; } + private calculateNetworkDifficulty(nBits: number) { + const mantissa: number = nBits & 0x007fffff; + const exponent: number = (nBits >> 24) & 0xff; + const target: number = mantissa * Math.pow(256, (exponent - 3)); + const maxTarget = Math.pow(2, 208) * 65535; + return maxTarget / target; + } + private buildRpcUrl(url: string, port: number): string { const normalizedUrl = /^https?:\/\//i.test(url) ? url : `http://${url}`; const rpcUrl = new URL(normalizedUrl); diff --git a/src/services/custom-work.service.spec.ts b/src/services/custom-work.service.spec.ts new file mode 100644 index 0000000..72ee3c7 --- /dev/null +++ b/src/services/custom-work.service.spec.ts @@ -0,0 +1,94 @@ +import { CustomWorkService } from './custom-work.service'; + +describe('CustomWorkService', () => { + const service = new CustomWorkService(); + + it('builds SV2 custom-job coinbase splits around the extranonce', () => { + const split = service.buildSv2CoinbaseSplit({ + channelId: 1, + requestId: 2, + token: Buffer.from('01', 'hex'), + version: 0x20000000, + prevHash: Buffer.alloc(32), + minNtime: 1, + nBits: 0x170fffff, + coinbaseTxVersion: 2, + coinbasePrefix: Buffer.from('51', 'hex'), + coinbaseTxInputNSequence: 0xffffffff, + coinbaseTxOutputs: Buffer.from('010000000000000000016a', 'hex'), + coinbaseTxLocktime: 0, + merklePath: [], + }, 12); + + expect(split.coinbasePrefix.subarray(0, 4).toString('hex')).toBe('02000000'); + expect(split.coinbasePrefix.includes(Buffer.from([13]))).toBe(true); + expect(split.coinbasePrefix.subarray(-1).toString('hex')).toBe('51'); + expect(split.coinbaseSuffix.subarray(0, 4).toString('hex')).toBe('ffffffff'); + expect(split.coinbaseSuffix.subarray(-4).toString('hex')).toBe('00000000'); + }); + + it('encodes Bitcoin varints at boundary values', () => { + expect(service.encodeBitcoinVarInt(0xfc).toString('hex')).toBe('fc'); + expect(service.encodeBitcoinVarInt(0xfd).toString('hex')).toBe('fdfd00'); + expect(service.encodeBitcoinVarInt(0xffff).toString('hex')).toBe('fdffff'); + expect(service.encodeBitcoinVarInt(0x10000).toString('hex')).toBe('fe00000100'); + }); + + it('validates a trivial custom share against a very easy target', () => { + const result = service.validateShare({ + coinbasePrefix: Buffer.from('0200000001000000000000000000000000000000000000000000000000000000000000000000000000ffffffff01', 'hex'), + coinbaseSuffix: Buffer.from('ffffffff010000000000000000016a00000000', 'hex'), + merklePath: [], + extranoncePrefix: Buffer.alloc(4), + extranonce: Buffer.alloc(8), + prevHash: Buffer.alloc(32), + nBits: 0x207fffff, + version: 0x20000000, + ntime: 1, + nonce: 1, + shareDifficulty: 0.00000001, + networkDifficulty: Number.MAX_SAFE_INTEGER, + }); + + expect(result.header).toHaveLength(80); + expect(result.merkleRoot).toHaveLength(32); + expect(result.submissionDifficulty).toBeGreaterThan(0); + expect(result.accepted).toBe(true); + expect(result.isBlockCandidate).toBe(false); + }); + + it('patches DATUM coinbase target byte before computing the merkle root', () => { + const baseInput = { + coinbasePrefix: Buffer.from('02000000ff', 'hex'), + coinbaseSuffix: Buffer.from('ffffffff010000000000000000016a00000000', 'hex'), + merklePath: [], + extranoncePrefix: Buffer.alloc(0), + extranonce: Buffer.alloc(12), + prevHash: Buffer.alloc(32), + nBits: 0x207fffff, + version: 0x20000000, + ntime: 1, + nonce: 1, + shareDifficulty: 0.00000001, + networkDifficulty: Number.MAX_SAFE_INTEGER, + }; + + const placeholder = service.validateShare(baseInput); + const patched = service.validateShare({ + ...baseInput, + coinbaseTargetByteIndex: 4, + coinbaseTargetByte: 0x12, + }); + const manuallyPatched = service.computeMerkleRoot( + Buffer.concat([ + Buffer.from('0200000012', 'hex'), + Buffer.alloc(12), + baseInput.coinbaseSuffix, + ]), + [], + ); + + expect(patched.merkleRoot.equals(placeholder.merkleRoot)).toBe(false); + expect(patched.merkleRoot.equals(manuallyPatched)).toBe(true); + }); +}); diff --git a/src/services/custom-work.service.ts b/src/services/custom-work.service.ts new file mode 100644 index 0000000..490a501 --- /dev/null +++ b/src/services/custom-work.service.ts @@ -0,0 +1,161 @@ +import { Injectable } from '@nestjs/common'; +import { BufferReader } from '../models/sv2/sv2-binary-codec'; +import { Sv2SetCustomMiningJob } from '../models/sv2/sv2-jdp-messages'; +import { DifficultyUtils } from '../utils/difficulty.utils'; +import { hash256 } from '../utils/hash.utils'; + +export interface CustomWorkCoinbaseSplit { + coinbasePrefix: Buffer; + coinbaseSuffix: Buffer; +} + +export interface CustomWorkShareValidationInput { + coinbasePrefix: Buffer; + coinbaseSuffix: Buffer; + coinbaseTargetByteIndex?: number; + coinbaseTargetByte?: number; + merklePath: Buffer[]; + extranoncePrefix: Buffer; + extranonce: Buffer; + prevHash: Buffer; + nBits: number; + version: number; + ntime: number; + nonce: number; + shareDifficulty: number; + networkDifficulty: number; +} + +export interface CustomWorkShareValidationResult { + header: Buffer; + merkleRoot: Buffer; + hashBuffer: Buffer; + submissionDifficulty: number; + accepted: boolean; + isBlockCandidate: boolean; +} + +@Injectable() +export class CustomWorkService { + public buildSv2CoinbaseSplit(job: Sv2SetCustomMiningJob, totalExtranonceSize: number): CustomWorkCoinbaseSplit { + const scriptSigLen = job.coinbasePrefix.length + totalExtranonceSize; + const scriptSigLenVarint = this.encodeBitcoinVarInt(scriptSigLen); + + const txVersion = Buffer.alloc(4); + txVersion.writeUInt32LE(job.coinbaseTxVersion >>> 0, 0); + + const nullOutpoint = Buffer.alloc(36); + nullOutpoint.writeUInt32LE(0xffffffff, 32); + + const sequence = Buffer.alloc(4); + sequence.writeUInt32LE(job.coinbaseTxInputNSequence >>> 0, 0); + + const locktime = Buffer.alloc(4); + locktime.writeUInt32LE(job.coinbaseTxLocktime >>> 0, 0); + + return { + coinbasePrefix: Buffer.concat([ + txVersion, + Buffer.from([0x01]), + nullOutpoint, + scriptSigLenVarint, + job.coinbasePrefix, + ]), + coinbaseSuffix: Buffer.concat([ + sequence, + job.coinbaseTxOutputs, + locktime, + ]), + }; + } + + public validateShare(input: CustomWorkShareValidationInput): CustomWorkShareValidationResult { + const coinbaseTx = Buffer.concat([ + input.coinbasePrefix, + input.extranoncePrefix, + input.extranonce, + input.coinbaseSuffix, + ]); + if (input.coinbaseTargetByteIndex != null || input.coinbaseTargetByte != null) { + if (input.coinbaseTargetByteIndex == null || input.coinbaseTargetByte == null) { + throw new RangeError('Both coinbase target byte index and value are required'); + } + if (input.coinbaseTargetByteIndex < 0 || input.coinbaseTargetByteIndex >= coinbaseTx.length) { + throw new RangeError(`Coinbase target byte index ${input.coinbaseTargetByteIndex} is outside coinbase length ${coinbaseTx.length}`); + } + coinbaseTx[input.coinbaseTargetByteIndex] = input.coinbaseTargetByte & 0xff; + } + const merkleRoot = this.computeMerkleRoot(coinbaseTx, input.merklePath); + const header = this.buildHeader(input.prevHash, merkleRoot, input.version, input.ntime, input.nBits, input.nonce); + const { submissionDifficulty, hashBuffer } = DifficultyUtils.calculateDifficulty(header); + + return { + header, + merkleRoot, + hashBuffer, + submissionDifficulty, + accepted: DifficultyUtils.meetsTarget(hashBuffer, DifficultyUtils.difficultyToTarget(input.shareDifficulty)), + isBlockCandidate: DifficultyUtils.meetsTarget(hashBuffer, DifficultyUtils.difficultyToTarget(input.networkDifficulty)), + }; + } + + public computeMerkleRoot(coinbaseTx: Buffer, merklePath: Buffer[]): Buffer { + let merkleRoot = hash256(coinbaseTx); + const pair = Buffer.alloc(64); + for (const sibling of merklePath) { + pair.fill(0); + merkleRoot.copy(pair, 0); + sibling.copy(pair, 32); + merkleRoot = hash256(pair); + } + return merkleRoot; + } + + public buildHeader(prevHash: Buffer, merkleRoot: Buffer, version: number, timestamp: number, nBits: number, nonce: number): Buffer { + const header = Buffer.alloc(80); + header.writeInt32LE(version, 0); + prevHash.copy(header, 4); + merkleRoot.copy(header, 36); + header.writeUInt32LE(timestamp >>> 0, 68); + header.writeUInt32LE(nBits >>> 0, 72); + header.writeUInt32LE(nonce >>> 0, 76); + return header; + } + + public encodeBitcoinVarInt(value: number): Buffer { + if (!Number.isSafeInteger(value) || value < 0) { + throw new RangeError(`Invalid Bitcoin varint value ${value}`); + } + if (value < 0xfd) { + return Buffer.from([value]); + } + if (value <= 0xffff) { + const result = Buffer.alloc(3); + result[0] = 0xfd; + result.writeUInt16LE(value, 1); + return result; + } + if (value <= 0xffffffff) { + const result = Buffer.alloc(5); + result[0] = 0xfe; + result.writeUInt32LE(value, 1); + return result; + } + const result = Buffer.alloc(9); + result[0] = 0xff; + result.writeBigUInt64LE(BigInt(value), 1); + return result; + } + + public readDatumNullTerminatedString(reader: BufferReader): string { + const bytes: number[] = []; + while (reader.remaining > 0) { + const byte = reader.readU8(); + if (byte === 0) { + break; + } + bytes.push(byte); + } + return Buffer.from(bytes).toString('utf8'); + } +} diff --git a/src/services/datum.service.spec.ts b/src/services/datum.service.spec.ts new file mode 100644 index 0000000..fd21060 --- /dev/null +++ b/src/services/datum.service.spec.ts @@ -0,0 +1,156 @@ +import { Socket } from 'net'; +import { DatumProtocolCommand } from '../models/datum/datum-codec'; +import { DatumService } from './datum.service'; +import { TemplateProviderService } from './template-provider.service'; + +function createService(): DatumService { + const templateProvider = { + validateTransactionData: jest.fn(({ transactionList, expectedCount, maxTotalBytes }) => { + if (expectedCount != null && transactionList.length !== expectedCount) { + return { valid: false, errorCode: 'transaction-count-mismatch' }; + } + const totalBytes = transactionList.reduce((sum: number, tx: Buffer) => sum + tx.length, 0); + if (maxTotalBytes != null && totalBytes > maxTotalBytes) { + return { valid: false, errorCode: 'transaction-bytes-exceed-limit' }; + } + return { valid: true, totalBytes }; + }), + }; + return new DatumService( + {} as any, + {} as any, + {} as any, + {} as any, + {} as any, + {} as any, + {} as any, + templateProvider as unknown as TemplateProviderService, + undefined, + ); +} + +function createSocket(): Socket { + return { + destroyed: false, + writableEnded: false, + write: jest.fn((_data: Buffer, callback: (error?: Error) => void) => callback()), + } as any; +} + +function createState(cache: any): any { + return { + sessionId: 'datum-test', + session: { + encryptChannelFrame: jest.fn((_command: DatumProtocolCommand, payload: Buffer) => payload), + }, + datumJobs: new Map([[7, cache]]), + }; +} + +describe('DatumService job validation', () => { + it('requests short transaction IDs once a DATUM job advertises transactions', async () => { + const service = createService() as any; + const socket = createSocket(); + const cache: any = { + transactionCount: 2, + coinbasePairs: new Map(), + }; + const state = createState(cache); + + await service.maybeRequestDatumJobValidation(socket, state, 7, cache); + + expect(cache.validationState).toBe('requested-short-txids'); + expect((socket.write as jest.Mock).mock.calls[0][0].toString('hex')).toBe('501007'); + expect(state.session.encryptChannelFrame).toHaveBeenCalledWith(DatumProtocolCommand.MINING, expect.any(Buffer)); + }); + + it('follows short transaction IDs with a full transaction blob request', async () => { + const service = createService() as any; + const socket = createSocket(); + const cache: any = { + transactionCount: 1, + coinbasePairs: new Map(), + }; + const state = createState(cache); + const shortIdsResponse = Buffer.concat([ + Buffer.from([0x90, 7, 0x01]), + Buffer.from('0100', 'hex'), + Buffer.from('010203040506', 'hex'), + Buffer.alloc(32, 0xaa), + Buffer.from([0xfe]), + ]); + + await service.handleJobValidationResponse(socket, state, shortIdsResponse); + + expect(cache.validationState).toBe('requested-full-transaction-blob'); + expect(cache.validationShortTxIds.map((id: Buffer) => id.toString('hex'))).toEqual(['010203040506']); + expect((socket.write as jest.Mock).mock.calls[0][0].toString('hex')).toBe('501207'); + }); + + it('fails DATUM job validation when short transaction ID counts disagree', async () => { + const service = createService() as any; + const socket = createSocket(); + const warn = jest.spyOn(console, 'warn').mockImplementation(() => undefined); + const cache: any = { + transactionCount: 2, + coinbasePairs: new Map(), + }; + const state = createState(cache); + const shortIdsResponse = Buffer.concat([ + Buffer.from([0x90, 7, 0x01]), + Buffer.from('0100', 'hex'), + Buffer.from('010203040506', 'hex'), + Buffer.alloc(32, 0xaa), + Buffer.from([0xfe]), + ]); + + await service.handleJobValidationResponse(socket, state, shortIdsResponse); + + expect(cache.validationState).toBe('failed'); + expect(cache.validationError).toBe('short txid count mismatch: advertised 2, received 1'); + expect(socket.write).not.toHaveBeenCalled(); + warn.mockRestore(); + }); + + it('validates and caches full DATUM transaction blobs', async () => { + const service = createService() as any; + const socket = createSocket(); + const tx = Buffer.from( + '0100000001' + + '0000000000000000000000000000000000000000000000000000000000000000' + + 'ffffffff00ffffffff' + + '01000000000000000000' + + '00000000', + 'hex', + ); + const txSize = Buffer.alloc(3); + txSize.writeUIntLE(tx.length, 0, 3); + const cache: any = { + transactionCount: 1, + totalSize: tx.length, + merkleBranches: [Buffer.alloc(32, 0xbb)], + coinbasePairs: new Map(), + }; + const state = createState(cache); + const fullBlobResponse = Buffer.concat([ + Buffer.from([0x92, 7, 0x01]), + Buffer.from('0100', 'hex'), + txSize, + tx, + Buffer.from([0xfe]), + ]); + + await service.handleJobValidationResponse(socket, state, fullBlobResponse); + + expect(cache.validationState).toBe('validated'); + expect(cache.validationTransactions).toEqual([tx]); + expect(cache.validationError).toBeUndefined(); + expect((service as any).templateProvider.validateTransactionData).toHaveBeenCalledWith({ + transactionList: [tx], + expectedCount: 1, + maxTotalBytes: tx.length, + expectedCoinbaseMerklePath: [Buffer.alloc(32, 0xbb)], + }); + expect(socket.write).not.toHaveBeenCalled(); + }); +}); diff --git a/src/services/datum.service.ts b/src/services/datum.service.ts new file mode 100644 index 0000000..3805709 --- /dev/null +++ b/src/services/datum.service.ts @@ -0,0 +1,722 @@ +import { Injectable, OnModuleInit } from '@nestjs/common'; +import { ConfigService } from '@nestjs/config'; +import { getAddressInfo } from 'bitcoin-address-validation'; +import * as bitcoinjs from 'bitcoinjs-lib'; +import { firstValueFrom } from 'rxjs'; +import { Server, Socket } from 'net'; + +import { BlocksService } from '../ORM/blocks/blocks.service'; +import { ClientEntity } from '../ORM/client/client.entity'; +import { ClientService } from '../ORM/client/client.service'; +import { ShareAccountingService } from '../ORM/share-accounting/share-accounting.service'; +import { + DATUM_INITIAL_HEADER_KEY, + DatumFrameReader, + DatumMiningCommand, + DatumProtocolCommand, + DatumRejectReason, + DatumShareResponseStatus, + DatumPowSubmit, + DatumJobValidationStatus, + DatumJobValidationResponse, + deserializeDatumCoinbaserFetch, + deserializeDatumJobValidationResponse, + deserializeDatumMiningCommand, + deserializeDatumPowSubmit, + encodeDatumFrame, + serializeDatumJobValidationFullTransactionBlobRequest, + serializeDatumJobValidationShortTxIdsRequest, + serializeDatumCoinbaserFetchResponse, + serializeDatumShareResponse, +} from '../models/datum/datum-codec'; +import { DatumCryptoSession } from '../models/datum/datum-crypto'; +import { BitcoinRpcService } from './bitcoin-rpc.service'; +import { CustomWorkService } from './custom-work.service'; +import { RedisMessagingService } from './redis-messaging.service'; +import { StratumV1JobsService } from './stratum-v1-jobs.service'; +import { DatumKeyPair } from '../models/datum/datum-crypto'; +import { TemplateProviderService } from './template-provider.service'; + +const DEFAULT_DATUM_SHARE_DIFFICULTY = 1; +const DEFAULT_DATUM_PING_INTERVAL_MS = 30_000; + +@Injectable() +export class DatumService implements OnModuleInit { + private readonly servers: Server[] = []; + private identityKeys: DatumKeyPair | null = null; + + constructor( + private readonly configService: ConfigService, + private readonly jobsService: StratumV1JobsService, + private readonly bitcoinRpcService: BitcoinRpcService, + private readonly clientService: ClientService, + private readonly blocksService: BlocksService, + private readonly shareAccountingService: ShareAccountingService, + private readonly customWorkService: CustomWorkService, + private readonly templateProvider: TemplateProviderService, + private readonly redisMessagingService?: RedisMessagingService, + ) {} + + public async onModuleInit(): Promise { + if (process.env.API_ONLY === 'true' || process.env.MASTER === 'true') { + return; + } + + const ports = this.getPorts(); + if (ports.length === 0) { + return; + } + + this.identityKeys = this.getIdentityKeys(); + console.log(`DATUM server public key: ${Buffer.concat([this.identityKeys.edPublicKey, this.identityKeys.xPublicKey]).toString('hex')}`); + + for (const port of ports) { + await this.startServer(port); + } + } + + private async startServer(port: number): Promise { + const server = new Server(socket => { + void this.handleSocket(socket); + }); + server.on('error', error => { + console.error(`DATUM server error on port ${port}: ${error.message}`); + }); + server.listen(port, () => { + console.log(`DATUM server is listening on port ${port}`); + }); + this.servers.push(server); + } + + private async handleSocket(socket: Socket): Promise { + socket.setKeepAlive(true, 60_000); + socket.setNoDelay(true); + + const session = await DatumCryptoSession.create(this.identityKeys ?? undefined); + const reader = new DatumFrameReader(DATUM_INITIAL_HEADER_KEY); + const state: DatumClientState = { + sessionId: Math.random().toString(16).slice(2, 10), + session, + clientEntity: null, + address: null, + workerName: 'default', + userAgent: 'datum/unknown', + pingTimer: null, + datumJobs: new Map(), + }; + console.log(`[DATUM ${state.sessionId}] connection accepted from ${socket.remoteAddress}:${socket.remotePort}`); + + const close = () => { + void this.destroyClient(state); + if (!socket.destroyed) { + socket.destroy(); + } + }; + + socket.on('data', data => { + void (async () => { + try { + reader.setHeaderXorKey(session.currentReceiveHeaderKey); + const frames = reader.feed(data); + for (const frame of frames) { + let payload = frame.payload; + if (frame.header.isEncryptedChannel) { + payload = session.decryptChannelPayload(payload); + } + await this.handleFrame(socket, state, frame.header.protoCmd, payload); + reader.setHeaderXorKey(session.currentReceiveHeaderKey); + } + } catch (error) { + console.error(`[DATUM ${state.sessionId}] ${error.message}`); + close(); + } + })(); + }); + socket.on('error', close); + socket.on('close', () => void this.destroyClient(state)); + } + + private async handleFrame(socket: Socket, state: DatumClientState, protoCmd: number, payload: Buffer): Promise { + if (protoCmd === DatumProtocolCommand.HANDSHAKE_INIT) { + const hello = state.session.openHandshake(payload); + state.userAgent = hello.userAgent || 'datum/unknown'; + console.log(`[DATUM ${state.sessionId}] handshake init from ${state.userAgent}`); + await this.writeRaw(socket, state.session.buildHandshakeResponse(hello, 'public-pool DATUM')); + console.log(`[DATUM ${state.sessionId}] handshake response sent`); + await this.sendDatumClientConfigure(socket, state); + console.log(`[DATUM ${state.sessionId}] client configure sent`); + this.startDatumPing(socket, state); + return; + } + + if (protoCmd !== DatumProtocolCommand.MINING) { + return; + } + + const mining = deserializeDatumMiningCommand(payload); + switch (mining.command) { + case DatumMiningCommand.FETCH_COINBASER: + await this.handleCoinbaserFetch(socket, state, mining.body); + break; + case DatumMiningCommand.SUBMIT_POW: + await this.handlePowSubmit(socket, state, mining.body); + break; + case DatumMiningCommand.JOB_VALIDATION: + await this.handleJobValidationResponse(socket, state, mining.body); + break; + case DatumMiningCommand.TEMPLATE_REFRESH: + break; + default: + console.warn(`[DATUM ${state.sessionId}] Ignoring unsupported mining command 0x${mining.command.toString(16)}`); + break; + } + } + + private async sendDatumClientConfigure(socket: Socket, state: DatumClientState): Promise { + const payoutAddress = this.getDatumPoolPayoutAddress(); + if (!payoutAddress) { + throw new Error('DATUM_POOL_PAYOUT_ADDRESS or DEV_FEE_ADDRESS must be set before DATUM client configuration can be sent'); + } + + const payoutScript = bitcoinjs.address.toOutputScript(payoutAddress, this.getNetwork()); + if (payoutScript.length > 255) { + throw new Error(`DATUM payout script is too long: ${payoutScript.length}`); + } + + const coinbaseTag = Buffer.from('public-pool', 'utf8'); + if (coinbaseTag.length > 255) { + throw new Error(`DATUM coinbase tag is too long: ${coinbaseTag.length}`); + } + + const minDifficulty = Math.max(1, Math.floor(this.getDatumShareDifficulty())); + const vardiffMin = 1n << BigInt(Math.ceil(Math.log2(minDifficulty))); + const vardiffMinBuffer = Buffer.alloc(8); + vardiffMinBuffer.writeBigUInt64LE(vardiffMin, 0); + + const payload = Buffer.concat([ + Buffer.from([DatumMiningCommand.CLIENT_CONFIGURE]), + Buffer.from([1, payoutScript.length]), + payoutScript, + uint32le(0x50554250), + Buffer.from([coinbaseTag.length]), + coinbaseTag, + vardiffMinBuffer, + Buffer.from([0, 0xfe]), + ]); + await this.writeRaw(socket, state.session.encryptChannelFrame(DatumProtocolCommand.MINING, payload, true)); + } + + private async handleCoinbaserFetch(socket: Socket, state: DatumClientState, payload: Buffer): Promise { + const fetch = deserializeDatumCoinbaserFetch(payload); + const payoutAddress = this.getDatumPoolPayoutAddress(); + if (!payoutAddress) { + throw new Error('DATUM_POOL_PAYOUT_ADDRESS or DEV_FEE_ADDRESS must be set before DATUM coinbaser fetches can be served'); + } + + const scriptPubKey = bitcoinjs.address.toOutputScript(payoutAddress, this.getNetwork()); + const response = serializeDatumCoinbaserFetchResponse(fetch.rewardValue, [{ + value: fetch.rewardValue, + scriptPubKey, + }]); + await this.writeRaw(socket, state.session.encryptChannelFrame(DatumProtocolCommand.MINING, response)); + } + + private async handlePowSubmit(socket: Socket, state: DatumClientState, payload: Buffer): Promise { + const pow = deserializeDatumPowSubmit(payload); + const { address, workerName } = this.parseUserIdentity(pow.username); + if (!this.isValidAddress(address)) { + await this.sendShareResponse(socket, state, DatumShareResponseStatus.REJECTED, DatumRejectReason.BAD_USERNAME, pow.nonce, pow.targetByte, pow.jobId); + return; + } + + state.address = address; + state.workerName = workerName; + await this.ensureClientEntity(state); + + const datumJob = this.updateDatumJobCache(state, pow); + await this.maybeRequestDatumJobValidation(socket, state, pow.jobId, datumJob); + const coinbase = this.getDatumCoinbase(datumJob, pow); + if (coinbase == null || datumJob.prevBlockHash == null || datumJob.nBits == null || datumJob.merkleBranches == null) { + await this.sendShareResponse(socket, state, DatumShareResponseStatus.REJECTED, DatumRejectReason.COINBASE_MISSING, pow.nonce, pow.targetByte, pow.jobId); + return; + } + if (datumJob.targetByteIndex == null) { + await this.sendShareResponse(socket, state, DatumShareResponseStatus.REJECTED, DatumRejectReason.BAD_TARGET, pow.nonce, pow.targetByte, pow.jobId); + return; + } + + const latestTemplate = await firstValueFrom(this.jobsService.newMiningJob$); + const templateValidation = this.templateProvider.validateDatumTemplateFastPath({ + prevBlockHash: datumJob.prevBlockHash, + nBits: datumJob.nBits, + height: datumJob.height, + version: pow.version, + coinbaseValue: datumJob.coinbaseValue, + totalWeight: datumJob.totalWeight, + totalSize: datumJob.totalSize, + totalSigops: datumJob.totalSigops, + merkleBranches: datumJob.merkleBranches, + }); + if (!templateValidation.valid) { + await this.sendShareResponse( + socket, + state, + DatumShareResponseStatus.REJECTED, + this.mapDatumTemplateRejectReason(templateValidation.errorCode), + pow.nonce, + pow.targetByte, + pow.jobId, + ); + return; + } + if (datumJob.validationState === 'failed') { + await this.sendShareResponse(socket, state, DatumShareResponseStatus.REJECTED, DatumRejectReason.OTHER, pow.nonce, pow.targetByte, pow.jobId); + return; + } + + const shareDifficulty = this.getDatumShareDifficulty(); + const nBits = datumJob.nBits.readUInt32LE(0); + const validation = this.customWorkService.validateShare({ + coinbasePrefix: coinbase.coinb1, + coinbaseSuffix: coinbase.coinb2, + coinbaseTargetByteIndex: datumJob.targetByteIndex, + coinbaseTargetByte: pow.targetByte, + merklePath: datumJob.merkleBranches, + extranoncePrefix: Buffer.alloc(0), + extranonce: pow.extranonce, + prevHash: datumJob.prevBlockHash, + nBits, + version: pow.version, + ntime: pow.ntime, + nonce: pow.nonce, + shareDifficulty, + networkDifficulty: latestTemplate.blockData.networkDifficulty, + }); + + if (!validation.accepted) { + await this.sendShareResponse(socket, state, DatumShareResponseStatus.REJECTED, DatumRejectReason.HIGH_HASH, pow.nonce, pow.targetByte, pow.jobId); + return; + } + + const isBlockCandidate = validation.isBlockCandidate || pow.isBlock; + const blockSubmissionResult = validation.isBlockCandidate ? 'datum-gateway-submit-expected' : null; + + await this.sendShareResponse(socket, state, DatumShareResponseStatus.ACCEPTED, 0, pow.nonce, pow.targetByte, pow.jobId); + if (isBlockCandidate) { + await this.blocksService.save({ + height: datumJob.height ?? latestTemplate.blockData.height, + minerAddress: address, + worker: workerName, + sessionId: state.sessionId, + blockData: validation.header.toString('hex'), + blockSubmissionResult, + payoutSnapshotId: null, + }); + } + await this.shareAccountingService.recordAcceptedShare({ + protocol: 'datum', + workSource: 'miner_template', + workProtocol: 'datum', + address, + clientName: workerName, + sessionId: state.sessionId, + clientId: state.clientEntity.id, + jobId: pow.jobId.toString(16), + jobTemplateId: latestTemplate.blockData.id, + blockHeight: datumJob.height ?? latestTemplate.blockData.height, + creditedDifficulty: shareDifficulty, + submissionDifficulty: validation.submissionDifficulty, + networkDifficulty: latestTemplate.blockData.networkDifficulty, + nonce: pow.nonce, + ntime: pow.ntime, + version: pow.version, + extraNonce2: pow.extranonce.toString('hex'), + isBlockCandidate, + blockSubmissionResult, + }); + await this.redisMessagingService?.setClientPresence({ + clientId: state.clientEntity.id, + address, + clientName: workerName, + sessionId: state.sessionId, + userAgent: state.userAgent, + startTime: new Date(state.clientEntity.startTime).toISOString(), + lastSeen: new Date().toISOString(), + hashRate: 0, + bestDifficulty: validation.submissionDifficulty, + }); + } + + private updateDatumJobCache(state: DatumClientState, pow: DatumPowSubmit): DatumJobCache { + let cache = state.datumJobs.get(pow.jobId); + if (cache == null || pow.prevBlockHash != null) { + cache = { + coinbasePairs: new Map(), + }; + state.datumJobs.set(pow.jobId, cache); + } + + if (pow.prevBlockHash != null) { + cache.prevBlockHash = pow.prevBlockHash; + } + if (pow.targetByteIndex != null) { + cache.targetByteIndex = pow.targetByteIndex; + } + if (pow.nBits != null) { + cache.nBits = pow.nBits; + } + if (pow.coinbaserId != null) { + cache.coinbaserId = pow.coinbaserId; + } + if (pow.height != null) { + cache.height = pow.height; + } + if (pow.coinbaseValue != null) { + cache.coinbaseValue = pow.coinbaseValue; + } + if (pow.transactionCount != null) { + cache.transactionCount = pow.transactionCount; + } + if (pow.totalWeight != null) { + cache.totalWeight = pow.totalWeight; + } + if (pow.totalSize != null) { + cache.totalSize = pow.totalSize; + } + if (pow.totalSigops != null) { + cache.totalSigops = pow.totalSigops; + } + if (pow.merkleBranches != null) { + cache.merkleBranches = pow.merkleBranches; + } + for (const [coinbaseId, coinbase] of pow.coinbasePairs) { + cache.coinbasePairs.set(coinbaseId, coinbase); + } + if (pow.subsidyOnlyCoinbase != null) { + cache.subsidyOnlyCoinbase = pow.subsidyOnlyCoinbase; + } + + return cache; + } + + private async maybeRequestDatumJobValidation( + socket: Socket, + state: DatumClientState, + jobId: number, + cache: DatumJobCache, + ): Promise { + if (cache.transactionCount == null || cache.validationState != null) { + return; + } + if (cache.transactionCount === 0) { + cache.validationState = 'validated'; + cache.validationTransactions = []; + return; + } + + cache.validationState = 'requested-short-txids'; + cache.validationRequestedAt = new Date(); + await this.writeRaw( + socket, + state.session.encryptChannelFrame( + DatumProtocolCommand.MINING, + serializeDatumJobValidationShortTxIdsRequest(jobId), + ), + ); + } + + private async handleJobValidationResponse(socket: Socket, state: DatumClientState, payload: Buffer): Promise { + const response = deserializeDatumJobValidationResponse(payload); + const cache = state.datumJobs.get(response.jobId); + if (cache == null) { + console.warn(`[DATUM ${state.sessionId}] Ignoring validation response for unknown job ${response.jobId}`); + return; + } + + if (response.status !== DatumJobValidationStatus.SUCCESS) { + cache.validationState = 'failed'; + cache.validationError = `gateway returned status 0x${response.status.toString(16)}`; + console.warn(`[DATUM ${state.sessionId}] Job ${response.jobId} validation failed: ${cache.validationError}`); + return; + } + + if (response.kind === 'short-txids') { + if (cache.transactionCount != null && response.transactionCount !== cache.transactionCount) { + cache.validationState = 'failed'; + cache.validationError = `short txid count mismatch: advertised ${cache.transactionCount}, received ${response.transactionCount}`; + console.warn(`[DATUM ${state.sessionId}] Job ${response.jobId} validation failed: ${cache.validationError}`); + return; + } + cache.validationShortTxIds = response.shortTxIds; + cache.validationShortTxCrosscheck = response.crosscheck ?? undefined; + cache.validationState = 'requested-full-transaction-blob'; + await this.writeRaw( + socket, + state.session.encryptChannelFrame( + DatumProtocolCommand.MINING, + serializeDatumJobValidationFullTransactionBlobRequest(response.jobId), + ), + ); + return; + } + + if (response.kind === 'full-transaction-blob' || response.kind === 'full-transactions') { + const validationError = this.validateDatumTransactionResponse(cache, response); + if (validationError != null) { + cache.validationState = 'failed'; + cache.validationError = validationError; + console.warn(`[DATUM ${state.sessionId}] Job ${response.jobId} transaction validation failed: ${validationError}`); + return; + } + + cache.validationState = 'validated'; + cache.validationError = undefined; + cache.validationTransactions = response.transactions; + cache.validationCompletedAt = new Date(); + } + } + + private validateDatumTransactionResponse( + cache: DatumJobCache, + response: DatumJobValidationResponse, + ): string | null { + if (response.kind === 'short-txids') { + return null; + } + if (cache.transactionCount != null && response.transactionCount !== cache.transactionCount) { + return `transaction count mismatch: advertised ${cache.transactionCount}, received ${response.transactionCount}`; + } + + const validation = this.templateProvider.validateTransactionData({ + transactionList: response.transactions, + expectedCount: cache.transactionCount, + maxTotalBytes: cache.totalSize, + expectedCoinbaseMerklePath: cache.merkleBranches, + }); + if (!validation.valid) { + return validation.errorCode ?? 'invalid-transaction-data'; + } + + return null; + } + + private getDatumCoinbase(cache: DatumJobCache, pow: DatumPowSubmit): { coinb1: Buffer; coinb2: Buffer } | undefined { + if (pow.subsidyOnly) { + return cache.subsidyOnlyCoinbase ?? cache.coinbasePairs.get(pow.coinbaseId); + } + return cache.coinbasePairs.get(pow.coinbaseId) ?? cache.subsidyOnlyCoinbase; + } + + private async sendShareResponse( + socket: Socket, + state: DatumClientState, + status: DatumShareResponseStatus, + reasonCode: number, + nonce: number, + targetByte: number, + jobId: number, + ): Promise { + const payload = serializeDatumShareResponse({ status, reasonCode, nonce, targetByte, jobId }); + await this.writeRaw(socket, state.session.encryptChannelFrame(DatumProtocolCommand.MINING, payload)); + } + + private async ensureClientEntity(state: DatumClientState): Promise { + if (state.clientEntity != null) { + return; + } + state.clientEntity = await this.clientService.insert({ + sessionId: state.sessionId, + address: state.address, + clientName: state.workerName, + userAgent: state.userAgent, + startTime: new Date(), + bestDifficulty: 0, + }); + } + + private async destroyClient(state: DatumClientState): Promise { + this.stopDatumPing(state); + if (state.clientEntity?.id == null) { + return; + } + await this.redisMessagingService?.removeClientPresence(state.clientEntity.id, state.clientEntity.address); + await this.clientService.delete(state.clientEntity.id); + state.clientEntity = null; + } + + private startDatumPing(socket: Socket, state: DatumClientState): void { + this.stopDatumPing(state); + state.pingTimer = setInterval(() => { + if (socket.destroyed || socket.writableEnded) { + this.stopDatumPing(state); + return; + } + void this.writeRaw( + socket, + state.session.encryptChannelFrame(DatumProtocolCommand.PING, Buffer.alloc(0)), + ).catch(error => { + console.error(`[DATUM ${state.sessionId}] ping failed: ${error.message}`); + if (!socket.destroyed) { + socket.destroy(); + } + }); + }, this.getDatumPingIntervalMs()); + state.pingTimer.unref?.(); + } + + private stopDatumPing(state: DatumClientState): void { + if (state.pingTimer != null) { + clearInterval(state.pingTimer); + state.pingTimer = null; + } + } + + private async writeRaw(socket: Socket, data: Buffer): Promise { + if (socket.destroyed || socket.writableEnded) { + return; + } + await new Promise((resolve, reject) => { + socket.write(data, error => error ? reject(error) : resolve()); + }); + } + + private parseUserIdentity(userIdentity: string): { address: string; workerName: string } { + const parts = userIdentity.split('.'); + const address = parts[0] ?? ''; + return { + address: this.normalizeAddress(address), + workerName: parts.length > 1 ? parts.slice(1).join('.') : 'default', + }; + } + + private normalizeAddress(address: string): string { + if (/^(bc1|tb1|bcrt1)/i.test(address)) { + return address.toLowerCase(); + } + return address; + } + + private isValidAddress(address: string): boolean { + try { + getAddressInfo(address); + return true; + } catch { + return false; + } + } + + private getDatumPoolPayoutAddress(): string { + return this.configService.get('DATUM_POOL_PAYOUT_ADDRESS') + || this.configService.get('PAYOUT_FEE_ADDRESS') + || this.configService.get('DEV_FEE_ADDRESS') + || ''; + } + + private getDatumShareDifficulty(): number { + const configured = parseFloat(this.configService.get('DATUM_SHARE_DIFFICULTY') ?? ''); + return Number.isFinite(configured) && configured > 0 ? configured : DEFAULT_DATUM_SHARE_DIFFICULTY; + } + + private mapDatumTemplateRejectReason(errorCode?: string): DatumRejectReason { + switch (errorCode) { + case 'prevhash-mismatch': + case 'height-mismatch': + return DatumRejectReason.STALE_BLOCK; + case 'nbits-mismatch': + case 'weight-limit-exceeded': + case 'size-limit-exceeded': + case 'sigop-limit-exceeded': + case 'coinbase-value-too-high': + return DatumRejectReason.OTHER; + case 'version-mismatch': + return DatumRejectReason.BAD_VERSION; + case 'bad-merkle-branch': + return DatumRejectReason.BAD_MERKLE_COUNT; + default: + return DatumRejectReason.OTHER; + } + } + + private getDatumPingIntervalMs(): number { + const configured = parseInt(this.configService.get('DATUM_PING_INTERVAL_MS') ?? '', 10); + return Number.isFinite(configured) && configured > 0 ? configured : DEFAULT_DATUM_PING_INTERVAL_MS; + } + + private getIdentityKeys(): DatumKeyPair { + const seedHex = this.configService.get('DATUM_IDENTITY_SEED')?.trim(); + if (!seedHex) { + const keys = DatumCryptoSession.generateKeyPair(); + console.warn('DATUM_IDENTITY_SEED is not set; generated an ephemeral DATUM server identity key'); + return keys; + } + if (!/^[0-9a-fA-F]{64}$/.test(seedHex)) { + throw new Error('DATUM_IDENTITY_SEED must be a 32-byte hex string'); + } + return DatumCryptoSession.generateKeyPairFromSeed(Buffer.from(seedHex, 'hex')); + } + + private getPorts(): number[] { + const configured = this.configService.get('DATUM_PORTS'); + if (!configured?.trim()) { + return []; + } + return Array.from(new Set(configured + .split(',') + .map(port => parseInt(port.trim(), 10)) + .filter(port => Number.isInteger(port) && port > 0 && port <= 65535))); + } + + private getNetwork(): bitcoinjs.networks.Network { + const networkConfig = this.configService.get('NETWORK'); + if (networkConfig === 'mainnet') { + return bitcoinjs.networks.bitcoin; + } + if (networkConfig === 'testnet') { + return bitcoinjs.networks.testnet; + } + if (networkConfig === 'regtest') { + return bitcoinjs.networks.regtest; + } + throw new Error('Invalid network configuration'); + } +} + +interface DatumClientState { + sessionId: string; + session: DatumCryptoSession; + clientEntity: ClientEntity | null; + address: string | null; + workerName: string; + userAgent: string; + pingTimer: NodeJS.Timeout | null; + datumJobs: Map; +} + +interface DatumJobCache { + prevBlockHash?: Buffer; + targetByteIndex?: number; + nBits?: Buffer; + coinbaserId?: number; + height?: number; + coinbaseValue?: bigint; + transactionCount?: number; + totalWeight?: number; + totalSize?: number; + totalSigops?: number; + merkleBranches?: Buffer[]; + coinbasePairs: Map; + subsidyOnlyCoinbase?: { coinb1: Buffer; coinb2: Buffer }; + validationState?: 'requested-short-txids' | 'requested-full-transaction-blob' | 'validated' | 'failed'; + validationRequestedAt?: Date; + validationCompletedAt?: Date; + validationShortTxIds?: Buffer[]; + validationShortTxCrosscheck?: Buffer; + validationTransactions?: Buffer[]; + validationError?: string; +} + +function uint32le(value: number): Buffer { + const result = Buffer.alloc(4); + result.writeUInt32LE(value >>> 0, 0); + return result; +} diff --git a/src/services/stratum-v1-jobs.service.spec.ts b/src/services/stratum-v1-jobs.service.spec.ts index b0a256e..6aa3654 100644 --- a/src/services/stratum-v1-jobs.service.spec.ts +++ b/src/services/stratum-v1-jobs.service.spec.ts @@ -91,6 +91,24 @@ describe('StratumV1JobsService', () => { expect(service.getJobTemplateById(jobTemplate.blockData.id)).toBe(jobTemplate); }); + it('should emit when payout snapshot metadata changes', async () => { + await firstValueFrom(service.newMiningJob$); + + const payoutTemplate = createTemplate(); + payoutTemplate.payoutSnapshotId = '42'; + payoutTemplate.payoutOutputs = [ + { address: 'tb1qumezefzdeqqwn5zfvgdrhxjzc5ylr39uhuxcz4', amountSats: 123456 }, + ]; + + const nextTemplate = firstValueFrom(service.newMiningJob$.pipe(skip(1))); + blockTemplate$.next(payoutTemplate); + const jobTemplate = await nextTemplate; + + expect(jobTemplate.blockData.clearJobs).toBe(false); + expect(jobTemplate.blockData.payoutSnapshotId).toBe('42'); + expect(jobTemplate.blockData.payoutOutputs).toEqual(payoutTemplate.payoutOutputs); + }); + it('should age old jobs and templates after five minutes', async () => { await firstValueFrom(service.newMiningJob$); const oldCreation = Date.now() - (1000 * 60 * 11); diff --git a/src/services/stratum-v1-jobs.service.ts b/src/services/stratum-v1-jobs.service.ts index 9247be2..828c251 100644 --- a/src/services/stratum-v1-jobs.service.ts +++ b/src/services/stratum-v1-jobs.service.ts @@ -4,7 +4,8 @@ import * as merkle from 'merkle-lib'; import * as merkleProof from 'merkle-lib/proof'; import { combineLatest, delay, filter, from, interval, map, Observable, shareReplay, startWith, switchMap, tap } from 'rxjs'; -import { MiningJob } from '../models/MiningJob'; +import { IBlockTemplateTx } from '../models/bitcoin-rpc/IBlockTemplate'; +import { AddressObject, MiningJob } from '../models/MiningJob'; import { hash256 } from '../utils/hash.utils'; import { BitcoinRpcService } from './bitcoin-rpc.service'; @@ -19,6 +20,12 @@ export interface IJobTemplate { networkDifficulty: number; height: number; clearJobs: boolean; + payoutSnapshotId?: string; + payoutOutputs?: AddressObject[]; + transactions?: IBlockTemplateTx[]; + sigoplimit?: number; + sizelimit?: number; + weightlimit?: number; }; } @@ -63,6 +70,12 @@ export class StratumV1JobsService { timestamp, blockTemplate.height, blockTemplate.coinbasevalue, + blockTemplate.payoutSnapshotId ?? '', + ...(blockTemplate.payoutOutputs ?? []).map(output => [ + output.address, + output.amountSats ?? '', + output.percent ?? '' + ].join(':')), ...blockTemplate.transactions.map(tx => tx.hash ?? tx.txid ?? tx.data) ].join('|'); @@ -80,11 +93,17 @@ export class StratumV1JobsService { timestamp, networkDifficulty: this.calculateNetworkDifficulty(parseInt(blockTemplate.bits, 16)), clearJobs, - height: blockTemplate.height + height: blockTemplate.height, + payoutSnapshotId: blockTemplate.payoutSnapshotId, + payoutOutputs: blockTemplate.payoutOutputs, + rawTransactions: blockTemplate.transactions, + sigoplimit: blockTemplate.sigoplimit, + sizelimit: blockTemplate.sizelimit, + weightlimit: blockTemplate.weightlimit, }; }), filter(next => next != null), - map(({ version, bits, prevHash, transactions, timestamp, coinbasevalue, networkDifficulty, clearJobs, height }) => { + map(({ version, bits, prevHash, transactions, timestamp, coinbasevalue, networkDifficulty, clearJobs, height, payoutSnapshotId, payoutOutputs, rawTransactions, sigoplimit, sizelimit, weightlimit }) => { const block = new bitcoinjs.Block(); //create an empty coinbase tx @@ -122,7 +141,13 @@ export class StratumV1JobsService { coinbasevalue, networkDifficulty, height, - clearJobs + clearJobs, + payoutSnapshotId, + payoutOutputs, + transactions: rawTransactions, + sigoplimit, + sizelimit, + weightlimit, } } }), diff --git a/src/services/stratum-v1.service.ts b/src/services/stratum-v1.service.ts index 758f45b..3833c95 100644 --- a/src/services/stratum-v1.service.ts +++ b/src/services/stratum-v1.service.ts @@ -9,6 +9,7 @@ import { UserAgentReportService } from '../ORM/_views/user-agent-report/user-age import { AddressSettingsService } from '../ORM/address-settings/address-settings.service'; import { BlocksService } from '../ORM/blocks/blocks.service'; import { ClientService } from '../ORM/client/client.service'; +import { PayoutSnapshotService } from '../ORM/payout-snapshot/payout-snapshot.service'; import { ShareAccountingService } from '../ORM/share-accounting/share-accounting.service'; import { BitcoinRpcService } from './bitcoin-rpc.service'; import { NotificationService } from './notification.service'; @@ -63,7 +64,8 @@ export class StratumV1Service implements OnModuleInit { private readonly stratumV2Service: StratumV2Service, private readonly userAgentReportService: UserAgentReportService, private readonly shareAccountingService?: ShareAccountingService, - private readonly redisMessagingService?: RedisMessagingService + private readonly redisMessagingService?: RedisMessagingService, + private readonly payoutSnapshotService?: PayoutSnapshotService ) { } @@ -186,7 +188,7 @@ export class StratumV1Service implements OnModuleInit { try { protocol = this.detectProtocol(firstChunk); if (protocol === 'v1') { - client = this.createV1Client(socket); + client = this.createV1Client(socket, 'sv1'); socket.emit('data', firstChunk); return; } @@ -218,7 +220,7 @@ export class StratumV1Service implements OnModuleInit { return server; } - private createV1Client(socket: Socket): StratumV1Client { + private createV1Client(socket: Socket, accountingProtocol: 'sv1' | 'sv1_tls'): StratumV1Client { return new StratumV1Client( socket, this.stratumV1JobsService, @@ -229,7 +231,9 @@ export class StratumV1Service implements OnModuleInit { this.configService, this.addressSettingsService, this.shareAccountingService, - this.redisMessagingService + this.redisMessagingService, + this.payoutSnapshotService, + accountingProtocol, ); } @@ -260,7 +264,7 @@ export class StratumV1Service implements OnModuleInit { socket.setTimeout(this.getSocketTimeoutMs()); socket.setKeepAlive(true, this.getTcpKeepAliveInitialDelayMs()); - const client = this.createV1Client(socket); + const client = this.createV1Client(socket, 'sv1_tls'); let cleanedUp = false; const cleanup = async (reason: string) => { diff --git a/src/services/stratum-v2.service.ts b/src/services/stratum-v2.service.ts index 286c550..89fed98 100644 --- a/src/services/stratum-v2.service.ts +++ b/src/services/stratum-v2.service.ts @@ -6,6 +6,7 @@ import { Server, Socket } from 'net'; import { AddressSettingsService } from '../ORM/address-settings/address-settings.service'; import { BlocksService } from '../ORM/blocks/blocks.service'; import { ClientService } from '../ORM/client/client.service'; +import { PayoutSnapshotService } from '../ORM/payout-snapshot/payout-snapshot.service'; import { ShareAccountingService } from '../ORM/share-accounting/share-accounting.service'; import { StratumV2Client } from '../models/StratumV2Client'; import { encodeSv2AuthorityPublicKey } from '../models/sv2/sv2-authority-key'; @@ -18,9 +19,11 @@ import { xOnlyPubKeyFromPriv, } from '../models/sv2/sv2-noise'; import { BitcoinRpcService } from './bitcoin-rpc.service'; +import { CustomWorkService } from './custom-work.service'; import { NotificationService } from './notification.service'; import { RedisMessagingService } from './redis-messaging.service'; import { StratumV1JobsService } from './stratum-v1-jobs.service'; +import { Sv2JobDeclarationRegistryService } from './sv2-job-declaration-registry.service'; const DEFAULT_SOCKET_TIMEOUT_MS = 1000 * 60 * 60; const DEFAULT_TCP_KEEPALIVE_INITIAL_DELAY_MS = 1000 * 60; @@ -44,8 +47,11 @@ export class StratumV2Service implements OnModuleInit { private readonly configService: ConfigService, private readonly stratumV1JobsService: StratumV1JobsService, private readonly addressSettingsService: AddressSettingsService, + private readonly customWorkService: CustomWorkService, + private readonly jobDeclarationRegistry: Sv2JobDeclarationRegistryService, private readonly shareAccountingService?: ShareAccountingService, private readonly redisMessagingService?: RedisMessagingService, + private readonly payoutSnapshotService?: PayoutSnapshotService, ) {} public async onModuleInit(): Promise { @@ -91,8 +97,11 @@ export class StratumV2Service implements OnModuleInit { this.blocksService, this.configService, this.addressSettingsService, + this.customWorkService, + this.jobDeclarationRegistry, this.shareAccountingService, this.redisMessagingService, + this.payoutSnapshotService, ); } @@ -132,6 +141,10 @@ export class StratumV2Service implements OnModuleInit { return this.extranonceManager.minerExtranonceSize; } + public getExtendedTotalExtranonceSize(): number { + return this.extranonceManager.totalSize; + } + private async initializeNoiseConfig(): Promise { const configuredAuthorityKey = this.configService.get('SV2_AUTHORITY_PRIVKEY'); this.authorityKeyConfigured = configuredAuthorityKey?.length === 64; diff --git a/src/services/sv2-job-declaration-registry.service.spec.ts b/src/services/sv2-job-declaration-registry.service.spec.ts new file mode 100644 index 0000000..a914ab4 --- /dev/null +++ b/src/services/sv2-job-declaration-registry.service.spec.ts @@ -0,0 +1,58 @@ +import { Sv2JobDeclarationRegistryService } from './sv2-job-declaration-registry.service'; + +describe('Sv2JobDeclarationRegistryService', () => { + it('tracks allocated and declared custom work tokens', () => { + const registry = new Sv2JobDeclarationRegistryService(); + const coinbaseOutputs = Buffer.from('010000000000000000016a', 'hex'); + const allocated = registry.allocateToken('bc1ptest.worker', coinbaseOutputs); + + expect(registry.hasKnownToken(allocated.token)).toBe(true); + expect(allocated.coinbaseOutputs).toEqual(coinbaseOutputs); + expect(allocated.coinbaseOutputs).not.toBe(coinbaseOutputs); + + const declared = registry.declareJob({ + requestId: 7, + miningJobToken: allocated.token, + version: 0x20000000, + coinbaseTxPrefix: Buffer.concat([Buffer.from('51', 'hex'), Buffer.from('6a', 'hex')]), + coinbaseTxSuffix: Buffer.from('52', 'hex'), + wtxidList: [], + excessData: Buffer.alloc(0), + }); + + expect(registry.hasKnownToken(declared.token)).toBe(true); + expect(registry.getDeclaredJob(declared.token)).toBe(declared); + expect(declared.originalToken).toEqual(allocated.token); + expect(declared.userIdentifier).toBe('bc1ptest.worker'); + }); + + it('rejects declared jobs that omit allocated coinbase output scripts', () => { + const registry = new Sv2JobDeclarationRegistryService(); + const coinbaseOutputs = Buffer.from('010000000000000000016a', 'hex'); + const allocated = registry.allocateToken('bc1ptest.worker', coinbaseOutputs); + + expect(() => registry.declareJob({ + requestId: 7, + miningJobToken: allocated.token, + version: 0x20000000, + coinbaseTxPrefix: Buffer.from('51', 'hex'), + coinbaseTxSuffix: Buffer.from('52', 'hex'), + wtxidList: [], + excessData: Buffer.alloc(0), + })).toThrow('coinbase-output-mismatch'); + }); + + it('rejects declarations using unknown mining job tokens', () => { + const registry = new Sv2JobDeclarationRegistryService(); + + expect(() => registry.declareJob({ + requestId: 8, + miningJobToken: Buffer.from('00', 'hex'), + version: 0x20000000, + coinbaseTxPrefix: Buffer.alloc(0), + coinbaseTxSuffix: Buffer.alloc(0), + wtxidList: [], + excessData: Buffer.alloc(0), + })).toThrow('invalid-mining-job-token'); + }); +}); diff --git a/src/services/sv2-job-declaration-registry.service.ts b/src/services/sv2-job-declaration-registry.service.ts new file mode 100644 index 0000000..483b56b --- /dev/null +++ b/src/services/sv2-job-declaration-registry.service.ts @@ -0,0 +1,161 @@ +import { Injectable } from '@nestjs/common'; +import * as crypto from 'crypto'; + +import { Sv2DeclareMiningJob } from '../models/sv2/sv2-jdp-messages'; +import { TemplateProviderTemplate } from './template-provider.service'; + +export interface Sv2AllocatedMiningJobToken { + token: Buffer; + userIdentifier: string; + coinbaseOutputs: Buffer; + createdAt: number; +} + +export interface Sv2DeclaredMiningJob { + originalToken: Buffer; + token: Buffer; + userIdentifier: string; + job: Sv2DeclareMiningJob; + createdAt: number; + templateId?: bigint; + template?: TemplateProviderTemplate; + validationMode: 'token_only' | 'full_template'; + providedTransactions: Buffer[]; +} + +@Injectable() +export class Sv2JobDeclarationRegistryService { + private readonly allocatedTokens = new Map(); + private readonly declaredJobs = new Map(); + private readonly tokenTtlMs = this.readPositiveInt('SV2_JDP_TOKEN_TTL_MS', 10 * 60 * 1000); + + public allocateToken(userIdentifier: string, coinbaseOutputs: Buffer): Sv2AllocatedMiningJobToken { + this.cleanup(); + const token = crypto.randomBytes(16); + const entry = { + token, + userIdentifier, + coinbaseOutputs: Buffer.from(coinbaseOutputs), + createdAt: Date.now(), + }; + this.allocatedTokens.set(token.toString('hex'), entry); + return entry; + } + + public declareJob(job: Sv2DeclareMiningJob, metadata: { + templateId?: bigint; + template?: TemplateProviderTemplate; + validationMode?: 'token_only' | 'full_template'; + providedTransactions?: Buffer[]; + } = {}): Sv2DeclaredMiningJob { + this.cleanup(); + const originalTokenHex = job.miningJobToken.toString('hex'); + const allocated = this.allocatedTokens.get(originalTokenHex); + if (allocated == null) { + throw new Error('invalid-mining-job-token'); + } + if (!this.coinbaseIncludesRequiredOutputs(job, allocated.coinbaseOutputs)) { + throw new Error('coinbase-output-mismatch'); + } + const token = crypto.randomBytes(16); + const declared = { + originalToken: Buffer.from(job.miningJobToken), + token, + userIdentifier: allocated.userIdentifier, + job, + createdAt: Date.now(), + templateId: metadata.templateId, + template: metadata.template, + validationMode: metadata.validationMode ?? 'token_only', + providedTransactions: (metadata.providedTransactions ?? []).map(tx => Buffer.from(tx)), + }; + this.declaredJobs.set(token.toString('hex'), declared); + return declared; + } + + public hasKnownToken(token: Buffer): boolean { + this.cleanup(); + const tokenHex = token.toString('hex'); + return this.allocatedTokens.has(tokenHex) || this.declaredJobs.has(tokenHex); + } + + public getDeclaredJob(token: Buffer): Sv2DeclaredMiningJob | undefined { + this.cleanup(); + return this.declaredJobs.get(token.toString('hex')); + } + + private cleanup(): void { + const cutoff = Date.now() - this.tokenTtlMs; + for (const [token, entry] of this.allocatedTokens.entries()) { + if (entry.createdAt < cutoff) { + this.allocatedTokens.delete(token); + } + } + for (const [token, entry] of this.declaredJobs.entries()) { + if (entry.createdAt < cutoff) { + this.declaredJobs.delete(token); + } + } + } + + private coinbaseIncludesRequiredOutputs(job: Sv2DeclareMiningJob, coinbaseOutputs: Buffer): boolean { + if (coinbaseOutputs.length === 0) { + return true; + } + const declaredCoinbase = Buffer.concat([job.coinbaseTxPrefix, job.coinbaseTxSuffix]); + for (const script of this.extractOutputScripts(coinbaseOutputs)) { + if (script.length === 0 || declaredCoinbase.indexOf(script) < 0) { + return false; + } + } + return true; + } + + private extractOutputScripts(outputs: Buffer): Buffer[] { + const { value: outputCount, offset } = this.readVarInt(outputs, 0); + const scripts: Buffer[] = []; + let cursor = offset; + for (let i = 0; i < outputCount; i++) { + if (cursor + 8 > outputs.length) { + throw new Error('invalid-coinbase-outputs'); + } + cursor += 8; + const scriptLength = this.readVarInt(outputs, cursor); + cursor = scriptLength.offset; + if (cursor + scriptLength.value > outputs.length) { + throw new Error('invalid-coinbase-outputs'); + } + scripts.push(Buffer.from(outputs.subarray(cursor, cursor + scriptLength.value))); + cursor += scriptLength.value; + } + return scripts; + } + + private readVarInt(buffer: Buffer, offset: number): { value: number; offset: number } { + if (offset >= buffer.length) { + throw new Error('invalid-coinbase-outputs'); + } + const first = buffer[offset]; + if (first < 0xfd) { + return { value: first, offset: offset + 1 }; + } + if (first === 0xfd) { + if (offset + 3 > buffer.length) { + throw new Error('invalid-coinbase-outputs'); + } + return { value: buffer.readUInt16LE(offset + 1), offset: offset + 3 }; + } + if (first === 0xfe) { + if (offset + 5 > buffer.length) { + throw new Error('invalid-coinbase-outputs'); + } + return { value: buffer.readUInt32LE(offset + 1), offset: offset + 5 }; + } + throw new Error('unsupported-large-coinbase-output-count'); + } + + private readPositiveInt(name: string, fallback: number): number { + const parsed = parseInt(process.env[name] ?? '', 10); + return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback; + } +} diff --git a/src/services/sv2-job-declaration.service.spec.ts b/src/services/sv2-job-declaration.service.spec.ts new file mode 100644 index 0000000..b2c393e --- /dev/null +++ b/src/services/sv2-job-declaration.service.spec.ts @@ -0,0 +1,503 @@ +import * as bitcoinjs from 'bitcoinjs-lib'; + +import { BufferReader } from '../models/sv2/sv2-binary-codec'; +import { Sv2JdpSetupFlags, Sv2MsgType, Sv2Protocol } from '../models/sv2/sv2-constants'; +import { + deserializeDeclareMiningJobError, + deserializeDeclareMiningJobSuccess, + deserializeProvideMissingTransactions, + serializeDeclareMiningJob, + serializeProvideMissingTransactionsSuccess, + serializePushSolution, +} from '../models/sv2/sv2-jdp-messages'; +import { + deserializeSetupConnectionSuccess, + serializeSetupConnection, +} from '../models/sv2/sv2-messages'; +import { Sv2JobDeclarationConnection } from './sv2-job-declaration.service'; +import { Sv2JobDeclarationRegistryService } from './sv2-job-declaration-registry.service'; + +describe('Sv2JobDeclarationConnection compliance', () => { + it('does not echo Job Declaration setup flags in SetupConnection.Success', async () => { + const { connection, sentFrames } = createConnection(); + + await (connection as any).handleSetupConnection(serializeSetupConnection({ + protocol: Sv2Protocol.JOB_DECLARATION, + minVersion: 2, + maxVersion: 2, + flags: Sv2JdpSetupFlags.DECLARE_TX_DATA, + endpoint_host: 'localhost', + endpoint_port: 34264, + vendor: 'jd-client', + hardwareVersion: '', + firmwareVersion: '', + deviceId: '', + })); + + const successFrame = sentFrames.find(frame => frame.msgType === Sv2MsgType.SETUP_CONNECTION_SUCCESS); + const success = deserializeSetupConnectionSuccess(new BufferReader(successFrame.payload)); + expect(success.flags).toBe(0); + expect(success.usedVersion).toBe(2); + }); + + it('rejects DeclareMiningJob unless DECLARE_TX_DATA was negotiated', async () => { + const { connection, sentFrames, registry } = createConnection(); + + await (connection as any).handleDeclareMiningJob(serializeDeclareMiningJob({ + requestId: 9, + miningJobToken: Buffer.from('aa', 'hex'), + version: 0x20000000, + coinbaseTxPrefix: Buffer.alloc(0), + coinbaseTxSuffix: Buffer.alloc(0), + wtxidList: [], + excessData: Buffer.alloc(0), + })); + + const errorFrame = sentFrames.find(frame => frame.msgType === Sv2MsgType.JDP_DECLARE_MINING_JOB_ERROR); + const error = deserializeDeclareMiningJobError(new BufferReader(errorFrame.payload)); + expect(error.requestId).toBe(9); + expect(error.errorCode).toBe('declare-tx-data-not-negotiated'); + expect(registry.declareJob).not.toHaveBeenCalled(); + }); + + it('accepts DeclareMiningJob when all wtxids are known to the template provider', async () => { + const registry = new Sv2JobDeclarationRegistryService(); + const allocated = registry.allocateToken('bc1ptest.worker', Buffer.alloc(0)); + const coinbaseTxPrefix = Buffer.from('010203', 'hex'); + const validateDeclaredWtxids = jest.fn().mockReturnValue({ + valid: true, + template: { templateId: 42n }, + }); + const { connection, sentFrames } = createConnection({ + registry, + templateProvider: { + validateDeclaredWtxids, + }, + }); + (connection as any).declareTxData = true; + + await (connection as any).handleDeclareMiningJob(serializeDeclareMiningJob({ + requestId: 10, + miningJobToken: allocated.token, + version: 0x20000000, + coinbaseTxPrefix, + coinbaseTxSuffix: Buffer.alloc(0), + wtxidList: [Buffer.alloc(32, 1)], + excessData: Buffer.alloc(0), + })); + + const successFrame = sentFrames.find(frame => frame.msgType === Sv2MsgType.JDP_DECLARE_MINING_JOB_SUCCESS); + const success = deserializeDeclareMiningJobSuccess(new BufferReader(successFrame.payload)); + expect(success.requestId).toBe(10); + expect(registry.getDeclaredJob(success.newMiningJobToken)?.validationMode).toBe('full_template'); + expect(registry.getDeclaredJob(success.newMiningJobToken)?.template).toEqual({ templateId: 42n }); + expect(validateDeclaredWtxids).toHaveBeenCalledWith({ + version: 0x20000000, + coinbaseTxPrefix, + wtxidList: [Buffer.alloc(32, 1)], + }); + }); + + it('rejects DeclareMiningJob when the declared coinbase prefix fails BIP34 validation', async () => { + const registry = new Sv2JobDeclarationRegistryService(); + const allocated = registry.allocateToken('bc1ptest.worker', Buffer.alloc(0)); + const { connection, sentFrames } = createConnection({ + registry, + templateProvider: { + validateDeclaredWtxids: jest.fn().mockReturnValue({ + valid: false, + errorCode: 'invalid-job-param-value-coinbase_tx_prefix', + template: { templateId: 42n }, + }), + }, + }); + (connection as any).declareTxData = true; + + await (connection as any).handleDeclareMiningJob(serializeDeclareMiningJob({ + requestId: 12, + miningJobToken: allocated.token, + version: 0x20000000, + coinbaseTxPrefix: Buffer.from('/public-pool-sri-jdc-e2e/', 'utf8'), + coinbaseTxSuffix: Buffer.alloc(0), + wtxidList: [], + excessData: Buffer.alloc(0), + })); + + const errorFrame = sentFrames.find(frame => frame.msgType === Sv2MsgType.JDP_DECLARE_MINING_JOB_ERROR); + const error = deserializeDeclareMiningJobError(new BufferReader(errorFrame.payload)); + expect(error.requestId).toBe(12); + expect(error.errorCode).toBe('invalid-job-param-value-coinbase_tx_prefix'); + expect(registry.getDeclaredJob(allocated.token)).toBeUndefined(); + }); + + it('requests missing transactions and finalizes after valid missing transaction response', async () => { + const registry = new Sv2JobDeclarationRegistryService(); + const allocated = registry.allocateToken('bc1ptest.worker', Buffer.alloc(0)); + const templateProvider = { + validateDeclaredWtxids: jest.fn().mockReturnValue({ + valid: false, + errorCode: 'missing-transactions', + template: { templateId: 42n }, + unknownTxPositionList: [1], + }), + validateProvidedTransactions: jest.fn().mockReturnValue({ valid: true }), + }; + const { connection, sentFrames } = createConnection({ registry, templateProvider }); + (connection as any).declareTxData = true; + + await (connection as any).handleDeclareMiningJob(serializeDeclareMiningJob({ + requestId: 11, + miningJobToken: allocated.token, + version: 0x20000000, + coinbaseTxPrefix: Buffer.alloc(0), + coinbaseTxSuffix: Buffer.alloc(0), + wtxidList: [Buffer.alloc(32, 1), Buffer.alloc(32, 2)], + excessData: Buffer.alloc(0), + })); + + const missingFrame = sentFrames.find(frame => frame.msgType === Sv2MsgType.JDP_PROVIDE_MISSING_TRANSACTIONS); + const missing = deserializeProvideMissingTransactions(new BufferReader(missingFrame.payload)); + expect(missing.requestId).toBe(11); + expect(missing.unknownTxPositionList).toEqual([1]); + + await (connection as any).handleProvideMissingTransactionsSuccess(serializeProvideMissingTransactionsSuccess({ + requestId: 11, + transactionList: [Buffer.from('0100000000', 'hex')], + })); + + const successFrame = sentFrames.find(frame => frame.msgType === Sv2MsgType.JDP_DECLARE_MINING_JOB_SUCCESS); + const success = deserializeDeclareMiningJobSuccess(new BufferReader(successFrame.payload)); + expect(registry.getDeclaredJob(success.newMiningJobToken)?.providedTransactions).toEqual([Buffer.from('0100000000', 'hex')]); + expect(registry.getDeclaredJob(success.newMiningJobToken)?.template).toEqual({ templateId: 42n }); + }); + + it('submits PushSolution before persisting the found block', async () => { + const block = createMockBlock('deadbeef'); + const templateProvider = { + getTemplate: jest.fn().mockReturnValue({ + height: 4990255, + jobTemplate: { + blockData: { + payoutSnapshotId: '17', + }, + }, + }), + buildBlockFromDeclaredJobSolution: jest.fn().mockReturnValue(block), + }; + const callOrder: string[] = []; + const bitcoinRpcService = { + SUBMIT_BLOCK: jest.fn().mockImplementation(async () => { + callOrder.push('submit'); + return null; + }), + }; + const blocksService = { + save: jest.fn().mockImplementation(async () => { + callOrder.push('save'); + }), + }; + const payoutSnapshotService = { + finalizeSnapshotForBlock: jest.fn().mockResolvedValue({ finalized: false, reason: 'disabled' }), + }; + const notificationService = { + notifySubscribersBlockFound: jest.fn().mockResolvedValue(undefined), + }; + const { connection } = createConnection({ + templateProvider, + bitcoinRpcService, + blocksService, + payoutSnapshotService, + notificationService, + }); + (connection as any).latestDeclaredJob = { + token: Buffer.from('0123456789abcdef0123456789abcdef', 'hex'), + userIdentifier: 'tb1qumezefzdeqqwn5zfvgdrhxjzc5ylr39uhuxcz4.worker1', + job: { requestId: 1 } as any, + templateId: 42n, + validationMode: 'full_template', + providedTransactions: [], + }; + + await (connection as any).handlePushSolution(serializePushSolution({ + extranonce: Buffer.from('aabbccdd', 'hex'), + prevHash: Buffer.alloc(32, 1), + nonce: 1, + ntime: 2, + nBits: 0x1d00ffff, + version: 0x20000000, + })); + + expect(bitcoinRpcService.SUBMIT_BLOCK).toHaveBeenCalledWith('deadbeef'); + expect(blocksService.save).toHaveBeenCalledWith(expect.objectContaining({ + height: 4990255, + minerAddress: 'tb1qumezefzdeqqwn5zfvgdrhxjzc5ylr39uhuxcz4', + worker: 'worker1', + sessionId: '01234567', + blockData: 'deadbeef', + payoutSnapshotId: '17', + })); + expect(callOrder).toEqual(['submit', 'save']); + expect(payoutSnapshotService.finalizeSnapshotForBlock).toHaveBeenCalledWith({ + payoutSnapshotId: '17', + blockHeight: 4990255, + blockSubmissionResult: null, + }); + expect(notificationService.notifySubscribersBlockFound).toHaveBeenCalledWith( + 'tb1qumezefzdeqqwn5zfvgdrhxjzc5ylr39uhuxcz4', + 4990255, + block, + null, + ); + }); + + it('submits PushSolution using the declared template snapshot after template cache cleanup', async () => { + const block = createMockBlock('feedface'); + const declaredTemplate = { + height: 4990300, + jobTemplate: { + blockData: { + payoutSnapshotId: '23', + }, + }, + }; + const templateProvider = { + getTemplate: jest.fn().mockReturnValue(undefined), + getLatestTemplate: jest.fn().mockReturnValue(undefined), + buildBlockFromDeclaredJobSolution: jest.fn().mockReturnValue(block), + }; + const bitcoinRpcService = { + SUBMIT_BLOCK: jest.fn().mockResolvedValue('SUCCESS!'), + }; + const blocksService = { + save: jest.fn().mockResolvedValue(undefined), + }; + const payoutSnapshotService = { + finalizeSnapshotForBlock: jest.fn().mockResolvedValue({ finalized: false, reason: 'disabled' }), + }; + const notificationService = { + notifySubscribersBlockFound: jest.fn().mockResolvedValue(undefined), + }; + const { connection } = createConnection({ + templateProvider, + bitcoinRpcService, + blocksService, + payoutSnapshotService, + notificationService, + }); + (connection as any).latestDeclaredJob = { + token: Buffer.from('0123456789abcdef0123456789abcdef', 'hex'), + userIdentifier: 'tb1qumezefzdeqqwn5zfvgdrhxjzc5ylr39uhuxcz4.worker1', + job: { requestId: 1 } as any, + templateId: 42n, + template: declaredTemplate, + validationMode: 'full_template', + providedTransactions: [], + }; + + await (connection as any).handlePushSolution(serializePushSolution({ + extranonce: Buffer.from('aabbccdd', 'hex'), + prevHash: Buffer.alloc(32, 1), + nonce: 1, + ntime: 2, + nBits: 0x1d00ffff, + version: 0x20000000, + })); + + expect(templateProvider.getTemplate).not.toHaveBeenCalled(); + expect(templateProvider.buildBlockFromDeclaredJobSolution).toHaveBeenCalledWith(expect.objectContaining({ + template: declaredTemplate, + })); + expect(bitcoinRpcService.SUBMIT_BLOCK).toHaveBeenCalledWith('feedface'); + expect(blocksService.save).toHaveBeenCalledWith(expect.objectContaining({ + height: 4990300, + blockSubmissionResult: 'SUCCESS!', + payoutSnapshotId: '23', + })); + }); + + it('does not resubmit duplicate PushSolution messages', async () => { + const block = createMockBlock('feedface'); + const templateProvider = { + buildBlockFromDeclaredJobSolution: jest.fn().mockReturnValue(block), + }; + const bitcoinRpcService = { + SUBMIT_BLOCK: jest.fn().mockResolvedValue('SUCCESS!'), + }; + const { connection } = createConnection({ + templateProvider, + bitcoinRpcService, + blocksService: { save: jest.fn().mockResolvedValue(undefined) }, + }); + (connection as any).latestDeclaredJob = { + token: Buffer.from('0123456789abcdef0123456789abcdef', 'hex'), + userIdentifier: 'tb1qumezefzdeqqwn5zfvgdrhxjzc5ylr39uhuxcz4.worker1', + job: { requestId: 1 } as any, + template: { + height: 4990300, + jobTemplate: { blockData: { payoutSnapshotId: null } }, + }, + validationMode: 'full_template', + providedTransactions: [], + }; + const payload = serializePushSolution({ + extranonce: Buffer.from('aabbccdd', 'hex'), + prevHash: Buffer.alloc(32, 1), + nonce: 1, + ntime: 2, + nBits: 0x1d00ffff, + version: 0x20000000, + }); + + await (connection as any).handlePushSolution(payload); + await (connection as any).handlePushSolution(payload); + + expect(bitcoinRpcService.SUBMIT_BLOCK).toHaveBeenCalledTimes(1); + }); + + it('does not submit PushSolution blocks with a coinbase missing the BIP34 height', async () => { + const warn = jest.spyOn(console, 'warn').mockImplementation(() => undefined); + const invalidBlock = createBlockWithCoinbaseScript(Buffer.from('/public-pool-sri-jdc-e2e/', 'utf8')); + const templateProvider = { + buildBlockFromDeclaredJobSolution: jest.fn().mockReturnValue(invalidBlock), + validateCoinbaseTransactionHeight: jest.fn().mockReturnValue({ + valid: false, + errorCode: 'invalid-job-param-value-coinbase_tx_prefix', + }), + }; + const bitcoinRpcService = { + SUBMIT_BLOCK: jest.fn().mockResolvedValue('SUCCESS!'), + }; + const blocksService = { + save: jest.fn().mockResolvedValue(undefined), + }; + const { connection } = createConnection({ + templateProvider, + bitcoinRpcService, + blocksService, + }); + (connection as any).latestDeclaredJob = { + token: Buffer.from('0123456789abcdef0123456789abcdef', 'hex'), + userIdentifier: 'tb1qumezefzdeqqwn5zfvgdrhxjzc5ylr39uhuxcz4.worker1', + job: { requestId: 1 } as any, + template: { + height: 4990300, + jobTemplate: { blockData: { payoutSnapshotId: null } }, + }, + validationMode: 'full_template', + providedTransactions: [], + }; + + await (connection as any).handlePushSolution(serializePushSolution({ + extranonce: Buffer.from('aabbccdd', 'hex'), + prevHash: Buffer.alloc(32, 1), + nonce: 1, + ntime: 2, + nBits: 0x1d00ffff, + version: 0x20000000, + })); + + expect(bitcoinRpcService.SUBMIT_BLOCK).not.toHaveBeenCalled(); + expect(blocksService.save).not.toHaveBeenCalled(); + expect(warn).toHaveBeenCalledWith(expect.stringContaining('invalid-job-param-value-coinbase_tx_prefix')); + warn.mockRestore(); + }); + + function createConnection(overrides: { + registry?: any; + templateProvider?: any; + bitcoinRpcService?: any; + blocksService?: any; + payoutSnapshotService?: any; + notificationService?: any; + } = {}): { + connection: Sv2JobDeclarationConnection; + sentFrames: any[]; + registry: { allocateToken: jest.Mock; declareJob: jest.Mock }; + } { + const socket = { + setKeepAlive: jest.fn(), + setNoDelay: jest.fn(), + on: jest.fn(), + write: jest.fn((data, callback) => callback?.()), + destroy: jest.fn(), + destroyed: false, + writableEnded: false, + }; + const registry = overrides.registry ?? { + allocateToken: jest.fn(), + declareJob: jest.fn(), + }; + const templateProvider = { + validateDeclaredWtxids: jest.fn().mockReturnValue({ valid: false, errorCode: 'template-not-found' }), + validateProvidedTransactions: jest.fn().mockReturnValue({ valid: false, errorCode: 'template-not-found' }), + validateCoinbaseTransactionHeight: jest.fn().mockReturnValue({ valid: true }), + ...overrides.templateProvider, + }; + const connection = new Sv2JobDeclarationConnection( + socket as any, + { + getNoiseConfig: () => ({ + staticKeypair: { + privateKey: Buffer.alloc(32), + publicKey: Buffer.alloc(64), + }, + certificateMessage: { + version: 0, + validFrom: 0, + notValidAfter: 0, + signature: Buffer.alloc(64), + }, + }), + } as any, + registry as any, + {} as any, + templateProvider, + overrides.bitcoinRpcService ?? { + SUBMIT_BLOCK: jest.fn().mockResolvedValue(null), + }, + overrides.blocksService ?? { + save: jest.fn().mockResolvedValue(undefined), + }, + overrides.payoutSnapshotService ?? { + finalizeSnapshotForBlock: jest.fn().mockResolvedValue({ finalized: false, reason: 'disabled' }), + }, + overrides.notificationService ?? { + notifySubscribersBlockFound: jest.fn().mockResolvedValue(undefined), + }, + 'tb1qumezefzdeqqwn5zfvgdrhxjzc5ylr39uhuxcz4', + bitcoinjs.networks.testnet, + ); + const sentFrames: any[] = []; + (connection as any).sendFrame = jest.fn((msgType: number, payload: Buffer) => { + sentFrames.push({ msgType, payload }); + return Promise.resolve(); + }); + return { connection, sentFrames, registry: registry as any }; + } + + function createBlockWithCoinbaseScript(script: Buffer): bitcoinjs.Block { + const tx = new bitcoinjs.Transaction(); + tx.version = 2; + tx.addInput(Buffer.alloc(32), 0xffffffff, 0xffffffff, script); + tx.addOutput(Buffer.from('6a', 'hex'), 0); + + const block = new bitcoinjs.Block(); + block.version = 0x20000000; + block.prevHash = Buffer.alloc(32); + block.timestamp = 2; + block.bits = 0x1d00ffff; + block.nonce = 1; + block.transactions = [tx]; + block.merkleRoot = bitcoinjs.Block.calculateMerkleRoot(block.transactions, false); + return block; + } + + function createMockBlock(hex: string): { toHex: jest.Mock; transactions: Array<{ toBuffer: jest.Mock }> } { + return { + toHex: jest.fn().mockReturnValue(hex), + transactions: [{ + toBuffer: jest.fn().mockReturnValue(Buffer.from('0100000000', 'hex')), + }], + }; + } +}); diff --git a/src/services/sv2-job-declaration.service.ts b/src/services/sv2-job-declaration.service.ts new file mode 100644 index 0000000..42a10cc --- /dev/null +++ b/src/services/sv2-job-declaration.service.ts @@ -0,0 +1,473 @@ +import { Injectable, OnModuleInit } from '@nestjs/common'; +import { ConfigService } from '@nestjs/config'; +import * as bitcoinjs from 'bitcoinjs-lib'; +import { Server, Socket } from 'net'; + +import { BlocksService } from '../ORM/blocks/blocks.service'; +import { PayoutSnapshotService } from '../ORM/payout-snapshot/payout-snapshot.service'; +import { BufferReader } from '../models/sv2/sv2-binary-codec'; +import { SV2_NOISE_ACT1_SIZE, Sv2JdpSetupFlags, Sv2MsgType, Sv2Protocol } from '../models/sv2/sv2-constants'; +import { Sv2FrameReader, Sv2FrameWriter } from '../models/sv2/sv2-frame'; +import { + deserializeAllocateMiningJobToken, + deserializeDeclareMiningJob, + deserializeProvideMissingTransactionsSuccess, + deserializePushSolution, + serializeAllocateMiningJobTokenSuccess, + serializeDeclareMiningJobError, + serializeDeclareMiningJobSuccess, + serializeProvideMissingTransactions, + Sv2DeclareMiningJob, +} from '../models/sv2/sv2-jdp-messages'; +import { + deserializeSetupConnection, + serializeSetupConnectionError, + serializeSetupConnectionSuccess, +} from '../models/sv2/sv2-messages'; +import { Sv2NoiseSession } from '../models/sv2/sv2-noise'; +import { BitcoinRpcService } from './bitcoin-rpc.service'; +import { CustomWorkService } from './custom-work.service'; +import { NotificationService } from './notification.service'; +import { StratumV2Service } from './stratum-v2.service'; +import { Sv2DeclaredMiningJob, Sv2JobDeclarationRegistryService } from './sv2-job-declaration-registry.service'; +import { TemplateProviderService } from './template-provider.service'; + +@Injectable() +export class Sv2JobDeclarationService implements OnModuleInit { + private readonly servers: Server[] = []; + + constructor( + private readonly configService: ConfigService, + private readonly stratumV2Service: StratumV2Service, + private readonly registry: Sv2JobDeclarationRegistryService, + private readonly customWorkService: CustomWorkService, + private readonly templateProvider: TemplateProviderService, + private readonly bitcoinRpcService: BitcoinRpcService, + private readonly blocksService: BlocksService, + private readonly payoutSnapshotService: PayoutSnapshotService, + private readonly notificationService: NotificationService, + ) {} + + public async onModuleInit(): Promise { + if (process.env.API_ONLY === 'true' || process.env.MASTER === 'true') { + return; + } + + const ports = this.getPorts(); + if (ports.length === 0) { + return; + } + + await this.stratumV2Service.ensureInitialized(); + for (const port of ports) { + this.startServer(port); + } + } + + private startServer(port: number): void { + const server = new Server(socket => { + void new Sv2JobDeclarationConnection( + socket, + this.stratumV2Service, + this.registry, + this.customWorkService, + this.templateProvider, + this.bitcoinRpcService, + this.blocksService, + this.payoutSnapshotService, + this.notificationService, + this.getPoolPayoutAddress(), + this.getNetwork(), + ).start(); + }); + server.on('error', error => console.error(`SV2 JDP server error on port ${port}: ${error.message}`)); + server.listen(port, () => console.log(`SV2 Job Declaration server is listening on port ${port}`)); + this.servers.push(server); + } + + private getPoolPayoutAddress(): string { + return this.configService.get('SV2_JDP_POOL_PAYOUT_ADDRESS') + || this.configService.get('DATUM_POOL_PAYOUT_ADDRESS') + || this.configService.get('PAYOUT_FEE_ADDRESS') + || this.configService.get('DEV_FEE_ADDRESS') + || ''; + } + + private getPorts(): number[] { + const configured = this.configService.get('SV2_JDP_PORTS'); + if (!configured?.trim()) { + return []; + } + return Array.from(new Set(configured + .split(',') + .map(port => parseInt(port.trim(), 10)) + .filter(port => Number.isInteger(port) && port > 0 && port <= 65535))); + } + + private getNetwork(): bitcoinjs.networks.Network { + const networkConfig = this.configService.get('NETWORK'); + if (networkConfig === 'mainnet') { + return bitcoinjs.networks.bitcoin; + } + if (networkConfig === 'testnet') { + return bitcoinjs.networks.testnet; + } + if (networkConfig === 'regtest') { + return bitcoinjs.networks.regtest; + } + throw new Error('Invalid network configuration'); + } +} + +export class Sv2JobDeclarationConnection { + private readonly noiseSession: Sv2NoiseSession; + private readonly frameReader = new Sv2FrameReader(null); + private readonly frameWriter = new Sv2FrameWriter(null); + private handshakeBuffer = Buffer.alloc(0); + private handshakeComplete = false; + private destroyed = false; + private declareTxData = false; + private latestDeclaredJob: Sv2DeclaredMiningJob | null = null; + private readonly submittedSolutions = new Set(); + private readonly pendingDeclarations = new Map; + unknownTxPositionList: number[]; + }>(); + + constructor( + private readonly socket: Socket, + stratumV2Service: StratumV2Service, + private readonly registry: Sv2JobDeclarationRegistryService, + private readonly customWorkService: CustomWorkService, + private readonly templateProvider: TemplateProviderService, + private readonly bitcoinRpcService: BitcoinRpcService, + private readonly blocksService: BlocksService, + private readonly payoutSnapshotService: PayoutSnapshotService, + private readonly notificationService: NotificationService, + private readonly poolPayoutAddress: string, + private readonly network: bitcoinjs.networks.Network, + ) { + this.noiseSession = new Sv2NoiseSession(stratumV2Service.getNoiseConfig()); + } + + public async start(): Promise { + this.socket.setKeepAlive(true, 60_000); + this.socket.setNoDelay(true); + this.socket.on('data', data => void this.handleSocketData(data)); + this.socket.on('error', () => this.close()); + this.socket.on('close', () => this.destroyed = true); + } + + private async handleSocketData(data: Buffer): Promise { + if (this.destroyed) { + return; + } + try { + if (!this.handshakeComplete) { + await this.handleHandshakeData(data); + return; + } + await this.handleEncryptedData(data); + } catch (error) { + console.error(`[SV2 JDP] ${error.message}`); + this.close(); + } + } + + private async handleHandshakeData(data: Buffer): Promise { + this.handshakeBuffer = Buffer.concat([this.handshakeBuffer, data]); + if (this.handshakeBuffer.length < SV2_NOISE_ACT1_SIZE) { + return; + } + const act1 = this.handshakeBuffer.subarray(0, SV2_NOISE_ACT1_SIZE); + const remainder = Buffer.from(this.handshakeBuffer.subarray(SV2_NOISE_ACT1_SIZE)); + this.handshakeBuffer = Buffer.alloc(0); + + await this.writeRaw(await this.noiseSession.processAct1(Buffer.from(act1))); + this.frameReader.setDecryptFn(ciphertext => this.noiseSession.decrypt(ciphertext)); + this.frameWriter.setEncryptFn(plaintext => this.noiseSession.encrypt(plaintext)); + this.handshakeComplete = true; + if (remainder.length > 0) { + await this.handleEncryptedData(remainder); + } + } + + private async handleEncryptedData(data: Buffer): Promise { + for (const frame of this.frameReader.feed(data)) { + switch (frame.header.msgType) { + case Sv2MsgType.SETUP_CONNECTION: + await this.handleSetupConnection(frame.payload); + break; + case Sv2MsgType.JDP_ALLOCATE_MINING_JOB_TOKEN: + await this.handleAllocateMiningJobToken(frame.payload); + break; + case Sv2MsgType.JDP_DECLARE_MINING_JOB: + await this.handleDeclareMiningJob(frame.payload); + break; + case Sv2MsgType.JDP_PROVIDE_MISSING_TRANSACTIONS_SUCCESS: + await this.handleProvideMissingTransactionsSuccess(frame.payload); + break; + case Sv2MsgType.JDP_PUSH_SOLUTION: + await this.handlePushSolution(frame.payload); + break; + default: + console.warn(`[SV2 JDP] Ignoring unsupported message type 0x${frame.header.msgType.toString(16)}`); + break; + } + } + } + + private async handleSetupConnection(payload: Buffer): Promise { + const setup = deserializeSetupConnection(new BufferReader(payload)); + if (setup.protocol !== Sv2Protocol.JOB_DECLARATION) { + await this.sendFrame(Sv2MsgType.SETUP_CONNECTION_ERROR, serializeSetupConnectionError({ + flags: 0, + errorCode: 'unsupported-protocol', + })); + this.close(); + return; + } + + this.declareTxData = (setup.flags & Sv2JdpSetupFlags.DECLARE_TX_DATA) !== 0; + await this.sendFrame(Sv2MsgType.SETUP_CONNECTION_SUCCESS, serializeSetupConnectionSuccess({ + usedVersion: 2, + flags: 0, + })); + } + + private async handleAllocateMiningJobToken(payload: Buffer): Promise { + const request = deserializeAllocateMiningJobToken(new BufferReader(payload)); + const token = this.registry.allocateToken( + request.userIdentifier, + this.buildPoolCoinbaseOutputs(request.userIdentifier), + ); + await this.sendFrame(Sv2MsgType.JDP_ALLOCATE_MINING_JOB_TOKEN_SUCCESS, serializeAllocateMiningJobTokenSuccess({ + requestId: request.requestId, + miningJobToken: token.token, + coinbaseOutputs: token.coinbaseOutputs, + })); + } + + private async handleDeclareMiningJob(payload: Buffer): Promise { + const job = deserializeDeclareMiningJob(new BufferReader(payload)); + if (!this.declareTxData) { + await this.sendFrame(Sv2MsgType.JDP_DECLARE_MINING_JOB_ERROR, serializeDeclareMiningJobError({ + requestId: job.requestId, + errorCode: 'declare-tx-data-not-negotiated', + errorDetails: Buffer.alloc(0), + })); + return; + } + + try { + const validation = this.templateProvider.validateDeclaredWtxids({ + version: job.version, + coinbaseTxPrefix: job.coinbaseTxPrefix, + wtxidList: job.wtxidList, + }); + if (validation.errorCode === 'missing-transactions' && validation.template != null) { + this.pendingDeclarations.set(job.requestId, { + job, + templateId: validation.template.templateId, + template: validation.template, + unknownTxPositionList: validation.unknownTxPositionList, + }); + await this.sendFrame(Sv2MsgType.JDP_PROVIDE_MISSING_TRANSACTIONS, serializeProvideMissingTransactions({ + requestId: job.requestId, + unknownTxPositionList: validation.unknownTxPositionList, + })); + return; + } + if (!validation.valid) { + await this.sendFrame(Sv2MsgType.JDP_DECLARE_MINING_JOB_ERROR, serializeDeclareMiningJobError({ + requestId: job.requestId, + errorCode: validation.errorCode ?? 'invalid-template', + errorDetails: Buffer.alloc(0), + })); + return; + } + + const declared = this.registry.declareJob(job, { + templateId: validation.template.templateId, + template: validation.template, + validationMode: 'full_template', + }); + this.latestDeclaredJob = declared; + await this.sendFrame(Sv2MsgType.JDP_DECLARE_MINING_JOB_SUCCESS, serializeDeclareMiningJobSuccess({ + requestId: job.requestId, + newMiningJobToken: declared.token, + })); + } catch (error) { + await this.sendFrame(Sv2MsgType.JDP_DECLARE_MINING_JOB_ERROR, serializeDeclareMiningJobError({ + requestId: job.requestId, + errorCode: error.message, + errorDetails: Buffer.alloc(0), + })); + } + } + + private async handleProvideMissingTransactionsSuccess(payload: Buffer): Promise { + const response = deserializeProvideMissingTransactionsSuccess(new BufferReader(payload)); + const pending = this.pendingDeclarations.get(response.requestId); + if (pending == null) { + await this.sendFrame(Sv2MsgType.JDP_DECLARE_MINING_JOB_ERROR, serializeDeclareMiningJobError({ + requestId: response.requestId, + errorCode: 'unknown-missing-transaction-request', + errorDetails: Buffer.alloc(0), + })); + return; + } + + const validation = await this.templateProvider.validateProvidedTransactions({ + expectedWtxids: pending.job.wtxidList, + unknownTxPositionList: pending.unknownTxPositionList, + transactionList: response.transactionList, + }); + if (!validation.valid) { + this.pendingDeclarations.delete(response.requestId); + await this.sendFrame(Sv2MsgType.JDP_DECLARE_MINING_JOB_ERROR, serializeDeclareMiningJobError({ + requestId: response.requestId, + errorCode: validation.errorCode ?? 'invalid-missing-transactions', + errorDetails: Buffer.alloc(0), + })); + return; + } + + try { + const declared = this.registry.declareJob(pending.job, { + templateId: pending.templateId, + template: pending.template, + validationMode: 'full_template', + providedTransactions: response.transactionList, + }); + this.latestDeclaredJob = declared; + this.pendingDeclarations.delete(response.requestId); + await this.sendFrame(Sv2MsgType.JDP_DECLARE_MINING_JOB_SUCCESS, serializeDeclareMiningJobSuccess({ + requestId: pending.job.requestId, + newMiningJobToken: declared.token, + })); + } catch (error) { + this.pendingDeclarations.delete(response.requestId); + await this.sendFrame(Sv2MsgType.JDP_DECLARE_MINING_JOB_ERROR, serializeDeclareMiningJobError({ + requestId: response.requestId, + errorCode: error.message, + errorDetails: Buffer.alloc(0), + })); + } + } + + private async handlePushSolution(payload: Buffer): Promise { + const solution = deserializePushSolution(new BufferReader(payload)); + const declared = this.latestDeclaredJob; + if (declared == null) { + console.warn('[SV2 JDP] PushSolution received before a mining job was declared'); + return; + } + + const solutionKey = [ + solution.prevHash.toString('hex'), + solution.extranonce.toString('hex'), + solution.nonce.toString(16), + solution.ntime.toString(16), + solution.nBits.toString(16), + solution.version.toString(16), + ].join(':'); + if (this.submittedSolutions.has(solutionKey)) { + return; + } + + const template = declared.template + ?? (declared.templateId == null + ? this.templateProvider.getLatestTemplate() + : this.templateProvider.getTemplate(declared.templateId)); + if (template == null) { + console.warn('[SV2 JDP] PushSolution skipped because the declared template is no longer available'); + return; + } + + try { + const block = this.templateProvider.buildBlockFromDeclaredJobSolution({ + template, + job: declared.job, + providedTransactions: declared.providedTransactions, + solution, + }); + const coinbaseValidation = this.templateProvider.validateCoinbaseTransactionHeight( + block.transactions[0].toBuffer(), + template.height, + ); + if (!coinbaseValidation.valid) { + console.warn(`[SV2 JDP] PushSolution rejected locally: ${coinbaseValidation.errorCode}`); + return; + } + const blockHex = block.toHex(false); + this.submittedSolutions.add(solutionKey); + const result = await this.bitcoinRpcService.SUBMIT_BLOCK(blockHex); + const { address, worker } = this.parseUserIdentifier(declared.userIdentifier); + await this.blocksService.save({ + height: template.height, + minerAddress: address, + worker, + sessionId: declared.token.toString('hex').slice(0, 8), + blockData: blockHex, + blockSubmissionResult: result, + payoutSnapshotId: template.jobTemplate.blockData.payoutSnapshotId ?? null, + }); + await this.payoutSnapshotService.finalizeSnapshotForBlock({ + payoutSnapshotId: template.jobTemplate.blockData.payoutSnapshotId, + blockHeight: template.height, + blockSubmissionResult: result, + }); + await this.notificationService.notifySubscribersBlockFound(address, template.height, block, result); + console.log(`[SV2 JDP] PushSolution submitted block at height ${template.height}: ${result ?? 'accepted'}`); + } catch (error) { + console.error(`[SV2 JDP] PushSolution failed: ${error.message ?? error}`); + } + } + + private buildPoolCoinbaseOutputs(userIdentifier: string): Buffer { + const address = this.poolPayoutAddress || userIdentifier.split('.')[0]; + const script = bitcoinjs.address.toOutputScript(address, this.network); + const value = Buffer.alloc(8); + return Buffer.concat([ + this.customWorkService.encodeBitcoinVarInt(1), + value, + this.customWorkService.encodeBitcoinVarInt(script.length), + script, + ]); + } + + private parseUserIdentifier(userIdentifier: string): { address: string; worker: string } { + const [address, ...workerParts] = userIdentifier.split('.'); + return { + address: address || userIdentifier, + worker: workerParts.join('.') || 'jdp', + }; + } + + private async sendFrame(msgType: number, payload: Buffer): Promise { + await this.writeRaw(this.frameWriter.writeFrame({ + extensionType: 0, + msgType, + msgLength: payload.length, + }, payload)); + } + + private async writeRaw(data: Buffer): Promise { + if (this.socket.destroyed || this.socket.writableEnded) { + return; + } + await new Promise((resolve, reject) => { + this.socket.write(data, error => error ? reject(error) : resolve()); + }); + } + + private close(): void { + this.destroyed = true; + if (!this.socket.destroyed) { + this.socket.destroy(); + } + } +} diff --git a/src/services/sv2-template-distribution.service.spec.ts b/src/services/sv2-template-distribution.service.spec.ts new file mode 100644 index 0000000..38565c7 --- /dev/null +++ b/src/services/sv2-template-distribution.service.spec.ts @@ -0,0 +1,290 @@ +import * as bitcoinjs from 'bitcoinjs-lib'; +import { BehaviorSubject, firstValueFrom } from 'rxjs'; + +import { MockRecording1 } from '../../test/models/MockRecording1'; +import { BufferReader } from '../models/sv2/sv2-binary-codec'; +import { MiningJob } from '../models/MiningJob'; +import { Sv2MsgType, Sv2Protocol } from '../models/sv2/sv2-constants'; +import { + deserializeSetupConnectionSuccess, + serializeSetupConnection, +} from '../models/sv2/sv2-messages'; +import { serializeTdpCoinbaseOutputConstraints } from '../models/sv2/sv2-tdp-messages'; +import { + deserializeTdpNewTemplate, + deserializeTdpRequestTransactionDataSuccess, + serializeTdpRequestTransactionData, + serializeTdpSubmitSolution, +} from '../models/sv2/sv2-tdp-messages'; +import { StratumV1JobsService } from './stratum-v1-jobs.service'; +import { Sv2TemplateDistributionConnection } from './sv2-template-distribution.service'; +import { TemplateProviderService } from './template-provider.service'; + +describe('Sv2TemplateDistributionConnection compliance', () => { + it('waits for client CoinbaseOutputConstraints before sending templates', async () => { + const { connection, sentFrames, templateProvider, jobTemplate } = await createConnection(); + + await (connection as any).handleSetupConnection(serializeSetupConnection({ + protocol: Sv2Protocol.TEMPLATE_DISTRIBUTION, + minVersion: 2, + maxVersion: 2, + flags: 0, + endpoint_host: 'localhost', + endpoint_port: 34265, + vendor: 'template-client', + hardwareVersion: '', + firmwareVersion: '', + deviceId: '', + })); + + expect(sentFrames.map(frame => frame.msgType)).toEqual([Sv2MsgType.SETUP_CONNECTION_SUCCESS]); + const success = deserializeSetupConnectionSuccess(new BufferReader(sentFrames[0].payload)); + expect(success.flags).toBe(0); + + sentFrames.length = 0; + await (connection as any).handleFrame( + Sv2MsgType.TDP_COINBASE_OUTPUT_CONSTRAINTS, + serializeTdpCoinbaseOutputConstraints({ + coinbaseOutputMaxAdditionalSize: 50_000, + coinbaseOutputMaxAdditionalSigops: 0, + }), + ); + await Promise.resolve(); + await Promise.resolve(); + + expect(sentFrames.some(frame => frame.msgType === Sv2MsgType.TDP_NEW_TEMPLATE)).toBe(true); + expect(sentFrames.some(frame => frame.msgType === Sv2MsgType.TDP_SET_NEW_PREV_HASH)).toBe(true); + const templateFrame = sentFrames.find(frame => frame.msgType === Sv2MsgType.TDP_NEW_TEMPLATE); + const template = deserializeTdpNewTemplate(new BufferReader(templateFrame.payload)); + const expectedJob = new MiningJob( + bitcoinjs.networks.testnet, + jobTemplate.blockData.id, + [{ address: 'tb1qumezefzdeqqwn5zfvgdrhxjzc5ylr39uhuxcz4', percent: 100 }], + jobTemplate, + ); + const expectedCoinbaseTx = expectedJob.cloneCoinbaseTransaction(); + expect(template.coinbasePrefix).toEqual(expectedJob.getCoinbasePrefixBuffer()); + expect(template.coinbasePrefix.includes(templateProvider.buildCoinbaseHeightPrefix(MockRecording1.BLOCK_TEMPLATE.height))).toBe(true); + expect(template.coinbaseTxVersion).toBe(expectedCoinbaseTx.version); + expect(template.coinbaseTxInputSequence).toBe(expectedCoinbaseTx.ins[0].sequence); + expect(template.coinbaseTxOutputsCount).toBe(expectedCoinbaseTx.outs.length); + expect(template.coinbaseTxOutputs).toEqual((connection as any).serializeCoinbaseOutputs(expectedCoinbaseTx)); + expect(template.coinbaseTxLocktime).toBe(expectedCoinbaseTx.locktime); + }); + + it('serves transaction data for known templates', async () => { + const { connection, sentFrames, templateProvider, jobTemplate } = await createConnection(); + const template = templateProvider.upsert(jobTemplate); + + await (connection as any).handleFrame( + Sv2MsgType.TDP_REQUEST_TRANSACTION_DATA, + serializeTdpRequestTransactionData({ templateId: template.templateId }), + ); + + const successFrame = sentFrames.find(frame => frame.msgType === Sv2MsgType.TDP_REQUEST_TRANSACTION_DATA_SUCCESS); + const success = deserializeTdpRequestTransactionDataSuccess(new BufferReader(successFrame.payload)); + expect(success.templateId).toBe(template.templateId); + expect(success.transactionList[0]).toEqual(Buffer.from(MockRecording1.BLOCK_TEMPLATE.transactions[0].data, 'hex')); + }); + + it('submits reconstructed blocks from SubmitSolution', async () => { + const { connection, templateProvider, jobTemplate, bitcoinRpcService, blocksService, payoutSnapshotService, notificationService } = await createConnection(); + const template = templateProvider.upsert(jobTemplate); + const callOrder: string[] = []; + bitcoinRpcService.SUBMIT_BLOCK.mockImplementation(async () => { + callOrder.push('submit'); + return null; + }); + blocksService.save.mockImplementation(async () => { + callOrder.push('save'); + }); + const coinbaseTx = createCoinbaseTransaction(templateProvider.buildCoinbaseHeightPrefix(template.height)); + + await (connection as any).handleFrame( + Sv2MsgType.TDP_SUBMIT_SOLUTION, + serializeTdpSubmitSolution({ + templateId: template.templateId, + version: template.version, + headerTimestamp: template.minNtime, + headerNonce: 123, + coinbaseTx: coinbaseTx.toBuffer(), + }), + ); + + expect(bitcoinRpcService.SUBMIT_BLOCK).toHaveBeenCalledWith(expect.stringMatching(/^[0-9a-f]+$/)); + expect(blocksService.save).toHaveBeenCalledWith(expect.objectContaining({ + height: MockRecording1.BLOCK_TEMPLATE.height, + minerAddress: 'tb1qumezefzdeqqwn5zfvgdrhxjzc5ylr39uhuxcz4', + worker: 'tdp', + blockData: expect.stringMatching(/^[0-9a-f]+$/), + })); + expect(callOrder).toEqual(['submit', 'save']); + expect(payoutSnapshotService.finalizeSnapshotForBlock).toHaveBeenCalledWith({ + payoutSnapshotId: jobTemplate.blockData.payoutSnapshotId, + blockHeight: MockRecording1.BLOCK_TEMPLATE.height, + blockSubmissionResult: null, + }); + expect(notificationService.notifySubscribersBlockFound).toHaveBeenCalledWith( + 'tb1qumezefzdeqqwn5zfvgdrhxjzc5ylr39uhuxcz4', + MockRecording1.BLOCK_TEMPLATE.height, + expect.any(bitcoinjs.Block), + null, + ); + }); + + it('does not resubmit duplicate SubmitSolution messages', async () => { + const { connection, templateProvider, jobTemplate, bitcoinRpcService } = await createConnection(); + const template = templateProvider.upsert(jobTemplate); + const coinbaseTx = createCoinbaseTransaction(templateProvider.buildCoinbaseHeightPrefix(template.height)); + const payload = serializeTdpSubmitSolution({ + templateId: template.templateId, + version: template.version, + headerTimestamp: template.minNtime, + headerNonce: 123, + coinbaseTx: coinbaseTx.toBuffer(), + }); + + await (connection as any).handleFrame(Sv2MsgType.TDP_SUBMIT_SOLUTION, payload); + await (connection as any).handleFrame(Sv2MsgType.TDP_SUBMIT_SOLUTION, payload); + + expect(bitcoinRpcService.SUBMIT_BLOCK).toHaveBeenCalledTimes(1); + }); + + it('rejects SubmitSolution locally when the coinbase is missing the BIP34 height', async () => { + const warn = jest.spyOn(console, 'warn').mockImplementation(() => undefined); + const { connection, templateProvider, jobTemplate, bitcoinRpcService, blocksService } = await createConnection(); + const template = templateProvider.upsert(jobTemplate); + const coinbaseTx = createCoinbaseTransaction(Buffer.from('/public-pool-sri-jdc-e2e/', 'utf8')); + + await (connection as any).handleFrame( + Sv2MsgType.TDP_SUBMIT_SOLUTION, + serializeTdpSubmitSolution({ + templateId: template.templateId, + version: template.version, + headerTimestamp: template.minNtime, + headerNonce: 123, + coinbaseTx: coinbaseTx.toBuffer(), + }), + ); + + expect(bitcoinRpcService.SUBMIT_BLOCK).not.toHaveBeenCalled(); + expect(blocksService.save).not.toHaveBeenCalled(); + expect(warn).toHaveBeenCalledWith(expect.stringContaining('invalid-job-param-value-coinbase_tx_prefix')); + warn.mockRestore(); + }); + + it('submits using a served template snapshot after provider cache cleanup', async () => { + const { connection, sentFrames, templateProvider, jobTemplate, bitcoinRpcService, blocksService } = await createConnection(); + await (connection as any).sendTemplate(jobTemplate); + const templateId = BigInt(parseInt(jobTemplate.blockData.id, 16)); + jest.spyOn(templateProvider, 'getTemplate').mockReturnValue(undefined); + sentFrames.length = 0; + const coinbaseTx = createCoinbaseTransaction(templateProvider.buildCoinbaseHeightPrefix(jobTemplate.blockData.height)); + + await (connection as any).handleFrame( + Sv2MsgType.TDP_SUBMIT_SOLUTION, + serializeTdpSubmitSolution({ + templateId, + version: jobTemplate.block.version, + headerTimestamp: jobTemplate.block.timestamp, + headerNonce: 123, + coinbaseTx: coinbaseTx.toBuffer(), + }), + ); + + expect(bitcoinRpcService.SUBMIT_BLOCK).toHaveBeenCalledWith(expect.stringMatching(/^[0-9a-f]+$/)); + expect(blocksService.save).toHaveBeenCalledWith(expect.objectContaining({ + height: MockRecording1.BLOCK_TEMPLATE.height, + blockSubmissionResult: null, + })); + expect(sentFrames.some(frame => frame.msgType === Sv2MsgType.TDP_REQUEST_TRANSACTION_DATA_ERROR)).toBe(false); + }); + + async function createConnection(): Promise<{ + connection: Sv2TemplateDistributionConnection; + sentFrames: any[]; + templateProvider: TemplateProviderService; + jobTemplate: any; + bitcoinRpcService: { SUBMIT_BLOCK: jest.Mock }; + blocksService: { save: jest.Mock }; + payoutSnapshotService: { finalizeSnapshotForBlock: jest.Mock }; + notificationService: { notifySubscribersBlockFound: jest.Mock }; + }> { + const blockTemplate$ = new BehaviorSubject(MockRecording1.BLOCK_TEMPLATE); + const jobsService = new StratumV1JobsService({ + newBlockTemplate$: blockTemplate$.asObservable(), + miningInfo: { blocks: MockRecording1.BLOCK_TEMPLATE.height }, + } as any); + const jobTemplate = await firstValueFrom(jobsService.newMiningJob$); + const templateProvider = new TemplateProviderService(jobsService); + const bitcoinRpcService = { + SUBMIT_BLOCK: jest.fn().mockResolvedValue(null), + }; + const blocksService = { + save: jest.fn().mockResolvedValue(undefined), + }; + const payoutSnapshotService = { + finalizeSnapshotForBlock: jest.fn().mockResolvedValue({ finalized: false, reason: 'disabled' }), + }; + const notificationService = { + notifySubscribersBlockFound: jest.fn().mockResolvedValue(undefined), + }; + const socket = { + setKeepAlive: jest.fn(), + setNoDelay: jest.fn(), + on: jest.fn(), + write: jest.fn((data, callback) => callback?.()), + destroy: jest.fn(), + destroyed: false, + writableEnded: false, + }; + const connection = new Sv2TemplateDistributionConnection( + socket as any, + { + getNoiseConfig: () => ({ + staticKeypair: { + privateKey: Buffer.alloc(32), + publicKey: Buffer.alloc(64), + }, + certificateMessage: { + version: 0, + validFrom: 0, + notValidAfter: 0, + signature: Buffer.alloc(64), + }, + }), + } as any, + jobsService, + templateProvider, + bitcoinRpcService as any, + blocksService as any, + payoutSnapshotService as any, + notificationService as any, + 'tb1qumezefzdeqqwn5zfvgdrhxjzc5ylr39uhuxcz4', + bitcoinjs.networks.testnet, + ); + const sentFrames: any[] = []; + (connection as any).sendFrame = jest.fn((msgType: number, payload: Buffer) => { + sentFrames.push({ msgType, payload }); + return Promise.resolve(); + }); + return { + connection, + sentFrames, + templateProvider, + jobTemplate, + bitcoinRpcService, + blocksService, + payoutSnapshotService, + notificationService, + }; + } + + function createCoinbaseTransaction(script: Buffer): bitcoinjs.Transaction { + const tx = new bitcoinjs.Transaction(); + tx.version = 2; + tx.addInput(Buffer.alloc(32), 0xffffffff, 0xffffffff, script); + tx.addOutput(Buffer.from('6a', 'hex'), 0); + return tx; + } +}); diff --git a/src/services/sv2-template-distribution.service.ts b/src/services/sv2-template-distribution.service.ts new file mode 100644 index 0000000..f29a33e --- /dev/null +++ b/src/services/sv2-template-distribution.service.ts @@ -0,0 +1,444 @@ +import { Injectable, OnModuleInit } from '@nestjs/common'; +import { ConfigService } from '@nestjs/config'; +import * as bitcoinjs from 'bitcoinjs-lib'; +import { Server, Socket } from 'net'; +import { Subscription } from 'rxjs'; + +import { BlocksService } from '../ORM/blocks/blocks.service'; +import { PayoutSnapshotService } from '../ORM/payout-snapshot/payout-snapshot.service'; +import { SV2_NOISE_ACT1_SIZE, Sv2MsgType, Sv2Protocol } from '../models/sv2/sv2-constants'; +import { Sv2FrameReader, Sv2FrameWriter } from '../models/sv2/sv2-frame'; +import { BufferReader } from '../models/sv2/sv2-binary-codec'; +import { + deserializeSetupConnection, + serializeSetupConnectionError, + serializeSetupConnectionSuccess, +} from '../models/sv2/sv2-messages'; +import { AddressObject, MiningJob } from '../models/MiningJob'; +import { Sv2NoiseSession } from '../models/sv2/sv2-noise'; +import { + deserializeTdpCoinbaseOutputConstraints, + deserializeTdpRequestTransactionData, + deserializeTdpSubmitSolution, + serializeTdpNewTemplate, + serializeTdpRequestTransactionDataError, + serializeTdpRequestTransactionDataSuccess, + serializeTdpSetNewPrevHash, +} from '../models/sv2/sv2-tdp-messages'; +import { BitcoinRpcService } from './bitcoin-rpc.service'; +import { NotificationService } from './notification.service'; +import { IJobTemplate, StratumV1JobsService } from './stratum-v1-jobs.service'; +import { StratumV2Service } from './stratum-v2.service'; +import { TemplateProviderService, TemplateProviderTemplate } from './template-provider.service'; + +interface TdpCoinbaseTemplateFields { + coinbaseTxVersion: number; + coinbasePrefix: Buffer; + coinbaseTxInputSequence: number; + coinbaseTxValueRemaining: bigint; + coinbaseTxOutputsCount: number; + coinbaseTxOutputs: Buffer; + coinbaseTxLocktime: number; +} + +@Injectable() +export class Sv2TemplateDistributionService implements OnModuleInit { + private readonly servers: Server[] = []; + + constructor( + private readonly configService: ConfigService, + private readonly stratumV2Service: StratumV2Service, + private readonly jobsService: StratumV1JobsService, + private readonly templateProvider: TemplateProviderService, + private readonly bitcoinRpcService: BitcoinRpcService, + private readonly blocksService: BlocksService, + private readonly payoutSnapshotService: PayoutSnapshotService, + private readonly notificationService: NotificationService, + ) {} + + public async onModuleInit(): Promise { + if (process.env.API_ONLY === 'true' || process.env.MASTER === 'true') { + return; + } + + const ports = this.getPorts(); + if (ports.length === 0) { + return; + } + + await this.stratumV2Service.ensureInitialized(); + for (const port of ports) { + this.startServer(port); + } + } + + private startServer(port: number): void { + const server = new Server(socket => { + void new Sv2TemplateDistributionConnection( + socket, + this.stratumV2Service, + this.jobsService, + this.templateProvider, + this.bitcoinRpcService, + this.blocksService, + this.payoutSnapshotService, + this.notificationService, + this.getPoolPayoutAddress(), + this.getNetwork(), + ).start(); + }); + server.on('error', error => console.error(`SV2 TDP server error on port ${port}: ${error.message}`)); + server.listen(port, () => console.log(`SV2 Template Distribution server is listening on port ${port}`)); + this.servers.push(server); + } + + private getPorts(): number[] { + const configured = this.configService.get('SV2_TDP_PORTS'); + if (!configured?.trim()) { + return []; + } + return Array.from(new Set(configured + .split(',') + .map(port => parseInt(port.trim(), 10)) + .filter(port => Number.isInteger(port) && port > 0 && port <= 65535))); + } + + private getPoolPayoutAddress(): string { + return this.configService.get('SV2_TDP_POOL_PAYOUT_ADDRESS') + || this.configService.get('SV2_JDP_POOL_PAYOUT_ADDRESS') + || this.configService.get('DATUM_POOL_PAYOUT_ADDRESS') + || this.configService.get('PAYOUT_FEE_ADDRESS') + || this.configService.get('DEV_FEE_ADDRESS') + || ''; + } + + private getNetwork(): bitcoinjs.networks.Network { + const networkConfig = this.configService.get('NETWORK'); + if (networkConfig === 'mainnet') { + return bitcoinjs.networks.bitcoin; + } + if (networkConfig === 'testnet') { + return bitcoinjs.networks.testnet; + } + if (networkConfig === 'regtest') { + return bitcoinjs.networks.regtest; + } + throw new Error('Invalid network configuration'); + } +} + +export class Sv2TemplateDistributionConnection { + private readonly noiseSession: Sv2NoiseSession; + private readonly frameReader = new Sv2FrameReader(null); + private readonly frameWriter = new Sv2FrameWriter(null); + private handshakeBuffer = Buffer.alloc(0); + private handshakeComplete = false; + private destroyed = false; + private subscription: Subscription | null = null; + private readonly submittedSolutions = new Set(); + private readonly servedTemplates = new Map(); + + constructor( + private readonly socket: Socket, + stratumV2Service: StratumV2Service, + private readonly jobsService: StratumV1JobsService, + private readonly templateProvider: TemplateProviderService, + private readonly bitcoinRpcService: BitcoinRpcService, + private readonly blocksService: BlocksService, + private readonly payoutSnapshotService: PayoutSnapshotService, + private readonly notificationService: NotificationService, + private readonly poolPayoutAddress: string, + private readonly network: bitcoinjs.networks.Network, + ) { + this.noiseSession = new Sv2NoiseSession(stratumV2Service.getNoiseConfig()); + } + + public async start(): Promise { + this.socket.setKeepAlive(true, 60_000); + this.socket.setNoDelay(true); + this.socket.on('data', data => void this.handleSocketData(data)); + this.socket.on('error', () => this.close()); + this.socket.on('close', () => this.close()); + } + + private async handleSocketData(data: Buffer): Promise { + if (this.destroyed) { + return; + } + try { + if (!this.handshakeComplete) { + await this.handleHandshakeData(data); + return; + } + for (const frame of this.frameReader.feed(data)) { + await this.handleFrame(frame.header.msgType, frame.payload); + } + } catch (error) { + console.error(`[SV2 TDP] ${error.message}`); + this.close(); + } + } + + private async handleHandshakeData(data: Buffer): Promise { + this.handshakeBuffer = Buffer.concat([this.handshakeBuffer, data]); + if (this.handshakeBuffer.length < SV2_NOISE_ACT1_SIZE) { + return; + } + const act1 = this.handshakeBuffer.subarray(0, SV2_NOISE_ACT1_SIZE); + const remainder = Buffer.from(this.handshakeBuffer.subarray(SV2_NOISE_ACT1_SIZE)); + this.handshakeBuffer = Buffer.alloc(0); + await this.writeRaw(await this.noiseSession.processAct1(Buffer.from(act1))); + this.frameReader.setDecryptFn(ciphertext => this.noiseSession.decrypt(ciphertext)); + this.frameWriter.setEncryptFn(plaintext => this.noiseSession.encrypt(plaintext)); + this.handshakeComplete = true; + if (remainder.length > 0) { + for (const frame of this.frameReader.feed(remainder)) { + await this.handleFrame(frame.header.msgType, frame.payload); + } + } + } + + private async handleFrame(msgType: number, payload: Buffer): Promise { + switch (msgType) { + case Sv2MsgType.SETUP_CONNECTION: + await this.handleSetupConnection(payload); + break; + case Sv2MsgType.TDP_COINBASE_OUTPUT_CONSTRAINTS: + deserializeTdpCoinbaseOutputConstraints(new BufferReader(payload)); + this.subscribeTemplates(); + break; + case Sv2MsgType.TDP_REQUEST_TRANSACTION_DATA: { + const request = deserializeTdpRequestTransactionData(new BufferReader(payload)); + await this.handleRequestTransactionData(request.templateId); + break; + } + case Sv2MsgType.TDP_SUBMIT_SOLUTION: { + const solution = deserializeTdpSubmitSolution(new BufferReader(payload)); + await this.handleSubmitSolution(solution); + break; + } + default: + break; + } + } + + private async handleSetupConnection(payload: Buffer): Promise { + const setup = deserializeSetupConnection(new BufferReader(payload)); + if (setup.protocol !== Sv2Protocol.TEMPLATE_DISTRIBUTION) { + await this.sendFrame(Sv2MsgType.SETUP_CONNECTION_ERROR, serializeSetupConnectionError({ + flags: 0, + errorCode: 'unsupported-protocol', + })); + this.close(); + return; + } + + await this.sendFrame(Sv2MsgType.SETUP_CONNECTION_SUCCESS, serializeSetupConnectionSuccess({ + usedVersion: 2, + flags: 0, + })); + } + + private subscribeTemplates(): void { + if (this.subscription != null) { + return; + } + this.subscription = this.jobsService.newMiningJob$.subscribe({ + next: template => void this.sendTemplate(template), + error: () => this.close(), + }); + } + + private async sendTemplate(template: IJobTemplate): Promise { + const cachedTemplate = this.templateProvider.upsert(template); + const templateId = BigInt(parseInt(template.blockData.id, 16)); + this.servedTemplates.set(templateId.toString(), cachedTemplate); + const merklePath = template.merkle_branch.map(branch => Buffer.from(branch, 'hex')); + const coinbase = this.buildCoinbaseTemplateFields(template); + await this.sendFrame(Sv2MsgType.TDP_NEW_TEMPLATE, serializeTdpNewTemplate({ + templateId, + futureTemplate: true, + version: template.block.version, + ...coinbase, + merklePath, + })); + await this.sendFrame(Sv2MsgType.TDP_SET_NEW_PREV_HASH, serializeTdpSetNewPrevHash({ + templateId, + prevHash: Buffer.from(template.block.prevHash), + headerTimestamp: template.block.timestamp, + nBits: template.block.bits, + target: cachedTemplate.target, + })); + } + + private buildCoinbaseTemplateFields(template: IJobTemplate): TdpCoinbaseTemplateFields { + const job = new MiningJob( + this.network, + template.blockData.id, + this.getPayoutInformation(template), + template, + ); + const coinbaseTx = job.cloneCoinbaseTransaction(); + + return { + coinbaseTxVersion: coinbaseTx.version, + coinbasePrefix: job.getCoinbasePrefixBuffer(), + coinbaseTxInputSequence: coinbaseTx.ins[0]?.sequence ?? 0xffffffff, + coinbaseTxValueRemaining: BigInt(template.blockData.coinbasevalue), + coinbaseTxOutputsCount: coinbaseTx.outs.length, + coinbaseTxOutputs: this.serializeCoinbaseOutputs(coinbaseTx), + coinbaseTxLocktime: coinbaseTx.locktime, + }; + } + + private getPayoutInformation(template: IJobTemplate): AddressObject[] { + if (template.blockData.payoutOutputs?.length > 0) { + return template.blockData.payoutOutputs; + } + + return [{ address: this.poolPayoutAddress, percent: 100 }]; + } + + private serializeCoinbaseOutputs(coinbaseTx: bitcoinjs.Transaction): Buffer { + return Buffer.concat(coinbaseTx.outs.map(output => { + const value = Buffer.alloc(8); + value.writeBigUInt64LE(BigInt(output.value), 0); + return Buffer.concat([ + value, + this.encodeBitcoinVarInt(output.script.length), + Buffer.from(output.script), + ]); + })); + } + + private encodeBitcoinVarInt(value: number): Buffer { + if (!Number.isSafeInteger(value) || value < 0) { + throw new RangeError(`Invalid Bitcoin varint value ${value}`); + } + if (value < 0xfd) { + return Buffer.from([value]); + } + if (value <= 0xffff) { + const result = Buffer.alloc(3); + result[0] = 0xfd; + result.writeUInt16LE(value, 1); + return result; + } + if (value <= 0xffffffff) { + const result = Buffer.alloc(5); + result[0] = 0xfe; + result.writeUInt32LE(value, 1); + return result; + } + const result = Buffer.alloc(9); + result[0] = 0xff; + result.writeBigUInt64LE(BigInt(value), 1); + return result; + } + + private async handleRequestTransactionData(templateId: bigint): Promise { + const template = this.getServedOrProviderTemplate(templateId); + if (template == null) { + await this.sendFrame(Sv2MsgType.TDP_REQUEST_TRANSACTION_DATA_ERROR, serializeTdpRequestTransactionDataError({ + templateId, + errorCode: 'template-not-found', + })); + return; + } + this.servedTemplates.set(templateId.toString(), template); + + await this.sendFrame(Sv2MsgType.TDP_REQUEST_TRANSACTION_DATA_SUCCESS, serializeTdpRequestTransactionDataSuccess({ + templateId, + excessData: Buffer.alloc(0), + transactionList: template.transactionList, + })); + } + + private async handleSubmitSolution(solution: ReturnType): Promise { + const template = this.getServedOrProviderTemplate(solution.templateId); + if (template == null) { + await this.sendFrame(Sv2MsgType.TDP_REQUEST_TRANSACTION_DATA_ERROR, serializeTdpRequestTransactionDataError({ + templateId: solution.templateId, + errorCode: 'template-not-found', + })); + return; + } + + const coinbaseValidation = this.templateProvider.validateCoinbaseTransactionHeight(solution.coinbaseTx, template.height); + if (!coinbaseValidation.valid) { + console.warn(`[SV2 TDP] SubmitSolution rejected locally: ${coinbaseValidation.errorCode}`); + return; + } + const block = this.templateProvider.buildBlockFromSolution({ + template, + coinbaseTx: solution.coinbaseTx, + version: solution.version, + headerTimestamp: solution.headerTimestamp, + headerNonce: solution.headerNonce, + }); + const blockHex = block.toHex(false); + const solutionKey = `${solution.templateId.toString()}:${block.getId()}`; + if (this.submittedSolutions.has(solutionKey)) { + return; + } + this.submittedSolutions.add(solutionKey); + const result = await this.bitcoinRpcService.SUBMIT_BLOCK(blockHex); + await this.blocksService.save({ + height: template.height, + minerAddress: this.poolPayoutAddress || 'sv2-tdp', + worker: 'tdp', + sessionId: solution.templateId.toString(16).slice(-8).padStart(8, '0'), + blockData: blockHex, + blockSubmissionResult: result, + payoutSnapshotId: template.jobTemplate.blockData.payoutSnapshotId ?? null, + }); + await this.payoutSnapshotService.finalizeSnapshotForBlock({ + payoutSnapshotId: template.jobTemplate.blockData.payoutSnapshotId, + blockHeight: template.height, + blockSubmissionResult: result, + }); + await this.notificationService.notifySubscribersBlockFound( + this.poolPayoutAddress || 'sv2-tdp', + template.height, + block, + result, + ); + if (result != null && result !== 'SUCCESS!') { + console.warn(`[SV2 TDP] SubmitSolution rejected: ${result}`); + } + } + + private getServedOrProviderTemplate(templateId: bigint): TemplateProviderTemplate | undefined { + return this.servedTemplates.get(templateId.toString()) + ?? this.templateProvider.getTemplate(templateId); + } + + private async sendFrame(msgType: number, payload: Buffer): Promise { + await this.writeRaw(this.frameWriter.writeFrame({ + extensionType: 0, + msgType, + msgLength: payload.length, + }, payload)); + } + + private async writeRaw(data: Buffer): Promise { + if (this.socket.destroyed || this.socket.writableEnded) { + return; + } + await new Promise((resolve, reject) => { + this.socket.write(data, error => error ? reject(error) : resolve()); + }); + } + + private close(): void { + if (this.destroyed) { + return; + } + this.destroyed = true; + this.subscription?.unsubscribe(); + this.subscription = null; + if (!this.socket.destroyed) { + this.socket.destroy(); + } + } +} diff --git a/src/services/template-provider.service.spec.ts b/src/services/template-provider.service.spec.ts new file mode 100644 index 0000000..642f163 --- /dev/null +++ b/src/services/template-provider.service.spec.ts @@ -0,0 +1,331 @@ +import { BehaviorSubject, firstValueFrom } from 'rxjs'; +import * as bitcoinjs from 'bitcoinjs-lib'; + +import { MockRecording1 } from '../../test/models/MockRecording1'; +import { CustomWorkService } from './custom-work.service'; +import { StratumV1JobsService } from './stratum-v1-jobs.service'; +import { TemplateProviderService } from './template-provider.service'; + +describe('TemplateProviderService', () => { + it('indexes template transactions and validates known wtxids', async () => { + const { provider, jobTemplate } = await createProvider(); + const template = provider.upsert(jobTemplate); + const firstTx = MockRecording1.BLOCK_TEMPLATE.transactions[0]; + + expect(template.transactionList[0]).toEqual(Buffer.from(firstTx.data, 'hex')); + expect(provider.getTemplate(template.templateId)).toBe(template); + + const validation = provider.validateDeclaredWtxids({ + version: MockRecording1.BLOCK_TEMPLATE.version, + wtxidList: [Buffer.from(firstTx.hash, 'hex')], + }); + expect(validation.valid).toBe(true); + expect(validation.template).toBe(template); + }); + + it('reports unknown transaction positions and validates supplied missing tx bytes', async () => { + const { provider, jobTemplate } = await createProvider(); + provider.upsert(jobTemplate); + const firstTx = MockRecording1.BLOCK_TEMPLATE.transactions[0]; + const unknownWtxid = Buffer.alloc(32, 0xaa); + + const validation = provider.validateDeclaredWtxids({ + version: MockRecording1.BLOCK_TEMPLATE.version, + wtxidList: [Buffer.from(firstTx.hash, 'hex'), unknownWtxid], + }); + expect(validation.valid).toBe(false); + expect(validation.errorCode).toBe('missing-transactions'); + expect(validation.unknownTxPositionList).toEqual([1]); + + const missingTxValidation = await provider.validateProvidedTransactions({ + expectedWtxids: [Buffer.from(firstTx.hash, 'hex'), unknownWtxid], + unknownTxPositionList: [1], + transactionList: [Buffer.from(firstTx.data, 'hex')], + }); + expect(missingTxValidation.valid).toBe(false); + expect(missingTxValidation.errorCode).toBe('transaction-wtxid-mismatch'); + }); + + it('rejects duplicate declared wtxids before requesting missing transactions', async () => { + const { provider, jobTemplate } = await createProvider(); + provider.upsert(jobTemplate); + const firstTx = MockRecording1.BLOCK_TEMPLATE.transactions[0]; + + const validation = provider.validateDeclaredWtxids({ + version: MockRecording1.BLOCK_TEMPLATE.version, + wtxidList: [ + Buffer.from(firstTx.hash, 'hex'), + Buffer.from(firstTx.hash, 'hex'), + ], + }); + + expect(validation.valid).toBe(false); + expect(validation.errorCode).toBe('duplicate-transactions'); + }); + + it('builds and validates the BIP34 coinbase height prefix for SV2 templates', async () => { + const { provider, jobTemplate } = await createProvider(); + const template = provider.upsert(jobTemplate); + const heightPrefix = provider.buildCoinbaseHeightPrefix(template.height); + const coinbaseTx = createCoinbaseTransaction(heightPrefix); + + expect(heightPrefix).toEqual(Buffer.concat([ + Buffer.from([bitcoinjs.script.number.encode(template.height).length]), + bitcoinjs.script.number.encode(template.height), + ])); + expect(provider.validateCoinbaseTransactionHeight(coinbaseTx.toBuffer(), template.height)).toEqual({ valid: true }); + }); + + it('rejects declared SV2 JDP coinbase prefixes missing the required BIP34 height', async () => { + const { provider, jobTemplate } = await createProvider(); + const template = provider.upsert(jobTemplate); + const badScript = Buffer.from('/public-pool-sri-jdc-e2e/', 'utf8'); + const badPrefix = createCoinbasePrefixForScript(badScript, badScript); + + const validation = provider.validateDeclaredWtxids({ + version: MockRecording1.BLOCK_TEMPLATE.version, + coinbaseTxPrefix: badPrefix, + wtxidList: [], + }); + + expect(validation).toMatchObject({ + valid: false, + errorCode: 'invalid-job-param-value-coinbase_tx_prefix', + template, + }); + }); + + it('accepts declared SV2 JDP coinbase prefixes that start with the required BIP34 height', async () => { + const { provider, jobTemplate } = await createProvider(); + const template = provider.upsert(jobTemplate); + const heightPrefix = provider.buildCoinbaseHeightPrefix(template.height); + const validPrefix = createCoinbasePrefixForScript( + Buffer.concat([heightPrefix, Buffer.from('/public-pool-sri-jdc-e2e/', 'utf8')]), + heightPrefix, + ); + + const validation = provider.validateDeclaredWtxids({ + version: MockRecording1.BLOCK_TEMPLATE.version, + coinbaseTxPrefix: validPrefix, + wtxidList: [], + }); + + expect(validation.valid).toBe(true); + expect(validation.template).toBe(template); + }); + + it('accepts DATUM template metadata matching the latest template fast path', async () => { + const { provider, jobTemplate } = await createProvider(); + const template = provider.upsert(jobTemplate); + const nBits = Buffer.alloc(4); + nBits.writeUInt32LE(template.nBits, 0); + + const validation = provider.validateDatumTemplateFastPath({ + prevBlockHash: Buffer.from(template.prevHash), + nBits, + height: template.height, + version: template.version ^ 0x2000, + coinbaseValue: template.coinbaseValue, + totalWeight: template.weightLimit, + totalSize: template.sizeLimit, + totalSigops: template.sigopLimit, + merkleBranches: [Buffer.alloc(32)], + }); + + expect(validation.valid).toBe(true); + expect(validation.template).toBe(template); + }); + + it('rejects stale or impossible DATUM template metadata in the fast path', async () => { + const { provider, jobTemplate } = await createProvider(); + const template = provider.upsert(jobTemplate); + const nBits = Buffer.alloc(4); + nBits.writeUInt32LE(template.nBits, 0); + + expect(provider.validateDatumTemplateFastPath({ + prevBlockHash: Buffer.alloc(32, 0xaa), + nBits, + height: template.height, + })).toMatchObject({ valid: false, errorCode: 'prevhash-mismatch' }); + + expect(provider.validateDatumTemplateFastPath({ + prevBlockHash: Buffer.from(template.prevHash), + nBits, + height: template.height + 1, + })).toMatchObject({ valid: false, errorCode: 'height-mismatch' }); + + expect(provider.validateDatumTemplateFastPath({ + prevBlockHash: Buffer.from(template.prevHash), + nBits, + height: template.height, + coinbaseValue: template.coinbaseValue + 1n, + })).toMatchObject({ valid: false, errorCode: 'coinbase-value-too-high' }); + + expect(provider.validateDatumTemplateFastPath({ + prevBlockHash: Buffer.from(template.prevHash), + nBits, + height: template.height, + totalWeight: template.weightLimit + 1, + })).toMatchObject({ valid: false, errorCode: 'weight-limit-exceeded' }); + }); + + it('validates supplied missing transactions against Bitcoin Core mempool policy', async () => { + const firstTx = MockRecording1.BLOCK_TEMPLATE.transactions[0]; + const { provider, bitcoinRpcService } = await createProvider({ + TEST_MEMPOOL_ACCEPT: jest.fn().mockResolvedValue([{ allowed: true }]), + }); + + const validation = await provider.validateProvidedTransactions({ + expectedWtxids: [Buffer.from(firstTx.hash, 'hex')], + unknownTxPositionList: [0], + transactionList: [Buffer.from(firstTx.data, 'hex')], + }); + + expect(validation.valid).toBe(true); + expect(bitcoinRpcService.TEST_MEMPOOL_ACCEPT).toHaveBeenCalledWith([Buffer.from(firstTx.data, 'hex')]); + }); + + it('rejects malformed supplied missing transactions before mempool policy checks', async () => { + const firstTx = MockRecording1.BLOCK_TEMPLATE.transactions[0]; + const { provider, bitcoinRpcService } = await createProvider({ + TEST_MEMPOOL_ACCEPT: jest.fn().mockResolvedValue([{ allowed: true }]), + }); + + const validation = await provider.validateProvidedTransactions({ + expectedWtxids: [Buffer.from(firstTx.hash, 'hex')], + unknownTxPositionList: [0], + transactionList: [Buffer.from('010203', 'hex')], + }); + + expect(validation.valid).toBe(false); + expect(validation.errorCode).toMatch(/^transaction-decode-failed:0:/); + expect(bitcoinRpcService.TEST_MEMPOOL_ACCEPT).not.toHaveBeenCalled(); + }); + + it('validates generic transaction data for SV2 and DATUM callers', async () => { + const firstTx = MockRecording1.BLOCK_TEMPLATE.transactions[0]; + const { provider } = await createProvider(); + + const valid = provider.validateTransactionData({ + transactionList: [Buffer.from(firstTx.data, 'hex')], + expectedCount: 1, + maxTotalBytes: Buffer.from(firstTx.data, 'hex').length, + }); + expect(valid.valid).toBe(true); + expect(valid.transactions?.[0].wtxid).toBe(firstTx.hash); + + expect(provider.validateTransactionData({ + transactionList: [Buffer.from(firstTx.data, 'hex')], + expectedCount: 2, + })).toMatchObject({ + valid: false, + errorCode: 'transaction-count-mismatch:2:1', + }); + + expect(provider.validateTransactionData({ + transactionList: [Buffer.from(firstTx.data, 'hex')], + maxTotalBytes: 1, + })).toMatchObject({ + valid: false, + errorCode: expect.stringMatching(/^transaction-bytes-exceed-limit:/), + }); + }); + + it('validates DATUM transaction blobs against the advertised coinbase merkle path', async () => { + const { provider } = await createProvider(); + const coinbaseTx = createTestTransaction(0); + const transactionList = [ + createTestTransaction(1), + createTestTransaction(2), + createTestTransaction(3), + ]; + const merklePath = provider.buildCoinbaseMerklePath(transactionList); + const merkleRoot = new CustomWorkService().computeMerkleRoot(coinbaseTx, merklePath); + const block = new bitcoinjs.Block(); + block.transactions = [ + bitcoinjs.Transaction.fromBuffer(coinbaseTx), + ...transactionList.map(tx => bitcoinjs.Transaction.fromBuffer(tx)), + ]; + + expect(merkleRoot).toEqual(bitcoinjs.Block.calculateMerkleRoot(block.transactions, false)); + expect(provider.validateTransactionData({ + transactionList, + expectedCount: transactionList.length, + expectedCoinbaseMerklePath: merklePath, + })).toMatchObject({ valid: true }); + + const wrongPath = merklePath.map(branch => Buffer.from(branch)); + wrongPath[0][0] ^= 0xff; + expect(provider.validateTransactionData({ + transactionList, + expectedCount: transactionList.length, + expectedCoinbaseMerklePath: wrongPath, + })).toMatchObject({ + valid: false, + errorCode: 'merkle-path-mismatch:0', + }); + }); + + it('rejects supplied missing transactions rejected by Bitcoin Core mempool policy', async () => { + const firstTx = MockRecording1.BLOCK_TEMPLATE.transactions[0]; + const { provider } = await createProvider({ + TEST_MEMPOOL_ACCEPT: jest.fn().mockResolvedValue([{ + allowed: false, + rejectReason: 'txn-mempool-conflict', + }]), + }); + + const validation = await provider.validateProvidedTransactions({ + expectedWtxids: [Buffer.from(firstTx.hash, 'hex')], + unknownTxPositionList: [0], + transactionList: [Buffer.from(firstTx.data, 'hex')], + }); + + expect(validation.valid).toBe(false); + expect(validation.errorCode).toBe('txn-mempool-conflict'); + }); + + async function createProvider(bitcoinRpcService?: any): Promise<{ + provider: TemplateProviderService; + jobTemplate: any; + bitcoinRpcService: any; + }> { + const blockTemplate$ = new BehaviorSubject(MockRecording1.BLOCK_TEMPLATE); + const jobsService = new StratumV1JobsService({ + newBlockTemplate$: blockTemplate$.asObservable(), + miningInfo: { blocks: MockRecording1.BLOCK_TEMPLATE.height }, + } as any); + const jobTemplate = await firstValueFrom(jobsService.newMiningJob$); + return { + provider: new TemplateProviderService(jobsService, bitcoinRpcService), + jobTemplate, + bitcoinRpcService, + }; + } + + function createTestTransaction(seed: number): Buffer { + const tx = new bitcoinjs.Transaction(); + tx.version = 2; + tx.addInput(Buffer.alloc(32, seed), 0xffffffff, 0xffffffff, Buffer.from([seed & 0xff])); + tx.addOutput(Buffer.from('6a', 'hex'), 0); + return tx.toBuffer(); + } + + function createCoinbaseTransaction(script: Buffer): bitcoinjs.Transaction { + const tx = new bitcoinjs.Transaction(); + tx.version = 2; + tx.addInput(Buffer.alloc(32), 0xffffffff, 0xffffffff, script); + tx.addOutput(Buffer.from('6a', 'hex'), 0); + return tx; + } + + function createCoinbasePrefixForScript(fullScript: Buffer, prefixScript: Buffer): Buffer { + const tx = createCoinbaseTransaction(fullScript); + const serialized = tx.toBuffer(); + const scriptStart = serialized.indexOf(fullScript); + if (scriptStart < 0) { + throw new Error('test coinbase script not found'); + } + return serialized.subarray(0, scriptStart + prefixScript.length); + } +}); diff --git a/src/services/template-provider.service.ts b/src/services/template-provider.service.ts new file mode 100644 index 0000000..c7a3521 --- /dev/null +++ b/src/services/template-provider.service.ts @@ -0,0 +1,656 @@ +import { Injectable, OnModuleInit } from '@nestjs/common'; +import * as bitcoinjs from 'bitcoinjs-lib'; +import { Subscription } from 'rxjs'; + +import { IBlockTemplateTx } from '../models/bitcoin-rpc/IBlockTemplate'; +import { Sv2DeclareMiningJob, Sv2PushSolution } from '../models/sv2/sv2-jdp-messages'; +import { hash256 } from '../utils/hash.utils'; +import { BitcoinRpcService } from './bitcoin-rpc.service'; +import { IJobTemplate, StratumV1JobsService } from './stratum-v1-jobs.service'; + +export interface TemplateProviderTransaction { + position: number; + data: Buffer; + txid: string; + wtxid: string; + fee: number; + sigops: number; + weight: number; +} + +export interface TemplateProviderTemplate { + templateId: bigint; + jobTemplate: IJobTemplate; + version: number; + prevHash: Buffer; + nBits: number; + minNtime: number; + coinbaseValue: bigint; + networkDifficulty: number; + target: Buffer; + height: number; + sigopLimit: number; + sizeLimit: number; + weightLimit: number; + transactions: TemplateProviderTransaction[]; + transactionList: Buffer[]; + txidMap: Map; + wtxidMap: Map; + createdAt: number; +} + +export interface DeclareMiningJobValidationResult { + valid: boolean; + errorCode?: string; + template?: TemplateProviderTemplate; + unknownTxPositionList?: number[]; +} + +export interface DatumTemplateValidationInput { + prevBlockHash?: Buffer; + nBits?: Buffer; + height?: number; + version?: number; + coinbaseValue?: bigint; + totalWeight?: number; + totalSize?: number; + totalSigops?: number; + merkleBranches?: Buffer[]; +} + +export interface TemplateFastPathValidationResult { + valid: boolean; + errorCode?: string; + template?: TemplateProviderTemplate; +} + +export interface TransactionDataValidationResult { + valid: boolean; + errorCode?: string; + transactions?: TemplateProviderTransaction[]; + totalBytes?: number; +} + +export interface CoinbaseHeightValidationResult { + valid: boolean; + errorCode?: string; +} + +const VERSION_ROLLING_MASK = 0x1fffe000; + +@Injectable() +export class TemplateProviderService implements OnModuleInit { + private readonly templates = new Map(); + private readonly retentionMs = this.readPositiveInt('SV2_TEMPLATE_RETENTION_MS', 10 * 60 * 1000); + private subscription: Subscription | null = null; + private latestTemplateId: string | null = null; + + constructor( + private readonly jobsService: StratumV1JobsService, + private readonly bitcoinRpcService?: BitcoinRpcService, + ) {} + + public onModuleInit(): void { + if (this.subscription != null) { + return; + } + this.subscription = this.jobsService.newMiningJob$.subscribe(template => { + this.upsert(template); + }); + } + + public upsert(jobTemplate: IJobTemplate): TemplateProviderTemplate { + this.cleanup(); + const templateId = BigInt(parseInt(jobTemplate.blockData.id, 16)); + const transactions = this.buildTransactions(jobTemplate.blockData.transactions ?? []); + const template: TemplateProviderTemplate = { + templateId, + jobTemplate, + version: jobTemplate.block.version, + prevHash: Buffer.from(jobTemplate.block.prevHash), + nBits: jobTemplate.block.bits, + minNtime: jobTemplate.block.timestamp, + coinbaseValue: BigInt(jobTemplate.blockData.coinbasevalue), + networkDifficulty: jobTemplate.blockData.networkDifficulty, + target: this.targetFromDifficulty(jobTemplate.blockData.networkDifficulty), + height: jobTemplate.blockData.height, + sigopLimit: jobTemplate.blockData.sigoplimit ?? 80_000, + sizeLimit: jobTemplate.blockData.sizelimit ?? 4_000_000, + weightLimit: jobTemplate.blockData.weightlimit ?? 4_000_000, + transactions, + transactionList: transactions.map(tx => Buffer.from(tx.data)), + txidMap: this.buildMap(transactions, tx => tx.txid), + wtxidMap: this.buildMap(transactions, tx => tx.wtxid), + createdAt: Date.now(), + }; + this.templates.set(template.templateId.toString(), template); + this.latestTemplateId = template.templateId.toString(); + return template; + } + + public getTemplate(templateId: bigint | number | string): TemplateProviderTemplate | undefined { + this.cleanup(); + return this.templates.get(BigInt(templateId).toString()); + } + + public getLatestTemplate(): TemplateProviderTemplate | undefined { + this.cleanup(); + return this.latestTemplateId == null ? undefined : this.templates.get(this.latestTemplateId); + } + + public validateDeclaredWtxids(input: { + version: number; + wtxidList: Buffer[]; + coinbaseTxPrefix?: Buffer; + }): DeclareMiningJobValidationResult { + const template = this.getLatestTemplate(); + if (template == null) { + return { valid: false, errorCode: 'template-not-found' }; + } + if (input.version !== template.version) { + return { valid: false, errorCode: 'version-mismatch', template }; + } + if (input.coinbaseTxPrefix != null) { + const coinbaseHeight = this.validateDeclaredCoinbasePrefixHeight(input.coinbaseTxPrefix, template.height); + if (!coinbaseHeight.valid) { + return { valid: false, errorCode: coinbaseHeight.errorCode, template }; + } + } + + const seen = new Set(); + const unknownTxPositionList: number[] = []; + let duplicateFound = false; + input.wtxidList.forEach((wtxid, index) => { + const aliases = this.hashAliases(wtxid.toString('hex')); + if (aliases.some(alias => seen.has(alias))) { + duplicateFound = true; + unknownTxPositionList.push(index); + return; + } + aliases.forEach(alias => seen.add(alias)); + if (!this.lookupWtxid(template, wtxid)) { + unknownTxPositionList.push(index); + } + }); + + if (duplicateFound) { + return { + valid: false, + errorCode: 'duplicate-transactions', + template, + unknownTxPositionList, + }; + } + + return { + valid: unknownTxPositionList.length === 0, + errorCode: unknownTxPositionList.length > 0 ? 'missing-transactions' : undefined, + template, + unknownTxPositionList, + }; + } + + public buildCoinbaseHeightPrefix(height: number): Buffer { + const encodedHeight = bitcoinjs.script.number.encode(height); + return Buffer.concat([Buffer.from([encodedHeight.length]), encodedHeight]); + } + + public validateCoinbaseTransactionHeight(coinbaseTx: Buffer, height: number): CoinbaseHeightValidationResult { + let tx: bitcoinjs.Transaction; + try { + tx = bitcoinjs.Transaction.fromBuffer(coinbaseTx); + } catch (error) { + return { + valid: false, + errorCode: `invalid-coinbase-transaction:${error instanceof Error ? error.message : String(error)}`, + }; + } + if (tx.ins.length !== 1) { + return { valid: false, errorCode: 'invalid-coinbase-input-count' }; + } + return this.validateCoinbaseScriptHeight(tx.ins[0].script, height); + } + + public validateDeclaredCoinbasePrefixHeight(coinbaseTxPrefix: Buffer, height: number): CoinbaseHeightValidationResult { + const scriptStart = this.getCoinbaseInputScriptStart(coinbaseTxPrefix); + if (scriptStart == null) { + return { valid: false, errorCode: 'invalid-job-param-value-coinbase_tx_prefix' }; + } + const expected = this.buildCoinbaseHeightPrefix(height); + if (coinbaseTxPrefix.length < scriptStart + expected.length) { + return { valid: false, errorCode: 'invalid-job-param-value-coinbase_tx_prefix' }; + } + return this.validateCoinbaseScriptHeight(coinbaseTxPrefix.subarray(scriptStart), height); + } + + public async validateProvidedTransactions(input: { + expectedWtxids: Buffer[]; + unknownTxPositionList: number[]; + transactionList: Buffer[]; + }): Promise<{ valid: boolean; errorCode?: string }> { + if (input.unknownTxPositionList.length !== input.transactionList.length) { + return { valid: false, errorCode: 'missing-transaction-count-mismatch' }; + } + const transactionValidation = this.validateTransactionData({ + transactionList: input.transactionList, + expectedCount: input.transactionList.length, + }); + if (!transactionValidation.valid) { + return { + valid: false, + errorCode: transactionValidation.errorCode, + }; + } + for (let i = 0; i < input.transactionList.length; i++) { + const position = input.unknownTxPositionList[i]; + const expected = input.expectedWtxids[position]; + const provided = transactionValidation.transactions![i]; + if (expected == null || !this.matchesHash(expected, provided.wtxid)) { + return { valid: false, errorCode: 'transaction-wtxid-mismatch' }; + } + } + const mempoolResult = await this.validateMempoolPolicy(input.transactionList); + if (!mempoolResult.valid) { + return mempoolResult; + } + return { valid: true }; + } + + public validateTransactionData(input: { + transactionList: Buffer[]; + expectedCount?: number; + maxTotalBytes?: number; + expectedCoinbaseMerklePath?: Buffer[]; + }): TransactionDataValidationResult { + if (input.expectedCount != null && input.transactionList.length !== input.expectedCount) { + return { + valid: false, + errorCode: `transaction-count-mismatch:${input.expectedCount}:${input.transactionList.length}`, + }; + } + + let totalBytes = 0; + const transactions: TemplateProviderTransaction[] = []; + for (let i = 0; i < input.transactionList.length; i++) { + const rawTransaction = input.transactionList[i]; + totalBytes += rawTransaction.length; + if (input.maxTotalBytes != null && totalBytes > input.maxTotalBytes) { + return { + valid: false, + errorCode: `transaction-bytes-exceed-limit:${totalBytes}:${input.maxTotalBytes}`, + totalBytes, + }; + } + try { + transactions.push(this.parseTransaction(rawTransaction, i)); + } catch (error) { + return { + valid: false, + errorCode: `transaction-decode-failed:${i}:${error instanceof Error ? error.message : String(error)}`, + totalBytes, + }; + } + } + if (input.expectedCoinbaseMerklePath != null) { + const merklePath = this.buildCoinbaseMerklePath(input.transactionList); + if (merklePath.length !== input.expectedCoinbaseMerklePath.length) { + return { + valid: false, + errorCode: `merkle-path-length-mismatch:${input.expectedCoinbaseMerklePath.length}:${merklePath.length}`, + totalBytes, + }; + } + for (let i = 0; i < merklePath.length; i++) { + if (!merklePath[i].equals(input.expectedCoinbaseMerklePath[i])) { + return { + valid: false, + errorCode: `merkle-path-mismatch:${i}`, + totalBytes, + }; + } + } + } + + return { + valid: true, + transactions, + totalBytes, + }; + } + + public buildCoinbaseMerklePath(transactionList: Buffer[]): Buffer[] { + const coinbasePlaceholderHash = Buffer.alloc(32); + let level = [ + coinbasePlaceholderHash, + ...transactionList.map(transaction => bitcoinjs.Transaction.fromBuffer(transaction).getHash(false)), + ]; + let index = 0; + const path: Buffer[] = []; + + while (level.length > 1) { + const siblingIndex = index ^ 1; + path.push(Buffer.from(level[siblingIndex] ?? level[index])); + + const nextLevel: Buffer[] = []; + for (let i = 0; i < level.length; i += 2) { + const left = level[i]; + const right = level[i + 1] ?? left; + nextLevel.push(hash256(Buffer.concat([left, right]))); + } + level = nextLevel; + index = Math.floor(index / 2); + } + + return path; + } + + public validateDatumTemplateFastPath(input: DatumTemplateValidationInput): TemplateFastPathValidationResult { + const template = this.getLatestTemplate(); + if (template == null) { + return { valid: false, errorCode: 'template-not-found' }; + } + if (input.prevBlockHash != null && !this.matchesHash(input.prevBlockHash, template.prevHash.toString('hex'))) { + return { valid: false, errorCode: 'prevhash-mismatch', template }; + } + if (input.nBits != null) { + if (input.nBits.length !== 4) { + return { valid: false, errorCode: 'nbits-mismatch', template }; + } + if (input.nBits.readUInt32LE(0) !== template.nBits) { + return { valid: false, errorCode: 'nbits-mismatch', template }; + } + } + if (input.height != null && input.height !== template.height) { + return { valid: false, errorCode: 'height-mismatch', template }; + } + if (input.version != null && !this.isCompatibleVersion(input.version, template.version)) { + return { valid: false, errorCode: 'version-mismatch', template }; + } + if (input.coinbaseValue != null && input.coinbaseValue > template.coinbaseValue) { + return { valid: false, errorCode: 'coinbase-value-too-high', template }; + } + if (input.totalWeight != null && input.totalWeight > template.weightLimit) { + return { valid: false, errorCode: 'weight-limit-exceeded', template }; + } + if (input.totalSize != null && input.totalSize > template.sizeLimit) { + return { valid: false, errorCode: 'size-limit-exceeded', template }; + } + if (input.totalSigops != null && input.totalSigops > template.sigopLimit) { + return { valid: false, errorCode: 'sigop-limit-exceeded', template }; + } + if (input.merkleBranches != null && input.merkleBranches.some(branch => branch.length !== 32)) { + return { valid: false, errorCode: 'bad-merkle-branch', template }; + } + return { valid: true, template }; + } + + public buildBlockFromSolution(input: { + template: TemplateProviderTemplate; + coinbaseTx: Buffer; + version: number; + headerTimestamp: number; + headerNonce: number; + }): bitcoinjs.Block { + const block = new bitcoinjs.Block(); + block.version = input.version; + block.prevHash = Buffer.from(input.template.prevHash); + block.timestamp = input.headerTimestamp; + block.bits = input.template.nBits; + block.nonce = input.headerNonce; + block.transactions = [ + bitcoinjs.Transaction.fromBuffer(input.coinbaseTx), + ...input.template.transactionList.map(tx => bitcoinjs.Transaction.fromBuffer(tx)), + ]; + block.merkleRoot = bitcoinjs.Block.calculateMerkleRoot(block.transactions, false); + try { + block.witnessCommit = bitcoinjs.Block.calculateMerkleRoot(block.transactions, true); + } catch { + block.witnessCommit = null; + } + return block; + } + + public buildBlockFromDeclaredJobSolution(input: { + template: TemplateProviderTemplate; + job: Sv2DeclareMiningJob; + providedTransactions?: Buffer[]; + solution: Sv2PushSolution; + }): bitcoinjs.Block { + const transactionMap = this.buildDeclaredTransactionMap(input.template, input.providedTransactions ?? []); + const coinbaseTx = Buffer.concat([ + input.job.coinbaseTxPrefix, + input.solution.extranonce, + input.job.coinbaseTxSuffix, + ]); + const transactions = [bitcoinjs.Transaction.fromBuffer(coinbaseTx)]; + for (const wtxid of input.job.wtxidList) { + const transaction = this.lookupDeclaredTransaction(transactionMap, wtxid); + if (transaction == null) { + throw new Error('declared-transaction-not-found'); + } + transactions.push(bitcoinjs.Transaction.fromBuffer(transaction)); + } + + const block = new bitcoinjs.Block(); + block.version = input.solution.version; + block.prevHash = Buffer.from(input.template.prevHash); + block.timestamp = input.solution.ntime; + block.bits = input.solution.nBits; + block.nonce = input.solution.nonce; + block.transactions = transactions; + block.merkleRoot = bitcoinjs.Block.calculateMerkleRoot(block.transactions, false); + try { + block.witnessCommit = bitcoinjs.Block.calculateMerkleRoot(block.transactions, true); + } catch { + block.witnessCommit = null; + } + return block; + } + + private buildTransactions(rawTransactions: IBlockTemplateTx[]): TemplateProviderTransaction[] { + return rawTransactions.map((tx, index) => { + const parsed = this.parseTransaction(Buffer.from(tx.data, 'hex'), index); + return { + ...parsed, + txid: tx.txid || parsed.txid, + wtxid: tx.hash || parsed.wtxid, + fee: tx.fee ?? 0, + sigops: tx.sigops ?? 0, + weight: tx.weight ?? 0, + }; + }); + } + + private parseTransaction(data: Buffer, position: number): TemplateProviderTransaction { + const tx = bitcoinjs.Transaction.fromBuffer(data); + return { + position, + data: Buffer.from(data), + txid: tx.getId(), + wtxid: Buffer.from(tx.getHash(true)).reverse().toString('hex'), + fee: 0, + sigops: 0, + weight: 0, + }; + } + + private validateCoinbaseScriptHeight(script: Buffer, height: number): CoinbaseHeightValidationResult { + const expected = this.buildCoinbaseHeightPrefix(height); + if (script.length < expected.length || !script.subarray(0, expected.length).equals(expected)) { + return { valid: false, errorCode: 'invalid-job-param-value-coinbase_tx_prefix' }; + } + return { valid: true }; + } + + private getCoinbaseInputScriptStart(txPrefix: Buffer): number | null { + let cursor = 4; + if (txPrefix.length < cursor + 1) { + return null; + } + + if (txPrefix.length >= 6 && txPrefix[4] === 0x00 && txPrefix[5] !== 0x00) { + cursor = 6; + } + + const inputCount = this.readBitcoinVarInt(txPrefix, cursor); + if (inputCount == null || inputCount.value !== 1n) { + return null; + } + cursor = inputCount.offset; + if (txPrefix.length < cursor + 36) { + return null; + } + cursor += 36; + + const scriptLength = this.readBitcoinVarInt(txPrefix, cursor); + if (scriptLength == null) { + return null; + } + return scriptLength.offset; + } + + private readBitcoinVarInt(buffer: Buffer, offset: number): { value: bigint; offset: number } | null { + if (offset >= buffer.length) { + return null; + } + const first = buffer[offset]; + if (first < 0xfd) { + return { value: BigInt(first), offset: offset + 1 }; + } + if (first === 0xfd) { + if (offset + 3 > buffer.length) { + return null; + } + return { value: BigInt(buffer.readUInt16LE(offset + 1)), offset: offset + 3 }; + } + if (first === 0xfe) { + if (offset + 5 > buffer.length) { + return null; + } + return { value: BigInt(buffer.readUInt32LE(offset + 1)), offset: offset + 5 }; + } + if (offset + 9 > buffer.length) { + return null; + } + return { value: buffer.readBigUInt64LE(offset + 1), offset: offset + 9 }; + } + + private buildDeclaredTransactionMap( + template: TemplateProviderTemplate, + providedTransactions: Buffer[], + ): Map { + const map = new Map(); + for (const tx of template.transactions) { + for (const alias of this.hashAliases(tx.wtxid)) { + map.set(alias, Buffer.from(tx.data)); + } + } + for (const provided of providedTransactions) { + const parsed = this.parseTransaction(provided, -1); + for (const alias of this.hashAliases(parsed.wtxid)) { + map.set(alias, Buffer.from(provided)); + } + } + return map; + } + + private lookupDeclaredTransaction(map: Map, hash: Buffer): Buffer | undefined { + for (const alias of this.hashAliases(hash.toString('hex'))) { + const found = map.get(alias); + if (found != null) { + return found; + } + } + return undefined; + } + + private buildMap( + transactions: TemplateProviderTransaction[], + selector: (tx: TemplateProviderTransaction) => string, + ): Map { + const map = new Map(); + for (const tx of transactions) { + for (const key of this.hashAliases(selector(tx))) { + map.set(key, tx); + } + } + return map; + } + + private lookupWtxid(template: TemplateProviderTemplate, hash: Buffer): TemplateProviderTransaction | undefined { + for (const alias of this.hashAliases(hash.toString('hex'))) { + const found = template.wtxidMap.get(alias); + if (found != null) { + return found; + } + } + return undefined; + } + + private matchesHash(candidate: Buffer, expectedHex: string): boolean { + const aliases = this.hashAliases(expectedHex); + return aliases.includes(candidate.toString('hex').toLowerCase()); + } + + private isCompatibleVersion(candidate: number, templateVersion: number): boolean { + return (((candidate >>> 0) ^ (templateVersion >>> 0)) & ~VERSION_ROLLING_MASK) === 0; + } + + private async validateMempoolPolicy(transactionList: Buffer[]): Promise<{ valid: boolean; errorCode?: string }> { + if (transactionList.length === 0 || this.bitcoinRpcService?.TEST_MEMPOOL_ACCEPT == null) { + return { valid: true }; + } + try { + const results = await this.bitcoinRpcService.TEST_MEMPOOL_ACCEPT(transactionList); + const rejected = results.find(result => !result.allowed); + if (rejected != null) { + return { + valid: false, + errorCode: rejected.rejectReason || rejected.rejectDetails || 'mempool-rejected-transaction', + }; + } + return { valid: true }; + } catch (error) { + return { + valid: false, + errorCode: `mempool-validation-failed:${error.message ?? error}`, + }; + } + } + + private hashAliases(hashHex: string): string[] { + const normalized = hashHex.toLowerCase(); + const reversed = Buffer.from(normalized, 'hex').reverse().toString('hex'); + return normalized === reversed ? [normalized] : [normalized, reversed]; + } + + private targetFromDifficulty(difficulty: number): Buffer { + const maxTarget = (BigInt(0xffff) << BigInt(8 * (0x1d - 3))); + const divisor = BigInt(Math.max(1, Math.floor(difficulty))); + const target = maxTarget / divisor; + const result = Buffer.alloc(32); + let remaining = target; + for (let i = 0; i < 32; i++) { + result[i] = Number(remaining & 0xffn); + remaining >>= 8n; + } + return result; + } + + private cleanup(): void { + const cutoff = Date.now() - this.retentionMs; + for (const [id, template] of this.templates.entries()) { + if (template.createdAt < cutoff) { + this.templates.delete(id); + if (this.latestTemplateId === id) { + this.latestTemplateId = null; + } + } + } + } + + private readPositiveInt(name: string, fallback: number): number { + const parsed = parseInt(process.env[name] ?? '', 10); + return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback; + } +} diff --git a/test/timescale-redis.integration-spec.ts b/test/timescale-redis.integration-spec.ts index d5227a2..a6ce0d2 100644 --- a/test/timescale-redis.integration-spec.ts +++ b/test/timescale-redis.integration-spec.ts @@ -6,6 +6,7 @@ import { UserAgentReportService } from '../src/ORM/_views/user-agent-report/user import { UserAgentReportView } from '../src/ORM/_views/user-agent-report/user-agent-report.view'; import { ClientStatisticsService } from '../src/ORM/client-statistics/client-statistics.service'; import { ClientEntity } from '../src/ORM/client/client.entity'; +import { PayoutSnapshotService } from '../src/ORM/payout-snapshot/payout-snapshot.service'; import { ShareAccountingService } from '../src/ORM/share-accounting/share-accounting.service'; import { RedisMessagingService } from '../src/services/redis-messaging.service'; @@ -42,9 +43,14 @@ describe('TimescaleDB and Redis integration', () => { }); beforeEach(async () => { + await dataSource.query(`DELETE FROM payout_history`); + await dataSource.query(`DELETE FROM payout_snapshot_entry`); + await dataSource.query(`DELETE FROM payout_snapshot`); + await dataSource.query(`DELETE FROM payout_balance`); await dataSource.query(`DELETE FROM share_rollup_batch_summary`); await dataSource.query(`DELETE FROM share_rollup_batch`); await dataSource.query(`DELETE FROM accepted_share_entity`); + await dataSource.query(`DELETE FROM blocks_entity`); await dataSource.query(`DELETE FROM client_entity`); await dataSource.query(`REFRESH MATERIALIZED VIEW user_agent_report_view`); await redisMessagingService.clearClientPresence(); @@ -117,6 +123,22 @@ describe('TimescaleDB and Redis integration', () => { summary_table: 'share_rollup_batch_summary', finalized_index: '"IDX_share_rollup_batch_finalized_end"', }); + + const payoutObjects = await dataSource.query(` + SELECT + to_regclass('public.payout_snapshot') AS snapshot_table, + to_regclass('public.payout_snapshot_entry') AS entry_table, + to_regclass('public.payout_balance') AS balance_table, + to_regclass('public.payout_history') AS history_table, + to_regclass('public."IDX_payout_snapshot_latest"') AS latest_index + `); + expect(payoutObjects[0]).toEqual({ + snapshot_table: 'payout_snapshot', + entry_table: 'payout_snapshot_entry', + balance_table: 'payout_balance', + history_table: 'payout_history', + latest_index: '"IDX_payout_snapshot_latest"', + }); }); it('should persist accepted shares and refresh the 10 minute aggregate', async () => { @@ -299,6 +321,120 @@ describe('TimescaleDB and Redis integration', () => { expect(rawRows[0].count).toBe(2); }); + it('should create payout snapshots from finalized share rollup batches', async () => { + process.env.PAYOUT_SNAPSHOT_ENABLED = 'true'; + process.env.PAYOUT_METHOD = 'pplns'; + process.env.PAYOUT_WINDOW_FACTOR = '4'; + process.env.PAYOUT_MAX_COINBASE_OUTPUTS = '10'; + process.env.PAYOUT_MIN_OUTPUT_SATS = '0'; + process.env.PAYOUT_FEE_PERCENT = '0'; + process.env.PAYOUT_FEE_ADDRESS = ''; + process.env.PAYOUT_COINBASE_WEIGHT_BUDGET = '50000'; + const accountingService = new ShareAccountingService(dataSource.getRepository(AcceptedShareEntity)); + const payoutSnapshotService = new PayoutSnapshotService(dataSource); + const acceptedAt = new Date('2026-06-07T12:30:00Z'); + const clients = await Promise.all([ + dataSource.getRepository(ClientEntity).save({ + address: 'tb1qumezefzdeqqwn5zfvgdrhxjzc5ylr39uhuxcz4', + clientName: 'payout-worker-a', + sessionId: '57a6f098', + userAgent: 'integration-test', + startTime: new Date(), + bestDifficulty: 0, + hashRate: 0, + }), + dataSource.getRepository(ClientEntity).save({ + address: 'tb1q99n3pu025yyu0jlywpmwzalyhm36tg5u37w20d', + clientName: 'payout-worker-b', + sessionId: '67a6f098', + userAgent: 'integration-test', + startTime: new Date(), + bestDifficulty: 0, + hashRate: 0, + }), + ]); + + for (const [index, client] of clients.entries()) { + await accountingService.recordAcceptedShare({ + protocol: 'sv1', + acceptedAt: new Date(acceptedAt.getTime() + index), + address: client.address, + clientName: client.clientName, + sessionId: client.sessionId, + clientId: client.id, + jobId: `payout-${index}`, + jobTemplateId: 'payout-template', + blockHeight: 900020, + creditedDifficulty: index === 0 ? 64 : 32, + submissionDifficulty: index === 0 ? 128 : 64, + networkDifficulty: 100000, + nonce: `payout-nonce-${index}`, + ntime: '64b3f3ec', + version: '20000000', + extraNonce2: `c70800000000000${index}`, + isBlockCandidate: false, + blockSubmissionResult: null, + }); + } + + await expect(accountingService.processPendingShareRollupBatch()).resolves.toEqual(expect.objectContaining({ + processed: true, + acceptedShareCount: 2, + creditedDifficulty: 96, + })); + + const snapshot = await payoutSnapshotService.createSnapshotForTemplate({ + blockHeight: 900020, + coinbaseValueSats: 100000, + networkDifficulty: 100000, + }); + + expect(snapshot).toEqual(expect.objectContaining({ + method: 'pplns', + blockHeight: 900020, + totalCreditedDifficulty: 96, + totalAcceptedShareCount: 2, + eligibleAddressCount: 2, + includedOutputCount: 2, + distributedSats: '100000', + })); + expect(snapshot.payoutOutputs).toEqual([ + { address: clients[0].address, amountSats: 66667, percent: 66.667 }, + { address: clients[1].address, amountSats: 33333, percent: 33.333 }, + ]); + + await expect(payoutSnapshotService.finalizeSnapshotForBlock({ + payoutSnapshotId: snapshot.id, + blockHeight: 900020, + blockSubmissionResult: 'SUCCESS!', + })).resolves.toEqual(expect.objectContaining({ + finalized: true, + payoutSnapshotId: snapshot.id, + historyRows: 2, + balanceRows: 2, + })); + + const historyRows = await dataSource.query(` + SELECT "address", "paidSats"::int AS "paidSats", "rowType" + FROM payout_history + WHERE "blockHeight" = $1 + ORDER BY "paidSats" DESC + `, [900020]); + expect(historyRows).toEqual([ + { address: clients[0].address, paidSats: 66667, rowType: 'coinbase' }, + { address: clients[1].address, paidSats: 33333, rowType: 'coinbase' }, + ]); + + await expect(payoutSnapshotService.finalizeSnapshotForBlock({ + payoutSnapshotId: snapshot.id, + blockHeight: 900020, + blockSubmissionResult: 'SUCCESS!', + })).resolves.toEqual(expect.objectContaining({ + finalized: false, + reason: 'already-finalized', + })); + }); + it('should exclude soft-deleted clients from the user-agent report', async () => { const userAgent = `integration-reconnect-${Date.now()}`; const repository = dataSource.getRepository(ClientEntity);