Migrate postgresql improvements to sqlite

2026-09-29 09:05:06 -07:00 · 2026-06-05 10:48:42 -04:00
parent b971e9ce4c
commit 95565eeec4
32 changed files with 6491 additions and 13481 deletions
@@ -1,7 +1,7 @@
 import { ConfigService } from '@nestjs/config';
 import * as bitcoinjs from 'bitcoinjs-lib';
 import { plainToInstance } from 'class-transformer';
-import { validate, ValidatorOptions } from 'class-validator';
+import { validate, ValidationError, ValidatorOptions } from 'class-validator';
 import * as crypto from 'crypto';
 import { Socket } from 'net';
 import { firstValueFrom, Subscription } from 'rxjs';
@@ -24,25 +24,30 @@ import { ConfigurationMessage } from './stratum-messages/ConfigurationMessage';
 import { MiningSubmitMessage } from './stratum-messages/MiningSubmitMessage';
 import { StratumErrorMessage } from './stratum-messages/StratumErrorMessage';
 import { SubscriptionMessage } from './stratum-messages/SubscriptionMessage';
+import { EXTRANONCE1_SIZE_BYTES } from './stratum.constants';
 import { SuggestDifficulty } from './stratum-messages/SuggestDifficultyMessage';
 import { StratumV1ClientStatistics } from './StratumV1ClientStatistics';
 import { ExternalSharesService } from '../services/external-shares.service';
-import { DifficultyUtils } from '../utils/difficulty.utils';

+const TRUE_DIFF_ONE = 2.695953529101131e67;
+const BLOCKED_USER_AGENT_LOG_INTERVAL_MS = 60 * 1000;
+const VALIDATION_ERROR_LOG_INTERVAL_MS = 60 * 1000;

 export class StratumV1Client {
+    private static blockedUserAgentLogState = new Map<string, { nextLogAt: number, suppressed: number }>();
+    private static validationErrorLogState = new Map<string, { nextLogAt: number, suppressed: number, sample: string }>();

    private clientSubscription: SubscriptionMessage;
    private clientConfiguration: ConfigurationMessage;
    private clientAuthorization: AuthorizationMessage;
    private clientSuggestedDifficulty: SuggestDifficulty;
    private stratumSubscription: Subscription;
-    private backgroundWork: NodeJS.Timer[] = [];
+    private backgroundWork: NodeJS.Timeout[] = [];

    private statistics: StratumV1ClientStatistics;
    private stratumInitialized = false;
    private usedSuggestedDifficulty = false;
-    private sessionDifficulty: number = 16384;
+    private sessionDifficulty: number = 100000;

    private entity: ClientEntity;
    private creatingEntity: Promise<void>;
@@ -53,6 +58,8 @@ export class StratumV1Client {
    public hashRate: number = 0;

    private buffer: string = '';
+    private connectionClosed = false;
+    private lastSentMiningJobTimestamp: number = null;

    private miningSubmissionHashes = new Set<string>()

@@ -74,16 +81,19 @@ export class StratumV1Client {
            let lines = this.buffer.split('\n');
            this.buffer = lines.pop() || ''; // Save the last part of the data (incomplete line) to the buffer

-            lines
-                .filter(m => m.length > 0)
-                .forEach(async (m) => {
+            (async () => {
+                for (const m of lines.filter(l => l.length > 0)) {
+                    if (this.connectionClosed || this.socket.destroyed || this.socket.writableEnded) {
+                        break;
+                    }
                    try {
                        await this.handleMessage(m);
                    } catch (e) {
                        await this.socket.end();
                        console.error(e);
                    }
-                });
+                }
+            })();
        });


@@ -105,10 +115,8 @@ export class StratumV1Client {
    }

    private getRandomHexString() {
-        const randomBytes = crypto.randomBytes(4); // 4 bytes = 32 bits
-        const randomNumber = randomBytes.readUInt32BE(0); // Convert bytes to a 32-bit unsigned integer
-        const hexString = randomNumber.toString(16).padStart(8, '0'); // Convert to hex and pad with zeros
-        return hexString;
+        const randomBytes = crypto.randomBytes(EXTRANONCE1_SIZE_BYTES);
+        return randomBytes.toString('hex');
    }


@@ -142,6 +150,11 @@ export class StratumV1Client {
                const errors = await validate(subscriptionMessage, validatorOptions);

                if (errors.length === 0) {
+                    if (this.isBlockedUserAgent(subscriptionMessage.userAgent)) {
+                        this.logBlockedUserAgent(subscriptionMessage.userAgent);
+                        this.closeSocket();
+                        return;
+                    }

                    if (this.sessionStart == null) {
                        this.sessionStart = new Date();
@@ -225,6 +238,9 @@ export class StratumV1Client {

                if (errors.length === 0) {
                    this.clientAuthorization = authorizationMessage;
+                    if (this.clientSuggestedDifficulty == null && this.clientAuthorization.startingDiff != null && this.clientAuthorization.startingDiff > this.sessionDifficulty) {
+                        this.sessionDifficulty = this.clientAuthorization.startingDiff;
+                    }
                    const success = await this.write(JSON.stringify(this.clientAuthorization.response()) + '\n');
                    if (!success) {
                        return;
@@ -318,17 +334,18 @@ export class StratumV1Client {


                } else {
-                    console.log('Mining Submit validation error');
+                    this.logValidationError('Mining Submit validation error', errors);
                    const err = new StratumErrorMessage(
                        miningSubmitMessage.id,
                        eStratumErrorCode.OtherUnknown,
                        'Mining Submit validation error',
                        errors).response();
-                    console.error(err);
                    const success = await this.write(err);
                    if (!success) {
                        return;
                    }
+                    this.closeSocket();
+                    return;
                }
                break;
            }
@@ -353,6 +370,12 @@ export class StratumV1Client {
    private async initStratum() {
        this.stratumInitialized = true;

+        if (this.isBlockedUserAgent(this.clientSubscription.userAgent)) {
+            this.logBlockedUserAgent(this.clientSubscription.userAgent);
+            this.closeSocket();
+            return;
+        }
+
        switch (this.clientSubscription.userAgent) {
            case 'cpuminer': {
                this.sessionDifficulty = 0.1;
@@ -438,6 +461,7 @@ export class StratumV1Client {
        if (!success) {
            return;
        }
+        this.lastSentMiningJobTimestamp = jobTemplate.block.timestamp;


        //console.log(`Sent new job to ${this.clientAuthorization.worker}.${this.extraNonceAndSessionId}. (clearJobs: ${jobTemplate.blockData.clearJobs}, fee?: ${!this.noFee})`)
@@ -445,46 +469,28 @@ export class StratumV1Client {
    }


-    private async handleMiningSubmission(submission: MiningSubmitMessage) {
+    private async ensureClientEntity() {
+        if (this.entity != null) {
+            return;
+        }

-        if (this.entity == null) {
-            if (this.creatingEntity == null) {
-                this.creatingEntity = new Promise(async (resolve, reject) => {
-                    try {
-                        this.entity = await this.clientService.insert({
-                            sessionId: this.extraNonceAndSessionId,
-                            address: this.clientAuthorization.address,
-                            clientName: this.clientAuthorization.worker,
-                            userAgent: this.clientSubscription.userAgent,
-                            startTime: new Date(),
-                            bestDifficulty: 0
-                        });
-                    } catch (e) {
-                        reject(e);
-                    }
-                    resolve();
+        if (this.creatingEntity == null) {
+            this.creatingEntity = (async () => {
+                this.entity = await this.clientService.insert({
+                    sessionId: this.extraNonceAndSessionId,
+                    address: this.clientAuthorization.address,
+                    clientName: this.clientAuthorization.worker,
+                    userAgent: this.clientSubscription.userAgent,
+                    startTime: new Date(),
+                    bestDifficulty: 0
                });
-                await this.creatingEntity;
-
-            } else {
-                await this.creatingEntity;
-            }
+            })();
        }

-        const submissionHash = submission.hash();
-        if(this.miningSubmissionHashes.has(submissionHash)){
-            const err = new StratumErrorMessage(
-                submission.id,
-                eStratumErrorCode.DuplicateShare,
-                'Duplicate share').response();
-            const success = await this.write(err);
-            if (!success) {
-                return false;
-            }
-            return false;
-        }else{
-            this.miningSubmissionHashes.add(submissionHash);
-        }
+        await this.creatingEntity;
+    }
+
+    private async handleMiningSubmission(submission: MiningSubmitMessage) {

        const job = this.stratumV1JobsService.getJobById(submission.jobId);

@@ -503,24 +509,73 @@ export class StratumV1Client {
        }
        const jobTemplate = this.stratumV1JobsService.getJobTemplateById(job.jobTemplateId);

-        const updatedJobBlock = job.copyAndUpdateBlock(
+        if (jobTemplate == null) {
+            const err = new StratumErrorMessage(
+                submission.id,
+                eStratumErrorCode.JobNotFound,
+                'Job Template not found').response();
+            //console.log(err);
+            const success = await this.write(err);
+            if (!success) {
+                return false;
+            }
+            return false;
+        }
+
+        const submissionHash = [
+            submission.jobId,
+            submission.extraNonce2,
+            submission.ntime,
+            submission.nonce,
+            submission.versionMask ?? ''
+        ].join(':');
+        if (this.miningSubmissionHashes.has(submissionHash)) {
+            const err = new StratumErrorMessage(
+                submission.id,
+                eStratumErrorCode.DuplicateShare,
+                'Duplicate share').response();
+            const success = await this.write(err);
+            if (!success) {
+                return false;
+            }
+            return false;
+        } else {
+            this.miningSubmissionHashes.add(submissionHash);
+        }
+
+        const versionMask = parseInt(submission.versionMask, 16);
+        const nonce = parseInt(submission.nonce, 16);
+        const timestamp = parseInt(submission.ntime, 16);
+
+        const header = job.buildHeaderBuffer(
            jobTemplate,
-            parseInt(submission.versionMask, 16),
-            parseInt(submission.nonce, 16),
+            versionMask,
+            nonce,
            this.extraNonceAndSessionId,
            submission.extraNonce2,
-            parseInt(submission.ntime, 16)
+            timestamp
        );
-        const header = updatedJobBlock.toBuffer(true);
-        const { submissionDifficulty } = DifficultyUtils.calculateDifficulty(header);
+        const { submissionDifficulty } = this.calculateDifficulty(header);

        //console.log(`DIFF: ${submissionDifficulty} of ${this.sessionDifficulty} from ${this.clientAuthorization.worker + '.' + this.extraNonceAndSessionId}`);


        if (submissionDifficulty >= this.sessionDifficulty) {
+            const success = await this.write(JSON.stringify(submission.response()) + '\n');
+            if (!success) {
+                return false;
+            }

            if (submissionDifficulty >= jobTemplate.blockData.networkDifficulty) {
                console.log('!!! BLOCK FOUND !!!');
+                const updatedJobBlock = job.copyAndUpdateBlock(
+                    jobTemplate,
+                    versionMask,
+                    nonce,
+                    this.extraNonceAndSessionId,
+                    submission.extraNonce2,
+                    timestamp
+                );
                const blockHex = updatedJobBlock.toHex(false);
                const result = await this.bitcoinRpcService.SUBMIT_BLOCK(blockHex);
                await this.blocksService.save({
@@ -537,6 +592,7 @@ export class StratumV1Client {
                    await this.addressSettingsService.resetBestDifficultyAndShares();
                }
            }
+            await this.ensureClientEntity();
            try {
                await this.statistics.addShares(this.entity, this.sessionDifficulty);
                const now = new Date();
@@ -551,11 +607,9 @@ export class StratumV1Client {
            }

            if (submissionDifficulty > this.entity.bestDifficulty) {
-                await this.clientService.updateBestDifficulty(this.extraNonceAndSessionId, submissionDifficulty);
+                await this.clientService.updateBestDifficultyIfHigher(this.extraNonceAndSessionId, submissionDifficulty);
                this.entity.bestDifficulty = submissionDifficulty;
-                if (submissionDifficulty > (await this.addressSettingsService.getSettings(this.clientAuthorization.address, true)).bestDifficulty) {
-                    await this.addressSettingsService.updateBestDifficulty(this.clientAuthorization.address, submissionDifficulty, this.entity.userAgent);
-                }
+                await this.addressSettingsService.updateBestDifficultyIfHigher(this.clientAuthorization.address, submissionDifficulty, this.entity.userAgent);
            }


@@ -587,7 +641,7 @@ export class StratumV1Client {
        }

        //await this.checkDifficulty();
-        return true;
+        return false;

    }

@@ -611,13 +665,130 @@ export class StratumV1Client {
            await this.socket.write(data);

            const jobTemplate = await firstValueFrom(this.stratumV1JobsService.newMiningJob$);
-            // we need to clear the jobs so that the difficulty set takes effect. Otherwise the different miner implementations can cause issues
-            jobTemplate.blockData.clearJobs = true;
-            await this.sendNewMiningJob(jobTemplate);
+            const nextTimestamp = Math.max(
+                jobTemplate.block.timestamp,
+                Math.floor(Date.now() / 1000),
+                (this.lastSentMiningJobTimestamp ?? 0) + 1
+            );
+            // We need to clear jobs so the difficulty takes effect, but avoid mutating or
+            // re-sending the shared cached template with byte-identical work.
+            const refreshedJobTemplate: IJobTemplate = {
+                ...jobTemplate,
+                block: Object.assign(new bitcoinjs.Block(), jobTemplate.block, {
+                    timestamp: nextTimestamp
+                }),
+                blockData: { ...jobTemplate.blockData, clearJobs: true }
+            };
+            await this.sendNewMiningJob(refreshedJobTemplate);

        }
    }

+    private calculateDifficulty(header: Buffer): { submissionDifficulty: number, submissionHash: string } {
+
+        const hashResult = bitcoinjs.crypto.hash256(header);
+
+        const target = this.le256todouble(hashResult);
+        const submissionDifficulty = target === 0 ? Number.POSITIVE_INFINITY : TRUE_DIFF_ONE / target;
+        return { submissionDifficulty, submissionHash: hashResult.toString('hex') };
+    }
+
+
+    private le256todouble(target: Buffer): number {
+
+        let number = 0;
+        for (let i = target.length - 1; i >= 0; i--) {
+            number = number * 256 + target[i];
+        }
+
+        return number;
+    }
+
+    private isBlockedUserAgent(userAgent: string): boolean {
+        const blockedUserAgents = this.configService.get<string>('NON_COMPLIANT_USER_AGENTS')
+            || this.configService.get<string>('BLOCKED_USER_AGENTS')
+            || this.configService.get<string>('COMPLIANT_HEADERS');
+        if (!blockedUserAgents || blockedUserAgents.trim() === '') {
+            return false;
+        }
+
+        const blockedList = blockedUserAgents.split(',').map(ua => ua.trim().toLowerCase());
+        const userAgentLower = userAgent.toLowerCase();
+
+        return blockedList.some(blocked => blocked.length > 0 && userAgentLower.includes(blocked));
+    }
+
+    private logBlockedUserAgent(userAgent: string) {
+        const now = Date.now();
+        const logState = StratumV1Client.blockedUserAgentLogState.get(userAgent);
+
+        if (logState != null && now < logState.nextLogAt) {
+            logState.suppressed += 1;
+            return;
+        }
+
+        const suppressed = logState?.suppressed ?? 0;
+        const suffix = suppressed > 0 ? ` (${suppressed} similar connections suppressed)` : '';
+        console.log(`Blocked non-compliant connection from userAgent: ${userAgent}${suffix}`);
+        StratumV1Client.blockedUserAgentLogState.set(userAgent, {
+            nextLogAt: now + BLOCKED_USER_AGENT_LOG_INTERVAL_MS,
+            suppressed: 0
+        });
+    }
+
+    private logValidationError(label: string, errors: ValidationError[]) {
+        const now = Date.now();
+        const signature = this.getValidationErrorSignature(errors);
+        const sample = this.getValidationErrorSample(errors);
+        const key = `${label}:${signature}`;
+        const logState = StratumV1Client.validationErrorLogState.get(key);
+
+        if (logState != null && now < logState.nextLogAt) {
+            logState.suppressed += 1;
+            return;
+        }
+
+        const suppressed = logState?.suppressed ?? 0;
+        const suffix = suppressed > 0 ? ` (${suppressed} similar validation errors suppressed)` : '';
+        console.warn(`${label}: ${signature}${sample}${suffix}`);
+        StratumV1Client.validationErrorLogState.set(key, {
+            nextLogAt: now + VALIDATION_ERROR_LOG_INTERVAL_MS,
+            suppressed: 0,
+            sample
+        });
+    }
+
+    private getValidationErrorSignature(errors: ValidationError[]): string {
+        if (errors.length === 0) {
+            return 'unknown';
+        }
+
+        return errors.map(error => {
+            const constraints = Object.keys(error.constraints ?? {}).sort().join('|') || 'invalid';
+            return `${error.property}:${constraints}`;
+        }).join(';');
+    }
+
+    private getValidationErrorSample(errors: ValidationError[]): string {
+        const values = errors
+            .map(error => error.value)
+            .filter(value => value != null)
+            .map(value => String(value).replace(/[\r\n]/g, '').slice(0, 64));
+
+        if (values.length === 0) {
+            return '';
+        }
+
+        return ` sample=${values.join(',')}`;
+    }
+
+    private closeSocket() {
+        this.connectionClosed = true;
+        if (!this.socket.destroyed) {
+            this.socket.destroy();
+        }
+    }
+
    private async write(message: string): Promise<boolean> {
        try {
            if (!this.socket.destroyed && !this.socket.writableEnded) {