From 8f6eeaba733e73742b96c1b58bb390105d8115c4 Mon Sep 17 00:00:00 2001 From: Seth For Privacy Date: Tue, 12 Dec 2023 20:11:36 -0500 Subject: [PATCH 01/15] Dockerfile improvements and automatic Github Actions builds (#14) * Initial GA push * Fix Actions * Remove test runs * Update Trivy scanner to latest syntax * Remove unnecessary build stage * Switch to slim base images * Minor Dockerfile improvements * Fix slim builds * Fix slim builds round 2 --- .github/workflows/build-image-on-push.yml | 35 ++++++++++++ .github/workflows/trivy-analysis.yml | 37 +++++++++++++ .github/workflows/update-base-image.yml | 61 +++++++++++++++++++++ .github/workflows/update-image-on-push.yml | 64 ++++++++++++++++++++++ Dockerfile | 22 +++++--- 5 files changed, 212 insertions(+), 7 deletions(-) create mode 100644 .github/workflows/build-image-on-push.yml create mode 100644 .github/workflows/trivy-analysis.yml create mode 100644 .github/workflows/update-base-image.yml create mode 100644 .github/workflows/update-image-on-push.yml diff --git a/.github/workflows/build-image-on-push.yml b/.github/workflows/build-image-on-push.yml new file mode 100644 index 0000000..27b6e92 --- /dev/null +++ b/.github/workflows/build-image-on-push.yml @@ -0,0 +1,35 @@ +name: "Test build of image when Dockerfile is changed" + +on: + push: + branches-ignore: + - master + pull_request: + workflow_dispatch: + +jobs: + rebuild-container: + name: "Build image with cache" + runs-on: ubuntu-latest + steps: + - + name: Set up QEMU + uses: docker/setup-qemu-action@v3.0.0 + with: + platforms: linux/arm64 + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3.0.0 + - + name: Checkout repository + uses: actions/checkout@v4 + - + name: Build image + id: docker_build_amd64 + uses: docker/build-push-action@v5.1.0 + with: + push: false + load: true + platforms: linux/amd64 + tags: public-pool:amd64 + cache-from: type=registry,ref=${{ secrets.DOCKER_USERNAME }}/public-pool:latest diff --git a/.github/workflows/trivy-analysis.yml b/.github/workflows/trivy-analysis.yml new file mode 100644 index 0000000..ae45fe2 --- /dev/null +++ b/.github/workflows/trivy-analysis.yml @@ -0,0 +1,37 @@ +name: Build and scan container for vulnerabilities with Trivy + +on: + push: + pull_request: + schedule: + - cron: '22 14 * * 0' + workflow_dispatch: + +jobs: + build: + name: Build and scan images + runs-on: ubuntu-latest + steps: + - + name: Checkout code + uses: actions/checkout@v4 + - + name: Build image from Dockerfile + uses: docker/build-push-action@v5.1.0 + with: + push: false + load: true + tags: ${{ secrets.DOCKER_USERNAME }}/public-pool:latest + - + name: Run Trivy vulnerability scanner against "latest" image + uses: aquasecurity/trivy-action@master + with: + image-ref: '${{ secrets.DOCKER_USERNAME }}/public-pool:latest' + format: 'sarif' + output: 'trivy-results.sarif' + severity: 'CRITICAL,HIGH' + - + name: Upload "latest" Trivy scan results to GitHub Security tab + uses: github/codeql-action/upload-sarif@v2 + with: + sarif_file: 'trivy-results.sarif' diff --git a/.github/workflows/update-base-image.yml b/.github/workflows/update-base-image.yml new file mode 100644 index 0000000..365e43a --- /dev/null +++ b/.github/workflows/update-base-image.yml @@ -0,0 +1,61 @@ +name: "Update image and push to Github Packages and Docker Hub weekly" + +on: + schedule: + - cron: "0 12 * * 1" # Run every Monday at noon. + workflow_dispatch: + +jobs: + rebuild-container: + name: "Rebuild Container with the latest base image" + runs-on: ubuntu-latest + steps: + - + name: Prepare outputs + id: prep + run: | + echo "::set-output name=created::$(date -u +'%Y-%m-%dT%H:%M:%SZ')" + - + name: Set up QEMU + uses: docker/setup-qemu-action@v3.0.0 + with: + platforms: linux/arm64 + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3.0.0 + - + name: Login to GitHub Container Registry + uses: docker/login-action@v3.0.0 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + - + name: Login to DockerHub + uses: docker/login-action@v3.0.0 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + - + name: Checkout repository + uses: actions/checkout@v4 + - + name: Get short SHA + id: get_short_sha + run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT + - + name: Build and push to Docker Hub and GitHub Packages Docker Registry + id: docker_build + uses: docker/build-push-action@v5.1.0 + with: + push: true + platforms: linux/amd64,linux/arm64 + tags: | + ghcr.io/${{ github.repository_owner }}/public-pool:latest + ghcr.io/${{ github.repository_owner }}/public-pool:${{ steps.get_short_sha.outputs.sha_short }} + ${{ secrets.DOCKER_USERNAME }}/public-pool:latest + ${{ secrets.DOCKER_USERNAME }}/public-pool:${{ steps.get_short_sha.outputs.sha_short }} + labels: | + org.opencontainers.image.source=${{ github.event.repository.html_url }} + org.opencontainers.image.created=${{ steps.prep.outputs.created }} + org.opencontainers.image.revision=${{ github.sha }} diff --git a/.github/workflows/update-image-on-push.yml b/.github/workflows/update-image-on-push.yml new file mode 100644 index 0000000..732ad40 --- /dev/null +++ b/.github/workflows/update-image-on-push.yml @@ -0,0 +1,64 @@ +name: "Update image when Dockerfile is changed" + +on: + push: + branches: + - master + workflow_dispatch: + +jobs: + rebuild-container: + name: "Rebuild Container with the latest base image" + runs-on: ubuntu-latest + steps: + - + name: Prepare outputs + id: prep + run: | + echo "::set-output name=created::$(date -u +'%Y-%m-%dT%H:%M:%SZ')" + - + name: Set up QEMU + uses: docker/setup-qemu-action@v3.0.0 + with: + platforms: linux/arm64 + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3.0.0 + - + name: Login to GitHub Container Registry + uses: docker/login-action@v3.0.0 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + - + name: Login to DockerHub + uses: docker/login-action@v3.0.0 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + - + name: Checkout repository + uses: actions/checkout@v4 + - + name: Get short SHA + id: get_short_sha + run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT + - + name: Build and push to Docker Hub and GitHub Packages Docker Registry + uses: docker/build-push-action@v5.1.0 + id: docker_build_push + with: + push: true + platforms: linux/amd64,linux/arm64 + tags: | + ghcr.io/${{ github.repository_owner }}/public-pool:latest + ghcr.io/${{ github.repository_owner }}/public-pool:${{ steps.get_short_sha.outputs.sha_short }} + ${{ secrets.DOCKER_USERNAME }}/public-pool:latest + ${{ secrets.DOCKER_USERNAME }}/public-pool:${{ steps.get_short_sha.outputs.sha_short }} + labels: | + org.opencontainers.image.source=${{ github.event.repository.html_url }} + org.opencontainers.image.created=${{ steps.prep.outputs.created }} + org.opencontainers.image.revision=${{ github.sha }} + cache-from: type=registry,ref=${{ secrets.DOCKER_USERNAME }}/public-pool:latest + cache-to: type=inline diff --git a/Dockerfile b/Dockerfile index 894bdbd..beaeb21 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,27 +2,35 @@ # Docker build environment # ############################ -FROM node:18.16.1-bookworm AS build +FROM node:18.16.1-bookworm-slim AS build + +# Upgrade all packages and install dependencies +RUN apt-get update \ + && apt-get upgrade -y +RUN DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + python3 \ + build-essential \ + && apt clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* WORKDIR /build COPY . . -RUN npm i -RUN npm run build +# Build Public Pool using NPM +RUN npm i && npm run build ############################ # Docker final environment # ############################ -FROM node:18.16.1-bookworm +FROM node:18.16.1-bookworm-slim -EXPOSE 3333 -EXPOSE 3334 -EXPOSE 8332 +# Expose ports for Stratum and Bitcoin RPC +EXPOSE 3333 3334 8332 WORKDIR /public-pool +# Copy built binaries into the final image COPY --from=build /build . #COPY .env.example .env From 38fda85f63f1494da9038aa5f2c8928aa0527e76 Mon Sep 17 00:00:00 2001 From: Ben Date: Sat, 23 Dec 2023 19:30:05 -0500 Subject: [PATCH 02/15] back port some changes from postgres --- .../address-settings.service.ts | 4 +-- .../client-statistics.entity.ts | 13 ++++---- .../client-statistics.service.ts | 24 ++++++++++---- src/app.controller.ts | 32 +++++++++++++------ src/controllers/client/client.controller.ts | 2 +- src/models/StratumV1Client.ts | 6 ++-- src/models/StratumV1ClientStatistics.ts | 7 ++-- 7 files changed, 55 insertions(+), 33 deletions(-) diff --git a/src/ORM/address-settings/address-settings.service.ts b/src/ORM/address-settings/address-settings.service.ts index da55722..ec3baab 100644 --- a/src/ORM/address-settings/address-settings.service.ts +++ b/src/ORM/address-settings/address-settings.service.ts @@ -14,9 +14,9 @@ export class AddressSettingsService { } - public async getSettings(address: string) { + public async getSettings(address: string, createIfNotFound: boolean) { const settings = await this.addressSettingsRepository.findOne({ where: { address } }); - if (settings == null) { + if (createIfNotFound == true && settings == null) { return await this.createNew(address); } return settings; diff --git a/src/ORM/client-statistics/client-statistics.entity.ts b/src/ORM/client-statistics/client-statistics.entity.ts index 111a60e..115cc26 100644 --- a/src/ORM/client-statistics/client-statistics.entity.ts +++ b/src/ORM/client-statistics/client-statistics.entity.ts @@ -1,15 +1,16 @@ -import { Column, Entity, Index, PrimaryColumn } from 'typeorm'; +import { Column, Entity, Index, PrimaryGeneratedColumn } from 'typeorm'; import { TrackedEntity } from '../utils/TrackedEntity.entity'; @Entity({ withoutRowid: true }) -//Index for the heartbeat update +//Index for getHashRateForSession +@Index(["address", "clientName", "sessionId"]) +//Index for statistics save @Index(["address", "clientName", "sessionId", "time"]) export class ClientStatisticsEntity extends TrackedEntity { - @PrimaryColumn({ length: 64, type: 'varchar' }) - submissionHash: string; - + @PrimaryGeneratedColumn() + id: number; @Column({ length: 62, type: 'varchar' }) address: string; @@ -31,6 +32,4 @@ export class ClientStatisticsEntity extends TrackedEntity { acceptedCount: number; - - } \ No newline at end of file diff --git a/src/ORM/client-statistics/client-statistics.service.ts b/src/ORM/client-statistics/client-statistics.service.ts index 028ce48..2dfdfb4 100644 --- a/src/ORM/client-statistics/client-statistics.service.ts +++ b/src/ORM/client-statistics/client-statistics.service.ts @@ -18,28 +18,38 @@ export class ClientStatisticsService { } public async save(clientStatistic: Partial) { - const res1 = await this.clientStatisticsRepository.createQueryBuilder() + // Attempt to update the existing record + const updateResult = await this.clientStatisticsRepository + .createQueryBuilder() .update(ClientStatisticsEntity) .set({ - shares: () => `"shares" + ${clientStatistic.shares}`, // Use the actual value of shares here + shares: () => `"shares" + :sharesIncrement`, acceptedCount: () => `"acceptedCount" + 1` }) - .where('address = :address AND clientName = :clientName AND sessionId = :sessionId AND time = :time', { address: clientStatistic.address, clientName: clientStatistic.clientName, sessionId: clientStatistic.sessionId, time: clientStatistic.time }) + .where('address = :address AND clientName = :clientName AND sessionId = :sessionId AND time = :time', { + address: clientStatistic.address, + clientName: clientStatistic.clientName, + sessionId: clientStatistic.sessionId, + time: clientStatistic.time, + sharesIncrement: clientStatistic.shares + }) .execute(); - if (res1.affected == 0) { + // Check if the update affected any rows + if (updateResult.affected === 0) { + // If no rows were updated, insert a new record await this.clientStatisticsRepository.insert(clientStatistic); } } public async deleteOldStatistics() { - const eightDays = new Date(Date.now() - 8 * 24 * 60 * 60 * 1000); + const oneDayAgo = new Date(Date.now() - 24 * 60 * 60 * 1000); return await this.clientStatisticsRepository .createQueryBuilder() .delete() .from(ClientStatisticsEntity) - .where('time < :time', { time: eightDays.getTime() }) + .where('time < :time', { time: oneDayAgo.getTime() }) .execute(); } @@ -60,7 +70,7 @@ export class ClientStatisticsService { ORDER BY time LIMIT 144; - + `; const result: any[] = await this.clientStatisticsRepository.query(query); diff --git a/src/app.controller.ts b/src/app.controller.ts index ff3eb5f..a46ffc4 100644 --- a/src/app.controller.ts +++ b/src/app.controller.ts @@ -21,17 +21,31 @@ export class AppController { private readonly bitcoinRpcService: BitcoinRpcService ) { } - @Get('info') public async info() { + + const CACHE_KEY = 'SITE_INFO'; + const cachedResult = await this.cacheManager.get(CACHE_KEY); + + if (cachedResult != null) { + return cachedResult; + } + + const blockData = await this.blocksService.getFoundBlocks(); const userAgents = await this.clientService.getUserAgents(); - return { + const data = { blockData, userAgents, uptime: this.uptime }; + + //1 min + await this.cacheManager.set(CACHE_KEY, data, 1 * 60 * 1000); + + return data; + } @Get('network') @@ -44,17 +58,17 @@ export class AppController { public async infoChart() { - // const CACHE_KEY = 'SITE_HASHRATE_GRAPH'; - // const cachedResult = await this.cacheManager.get(CACHE_KEY); + const CACHE_KEY = 'SITE_HASHRATE_GRAPH'; + const cachedResult = await this.cacheManager.get(CACHE_KEY); - // if (cachedResult != null) { - // return cachedResult; - // } + if (cachedResult != null) { + return cachedResult; + } const chartData = await this.clientStatisticsService.getChartDataForSite(); - //5 min - //await this.cacheManager.set(CACHE_KEY, chartData, 5 * 60 * 1000); + //10 min + await this.cacheManager.set(CACHE_KEY, chartData, 10 * 60 * 1000); return chartData; diff --git a/src/controllers/client/client.controller.ts b/src/controllers/client/client.controller.ts index 7a3d2c2..4fe77ac 100644 --- a/src/controllers/client/client.controller.ts +++ b/src/controllers/client/client.controller.ts @@ -20,7 +20,7 @@ export class ClientController { const workers = await this.clientService.getByAddress(address); - const addressSettings = await this.addressSettingsService.getSettings(address); + const addressSettings = await this.addressSettingsService.getSettings(address, false); return { bestDifficulty: addressSettings?.bestDifficulty, diff --git a/src/models/StratumV1Client.ts b/src/models/StratumV1Client.ts index 0d1edde..4cd9ded 100644 --- a/src/models/StratumV1Client.ts +++ b/src/models/StratumV1Client.ts @@ -479,7 +479,7 @@ export class StratumV1Client { parseInt(submission.ntime, 16) ); const header = updatedJobBlock.toBuffer(true); - const { submissionDifficulty, submissionHash } = this.calculateDifficulty(header); + const { submissionDifficulty } = this.calculateDifficulty(header); //console.log(`DIFF: ${submissionDifficulty} of ${this.sessionDifficulty} from ${this.clientAuthorization.worker + '.' + this.extraNonceAndSessionId}`); @@ -505,7 +505,7 @@ export class StratumV1Client { } } try { - await this.statistics.addSubmission(this.entity, submissionHash, this.sessionDifficulty); + await this.statistics.addSubmission(this.entity, this.sessionDifficulty); await this.clientService.heartbeat(this.entity.address, this.entity.clientName, this.entity.sessionId, this.hashRate); } catch (e) { console.log(e); @@ -524,7 +524,7 @@ export class StratumV1Client { if (submissionDifficulty > this.entity.bestDifficulty) { await this.clientService.updateBestDifficulty(this.extraNonceAndSessionId, submissionDifficulty); this.entity.bestDifficulty = submissionDifficulty; - if (submissionDifficulty > (await this.addressSettingsService.getSettings(this.clientAuthorization.address)).bestDifficulty) { + if (submissionDifficulty > (await this.addressSettingsService.getSettings(this.clientAuthorization.address, true)).bestDifficulty) { await this.addressSettingsService.updateBestDifficulty(this.clientAuthorization.address, submissionDifficulty); } } diff --git a/src/models/StratumV1ClientStatistics.ts b/src/models/StratumV1ClientStatistics.ts index 8aae0bf..9f72c4f 100644 --- a/src/models/StratumV1ClientStatistics.ts +++ b/src/models/StratumV1ClientStatistics.ts @@ -17,7 +17,7 @@ export class StratumV1ClientStatistics { this.submissionCacheStart = new Date(); } - public async addSubmission(client: ClientEntity, submissionHash: string, targetDifficulty: number) { + public async addSubmission(client: ClientEntity, targetDifficulty: number) { if (this.submissionCache.length > CACHE_SIZE) { this.submissionCache.shift(); @@ -31,15 +31,14 @@ export class StratumV1ClientStatistics { // 10 min var coeff = 1000 * 60 * 10; var date = new Date(); - var rounded = new Date(Math.round(date.getTime() / coeff) * coeff); + var rounded = new Date(Math.floor(date.getTime() / coeff) * coeff); await this.clientStatisticsService.save({ time: rounded.getTime(), shares: targetDifficulty, address: client.address, clientName: client.clientName, - sessionId: client.sessionId, - submissionHash + sessionId: client.sessionId }); } From 76285ba728341334f0ef0d0655ec2ec5d6b49e5f Mon Sep 17 00:00:00 2001 From: Ben Wilson Date: Fri, 15 Dec 2023 00:56:22 -0500 Subject: [PATCH 03/15] stratum jobs array to dictionary --- src/services/stratum-v1-jobs.service.ts | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/src/services/stratum-v1-jobs.service.ts b/src/services/stratum-v1-jobs.service.ts index a3cea4c..a28b544 100644 --- a/src/services/stratum-v1-jobs.service.ts +++ b/src/services/stratum-v1-jobs.service.ts @@ -30,7 +30,7 @@ export class StratumV1JobsService { public latestJobId: number = 1; public latestJobTemplateId: number = 1; - public jobs: MiningJob[] = []; + public jobs: { [jobId: string]: MiningJob } = {}; public blocks: { [id: number]: IJobTemplate } = {}; @@ -123,7 +123,7 @@ export class StratumV1JobsService { tap((data) => { if (data.blockData.clearJobs) { this.blocks = {}; - this.jobs = []; + this.jobs = {}; } this.blocks[data.blockData.id] = data; }), @@ -153,16 +153,12 @@ export class StratumV1JobsService { } public addJob(job: MiningJob) { - this.jobs.push(job); + this.jobs[job.jobId] = job; this.latestJobId++; } - public getLatestJob() { - return this.jobs[this.jobs.length - 1]; - } - public getJobById(jobId: string) { - return this.jobs.find(job => job.jobId == jobId); + return this.jobs[jobId]; } public getNextTemplateId() { From ab1e7bb8a0eea78f49c4ccf0fdfa3b7c201a4c34 Mon Sep 17 00:00:00 2001 From: Ben Wilson Date: Fri, 15 Dec 2023 00:25:01 -0500 Subject: [PATCH 04/15] batch save share statistics --- src/models/StratumV1Client.ts | 5 ++- src/models/StratumV1ClientStatistics.ts | 50 +++++++++++++++---------- 2 files changed, 34 insertions(+), 21 deletions(-) diff --git a/src/models/StratumV1Client.ts b/src/models/StratumV1Client.ts index 4cd9ded..8619bb7 100644 --- a/src/models/StratumV1Client.ts +++ b/src/models/StratumV1Client.ts @@ -133,7 +133,7 @@ export class StratumV1Client { if (this.sessionStart == null) { this.sessionStart = new Date(); - this.statistics = new StratumV1ClientStatistics(this.clientStatisticsService, this.clientService); + this.statistics = new StratumV1ClientStatistics(this.clientStatisticsService); this.extraNonceAndSessionId = this.getRandomHexString(); console.log(`New client ID: : ${this.extraNonceAndSessionId}, ${this.socket.remoteAddress}:${this.socket.remotePort}`); } @@ -363,6 +363,7 @@ export class StratumV1Client { this.backgroundWork = setInterval(async () => { await this.checkDifficulty(); + await this.statistics.saveShares(this.entity); }, 60 * 1000); } @@ -505,7 +506,7 @@ export class StratumV1Client { } } try { - await this.statistics.addSubmission(this.entity, this.sessionDifficulty); + await this.statistics.addShares(this.sessionDifficulty); await this.clientService.heartbeat(this.entity.address, this.entity.clientName, this.entity.sessionId, this.hashRate); } catch (e) { console.log(e); diff --git a/src/models/StratumV1ClientStatistics.ts b/src/models/StratumV1ClientStatistics.ts index 9f72c4f..134a671 100644 --- a/src/models/StratumV1ClientStatistics.ts +++ b/src/models/StratumV1ClientStatistics.ts @@ -1,23 +1,48 @@ import { ClientStatisticsService } from '../ORM/client-statistics/client-statistics.service'; import { ClientEntity } from '../ORM/client/client.entity'; -import { ClientService } from '../ORM/client/client.service'; const CACHE_SIZE = 30; const TARGET_SUBMISSION_PER_SECOND = 10; const MIN_DIFF = 0.00001; export class StratumV1ClientStatistics { + private shareBacklog: number = 0; + private submissionCacheStart: Date; private submissionCache = []; constructor( - private readonly clientStatisticsService: ClientStatisticsService, - private readonly clientService: ClientService + private readonly clientStatisticsService: ClientStatisticsService ) { this.submissionCacheStart = new Date(); } - public async addSubmission(client: ClientEntity, targetDifficulty: number) { + + public async saveShares(client: ClientEntity) { + + if (client == null || client.address == null || client.clientName == null || client.sessionId == null) { + return; + } + + // 10 min + var coeff = 1000 * 60 * 10; + var date = new Date(); + var rounded = new Date(Math.floor(date.getTime() / coeff) * coeff); + + await this.clientStatisticsService.save({ + time: rounded.getTime(), + shares: this.shareBacklog, + address: client.address, + clientName: client.clientName, + sessionId: client.sessionId + }); + + this.shareBacklog = 0; + } + + // We don't want to save them here because it can be DB intensive, stead do it every once in + // awhile with saveShares() + public async addShares(targetDifficulty: number) { if (this.submissionCache.length > CACHE_SIZE) { this.submissionCache.shift(); @@ -28,23 +53,10 @@ export class StratumV1ClientStatistics { difficulty: targetDifficulty, }); - // 10 min - var coeff = 1000 * 60 * 10; - var date = new Date(); - var rounded = new Date(Math.floor(date.getTime() / coeff) * coeff); - - await this.clientStatisticsService.save({ - time: rounded.getTime(), - shares: targetDifficulty, - address: client.address, - clientName: client.clientName, - sessionId: client.sessionId - }); + this.shareBacklog += targetDifficulty; } - public getLastSubmissionTime(): Date | null { - return this.submissionCache[this.submissionCache.length - 1]?.time; - } + public getSuggestedDifficulty(clientDifficulty: number) { // miner hasn't submitted shares in one minute From d97dafe1850e5c4c247c264bd8a75ebe018e3958 Mon Sep 17 00:00:00 2001 From: Ben Date: Sun, 24 Dec 2023 17:54:40 -0500 Subject: [PATCH 05/15] separate background tasks --- src/models/MiningJob.ts | 4 +- src/models/StratumV1Client.ts | 91 +++++++++++++++++++---------------- 2 files changed, 52 insertions(+), 43 deletions(-) diff --git a/src/models/MiningJob.ts b/src/models/MiningJob.ts index 02bccf0..775842e 100644 --- a/src/models/MiningJob.ts +++ b/src/models/MiningJob.ts @@ -27,9 +27,9 @@ export class MiningJob { jobTemplate: IJobTemplate ) { - this.jobTemplateId = jobTemplate.blockData.id, + this.jobTemplateId = jobTemplate.blockData.id; - this.coinbaseTransaction = this.createCoinbaseTransaction(payoutInformation, jobTemplate.blockData.coinbasevalue); + this.coinbaseTransaction = this.createCoinbaseTransaction(payoutInformation, jobTemplate.blockData.coinbasevalue); //The commitment is recorded in a scriptPubKey of the coinbase transaction. It must be at least 38 bytes, with the first 6-byte of 0x6a24aa21a9ed, that is: // 1-byte - OP_RETURN (0x6a) diff --git a/src/models/StratumV1Client.ts b/src/models/StratumV1Client.ts index 8619bb7..52d0ce2 100644 --- a/src/models/StratumV1Client.ts +++ b/src/models/StratumV1Client.ts @@ -6,6 +6,7 @@ import { validate, ValidatorOptions } from 'class-validator'; import * as crypto from 'crypto'; import { Socket } from 'net'; import { firstValueFrom, Subscription } from 'rxjs'; +import { clearInterval } from 'timers'; import { AddressSettingsService } from '../ORM/address-settings/address-settings.service'; import { BlocksService } from '../ORM/blocks/blocks.service'; @@ -35,7 +36,7 @@ export class StratumV1Client { private clientAuthorization: AuthorizationMessage; private clientSuggestedDifficulty: SuggestDifficulty; private stratumSubscription: Subscription; - private backgroundWork: NodeJS.Timer; + private backgroundWork: NodeJS.Timer[] = []; private statistics: StratumV1ClientStatistics; private stratumInitialized = false; @@ -87,9 +88,10 @@ export class StratumV1Client { if (this.stratumSubscription != null) { this.stratumSubscription.unsubscribe(); } - if (this.backgroundWork != null) { - clearInterval(this.backgroundWork); - } + + this.backgroundWork.forEach(work => { + clearInterval(work); + }); } private getRandomHexString() { @@ -333,55 +335,62 @@ export class StratumV1Client { && this.clientAuthorization != null && this.stratumInitialized == false) { - this.stratumInitialized = true; - - switch (this.clientSubscription.userAgent) { - case 'cpuminer': { - this.sessionDifficulty = 0.1; - } - } - - - if (this.clientSuggestedDifficulty == null) { - //console.log(`Setting difficulty to ${this.sessionDifficulty}`) - const setDifficulty = JSON.stringify(new SuggestDifficulty().response(this.sessionDifficulty)); - const success = await this.write(setDifficulty + '\n'); - if (!success) { - return; - } - } - - - this.stratumSubscription = this.stratumV1JobsService.newMiningJob$.subscribe(async (jobTemplate) => { - try { - await this.sendNewMiningJob(jobTemplate); - } catch (e) { - await this.socket.end(); - console.error(e); - } - }); - - this.backgroundWork = setInterval(async () => { - await this.checkDifficulty(); - await this.statistics.saveShares(this.entity); - }, 60 * 1000); + await this.initStratum(); } } + private async initStratum() { + this.stratumInitialized = true; + + switch (this.clientSubscription.userAgent) { + case 'cpuminer': { + this.sessionDifficulty = 0.1; + } + } + + if (this.clientSuggestedDifficulty == null) { + //console.log(`Setting difficulty to ${this.sessionDifficulty}`) + const setDifficulty = JSON.stringify(new SuggestDifficulty().response(this.sessionDifficulty)); + const success = await this.write(setDifficulty + '\n'); + if (!success) { + return; + } + } + + this.stratumSubscription = this.stratumV1JobsService.newMiningJob$.subscribe(async (jobTemplate) => { + try { + await this.sendNewMiningJob(jobTemplate); + } catch (e) { + await this.socket.end(); + console.error(e); + } + }); + + this.backgroundWork.push( + setInterval(async () => { + await this.checkDifficulty(); + }, 60 * 1000) + ); + + this.backgroundWork.push( + setInterval(async () => { + await this.statistics.saveShares(this.entity); + }, 60 * 1000) + ); + } + private async sendNewMiningJob(jobTemplate: IJobTemplate) { - - let payoutInformation; const devFeeAddress = this.configService.get('DEV_FEE_ADDRESS'); //50Th/s - this.noFee = false; - if (this.entity) { + this.noFee = devFeeAddress == null || devFeeAddress.length < 1; + if (this.noFee == false && this.entity) { this.hashRate = await this.clientStatisticsService.getHashRateForSession(this.clientAuthorization.address, this.clientAuthorization.worker, this.extraNonceAndSessionId); this.noFee = this.hashRate != 0 && this.hashRate < 50000000000000; } - if (this.noFee || devFeeAddress == null || devFeeAddress.length < 1) { + if (this.noFee) { payoutInformation = [ { address: this.clientAuthorization.address, percent: 100 } ]; From 982194b4e9a4e295149cab066bba158960ce8b0b Mon Sep 17 00:00:00 2001 From: Ben Date: Sun, 24 Dec 2023 17:57:03 -0500 Subject: [PATCH 06/15] ClientStatisticsEntity rowid --- src/ORM/client-statistics/client-statistics.entity.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ORM/client-statistics/client-statistics.entity.ts b/src/ORM/client-statistics/client-statistics.entity.ts index 115cc26..6976b63 100644 --- a/src/ORM/client-statistics/client-statistics.entity.ts +++ b/src/ORM/client-statistics/client-statistics.entity.ts @@ -2,7 +2,7 @@ import { Column, Entity, Index, PrimaryGeneratedColumn } from 'typeorm'; import { TrackedEntity } from '../utils/TrackedEntity.entity'; -@Entity({ withoutRowid: true }) +@Entity() //Index for getHashRateForSession @Index(["address", "clientName", "sessionId"]) //Index for statistics save From 1d6670051f76663a606fb221dbe229000493cfcf Mon Sep 17 00:00:00 2001 From: Ben Wilson Date: Sun, 14 Jan 2024 14:10:47 -0500 Subject: [PATCH 07/15] reduce heartbeat updates --- src/ORM/client/client.service.ts | 4 ++-- src/models/StratumV1Client.ts | 8 +++++++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/ORM/client/client.service.ts b/src/ORM/client/client.service.ts index 8798879..65a4995 100644 --- a/src/ORM/client/client.service.ts +++ b/src/ORM/client/client.service.ts @@ -45,8 +45,8 @@ export class ClientService { .execute(); } - public async heartbeat(address: string, clientName: string, sessionId: string, hashRate: number) { - return await this.clientRepository.update({ address, clientName, sessionId }, { hashRate, deletedAt: null, updatedAt: new Date() }); + public async heartbeat(address: string, clientName: string, sessionId: string, hashRate: number, updatedAt: Date) { + return await this.clientRepository.update({ address, clientName, sessionId }, { hashRate, deletedAt: null, updatedAt }); } // public async save(client: Partial) { diff --git a/src/models/StratumV1Client.ts b/src/models/StratumV1Client.ts index 52d0ce2..ed03bc1 100644 --- a/src/models/StratumV1Client.ts +++ b/src/models/StratumV1Client.ts @@ -516,7 +516,13 @@ export class StratumV1Client { } try { await this.statistics.addShares(this.sessionDifficulty); - await this.clientService.heartbeat(this.entity.address, this.entity.clientName, this.entity.sessionId, this.hashRate); + const now = new Date(); + // only update every minute + if (now.getTime() - this.entity.updatedAt.getTime() > 1000 * 60) { + await this.clientService.heartbeat(this.entity.address, this.entity.clientName, this.entity.sessionId, this.hashRate, now); + this.entity.updatedAt = now; + } + } catch (e) { console.log(e); const err = new StratumErrorMessage( From 300fd99067c513ef6238db22370b2926ffe1d55f Mon Sep 17 00:00:00 2001 From: Ben Wilson Date: Mon, 15 Jan 2024 20:03:55 -0500 Subject: [PATCH 08/15] bug fixes --- src/app.controller.ts | 1 + src/models/StratumV1Client.ts | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/app.controller.ts b/src/app.controller.ts index a46ffc4..6af196b 100644 --- a/src/app.controller.ts +++ b/src/app.controller.ts @@ -21,6 +21,7 @@ export class AppController { private readonly bitcoinRpcService: BitcoinRpcService ) { } + @Get('info') public async info() { diff --git a/src/models/StratumV1Client.ts b/src/models/StratumV1Client.ts index ed03bc1..2726b88 100644 --- a/src/models/StratumV1Client.ts +++ b/src/models/StratumV1Client.ts @@ -385,12 +385,12 @@ export class StratumV1Client { let payoutInformation; const devFeeAddress = this.configService.get('DEV_FEE_ADDRESS'); //50Th/s - this.noFee = devFeeAddress == null || devFeeAddress.length < 1; - if (this.noFee == false && this.entity) { + this.noFee = false; + if (this.entity) { this.hashRate = await this.clientStatisticsService.getHashRateForSession(this.clientAuthorization.address, this.clientAuthorization.worker, this.extraNonceAndSessionId); this.noFee = this.hashRate != 0 && this.hashRate < 50000000000000; } - if (this.noFee) { + if (this.noFee || devFeeAddress == null || devFeeAddress.length < 1) { payoutInformation = [ { address: this.clientAuthorization.address, percent: 100 } ]; @@ -518,7 +518,7 @@ export class StratumV1Client { await this.statistics.addShares(this.sessionDifficulty); const now = new Date(); // only update every minute - if (now.getTime() - this.entity.updatedAt.getTime() > 1000 * 60) { + if (this.entity.updatedAt == null || now.getTime() - this.entity.updatedAt.getTime() > 1000 * 60) { await this.clientService.heartbeat(this.entity.address, this.entity.clientName, this.entity.sessionId, this.hashRate, now); this.entity.updatedAt = now; } From 537e06ef50963df4de17bd1ba8f7cb137670c4f6 Mon Sep 17 00:00:00 2001 From: Ben Wilson Date: Mon, 22 Jan 2024 17:34:46 -0500 Subject: [PATCH 09/15] turn back cache sizes to default for sqlite --- src/services/app.service.ts | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/services/app.service.ts b/src/services/app.service.ts index 6897424..78fc73f 100644 --- a/src/services/app.service.ts +++ b/src/services/app.service.ts @@ -20,13 +20,14 @@ export class AppService implements OnModuleInit { // if (process.env.NODE_APP_INSTANCE == '0') { // await this.dataSource.query(`VACUUM;`); // } + //https://phiresky.github.io/blog/2020/sqlite-performance-tuning/ - //500 MB DB cache - await this.dataSource.query(`PRAGMA cache_size = -500000;`); + // //500 MB DB cache + // await this.dataSource.query(`PRAGMA cache_size = -500000;`); //Normal is still completely corruption safe in WAL mode, and means only WAL checkpoints have to wait for FSYNC. await this.dataSource.query(`PRAGMA synchronous = off;`); - //6Gb - await this.dataSource.query(`PRAGMA mmap_size = 6000000000;`); + // //6Gb + // await this.dataSource.query(`PRAGMA mmap_size = 6000000000;`); } @Interval(1000 * 60 * 60) From e29caad3771d1594f05bac1fc7bf7c376794077b Mon Sep 17 00:00:00 2001 From: Ben Wilson Date: Mon, 22 Jan 2024 17:38:08 -0500 Subject: [PATCH 10/15] remove cache typeorm option --- src/app.module.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/app.module.ts b/src/app.module.ts index bb10740..b904679 100644 --- a/src/app.module.ts +++ b/src/app.module.ts @@ -44,7 +44,6 @@ const ORMModules = [ database: './DB/public-pool.sqlite', synchronize: true, autoLoadEntities: true, - cache: true, logging: false, enableWAL: true, busyTimeout: 30 * 1000, From a98ec3879ee06b1a3c51b7127d77f24bedc3f5f0 Mon Sep 17 00:00:00 2001 From: Ben Wilson Date: Tue, 23 Jan 2024 18:40:50 -0500 Subject: [PATCH 11/15] block template fix for NULL NODE_APP_INSTANCE --- src/services/bitcoin-rpc.service.ts | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/src/services/bitcoin-rpc.service.ts b/src/services/bitcoin-rpc.service.ts index 38c9574..6c6e869 100644 --- a/src/services/bitcoin-rpc.service.ts +++ b/src/services/bitcoin-rpc.service.ts @@ -30,7 +30,11 @@ export class BitcoinRpcService { this.client = new RPCClient({ url, port, timeout, user, pass }); - console.log('Bitcoin RPC connected'); + this.client.getrpcinfo().then((res) => { + console.log('Bitcoin RPC connected'); + }, () => { + console.error('Could not reach RPC host'); + }); if (this.configService.get('BITCOIN_ZMQ_HOST')) { // const sock = zmq.socket("sub"); @@ -77,11 +81,14 @@ export class BitcoinRpcService { if (block != null && block.data != null) { return Promise.resolve(JSON.parse(block.data)); } else if (block == null) { - // There is a unique constraint on the block height so if another process tries to lock, it'll throw - try { - await this.rpcBlockService.lockBlock(blockHeight, process.env.NODE_APP_INSTANCE); - } catch (e) { - result = await this.waitForBlock(blockHeight); + + if (process.env.NODE_APP_INSTANCE != null) { + // There is a unique constraint on the block height so if another process tries to lock, it'll throw + try { + await this.rpcBlockService.lockBlock(blockHeight, process.env.NODE_APP_INSTANCE); + } catch (e) { + result = await this.waitForBlock(blockHeight); + } } result = await this.client.getblocktemplate({ From dadb03ed1cba6334acc009b79a389fb7b71e8567 Mon Sep 17 00:00:00 2001 From: Ben Wilson Date: Tue, 23 Jan 2024 21:24:33 -0500 Subject: [PATCH 12/15] re-add zmq with pre-built binaries and pinned version --- package-lock.json | 170 +++++++++++++++++----------- package.json | 4 +- src/services/bitcoin-rpc.service.ts | 17 ++- 3 files changed, 114 insertions(+), 77 deletions(-) diff --git a/package-lock.json b/package-lock.json index d048806..c81a0de 100644 --- a/package-lock.json +++ b/package-lock.json @@ -35,7 +35,7 @@ "sqlite3": "^5.1.6", "tiny-secp256k1": "^2.2.3", "typeorm": "^0.3.17", - "zeromq": "^5.3.1" + "zeromq": "6.0.0-beta.19" }, "devDependencies": { "@nestjs/cli": "^9.0.0", @@ -64,6 +64,16 @@ "typescript": "^5.0.0" } }, + "node_modules/@aminya/node-gyp-build": { + "version": "4.5.0-aminya.5", + "resolved": "https://registry.npmjs.org/@aminya/node-gyp-build/-/node-gyp-build-4.5.0-aminya.5.tgz", + "integrity": "sha512-TO7GldxDfSeSRNZVmhlm0liS2GX2o2Q/qTlcD3iD4ltTM6dir568LTRZ+ZDsDbLfMAkfhrbU+VuzNYImwYfczg==", + "bin": { + "aminya-node-gyp-build": "bin.js", + "aminya-node-gyp-build-optional": "optional.js", + "aminya-node-gyp-build-test": "build-test.js" + } + }, "node_modules/@ampproject/remapping": { "version": "2.2.1", "resolved": "https://registry.npmjs.org/@ampproject/remapping/-/remapping-2.2.1.tgz", @@ -4308,11 +4318,27 @@ "luxon": "^3.2.1" } }, + "node_modules/cross-env": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/cross-env/-/cross-env-7.0.3.tgz", + "integrity": "sha512-+/HKd6EgcQCJGh2PSjZuUitQBQynKor4wrFbRg4DtAgS1aWO+gU52xpH7M9ScGgXSYmAVS9bIJ8EzuaGw0oNAw==", + "dependencies": { + "cross-spawn": "^7.0.1" + }, + "bin": { + "cross-env": "src/bin/cross-env.js", + "cross-env-shell": "src/bin/cross-env-shell.js" + }, + "engines": { + "node": ">=10.14", + "npm": ">=6", + "yarn": ">=1" + } + }, "node_modules/cross-spawn": { "version": "7.0.3", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==", - "dev": true, "dependencies": { "path-key": "^3.1.0", "shebang-command": "^2.0.0", @@ -6265,7 +6291,6 @@ "version": "1.4.0", "resolved": "https://registry.npmjs.org/interpret/-/interpret-1.4.0.tgz", "integrity": "sha512-agE4QfB2Lkp9uICn7BAqoscw4SZP9kTE2hxiFI3jBPmXJfdqiahTbUuKGsMoN2GtqL9AxhYioAcVvgsb1HvRbA==", - "dev": true, "engines": { "node": ">= 0.10" } @@ -6356,7 +6381,6 @@ "version": "2.12.1", "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.12.1.tgz", "integrity": "sha512-Q4ZuBAe2FUsKtyQJoQHlvP8OvBERxO3jEmy1I7hcRXcJBGGHFh/aJBswbXuS9sgrDH2QUO8ilkwNPHvHMd8clg==", - "dev": true, "dependencies": { "has": "^1.0.3" }, @@ -6594,8 +6618,7 @@ "node_modules/isexe": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", - "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", - "devOptional": true + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==" }, "node_modules/isstream": { "version": "0.1.2", @@ -7896,7 +7919,6 @@ "version": "1.2.8", "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", - "dev": true, "funding": { "url": "https://github.com/sponsors/ljharb" } @@ -8136,6 +8158,14 @@ "integrity": "sha512-AGK2yQKIjRuqnc6VkX2Xj5d+QW8xZ87pa1UK6yA6ouUyuxfHuMP6umE5QK7UmTeOAymo+Zx1Fxiuw9rVx8taHQ==", "dev": true }, + "node_modules/node-addon-api": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-7.1.0.tgz", + "integrity": "sha512-mNcltoe1R8o7STTegSOHdnJNN7s5EUvhoS7ShnTHDyOSd+8H+UdWODq6qSv67PjC8Zc5JRT8+oLAMCr0SIXw7g==", + "engines": { + "node": "^16 || ^18 || >= 20" + } + }, "node_modules/node-emoji": { "version": "1.11.0", "resolved": "https://registry.npmjs.org/node-emoji/-/node-emoji-1.11.0.tgz", @@ -8188,16 +8218,6 @@ "node": ">= 10.12.0" } }, - "node_modules/node-gyp-build": { - "version": "4.6.1", - "resolved": "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-4.6.1.tgz", - "integrity": "sha512-24vnklJmyRS8ViBNI8KbtK/r/DmXQMRiOMXTNz2nrTnAYUwjmEEbnnpB/+kt+yWRv73bPsSPRFddrcIbAxSiMQ==", - "bin": { - "node-gyp-build": "bin.js", - "node-gyp-build-optional": "optional.js", - "node-gyp-build-test": "build-test.js" - } - }, "node_modules/node-gyp/node_modules/are-we-there-yet": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/are-we-there-yet/-/are-we-there-yet-3.0.1.tgz", @@ -8705,7 +8725,6 @@ "version": "3.1.1", "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", - "dev": true, "engines": { "node": ">=8" } @@ -8713,8 +8732,7 @@ "node_modules/path-parse": { "version": "1.0.7", "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz", - "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==", - "dev": true + "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==" }, "node_modules/path-scurry": { "version": "1.9.2", @@ -9198,7 +9216,6 @@ "version": "0.6.2", "resolved": "https://registry.npmjs.org/rechoir/-/rechoir-0.6.2.tgz", "integrity": "sha512-HFM8rkZ+i3zrV+4LQjwQ0W+ez98pApMGM3HUrN04j3CqzPOzl9nmP15Y8YXNm8QHGv/eacOVEjqhmWpkRV0NAw==", - "dev": true, "dependencies": { "resolve": "^1.1.6" }, @@ -9345,7 +9362,6 @@ "version": "1.22.2", "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.2.tgz", "integrity": "sha512-Sb+mjNHOULsBv818T40qSPeRiuWLyaGMa5ewydRLFimneixmVy2zdivRl+AF6jaYPC8ERxGDmFSiqui6SfPd+g==", - "dev": true, "dependencies": { "is-core-module": "^2.11.0", "path-parse": "^1.0.7", @@ -9689,7 +9705,6 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", - "dev": true, "dependencies": { "shebang-regex": "^3.0.0" }, @@ -9701,7 +9716,6 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", - "dev": true, "engines": { "node": ">=8" } @@ -9710,7 +9724,6 @@ "version": "0.8.5", "resolved": "https://registry.npmjs.org/shelljs/-/shelljs-0.8.5.tgz", "integrity": "sha512-TiwcRcrkhHvbrZbnRcFYMLl30Dfov3HKqzp5tO5b4pt6G/SezKcYhmDg15zXVBswHmctSAQKznqNW2LO5tTDow==", - "dev": true, "dependencies": { "glob": "^7.0.0", "interpret": "^1.0.0", @@ -9727,7 +9740,6 @@ "version": "7.2.3", "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", - "dev": true, "dependencies": { "fs.realpath": "^1.0.0", "inflight": "^1.0.4", @@ -9743,6 +9755,21 @@ "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/shx": { + "version": "0.3.4", + "resolved": "https://registry.npmjs.org/shx/-/shx-0.3.4.tgz", + "integrity": "sha512-N6A9MLVqjxZYcVn8hLmtneQWIJtp8IKzMP4eMnx+nqkvXoqinUPCbUFLp2UcWTEIUONhlk0ewxr/jaVGlc+J+g==", + "dependencies": { + "minimist": "^1.2.3", + "shelljs": "^0.8.5" + }, + "bin": { + "shx": "lib/cli.js" + }, + "engines": { + "node": ">=6" + } + }, "node_modules/side-channel": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.4.tgz", @@ -10187,7 +10214,6 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz", "integrity": "sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==", - "dev": true, "engines": { "node": ">= 0.4" }, @@ -11168,7 +11194,6 @@ "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", - "devOptional": true, "dependencies": { "isexe": "^2.0.0" }, @@ -11368,20 +11393,28 @@ } }, "node_modules/zeromq": { - "version": "5.3.1", - "resolved": "https://registry.npmjs.org/zeromq/-/zeromq-5.3.1.tgz", - "integrity": "sha512-4WDF9bNWWXe8OAI319bVw5dmG4BklEk8wzFGwRQxEzKb+0mgDU5J/jtyZPo0BEusVIU1+3mRQIEdT5LtQn+aAw==", + "version": "6.0.0-beta.19", + "resolved": "https://registry.npmjs.org/zeromq/-/zeromq-6.0.0-beta.19.tgz", + "integrity": "sha512-2eU6H7Z4r9LmTkseGdIfEqp0k3rJr5EpXQw2oC0bUqy3wpoj1M83IU/c3qHPXp0z8IklNMoVmVm130d3g2Xf3g==", "hasInstallScript": true, "dependencies": { - "nan": "2.17.0", - "node-gyp-build": "^4.5.0" + "@aminya/node-gyp-build": "4.5.0-aminya.5", + "cross-env": "^7.0.3", + "node-addon-api": "^7.0.0", + "shelljs": "^0.8.5", + "shx": "^0.3.4" }, "engines": { - "node": ">=6.0" + "node": ">= 10.2" } } }, "dependencies": { + "@aminya/node-gyp-build": { + "version": "4.5.0-aminya.5", + "resolved": "https://registry.npmjs.org/@aminya/node-gyp-build/-/node-gyp-build-4.5.0-aminya.5.tgz", + "integrity": "sha512-TO7GldxDfSeSRNZVmhlm0liS2GX2o2Q/qTlcD3iD4ltTM6dir568LTRZ+ZDsDbLfMAkfhrbU+VuzNYImwYfczg==" + }, "@ampproject/remapping": { "version": "2.2.1", "resolved": "https://registry.npmjs.org/@ampproject/remapping/-/remapping-2.2.1.tgz", @@ -14672,11 +14705,18 @@ "luxon": "^3.2.1" } }, + "cross-env": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/cross-env/-/cross-env-7.0.3.tgz", + "integrity": "sha512-+/HKd6EgcQCJGh2PSjZuUitQBQynKor4wrFbRg4DtAgS1aWO+gU52xpH7M9ScGgXSYmAVS9bIJ8EzuaGw0oNAw==", + "requires": { + "cross-spawn": "^7.0.1" + } + }, "cross-spawn": { "version": "7.0.3", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==", - "dev": true, "requires": { "path-key": "^3.1.0", "shebang-command": "^2.0.0", @@ -16145,8 +16185,7 @@ "interpret": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/interpret/-/interpret-1.4.0.tgz", - "integrity": "sha512-agE4QfB2Lkp9uICn7BAqoscw4SZP9kTE2hxiFI3jBPmXJfdqiahTbUuKGsMoN2GtqL9AxhYioAcVvgsb1HvRbA==", - "dev": true + "integrity": "sha512-agE4QfB2Lkp9uICn7BAqoscw4SZP9kTE2hxiFI3jBPmXJfdqiahTbUuKGsMoN2GtqL9AxhYioAcVvgsb1HvRbA==" }, "ip": { "version": "2.0.0", @@ -16210,7 +16249,6 @@ "version": "2.12.1", "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.12.1.tgz", "integrity": "sha512-Q4ZuBAe2FUsKtyQJoQHlvP8OvBERxO3jEmy1I7hcRXcJBGGHFh/aJBswbXuS9sgrDH2QUO8ilkwNPHvHMd8clg==", - "dev": true, "requires": { "has": "^1.0.3" } @@ -16364,8 +16402,7 @@ "isexe": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", - "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", - "devOptional": true + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==" }, "isstream": { "version": "0.1.2", @@ -17377,8 +17414,7 @@ "minimist": { "version": "1.2.8", "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", - "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", - "dev": true + "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==" }, "minipass": { "version": "4.2.8", @@ -17577,6 +17613,11 @@ "integrity": "sha512-AGK2yQKIjRuqnc6VkX2Xj5d+QW8xZ87pa1UK6yA6ouUyuxfHuMP6umE5QK7UmTeOAymo+Zx1Fxiuw9rVx8taHQ==", "dev": true }, + "node-addon-api": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-7.1.0.tgz", + "integrity": "sha512-mNcltoe1R8o7STTegSOHdnJNN7s5EUvhoS7ShnTHDyOSd+8H+UdWODq6qSv67PjC8Zc5JRT8+oLAMCr0SIXw7g==" + }, "node-emoji": { "version": "1.11.0", "resolved": "https://registry.npmjs.org/node-emoji/-/node-emoji-1.11.0.tgz", @@ -17686,11 +17727,6 @@ } } }, - "node-gyp-build": { - "version": "4.6.1", - "resolved": "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-4.6.1.tgz", - "integrity": "sha512-24vnklJmyRS8ViBNI8KbtK/r/DmXQMRiOMXTNz2nrTnAYUwjmEEbnnpB/+kt+yWRv73bPsSPRFddrcIbAxSiMQ==" - }, "node-int64": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/node-int64/-/node-int64-0.4.0.tgz", @@ -18006,14 +18042,12 @@ "path-key": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", - "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", - "dev": true + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==" }, "path-parse": { "version": "1.0.7", "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz", - "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==", - "dev": true + "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==" }, "path-scurry": { "version": "1.9.2", @@ -18365,7 +18399,6 @@ "version": "0.6.2", "resolved": "https://registry.npmjs.org/rechoir/-/rechoir-0.6.2.tgz", "integrity": "sha512-HFM8rkZ+i3zrV+4LQjwQ0W+ez98pApMGM3HUrN04j3CqzPOzl9nmP15Y8YXNm8QHGv/eacOVEjqhmWpkRV0NAw==", - "dev": true, "requires": { "resolve": "^1.1.6" } @@ -18472,7 +18505,6 @@ "version": "1.22.2", "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.2.tgz", "integrity": "sha512-Sb+mjNHOULsBv818T40qSPeRiuWLyaGMa5ewydRLFimneixmVy2zdivRl+AF6jaYPC8ERxGDmFSiqui6SfPd+g==", - "dev": true, "requires": { "is-core-module": "^2.11.0", "path-parse": "^1.0.7", @@ -18707,7 +18739,6 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", - "dev": true, "requires": { "shebang-regex": "^3.0.0" } @@ -18715,14 +18746,12 @@ "shebang-regex": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", - "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", - "dev": true + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==" }, "shelljs": { "version": "0.8.5", "resolved": "https://registry.npmjs.org/shelljs/-/shelljs-0.8.5.tgz", "integrity": "sha512-TiwcRcrkhHvbrZbnRcFYMLl30Dfov3HKqzp5tO5b4pt6G/SezKcYhmDg15zXVBswHmctSAQKznqNW2LO5tTDow==", - "dev": true, "requires": { "glob": "^7.0.0", "interpret": "^1.0.0", @@ -18733,7 +18762,6 @@ "version": "7.2.3", "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", - "dev": true, "requires": { "fs.realpath": "^1.0.0", "inflight": "^1.0.4", @@ -18745,6 +18773,15 @@ } } }, + "shx": { + "version": "0.3.4", + "resolved": "https://registry.npmjs.org/shx/-/shx-0.3.4.tgz", + "integrity": "sha512-N6A9MLVqjxZYcVn8hLmtneQWIJtp8IKzMP4eMnx+nqkvXoqinUPCbUFLp2UcWTEIUONhlk0ewxr/jaVGlc+J+g==", + "requires": { + "minimist": "^1.2.3", + "shelljs": "^0.8.5" + } + }, "side-channel": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.4.tgz", @@ -19077,8 +19114,7 @@ "supports-preserve-symlinks-flag": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz", - "integrity": "sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==", - "dev": true + "integrity": "sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==" }, "symbol-observable": { "version": "4.0.0", @@ -19717,7 +19753,6 @@ "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", - "devOptional": true, "requires": { "isexe": "^2.0.0" } @@ -19849,12 +19884,15 @@ "dev": true }, "zeromq": { - "version": "5.3.1", - "resolved": "https://registry.npmjs.org/zeromq/-/zeromq-5.3.1.tgz", - "integrity": "sha512-4WDF9bNWWXe8OAI319bVw5dmG4BklEk8wzFGwRQxEzKb+0mgDU5J/jtyZPo0BEusVIU1+3mRQIEdT5LtQn+aAw==", + "version": "6.0.0-beta.19", + "resolved": "https://registry.npmjs.org/zeromq/-/zeromq-6.0.0-beta.19.tgz", + "integrity": "sha512-2eU6H7Z4r9LmTkseGdIfEqp0k3rJr5EpXQw2oC0bUqy3wpoj1M83IU/c3qHPXp0z8IklNMoVmVm130d3g2Xf3g==", "requires": { - "nan": "2.17.0", - "node-gyp-build": "^4.5.0" + "@aminya/node-gyp-build": "4.5.0-aminya.5", + "cross-env": "^7.0.3", + "node-addon-api": "^7.0.0", + "shelljs": "^0.8.5", + "shx": "^0.3.4" } } } diff --git a/package.json b/package.json index 0c528c7..acf577c 100644 --- a/package.json +++ b/package.json @@ -46,7 +46,7 @@ "sqlite3": "^5.1.6", "tiny-secp256k1": "^2.2.3", "typeorm": "^0.3.17", - "zeromq": "^5.3.1" + "zeromq": "6.0.0-beta.19" }, "devDependencies": { "@nestjs/cli": "^9.0.0", @@ -91,4 +91,4 @@ "coverageDirectory": "../coverage", "testEnvironment": "node" } -} +} \ No newline at end of file diff --git a/src/services/bitcoin-rpc.service.ts b/src/services/bitcoin-rpc.service.ts index 6c6e869..def4ea1 100644 --- a/src/services/bitcoin-rpc.service.ts +++ b/src/services/bitcoin-rpc.service.ts @@ -3,12 +3,11 @@ import { ConfigService } from '@nestjs/config'; import { RPCClient } from 'rpc-bitcoin'; import { BehaviorSubject, filter, shareReplay } from 'rxjs'; import { RpcBlockService } from 'src/ORM/rpc-block/rpc-block.service'; +import * as zmq from 'zeromq/v5-compat'; import { IBlockTemplate } from '../models/bitcoin-rpc/IBlockTemplate'; import { IMiningInfo } from '../models/bitcoin-rpc/IMiningInfo'; -// import * as zmq from 'zeromq'; - @Injectable() export class BitcoinRpcService { @@ -37,13 +36,13 @@ export class BitcoinRpcService { }); if (this.configService.get('BITCOIN_ZMQ_HOST')) { - // const sock = zmq.socket("sub"); - // sock.connect(this.configService.get('BITCOIN_ZMQ_HOST')); - // sock.subscribe("rawblock"); - // sock.on("message", async (topic: Buffer, message: Buffer) => { - // console.log("new block zmq"); - // this.pollMiningInfo(); - // }); + const sock = zmq.socket("sub"); + sock.connect(this.configService.get('BITCOIN_ZMQ_HOST')); + sock.subscribe("rawblock"); + sock.on("message", async (topic: Buffer, message: Buffer) => { + console.log("new block zmq"); + this.pollMiningInfo(); + }); this.pollMiningInfo(); } else { setInterval(this.pollMiningInfo.bind(this), 500); From 3fac0dfaa9ab813c108d72dd99798287dc20b9f7 Mon Sep 17 00:00:00 2001 From: Ben Wilson Date: Wed, 24 Jan 2024 11:50:59 -0500 Subject: [PATCH 13/15] zmq await --- src/services/bitcoin-rpc.service.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/services/bitcoin-rpc.service.ts b/src/services/bitcoin-rpc.service.ts index def4ea1..dff9883 100644 --- a/src/services/bitcoin-rpc.service.ts +++ b/src/services/bitcoin-rpc.service.ts @@ -41,9 +41,9 @@ export class BitcoinRpcService { sock.subscribe("rawblock"); sock.on("message", async (topic: Buffer, message: Buffer) => { console.log("new block zmq"); - this.pollMiningInfo(); + await this.pollMiningInfo(); }); - this.pollMiningInfo(); + this.pollMiningInfo().then(() => { }); } else { setInterval(this.pollMiningInfo.bind(this), 500); } From 1bf32d075aab5ed3519b3e57e2d3b6d567799dc0 Mon Sep 17 00:00:00 2001 From: WantClue <86001033+WantClue@users.noreply.github.com> Date: Thu, 8 Feb 2024 16:31:52 +0100 Subject: [PATCH 14/15] adding pool endpoint (#23) --- src/app.controller.ts | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/src/app.controller.ts b/src/app.controller.ts index 6af196b..7cb2f74 100644 --- a/src/app.controller.ts +++ b/src/app.controller.ts @@ -49,6 +49,37 @@ export class AppController { } + @Get('pool') + public async pool() { + + const CACHE_KEY = 'POOL_INFO'; + const cachedResult = await this.cacheManager.get(CACHE_KEY); + + if (cachedResult != null) { + return cachedResult; + } + + + const userAgents = await this.clientService.getUserAgents(); + const totalHashRate = userAgents.reduce((acc, userAgent) => acc + parseFloat(userAgent.totalHashRate), 0); + const totalMiners = userAgents.reduce((acc, userAgent) => acc + parseInt(userAgent.count), 0); + const blockHeight = (await firstValueFrom(this.bitcoinRpcService.newBlock$)).blocks; + const blocksFound = await this.blocksService.getFoundBlocks(); + + const data = { + totalHashRate, + blockHeight, + totalMiners, + blocksFound, + fee: 0 + } + + //5 min + await this.cacheManager.set(CACHE_KEY, data, 5 * 60 * 1000); + + return data; + } + @Get('network') public async network() { const miningInfo = await firstValueFrom(this.bitcoinRpcService.newBlock$); From d486c778a326d77bf48b67014748d1dff6b37c10 Mon Sep 17 00:00:00 2001 From: Arman The Parman <77603167+ArmanTheParman@users.noreply.github.com> Date: Wed, 14 Feb 2024 06:41:06 +1100 Subject: [PATCH 15/15] Docker changes (#29) * Update README.md I needed to change -v .env:.env to -v .env:/.env to get it to work. Docker (Linux) was not happy otherwise. * Update Dockerfile Added cmake to Dockerfile, so Pi4 (ARM chip) will work. * Update README.md Corrected volume mount --- Dockerfile | 1 + README.md | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index beaeb21..e902ca9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,6 +10,7 @@ RUN apt-get update \ RUN DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ python3 \ build-essential \ + cmake \ && apt clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* WORKDIR /build diff --git a/README.md b/README.md index c083953..2ed731e 100644 --- a/README.md +++ b/README.md @@ -56,7 +56,7 @@ $ docker build -t public-pool . Run container: ```bash -$ docker container run --name public-pool --rm -p 3333:3333 -p 3334:3334 -p 8332:8332 -v .env:.env public-pool +$ docker container run --name public-pool --rm -p 3333:3333 -p 3334:3334 -p 8332:8332 -v .env:/public-pool/.env public-pool ``` ### Docker Compose