From 87508bfb69a39c6cb0e5124e1b202239bd335e52 Mon Sep 17 00:00:00 2001 From: Ben Wilson Date: Fri, 16 Feb 2024 23:49:47 -0500 Subject: [PATCH] user agent report --- .../user-agent-report.module.ts | 15 +++++ .../user-agent-report.service.ts | 35 +++++++++++ .../user-agent-report.view.ts | 27 ++++++++ src/ORM/client/client.service.ts | 22 +++---- src/app.controller.ts | 16 ++--- src/app.module.ts | 8 ++- src/services/app.service.ts | 61 +++++++++++-------- 7 files changed, 140 insertions(+), 44 deletions(-) create mode 100644 src/ORM/_views/user-agent-report/user-agent-report.module.ts create mode 100644 src/ORM/_views/user-agent-report/user-agent-report.service.ts create mode 100644 src/ORM/_views/user-agent-report/user-agent-report.view.ts diff --git a/src/ORM/_views/user-agent-report/user-agent-report.module.ts b/src/ORM/_views/user-agent-report/user-agent-report.module.ts new file mode 100644 index 0000000..3b3d6ba --- /dev/null +++ b/src/ORM/_views/user-agent-report/user-agent-report.module.ts @@ -0,0 +1,15 @@ +import { Global, Module } from '@nestjs/common'; +import { TypeOrmModule } from '@nestjs/typeorm'; + +import { UserAgentReportService } from './user-agent-report.service'; +import { UserAgentReportView } from './user-agent-report.view'; + + + +@Global() +@Module({ + imports: [TypeOrmModule.forFeature([UserAgentReportView])], + providers: [UserAgentReportService], + exports: [TypeOrmModule, UserAgentReportService], +}) +export class UserAgentReportModule { } \ No newline at end of file diff --git a/src/ORM/_views/user-agent-report/user-agent-report.service.ts b/src/ORM/_views/user-agent-report/user-agent-report.service.ts new file mode 100644 index 0000000..a476be1 --- /dev/null +++ b/src/ORM/_views/user-agent-report/user-agent-report.service.ts @@ -0,0 +1,35 @@ +import { Injectable } from '@nestjs/common'; +import { InjectRepository } from '@nestjs/typeorm'; +import { Repository } from 'typeorm'; + +import { UserAgentReportView } from './user-agent-report.view'; + +@Injectable() +export class UserAgentReportService { + + constructor( + @InjectRepository(UserAgentReportView) + private userAgentReport: Repository, + ) { + + } + + public async getReport() { + return await this.userAgentReport.find(); + } + + public async refreshReport() { + try { + return await this.userAgentReport.query(` + COMMIT; + BEGIN TRANSACTION ISOLATION LEVEL REPEATABLE READ; + REFRESH MATERIALIZED VIEW user_agent_report_view; + COMMIT; + `); + } catch (e) { + + console.log(e) + } + + } +} \ No newline at end of file diff --git a/src/ORM/_views/user-agent-report/user-agent-report.view.ts b/src/ORM/_views/user-agent-report/user-agent-report.view.ts new file mode 100644 index 0000000..c8fa5c1 --- /dev/null +++ b/src/ORM/_views/user-agent-report/user-agent-report.view.ts @@ -0,0 +1,27 @@ +import { DataSource, ViewColumn, ViewEntity } from 'typeorm'; + +import { ClientEntity } from '../../client/client.entity'; + +@ViewEntity({ + materialized: true, + expression: (dataSource: DataSource) => + dataSource + .createQueryBuilder() + .select('client.userAgent as "userAgent"') + .addSelect('COUNT(client.userAgent)', 'count') + .addSelect('MAX(client.bestDifficulty)', 'bestDifficulty') + .addSelect('SUM(client.hashRate)', 'totalHashRate') + .from(ClientEntity, 'client') + .groupBy('client.userAgent') + .orderBy('"totalHashRate"', 'DESC') +}) +export class UserAgentReportView { + @ViewColumn() + userAgent: string; + @ViewColumn() + count: number; + @ViewColumn() + bestDifficulty: number; + @ViewColumn() + totalHashRate: number; +} \ No newline at end of file diff --git a/src/ORM/client/client.service.ts b/src/ORM/client/client.service.ts index 05d597c..702f966 100644 --- a/src/ORM/client/client.service.ts +++ b/src/ORM/client/client.service.ts @@ -104,16 +104,16 @@ export class ClientService { return await this.clientRepository.softDelete({}) } - public async getUserAgents() { - const result = await this.clientRepository.createQueryBuilder('client') - .select('client.userAgent as "userAgent"') - .addSelect('COUNT(client.userAgent)', 'count') - .addSelect('MAX(client.bestDifficulty)', 'bestDifficulty') - .addSelect('SUM(client.hashRate)', 'totalHashRate') - .groupBy('client.userAgent') - .orderBy('"totalHashRate"', 'DESC') - .getRawMany(); - return result; - } + // public async getUserAgents() { + // const result = await this.clientRepository.createQueryBuilder('client') + // .select('client.userAgent as "userAgent"') + // .addSelect('COUNT(client.userAgent)', 'count') + // .addSelect('MAX(client.bestDifficulty)', 'bestDifficulty') + // .addSelect('SUM(client.hashRate)', 'totalHashRate') + // .groupBy('client.userAgent') + // .orderBy('"totalHashRate"', 'DESC') + // .getRawMany(); + // return result; + // } } \ No newline at end of file diff --git a/src/app.controller.ts b/src/app.controller.ts index de24379..919bfe1 100644 --- a/src/app.controller.ts +++ b/src/app.controller.ts @@ -3,6 +3,7 @@ import { Controller, Get, Inject } from '@nestjs/common'; import { Cache } from 'cache-manager'; import { firstValueFrom } from 'rxjs'; +import { UserAgentReportService } from './ORM/_views/user-agent-report/user-agent-report.service'; import { AddressSettingsService } from './ORM/address-settings/address-settings.service'; import { BlocksService } from './ORM/blocks/blocks.service'; import { ClientStatisticsService } from './ORM/client-statistics/client-statistics.service'; @@ -22,7 +23,8 @@ export class AppController { private readonly blocksService: BlocksService, private readonly bitcoinRpcService: BitcoinRpcService, private readonly homeGraphService: HomeGraphService, - private readonly addressSettingsService: AddressSettingsService + private readonly addressSettingsService: AddressSettingsService, + private readonly userAgentReportService: UserAgentReportService ) { } @Get('info') @@ -38,7 +40,7 @@ export class AppController { const blockData = await this.blocksService.getFoundBlocks(); - const userAgents = await this.clientService.getUserAgents(); + const userAgents = await this.userAgentReportService.getReport(); const highScores = await this.addressSettingsService.getHighScores(); const data = { @@ -48,8 +50,8 @@ export class AppController { uptime: this.uptime }; - //1 min - await this.cacheManager.set(CACHE_KEY, data, 1 * 60 * 1000); + //5 min + await this.cacheManager.set(CACHE_KEY, data, 5 * 60 * 1000); return data; @@ -66,9 +68,9 @@ export class AppController { } - const userAgents = await this.clientService.getUserAgents(); - const totalHashRate = userAgents.reduce((acc, userAgent) => acc + parseFloat(userAgent.totalHashRate), 0); - const totalMiners = userAgents.reduce((acc, userAgent) => acc + parseInt(userAgent.count), 0); + const userAgents = await this.userAgentReportService.getReport(); + const totalHashRate = userAgents.reduce((acc, userAgent) => acc + userAgent.totalHashRate, 0); + const totalMiners = userAgents.reduce((acc, userAgent) => acc + userAgent.count, 0); const blockHeight = (await firstValueFrom(this.bitcoinRpcService.newBlock$)).blocks; const blocksFound = await this.blocksService.getFoundBlocks(); diff --git a/src/app.module.ts b/src/app.module.ts index 434cae8..c1f812c 100644 --- a/src/app.module.ts +++ b/src/app.module.ts @@ -10,6 +10,8 @@ import { AddressController } from './controllers/address/address.controller'; import { ClientController } from './controllers/client/client.controller'; import { BitcoinAddressValidator } from './models/validators/bitcoin-address.validator'; import { UniqueNonceIndex } from './ORM/_migrations/UniqueNonceIndex'; +import { UserAgentReportModule } from './ORM/_views/user-agent-report/user-agent-report.module'; +import { UserAgentReportView } from './ORM/_views/user-agent-report/user-agent-report.view'; import { AddressSettingsEntity } from './ORM/address-settings/address-settings.entity'; import { AddressSettingsModule } from './ORM/address-settings/address-settings.module'; import { BlocksEntity } from './ORM/blocks/blocks.entity'; @@ -43,7 +45,8 @@ const ORMModules = [ TelegramSubscriptionsModule, BlocksModule, RpcBlocksModule, - HomeGraphModule + HomeGraphModule, + UserAgentReportModule ] @Module({ @@ -65,7 +68,8 @@ const ORMModules = [ ClientStatisticsEntity, RpcBlockEntity, TelegramSubscriptionsEntity, - HomeGraphEntity + HomeGraphEntity, + UserAgentReportView ], synchronize: configService.get('PRODUCTION') != 'true', logging: false, diff --git a/src/services/app.service.ts b/src/services/app.service.ts index 16825e8..0da6ddb 100644 --- a/src/services/app.service.ts +++ b/src/services/app.service.ts @@ -1,7 +1,7 @@ import { Injectable, OnModuleInit } from '@nestjs/common'; -import { Interval } from '@nestjs/schedule'; import { DataSource } from 'typeorm'; +import { UserAgentReportService } from '../ORM/_views/user-agent-report/user-agent-report.service'; import { ClientStatisticsService } from '../ORM/client-statistics/client-statistics.service'; import { ClientService } from '../ORM/client/client.service'; import { HomeGraphService } from '../ORM/home-graph/home-graph.service'; @@ -16,41 +16,54 @@ export class AppService implements OnModuleInit { private readonly rpcBlockService: RpcBlockService, private readonly homeGraphService: HomeGraphService, private readonly dataSource: DataSource, + private readonly userAgentReportService: UserAgentReportService ) { } async onModuleInit() { + if (process.env.ENABLE_SOLO == 'true' && (process.env.NODE_APP_INSTANCE == null || process.env.NODE_APP_INSTANCE == '0')) { + + setInterval(async () => { + await this.deleteOldStatistics(); + }, 1000 * 60 * 60); + + setInterval(async () => { + console.log('Killing dead clients'); + await this.clientService.killDeadClients(); + }, 1000 * 60 * 5); + + setInterval(async () => { + console.log('Deleting Old Blocks'); + await this.rpcBlockService.deleteOldBlocks(); + }, 1000 * 60 * 60 * 24); + + setInterval(async () => { + await this.updateChart(); + }, 1000 * 60 * 10); + + + setInterval(async () => { + console.log('Refreshing user agent report view') + await this.userAgentReportService.refreshReport(); + console.log('Finished Refreshing user agent report view') + }, 1000 * 60 * 5); + + } + } - @Interval(1000 * 60 * 60) private async deleteOldStatistics() { console.log('Deleting statistics'); - if (process.env.ENABLE_SOLO == 'true' && (process.env.NODE_APP_INSTANCE == null || process.env.NODE_APP_INSTANCE == '0')) { - const deletedStatistics = await this.clientStatisticsService.deleteOldStatistics(); - console.log(`Deleted ${deletedStatistics.affected} old statistics`); - const deletedClients = await this.clientService.deleteOldClients(); - console.log(`Deleted ${deletedClients.affected} old clients`); - } + + const deletedStatistics = await this.clientStatisticsService.deleteOldStatistics(); + console.log(`Deleted ${deletedStatistics.affected} old statistics`); + const deletedClients = await this.clientService.deleteOldClients(); + console.log(`Deleted ${deletedClients.affected} old clients`); + } - @Interval(1000 * 60 * 5) - private async killDeadClients() { - console.log('Killing dead clients'); - if (process.env.ENABLE_SOLO == 'true' && (process.env.NODE_APP_INSTANCE == null || process.env.NODE_APP_INSTANCE == '0')) { - await this.clientService.killDeadClients(); - } - } - @Interval(1000 * 60 * 60 * 24) - private async deleteOldBlocks() { - console.log('Deleting Old Blocks'); - if (process.env.ENABLE_SOLO == 'true' && (process.env.NODE_APP_INSTANCE == null || process.env.NODE_APP_INSTANCE == '0')) { - await this.rpcBlockService.deleteOldBlocks(); - } - } - - @Interval(1000 * 60 * 10) private async updateChart() { console.log('Updating Chart');