gravatite/public-pool
1
0
Fork 0
mirror of https://github.com/benjamin-wilson/public-pool.git synced 2026-09-29 09:05:06 -07:00
Code Issues Packages Projects Releases Wiki Activity

Migrate pool storage to TimescaleDB and Redis

Browse Source
This commit is contained in:
Ben
2026-06-07 20:33:13 -04:00
parent fb367ac5f0
commit ec69f5fb6c
64 changed files with 4751 additions and 1146 deletions
Download Patch File Download Diff File Expand all files Collapse all files
full-setup/README.md
+25 -1
View File
@@ -1,6 +1,7 @@
# Full Setup for public pool
This setup provides a docker-compose setup consisting of Bitcoin Core Node and Public-Pool running in Mainnet or Testnet.
This setup provides Docker Compose stacks for Bitcoin Core, Public Pool,
TimescaleDB, and Redis on mainnet, testnet, and regtest.
It exposes following ports:
@@ -8,6 +9,7 @@ It exposes following ports:
- `8333/18333` Bitcoin peering on `0.0.0.0`
- `3333/13333` Public-Pool Stratum port on `0.0.0.0`
- `3334/13334` Public-Pool API port on `localhost`
- TimescaleDB and Redis only on the internal compose network
The docker-compose setups for Mainnet and Testnet can be run in parallel without any problems.
@@ -42,6 +44,9 @@ There are 4 config files for Mainnet and Testnet
- `bitcoin-testnet.conf`
**note: pruning (`prune=550`) is enabled by default in the config**
**note: Bitcoin ZMQ raw block publishing is enabled on the internal compose
network so the Public Pool master can refresh templates immediately after new
blocks.**
# Running the setup
To start the setup in foreground mode:
@@ -65,6 +70,25 @@ To stop the setup use:
docker compose -f docker-compose-mainnet.yml down
```
Database and Redis data are stored under `full-setup/data/<network>/`. To remove
all runtime data, stop the stack and delete the relevant `bitcoin`,
`timescaledb`, and `redis` directories.
# Migrations and backups
The Public Pool container waits for TimescaleDB and Redis, runs migrations, and
then starts PM2. To run migrations manually:
```bash
docker compose -f docker-compose-mainnet.yml run --rm public-pool npm run migration:run:prod
```
To back up mainnet accounting data:
```bash
docker compose -f docker-compose-mainnet.yml exec timescaledb pg_dump -U public_pool public_pool_mainnet > public-pool-mainnet.sql
```
# Regtest
After running the `regtest` setup a couple of blocks need to be generated:
full-setup/bitcoin-mainnet.conf
+3
View File
@@ -18,6 +18,9 @@ rpcuser=bitcoin
# enable server
server=1
# notify pool master about new blocks
zmqpubrawblock=tcp://0.0.0.0:3000
[main]
[test]
rpcbind=0.0.0.0
full-setup/bitcoin-regtest.conf
+3
View File
@@ -20,3 +20,6 @@ rpcuser=bitcoin
# set pruning
prune=550
# notify pool master about new blocks
zmqpubrawblock=tcp://0.0.0.0:3000
full-setup/bitcoin-testnet.conf
+4 -1
View File
@@ -18,6 +18,9 @@ rpcuser=bitcoin
# enable server
server=1
# notify pool master about new blocks
zmqpubrawblock=tcp://0.0.0.0:3000
[main]
[test]
rpcbind=0.0.0.0
@@ -25,4 +28,4 @@ rpcport=18332
# no additional testnet settings
[signet]
[regtest]
[regtest]
full-setup/docker-compose-mainnet.yml
+66 -4
View File
@@ -1,5 +1,3 @@
version: '3.8'
services:
bitcoin:
container_name: bitcoin
@@ -20,6 +18,50 @@ services:
- "./data/mainnet/bitcoin:/app/data"
- "./bitcoin-mainnet.conf:/app/data/bitcoin.conf:ro"
timescaledb:
image: timescale/timescaledb:latest-pg17
container_name: public-pool-mainnet-timescaledb
restart: unless-stopped
networks:
- bitcoin
environment:
POSTGRES_DB: public_pool_mainnet
POSTGRES_USER: public_pool
POSTGRES_PASSWORD: public_pool
command:
- postgres
- -c
- max_wal_size=16GB
- -c
- min_wal_size=2GB
- -c
- checkpoint_timeout=15min
- -c
- checkpoint_completion_target=0.9
- -c
- wal_compression=on
volumes:
- "./data/mainnet/timescaledb:/var/lib/postgresql/data"
healthcheck:
test: ["CMD-SHELL", "pg_isready -U public_pool -d public_pool_mainnet"]
interval: 10s
timeout: 5s
retries: 10
redis:
image: redis:8-alpine
container_name: public-pool-mainnet-redis
restart: unless-stopped
networks:
- bitcoin
volumes:
- "./data/mainnet/redis:/data"
healthcheck:
test: ["CMD", "redis-cli", "ping"]
interval: 10s
timeout: 5s
retries: 10
public-pool:
container_name: public-pool
build:
@@ -27,16 +69,36 @@ services:
dockerfile: Dockerfile
restart: unless-stopped
stop_grace_period: 30s
depends_on:
timescaledb:
condition: service_healthy
redis:
condition: service_healthy
networks:
- bitcoin
ports:
- "0.0.0.0:3333:3333/tcp"
- "0.0.0.0:3332:3332/tcp"
- "0.0.0.0:3331:3331/tcp"
- "0.0.0.0:3330:3330/tcp"
- "0.0.0.0:4333:4333/tcp"
- "0.0.0.0:4332:4332/tcp"
- "0.0.0.0:4331:4331/tcp"
- "0.0.0.0:4330:4330/tcp"
- "127.0.0.1:3334:3334/tcp"
volumes:
- "./data/mainnet/public-pool:/public-pool/DB"
- "./public-pool-mainnet.env:/public-pool/.env:ro"
- "../secrets:/public-pool/secrets:ro"
environment:
- NODE_ENV=production
NODE_ENV: production
DB_HOST: timescaledb
DB_PORT: 5432
DB_USERNAME: public_pool
DB_PASSWORD: public_pool
DB_DATABASE: public_pool_mainnet
REDIS_URL: redis://redis:6379
PM2_ENABLED: "true"
STRATUM_WORKERS: ${STRATUM_WORKERS:-2}
networks:
bitcoin:
full-setup/docker-compose-regtest.yml
+62 -4
View File
@@ -1,5 +1,3 @@
version: '3.8'
services:
bitcoin-regtest:
container_name: bitcoin-regtest
@@ -22,6 +20,50 @@ services:
- "./data/regtest/bitcoin:/app/data"
- "./bitcoin-regtest.conf:/app/data/bitcoin.conf:ro"
timescaledb-regtest:
image: timescale/timescaledb:latest-pg17
container_name: public-pool-regtest-timescaledb
restart: unless-stopped
networks:
- bitcoin-regtest
environment:
POSTGRES_DB: public_pool_regtest
POSTGRES_USER: public_pool
POSTGRES_PASSWORD: public_pool
command:
- postgres
- -c
- max_wal_size=16GB
- -c
- min_wal_size=2GB
- -c
- checkpoint_timeout=15min
- -c
- checkpoint_completion_target=0.9
- -c
- wal_compression=on
volumes:
- "./data/regtest/timescaledb:/var/lib/postgresql/data"
healthcheck:
test: ["CMD-SHELL", "pg_isready -U public_pool -d public_pool_regtest"]
interval: 10s
timeout: 5s
retries: 10
redis-regtest:
image: redis:8-alpine
container_name: public-pool-regtest-redis
restart: unless-stopped
networks:
- bitcoin-regtest
volumes:
- "./data/regtest/redis:/data"
healthcheck:
test: ["CMD", "redis-cli", "ping"]
interval: 10s
timeout: 5s
retries: 10
public-pool-regtest:
container_name: public-pool-regtest
build:
@@ -29,16 +71,32 @@ services:
dockerfile: Dockerfile
restart: unless-stopped
stop_grace_period: 30s
depends_on:
timescaledb-regtest:
condition: service_healthy
redis-regtest:
condition: service_healthy
networks:
- bitcoin-regtest
ports:
- "0.0.0.0:23333:23333/tcp"
- "0.0.0.0:23332:23332/tcp"
- "0.0.0.0:23331:23331/tcp"
- "0.0.0.0:23330:23330/tcp"
- "127.0.0.1:23334:23334/tcp"
volumes:
- "./data/regtest/public-pool:/public-pool/DB"
- "./public-pool-regtest.env:/public-pool/.env:ro"
- "../secrets:/public-pool/secrets:ro"
environment:
- NODE_ENV=production
NODE_ENV: production
DB_HOST: timescaledb-regtest
DB_PORT: 5432
DB_USERNAME: public_pool
DB_PASSWORD: public_pool
DB_DATABASE: public_pool_regtest
REDIS_URL: redis://redis-regtest:6379
PM2_ENABLED: "true"
STRATUM_WORKERS: ${STRATUM_WORKERS:-2}
networks:
bitcoin-regtest:
full-setup/docker-compose-testnet.yml
+62 -4
View File
@@ -1,5 +1,3 @@
version: '3.8'
services:
bitcoin-testnet:
container_name: bitcoin-testnet
@@ -21,6 +19,50 @@ services:
- "./data/testnet/bitcoin:/app/data"
- "./bitcoin-testnet.conf:/app/data/bitcoin.conf:ro"
timescaledb-testnet:
image: timescale/timescaledb:latest-pg17
container_name: public-pool-testnet-timescaledb
restart: unless-stopped
networks:
- bitcoin-testnet
environment:
POSTGRES_DB: public_pool_testnet
POSTGRES_USER: public_pool
POSTGRES_PASSWORD: public_pool
command:
- postgres
- -c
- max_wal_size=16GB
- -c
- min_wal_size=2GB
- -c
- checkpoint_timeout=15min
- -c
- checkpoint_completion_target=0.9
- -c
- wal_compression=on
volumes:
- "./data/testnet/timescaledb:/var/lib/postgresql/data"
healthcheck:
test: ["CMD-SHELL", "pg_isready -U public_pool -d public_pool_testnet"]
interval: 10s
timeout: 5s
retries: 10
redis-testnet:
image: redis:8-alpine
container_name: public-pool-testnet-redis
restart: unless-stopped
networks:
- bitcoin-testnet
volumes:
- "./data/testnet/redis:/data"
healthcheck:
test: ["CMD", "redis-cli", "ping"]
interval: 10s
timeout: 5s
retries: 10
public-pool-testnet:
container_name: public-pool-testnet
build:
@@ -28,16 +70,32 @@ services:
dockerfile: Dockerfile
restart: unless-stopped
stop_grace_period: 30s
depends_on:
timescaledb-testnet:
condition: service_healthy
redis-testnet:
condition: service_healthy
networks:
- bitcoin-testnet
ports:
- "0.0.0.0:13333:13333/tcp"
- "0.0.0.0:13332:13332/tcp"
- "0.0.0.0:13331:13331/tcp"
- "0.0.0.0:13330:13330/tcp"
- "127.0.0.1:13334:13334/tcp"
volumes:
- "./data/testnet/public-pool:/public-pool/DB"
- "./public-pool-testnet.env:/public-pool/.env:ro"
- "../secrets:/public-pool/secrets:ro"
environment:
- NODE_ENV=production
NODE_ENV: production
DB_HOST: timescaledb-testnet
DB_PORT: 5432
DB_USERNAME: public_pool
DB_PASSWORD: public_pool
DB_DATABASE: public_pool_testnet
REDIS_URL: redis://redis-testnet:6379
PM2_ENABLED: "true"
STRATUM_WORKERS: ${STRATUM_WORKERS:-2}
networks:
bitcoin-testnet:
full-setup/public-pool-mainnet.env
+14 -4
View File
@@ -4,12 +4,14 @@ BITCOIN_RPC_PASSWORD=bitcoin
BITCOIN_RPC_PORT=8332
BITCOIN_RPC_TIMEOUT=10000
# Enable in bitcoin.conf with
# zmqpubrawblock=tcp://*:3000
# BITCOIN_ZMQ_HOST="tcp://192.168.1.100:3000"
BITCOIN_ZMQ_HOST=tcp://bitcoin:3000
API_PORT=3334
STRATUM_PORT=3333
STRATUM_PORTS=3333,3332,3331,3330
STRATUM_SECURE=true
SECURE_STRATUM_PORTS=4333,4332,4331,4330
STRATUM_MAX_CONNECTIONS_PER_LISTENER=10000
STRATUM_TLS_HANDSHAKE_TIMEOUT_MS=10000
#optional telegram bot
#TELEGRAM_BOT_TOKEN=
@@ -25,3 +27,11 @@ DEV_FEE_ADDRESS=
NETWORK=mainnet
API_SECURE=false
DB_HOST=timescaledb
DB_PORT=5432
DB_USERNAME=public_pool
DB_PASSWORD=public_pool
DB_DATABASE=public_pool_mainnet
REDIS_URL=redis://redis:6379
PRODUCTION=true
full-setup/public-pool-regtest.env
+14 -4
View File
@@ -4,12 +4,14 @@ BITCOIN_RPC_PASSWORD=bitcoin
BITCOIN_RPC_PORT=28332
BITCOIN_RPC_TIMEOUT=10000
# Enable in bitcoin.conf with
# zmqpubrawblock=tcp://*:3000
# BITCOIN_ZMQ_HOST="tcp://192.168.1.100:3000"
BITCOIN_ZMQ_HOST=tcp://bitcoin-regtest:3000
API_PORT=23334
STRATUM_PORT=23333
STRATUM_PORTS=23333,23332,23331,23330
STRATUM_SECURE=false
SECURE_STRATUM_PORTS=
STRATUM_MAX_CONNECTIONS_PER_LISTENER=10000
STRATUM_TLS_HANDSHAKE_TIMEOUT_MS=10000
#optional telegram bot
#TELEGRAM_BOT_TOKEN=
@@ -25,3 +27,11 @@ DEV_FEE_ADDRESS=
NETWORK=regtest
API_SECURE=false
DB_HOST=timescaledb-regtest
DB_PORT=5432
DB_USERNAME=public_pool
DB_PASSWORD=public_pool
DB_DATABASE=public_pool_regtest
REDIS_URL=redis://redis-regtest:6379
PRODUCTION=true
full-setup/public-pool-testnet.env
+15 -5
View File
@@ -4,12 +4,14 @@ BITCOIN_RPC_PASSWORD=bitcoin
BITCOIN_RPC_PORT=18332
BITCOIN_RPC_TIMEOUT=10000
# Enable in bitcoin.conf with
# zmqpubrawblock=tcp://*:3000
# BITCOIN_ZMQ_HOST="tcp://192.168.1.100:3000"
BITCOIN_ZMQ_HOST=tcp://bitcoin-testnet:3000
API_PORT=13334
STRATUM_PORT=13333
STRATUM_PORTS=13333,13332,13331,13330
STRATUM_SECURE=false
SECURE_STRATUM_PORTS=
STRATUM_MAX_CONNECTIONS_PER_LISTENER=10000
STRATUM_TLS_HANDSHAKE_TIMEOUT_MS=10000
#optional telegram bot
#TELEGRAM_BOT_TOKEN=
@@ -24,4 +26,12 @@ DEV_FEE_ADDRESS=
# mainnet | testnet
NETWORK=testnet
API_SECURE=false
API_SECURE=false
DB_HOST=timescaledb-testnet
DB_PORT=5432
DB_USERNAME=public_pool
DB_PASSWORD=public_pool
DB_DATABASE=public_pool_testnet
REDIS_URL=redis://redis-testnet:6379
PRODUCTION=true