Provisioning Bitcoin Network Crawler

Launch Server

Hetzner AX41-NVMe Dedicated Server 64GB Debian 11 64-bit

Install Packages

apt update && apt -y upgrade
apt install -y apt-transport-https
apt install -y build-essential
apt install -y git
apt install -y libbz2-dev
apt install -y libevent-dev
apt install -y libffi-dev
apt install -y liblzma-dev
apt install -y libncursesw5-dev
apt install -y libreadline-dev
apt install -y libsqlite3-dev
apt install -y libssl-dev
apt install -y libxml2-dev
apt install -y libxmlsec1-dev
apt install -y libzstd-dev
apt install -y tcpdump
apt install -y tk-dev
apt install -y vim
apt install -y zlib1g-dev

Update /etc/sysctl.conf

net.ipv4.conf.default.rp_filter=1
net.ipv4.conf.all.rp_filter=1
net.ipv4.tcp_syncookies=1
net.ipv4.conf.all.accept_redirects=0
net.ipv6.conf.all.accept_redirects=0
net.ipv4.conf.all.send_redirects=0
net.ipv4.conf.all.accept_source_route=0
net.ipv6.conf.all.accept_source_route=0
net.ipv4.conf.all.log_martians=1
net.core.rmem_default=16777216
net.core.wmem_default=16777216
net.core.rmem_max=16777216
net.core.wmem_max=16777216
net.core.optmem_max=16777216
# tcp_rmem = 1460 (MSS) * 60 = 87600
net.ipv4.tcp_rmem=8192 87600 16777216
net.ipv4.tcp_wmem=8192 87600 16777216
# tcp_mem = 16777216 * 2 * 10000 (connections) / 4096 (page size) = 98304000
net.ipv4.tcp_mem=98304000 98304000 98304000
net.ipv4.ip_local_port_range=2000 65535
net.core.netdev_max_backlog=100000
net.ipv4.tcp_max_syn_backlog=80000
net.ipv4.tcp_max_tw_buckets=2000000
net.ipv4.tcp_tw_reuse=1
net.ipv4.tcp_fin_timeout=5
net.ipv4.tcp_slow_start_after_idle=0
net.core.somaxconn=60000
fs.file-max=10000000
fs.nr_open=20000000
vm.swappiness=10
vm.min_free_kbytes=1048576
vm.overcommit_memory=1

Update /etc/security/limits.conf

* soft nofile 10000000
* hard nofile 10000000

Reboot Server

reboot

Install pyenv

git clone https://github.com/pyenv/pyenv.git ~/.pyenv
cd ~/.pyenv && src/configure && make -C src

echo 'export PYENV_ROOT="$HOME/.pyenv"' >> ~/.bashrc
echo 'command -v pyenv >/dev/null || export PATH="$PYENV_ROOT/bin:$PATH"' >> ~/.bashrc
echo 'eval "$(pyenv init -)"' >> ~/.bashrc

echo 'export PYENV_ROOT="$HOME/.pyenv"' >> ~/.profile
echo 'command -v pyenv >/dev/null || export PATH="$PYENV_ROOT/bin:$PATH"' >> ~/.profile
echo 'eval "$(pyenv init -)"' >> ~/.profile

Install Redis

cd && wget https://github.com/redis/redis/archive/7.0.10.tar.gz && tar xzf 7.0.10.tar.gz && cd redis-7.0.10 && make
sudo make install

sudo vi /etc/init.d/redis_0
***************************************************************************
#!/bin/sh
EXEC=/usr/local/bin/redis-server
CLIEXEC=/usr/local/bin/redis-cli
PIDFILE=/var/run/redis_0.pid
CONF="/etc/redis/0.conf"
REDISSOCKET="/run/redis.sock"
REDISPASSWORD="[PASSWORD]"

case "$1" in
    start)
        if [ -f $PIDFILE ]
        then
            echo "$PIDFILE exists, process is already running or crashed"
        else
            echo "Starting Redis server..."
            $EXEC $CONF
        fi
        ;;
    stop)
        if [ ! -f $PIDFILE ]
        then
            echo "$PIDFILE does not exist, process is not running"
        else
            PID=$(cat $PIDFILE)
            echo "Stopping ..."
            $CLIEXEC -s $REDISSOCKET -a $REDISPASSWORD shutdown
            while [ -x /proc/${PID} ]
            do
                echo "Waiting for Redis to shutdown ..."
                sleep 1
            done
            echo "Redis stopped"
        fi
        ;;
    status)
        PID=$(cat $PIDFILE)
        if [ ! -x /proc/${PID} ]
        then
            echo 'Redis is not running'
        else
            echo "Redis is running ($PID)"
        fi
        ;;
    restart)
        $0 stop
        $0 start
        ;;
    *)
        echo "Please use start, stop, restart or status as first argument"
        ;;
esac
***************************************************************************

sudo chmod 755 /etc/init.d/redis_0
sudo update-rc.d redis_0 defaults
sudo mkdir /etc/redis
sudo cp ~/redis-7.0.10/redis.conf /etc/redis/0.conf
sudo vi /etc/redis/0.conf
    pidfile /var/run/redis_0.pid
    daemonize yes
    port 0
    unixsocket /run/redis.sock
    unixsocketperm 777
    save ""
    requirepass [PASSWORD]
    maxclients 50000
    maxmemory 34326183936
    maxmemory-policy volatile-lru
    notify-keyspace-events K$lz
    activerehashing no
    client-output-buffer-limit normal 512mb 256mb 300
    client-output-buffer-limit replica 512mb 256mb 300
    client-output-buffer-limit pubsub 512mb 256mb 300

sudo service redis_0 restart

Launch Crawler

pyenv install 3.11.2
cd && git clone https://github.com/ayeowch/bitnodes.git && cd bitnodes
~/.pyenv/versions/3.11.2/bin/python -m venv venv
source venv/bin/activate
pip install -r requirements.txt
vi geoip/.maxmind_license_key
    [MAXMIND_LICENSE_KEY]
bash geoip/update.sh
cp conf/crawl.conf.default conf/crawl.f9beb4d9.conf
cp conf/ping.conf.default conf/ping.f9beb4d9.conf
cp conf/resolve.conf.default conf/resolve.f9beb4d9.conf
cp conf/export.conf.default conf/export.f9beb4d9.conf
cp conf/seeder.conf.default conf/seeder.f9beb4d9.conf
cp conf/cache_inv.conf.default conf/cache_inv.f9beb4d9.conf
bash start.sh
cd data/pcap/f9beb4d9
sudo rm *.pcap; sudo tcpdump -i [INTERFACE] -w %s.[INTERFACE].pcap -v -n -G 2 -B 65536 -Z [USERNAME] 'tcp and not portrange 0-2000 and not (src host [IPV4_ADDRESS] or src host [IPV6_ADDRESS])' > [INTERFACE] 2>&1 &
sudo tcpdump -i lo -w %s.lo.pcap -v -n -G 2 -B 65536 -Z [USERNAME] 'tcp and port 9050' > lo 2>&1 &