Skip to content

Provisioning Bitcoin Network Crawler

Jump to bottom
Addy Yeow edited this page Mar 21, 2022 · 61 revisions

Launch Server

Hetzner PX61-NVMe Dedicated Server 64GB Debian 9 64-bit (Falkenstein, DE)

Initial Login

ssh root@[IP_ADDRESS]
    [PASSWORD]

Change Password

passwd
    [PASSWORD]

Configure Hostname

vi /etc/hostname
    [HOSTNAME]

vi /etc/hosts
    127.0.0.1 localhost
    [IP_ADDRESS] [HOSTNAME]
    ::1 ip6-localhost ip6-loopback
    fe00::0 ip6-localnet
    ff00::0 ip6-mcastprefix
    ff02::1 ip6-allnodes
    ff02::2 ip6-allrouters
    ff02::3 ip6-allhosts
    [IPV6_ADDRESS] [HOSTNAME]

Configure Locales

dpkg-reconfigure locales
    en_US.UTF-8 UTF-8
    en_US.UTF-8

vi /etc/environment
    LC_CTYPE=en_US.UTF-8

Configure Timezone

dpkg-reconfigure tzdata
    None of the above
    UTC

Update Packages

apt-get update; apt-get upgrade

Install Packages

apt-get -y install apt-transport-https build-essential dirmngr htop python-dev python-virtualenv sudo tcl tcpdump unzip

Add User

adduser [USERNAME]
    [PASSWORD]
adduser [USERNAME] sudo
mkdir -p /home/[USERNAME]/.ssh; chmod 700 /home/[USERNAME]/.ssh
vi /home/[USERNAME]/.ssh/authorized_keys
    [PUBLICKEY]
chmod 600 /home/[USERNAME]/.ssh/authorized_keys; chown -R [USERNAME]:[USERNAME] /home/[USERNAME]/.ssh

Update /home/[USERNAME]/.bashrc

export REDIS_SOCKET=/tmp/redis.sock
export REDIS_PASSWORD=[PASSWORD]

Update /etc/ssh/sshd_config

Port [PORT]
LoginGraceTime 60
PermitRootLogin no
AllowUsers [USERNAME]
PasswordAuthentication no

Update /etc/sysctl.conf

net.ipv4.conf.default.rp_filter=1
net.ipv4.conf.all.rp_filter=1
net.ipv4.tcp_syncookies=1
net.ipv4.conf.all.accept_redirects=0
net.ipv6.conf.all.accept_redirects=0
net.ipv4.conf.all.send_redirects=0
net.ipv4.conf.all.accept_source_route=0
net.ipv6.conf.all.accept_source_route=0
net.ipv4.conf.all.log_martians=1
net.core.rmem_default=33554432
net.core.wmem_default=33554432
net.core.rmem_max=33554432
net.core.wmem_max=33554432
net.core.optmem_max=33554432
net.ipv4.tcp_rmem=10240 87380 33554432
net.ipv4.tcp_wmem=10240 87380 33554432
net.ipv4.ip_local_port_range=2000 65500
net.core.netdev_max_backlog=100000
net.ipv4.tcp_max_syn_backlog=80000
net.ipv4.tcp_max_tw_buckets=2000000
net.ipv4.tcp_tw_reuse=1
net.ipv4.tcp_fin_timeout=5
net.ipv4.tcp_slow_start_after_idle=0
net.core.somaxconn=60000
fs.file-max=1000000
vm.swappiness=10
vm.min_free_kbytes=1048576
vm.overcommit_memory=1

Update /etc/security/limits.conf

* soft nofile 1000000
* hard nofile 1000000

Disable THP

vi /etc/systemd/system/disable-transparent-huge-pages.service
    [Unit]
    Description=disable-transparent-huge-pages

    [Service]
    Type=oneshot
    ExecStart=/bin/sh -c "echo "never" | tee /sys/kernel/mm/transparent_hugepage/enabled"
    ExecStart=/bin/sh -c "echo "never" | tee /sys/kernel/mm/transparent_hugepage/defrag"

    [Install]
    WantedBy=multi-user.target

systemctl enable disable-transparent-huge-pages.service
systemctl start disable-transparent-huge-pages.service

Set txqueuelen

vi /etc/systemd/system/set-txqueuelen.service
    [Unit]
    Description=set-txqueuelen

    [Service]
    Type=oneshot
    ExecStart=/sbin/ifconfig [INTERFACE] txqueuelen 5000

    [Install]
    WantedBy=multi-user.target

systemctl enable set-txqueuelen.service
systemctl start set-txqueuelen.service

Reboot Server

reboot

Normal User Login

ssh -p [PORT] -i [PRIVATEKEY] [USERNAME]@[IP_ADDRESS]

Install Redis

cd; wget http://download.redis.io/releases/redis-6.2.6.tar.gz
tar xzf redis-6.2.6.tar.gz; cd redis-6.2.6; make; make test
sudo make install
cd utils; sudo bash install_server.sh (works for 5.x only; if upgrading to 6.x, continue below)

sudo vi /etc/init.d/redis_0
***************************************************************************
#!/bin/sh
EXEC=/usr/local/bin/redis-server
CLIEXEC=/usr/local/bin/redis-cli
PIDFILE=/var/run/redis_0.pid
CONF="/etc/redis/0.conf"
REDISSOCKET="/tmp/redis.sock"
REDISPASSWORD="[PASSWORD]"

###############
# SysV Init Information
# chkconfig: - 58 74
# description: redis_0 is the redis daemon.
### BEGIN INIT INFO
# Provides: redis_0
# Required-Start: $network $local_fs $remote_fs
# Required-Stop: $network $local_fs $remote_fs
# Default-Start: 2 3 4 5
# Default-Stop: 0 1 6
# Should-Start: $syslog $named
# Should-Stop: $syslog $named
# Short-Description: start and stop redis_0
# Description: Redis daemon
### END INIT INFO

case "$1" in
    start)
        if [ -f $PIDFILE ]
        then
            echo "$PIDFILE exists, process is already running or crashed"
        else
            echo "Starting Redis server..."
            $EXEC $CONF
        fi
        ;;
    stop)
        if [ ! -f $PIDFILE ]
        then
            echo "$PIDFILE does not exist, process is not running"
        else
            PID=$(cat $PIDFILE)
            echo "Stopping ..."
            $CLIEXEC -s $REDISSOCKET -a $REDISPASSWORD shutdown
            while [ -x /proc/${PID} ]
            do
                echo "Waiting for Redis to shutdown ..."
                sleep 1
            done
            echo "Redis stopped"
        fi
        ;;
    status)
        PID=$(cat $PIDFILE)
        if [ ! -x /proc/${PID} ]
        then
            echo 'Redis is not running'
        else
            echo "Redis is running ($PID)"
        fi
        ;;
    restart)
        $0 stop
        $0 start
        ;;
    *)
        echo "Please use start, stop, restart or status as first argument"
        ;;
esac
***************************************************************************

sudo chmod 755 /etc/init.d/redis_0

sudo update-rc.d redis_0 defaults

sudo vi /etc/redis/0.conf
    pidfile /var/run/redis_0.pid
    daemonize yes
    unixsocket /tmp/redis.sock
    unixsocketperm 777
    save ""
    requirepass [PASSWORD]
    maxclients 50000
    maxmemory 34326183936
    maxmemory-policy volatile-lru
    notify-keyspace-events K$z
    activerehashing no
    client-output-buffer-limit normal 512mb 256mb 300
    client-output-buffer-limit replica 512mb 256mb 300
    client-output-buffer-limit pubsub 512mb 256mb 300

Reboot Server

sudo reboot

Normal User Login

ssh -p [PORT] -i [PRIVATEKEY] [USERNAME]@[IP_ADDRESS]

Launch Crawler

cd; wget --no-check-certificate https://github.com/ayeowch/bitnodes/archive/master.zip
unzip master.zip
virtualenv ~/.virtualenvs/bitnodes
source ~/.virtualenvs/bitnodes/bin/activate
cd bitnodes-master
pip install -r requirements.txt
vi geoip/.maxmind_license_key
    [MAXMIND_LICENSE_KEY]
bash geoip/update.sh
cp conf/crawl.conf.default conf/crawl.f9beb4d9.conf
cp conf/ping.conf.default conf/ping.f9beb4d9.conf
cp conf/resolve.conf.default conf/resolve.f9beb4d9.conf
cp conf/export.conf.default conf/export.f9beb4d9.conf
cp conf/seeder.conf.default conf/seeder.f9beb4d9.conf
cp conf/cache_inv.conf.default conf/cache_inv.f9beb4d9.conf
bash start.sh
cd data/pcap/f9beb4d9
sudo rm *.pcap; sudo tcpdump -i [INTERFACE] -w %s.[INTERFACE.pcap -v -n -G 2 -B 65536 -Z [USERNAME] 'tcp and not src host [IP_ADDRESS] and not src host [IPV6_ADDRESS]' > [INTERFACE] 2>&1 &
sudo tcpdump -i lo -w %s.lo.pcap -v -n -G 2 -B 65536 -Z [USERNAME] 'tcp and port 9050' > lo 2>&1 &
Clone this wiki locally