Banning

Makefile

@@ -15,7 +15,7 @@ darwin: prepare
 	GOOS=darwin GOARCH=amd64 go build -mod=vendor -o artifacts/filebin2-darwin-amd64 -trimpath -buildvcs=false
 
 run: linux
-	artifacts/filebin2-linux-amd64 --listen-host 0.0.0.0 --lurker-interval 10 --expiration 3600 --access-log=access.log --s3-secure=false --db-host=db --limit-storage 1G --admin-username admin --admin-password changeme --metrics
+	artifacts/filebin2-linux-amd64 --listen-host 0.0.0.0 --lurker-interval 10 --expiration 3600 --access-log=access.log --s3-secure=false --db-host=db --limit-storage 1G --admin-username admin --admin-password changeme --metrics --mmdb-city mmdb/GeoLite2-City.mmdb --mmdb-asn mmdb/GeoLite2-ASN.mmdb
 
 fmt:
 	gofmt -w -s *.go

dbl/client.go

@@ -0,0 +1,147 @@
+package dbl
+
+import (
+	"database/sql"
+	"errors"
+	"fmt"
+	"net"
+	//"net/http"
+	"time"
+
+	"github.com/espebra/filebin2/ds"
+
+	"github.com/dustin/go-humanize"
+)
+
+type ClientDao struct {
+	db *sql.DB
+}
+
+func (c *ClientDao) GetByIP(ip net.IP) (client ds.Client, found bool, err error) {
+	sqlStatement := "SELECT ip, asn, network, city, country, continent, proxy, requests, first_active_at, last_active_at, banned_at, banned_by FROM client WHERE ip = $1 LIMIT 1"
+	err = c.db.QueryRow(sqlStatement, ip.String()).Scan(&client.IP, &client.ASN, &client.Network, &client.City, &client.Country, &client.Continent, &client.Proxy, &client.Requests, &client.FirstActiveAt, &client.LastActiveAt, &client.BannedAt, &client.BannedBy)
+	if err != nil {
+		if err == sql.ErrNoRows {
+			client.IP = ip.String()
+			//client.Requests = 0
+			//now := time.Now().UTC()
+			//client.FirstActiveAt = now
+			//client.LastActiveAt = now
+			return client, false, nil
+		}
+		return client, false, err
+	}
+	// https://github.com/lib/pq/issues/329
+	client.FirstActiveAt = client.FirstActiveAt.UTC()
+	client.LastActiveAt = client.LastActiveAt.UTC()
+	client.FirstActiveAtRelative = humanize.Time(client.FirstActiveAt)
+	client.LastActiveAtRelative = humanize.Time(client.LastActiveAt)
+	if client.IsBanned() {
+		client.BannedAt.Time = client.BannedAt.Time.UTC()
+		client.BannedAtRelative = humanize.Time(client.BannedAt.Time)
+	}
+	return client, true, nil
+}
+
+func (c *ClientDao) GetByRemoteAddr(remoteAddr string) (client ds.Client, found bool, err error) {
+	// Parse the client IP address
+	host, _, err := net.SplitHostPort(remoteAddr)
+	var ip net.IP
+	if err == nil {
+		ip = net.ParseIP(host)
+	} else {
+		ip = net.ParseIP(remoteAddr)
+	}
+	if ip == nil {
+		return client, false, errors.New(fmt.Sprintf("Unable to parse remote addr %s", remoteAddr))
+	}
+
+	// Look up the client by IP
+	client, found, err = c.GetByIP(ip)
+	return client, found, err
+}
+
+func (c *ClientDao) Update(client *ds.Client) (err error) {
+	now := time.Now().UTC()
+	sqlStatement := "INSERT INTO client (ip, asn, network, city, country, continent, proxy, requests, first_active_at, last_active_at, banned_by) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, '') ON CONFLICT(ip) DO UPDATE SET asn=$11, network=$12, city=$13, country=$14, continent=$15, proxy=$16, requests=client.requests+1, last_active_at=$17 RETURNING ip, requests"
+	if err := c.db.QueryRow(sqlStatement, client.IP, client.ASN, client.Network, client.City, client.Country, client.Continent, client.Proxy, 1, now, now, client.ASN, client.Network, client.City, client.Country, client.Continent, client.Proxy, now).Scan(&client.IP, &client.Requests); err != nil {
+		return err
+	}
+	return err
+}
+
+func (c *ClientDao) GetAll() (clients []ds.Client, err error) {
+	sqlStatement := "SELECT ip, asn, network, city, country, continent, proxy, requests, first_active_at, last_active_at, banned_at, banned_by FROM client ORDER BY last_active_at ASC"
+	clients, err = c.clientQuery(sqlStatement)
+	return clients, err
+}
+
+func (c *ClientDao) GetByLastActiveAt(limit int) (clients []ds.Client, err error) {
+	sqlStatement := "SELECT ip, asn, network, city, country, continent, proxy, requests, first_active_at, last_active_at, banned_at, banned_by FROM client ORDER BY last_active_at ASC LIMIT $1"
+	clients, err = c.clientQuery(sqlStatement, limit)
+	return clients, err
+}
+
+func (c *ClientDao) GetByRequests(limit int) (clients []ds.Client, err error) {
+	sqlStatement := "SELECT ip, asn, network, city, country, continent, proxy, requests, first_active_at, last_active_at, banned_at, banned_by FROM client ORDER BY requests ASC LIMIT $1"
+	clients, err = c.clientQuery(sqlStatement, limit)
+	return clients, err
+}
+
+func (c *ClientDao) GetByBannedAt(limit int) (clients []ds.Client, err error) {
+	sqlStatement := "SELECT ip, asn, network, city, country, continent, proxy, requests, first_active_at, last_active_at, banned_at, banned_by FROM client ORDER BY banned_at ASC LIMIT $1"
+	clients, err = c.clientQuery(sqlStatement, limit)
+	return clients, err
+}
+
+func (c *ClientDao) clientQuery(sqlStatement string, params ...interface{}) (clients []ds.Client, err error) {
+	rows, err := c.db.Query(sqlStatement, params...)
+	if err != nil {
+		return clients, err
+	}
+	defer rows.Close()
+	for rows.Next() {
+		var client ds.Client
+		err = rows.Scan(&client.IP, &client.ASN, &client.Network, &client.City, &client.Country, &client.Continent, &client.Proxy, &client.Requests, &client.FirstActiveAt, &client.LastActiveAt, &client.BannedAt, &client.BannedBy)
+		if err != nil {
+			return clients, err
+		}
+		// https://github.com/lib/pq/issues/329
+		client.FirstActiveAt = client.FirstActiveAt.UTC()
+		client.LastActiveAt = client.LastActiveAt.UTC()
+		client.FirstActiveAtRelative = humanize.Time(client.FirstActiveAt)
+		client.LastActiveAtRelative = humanize.Time(client.LastActiveAt)
+		if client.IsBanned() {
+			client.BannedAt.Time = client.BannedAt.Time.UTC()
+			client.BannedAtRelative = humanize.Time(client.BannedAt.Time)
+		}
+		clients = append(clients, client)
+	}
+	return clients, nil
+}
+
+func (c *ClientDao) Ban(IPsToBan []string, banByRemoteAddr string) (err error) {
+	// Parse the IP address of the client that requested the ban
+	host, _, err := net.SplitHostPort(banByRemoteAddr)
+	var ip net.IP
+	if err == nil {
+		ip = net.ParseIP(host)
+	} else {
+		ip = net.ParseIP(banByRemoteAddr)
+	}
+	if ip == nil {
+		return errors.New(fmt.Sprintf("Unable to parse remote addr %s", banByRemoteAddr))
+	}
+	banByIP := ip.String()
+
+	// Loop over the IP addresses that will be banned
+	now := time.Now().UTC()
+	sqlStatement := "UPDATE client SET banned_at=$1, banned_by=$2 WHERE ip=$3 RETURNING ip"
+	var ret string
+	for _, ipToBan := range IPsToBan {
+		if err := c.db.QueryRow(sqlStatement, now, banByIP, ipToBan).Scan(&ret); err != nil {
+			return err
+		}
+	}
+	return err
+}

dbl/client_test.go

@@ -0,0 +1,88 @@
+package dbl
+
+import (
+	//"fmt"
+	"github.com/espebra/filebin2/ds"
+	"testing"
+	//"time"
+	"net"
+)
+
+func TestGetClientByIP(t *testing.T) {
+	dao, err := tearUp()
+	if err != nil {
+		t.Error(err)
+	}
+	defer tearDown(dao)
+
+	ip := "1.2.3.4"
+	client := &ds.Client{}
+	client.IP = ip
+	err = dao.Client().Update(client)
+	if err != nil {
+		t.Error(err)
+	}
+	if client.IP != ip {
+		t.Errorf("Was expecting client ip %s, got %s instead.", ip, client.IP)
+	}
+
+	dbClient, found, err := dao.Client().GetByIP(net.ParseIP(ip))
+	if err != nil {
+		t.Error(err)
+	}
+	if found == false {
+		t.Errorf("Expected found to be true as the client exists.")
+	}
+	if dbClient.IP != ip {
+		t.Errorf("Was expecting client ip %s, got %s instead.", ip, dbClient.IP)
+	}
+	if dbClient.Requests != 1 {
+		t.Errorf("Was expecting number of requests to be 1, got %d\n", dbClient.Requests)
+	}
+
+	err = dao.Client().Update(&dbClient)
+	if err != nil {
+		t.Error(err)
+	}
+	if dbClient.Requests != 2 {
+		t.Errorf("Was expecting the number of requests to be 2, not %d\n", dbClient.Requests)
+	}
+
+	dbClient2, found, err := dao.Client().GetByRemoteAddr(ip)
+	if err != nil {
+		t.Error(err)
+	}
+	err = dao.Client().Update(&dbClient2)
+	if err != nil {
+		t.Error(err)
+	}
+	if dbClient2.Requests != 3 {
+		t.Errorf("Was expecting number of requests to be 3, got %d\n", dbClient2.Requests)
+	}
+}
+
+func TestClientsGetAll(t *testing.T) {
+	dao, err := tearUp()
+	if err != nil {
+		t.Error(err)
+	}
+	defer tearDown(dao)
+
+	ips := []string{"192.168.0.1", "192.168.0.2", "10.0.0.10"}
+	for _, ip := range ips {
+		client := &ds.Client{}
+		client.IP = ip
+		err = dao.Client().Update(client)
+		if err != nil {
+			t.Error(err)
+		}
+	}
+
+	clients, err := dao.Client().GetAll()
+	if err != nil {
+		t.Error(err)
+	}
+	if len(clients) != len(ips) {
+		t.Errorf("Was expecting %d clients, got %d\n", len(ips), len(clients))
+	}
+}

dbl/db.go

@@ -15,6 +15,7 @@ type DAO struct {
 	fileDao        *FileDao
 	metricsDao     *MetricsDao
 	transactionDao *TransactionDao
+	clientDao      *ClientDao
 }
 
 // Init a database connection given
@@ -43,6 +44,7 @@ func Init(dbHost string, dbPort int, dbName, dbUser, dbPassword string) (DAO, er
 	dao.fileDao = &FileDao{db: db}
 	dao.metricsDao = &MetricsDao{db: db}
 	dao.transactionDao = &TransactionDao{db: db}
+	dao.clientDao = &ClientDao{db: db}
 	return dao, nil
 }
 
@@ -59,6 +61,8 @@ func (dao DAO) ResetDB() error {
 	sqlStatements := []string{
 		"DELETE FROM file",
 		"DELETE FROM bin",
+		"DELETE FROM client",
+		"DELETE FROM autonomous_system",
 		"DELETE FROM transaction"}
 
 	for _, s := range sqlStatements {
@@ -86,6 +90,10 @@ func (dao DAO) Transaction() *TransactionDao {
 	return dao.transactionDao
 }
 
+func (dao DAO) Client() *ClientDao {
+	return dao.clientDao
+}
+
 func (dao DAO) Status() bool {
 	if err := dao.db.Ping(); err != nil {
 		fmt.Printf("Database status check returned: %s\n", err.Error())

ds/client.go

@@ -1,28 +1,57 @@
 package ds
 
 import (
-	"fmt"
-	"net"
+	"database/sql"
+	//"fmt"
+	//"net"
+	"time"
 )
 
 type Client struct {
-	IP          net.IP `json:"ip"`
-	Fingerprint string `json:"fingerprint"`
-	Network     string `json:"network"`
-	City        string `json:"city"`
-	Country     string `json:"country"`
-	Continent   string `json:"continent"`
-	Proxy       bool   `json:"proxy"`
+	IP                    string       `json:"ip"`
+	ASN                   int          `json:"asn"`
+	Network               string       `json:"network"`
+	City                  string       `json:"city"`
+	Country               string       `json:"country"`
+	Continent             string       `json:"continent"`
+	Proxy                 bool         `json:"proxy"`
+	Requests              uint64       `json:"requests"`
+	FirstActiveAt         time.Time    `json:"first_active_at"`
+	FirstActiveAtRelative string       `json:"first_active_at_relative"`
+	LastActiveAt          time.Time    `json:"last_active_at"`
+	LastActiveAtRelative  string       `json:"last_active_at_relative"`
+	BannedAt              sql.NullTime `json:"banned_at"`
+	BannedAtRelative      string       `json:"banned_at_relative"`
+	BannedBy              string       `json:"banned_by"`
 }
 
-func (c *Client) String() string {
-	var s string
-	s += fmt.Sprintf("IP: %s, network: %s, continent: %s, country: %s, city: %s.", c.IP.String(), c.Network, c.Continent, c.Country, c.City)
-	if c.Fingerprint != "" {
-		s += fmt.Sprintf(" Fingerprint: %s.", c.Fingerprint)
+func (c *Client) IsBanned() bool {
+	if c.BannedAt.Valid {
+		if c.BannedAt.Time.IsZero() == false {
+			return true
+		}
 	}
-	if c.Proxy {
-		s += fmt.Sprintf(" IP is an anonymous proxy.")
+	return false
+}
+
+type AutonomousSystem struct {
+	ASN                   int          `json:"asn"`
+	Organization          string       `json:"organization"`
+	Network               string       `json:"network"`
+	Requests              uint64       `json:"requests"`
+	FirstActiveAt         time.Time    `json:"first_active_at"`
+	FirstActiveAtRelative string       `json:"first_active_at_relative"`
+	LastActiveAt          time.Time    `json:"last_active_at"`
+	LastActiveAtRelative  string       `json:"last_active_at_relative"`
+	BannedAt              sql.NullTime `json:"banned_at"`
+	BannedAtRelative      string       `json:"banned_at_relative"`
+}
+
+func (a *AutonomousSystem) IsBanned() bool {
+	if a.BannedAt.Valid {
+		if a.BannedAt.Time.IsZero() == false {
+			return true
+		}
 	}
-	return s
+	return false
 }

ds/metrics.go

@@ -32,6 +32,8 @@ type Metrics struct {
 	FileDownloadCount         uint64 `json:"file-download-count"`
 	FileDeleteCount           uint64 `json:"file-delete-count"`
 	BinDeleteCount            uint64 `json:"bin-delete-count"`
+	BinBanCount               uint64 `json:"bin-ban-count"`
+	BinLockCount              uint64 `json:"bin-lock-count"`
 	TarArchiveDownloadCount   uint64 `json:"tar-archive-download-count"`
 	ZipArchiveDownloadCount   uint64 `json:"zip-archive-download-count"`
 	FrontPageViewCount        uint64 `json:"front-page-view-count"`
@@ -134,6 +136,18 @@ func (m *Metrics) IncrBinDeleteCount() {
 	m.Unlock()
 }
 
+func (m *Metrics) IncrBinBanCount() {
+	m.Lock()
+	m.BinBanCount = m.BinBanCount + 1
+	m.Unlock()
+}
+
+func (m *Metrics) IncrBinLockCount() {
+	m.Lock()
+	m.BinLockCount = m.BinLockCount + 1
+	m.Unlock()
+}
+
 func (m *Metrics) IncrFileUploadInProgress() {
 	m.Lock()
 	m.FileUploadInProgress = m.FileUploadInProgress + 1