-
Notifications
You must be signed in to change notification settings - Fork 1
/
Copy pathT2_ipfs_pinner_0_0_3_beta.py
1476 lines (1225 loc) · 60.7 KB
/
T2_ipfs_pinner_0_0_3_beta.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
# Tool 002 - IPFS Pinner Script
# This script is to help people generally more easily bridge between local nodes (or servers) running on the same machine.
# Version: 0.0.3 beta
# Initial Author: LSJI07
# Contributor:
# Update 0.0.2
# Revised retrieve_small asset function.
## More informative logging and initial database revised to check includes file extension, pinned and added statuses. If present the script moves on.
# Revised retrieve_large asset function.
## More informative logging and initial database revised to check includes file extension, pinned and added statuses. If present the script moves on.
## Updated the script to run dag get before trying to pin add the contents.
## Update the script to honour the max file size threshold when retrieving large files.
## Update the function to give a brief timeout error and longer traceback on other errors.
# Update 0.0.3
# Updated the saved block function logic as that was not updating when restarting the script.
# General Information.
# The script takes some user input and setup and is intended to help most computer minded people setup the node local communications over RPC.
# When running the script, it will ask the user questions and setup a config file specific to that machine and user in the folder the script is run in.
# This script should work on UNIX and windows systems. I specifically avoided using imports that are not cross compatible.
# The intent is to assist users to pin IPFS content directly from their Ravencoin node to their local IPFS node.
# Install the required depends using the below. On windows systems ensure python is installed.
# Tested using Python version 3.10.
# Ensure the below dependancies are available.
# pip install bitcoinrpc
# pip install sqlite3
import sys
import argparse
import zmq
import struct
import binascii
import os
import logging # for info and debugging management.
import time # for debugging.
import bitcoinrpc
import traceback
import sqlite3
import select
import msvcrt # For Windows
import signal # For Unix-like systems
import json # For error handling
import re # for manipulating regular expressions.
import socket # Used to check for valid ipv4 addresses in configuration.
import requests # Used for making HTTP requests to the local IPFS node.
from tqdm import tqdm # for progress indication.
import configparser # for helping set ipfspinner.config on first run and thereafter.
from bitcoinrpc.authproxy import AuthServiceProxy, JSONRPCException
from datetime import datetime # for timestamping logs.
from requests.exceptions import RequestException
# Global variable to track script pause state
paused = False
# Signal handler for Ctrl+C on Unix-like systems
def handle_stop_signal(signal, frame):
global paused
if paused:
sys.exit(0)
else:
paused = True
print("\nPausing the script... Press 's' again to resume.")
# Function to handle keyboard input on Unix-like systems
def handle_keyboard_input_unix(args):
global paused
while True:
if not paused:
# Wait for keyboard input
r, _, _ = select.select([sys.stdin], [], [], 1)
if r:
key = sys.stdin.readline().strip()
if key == 's':
paused = True
print("\nPausing the script... Press 's' again to resume.")
else:
# Wait for keyboard input
r, _, _ = select.select([sys.stdin], [], [], 1)
if r:
key = sys.stdin.readline().strip()
if key == 's':
paused = False
print("\nResuming the script...")
break
# Function to handle keyboard input on Windows
def handle_keyboard_input_windows():
global paused
if msvcrt.kbhit():
key = msvcrt.getch()
if key == b's':
if paused:
paused = False
print("Resuming the script...")
else:
paused = True
print("Pausing the script... Press 's' again to resume.")
elif key == b'q':
print("Stopping the script...")
sys.exit(0)
def is_valid_ipv4_address(address):
try:
socket.inet_pton(socket.AF_INET, address)
except AttributeError:
try:
socket.inet_aton(address)
except socket.error:
return False
return address.count('.') == 3
except socket.error:
return False
return True
def setup_config_settings():
logger = logging.getLogger("ipfspinner")
config = configparser.ConfigParser()
if not os.path.exists('ipfspinner.config'):
print(r"""
@@@@@@@@@@@@@@@@@@@@@@@@@@@@((((((@@@@@@@@@@@@@@@
@@@@@@@@@@@@@@@@@@@@@@@(((((((***@@(@@@@@@@@@@@@@
@@@@@@@@@@@@@@@@@@@@(((((((@((((%***%*****@@@@@@@
@@@@@@@@@@@@@@@@@@@%%%%%%%%@(@%%%@@@((*********@@
@@@@@@@@@@@@@@@@@@%%%@(((@%%%%%%%%%%%%@@@@@@@@@@@
@@@@@@@@@@@@@@@@@(((((&%%%%%%%%@@******@@@@@@@@@@
@@@@@@@@@@@@@@@@((@%%%%@@***************@@@@@@@@@
@@@@@@@@@@@@@@@@************************@@@@@@@@@
@@@@@@@@@@@@@%%(((((((((((((*************@@@@@@@@
@@@@@@@@@@@@@%@%(((((((((((((((((((((((((@@@@@@@@
@@@@@@@@@@@@%%%%%((((((((((((((((((((((((%%%@@@@@
@@@@@@@@@@@@%%%@%%(((((((((((((((((((((((%%%%@@@@
@@@@@@@@@@@%%%%@%%%@((((((((((((((((((((@%%%%@@@@
@@@@@@@@@@@%%%%%%%%%@(((((((((((((((((((%%%%@@@@@
@@@@@@@@@@%%%%%%@%%%%@(((((((((((((((((@%%%@@@@@@
@@@@@@@@@%%%%%%%%%%%%%@((((((((((((((((@%%%@@@@@@
@@@@@@@@@%%%%%%%%@%%%%%@(((((((((((((((%%%@@@@@@@
@@@@@@@@%%%%%%%%%%%%%%%%%(((((((((((((@%%@@@@@@@@
@@@@@@@@%%%%%%%%%%%%%%%%%%((((((((((((%%%@@@@@@@@
@@@@@@@%%%%%%%%%%%@%%%%%%%%(((((((((((%%@@@@@@@@@
@@@@@@@%%%%%%%%%%%%%%%%%%%%%(((((((((@%@@@@@@@@@@
@@@@@@%%%%%%%%%%%%%@%%%%%%%%%@(((((((%%@@@@@@@@@@
@@@@@@%%%%%%%%%%%%%%%%%%%%%%%%@(((((&%@@@@@@@@@@@
@@@@@%%%%%%%%%%%%%%%%%%%%%%%%%%@((((@@@@@@@@@@@@@
@@@@@%%%%%%%%%%%%%%%@%%%%%%%%%%%@(((%@@@@@@@@@@@@
@@@@%%%%%%%%%%%%%%%%%%%%%%%%%%%%%@(@@@@@@@@@@@@@@
@@@%%%%%%%%%%%%%%%%%%@@@@@@@@@@@@@@@@@@@@@@@@@@@@
@@@%%%%%%%%%%%%%%%@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
@@%%%%%%%%%%%%@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
@@%%%%%%%%%@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
@%%%%%%@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
@%%%@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
(Script courtesy of RavenAngels)
Made by community for community.
""")
print()
print("Welcome to the Ravencoin IPFS Pinner. Let's get started!")
print()
print()
print("We need to get your IPFS Pinner Settings configured.")
print()
config = prompt_ravencoin_settings(config)
prompt_ravencoin_mode(config)
prompt_ravencoin_rpc(config)
prompt_ipfs_settings(config)
print()
print("You can view the ipfspinner.config information in the generated file. Delete the ipfspinner.config file to reset the config information and reset this initial configuration process.")
print()
with open('ipfspinner.config', 'w') as f:
config.write(f)
else:
config.read('ipfspinner.config')
return config
def prompt_ravencoin_settings(config):
print()
print("Welcome to the Ravencoin IPFS Pinner. Let's get started!")
print()
print()
print("We need to get your IPFS Pinner Settings configured.")
print()
print("Enter 'true', 'false' or the required setting. 'default' is also valid.")
print()
config['ipfspinner'] = {}
noblockscan_input = input("01. Ravencoin node block scanning over RPC for Asset Information (True/False/Default): ")
if noblockscan_input.lower() == 'true':
config['ipfspinner']['noblockscan'] = 'True'
elif noblockscan_input.lower() == 'false':
config['ipfspinner']['noblockscan'] = 'False'
elif noblockscan_input.lower() == 'default':
config['ipfspinner']['noblockscan'] = 'False'
else:
print("Invalid input. You must enter 'True', 'False', or 'Default'")
print()
print("*INFO* You must run ravend or raven-qt using '--zmqpubrawtx=tcp://127.0.0.1:28766' as well as an IPFS node for ongoing digital asset IPFS hash monitoring: ")
print()
config['ipfspinner'] = {}
nozmqwatch_input = input("02. Ravencoin node using ZMQ to watch Assets and pass new asset IPFS hash information (true/false/default): ")
if nozmqwatch_input.lower() == 'true':
config['ipfspinner']['nozmqwatch'] = 'True'
elif nozmqwatch_input.lower() == 'false':
config['ipfspinner']['nozmqwatch'] = 'False'
elif nozmqwatch_input.lower() == 'default':
config['ipfspinner']['nozmqwatch'] = 'False'
else:
print("Invalid input. You must enter 'True', 'False', or 'Default'")
config['ipfspinner'] = {}
safemode_input = input("03. Safe mode (True/False/Default): ")
if safemode_input.lower() == 'true':
config['ipfspinner']['safemode'] = 'True'
elif safemode_input.lower() == 'false':
config['ipfspinner']['safemode'] = 'False'
elif safemode_input.lower() == 'default':
config['ipfspinner']['safemode'] = 'False'
else:
print("Invalid input. You must enter 'True', 'False', or 'Default'")
config['ipfspinner'] = {}
debug_input = input("04. Debug mode (True/False/Default): ")
if debug_input.lower() == 'true':
config['ipfspinner']['debug'] = 'True'
elif debug_input.lower() == 'false':
config['ipfspinner']['debug'] = 'False'
elif debug_input.lower() == 'default':
config['ipfspinner']['debug'] = 'False'
else:
print("Invalid input. You must enter 'True', 'False', or 'Default'")
while True:
block_input = input("05. Enter the block to start scanning Ravencoin FROM for the initial sync (default block: 435456 for Mainnet, testnet varies): ")
if block_input == '':
block_input = '435456'
elif block_input.lower() == 'default':
block_input = '435456'
elif not block_input.isdigit():
print("Invalid input. Block must be a number.")
continue
config['ipfspinner']['block'] = block_input
break
while True:
folder_input = input("06. Input folder path to save files related to digital asset to (default: /ipfsfiles): ")
if folder_input == '':
folder_input = '/ipfsfiles'
elif folder_input.lower() == 'default':
folder_input = '/ipfsfiles'
elif not os.path.isdir(folder_input):
print("Invalid input. Folder path must be an existing directory.")
continue
config['ipfspinner']['folder'] = folder_input
break
while True:
threshold_input = input("07. Enter the new filesize waterline in MB to differentiate between large and small files. (default is circa 15MB): ")
if threshold_input == '':
threshold_input = '15'
elif not threshold_input.isdigit():
print("Invalid input. Threshold must be a number.")
continue
threshold_bytes = int(threshold_input) * 1024 * 1024 # Convert MB to bytes
config['ipfspinner']['FILESIZE_THRESHOLD'] = str(threshold_bytes)
break
while True:
large_threshold_input = input("08. Enter the new max large file threshold in MB (default is 500MB): ")
if large_threshold_input == '':
large_threshold_input = '500'
elif not large_threshold_input.isdigit():
print("Invalid input. Threshold must be a number.")
continue
large_threshold_bytes = int(large_threshold_input) * 1024 * 1024 # Convert MB to bytes
config['ipfspinner']['MAX_LARGE_FILE_THRESHOLD'] = str(large_threshold_bytes)
break
return config
def prompt_ravencoin_mode(config):
print("Ravencoin Node Settings")
while True:
mode_input = input("01. Mainnet or Testnet mode? Enter 'default' or 'testnet': ")
if mode_input == '':
mode_input = ''
elif mode_input.lower() == 'default':
mode_input = ''
elif mode_input.lower() == 'testnet':
mode_input = '-testnet'
elif mode_input.lower() == '-testnet':
pass
else:
print("Invalid input. Enter '-testnet' for testnet, 'default' for main.")
continue
config['ravencoin'] = {}
config['ravencoin']['mode'] = mode_input
break
def prompt_ravencoin_rpc(config):
print("***INFO***")
print()
print("*INFO* The below settings must be entered exactly as per raven.config.")
print("*INFO* We do recommend the the below indexes are enable in your Ravencoin node in raven.config to allow the node access to the full dataset")
print("*INFO* raven.config can be found in raven-qt by going to the following menus and options.")
print("'Menu (Wallet) > Menu (Options) > Tab (Window) > Button (Open Configuration File)'")
print()
print("txindex=1")
print("assetindex=1")
print("timestampindex=1")
print("addressindex=1")
print("spentindex=1")
print()
print("*INFO* We do recommend the the below are enabled, or '=1', in Ravencoin node using raven-qt node in raven.config to enable RPC calls. This is not normally required using ravend.")
print()
print("server=1")
print()
print("*INFO* We would suggest you consider adding 'rpcallowip' as well. Most people will be running RPC on the local machine and this binds the RPC node to the localhost.")
print()
print("rpcallowip=127.0.0.1")
print()
print("*INFO* Note this script is designed to be used on a single machine and interface between the local Ravencoin node and local IPFS node.")
print()
print("*INFO* We do not recommend using unencrypted RPC over the external internet as user names and passwords would be exposed.")
print()
while True:
rpc_port_input = input("02. Enter the Ravencoin node rpcport (default for mainnet: 8766): ")
if rpc_port_input == '':
rpc_port_input = '8766'
elif rpc_port_input.lower() == 'default':
rpc_port_input = '8766'
elif not rpc_port_input.isdigit() or int(rpc_port_input) not in range(1, 65536):
print("Invalid input. Port number must be an integer between 1 and 65535.")
continue
config['ravencoin']['rpc_port'] = rpc_port_input
break
while True:
rpc_user_input = input("03. Enter the rpcuser: ")
if rpc_user_input == '':
print("Invalid input. rpcuser cannot be empty.")
continue
rpc_pass_input = input("04. Enter the rpcpassword: ")
if rpc_pass_input == '':
print("Invalid input. rpcpassword cannot be empty.")
continue
config['ravencoin']['rpc_user'] = rpc_user_input
config['ravencoin']['rpc_pass'] = rpc_pass_input
break
def prompt_ipfs_settings(config):
print()
print("Now we need to set the IPFS Node Settings")
print()
config['ipfs'] = {}
while True:
ipfs_host_input = input("01. Enter the IPFS node IP address (default: localhost): ")
if ipfs_host_input == '':
ipfs_host_input = 'localhost'
elif ipfs_host_input.lower() == 'default':
ipfs_host_input = 'localhost'
elif not is_valid_ipv4_address(ipfs_host_input):
print("Invalid input. IP address must be in IPv4 format or 'default'.")
continue
config['ipfs']['host'] = ipfs_host_input
break
while True:
ipfs_port_input = input("02. Enter the IPFS port (default: 5001): ")
if ipfs_port_input == '':
ipfs_port_input = '5001'
elif ipfs_port_input.lower() == 'default':
ipfs_port_input = '5001'
elif not ipfs_port_input.isdigit() or not (0 <= int(ipfs_port_input) <= 65535):
print("Invalid input. Port must be an integer between 0 and 65535.")
continue
config['ipfs']['port'] = ipfs_port_input
break
while True:
ipfs_timeout_input = input("03. Enter the desired timeout to be used when waiting for IPFS files to retrieve. In seconds (default: 60): ")
if ipfs_timeout_input == '':
ipfs_timeout_input = '60'
elif ipfs_timeout_input.lower() == 'default':
ipfs_timeout_input = '60'
elif not ipfs_timeout_input.isdigit() or int(ipfs_timeout_input) <= 0:
print("Invalid input. Timeout must be a positive integer or 'default'.")
continue
config['ipfs']['timeout'] = ipfs_timeout_input
break
def read_config_settings():
logger = logging.getLogger("ipfspinner")
config = configparser.ConfigParser()
config.read('ipfspinner.config')
settings = {}
settings['noblockscan'] = config.getboolean('ipfspinner', 'noblockscan', fallback=False)
settings['nozmqwatch'] = config.getboolean('ipfspinner', 'nozmqwatch', fallback=False)
settings['safemode'] = config.getboolean('ipfspinner', 'safemode', fallback=False)
settings['FIRST_ASSET_BLOCK'] = config.getint('ipfspinner', 'block', fallback=435456)
settings['folder'] = config.get('ipfspinner', 'folder', fallback=None)
settings['debug'] = config.getboolean('ipfspinner', 'debug', fallback=False)
settings['mode'] = config.get('ravencoin', 'mode', fallback='')
settings['rpc_port'] = config.getint('ravencoin', 'rpc_port', fallback='8766')
settings['rpc_user'] = config.get('ravencoin', 'rpc_user', fallback='rpcuser')
settings['rpc_pass'] = config.get('ravencoin', 'rpc_pass', fallback='rpcpass555')
settings['ipfs_host'] = config.get('ipfs', 'host', fallback='localhost')
settings['ipfs_port'] = config.getint('ipfs', 'port', fallback=5001)
settings['ipfs_timeout'] = int(config.get('ipfs', 'timeout', fallback=60))
settings['filesize_threshold'] = int(config.get('ipfspinner', 'FILESIZE_THRESHOLD', fallback=15000000))
return settings
logger = logging.getLogger("ipfspinner")
config = setup_config_settings()
# INFO Ravencoin RPC functions
def get_rpc_connection(config):
from bitcoinrpc.authproxy import AuthServiceProxy, JSONRPCException
rpc_user = config['ravencoin']['rpc_user']
rpc_pass = config['ravencoin']['rpc_pass']
rpc_port = config['ravencoin']['rpc_port']
connection = "http://%s:%[email protected]:%s" % (rpc_user, rpc_pass, rpc_port)
rpc_conn = AuthServiceProxy(connection)
return rpc_conn
rpc_connection = get_rpc_connection(config)
def get_blockinfo(num):
hash = rpc_connection.getblockhash(num)
blockinfo = rpc_connection.getblock(hash)
return(blockinfo)
def get_block(hash):
blockinfo = rpc_connection.getblock(hash)
return(blockinfo)
def get_rawtx(tx):
txinfo = rpc_connection.getrawtransaction(tx)
return(txinfo)
def get_bci():
bci = rpc_connection.getblockchaininfo()
return(bci)
def decode_rawtx(txdata):
logger.debug("Decoding transaction: %s", txdata)
try:
txjson = rpc_connection.decoderawtransaction(txdata)
return txjson
except Exception as e:
logger.exception("Error occurred while decoding raw transaction: %s", e)
raise
def decode_script(script):
try:
scriptinfo = rpc_connection.decodescript(script)
return scriptinfo
except Exception as e:
logger.exception("Error occurred while decoding script: %s", e)
raise
# Create a new SQLite database and table
def setup_database():
dbconn = sqlite3.connect('ravencoin.db')
dbc = dbconn.cursor()
dbc.execute('''
CREATE TABLE IF NOT EXISTS assets (
id INTEGER PRIMARY KEY,
block_height INTEGER,
asset_name TEXT,
ipfs_hash TEXT,
filename TEXT,
file_extension TEXT,
file_size INTEGER,
downloaded BOOLEAN,
added BOOLEAN,
pinned BOOLEAN,
UNIQUE(asset_name, ipfs_hash)
)
''')
dbconn.commit()
return dbconn, dbc
# Used to show better human readable size indication than bytes in logging.
def format_file_size(size_in_bytes):
if size_in_bytes < 1024:
return f"{size_in_bytes} B"
elif size_in_bytes < 1024 ** 2:
return f"{size_in_bytes / 1024:.2f} KB"
elif size_in_bytes < 1024 ** 3:
return f"{size_in_bytes / (1024 ** 2):.2f} MB"
elif size_in_bytes < 1024 ** 4:
return f"{size_in_bytes / (1024 ** 3):.2f} GB"
else:
return f"{size_in_bytes / (1024 ** 4):.2f} TB"
def estimate_ipfs_storage_size(filesize_threshold, dbc):
# Read the IPFSpinner configuration from ipfspinner.config file
config = configparser.ConfigParser()
config.read("ipfspinner.config")
dbc.execute('SELECT file_size FROM assets WHERE file_size IS NOT NULL AND file_size <= ?', (config['ipfspinner'].getint('max_large_file_threshold'),))
file_sizes = dbc.fetchall()
total_size = sum(file_size[0] for file_size in file_sizes)
# Apply 10% increase for recursive files and folders
total_size *= 1.1
# If the total size is less than the file size threshold, use the file size threshold
if total_size < filesize_threshold:
total_size = filesize_threshold
logging.info(f"Estimated total IPFS storage size: {format_file_size(total_size)}")
def asset_handler(asset_script, block_num, config, args):
global dbc, dbconn
asset_file = asset_to_file(asset_script.get('asset_name'))
try:
log_asset_details(asset_script, asset_script.get('asset_name'), asset_file)
if asset_script.get('hasIPFS'):
ipfs_hash = asset_script.get('ipfs_hash')
log_ipfs_hash(ipfs_hash)
start_time = time.time()
# Open the database connection
with sqlite3.connect('ravencoin.db') as dbconn:
dbc = dbconn.cursor()
# Check if the size is already available in the database
dbc.execute('SELECT file_size FROM assets WHERE asset_name = ? AND ipfs_hash = ?', (asset_script.get('asset_name'), ipfs_hash))
result = dbc.fetchone()
if result is not None and result[0] is not None and result[0] > 0:
size = result[0]
else:
# Check if the IPFS hash has been encountered multiple times with None size
dbc.execute('SELECT COUNT(*) FROM assets WHERE ipfs_hash = ? AND file_size IS NULL', (ipfs_hash,))
count_result = dbc.fetchone()
if count_result is not None and count_result[0] >= 2:
size = None # Skip checking the IPFS hash again if encountered multiple times with None size
else:
size = get_ipfs_file_size(ipfs_hash, config)
dbc.execute('''
UPDATE assets SET file_size = ? WHERE asset_name = ? AND ipfs_hash = ?
''', (size, asset_script.get('asset_name'), ipfs_hash))
dbconn.commit()
# Format the size for logging
if size is not None:
formatted_size = format_file_size(size)
logging.info(f"Size of IPFS file for {asset_script.get('asset_name')}: {formatted_size}")
else:
logging.info(f"Size of IPFS file for {asset_script.get('asset_name')}: Missing")
filename = generate_file_name(asset_file, ipfs_hash)
logging.info(f"Filename: {filename}")
# Insert or update the database record
dbc.execute('''
INSERT INTO assets (block_height, asset_name, ipfs_hash, filename, file_size, downloaded, added, pinned)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(asset_name, ipfs_hash) DO UPDATE SET
file_size = excluded.file_size,
downloaded = excluded.downloaded,
added = excluded.added,
pinned = excluded.pinned
''', (block_num, asset_script.get('asset_name'), ipfs_hash, filename, size, False, False, False))
dbconn.commit()
end_time = time.time()
elapsed_time = end_time - start_time
logging.info(f"Elapsed time for {asset_script.get('asset_name')}: {elapsed_time} seconds.")
except Exception as e:
logging.error("An error occurred in asset_handler: %s", str(e))
logging.error(traceback.format_exc())
def log_asset_details(asset_script, asset_name, asset_file):
logging.debug("Type: %s", asset_script.get('type'))
logging.debug("Asset: %s", asset_name)
logging.debug("Asset File: %s", asset_file)
logging.debug(asset_script.get('amount'))
logging.debug(asset_script.get('units'))
logging.debug("Reissuable: %s", str(asset_script.get('reissuable')))
logging.debug("Has IPFS: %s", str(asset_script.get('hasIPFS')))
def log_ipfs_hash(ipfs_hash):
logging.debug(ipfs_hash)
def insert_asset_to_database(asset_name, ipfs_hash, block_num, filename, size):
dbc.execute('''
SELECT * FROM assets WHERE asset_name = ? AND ipfs_hash = ?
''', (asset_name, ipfs_hash))
result = dbc.fetchone()
if result is None:
file_extension = ""
dbc.execute('''
INSERT INTO assets (block_height, asset_name, ipfs_hash, filename, file_extension, file_size, downloaded, added, pinned)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
''', (block_num, asset_name, ipfs_hash, filename, file_extension, size, False, False, False))
dbconn.commit()
return True
else:
return False
# Retrieval functions
def update_asset_status(asset_name, ipfs_hash, dbc, dbconn, pinned=False, added=False):
"""Update the status of an asset in the database.
Args:
asset_name (str): The name of the asset.
ipfs_hash (str): The IPFS hash of the asset.
dbc: The database connection object.
dbconn: The database connection.
pinned (bool): The status of whether the asset is pinned. Default: False.
added (bool): The status of whether the asset is added. Default: False.
"""
dbc.execute('UPDATE assets SET pinned = ?, added = ? WHERE asset_name = ? AND ipfs_hash = ?', (pinned, added, asset_name, ipfs_hash))
dbconn.commit()
def retrieve_asset(asset_name, ipfs_hash, args, json_only_check, filesize_threshold, dbc, dbconn):
try:
if not ipfs_hash.startswith("Qm") or len(ipfs_hash) != 46:
logging.error(f"Error: Invalid IPFS hash format for asset {asset_name}, IPFS hash {ipfs_hash}. Skipping asset.")
update_asset_status(asset_name, ipfs_hash, dbc, dbconn, pinned=False, added=False)
return
dbc.execute('SELECT file_size FROM assets WHERE ipfs_hash = ?', (ipfs_hash,))
result = dbc.fetchone()
if result is not None:
size = result[0]
else:
size = get_ipfs_file_size(ipfs_hash, config)
dbc.execute('UPDATE assets SET file_size = ? WHERE ipfs_hash = ?', (size, ipfs_hash))
dbconn.commit()
if size is not None:
dbc.execute('SELECT pinned, added FROM assets WHERE ipfs_hash = ?', (ipfs_hash,))
result = dbc.fetchone()
if result is not None and result[0] and result[1]:
logging.debug(f"IPFS hash {ipfs_hash} is already pinned and added. Skipping pinning.")
return
pinned_objects = ipfs_pin_ls(ipfs_hash)
if pinned_objects is not None and 'Keys' in pinned_objects:
keys = pinned_objects['Keys']
if ipfs_hash in keys:
identical_hashes = [k for k, v in keys.items() if v['Type'] == keys[ipfs_hash]['Type']]
dbc.execute('UPDATE assets SET pinned = ?, added = ? WHERE ipfs_hash IN ({})'.format(','.join(['?'] * len(identical_hashes))), (True, True, *identical_hashes))
dbconn.commit()
logging.info(f"IPFS hash for asset {asset_name} has already been pinned and added.")
return
else:
logging.info(f"IPFS hash {ipfs_hash} is not currently pinned. Continuing to retrieve content...")
if args.safemode:
print(f"Retrieving small file content for {asset_name} (Safe Mode)...")
retrieve_small_file(asset_name, ipfs_hash, json_only_check, args, dbc, dbconn)
else:
if size <= filesize_threshold:
print(f"Retrieving small file content for {asset_name}...")
retrieve_small_file(asset_name, ipfs_hash, json_only_check, args, dbc, dbconn)
else:
print(f"Retrieving large file content for {asset_name}...")
retrieve_large_file(asset_name, ipfs_hash, dbc, dbconn)
else:
logging.error(f"Error: File size is not available for IPFS hash {ipfs_hash}. Skipping asset.")
update_asset_status(asset_name, ipfs_hash, dbc, dbconn, pinned=False, added=False)
except IPFSException as e:
logging.error(f"Error: Failed to retrieve IPFS content for {ipfs_hash}")
logging.error(str(e))
update_asset_status(asset_name, ipfs_hash, dbc, dbconn, pinned=False, added=False)
except Exception as ex:
logging.error(f"Error: Failed to retrieve IPFS content for {ipfs_hash}")
logging.error(str(ex))
update_asset_status(asset_name, ipfs_hash, dbc, dbconn, pinned=False, added=False)
def retrieve_small_file(asset_name, ipfs_hash, json_only_check, args, dbc, dbconn):
try:
# Get asset name from the asset script
asset_name = asset_name
# Check if the file extension already exists in the database and if it is pinned and added
dbc.execute('SELECT file_extension, pinned, added FROM assets WHERE asset_name = ? AND ipfs_hash = ?', (asset_name, ipfs_hash))
result = dbc.fetchone()
if result is not None:
file_extension, pinned, added = result
if file_extension is not None and pinned and added:
logging.info(f"File contents is marked as {file_extension} and already exists, the asset contents is pinned and added for asset {asset_name} and IPFS hash {ipfs_hash}.")
return
# Retrieve content from IPFS using ipfs_dag_get
try:
content_bytes = ipfs_dag_get(ipfs_hash)
content = content_bytes.decode('utf-8')
logging.debug(f"Content retrieved successfully:\n{content}")
except IPFSException as e:
raise Exception(f"Error retrieving content from IPFS: {str(e)}")
# Fetch IPFS size from the database
dbc.execute('SELECT file_size FROM assets WHERE asset_name = ? AND ipfs_hash = ?', (asset_name, ipfs_hash))
result = dbc.fetchone()
if result is None:
raise Exception(f"Asset {asset_name} with IPFS hash {ipfs_hash} not found in the database.")
size = result[0]
# Determine the content type based on its structure
if content.startswith(('{', '[')) and content.rstrip().endswith(('}', ']')):
logging.info("Content is JSON")
new_file_extension = 'json'
else:
logging.info("Content is CAR")
new_file_extension = 'car'
# Update the database with the retrieved file extension and mark as added
logging.info(f"Updating database with file extension {new_file_extension} and marking as added for asset {asset_name} and IPFS hash {ipfs_hash}")
dbc.execute('UPDATE assets SET file_extension = ?, added = ? WHERE asset_name = ? AND ipfs_hash = ?', (new_file_extension, True, asset_name, ipfs_hash))
dbconn.commit()
# Pin the asset if it is not already pinned
dbc.execute('SELECT pinned FROM assets WHERE asset_name = ? AND ipfs_hash = ?', (asset_name, ipfs_hash))
result = dbc.fetchone()
if result is not None and result[0]:
logging.info(f"Asset {asset_name} with IPFS hash {ipfs_hash} is already pinned.")
else:
logging.info(f"Pinning asset {asset_name} with IPFS hash {ipfs_hash}")
try:
ipfs_pin_add(ipfs_hash)
dbc.execute('UPDATE assets SET pinned = ? WHERE asset_name = ? AND ipfs_hash = ?', (True, asset_name, ipfs_hash))
dbconn.commit()
except Exception as e:
logging.error(f"Error pinning asset {asset_name} with IPFS hash {ipfs_hash}: {str(e)}")
except Exception as e:
logging.error(f"Error: Unable to retrieve IPFS file for asset {asset_name}. Reason: {str(e)}")
check_missing_asset_info(asset_name, ipfs_hash)
def retrieve_large_file(asset_name, ipfs_hash, dbc, dbconn):
try:
# Read the IPFS configuration from ipfspinner.config file
config = configparser.ConfigParser()
config.read("ipfspinner.config")
ipfs_timeout = config.getint("ipfs", "timeout")
max_large_file_threshold = int(config.get("ipfspinner", "max_large_file_threshold"))
ipfs_timeout = int(ipfs_timeout) # Ensure ipfs_timeout is an integer
# Check if the file extension already exists in the database and if it is pinned and added
dbc.execute('SELECT file_extension, pinned, added, file_size FROM assets WHERE asset_name = ? AND ipfs_hash = ?', (asset_name, ipfs_hash))
result = dbc.fetchone()
if result is not None:
file_extension, pinned, added, file_size = result
if file_extension is not None and pinned and added:
logging.info(f"File content is already marked as {file_extension}, and the asset is pinned and added for asset {asset_name} and IPFS hash {ipfs_hash}.")
return
# Check if the file size exceeds the maximum threshold
if file_size is not None and file_size > max_large_file_threshold:
logging.info(f"Skipping retrieval of asset {asset_name} with IPFS hash {ipfs_hash} due to its large size.")
logging.info(f"File size: {format_file_size(file_size)}, Maximum threshold: {format_file_size(max_large_file_threshold)}")
dbc.execute('UPDATE assets SET pinned = ?, added = ? WHERE asset_name = ? AND ipfs_hash = ?', (False, False, asset_name, ipfs_hash))
dbconn.commit()
return
# Retrieve file content from IPFS using ipfs_dag_get()
file_content = ipfs_dag_get(ipfs_hash)
# Pin the asset if it is not already pinned
dbc.execute('SELECT pinned FROM assets WHERE asset_name = ? AND ipfs_hash = ?', (asset_name, ipfs_hash))
result = dbc.fetchone()
if result is not None and result[0]:
logging.info(f"Asset {asset_name} with IPFS hash {ipfs_hash} is already pinned.")
else:
logging.info(f"Pinning asset {asset_name} with IPFS hash {ipfs_hash}")
ipfs_pin_add(ipfs_hash)
dbc.execute('UPDATE assets SET pinned = ? WHERE asset_name = ? AND ipfs_hash = ?', (True, asset_name, ipfs_hash))
dbconn.commit()
# Set the file extension to 'car'
file_extension = 'car'
logging.info("File content is CAR")
# Update the database to indicate that the file has been pinned and added
dbc.execute('UPDATE assets SET pinned = ?, added = ?, file_extension = ? WHERE asset_name = ? AND ipfs_hash = ?', (True, True, file_extension, asset_name, ipfs_hash))
dbconn.commit()
except IPFSException as e:
logging.error(f"Error: Unable to retrieve IPFS file for asset {asset_name}. Reason: {str(e)}")
except Exception as e:
logging.error(f"Error: Unable to retrieve IPFS file for asset {asset_name}. Reason: {str(e)}")
logging.error("Error details:", exc_info=True)
# Update the database to indicate that the file has not been retrieved
dbc.execute('UPDATE assets SET pinned = ?, added = ? WHERE asset_name = ? AND ipfs_hash = ?', (False, False, asset_name, ipfs_hash))
dbconn.commit()
def generate_file_name(asset_file, ipfs_hash):
filename = asset_file + '_' + ipfs_hash
# Replace specific characters with their URL-encoded values
filename = filename.replace('/', '%2F')
filename = filename.replace('*', '%2A')
filename = filename.replace('&', '%26')
filename = filename.replace('?', '%3F')
filename = filename.replace(':', '%3A')
filename = filename.replace('=', '%3D')
# Remove potential special characters from the filename
filename = re.sub(r'[^\w\.-]', '_', filename)
return filename
# This function is called when the script is unable to fetch an asset from IPFS and creates a text file list of missing hashes, avoiding duplicates.
def check_missing_asset_info(asset_name, ipfs_hash):
missing_hashes_path = os.path.join(os.getcwd(), "missing_hashes")
os.makedirs(missing_hashes_path, exist_ok=True)
filename = os.path.join(missing_hashes_path, "missing_IPFS_hashes.txt")
dbc.execute('''
SELECT downloaded, added, pinned FROM assets WHERE asset_name = ? AND ipfs_hash = ?
''', (asset_name, ipfs_hash))
result = dbc.fetchone()
if result is not None:
downloaded, added, pinned = result
else:
logging.error(f"Error: Asset {asset_name} with IPFS hash {ipfs_hash} not found in the database.")
return
updated_lines = []
status_line = f"{asset_name}: {ipfs_hash} - Downloaded: {downloaded}, Added: {added}, Pinned: {pinned}"
if os.path.isfile(filename):
with open(filename, 'r') as f:
for line in f:
line = line.strip()
if line:
parts = line.split(':')
if len(parts) >= 2:
file_asset_name, file_ipfs_hash = parts[0].strip(), parts[1].strip()
if file_asset_name != asset_name or file_ipfs_hash != ipfs_hash:
updated_lines.append(line)
updated_lines.append(status_line)
updated_lines.append(f"Size not found for asset {asset_name} with IPFS hash {ipfs_hash}")
with open(filename, 'w') as f:
f.write('\n'.join(updated_lines))
logging.info(f"Updated status for {asset_name}: {ipfs_hash} in {filename}")
# INFO IPFS HTTP based functions
class IPFSException(Exception):
"""Exception raised for errors related to IPFS operations."""
pass
def add_ipfs_peers(config):
peers = [
("RavencoinIPFS", "12D3KooWBNqVomfLbFk16gdu8azcQEyg6RcRFWfp2QxSztdiC7iM", "/dnsaddr/ravencoinipfs.com/p2p/12D3KooWBNqVomfLbFk16gdu8azcQEyg6RcRFWfp2QxSztdiC7iM"),
("MangoFarmAssets", "12D3KooWRjhU28ez8xGN7477oiG67NbUpL9owAkWpwGZct3PdUpe", "/dnsaddr/ravencoinipfs.com/p2p/12D3KooWRjhU28ez8xGN7477oiG67NbUpL9owAkWpwGZct3PdUpe"),
("RavencoinIPFS Gateway 1", "12D3KooWSxmtT3azrbtDnadxC196HS6QoU2dNJvvyzAFGfkZPiPB", "/dnsaddr/ravencoinipfs-gateway.com/p2p/12D3KooWSxmtT3azrbtDnadxC196HS6QoU2dNJvvyzAFGfkZPiPB"),
("RavencoinIPFS Gateway 2", "12D3KooWAw2gTLa2LXhnx6tkhJcdPvvaYH82R6m8dxttDCPEkCtm", "/dnsaddr/ravencoinipfs-gateway.com/p2p/12D3KooWAw2gTLa2LXhnx6tkhJcdPvvaYH82R6m8dxttDCPEkCtm"),
("RavencoinIPFS Gateway 3", "12D3KooWS1ehpyirKUJma8tkVaMennGTCG9E822CXuWXnJ37YBqL", "/dnsaddr/ravencoinipfs-gateway.com/p2p/12D3KooWS1ehpyirKUJma8tkVaMennGTCG9E822CXuWXnJ37YBqL")
]
existing_peers = set()
try:
url = f"http://{config['ipfs']['host']}:{config['ipfs']['port']}/api/v0/bootstrap/list"
response = requests.post(url)
response.raise_for_status()
result = response.json()
if 'Peers' in result:
existing_peers = set(result['Peers'])
except requests.exceptions.RequestException as e:
logging.warning("Failed to retrieve existing IPFS peers from the bootstrap list")
logging.warning("Error: %s", str(e))
for peer_name, peer_id, peer_address in peers:
if peer_address not in existing_peers:
try:
url = f"http://{config['ipfs']['host']}:{config['ipfs']['port']}/api/v0/bootstrap/add?arg={peer_address}"
response = requests.post(url)
if response.status_code == 200:
logging.info(f"Added {peer_name} ({peer_id}) to the bootstrap list")
else:
logging.warning(f"Failed to add {peer_name} ({peer_id}) to the bootstrap list")
logging.warning(f"Error: {response.text}")
except requests.exceptions.RequestException as e:
logging.warning(f"Failed to add {peer_name} ({peer_id}) to the bootstrap list")
logging.warning(f"Error: {str(e)}")
raise IPFSException(f"Failed to add {peer_name} ({peer_id}) to the bootstrap list")
else:
logging.info(f"{peer_name} ({peer_id}) is already in the bootstrap list")
def get_ipfs_file_size(ipfs_hash, config):
MAX_RETRIES = 2
RETRY_DELAY = 1
TIMEOUT = 10
retries = 0
cumulative_size = 0
while retries < MAX_RETRIES:
try:
logging.info("Checking size in IPFS")
url = f"http://{config['ipfs']['host']}:{config['ipfs']['port']}/api/v0/object/stat?arg={ipfs_hash}"
response = requests.post(url, timeout=TIMEOUT)
response.raise_for_status()
res = response.json()
logging.debug(res)
cumulative_size = res['CumulativeSize']
break # Break out of the loop if the size is successfully obtained
except (requests.exceptions.RequestException, KeyError) as e:
logging.error("Error checking IPFS file size: %s", str(e))
retries += 1
if retries < MAX_RETRIES:
logging.info(f"Retrying in {RETRY_DELAY} seconds...")
time.sleep(RETRY_DELAY)
else:
logging.info("Maximum number of retries reached. Couldn't get size - skipping asset.")
return cumulative_size if cumulative_size > 0 else None
def ipfs_cat(ipfs_hash):
"""Retrieve file content from IPFS using the 'cat' command.
Args:
ipfs_hash (str): The IPFS hash of the file.
Returns:
str: The contents of the file.
Raises:
Exception: If an error occurs while retrieving the file content.
"""
logging.info("Retrieving file content from IPFS using 'cat'...")
config = configparser.ConfigParser()
config.read("ipfspinner.config")
ipfs_host = config.get("ipfs", "host")