diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e1768724a..35fdccf4b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -321,6 +321,7 @@ jobs: ar_mining_io_tests, ar_poller_tests, ar_reject_chunks_tests, + ar_replica_2_9_nif_tests, ar_semaphore_tests, ar_start_from_block_tests, ar_tx_blacklist_tests, diff --git a/apps/arweave/c_src/randomx/pack_randomx_square.cpp b/apps/arweave/c_src/randomx/pack_randomx_square.cpp deleted file mode 100644 index f3e846380..000000000 --- a/apps/arweave/c_src/randomx/pack_randomx_square.cpp +++ /dev/null @@ -1,160 +0,0 @@ -#include -#include "crc32.h" -#include "pack_randomx_square.h" -#include "feistel_msgsize_key_cipher.h" - -// imports from randomx -#include "vm_compiled.hpp" -#include "blake2/blake2.h" -#include "aes_hash.hpp" - -extern "C" { - void randomx_squared_exec( - randomx_vm *machine, - const unsigned char *inHash, const unsigned char *inScratchpad, - unsigned char *outHash, unsigned char *outScratchpad, - const int randomxProgramCount) { - assert(machine != nullptr); - alignas(16) uint64_t tempHash[8]; - memcpy(tempHash, inHash, sizeof(tempHash)); - void* scratchpad = (void*)machine->getScratchpad(); - memcpy(scratchpad, inScratchpad, randomx_get_scratchpad_size()); - machine->resetRoundingMode(); - int blakeResult; - for (int chain = 0; chain < randomxProgramCount - 1; ++chain) { - machine->run(&tempHash); - blakeResult = randomx_blake2b( - tempHash, sizeof(tempHash), machine->getRegisterFile(), - sizeof(randomx::RegisterFile), nullptr, 0); - assert(blakeResult == 0); - } - machine->run(&tempHash); - - blakeResult = randomx_blake2b( - tempHash, sizeof(tempHash), machine->getRegisterFile(), - sizeof(randomx::RegisterFile), nullptr, 0); - assert(blakeResult == 0); - - memcpy(outHash, tempHash, sizeof(tempHash)); - - packing_mix_entropy_crc32( - (const unsigned char*)machine->getScratchpad(), - outScratchpad, randomx_get_scratchpad_size()); - } - - void randomx_squared_exec_test( - randomx_vm *machine, - const unsigned char *inHash, const unsigned char *inScratchpad, - unsigned char *outHash, unsigned char *outScratchpad, - const int randomxProgramCount) { - assert(machine != nullptr); - alignas(16) uint64_t tempHash[8]; - memcpy(tempHash, inHash, sizeof(tempHash)); - void* scratchpad = (void*)machine->getScratchpad(); - memcpy(scratchpad, inScratchpad, randomx_get_scratchpad_size()); - machine->resetRoundingMode(); - int blakeResult; - for (int chain = 0; chain < randomxProgramCount - 1; ++chain) { - machine->run(&tempHash); - blakeResult = randomx_blake2b( - tempHash, sizeof(tempHash), machine->getRegisterFile(), - sizeof(randomx::RegisterFile), nullptr, 0); - assert(blakeResult == 0); - } - machine->run(&tempHash); - - blakeResult = randomx_blake2b( - tempHash, sizeof(tempHash), machine->getRegisterFile(), - sizeof(randomx::RegisterFile), nullptr, 0); - assert(blakeResult == 0); - - memcpy(outHash, tempHash, sizeof(tempHash)); - memcpy(outScratchpad, machine->getScratchpad(), randomx_get_scratchpad_size()); - } - - // init_msg + hash - void randomx_squared_init_scratchpad( - randomx_vm *machine, const unsigned char *input, const size_t inputSize, - unsigned char *outHash, unsigned char *outScratchpad, - const int randomxProgramCount) { - assert(machine != nullptr); - assert(inputSize == 0 || input != nullptr); - alignas(16) uint64_t tempHash[8]; - int blakeResult = randomx_blake2b( - tempHash, sizeof(tempHash), input, inputSize, nullptr, 0); - assert(blakeResult == 0); - void* scratchpad = (void*)machine->getScratchpad(); - // bool softAes = false - fillAes1Rx4(tempHash, randomx_get_scratchpad_size(), scratchpad); - - memcpy(outHash, tempHash, sizeof(tempHash)); - memcpy(outScratchpad, machine->getScratchpad(), randomx_get_scratchpad_size()); - } - - void packing_mix_entropy_crc32( - const unsigned char *inEntropy, - unsigned char *outEntropy, const size_t entropySize) { - // NOTE we can't use _mm_crc32_u64, because it output only final 32-bit result - // NOTE commented variant is more readable but unoptimized - unsigned int state = ~0; - // unsigned int state = 0; - const unsigned int *inEntropyPtr = (const unsigned int*)inEntropy; - unsigned int *outEntropyPtr = (unsigned int*)outEntropy; - for(size_t i=0;i 0) { - for (size_t i = 0; i < numJumps; ++i) { - size_t srcPos = i * jumpSize + numBlocksPerJump * blockSize; - memcpy(outEntropyPtr, &inEntropy[srcPos], leftover); - outEntropyPtr += leftover; - } - } - } - - // TODO optimized packing_apply_to_subchunk (NIF only uses slice) -} diff --git a/apps/arweave/c_src/randomx/pack_randomx_square.h b/apps/arweave/c_src/randomx/pack_randomx_square.h deleted file mode 100755 index 47038e839..000000000 --- a/apps/arweave/c_src/randomx/pack_randomx_square.h +++ /dev/null @@ -1,34 +0,0 @@ -#ifndef PACK_RANDOMX_SQUARE_H -#define PACK_RANDOMX_SQUARE_H - -#include "randomx.h" - -#if defined(__cplusplus) -extern "C" { -#endif - -RANDOMX_EXPORT void randomx_squared_exec( - randomx_vm *machine, const unsigned char *inHash, const unsigned char *inScratchpad, - unsigned char *outHash, unsigned char *outScratchpad, const int randomxProgramCount); -RANDOMX_EXPORT void randomx_squared_exec_test( - randomx_vm *machine, const unsigned char *inHash, const unsigned char *inScratchpad, - unsigned char *outHash, unsigned char *outScratchpad, const int randomxProgramCount); -// init_msg + hash -RANDOMX_EXPORT void randomx_squared_init_scratchpad( - randomx_vm *machine, const unsigned char *input, const size_t inputSize, - unsigned char *outHash, unsigned char *outScratchpad, const int randomxProgramCount); - -RANDOMX_EXPORT void packing_mix_entropy_crc32( - const unsigned char *inEntropy, unsigned char *outEntropy, const size_t entropySize); - -RANDOMX_EXPORT void packing_mix_entropy_far( - const unsigned char *inEntropy, unsigned char *outEntropy, const size_t entropySize, - const size_t jumpSize, const size_t blockSize); - -// TODO optimized packing_apply_to_subchunk (NIF only uses slice) - -#if defined(__cplusplus) -} -#endif - -#endif // PACK_RANDOMX_SQUARE_H diff --git a/apps/arweave/c_src/randomx/randomx_squared.cpp b/apps/arweave/c_src/randomx/randomx_squared.cpp new file mode 100644 index 000000000..473370cfb --- /dev/null +++ b/apps/arweave/c_src/randomx/randomx_squared.cpp @@ -0,0 +1,256 @@ +#include +#include +#include "crc32.h" +#include "randomx_squared.h" +#include "feistel_msgsize_key_cipher.h" + +// imports from randomx +#include "vm_compiled.hpp" +#include "blake2/blake2.h" + +extern "C" { + + void _rsp_mix_entropy_near( + const unsigned char *inEntropy, + unsigned char *outEntropy, + const size_t entropySize + ) { + // NOTE we can't use _mm_crc32_u64, because it output only final 32-bit result + // NOTE commented variant is more readable but unoptimized + unsigned int state = ~0; + // unsigned int state = 0; + const unsigned int *inEntropyPtr = (const unsigned int*)inEntropy; + unsigned int *outEntropyPtr = (unsigned int*)outEntropy; + for(size_t i=0;iresetRoundingMode(); + for (int chain = 0; chain < programCount-1; chain++) { + machine->run(tempHash); + int blakeResult = randomx_blake2b( + tempHash, 64, + machine->getRegisterFile(), + sizeof(randomx::RegisterFile), + nullptr, 0 + ); + assert(blakeResult == 0); + } + machine->run(tempHash); + int blakeResult = randomx_blake2b( + tempHash, 64, + machine->getRegisterFile(), + sizeof(randomx::RegisterFile), + nullptr, 0 + ); + assert(blakeResult == 0); + _rsp_mix_entropy_near( + (const unsigned char*)machine->getScratchpad(), + (unsigned char*)(void*)machine->getScratchpad(), + scratchpadSize); + } + + void _copy_chunk_cross_lane( + randomx_vm** inSet, + randomx_vm** outSet, + size_t srcPos, + size_t dstPos, + size_t length, + size_t scratchpadSize + ) { + while (length > 0) { + int srcLane = (int)(srcPos / scratchpadSize); + size_t offsetInSrcLane = srcPos % scratchpadSize; + + int dstLane = (int)(dstPos / scratchpadSize); + size_t offsetInDstLane = dstPos % scratchpadSize; + + size_t srcLaneRemain = scratchpadSize - offsetInSrcLane; + size_t dstLaneRemain = scratchpadSize - offsetInDstLane; + + size_t chunkSize = length; + if (chunkSize > srcLaneRemain) { + chunkSize = srcLaneRemain; + } + if (chunkSize > dstLaneRemain) { + chunkSize = dstLaneRemain; + } + + unsigned char* srcSp = (unsigned char*)(void*) inSet[srcLane]->getScratchpad(); + unsigned char* dstSp = (unsigned char*)(void*) outSet[dstLane]->getScratchpad(); + memcpy(dstSp + offsetInDstLane, srcSp + offsetInSrcLane, chunkSize); + + srcPos += chunkSize; + dstPos += chunkSize; + length -= chunkSize; + } + } + + void _rsp_mix_entropy_far( + randomx_vm** inSet, + randomx_vm** outSet, + int count, + size_t scratchpadSize, + size_t jumpSize, + size_t blockSize) + { + size_t totalSize = (size_t)count * scratchpadSize; + + size_t entropySize = totalSize; + size_t numJumps = entropySize / jumpSize; + size_t numBlocksPerJump = jumpSize / blockSize; + size_t leftover = jumpSize % blockSize; + + size_t outOffset = 0; + for (size_t offset = 0; offset < numBlocksPerJump; ++offset) { + for (size_t i = 0; i < numJumps; ++i) { + size_t srcPos = i * jumpSize + offset * blockSize; + _copy_chunk_cross_lane(inSet, outSet, srcPos, outOffset, blockSize, scratchpadSize); + outOffset += blockSize; + } + } + + if (leftover > 0) { + for (size_t i = 0; i < numJumps; ++i) { + size_t srcPos = i * jumpSize + numBlocksPerJump * blockSize; + _copy_chunk_cross_lane(inSet, outSet, srcPos, outOffset, leftover, scratchpadSize); + outOffset += leftover; + } + } + } + + int rsp_fused_entropy( + randomx_vm** vmList, + size_t scratchpadSize, + int subChunkCount, + int subChunkSize, + int laneCount, + int rxDepth, + int randomxProgramCount, + int blockSize, + const unsigned char* keyData, + size_t keySize, + unsigned char* outEntropy + ) { + struct vm_hash_t { + alignas(16) uint64_t tempHash[8]; // 64 bytes + }; + + vm_hash_t* vmHashes = new (std::nothrow) vm_hash_t[laneCount]; + if (!vmHashes) { + return 0; + } + + // Initialize the scratchaps for each lane + for (int i = 0; i < laneCount; i++) { + // laneSeed = sha256(<>) + // laneSeed should be unique - i.e. now two lanes across all entropies and all + // replicas should have the same seed. Current key (as off 2025-01-01) is + // <> where entropy index is unique within + // a given partition. + unsigned char laneSeed[32]; + { + SHA256_CTX sha256; + SHA256_Init(&sha256); + SHA256_Update(&sha256, keyData, keySize); + unsigned char laneIndex = (unsigned char)i + 1; + SHA256_Update(&sha256, &laneIndex, 1); + SHA256_Final(laneSeed, &sha256); + } + int blakeResult = randomx_blake2b( + vmHashes[i].tempHash, sizeof(vmHashes[i].tempHash), + laneSeed, 32, + nullptr, 0 + ); + if (blakeResult != 0) { + delete[] vmHashes; + return 0; + } + vmList[i]->initScratchpad(&vmHashes[i].tempHash); + } + + for (int d = 0; d < rxDepth; d++) { + for (int lane = 0; lane < laneCount; lane++) { + _rsp_exec_inplace( + vmList[lane], + vmHashes[lane].tempHash, + randomxProgramCount, scratchpadSize); + } + _rsp_mix_entropy_far(&vmList[0], &vmList[laneCount], + laneCount, scratchpadSize, scratchpadSize, + blockSize); + + if (d + 1 < rxDepth) { + d++; + for (int lane = 0; lane < laneCount; lane++) { + _rsp_exec_inplace( + vmList[lane+laneCount], + vmHashes[lane].tempHash, + randomxProgramCount, scratchpadSize); + } + _rsp_mix_entropy_far(&vmList[laneCount], &vmList[0], + laneCount, scratchpadSize, scratchpadSize, + blockSize); + } + } + // NOTE still unoptimal. Last copy can be performed from scratchpad to output. + // But requires +1 variation (set to buffer) + + if ((rxDepth % 2) == 0) { + unsigned char* outEntropyPtr = outEntropy; + for (int i = 0; i < laneCount; i++) { + void* sp = (void*)vmList[i]->getScratchpad(); + memcpy(outEntropyPtr, sp, scratchpadSize); + outEntropyPtr += scratchpadSize; + } + } else { + unsigned char* outEntropyPtr = outEntropy; + for (int i = laneCount; i < 2*laneCount; i++) { + void* sp = (void*)vmList[i]->getScratchpad(); + memcpy(outEntropyPtr, sp, scratchpadSize); + outEntropyPtr += scratchpadSize; + } + } + + delete[] vmHashes; + + return 1; + } + + // TODO optimized packing_apply_to_subchunk (NIF only uses slice) +} diff --git a/apps/arweave/c_src/randomx/randomx_squared.h b/apps/arweave/c_src/randomx/randomx_squared.h new file mode 100755 index 000000000..d98560333 --- /dev/null +++ b/apps/arweave/c_src/randomx/randomx_squared.h @@ -0,0 +1,30 @@ +#ifndef RANDOMX_SQUARED_H +#define RANDOMX_SQUARED_H + +#include "randomx.h" + +#if defined(__cplusplus) +extern "C" { +#endif + +RANDOMX_EXPORT int rsp_fused_entropy( + randomx_vm** vmList, + size_t scratchpadSize, + int subChunkCount, + int subChunkSize, + int laneCount, + int rxDepth, + int randomxProgramCount, + int blockSize, + const unsigned char* keyData, + size_t keySize, + unsigned char* outEntropy // We'll pass in a pointer for final scratchpad data +); + +// TODO optimized packing_apply_to_subchunk (NIF only uses slice) + +#if defined(__cplusplus) +} +#endif + +#endif // RANDOMX_SQUARED_H diff --git a/apps/arweave/c_src/randomx/rxsquared/ar_rxsquared_nif.c b/apps/arweave/c_src/randomx/rxsquared/ar_rxsquared_nif.c index 810ea8134..c555139e3 100755 --- a/apps/arweave/c_src/randomx/rxsquared/ar_rxsquared_nif.c +++ b/apps/arweave/c_src/randomx/rxsquared/ar_rxsquared_nif.c @@ -3,7 +3,7 @@ #include #include "../randomx_long_with_entropy.h" #include "../feistel_msgsize_key_cipher.h" -#include "../pack_randomx_square.h" +#include "../randomx_squared.h" #include "../ar_randomx_impl.h" @@ -15,437 +15,246 @@ static ERL_NIF_TERM rxsquared_info_nif(ErlNifEnv* envPtr, int argc, const ERL_NI static ERL_NIF_TERM rxsquared_init_nif(ErlNifEnv* envPtr, int argc, const ERL_NIF_TERM argv[]); static ERL_NIF_TERM rxsquared_hash_nif(ErlNifEnv* envPtr, int argc, const ERL_NIF_TERM argv[]); -static int rxsquared_load(ErlNifEnv* envPtr, void** priv, ERL_NIF_TERM info) -{ +static int rxsquared_load(ErlNifEnv* envPtr, void** priv, ERL_NIF_TERM info) { return load(envPtr, priv, info); } -static ERL_NIF_TERM rxsquared_info_nif(ErlNifEnv* envPtr, int argc, const ERL_NIF_TERM argv[]) -{ +static ERL_NIF_TERM rxsquared_info_nif(ErlNifEnv* envPtr, int argc, const ERL_NIF_TERM argv[]) { return info_nif("rxsquared", envPtr, argc, argv); } -static ERL_NIF_TERM rxsquared_init_nif(ErlNifEnv* envPtr, int argc, const ERL_NIF_TERM argv[]) -{ +static ERL_NIF_TERM rxsquared_init_nif(ErlNifEnv* envPtr, int argc, const ERL_NIF_TERM argv[]) { return init_nif(envPtr, argc, argv); } -static ERL_NIF_TERM rxsquared_hash_nif(ErlNifEnv* envPtr, int argc, const ERL_NIF_TERM argv[]) -{ +static ERL_NIF_TERM rxsquared_hash_nif(ErlNifEnv* envPtr, int argc, const ERL_NIF_TERM argv[]) { return hash_nif(envPtr, argc, argv); } -static ERL_NIF_TERM rsp_exec_nif( - ErlNifEnv* envPtr, int argc, const ERL_NIF_TERM argv[]) { - if (argc != 7) { - return enif_make_badarg(envPtr); - } - int randomxProgramCount; - int jitEnabled, largePagesEnabled, hardwareAESEnabled; - struct state* statePtr; - ErlNifBinary inHashBin; - ErlNifBinary inScratchpadBin; - ERL_NIF_TERM outHashTerm; - unsigned char* outHashData; - ERL_NIF_TERM outScratchpadTerm; - unsigned char* outScratchpadData; - - if (!enif_get_resource(envPtr, argv[0], stateType, (void**) &statePtr)) { - return error_tuple(envPtr, "failed to read state"); - } - if (!enif_inspect_binary(envPtr, argv[1], &inHashBin)) { - return enif_make_badarg(envPtr); - } - if (!enif_inspect_binary(envPtr, argv[2], &inScratchpadBin)) { - return enif_make_badarg(envPtr); - } - if (!enif_get_int(envPtr, argv[3], &jitEnabled)) { - return enif_make_badarg(envPtr); - } - if (!enif_get_int(envPtr, argv[4], &largePagesEnabled)) { - return enif_make_badarg(envPtr); - } - if (!enif_get_int(envPtr, argv[5], &hardwareAESEnabled)) { - return enif_make_badarg(envPtr); - } - if (!enif_get_int(envPtr, argv[6], &randomxProgramCount)) { - return enif_make_badarg(envPtr); - } - - if (inHashBin.size != 64) { - return enif_make_badarg(envPtr); - } - if (inScratchpadBin.size != randomx_get_scratchpad_size()) { - return enif_make_badarg(envPtr); - } - - int isRandomxReleased; - randomx_vm *vmPtr = create_vm(statePtr, (statePtr->mode == HASHING_MODE_FAST), - jitEnabled, largePagesEnabled, hardwareAESEnabled, &isRandomxReleased); - if (vmPtr == NULL) { - if (isRandomxReleased != 0) { - return error_tuple(envPtr, "state has been released"); - } - return error_tuple(envPtr, "randomx_create_vm failed"); - } - - outScratchpadData = enif_make_new_binary( - envPtr, randomx_get_scratchpad_size(), &outScratchpadTerm); - if (outScratchpadData == NULL) { - return enif_make_badarg(envPtr); - } - - outHashData = enif_make_new_binary(envPtr, 64, &outHashTerm); - if (outHashData == NULL) { - return enif_make_badarg(envPtr); - } - - randomx_squared_exec( - vmPtr, inHashBin.data, inScratchpadBin.data, outHashData, outScratchpadData, - randomxProgramCount); - - destroy_vm(statePtr, vmPtr); - - return ok_tuple2(envPtr, outHashTerm, outScratchpadTerm); -} - -static ERL_NIF_TERM rsp_exec_test_nif( +static ERL_NIF_TERM rsp_feistel_encrypt_nif( ErlNifEnv* envPtr, int argc, const ERL_NIF_TERM argv[]) { - if (argc != 7) { - return enif_make_badarg(envPtr); - } - - int randomxProgramCount; - int jitEnabled, largePagesEnabled, hardwareAESEnabled; - struct state* statePtr; - ErlNifBinary inHashBin; - ErlNifBinary inScratchpadBin; - ERL_NIF_TERM outHashTerm; - unsigned char* outHashData; - ERL_NIF_TERM outScratchpadTerm; - unsigned char* outScratchpadData; - - if (!enif_get_resource(envPtr, argv[0], stateType, (void**) &statePtr)) { - return error_tuple(envPtr, "failed to read state"); - } - if (!enif_inspect_binary(envPtr, argv[1], &inHashBin)) { - return enif_make_badarg(envPtr); - } - if (!enif_inspect_binary(envPtr, argv[2], &inScratchpadBin)) { - return enif_make_badarg(envPtr); - } - if (!enif_get_int(envPtr, argv[3], &jitEnabled)) { - return enif_make_badarg(envPtr); - } - if (!enif_get_int(envPtr, argv[4], &largePagesEnabled)) { - return enif_make_badarg(envPtr); - } - if (!enif_get_int(envPtr, argv[5], &hardwareAESEnabled)) { - return enif_make_badarg(envPtr); - } - if (!enif_get_int(envPtr, argv[6], &randomxProgramCount)) { - return enif_make_badarg(envPtr); - } + ErlNifBinary inMsgBin; + ErlNifBinary inKeyBin; + ERL_NIF_TERM outMsgTerm; + unsigned char* outMsgData; - if (inHashBin.size != 64) { - return enif_make_badarg(envPtr); - } - if (inScratchpadBin.size != randomx_get_scratchpad_size()) { + if (argc != 2) { return enif_make_badarg(envPtr); } - int isRandomxReleased; - randomx_vm *vmPtr = create_vm(statePtr, (statePtr->mode == HASHING_MODE_FAST), - jitEnabled, largePagesEnabled, hardwareAESEnabled, &isRandomxReleased); - if (vmPtr == NULL) { - if (isRandomxReleased != 0) { - return error_tuple(envPtr, "state has been released"); - } - return error_tuple(envPtr, "randomx_create_vm failed"); - } - - outScratchpadData = enif_make_new_binary( - envPtr, randomx_get_scratchpad_size(), &outScratchpadTerm); - if (outScratchpadData == NULL) { + if (!enif_inspect_binary(envPtr, argv[0], &inMsgBin)) { return enif_make_badarg(envPtr); } - outHashData = enif_make_new_binary(envPtr, 64, &outHashTerm); - if (outHashData == NULL) { + if (!enif_inspect_binary(envPtr, argv[1], &inKeyBin)) { return enif_make_badarg(envPtr); } - randomx_squared_exec_test( - vmPtr, inHashBin.data, inScratchpadBin.data, outHashData, outScratchpadData, - randomxProgramCount); - - destroy_vm(statePtr, vmPtr); - - return ok_tuple2(envPtr, outHashTerm, outScratchpadTerm); -} - -static ERL_NIF_TERM rsp_init_scratchpad_nif( - ErlNifEnv* envPtr, int argc, const ERL_NIF_TERM argv[]) { - if (argc != 6) { - return enif_make_badarg(envPtr); - } + size_t msgSize = inMsgBin.size; - int randomxProgramCount; - int jitEnabled, largePagesEnabled, hardwareAESEnabled; - struct state* statePtr; - ErlNifBinary inputBin; - ERL_NIF_TERM outHashTerm; - unsigned char* outHashData; - ERL_NIF_TERM outScratchpadTerm; - unsigned char* outScratchpadData; - - if (!enif_get_resource(envPtr, argv[0], stateType, (void**) &statePtr)) { - return error_tuple(envPtr, "failed to read state"); - } - if (!enif_inspect_binary(envPtr, argv[1], &inputBin)) { - return enif_make_badarg(envPtr); - } - if (!enif_get_int(envPtr, argv[2], &jitEnabled)) { - return enif_make_badarg(envPtr); - } - if (!enif_get_int(envPtr, argv[3], &largePagesEnabled)) { - return enif_make_badarg(envPtr); - } - if (!enif_get_int(envPtr, argv[4], &hardwareAESEnabled)) { - return enif_make_badarg(envPtr); - } - if (!enif_get_int(envPtr, argv[5], &randomxProgramCount)) { + if (inKeyBin.size != msgSize) { return enif_make_badarg(envPtr); } - int isRandomxReleased; - randomx_vm *vmPtr = create_vm(statePtr, (statePtr->mode == HASHING_MODE_FAST), - jitEnabled, largePagesEnabled, hardwareAESEnabled, &isRandomxReleased); - if (vmPtr == NULL) { - if (isRandomxReleased != 0) { - return error_tuple(envPtr, "state has been released"); - } - return error_tuple(envPtr, "randomx_create_vm failed"); - } - - outScratchpadData = enif_make_new_binary( - envPtr, randomx_get_scratchpad_size(), &outScratchpadTerm); - if (outScratchpadData == NULL) { + if (msgSize % 64 != 0) { return enif_make_badarg(envPtr); } - outHashData = enif_make_new_binary(envPtr, 64, &outHashTerm); - if (outHashData == NULL) { + outMsgData = enif_make_new_binary(envPtr, msgSize, &outMsgTerm); + if (outMsgData == NULL) { return enif_make_badarg(envPtr); } - randomx_squared_init_scratchpad( - vmPtr, inputBin.data, inputBin.size, outHashData, outScratchpadData, - randomxProgramCount); - - destroy_vm(statePtr, vmPtr); + feistel_encrypt(inMsgBin.data, msgSize, inKeyBin.data, outMsgData); - return ok_tuple2(envPtr, outHashTerm, outScratchpadTerm); + return ok_tuple(envPtr, outMsgTerm); } - -// pack randomx square randomx independent -static ERL_NIF_TERM rsp_mix_entropy_crc32_nif( +static ERL_NIF_TERM rsp_feistel_decrypt_nif( ErlNifEnv* envPtr, int argc, const ERL_NIF_TERM argv[]) { - ErlNifBinary inEntropyBin; - ERL_NIF_TERM outEntropyTerm; - unsigned char* outEntropyData; - - if (argc != 1) { - return enif_make_badarg(envPtr); - } + ErlNifBinary inMsgBin; + ErlNifBinary inKeyBin; + ERL_NIF_TERM outMsgTerm; + unsigned char* outMsgData; - if (!enif_inspect_binary(envPtr, argv[0], &inEntropyBin)) { + if (argc != 2) { return enif_make_badarg(envPtr); } - size_t entropySize = inEntropyBin.size; - - if (entropySize % 8 != 0) { + if (!enif_inspect_binary(envPtr, argv[0], &inMsgBin)) { return enif_make_badarg(envPtr); } - outEntropyData = enif_make_new_binary(envPtr, entropySize, &outEntropyTerm); - if (outEntropyData == NULL) { + if (!enif_inspect_binary(envPtr, argv[1], &inKeyBin)) { return enif_make_badarg(envPtr); } - packing_mix_entropy_crc32(inEntropyBin.data, outEntropyData, entropySize); - - return ok_tuple(envPtr, outEntropyTerm); -} - -static ERL_NIF_TERM rsp_mix_entropy_far_nif( - ErlNifEnv* envPtr, int argc, const ERL_NIF_TERM argv[]) { - ErlNifBinary inEntropyBin; - ERL_NIF_TERM outEntropyTerm; - unsigned char* outEntropyData; - - if (argc != 1) { - return enif_make_badarg(envPtr); - } + size_t msgSize = inMsgBin.size; - if (!enif_inspect_binary(envPtr, argv[0], &inEntropyBin)) { + if (inKeyBin.size != msgSize) { return enif_make_badarg(envPtr); } - size_t entropySize = inEntropyBin.size; - - if (entropySize % 8 != 0) { + if (msgSize % 64 != 0) { return enif_make_badarg(envPtr); } - outEntropyData = enif_make_new_binary(envPtr, entropySize, &outEntropyTerm); - if (outEntropyData == NULL) { + outMsgData = enif_make_new_binary(envPtr, msgSize, &outMsgTerm); + if (outMsgData == NULL) { return enif_make_badarg(envPtr); } - packing_mix_entropy_far(inEntropyBin.data, outEntropyData, entropySize, - randomx_get_scratchpad_size(), 6); + feistel_decrypt(inMsgBin.data, msgSize, inKeyBin.data, outMsgData); - return ok_tuple(envPtr, outEntropyTerm); + return ok_tuple(envPtr, outMsgTerm); } -static ERL_NIF_TERM rsp_mix_entropy_far_test_nif( - ErlNifEnv* envPtr, int argc, const ERL_NIF_TERM argv[]) { - ErlNifBinary inEntropyBin; - ERL_NIF_TERM outEntropyTerm; - unsigned char* outEntropyData; - unsigned int jumpSize; - unsigned int blockSize; - - if (argc != 3) { +static ERL_NIF_TERM rsp_fused_entropy_nif(ErlNifEnv* envPtr, int argc, const ERL_NIF_TERM argv[]) { + if (argc != 10) { return enif_make_badarg(envPtr); } - if (!enif_inspect_binary(envPtr, argv[0], &inEntropyBin)) { - return enif_make_badarg(envPtr); - } - if (!enif_get_uint(envPtr, argv[1], &jumpSize)) { - return enif_make_badarg(envPtr); - } - if (!enif_get_uint(envPtr, argv[2], &blockSize)) { - return enif_make_badarg(envPtr); + // 1. Parse the state resource + struct state* statePtr; + if (!enif_get_resource(envPtr, argv[0], stateType, (void**)&statePtr)) { + return error_tuple(envPtr, "failed_to_read_state"); } - size_t entropySize = inEntropyBin.size; - - if (entropySize % 8 != 0) { + // 2. Parse each integer + int subChunkCount; + if (!enif_get_int(envPtr, argv[1], &subChunkCount)) { return enif_make_badarg(envPtr); } - outEntropyData = enif_make_new_binary(envPtr, entropySize, &outEntropyTerm); - if (outEntropyData == NULL) { + int subChunkSize; + if (!enif_get_int(envPtr, argv[2], &subChunkSize)) { return enif_make_badarg(envPtr); } - packing_mix_entropy_far(inEntropyBin.data, outEntropyData, entropySize, - jumpSize, blockSize); - - return ok_tuple(envPtr, outEntropyTerm); -} - -static ERL_NIF_TERM rsp_feistel_encrypt_nif( - ErlNifEnv* envPtr, int argc, const ERL_NIF_TERM argv[]) { - ErlNifBinary inMsgBin; - ErlNifBinary inKeyBin; - ERL_NIF_TERM outMsgTerm; - unsigned char* outMsgData; - - if (argc != 2) { - return enif_make_badarg(envPtr); - } - - if (!enif_inspect_binary(envPtr, argv[0], &inMsgBin)) { + int laneCount; + if (!enif_get_int(envPtr, argv[3], &laneCount)) { return enif_make_badarg(envPtr); } - if (!enif_inspect_binary(envPtr, argv[1], &inKeyBin)) { + int rxDepth; + if (!enif_get_int(envPtr, argv[4], &rxDepth)) { return enif_make_badarg(envPtr); } - size_t msgSize = inMsgBin.size; - - if (inKeyBin.size != msgSize) { + int jitEnabled; + if (!enif_get_int(envPtr, argv[5], &jitEnabled)) { return enif_make_badarg(envPtr); } - if (msgSize % 64 != 0) { + int largePagesEnabled; + if (!enif_get_int(envPtr, argv[6], &largePagesEnabled)) { return enif_make_badarg(envPtr); } - outMsgData = enif_make_new_binary(envPtr, msgSize, &outMsgTerm); - if (outMsgData == NULL) { + int hardwareAESEnabled; + if (!enif_get_int(envPtr, argv[7], &hardwareAESEnabled)) { return enif_make_badarg(envPtr); } - feistel_encrypt(inMsgBin.data, msgSize, inKeyBin.data, outMsgData); - - return ok_tuple(envPtr, outMsgTerm); -} - -static ERL_NIF_TERM rsp_feistel_decrypt_nif( - ErlNifEnv* envPtr, int argc, const ERL_NIF_TERM argv[]) { - ErlNifBinary inMsgBin; - ErlNifBinary inKeyBin; - ERL_NIF_TERM outMsgTerm; - unsigned char* outMsgData; - - if (argc != 2) { + int randomxProgramCount; + if (!enif_get_int(envPtr, argv[8], &randomxProgramCount)) { return enif_make_badarg(envPtr); } - if (!enif_inspect_binary(envPtr, argv[0], &inMsgBin)) { + // 3. Parse key as a binary + ErlNifBinary keyBin; + if (!enif_inspect_binary(envPtr, argv[9], &keyBin)) { return enif_make_badarg(envPtr); } - if (!enif_inspect_binary(envPtr, argv[1], &inKeyBin)) { - return enif_make_badarg(envPtr); + // 4. Create VMs + int totalVMs = 2 * laneCount; + randomx_vm** vmList = (randomx_vm**)calloc(totalVMs, sizeof(randomx_vm*)); + if (!vmList) { + return error_tuple(envPtr, "vmList_alloc_failed"); } - size_t msgSize = inMsgBin.size; + size_t scratchpadSize = randomx_get_scratchpad_size(); - if (inKeyBin.size != msgSize) { - return enif_make_badarg(envPtr); + // 5. Pre-allocate the final output binary to store all scratchpads + size_t outEntropySize = scratchpadSize * laneCount; + ERL_NIF_TERM outEntropyTerm; + unsigned char* outEntropy = + enif_make_new_binary(envPtr, outEntropySize, &outEntropyTerm); + if (!outEntropy) { + free(vmList); + return enif_make_badarg(envPtr); + } + + // 6. Create the randomx_vm objects + int isRandomxReleased = 0; + for (int i = 0; i < totalVMs; i++) { + vmList[i] = create_vm( + statePtr, + (statePtr->mode == HASHING_MODE_FAST), + jitEnabled, + largePagesEnabled, + hardwareAESEnabled, + &isRandomxReleased + ); + if (!vmList[i]) { + // Clean up partial + for (int j = 0; j < i; j++) { + destroy_vm(statePtr, vmList[j]); + } + free(vmList); + if (isRandomxReleased != 0) { + return error_tuple(envPtr, "state_has_been_released"); + } + return error_tuple(envPtr, "randomx_create_vm_failed"); + } } - if (msgSize % 64 != 0) { - return enif_make_badarg(envPtr); + // 7. Call the pure C++ function that does the heavy logic and returns bool + int success = rsp_fused_entropy( + vmList, + scratchpadSize, + subChunkCount, + subChunkSize, + laneCount, + rxDepth, + randomxProgramCount, + 6, + keyBin.data, + keyBin.size, + outEntropy // final buffer for the output entropy + ); + + // 8. If the function returned false, we interpret that as an error + if (!success) { + // Cleanup + for (int i = 0; i < totalVMs; i++) { + if (vmList[i]) { + destroy_vm(statePtr, vmList[i]); + } + } + free(vmList); + return error_tuple(envPtr, "cxx_fused_entropy_failed"); } - outMsgData = enif_make_new_binary(envPtr, msgSize, &outMsgTerm); - if (outMsgData == NULL) { - return enif_make_badarg(envPtr); + // 9. If success, destroy VMs and return {ok, outEntropyTerm} + for (int i = 0; i < totalVMs; i++) { + destroy_vm(statePtr, vmList[i]); } + free(vmList); - feistel_decrypt(inMsgBin.data, msgSize, inKeyBin.data, outMsgData); - - return ok_tuple(envPtr, outMsgTerm); + return ok_tuple(envPtr, outEntropyTerm); } + static ErlNifFunc rxsquared_funcs[] = { {"rxsquared_info_nif", 1, rxsquared_info_nif, ERL_NIF_DIRTY_JOB_CPU_BOUND}, {"rxsquared_init_nif", 5, rxsquared_init_nif, ERL_NIF_DIRTY_JOB_CPU_BOUND}, {"rxsquared_hash_nif", 5, rxsquared_hash_nif, ERL_NIF_DIRTY_JOB_CPU_BOUND}, - {"rsp_exec_nif", 7, - rsp_exec_nif, ERL_NIF_DIRTY_JOB_CPU_BOUND}, - {"rsp_exec_test_nif", 7, - rsp_exec_test_nif, ERL_NIF_DIRTY_JOB_CPU_BOUND}, - {"rsp_init_scratchpad_nif", 6, - rsp_init_scratchpad_nif, ERL_NIF_DIRTY_JOB_CPU_BOUND}, - {"rsp_mix_entropy_crc32_nif", 1, - rsp_mix_entropy_crc32_nif, ERL_NIF_DIRTY_JOB_CPU_BOUND}, - {"rsp_mix_entropy_far_nif", 1, - rsp_mix_entropy_far_nif, ERL_NIF_DIRTY_JOB_CPU_BOUND}, - {"rsp_mix_entropy_far_test_nif", 3, - rsp_mix_entropy_far_test_nif, ERL_NIF_DIRTY_JOB_CPU_BOUND}, + {"rsp_fused_entropy_nif", 10, + rsp_fused_entropy_nif, ERL_NIF_DIRTY_JOB_CPU_BOUND}, {"rsp_feistel_encrypt_nif", 2, rsp_feistel_encrypt_nif, ERL_NIF_DIRTY_JOB_CPU_BOUND}, {"rsp_feistel_decrypt_nif", 2, rsp_feistel_decrypt_nif, ERL_NIF_DIRTY_JOB_CPU_BOUND} }; diff --git a/apps/arweave/include/ar_consensus.hrl b/apps/arweave/include/ar_consensus.hrl index 44b42ab8f..01f9b424e 100755 --- a/apps/arweave/include/ar_consensus.hrl +++ b/apps/arweave/include/ar_consensus.hrl @@ -21,15 +21,20 @@ %% The number of times we apply an RX hash in each RX2 lane in-between every pair %% of mixings. --define(REPLICA_2_9_RANDOMX_ROUND_COUNT, 6). +-define(REPLICA_2_9_RANDOMX_PROGRAM_COUNT, 6). %% The number of RX2 lanes. -define(REPLICA_2_9_RANDOMX_LANE_COUNT, 4). -%% The RX2 depth. +%% The RX2 depth: the number of RX2 rounds. A round of RX2 has: +%% 1. REPLICA_2_9_RANDOMX_LANE_COUNT lanes +%% 2. Each lane evaluates REPLICA_2_9_RANDOMX_PROGRAM_COUNT RandomX programs +%% 3. The output entropy of each lane is then mixed with crc32 (aka "near mix") +%% 4. The the mixed output from all lanes is then shuffled (aka "far mix") -define(REPLICA_2_9_RANDOMX_DEPTH, 3). -%% The size in bytes of the component (NOT the total) RX2 scratchpad. +%% The size in bytes of the component RX2 scratchpad (aka the output from each RX2 lane). This +%% is NOT the total output entropy (that size is defined in REPLICA_2_9_ENTROPY_SIZE). -define(RANDOMX_SCRATCHPAD_SIZE, 2097152). %% The size in bytes of the total RX2 entropy (# of lanes * scratchpad size). diff --git a/apps/arweave/src/ar_bench_2_9.erl b/apps/arweave/src/ar_bench_2_9.erl index a9f4797b2..d1d3e7ac7 100644 --- a/apps/arweave/src/ar_bench_2_9.erl +++ b/apps/arweave/src/ar_bench_2_9.erl @@ -6,7 +6,7 @@ -include_lib("arweave/include/ar_config.hrl"). run_benchmark_from_cli(Args) -> - Threads = list_to_integer(get_flag_value(Args, "threads", "1")), + Threads = list_to_integer(get_flag_value(Args, "threads", "1")), DataMiB = list_to_integer(get_flag_value(Args, "mib", "1024")), Format= case get_flag_value(Args, "format", "replica_2_9") of @@ -17,35 +17,34 @@ run_benchmark_from_cli(Args) -> _ -> show_help() end, - % Collect all directory values - Dirs = collect_dirs(Args), - - % Ensure each directory exists - lists:foreach(fun(Dir) -> - case filelib:ensure_dir(filename:join(Dir, "dummy")) of - ok -> ok; - {error, Reason} -> - io:format("Error: Could not ensure directory ~p exists. Reason: ~p~n", [Dir, Reason]), - show_help(), - erlang:halt(1) - end - end, Dirs), - - run_benchmark({Format, Dirs, Threads, DataMiB}). + % Collect all directory values + Dirs = collect_dirs(Args), + + % Ensure each directory exists + lists:foreach(fun(Dir) -> + case filelib:ensure_dir(filename:join(Dir, "dummy")) of + ok -> ok; + {error, Reason} -> + io:format("Error: Could not ensure directory ~p exists. Reason: ~p~n", [Dir, Reason]), + show_help(), + erlang:halt(1) + end + end, Dirs), + run_benchmark({Format, Dirs, Threads, DataMiB}). collect_dirs([]) -> - []; + []; collect_dirs(["dir", Dir | Tail]) -> - [Dir | collect_dirs(Tail)]; + [Dir | collect_dirs(Tail)]; collect_dirs([_ | Tail]) -> - collect_dirs(Tail). + collect_dirs(Tail). get_flag_value([], _, DefaultValue) -> - DefaultValue; + DefaultValue; get_flag_value([Flag, Value | Tail], TargetFlag, _DefaultValue) when Flag == TargetFlag -> - Value; + Value; get_flag_value([_ | Tail], TargetFlag, DefaultValue) -> - get_flag_value(Tail, TargetFlag, DefaultValue). + get_flag_value(Tail, TargetFlag, DefaultValue). show_help() -> io:format("~nUsage: benchmark-2.9 [format replica_2_9|composite.1|composite.10|spora_2_6] [threads N] [mib N] [dir path1 dir path2 dir path3 ...]~n~n"), @@ -167,6 +166,7 @@ pack_chunks(replica_2_9, Thread, Dir, Context, Count) -> file:write_file(Path, PackedSubChunks) end, pack_chunks(replica_2_9, Thread, Dir, Context, Count-1); + pack_chunks(spora_2_6, Thread, Dir, Context, Count) -> {RandomXState, Chunk, Key, _ChunksPerThread} = Context, {ok, PackedChunk} = ar_rx512_nif:rx512_encrypt_chunk_nif( diff --git a/apps/arweave/src/ar_mine_randomx.erl b/apps/arweave/src/ar_mine_randomx.erl index 635f43096..e71ce9648 100755 --- a/apps/arweave/src/ar_mine_randomx.erl +++ b/apps/arweave/src/ar_mine_randomx.erl @@ -126,59 +126,19 @@ randomx_generate_replica_2_9_entropy({_, {stub_state, _}}, Key) -> %% Non-TEST implementation randomx_generate_replica_2_9_entropy({rxsquared, RandomxState}, Key) -> - Inputs = [crypto:hash(sha256, << Key/binary, LaneNumber:8 >>) - || LaneNumber <- lists:seq(1, ?REPLICA_2_9_RANDOMX_LANE_COUNT)], - HashesScratchpads0 = randomx_initialize_replica_2_9_scratchpads(RandomxState, Key, Inputs), - randomx_generate_replica_2_9_entropy(RandomxState, Key, HashesScratchpads0, - 1, ?REPLICA_2_9_RANDOMX_DEPTH). - -write_scratchpad_to_disk(Type, Hash0, Scratchpad0) -> - HashHex = ar_util:encode(Hash0), - FileName = io_lib:format("~s_~s.bin", [Type, HashHex]), - file:write_file(FileName, Scratchpad0). - -randomx_initialize_replica_2_9_scratchpads(_RandomxState, _Key, []) -> - []; -randomx_initialize_replica_2_9_scratchpads(RandomxState, Key, [Input | Inputs]) -> - {ok, Hash0, Scratchpad0} = - ar_rxsquared_nif:rsp_init_scratchpad_nif(RandomxState, Input, - jit(), large_pages(), hardware_aes(), ?REPLICA_2_9_RANDOMX_ROUND_COUNT), - [{Hash0, Scratchpad0} | randomx_initialize_replica_2_9_scratchpads(RandomxState, Key, - Inputs)]. - -randomx_generate_replica_2_9_entropy(RandomxState, Key, HashesScratchpads, - Depth, MaxDepth) -> - HashesScratchpads2 = randomx_process_replica_2_9_scratchpads( - RandomxState, HashesScratchpads), - - Scratchpad = iolist_to_binary([S || {_H, S} <- HashesScratchpads2]), - {ok, MixedScratchpad} = ar_rxsquared_nif:rsp_mix_entropy_far_nif(Scratchpad), - - case Depth == MaxDepth of - true -> - MixedScratchpad; - false -> - Scratchpads = split_scratchpads(MixedScratchpad), - HashesScratchpads3 = lists:zip([H || {H, _S} <- HashesScratchpads2], Scratchpads), - randomx_generate_replica_2_9_entropy(RandomxState, Key, HashesScratchpads3, - Depth + 1, MaxDepth) - end. - -split_scratchpads(<<>>) -> - []; -split_scratchpads(<< Scratchpad:(?RANDOMX_SCRATCHPAD_SIZE)/binary, Rest/binary >>) -> - [Scratchpad | split_scratchpads(Rest)]. - -randomx_process_replica_2_9_scratchpads(_RandomxState, []) -> - []; -randomx_process_replica_2_9_scratchpads(RandomxState, - [{Input, Scratchpad} | HashesScratchpads]) -> - {ok, Hash2, Scratchpad2} = - ar_rxsquared_nif:rsp_exec_nif(RandomxState, - Input, Scratchpad, jit(), large_pages(), hardware_aes(), - ?REPLICA_2_9_RANDOMX_ROUND_COUNT), - [{Hash2, Scratchpad2} | randomx_process_replica_2_9_scratchpads( - RandomxState, HashesScratchpads)]. + {ok, EntropyFused} = ar_rxsquared_nif:rsp_fused_entropy_nif( + RandomxState, + ?COMPOSITE_PACKING_SUB_CHUNK_COUNT, + ?COMPOSITE_PACKING_SUB_CHUNK_SIZE, + ?REPLICA_2_9_RANDOMX_LANE_COUNT, + ?REPLICA_2_9_RANDOMX_DEPTH, + jit(), + large_pages(), + hardware_aes(), + ?REPLICA_2_9_RANDOMX_PROGRAM_COUNT, + Key + ), + EntropyFused. randomx_decrypt_replica_2_9_sub_chunk({PackingState, Key, SubChunk, EntropySubChunkIndex}) -> diff --git a/apps/arweave/src/ar_rxsquared_nif.erl b/apps/arweave/src/ar_rxsquared_nif.erl index 0bce83374..d0a5dce65 100755 --- a/apps/arweave/src/ar_rxsquared_nif.erl +++ b/apps/arweave/src/ar_rxsquared_nif.erl @@ -5,12 +5,7 @@ -on_load(init_nif/0). -export([rxsquared_hash_nif/5, rxsquared_info_nif/1, rxsquared_init_nif/5, - rsp_exec_nif/7, - rsp_exec_test_nif/7, - rsp_init_scratchpad_nif/6, - rsp_mix_entropy_crc32_nif/1, - rsp_mix_entropy_far_nif/1, - rsp_mix_entropy_far_test_nif/3, + rsp_fused_entropy_nif/10, rsp_feistel_encrypt_nif/2, rsp_feistel_decrypt_nif/2]). @@ -38,29 +33,19 @@ init_nif() -> %%% Randomx square packing %%%=================================================================== -rsp_exec_nif(_State, _Hash, _Scratchpad, _JIT, _LargePages, _HardwareAES, _RoundCount) -> - ?LOG_ERROR("rsp_exec_nif"), - erlang:nif_error(nif_not_loaded). - -rsp_exec_test_nif(_State, _Hash, _Scratchpad, _JIT, _LargePages, _HardwareAES, _RoundCount) -> - ?LOG_ERROR("rsp_exec_test_nif"), - erlang:nif_error(nif_not_loaded). - -rsp_init_scratchpad_nif(_State, _Input, _JIT, _LargePages, _HardwareAES, _RoundCount) -> - ?LOG_ERROR("rsp_init_scratchpad_nif"), - erlang:nif_error(nif_not_loaded). - -rsp_mix_entropy_crc32_nif(_Entropy) -> - ?LOG_ERROR("rsp_mix_entropy_crc32_nif"), - erlang:nif_error(nif_not_loaded). - -rsp_mix_entropy_far_nif(_Entropy) -> - ?LOG_ERROR("rsp_mix_entropy_far_nif"), - erlang:nif_error(nif_not_loaded). - -% NOTE maybe this impl will replace rsp_mix_entropy_far_nif -rsp_mix_entropy_far_test_nif(_Entropy, _JumpSize, _BlockSize) -> - ?LOG_ERROR("rsp_mix_entropy_far_test_nif"), +rsp_fused_entropy_nif( + _RandomxState, + _ReplicaEntropySubChunkCount, + _CompositePackingSubChunkSize, + _LaneCount, + _RxDepth, + _JitEnabled, + _LargePagesEnabled, + _HardwareAESEnabled, + _RandomxProgramCount, + _Key +) -> + ?LOG_ERROR("rsp_fused_entropy_nif"), erlang:nif_error(nif_not_loaded). rsp_feistel_encrypt_nif(_InMsg, _Key) -> diff --git a/apps/arweave/test/ar_audit_tests.erl b/apps/arweave/test/ar_audit_tests.erl deleted file mode 100644 index 6414a0123..000000000 --- a/apps/arweave/test/ar_audit_tests.erl +++ /dev/null @@ -1,136 +0,0 @@ --module(ar_audit_tests). - --include_lib("eunit/include/eunit.hrl"). - --include_lib("arweave/include/ar_consensus.hrl"). - -setup_replica_2_9() -> - FastState = ar_mine_randomx:init_fast2(rxsquared, ?RANDOMX_PACKING_KEY, 0, 0, - erlang:system_info(dirty_cpu_schedulers_online)), - LightState = ar_mine_randomx:init_light2(rxsquared, ?RANDOMX_PACKING_KEY, 0, 0), - {FastState, LightState}. - -test_register(TestFun, Fixture) -> - {timeout, 120, {with, Fixture, [TestFun]}}. - -randomx_replica_2_9_suite_test_() -> - {setup, fun setup_replica_2_9/0, - fun (SetupData) -> - [ - test_register(fun test_vectors/1, SetupData), % TODO move bottom - test_register(fun test_state/1, SetupData), - test_register(fun test_quick/1, SetupData), - test_register(fun test_pack_unpack_sub_chunks/1, SetupData) - ] - end - }. - - -%% ------------------------------------------------------------------------------------------- -%% replica_2_9 tests -%% ------------------------------------------------------------------------------------------- -test_state({FastState, LightState}) -> - - ?assertEqual( - {ok, {rxsquared, fast, 67602036, 2097152}}, - ar_mine_randomx:info(FastState) - ), - ?assertEqual( - {ok, {rxsquared, light, 0, 2097152}}, - ar_mine_randomx:info(LightState) - ), - - {ok, {_, _, _, ScratchpadSize}} = ar_mine_randomx:info(FastState), - ?assertEqual(?RANDOMX_SCRATCHPAD_SIZE, ScratchpadSize). - -test_vectors({FastState, _LightState}) -> - Hash = << 255:(8*64) >>, - Scratchpad = << 255:(8*2097152) >>, - {ok, OutHash1Real, Output1} = ar_rxsquared_nif:rsp_exec_test_nif(element(2, FastState), Hash, Scratchpad, 0, 0, 0, 8), - Output1HashReal = crypto:hash(sha256, Output1), - Output1HashExpd = << 23,173,31,182,17,62,103,254,86,234,161,194,62,234, - 176,71,171,120,18,186,252,150,107,106,65,5,197,85, - 108,100,151,250 >>, - - {ok, Output2v1} = ar_rxsquared_nif:rsp_mix_entropy_crc32_nif(Output1), - Output2v1HashReal = crypto:hash(sha256, Output2v1), - {ok, OutHash2Real, Output2v2} = ar_rxsquared_nif:rsp_exec_nif(element(2, FastState), Hash, Scratchpad, 0, 0, 0, 8), - Output2v2HashReal = crypto:hash(sha256, Output2v2), - Output2HashExpd = << 133,226,122,189,170,63,128,182,242,28,50,204,85,179, - 230,105,98,187,39,24,30,133,84,135,70,85,220,145,30, - 165,161,242 >>, - OutHashExpd = << 137,100,229,43,87,136,2,64,101,172,17,65,106,94,24, - 209,195,201,194,250,35,211,175,73,15,102,11,25,12, - 147,231,196,194,48,55,194,181,47,41,136,101,215,179, - 124,181,223,195,140,217,56,206,55,144,184,44,131,86, - 206,247,3,124,167,34,75 >>, - - ?assertEqual(Output1HashExpd, Output1HashReal), - ?assertEqual(Output2HashExpd, Output2v1HashReal), - ?assertEqual(Output2HashExpd, Output2v2HashReal), - ?assertEqual(OutHashExpd, OutHash1Real), - ?assertEqual(OutHashExpd, OutHash2Real), - - Key = << 1 >>, - Entropy = ar_mine_randomx:randomx_generate_replica_2_9_entropy(FastState, Key), - EntropyHash = crypto:hash(sha256, Entropy), - EntropyHashExpd = <<56,199,231,119,170,151,220,154,45,204,70,193,80,68, - 46,50,136,31,35,102,141,77,19,66,191,127,97,183,230, - 119,243,151 >>, - ?assertEqual(EntropyHashExpd, EntropyHash), - - SubChunk = << 255:(8*8192) >>, - EntropySubChunkIndex = 1, - {ok, PackedOut} = ar_mine_randomx:randomx_encrypt_replica_2_9_sub_chunk({FastState, Entropy, SubChunk, - EntropySubChunkIndex}), - PackedOutHashReal = crypto:hash(sha256, PackedOut), - PackedOutHashExpd = << 25,148,72,35,27,27,6,222,247,71,104,10,58,78,178,211, - 204,199,238,124,237,101,100,96,27,64,234,145,250,78, - 75,207>>, - ?assertEqual(PackedOutHashExpd, PackedOutHashReal), - {ok, SubChunkReal} = ar_mine_randomx:randomx_decrypt_replica_2_9_sub_chunk({FastState, Key, PackedOut, - EntropySubChunkIndex}), - ?assertEqual(SubChunk, SubChunkReal), - - ok. - -test_quick({FastState, _LightState}) -> - {ok, _, _} = ar_rxsquared_nif:rsp_exec_nif( - element(2, FastState), << 0:(8*64) >>, << 0:(8*2097152) >>, 0, 0, 0, 8), - {ok, _, _} = ar_rxsquared_nif:rsp_init_scratchpad_nif( - element(2, FastState), <<"Some input">>, 0, 0, 0, 8), - {ok, _} = ar_rxsquared_nif:rsp_mix_entropy_crc32_nif(<< 0:(8*2097152) >>), - {ok, _} = ar_rxsquared_nif:rsp_mix_entropy_far_nif(<< 0:(8*2097152) >>), - {ok, _} = ar_rxsquared_nif:rsp_feistel_encrypt_nif( - << 0:(8*2097152) >>, << 0:(8*2097152) >>), - {ok, _} = ar_rxsquared_nif:rsp_feistel_decrypt_nif( - << 0:(8*2097152) >>, << 0:(8*2097152) >>). - -test_pack_unpack_sub_chunks({State, _LightState}) -> - Key = << 0:256 >>, - SubChunk = << 0:(8192 * 8) >>, - Entropy = ar_mine_randomx:randomx_generate_replica_2_9_entropy(State, Key), - ?assertEqual(268435456, byte_size(Entropy)), - PackedSubChunks = pack_sub_chunks(SubChunk, Entropy, 0, SubChunk, State), - ?assert(lists:all(fun(PackedSubChunk) -> byte_size(PackedSubChunk) == 8192 end, - PackedSubChunks)), - unpack_sub_chunks(PackedSubChunks, 0, SubChunk, Entropy). - -pack_sub_chunks(_SubChunk, _Entropy, Index, _PreviousSubChunk, _State) - when Index == 32768 -> - []; -pack_sub_chunks(SubChunk, Entropy, Index, PreviousSubChunk, State) -> - {ok, PackedSubChunk} = ar_mine_randomx:randomx_encrypt_replica_2_9_sub_chunk( - {State, Entropy, SubChunk, Index}), - Note = io_lib:format("Packed a sub-chunk, index=~B.~n", [Index]), - ?assertNotEqual(PackedSubChunk, PreviousSubChunk, Note), - [PackedSubChunk | pack_sub_chunks(SubChunk, Entropy, Index + 1, PackedSubChunk, State)]. - -unpack_sub_chunks([], _Index, _SubChunk, _Entropy) -> - ok; -unpack_sub_chunks([PackedSubChunk | PackedSubChunks], Index, SubChunk, Entropy) -> - {ok, UnpackedSubChunk} = ar_mine_randomx:randomx_decrypt_replica_2_9_sub_chunk2( - {Entropy, PackedSubChunk, Index}), - Note = io_lib:format("Unpacked a sub-chunk, index=~B.~n", [Index]), - ?assertEqual(SubChunk, UnpackedSubChunk, Note), - unpack_sub_chunks(PackedSubChunks, Index + 1, SubChunk, Entropy). diff --git a/apps/arweave/test/ar_packing_tests.erl b/apps/arweave/test/ar_packing_tests.erl index 6222bd0f6..1dc53a98a 100644 --- a/apps/arweave/test/ar_packing_tests.erl +++ b/apps/arweave/test/ar_packing_tests.erl @@ -28,8 +28,7 @@ packing_test_() -> {setup, fun setup/0, fun teardown/1, - [fun test_mix_crc/0, - fun test_mix_far/0, + [fun test_feistel/0, fun test_full_chunk/0, fun test_partial_chunk/0, fun test_full_chunk_repack/0, @@ -48,40 +47,28 @@ teardown(_) -> % optional cleanup code ok. -test_mix_crc() -> - Input1 = << 0:(8*8)>>, - {ok, RealOutput1} = ar_rxsquared_nif:rsp_mix_entropy_crc32_nif(Input1), - ExpdOutput1 = << 199,75,103,72,178,6,176,59 >>, - ?assertEqual(ExpdOutput1, RealOutput1), - - Input2 = << 1,2,3,4,5,6,7,8 >>, - {ok, RealOutput2} = ar_rxsquared_nif:rsp_mix_entropy_crc32_nif(Input2), - ExpdOutput2 = << 245,142,51,45,188,173,22,249 >>, - ?assertEqual(ExpdOutput2, RealOutput2), - ok. - -test_mix_far() -> - % divisible - Input1 = << 11, 12, 21, 22, 31, 32, 41, 42 >>, - ExodOutput1 = << 11, 21, 31, 41, 12, 22, 32, 42 >>, - {ok, RealOutput1} = ar_rxsquared_nif:rsp_mix_entropy_far_test_nif(Input1, 2, 1), - ?assertEqual(ExodOutput1, RealOutput1), - - Input2 = << 11, 12, 13, 14, 21, 22, 23, 24 >>, - ExodOutput2 = << 11, 21, 12, 22, 13, 23, 14, 24 >>, - {ok, RealOutput2} = ar_rxsquared_nif:rsp_mix_entropy_far_test_nif(Input2, 4, 1), - ?assertEqual(ExodOutput2, RealOutput2), - - Input3 = << 11, 12, 13, 14, 21, 22, 23, 24 >>, - ExodOutput3 = << 11, 12, 21, 22, 13, 14, 23, 24 >>, - {ok, RealOutput3} = ar_rxsquared_nif:rsp_mix_entropy_far_test_nif(Input3, 4, 2), - ?assertEqual(ExodOutput3, RealOutput3), - - % not divisible - Input4 = << 11, 12, 13, 14, 21, 22, 23, 24 >>, - ExodOutput4 = << 11, 12, 13, 21, 22, 23, 14, 24 >>, - {ok, RealOutput4} = ar_rxsquared_nif:rsp_mix_entropy_far_test_nif(Input4, 4, 3), - ?assertEqual(ExodOutput4, RealOutput4), +test_feistel()-> + Unpacked = << 1:(8*2097152) >>, + Entropy = << 2:(8*2097152) >>, + {ok, Packed} = ar_rxsquared_nif:rsp_feistel_encrypt_nif(Unpacked, Entropy), + PackedHashReal = crypto:hash(sha256, Packed), + PackedHashExpd = << 73,123,99,202,146,24,95,220,127,228,210,8,106,220,94, + 251,234,166,63,206,16,213,64,208,35,104,15,144,215, + 139,183,59 >>, + ?assertEqual(PackedHashExpd, PackedHashReal), + {ok, UnpackedReal} = ar_rxsquared_nif:rsp_feistel_decrypt_nif(Packed, Entropy), + ?assertEqual(Unpacked, UnpackedReal), + + Unpacked2 = << 3:(8*2097152) >>, + Entropy2 = << 4:(8*2097152) >>, + {ok, Packed2} = ar_rxsquared_nif:rsp_feistel_encrypt_nif(Unpacked2, Entropy2), + PackedHashReal2 = crypto:hash(sha256, Packed2), + PackedHashExpd2 = << 226,95,254,246,118,154,133,215,229,243,245,255,18,48, + 130,246,98,240,207,197,188,161,222,66,140,47,110,18, + 193,145,96,210 >>, + ?assertEqual(PackedHashExpd2, PackedHashReal2), + {ok, UnpackedReal2} = ar_rxsquared_nif:rsp_feistel_decrypt_nif(Packed2, Entropy2), + ?assertEqual(Unpacked2, UnpackedReal2), ok. test_full_chunk() -> diff --git a/apps/arweave/test/ar_replica_2_9_nif_tests.erl b/apps/arweave/test/ar_replica_2_9_nif_tests.erl new file mode 100644 index 000000000..607404240 --- /dev/null +++ b/apps/arweave/test/ar_replica_2_9_nif_tests.erl @@ -0,0 +1,104 @@ +-module(ar_replica_2_9_nif_tests). + +-include_lib("eunit/include/eunit.hrl"). + +-include_lib("arweave/include/ar_consensus.hrl"). + +setup_replica_2_9() -> + FastState = ar_mine_randomx:init_fast2(rxsquared, ?RANDOMX_PACKING_KEY, 0, 0, + erlang:system_info(dirty_cpu_schedulers_online)), + LightState = ar_mine_randomx:init_light2(rxsquared, ?RANDOMX_PACKING_KEY, 0, 0), + {FastState, LightState}. + +test_register(TestFun, Fixture) -> + {timeout, 120, {with, Fixture, [TestFun]}}. + +randomx_replica_2_9_suite_test_() -> + {setup, fun setup_replica_2_9/0, + fun (SetupData) -> + [ + test_register(fun test_vectors/1, SetupData), % TODO move bottom + test_register(fun test_state/1, SetupData), + test_register(fun test_pack_unpack_sub_chunks/1, SetupData) + ] + end + }. + + +%% ------------------------------------------------------------------------------------------- +%% replica_2_9 tests +%% ------------------------------------------------------------------------------------------- +test_state({FastState, LightState}) -> + + ?assertEqual( + {ok, {rxsquared, fast, 34047604, 2097152}}, + ar_mine_randomx:info(FastState) + ), + ?assertEqual( + {ok, {rxsquared, light, 0, 2097152}}, + ar_mine_randomx:info(LightState) + ), + + {ok, {_, _, _, ScratchpadSize}} = ar_mine_randomx:info(FastState), + ?assertEqual(?RANDOMX_SCRATCHPAD_SIZE, ScratchpadSize). + +test_vectors({FastState, _LightState}) -> + Key = << 1 >>, + Entropy = ar_mine_randomx:randomx_generate_replica_2_9_entropy(FastState, Key), + EntropyHash = crypto:hash(sha256, Entropy), + EntropyHashExpd = << 56,199,231,119,170,151,220,154,45,204,70,193,80,68, + 46,50,136,31,35,102,141,77,19,66,191,127,97,183,230, + 119,243,151 >>, + ?assertEqual(EntropyHashExpd, EntropyHash), + + Key2 = << 2 >>, + Entropy2 = ar_mine_randomx:randomx_generate_replica_2_9_entropy(FastState, Key2), + EntropyHash2 = crypto:hash(sha256, Entropy2), + EntropyHashExpd2 = << 206,47,133,111,139,20,31,64,185,33,107,29,14,10,252, + 76,201,75,203,186,131,32,20,45,34,125,76,248,64,90, + 220,196 >>, + ?assertEqual(EntropyHashExpd2, EntropyHash2), + + SubChunk = << 255:(8*8192) >>, + EntropySubChunkIndex = 1, + {ok, Packed} = ar_mine_randomx:randomx_encrypt_replica_2_9_sub_chunk( + {FastState, Entropy, SubChunk, EntropySubChunkIndex}), + PackedHashReal = crypto:hash(sha256, Packed), + PackedHashExpd = << 15,46,184,11,124,31,150,77,199,107,221,0,136,154,61, + 146,193,198,126,52,19,7,211,28,121,108,176,15,124,33, + 48,99 >>, + ?assertEqual(PackedHashExpd, PackedHashReal), + {ok, Unpacked} = ar_mine_randomx:randomx_decrypt_replica_2_9_sub_chunk( + {FastState, Key, Packed, EntropySubChunkIndex}), + ?assertEqual(SubChunk, Unpacked), + + ok. + +test_pack_unpack_sub_chunks({State, _LightState}) -> + Key = << 0:256 >>, + SubChunk = << 0:(8192 * 8) >>, + Entropy = ar_mine_randomx:randomx_generate_replica_2_9_entropy(State, Key), + ?assertEqual(8388608, byte_size(Entropy)), + PackedSubChunks = pack_sub_chunks(SubChunk, Entropy, 0, SubChunk, State), + ?assert(lists:all(fun(PackedSubChunk) -> byte_size(PackedSubChunk) == 8192 end, + PackedSubChunks)), + unpack_sub_chunks(PackedSubChunks, 0, SubChunk, Entropy). + +pack_sub_chunks(_SubChunk, _Entropy, Index, _PreviousSubChunk, _State) + when Index == 1024 -> + []; +pack_sub_chunks(SubChunk, Entropy, Index, PreviousSubChunk, State) -> + {ok, PackedSubChunk} = ar_mine_randomx:randomx_encrypt_replica_2_9_sub_chunk( + {State, Entropy, SubChunk, Index}), + Note = io_lib:format("Packed a sub-chunk, index=~B.~n", [Index]), + ?assertNotEqual(PackedSubChunk, PreviousSubChunk, Note), + [PackedSubChunk | pack_sub_chunks(SubChunk, Entropy, Index + 1, PackedSubChunk, State)]. + +unpack_sub_chunks([], _Index, _SubChunk, _Entropy) -> + ok; +unpack_sub_chunks([PackedSubChunk | PackedSubChunks], Index, SubChunk, Entropy) -> + {ok, UnpackedSubChunk} = ar_mine_randomx:randomx_decrypt_replica_2_9_sub_chunk2( + {Entropy, PackedSubChunk, Index}), + Note = io_lib:format("Unpacked a sub-chunk, index=~B.~n", [Index]), + ?assertEqual(SubChunk, UnpackedSubChunk, Note), + unpack_sub_chunks(PackedSubChunks, Index + 1, SubChunk, Entropy). diff --git a/apps/randomx_square_latency_tester/main.cpp b/apps/randomx_square_latency_tester/main.cpp index 0b7ff4f94..a9f603f80 100755 --- a/apps/randomx_square_latency_tester/main.cpp +++ b/apps/randomx_square_latency_tester/main.cpp @@ -2,7 +2,7 @@ #include #include #include -#include "pack_randomx_square.h" +#include "randomx_squared.h" int main() { return 0;