Merge branch 'feature/2.1-chunk-uploading'

.gitignore

@@ -3,3 +3,4 @@
 /node
 /web
 /bundles
+/debug-cli/testfiles/**/*

README.md

@@ -23,6 +23,7 @@ Arweave JS is the JavaScript/TypeScript SDK for interacting with the Arweave net
       - [Add tags to a transaction](#add-tags-to-a-transaction)
       - [Sign a transaction](#sign-a-transaction)
       - [Submit a transaction](#submit-a-transaction)
+        - [Chunked uploading advanced options](#chunked-uploading-advanced-options)
       - [Get a transaction status](#get-a-transaction-status)
       - [Get a transaction](#get-a-transaction)
       - [Get transaction data](#get-transaction-data)
@@ -312,7 +313,28 @@ console.log(transaction);
 
 #### Submit a transaction
 
-Once a transaction is submitted to the network it'll be broadcast around all nodes and mined into a block.
+The preferred method of submitting a data transaction is to use chunk uploading. This method will allow larger transaction sizes, resuming a transaction upload if its interrupted and give progress updates while uploading.
+
+Simple example:
+
+```js
+
+let data = fs.readFileSync('path/to/file.pdf');
+
+let transaction = await arweave.createTransaction({ data: data }, key);
+transaction.addTag('Content-Type', 'application/pdf');
+
+await arweave.transaction.sign(transaction, key);
+
+let uploader = arweave.transactions.getUploader(transaction);
+
+while (!uploader.isComplete) {
+  await uploader.uploadChunk();
+  console.log(`${uploader.pctComplete}% complete, ${uploader.uploadedChunks}/${uploader.totalChunks}`);
+}
+```
+
+You can also submit transactions using `transactions.post()` which is suitable for small transactions or token transfers:
 
 ```js
 let key = await arweave.wallets.generate();
@@ -332,6 +354,47 @@ console.log(response.status);
 // HTTP response codes (200 - ok, 400 - invalid transaction, 500 - error)
 ```
 
+##### Chunked uploading advanced options
+
+You can resume an upload from a saved uploader object, that you have persisted in storage some using `JSON.stringify(uploader)` at any stage of the upload. To resume, parse it back into an object pass it to `getUploader()` along with the transactions data:
+
+```js
+
+let data = fs.readFileSync('path/to/file.pdf'); // get the same data
+let resumeObject = JSON.parse(savedUploader); // get uploader object from where you stored it.
+
+let uploader = arweave.transactions.getUploader(resumeObject, data);
+while (!uploader.isComplete) {
+  await uploader.uploadChunk();
+}
+
+```
+
+When resuming the upload, you *must provide the same data* as the original upload. When you serialize the uploader object with `JSON.stringify()` to save it somewhere, it will not include the data.
+
+You can also resume an upload from just the transaction ID and data, once it has been mined into a block. This can be useful if you didn't save the uploader somewhere but the upload got interrupted. This will re-upload all of the data from the beginning, since we don't know which parts have been uploaded:
+
+```js
+
+let data = fs.readFileSync('path/to/file.pdf'); // get the same data
+let resumeTxId = 'mytxid' // a transaction id for a mined transaction that didn't complete the upload.
+
+let uploader = arweave.transactions.getUploader(resumeTxId, data);
+while (!uploader.isComplete) {
+  await uploader.uploadChunks();
+  console.log(`${progress.pctComplete}% complete`);
+}
+```
+
+There is also a async iterator interface to chunk uploading, but this method means you'll need to ensure you are using a transpiler and polyfill for the asyncIterator symbol for some environments. (Safari on iOS in particular). This method takes the same arguments for uploading/resuming a transaction as `getUploader()` and just has a slightly shorter syntax:
+
+```js
+for await (const uploader of arweave.transactions.upload(tx) {
+  console.log(`${uploader.pctComplete}% Complete`);
+}
+// done.
+```
+
 #### Get a transaction status
 
 ```js

debug-cli/README.md

@@ -0,0 +1,33 @@
+# debug-cli
+
+Just some very quick scripts to test things around chunk uploading and downloading. 
+
+For many of these, you'll need a wallet with funds, just store it as an environment var:
+
+```bash
+export WALLET_JSON=$(cat path/to/keyfile.json)
+``` 
+
+## Upload a file via chunks and print progress.
+
+```bash
+./test-chunk-upload.js <file> 
+```
+
+## Redo an upload from file and txid ( tx must be mined)
+
+```bash
+./test-chunk-resume-id.js <file> <txid>
+```
+
+## Resume an upload from a progress.json file
+
+```bash
+./test-chunk-resume-js <file> <progress.file.json> 
+```
+
+## Download all chunks from a txid
+
+```bash
+./test-chunk-dl.js <txid>
+```

debug-cli/test-chunk-dl.js

@@ -0,0 +1,25 @@
+#!/usr/bin/env node
+
+const Arweave = require('../node');
+const arweave = Arweave.init({ host: 'arweave.net', port: 443, protocol: 'https' });
+
+async function testIt(id) {
+
+  const offsetResponse = await arweave.chunks.getTransactionOffset(id); 
+  console.log(offsetResponse);
+  let offset = arweave.chunks.firstChunkOffset(offsetResponse);
+  let totalSize = 0;
+  while (offset < offsetResponse.offset) {
+    const chunk = await arweave.chunks.getChunk(offset);
+    const data = Arweave.utils.b64UrlToBuffer(chunk.chunk);
+    console.log(`Read chunk of size: ${(data.byteLength / 1024).toFixed(2)}KiB`);
+    offset += data.byteLength;
+    totalSize += data.byteLength;
+  }
+  console.log(`Finished, read: ${totalSize}.`);
+}
+
+const id = process.argv.slice(-1)[0];
+
+testIt(id)
+  .catch(e => console.error(e))

debug-cli/test-chunk-resume-id.js

@@ -0,0 +1,26 @@
+#!/usr/bin/env node
+
+const Arweave = require('../node');
+const fs = require('fs');
+const arweave = Arweave.init({ host: 'lon-1.eu-west-1.arweave.net', port: 1984, protocol: 'http' });
+
+const jwk = JSON.parse(process.env.WALLET_JSON)
+
+async function testIt(file, id) {
+  const data = fs.readFileSync(file);
+
+  for await (const progress of arweave.transactions.upload(id, data)) {
+    fs.writeFileSync(`${progress.transaction.id}.progress.json`, JSON.stringify(progress));
+    console.log(`${progress.transaction.id} - ${progress.pctComplete}% - ${progress.uploadedChunks}/${progress.totalChunks} - ${progress.lastResponseStatus} ${progress.lastResponseError}`)
+    //await new Promise(res => setTimeout(res, 1000 * 1)); 
+  }
+
+  return
+}
+
+const file = process.argv.slice(-2)[0];
+const id = process.argv.slice(-1)[0];
+
+testIt(file, id)
+  .then(x => console.log(x))
+  .catch(e => console.error(e))

debug-cli/test-chunk-resume.js

@@ -0,0 +1,25 @@
+#!/usr/bin/env node
+
+const Arweave = require('../node');
+const fs = require('fs');
+const arweave = Arweave.init({ host: 'arweave.net', port: 443, protocol: 'https' });
+
+const jwk = JSON.parse(process.env.WALLET_JSON)
+
+async function testIt(file, resume) {
+  const data = fs.readFileSync(file);
+
+  for await (const progress of arweave.transactions.upload(resume, data)) {
+    fs.writeFileSync(`${progress.transaction.id}.progress.json`, JSON.stringify(progress));
+    console.log(`${progress.transaction.id} - ${progress.pctComplete}% - ${progress.uploadedChunks}/${progress.totalChunks}`)
+  }
+
+  return resume.transaction.id;
+}
+
+const file = process.argv.slice(-2)[0];
+const resume = JSON.parse(fs.readFileSync(process.argv.slice(-1)[0]).toString());
+
+testIt(file, resume)
+  .then(x => console.log(x))
+  .catch(e => console.error(e))

debug-cli/test-chunk-sizing.js

@@ -0,0 +1,28 @@
+#!/usr/bin/env node
+
+const Arweave = require('../node');
+const fs = require('fs');
+const arweave = Arweave.init({ host: 'lon-1.eu-west-1.arweave.net', port: 1984, protocol: 'http' });
+
+const jwk = JSON.parse(process.env.WALLET_JSON)
+
+async function testIt(file) {
+  const data = fs.readFileSync(file);
+  const tx = await arweave.createTransaction({ data }, jwk);
+  tx.addTag('Test', 'Yes');
+  await arweave.transactions.sign(tx, jwk);
+
+  tx.chunks.chunks.forEach((chunk, idx) => {
+    const size = chunk.maxByteRange - chunk.minByteRange
+    console.log(`Chunk: ${idx} - ${size} - ${(size / 1024).toFixed(3)}, ${tx.chunks.proofs[idx].offset}`);
+  })
+  console.log(tx.data_root);
+  console.log(tx.data_size);
+  return tx.id;
+}
+
+const file = process.argv.slice(-1)[0];
+
+testIt(file)
+  .then(x => console.log(x))
+  .catch(e => console.error(e))

debug-cli/test-chunk-uploader.js

@@ -0,0 +1,29 @@
+#!/usr/bin/env node
+
+const Arweave = require('../node');
+const fs = require('fs');
+const arweave = Arweave.init({ host: 'arweave.net', port: 443, protocol: 'https' });
+
+const jwk = JSON.parse(process.env.WALLET_JSON)
+
+async function testIt(file) {
+  const data = fs.readFileSync(file);
+  const tx = await arweave.createTransaction({ data }, jwk);
+  tx.addTag('Test', 'Yes');
+  await arweave.transactions.sign(tx, jwk);
+
+  console.log(`uploading tx ${tx.id}`);
+
+  for await (const progress of arweave.transactions.upload(tx)) {
+    fs.writeFileSync(`${tx.id}.progress.json`, JSON.stringify(progress));
+    console.log(`${tx.id} - ${progress.pctComplete}% - ${progress.uploadedChunks}/${progress.totalChunks} - ${progress.lastResponseStatus} - ${progress.lastResponseError}`)
+  }
+
+  return tx.id;
+}
+
+const file = process.argv.slice(-1)[0];
+
+testIt(file)
+  .then(x => console.log(x))
+  .catch(e => console.error(e))

debug-cli/test-merkle-gen-time.js

@@ -0,0 +1,31 @@
+#!/usr/bin/env node
+
+const { chunkData, generateLeaves , buildLayers, generateProofs } = require('./node/lib/merkle');
+const fs = require('fs');
+
+async function testIt(file) {
+  const data = fs.readFileSync(file);
+
+  const t0 = Date.now()
+  const chunks = await chunkData(data);
+  const t1 = Date.now()
+  const leaves = await generateLeaves(chunks);
+  const t2 = Date.now()
+  const root = await buildLayers(leaves);
+  const t3 = Date.now()
+  const proofs = await generateProofs(root);
+  const t4 = Date.now()
+
+  console.log(`Chunking: ${(t1-t0)/1000}`);
+  console.log(`Leaves: ${(t2-t1)/1000}`);
+  console.log(`Layers: ${(t3-t2)/1000}`);
+  console.log(`Proofs: ${(t4-t3)/1000}`);
+  console.log(process.memoryUsage());
+  //console.log(`The script uses approximately ${Math.round(used * 100) / 100} MB`);
+}
+
+const file = process.argv.slice(-1)[0];
+
+testIt(file)
+  .then(x => console.log(x))
+  .catch(e => console.error(e))

debug-cli/test-non-generator-interface.js

@@ -0,0 +1,27 @@
+#!/usr/bin/env node
+
+const Arweave = require('../node');
+const fs = require('fs');
+const arweave = Arweave.init({ host: 'lon-1.eu-west-1.arweave.net', port: 1984, protocol: 'http' });
+
+const jwk = JSON.parse(process.env.WALLET_JSON)
+
+
+async function testIt(file, id) {
+  const data = fs.readFileSync(file);
+
+  let uploader = await arweave.transactions.getUploader(id, data);
+  while (!uploader.isComplete) {
+    await uploader.uploadChunk();
+    console.log(`${uploader.transaction.id} - ${uploader.pctComplete}% - ${uploader.uploadedChunks}/${uploader.totalChunks}`)
+  }
+
+  return uploader.transaction.id;
+}
+
+const file = process.argv.slice(-2)[0];
+const id = process.argv.slice(-1)[0];
+
+testIt(file, id)
+  .then(x => console.log(x))
+  .catch(e => console.error(e))

debug-cli/test-post-upload.js

@@ -0,0 +1,27 @@
+#!/usr/bin/env node
+
+const Arweave = require('../node');
+const fs = require('fs');
+const arweave = Arweave.init({ host: 'arweave.net', port: 443, protocol: 'https' });
+
+const jwk = JSON.parse(process.env.WALLET_JSON)
+
+
+async function testIt(file) {
+  const data = fs.readFileSync(file);
+  const tx = await arweave.createTransaction({ data }, jwk);
+  tx.addTag('Test', 'Yes');
+  await arweave.transactions.sign(tx, jwk);
+  tx.addTag('Foo', 'whut');
+
+  const resp = await arweave.transactions.post(tx);
+  console.log(resp.status);
+  console.log(resp.statusText);
+  return tx.id;
+}
+
+const file = process.argv.slice(-1)[0];
+
+testIt(file)
+  .then(x => console.log(x))
+  .catch(e => console.error(e))

src/common/chunks.ts

@@ -0,0 +1,40 @@
+import Api from "./lib/api";
+import { getError } from "./lib/error";
+
+export interface TransactionOffsetResponse {
+  size: string
+  offset: string
+}
+
+export interface TransactionChunkResponse {
+  chunk: string
+  data_path: string 
+  tx_path: string
+}
+
+export default class Chunks {
+
+  constructor(private api: Api) {
+  }
+
+  async getTransactionOffset(tx: string): Promise<TransactionOffsetResponse> {
+    const resp = await this.api.request().get(`/tx/${tx}/offset`)
+    if (resp.status === 200) {
+      return resp.data
+    }
+    throw new Error(`Unable to get transaction offset: ${getError(resp)}`);
+  }
+
+  async getChunk(offset: string | number | BigInt): Promise<TransactionChunkResponse> {
+    const resp = await this.api.request().get(`/chunk/${offset}`);
+    if (resp.status === 200) {
+      return resp.data
+    }
+    throw new Error(`Unable to get chunk: ${getError(resp)}`);
+  }
+
+  firstChunkOffset(offsetResponse: TransactionOffsetResponse): number {
+    return parseInt(offsetResponse.offset) - parseInt(offsetResponse.size) + 1;
+  }
+
+}