Skip to content

Commit

Permalink
merge main
Browse files Browse the repository at this point in the history
  • Loading branch information
paulo-ocean committed Jan 13, 2025
2 parents 43b3823 + c94b0dd commit 12f06db
Show file tree
Hide file tree
Showing 18 changed files with 619 additions and 141 deletions.
2 changes: 1 addition & 1 deletion .env.example
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ export INDEXER_INTERVAL=
export ALLOWED_ADMINS=
export DASHBOARD=true
export RATE_DENY_LIST=
export MAX_REQ_PER_SECOND=
export MAX_REQ_PER_MINUTE=
export MAX_CHECKSUM_LENGTH=
export LOG_LEVEL=
export HTTP_API_PORT=
Expand Down
20 changes: 2 additions & 18 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -132,15 +132,6 @@ jobs:
- run: docker image ls
- name: Delete default runner images
run: |
docker image rm node:20
docker image rm node:20-alpine
docker image rm node:18
docker image rm node:18-alpine
docker image rm debian:10
docker image rm debian:11
docker image rm ubuntu:22.04
docker image rm ubuntu:20.04
docker image rm moby/buildkit:latest
rm -rf /usr/share/swift/
- name: Wait for contracts deployment and C2D cluster to be ready
working-directory: ${{ github.workspace }}/barge
Expand Down Expand Up @@ -226,15 +217,6 @@ jobs:
- run: docker image ls
- name: Delete default runner images
run: |
docker image rm node:20
docker image rm node:20-alpine
docker image rm node:18
docker image rm node:18-alpine
docker image rm debian:10
docker image rm debian:11
docker image rm ubuntu:22.04
docker image rm ubuntu:20.04
docker image rm moby/buildkit:latest
rm -rf /usr/share/swift/
- name: Wait for contracts deployment and C2D cluster to be ready
Expand Down Expand Up @@ -278,6 +260,8 @@ jobs:
P2P_ENABLE_AUTONAT: 'false'
ALLOWED_ADMINS: '["0xe2DD09d719Da89e5a3D0F2549c7E24566e947260"]'
DB_TYPE: 'elasticsearch'
MAX_REQ_PER_MINUTE: 320
MAX_CONNECTIONS_PER_MINUTE: 320
- name: Check Ocean Node is running
run: |
for i in $(seq 1 90); do
Expand Down
47 changes: 40 additions & 7 deletions docs/API.md
Original file line number Diff line number Diff line change
Expand Up @@ -449,6 +449,38 @@ returns P2P peer

---

## find peer multiaddress

### `HTTP` GET /findPeer/?

#### Description

returns P2P peer multiaddresses if found in DHT

#### Query Parameters

| name | type | required | description |
| ------- | ------ | -------- | ----------- |
| peerId | string | v | peer id |
| timeout | int | optional | timeout |

#### Response

```
{
"id": "16Uiu2HAmLhRDqfufZiQnxvQs2XHhd6hwkLSPfjAQg1gH8wgRixiP",
"multiaddrs": [
"/ip4/127.0.0.1/tcp/9000",
"/ip4/127.0.0.1/tcp/9001/ws",
"/ip4/172.18.0.2/tcp/9000",
"/ip4/172.18.0.2/tcp/9001/ws",
"/ip6/::1/tcp/9002"
]
}
```

---

## Get P2P Peers

### `HTTP` GET /getP2PPeers
Expand Down Expand Up @@ -501,13 +533,14 @@ returns an empty object if it is valid otherwise an array with error

#### Parameters

| name | type | required | description |
| ---------- | ------ | -------- | ------------------------------------------------- |
| command | string | v | command name |
| node | string | | if not present it means current node |
| id | string | v | document id or did |
| chainId | number | v | chain id of network on which document is provided |
| nftAddress | string | v | address of nft token |
| name | type | required | description |
| ---------- | -------- | -------- | ------------------------------------------------- |
| command | string | v | command name |
| node | string | | if not present it means current node |
| multiAddrs | string[] | | if passed, use this instead of peerStore & DHT |
| id | string | v | document id or did |
| chainId | number | v | chain id of network on which document is provided |
| nftAddress | string | v | address of nft token |

#### Request

Expand Down
2 changes: 1 addition & 1 deletion docs/dockerDeployment.md
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,7 @@ services:
# INDEXER_INTERVAL: ''
DASHBOARD: 'true'
# RATE_DENY_LIST: ''
# MAX_REQ_PER_SECOND: ''
# MAX_REQ_PER_MINUTE: ''
# MAX_CHECKSUM_LENGTH: ''
# LOG_LEVEL: ''
HTTP_API_PORT: '8000'
Expand Down
5 changes: 3 additions & 2 deletions docs/env.md
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,8 @@ Environmental variables are also tracked in `ENVIRONMENT_VARIABLES` within `src/
- `ALLOWED_ADMINS`: Sets the public address of accounts which have access to admin endpoints e.g. shutting down the node. Example: `"[\"0x967da4048cD07aB37855c090aAF366e4ce1b9F48\",\"0x388C818CA8B9251b393131C08a736A67ccB19297\"]"`
- `DASHBOARD`: If `false` the dashboard will not run. If not set or `true` the dashboard will start with the node. Example: `false`
- `RATE_DENY_LIST`: Blocked list of IPs and peer IDs. Example: `"{ \"peers\": [\"16Uiu2HAkuYfgjXoGcSSLSpRPD6XtUgV71t5RqmTmcqdbmrWY9MJo\"], \"ips\": [\"127.0.0.1\"] }"`
- `MAX_REQ_PER_SECOND`: Number of requests per second allowed by the same client. Example: `3`
- `MAX_REQ_PER_MINUTE`: Number of requests per minute allowed by the same client (IP or Peer id). Example: `30`
- `MAX_CONNECTIONS_PER_MINUTE`: Max number of requests allowed per minute (all clients). Example: `120`
- `MAX_CHECKSUM_LENGTH`: Define the maximum length for a file if checksum is required (Mb). Example: `10`
- `IS_BOOTSTRAP`: Is this node to be used as bootstrap node or not. Default is `false`.

Expand Down Expand Up @@ -54,7 +55,7 @@ Environmental variables are also tracked in `ENVIRONMENT_VARIABLES` within `src/
- `P2P_pubsubPeerDiscoveryInterval`: Interval (in ms) for discovery using pubsub. Defaults to `10000` (three seconds). Example: `10000`
- `P2P_dhtMaxInboundStreams`: Maximum number of DHT inbound streams. Defaults to `500`. Example: `500`
- `P2P_dhtMaxOutboundStreams`: Maximum number of DHT outbound streams. Defaults to `500`. Example: `500`
- `P2P_ENABLE_DHT_SERVER`: Enable DHT server mode. This should be enabled for bootstrapers & well established nodes. Default: `false`
- `P2P_DHT_FILTER`: Filter address in DHT. 0 = (Default) No filter 1. Filter private ddresses. 2. Filter public addresses
- `P2P_mDNSInterval`: Interval (in ms) for discovery using mDNS. Defaults to `20000` (20 seconds). Example: `20000`
- `P2P_connectionsMaxParallelDials`: Maximum number of parallel dials. Defaults to `150`. Example: `150`
- `P2P_connectionsDialTimeout`: Timeout for dial commands. Defaults to `10000` (10 seconds). Example: `10000`
Expand Down
3 changes: 2 additions & 1 deletion scripts/ocean-node-quickstart.sh
Original file line number Diff line number Diff line change
Expand Up @@ -192,7 +192,8 @@ services:
# INDEXER_INTERVAL: ''
DASHBOARD: 'true'
# RATE_DENY_LIST: ''
# MAX_REQ_PER_SECOND: ''
# MAX_REQ_PER_MINUTE: ''
# MAX_CONNECTIONS_PER_MINUTE: ''
# MAX_CHECKSUM_LENGTH: ''
# LOG_LEVEL: ''
HTTP_API_PORT: '$HTTP_API_PORT'
Expand Down
203 changes: 203 additions & 0 deletions src/@types/C2D.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,203 @@
import type { MetadataAlgorithm } from './DDO/Metadata.js'

export enum C2DClusterType {
// eslint-disable-next-line no-unused-vars
OPF_K8 = 0,
// eslint-disable-next-line no-unused-vars
NODE_LOCAL = 1,
// eslint-disable-next-line no-unused-vars
DOCKER = 2
}

export interface C2DClusterInfo {
/** Type of cluster: K8, Node local, etc */
type: C2DClusterType
/** Hash of cluster. hash(url) for remote, hash(nodeId) for local */
hash: string
/** Connection URI */
connection?: string
/** Folder for storing data */
tempFolder?: string
}

export interface ComputeEnvironment {
id: string
cpuNumber: number
cpuType: string
gpuNumber: number
gpuType: string
ramGB: number
diskGB: number
priceMin: number
desc: string
currentJobs: number
maxJobs: number
consumerAddress: string
storageExpiry: number
maxJobDuration: number
lastSeen: number
chainId?: number
feeToken: string
}

export interface ComputeEnvByChain {
[chainId: number]: ComputeEnvironment[]
}

export type ComputeResultType =
| 'algorithmLog'
| 'output'
| 'configrationLog'
| 'publishLog'

export interface ComputeResult {
filename: string
filesize: number
type: ComputeResultType
index?: number
}

export interface ComputeJob {
owner: string
did?: string
jobId: string
dateCreated: string
dateFinished: string
status: number
statusText: string
results: ComputeResult[]
inputDID?: string[]
algoDID?: string
agreementId?: string
expireTimestamp: number
}

export interface ComputeOutput {
publishAlgorithmLog?: boolean
publishOutput?: boolean
providerAddress?: string
providerUri?: string
metadataUri?: string
nodeUri?: string
owner?: string
secretStoreUri?: string
whitelist?: string[]
}

export interface ComputeAsset {
url?: string
documentId: string
serviceId: string
transferTxId?: string
userdata?: { [key: string]: any }
}

export interface ComputeAlgorithm {
documentId?: string
serviceId?: string
url?: string
meta?: MetadataAlgorithm
transferTxId?: string
algocustomdata?: { [key: string]: any }
userdata?: { [key: string]: any }
}

/* The following are specific to OPF_k8 compute engine */
export interface OPFK8ComputeStageInput {
index: number
id?: string
remote?: any
url?: string[]
}
export interface OPFK8ComputeStageAlgorithm {
id?: string
url?: string
remote?: any
rawcode?: string
container?: {
/**
* The command to execute, or script to run inside the Docker image.
* @type {string}
*/
entrypoint: string

/**
* Name of the Docker image.
* @type {string}
*/
image: string

/**
* Tag of the Docker image.
* @type {string}
*/
tag: string
}
}

export interface OPFK8ComputeOutput {
// this is a copy of ComputeOutput, but they could diverge in the future
publishAlgorithmLog?: boolean
publishOutput?: boolean
providerAddress?: string
providerUri?: string
metadataUri?: string
nodeUri?: string
owner?: string
secretStoreUri?: string
whitelist?: string[]
}
export interface OPFK8ComputeStage {
index: number
input: OPFK8ComputeStageInput[]
algorithm: OPFK8ComputeStageAlgorithm
compute?: {}
output: OPFK8ComputeOutput
}

export interface OPFK8ComputeWorkflow {
stages: OPFK8ComputeStage[]
}
export interface OPFK8ComputeStart {
workflow: OPFK8ComputeWorkflow
owner: string
agreementId: string
providerSignature: string
providerAddress: string
environment: string
validUntil: number
nonce: number
chainId: number
}

export interface OPFK8ComputeStop {
jobId: string
owner: string
agreementId?: string
providerSignature: string // message=owner+jobId
providerAddress: string
nonce: number
}

export interface OPFK8ComputeGetStatus {
agreementId?: string
jobId?: string
owner?: string
providerSignature: string // message=owner+jobId(if any)
providerAddress: string
nonce: number
}

export interface OPFK8ComputeGetResult {
jobId: string
owner: string
index: number
providerSignature: string // message=owner+jobId
providerAddress: string
nonce: number
}

export interface AlgoChecksums {
files: string
container: string
}
12 changes: 9 additions & 3 deletions src/@types/OceanNode.ts
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,12 @@ export interface OceanNodeKeys {
privateKey: any
ethAddress: string
}

/* eslint-disable no-unused-vars */
export enum dhtFilterMethod {
filterPrivate = 'filterPrivate', // default, remove all private addresses from DHT
filterPublic = 'filterPublic', // remove all public addresses from DHT
filterNone = 'filterNone' // do not remove all any addresses from DHT
}
export interface OceanNodeP2PConfig {
bootstrapNodes: string[]
bootstrapTimeout: number
Expand All @@ -41,7 +46,7 @@ export interface OceanNodeP2PConfig {
pubsubPeerDiscoveryInterval: number
dhtMaxInboundStreams: number
dhtMaxOutboundStreams: number
enableDHTServer: boolean
dhtFilter: dhtFilterMethod
mDNSInterval: number
connectionsMaxParallelDials: number
connectionsDialTimeout: number
Expand Down Expand Up @@ -81,7 +86,8 @@ export interface OceanNodeConfig {
assetPurgatoryUrl: string
allowedAdmins?: string[]
codeHash?: string
rateLimit?: number
rateLimit?: number // per request ip or peer
maxConnections?: number // global, regardless of client address(es)
denyList?: DenyList
unsafeURLs?: string[]
isBootstrap?: boolean
Expand Down
Loading

0 comments on commit 12f06db

Please sign in to comment.