Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(sdk): adds Collections API #212

Merged
merged 6 commits into from
Nov 20, 2024
Merged

feat(sdk): adds Collections API #212

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
ec22e09
feat: adds Collections API to SDK
imdominicreed Nov 19, 2024
1aebc4f
adds NanoTDF Collection on Encrypt
imdominicreed Nov 20, 2024
72b8732
adds CollectionStore
imdominicreed Nov 20, 2024
289cb03
creates collection examples
imdominicreed Nov 20, 2024
e78dbdb
adds collection test
imdominicreed Nov 20, 2024
62761f9
Add NoOpCollectionStore
imdominicreed Nov 20, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
41 changes: 41 additions & 0 deletions examples/src/main/java/io/opentdf/platform/DecryptCollectionExample.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
package io.opentdf.platform;

import io.opentdf.platform.sdk.Config;
import io.opentdf.platform.sdk.NanoTDF;
import io.opentdf.platform.sdk.SDK;
import io.opentdf.platform.sdk.SDKBuilder;

import java.io.ByteArrayInputStream;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.security.NoSuchAlgorithmException;

public class DecryptCollectionExample {
public static void main(String[] args) throws IOException, NanoTDF.NanoTDFMaxSizeLimit, NanoTDF.UnsupportedNanoTDFFeature, NanoTDF.InvalidNanoTDFConfig, NoSuchAlgorithmException, InterruptedException {
String clientId = "opentdf-sdk";
String clientSecret = "secret";
String platformEndpoint = "localhost:8080";

SDKBuilder builder = new SDKBuilder();
SDK sdk = builder.platformEndpoint(platformEndpoint)
.clientSecret(clientId, clientSecret).useInsecurePlaintextConnection(true)
.build();

var kasInfo = new Config.KASInfo();
kasInfo.URL = "http://localhost:8080/kas";


// Convert String to InputStream
NanoTDF nanoTDFClient = new NanoTDF(true);

for (int i = 0; i < 50; i++) {
FileInputStream fis = new FileInputStream(String.format("out/my.%d_ciphertext", i));
nanoTDFClient.readNanoTDF(ByteBuffer.wrap(fis.readAllBytes()), System.out, sdk.getServices().kas());
fis.close();
}

}
}
49 changes: 49 additions & 0 deletions examples/src/main/java/io/opentdf/platform/EncryptCollectionExample.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
package io.opentdf.platform;

import io.opentdf.platform.sdk.Config;
import io.opentdf.platform.sdk.NanoTDF;
import io.opentdf.platform.sdk.SDK;
import io.opentdf.platform.sdk.SDKBuilder;

import java.io.ByteArrayInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.security.NoSuchAlgorithmException;

public class EncryptCollectionExample {
public static void main(String[] args) throws IOException, NanoTDF.NanoTDFMaxSizeLimit, NanoTDF.UnsupportedNanoTDFFeature, NanoTDF.InvalidNanoTDFConfig, NoSuchAlgorithmException, InterruptedException {
String clientId = "opentdf-sdk";
String clientSecret = "secret";
String platformEndpoint = "localhost:8080";

SDKBuilder builder = new SDKBuilder();
SDK sdk = builder.platformEndpoint(platformEndpoint)
.clientSecret(clientId, clientSecret).useInsecurePlaintextConnection(true)
.build();

var kasInfo = new Config.KASInfo();
kasInfo.URL = "http://localhost:8080/kas";

var tdfConfig = Config.newNanoTDFConfig(
Config.withNanoKasInformation(kasInfo),
Config.witDataAttributes("https://example.com/attr/attr1/value/value1"),
Config.withCollection()
);

String str = "Hello, World!";

// Convert String to InputStream
var in = new ByteArrayInputStream(str.getBytes(StandardCharsets.UTF_8));
NanoTDF nanoTDFClient = new NanoTDF();

for (int i = 0; i < 50; i++) {
FileOutputStream fos = new FileOutputStream(String.format("out/my.%d_ciphertext", i));
nanoTDFClient.createNanoTDF(ByteBuffer.wrap(str.getBytes()), fos, tdfConfig,
sdk.getServices().kas());
}

}
}
68 changes: 68 additions & 0 deletions sdk/src/main/java/io/opentdf/platform/sdk/Config.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,15 @@

import io.opentdf.platform.sdk.Autoconfigure.AttributeValueFQN;
import io.opentdf.platform.sdk.nanotdf.ECCMode;
import io.opentdf.platform.sdk.nanotdf.Header;
import io.opentdf.platform.sdk.nanotdf.NanoTDFType;
import io.opentdf.platform.sdk.nanotdf.SymmetricAndPayloadConfig;

import io.opentdf.platform.policy.Value;
import org.bouncycastle.oer.its.ieee1609dot2.HeaderInfo;

import java.util.*;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Consumer;

/**
Expand All @@ -20,6 +23,7 @@ public class Config {
public static final int DEFAULT_SEGMENT_SIZE = 2 * 1024 * 1024; // 2mb
public static final String KAS_PUBLIC_KEY_PATH = "/kas_public_key";
public static final String DEFAULT_MIME_TYPE = "application/octet-stream";
public static final int MAX_COLLECTION_ITERATION = (1 << 24) - 1;

public enum TDFFormat {
JSONFormat,
Expand Down Expand Up @@ -248,6 +252,7 @@ public static class NanoTDFConfig {
public SymmetricAndPayloadConfig config;
public List<String> attributes;
public List<KASInfo> kasInfoList;
public CollectionConfig collectionConfig;

public NanoTDFConfig() {
this.eccMode = new ECCMode();
Expand All @@ -262,6 +267,7 @@ public NanoTDFConfig() {

this.attributes = new ArrayList<>();
this.kasInfoList = new ArrayList<>();
this.collectionConfig = new CollectionConfig(false);
}
}

Expand All @@ -273,6 +279,12 @@ public static NanoTDFConfig newNanoTDFConfig(Consumer<NanoTDFConfig>... options)
return config;
}

public static Consumer<NanoTDFConfig> withCollection() {
return (NanoTDFConfig config) -> {
config.collectionConfig = new CollectionConfig(true);
};
}

public static Consumer<NanoTDFConfig> witDataAttributes(String... attributes) {
return (NanoTDFConfig config) -> {
Collections.addAll(config.attributes, attributes);
Expand Down Expand Up @@ -304,4 +316,60 @@ public static Consumer<NanoTDFConfig> withEllipticCurve(String curve) {
public static Consumer<NanoTDFConfig> WithECDSAPolicyBinding() {
return (NanoTDFConfig config) -> config.eccMode.setECDSABinding(false);
}

public static class HeaderInfo {
private final Header header;
private final AesGcm key;
private final int iteration;

public HeaderInfo(Header header,AesGcm key, int iteration) {
this.header = header;
this.key = key;
this.iteration = iteration;
}

public Header getHeader() {
return header;
}

public int getIteration() {
return iteration;
}

public AesGcm getKey() {
return key;
}
}

public static class CollectionConfig {
private int iterationCounter;
private HeaderInfo headerInfo;
public final boolean useCollection;
private Boolean updatedHeaderInfo;


public CollectionConfig(boolean useCollection) {
this.useCollection = useCollection;
}

public synchronized HeaderInfo getHeaderInfo() throws InterruptedException {
int iteration = iterationCounter;
iterationCounter = (iterationCounter + 1) % MAX_COLLECTION_ITERATION;

if (iteration == 0) {
updatedHeaderInfo = false;
return null;
}
while (!updatedHeaderInfo) {
this.wait();
}
return new HeaderInfo(headerInfo.getHeader(), headerInfo.getKey(), iteration);
}

public synchronized void updateHeaderInfo(HeaderInfo headerInfo) {
this.headerInfo = headerInfo;
updatedHeaderInfo = true;
this.notifyAll();
}
}
}
112 changes: 86 additions & 26 deletions sdk/src/main/java/io/opentdf/platform/sdk/NanoTDF.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import java.io.IOException;
import java.io.OutputStream;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.nio.charset.StandardCharsets;
import java.security.*;
import java.util.*;
Expand All @@ -31,6 +32,19 @@ public class NanoTDF {
private static final int kIvPadding = 9;
private static final int kNanoTDFIvSize = 3;
private static final byte[] kEmptyIV = new byte[] { 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 };
private final CollectionStore collectionStore;

public NanoTDF() {
this(new CollectionStore.NoOpCollectionStore());
}

public NanoTDF(boolean collectionStoreEnabled) {
this(collectionStoreEnabled ? new CollectionStoreImpl() : null);
}

public NanoTDF(CollectionStore collectionStore) {
this.collectionStore = collectionStore;
}

public static class NanoTDFMaxSizeLimit extends Exception {
public NanoTDFMaxSizeLimit(String errorMessage) {
Expand All @@ -50,19 +64,16 @@ public InvalidNanoTDFConfig(String errorMessage) {
}
}

public int createNanoTDF(ByteBuffer data, OutputStream outputStream,
Config.NanoTDFConfig nanoTDFConfig,
SDK.KAS kas) throws IOException, NanoTDFMaxSizeLimit, InvalidNanoTDFConfig,
NoSuchAlgorithmException, UnsupportedNanoTDFFeature {

int nanoTDFSize = 0;
Gson gson = new GsonBuilder().create();

int dataSize = data.limit();
if (dataSize > kMaxTDFSize) {
throw new NanoTDFMaxSizeLimit("exceeds max size for nano tdf");
private Config.HeaderInfo getHeaderInfo(Config.NanoTDFConfig nanoTDFConfig, SDK.KAS kas)
throws InvalidNanoTDFConfig, UnsupportedNanoTDFFeature, NoSuchAlgorithmException, InterruptedException {
if (nanoTDFConfig.collectionConfig.useCollection) {
Config.HeaderInfo headerInfo = nanoTDFConfig.collectionConfig.getHeaderInfo();
if (headerInfo != null) {
return headerInfo;
}
}

Gson gson = new GsonBuilder().create();
if (nanoTDFConfig.kasInfoList.isEmpty()) {
throw new InvalidNanoTDFConfig("kas url is missing");
}
Expand Down Expand Up @@ -120,9 +131,32 @@ public int createNanoTDF(ByteBuffer data, OutputStream outputStream,
header.setPayloadConfig(nanoTDFConfig.config);
header.setEphemeralKey(compressedPubKey);
header.setKasLocator(kasURL);

header.setPolicyInfo(policyInfo);

Config.HeaderInfo headerInfo = new Config.HeaderInfo(header, gcm, 0);
if (nanoTDFConfig.collectionConfig.useCollection) {
nanoTDFConfig.collectionConfig.updateHeaderInfo(headerInfo);
}

return headerInfo;
}

public int createNanoTDF(ByteBuffer data, OutputStream outputStream,
Config.NanoTDFConfig nanoTDFConfig,
SDK.KAS kas) throws IOException, NanoTDFMaxSizeLimit, InvalidNanoTDFConfig,
NoSuchAlgorithmException, UnsupportedNanoTDFFeature, InterruptedException {
int nanoTDFSize = 0;

int dataSize = data.limit();
if (dataSize > kMaxTDFSize) {
throw new NanoTDFMaxSizeLimit("exceeds max size for nano tdf");
}

Config.HeaderInfo headerKeyPair = getHeaderInfo(nanoTDFConfig, kas);
Header header = headerKeyPair.getHeader();
AesGcm gcm = headerKeyPair.getKey();
int iteration = headerKeyPair.getIteration();

int headerSize = header.getTotalSize();
ByteBuffer bufForHeader = ByteBuffer.allocate(headerSize);
header.writeIntoBuffer(bufForHeader);
Expand All @@ -132,13 +166,21 @@ public int createNanoTDF(ByteBuffer data, OutputStream outputStream,
nanoTDFSize += headerSize;
logger.debug("createNanoTDF header length {}", headerSize);

int authTagSize = SymmetricAndPayloadConfig.sizeOfAuthTagForCipher(nanoTDFConfig.config.getCipherType());
// Encrypt the data
byte[] actualIV = new byte[kIvPadding + kNanoTDFIvSize];
do {
byte[] iv = new byte[kNanoTDFIvSize];
SecureRandom.getInstanceStrong().nextBytes(iv);
System.arraycopy(iv, 0, actualIV, kIvPadding, iv.length);
} while (Arrays.equals(actualIV, kEmptyIV)); // if match, we need to retry to prevent key + iv reuse with the policy
if (nanoTDFConfig.collectionConfig.useCollection) {
ByteBuffer b = ByteBuffer.allocate(4);
b.order(ByteOrder.LITTLE_ENDIAN);
b.putInt(iteration);
System.arraycopy(b.array(), 0, actualIV, kIvPadding, kNanoTDFIvSize);
} else {
do {
byte[] iv = new byte[kNanoTDFIvSize];
SecureRandom.getInstanceStrong().nextBytes(iv);
System.arraycopy(iv, 0, actualIV, kIvPadding, iv.length);
} while (Arrays.equals(actualIV, kEmptyIV)); // if match, we need to retry to prevent key + iv reuse with the policy
}

byte[] cipherData = gcm.encrypt(actualIV, authTagSize, data.array(), data.arrayOffset(), dataSize);

Expand All @@ -157,23 +199,30 @@ public int createNanoTDF(ByteBuffer data, OutputStream outputStream,
return nanoTDFSize;
}


public void readNanoTDF(ByteBuffer nanoTDF, OutputStream outputStream,
SDK.KAS kas) throws IOException {

Header header = new Header(nanoTDF);
CollectionKey cachedKey = collectionStore.getKey(header);
byte[] key = cachedKey.getKey();

// create base64 encoded
byte[] headerData = new byte[header.getTotalSize()];
header.writeIntoBuffer(ByteBuffer.wrap(headerData));
String base64HeaderData = Base64.getEncoder().encodeToString(headerData);
// perform unwrap is not in collectionStore;
if (key == null) {
// create base64 encoded
byte[] headerData = new byte[header.getTotalSize()];
header.writeIntoBuffer(ByteBuffer.wrap(headerData));
String base64HeaderData = Base64.getEncoder().encodeToString(headerData);

logger.debug("readNanoTDF header length {}", headerData.length);
logger.debug("readNanoTDF header length {}", headerData.length);

String kasUrl = header.getKasLocator().getResourceUrl();
String kasUrl = header.getKasLocator().getResourceUrl();

byte[] key = kas.unwrapNanoTDF(header.getECCMode().getEllipticCurveType(),
base64HeaderData,
kasUrl);
key = kas.unwrapNanoTDF(header.getECCMode().getEllipticCurveType(),
base64HeaderData,
kasUrl);
collectionStore.store(header, new CollectionKey(key));
}

byte[] payloadLengthBuf = new byte[4];
nanoTDF.get(payloadLengthBuf, 1, 3);
Expand Down Expand Up @@ -213,4 +262,15 @@ PolicyObject createPolicyObject(List<String> attributes) {
}
return policyObject;
}

public static class CollectionKey {
private final byte[] key;

public CollectionKey(byte[] key) {
this.key = key;
}
protected byte[] getKey() {
return key;
}
}
}
Loading