Skip to content

Commit

Permalink
corrections
Browse files Browse the repository at this point in the history
  • Loading branch information
@Baunsgaard
Baunsgaard committed Jan 24, 2025
1 parent 81c5a51 commit 174ed30
Show file tree
Hide file tree
Showing 10 changed files with 325 additions and 144 deletions.
47 changes: 35 additions & 12 deletions src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/IdentityDictionary.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,21 +51,48 @@ public class IdentityDictionary extends AIdentityDictionary {
*
* @param nRowCol The number of rows and columns in this identity matrix.
*/
public IdentityDictionary(int nRowCol) {
private IdentityDictionary(int nRowCol) {
super(nRowCol);
}

/**
* Create an identity matrix dictionary. It behaves as if allocated a Sparse Matrix block but exploits that the
* structure is known to have certain properties.
*
* @param nRowCol The number of rows and columns in this identity matrix.
*/
public static IDictionary create(int nRowCol) {
return create(nRowCol, false);
}

/**
* Create an identity matrix dictionary, It behaves as if allocated a Sparse Matrix block but exploits that the
* structure is known to have certain properties.
*
* @param nRowCol The number of rows and columns in this identity matrix.
* @param withEmpty If the matrix should contain an empty row in the end.
*/
public IdentityDictionary(int nRowCol, boolean withEmpty) {
private IdentityDictionary(int nRowCol, boolean withEmpty) {
super(nRowCol, withEmpty);
}

/**
* Create an identity matrix dictionary, It behaves as if allocated a Sparse Matrix block but exploits that the
* structure is known to have certain properties.
*
* @param nRowCol The number of rows and columns in this identity matrix.
* @param withEmpty If the matrix should contain an empty row in the end.
*/
public static IDictionary create(int nRowCol, boolean withEmpty) {
if(nRowCol == 1) {
if(withEmpty)
return new Dictionary(new double[] {1, 0});
else
return new Dictionary(new double[] {1});
}
return new IdentityDictionary(nRowCol, withEmpty);
}

@Override
public double[] getValues() {
if(nRowCol < 3) {
Expand Down Expand Up @@ -129,7 +156,6 @@ public void aggregateCols(double[] c, Builtin fn, IColIndex colIndexes) {
}
}


@Override
public IDictionary binOpRight(BinaryOperator op, double[] v, IColIndex colIndexes) {
boolean same = false;
Expand Down Expand Up @@ -233,7 +259,6 @@ public void colProduct(double[] res, int[] counts, IColIndex colIndexes) {
}
}


@Override
public double sum(int[] counts, int ncol) {
// number of rows, change this.
Expand All @@ -255,7 +280,7 @@ public IDictionary sliceOutColumnRange(int idxStart, int idxEnd, int previousNum
if(idxStart == 0 && idxEnd == nRowCol)
return new IdentityDictionary(nRowCol, withEmpty);
else
return new IdentityDictionarySlice(nRowCol, withEmpty, idxStart, idxEnd);
return IdentityDictionarySlice.create(nRowCol, withEmpty, idxStart, idxEnd);
}

@Override
Expand All @@ -265,6 +290,8 @@ public long getNumberNonZeros(int[] counts, int nCol) {

@Override
public int[] countNNZZeroColumns(int[] counts) {
if(withEmpty)
return Arrays.copyOf(counts, nRowCol); // one less.
return counts; // interesting ... but true.
}

Expand Down Expand Up @@ -322,28 +349,25 @@ private void addToEntryVectorizedNorm(double[] v, int f1, int f2, int f3, int f4
v[t8 * nCol + f8] += 1;
}

@Override
public MatrixBlockDictionary getMBDict(){
@Override
public MatrixBlockDictionary getMBDict() {
return getMBDict(nRowCol);
}

@Override
public MatrixBlockDictionary createMBDict(int nCol) {

if(withEmpty) {
final SparseBlock sb = SparseBlockFactory.createIdentityMatrixWithEmptyRow(nRowCol);
final MatrixBlock identity = new MatrixBlock(nRowCol + 1, nRowCol, nRowCol, sb);
return new MatrixBlockDictionary(identity);
}
else {

final SparseBlock sb = SparseBlockFactory.createIdentityMatrix(nRowCol);
final MatrixBlock identity = new MatrixBlock(nRowCol, nRowCol, nRowCol, sb);
return new MatrixBlockDictionary(identity);
}
}


@Override
public void write(DataOutput out) throws IOException {
out.writeByte(DictionaryFactory.Type.IDENTITY.ordinal());
Expand Down Expand Up @@ -403,7 +427,7 @@ public void multiplyScalar(double v, double[] ret, int off, int dictIdx, IColInd
@Override
public void MMDictDense(double[] left, IColIndex rowsLeft, IColIndex colsRight, MatrixBlock result) {
// similar to fused transpose left into right locations.

final int leftSide = rowsLeft.size();
final int colsOut = result.getNumColumns();
final int commonDim = Math.min(left.length / leftSide, nRowCol);
Expand Down Expand Up @@ -431,7 +455,6 @@ public void MMDictScalingDense(double[] left, IColIndex rowsLeft, IColIndex cols
}
}


@Override
public boolean equals(IDictionary o) {
if(o instanceof IdentityDictionary && //
Expand Down
66 changes: 45 additions & 21 deletions ...n/java/org/apache/sysds/runtime/compress/colgroup/dictionary/IdentityDictionarySlice.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,12 +49,34 @@ public class IdentityDictionarySlice extends AIdentityDictionary {
*/
public IdentityDictionarySlice(int nRowCol, boolean withEmpty, int l, int u) {
super(nRowCol, withEmpty);
if(u > nRowCol || l < 0 || l >= u)
throw new DMLRuntimeException("Invalid slice Identity: " + nRowCol + " range: " + l + "--" + u);
this.l = l;
this.u = u;
}

/**
* Create a Identity matrix dictionary slice (if other groups are not more applicable). It behaves as if allocated a
* Sparse Matrix block but exploits that the structure is known to have certain properties.
*
* @param nRowCol the number of rows and columns in this identity matrix.
* @param withEmpty If the matrix should contain an empty row in the end.
* @param l the index lower to start at
* @param u the index upper to end at (not inclusive)
*/
public static IDictionary create(int nRowCol, boolean withEmpty, int l, int u) {
if(u > nRowCol || l < 0 || l >= u)
throw new DMLRuntimeException("Invalid slice Identity: " + nRowCol + " range: " + l + "--" + u);
if(nRowCol == 1) {
if(withEmpty)
return new Dictionary(new double[] {1, 0});
else
return new Dictionary(new double[] {1});
}
else if(l == 0 && u == nRowCol)
return IdentityDictionary.create(nRowCol, withEmpty);
else
return new IdentityDictionarySlice(nRowCol, withEmpty, l, u);
}

@Override
public double[] getValues() {
LOG.warn("Should not call getValues on Identity Dictionary");
Expand Down Expand Up @@ -96,9 +118,15 @@ public static long getInMemorySize(int numberColumns) {

@Override
public double[] aggregateRows(Builtin fn, int nCol) {
double[] ret = new double[nRowCol];
Arrays.fill(ret, l, u, fn.execute(1, 0));
return ret;
double[] ret = new double[nRowCol + (withEmpty ? 1 : 0)];
if(l + 1 == u) {
ret[l] = 1;
return ret;
}
else {
Arrays.fill(ret, l, u, fn.execute(1, 0));
return ret;
}
}

@Override
Expand Down Expand Up @@ -139,19 +167,16 @@ public double[] sumAllRowsToDoubleWithDefault(double[] defaultTuple) {

@Override
public double[] sumAllRowsToDoubleWithReference(double[] reference) {
double[] ret = new double[getNumberOfValues(reference.length)];
final double[] ret = new double[getNumberOfValues(reference.length)];
double refSum = 0;
for(int i = 0; i < reference.length; i++)
refSum += reference[i];
for(int i = 0; i < ret.length; i++) {
if(i < l || i > u)
ret[i] = refSum;
else
ret[i] = 1 + refSum;
}

if(withEmpty)
ret[ret.length - 1] += -1;
for(int i = 0; i < l; i++)
ret[i] = refSum;
for(int i = l; i < u; i++)
ret[i] = 1 + refSum;
for(int i = u; i < ret.length; i++)
ret[i] = refSum;
return ret;
}

Expand Down Expand Up @@ -180,9 +205,8 @@ public void colSum(double[] c, int[] counts, IColIndex colIndexes) {

@Override
public double sum(int[] counts, int ncol) {
int end = withEmpty && u == ncol ? u - 1 : u;
double s = 0.0;
for(int i = l; i < end; i++)
for(int i = l; i < u; i++)
s += counts[i];
return s;
}
Expand Down Expand Up @@ -241,22 +265,22 @@ public void addToEntry(final double[] v, final int fr, final int to, final int n
public boolean equals(IDictionary o) {
if(o instanceof IdentityDictionarySlice) {
IdentityDictionarySlice os = ((IdentityDictionarySlice) o);
return os.nRowCol == nRowCol && os.l == l && os.u == u;
return os.nRowCol == nRowCol && os.l == l && os.u == u && withEmpty == os.withEmpty;
}
else if(o instanceof IdentityDictionary)
return false;
else
return getMBDict().equals(o);
}

@Override
public MatrixBlockDictionary getMBDict(){
@Override
public MatrixBlockDictionary getMBDict() {
return getMBDict(nRowCol);
}

@Override
public MatrixBlockDictionary createMBDict(int nCol) {
MatrixBlock identity = new MatrixBlock(nRowCol + (withEmpty ? 1 : 0), u - l, true);
MatrixBlock identity = new MatrixBlock(nRowCol + (withEmpty ? 1 : 0), u - l, true);
for(int i = l; i < u; i++)
identity.set(i, i - l, 1.0);
return new MatrixBlockDictionary(identity);
Expand Down
2 changes: 1 addition & 1 deletion ...ain/java/org/apache/sysds/runtime/compress/colgroup/dictionary/MatrixBlockDictionary.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1580,7 +1580,7 @@ public int[] countNNZZeroColumns(int[] counts) {
final int aix[] = sb.indexes(i);
for(int j = apos; j < alen; j++) {

ret[aix[i]] += counts[i];
ret[aix[j]] += counts[i];
}
}
}
Expand Down
70 changes: 22 additions & 48 deletions src/main/java/org/apache/sysds/runtime/compress/colgroup/dictionary/QDictionary.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@
import java.io.DataOutput;
import java.io.IOException;

import org.apache.commons.lang3.NotImplementedException;
import org.apache.sysds.runtime.functionobjects.Builtin;
import org.apache.sysds.runtime.matrix.data.MatrixBlock;
import org.apache.sysds.utils.MemoryEstimates;
Expand All @@ -47,11 +46,24 @@ protected QDictionary(byte[] values, double scale, int nCol) {
_nCol = nCol;
}

public static QDictionary create(byte[] values, double scale, int nCol, boolean check) {
if(scale == 0)
return null;
if(check) {
boolean containsOnlyZero = true;
for(int i = 0; i < values.length && containsOnlyZero; i++) {
if(values[i] != 0)
containsOnlyZero = false;
}
if(containsOnlyZero)
return null;
}
return new QDictionary(values, scale, nCol);
}

@Override
public double[] getValues() {
if(_values == null) {
return new double[0];
}

double[] res = new double[_values.length];
for(int i = 0; i < _values.length; i++) {
res[i] = getValue(i);
Expand All @@ -69,18 +81,6 @@ public final double getValue(int r, int c, int nCol) {
return _values[r * nCol + c] * _scale;
}

public byte getValueByte(int i) {
return _values[i];
}

public byte[] getValuesByte() {
return _values;
}

public double getScale() {
return _scale;
}

@Override
public long getInMemorySize() {
// object + values array + double
Expand All @@ -102,26 +102,6 @@ public double aggregate(double init, Builtin fn) {
return ret;
}

@Override
public double aggregateWithReference(double init, Builtin fn, double[] reference, boolean def) {
throw new NotImplementedException();
}

@Override
public double[] aggregateRows(Builtin fn, final int nCol) {
if(nCol == 1)
return getValues();
final int nRows = _values.length / nCol;
double[] res = new double[nRows];
for(int i = 0; i < nRows; i++) {
final int off = i * nCol;
res[i] = _values[off];
for(int j = off + 1; j < off + nCol; j++)
res[i] = fn.execute(res[i], _values[j] * _scale);
}
return res;
}

private int size() {
return _values.length;
}
Expand Down Expand Up @@ -159,7 +139,7 @@ public long getExactSizeOnDisk() {

@Override
public int getNumberOfValues(int nCol) {
return (_values == null) ? 0 : _values.length / nCol;
return _values.length / nCol;
}

@Override
Expand All @@ -185,10 +165,7 @@ public double[] sumAllRowsToDoubleSq(int nrColumns) {
}

private double sumRow(int k, int nrColumns) {
if(_values == null)
return 0;
int valOff = k * nrColumns;

int res = 0;
for(int i = 0; i < nrColumns; i++) {
res += _values[valOff + i];
Expand All @@ -197,8 +174,6 @@ private double sumRow(int k, int nrColumns) {
}

private double sumRowSq(int k, int nrColumns) {
if(_values == null)
return 0;
int valOff = k * nrColumns;
double res = 0.0;
for(int i = 0; i < nrColumns; i++)
Expand All @@ -215,11 +190,6 @@ public String getString(int colIndexes) {
return sb.toString();
}

public Dictionary makeDoubleDictionary() {
double[] doubleValues = getValues();
return Dictionary.create(doubleValues);
}

public IDictionary sliceOutColumnRange(int idxStart, int idxEnd, int previousNumberOfColumns) {
int numberTuples = getNumberOfValues(previousNumberOfColumns);
int tupleLengthAfter = idxEnd - idxStart;
Expand Down Expand Up @@ -274,7 +244,11 @@ public DictType getDictType() {

@Override
public double getSparsity() {
throw new NotImplementedException();
int nnz = 0;
for(int i = 0; i < _values.length; i++) {
nnz += _values[i] == 0 ? 0 : 1;
}
return (double) nnz / _values.length;
}

@Override
Expand Down
Loading

0 comments on commit 174ed30

Please sign in to comment.