Skip to content

Commit

Permalink
Allow reading ViewVarCharVector and ViewVarBinaryVector from Arrow files
Browse files Browse the repository at this point in the history
  • Loading branch information
Kopilov committed Sep 2, 2024
1 parent fdec9a1 commit e46883b
Show file tree
Hide file tree
Showing 2 changed files with 28 additions and 3 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,8 @@ import org.apache.arrow.vector.UInt8Vector
import org.apache.arrow.vector.VarBinaryVector
import org.apache.arrow.vector.VarCharVector
import org.apache.arrow.vector.VectorSchemaRoot
import org.apache.arrow.vector.ViewVarBinaryVector
import org.apache.arrow.vector.ViewVarCharVector
import org.apache.arrow.vector.complex.StructVector
import org.apache.arrow.vector.ipc.ArrowFileReader
import org.apache.arrow.vector.ipc.ArrowReader
Expand Down Expand Up @@ -207,6 +209,25 @@ private fun VarCharVector.values(range: IntRange): List<String?> =
}
}

private fun LargeVarCharVector.values(range: IntRange): List<String?> =
range.map {
if (isNull(it)) {
null
} else {
String(get(it))
}
}

private fun ViewVarCharVector.values(range: IntRange): List<String?> =
range.map {
if (isNull(it)) {
null
} else {
String(get(it))
}
}


private fun VarBinaryVector.values(range: IntRange): List<ByteArray?> =
range.map {
if (isNull(it)) {
Expand All @@ -225,12 +246,12 @@ private fun LargeVarBinaryVector.values(range: IntRange): List<ByteArray?> =
}
}

private fun LargeVarCharVector.values(range: IntRange): List<String?> =
private fun ViewVarBinaryVector.values(range: IntRange): List<ByteArray?> =
range.map {
if (isNull(it)) {
null
} else {
String(get(it))
get(it)
}
}

Expand Down Expand Up @@ -266,10 +287,14 @@ private fun readField(root: VectorSchemaRoot, field: Field, nullability: Nullabi

is LargeVarCharVector -> vector.values(range).withTypeNullable(field.isNullable, nullability)

is ViewVarCharVector -> vector.values(range).withTypeNullable(field.isNullable, nullability)

is VarBinaryVector -> vector.values(range).withTypeNullable(field.isNullable, nullability)

is LargeVarBinaryVector -> vector.values(range).withTypeNullable(field.isNullable, nullability)

is ViewVarBinaryVector -> vector.values(range).withTypeNullable(field.isNullable, nullability)

is BitVector -> vector.values(range).withTypeNullable(field.isNullable, nullability)

is SmallIntVector -> vector.values(range).withTypeNullable(field.isNullable, nullability)
Expand Down
2 changes: 1 addition & 1 deletion gradle/libs.versions.toml
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ junit-platform = "1.10.2"
kotestAsserions = "5.5.4"

jsoup = "1.17.2"
arrow = "15.0.0"
arrow = "17.0.0"
docProcessor = "0.3.10"
simpleGit = "2.0.3"
dependencyVersions = "0.51.0"
Expand Down

0 comments on commit e46883b

Please sign in to comment.