CALCITE-6465: Rework code generator to use Flink code splitter

bom/build.gradle.kts

@@ -111,6 +111,7 @@ dependencies {
         apiv("org.apache.commons:commons-pool2")
         apiv("org.apache.commons:commons-collections4")
         apiv("org.apache.commons:commons-text")
+        apiv("org.apache.flink:flink-table-code-splitter")
         apiv("org.apache.geode:geode-core")
         apiv("org.apache.hadoop:hadoop-client", "hadoop")
         apiv("org.apache.hadoop:hadoop-common", "hadoop")

core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableInterpretable.java

@@ -111,7 +111,10 @@ public static Bindable toBindable(Map<String, Object> parameters,
             parameters);
 
     final ClassDeclaration expr = relImplementor.implementRoot(rel, prefer);
-    String s = Expressions.toString(expr.memberDeclarations, "\n", false);
+    String s =
+        Expressions.toString(expr.memberDeclarations,
+            "\n", false,
+            CalciteSystemProperty.ENABLE_METHOD_SPLITTING.value());
 
     if (CalciteSystemProperty.DEBUG.value()) {
       Util.debugCode(System.out, s);

core/src/main/java/org/apache/calcite/config/CalciteSystemProperty.java

@@ -420,6 +420,13 @@ public final class CalciteSystemProperty<T> {
   public static final CalciteSystemProperty<Integer> FUNCTION_LEVEL_CACHE_MAX_SIZE =
       intProperty("calcite.function.cache.maxSize", 0, v -> v >= 0);
 
+  /** Whether to enable automatic method splitting when generating Java code.
+   *
+   * <p>Some queries can generate methods exceeding the JVM limit of 4000 characters per method.
+   * Enable this feature to automatically detect and split methods larger than the limit. */
+  public static final CalciteSystemProperty<Boolean> ENABLE_METHOD_SPLITTING =
+      booleanProperty("calcite.linq.enable_method_splitting", false);
+
   private static CalciteSystemProperty<Boolean> booleanProperty(String key,
       boolean defaultValue) {
     // Note that "" -> true (convenient for command-lines flags like '-Dflag')

core/src/main/java/org/apache/calcite/interpreter/JaninoRexCompiler.java

@@ -186,7 +186,9 @@ static Scalar.Producer baz(ParameterExpression context_,
     final ClassDeclaration classDeclaration =
         Expressions.classDecl(Modifier.PUBLIC, "Buzz", null,
             ImmutableList.of(Scalar.Producer.class), declarations);
-    String s = Expressions.toString(declarations, "\n", false);
+    String s =
+        Expressions.toString(declarations, "\n", false,
+            CalciteSystemProperty.ENABLE_METHOD_SPLITTING.value());
     if (CalciteSystemProperty.DEBUG.value()) {
       Util.debugCode(System.out, s);
     }

core/src/main/java/org/apache/calcite/rex/RexExecutorImpl.java

@@ -100,7 +100,8 @@ private static String compile(RexBuilder rexBuilder, List<RexNode> constExps,
         Expressions.methodDecl(Modifier.PUBLIC, Object[].class,
             BuiltInMethod.FUNCTION1_APPLY.method.getName(),
             ImmutableList.of(root0_), blockBuilder.toBlock());
-    String code = Expressions.toString(methodDecl);
+    String code =
+        Expressions.toString(methodDecl, CalciteSystemProperty.ENABLE_METHOD_SPLITTING.value());
     if (CalciteSystemProperty.DEBUG.value()) {
       Util.debugCode(System.out, code);
     }

core/src/test/java/org/apache/calcite/adapter/enumerable/EnumUtilsTest.java

@@ -47,9 +47,9 @@ public final class EnumUtilsTest {
         EnumUtils.convert(date, int.class);
     final Expression dateToInteger =
         EnumUtils.convert(date, Integer.class);
-    assertThat(Expressions.toString(dateToInt),
+    assertThat(Expressions.toString(dateToInt, true),
         is("org.apache.calcite.runtime.SqlFunctions.toInt(x)"));
-    assertThat(Expressions.toString(dateToInteger),
+    assertThat(Expressions.toString(dateToInteger, true),
         is("org.apache.calcite.runtime.SqlFunctions.toIntOptional(x)"));
 
     // java.sql.Time x;
@@ -59,9 +59,9 @@ public final class EnumUtilsTest {
         EnumUtils.convert(time, int.class);
     final Expression timeToInteger =
         EnumUtils.convert(time, Integer.class);
-    assertThat(Expressions.toString(timeToInt),
+    assertThat(Expressions.toString(timeToInt, true),
         is("org.apache.calcite.runtime.SqlFunctions.toInt(x)"));
-    assertThat(Expressions.toString(timeToInteger),
+    assertThat(Expressions.toString(timeToInteger, true),
         is("org.apache.calcite.runtime.SqlFunctions.toIntOptional(x)"));
 
     // java.sql.TimeStamp x;
@@ -71,9 +71,9 @@ public final class EnumUtilsTest {
         EnumUtils.convert(timestamp, long.class);
     final Expression timeStampToLong =
         EnumUtils.convert(timestamp, Long.class);
-    assertThat(Expressions.toString(timeStampToLongPrimitive),
+    assertThat(Expressions.toString(timeStampToLongPrimitive, true),
         is("org.apache.calcite.runtime.SqlFunctions.toLong(x)"));
-    assertThat(Expressions.toString(timeStampToLong),
+    assertThat(Expressions.toString(timeStampToLong, true),
         is("org.apache.calcite.runtime.SqlFunctions.toLongOptional(x)"));
   }
 
@@ -86,77 +86,77 @@ public final class EnumUtilsTest {
         Expressions.convert_(intVariable, byte.class);
     final Expression converted0 =
         EnumUtils.convert(bytePrimitiveConverted, Byte.class);
-    assertThat(Expressions.toString(converted0),
+    assertThat(Expressions.toString(converted0, true),
         is("Byte.valueOf((byte) intV)"));
 
     // (char)(int) -> Character: Character.valueOf((char) intV)
     final Expression characterPrimitiveConverted =
         Expressions.convert_(intVariable, char.class);
     final Expression converted1 =
         EnumUtils.convert(characterPrimitiveConverted, Character.class);
-    assertThat(Expressions.toString(converted1),
+    assertThat(Expressions.toString(converted1, true),
         is("Character.valueOf((char) intV)"));
 
     // (short)(int) -> Short: Short.valueOf((short) intV)
     final Expression shortPrimitiveConverted =
         Expressions.convert_(intVariable, short.class);
     final Expression converted2 =
         EnumUtils.convert(shortPrimitiveConverted, Short.class);
-    assertThat(Expressions.toString(converted2),
+    assertThat(Expressions.toString(converted2, true),
         is("Short.valueOf((short) intV)"));
 
     // (long)(int) -> Long: Long.valueOf(intV)
     final Expression longPrimitiveConverted =
         Expressions.convert_(intVariable, long.class);
     final Expression converted3 =
         EnumUtils.convert(longPrimitiveConverted, Long.class);
-    assertThat(Expressions.toString(converted3),
+    assertThat(Expressions.toString(converted3, true),
         is("Long.valueOf(intV)"));
 
     // (float)(int) -> Float: Float.valueOf(intV)
     final Expression floatPrimitiveConverted =
         Expressions.convert_(intVariable, float.class);
     final Expression converted4 =
         EnumUtils.convert(floatPrimitiveConverted, Float.class);
-    assertThat(Expressions.toString(converted4),
+    assertThat(Expressions.toString(converted4, true),
         is("Float.valueOf(intV)"));
 
     // (double)(int) -> Double: Double.valueOf(intV)
     final Expression doublePrimitiveConverted =
         Expressions.convert_(intVariable, double.class);
     final Expression converted5 =
         EnumUtils.convert(doublePrimitiveConverted, Double.class);
-    assertThat(Expressions.toString(converted5),
+    assertThat(Expressions.toString(converted5, true),
         is("Double.valueOf(intV)"));
 
     final Expression byteConverted =
         EnumUtils.convert(intVariable, Byte.class);
-    assertThat(Expressions.toString(byteConverted),
+    assertThat(Expressions.toString(byteConverted, true),
         is("Byte.valueOf((byte) intV)"));
 
     final Expression shortConverted =
         EnumUtils.convert(intVariable, Short.class);
-    assertThat(Expressions.toString(shortConverted),
+    assertThat(Expressions.toString(shortConverted, true),
         is("Short.valueOf((short) intV)"));
 
     final Expression integerConverted =
         EnumUtils.convert(intVariable, Integer.class);
-    assertThat(Expressions.toString(integerConverted),
+    assertThat(Expressions.toString(integerConverted, true),
         is("Integer.valueOf(intV)"));
 
     final Expression longConverted =
         EnumUtils.convert(intVariable, Long.class);
-    assertThat(Expressions.toString(longConverted),
+    assertThat(Expressions.toString(longConverted, true),
         is("Long.valueOf((long) intV)"));
 
     final Expression floatConverted =
         EnumUtils.convert(intVariable, Float.class);
-    assertThat(Expressions.toString(floatConverted),
+    assertThat(Expressions.toString(floatConverted, true),
         is("Float.valueOf((float) intV)"));
 
     final Expression doubleConverted =
         EnumUtils.convert(intVariable, Double.class);
-    assertThat(Expressions.toString(doubleConverted),
+    assertThat(Expressions.toString(doubleConverted, true),
         is("Double.valueOf((double) intV)"));
   }
 
@@ -167,8 +167,8 @@ public final class EnumUtilsTest {
     final ConstantExpression nullLiteral2 = Expressions.constant(null, Object.class);
     final Expression e1 = EnumUtils.convert(nullLiteral1, String.class);
     final Expression e2 = EnumUtils.convert(nullLiteral2, String.class);
-    assertThat(Expressions.toString(e1), is("(String) null"));
-    assertThat(Expressions.toString(e2), is("(String) (Object) null"));
+    assertThat(Expressions.toString(e1, true), is("(String) null"));
+    assertThat(Expressions.toString(e2, true), is("(String) (Object) null"));
   }
 
   @Test void testMethodCallExpression() {
@@ -178,7 +178,7 @@ public final class EnumUtilsTest {
     final MethodCallExpression arrayMethodCall =
         EnumUtils.call(null, SqlFunctions.class,
             BuiltInMethod.ARRAY.getMethodName(), Arrays.asList(arg0, arg1));
-    assertThat(Expressions.toString(arrayMethodCall),
+    assertThat(Expressions.toString(arrayMethodCall, true),
         is("org.apache.calcite.runtime.SqlFunctions.array(1, \"x\")"));
 
     // test for Object.class argument type
@@ -187,7 +187,7 @@ public final class EnumUtilsTest {
         EnumUtils.call(null, XmlFunctions.class,
             BuiltInMethod.EXTRACT_VALUE.getMethodName(),
             Arrays.asList(arg1, nullLiteral));
-    assertThat(Expressions.toString(xmlExtractMethodCall),
+    assertThat(Expressions.toString(xmlExtractMethodCall, true),
         is("org.apache.calcite.runtime.XmlFunctions.extractValue(\"x\", (String) null)"));
 
     // test "mod(decimal, long)" match to "mod(decimal, decimal)"
@@ -196,7 +196,7 @@ public final class EnumUtilsTest {
     final MethodCallExpression modMethodCall =
         EnumUtils.call(null, SqlFunctions.class, "mod",
             Arrays.asList(arg2, arg3));
-    assertThat(Expressions.toString(modMethodCall),
+    assertThat(Expressions.toString(modMethodCall, true),
         is("org.apache.calcite.runtime.SqlFunctions.mod("
             + "java.math.BigDecimal.valueOf(125L, 1), "
             + "new java.math.BigDecimal(\n  3L))"));
@@ -207,7 +207,7 @@ public final class EnumUtilsTest {
     final MethodCallExpression geoMethodCall =
         EnumUtils.call(null, SpatialTypeFunctions.class, "ST_MakePoint",
             Arrays.asList(arg4, arg5));
-    assertThat(Expressions.toString(geoMethodCall),
+    assertThat(Expressions.toString(geoMethodCall, true),
         is("org.apache.calcite.runtime.SpatialTypeFunctions.ST_MakePoint("
             + "new java.math.BigDecimal(\n  1), "
             + "new java.math.BigDecimal(\n  2))"));

core/src/test/java/org/apache/calcite/adapter/enumerable/PhysTypeTest.java

@@ -95,6 +95,6 @@ public final class PhysTypeTest {
         + "  }\n"
         + "}\n"
         + ")";
-    assertEquals(Expressions.toString(e), expected);
+    assertEquals(Expressions.toString(e, true), expected);
   }
 }

core/src/test/java/org/apache/calcite/adapter/enumerable/TypeFinderTest.java

@@ -97,7 +97,7 @@ private void assertJavaCodeContains(String expected, Node node) {
   }
 
   private void assertJavaCodeContains(String expected, List<Node> nodes) {
-    final String javaCode = Expressions.toString(nodes, "\n", false);
+    final String javaCode = Expressions.toString(nodes, "\n", false, true);
     assertThat(javaCode, containsString(expected));
   }
 

gradle.properties

@@ -102,6 +102,7 @@ dropwizard-metrics.version=4.0.5
 # do not upgrade this, new versions are Category X license.
 elasticsearch.version=7.10.2
 embedded-redis.version=0.6
+flink-table-code-splitter.version=1.20.0
 jts-core.version=1.19.0
 jts-io-common.version=1.19.0
 proj4j.version=1.2.2

innodb/src/main/java/org/apache/calcite/adapter/innodb/InnodbToEnumerableConverter.java

@@ -150,7 +150,9 @@ static List<String> innodbFieldNames(final RelDataType rowType) {
                 InnodbMethod.INNODB_QUERYABLE_QUERY.method, fields,
                 selectFields, cond, ascOrder));
     if (CalciteSystemProperty.DEBUG.value()) {
-      System.out.println("Innodb: " + Expressions.toString(enumerable));
+      System.out.println(
+          "Innodb: " + Expressions.toString(enumerable,
+          CalciteSystemProperty.ENABLE_METHOD_SPLITTING.value()));
     }
     list.add(Expressions.return_(null, enumerable));
     return implementor.result(physType, list.toBlock());

linq4j/build.gradle.kts

@@ -20,4 +20,5 @@ dependencies {
 
     implementation("com.google.guava:guava")
     implementation("org.apache.calcite.avatica:avatica-core")
+    implementation("org.apache.flink:flink-table-code-splitter")
 }

linq4j/src/main/java/org/apache/calcite/linq4j/tree/AbstractNode.java

@@ -51,7 +51,7 @@ public Type getType() {
   }
 
   @Override public String toString() {
-    ExpressionWriter writer = new ExpressionWriter(true);
+    ExpressionWriter writer = new ExpressionWriter(true, true);
     accept(writer, 0, 0);
     return writer.toString();
   }

linq4j/src/main/java/org/apache/calcite/linq4j/tree/ExpressionWriter.java

@@ -32,15 +32,24 @@ class ExpressionWriter {
 
   private final Spacer spacer = new Spacer(0);
   private final StringBuilder buf = new StringBuilder();
-  private boolean indentPending;
   private final boolean generics;
+  private final boolean methodSplitting;
+  private boolean indentPending;
 
   ExpressionWriter() {
-    this(true);
+    this(true, true);
   }
 
-  ExpressionWriter(boolean generics) {
+  ExpressionWriter(boolean generics, boolean methodSplitting) {
     this.generics = generics;
+    this.methodSplitting = methodSplitting;
+  }
+
+  public ExpressionWriter duplicateState() {
+    final ExpressionWriter writer = new ExpressionWriter(this.generics, this.methodSplitting);
+    writer.indentPending = this.indentPending;
+    writer.spacer.add(this.spacer.get());
+    return writer;
   }
 
   public void write(Node expression) {
@@ -73,6 +82,10 @@ public boolean requireParentheses(Expression expression, int lprec,
     return true;
   }
 
+  public boolean usesMethodSplitting() {
+    return methodSplitting;
+  }
+
   /**
    * Increases the indentation level.
    */

linq4j/src/main/java/org/apache/calcite/linq4j/tree/Expressions.java

@@ -58,8 +58,8 @@ private Expressions() {}
    * extra type information in generics.
    */
   public static String toString(List<? extends Node> expressions, String sep,
-      boolean generics) {
-    final ExpressionWriter writer = new ExpressionWriter(generics);
+      boolean generics, boolean methodSplit) {
+    final ExpressionWriter writer = new ExpressionWriter(generics, methodSplit);
     for (Node expression : expressions) {
       writer.write(expression);
       writer.append(sep);
@@ -70,8 +70,8 @@ public static String toString(List<? extends Node> expressions, String sep,
   /**
    * Converts an expression to Java source code.
    */
-  public static String toString(Node expression) {
-    return toString(Collections.singletonList(expression), "", true);
+  public static String toString(Node expression, boolean methodSplit) {
+    return toString(Collections.singletonList(expression), "", true, methodSplit);
   }
 
   /**

linq4j/src/main/java/org/apache/calcite/linq4j/tree/MethodDeclaration.java

@@ -16,6 +16,9 @@
  */
 package org.apache.calcite.linq4j.tree;
 
+import org.apache.commons.lang3.StringUtils;
+import org.apache.flink.table.codesplit.JavaCodeSplitter;
+
 import org.checkerframework.checker.nullness.qual.Nullable;
 
 import java.lang.reflect.Modifier;
@@ -57,20 +60,39 @@ public MethodDeclaration(int modifier, String name, Type resultType,
   }
 
   @Override public void accept(ExpressionWriter writer) {
-    String modifiers = Modifier.toString(modifier);
-    writer.append(modifiers);
+    final ExpressionWriter writerForUnsplitMethod = writer.usesMethodSplitting()
+        ? writer.duplicateState() : writer;
+
+    final String modifiers = Modifier.toString(modifier);
+    writerForUnsplitMethod.append(modifiers);
     if (!modifiers.isEmpty()) {
-      writer.append(' ');
+      writerForUnsplitMethod.append(' ');
     }
     //noinspection unchecked
-    writer
+    writerForUnsplitMethod
         .append(resultType)
         .append(' ')
         .append(name)
         .list("(", ", ", ")",
             () -> (Iterator) parameters.stream().map(ParameterExpression::declString).iterator())
         .append(' ')
         .append(body);
+
+    if (writer.usesMethodSplitting()) {
+      //  Specifies a threshold where generated code will be split into sub-function calls.
+      //  Java has a maximum method length of 64 KB. This setting allows for finer granularity if
+      //  necessary.
+      //  Default value is 4000 instead of 64KB as by default JIT refuses to work on methods with
+      //  more than 8K byte code.
+      final int defaultMaxGeneratedCodeLength = 4000;
+      final int defaultMaxMembersGeneratedCode = 10000;
+
+      writer.append(
+          StringUtils.stripStart(
+              JavaCodeSplitter.split(writerForUnsplitMethod.toString(),
+                  defaultMaxGeneratedCodeLength, defaultMaxMembersGeneratedCode),
+              " "));
+    }
     writer.newlineAndIndent();
   }