wip

Author: g31pranjal

fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/ddl/MaterializedViewIndexGenerator.java

@@ -200,20 +200,6 @@ public GenerationResult generate(@Nonnull RecordLayerSchemaTemplate.Builder sche
         final var indexBuilder = RecordLayerIndex.newBuilder()
                 .setName(indexName)
                 .setUnique(isUnique);
-        // If the plan unnests an array field *of a struct type*, the index is defined on a synthetic
-        // (unnested) type: each such array becomes a nested constituent. Scalar arrays (e.g.
-        // STRING ARRAY) are not required to be represented in the synthetic type since they can directly
-        // fan-out key expression within their owning constituent.
-        if (!unnestedConstituents.isEmpty()) {
-            final String syntheticTableName = "__unnested_" + recordTypeName + "_" + indexName;
-            indexBuilder
-                    .setTableName(syntheticTableName)
-                    .setTableStorageName(syntheticTableName);
-            syntheticTableBuilder = Optional.of(buildUnnestedTypeMetadata(
-                    schemaTemplateBuilder, syntheticTableName, recordTypeName, findParentConstituentAlias()));
-        } else {
-            indexBuilder.setTableType(tableType);
-        }
 
         // add predicates
         final var predicate = getTopLevelPredicate(Lists.reverse(expressionRefs));
@@ -245,8 +231,17 @@ public GenerationResult generate(@Nonnull RecordLayerSchemaTemplate.Builder sche
             if (orderByValues.isEmpty() && !generateKeyValueExpressionWithEmptyKey) {
                 splitPoint = -1;
             }
+            // The choice of representation can only be made once the key columns and their order are
+            // known, since it turns on whether one array element's columns are contiguous in the key.
+            final boolean useSyntheticType = requiresSyntheticType(reordered);
             KeyExpression keyExpression;
-            if (!unnestedConstituents.isEmpty()) {
+            if (useSyntheticType) {
+                final String syntheticTableName = "__unnested_" + recordTypeName + "_" + indexName;
+                indexBuilder
+                        .setTableName(syntheticTableName)
+                        .setTableStorageName(syntheticTableName);
+                syntheticTableBuilder = Optional.of(buildUnnestedTypeMetadata(
+                        schemaTemplateBuilder, syntheticTableName, recordTypeName, findParentConstituentAlias()));
                 // For unnested synthetic types the key expression uses constituent-alias paths
                 // (e.g. field("SQ").nest(field("a"))) rather than stored-table field paths.
                 // Build it directly from the dereferenced FieldValues.
@@ -255,6 +250,7 @@ public GenerationResult generate(@Nonnull RecordLayerSchemaTemplate.Builder sche
                 keyExpression = splitPoint != -1 && splitPoint < fieldValues.size() ?
                                 keyWithValue(fullExpr, splitPoint) : fullExpr;
             } else {
+                indexBuilder.setTableType(tableType);
                 final var expression = generate(reordered, orderingFunctions);
                 final var unwrappedKeyExpression = splitPoint != -1 && splitPoint < fieldValues.size() ?
                                                    keyWithValue(expression, splitPoint) : expression;
@@ -263,8 +259,9 @@ public GenerationResult generate(@Nonnull RecordLayerSchemaTemplate.Builder sche
             }
             indexBuilder.setKeyExpression(keyExpression);
         } else {
-            Assert.thatUnchecked(unnestedConstituents.isEmpty(), ErrorCode.UNSUPPORTED_OPERATION,
-                    "Aggregate indexes cannot be defined on unnested synthetic types");
+            // Aggregate indexes always use the stored table: their grouping columns are emitted with a
+            // fan-out key expression, as before synthetic types existed.
+            indexBuilder.setTableType(tableType);
             final var aggregateValue = (AggregateValue) aggregateValues.get(0);
             int aggregateOrderIndex = -1;
             if (!orderByValues.isEmpty()) {
@@ -783,7 +780,7 @@ private void collectQuantifiersInternal(@Nonnull RelationalExpression relational
                     final var accessors = field.getFieldPath().getFieldAccessors();
                     final String arrayFieldStorageName =
                             accessors.get(accessors.size() - 1).getField().getFieldStorageName();
-                    final String owningAlias = Iterables.getOnlyElement(field.getCorrelatedTo()).toString();
+                    final String owningAlias = resolveOwningAlias(field);
                     final var arrayType = (Type.Array) field.getResultType();
                     if (arrayType.getElementType() instanceof Type.Record) {
                         unnestedConstituents.put(explodeCounter.get(),
@@ -802,6 +799,36 @@ private void collectQuantifiersInternal(@Nonnull RelationalExpression relational
         }
     }
 
+    /**
+     * Returns the alias of the constituent that owns the array an explode reads from.
+     *
+     * <p>The raw correlation of the collection value is not enough. A first-level explode reads its array
+     * off the table quantifier, {@code FieldValue(QOV(a), [P])} for {@code a.p}, and that quantifier is
+     * the parent constituent — so the correlation happens to be right. A chained explode reads its array
+     * off the <em>preceding subquery's</em> alias, {@code FieldValue(QOV(b), [Q])} for {@code b.q} where
+     * {@code b} is {@code (SELECT * FROM a.p)}; that subquery quantifier is not a constituent, the explode
+     * inside it is. Using the correlation directly would name a non-existent constituent.
+     *
+     * <p>So the collection value is dereferenced first: any enclosing unnestings then show up as
+     * {@link AnnotatedAccessor}s in the resulting path, and the innermost of those identifies the
+     * constituent to parent to. When there are none the array hangs directly off the stored record and the
+     * correlation is already the right answer.
+     *
+     * @param collectionValue the array a newly seen explode ranges over
+     * @return the alias of the owning constituent
+     */
+    @Nonnull
+    private String resolveOwningAlias(@Nonnull final FieldValue collectionValue) {
+        final var markers = unnestingMarkers(dereference(collectionValue));
+        for (int i = markers.size() - 1; i >= 0; i--) {
+            final NestedConstituentInfo enclosing = unnestedConstituents.get(markers.get(i));
+            if (enclosing != null) {
+                return enclosing.alias();
+            }
+        }
+        return Iterables.getOnlyElement(collectionValue.getCorrelatedTo()).toString();
+    }
+
     /**
      * One nested constituent of an unnested synthetic type, as discovered from an
      * {@link ExplodeExpression} over a struct array during {@link #collectQuantifiers}.
@@ -981,6 +1008,98 @@ private KeyExpression buildConstituentKeyExpression(
         return parts.size() == 1 ? parts.get(0) : concat(parts);
     }
 
+    /**
+     * Returns the position of the innermost unnesting boundary in a dereferenced field path, or
+     * {@code -1} if the path crosses none. The <em>last</em> {@link AnnotatedAccessor} is the innermost
+     * one: for {@code FROM T AS r, r.a AS x, x.b AS y} the value {@code y.c} dereferences to
+     * {@code [ann(A), ann(B), C]}, and it is {@code ann(B)} that says where {@code c} lives.
+     *
+     * @param accessors the dereferenced field path
+     * @return the index of the innermost {@link AnnotatedAccessor}, or {@code -1}
+     */
+    private static int innermostUnnestingIndex(@Nonnull final List<FieldValue.ResolvedAccessor> accessors) {
+        for (int i = accessors.size() - 1; i >= 0; i--) {
+            if (accessors.get(i) instanceof AnnotatedAccessor) {
+                return i;
+            }
+        }
+        return -1;
+    }
+
+    /**
+     * Returns the markers of every unnesting a value is read through, outermost first. For
+     * {@code FROM A AS a, (SELECT * FROM a.p) AS b, (SELECT * FROM b.q) AS c} the value {@code c.y}
+     * dereferences to {@code [ann(P), ann(Q), Y]} and so traverses both unnestings, while {@code b.x}
+     * dereferences to {@code [ann(P), X]} and traverses only the outer one.
+     *
+     * @param value the dereferenced value
+     * @return the markers of the unnestings traversed, outermost first, empty if there are none
+     */
+    @Nonnull
+    private static List<Integer> unnestingMarkers(@Nonnull final Value value) {
+        if (!(value instanceof FieldValue fieldValue)) {
+            return List.of();
+        }
+        final var markers = ImmutableList.<Integer>builder();
+        for (final FieldValue.ResolvedAccessor accessor : fieldValue.getFieldPath().getFieldAccessors()) {
+            if (accessor instanceof AnnotatedAccessor annotatedAccessor) {
+                markers.add(annotatedAccessor.getMarker());
+            }
+        }
+        return markers.build();
+    }
+
+    /**
+     * Returns whether this index has to be defined on an unnested synthetic type, rather than on the
+     * stored table with a fan-out key expression.
+     *
+     * <p>A fan-out expresses an unnesting perfectly well as long as every column read through one
+     * unnesting sits in a contiguous run of the index key: those columns are then emitted under a single
+     * navigation into the array, {@code field("A").nest(field("values", FanOut).nest(concat(...)))},
+     * which yields one index entry per element. That covers a single column from an element, and
+     * several adjacent columns of the same element. It also covers two independent unnestings, each
+     * with its own fan-out — as in {@code FROM T1, (SELECT col3 FROM T1.A) X, (SELECT col4 FROM T1.A) Y}
+     * — where the resulting cross-product is the intended meaning of the cross join and no correlation
+     * between {@code X} and {@code Y} is wanted.
+     *
+     * <p>What a fan-out cannot express is two columns reached through the <em>same</em> unnesting,
+     * separated by a column that is not, as in {@code ORDER BY X.col2, T1.col5, X.col3}. There is no
+     * single navigation covering both, and emitting two would fan out twice and cross-multiply, so this
+     * was previously rejected outright. A synthetic type handles it: a constituent is navigated with
+     * {@link KeyExpression.FanType#None} and holds one element per synthetic record, so it can be
+     * referenced at as many key positions as needed.
+     *
+     * <p>Every unnesting a column is read through counts, not just the innermost one. Under chained
+     * unnesting, {@code SELECT b.x, a.k, c.y} reads {@code b.x} and {@code c.y} through different
+     * innermost unnestings, but both traverse the outer {@code b}; with {@code a.k} between them that
+     * outer navigation would have to be emitted twice, so a synthetic type is required even though
+     * neither innermost unnesting is itself split.
+     *
+     * <p>Only struct arrays are considered: a scalar array cannot be a constituent at all, so repeated
+     * non-adjacent references to one keep failing as before.
+     *
+     * @param keyValues the index key columns, in key order
+     * @return whether a synthetic type is required
+     */
+    private boolean requiresSyntheticType(@Nonnull final List<Value> keyValues) {
+        final Map<Integer, Integer> firstPositions = new LinkedHashMap<>();
+        final Map<Integer, Integer> lastPositions = new LinkedHashMap<>();
+        final Map<Integer, Integer> counts = new LinkedHashMap<>();
+        for (int i = 0; i < keyValues.size(); i++) {
+            for (final Integer marker : unnestingMarkers(keyValues.get(i))) {
+                // Skip scalar arrays, which cannot be constituents.
+                if (!unnestedConstituents.containsKey(marker)) {
+                    continue;
+                }
+                firstPositions.putIfAbsent(marker, i);
+                lastPositions.put(marker, i);
+                counts.merge(marker, 1, Integer::sum);
+            }
+        }
+        return counts.entrySet().stream().anyMatch(entry ->
+                lastPositions.get(entry.getKey()) - firstPositions.get(entry.getKey()) + 1 != entry.getValue());
+    }
+
     /**
      * Translates a single dereferenced {@link FieldValue} into a constituent-alias key expression.
      * {@link AnnotatedAccessor}s in the path mark unnesting boundaries.
@@ -1002,29 +1121,29 @@ private KeyExpression toKeyExpressionOnNestedConstituent(@Nonnull Value value, @
             if (accessors.isEmpty()) {
                 return field(parentAlias, KeyExpression.FanType.None);
             }
-            for (int i = accessors.size() - 1; i >= 0; i--) {
-                if (accessors.get(i) instanceof AnnotatedAccessor annotatedAccessor) {
-                    final int marker = annotatedAccessor.getMarker();
-                    final var remaining = accessors.subList(i + 1, accessors.size());
-                    final NestedConstituentInfo info = unnestedConstituents.get(marker);
-                    if (info != null) {
-                        return remaining.isEmpty() ?
-                               field(info.alias(), KeyExpression.FanType.None) :
-                               field(info.alias(), KeyExpression.FanType.None)
-                                .nest(toKeyExpression(remaining.iterator(), KeyExpression.FanType.FanOut));
-                    }
-                    // Scalar array — not a constituent. Fan out over the array field within the
-                    // constituent that owns it. Scalar elements have no sub-fields, so nothing remains.
-                    final ScalarUnnestingInfo scalarInfo = scalarFanouts.get(marker);
-                    Assert.notNullUnchecked(scalarInfo, "unknown unnesting in index definition");
-                    Assert.thatUnchecked(remaining.isEmpty(), ErrorCode.UNSUPPORTED_OPERATION,
-                            "Unsupported index definition, cannot dereference a field of a scalar array element");
-                    return field(scalarInfo.owningAlias(), KeyExpression.FanType.None)
-                            .nest(scalarInfo.toFanOutExpression());
-                }
+            final int boundary = innermostUnnestingIndex(accessors);
+            if (boundary < 0) {
+                // No AnnotatedAccessor — field comes from the parent constituent.
+                return field(parentAlias, KeyExpression.FanType.None)
+                        .nest(toKeyExpression(accessors.iterator(), KeyExpression.FanType.FanOut));
+            }
+            final int marker = ((AnnotatedAccessor) accessors.get(boundary)).getMarker();
+            final var remaining = accessors.subList(boundary + 1, accessors.size());
+            final NestedConstituentInfo info = unnestedConstituents.get(marker);
+            if (info != null) {
+                return remaining.isEmpty() ?
+                       field(info.alias(), KeyExpression.FanType.None) :
+                       field(info.alias(), KeyExpression.FanType.None)
+                        .nest(toKeyExpression(remaining.iterator(), KeyExpression.FanType.FanOut));
             }
-            return field(parentAlias, KeyExpression.FanType.None)
-                    .nest(toKeyExpression(accessors.iterator(), KeyExpression.FanType.FanOut));
+            // Scalar array — not a constituent. Fan out over the array field within the
+            // constituent that owns it. Scalar elements have no sub-fields, so nothing remains.
+            final ScalarUnnestingInfo scalarInfo = scalarFanouts.get(marker);
+            Assert.notNullUnchecked(scalarInfo, "unknown unnesting in index definition");
+            Assert.thatUnchecked(remaining.isEmpty(), ErrorCode.UNSUPPORTED_OPERATION,
+                    "Unsupported index definition, cannot dereference a field of a scalar array element");
+            return field(scalarInfo.owningAlias(), KeyExpression.FanType.None)
+                    .nest(scalarInfo.toFanOutExpression());
         }
     }