Match a plain explode against an explode with ordinality

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaData.java

@@ -775,10 +775,27 @@ public List<String> getTempFunctions() {
         }
     }
 
-
+    /**
+     * Get the planner's view of a stored record type. This resolves the name against the stored record types only;
+     * use {@link #getPlannerTypeForRecordType(RecordType)} for a {@link SyntheticRecordType}.
+     * @param recordTypeName the name of a stored record type
+     * @return the type the planner uses to represent records of that type
+     */
     @Nonnull
     public Type.Record getPlannerType(@Nonnull String recordTypeName) {
-        final RecordType recordType = getRecordType(recordTypeName);
+        return getPlannerTypeForRecordType(getRecordType(recordTypeName));
+    }
+
+    /**
+     * Get the planner's view of an already-resolved record type. Unlike {@link #getPlannerType(String)} this does not
+     * resolve a name, so it works for a {@link SyntheticRecordType} as well: an index defined on a synthetic record
+     * type names that type, and {@link #recordTypesForIndex(Index)} hands back the type itself, so expanding such an
+     * index does not need to look the name up again.
+     * @param recordType a record type, possibly synthetic
+     * @return the type the planner uses to represent records of that type
+     */
+    @Nonnull
+    public Type.Record getPlannerTypeForRecordType(@Nonnull RecordType recordType) {
         Type.Record plannerType = Type.Record.fromDescriptor(recordType.getDescriptor());
         if (storeRecordVersions) {
             plannerType = plannerType.addPseudoFields();
@@ -788,13 +805,25 @@ public Type.Record getPlannerType(@Nonnull String recordTypeName) {
 
     @Nonnull
     public Type.Record getPlannerType(@Nonnull Collection<String> recordTypeNames) {
-        if (recordTypeNames.size() == 1) {
-            final String recordTypeName = Iterables.getOnlyElement(recordTypeNames);
-            return getPlannerType(recordTypeName);
+        return getPlannerTypeForRecordTypes(recordTypeNames.stream()
+                .map(this::getRecordType)
+                .collect(Collectors.toList()));
+    }
+
+    /**
+     * As {@link #getPlannerType(Collection)}, but for already-resolved record types, so it also accepts
+     * {@link SyntheticRecordType}s.
+     * @param recordTypes the record types the planner type should cover
+     * @return the type the planner uses to represent records of those types
+     */
+    @Nonnull
+    public Type.Record getPlannerTypeForRecordTypes(@Nonnull Collection<RecordType> recordTypes) {
+        if (recordTypes.size() == 1) {
+            return getPlannerTypeForRecordType(Iterables.getOnlyElement(recordTypes));
         }
         // todo: should be removed https://github.com/FoundationDB/fdb-record-layer/issues/1884
-        LinkedHashMap<String, Type.Record.Field> fieldsByName = recordTypeNames.stream()
-                .map(this::getPlannerType)
+        LinkedHashMap<String, Type.Record.Field> fieldsByName = recordTypes.stream()
+                .map(this::getPlannerTypeForRecordType)
                 .flatMap(type -> type.getFields().stream())
                 .collect(Collectors.groupingBy(Type.Record.Field::getFieldName,
                         LinkedHashMap::new,

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/expressions/ExplodeExpression.java

@@ -23,19 +23,23 @@
 import com.apple.foundationdb.annotation.API;
 import com.apple.foundationdb.record.EvaluationContext;
 import com.apple.foundationdb.record.query.plan.cascades.AliasMap;
+import com.apple.foundationdb.record.query.plan.cascades.Column;
 import com.apple.foundationdb.record.query.plan.cascades.ComparisonRange;
 import com.apple.foundationdb.record.query.plan.cascades.Compensation;
 import com.apple.foundationdb.record.query.plan.cascades.CorrelationIdentifier;
 import com.apple.foundationdb.record.query.plan.cascades.IdentityBiMap;
 import com.apple.foundationdb.record.query.plan.cascades.MatchInfo;
 import com.apple.foundationdb.record.query.plan.cascades.PartialMatch;
 import com.apple.foundationdb.record.query.plan.cascades.Quantifier;
+import com.apple.foundationdb.record.query.plan.cascades.Quantifiers;
 import com.apple.foundationdb.record.query.plan.cascades.explain.InternalPlannerGraphRewritable;
 import com.apple.foundationdb.record.query.plan.cascades.explain.PlannerGraph;
 import com.apple.foundationdb.record.query.plan.cascades.typing.Type;
 import com.apple.foundationdb.record.query.plan.cascades.values.FieldValue;
 import com.apple.foundationdb.record.query.plan.cascades.values.QueriedValue;
+import com.apple.foundationdb.record.query.plan.cascades.values.RecordConstructorValue;
 import com.apple.foundationdb.record.query.plan.cascades.values.Value;
+import com.apple.foundationdb.record.query.plan.cascades.values.translation.MaxMatchMap;
 import com.apple.foundationdb.record.query.plan.cascades.values.translation.PullUp;
 import com.apple.foundationdb.record.query.plan.cascades.values.translation.TranslationMap;
 import com.google.common.base.Verify;
@@ -85,6 +89,12 @@ public class ExplodeExpression extends AbstractRelationalExpressionWithoutChildr
     @Nonnull
     private final Type explodeResultType;
 
+    /**
+     * The result value of the explode.
+     */
+    @Nonnull
+    private final Value resultValue;
+
     public ExplodeExpression(@Nonnull final Value collectionValue, final boolean withOrdinality,
                              final boolean zeroBasedOrdinality) {
         Verify.verify(withOrdinality || !zeroBasedOrdinality, "cannot base ordinals that are not produced");
@@ -94,6 +104,8 @@ public ExplodeExpression(@Nonnull final Value collectionValue, final boolean wit
         Verify.verify(collectionValue.getResultType().isArray());
         this.elementType = Objects.requireNonNull(((Type.Array)collectionValue.getResultType()).getElementType());
         this.explodeResultType = explodeResultType(elementType, withOrdinality);
+        this.resultValue = explodeResultValue(elementType, withOrdinality);
+        Verify.verify(explodeResultType.equals(resultValue.getResultType()));
     }
 
     public ExplodeExpression(@Nonnull final Value collectionValue, final boolean withOrdinality) {
@@ -135,10 +147,45 @@ public Type getExplodeResultType() {
         return explodeResultType;
     }
 
+    /**
+     * Returns the value an explode of {@code elementType} flows. For the plain variant that is an opaque
+     * {@link QueriedValue} standing for the element. For the {@code WITH ORDINALITY} variant it is a
+     * {@link RecordConstructorValue} of two such values, the element and the ordinal, rather than a single opaque value
+     * of the struct type.
+     *
+     * <p>The distinction matters for matching, not for evaluation. {@link com.apple.foundationdb.record.query.plan.cascades.values.translation.MaxMatchMap}
+     * descends into record constructors but not into opaque values, so building the struct explicitly makes the element
+     * a <em>reachable</em> sub-value of the result: a plain explode on the query side can then be related to a
+     * {@code WITH ORDINALITY} explode on the candidate side, and the correspondence pulls up through the enclosing
+     * quantifiers as {@code q._0} rather than being lost. An opaque value of the struct type offers nothing to match
+     * against but itself.
+     *
+     * @param elementType the element type of the collection being exploded
+     * @param withOrdinality whether ordinals are produced alongside the elements
+     * @return the value flowed by such an explode
+     */
+    @Nonnull
+    public static Value explodeResultValue(@Nonnull final Type elementType, final boolean withOrdinality) {
+        final var elementValue = new QueriedValue(elementType);
+        if (!withOrdinality) {
+            return elementValue;
+        }
+        // Note: the element must stay the first column. `MaxMatchMap` returns the first reachable candidate value that
+        // compares equal, and a `QueriedValue` compares equal to any other `QueriedValue`, so an element on the query
+        // side would just as happily match the ordinal if the ordinal came first.
+        //
+        // The record is built nullable so that its type is the one `explodeResultType` already declares -- that type
+        // backs the protobuf descriptor the plan builds at run time, so it is the type that must not move.
+        return RecordConstructorValue.ofColumns(
+                ImmutableList.of(Column.unnamedOf(elementValue),
+                        Column.unnamedOf(new QueriedValue(Type.primitiveType(Type.TypeCode.INT, false)))),
+                true);
+    }
+
     @Nonnull
     @Override
     public Value getResultValue() {
-        return new QueriedValue(getExplodeResultType());
+        return resultValue;
     }
 
     @Nonnull
@@ -220,9 +267,50 @@ public Iterable<MatchInfo> subsumedBy(@Nonnull final RelationalExpression candid
             return ImmutableList.of();
         }
 
+        if (!withOrdinality
+                && candidateExpression instanceof final ExplodeExpression candidateExplodeExpression
+                && candidateExplodeExpression.isWithOrdinality()) {
+            return subsumedByWithOrdinality(candidateExplodeExpression, bindingAliasMap, partialMatchMap);
+        }
+
         return exactlySubsumedBy(candidateExpression, bindingAliasMap, partialMatchMap, TranslationMap.empty());
     }
 
+    /**
+     * Establishes that an explode <em>without</em> ordinality is subsumed by an explode <em>with</em> ordinality over
+     * the same collection. The candidate emits one {@code (element, ordinal)} struct per element this expression
+     * emits just element. That satisfies subsumption: the candidate produces at least everything the query may produce.
+     * This case cannot be dealt with by {@link #exactlySubsumedBy}, whose {@code equalsWithoutChildren} compares
+     * {@link #isWithOrdinality()}.
+     *
+     * <p>No {@link com.apple.foundationdb.record.query.plan.cascades.ValueEquivalence} is needed to relate the two
+     * result values: {@link #explodeResultValue} builds the candidate's as a record constructor, so this expression's
+     * element value is a reachable sub-value of it and the correspondence is found structurally. The resulting mapping
+     * points at the candidate's element column, which is what lets the enclosing select express a navigation into the
+     * element as {@code q._0.field}.
+     *
+     * @param candidateExpression the candidate explode, which must be {@code WITH ORDINALITY}
+     * @param bindingAliasMap a map of aliases defining the equivalence between quantifiers
+     * @param partialMatchMap a map from quantifier to the {@link PartialMatch} pulled up along that quantifier
+     * @return an iterable containing a {@link MatchInfo} if subsumption holds, empty otherwise
+     */
+    @Nonnull
+    private Iterable<MatchInfo> subsumedByWithOrdinality(@Nonnull final ExplodeExpression candidateExpression,
+                                                         @Nonnull final AliasMap bindingAliasMap,
+                                                         @Nonnull final IdentityBiMap<Quantifier, PartialMatch> partialMatchMap) {
+        if (!collectionValue.semanticEquals(candidateExpression.getCollectionValue(), bindingAliasMap)) {
+            return ImmutableList.of();
+        }
+
+        final var maxMatchMap =
+                MaxMatchMap.compute(getResultValue(), candidateExpression.getResultValue(),
+                        Quantifiers.aliases(candidateExpression.getQuantifiers()));
+
+        return MatchInfo.RegularMatchInfo.tryFromMatchMap(bindingAliasMap, partialMatchMap, maxMatchMap)
+                .map(ImmutableList::of)
+                .orElse(ImmutableList.of());
+    }
+
     @Nonnull
     @Override
     public Compensation compensate(@Nonnull final PartialMatch partialMatch,

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/QueriedValue.java

@@ -29,6 +29,7 @@
 import com.apple.foundationdb.record.planprotos.PQueriedValue;
 import com.apple.foundationdb.record.planprotos.PValue;
 import com.apple.foundationdb.record.query.plan.cascades.AliasMap;
+import com.apple.foundationdb.record.query.plan.cascades.ConstrainedBoolean;
 import com.apple.foundationdb.record.query.plan.explain.ExplainTokens;
 import com.apple.foundationdb.record.query.plan.explain.ExplainTokensWithPrecedence;
 import com.apple.foundationdb.record.query.plan.cascades.typing.Type;
@@ -90,6 +91,16 @@ public boolean isFunctionallyDependentOn(@Nonnull final Value otherValue) {
         return false;
     }
 
+    /**
+     * Two queried values are equal if they stand for streams of the same type.
+     */
+    @Nonnull
+    @Override
+    public ConstrainedBoolean equalsWithoutChildren(@Nonnull final Value other) {
+        return LeafValue.super.equalsWithoutChildren(other)
+                .filter(ignored -> resultType.equals(((QueriedValue)other).resultType));
+    }
+
     @Override
     public int hashCodeWithoutChildren() {
         return PlanHashable.objectPlanHash(PlanHashable.CURRENT_FOR_CONTINUATION, BASE_HASH);

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryExplodePlan.java

@@ -46,7 +46,6 @@
 import com.apple.foundationdb.record.query.plan.cascades.expressions.RelationalExpression;
 import com.apple.foundationdb.record.query.plan.cascades.typing.Type;
 import com.apple.foundationdb.record.query.plan.cascades.typing.TypeRepository;
-import com.apple.foundationdb.record.query.plan.cascades.values.QueriedValue;
 import com.apple.foundationdb.record.query.plan.cascades.values.RecordConstructorValue;
 import com.apple.foundationdb.record.query.plan.cascades.values.Value;
 import com.apple.foundationdb.record.query.plan.cascades.values.translation.TranslationMap;
@@ -267,7 +266,7 @@ public Type getExplodeResultType() {
     @Nonnull
     @Override
     public Value getResultValue() {
-        return new QueriedValue(getExplodeResultType());
+        return ExplodeExpression.explodeResultValue(elementType, withOrdinality);
     }
 
     @Nonnull

fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/metadata/UnnestedRecordTypeExpansionTest.java

@@ -37,6 +37,7 @@
 import com.apple.foundationdb.record.query.plan.cascades.expressions.LogicalTypeFilterExpression;
 import com.apple.foundationdb.record.query.plan.cascades.expressions.RelationalExpression;
 import com.apple.foundationdb.record.query.plan.cascades.expressions.SelectExpression;
+import com.apple.foundationdb.record.query.plan.cascades.typing.PseudoField;
 import com.apple.foundationdb.record.query.plan.cascades.typing.Type;
 import com.apple.foundationdb.record.query.plan.cascades.values.FieldValue;
 import com.apple.foundationdb.record.query.plan.cascades.values.Value;
@@ -257,6 +258,56 @@ void expandOnAJoinedRecordTypeIsUnsupported() {
         assertEquals("cannot expand an index defined on a JoinedRecordType", exception.getMessage());
     }
 
+    @Test
+    void getPlannerTypeForRecordTypeDescribesASyntheticType() {
+        final RecordMetaData metaData = mapMetaData(addTwoMapsType());
+        final UnnestedRecordType type = unnestedType(metaData, TWO_UNNESTED_MAPS);
+
+        // A synthetic type's name cannot be resolved against the stored types, so only the type-taking overload can
+        // describe it.
+        assertThrows(MetaDataException.class, () -> metaData.getPlannerType(TWO_UNNESTED_MAPS));
+        final Type.Record plannerType = metaData.getPlannerTypeForRecordType(type);
+        assertEquals(Type.Record.fromDescriptor(type.getDescriptor()), plannerType);
+        assertThat(plannerType.getFields().stream().map(Type.Record.Field::getFieldName).collect(Collectors.toList()),
+                contains(PARENT, "entry_one", "entry_two", UnnestedRecordType.POSITIONS_FIELD));
+    }
+
+    @Test
+    void getPlannerTypeForRecordTypeMatchesTheNameTakingOverloads() {
+        final RecordMetaData metaData = mapMetaData(metaDataBuilder -> metaDataBuilder.setStoreRecordVersions(true));
+
+        assertEquals(metaData.getPlannerType(OUTER),
+                metaData.getPlannerTypeForRecordType(metaData.getRecordType(OUTER)));
+        assertEquals(metaData.getPlannerType(List.of(OUTER)),
+                metaData.getPlannerTypeForRecordTypes(List.of(metaData.getRecordType(OUTER))));
+        assertEquals(metaData.getPlannerType(List.of(OUTER, OTHER)),
+                metaData.getPlannerTypeForRecordTypes(
+                        List.of(metaData.getRecordType(OUTER), metaData.getRecordType(OTHER))));
+
+        // Storing record versions adds the pseudo field to every stored type, including the union of several of them.
+        final String versionField = PseudoField.ROW_VERSION.getFieldName();
+        assertTrue(metaData.getPlannerTypeForRecordType(metaData.getRecordType(OUTER))
+                .getFieldNameFieldMap().containsKey(versionField));
+        assertTrue(metaData.getPlannerTypeForRecordTypes(
+                        List.of(metaData.getRecordType(OUTER), metaData.getRecordType(OTHER)))
+                .getFieldNameFieldMap().containsKey(versionField));
+    }
+
+    @Test
+    void getPlannerTypeForRecordTypesUnionsTheirFields() {
+        final RecordMetaData metaData = mapMetaData(metaDataBuilder -> { });
+        final Type.Record unionType = metaData.getPlannerTypeForRecordTypes(
+                List.of(metaData.getRecordType(OUTER), metaData.getRecordType(OTHER)));
+
+        // `rec_id` and `other_id` are shared, so the union has them once; the remaining fields come from one type each.
+        assertThat(unionType.getFields().stream().map(Type.Record.Field::getFieldName).collect(Collectors.toList()),
+                contains("rec_id", "other_id", "map", "other_value"));
+
+        // A single type is described exactly as the type-taking overload would describe it on its own.
+        assertEquals(metaData.getPlannerTypeForRecordType(metaData.getRecordType(OUTER)),
+                metaData.getPlannerTypeForRecordTypes(List.of(metaData.getRecordType(OUTER))));
+    }
+
     @Nonnull
     private static RecordMetaData mapMetaData(@Nonnull Consumer<RecordMetaDataBuilder> hook) {
         final RecordMetaDataBuilder metaDataBuilder = RecordMetaData.newBuilder()