Build a match candidate from an index defined on an unnested record type

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/Compensation.java

@@ -1031,20 +1031,28 @@ public RelationalExpression apply(@Nonnull final Memoizer memoizer,
 
         @Nonnull
         private CorrelationIdentifier getMatchedForEachAlias() {
+            return Iterables.getOnlyElement(getMatchedForEachAliases());
+        }
+
+        /**
+         * Returns the aliases of the matched for-each quantifiers. There is more than one exactly when the match
+         * stands in for a join, which is what a candidate over a synthetic record type produces.
+         *
+         * @return the aliases of the matched for-each quantifiers
+         */
+        @Nonnull
+        private Set<CorrelationIdentifier> getMatchedForEachAliases() {
             final var matchedQuantifierMap =
                     Quantifiers.aliasToQuantifierMap(matchedQuantifiers);
 
             final var matchedAliases =
                     matchedQuantifierMap.keySet();
             Verify.verify(compensatedAliases.equals(matchedAliases));
 
-            final var matchedForEachQuantifierAliases =
-                    matchedAliases
-                            .stream()
-                            .filter(alias -> matchedQuantifierMap.get(alias) instanceof Quantifier.ForEach)
-                            .collect(ImmutableSet.toImmutableSet());
-
-            return Iterables.getOnlyElement(matchedForEachQuantifierAliases);
+            return matchedAliases
+                    .stream()
+                    .filter(alias -> matchedQuantifierMap.get(alias) instanceof Quantifier.ForEach)
+                    .collect(ImmutableSet.toImmutableSet());
         }
 
         @Nonnull
@@ -1055,7 +1063,13 @@ public RelationalExpression applyFinal(@Nonnull final Memoizer memoizer,
             Verify.verify(!isImpossible());
             Verify.verify(resultCompensationFunction.isNeeded());
 
-            final var matchedForEachAlias = getMatchedForEachAlias();
+            //
+            // Any of the matched for-each aliases will do here, and when the match stands in for a join there is more
+            // than one with none of them distinguished. The compensated result value is expressed over the top of the
+            // match, the translation below re-targets it onto the alias picked here, and that alias then names the
+            // single new quantifier over the data access -- so the choice only decides a name.
+            //
+            final var matchedForEachAlias = Iterables.get(getMatchedForEachAliases(), 0);
 
             final var resultValue =
                     resultCompensationFunction.applyCompensationForResult(

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/IndexExpansionInfo.java

@@ -51,22 +51,18 @@ public final class IndexExpansionInfo {
     @Nonnull
     private final Collection<RecordType> indexedRecordTypes;
     @Nonnull
-    private final Set<String> indexedRecordTypeNames;
-    @Nonnull
     private final Type.Record baseType;
 
     private IndexExpansionInfo(@Nonnull RecordMetaData metaData,
                                @Nonnull Index index,
                                boolean reverse,
                                @Nonnull Collection<RecordType> indexedRecordTypes,
-                               @Nonnull Set<String> indexedRecordTypeNames,
                                @Nonnull Type.Record baseType,
                                @Nullable KeyExpression commonPrimaryKeyForTypes) {
         this.metaData = metaData;
         this.index = index;
         this.reverse = reverse;
         this.indexedRecordTypes = indexedRecordTypes;
-        this.indexedRecordTypeNames = indexedRecordTypeNames;
         this.baseType = baseType;
         this.commonPrimaryKeyForTypes = commonPrimaryKeyForTypes;
     }
@@ -97,7 +93,9 @@ public Collection<RecordType> getIndexedRecordTypes() {
 
     @Nonnull
     public Set<String> getIndexedRecordTypeNames() {
-        return indexedRecordTypeNames;
+        return indexedRecordTypes.stream()
+                .map(RecordType::getName)
+                .collect(ImmutableSet.toImmutableSet());
     }
 
     @Nullable
@@ -133,15 +131,12 @@ public static IndexExpansionInfo createInfo(@Nonnull RecordMetaData metaData,
         @Nonnull
         final Collection<RecordType> indexedRecordTypes = Collections.unmodifiableCollection(metaData.recordTypesForIndex(index));
         @Nonnull
-        final Set<String> indexedRecordTypeNames = indexedRecordTypes.stream()
-                .map(RecordType::getName)
-                .collect(ImmutableSet.toImmutableSet());
-        @Nonnull
-        final Type.Record baseType = metaData.getPlannerType(indexedRecordTypeNames);
+        final Type.Record baseType = metaData.getPlannerTypeForRecordTypes(indexedRecordTypes);
         @Nullable
         final KeyExpression commonPrimaryKeyForTypes = RecordMetaData.commonPrimaryKey(indexedRecordTypes);
 
-        return new IndexExpansionInfo(metaData, index, reverse, indexedRecordTypes, indexedRecordTypeNames, baseType, commonPrimaryKeyForTypes);
+        return new IndexExpansionInfo(metaData, index, reverse, indexedRecordTypes, baseType,
+                commonPrimaryKeyForTypes);
     }
 
 }

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/KeyExpressionExpansionVisitor.java

@@ -39,6 +39,7 @@
 import com.apple.foundationdb.record.query.plan.cascades.predicates.PredicateWithValueAndRanges;
 import com.apple.foundationdb.record.query.plan.cascades.values.EmptyValue;
 import com.apple.foundationdb.record.query.plan.cascades.values.FieldValue;
+import com.apple.foundationdb.record.query.plan.cascades.values.RecordConstructorValue;
 import com.apple.foundationdb.record.query.plan.cascades.values.Value;
 import com.apple.foundationdb.record.util.ProtoUtils;
 import com.google.common.base.Verify;
@@ -441,7 +442,63 @@ public GraphExpansion visitExpression(@Nonnull final ThenKeyExpression thenKeyEx
     @Nonnull
     @Override
     public GraphExpansion visitExpression(@Nonnull final ListKeyExpression listKeyExpression) {
-        throw new UnsupportedOperationException("visitor method for this key expression is not implemented");
+        final ImmutableList.Builder<GraphExpansion> expandedPredicatesBuilder = ImmutableList.builder();
+        final VisitorState state = getCurrentState();
+        int currentOrdinal = state.getCurrentOrdinal();
+        for (KeyExpression child : listKeyExpression.getChildren()) {
+            final VisitorState childState = state.withCurrentOrdinal(currentOrdinal);
+            final Value value = childState.registerValue(listChildValue(child, childState));
+            expandedPredicatesBuilder.add(expansionForListChild(childState, value));
+            currentOrdinal++;
+        }
+        return GraphExpansion.ofOthers(expandedPredicatesBuilder.build());
+    }
+
+    /**
+     * Computes the single {@link Value} a child of a {@link ListKeyExpression} contributes. The child is expanded as an
+     * internal expansion so that it neither registers its own values nor creates its own placeholders, and the values
+     * of the columns it yields are then collapsed into one.
+     *
+     * @param child the child of the list
+     * @param state the state to expand the child under, whose ordinal is the child's key position
+     * @return the value the child contributes at its position
+     */
+    @Nonnull
+    private Value listChildValue(@Nonnull final KeyExpression child, @Nonnull final VisitorState state) {
+        final GraphExpansion childExpansion = pop(child.expand(push(state.forFunctionalExpansion())));
+        if (!childExpansion.getQuantifiers().isEmpty()) {
+            throw new UnsupportedOperationException("cannot expand a list whose child introduces quantifiers");
+        }
+        final var childValues = childExpansion.getResultColumns()
+                .stream()
+                .map(Column::getValue)
+                .collect(ImmutableList.toImmutableList());
+        if (childValues.size() == 1) {
+            return Iterables.getOnlyElement(childValues);
+        }
+        return RecordConstructorValue.ofUnnamed(childValues);
+    }
+
+    /**
+     * Emits the expansion for one child of a {@link ListKeyExpression}, mirroring what
+     * {@link #visitExpression(FieldKeyExpression)} does for a scalar field.
+     *
+     * @param state the state whose ordinal is the child's key position
+     * @param value the value the child contributes
+     * @return the expansion contributed by that child
+     */
+    @Nonnull
+    private GraphExpansion expansionForListChild(@Nonnull final VisitorState state, @Nonnull final Value value) {
+        final boolean isSargable = state.isKey() && !state.isInternalExpansion();
+        if (state.isSelectStar()) {
+            return isSargable
+                   ? GraphExpansion.ofPlaceholder(value.asPlaceholder(newParameterAlias()))
+                   : GraphExpansion.empty();
+        }
+        final Column<?> column = Column.unnamedOf(value);
+        return isSargable
+               ? GraphExpansion.ofResultColumnAndPlaceholder(column, value.asPlaceholder(newParameterAlias()))
+               : GraphExpansion.ofResultColumn(column);
     }
 
     /**

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/MetaDataPlanContext.java

@@ -102,6 +102,28 @@ private static List<Index> readableOf(@Nonnull RecordStoreState recordStoreState
         }
     }
 
+    /**
+     * Collects the indexes defined on a {@link com.apple.foundationdb.record.metadata.SyntheticRecordType} that the
+     * query could use.
+     */
+    @Nonnull
+    private static List<Index> syntheticIndexesOf(@Nonnull final RecordMetaData metaData,
+                                                  @Nonnull final RecordStoreState recordStoreState,
+                                                  @Nonnull final Collection<String> queriedRecordTypeNames) {
+        final var indexes = Lists.<Index>newArrayList();
+        for (final var syntheticRecordType : metaData.getSyntheticRecordTypes().values()) {
+            final var storedConstituentNames =
+                    syntheticRecordType.getConstituents().stream()
+                            .map(constituent -> constituent.getRecordType().getName())
+                            .filter(metaData.getRecordTypes()::containsKey)
+                            .collect(ImmutableSet.toImmutableSet());
+            if (!storedConstituentNames.isEmpty() && queriedRecordTypeNames.containsAll(storedConstituentNames)) {
+                indexes.addAll(readableOf(recordStoreState, syntheticRecordType.getIndexes()));
+            }
+        }
+        return indexes;
+    }
+
     @Nonnull
     public static PlanContext forRecordQuery(@Nonnull RecordQueryPlannerConfiguration plannerConfiguration,
                                              @Nonnull RecordMetaData metaData,
@@ -144,6 +166,7 @@ public static PlanContext forRecordQuery(@Nonnull RecordQueryPlannerConfiguratio
             }
 
             indexList.addAll(readableOf(recordStoreState, metaData.getUniversalIndexes()));
+            indexList.addAll(syntheticIndexesOf(metaData, recordStoreState, queriedRecordTypeNames));
         } finally {
             recordStoreState.endRead();
         }
@@ -187,6 +210,7 @@ public static PlanContext forRootReference(@Nonnull final RecordQueryPlannerConf
             for (final var recordType : queriedRecordTypes) {
                 indexList.addAll(readableOf(recordStoreState, recordType.getAllIndexes()));
             }
+            indexList.addAll(syntheticIndexesOf(metaData, recordStoreState, queriedRecordTypeNames));
         } finally {
             recordStoreState.endRead();
         }

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/ScalarTranslationVisitor.java

@@ -34,6 +34,7 @@
 import com.apple.foundationdb.record.query.plan.cascades.values.EmptyValue;
 import com.apple.foundationdb.record.query.plan.cascades.values.FieldValue;
 import com.apple.foundationdb.record.query.plan.cascades.values.QuantifiedObjectValue;
+import com.apple.foundationdb.record.query.plan.cascades.values.RecordConstructorValue;
 import com.apple.foundationdb.record.query.plan.cascades.values.Value;
 import com.apple.foundationdb.record.util.ProtoUtils;
 import com.google.common.collect.ImmutableList;
@@ -203,7 +204,29 @@ public Value visitExpression(@Nonnull final ThenKeyExpression thenKeyExpression)
     @Nonnull
     @Override
     public Value visitExpression(@Nonnull final ListKeyExpression listKeyExpression) {
-        throw new UnsupportedOperationException("visitor method for this key expression is not implemented");
+        // A list places each child into its own nested tuple, so a list of more than one child spans more than one
+        // key position and cannot be scalar. `normalizeKeyForPositions()` splits a longer list into single-child
+        // lists, which is the shape that reaches here.
+        if (listKeyExpression.getColumnSize() > 1) {
+            throw new RecordCoreException("cannot expand ListKeyExpression in scalar expansion");
+        }
+
+        final ScalarVisitorState state = getCurrentState();
+        final KeyExpression child = Iterables.getOnlyElement(listKeyExpression.getChildren());
+
+        // The child occupies a single position whose value is the nested tuple of the child's own columns, so a
+        // multi-column child collapses into one record. This has to agree with what
+        // `KeyExpressionExpansionVisitor#visitExpression(ListKeyExpression)` registers for the same position, since
+        // the ordering parts computed from these values are matched to the candidate's parameters by ordinal.
+        final List<KeyExpression> positions = child.normalizeKeyForPositions();
+        if (positions.size() == 1) {
+            return pop(Iterables.getOnlyElement(positions).expand(push(state)));
+        }
+        final ImmutableList.Builder<Value> valuesBuilder = ImmutableList.builder();
+        for (final KeyExpression position : positions) {
+            valuesBuilder.add(pop(position.expand(push(state))));
+        }
+        return RecordConstructorValue.ofUnnamed(valuesBuilder.build());
     }
 
     @Nonnull

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/ScanWithFetchMatchCandidate.java

@@ -22,6 +22,8 @@
 
 import com.apple.foundationdb.record.EvaluationContext;
 import com.apple.foundationdb.record.metadata.RecordType;
+import com.apple.foundationdb.record.metadata.expressions.KeyExpression;
+import com.apple.foundationdb.record.metadata.expressions.ListKeyExpression;
 import com.apple.foundationdb.record.query.plan.IndexKeyValueToPartialRecord;
 import com.apple.foundationdb.record.query.plan.cascades.typing.Type;
 import com.apple.foundationdb.record.query.plan.cascades.values.FieldValue;
@@ -138,7 +140,8 @@ static Optional<ScanWithFetchMatchCandidate.IndexEntryToLogicalRecord> computeIn
                                                                                                             @Nonnull final CorrelationIdentifier baseAlias,
                                                                                                             @Nonnull final Type baseType,
                                                                                                             @Nonnull final List<Value> indexKeyValues,
-                                                                                                            @Nonnull final List<Value> indexValueValues) {
+                                                                                                            @Nonnull final List<Value> indexValueValues,
+                                                                                                            @Nonnull final List<KeyExpression> normalizedKeyExpressions) {
         if (queriedRecordTypes.size() > 1) {
             return Optional.empty();
         }
@@ -152,7 +155,8 @@ static Optional<ScanWithFetchMatchCandidate.IndexEntryToLogicalRecord> computeIn
 
             final var extractFromIndexEntryPairOptional =
                     keyValue.extractFromIndexEntryMaybe(baseObjectValue, EvaluationContext.empty(), AliasMap.emptyMap(),
-                            ImmutableSet.of(), IndexKeyValueToPartialRecord.TupleSource.KEY, ImmutableIntArray.of(i));
+                            ImmutableSet.of(), IndexKeyValueToPartialRecord.TupleSource.KEY,
+                            keyOrdinalPath(normalizedKeyExpressions, i));
             if (extractFromIndexEntryPairOptional.isPresent()) {
                 final var extractFromIndexEntryPair = extractFromIndexEntryPairOptional.get();
                 if (!addCoveringField(builder, extractFromIndexEntryPair.getKey(),
@@ -192,6 +196,28 @@ static Optional<ScanWithFetchMatchCandidate.IndexEntryToLogicalRecord> computeIn
                         indexEntryToRecordValue(baseType, covered)));
     }
 
+    /**
+     * Computes the path into the index entry's key tuple at which the data for key position {@code ordinal} is found.
+     *
+     * <p>For almost every {@link KeyExpression} that is a single element, {@code tuple.get(ordinal)}. A
+     * {@link ListKeyExpression} is the exception: it places each of its children into a nested tuple of its own rather
+     * than flattening them, so the datum sits one level deeper. {@code normalizeKeyForPositions()} yields single-child
+     * lists, so that level is entered at ordinal zero.
+     *
+     * @param normalizedKeyExpressions the positions of the full key, aligned with the candidate's key values
+     * @param ordinal the key position
+     * @return the ordinal path to extract that position with
+     */
+    @Nonnull
+    private static ImmutableIntArray keyOrdinalPath(@Nonnull final List<KeyExpression> normalizedKeyExpressions,
+                                                    final int ordinal) {
+        if (ordinal < normalizedKeyExpressions.size() &&
+                normalizedKeyExpressions.get(ordinal) instanceof ListKeyExpression) {
+            return ImmutableIntArray.of(ordinal, 0);
+        }
+        return ImmutableIntArray.of(ordinal);
+    }
+
     /**
      * Records what the given extraction covers, descending a node per field of the path it fills, which may run into
      * nested messages. Every name is present here, {@code addCoveringField} having already refused the extraction