diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaData.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaData.java index 25f81968138..8eda82ee1ca 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaData.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaData.java @@ -775,10 +775,27 @@ public List getTempFunctions() { } } - + /** + * Get the planner's view of a stored record type. This resolves the name against the stored record types only. + * + * @param recordTypeName the name of a stored record type + * @return the type the planner uses to represent records of that type + */ @Nonnull public Type.Record getPlannerType(@Nonnull String recordTypeName) { - final RecordType recordType = getRecordType(recordTypeName); + return getPlannerTypeForRecordType(getRecordType(recordTypeName)); + } + + /** + * Get the planner's view of an already-resolved record type. Unlike {@link #getPlannerType(String)} this does not + * resolve a name, so it works for a {@link SyntheticRecordType} as well: an index defined on a synthetic record + * type names that type, and {@link #recordTypesForIndex(Index)} hands back the type itself, so expanding such an + * index does not need to look the name up again. + * @param recordType a record type, possibly synthetic + * @return the type the planner uses to represent records of that type + */ + @Nonnull + public Type.Record getPlannerTypeForRecordType(@Nonnull RecordType recordType) { Type.Record plannerType = Type.Record.fromDescriptor(recordType.getDescriptor()); if (storeRecordVersions) { plannerType = plannerType.addPseudoFields(); @@ -788,13 +805,25 @@ public Type.Record getPlannerType(@Nonnull String recordTypeName) { @Nonnull public Type.Record getPlannerType(@Nonnull Collection recordTypeNames) { - if (recordTypeNames.size() == 1) { - final String recordTypeName = Iterables.getOnlyElement(recordTypeNames); - return getPlannerType(recordTypeName); + return getPlannerTypeForRecordTypes(recordTypeNames.stream() + .map(this::getRecordType) + .collect(Collectors.toList())); + } + + /** + * As {@link #getPlannerType(Collection)}, but for already-resolved record types, so it also accepts + * {@link SyntheticRecordType}s. + * @param recordTypes the record types the planner type should cover + * @return the type the planner uses to represent records of those types + */ + @Nonnull + public Type.Record getPlannerTypeForRecordTypes(@Nonnull Collection recordTypes) { + if (recordTypes.size() == 1) { + return getPlannerTypeForRecordType(Iterables.getOnlyElement(recordTypes)); } // todo: should be removed https://github.com/FoundationDB/fdb-record-layer/issues/1884 - LinkedHashMap fieldsByName = recordTypeNames.stream() - .map(this::getPlannerType) + LinkedHashMap fieldsByName = recordTypes.stream() + .map(this::getPlannerTypeForRecordType) .flatMap(type -> type.getFields().stream()) .collect(Collectors.groupingBy(Type.Record.Field::getFieldName, LinkedHashMap::new, diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/IndexPredicate.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/IndexPredicate.java index 36d7699e5c8..ac4f21303ad 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/IndexPredicate.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/IndexPredicate.java @@ -705,7 +705,7 @@ public List> getPartitionFieldPaths() { @Nonnull public KeyExpression getOrderingKey() { - return fieldPathToKeyExpression(orderingField); + return KeyExpression.fromPath(orderingField); } /** @@ -718,11 +718,11 @@ public KeyExpression getPartitionKey() { return null; } if (partitionFieldPaths.size() == 1) { - return fieldPathToKeyExpression(partitionFieldPaths.get(0)); + return KeyExpression.fromPath(partitionFieldPaths.get(0)); } - KeyExpression result = fieldPathToKeyExpression(partitionFieldPaths.get(0)); + KeyExpression result = KeyExpression.fromPath(partitionFieldPaths.get(0)); for (int i = 1; i < partitionFieldPaths.size(); i++) { - result = Key.Expressions.concat(result, fieldPathToKeyExpression(partitionFieldPaths.get(i))); + result = Key.Expressions.concat(result, KeyExpression.fromPath(partitionFieldPaths.get(i))); } return result; } @@ -804,14 +804,5 @@ public boolean equals(Object o) { public int hashCode() { return Objects.hash(orderingField, direction, size, partitionFieldPaths); } - - @Nonnull - private static KeyExpression fieldPathToKeyExpression(@Nonnull List path) { - KeyExpression result = Key.Expressions.field(path.get(path.size() - 1)); - for (int i = path.size() - 2; i >= 0; i--) { - result = Key.Expressions.field(path.get(i)).nest(result); - } - return result; - } } } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/SyntheticRecordType.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/SyntheticRecordType.java index 6f20100f2c7..152950bb049 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/SyntheticRecordType.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/SyntheticRecordType.java @@ -26,6 +26,8 @@ import com.apple.foundationdb.record.provider.foundationdb.FDBRecordStore; import com.apple.foundationdb.record.provider.foundationdb.FDBSyntheticRecord; import com.apple.foundationdb.record.provider.foundationdb.IndexOrphanBehavior; +import com.apple.foundationdb.record.query.plan.cascades.AccessHint; +import com.apple.foundationdb.record.query.plan.cascades.GraphExpansion; import com.apple.foundationdb.tuple.Tuple; import com.google.protobuf.Descriptors; @@ -102,6 +104,20 @@ public CompletableFuture loadByPrimaryKeyAsync(FDBRecordStor @Nonnull public abstract CompletableFuture loadByPrimaryKeyAsync(FDBRecordStore store, Tuple primaryKey, IndexOrphanBehavior orphanBehavior); + /** + * Expands this type into the graph that assembles its records, so that an index defined on it can be matched + * against a query that performs the same assembly. + * + * @param accessHint an access hint to apply to the stored records the expansion reads + * @return an unsealed expansion assembling records of this type + */ + @Nonnull + @API(API.Status.INTERNAL) + public GraphExpansion expand(@Nonnull final AccessHint accessHint) { + throw new UnsupportedOperationException("cannot expand an index defined on a " + + getClass().getSimpleName()); + } + @Override public String toString() { StringBuilder str = new StringBuilder(); diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/UnnestedRecordType.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/UnnestedRecordType.java index fb69ba74a8a..b03611a2d2c 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/UnnestedRecordType.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/UnnestedRecordType.java @@ -25,14 +25,32 @@ import com.apple.foundationdb.record.RecordMetaData; import com.apple.foundationdb.record.RecordMetaDataProto; import com.apple.foundationdb.record.logging.LogMessageKeys; +import com.apple.foundationdb.record.metadata.expressions.FieldKeyExpression; import com.apple.foundationdb.record.metadata.expressions.KeyExpression; import com.apple.foundationdb.record.metadata.expressions.LiteralKeyExpression; +import com.apple.foundationdb.record.metadata.expressions.NestingKeyExpression; import com.apple.foundationdb.record.provider.foundationdb.FDBRecordStore; import com.apple.foundationdb.record.provider.foundationdb.FDBStoredRecord; import com.apple.foundationdb.record.provider.foundationdb.FDBSyntheticRecord; import com.apple.foundationdb.record.provider.foundationdb.IndexOrphanBehavior; import com.apple.foundationdb.record.provider.foundationdb.RecordDoesNotExistException; +import com.apple.foundationdb.record.query.plan.cascades.AccessHint; +import com.apple.foundationdb.record.query.plan.cascades.Column; +import com.apple.foundationdb.record.query.plan.cascades.ExpansionVisitor; +import com.apple.foundationdb.record.query.plan.cascades.GraphExpansion; +import com.apple.foundationdb.record.query.plan.cascades.NullableArrayTypeUtils; +import com.apple.foundationdb.record.query.plan.cascades.Quantifier; +import com.apple.foundationdb.record.query.plan.cascades.Reference; +import com.apple.foundationdb.record.query.plan.cascades.expressions.ExplodeExpression; +import com.apple.foundationdb.record.query.plan.cascades.typing.Type; +import com.apple.foundationdb.record.query.plan.cascades.values.FieldValue; +import com.apple.foundationdb.record.query.plan.cascades.values.PromoteValue; +import com.apple.foundationdb.record.query.plan.cascades.values.RecordConstructorValue; +import com.apple.foundationdb.record.query.plan.cascades.values.Value; +import com.apple.foundationdb.record.util.ProtoUtils; import com.apple.foundationdb.tuple.Tuple; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import com.google.protobuf.Descriptors; import com.google.protobuf.Message; @@ -42,8 +60,10 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.concurrent.CompletableFuture; + /** * A {@linkplain SyntheticRecordType synthetic record type} representing an unnesting of some kind of * repeated, nested structure. Each record of this type should be associated with one stored record @@ -357,4 +377,84 @@ public RecordMetaDataProto.UnnestedRecordType toProto() { } return builder.build(); } + + @Nonnull + @Override + @API(API.Status.INTERNAL) + public GraphExpansion expand(@Nonnull final AccessHint accessHint) { + final RecordMetaData metaData = getRecordMetaData(); + final NestedConstituent parentConstituent = getParentConstituent(); + final RecordType parentRecordType = parentConstituent.getRecordType(); + final Quantifier.ForEach parentQuantifier = + Quantifier.forEach(ExpansionVisitor.createBaseRef(metaData.getRecordTypes().keySet(), + ImmutableSet.of(parentRecordType.getName()), + metaData.getPlannerType(parentRecordType.getName()), null, + accessHint)); + + final Map elementValuesByConstituent = new HashMap<>(); + final GraphExpansion.Builder builder = GraphExpansion.builder(); + final ImmutableList.Builder> positionColumns = ImmutableList.builder(); + for (final NestedConstituent constituent : getConstituents()) { + final Value elementValue; + if (constituent.isParent()) { + builder.addQuantifier(parentQuantifier); + elementValue = parentQuantifier.getFlowedObjectValue(); + } else { + final Value ownerElementValue = + Objects.requireNonNull(elementValuesByConstituent.get(constituent.getParentName())); + final Quantifier.ForEach constituentQuantifier = + constituentQuantifier(ownerElementValue, constituent.getNestingExpression()); + builder.addQuantifier(constituentQuantifier); + final Value flowedValue = constituentQuantifier.getFlowedObjectValue(); + elementValue = FieldValue.ofOrdinalNumber(flowedValue, 0); + final Value positionValue = PromoteValue.inject(FieldValue.ofOrdinalNumber(flowedValue, 1), + Type.primitiveType(Type.TypeCode.LONG, false)); + positionColumns.add(Column.of(Optional.of(constituent.getName()), positionValue)); + } + elementValuesByConstituent.put(constituent.getName(), elementValue); + builder.addResultColumn(Column.of(Optional.of(constituent.getName()), elementValue)); + } + builder.addResultColumn(Column.of(Optional.of(POSITIONS_FIELD), + RecordConstructorValue.ofColumns(positionColumns.build()))); + return builder.build(); + } + + /** + * Builds the quantifier standing for one constituent: a select over an explode of the constituent's array. + * + *

The explode is created {@code WITH ORDINALITY}, so it flows an anonymous {@code (element, ordinal)} struct. + * The ordinal is needed to reconstruct the {@code __positions} field of a synthetic record, without which the + * synthetic primary key cannot be expressed. It is asked for 0-based. + * + * @param ownerElementValue the record the array hangs off + * @param nestingExpression the constituent's nesting expression + * @return a quantifier flowing {@code (element, ordinal)} structs for the constituent's array + */ + @Nonnull + private static Quantifier.ForEach constituentQuantifier(@Nonnull final Value ownerElementValue, + @Nonnull final KeyExpression nestingExpression) { + final Quantifier.ForEach explodeQuantifier = + Quantifier.forEach(Reference.initialOf(new ExplodeExpression( + FieldValue.ofFieldNames(ownerElementValue, arrayFieldPath(nestingExpression)), true, true))); + return Quantifier.forEach(Reference.initialOf(GraphExpansion.ofQuantifier(explodeQuantifier) + .seal() + .buildSimpleSelectOverQuantifier(explodeQuantifier))); + } + + @Nonnull + private static List arrayFieldPath(@Nonnull final KeyExpression nestingExpression) { + final ImmutableList.Builder pathBuilder = ImmutableList.builder(); + KeyExpression current = nestingExpression; + while (current instanceof final NestingKeyExpression nesting) { + final FieldKeyExpression parent = nesting.getParent(); + pathBuilder.add(ProtoUtils.toUserIdentifier(parent.getFieldName())); + final var wrapperFanType = NullableArrayTypeUtils.matchArrayWrapper(nesting); + if (wrapperFanType.isPresent()) { + return pathBuilder.build(); + } + current = nesting.getChild(); + } + pathBuilder.add(ProtoUtils.toUserIdentifier(((FieldKeyExpression)current).getFieldName())); + return pathBuilder.build(); + } } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/KeyExpression.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/KeyExpression.java index 7eeafb07b52..dc1e17bf1b4 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/KeyExpression.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/KeyExpression.java @@ -418,11 +418,22 @@ static KeyExpression fromPath(@Nonnull final List path) { if (path.isEmpty()) { throw new InvalidExpressionException("attempt to create key expression using empty path"); } - final String fieldName = path.get(path.size() - 1); - KeyExpression keyExpression = Key.Expressions.field(fieldName); - final List fieldPrefix = path.subList(0, path.size() - 1); - for (int i = fieldPrefix.size() - 1; i >= 0; i --) { - keyExpression = Key.Expressions.field(fieldPrefix.get(i)).nest(keyExpression); + return fromPath(path.subList(0, path.size() - 1), Key.Expressions.field(path.get(path.size() - 1))); + } + + /** + * Nests an expression under a path of scalar field hops, outermost hop first. An empty path returns the expression + * unchanged. + * + * @param path the fields to navigate before reaching {@code nested} + * @param nested the expression to evaluate against the record the path ends at + * @return the resulting {@link KeyExpression} + */ + @Nonnull + static KeyExpression fromPath(@Nonnull final List path, @Nonnull final KeyExpression nested) { + KeyExpression keyExpression = nested; + for (int i = path.size() - 1; i >= 0; i--) { + keyExpression = Key.Expressions.field(path.get(i)).nest(keyExpression); } return keyExpression; } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/Compensation.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/Compensation.java index ed665eb8277..823ed427cde 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/Compensation.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/Compensation.java @@ -1031,6 +1031,17 @@ public RelationalExpression apply(@Nonnull final Memoizer memoizer, @Nonnull private CorrelationIdentifier getMatchedForEachAlias() { + return Iterables.getOnlyElement(getMatchedForEachAliases()); + } + + /** + * Returns the aliases of the matched for-each quantifiers. There is more than one exactly when the match + * stands in for a join, which is what a candidate over a synthetic record type produces. + * + * @return the aliases of the matched for-each quantifiers + */ + @Nonnull + private Set getMatchedForEachAliases() { final var matchedQuantifierMap = Quantifiers.aliasToQuantifierMap(matchedQuantifiers); @@ -1038,13 +1049,10 @@ private CorrelationIdentifier getMatchedForEachAlias() { matchedQuantifierMap.keySet(); Verify.verify(compensatedAliases.equals(matchedAliases)); - final var matchedForEachQuantifierAliases = - matchedAliases - .stream() - .filter(alias -> matchedQuantifierMap.get(alias) instanceof Quantifier.ForEach) - .collect(ImmutableSet.toImmutableSet()); - - return Iterables.getOnlyElement(matchedForEachQuantifierAliases); + return matchedAliases + .stream() + .filter(alias -> matchedQuantifierMap.get(alias) instanceof Quantifier.ForEach) + .collect(ImmutableSet.toImmutableSet()); } @Nonnull @@ -1055,7 +1063,13 @@ public RelationalExpression applyFinal(@Nonnull final Memoizer memoizer, Verify.verify(!isImpossible()); Verify.verify(resultCompensationFunction.isNeeded()); - final var matchedForEachAlias = getMatchedForEachAlias(); + // + // Any of the matched for-each aliases will do here, and when the match stands in for a join there is more + // than one with none of them distinguished. The compensated result value is expressed over the top of the + // match, the translation below re-targets it onto the alias picked here, and that alias then names the + // single new quantifier over the data access -- so the choice only decides a name. + // + final var matchedForEachAlias = Iterables.get(getMatchedForEachAliases(), 0); final var resultValue = resultCompensationFunction.applyCompensationForResult( diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/IndexExpansionInfo.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/IndexExpansionInfo.java index 86d9b64f543..11a8cb348db 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/IndexExpansionInfo.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/IndexExpansionInfo.java @@ -51,22 +51,18 @@ public final class IndexExpansionInfo { @Nonnull private final Collection indexedRecordTypes; @Nonnull - private final Set indexedRecordTypeNames; - @Nonnull private final Type.Record baseType; private IndexExpansionInfo(@Nonnull RecordMetaData metaData, @Nonnull Index index, boolean reverse, @Nonnull Collection indexedRecordTypes, - @Nonnull Set indexedRecordTypeNames, @Nonnull Type.Record baseType, @Nullable KeyExpression commonPrimaryKeyForTypes) { this.metaData = metaData; this.index = index; this.reverse = reverse; this.indexedRecordTypes = indexedRecordTypes; - this.indexedRecordTypeNames = indexedRecordTypeNames; this.baseType = baseType; this.commonPrimaryKeyForTypes = commonPrimaryKeyForTypes; } @@ -97,7 +93,9 @@ public Collection getIndexedRecordTypes() { @Nonnull public Set getIndexedRecordTypeNames() { - return indexedRecordTypeNames; + return indexedRecordTypes.stream() + .map(RecordType::getName) + .collect(ImmutableSet.toImmutableSet()); } @Nullable @@ -133,15 +131,12 @@ public static IndexExpansionInfo createInfo(@Nonnull RecordMetaData metaData, @Nonnull final Collection indexedRecordTypes = Collections.unmodifiableCollection(metaData.recordTypesForIndex(index)); @Nonnull - final Set indexedRecordTypeNames = indexedRecordTypes.stream() - .map(RecordType::getName) - .collect(ImmutableSet.toImmutableSet()); - @Nonnull - final Type.Record baseType = metaData.getPlannerType(indexedRecordTypeNames); + final Type.Record baseType = metaData.getPlannerTypeForRecordTypes(indexedRecordTypes); @Nullable final KeyExpression commonPrimaryKeyForTypes = RecordMetaData.commonPrimaryKey(indexedRecordTypes); - return new IndexExpansionInfo(metaData, index, reverse, indexedRecordTypes, indexedRecordTypeNames, baseType, commonPrimaryKeyForTypes); + return new IndexExpansionInfo(metaData, index, reverse, indexedRecordTypes, baseType, + commonPrimaryKeyForTypes); } } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/KeyExpressionExpansionVisitor.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/KeyExpressionExpansionVisitor.java index 23dcd7b5878..8b7a2be0e85 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/KeyExpressionExpansionVisitor.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/KeyExpressionExpansionVisitor.java @@ -39,6 +39,7 @@ import com.apple.foundationdb.record.query.plan.cascades.predicates.PredicateWithValueAndRanges; import com.apple.foundationdb.record.query.plan.cascades.values.EmptyValue; import com.apple.foundationdb.record.query.plan.cascades.values.FieldValue; +import com.apple.foundationdb.record.query.plan.cascades.values.RecordConstructorValue; import com.apple.foundationdb.record.query.plan.cascades.values.Value; import com.apple.foundationdb.record.util.ProtoUtils; import com.google.common.base.Verify; @@ -441,7 +442,63 @@ public GraphExpansion visitExpression(@Nonnull final ThenKeyExpression thenKeyEx @Nonnull @Override public GraphExpansion visitExpression(@Nonnull final ListKeyExpression listKeyExpression) { - throw new UnsupportedOperationException("visitor method for this key expression is not implemented"); + final ImmutableList.Builder expandedPredicatesBuilder = ImmutableList.builder(); + final VisitorState state = getCurrentState(); + int currentOrdinal = state.getCurrentOrdinal(); + for (KeyExpression child : listKeyExpression.getChildren()) { + final VisitorState childState = state.withCurrentOrdinal(currentOrdinal); + final Value value = childState.registerValue(listChildValue(child, childState)); + expandedPredicatesBuilder.add(expansionForListChild(childState, value)); + currentOrdinal++; + } + return GraphExpansion.ofOthers(expandedPredicatesBuilder.build()); + } + + /** + * Computes the single {@link Value} a child of a {@link ListKeyExpression} contributes. The child is expanded as an + * internal expansion so that it neither registers its own values nor creates its own placeholders, and the values + * of the columns it yields are then collapsed into one. + * + * @param child the child of the list + * @param state the state to expand the child under, whose ordinal is the child's key position + * @return the value the child contributes at its position + */ + @Nonnull + private Value listChildValue(@Nonnull final KeyExpression child, @Nonnull final VisitorState state) { + final GraphExpansion childExpansion = pop(child.expand(push(state.forFunctionalExpansion()))); + if (!childExpansion.getQuantifiers().isEmpty()) { + throw new UnsupportedOperationException("cannot expand a list whose child introduces quantifiers"); + } + final var childValues = childExpansion.getResultColumns() + .stream() + .map(Column::getValue) + .collect(ImmutableList.toImmutableList()); + if (childValues.size() == 1) { + return Iterables.getOnlyElement(childValues); + } + return RecordConstructorValue.ofUnnamed(childValues); + } + + /** + * Emits the expansion for one child of a {@link ListKeyExpression}, mirroring what + * {@link #visitExpression(FieldKeyExpression)} does for a scalar field. + * + * @param state the state whose ordinal is the child's key position + * @param value the value the child contributes + * @return the expansion contributed by that child + */ + @Nonnull + private GraphExpansion expansionForListChild(@Nonnull final VisitorState state, @Nonnull final Value value) { + final boolean isSargable = state.isKey() && !state.isInternalExpansion(); + if (state.isSelectStar()) { + return isSargable + ? GraphExpansion.ofPlaceholder(value.asPlaceholder(newParameterAlias())) + : GraphExpansion.empty(); + } + final Column column = Column.unnamedOf(value); + return isSargable + ? GraphExpansion.ofResultColumnAndPlaceholder(column, value.asPlaceholder(newParameterAlias())) + : GraphExpansion.ofResultColumn(column); } /** diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/MetaDataPlanContext.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/MetaDataPlanContext.java index cb3b44cd3d0..087265e3b81 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/MetaDataPlanContext.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/MetaDataPlanContext.java @@ -102,6 +102,28 @@ private static List readableOf(@Nonnull RecordStoreState recordStoreState } } + /** + * Collects the indexes defined on a {@link com.apple.foundationdb.record.metadata.SyntheticRecordType} that the + * query could use. + */ + @Nonnull + private static List syntheticIndexesOf(@Nonnull final RecordMetaData metaData, + @Nonnull final RecordStoreState recordStoreState, + @Nonnull final Collection queriedRecordTypeNames) { + final var indexes = Lists.newArrayList(); + for (final var syntheticRecordType : metaData.getSyntheticRecordTypes().values()) { + final var storedConstituentNames = + syntheticRecordType.getConstituents().stream() + .map(constituent -> constituent.getRecordType().getName()) + .filter(metaData.getRecordTypes()::containsKey) + .collect(ImmutableSet.toImmutableSet()); + if (!storedConstituentNames.isEmpty() && queriedRecordTypeNames.containsAll(storedConstituentNames)) { + indexes.addAll(readableOf(recordStoreState, syntheticRecordType.getIndexes())); + } + } + return indexes; + } + @Nonnull public static PlanContext forRecordQuery(@Nonnull RecordQueryPlannerConfiguration plannerConfiguration, @Nonnull RecordMetaData metaData, @@ -144,6 +166,7 @@ public static PlanContext forRecordQuery(@Nonnull RecordQueryPlannerConfiguratio } indexList.addAll(readableOf(recordStoreState, metaData.getUniversalIndexes())); + indexList.addAll(syntheticIndexesOf(metaData, recordStoreState, queriedRecordTypeNames)); } finally { recordStoreState.endRead(); } @@ -187,6 +210,7 @@ public static PlanContext forRootReference(@Nonnull final RecordQueryPlannerConf for (final var recordType : queriedRecordTypes) { indexList.addAll(readableOf(recordStoreState, recordType.getAllIndexes())); } + indexList.addAll(syntheticIndexesOf(metaData, recordStoreState, queriedRecordTypeNames)); } finally { recordStoreState.endRead(); } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/ScalarTranslationVisitor.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/ScalarTranslationVisitor.java index e0ba7a9b0f8..4e74d123608 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/ScalarTranslationVisitor.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/ScalarTranslationVisitor.java @@ -34,6 +34,7 @@ import com.apple.foundationdb.record.query.plan.cascades.values.EmptyValue; import com.apple.foundationdb.record.query.plan.cascades.values.FieldValue; import com.apple.foundationdb.record.query.plan.cascades.values.QuantifiedObjectValue; +import com.apple.foundationdb.record.query.plan.cascades.values.RecordConstructorValue; import com.apple.foundationdb.record.query.plan.cascades.values.Value; import com.apple.foundationdb.record.util.ProtoUtils; import com.google.common.collect.ImmutableList; @@ -203,7 +204,29 @@ public Value visitExpression(@Nonnull final ThenKeyExpression thenKeyExpression) @Nonnull @Override public Value visitExpression(@Nonnull final ListKeyExpression listKeyExpression) { - throw new UnsupportedOperationException("visitor method for this key expression is not implemented"); + // A list places each child into its own nested tuple, so a list of more than one child spans more than one + // key position and cannot be scalar. `normalizeKeyForPositions()` splits a longer list into single-child + // lists, which is the shape that reaches here. + if (listKeyExpression.getColumnSize() > 1) { + throw new RecordCoreException("cannot expand ListKeyExpression in scalar expansion"); + } + + final ScalarVisitorState state = getCurrentState(); + final KeyExpression child = Iterables.getOnlyElement(listKeyExpression.getChildren()); + + // The child occupies a single position whose value is the nested tuple of the child's own columns, so a + // multi-column child collapses into one record. This has to agree with what + // `KeyExpressionExpansionVisitor#visitExpression(ListKeyExpression)` registers for the same position, since + // the ordering parts computed from these values are matched to the candidate's parameters by ordinal. + final List positions = child.normalizeKeyForPositions(); + if (positions.size() == 1) { + return pop(Iterables.getOnlyElement(positions).expand(push(state))); + } + final ImmutableList.Builder valuesBuilder = ImmutableList.builder(); + for (final KeyExpression position : positions) { + valuesBuilder.add(pop(position.expand(push(state)))); + } + return RecordConstructorValue.ofUnnamed(valuesBuilder.build()); } @Nonnull diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/ScanWithFetchMatchCandidate.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/ScanWithFetchMatchCandidate.java index 19901f4662d..91755853966 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/ScanWithFetchMatchCandidate.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/ScanWithFetchMatchCandidate.java @@ -22,6 +22,8 @@ import com.apple.foundationdb.record.EvaluationContext; import com.apple.foundationdb.record.metadata.RecordType; +import com.apple.foundationdb.record.metadata.expressions.KeyExpression; +import com.apple.foundationdb.record.metadata.expressions.ListKeyExpression; import com.apple.foundationdb.record.query.plan.IndexKeyValueToPartialRecord; import com.apple.foundationdb.record.query.plan.cascades.typing.Type; import com.apple.foundationdb.record.query.plan.cascades.values.FieldValue; @@ -138,7 +140,8 @@ static Optional computeIn @Nonnull final CorrelationIdentifier baseAlias, @Nonnull final Type baseType, @Nonnull final List indexKeyValues, - @Nonnull final List indexValueValues) { + @Nonnull final List indexValueValues, + @Nonnull final List normalizedKeyExpressions) { if (queriedRecordTypes.size() > 1) { return Optional.empty(); } @@ -152,7 +155,8 @@ static Optional computeIn final var extractFromIndexEntryPairOptional = keyValue.extractFromIndexEntryMaybe(baseObjectValue, EvaluationContext.empty(), AliasMap.emptyMap(), - ImmutableSet.of(), IndexKeyValueToPartialRecord.TupleSource.KEY, ImmutableIntArray.of(i)); + ImmutableSet.of(), IndexKeyValueToPartialRecord.TupleSource.KEY, + keyOrdinalPath(normalizedKeyExpressions, i)); if (extractFromIndexEntryPairOptional.isPresent()) { final var extractFromIndexEntryPair = extractFromIndexEntryPairOptional.get(); if (!addCoveringField(builder, extractFromIndexEntryPair.getKey(), @@ -192,6 +196,28 @@ static Optional computeIn indexEntryToRecordValue(baseType, covered))); } + /** + * Computes the path into the index entry's key tuple at which the data for key position {@code ordinal} is found. + * + *

For almost every {@link KeyExpression} that is a single element, {@code tuple.get(ordinal)}. A + * {@link ListKeyExpression} is the exception: it places each of its children into a nested tuple of its own rather + * than flattening them, so the datum sits one level deeper. {@code normalizeKeyForPositions()} yields single-child + * lists, so that level is entered at ordinal zero. + * + * @param normalizedKeyExpressions the positions of the full key, aligned with the candidate's key values + * @param ordinal the key position + * @return the ordinal path to extract that position with + */ + @Nonnull + private static ImmutableIntArray keyOrdinalPath(@Nonnull final List normalizedKeyExpressions, + final int ordinal) { + if (ordinal < normalizedKeyExpressions.size() && + normalizedKeyExpressions.get(ordinal) instanceof ListKeyExpression) { + return ImmutableIntArray.of(ordinal, 0); + } + return ImmutableIntArray.of(ordinal); + } + /** * Records what the given extraction covers, descending a node per field of the path it fills, which may run into * nested messages. Every name is present here, {@code addCoveringField} having already refused the extraction diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/ValueIndexExpansionVisitor.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/ValueIndexExpansionVisitor.java index 163274cc4d7..31bb33b2a78 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/ValueIndexExpansionVisitor.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/ValueIndexExpansionVisitor.java @@ -24,6 +24,7 @@ import com.apple.foundationdb.record.metadata.Index; import com.apple.foundationdb.record.metadata.IndexTypes; import com.apple.foundationdb.record.metadata.RecordType; +import com.apple.foundationdb.record.metadata.SyntheticRecordType; import com.apple.foundationdb.record.metadata.expressions.GroupingKeyExpression; import com.apple.foundationdb.record.metadata.expressions.KeyExpression; import com.apple.foundationdb.record.metadata.expressions.KeyWithValueExpression; @@ -32,9 +33,12 @@ import com.apple.foundationdb.record.query.plan.cascades.predicates.Placeholder; import com.apple.foundationdb.record.query.plan.cascades.predicates.PredicateWithValueAndRanges; import com.apple.foundationdb.record.query.plan.cascades.typing.Type; +import com.apple.foundationdb.record.query.plan.cascades.values.RecordConstructorValue; import com.apple.foundationdb.record.query.plan.cascades.values.Value; +import com.apple.foundationdb.record.query.plan.cascades.values.translation.TranslationMap; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; import com.google.common.collect.Lists; import javax.annotation.Nonnull; @@ -43,6 +47,7 @@ import java.util.Collection; import java.util.List; import java.util.Objects; +import java.util.Optional; import java.util.Set; import static com.apple.foundationdb.record.metadata.Key.Expressions.concat; @@ -72,6 +77,68 @@ public ValueIndexExpansionVisitor(@Nonnull Index index, @Nonnull Collection> syntheticRecordTypeMaybe( + @Nonnull final Collection indexedRecordTypes) { + if (indexedRecordTypes.size() != 1) { + return Optional.empty(); + } + final RecordType indexedRecordType = Iterables.getOnlyElement(indexedRecordTypes); + return indexedRecordType instanceof final SyntheticRecordType syntheticRecordType + ? Optional.of(syntheticRecordType) + : Optional.empty(); + } + + /** + * Returns the record a base expansion assembles -- a value of the synthetic type it stands for. + */ + @Nonnull + private static Value recordOf(@Nonnull final GraphExpansion baseExpansion) { + return RecordConstructorValue.ofColumns(baseExpansion.getResultColumns()); + } + + /** + * Merges the expansion of an index key into the expansion of the synthetic type the index is defined on. + * + *

The key's values are expressed over {@code baseQuantifier}, which does not appear in the resulting graph, so + * every predicate is re-expressed over the record the base expansion assembles. Simplification then composes the + * navigations away -- {@code base.unnesting_0.reviewer} over + * {@code (parent AS parent, explode._0 AS unnesting_0, ...)} becomes {@code explode._0.reviewer}, which is exactly + * the value a query navigating the same unnesting flows. + * + *

Predicates and placeholders have to be translated separately because a placeholder is held in both lists, and + * {@link GraphExpansion#seal()} pairs the two up by comparing their values. Translating only one of them would drop + * the placeholder from the select's predicates while leaving it among the candidate's parameters. + * + * @param baseExpansion the expansion assembling the synthetic type's records + * @param keyExpansion the expansion of the index key, expressed over {@code baseQuantifier} + * @param baseQuantifier the quantifier {@code keyExpansion} is expressed over + * @return a single expansion holding the base expansion's quantifiers and the key's placeholders + */ + @Nonnull + private static GraphExpansion mergeIntoBase(@Nonnull final GraphExpansion baseExpansion, + @Nonnull final GraphExpansion keyExpansion, + @Nonnull final Quantifier.ForEach baseQuantifier) { + final var ontoBaseRecord = TranslationMap.regularBuilder() + .when(baseQuantifier.getAlias()).then((sourceAlias, leafValue) -> recordOf(baseExpansion)) + .build(); + return GraphExpansion.ofOthers( + // the base expansion's result columns live on as the select's result value, and + // buildSelectWithResultValue() insists on there being no result columns of its own + baseExpansion.toBuilder().removeAllResultColumns().build(), + keyExpansion.toBuilder() + .removeAllPredicates() + .addAllPredicates(keyExpansion.getPredicates().stream() + .map(predicate -> predicate.translateCorrelations(ontoBaseRecord, true)) + .collect(ImmutableList.toImmutableList())) + .replacePlaceholder(placeholder -> + placeholder.translateLeafPredicate(ontoBaseRecord, true)) + .build()); + } + @Nonnull @Override public MatchCandidate expand(@Nonnull final Set availableRecordTypeNames, @@ -87,12 +154,18 @@ public MatchCandidate expand(@Nonnull final Set availableRecordTypeNames // the instantiation of the type filter below to create a placeholder for the record type key parameter // alias and reuse it here. Similar to what we currently do for primary scans. // - final var baseQuantifier = Quantifier.forEach(ExpansionVisitor.createBaseRef(availableRecordTypeNames, - queriedRecordTypeNames, baseType, null, accessHint)); + @Nullable final GraphExpansion baseExpansion = + syntheticRecordTypeMaybe(queriedRecordTypes).map(recordType -> recordType.expand(accessHint)).orElse(null); + final Quantifier.ForEach baseQuantifier = baseExpansion == null + ? Quantifier.forEach(ExpansionVisitor.createBaseRef(availableRecordTypeNames, + queriedRecordTypeNames, baseType, null, accessHint)) + : Quantifier.forEach(Reference.initialOf(baseExpansion.seal().buildSelect())); final var allExpansionsBuilder = ImmutableList.builder(); - // add the value for the flow of records - allExpansionsBuilder.add(GraphExpansion.ofQuantifier(baseQuantifier)); + if (baseExpansion == null) { + // add the value for the flow of records + allExpansionsBuilder.add(GraphExpansion.ofQuantifier(baseQuantifier)); + } var rootExpression = index.getRootExpression(); @@ -105,8 +178,7 @@ public MatchCandidate expand(@Nonnull final Set availableRecordTypeNames } final int keyValueSplitPoint; - if (rootExpression instanceof KeyWithValueExpression) { - final KeyWithValueExpression keyWithValueExpression = (KeyWithValueExpression)rootExpression; + if (rootExpression instanceof final KeyWithValueExpression keyWithValueExpression) { keyValueSplitPoint = keyWithValueExpression.getSplitPoint(); rootExpression = keyWithValueExpression.getInnerKey(); } else { @@ -191,14 +263,19 @@ public MatchCandidate expand(@Nonnull final Set availableRecordTypeNames } final var completeExpansion = GraphExpansion.ofOthers(allExpansionsBuilder.build()); - final var sealedExpansion = completeExpansion.seal(); + final var sealedExpansion = baseExpansion == null + ? completeExpansion.seal() + : mergeIntoBase(baseExpansion, completeExpansion, baseQuantifier).seal(); + final var baseObjectValue = baseExpansion == null + ? baseQuantifier.getFlowedObjectValue() + : recordOf(baseExpansion); final var parameters = sealedExpansion.getPlaceholders() .stream() .map(Placeholder::getParameterAlias) .collect(ImmutableList.toImmutableList()); final var matchableSortExpression = new MatchableSortExpression(parameters, isReverse, - sealedExpansion.buildSelectWithResultValue(baseQuantifier.getFlowedObjectValue())); + sealedExpansion.buildSelectWithResultValue(baseObjectValue)); return new ValueIndexScanMatchCandidate(index, queriedRecordTypes, Traversal.withRoot(Reference.initialOf(matchableSortExpression)), diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/ValueIndexScanMatchCandidate.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/ValueIndexScanMatchCandidate.java index a7e3d90f5d5..89e1886174a 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/ValueIndexScanMatchCandidate.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/ValueIndexScanMatchCandidate.java @@ -133,7 +133,8 @@ public ValueIndexScanMatchCandidate(@Nonnull final Index index, Suppliers.memoize(() -> MatchCandidate.computePrimaryKeyValuesMaybe(primaryKey, baseType)); this.indexEntryToLogicalRecordOptionalSupplier = Suppliers.memoize(() -> ScanWithFetchMatchCandidate.computeIndexEntryToLogicalRecord(queriedRecordTypes, - baseAlias, baseType, indexKeyValues, indexValueValues)); + baseAlias, baseType, indexKeyValues, indexValueValues, + fullKeyExpression.normalizeKeyForPositions())); } @Override diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/WindowedIndexScanMatchCandidate.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/WindowedIndexScanMatchCandidate.java index 13be511c3f4..30f7b984c8d 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/WindowedIndexScanMatchCandidate.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/WindowedIndexScanMatchCandidate.java @@ -159,7 +159,8 @@ public WindowedIndexScanMatchCandidate(@Nonnull Index index, this.primaryKeyValuesSupplier = Suppliers.memoize(() -> MatchCandidate.computePrimaryKeyValuesMaybe(primaryKey, baseType)); this.indexEntryToLogicalRecordOptionalSupplier = Suppliers.memoize(() -> ScanWithFetchMatchCandidate.computeIndexEntryToLogicalRecord(queriedRecordTypes, - baseAlias, baseType, indexKeyValues, ImmutableList.of())); + baseAlias, baseType, indexKeyValues, ImmutableList.of(), + fullKeyExpression.normalizeKeyForPositions())); } @Override diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/expressions/ExplodeExpression.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/expressions/ExplodeExpression.java index 5d4f9faaf69..a0f6e95b7fa 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/expressions/ExplodeExpression.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/expressions/ExplodeExpression.java @@ -23,6 +23,7 @@ import com.apple.foundationdb.annotation.API; import com.apple.foundationdb.record.EvaluationContext; import com.apple.foundationdb.record.query.plan.cascades.AliasMap; +import com.apple.foundationdb.record.query.plan.cascades.Column; import com.apple.foundationdb.record.query.plan.cascades.ComparisonRange; import com.apple.foundationdb.record.query.plan.cascades.Compensation; import com.apple.foundationdb.record.query.plan.cascades.CorrelationIdentifier; @@ -30,12 +31,15 @@ import com.apple.foundationdb.record.query.plan.cascades.MatchInfo; import com.apple.foundationdb.record.query.plan.cascades.PartialMatch; import com.apple.foundationdb.record.query.plan.cascades.Quantifier; +import com.apple.foundationdb.record.query.plan.cascades.Quantifiers; import com.apple.foundationdb.record.query.plan.cascades.explain.InternalPlannerGraphRewritable; import com.apple.foundationdb.record.query.plan.cascades.explain.PlannerGraph; import com.apple.foundationdb.record.query.plan.cascades.typing.Type; import com.apple.foundationdb.record.query.plan.cascades.values.FieldValue; import com.apple.foundationdb.record.query.plan.cascades.values.QueriedValue; +import com.apple.foundationdb.record.query.plan.cascades.values.RecordConstructorValue; import com.apple.foundationdb.record.query.plan.cascades.values.Value; +import com.apple.foundationdb.record.query.plan.cascades.values.translation.MaxMatchMap; import com.apple.foundationdb.record.query.plan.cascades.values.translation.PullUp; import com.apple.foundationdb.record.query.plan.cascades.values.translation.TranslationMap; import com.google.common.base.Verify; @@ -85,6 +89,12 @@ public class ExplodeExpression extends AbstractRelationalExpressionWithoutChildr @Nonnull private final Type explodeResultType; + /** + * The result value of the explode. + */ + @Nonnull + private final Value resultValue; + public ExplodeExpression(@Nonnull final Value collectionValue, final boolean withOrdinality, final boolean zeroBasedOrdinality) { Verify.verify(withOrdinality || !zeroBasedOrdinality, "cannot base ordinals that are not produced"); @@ -94,6 +104,8 @@ public ExplodeExpression(@Nonnull final Value collectionValue, final boolean wit Verify.verify(collectionValue.getResultType().isArray()); this.elementType = Objects.requireNonNull(((Type.Array)collectionValue.getResultType()).getElementType()); this.explodeResultType = explodeResultType(elementType, withOrdinality); + this.resultValue = explodeResultValue(elementType, withOrdinality); + Verify.verify(explodeResultType.equals(resultValue.getResultType())); } public ExplodeExpression(@Nonnull final Value collectionValue, final boolean withOrdinality) { @@ -135,10 +147,39 @@ public Type getExplodeResultType() { return explodeResultType; } + /** + * Returns the value an explode of {@code elementType} flows. For the plain variant that is an opaque + * {@link QueriedValue} standing for the element. For the {@code WITH ORDINALITY} variant it is a + * {@link RecordConstructorValue} of two such values, the element and the ordinal, rather than a single opaque value + * of the struct type. + * + * @param elementType the element type of the collection being exploded + * @param withOrdinality whether ordinals are produced alongside the elements + * @return the value flowed by such an explode + */ + @Nonnull + public static Value explodeResultValue(@Nonnull final Type elementType, final boolean withOrdinality) { + final var elementValue = new QueriedValue(elementType); + if (!withOrdinality) { + return elementValue; + } + // Note: the element must stay the first column. `MaxMatchMap` returns the first reachable candidate value that + // compares equal, and a `QueriedValue` has no identity beyond its class and result type, so an element whose + // type is also a non-nullable `INT` -- the ordinal's type -- would just as happily match the ordinal if the + // ordinal came first. + // + // The record is built nullable because `explodeResultType` declares it that way, and the constructor verifies + // that the two agree: that declared type is what the plan looks its protobuf descriptor up by at run time. + return RecordConstructorValue.ofColumns( + ImmutableList.of(Column.unnamedOf(elementValue), + Column.unnamedOf(new QueriedValue(Type.primitiveType(Type.TypeCode.INT, false)))), + true); + } + @Nonnull @Override public Value getResultValue() { - return new QueriedValue(getExplodeResultType()); + return resultValue; } @Nonnull @@ -219,10 +260,41 @@ public Iterable subsumedBy(@Nonnull final RelationalExpression candid if (!isCompatiblyAndCompletelyBound(bindingAliasMap, candidateExpression.getQuantifiers())) { return ImmutableList.of(); } - + if (!withOrdinality + && candidateExpression instanceof final ExplodeExpression candidateExplodeExpression + && candidateExplodeExpression.isWithOrdinality()) { + return subsumedByWithOrdinality(candidateExplodeExpression, bindingAliasMap, partialMatchMap); + } return exactlySubsumedBy(candidateExpression, bindingAliasMap, partialMatchMap, TranslationMap.empty()); } + /** + * Establishes that an explode without ordinality is subsumed by an explode with ordinality over + * the same collection. The candidate emits one {@code (element, ordinal)} struct per element this expression + * emits just element. That satisfies subsumption: the candidate produces at least everything the query may produce. + * This case cannot be dealt with by {@link #exactlySubsumedBy}, whose {@code equalsWithoutChildren} compares + * {@link #isWithOrdinality()}. + * + * @param candidateExpression the candidate explode, which must be {@code WITH ORDINALITY} + * @param bindingAliasMap a map of aliases defining the equivalence between quantifiers + * @param partialMatchMap a map from quantifier to the {@link PartialMatch} pulled up along that quantifier + * @return an iterable containing a {@link MatchInfo} if subsumption holds, empty otherwise + */ + @Nonnull + private Iterable subsumedByWithOrdinality(@Nonnull final ExplodeExpression candidateExpression, + @Nonnull final AliasMap bindingAliasMap, + @Nonnull final IdentityBiMap partialMatchMap) { + if (!collectionValue.semanticEquals(candidateExpression.getCollectionValue(), bindingAliasMap)) { + return ImmutableList.of(); + } + final var maxMatchMap = + MaxMatchMap.compute(getResultValue(), candidateExpression.getResultValue(), + Quantifiers.aliases(candidateExpression.getQuantifiers())); + return MatchInfo.RegularMatchInfo.tryFromMatchMap(bindingAliasMap, partialMatchMap, maxMatchMap) + .map(ImmutableList::of) + .orElse(ImmutableList.of()); + } + @Nonnull @Override public Compensation compensate(@Nonnull final PartialMatch partialMatch, diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/expressions/SelectExpression.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/expressions/SelectExpression.java index adff74cce78..ec2bad8ae5f 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/expressions/SelectExpression.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/expressions/SelectExpression.java @@ -911,16 +911,25 @@ public Compensation compensate(@Nonnull final PartialMatch partialMatch, } // - // We now know we need compensation, and if we have more than one quantifier, we would have to translate - // the references of the values from the query graph to values operating on the MQT in order to do that - // compensation. We cannot do that (yet). If we, however, do not have to worry about compensation we just do - // this select entirely with the scan. + // We now know we need compensation. If more than one for-each quantifier was matched, reapplying that + // compensation means translating references from the query graph onto values over the MQT. For predicates and + // for unmatched quantifiers we cannot do that (yet), because such a reference may reach a quantifier whose + // correspondence to the candidate was never established. + // + // For the result value we can: `computeResultCompensation` above already expressed it over the root of the + // match. That is the case a candidate over a synthetic record type produces -- the match stands in for the + // whole join, and all that is left is to project the query's result out of the record the scan flows. // final var partialMatchMap = regularMatchInfo.getPartialMatchMap(); - if (quantifiers.stream() - .filter(quantifier -> quantifier instanceof Quantifier.ForEach && - partialMatchMap.containsKeyUnwrapped(quantifier)) - .count() > 1) { + final var isResultCompensationOnly = + !childCompensation.isNeeded() && + unmatchedQuantifiers.isEmpty() && + !isAnyCompensationFunctionNeeded; + if (!isResultCompensationOnly && + quantifiers.stream() + .filter(quantifier -> quantifier instanceof Quantifier.ForEach && + partialMatchMap.containsKeyUnwrapped(quantifier)) + .count() > 1) { return Compensation.impossibleCompensation(); } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/matching/graph/BaseMatcher.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/matching/graph/BaseMatcher.java index 77a921e6efe..9a62aa502fb 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/matching/graph/BaseMatcher.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/matching/graph/BaseMatcher.java @@ -270,17 +270,21 @@ protected Iterable match(@Nonnull final EnumerationFunction enumeratio return IterableHelpers .flatMap(otherCombinationsIterable, otherCombination -> { + // other permutation -> filtered by current combination final List otherFilteredPermutation = otherPermutation .stream() .filter(otherCombination::contains) .collect(ImmutableList.toImmutableList()); + // this' combination -> bound to the size of others' size final Iterable> combinationsIterable = isCompleteMatchesOnly ? ImmutableList.of(getAliases()) : soundCombinations(getAliases(), getDependsOnMap(), otherCombination.size(), otherCombination.size()); // limit to the other combination's size + // at this point, we have any 1 permutation of other, and all combinations of this. + return IterableHelpers.flatMap(combinationsIterable, combination -> { final EnumeratingIterable permutationsIterable = diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/predicates/Placeholder.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/predicates/Placeholder.java index a43f08c8482..edd544d06ff 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/predicates/Placeholder.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/predicates/Placeholder.java @@ -113,7 +113,7 @@ public Placeholder withExtraRanges(@Nonnull final Set ranges) @Nonnull @Override public Placeholder translateLeafPredicate(@Nonnull final TranslationMap translationMap, final boolean shouldSimplifyValues) { - return new Placeholder(getValue().translateCorrelations(translationMap), + return new Placeholder(getValue().translateCorrelations(translationMap, shouldSimplifyValues), getRanges().stream() .map(range -> range.translateCorrelations(translationMap, shouldSimplifyValues)) .collect(ImmutableSet.toImmutableSet()), getParameterAlias()); diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/rules/AbstractDataAccessRule.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/rules/AbstractDataAccessRule.java index 0482fb55d1a..24dee0f8d30 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/rules/AbstractDataAccessRule.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/rules/AbstractDataAccessRule.java @@ -68,7 +68,6 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.ImmutableSetMultimap; -import com.google.common.collect.Iterables; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import org.slf4j.Logger; @@ -157,7 +156,7 @@ public void onMatch(@Nonnull final CascadesRuleCall call) { final var aliasToQuantifierMap = Quantifiers.aliasToQuantifierMap(expression.getQuantifiers()); final var aliases = aliasToQuantifierMap.keySet(); - // group all successful matches by their sets of compensated aliases + // group all successful matches by the set of for-each quantifiers they compensate final var matchPartitionByMatchAliasMap = completeMatches .stream() @@ -171,10 +170,14 @@ public void onMatch(@Nonnull final CascadesRuleCall call) { .filter(matchedAlias -> Objects.requireNonNull(aliasToQuantifierMap.get(matchedAlias)) instanceof Quantifier.ForEach) .collect(ImmutableSet.toImmutableSet()); - if (matchedForEachAliases.size() == 1) { - return Stream.of(NonnullPair.of(Iterables.getOnlyElement(matchedForEachAliases), match)); + if (matchedForEachAliases.isEmpty()) { + // + // The match covers only existential quantifiers, so it does not account for anything + // this expression flows and cannot stand in for it. + // + return Stream.empty(); } - return Stream.empty(); + return Stream.of(NonnullPair.of(matchedForEachAliases, match)); }) .collect(Collectors.groupingBy( Pair::getLeft, @@ -203,11 +206,11 @@ public void onMatch(@Nonnull final CascadesRuleCall call) { ImmutableList.toImmutableList())); // - // Note that this works because there is only one for-each and potentially 0 - n existential quantifiers - // that are covered by the match partition. Even though that logically forms a join, the existential - // quantifiers do not mutate the result of the join, they only cause filtering, that is, the resulting - // record is exactly what the for each quantifier produced filtered by the predicates expressed on the - // existential quantifiers. + // Note that the matches in this partition all compensate the same set of for-each quantifiers, plus + // potentially 0 - n existential quantifiers. A match may therefore stand in for a join, which is the case + // for a candidate over a synthetic record type. That is sound because a match reaching here compensates + // every quantifier the expression owns, so the data access accounts for the whole expression rather than + // for one of its legs; existential quantifiers only cause filtering and do not contribute to the result. // for (final var matchPartitionEntry : matchPartitionsForAliasesByPredicates.entrySet()) { final var matchPartition = matchPartitionEntry.getValue(); diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/QueriedValue.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/QueriedValue.java index ae231efd462..e0d6a895834 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/QueriedValue.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/QueriedValue.java @@ -29,6 +29,7 @@ import com.apple.foundationdb.record.planprotos.PQueriedValue; import com.apple.foundationdb.record.planprotos.PValue; import com.apple.foundationdb.record.query.plan.cascades.AliasMap; +import com.apple.foundationdb.record.query.plan.cascades.ConstrainedBoolean; import com.apple.foundationdb.record.query.plan.explain.ExplainTokens; import com.apple.foundationdb.record.query.plan.explain.ExplainTokensWithPrecedence; import com.apple.foundationdb.record.query.plan.cascades.typing.Type; @@ -90,6 +91,16 @@ public boolean isFunctionallyDependentOn(@Nonnull final Value otherValue) { return false; } + /** + * Two queried values are equal if they stand for streams of the same type. + */ + @Nonnull + @Override + public ConstrainedBoolean equalsWithoutChildren(@Nonnull final Value other) { + return LeafValue.super.equalsWithoutChildren(other) + .filter(ignored -> resultType.equals(((QueriedValue)other).resultType)); + } + @Override public int hashCodeWithoutChildren() { return PlanHashable.objectPlanHash(PlanHashable.CURRENT_FOR_CONTINUATION, BASE_HASH); diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryExplodePlan.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryExplodePlan.java index 8f4a09e8a2d..32a5064356e 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryExplodePlan.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryExplodePlan.java @@ -46,7 +46,6 @@ import com.apple.foundationdb.record.query.plan.cascades.expressions.RelationalExpression; import com.apple.foundationdb.record.query.plan.cascades.typing.Type; import com.apple.foundationdb.record.query.plan.cascades.typing.TypeRepository; -import com.apple.foundationdb.record.query.plan.cascades.values.QueriedValue; import com.apple.foundationdb.record.query.plan.cascades.values.RecordConstructorValue; import com.apple.foundationdb.record.query.plan.cascades.values.Value; import com.apple.foundationdb.record.query.plan.cascades.values.translation.TranslationMap; @@ -267,7 +266,7 @@ public Type getExplodeResultType() { @Nonnull @Override public Value getResultValue() { - return new QueriedValue(getExplodeResultType()); + return ExplodeExpression.explodeResultValue(elementType, withOrdinality); } @Nonnull diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/metadata/UnnestedRecordTypeExpansionTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/metadata/UnnestedRecordTypeExpansionTest.java new file mode 100644 index 00000000000..d81e118facd --- /dev/null +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/metadata/UnnestedRecordTypeExpansionTest.java @@ -0,0 +1,419 @@ +/* + * UnnestedRecordTypeExpansionTest.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2015-2026 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apple.foundationdb.record.metadata; + +import com.apple.foundationdb.record.RecordMetaData; +import com.apple.foundationdb.record.RecordMetaDataBuilder; +import com.apple.foundationdb.record.TestRecords4WrapperProto; +import com.apple.foundationdb.record.TestRecordsNestedChainProto; +import com.apple.foundationdb.record.TestRecordsNestedMapProto; +import com.apple.foundationdb.record.metadata.expressions.KeyExpression; +import com.apple.foundationdb.record.metadata.expressions.KeyExpression.FanType; +import com.apple.foundationdb.record.query.plan.cascades.AccessHint; +import com.apple.foundationdb.record.query.plan.cascades.Column; +import com.apple.foundationdb.record.query.plan.cascades.GraphExpansion; +import com.apple.foundationdb.record.query.plan.cascades.PrimaryAccessHint; +import com.apple.foundationdb.record.query.plan.cascades.Quantifier; +import com.apple.foundationdb.record.query.plan.cascades.expressions.ExplodeExpression; +import com.apple.foundationdb.record.query.plan.cascades.expressions.FullUnorderedScanExpression; +import com.apple.foundationdb.record.query.plan.cascades.expressions.LogicalTypeFilterExpression; +import com.apple.foundationdb.record.query.plan.cascades.expressions.RelationalExpression; +import com.apple.foundationdb.record.query.plan.cascades.expressions.SelectExpression; +import com.apple.foundationdb.record.query.plan.cascades.typing.PseudoField; +import com.apple.foundationdb.record.query.plan.cascades.typing.Type; +import com.apple.foundationdb.record.query.plan.cascades.values.FieldValue; +import com.apple.foundationdb.record.query.plan.cascades.values.Value; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.protobuf.Descriptors; +import org.junit.jupiter.api.Test; + +import javax.annotation.Nonnull; +import java.util.List; +import java.util.Optional; +import java.util.function.Consumer; +import java.util.stream.Collectors; + +import static com.apple.foundationdb.record.metadata.Key.Expressions.field; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.instanceOf; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Tests of {@link UnnestedRecordType#expand(AccessHint)} and of the planner types the expansion is built from. All of + * this is metadata-to-graph work, so none of these tests needs a record store. + */ +class UnnestedRecordTypeExpansionTest { + @Nonnull + private static final String OUTER = "OuterRecord"; + @Nonnull + private static final String OTHER = "OtherRecord"; + @Nonnull + private static final String PARENT = "parent"; + @Nonnull + private static final String UNNESTED_MAP = "UnnestedMap"; + @Nonnull + private static final String TWO_UNNESTED_MAPS = "TwoUnnestedMaps"; + @Nonnull + private static final String NESTED_CHAIN = "NestedChain"; + @Nonnull + private static final String UNNESTED_REVIEWS = "UnnestedReviews"; + @Nonnull + private static final String ESCAPED_NAMES = "EscapedNames"; + @Nonnull + private static final String OUTER_OTHER_JOINED = "OuterOtherJoined"; + @Nonnull + private static final Descriptors.Descriptor INNER_DESCRIPTOR = + TestRecordsNestedChainProto.OuterRecord.MiddleRecord.InnerRecord.getDescriptor(); + @Nonnull + private static final KeyExpression ENTRIES_FAN_OUT = field("map").nest(field("entry", FanType.FanOut)); + @Nonnull + private static final AccessHint ACCESS_HINT = new PrimaryAccessHint(); + + @Test + void expandUnnestsAConstituentWithOrdinality() { + final RecordMetaData metaData = mapMetaData(metaDataBuilder -> { + final UnnestedRecordTypeBuilder typeBuilder = metaDataBuilder.addUnnestedRecordType(UNNESTED_MAP); + typeBuilder.addParentConstituent(PARENT, metaDataBuilder.getRecordType(OUTER)); + typeBuilder.addNestedConstituent("map_entry", TestRecordsNestedMapProto.MapRecord.Entry.getDescriptor(), + PARENT, ENTRIES_FAN_OUT); + }); + final UnnestedRecordType type = unnestedType(metaData, UNNESTED_MAP); + + final GraphExpansion expansion = type.expand(ACCESS_HINT); + assertThat(columnNames(expansion), contains(PARENT, "map_entry", UnnestedRecordType.POSITIONS_FIELD)); + assertEquals(2, expansion.getQuantifiers().size()); + + // The parent constituent is a scan of the stored type, restricted to it and typed as the planner sees it. + final Quantifier parentQuantifier = expansion.getQuantifiers().get(0); + final LogicalTypeFilterExpression typeFilter = assertTypeFilter(parentQuantifier, OUTER); + assertEquals(metaData.getPlannerType(OUTER), typeFilter.getResultValue().getResultType()); + final FullUnorderedScanExpression scan = + (FullUnorderedScanExpression)Iterables.getOnlyElement(typeFilter.getQuantifiers()).getRangesOver().get(); + assertEquals(metaData.getRecordTypes().keySet(), scan.getRecordTypes()); + assertThat(scan.getAccessHints().getAccessHintSet(), contains(ACCESS_HINT)); + assertEquals(parentQuantifier.getFlowedObjectValue(), columnValue(expansion, PARENT)); + + // The nested constituent explodes `parent.map.entry`, with ordinality so that positions can be recovered. The + // ordinals are asked for 0-based, so that an ordinal is a position rather than one more than it. + final Quantifier entryQuantifier = expansion.getQuantifiers().get(1); + final ExplodeExpression explode = assertSelectOverExplode(entryQuantifier); + assertTrue(explode.isWithOrdinality()); + assertTrue(explode.isZeroBasedOrdinality()); + assertEquals(FieldValue.ofFieldNames(parentQuantifier.getFlowedObjectValue(), List.of("map", "entry")), + explode.getCollectionValue()); + + // The constituent's column is the element the explode flows, not the `(element, ordinal)` pair, and it is typed + // as the synthetic record type declares that constituent. + assertEquals(FieldValue.ofOrdinalNumber(entryQuantifier.getFlowedObjectValue(), 0), + columnValue(expansion, "map_entry")); + assertEquals(Type.Record.fromDescriptor(type.getDescriptor()).getFieldNameFieldMap().get("map_entry").getFieldType(), + columnValue(expansion, "map_entry").getResultType()); + } + + @Test + void expandFlowsOnePositionPerNestedConstituent() { + final RecordMetaData metaData = mapMetaData(addTwoMapsType()); + final UnnestedRecordType type = unnestedType(metaData, TWO_UNNESTED_MAPS); + + final GraphExpansion expansion = type.expand(ACCESS_HINT); + assertThat(columnNames(expansion), + contains(PARENT, "entry_one", "entry_two", UnnestedRecordType.POSITIONS_FIELD)); + assertEquals(3, expansion.getQuantifiers().size()); + + // The positions record has a field per nested constituent, and no field for the parent, which has no position. + final Value positionsValue = columnValue(expansion, UnnestedRecordType.POSITIONS_FIELD); + assertEquals(Type.Record.fromFields(false, + ImmutableList.of(longField("entry_one"), longField("entry_two"))), + positionsValue.getResultType()); + + // Each position is the ordinal its own explode flows, widened to the `LONG` the positions field declares. + for (int i = 0; i < 2; i++) { + final Quantifier constituentQuantifier = expansion.getQuantifiers().get(i + 1); + final Value positionValue = Iterables.get(positionsValue.getChildren(), i); + assertEquals(constituentQuantifier.getAlias(), + Iterables.getOnlyElement(positionValue.getCorrelatedTo())); + assertEquals(FieldValue.ofOrdinalNumber(constituentQuantifier.getFlowedObjectValue(), 1), + Iterables.getOnlyElement(positionValue.getChildren())); + } + + // Both constituents unnest the same array, but each gets its own explode, so that they range independently. + final ExplodeExpression explodeOne = assertSelectOverExplode(expansion.getQuantifiers().get(1)); + final ExplodeExpression explodeTwo = assertSelectOverExplode(expansion.getQuantifiers().get(2)); + assertNotEquals(expansion.getQuantifiers().get(1).getAlias(), expansion.getQuantifiers().get(2).getAlias()); + assertEquals(explodeOne.getCollectionValue(), explodeTwo.getCollectionValue()); + } + + @Test + void expandChainsNestedConstituentsOntoTheirOwner() { + final RecordMetaData metaData = nestedChainMetaData(addNestedChainType()); + final UnnestedRecordType type = unnestedType(metaData, NESTED_CHAIN); + + final GraphExpansion expansion = type.expand(ACCESS_HINT); + assertThat(columnNames(expansion), + contains(PARENT, "middle", "inner", "outer_inner", UnnestedRecordType.POSITIONS_FIELD)); + assertEquals(4, expansion.getQuantifiers().size()); + + final Quantifier parentQuantifier = expansion.getQuantifiers().get(0); + final Quantifier middleQuantifier = expansion.getQuantifiers().get(1); + + // `middle` and `outer_inner` hang off the stored record, ... + assertEquals(FieldValue.ofFieldNames(parentQuantifier.getFlowedObjectValue(), List.of("many_middle")), + assertSelectOverExplode(middleQuantifier).getCollectionValue()); + assertEquals(FieldValue.ofFieldNames(parentQuantifier.getFlowedObjectValue(), List.of("inner")), + assertSelectOverExplode(expansion.getQuantifiers().get(3)).getCollectionValue()); + + // ... while `inner` hangs off the element `middle` flows, which is what makes this a chain rather than a fan. + final Value innerCollectionValue = assertSelectOverExplode(expansion.getQuantifiers().get(2)) + .getCollectionValue(); + assertEquals(FieldValue.ofFieldNames( + FieldValue.ofOrdinalNumber(middleQuantifier.getFlowedObjectValue(), 0), List.of("inner")), + innerCollectionValue); + assertEquals(middleQuantifier.getAlias(), Iterables.getOnlyElement(innerCollectionValue.getCorrelatedTo())); + + // Every nested constituent still contributes its own position, chained or not. + assertEquals(Type.Record.fromFields(false, ImmutableList.of( + longField("middle"), longField("inner"), longField("outer_inner"))), + columnValue(expansion, UnnestedRecordType.POSITIONS_FIELD).getResultType()); + } + + @Test + void expandUnnestsThroughANullableArrayWrapper() { + final RecordMetaData metaData = wrapperMetaData(metaDataBuilder -> { + final UnnestedRecordTypeBuilder typeBuilder = metaDataBuilder.addUnnestedRecordType(UNNESTED_REVIEWS); + typeBuilder.addParentConstituent(PARENT, metaDataBuilder.getRecordType("RestaurantRecord")); + typeBuilder.addNestedConstituent("review", TestRecords4WrapperProto.RestaurantReview.getDescriptor(), + PARENT, field("reviews").nest(field("values", FanType.FanOut))); + }); + final UnnestedRecordType type = unnestedType(metaData, UNNESTED_REVIEWS); + + final GraphExpansion expansion = type.expand(ACCESS_HINT); + // The planner models a nullable array as an array, not as the wrapper message the records store it in, so the + // path to the exploded array stops at the wrapper field and must not descend into its repeated field. + final Value collectionValue = assertSelectOverExplode(expansion.getQuantifiers().get(1)).getCollectionValue(); + assertEquals(FieldValue.ofFieldNames(expansion.getQuantifiers().get(0).getFlowedObjectValue(), + List.of("reviews")), collectionValue); + assertEquals(Type.Record.fromDescriptor(type.getDescriptor()).getFieldNameFieldMap().get("review").getFieldType(), + columnValue(expansion, "review").getResultType()); + } + + @Test + void expandUnnestsThroughEscapedFieldNames() { + final RecordMetaData metaData = nestedChainMetaData(metaDataBuilder -> { + final UnnestedRecordTypeBuilder typeBuilder = metaDataBuilder.addUnnestedRecordType(ESCAPED_NAMES); + typeBuilder.addParentConstituent(PARENT, metaDataBuilder.getRecordType(OUTER)); + typeBuilder.addNestedConstituent("direct", INNER_DESCRIPTOR, PARENT, + field("escaped__2inner", FanType.FanOut)); + typeBuilder.addNestedConstituent("nested", INNER_DESCRIPTOR, PARENT, + field("nested__2holder").nest(field("inner", FanType.FanOut))); + }); + final UnnestedRecordType type = unnestedType(metaData, ESCAPED_NAMES); + + final GraphExpansion expansion = type.expand(ACCESS_HINT); + // A key expression addresses protobuf fields, so it carries the names the descriptor uses; a `FieldValue` + // resolves against the planner's type, whose field names those have been decoded into. The exploded path is + // therefore expressed in the decoded names, and a field whose descriptor name carries an escape sequence + // cannot be found under the name the key expression holds. + final Value parentValue = expansion.getQuantifiers().get(0).getFlowedObjectValue(); + assertEquals(FieldValue.ofFieldNames(parentValue, List.of("escaped.inner")), + assertSelectOverExplode(expansion.getQuantifiers().get(1)).getCollectionValue()); + assertEquals(FieldValue.ofFieldNames(parentValue, List.of("nested.holder", "inner")), + assertSelectOverExplode(expansion.getQuantifiers().get(2)).getCollectionValue()); + } + + @Test + void expandOnAJoinedRecordTypeIsUnsupported() { + final RecordMetaData metaData = mapMetaData(metaDataBuilder -> { + final JoinedRecordTypeBuilder typeBuilder = metaDataBuilder.addJoinedRecordType(OUTER_OTHER_JOINED); + typeBuilder.addConstituent("outer", OUTER); + typeBuilder.addConstituent("other", OTHER); + typeBuilder.addJoin("outer", field("other_id"), "other", field("other_id")); + }); + final SyntheticRecordType type = metaData.getSyntheticRecordType(OUTER_OTHER_JOINED); + + final UnsupportedOperationException exception = + assertThrows(UnsupportedOperationException.class, () -> type.expand(ACCESS_HINT)); + assertEquals("cannot expand an index defined on a JoinedRecordType", exception.getMessage()); + } + + @Test + void getPlannerTypeForRecordTypeDescribesASyntheticType() { + final RecordMetaData metaData = mapMetaData(addTwoMapsType()); + final UnnestedRecordType type = unnestedType(metaData, TWO_UNNESTED_MAPS); + + // A synthetic type's name cannot be resolved against the stored types, so only the type-taking overload can + // describe it. + assertThrows(MetaDataException.class, () -> metaData.getPlannerType(TWO_UNNESTED_MAPS)); + final Type.Record plannerType = metaData.getPlannerTypeForRecordType(type); + assertEquals(Type.Record.fromDescriptor(type.getDescriptor()), plannerType); + assertThat(plannerType.getFields().stream().map(Type.Record.Field::getFieldName).collect(Collectors.toList()), + contains(PARENT, "entry_one", "entry_two", UnnestedRecordType.POSITIONS_FIELD)); + } + + @Test + void getPlannerTypeForRecordTypeMatchesTheNameTakingOverloads() { + final RecordMetaData metaData = mapMetaData(metaDataBuilder -> metaDataBuilder.setStoreRecordVersions(true)); + + assertEquals(metaData.getPlannerType(OUTER), + metaData.getPlannerTypeForRecordType(metaData.getRecordType(OUTER))); + assertEquals(metaData.getPlannerType(List.of(OUTER)), + metaData.getPlannerTypeForRecordTypes(List.of(metaData.getRecordType(OUTER)))); + assertEquals(metaData.getPlannerType(List.of(OUTER, OTHER)), + metaData.getPlannerTypeForRecordTypes( + List.of(metaData.getRecordType(OUTER), metaData.getRecordType(OTHER)))); + + // Storing record versions adds the pseudo field to every stored type, including the union of several of them. + final String versionField = PseudoField.ROW_VERSION.getFieldName(); + assertTrue(metaData.getPlannerTypeForRecordType(metaData.getRecordType(OUTER)) + .getFieldNameFieldMap().containsKey(versionField)); + assertTrue(metaData.getPlannerTypeForRecordTypes( + List.of(metaData.getRecordType(OUTER), metaData.getRecordType(OTHER))) + .getFieldNameFieldMap().containsKey(versionField)); + } + + @Test + void getPlannerTypeForRecordTypesUnionsTheirFields() { + final RecordMetaData metaData = mapMetaData(metaDataBuilder -> { }); + final Type.Record unionType = metaData.getPlannerTypeForRecordTypes( + List.of(metaData.getRecordType(OUTER), metaData.getRecordType(OTHER))); + + // `rec_id` and `other_id` are shared, so the union has them once; the remaining fields come from one type each. + assertThat(unionType.getFields().stream().map(Type.Record.Field::getFieldName).collect(Collectors.toList()), + contains("rec_id", "other_id", "map", "other_value")); + + // A single type is described exactly as the type-taking overload would describe it on its own. + assertEquals(metaData.getPlannerTypeForRecordType(metaData.getRecordType(OUTER)), + metaData.getPlannerTypeForRecordTypes(List.of(metaData.getRecordType(OUTER)))); + } + + @Nonnull + private static RecordMetaData mapMetaData(@Nonnull Consumer hook) { + final RecordMetaDataBuilder metaDataBuilder = RecordMetaData.newBuilder() + .setRecords(TestRecordsNestedMapProto.getDescriptor()); + hook.accept(metaDataBuilder); + return metaDataBuilder.build(); + } + + @Nonnull + private static RecordMetaData nestedChainMetaData(@Nonnull Consumer hook) { + final RecordMetaDataBuilder metaDataBuilder = RecordMetaData.newBuilder() + .setRecords(TestRecordsNestedChainProto.getDescriptor()); + hook.accept(metaDataBuilder); + return metaDataBuilder.build(); + } + + @Nonnull + private static RecordMetaData wrapperMetaData(@Nonnull Consumer hook) { + final RecordMetaDataBuilder metaDataBuilder = RecordMetaData.newBuilder() + .setRecords(TestRecords4WrapperProto.getDescriptor()); + hook.accept(metaDataBuilder); + return metaDataBuilder.build(); + } + + @Nonnull + private static Consumer addTwoMapsType() { + return metaDataBuilder -> { + final UnnestedRecordTypeBuilder typeBuilder = metaDataBuilder.addUnnestedRecordType(TWO_UNNESTED_MAPS); + typeBuilder.addParentConstituent(PARENT, metaDataBuilder.getRecordType(OUTER)); + typeBuilder.addNestedConstituent("entry_one", TestRecordsNestedMapProto.MapRecord.Entry.getDescriptor(), + PARENT, ENTRIES_FAN_OUT); + typeBuilder.addNestedConstituent("entry_two", TestRecordsNestedMapProto.MapRecord.Entry.getDescriptor(), + PARENT, ENTRIES_FAN_OUT); + }; + } + + @Nonnull + private static Consumer addNestedChainType() { + return metaDataBuilder -> { + final UnnestedRecordTypeBuilder typeBuilder = metaDataBuilder.addUnnestedRecordType(NESTED_CHAIN); + typeBuilder.addParentConstituent(PARENT, metaDataBuilder.getRecordType(OUTER)); + typeBuilder.addNestedConstituent("middle", + TestRecordsNestedChainProto.OuterRecord.MiddleRecord.getDescriptor(), + PARENT, field("many_middle", FanType.FanOut)); + typeBuilder.addNestedConstituent("inner", + TestRecordsNestedChainProto.OuterRecord.MiddleRecord.InnerRecord.getDescriptor(), + "middle", field("inner", FanType.FanOut)); + typeBuilder.addNestedConstituent("outer_inner", + TestRecordsNestedChainProto.OuterRecord.MiddleRecord.InnerRecord.getDescriptor(), + PARENT, field("inner", FanType.FanOut)); + }; + } + + @Nonnull + private static Type.Record.Field longField(@Nonnull String fieldName) { + return Type.Record.Field.of(Type.primitiveType(Type.TypeCode.LONG, false), Optional.of(fieldName)); + } + + @Nonnull + private static UnnestedRecordType unnestedType(@Nonnull RecordMetaData metaData, @Nonnull String typeName) { + final SyntheticRecordType syntheticRecordType = metaData.getSyntheticRecordType(typeName); + assertThat(syntheticRecordType, instanceOf(UnnestedRecordType.class)); + return (UnnestedRecordType)syntheticRecordType; + } + + @Nonnull + private static List columnNames(@Nonnull GraphExpansion expansion) { + return expansion.getResultColumns().stream() + .map(column -> column.getField().getFieldName()) + .collect(Collectors.toList()); + } + + @Nonnull + private static Value columnValue(@Nonnull GraphExpansion expansion, @Nonnull String columnName) { + return expansion.getResultColumns().stream() + .filter(column -> columnName.equals(column.getField().getFieldNameOptional().orElse(null))) + .map(Column::getValue) + .findFirst() + .orElseThrow(() -> new AssertionError("no column named " + columnName)); + } + + @Nonnull + private static LogicalTypeFilterExpression assertTypeFilter(@Nonnull Quantifier quantifier, + @Nonnull String recordTypeName) { + final RelationalExpression expression = quantifier.getRangesOver().get(); + assertThat(expression, instanceOf(LogicalTypeFilterExpression.class)); + final LogicalTypeFilterExpression typeFilter = (LogicalTypeFilterExpression)expression; + assertThat(typeFilter.getRecordTypes(), contains(recordTypeName)); + assertThat(Iterables.getOnlyElement(typeFilter.getQuantifiers()).getRangesOver().get(), + instanceOf(FullUnorderedScanExpression.class)); + return typeFilter; + } + + /** + * Asserts that the given quantifier ranges over a select that returns the exploded struct itself, and returns the + * explode underneath it. + */ + @Nonnull + private static ExplodeExpression assertSelectOverExplode(@Nonnull Quantifier quantifier) { + final RelationalExpression expression = quantifier.getRangesOver().get(); + assertThat(expression, instanceOf(SelectExpression.class)); + final SelectExpression select = (SelectExpression)expression; + final Quantifier explodeQuantifier = Iterables.getOnlyElement(select.getQuantifiers()); + assertEquals(explodeQuantifier.getFlowedObjectValue(), select.getResultValue()); + final RelationalExpression explode = explodeQuantifier.getRangesOver().get(); + assertThat(explode, instanceOf(ExplodeExpression.class)); + return (ExplodeExpression)explode; + } +} diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/IndexEntryTranslatorEquivalenceTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/IndexEntryTranslatorEquivalenceTest.java index e11547f03b9..701d3471631 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/IndexEntryTranslatorEquivalenceTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/IndexEntryTranslatorEquivalenceTest.java @@ -89,8 +89,12 @@ void bothTranslatorsDecodeANestedEntryAlike() { FieldValue.ofFieldNames(baseObjectValue, List.of("header", "num")), FieldValue.ofFieldNames(baseObjectValue, List.of("header", "rec_no"))); + // The positions of the full key the candidate scans, index columns then primary key, as the candidates pass them. + final var keyPositions = concat(metaDataBuilder.build().getIndex("multi").getRootExpression(), + recordType.getPrimaryKey()).normalizeKeyForPositions(); + final var translators = ScanWithFetchMatchCandidate.computeIndexEntryToLogicalRecord(List.of(recordType), - baseAlias, baseType, indexKeyValues, ImmutableList.of()).orElseThrow(); + baseAlias, baseType, indexKeyValues, ImmutableList.of(), keyPositions).orElseThrow(); final var copiers = translators.indexKeyValueToPartialRecord(); final var recordValue = translators.indexEntryToRecordValue(); assertNotNull(recordValue, "a nested index should still yield a value based translator"); @@ -133,8 +137,11 @@ void bothTranslatorsKeepTheFirstColumnOfARepeatedField() { FieldValue.ofFieldNames(baseObjectValue, List.of("num_value_2")), FieldValue.ofFieldNames(baseObjectValue, List.of("rec_no"))); + final var keyPositions = concat(metaData.getIndex("dup").getRootExpression(), recordType.getPrimaryKey()) + .normalizeKeyForPositions(); + final var translators = ScanWithFetchMatchCandidate.computeIndexEntryToLogicalRecord(List.of(recordType), - baseAlias, baseType, indexKeyValues, ImmutableList.of()).orElseThrow(); + baseAlias, baseType, indexKeyValues, ImmutableList.of(), keyPositions).orElseThrow(); final var recordValue = translators.indexEntryToRecordValue(); assertNotNull(recordValue, "a repeated field should still yield a value based translator"); diff --git a/fdb-record-layer-core/src/test/proto/test_records_nested_chain.proto b/fdb-record-layer-core/src/test/proto/test_records_nested_chain.proto new file mode 100644 index 00000000000..269f0c6dde8 --- /dev/null +++ b/fdb-record-layer-core/src/test/proto/test_records_nested_chain.proto @@ -0,0 +1,52 @@ +/* + * test_records_nested_chain.proto + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2015-2025 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +syntax = "proto2"; + +package com.apple.foundationdb.record.test.nestedchain; + +option java_package = "com.apple.foundationdb.record"; +option java_outer_classname = "TestRecordsNestedChainProto"; + +import "record_metadata_options.proto"; + +// A record type with two levels of repeated, nested structure. This is the same shape as +// `test_records_double_nested.proto`, except that none of the types here refers back to a type that contains it. +message OuterRecord { + message MiddleRecord { + message InnerRecord { + optional int64 foo = 1; + optional string bar = 2; + } + repeated InnerRecord inner = 1; + optional int64 other_int = 2; + } + optional int64 rec_no = 1 [(field).primary_key = true]; + repeated MiddleRecord many_middle = 2; + repeated MiddleRecord.InnerRecord inner = 3; + + // Fields whose descriptor names carry escape sequences. `__2` decodes to `.`, so the planner knows these fields as + // `nested.holder` and `escaped.inner`, while a key expression addressing them carries the names below. + optional MiddleRecord nested__2holder = 4; + repeated MiddleRecord.InnerRecord escaped__2inner = 5; +} + +message RecordTypeUnion { + optional OuterRecord _OuterRecord = 1; +} diff --git a/fdb-relational-api/src/main/java/com/apple/foundationdb/relational/api/metadata/SchemaTemplate.java b/fdb-relational-api/src/main/java/com/apple/foundationdb/relational/api/metadata/SchemaTemplate.java index 7745b0a1ff7..4af5ba5af8b 100644 --- a/fdb-relational-api/src/main/java/com/apple/foundationdb/relational/api/metadata/SchemaTemplate.java +++ b/fdb-relational-api/src/main/java/com/apple/foundationdb/relational/api/metadata/SchemaTemplate.java @@ -76,6 +76,15 @@ public interface SchemaTemplate extends Metadata { @Nonnull Set getViews() throws RelationalException; + /** + * Returns the {@link SyntheticTable}s inside the schema template. + * + * @return The {@link SyntheticTable}s inside the schema template. + * @throws RelationalException if it is a NoOpSchemaTemplate + */ + @Nonnull + Set getSyntheticTables() throws RelationalException; + /** * Retrieves a {@link Table} by looking up its name. * diff --git a/fdb-relational-api/src/main/java/com/apple/foundationdb/relational/api/metadata/SyntheticTable.java b/fdb-relational-api/src/main/java/com/apple/foundationdb/relational/api/metadata/SyntheticTable.java new file mode 100644 index 00000000000..509255bd75e --- /dev/null +++ b/fdb-relational-api/src/main/java/com/apple/foundationdb/relational/api/metadata/SyntheticTable.java @@ -0,0 +1,56 @@ +/* + * SyntheticTable.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2021-2026 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apple.foundationdb.relational.api.metadata; + +import javax.annotation.Nonnull; +import java.util.Set; + +/** + * Metadata for a synthetic table: a named, virtual table whose rows are derived from those of one or more stored + * {@link Table}s, and which carries {@link Index}es maintained from writes to those tables. + *

+ */ +public interface SyntheticTable extends Metadata { + + /** + * {@return the indexes of this synthetic table} + */ + @Nonnull + Set getIndexes(); + + /** + * Names of the stored tables this synthetic table is derived from. Its indexes are maintained from writes to those + * tables. + * + * @return the names of the underlying stored tables + */ + @Nonnull + Set getUnderlyingTableNames(); + + @Override + default void accept(@Nonnull final Visitor visitor) { + visitor.visit(this); + + for (final var index : getIndexes()) { + index.accept(visitor); + } + } +} diff --git a/fdb-relational-api/src/main/java/com/apple/foundationdb/relational/api/metadata/Visitor.java b/fdb-relational-api/src/main/java/com/apple/foundationdb/relational/api/metadata/Visitor.java index 78ce737571f..309642844be 100644 --- a/fdb-relational-api/src/main/java/com/apple/foundationdb/relational/api/metadata/Visitor.java +++ b/fdb-relational-api/src/main/java/com/apple/foundationdb/relational/api/metadata/Visitor.java @@ -51,4 +51,6 @@ default void visit(@Nonnull final Metadata metadata) { void visit(@Nonnull InvokedRoutine invokedRoutine); void visit(@Nonnull View view); + + void visit(@Nonnull SyntheticTable syntheticTable); } diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/NoOpSchemaTemplate.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/NoOpSchemaTemplate.java index a78a3dedd8a..435423f5ec3 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/NoOpSchemaTemplate.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/NoOpSchemaTemplate.java @@ -28,6 +28,7 @@ import com.apple.foundationdb.relational.api.metadata.Schema; import com.apple.foundationdb.relational.api.metadata.SchemaTemplate; import com.apple.foundationdb.relational.api.metadata.StoredQuery; +import com.apple.foundationdb.relational.api.metadata.SyntheticTable; import com.apple.foundationdb.relational.api.metadata.Table; import com.apple.foundationdb.relational.api.metadata.View; @@ -93,6 +94,12 @@ public Set getViews() throws RelationalException { throw new RelationalException("NoOpSchemaTemplate doesn't have views!", ErrorCode.INVALID_PARAMETER); } + @Nonnull + @Override + public Set getSyntheticTables() throws RelationalException { + throw new RelationalException("NoOpSchemaTemplate doesn't have synthetic tables!", ErrorCode.INVALID_PARAMETER); + } + @Nonnull @Override public Optional findTableByName(@Nonnull final String tableName) throws RelationalException { diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/RecordLayerSchemaTemplate.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/RecordLayerSchemaTemplate.java index b8717bcec7a..6d36df8d9c7 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/RecordLayerSchemaTemplate.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/RecordLayerSchemaTemplate.java @@ -81,6 +81,9 @@ public final class RecordLayerSchemaTemplate implements SchemaTemplate { @Nonnull private final Set views; + @Nonnull + private final Set syntheticTables; + @Nonnull private final Map storedQueries; @@ -111,6 +114,7 @@ private RecordLayerSchemaTemplate(@Nonnull final String name, @Nonnull final Set tables, @Nonnull final Set invokedRoutines, @Nonnull final Set views, + @Nonnull final Set syntheticTables, @Nonnull final Map storedQueries, int version, boolean enableLongRows, @@ -120,6 +124,7 @@ private RecordLayerSchemaTemplate(@Nonnull final String name, this.tables = ImmutableSet.copyOf(tables); this.invokedRoutines = ImmutableSet.copyOf(invokedRoutines); this.views = ImmutableSet.copyOf(views); + this.syntheticTables = ImmutableSet.copyOf(syntheticTables); this.storedQueries = ImmutableMap.copyOf(storedQueries); this.version = version; this.enableLongRows = enableLongRows; @@ -136,6 +141,7 @@ private RecordLayerSchemaTemplate(@Nonnull final String name, @Nonnull final Set tables, @Nonnull final Set invokedRoutines, @Nonnull final Set views, + @Nonnull final Set syntheticTables, @Nonnull final Map storedQueries, int version, boolean enableLongRows, @@ -147,6 +153,7 @@ private RecordLayerSchemaTemplate(@Nonnull final String name, this.tables = ImmutableSet.copyOf(tables); this.invokedRoutines = ImmutableSet.copyOf(invokedRoutines); this.views = ImmutableSet.copyOf(views); + this.syntheticTables = ImmutableSet.copyOf(syntheticTables); this.storedQueries = ImmutableMap.copyOf(storedQueries); this.enableLongRows = enableLongRows; this.storeRowVersions = storeRowVersions; @@ -264,6 +271,13 @@ private Multimap computeTableIndexMapping() { result.put(table.getName(), index.getName()); } } + for (final var syntheticTable : getSyntheticTables()) { + for (final var index : syntheticTable.getIndexes()) { + for (final var tableName : syntheticTable.getUnderlyingTableNames()) { + result.put(tableName, index.getName()); + } + } + } return result.build(); } @@ -283,7 +297,7 @@ private Set computeIndexes() { final Set result = new TreeSet<>(); // TODO: There are few index types that we currently don't handle - // Namely, universal, multi-type, and synthetic indexes. Once those are handled, we + // Namely, universal and multi-type indexes. Once those are handled, we // should be able to replace this with logic that gets the indexes from the // schema template directly instead of converting it to meta-data. final RecordMetaData metaData = toRecordMetadata(); @@ -346,6 +360,31 @@ public Set getViews() { return views; } + /** + * Returns the synthetic tables of this template, of any kind. + * + * @return the synthetic tables + */ + @Nonnull + @Override + public Set getSyntheticTables() { + return syntheticTables; + } + + /** + * Returns the unnested synthetic tables of this template. + * + * @return the unnested synthetic tables + */ + @VisibleForTesting + @Nonnull + public Set getUnnestedSyntheticTables() { + return syntheticTables.stream() + .filter(RecordLayerUnnestedSyntheticTable.class::isInstance) + .map(RecordLayerUnnestedSyntheticTable.class::cast) + .collect(ImmutableSet.toImmutableSet()); + } + @Nonnull @Override public Map getStoredQueries() { @@ -401,6 +440,9 @@ public void accept(@Nonnull final Visitor visitor) { for (final var view : getViews()) { view.accept(visitor); } + for (final var syntheticTable : syntheticTables) { + syntheticTable.accept(visitor); + } visitor.finishVisit(this); } @@ -428,6 +470,9 @@ public static final class Builder { @Nonnull private final Map views; + @Nonnull + private final Map syntheticTables; + @Nonnull private final Map storedQueries; @@ -439,6 +484,7 @@ private Builder() { auxiliaryTypes = new LinkedHashMap<>(); invokedRoutines = new LinkedHashMap<>(); views = new LinkedHashMap<>(); + syntheticTables = new LinkedHashMap<>(); storedQueries = new LinkedHashMap<>(); // enable long rows is TRUE by default enableLongRows = true; @@ -552,6 +598,19 @@ public Builder removeView(@Nonnull final String viewName) { return this; } + @Nonnull + public Builder addSyntheticTable(@Nonnull final RecordLayerSyntheticTable table) { + verifyNameIsNotUsed(table.getName()); + syntheticTables.put(table.getName(), table); + return this; + } + + @Nonnull + public Builder addSyntheticTables(@Nonnull final Collection syntheticTables) { + syntheticTables.forEach(this::addSyntheticTable); + return this; + } + @Nonnull public Builder addViews(@Nonnull final Collection views) { views.forEach(this::addView); @@ -663,10 +722,16 @@ public RecordLayerSchemaTemplate build() { if (cachedMetadata != null) { return new RecordLayerSchemaTemplate(name, new LinkedHashSet<>(tables.values()), - new LinkedHashSet<>(invokedRoutines.values()), new LinkedHashSet<>(views.values()), storedQueries, version, enableLongRows, storeRowVersions, intermingleTables, cachedMetadata); + new LinkedHashSet<>(invokedRoutines.values()), + new LinkedHashSet<>(views.values()), + new LinkedHashSet<>(syntheticTables.values()), + storedQueries, version, enableLongRows, storeRowVersions, intermingleTables, cachedMetadata); } else { return new RecordLayerSchemaTemplate(name, new LinkedHashSet<>(tables.values()), - new LinkedHashSet<>(invokedRoutines.values()), new LinkedHashSet<>(views.values()), storedQueries, version, enableLongRows, storeRowVersions, intermingleTables); + new LinkedHashSet<>(invokedRoutines.values()), + new LinkedHashSet<>(views.values()), + new LinkedHashSet<>(syntheticTables.values()), + storedQueries, version, enableLongRows, storeRowVersions, intermingleTables); } } @@ -745,6 +810,7 @@ private void verifyNameIsNotUsed(@Nonnull final String name) { Assert.thatUnchecked(!auxiliaryTypes.containsKey(name), ErrorCode.INVALID_SCHEMA_TEMPLATE, () -> "type with name '" + name + "' already exists"); Assert.thatUnchecked(!invokedRoutines.containsKey(name), ErrorCode.INVALID_SCHEMA_TEMPLATE, () -> "routine with name '" + name + "' already exists"); Assert.thatUnchecked(!views.containsKey(name), ErrorCode.INVALID_SCHEMA_TEMPLATE, () -> "view with name '" + name + "' already exists"); + Assert.thatUnchecked(!syntheticTables.containsKey(name), ErrorCode.INVALID_SCHEMA_TEMPLATE, () -> "synthetic table with name '" + name + "' already exists"); } @Nonnull @@ -793,8 +859,9 @@ public Builder toBuilder() { .setEnableLongRows(enableLongRows) .setIntermingleTables(intermingleTables) .addTables(getTables()) - .addInvokedRoutines(getInvokedRoutines()) + .addSyntheticTables(getSyntheticTables()) .addViews(getViews()) + .addInvokedRoutines(getInvokedRoutines()) .addStoredQueries(getStoredQueries()); } } diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/RecordLayerSyntheticTable.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/RecordLayerSyntheticTable.java new file mode 100644 index 00000000000..1390ffcd20d --- /dev/null +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/RecordLayerSyntheticTable.java @@ -0,0 +1,94 @@ +/* + * RecordLayerSyntheticTable.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2021-2026 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apple.foundationdb.relational.recordlayer.metadata; + +import com.apple.foundationdb.annotation.API; +import com.apple.foundationdb.record.query.plan.cascades.typing.Type; +import com.apple.foundationdb.relational.api.metadata.SyntheticTable; + +import javax.annotation.Nonnull; +import java.util.Objects; +import java.util.Set; + +/** + * Base class for synthetic tables in the relational layer: a virtual table backed by a record-layer synthetic record + * type ({@code UnnestedRecordType} and, eventually, joined types) with one or more indexes maintained on it. + * + * @see RecordLayerUnnestedSyntheticTable + */ +@API(API.Status.EXPERIMENTAL) +public abstract sealed class RecordLayerSyntheticTable implements SyntheticTable + permits RecordLayerUnnestedSyntheticTable { + + @Nonnull + final Type.Record type; + + @Nonnull + private final Set indexes; + + protected RecordLayerSyntheticTable(@Nonnull final Set indexes, + @Nonnull final Type.Record type) { + this.indexes = Set.copyOf(indexes); + this.type = type; + } + + @Nonnull + @Override + public String getName() { + return Objects.requireNonNull(type.getName()); + } + + @Nonnull + public Type.Record getType() { + return type; + } + + @Nonnull + @Override + public Set getIndexes() { + return indexes; + } + + @Override + public boolean equals(final Object o) { + if (o == null) { + return false; + } + if (getClass() != o.getClass()) { + return false; + } + final RecordLayerSyntheticTable that = (RecordLayerSyntheticTable) o; + return Objects.equals(type, that.type) && Objects.equals(indexes, that.indexes); + } + + @Override + public int hashCode() { + return Objects.hash(type, indexes); + } + + public interface Builder { + @Nonnull + Builder addIndex(@Nonnull RecordLayerIndex index); + + @Nonnull + RecordLayerSyntheticTable build(); + } +} diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/RecordLayerUnnestedSyntheticTable.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/RecordLayerUnnestedSyntheticTable.java new file mode 100644 index 00000000000..b401a137451 --- /dev/null +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/RecordLayerUnnestedSyntheticTable.java @@ -0,0 +1,292 @@ +/* + * RecordLayerUnnestedSyntheticTable.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2021-2026 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apple.foundationdb.relational.recordlayer.metadata; + +import com.apple.foundationdb.annotation.API; +import com.apple.foundationdb.record.metadata.expressions.FieldKeyExpression; +import com.apple.foundationdb.record.metadata.expressions.KeyExpression; +import com.apple.foundationdb.record.metadata.expressions.NestingKeyExpression; +import com.apple.foundationdb.record.query.plan.cascades.typing.Type; +import com.apple.foundationdb.relational.api.exceptions.ErrorCode; +import com.apple.foundationdb.relational.util.Assert; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.util.ArrayList; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Objects; +import java.util.Set; + +/** + * A synthetic record type that unnests one or more struct array fields of a stored record type. Each + * combination of a stored record and one element from each unnested array forms a single synthetic record, + * so an index key can reference several fields of the same element without fanning out over the array once + * per reference. + * + *

Only struct arrays become constituents; a scalar array contributes at most one key column, so it stays + * a fan-out in the index key expression. A constituent's array lives either on the stored record or, for + * chained unnesting, on another constituent's element type. The nesting expression that reaches the elements + * carries the {@code FanOut}, and its shape follows how the array is stored: a nullable array is wrapped as + * {@code { repeated T values; }}, while a non-nullable one is a plain repeated field. + */ +@API(API.Status.EXPERIMENTAL) +public final class RecordLayerUnnestedSyntheticTable extends RecordLayerSyntheticTable { + + @Nonnull + private final String alias; + + @Nonnull + private final String parentTableName; + + @Nonnull + private final String parentTableStorageName; + + @Nonnull + private final List constituents; + + private RecordLayerUnnestedSyntheticTable(@Nonnull final String alias, + @Nonnull final String parentTableName, + @Nonnull final String parentTableStorageName, + @Nonnull final List constituents, + @Nonnull final Set indexes, + @Nonnull final Type.Record recordType) { + super(indexes, recordType); + this.alias = alias; + this.parentTableName = parentTableName; + this.parentTableStorageName = parentTableStorageName; + this.constituents = ImmutableList.copyOf(constituents); + } + + @Nonnull + public String getAlias() { + return alias; + } + + @Nonnull + public String getParentTableStorageName() { + return parentTableStorageName; + } + + @Nonnull + public List getConstituents() { + return constituents; + } + + @Nonnull + @Override + public Set getUnderlyingTableNames() { + return Set.of(parentTableName); + } + + @Override + public boolean equals(final Object o) { + return o instanceof RecordLayerUnnestedSyntheticTable that + && super.equals(o) + && Objects.equals(alias, that.alias) + && Objects.equals(parentTableName, that.parentTableName) + && Objects.equals(parentTableStorageName, that.parentTableStorageName) + && Objects.equals(constituents, that.constituents); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), alias, parentTableName, parentTableStorageName, constituents); + } + + /** + * A nested constituent of an {@link RecordLayerUnnestedSyntheticTable}, representing one array fan-out. + * The {@code parentAlias} is the alias of the constituent from which this one is unnested. For a + * single-level unnesting this is the stored-record parent alias; for chained unnesting it is the + * alias of the immediately preceding nested constituent. + */ + public static final class NestedConstituent { + + @Nonnull + private final String alias; + + @Nonnull + private final String parentAlias; + + @Nonnull + private final KeyExpression nestingExpression; + + @Nonnull + private final List fieldPath; + + public NestedConstituent(@Nonnull final String alias, + @Nonnull final String parentAlias, + @Nonnull final KeyExpression nestingExpression) { + this.alias = alias; + this.parentAlias = parentAlias; + this.nestingExpression = nestingExpression; + this.fieldPath = computeFieldPath(alias, nestingExpression); + } + + @Nonnull + public String getAlias() { + return alias; + } + + @Nonnull + public String getParentAlias() { + return parentAlias; + } + + /** + * Expression navigating from the owning constituent's record down to this constituent's elements. This is + * the same expression the record layer stores and evaluates. + */ + @Nonnull + public KeyExpression getNestingExpression() { + return nestingExpression; + } + + @Override + public boolean equals(final Object o) { + return o instanceof NestedConstituent that + && Objects.equals(alias, that.alias) + && Objects.equals(parentAlias, that.parentAlias) + && Objects.equals(nestingExpression, that.nestingExpression); + } + + @Override + public int hashCode() { + return Objects.hash(alias, parentAlias, nestingExpression); + } + + /** + * The chain of proto field names that {@link #getNestingExpression()} walks, e.g. {@code [scores]} for a + * plain repeated field, or {@code [scores, values]} for a nullable array stored wrapped. + */ + @Nonnull + public List getFieldPath() { + return fieldPath; + } + + /** + * Walks the nesting expression into the chain of field names the serializer follows to reach the element + * descriptor. + * + * @param alias the constituent's alias, for the rejection message + * @param nestingExpression the expression navigating to the constituent's elements + * + * @return the field names the expression walks, outermost first + */ + @Nonnull + private static List computeFieldPath(@Nonnull final String alias, + @Nonnull final KeyExpression nestingExpression) { + final ImmutableList.Builder fieldPath = ImmutableList.builder(); + KeyExpression remaining = nestingExpression; + while (remaining instanceof NestingKeyExpression nesting) { + fieldPath.add(nesting.getParent().getFieldName()); + remaining = nesting.getChild(); + } + Assert.thatUnchecked(remaining instanceof FieldKeyExpression, ErrorCode.INVALID_SCHEMA_TEMPLATE, + "unsupported nesting expression '%s' for constituent '%s'", nestingExpression, alias); + fieldPath.add(((FieldKeyExpression)remaining).getFieldName()); + return fieldPath.build(); + } + } + + @Nonnull + public static Builder newBuilder(Type.Record type) { + return new Builder().setType(type); + } + + /** + * Builder for {@link RecordLayerUnnestedSyntheticTable}. + *

+ * {@link #build()} checks the constituents to form a tree rooted at the stored record. It rejects a + * table with no nested constituent, a duplicate alias, and a constituent whose parent alias is not the parent + * constituent's or an earlier constituent's. + */ + public static final class Builder implements RecordLayerSyntheticTable.Builder { + + @Nullable + private String alias; + @Nullable + private String parentTableName; + @Nullable + private String parentTableStorageName; + @Nonnull + private final List constituents = new ArrayList<>(); + @Nonnull + private final ImmutableSet.Builder indexes = ImmutableSet.builder(); + @Nullable + private Type.Record type; + + @Nonnull + public Builder setType(@Nonnull final Type.Record type) { + this.type = type; + return this; + } + + @Nonnull + public Builder setAlias(@Nonnull final String alias) { + this.alias = alias; + return this; + } + + @Nonnull + public Builder setParentTableType(@Nonnull final Type.Record tableType) { + this.parentTableName = tableType.getName(); + this.parentTableStorageName = tableType.getStorageName(); + return this; + } + + @Nonnull + public Builder addConstituent(@Nonnull final NestedConstituent constituent) { + constituents.add(constituent); + return this; + } + + @Nonnull + @Override + public Builder addIndex(@Nonnull final RecordLayerIndex index) { + indexes.add(index); + return this; + } + + @Nonnull + @Override + public RecordLayerUnnestedSyntheticTable build() { + Assert.notNullUnchecked(alias, "parent constituent alias is not set"); + Assert.notNullUnchecked(parentTableName, "parent table type is not set"); + Assert.notNullUnchecked(type, "type is not set"); + Assert.thatUnchecked(!constituents.isEmpty(), "unnested type has no nested constituents"); + final Set aliases = new LinkedHashSet<>(); + aliases.add(alias); + for (final NestedConstituent constituent : constituents) { + Assert.thatUnchecked(aliases.contains(constituent.getParentAlias()), + ErrorCode.INVALID_SCHEMA_TEMPLATE, + "constituent parent alias '%s' is not a known alias", constituent.getParentAlias()); + final var isUnique = aliases.add(constituent.getAlias()); + Assert.thatUnchecked(isUnique, ErrorCode.INVALID_SCHEMA_TEMPLATE, + "duplicate constituent alias '%s' in unnested type", constituent.getAlias()); + } + return new RecordLayerUnnestedSyntheticTable(alias, parentTableName, parentTableStorageName, + constituents, indexes.build(), type); + } + } +} diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/SkeletonVisitor.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/SkeletonVisitor.java index 89f438b208f..0ea21c9b7aa 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/SkeletonVisitor.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/SkeletonVisitor.java @@ -27,6 +27,7 @@ import com.apple.foundationdb.relational.api.metadata.InvokedRoutine; import com.apple.foundationdb.relational.api.metadata.Schema; import com.apple.foundationdb.relational.api.metadata.SchemaTemplate; +import com.apple.foundationdb.relational.api.metadata.SyntheticTable; import com.apple.foundationdb.relational.api.metadata.Table; import com.apple.foundationdb.relational.api.metadata.View; import com.apple.foundationdb.relational.api.metadata.Visitor; @@ -83,4 +84,9 @@ public void visit(@Nonnull final InvokedRoutine invokedRoutine) { public void visit(@Nonnull final View view) { // no-op } + + @Override + public void visit(@Nonnull final SyntheticTable syntheticTable) { + // no-op + } } diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/serde/RecordMetadataDeserializer.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/serde/RecordMetadataDeserializer.java index d3cc95a11d8..b7b346d5bf3 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/serde/RecordMetadataDeserializer.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/serde/RecordMetadataDeserializer.java @@ -22,7 +22,9 @@ import com.apple.foundationdb.annotation.API; import com.apple.foundationdb.record.RecordMetaData; +import com.apple.foundationdb.record.metadata.Index; import com.apple.foundationdb.record.metadata.RecordType; +import com.apple.foundationdb.record.metadata.UnnestedRecordType; import com.apple.foundationdb.record.query.plan.cascades.RawSqlFunction; import com.apple.foundationdb.record.query.plan.cascades.UserDefinedFunction; import com.apple.foundationdb.record.query.plan.cascades.UserDefinedMacroFunction; @@ -34,6 +36,7 @@ import com.apple.foundationdb.relational.recordlayer.metadata.RecordLayerInvokedRoutine; import com.apple.foundationdb.relational.recordlayer.metadata.RecordLayerSchemaTemplate; import com.apple.foundationdb.relational.recordlayer.metadata.RecordLayerTable; +import com.apple.foundationdb.relational.recordlayer.metadata.RecordLayerUnnestedSyntheticTable; import com.apple.foundationdb.relational.recordlayer.metadata.RecordLayerView; import com.apple.foundationdb.relational.recordlayer.query.LogicalOperator; import com.apple.foundationdb.relational.util.Assert; @@ -123,6 +126,13 @@ private static RecordLayerSchemaTemplate.Builder deserializeRecordMetaData(@Nonn schemaTemplateBuilder.addView(generateViewBuilder(metadataProvider, view.getKey(), view.getValue().getDefinition()).build()); } } + for (final var syntheticType : recordMetaData.getSyntheticRecordTypes().values()) { + // knowingly ignoring other for now since they are not supported + if (syntheticType instanceof UnnestedRecordType unnestedRecordType) { + schemaTemplateBuilder.addSyntheticTable( + generateUnnestedSyntheticTableBuilder(recordMetaData, unnestedRecordType).build()); + } + } for (final var entry : recordMetaData.getStoredQueries().entrySet()) { final RecordMetaData.StoredQuery storedQuery = entry.getValue(); schemaTemplateBuilder.addStoredQuery(entry.getKey(), storedQuery.getQuery(), storedQuery.getTempFunctions()); @@ -197,6 +207,33 @@ private static RecordLayerView.Builder generateViewBuilder(@Nonnull final Suppli .setViewCompiler(getViewCompiler(name, metadata, definition)); } + @Nonnull + private static RecordLayerUnnestedSyntheticTable.Builder generateUnnestedSyntheticTableBuilder( + @Nonnull final RecordMetaData recordMetaData, + @Nonnull final UnnestedRecordType unnestedRecordType) { + final var type = Type.Record.fromDescriptorPreservingName(unnestedRecordType.getDescriptor()); + final UnnestedRecordType.NestedConstituent parentConstituent = unnestedRecordType.getParentConstituent(); + final String parentStorageName = parentConstituent.getRecordType().getName(); + final Type.Record parentType = Type.Record.fromDescriptorPreservingName( + recordMetaData.getRecordType(parentStorageName).getDescriptor()); + final RecordLayerUnnestedSyntheticTable.Builder builder = RecordLayerUnnestedSyntheticTable.newBuilder(type) + .setAlias(parentConstituent.getName()) + .setParentTableType(parentType); + for (final UnnestedRecordType.NestedConstituent constituent : unnestedRecordType.getConstituents()) { + if (constituent.isParent()) { + continue; + } + builder.addConstituent(new RecordLayerUnnestedSyntheticTable.NestedConstituent( + constituent.getName(), Objects.requireNonNull(constituent.getParentName()), + constituent.getNestingExpression())); + } + // add indexes + for (final Index index : unnestedRecordType.getIndexes()) { + builder.addIndex(RecordLayerIndex.from(type.getName(), type.getStorageName(), index)); + } + return builder; + } + @Nonnull public RecordMetaData getRecordMetaData() { return recordMetaData; diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/serde/RecordMetadataSerializer.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/serde/RecordMetadataSerializer.java index abea5f55004..ad76c5fe82f 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/serde/RecordMetadataSerializer.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/serde/RecordMetadataSerializer.java @@ -27,15 +27,18 @@ import com.apple.foundationdb.record.metadata.Index; import com.apple.foundationdb.record.metadata.IndexPredicate; import com.apple.foundationdb.record.metadata.RecordTypeBuilder; +import com.apple.foundationdb.record.metadata.UnnestedRecordTypeBuilder; import com.apple.foundationdb.record.metadata.expressions.KeyExpression; import com.apple.foundationdb.relational.api.metadata.InvokedRoutine; import com.apple.foundationdb.relational.api.metadata.SchemaTemplate; +import com.apple.foundationdb.relational.api.metadata.SyntheticTable; import com.apple.foundationdb.relational.api.metadata.Table; import com.apple.foundationdb.relational.api.metadata.View; import com.apple.foundationdb.relational.recordlayer.metadata.RecordLayerIndex; import com.apple.foundationdb.relational.recordlayer.metadata.RecordLayerInvokedRoutine; import com.apple.foundationdb.relational.recordlayer.metadata.RecordLayerSchemaTemplate; import com.apple.foundationdb.relational.recordlayer.metadata.RecordLayerTable; +import com.apple.foundationdb.relational.recordlayer.metadata.RecordLayerUnnestedSyntheticTable; import com.apple.foundationdb.relational.recordlayer.metadata.RecordLayerView; import com.apple.foundationdb.relational.recordlayer.metadata.SkeletonVisitor; import com.apple.foundationdb.relational.util.Assert; @@ -43,6 +46,8 @@ import com.google.protobuf.Descriptors; import javax.annotation.Nonnull; +import java.util.LinkedHashMap; +import java.util.Map; @API(API.Status.EXPERIMENTAL) public class RecordMetadataSerializer extends SkeletonVisitor { @@ -71,6 +76,34 @@ public void visit(@Nonnull Table table) { recordType.setPrimaryKey(keyExpression); } + private void visit(@Nonnull final RecordLayerUnnestedSyntheticTable unnestedSyntheticTable) { + final UnnestedRecordTypeBuilder builder = + getBuilder().addUnnestedRecordType(unnestedSyntheticTable.getType().getStorageName()); + final RecordTypeBuilder recordTypeBuilder = getBuilder().getRecordType(unnestedSyntheticTable.getParentTableStorageName()); + builder.addParentConstituent(unnestedSyntheticTable.getAlias(), recordTypeBuilder); + final Map descriptorsByAlias = new LinkedHashMap<>(); + descriptorsByAlias.put(unnestedSyntheticTable.getAlias(), recordTypeBuilder.getDescriptor()); + for (final RecordLayerUnnestedSyntheticTable.NestedConstituent nested : unnestedSyntheticTable.getConstituents()) { + final Descriptors.Descriptor owningProto = descriptorsByAlias.get(nested.getParentAlias()); + Assert.notNullUnchecked(owningProto, "unknown parent constituent '" + nested.getParentAlias() + + "' for constituent '" + nested.getAlias() + "'"); + Descriptors.Descriptor constituentDescriptor = owningProto; + for (final String fieldName : nested.getFieldPath()) { + final Descriptors.FieldDescriptor pathField = constituentDescriptor.findFieldByName(fieldName); + Assert.notNullUnchecked(pathField, "field '" + fieldName + "' on the path to constituent '" + + nested.getAlias() + "' not found on '" + constituentDescriptor.getName() + "'"); + Assert.thatUnchecked(pathField.getType() == Descriptors.FieldDescriptor.Type.MESSAGE, + "field '" + fieldName + "' on the path to constituent '" + nested.getAlias() + + "' is not a nested type, so it cannot be navigated through; a constituent has to " + + "unnest a struct array, and scalar arrays are not supported"); + constituentDescriptor = pathField.getMessageType(); + } + builder.addNestedConstituent(nested.getAlias(), constituentDescriptor, + nested.getParentAlias(), nested.getNestingExpression()); + descriptorsByAlias.put(nested.getAlias(), constituentDescriptor); + } + } + @Override public void visit(@Nonnull com.apple.foundationdb.relational.api.metadata.Index index) { // Note: this does not preserve the index added and lest modified version, necessary @@ -100,8 +133,17 @@ public void visit(@Nonnull final InvokedRoutine invokedRoutine) { @Override public void visit(@Nonnull final View view) { - Assert.thatUnchecked(view instanceof RecordLayerView); - getBuilder().addView(((RecordLayerView)view).asRawView()); + final var recordLayerView = Assert.castUnchecked(view, RecordLayerView.class); + getBuilder().addView(recordLayerView.asRawView()); + } + + @Override + public void visit(@Nonnull final SyntheticTable syntheticTable) { + if (syntheticTable instanceof RecordLayerUnnestedSyntheticTable unnestedSyntheticTable) { + visit(unnestedSyntheticTable); + } else { + Assert.failUnchecked("synthetic table kind not supported"); + } } @Override diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/ddl/IndexGenerationResult.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/ddl/IndexGenerationResult.java new file mode 100644 index 00000000000..9931a38e8cf --- /dev/null +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/ddl/IndexGenerationResult.java @@ -0,0 +1,64 @@ +/* + * IndexGenerationResult.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2021-2026 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apple.foundationdb.relational.recordlayer.query.ddl; + +import com.apple.foundationdb.annotation.API; +import com.apple.foundationdb.relational.recordlayer.metadata.RecordLayerIndex; +import com.apple.foundationdb.relational.recordlayer.metadata.RecordLayerSchemaTemplate; +import com.apple.foundationdb.relational.recordlayer.metadata.RecordLayerTable; +import com.apple.foundationdb.relational.recordlayer.metadata.RecordLayerSyntheticTable; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +/** + * The outcome of generating an index from a DDL definition. + * + *

Most indexes are defined on a stored table, in which case only the index definition is produced. An index + * whose key spans a record composed of several others is instead defined on a synthetic record type, which is + * returned alongside it: an unnested type for a stored record and the elements of an array it unnests, a joined + * type for the sides of a join. Callers must register that type, since the index names it, so leaving it out + * would produce an index on a type that does not exist. + * + * @param indexBuilder the index definition + * @param syntheticTable the synthetic table to register, {@code null} for an index on a stored table + */ +@API(API.Status.INTERNAL) +public record IndexGenerationResult(@Nonnull RecordLayerIndex.Builder indexBuilder, + @Nullable RecordLayerSyntheticTable.Builder syntheticTable) { + + /** + * Registers the generated index on the given schema template, together with the synthetic table it is defined + * on when there is one. Doing this here rather than at each call site keeps the two from drifting apart: an index + * whose synthetic table is not registered names a type that does not exist. + * + * @param metadataBuilder the schema template being built + */ + public void registerOn(@Nonnull final RecordLayerSchemaTemplate.Builder metadataBuilder) { + if (syntheticTable != null) { + metadataBuilder.addSyntheticTable(syntheticTable.addIndex(indexBuilder.build()).build()); + } else { + final RecordLayerIndex index = indexBuilder.build(); + final var table = metadataBuilder.extractTable(index.getTableName()); + metadataBuilder.addTable(RecordLayerTable.Builder.from(table).addIndex(index).build()); + } + } +} diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/ddl/IndexGenerator.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/ddl/IndexGenerator.java index a84bbba288c..99386379f17 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/ddl/IndexGenerator.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/ddl/IndexGenerator.java @@ -59,12 +59,16 @@ public interface IndexGenerator { /** * Builds the index this generator was configured with. *

- * The returned builder is not yet built, so a caller that knows something the definition does not carry can still - * add it -- a vector index, for instance, sets its index type and engine options afterwards. Nothing is written to - * the schema template until the caller builds it and adds it. + * The returned index builder is not yet built, so a caller that knows something the definition does not carry can + * still add it -- a vector index, for instance, sets its index type and engine options afterwards. Nothing is + * written to the schema template until the caller registers the result. + *

+ * An index whose key spans a record composed of several others is defined on a synthetic table, which the result + * carries alongside the index. {@link IndexGenerationResult#registerOn} registers both, since an index whose + * synthetic table is not registered names a type that does not exist. *

* - * @return the index the definition asks for + * @return the index the definition asks for, with the synthetic table to define it on when there is one * * @throws com.apple.foundationdb.relational.api.exceptions.UncheckedRelationalException with * {@link com.apple.foundationdb.relational.api.exceptions.ErrorCode#UNSUPPORTED_OPERATION} if the definition @@ -72,5 +76,5 @@ public interface IndexGenerator { * key expression can express */ @Nonnull - RecordLayerIndex.Builder generate(); + IndexGenerationResult generate(); } diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/ddl/IndexSpec.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/ddl/IndexSpec.java index 9befa247132..32cdc24cbcc 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/ddl/IndexSpec.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/ddl/IndexSpec.java @@ -46,6 +46,8 @@ import com.apple.foundationdb.record.query.plan.cascades.values.Value; import com.apple.foundationdb.record.query.plan.cascades.values.Values; import com.apple.foundationdb.relational.api.exceptions.ErrorCode; +import com.apple.foundationdb.relational.recordlayer.metadata.RecordLayerSchemaTemplate; +import com.apple.foundationdb.relational.recordlayer.metadata.RecordLayerTable; import com.apple.foundationdb.relational.util.Assert; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -66,7 +68,7 @@ * bottom-up pass, stating each rule at the node it concerns; {@link #checkValidity} rejects the plans that cannot become * an index. */ -record IndexSpec(int scanCount, @Nullable String recordTypeName, @Nullable QueryPredicate predicate, +record IndexSpec(int scanCount, @Nullable RecordLayerTable table, @Nullable QueryPredicate predicate, @Nullable GroupByExpression groupBy, @Nullable OrderBy orderBy, @Nullable Projection projection) { @@ -75,26 +77,21 @@ record IndexSpec(int scanCount, @Nullable String recordTypeName, @Nullable Query * * @param expression the root of the plan * @param quantifierValues what the plan's quantifiers stand for, from the preceding pass + * @param schemaTemplateBuilder the metadata the plan's type filter is resolved against * * @return what the index is made of */ @Nonnull public static IndexSpec collect(@Nonnull final RelationalExpression expression, - @Nonnull final QuantifierValues quantifierValues) { - final var visitor = new Visitor(quantifierValues); + @Nonnull final QuantifierValues quantifierValues, + @Nonnull final RecordLayerSchemaTemplate.Builder schemaTemplateBuilder) { + final var visitor = new Visitor(quantifierValues, schemaTemplateBuilder); final var indexSpec = Assert.notNullUnchecked(visitor.visit(expression)); // the projection belongs to the root, which the bottom-up traversal cannot single out return indexSpec.withProjection(new ProjectionResolver(quantifierValues) .resolve(expression.getResultValue(), indexSpec.groupBy())); } - @Override - @Nonnull - public String recordTypeName() { - return Assert.notNullUnchecked(recordTypeName, ErrorCode.UNSUPPORTED_OPERATION, - "Unsupported query, expected to find exactly one type filter operator"); - } - /** * The columns the index is ordered by, resolved down to the base record, empty when the definition had no * {@code ORDER BY}. @@ -112,6 +109,41 @@ public Map getOrderingFunctions() { return orderBy == null ? ImmutableMap.of() : orderBy.orderingFunctions(); } + /** + * The index key columns in key order: the order-by columns lead, then whatever the projection holds beyond them. + * Empty ordering, and an aggregate index, keep the projection's own order. + * + * @return the key columns, in key order + */ + @Nonnull + public List rootValues() { + if (projection().aggregate() != null) { + return projection().values(); + } + return reorderValues(projection().fieldValues(), getOrderByValues()); + } + + /** + * Puts the key columns first, followed by the projected columns that are not part of the key. The result is the + * values in the order they need to be traversed to come up with the equivalent {@code KeyExpression}. + * + * @param allValues every projected column + * @param keyValues the columns the index is ordered by, a subset of {@code allValues} + * + * @return the columns in traversal order + */ + @Nonnull + private static List reorderValues(@Nonnull final List allValues, @Nonnull final List keyValues) { + Assert.thatUnchecked(allValues.size() >= keyValues.size()); + if (keyValues.isEmpty()) { + return allValues; + } + final var valueValues = allValues.stream() + .filter(value -> !keyValues.contains(value)) + .collect(ImmutableList.toImmutableList()); + return ImmutableList.builder().addAll(keyValues).addAll(valueValues).build(); + } + /** * The projection the index is defined over, resolved down to the base record. */ @@ -123,50 +155,53 @@ public Projection projection() { @Nonnull private IndexSpec withProjection(@Nonnull final Projection newProjection) { - return new IndexSpec(scanCount, recordTypeName, predicate, groupBy, orderBy, newProjection); + return new IndexSpec(scanCount, table, predicate, groupBy, orderBy, newProjection); } @Nonnull private IndexSpec withOrderBy(@Nonnull final OrderBy newOrderBy) { Assert.thatUnchecked(orderBy == null, ErrorCode.UNSUPPORTED_OPERATION, "Unsupported index definition, more than one sort expression found"); - return new IndexSpec(scanCount, recordTypeName, predicate, groupBy, newOrderBy, projection); + return new IndexSpec(scanCount, table, predicate, groupBy, newOrderBy, projection); } @Nonnull private IndexSpec withScan() { - return new IndexSpec(scanCount + 1, recordTypeName, predicate, groupBy, orderBy, projection); + return new IndexSpec(scanCount + 1, table, predicate, groupBy, orderBy, projection); } @Nonnull - private IndexSpec withRecordTypeName(@Nonnull final String newRecordTypeName) { - Assert.thatUnchecked(recordTypeName == null, ErrorCode.UNSUPPORTED_OPERATION, + private IndexSpec withTable(@Nonnull final RecordLayerTable newTable) { + Assert.thatUnchecked(table == null, ErrorCode.UNSUPPORTED_OPERATION, "Unsupported query, expected to find exactly one type filter operator"); - return new IndexSpec(scanCount, newRecordTypeName, predicate, groupBy, orderBy, projection); + return new IndexSpec(scanCount, newTable, predicate, groupBy, orderBy, projection); } @Nonnull private IndexSpec withPredicate(@Nonnull final QueryPredicate newPredicate) { - return new IndexSpec(scanCount, recordTypeName, newPredicate, groupBy, orderBy, projection); + return new IndexSpec(scanCount, table, newPredicate, groupBy, orderBy, projection); } @Nonnull private IndexSpec withGroupBy(@Nonnull final GroupByExpression newGroupBy) { Assert.thatUnchecked(groupBy == null, ErrorCode.UNSUPPORTED_OPERATION, "Unsupported index definition, multiple group by expressions found"); - return new IndexSpec(scanCount, recordTypeName, predicate, newGroupBy, orderBy, projection); + return new IndexSpec(scanCount, table, predicate, newGroupBy, orderBy, projection); } /** - * Rejects every definition the generator cannot turn into an index, apart from two: the predicate, checked as it is - * collected, and ordering by the aggregate, checked once the index type is known. + * Rejects every definition that cannot become an index at all, apart from two: the predicate, checked as it is + * collected, and ordering by the aggregate, checked once the index type is known. What a stored table can express but + * an unnested synthetic table cannot is a separate question, answered by + * {@code RecordLayerUnnestedSyntheticTableGenerator#checkSupported} once it is known that one is needed. */ public void checkValidity() { // the traversal rejects a second scan as a join, leaving none to reject here Assert.thatUnchecked(scanCount == 1, ErrorCode.UNSUPPORTED_OPERATION, "Unsupported index definition, no iteration generator found"); // throws unless exactly one type filter was found - recordTypeName(); + Assert.notNullUnchecked(table, ErrorCode.UNSUPPORTED_OPERATION, + "Unsupported query, expected to find exactly one type filter operator"); final var projection = projection(); reject(projection.values().stream() @@ -256,12 +291,12 @@ private static IndexSpec merge(@Nonnull final List childSpecs) { var merged = new IndexSpec(0, null, null, null, null, null); for (final var childSpec : childSpecs) { // the record type comes first: a join trips this before the scan below, which is the message callers see - final var recordTypeName = pickOneRecordTypeName(merged.recordTypeName, childSpec.recordTypeName); + final var table = pickOneTable(merged.table, childSpec.table); Assert.thatUnchecked(merged.scanCount == 0 || childSpec.scanCount == 0, ErrorCode.UNSUPPORTED_OPERATION, "Unsupported index definition, join indexes are not supported"); merged = new IndexSpec(merged.scanCount + childSpec.scanCount, - recordTypeName, + table, pickOne(merged.predicate, childSpec.predicate, "predicate"), pickOne(merged.groupBy, childSpec.groupBy, "group by expression"), pickOne(merged.orderBy, childSpec.orderBy, "sort expression"), null); @@ -270,7 +305,8 @@ private static IndexSpec merge(@Nonnull final List childSpecs) { } @Nullable - private static String pickOneRecordTypeName(@Nullable final String left, @Nullable final String right) { + private static RecordLayerTable pickOneTable(@Nullable final RecordLayerTable left, + @Nullable final RecordLayerTable right) { Assert.thatUnchecked(left == null || right == null, ErrorCode.UNSUPPORTED_OPERATION, "Unsupported query, expected to find exactly one type filter operator"); return left == null ? right : left; @@ -323,7 +359,7 @@ public List fieldValues() { } @Nonnull - private List versionValues() { + List versionValues() { return values.stream() .filter(value -> value instanceof FieldValue && value.getResultType().equals(PseudoField.ROW_VERSION.getType())) @@ -335,7 +371,9 @@ private List versionValues() { * The traversal. Each override visits its children, then applies what the node contributes; the checks every node * shares live in {@link #evaluateAtExpression}. */ - private record Visitor(@Nonnull QuantifierValues quantifierValues) implements SimpleExpressionVisitor { + private record Visitor(@Nonnull QuantifierValues quantifierValues, + @Nonnull RecordLayerSchemaTemplate.Builder schemaTemplateBuilder) + implements SimpleExpressionVisitor { @Nonnull @Override @@ -365,8 +403,9 @@ public IndexSpec visitLogicalTypeFilterExpression(@Nonnull final LogicalTypeFilt () -> String.format(Locale.ROOT, "Unsupported query, expected to find exactly one record type in type filter operator, however found %s", recordTypes.isEmpty() ? "nothing" : String.join(",", recordTypes))); + final var storageName = recordTypes.stream().findFirst().orElseThrow(); return evaluateAtExpression(expression, visitQuantifiers(expression)) - .withRecordTypeName(recordTypes.stream().findFirst().orElseThrow()); + .withTable(schemaTemplateBuilder.findTableByStorageName(storageName)); } @Nonnull diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/ddl/MaterializedViewIndexGenerator.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/ddl/MaterializedViewIndexGenerator.java index f17b670f795..102c6c97855 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/ddl/MaterializedViewIndexGenerator.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/ddl/MaterializedViewIndexGenerator.java @@ -26,6 +26,7 @@ import com.apple.foundationdb.record.metadata.IndexTypes; import com.apple.foundationdb.record.metadata.expressions.KeyExpression; import com.apple.foundationdb.record.query.plan.cascades.expressions.RelationalExpression; +import com.apple.foundationdb.record.query.plan.cascades.typing.Type; import com.apple.foundationdb.record.query.plan.cascades.values.RecordConstructorValue; import com.apple.foundationdb.record.query.plan.cascades.values.Value; import com.apple.foundationdb.relational.api.exceptions.ErrorCode; @@ -33,10 +34,8 @@ import com.apple.foundationdb.relational.recordlayer.metadata.RecordLayerSchemaTemplate; import com.apple.foundationdb.relational.util.Assert; import com.apple.foundationdb.relational.util.NullableArrayUtils; -import com.google.common.collect.ImmutableList; import javax.annotation.Nonnull; -import java.util.List; import java.util.Map; import static com.apple.foundationdb.record.metadata.Key.Expressions.keyWithValue; @@ -91,16 +90,23 @@ public static MaterializedViewIndexGenerator newInstance(@Nonnull RelationalExpr @Nonnull @Override - public RecordLayerIndex.Builder generate() { - final var spec = IndexSpec.collect(relationalExpression, - QuantifierValues.collect(relationalExpression)); + public IndexGenerationResult generate() { + final var quantifierValues = QuantifierValues.collect(relationalExpression); + var spec = IndexSpec.collect(relationalExpression, quantifierValues, schemaTemplateBuilder); spec.checkValidity(); - - final var translation = translateToKeyExpression(spec); + final var unnestedTableGeneratorMaybe = RecordLayerUnnestedSyntheticTableGenerator.initIfNeeded( + spec, indexName, quantifierValues); + final Type.Record tableType; + if (unnestedTableGeneratorMaybe.isPresent()) { + final var unnestedTableGenerator = unnestedTableGeneratorMaybe.get(); + unnestedTableGenerator.checkSupported(spec); + spec = unnestedTableGenerator.rewrite(spec); + tableType = unnestedTableGenerator.getSyntheticType(); + } else { + tableType = spec.table().getType(); + } + final var translation = translateToKeyExpression(spec, unnestedTableGeneratorMaybe.isEmpty()); final var indexType = translation.indexType(); - // the record layer indexes by storage name - final var tableType = schemaTemplateBuilder.findTableByStorageName(spec.recordTypeName()).getType(); - final var indexBuilder = RecordLayerIndex.newBuilder() .setName(indexName) .setTableType(tableType) @@ -119,33 +125,28 @@ public RecordLayerIndex.Builder generate() { } indexBuilder.setKeyExpression(KeyExpression.fromProto( NullableArrayUtils.wrapArray(keyExpression.toKeyExpression(), tableType, options.containsNullableArray()))); - return indexBuilder; + return new IndexGenerationResult(indexBuilder, + unnestedTableGeneratorMaybe.map(RecordLayerUnnestedSyntheticTableGenerator::generate).orElse(null)); } /** * Translates the projection into the index key, columns in key order: the order-by columns lead a value index, while * an aggregate index keeps the projection's order. + * + * @param spec what the index is made of, already rewritten onto the synthetic table if there is one + * @param allowCollapsing whether a run of adjacent field paths may merge into a single navigation, which is what a + * fan-out on a stored table wants and what an index over constituent aliases must not do + * + * @return the index key and the index type it implies */ @Nonnull - private ValueToKeyExpressionVisitor.Result translateToKeyExpression(@Nonnull final IndexSpec spec) { - final var projection = spec.projection(); - final var isAggregate = projection.aggregate() != null; - final var reorderedValues = isAggregate ? projection.values() - : reorderValues(projection.fieldValues(), spec.getOrderByValues()); - return ValueToKeyExpressionVisitor.translate(RecordConstructorValue.ofUnnamed(reorderedValues), - isAggregate ? Map.of() : spec.getOrderingFunctions(), options.extremumEverStorage()); - } - - @Nonnull - private static List reorderValues(@Nonnull final List allValues, @Nonnull final List keyValues) { - Assert.thatUnchecked(allValues.size() >= keyValues.size()); - if (keyValues.isEmpty()) { - return allValues; - } - final var valueValues = allValues.stream() - .filter(value -> !keyValues.contains(value)) - .collect(ImmutableList.toImmutableList()); - return ImmutableList.builder().addAll(keyValues).addAll(valueValues).build(); + private ValueToKeyExpressionVisitor.Result translateToKeyExpression(@Nonnull IndexSpec spec, boolean allowCollapsing) { + final var projectionValue = RecordConstructorValue.ofUnnamed(spec.rootValues()); + final var orderingFunctions = spec.projection().aggregate() != null ? + Map.of() : + spec.getOrderingFunctions(); + return ValueToKeyExpressionVisitor.translate(projectionValue, orderingFunctions, + options.extremumEverStorage(), allowCollapsing); } /** diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/ddl/OnSourceIndexGenerator.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/ddl/OnSourceIndexGenerator.java index ca50f21468f..460ea038a17 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/ddl/OnSourceIndexGenerator.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/ddl/OnSourceIndexGenerator.java @@ -158,11 +158,12 @@ private OnSourceIndexGenerator(@Nonnull final Identifier indexName, @Nonnull fin * The generated index will be ordered according to the key columns and can optionally enforce uniqueness * if configured via {@link IndexGenerationOptions#unique()} flag. * - * @return a fully configured {@link RecordLayerIndex} ready to be added to the schema + * @return the generated index, together with the unnested synthetic table to define it on when the + * source unnests a struct array in a way a fan-out cannot express */ @Nonnull @Override - public RecordLayerIndex.Builder generate() { + public IndexGenerationResult generate() { final var keyIdentifiers = keyColumns.stream().map(IndexedColumn::identifier).collect(ImmutableList.toImmutableList()); final var keyIdentifiersAsSet = ImmutableSet.copyOf(keyIdentifiers); final var valueIdentifiers = valueColumns.stream().map(IndexedColumn::identifier) @@ -219,9 +220,9 @@ public RecordLayerIndex.Builder generate() { final var indexGenerator = MaterializedViewIndexGenerator.newInstance( indexPlan.getQuantifier().getRangesOver().get(), metadataBuilder, indexName.toString(), options); - final var indexMetadata = indexGenerator.generate(); - indexMetadata.addAllOptions(indexOptions); - return indexMetadata; + final var result = indexGenerator.generate(); + result.indexBuilder().addAllOptions(indexOptions); + return result; } public record IndexedColumn(@Nonnull Identifier identifier, boolean isDescending, boolean isNullsLast) { diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/ddl/QuantifierValues.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/ddl/QuantifierValues.java index 1e98cfa5aa1..e6af76484fe 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/ddl/QuantifierValues.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/ddl/QuantifierValues.java @@ -41,7 +41,6 @@ import java.util.Map; import java.util.Objects; import java.util.Set; -import java.util.concurrent.atomic.AtomicInteger; /** * What every quantifier of an index-defining plan stands for, and with it the ability to resolve a value written in terms @@ -52,8 +51,27 @@ final class QuantifierValues { @Nonnull private final Map valuesByQuantifier; - private QuantifierValues(@Nonnull final Map valuesByQuantifier) { + /** + * The array each explode of the plan ranges over, in the order they were found. An explode's {@link AnnotatedAccessor} + * marker is its position here. + */ + @Nonnull + private final List explodes; + + private QuantifierValues(@Nonnull final Map valuesByQuantifier, + @Nonnull final List explodes) { this.valuesByQuantifier = valuesByQuantifier; + this.explodes = explodes; + } + + /** + * The array each explode of the plan ranges over, in the order they were found. + * + * @return what the traversal saw at each explode + */ + @Nonnull + public List getExplodes() { + return explodes; } /** @@ -65,7 +83,8 @@ private QuantifierValues(@Nonnull final Map values */ @Nonnull public static QuantifierValues collect(@Nonnull final RelationalExpression expression) { - return new QuantifierValues(Assert.notNullUnchecked(new Collector().visit(expression))); + final var collector = new Collector(); + return new QuantifierValues(collector.visit(expression), collector.explodes); } /** @@ -112,11 +131,12 @@ public Value visitQuantifiedObjectValue(@Nonnull final QuantifiedObjectValue ele private static final class Collector implements SimpleExpressionVisitor> { /** - * Numbers the unnestings, so that two unnestings of the same array field compare unequal. Only distinctness - * matters; the number never reaches the key expression. + * The array each explode ranges over, in the order they were found. A position doubles as the explode's marker, + * which only has to be distinct -- it numbers the unnestings so two unnestings of one array field compare + * unequal, and never reaches the key expression. */ @Nonnull - private final AtomicInteger explodeCounter = new AtomicInteger(0); + private final List explodes = new ArrayList<>(); @Nonnull @Override @@ -142,16 +162,17 @@ public Map evaluateAtRef(@Nonnull final Reference @Nonnull private Value unnestedCollectionValue(@Nonnull final ExplodeExpression explode) { - final var marker = explodeCounter.incrementAndGet(); final var collectionValue = explode.getCollectionValue(); - if (!(collectionValue instanceof FieldValue)) { + if (!(collectionValue instanceof final FieldValue field)) { return collectionValue; } - final var field = (FieldValue)collectionValue; + final var marker = explodes.size(); final var fieldAccessors = new ArrayList<>(field.getFieldPath().getFieldAccessors()); fieldAccessors.set(fieldAccessors.size() - 1, AnnotatedAccessor.of(fieldAccessors.get(fieldAccessors.size() - 1), marker)); - return FieldValue.ofFields(field.getChild(), new FieldValue.FieldPath(fieldAccessors)); + final var annotated = FieldValue.ofFields(field.getChild(), new FieldValue.FieldPath(fieldAccessors)); + explodes.add(annotated); + return annotated; } @Nonnull @@ -165,6 +186,14 @@ private static Map merge(@Nonnull final List + * That tag is the marker: a small integer handed to each explode of the plan in the order the traversal finds them, + * so that it is also the explode's position in {@link QuantifierValues#getExplodes()}. It is stamped onto the last + * accessor of the field path that reaches the array, and so travels along with every value later built from that + * path; a plain {@link FieldValue.ResolvedAccessor} in that position means the field was not reached through an + * unnest at all. Two markers therefore mean two unnestings even where the field paths are identical, as in + * {@code FROM T1, T1.A X, T1.A Y}. A consumer recovers the markers a key column reads through and looks each one up + * against the explode it names, which is how it tells what a column was unnested from. */ static final class AnnotatedAccessor extends FieldValue.ResolvedAccessor { @@ -175,6 +204,15 @@ private AnnotatedAccessor(@Nonnull final Type.Record.Field field, final int ordi this.marker = marker; } + /** + * Which unnesting of the plan the field this accessor reaches was unnested by. + * + * @return the marker, which is that unnesting's position in {@link QuantifierValues#getExplodes()} + */ + int getMarker() { + return marker; + } + @Nonnull static AnnotatedAccessor of(@Nonnull final FieldValue.ResolvedAccessor accessor, final int marker) { return new AnnotatedAccessor(accessor.getField(), accessor.getOrdinal(), marker); diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/ddl/RecordLayerUnnestedSyntheticTableGenerator.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/ddl/RecordLayerUnnestedSyntheticTableGenerator.java new file mode 100644 index 00000000000..4b7c8468008 --- /dev/null +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/ddl/RecordLayerUnnestedSyntheticTableGenerator.java @@ -0,0 +1,556 @@ +/* + * RecordLayerUnnestedSyntheticTableGenerator.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2015-2026 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apple.foundationdb.relational.recordlayer.query.ddl; + +import com.apple.foundationdb.record.metadata.UnnestedRecordType; +import com.apple.foundationdb.record.metadata.expressions.KeyExpression; +import com.apple.foundationdb.record.query.plan.cascades.typing.Type; +import com.apple.foundationdb.record.query.plan.cascades.values.FieldValue; +import com.apple.foundationdb.record.query.plan.cascades.values.QueriedValue; +import com.apple.foundationdb.record.query.plan.cascades.values.SimpleValueVisitor; +import com.apple.foundationdb.record.query.plan.cascades.values.Value; +import com.apple.foundationdb.relational.api.exceptions.ErrorCode; +import com.apple.foundationdb.relational.api.exceptions.RelationalException; +import com.apple.foundationdb.relational.api.metadata.DataType; +import com.apple.foundationdb.relational.recordlayer.metadata.DataTypeUtils; +import com.apple.foundationdb.relational.recordlayer.metadata.RecordLayerTable; +import com.apple.foundationdb.relational.recordlayer.metadata.RecordLayerSyntheticTable; +import com.apple.foundationdb.relational.recordlayer.metadata.RecordLayerUnnestedSyntheticTable; + +import java.util.function.Supplier; +import com.google.common.base.Suppliers; +import com.apple.foundationdb.relational.util.Assert; +import com.apple.foundationdb.relational.util.NullableArrayUtils; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.util.IdentityHashMap; +import java.util.LinkedHashMap; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; + +/** + * Generates the unnested synthetic table an index has to be defined on. + *

+ * An index with an unnesting can be maintained from the stored table, with a fan-out key expression. This only works + * while every column read through one unnesting sits in a contiguous run of the index key, because those columns are + * emitted under a single navigation into the array. Two or more columns reached through the same unnesting at + * non-adjacent positions cannot be: a fan-out would have to be emitted twice and would range over the array twice. + * Those indexes are defined on a synthetic table instead, whose constituents are navigated with + * {@link KeyExpression.FanType#None} and so may be referenced at any number of key positions. + *

+ * {@link #initIfNeeded} answers whether an index needs one at all, and yields a generator only when it does. Nothing is + * built until {@link #generate} is called, so an index on a stored table costs no more than the decision. What cannot be + * defined on a synthetic table is rejected by {@link #checkSupported}, separately from the definitions + * {@link IndexSpec#checkValidity} rejects for any index. + */ +final class RecordLayerUnnestedSyntheticTableGenerator { + + /** + * Prefixes the name of a synthetic table, keeping it out of the space of names a user can declare. + */ + private static final String UNNESTED_TABLE_NAME_PREFIX = "__unnested_"; + + /** + * Alias of the parent (stored record) constituent. Constituent aliases are persisted in the metadata, so they are + * derived from the index definition alone rather than taken from the plan's correlation identifiers, whose values + * depend on how many quantifiers the JVM has allocated and so differ between runs of the same DDL. + */ + private static final String PARENT_CONSTITUENT_ALIAS = "parent"; + + /** + * Prefixes the alias of each nested constituent, which is numbered by the order its unnesting was found in. The + * record layer reserves {@code "__"} for constituent names of its own, so this cannot carry that prefix. + */ + private static final String NESTED_CONSTITUENT_ALIAS_PREFIX = "unnesting_"; + + /** + * Name of the struct that holds the constituent positions. The descriptor nests it inside the synthetic message, so + * the name only has to be unique within it; it matches what {@code UnnestedRecordTypeBuilder} calls it. + */ + private static final String POSITIONS_TYPE_NAME = "Positions"; + + /** + * Every unnesting the plan performs, composed from what {@link QuantifierValues} recorded, keyed by marker. + */ + @Nonnull + private final Map unnestings; + + @Nonnull + private final String parentAlias; + + /** + * The stored table the synthetic table's parent constituent stands for. + */ + @Nonnull + private final RecordLayerTable parentTable; + + @Nonnull + private final String syntheticTableName; + + /** + * Computed once: the key is resolved against this type and it also names the table the index is on, so it is asked + * for more than once per index. + */ + @Nonnull + private final Supplier syntheticType; + + private RecordLayerUnnestedSyntheticTableGenerator(@Nonnull final Map unnestings, + @Nonnull final String parentAlias, + @Nonnull final RecordLayerTable parentTable, + @Nonnull final String syntheticTableName) { + // ImmutableMap, not Map.copyOf: the constituents are registered and the synthetic type's fields are + // numbered by iterating this map, so its insertion (marker) order is load-bearing, and Map.copyOf makes + // no order guarantee. + this.unnestings = ImmutableMap.copyOf(unnestings); + this.parentAlias = parentAlias; + this.parentTable = parentTable; + this.syntheticTableName = syntheticTableName; + this.syntheticType = Suppliers.memoize(this::computeSyntheticType); + } + + /** + * A generator for the index's synthetic table, if it needs one at all. + *

+ * Every unnesting a column is read through counts, not only the innermost, so chained unnesting can require a + * synthetic table even when no innermost unnesting is itself split. Only struct arrays are considered, since a + * scalar array cannot be a constituent. + * + * @param spec what the index is made of + * @param indexName the name the definition gives the index + * @param quantifierValues what the plan's quantifiers stand for, and the unnestings it performs + * + * @return a generator for the synthetic table, empty when the index is maintained from the stored table + */ + @Nonnull + static Optional initIfNeeded(@Nonnull final IndexSpec spec, + @Nonnull final String indexName, + @Nonnull final QuantifierValues quantifierValues) { + final var unnestings = composeUnnestings(quantifierValues); + if (!isNeededFor(spec, unnestings)) { + return Optional.empty(); + } + final var parentTable = spec.table(); + // Currently, the synthetic table name is derived from the record type. This may or may not be true in the + // future. + final var syntheticTableName = UNNESTED_TABLE_NAME_PREFIX + parentTable.getType().getName() + "_" + indexName; + return Optional.of(new RecordLayerUnnestedSyntheticTableGenerator(unnestings, PARENT_CONSTITUENT_ALIAS, + parentTable, syntheticTableName)); + } + + /** + * The markers of every unnesting a value is read through, transitively, outermost first. A column carries the + * marker of each unnesting on its path. + *

+ * For a table {@code A(k, p P array)} whose element type holds {@code Q(y) array}, and the definition + * 

+     * FROM A AS a, (SELECT * FROM a.p) AS b, (SELECT * FROM b.q) AS c
+     *
+ * If {@code a.p} is marked 1 and {@code b.q} is marked 2, each column resolves to a path whose + * {@link QuantifierValues.AnnotatedAccessor}s give, in the order the path walks them rather than in marker order: + * 
+     * c.y  resolves to  base().P.Q.Y  ->  [1, 2]
+     * b.x  resolves to  base().P.X    ->  [1]
+     * a.k  resolves to  base().K      ->  []
+     *
+ * + * @param value a value resolved down to the base record + * + * @return the markers traversed, empty if there are none + */ + @Nonnull + private static List unnestingMarkers(@Nonnull final Value value) { + final var markers = ImmutableList.builder(); + if (value instanceof FieldValue) { + for (final var accessor : ((FieldValue)value).getFieldPath().getFieldAccessors()) { + if (accessor instanceof QuantifierValues.AnnotatedAccessor) { + markers.add(((QuantifierValues.AnnotatedAccessor)accessor).getMarker()); + } + } + } + for (final var child : value.getChildren()) { + markers.addAll(unnestingMarkers(child)); + } + return markers.build(); + } + + /** + * Composes what {@link QuantifierValues} recorded at each explode into what the unnesting means. + * + * @param quantifierValues what the plan's quantifiers stand for, and what it explodes + * + * @return every unnesting the plan performs, keyed by marker + */ + @Nonnull + private static Map composeUnnestings(@Nonnull final QuantifierValues quantifierValues) { + // Only a struct array becomes a constituent, so only those are named, and they are numbered by their order among + // the constituents rather than by marker: markers count scalar unnestings too, which would leave gaps. + final var explodes = quantifierValues.getExplodes(); + final Map aliasByMarker = new LinkedHashMap<>(); + for (int marker = 0; marker < explodes.size(); marker++) { + if (arrayTypeOf(explodes.get(marker)).getElementType() instanceof Type.Record) { + aliasByMarker.put(marker, NESTED_CONSTITUENT_ALIAS_PREFIX + aliasByMarker.size()); + } + } + final var result = ImmutableMap.builder(); + // a marker is an explode's position, so the index is the key + for (int marker = 0; marker < explodes.size(); marker++) { + final var collectionValue = explodes.get(marker); + result.put(marker, new UnnestingInfo(aliasByMarker.get(marker), + owningAlias(marker, collectionValue, quantifierValues, aliasByMarker), + collectionValue.getFieldPath())); + } + return result.build(); + } + + /** + * The alias of the constituent that owns the array an explode ranges over: the innermost struct-array unnesting + * enclosing it, or the parent constituent when the array hangs off the stored record. Only a struct array is named, + * so an unnamed enclosing unnesting is a scalar one and cannot be the owner. + */ + @Nonnull + private static String owningAlias(final int marker, + @Nonnull final FieldValue collectionValue, + @Nonnull final QuantifierValues quantifierValues, + @Nonnull final Map aliasByMarker) { + final var markers = unnestingMarkers(quantifierValues.resolve(collectionValue)); + for (int i = markers.indexOf(marker) - 1; i >= 0; i--) { + final var enclosing = aliasByMarker.get(markers.get(i)); + if (enclosing != null) { + return enclosing; + } + } + return PARENT_CONSTITUENT_ALIAS; + } + + @Nonnull + private static Type.Array arrayTypeOf(@Nonnull final FieldValue collectionValue) { + return (Type.Array)collectionValue.getFieldPath().getLastFieldType(); + } + + /** + * One unnesting the plan performs. + * + *

A struct array becomes a constituent of the synthetic table, navigated by {@link #arrayElements()} from + * {@code owningAlias}. A scalar array cannot be a constituent, since its elements have no fields to reference, so + * the same expression is instead emitted as a fan-out inside the owning constituent. + * + * @param alias the constituent's alias, or {@code null} for a scalar array, which is not a constituent + * @param owningAlias the constituent the unnested array lives on + * @param arrayPath the path to the array being unnested, relative to the record that owns it -- the owning + * constituent, which is not necessarily the stored record. An array reached through non-repeated fields makes this + * more than one hop. + */ + private record UnnestingInfo(@Nullable String alias, @Nonnull String owningAlias, + @Nonnull FieldValue.FieldPath arrayPath) { + + @Nonnull + private Type.Array arrayType() { + return (Type.Array)arrayPath.getLastFieldType(); + } + + /** + * Navigates from the record owning the array to the array's elements. Unlike the index key, which is wrapped + * afterwards by {@link NullableArrayUtils#wrapArray}, this expression is metadata of its own and so is built + * with the array already in the form it is stored in. + * + * @return an expression reaching the array's elements from the record the path starts at + */ + @Nonnull + public KeyExpression arrayElements() { + final var accessors = arrayPath.getFieldAccessors(); + final var prefix = accessors.subList(0, accessors.size() - 1).stream() + .map(ValueToKeyExpressionVisitor::storageName) + .collect(ImmutableList.toImmutableList()); + return KeyExpression.fromPath(prefix, NullableArrayUtils.arrayElements( + ValueToKeyExpressionVisitor.storageName(accessors.get(accessors.size() - 1)), + arrayType().isNullable())); + } + + public boolean structArray() { + return arrayType().getElementType() instanceof Type.Record; + } + + @Nonnull + public DataType.StructType structElementType() { + return (DataType.StructType)DataTypeUtils.toRelationalType( + Objects.requireNonNull(arrayType().getElementType())); + } + } + + /** + * The same index, made of the synthetic table rather than the stored record: it is the record type the index is on, + * and the projection and ordering are resolved against it. + * + * @param spec what the index is made of, resolved against the stored record + * + * @return the same index, in the synthetic table's coordinates + */ + @Nonnull + public IndexSpec rewrite(@Nonnull final IndexSpec spec) { + Assert.isNullUnchecked(spec.predicate(), ErrorCode.UNSUPPORTED_OPERATION, + "predicate on an index over an unnested synthetic table"); + Assert.isNullUnchecked(spec.groupBy(), ErrorCode.UNSUPPORTED_OPERATION, + "group by on an index over an unnested synthetic table"); + // The slot names the stored table the index reads from, which the synthetic table is built over, so it carries + // through unchanged; what the index is defined on is the synthetic type, which the caller takes from here. + return new IndexSpec(spec.scanCount(), spec.table(), null, null, + spec.orderBy() == null ? null : rewrite(spec.orderBy()), + new IndexSpec.Projection(rewrite(spec.projection().values()))); + } + + /** + * Ordering, resolved against the synthetic table. + */ + @Nonnull + private IndexSpec.OrderBy rewrite(@Nonnull final IndexSpec.OrderBy orderBy) { + final var values = orderBy.values(); + final var rewritten = rewrite(values); + final Map functions = new IdentityHashMap<>(); + for (int i = 0; i < values.size(); i++) { + final var function = orderBy.orderingFunctions().get(values.get(i)); + if (function != null) { + functions.put(rewritten.get(i), function); + } + } + return new IndexSpec.OrderBy(rewritten, functions); + } + + /** + * Rewrites the columns from the stored record's coordinates into the synthetic table's, so that they can be + * translated as ordinary field paths. + * + * @param values the index key columns, resolved against the stored record + * + * @return the same columns, resolved against the synthetic table + */ + @Nonnull + private List rewrite(@Nonnull final List values) { + final var rewriter = new Rewriter(new QueriedValue(getSyntheticType())); + return values.stream() + .map(value -> Objects.requireNonNull(value.acceptVisitor(rewriter))) + .collect(ImmutableList.toImmutableList()); + } + + /** + * Re-roots a column at the synthetic table. Anything that is not a plain column reference reaches + * {@link #evaluateAtValue} and is rejected there. + */ + private final class Rewriter implements SimpleValueVisitor { + + @Nonnull + private final Value root; + + private Rewriter(@Nonnull final Value root) { + this.root = root; + } + + @Nonnull + @Override + public Value evaluateAtValue(@Nonnull final Value value, @Nonnull final List childResults) { + throw new RelationalException( + "Unsupported index definition, an index over an unnested synthetic table supports only plain column references", + ErrorCode.UNSUPPORTED_OPERATION).toUncheckedWrappedException(); + } + + @Nonnull + @Override + public Value visitFieldValue(@Nonnull final FieldValue fieldValue) { + final var accessors = fieldValue.getFieldPath().getFieldAccessors(); + int innermostIdx = -1; + for (int i = accessors.size() - 1; i >= 0; i--) { + if (accessors.get(i) instanceof QuantifierValues.AnnotatedAccessor) { + innermostIdx = i; + break; + } + } + final var names = ImmutableList.builder(); + final List remaining; + @Nullable final UnnestingInfo innermost; + if (innermostIdx < 0) { + // a column of the stored record itself + innermost = null; + names.add(parentAlias); + remaining = accessors; + } else { + final var marker = ((QuantifierValues.AnnotatedAccessor)accessors.get(innermostIdx)).getMarker(); + innermost = Assert.notNullUnchecked(unnestings.get(marker), "unknown unnesting in index definition"); + if (innermost.structArray()) { + // the unnesting is a constituent: name it, and continue from its element type + names.add(innermost.alias()); + remaining = accessors.subList(innermostIdx + 1, accessors.size()); + } else { + // A scalar array is not a constituent: it stays a field of whichever constituent owns it, reached by + // the path the unnesting itself walks from that constituent. That path is the array's own field only + // when the array hangs directly off the owning record -- a non-repeated struct in between + // contributes hops that taking the array's accessor alone would drop. Nothing follows the array, + // since a scalar element has no fields to reference. + names.add(innermost.owningAlias()); + remaining = innermost.arrayPath().getFieldAccessors(); + } + } + remaining.forEach(accessor -> names.add(accessor.getField().getFieldName())); + final var rewritten = FieldValue.ofFieldNames(root, names.build()); + if (innermost == null || innermost.structArray()) { + return rewritten; + } + // Re-tag the scalar array's accessor: the marker is what says this array is reached through an unnest, which is + // what lets it be emitted as a fan-out rather than rejected. + final var rewrittenAccessors = new ArrayList<>(rewritten.getFieldPath().getFieldAccessors()); + final var last = rewrittenAccessors.size() - 1; + rewrittenAccessors.set(last, QuantifierValues.AnnotatedAccessor.of(rewrittenAccessors.get(last), + ((QuantifierValues.AnnotatedAccessor)accessors.get(innermostIdx)).getMarker())); + return FieldValue.ofFields(rewritten.getChild(), new FieldValue.FieldPath(rewrittenAccessors)); + } + } + + @Nonnull + Type.Record getSyntheticType() { + return syntheticType.get(); + } + + @Nonnull + private Type.Record computeSyntheticType() { + final var parentType = parentTable.getDatatype(); + final var fields = ImmutableList.builder(); + int fieldNumber = 1; + fields.add(DataType.StructType.Field.from(parentAlias, parentType, fieldNumber++)); + final var positions = ImmutableList.builder(); + int positionNumber = 1; + for (final var info : unnestings.values()) { + if (info.structArray()) { + // Nullable, because the descriptor declares every constituent field optional. The composed type has to + // agree with it, or a reloaded template's synthetic table is unequal to the one the DDL built. + fields.add(DataType.StructType.Field.from(info.alias(), + info.structElementType().withNullable(true), fieldNumber++)); + positions.add(DataType.StructType.Field.from(info.alias(), + DataType.Primitives.NULLABLE_LONG.type(), positionNumber++)); + } + } + fields.add(DataType.StructType.Field.from(UnnestedRecordType.POSITIONS_FIELD, + DataType.StructType.from(POSITIONS_TYPE_NAME, positions.build(), true), fieldNumber)); + return (Type.Record)DataTypeUtils.toRecordLayerType( + DataType.StructType.from(syntheticTableName, fields.build(), false)); + } + + /** + * Builds the synthetic table: the stored record as parent constituent, and one nested constituent per unnested + * struct array, each navigated from the constituent that owns its array. + * + * @return the synthetic table, which the caller has to register alongside the index + */ + @Nonnull + public RecordLayerSyntheticTable.Builder generate() { + final var builder = RecordLayerUnnestedSyntheticTable.newBuilder(syntheticType.get()) + .setAlias(parentAlias) + .setParentTableType(parentTable.getType()); + unnestings.values().stream() + .filter(UnnestingInfo::structArray) + .forEach(info -> builder.addConstituent(new RecordLayerUnnestedSyntheticTable.NestedConstituent( + info.alias(), info.owningAlias(), info.arrayElements()))); + return builder; + } + + /** + * Rejects what a stored table can express but an unnested synthetic table cannot. Separate from + * {@link IndexSpec#checkValidity}, which rejects what cannot become an index at all: everything here is a definition + * that would be accepted on index over stored table, so it is only asked once the shape is known to need a + * synthetic table. + * + * @param spec what the index is made of + */ + void checkSupported(@Nonnull final IndexSpec spec) { + final var projection = spec.projection(); + Assert.thatUnchecked(projection.aggregate() == null, + ErrorCode.UNSUPPORTED_OPERATION, + "Unsupported index definition, an aggregate cannot be defined on an unnested synthetic table"); + // The row version can technically refer to that of the parent here. However, disallowing for now. + Assert.thatUnchecked(projection.versionValues().isEmpty(), + ErrorCode.UNSUPPORTED_OPERATION, + "Unsupported index definition, a version column cannot be part of an index over an unnested synthetic table"); + Assert.thatUnchecked(scalarUnnestingsReferencedOnce(spec.rootValues()), + ErrorCode.UNSUPPORTED_OPERATION, + "Unsupported index definition, a scalar array cannot be referenced at more than one index key position"); + // A predicate would have to be evaluated against the synthetic record rather than the stored one, which is + // not worked out yet. Rejected rather than falling back to a fan-out, which cannot express these shapes and + // so would fail later with a less clear error. + Assert.thatUnchecked(spec.predicate() == null, ErrorCode.UNSUPPORTED_OPERATION, + "Unsupported index definition, a predicate is not supported on an index over an unnested synthetic table"); + } + + /** + * Whether every scalar unnesting the key reads through is referenced at no more than one key position. A scalar + * array cannot be a constituent, so each reference is emitted as its own fan-out over the array; two of them would + * range over it independently and yield a cross-product of one view column against itself, which nothing else on + * this path would catch. An index on the stored table is left to the trie's disconnected-reference guard, which + * rejects the same shape there. + * + * @param values the index key columns + * + * @return whether no scalar unnesting is referenced twice + */ + private boolean scalarUnnestingsReferencedOnce(@Nonnull final List values) { + // Distinct per position, since one value can read through the same unnesting more than once (e.g. `M.x + M.y`); + // what is counted has to be a number of key positions. + final var scalarMarkers = values.stream() + .flatMap(value -> unnestingMarkers(value).stream().distinct()) + .filter(marker -> { + final var info = unnestings.get(marker); + return info == null || !info.structArray(); + }) + .collect(ImmutableList.toImmutableList()); + return scalarMarkers.size() == ImmutableSet.copyOf(scalarMarkers).size(); + } + + /** + * Whether the index key reads two or more columns through one unnesting at non-adjacent positions, and so has to be + * defined on a synthetic table. + */ + private static boolean isNeededFor(@Nonnull final IndexSpec spec, + @Nonnull final Map unnestings) { + final Map firstPositions = new LinkedHashMap<>(); + final Map lastPositions = new LinkedHashMap<>(); + final Map counts = new LinkedHashMap<>(); + final List keyValues = spec.rootValues(); + for (int i = 0; i < keyValues.size(); i++) { + // Distinct markers per position: the counts below must be a number of key positions, and one value can + // read through the same unnesting more than once (e.g. `M.x + M.y`). + for (final var marker : ImmutableSet.copyOf(unnestingMarkers(keyValues.get(i)))) { + final var info = unnestings.get(marker); + // skip scalar arrays, which cannot be constituents + if (info == null || !info.structArray()) { + continue; + } + firstPositions.putIfAbsent(marker, i); + lastPositions.put(marker, i); + counts.merge(marker, 1, Integer::sum); + } + } + return counts.entrySet().stream().anyMatch(entry -> + lastPositions.get(entry.getKey()) - firstPositions.get(entry.getKey()) + 1 != entry.getValue()); + } +} diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/ddl/ValueToKeyExpressionVisitor.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/ddl/ValueToKeyExpressionVisitor.java index 476d6fc35b9..30fc80358db 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/ddl/ValueToKeyExpressionVisitor.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/ddl/ValueToKeyExpressionVisitor.java @@ -76,6 +76,26 @@ * aggregate index is everything in the projection other than the single aggregate. Sort direction comes from the caller, * as an ordering function per column. *

+ *

+ * Collapsing. Key columns whose field paths share a prefix can be written in either of two ways. Collapsed, the + * shared prefix is navigated once and the columns hang underneath it; uncollapsed, every column is navigated from the + * root on its own: + *

+ * 
{@code
+ * // collapsed
+ * field("A", FanOut).nest(concat(field("COL2"), field("COL3")))
+ * // uncollapsed
+ * concat(field("A", FanOut).nest(field("COL2")),
+ *        field("A", FanOut).nest(field("COL3")))
+ * }
+ *

+ * The two diverge as soon as the shared prefix crosses a repeated field, since a navigation carrying a {@code FanOut} + * enumerates the array it steps into: collapsed, a single enumeration puts {@code COL2} and {@code COL3} of the same + * element in one entry; uncollapsed, two enumerations range independently and the entries are their cross product. + * Crossing no repeated field, the two stand for the same entries and differ only in shape. Collapsing also groups by + * prefix rather than by position, so it keeps the requested key order only where the columns sharing a prefix are + * adjacent. Which form is wanted does not follow from the values alone, so it is the caller's to choose. + *

*/ final class ValueToKeyExpressionVisitor implements SimpleValueVisitor { @@ -108,10 +128,18 @@ final class ValueToKeyExpressionVisitor implements SimpleValueVisitor orderingFunctions, - @Nonnull final ExtremumEverStorage extremumEverStorage) { + @Nonnull final ExtremumEverStorage extremumEverStorage, + final boolean allowCollapsing) { this.orderingFunctions = orderingFunctions; this.extremumEverStorage = extremumEverStorage; + this.allowCollapsing = allowCollapsing; } // @@ -119,7 +147,10 @@ private ValueToKeyExpressionVisitor(@Nonnull final Map orderingFu // /** - * Translates the result value of an index-defining select to the corresponding key expression and index type. + * Translates the result value of an index-defining select to the corresponding key expression and index type, for an + * index maintained from a stored table. A run of adjacent field paths is collapsed under its shared prefix, which is + * what a fan-out over a repeated field wants; {@link #translate(Value, Map, ExtremumEverStorage, boolean)} is the + * form to use when it must not be. * * @param value the result value of the select * @param orderingFunctions the ordering function per column, keyed by identity on the columns of {@code value} @@ -131,7 +162,27 @@ private ValueToKeyExpressionVisitor(@Nonnull final Map orderingFu public static Result translate(@Nonnull final Value value, @Nonnull final Map orderingFunctions, @Nonnull final ExtremumEverStorage extremumEverStorage) { - final var visitor = new ValueToKeyExpressionVisitor(orderingFunctions, extremumEverStorage); + return translate(value, orderingFunctions, extremumEverStorage, true); + } + + /** + * Translates the result value of an index-defining select to the corresponding key expression and index type. + * + * @param value the result value of the select + * @param orderingFunctions the ordering function per column, keyed by identity on the columns of {@code value} + * @param extremumEverStorage which form an extremum-ever aggregate is stored in + * @param allowCollapsing whether a run of adjacent field paths may be collapsed under its shared prefix, as the class + * javadoc describes: true for an index on a stored table, where the collapsed navigation is the fan-out; false for + * one on an unnested synthetic table, whose constituents already hold one element each + * + * @return the key expression and the index type + */ + @Nonnull + public static Result translate(@Nonnull final Value value, + @Nonnull final Map orderingFunctions, + @Nonnull final ExtremumEverStorage extremumEverStorage, + final boolean allowCollapsing) { + final var visitor = new ValueToKeyExpressionVisitor(orderingFunctions, extremumEverStorage, allowCollapsing); return new Result(Objects.requireNonNull(value.acceptVisitor(visitor)), visitor.indexType); } @@ -277,8 +328,10 @@ public KeyExpression evaluateAtValue(@Nonnull final Value value, @Nonnull final // /** - * Combines sibling columns into one key expression. Adjacent {@link FieldValue}s nest under their shared prefix: - * {@code r.s.a, r.s.b} becomes {@code field("R").nest(concat(A, B))}. + * Combines sibling columns into one key expression, one component per column in key order, except that a run of + * adjacent {@link FieldValue}s may be collapsed under its shared prefix: {@code r.s.a, r.s.b} becomes + * {@code field("R").nest(concat(A, B))}. See the class javadoc for why collapsing is what an index on a stored table + * wants and what an index on an unnested synthetic table must not do. */ @Nonnull private KeyExpression combine(@Nonnull final List values) { @@ -288,14 +341,26 @@ private KeyExpression combine(@Nonnull final List values) { if (values.size() == 1) { return ordered(values.get(0)); } - // a run of adjacent field values forms one component; any other value forms its own - final List tries = new ArrayList<>(values.size()); final List components = new ArrayList<>(values.size()); - final PeekingIterator valueIterator = Iterators.peekingIterator(values.iterator()); - while (valueIterator.hasNext()) { - components.add(valueIterator.peek() instanceof FieldValue - ? nextFieldPaths(valueIterator, tries) - : ordered(valueIterator.next())); + if (allowCollapsing) { + // a run of adjacent field values forms one component; any other value forms its own + final List tries = new ArrayList<>(values.size()); + final PeekingIterator valueIterator = Iterators.peekingIterator(values.iterator()); + while (valueIterator.hasNext()) { + components.add(valueIterator.peek() instanceof FieldValue + ? nextFieldPaths(valueIterator, tries) + : ordered(valueIterator.next())); + } + } else { + // Without the tries, the check they carry out -- that no field path is referenced from two disconnected key + // positions -- is not made here. For a constituent it is inverted anyway: referencing one twice is the whole + // point of the synthetic table. For a scalar array, which cannot be a constituent and so stays a fan-out, it + // still has to hold, and RecordLayerUnnestedSyntheticTableGenerator#checkSupported makes it instead. That + // leaves a non-repeated column at two key positions, which the tries reject only as a collision of map keys: + // with no fan-out involved, it is redundant rather than wrong. + for (final Value value : values) { + components.add(ordered(value)); + } } return concatOf(components); } @@ -383,10 +448,8 @@ private KeyExpression fieldAccessorToKeyExpression(@Nonnull final FieldValue.Res indexType = IndexTypes.VERSION; return VersionKeyExpression.VERSION; } - // Protobuf storage references the storage name - final var storageName = Assert.notNullUnchecked(recordField.getFieldStorageName()); if (!recordField.getFieldType().isArray()) { - return field(storageName, KeyExpression.FanType.None); + return field(storageName(accessor), KeyExpression.FanType.None); } // an array is indexable only through an unnest, which tags its accessor, or materialized whole Assert.thatUnchecked(accessor instanceof QuantifierValues.AnnotatedAccessor @@ -394,7 +457,7 @@ private KeyExpression fieldAccessorToKeyExpression(@Nonnull final FieldValue.Res ErrorCode.UNSUPPORTED_OPERATION, "Unsupported index definition, cannot create index on array field '" + recordField.getFieldName() + "' without unnesting"); - return field(storageName, fanTypeForArray); + return field(storageName(accessor), fanTypeForArray); } private static boolean isRowVersion(@Nonnull final Type.Record.Field recordField) { @@ -402,6 +465,19 @@ private static boolean isRowVersion(@Nonnull final Type.Record.Field recordField && PseudoField.ROW_VERSION.getFieldName().equals(recordField.getFieldName()); } + /** + * The name a field is referenced by in a key expression, which is its protobuf storage name rather than the name it + * was declared with. + * + * @param accessor one step of a field path + * + * @return the name to reference that field by + */ + @Nonnull + static String storageName(@Nonnull final FieldValue.ResolvedAccessor accessor) { + return Assert.notNullUnchecked(accessor.getField().getFieldStorageName()); + } + @Nonnull private static KeyExpression arguments(@Nonnull final List argumentList) { return argumentList.isEmpty() ? empty() : concatOf(argumentList); diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/BaseVisitor.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/BaseVisitor.java index 2525e5faf00..27454639207 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/BaseVisitor.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/BaseVisitor.java @@ -31,7 +31,6 @@ import com.apple.foundationdb.relational.api.metadata.DataType; import com.apple.foundationdb.relational.generated.RelationalParser; import com.apple.foundationdb.relational.generated.RelationalParserBaseVisitor; -import com.apple.foundationdb.relational.recordlayer.metadata.RecordLayerIndex; import com.apple.foundationdb.relational.recordlayer.metadata.RecordLayerInvokedRoutine; import com.apple.foundationdb.relational.recordlayer.metadata.RecordLayerSchemaTemplate; import com.apple.foundationdb.relational.recordlayer.metadata.RecordLayerTable; @@ -49,6 +48,7 @@ import com.apple.foundationdb.relational.recordlayer.query.ProceduralPlan; import com.apple.foundationdb.relational.recordlayer.query.QueryPlan; import com.apple.foundationdb.relational.recordlayer.query.SemanticAnalyzer; +import com.apple.foundationdb.relational.recordlayer.query.ddl.IndexGenerationResult; import com.apple.foundationdb.relational.recordlayer.query.functions.CompiledSqlFunction; import com.apple.foundationdb.relational.recordlayer.query.functions.SqlFunctionCatalog; import com.apple.foundationdb.relational.util.Assert; @@ -434,19 +434,19 @@ public DataType.Named visitEnumDefinition(@Nonnull RelationalParser.EnumDefiniti @Nonnull @Override - public RecordLayerIndex visitIndexAsSelectDefinition(@Nonnull RelationalParser.IndexAsSelectDefinitionContext ctx) { + public IndexGenerationResult visitIndexAsSelectDefinition(@Nonnull RelationalParser.IndexAsSelectDefinitionContext ctx) { return ddlVisitor.visitIndexAsSelectDefinition(ctx); } @Nonnull @Override - public RecordLayerIndex visitIndexOnSourceDefinition(@Nonnull RelationalParser.IndexOnSourceDefinitionContext ctx) { + public IndexGenerationResult visitIndexOnSourceDefinition(@Nonnull RelationalParser.IndexOnSourceDefinitionContext ctx) { return ddlVisitor.visitIndexOnSourceDefinition(ctx); } @Nonnull @Override - public RecordLayerIndex visitVectorIndexDefinition(final RelationalParser.VectorIndexDefinitionContext ctx) { + public IndexGenerationResult visitVectorIndexDefinition(final RelationalParser.VectorIndexDefinitionContext ctx) { return ddlVisitor.visitVectorIndexDefinition(ctx); } diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DdlVisitor.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DdlVisitor.java index 50161de3e04..7289f2410dc 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DdlVisitor.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DdlVisitor.java @@ -39,7 +39,6 @@ import com.apple.foundationdb.relational.generated.RelationalParser; import com.apple.foundationdb.relational.recordlayer.metadata.DataTypeUtils; import com.apple.foundationdb.relational.recordlayer.metadata.RecordLayerColumn; -import com.apple.foundationdb.relational.recordlayer.metadata.RecordLayerIndex; import com.apple.foundationdb.relational.recordlayer.metadata.RecordLayerInvokedRoutine; import com.apple.foundationdb.relational.recordlayer.metadata.RecordLayerSchemaTemplate; import com.apple.foundationdb.relational.recordlayer.metadata.RecordLayerTable; @@ -55,6 +54,7 @@ import com.apple.foundationdb.relational.recordlayer.query.SemanticAnalyzer; import com.apple.foundationdb.relational.recordlayer.query.ddl.ExtremumEverStorage; import com.apple.foundationdb.relational.recordlayer.query.ddl.IndexGenerationOptions; +import com.apple.foundationdb.relational.recordlayer.query.ddl.IndexGenerationResult; import com.apple.foundationdb.relational.recordlayer.query.ddl.IndexGenerator; import com.apple.foundationdb.relational.recordlayer.query.ddl.MaterializedViewIndexGenerator; import com.apple.foundationdb.relational.recordlayer.query.ddl.OnSourceIndexGenerator; @@ -257,7 +257,7 @@ public RecordLayerTable visitStructDefinition(@Nonnull RelationalParser.StructDe @Nonnull @Override - public RecordLayerIndex visitIndexAsSelectDefinition(@Nonnull RelationalParser.IndexAsSelectDefinitionContext indexDefinitionContext) { + public IndexGenerationResult visitIndexAsSelectDefinition(@Nonnull RelationalParser.IndexAsSelectDefinitionContext indexDefinitionContext) { final var indexId = visitUid(indexDefinitionContext.indexName); final var ddlCatalog = metadataBuilder.build(); @@ -272,12 +272,12 @@ public RecordLayerIndex visitIndexAsSelectDefinition(@Nonnull RelationalParser.I indexId.getName(), new IndexGenerationOptions(isUnique, containsNullableArray, false, ExtremumEverStorage.ofLegacyAttribute(useLegacyBasedExtremumEver))); Assert.thatUnchecked(viewPlan instanceof LogicalSortExpression, ErrorCode.INVALID_COLUMN_REFERENCE, "Cannot create index and order by an expression that is not present in the projection list"); - return generator.generate().build(); + return generator.generate(); } @Nonnull @Override - public RecordLayerIndex visitIndexOnSourceDefinition(@Nonnull final RelationalParser.IndexOnSourceDefinitionContext indexDefinitionContext) { + public IndexGenerationResult visitIndexOnSourceDefinition(@Nonnull final RelationalParser.IndexOnSourceDefinitionContext indexDefinitionContext) { final var ddlCatalog = metadataBuilder.build(); getDelegate().replaceSchemaTemplate(ddlCatalog); getDelegate().pushPlanFragment(); @@ -310,12 +310,12 @@ public RecordLayerIndex visitIndexOnSourceDefinition(@Nonnull final RelationalPa } getDelegate().popPlanFragment(); - return indexGeneratorBuilder.build().generate().build(); + return indexGeneratorBuilder.build().generate(); } @Nonnull @Override - public RecordLayerIndex visitVectorIndexDefinition(final RelationalParser.VectorIndexDefinitionContext indexDefinitionContext) { + public IndexGenerationResult visitVectorIndexDefinition(final RelationalParser.VectorIndexDefinitionContext indexDefinitionContext) { final var ddlCatalog = metadataBuilder.build(); getDelegate().replaceSchemaTemplate(ddlCatalog); getDelegate().pushPlanFragment(); @@ -362,7 +362,9 @@ public RecordLayerIndex visitVectorIndexDefinition(final RelationalParser.Vector } getDelegate().popPlanFragment(); - return indexGeneratorBuilder.build().generate().setIndexType(IndexTypes.VECTOR).build(); + final var result = indexGeneratorBuilder.build().generate(); + result.indexBuilder().setIndexType(IndexTypes.VECTOR); + return result; } @Nonnull @@ -556,12 +558,8 @@ public ProceduralPlan visitCreateSchemaTemplateStatement(@Nonnull RelationalPars final var view = getViewMetadata(viewClause, metadataBuilder.build()); metadataBuilder.addView(view); }); - final var indexes = indexClauses.build().stream().map(clause -> Assert.castUnchecked(visit(clause), RecordLayerIndex.class)).collect(ImmutableList.toImmutableList()); - for (final RecordLayerIndex index : indexes) { - final var table = metadataBuilder.extractTable(index.getTableName()); - final var tableWithIndex = RecordLayerTable.Builder.from(table).addIndex(index).build(); - metadataBuilder.addTable(tableWithIndex); - } + indexClauses.build().forEach(clause -> + Assert.castUnchecked(visit(clause), IndexGenerationResult.class).registerOn(metadataBuilder)); return ProceduralPlan.of(metadataOperationsFactory.getSaveSchemaTemplateConstantAction(metadataBuilder.build(), Options.NONE)); } diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DelegatingVisitor.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DelegatingVisitor.java index 1630c5c86c2..bab3fd4d455 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DelegatingVisitor.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DelegatingVisitor.java @@ -26,7 +26,6 @@ import com.apple.foundationdb.record.util.pair.NonnullPair; import com.apple.foundationdb.relational.api.metadata.DataType; import com.apple.foundationdb.relational.generated.RelationalParser; -import com.apple.foundationdb.relational.recordlayer.metadata.RecordLayerIndex; import com.apple.foundationdb.relational.recordlayer.metadata.RecordLayerInvokedRoutine; import com.apple.foundationdb.relational.recordlayer.metadata.RecordLayerTable; import com.apple.foundationdb.relational.recordlayer.query.Expression; @@ -37,6 +36,7 @@ import com.apple.foundationdb.relational.recordlayer.query.WindowSpecExpression; import com.apple.foundationdb.relational.recordlayer.query.ProceduralPlan; import com.apple.foundationdb.relational.recordlayer.query.QueryPlan; +import com.apple.foundationdb.relational.recordlayer.query.ddl.IndexGenerationResult; import com.apple.foundationdb.relational.recordlayer.query.functions.CompiledSqlFunction; import org.antlr.v4.runtime.tree.ErrorNode; import org.antlr.v4.runtime.tree.ParseTree; @@ -577,19 +577,19 @@ public OrderByExpression visitOrderByExpression(@Nonnull RelationalParser.OrderB @Nonnull @Override - public RecordLayerIndex visitIndexAsSelectDefinition(@Nonnull RelationalParser.IndexAsSelectDefinitionContext ctx) { + public IndexGenerationResult visitIndexAsSelectDefinition(@Nonnull RelationalParser.IndexAsSelectDefinitionContext ctx) { return getDelegate().visitIndexAsSelectDefinition(ctx); } @Nonnull @Override - public RecordLayerIndex visitIndexOnSourceDefinition(@Nonnull RelationalParser.IndexOnSourceDefinitionContext ctx) { + public IndexGenerationResult visitIndexOnSourceDefinition(@Nonnull RelationalParser.IndexOnSourceDefinitionContext ctx) { return getDelegate().visitIndexOnSourceDefinition(ctx); } @Nonnull @Override - public RecordLayerIndex visitVectorIndexDefinition(final RelationalParser.VectorIndexDefinitionContext ctx) { + public IndexGenerationResult visitVectorIndexDefinition(final RelationalParser.VectorIndexDefinitionContext ctx) { return getDelegate().visitVectorIndexDefinition(ctx); } diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/TypedVisitor.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/TypedVisitor.java index e7870b56efd..f0e798b9ec6 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/TypedVisitor.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/TypedVisitor.java @@ -26,7 +26,6 @@ import com.apple.foundationdb.relational.api.metadata.DataType; import com.apple.foundationdb.relational.generated.RelationalParser; import com.apple.foundationdb.relational.generated.RelationalParserVisitor; -import com.apple.foundationdb.relational.recordlayer.metadata.RecordLayerIndex; import com.apple.foundationdb.relational.recordlayer.metadata.RecordLayerInvokedRoutine; import com.apple.foundationdb.relational.recordlayer.metadata.RecordLayerTable; import com.apple.foundationdb.relational.recordlayer.query.Expression; @@ -37,6 +36,7 @@ import com.apple.foundationdb.relational.recordlayer.query.WindowSpecExpression; import com.apple.foundationdb.relational.recordlayer.query.ProceduralPlan; import com.apple.foundationdb.relational.recordlayer.query.QueryPlan; +import com.apple.foundationdb.relational.recordlayer.query.ddl.IndexGenerationResult; import com.apple.foundationdb.relational.recordlayer.query.functions.CompiledSqlFunction; import javax.annotation.Nonnull; @@ -165,15 +165,15 @@ public interface TypedVisitor extends RelationalParserVisitor { @Nonnull @Override - RecordLayerIndex visitIndexAsSelectDefinition(@Nonnull RelationalParser.IndexAsSelectDefinitionContext ctx); + IndexGenerationResult visitIndexAsSelectDefinition(@Nonnull RelationalParser.IndexAsSelectDefinitionContext ctx); @Nonnull @Override - RecordLayerIndex visitIndexOnSourceDefinition(@Nonnull RelationalParser.IndexOnSourceDefinitionContext ctx); + IndexGenerationResult visitIndexOnSourceDefinition(@Nonnull RelationalParser.IndexOnSourceDefinitionContext ctx); @Nonnull @Override - RecordLayerIndex visitVectorIndexDefinition(RelationalParser.VectorIndexDefinitionContext ctx); + IndexGenerationResult visitVectorIndexDefinition(RelationalParser.VectorIndexDefinitionContext ctx); @Nonnull @Override diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/util/NullableArrayUtils.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/util/NullableArrayUtils.java index 65e85406c57..9e872f691b2 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/util/NullableArrayUtils.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/util/NullableArrayUtils.java @@ -22,6 +22,8 @@ import com.apple.foundationdb.annotation.API; import com.apple.foundationdb.record.expressions.RecordKeyExpressionProto; +import com.apple.foundationdb.record.metadata.Key; +import com.apple.foundationdb.record.metadata.expressions.KeyExpression; import com.apple.foundationdb.record.query.plan.cascades.typing.Type; import com.apple.foundationdb.record.query.plan.cascades.typing.TypeRepository; @@ -42,6 +44,23 @@ private NullableArrayUtils() { throw new IllegalStateException("Utility class"); } + /** + * Navigates from the record owning an array field to that array's elements. A nullable array is stored wrapped + * in a {@code { repeated T values; }} message, so the fan-out sits on {@code values} in that case; a + * non-nullable one is a plain repeated field. + * + * @param arrayFieldName the proto storage name of the array field + * @param nullableArray whether the array is stored wrapped + * @return an expression reaching the array's elements + */ + @Nonnull + public static KeyExpression arrayElements(@Nonnull final String arrayFieldName, final boolean nullableArray) { + return nullableArray + ? Key.Expressions.field(arrayFieldName) + .nest(Key.Expressions.field(REPEATED_FIELD_NAME, KeyExpression.FanType.FanOut)) + : Key.Expressions.field(arrayFieldName, KeyExpression.FanType.FanOut); + } + public static String getRepeatedFieldName() { return REPEATED_FIELD_NAME; } diff --git a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/api/ddl/DdlStatementParsingTest.java b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/api/ddl/DdlStatementParsingTest.java index 0b544bfdddd..77004fca9b5 100644 --- a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/api/ddl/DdlStatementParsingTest.java +++ b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/api/ddl/DdlStatementParsingTest.java @@ -54,7 +54,6 @@ import com.apple.foundationdb.relational.recordlayer.metadata.RecordLayerTable; import com.apple.foundationdb.relational.recordlayer.metric.StoreTimerMetricCollector; import com.apple.foundationdb.relational.recordlayer.query.Plan; -import com.apple.foundationdb.relational.recordlayer.query.PreparedParams; import com.apple.foundationdb.relational.recordlayer.util.ExceptionUtil; import com.apple.foundationdb.relational.util.Assert; import com.apple.foundationdb.relational.utils.PermutationIterator; @@ -166,17 +165,8 @@ void shouldFailWithInjectedFactory(@Nonnull final String query, @Nullable final void shouldWorkWithInjectedFactory(@Nonnull final String query, @Nonnull final MetadataOperationsFactory metadataOperationsFactory) throws Exception { - connection.setAutoCommit(false); - (connection.getUnderlyingEmbeddedConnection()).createNewTransaction(); - final var transaction = connection.getUnderlyingEmbeddedConnection().getTransaction(); - final var plan = DdlTestUtil.getPlanGenerator(connection.getUnderlyingEmbeddedConnection(), database.getSchemaTemplateName(), - "/DdlStatementParsingTest", metadataOperationsFactory, PreparedParams.empty(), - Options.builder().withOption(Options.Name.CASE_SENSITIVE_IDENTIFIERS, true).build()).getPlan(query); - // execute the plan so we run any extra test-driven verifications within the transactional closure. - plan.execute(Plan.ExecutionContext.of(transaction, Options.NONE, connection, - StoreTimerMetricCollector.fromFDBRecordContext(transaction.unwrap(RecordContextTransaction.class).getContext()))); - connection.rollback(); - connection.setAutoCommit(true); + DdlTestUtil.shouldWorkWithInjectedFactory(connection, database.getSchemaTemplateName(), + "/DdlStatementParsingTest", query, metadataOperationsFactory); } void shouldFailWithInjectedQueryFactory(@Nonnull final String query, @Nullable ErrorCode errorCode, @@ -1533,68 +1523,6 @@ public ConstantAction getSaveSchemaTemplateConstantAction(@Nonnull SchemaTemplat }); } - @Test - void createIndexOnRepeated() throws Exception { - final String schemaStatement = "CREATE SCHEMA TEMPLATE test_template " + - "CREATE TYPE AS STRUCT A(x bigint) " + - "CREATE TABLE T(p bigint, a A array, primary key(p)) " + - "CREATE VIEW mv1 AS SELECT SQ.x, t.p from T AS t, (select M.x from t.a AS M) SQ " + - "CREATE INDEX i1 on mv1(x, p)"; - - shouldWorkWithInjectedFactory(schemaStatement, new AbstractMetadataOperationsFactory() { - @Nonnull - @Override - public ConstantAction getSaveSchemaTemplateConstantAction(@Nonnull SchemaTemplate template, - @Nonnull Options templateProperties) { - final var tableMaybe = Assertions.assertDoesNotThrow(() -> template.findTableByName("T")); - assertThat(tableMaybe).isPresent(); - final var table = Assert.optionalUnchecked(tableMaybe); - assertThat(table.getIndexes().size()).isEqualTo(1); - final var index = Assert.optionalUnchecked(table.getIndexes().stream().findFirst()); - assertThat(index.getIndexType()).isEqualTo(IndexTypes.VALUE); - assertThat(index.getName()).isEqualTo("i1"); - assertThat(index).isInstanceOf(RecordLayerIndex.class); - final var recordLayerIndex = Assert.castUnchecked(index, RecordLayerIndex.class); - assertThat(recordLayerIndex.getKeyExpression()).isEqualTo( - Key.Expressions.concat(Key.Expressions.field("a", KeyExpression.FanType.None) - .nest(Key.Expressions.field("values", KeyExpression.FanType.FanOut).nest("x")), Key.Expressions.field("p"))); - return txn -> { - }; - } - }); - } - - @Test - void createIndexOnRepeatedUsingMatViewSyntax() throws Exception { - final String schemaStatement = "CREATE SCHEMA TEMPLATE test_template " + - "CREATE TYPE AS STRUCT A(x bigint) " + - "CREATE TABLE T(p bigint, a A array, primary key(p)) " + - "CREATE INDEX mv1 AS SELECT SQ.x, t.p from T AS t, (select M.x from t.a AS M) SQ order by SQ.x, t.p "; - - shouldWorkWithInjectedFactory(schemaStatement, new AbstractMetadataOperationsFactory() { - @Nonnull - @Override - public ConstantAction getSaveSchemaTemplateConstantAction(@Nonnull SchemaTemplate template, - @Nonnull Options templateProperties) { - final var tableMaybe = Assertions.assertDoesNotThrow(() -> template.findTableByName("T")); - assertThat(tableMaybe).isPresent(); - final var table = Assert.optionalUnchecked(tableMaybe); - assertThat(table.getIndexes().size()).isEqualTo(1); - final var index = Assert.optionalUnchecked(table.getIndexes().stream().findFirst()); - assertThat(index.getIndexType()).isEqualTo(IndexTypes.VALUE); - assertThat(index.getName()).isEqualTo("mv1"); - assertThat(index).isInstanceOf(RecordLayerIndex.class); - final var recordLayerIndex = Assert.castUnchecked(index, RecordLayerIndex.class); - assertThat(recordLayerIndex.getKeyExpression()).isEqualTo( - Key.Expressions.concat(Key.Expressions.field("a", KeyExpression.FanType.None) - .nest(Key.Expressions.field("values", KeyExpression.FanType.FanOut).nest("x")), Key.Expressions.field("p"))); - return txn -> { - }; - } - }); - } - - @Test void createIndexOnAggregate() throws Exception { final String schemaStatement = "CREATE SCHEMA TEMPLATE test_template " + diff --git a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/api/ddl/DdlTestUtil.java b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/api/ddl/DdlTestUtil.java index 7a6d7ae4560..6065afca949 100644 --- a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/api/ddl/DdlTestUtil.java +++ b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/api/ddl/DdlTestUtil.java @@ -26,8 +26,13 @@ import com.apple.foundationdb.record.RecordStoreState; import com.apple.foundationdb.record.provider.foundationdb.IndexMaintainerFactoryRegistryImpl; import com.apple.foundationdb.relational.api.Options; +import com.apple.foundationdb.relational.api.exceptions.ErrorCode; import com.apple.foundationdb.relational.api.exceptions.RelationalException; import com.apple.foundationdb.relational.recordlayer.EmbeddedRelationalConnection; +import com.apple.foundationdb.relational.recordlayer.RecordContextTransaction; +import com.apple.foundationdb.relational.recordlayer.RelationalConnectionRule; +import com.apple.foundationdb.relational.recordlayer.metric.StoreTimerMetricCollector; +import com.apple.foundationdb.relational.recordlayer.query.Plan; import com.apple.foundationdb.relational.recordlayer.ddl.NoOpMetadataOperationsFactory; import com.apple.foundationdb.relational.recordlayer.metadata.RecordLayerSchemaTemplate; import com.apple.foundationdb.relational.recordlayer.query.PlanContext; @@ -45,13 +50,78 @@ import java.util.ArrayList; import java.util.HashSet; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + public class DdlTestUtil { + /** + * Asserts that planning the given DDL is rejected, with the given error code and a message containing + * {@code errorMessage}. + * + * @param connection the connection to plan against + * @param schemaTemplateName the name of the schema template in the catalog + * @param databaseUri the database URI to plan against + * @param query the DDL statement expected to be rejected + * @param errorCode the expected error code + * @param errorMessage a substring the rejection message has to contain + * @throws Exception if anything other than planning fails + */ + static void shouldFailWith(@Nonnull final RelationalConnectionRule connection, + @Nonnull final String schemaTemplateName, + @Nonnull final String databaseUri, + @Nonnull final String query, + @Nonnull final ErrorCode errorCode, + @Nonnull final String errorMessage) throws Exception { + connection.setAutoCommit(false); + connection.getUnderlyingEmbeddedConnection().createNewTransaction(); + final RelationalException ve = assertThrows(RelationalException.class, () -> + getPlanGenerator(connection.getUnderlyingEmbeddedConnection(), schemaTemplateName, databaseUri) + .getPlan(query)); + assertEquals(errorCode, ve.getErrorCode()); + assertTrue(ve.getMessage().contains(errorMessage), + String.format(Locale.ROOT, "expected error message '%s' to contain '%s' but it didn't", + ve.getMessage(), errorMessage)); + connection.rollback(); + connection.setAutoCommit(true); + } + + /** + * Plans and executes the given DDL with an injected metadata factory, so that any assertions the + * factory makes run against the schema template the statement builds, inside the transaction. + * + * @param connection the connection to plan against + * @param schemaTemplateName the name of the schema template in the catalog + * @param databaseUri the database URI to plan against + * @param query the DDL statement + * @param metadataOperationsFactory the factory holding the assertions + * @throws Exception if planning or execution fails + */ + static void shouldWorkWithInjectedFactory(@Nonnull final RelationalConnectionRule connection, + @Nonnull final String schemaTemplateName, + @Nonnull final String databaseUri, + @Nonnull final String query, + @Nonnull final MetadataOperationsFactory metadataOperationsFactory) throws Exception { + connection.setAutoCommit(false); + connection.getUnderlyingEmbeddedConnection().createNewTransaction(); + final var transaction = connection.getUnderlyingEmbeddedConnection().getTransaction(); + final var plan = getPlanGenerator(connection.getUnderlyingEmbeddedConnection(), schemaTemplateName, databaseUri, + metadataOperationsFactory, PreparedParams.empty(), + Options.builder().withOption(Options.Name.CASE_SENSITIVE_IDENTIFIERS, true).build()).getPlan(query); + // execute the plan so we run any extra test-driven verifications within the transactional closure. + plan.execute(Plan.ExecutionContext.of(transaction, Options.NONE, connection, + StoreTimerMetricCollector.fromFDBRecordContext(transaction.unwrap(RecordContextTransaction.class).getContext()))); + connection.rollback(); + connection.setAutoCommit(true); + } + @Nonnull static PlanContext createVanillaPlanContext(@Nonnull final EmbeddedRelationalConnection connection, @Nonnull final String schemaTemplateName, diff --git a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/api/ddl/IndexTest.java b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/api/ddl/IndexTest.java index 4f16c260811..9de1e369b37 100644 --- a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/api/ddl/IndexTest.java +++ b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/api/ddl/IndexTest.java @@ -30,7 +30,6 @@ import com.apple.foundationdb.record.metadata.expressions.KeyWithValueExpression; import com.apple.foundationdb.relational.api.Options; import com.apple.foundationdb.relational.api.exceptions.ErrorCode; -import com.apple.foundationdb.relational.api.exceptions.RelationalException; import com.apple.foundationdb.relational.api.metadata.Index; import com.apple.foundationdb.relational.api.metadata.SchemaTemplate; import com.apple.foundationdb.relational.api.metadata.Table; @@ -81,7 +80,7 @@ public class IndexTest { @RegisterExtension @Order(2) - public final SimpleDatabaseRule database = new SimpleDatabaseRule(DdlStatementParsingTest.class, TestSchemas.books()); + public final SimpleDatabaseRule database = new SimpleDatabaseRule(IndexTest.class, TestSchemas.books()); @RegisterExtension @Order(3) @@ -93,17 +92,9 @@ public static void setup() { Utils.enableCascadesDebugger(); } - void shouldFailWith(@Nonnull final String query, @Nonnull final ErrorCode errorCode, @Nonnull final String errorMessage) throws Exception { - connection.setAutoCommit(false); - connection.getUnderlyingEmbeddedConnection().createNewTransaction(); - final RelationalException ve = Assertions.assertThrows(RelationalException.class, () -> - DdlTestUtil.getPlanGenerator(connection.getUnderlyingEmbeddedConnection(), database.getSchemaTemplateName(), - "/IndexTest").getPlan(query)); - Assertions.assertEquals(errorCode, ve.getErrorCode()); - Assertions.assertTrue(ve.getMessage().contains(errorMessage), String.format(Locale.ROOT, - "expected error message '%s' to contain '%s' but it didn't", ve.getMessage(), errorMessage)); - connection.rollback(); - connection.setAutoCommit(true); + void shouldFailWith(@Nonnull final String query, @Nonnull final ErrorCode errorCode, + @Nonnull final String errorMessage) throws Exception { + DdlTestUtil.shouldFailWith(connection, database.getSchemaTemplateName(), "/IndexTest", query, errorCode, errorMessage); } void shouldWorkWithInjectedFactory(@Nonnull final String query, @Nonnull final MetadataOperationsFactory metadataOperationsFactory) @@ -112,7 +103,8 @@ void shouldWorkWithInjectedFactory(@Nonnull final String query, @Nonnull final M connection.getUnderlyingEmbeddedConnection().createNewTransaction(); Assertions.assertDoesNotThrow(() -> DdlTestUtil.getPlanGenerator(connection.getUnderlyingEmbeddedConnection(), database.getSchemaTemplateName(), - "/IndexTest", metadataOperationsFactory).getPlan(query)); + "/IndexTest", metadataOperationsFactory) + .getPlan(query)); connection.rollback(); connection.setAutoCommit(true); } @@ -286,6 +278,21 @@ void createdIndexWorksDeepNestingAndNestedCartesianConcat() throws Exception { IndexTypes.VALUE); } + @Test + void createIndexWithChainedUnnestingAdjacentKeepsFanOut() throws Exception { + final String stmt = "CREATE SCHEMA TEMPLATE test_template " + + "CREATE TYPE AS STRUCT Q(y bigint, y2 bigint) " + + "CREATE TYPE AS STRUCT P(x bigint, x2 bigint, q Q array) " + + "CREATE TABLE A(k bigint, p P array, primary key(k)) " + + "CREATE INDEX mv1 AS SELECT b.x, c.y FROM A AS a, (select * from a.p) as b, (select * from b.q) as c " + + "ORDER BY b.x, c.y"; + indexIs(stmt, field("P", KeyExpression.FanType.None) + .nest(field(NullableArrayUtils.getRepeatedFieldName(), KeyExpression.FanType.FanOut) + .nest(concat(field("X"), field("Q", KeyExpression.FanType.None) + .nest(field(NullableArrayUtils.getRepeatedFieldName(), KeyExpression.FanType.FanOut) + .nest(field("Y")))))), IndexTypes.VALUE); + } + /** * Scalar array unnesting via correlated subquery: STRING ARRAY, INDEX…AS syntax. */ @@ -736,6 +743,20 @@ void createIndexWithNestedRepeatedSameParent() throws Exception { indexIs(stmt, keyWithValue(concat(field("COL5"), field("A").nest(field("values", KeyExpression.FanType.FanOut).nest(concatenateFields("COL3", "COL4")))), 2), IndexTypes.VALUE); } + @Test + void createIndexOverNestedRepeatedUnderPathAdjacentKeepsFanOut() throws Exception { + final String stmt = "CREATE SCHEMA TEMPLATE test_template " + + "CREATE TYPE AS STRUCT S1(a string, b string) " + + "CREATE TYPE AS STRUCT S2(x S1 array, y S1) " + + "CREATE TYPE AS STRUCT S3(alpha S2, beta S2) " + + "CREATE TABLE T(id bigint, fizz S3, buzz bigint, primary key(id)) " + + "CREATE INDEX mv1 AS SELECT u.a, u.b, T.buzz " + + "FROM T, (SELECT a, b FROM T.fizz.alpha.x) AS u ORDER BY u.a, u.b, T.buzz"; + indexIs(stmt, concat(field("FIZZ").nest(field("ALPHA").nest(field("X") + .nest(field("values", KeyExpression.FanType.FanOut).nest(concatenateFields("A", "B"))))), + field("BUZZ")), IndexTypes.VALUE); + } + @Test void createIndexWithNestedRepeatedCartesianProduct() throws Exception { final String stmt = "CREATE SCHEMA TEMPLATE test_template " + @@ -746,21 +767,60 @@ void createIndexWithNestedRepeatedCartesianProduct() throws Exception { } @Test - void createIndexWithRepeatedNestedSplitByField() throws Exception { + void createIndexWithRepeatedNestedCartesianSplitByField() throws Exception { final String stmt = "CREATE SCHEMA TEMPLATE test_template " + "CREATE TYPE AS STRUCT A(col2 string, col3 bigint, col4 bigint) " + "CREATE TABLE T1(col1 bigint, a A Array, col5 bigint, primary key(col1)) " + - "CREATE INDEX mv1 AS SELECT X.col2, T1.col5, X.col3, X.col4 FROM T1, (SELECT col2, col3, col4 FROM T1.A) X ORDER BY X.col2, T1.col5, X.col3"; - shouldFailWith(stmt, ErrorCode.UNSUPPORTED_OPERATION, "Index with multiple disconnected references to the same column are not supported"); + "CREATE INDEX mv1 AS SELECT Y.col2, T1.col5, X.col3, X.col4 FROM T1, (SELECT col3, col4 FROM T1.A) X, (SELECT col2 FROM T1.A) Y ORDER BY Y.col2, T1.col5, X.col3"; + indexIs(stmt, keyWithValue(concat(field("A").nest(field("values", KeyExpression.FanType.FanOut).nest("COL2")), field("COL5"), field("A").nest(field("values", KeyExpression.FanType.FanOut).nest(concatenateFields("COL3", "COL4")))), 3), IndexTypes.VALUE); } + /** + * The same columns and the same predicate as + * {@link UnnestedSyntheticTableIndexTest#createIndexWithPredicateOverUnnestedSyntheticTableIsNotSupported()}, but + * with the two columns of {@code X} made adjacent. That is expressible as a fan-out, so no synthetic type is needed + * and the predicate is accepted -- reordering the key alone decides whether the predicate is allowed. + */ @Test - void createIndexWithRepeatedNestedCartesianSplitByField() throws Exception { + void createIndexWithPredicateIsSupportedWhenUnnestingNeedsNoSyntheticTable() throws Exception { final String stmt = "CREATE SCHEMA TEMPLATE test_template " + "CREATE TYPE AS STRUCT A(col2 string, col3 bigint, col4 bigint) " + "CREATE TABLE T1(col1 bigint, a A Array, col5 bigint, primary key(col1)) " + - "CREATE INDEX mv1 AS SELECT Y.col2, T1.col5, X.col3, X.col4 FROM T1, (SELECT col3, col4 FROM T1.A) X, (SELECT col2 FROM T1.A) Y ORDER BY Y.col2, T1.col5, X.col3"; - indexIs(stmt, keyWithValue(concat(field("A").nest(field("values", KeyExpression.FanType.FanOut).nest("COL2")), field("COL5"), field("A").nest(field("values", KeyExpression.FanType.FanOut).nest(concatenateFields("COL3", "COL4")))), 3), IndexTypes.VALUE); + "CREATE INDEX mv1 AS SELECT X.col2, X.col3, T1.col5 FROM T1, (SELECT col2, col3 FROM T1.A) X " + + "WHERE T1.col5 > 10 ORDER BY X.col2, X.col3, T1.col5"; + indexIs(stmt, concat(field("A").nest(field("values", KeyExpression.FanType.FanOut) + .nest(concatenateFields("COL2", "COL3"))), field("COL5")), IndexTypes.VALUE, + index -> assertThat(index.getPredicate()).isEqualTo(greaterThanTen("COL5"))); + } + + /** + * The same predicate is fine when the shape does not need a synthetic type: one column per unnesting + * keeps the index on the stored table with a fan-out. + */ + @Test + void createIndexWithPredicateOverUnnestingIsSupported() throws Exception { + final String stmt = "CREATE SCHEMA TEMPLATE test_template " + + "CREATE TYPE AS STRUCT A(col2 string, col3 bigint, col4 bigint) " + + "CREATE TABLE T1(col1 bigint, a A Array, col5 bigint, primary key(col1)) " + + "CREATE INDEX mv1 AS SELECT X.col3, T1.col5 FROM T1, (SELECT col3 FROM T1.A) X " + + "WHERE T1.col5 > 10 ORDER BY X.col3, T1.col5"; + indexIs(stmt, concat(field("A").nest(field("values", KeyExpression.FanType.FanOut).nest("COL3")), + field("COL5")), IndexTypes.VALUE, + index -> assertThat(index.getPredicate()).isEqualTo(greaterThanTen("COL5"))); + } + + @Nonnull + private static Predicate greaterThanTen(@Nonnull final String column) { + return Predicate.newBuilder() + .setValuePredicate(ValuePredicate.newBuilder().addValue(column) + .setComparison(Comparison.newBuilder() + .setSimpleComparison(SimpleComparison.newBuilder() + .setType(ComparisonType.GREATER_THAN) + .setOperand(Value.newBuilder().setLongValue(10L).build()) + .build()) + .build()) + .build()) + .build(); } @Test @@ -927,16 +987,6 @@ void createVersionIndexWithRepeatedNested() throws Exception { indexIs(stmt, keyWithValue(concat(version(), field("A").nest(field("values", KeyExpression.FanType.FanOut).nest(concatenateFields("COL3", "COL4")))), 2), IndexTypes.VERSION); } - @Test - void createVersionIndexWithRepeatedNestedSplitByVersion() throws Exception { - final String stmt = "CREATE SCHEMA TEMPLATE test_template " + - "CREATE TYPE AS STRUCT A(col2 string, col3 bigint, col4 bigint) " + - "CREATE TABLE T1(col1 bigint, a A Array, primary key(col1)) " + - "CREATE INDEX mv1 AS SELECT X.col2, T1.\"__ROW_VERSION\", X.col3, X.col4 FROM T1, (SELECT col2, col3, col4 FROM T1.A) X ORDER BY X.col2, T1.\"__ROW_VERSION\", X.col3 " + - "WITH OPTIONS(store_row_versions=true)"; - shouldFailWith(stmt, ErrorCode.UNSUPPORTED_OPERATION, "Index with multiple disconnected references to the same column are not supported"); - } - @Test void createVersionIndexWithoutQualifyingTableName() throws Exception { final String stmt = "CREATE SCHEMA TEMPLATE test_template " + diff --git a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/api/ddl/SqlFunctionTest.java b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/api/ddl/SqlFunctionTest.java index db6e54f25ce..792fdecaf43 100644 --- a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/api/ddl/SqlFunctionTest.java +++ b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/api/ddl/SqlFunctionTest.java @@ -64,7 +64,7 @@ public class SqlFunctionTest { @RegisterExtension @Order(2) - public final SimpleDatabaseRule database = new SimpleDatabaseRule(DdlStatementParsingTest.class, TestSchemas.books()); + public final SimpleDatabaseRule database = new SimpleDatabaseRule(SqlFunctionTest.class, TestSchemas.books()); @RegisterExtension @Order(3) diff --git a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/api/ddl/UnnestedSyntheticTableIndexTest.java b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/api/ddl/UnnestedSyntheticTableIndexTest.java new file mode 100644 index 00000000000..55181d47e1c --- /dev/null +++ b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/api/ddl/UnnestedSyntheticTableIndexTest.java @@ -0,0 +1,963 @@ +/* + * UnnestedSyntheticTableIndexTest.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2021-2026 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apple.foundationdb.relational.api.ddl; + +import com.apple.foundationdb.record.RecordMetaData; +import com.apple.foundationdb.record.metadata.IndexTypes; +import com.apple.foundationdb.record.metadata.Key; +import com.apple.foundationdb.record.metadata.UnnestedRecordType; +import com.apple.foundationdb.record.metadata.expressions.KeyExpression; +import com.apple.foundationdb.record.query.plan.cascades.typing.Type; +import com.apple.foundationdb.relational.api.Options; +import com.apple.foundationdb.relational.api.exceptions.ErrorCode; +import com.apple.foundationdb.relational.api.metadata.SchemaTemplate; +import com.apple.foundationdb.relational.recordlayer.EmbeddedRelationalExtension; +import com.apple.foundationdb.relational.recordlayer.RelationalConnectionRule; +import com.apple.foundationdb.relational.recordlayer.Utils; +import com.apple.foundationdb.relational.recordlayer.ddl.AbstractMetadataOperationsFactory; +import com.apple.foundationdb.relational.recordlayer.metadata.RecordLayerSchemaTemplate; +import com.apple.foundationdb.relational.recordlayer.metadata.RecordLayerUnnestedSyntheticTable; +import com.apple.foundationdb.relational.util.Assert; +import com.apple.foundationdb.relational.utils.SimpleDatabaseRule; +import com.google.common.collect.Iterables; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Order; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; +import javax.annotation.Nonnull; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.function.BiConsumer; +import java.util.function.BiFunction; +import java.util.stream.Collectors; +import static com.apple.foundationdb.record.metadata.Key.Expressions.concat; +import static com.apple.foundationdb.record.metadata.Key.Expressions.field; +import static com.apple.foundationdb.record.metadata.Key.Expressions.function; +import static com.apple.foundationdb.record.metadata.Key.Expressions.keyWithValue; +import static com.apple.foundationdb.relational.util.NullableArrayUtils.REPEATED_FIELD_NAME; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import java.util.stream.Stream; + +/** + * Indexes over an unnesting that need an unnested synthetic table, and the shapes that are rejected. A synthetic table + * is needed exactly when two or more columns read through the same unnesting are non-adjacent in the index key; the + * shapes that stay a fan-out on the stored table are covered by {@link IndexTest}. + */ +public class UnnestedSyntheticTableIndexTest { + /** + * The plan generator loads a store for the connection's schema to read its metadata, so a schema has to exist before + * any of these statements can be planned. Every test then declares its own schema template, so nothing here reads + * this table -- it is the smallest one that makes the connection usable. + */ + private static final String PLACEHOLDER_SCHEMA = "CREATE TABLE placeholder(id bigint, primary key(id))"; + + @RegisterExtension + @Order(0) + public final EmbeddedRelationalExtension relationalExtension = new EmbeddedRelationalExtension(); + + @RegisterExtension + @Order(2) + public final SimpleDatabaseRule database = + new SimpleDatabaseRule(UnnestedSyntheticTableIndexTest.class, PLACEHOLDER_SCHEMA); + + @RegisterExtension + @Order(3) + public final RelationalConnectionRule connection = new RelationalConnectionRule(database::getConnectionUri) + .withSchema("TEST_SCHEMA"); + + @BeforeAll + public static void setup() { + Utils.enableCascadesDebugger(); + } + + /** + * Asserts that the definition is rejected, with a message containing {@code errorMessage}. Every shape this file + * rejects is an unsupported one, so the code is always {@link ErrorCode#UNSUPPORTED_OPERATION}. + * + * @param query the DDL statement expected to be rejected + * @param errorMessage a substring the rejection message has to contain + * @throws Exception if anything other than planning fails + */ + void shouldFailWith(@Nonnull final String query, @Nonnull final String errorMessage) throws Exception { + DdlTestUtil.shouldFailWith(connection, database.getSchemaTemplateName(), "/UnnestedSyntheticTableIndexTest", query, + ErrorCode.UNSUPPORTED_OPERATION, errorMessage); + } + + void shouldWorkWithInjectedFactory(@Nonnull final String query, @Nonnull final MetadataOperationsFactory metadataOperationsFactory) + throws Exception { + connection.setAutoCommit(false); + connection.getUnderlyingEmbeddedConnection().createNewTransaction(); + Assertions.assertDoesNotThrow(() -> + DdlTestUtil.getPlanGenerator(connection.getUnderlyingEmbeddedConnection(), database.getSchemaTemplateName(), + "/UnnestedSyntheticTableIndexTest", metadataOperationsFactory) + .getPlan(query)); + connection.rollback(); + connection.setAutoCommit(true); + } + + /** + * Asserts that the statement defines its index on an unnested synthetic table with a single nested + * constituent, and that the index key matches. + * + * @param stmt the DDL statement + * @param indexType the expected index type + * @param expectedKey builds the expected key from the (parent alias, constituent alias) + * @throws Exception if planning fails + */ + private void syntheticIndexIs(@Nonnull final String stmt, @Nonnull final String indexType, + @Nonnull final BiFunction expectedKey) throws Exception { + syntheticIndexIs(stmt, indexType, 1, (parent, constituents) -> expectedKey.apply(parent, constituents.get(0))); + } + + /** + * Asserts that the statement defines its index on an unnested synthetic table, and that the index key + * matches. The parent and constituent aliases are generated, so the expected key is built from the + * aliases found in the metadata; constituents are given in registration order, outermost first. + * + * @param stmt the DDL statement + * @param indexType the expected index type + * @param constituentCount the expected number of nested constituents + * @param expectedKey builds the expected key from the (parent alias, constituent aliases) + * @throws Exception if planning fails + */ + private void syntheticIndexIs(@Nonnull final String stmt, @Nonnull final String indexType, final int constituentCount, + @Nonnull final BiFunction, KeyExpression> expectedKey) throws Exception { + syntheticIndexIs(stmt, indexType, constituentCount, expectedKey, (syntheticTable, metaData) -> { }); + } + + /** + * As {@link #syntheticIndexIs(String, String, int, BiFunction)}, with an extra validator for assertions + * that go beyond the index key, such as the constituent tree or the synthetic primary key. + * + * @param stmt the DDL statement + * @param indexType the expected index type + * @param constituentCount the expected number of nested constituents + * @param expectedKey builds the expected key from the (parent alias, constituent aliases) + * @param validator further assertions on the synthetic table and the metadata it serializes to + * @throws Exception if planning fails + */ + private void syntheticIndexIs(@Nonnull final String stmt, @Nonnull final String indexType, final int constituentCount, + @Nonnull final BiFunction, KeyExpression> expectedKey, + @Nonnull final BiConsumer validator) throws Exception { + shouldWorkWithInjectedFactory(stmt, new AbstractMetadataOperationsFactory() { + @Nonnull + @Override + public ConstantAction getSaveSchemaTemplateConstantAction(@Nonnull final SchemaTemplate template, + @Nonnull final Options templateProperties) { + final var syntheticTables = Assert.castUnchecked(template, RecordLayerSchemaTemplate.class) + .getUnnestedSyntheticTables(); + Assertions.assertEquals(1, syntheticTables.size(), "Incorrect number of synthetic tables!"); + final var syntheticTable = syntheticTables.stream().findFirst().orElseThrow(); + Assertions.assertEquals(constituentCount, syntheticTable.getConstituents().size(), + "Incorrect number of nested constituents!"); + final var constituentAliases = syntheticTable.getConstituents().stream() + .map(RecordLayerUnnestedSyntheticTable.NestedConstituent::getAlias) + .collect(Collectors.toList()); + syntheticTable.getConstituents().forEach(constituent -> + Assertions.assertTrue(constituent.getParentAlias().equals(syntheticTable.getAlias()) + || constituentAliases.contains(constituent.getParentAlias()), + () -> "constituent '" + constituent.getAlias() + "' has unknown parent '" + + constituent.getParentAlias() + "'")); + Assertions.assertEquals(1, syntheticTable.getIndexes().size(), "Incorrect number of indexes!"); + final var index = syntheticTable.getIndexes().stream().findFirst().orElseThrow(); + Assertions.assertEquals(indexType, index.getIndexType()); + Assertions.assertEquals(expectedKey.apply(syntheticTable.getAlias(), constituentAliases), + KeyExpression.fromProto(index.getKeyExpression().toKeyExpression())); + final var metaData = Assert.castUnchecked(template, RecordLayerSchemaTemplate.class).toRecordMetadata(); + // the descriptor is keyed by the storage name, which is the protobuf-compliant form of the declared one + Assertions.assertTrue(metaData.getSyntheticRecordTypes().containsKey(syntheticTable.getType().getStorageName()), + () -> "synthetic type '" + syntheticTable.getType().getStorageName() + "' missing from serialized metadata, got " + + metaData.getSyntheticRecordTypes().keySet()); + validator.accept(syntheticTable, metaData); + return txn -> { + }; + } + }); + } + + @Test + void createIndexWithRepeatedNestedSplitByField() throws Exception { + final String stmt = "CREATE SCHEMA TEMPLATE test_template " + + "CREATE TYPE AS STRUCT A(col2 string, col3 bigint, col4 bigint) " + + "CREATE TABLE T1(col1 bigint, a A Array, col5 bigint, primary key(col1)) " + + "CREATE INDEX mv1 AS SELECT X.col2, T1.col5, X.col3, X.col4 FROM T1, (SELECT col2, col3, col4 FROM T1.A) X ORDER BY X.col2, T1.col5, X.col3"; + syntheticIndexIs(stmt, IndexTypes.VALUE, (parent, x) -> keyWithValue(concat( + field(x).nest("COL2"), + field(parent).nest("COL5"), + field(x).nest("COL3"), + field(x).nest("COL4")), 3)); + } + + /** + * As {@link #createIndexWithRepeatedNestedSplitByField()}, but over a table with a composite primary key, + * one of whose columns is also an index key column. + */ + @Test + void createIndexWithRepeatedNestedSplitByFieldOverCompositePrimaryKey() throws Exception { + final String stmt = "CREATE SCHEMA TEMPLATE test_template " + + "CREATE TYPE AS STRUCT A(col2 string, col3 bigint, col4 bigint) " + + "CREATE TABLE T1(col1 bigint, a A Array, col5 bigint, primary key(col1, col5)) " + + "CREATE INDEX mv1 AS SELECT X.col2, T1.col5, X.col3, X.col4 FROM T1, (SELECT col2, col3, col4 FROM T1.A) X ORDER BY X.col2, T1.col5, X.col3"; + syntheticIndexIs(stmt, IndexTypes.VALUE, 1, (parent, constituents) -> keyWithValue(concat( + field(constituents.get(0)).nest("COL2"), + field(parent).nest("COL5"), + field(constituents.get(0)).nest("COL3"), + field(constituents.get(0)).nest("COL4")), 3), + (syntheticTable, metaData) -> { + final var unnestedType = (UnnestedRecordType) metaData.getSyntheticRecordTypes() + .get(syntheticTable.getName()); + final String constituent = syntheticTable.getConstituents().get(0).getAlias(); + Assertions.assertEquals( + concat(Key.Expressions.recordType(), Key.Expressions.list(List.of( + field(syntheticTable.getAlias()).nest(concat(Key.Expressions.recordType(), + field("COL1"), field("COL5"))), + field(UnnestedRecordType.POSITIONS_FIELD).nest(constituent)))), + unnestedType.getPrimaryKey(), + "the parent's full composite primary key should reach the synthetic type"); + }); + } + + /** + * As {@link #createIndexWithRepeatedNestedSplitByField()}, but over an {@code ARRAY NOT NULL} column. + */ + @Test + void createIndexWithNestedRepeatedSplitOverNonNullableRepeated() throws Exception { + final String stmt = "CREATE SCHEMA TEMPLATE test_template " + + "CREATE TYPE AS STRUCT A(col2 string, col3 bigint, col4 bigint) " + + "CREATE TABLE T1(col1 bigint, a A Array not null, col5 bigint, primary key(col1)) " + + "CREATE INDEX mv1 AS SELECT X.col2, T1.col5, X.col3, X.col4 FROM T1, (SELECT col2, col3, col4 FROM T1.A) X ORDER BY X.col2, T1.col5, X.col3"; + syntheticIndexIs(stmt, IndexTypes.VALUE, (parent, x) -> keyWithValue(concat( + field(x).nest("COL2"), + field(parent).nest("COL5"), + field(x).nest("COL3"), + field(x).nest("COL4")), 3)); + } + + /** + * The synthetic table's name is composed from the index name, which is a user identifier and so need not be a legal + * protobuf message name -- and the name does become one, in the synthetic record type's descriptor. + */ + @Test + void createIndexWithNonProtoCompliantNameOverUnnestedSyntheticTable() throws Exception { + final String stmt = "CREATE SCHEMA TEMPLATE test_template " + + "CREATE TYPE AS STRUCT A(col2 string, col3 bigint, col4 bigint) " + + "CREATE TABLE T1(col1 bigint, a A Array, col5 bigint, primary key(col1)) " + + "CREATE INDEX \"mv.1\" AS SELECT X.col2, T1.col5, X.col3 FROM T1, (SELECT col2, col3 FROM T1.A) X " + + "ORDER BY X.col2, T1.col5, X.col3"; + syntheticIndexIs(stmt, IndexTypes.VALUE, 1, + (parent, constituents) -> concat( + field(constituents.get(0)).nest("COL2"), + field(parent).nest("COL5"), + field(constituents.get(0)).nest("COL3")), + (syntheticTable, metaData) -> { + Assertions.assertEquals("__unnested_T1_mv.1", syntheticTable.getName()); + Assertions.assertTrue(metaData.getSyntheticRecordTypes().containsKey("__unnested_T1_mv__21")); + }); + } + + /** + * The same split, ordered by an explicit direction. The ordering functions are keyed by identity on the order-by + * columns, so rewriting those columns onto the synthetic table has to re-key the map onto the rewritten values -- + * a column whose key is stale simply loses its direction, which no other assertion here would notice. + */ + @Test + void createIndexWithRepeatedNestedSplitByFieldRetainsOrderingFunctions() throws Exception { + final String stmt = "CREATE SCHEMA TEMPLATE test_template " + + "CREATE TYPE AS STRUCT A(col2 string, col3 bigint, col4 bigint) " + + "CREATE TABLE T1(col1 bigint, a A Array, col5 bigint, primary key(col1)) " + + "CREATE INDEX mv1 AS SELECT X.col2, T1.col5, X.col3 FROM T1, (SELECT col2, col3 FROM T1.A) X " + + "ORDER BY X.col2 DESC, T1.col5, X.col3 NULLS LAST"; + syntheticIndexIs(stmt, IndexTypes.VALUE, (parent, x) -> concat( + function("order_desc_nulls_last", field(x).nest("COL2")), + field(parent).nest("COL5"), + function("order_asc_nulls_last", field(x).nest("COL3")))); + } + + /** + * A scalar repeated field cannot be a constituent, so every reference to it is emitted as its own fan-out. Two + * references to one unnesting would then range over the repeated field independently, and their cross + * product holds entries where the two differ, which no view row does. No representation exists, so it is rejected. + */ + @Test + void createIndexWithScalarRepeatedReferencedTwiceIsNotSupported() throws Exception { + final String stmt = "CREATE SCHEMA TEMPLATE test_template " + + "CREATE TYPE AS STRUCT A(col2 string, col3 bigint) " + + "CREATE TABLE T1(col1 bigint, a A Array, s string array, primary key(col1)) " + + "CREATE INDEX mv1 AS SELECT X.col2, V.s AS v1, X.col3, V.s AS v2 FROM T1, (SELECT col2, col3 FROM T1.A) X, (SELECT s FROM T1.S) V ORDER BY X.col2, v1, X.col3, v2"; + shouldFailWith(stmt, "a scalar array cannot be referenced at more than one index key position"); + } + + /** + * The single-reference cases the check above must not disturb: one scalar reference stays a fan-out, whether + * the repeated field hangs off the stored record or off an unnested element. + */ + @Test + void createIndexWithScalarRepeatedReferencedOnceKeepsFanOut() throws Exception { + final String stmt = "CREATE SCHEMA TEMPLATE test_template " + + "CREATE TYPE AS STRUCT A(col2 string, col3 bigint) " + + "CREATE TABLE T1(col1 bigint, a A Array, s string array, primary key(col1)) " + + "CREATE INDEX mv1 AS SELECT X.col2, V.s AS v1, X.col3 FROM T1, (SELECT col2, col3 FROM T1.A) X, (SELECT s FROM T1.S) V ORDER BY X.col2, v1, X.col3"; + syntheticIndexIs(stmt, IndexTypes.VALUE, (parent, x) -> concat( + field(x).nest("COL2"), + field(parent).nest(field("S").nest(field("values", KeyExpression.FanType.FanOut))), + field(x).nest("COL3"))); + } + + /** + * A scalar repeated field reached through a non-repeated struct. Re-rooting the reference onto the constituent that + * owns the array has to carry every hop of that path, not just the array's own field: the table here also declares a + * top-level {@code s}, so dropping the {@code FIZZ} hop would silently index the wrong column rather than fail. + */ + @Test + void createIndexWithScalarRepeatedUnderPathKeepsFanOut() throws Exception { + final String stmt = "CREATE SCHEMA TEMPLATE test_template " + + "CREATE TYPE AS STRUCT A(col2 string, col3 bigint) " + + "CREATE TYPE AS STRUCT FIZZ(s string array) " + + "CREATE TABLE T1(col1 bigint, a A Array, fizz FIZZ, s string array, primary key(col1)) " + + "CREATE INDEX mv1 AS SELECT X.col2, V.s AS v1, X.col3 FROM T1, (SELECT col2, col3 FROM T1.A) X, (SELECT s FROM T1.FIZZ.S) V ORDER BY X.col2, v1, X.col3"; + syntheticIndexIs(stmt, IndexTypes.VALUE, (parent, x) -> concat( + field(x).nest("COL2"), + field(parent).nest(field("FIZZ").nest( + field("S").nest(field("values", KeyExpression.FanType.FanOut)))), + field(x).nest("COL3"))); + } + + /** + * Two separate explodes over the same scalar repeated field are distinct unnestings, so their cross-product is + * the intended meaning of the cross join and each is referenced once. + */ + @Test + void createIndexWithTwoIndependentScalarUnnestingsIsSupported() throws Exception { + final String stmt = "CREATE SCHEMA TEMPLATE test_template " + + "CREATE TYPE AS STRUCT A(col2 string, col3 bigint) " + + "CREATE TABLE T1(col1 bigint, a A Array, s string array, primary key(col1)) " + + "CREATE INDEX mv1 AS SELECT X.col2, V.s AS v1, X.col3, W.s AS v2 FROM T1, (SELECT col2, col3 FROM T1.A) X, (SELECT s FROM T1.S) V, (SELECT s FROM T1.S) W ORDER BY X.col2, v1, X.col3, v2"; + syntheticIndexIs(stmt, IndexTypes.VALUE, (parent, x) -> concat( + field(x).nest("COL2"), + field(parent).nest(field("S").nest(field("values", KeyExpression.FanType.FanOut))), + field(x).nest("COL3"), + field(parent).nest(field("S").nest(field("values", KeyExpression.FanType.FanOut))))); + } + + /** + * Two indexes in one template each need their own synthetic type, alongside a plain index on the stored table. + */ + @Test + void createTwoIndexesEachRequiringSyntheticTableKeepsThemSeparate() throws Exception { + final String stmt = "CREATE SCHEMA TEMPLATE test_template " + + "CREATE TYPE AS STRUCT A(col2 string, col3 bigint, col4 bigint) " + + "CREATE TABLE T1(col1 bigint, a A Array, col5 bigint, primary key(col1)) " + + "CREATE INDEX mv1 AS SELECT X.col2, T1.col5, X.col3 FROM T1, (SELECT col2, col3 FROM T1.A) X ORDER BY X.col2, T1.col5, X.col3 " + + "CREATE INDEX mv2 AS SELECT Y.col3, T1.col5, Y.col4 FROM T1, (SELECT col3, col4 FROM T1.A) Y ORDER BY Y.col3, T1.col5, Y.col4 " + + "CREATE INDEX i3 AS SELECT T1.col5, T1.col1 FROM T1 ORDER BY T1.col5, T1.col1"; + shouldWorkWithInjectedFactory(stmt, new AbstractMetadataOperationsFactory() { + @Nonnull + @Override + public ConstantAction getSaveSchemaTemplateConstantAction(@Nonnull final SchemaTemplate template, + @Nonnull final Options templateProperties) { + final var recLayer = Assert.castUnchecked(template, RecordLayerSchemaTemplate.class); + + final var syntheticNames = recLayer.getUnnestedSyntheticTables().stream() + .map(RecordLayerUnnestedSyntheticTable::getName) + .collect(Collectors.toSet()); + Assertions.assertEquals(Set.of("__unnested_T1_MV1", "__unnested_T1_MV2"), syntheticNames); + + // one index each, and the plain index stays on the stored table + recLayer.getUnnestedSyntheticTables().forEach(synthetic -> + Assertions.assertEquals(1, synthetic.getIndexes().size(), + () -> "expected one index on " + synthetic.getName())); + final var storedTableIndexes = Assertions.assertDoesNotThrow(() -> + Assert.optionalUnchecked(template.findTableByName("T1")).getIndexes().stream() + .map(com.apple.foundationdb.relational.api.metadata.Index::getName) + .collect(Collectors.toSet())); + Assertions.assertEquals(Set.of("I3"), storedTableIndexes); + + // both reach RecordMetaData, as distinct types with distinct record type keys + final var metaData = recLayer.toRecordMetadata(); + Assertions.assertTrue(metaData.getSyntheticRecordTypes().keySet() + .containsAll(Set.of("__unnested_T1_MV1", "__unnested_T1_MV2")), + () -> "got " + metaData.getSyntheticRecordTypes().keySet()); + final var recordTypeKeys = metaData.getSyntheticRecordTypes().values().stream() + .map(com.apple.foundationdb.record.metadata.SyntheticRecordType::getRecordTypeKey) + .collect(Collectors.toSet()); + Assertions.assertEquals(2, recordTypeKeys.size(), + () -> "the two synthetic types share a record type key: " + recordTypeKeys); + return txn -> { + }; + } + }); + } + + /** + * The scalar repeated field lives on the unnested element type, not on the stored record, so its fan-out is + * rooted at the constituent that owns it rather than at the parent. + */ + @Test + void createIndexWithScalarRepeatedInsideNestedRepeatedRootsFanOutAtConstituent() throws Exception { + // A parent column between the struct's own columns is what splits the outer unnesting: `tg` sits inside + // the struct element, so it traverses that unnesting too and cannot split it. + final String stmt = "CREATE SCHEMA TEMPLATE test_template " + + "CREATE TYPE AS STRUCT A(x bigint, tags string array, y bigint) " + + "CREATE TABLE T1(col1 bigint, a A Array, primary key(col1)) " + + "CREATE INDEX mv1 AS SELECT M.x, t.col1, M.y, tg FROM T1 AS t, t.a AS M, M.tags AS tg ORDER BY M.x, t.col1, M.y, tg"; + syntheticIndexIs(stmt, IndexTypes.VALUE, (parent, x) -> concat( + field(x).nest("X"), + field(parent).nest("COL1"), + field(x).nest("Y"), + field(x).nest(field("TAGS").nest(field("values", KeyExpression.FanType.FanOut))))); + } + + /** + * Aggregate Index is not supported. + */ + @Test + void createAggregateIndexOverUnnestedSyntheticTableIsNotSupported() throws Exception { + final String stmt = "CREATE SCHEMA TEMPLATE test_template " + + "CREATE TYPE AS STRUCT A(x bigint, x2 bigint) " + + "CREATE TABLE T(p bigint, a A array, primary key(p)) " + + "CREATE INDEX mv1 AS SELECT M.x, t.p, SUM(M.x2) FROM T AS t, t.a AS M GROUP BY M.x, t.p " + + "ORDER BY M.x, t.p"; + shouldFailWith(stmt, "cannot be defined on an unnested synthetic table"); + } + + /** + * Predicates are not supported. + */ + @Test + void createIndexWithPredicateOverUnnestedSyntheticTableIsNotSupported() throws Exception { + final String stmt = "CREATE SCHEMA TEMPLATE test_template " + + "CREATE TYPE AS STRUCT A(col2 string, col3 bigint, col4 bigint) " + + "CREATE TABLE T1(col1 bigint, a A Array, col5 bigint, primary key(col1)) " + + "CREATE INDEX mv1 AS SELECT X.col2, T1.col5, X.col3 FROM T1, (SELECT col2, col3 FROM T1.A) X " + + "WHERE T1.col5 > 10 ORDER BY X.col2, T1.col5, X.col3"; + shouldFailWith(stmt, "a predicate is not supported on an index over an unnested synthetic table"); + } + + /** + * Two columns of the same unnesting separated by a column of a different unnesting, rather + * than by a parent column. Still no single fan-out covers X, so this needs a synthetic type. + */ + @Test + void createIndexWithRepeatedNestedSplitByOtherRepeated() throws Exception { + final String stmt = "CREATE SCHEMA TEMPLATE test_template " + + "CREATE TYPE AS STRUCT A(col2 string, col3 bigint, col4 bigint) " + + "CREATE TABLE T1(col1 bigint, a A Array, primary key(col1)) " + + "CREATE INDEX mv1 AS SELECT X.col3, Y.col2, X.col4 FROM T1, (SELECT col3, col4 FROM T1.A) X, " + + "(SELECT col2 FROM T1.A) Y ORDER BY X.col3, Y.col2, X.col4"; + syntheticIndexIs(stmt, IndexTypes.VALUE, 2, (parent, constituents) -> concat( + field(constituents.get(0)).nest("COL3"), + field(constituents.get(1)).nest("COL2"), + field(constituents.get(0)).nest("COL4"))); + } + + /** + * Constituents branch as well as chain: {@code b} and {@code d} both hang off the stored record, while + * {@code c} hangs off {@code b}. + */ + @Test + void createIndexWithBranchingAndChainedUnnestingUsesSyntheticTable() throws Exception { + final String stmt = "CREATE SCHEMA TEMPLATE test_template " + + "CREATE TYPE AS STRUCT Q(y bigint, y2 bigint) " + + "CREATE TYPE AS STRUCT P(x bigint, x2 bigint, q Q array) " + + "CREATE TYPE AS STRUCT R(z bigint, z2 bigint) " + + "CREATE TABLE A(k bigint, p P array, r R array, primary key(k)) " + + "CREATE INDEX mv1 AS SELECT b.x, a.k, c.y, d.z FROM A AS a, (select * from a.p) as b, " + + "(select * from b.q) as c, (select * from a.r) as d ORDER BY b.x, a.k, c.y, d.z"; + syntheticIndexIs(stmt, IndexTypes.VALUE, 3, (parent, constituents) -> concat( + field(constituents.get(0)).nest("X"), + field(parent).nest("K"), + field(constituents.get(1)).nest("Y"), + field(constituents.get(2)).nest("Z")), + (syntheticTable, metaData) -> { + // The generic helper only checks that each parent alias is known, which cannot tell this + // tree apart from a chain, so pin the actual parent of each constituent. + final var constituents = syntheticTable.getConstituents(); + Assertions.assertEquals( + List.of(syntheticTable.getAlias(), constituents.get(0).getAlias(), syntheticTable.getAlias()), + constituents.stream() + .map(RecordLayerUnnestedSyntheticTable.NestedConstituent::getParentAlias) + .collect(Collectors.toList()), + "constituents should branch at the stored record, with only the second one chained"); + Assertions.assertEquals( + List.of(List.of("P", REPEATED_FIELD_NAME), List.of("Q", REPEATED_FIELD_NAME), + List.of("R", REPEATED_FIELD_NAME)), + constituents.stream() + .map(RecordLayerUnnestedSyntheticTable.NestedConstituent::getFieldPath) + .collect(Collectors.toList())); + }); + } + + /** + * ROW_VERSION is not supported. + */ + @Test + void createVersionIndexWithRepeatedNestedSplitByVersion() throws Exception { + final String stmt = "CREATE SCHEMA TEMPLATE test_template " + + "CREATE TYPE AS STRUCT A(col2 string, col3 bigint, col4 bigint) " + + "CREATE TABLE T1(col1 bigint, a A Array, primary key(col1)) " + + "CREATE INDEX mv1 AS SELECT X.col2, T1.\"__ROW_VERSION\", X.col3, X.col4 FROM T1, (SELECT col2, col3, col4 FROM T1.A) X ORDER BY X.col2, T1.\"__ROW_VERSION\", X.col3 " + + "WITH OPTIONS(store_row_versions=true)"; + shouldFailWith(stmt, "a version column cannot be part of an index over an unnested synthetic table"); + } + + /** + * Complex values (anything other than column references) is not supported. + */ + @Test + void createIndexOverUnnestedSyntheticTableWithArithmeticColumnIsNotSupported() throws Exception { + final String stmt = "CREATE SCHEMA TEMPLATE test_template " + + "CREATE TYPE AS STRUCT A(col2 string, col3 bigint, col4 bigint) " + + "CREATE TABLE T1(col1 bigint, a A Array, col5 bigint, primary key(col1)) " + + "CREATE INDEX mv1 AS SELECT X.col2, T1.col5, X.col3, T1.col5 + 1 AS pp FROM T1, (SELECT col2, col3 FROM T1.A) X ORDER BY X.col2, T1.col5, X.col3, pp"; + shouldFailWith(stmt, "supports only plain column references"); + } + + // Spellings + + private static final String VIEW_SUBQUERY = "view + correlated subquery"; + private static final String VIEW_PARTIQL = "view + PartiQL path"; + private static final String AS_SELECT_SUBQUERY = "index as select + correlated subquery"; + private static final String AS_SELECT_PARTIQL = "index as select + PartiQL path"; + + // Schema Templates + + private static final String SINGLE_NESTED_REPEATED_SCHEMA = "CREATE SCHEMA TEMPLATE test_template " + + "CREATE TYPE AS STRUCT A(x bigint, y bigint) " + + "CREATE TABLE T(p bigint, a A array, primary key(p)) "; + + private static final String MULTIPLE_NESTED_REPEATED_SCHEMA = "CREATE SCHEMA TEMPLATE test_template " + + "CREATE TYPE AS STRUCT A(x bigint, x2 bigint) CREATE TYPE AS STRUCT B(y bigint, y2 bigint) " + + "CREATE TYPE AS STRUCT C(z bigint, z2 bigint) " + + "CREATE TABLE T(p bigint, a A array, b B array, c C array, primary key(p)) "; + + private static final String MIXED_REPEATED_SCHEMA = "CREATE SCHEMA TEMPLATE test_template " + + "CREATE TYPE AS STRUCT A(x bigint, y bigint) " + + "CREATE TABLE T(p bigint, a A array, s string array, primary key(p)) "; + + private static final String DEEP_NESTED_REPEATED_SCHEMA = "CREATE SCHEMA TEMPLATE test_template " + + "CREATE TYPE AS STRUCT S1(a string, b string) " + + "CREATE TYPE AS STRUCT S2(x S1 array, y S1) " + + "CREATE TYPE AS STRUCT S3(alpha S2, beta S2) " + + "CREATE TABLE T(id bigint, fizz S3, buzz bigint, primary key(id)) "; + + private static final String CHAINED_NESTED_REPEATED_SCHEMA = "CREATE SCHEMA TEMPLATE test_template " + + "CREATE TYPE AS STRUCT Q(y bigint, y2 bigint) " + + "CREATE TYPE AS STRUCT P(x bigint, x2 bigint, q Q array) " + + "CREATE TABLE A(k bigint, p P array, primary key(k)) "; + + // ─── A single nested repeated field ────────────────────────────────────────────────────────────── + + @Nonnull + private static Stream singleNestedRepeatedSpellings() { + return Stream.of( + Arguments.of(VIEW_SUBQUERY, "I1", + "CREATE VIEW mv1 AS SELECT SQ.x, t.p, SQ.y from T AS t, (select M.x, M.y from t.a AS M) SQ " + + "CREATE INDEX i1 on mv1(x, p, y)"), + Arguments.of(VIEW_PARTIQL, "I1", + "CREATE VIEW mv1 AS SELECT M.x, t.p, M.y from T AS t, t.a AS M " + + "CREATE INDEX i1 on mv1(x, p, y)"), + Arguments.of(AS_SELECT_SUBQUERY, "MV1", + "CREATE INDEX mv1 AS SELECT SQ.x, t.p, SQ.y from T AS t, (select M.x, M.y from t.a AS M) SQ " + + "order by SQ.x, t.p, SQ.y "), + Arguments.of(AS_SELECT_PARTIQL, "MV1", + "CREATE INDEX mv1 AS SELECT M.x, t.p, M.y from T AS t, t.a AS M order by M.x, t.p, M.y ")); + } + + /** + * What every spelling of a shape has to agree on beyond the index key: the name derived for the synthetic table, the + * stored table it is built over, one nesting expression per constituent in registration order, the alias each + * constituent hangs off, and the index it carries. The key itself, the constituent count and serialization are + * asserted by {@link #syntheticIndexIs(String, String, int, BiFunction, BiConsumer)}. + * + * @param syntheticTable the table the index was defined on + * @param storedTableName the stored table the synthetic table is built over + * @param indexName the name the definition gives the index + * @param nestingExpressions the expected nesting expression of each constituent, in order + * @param parentAliases the alias each constituent is expected to hang off, in the same order + */ + private static void assertUnnestedTableIs(@Nonnull final RecordLayerUnnestedSyntheticTable syntheticTable, + @Nonnull final String storedTableName, + @Nonnull final String indexName, + @Nonnull final List nestingExpressions, + @Nonnull final List parentAliases) { + Assertions.assertEquals("__unnested_" + storedTableName + "_" + indexName, syntheticTable.getName()); + Assertions.assertEquals(Set.of(storedTableName), syntheticTable.getUnderlyingTableNames()); + Assertions.assertEquals(nestingExpressions, syntheticTable.getConstituents().stream() + .map(RecordLayerUnnestedSyntheticTable.NestedConstituent::getNestingExpression) + .collect(Collectors.toList())); + Assertions.assertEquals(parentAliases, syntheticTable.getConstituents().stream() + .map(RecordLayerUnnestedSyntheticTable.NestedConstituent::getParentAlias) + .collect(Collectors.toList())); + final var index = syntheticTable.getIndexes().stream().findFirst().orElseThrow(); + Assertions.assertEquals(indexName, index.getName()); + Assertions.assertEquals(syntheticTable.getName(), index.getTableName()); + } + + /** + * As {@link #assertUnnestedTableIs(RecordLayerUnnestedSyntheticTable, String, String, List, List)}, for a table over + * {@code T} whose constituents all hang directly off the stored record. + * + * @param syntheticTable the table the index was defined on + * @param indexName the name the definition gives the index + * @param nestingExpressions the expected nesting expression of each constituent, in order + */ + private static void assertUnnestedTableIs(@Nonnull final RecordLayerUnnestedSyntheticTable syntheticTable, + @Nonnull final String indexName, + @Nonnull final List nestingExpressions) { + assertUnnestedTableIs(syntheticTable, "T", indexName, nestingExpressions, + Collections.nCopies(nestingExpressions.size(), syntheticTable.getAlias())); + } + + @ParameterizedTest(name = "{displayName} - {0}") + @MethodSource("singleNestedRepeatedSpellings") + void createIndexOnNestedRepeatedSplitUsesSyntheticTable(@Nonnull final String spelling, @Nonnull final String indexName, + @Nonnull final String indexDdl) throws Exception { + // constituent-alias paths with no fan-out: the fan-out lives in the constituent's nesting expression, and the + // ORDER BY column order is preserved + syntheticIndexIs(SINGLE_NESTED_REPEATED_SCHEMA + indexDdl, IndexTypes.VALUE, 1, + (parent, constituents) -> concat( + field(constituents.get(0)).nest("X"), + field(parent).nest("P"), + field(constituents.get(0)).nest("Y")), + (syntheticTable, metaData) -> + assertUnnestedTableIs(syntheticTable, indexName, List.of(wrappedRepeatedElements("A")))); + } + + // ─── Multiple nested repeated fields ───────────────────────────────────────────────────────────── + + /** Navigates to the elements of a nullable array, which is how the DDL layer stores {@code array}. */ + @Nonnull + private static KeyExpression wrappedRepeatedElements(@Nonnull final String arrayFieldName) { + return field(arrayFieldName) + .nest(field("values", KeyExpression.FanType.FanOut)); + } + + @Nonnull + private static Stream multipleNestedRepeatedSpellings() { + return Stream.of( + Arguments.of(VIEW_SUBQUERY, "I1", + "CREATE VIEW v1 AS SELECT SQ1.x, SQ2.y, SQ3.z, t.p, SQ1.x2, SQ2.y2, SQ3.z2 from T AS t, " + + "(select M.x, M.x2 from t.a AS M) SQ1, (select N.y, N.y2 from t.b AS N) SQ2, " + + "(select O.z, O.z2 from t.c AS O) SQ3 " + + "CREATE INDEX i1 on v1(x, y, z, p, x2, y2, z2)"), + Arguments.of(VIEW_PARTIQL, "I1", + "CREATE VIEW v1 AS SELECT M.x, N.y, O.z, t.p, M.x2, N.y2, O.z2 from T AS t, " + + "t.a AS M, t.b AS N, t.c AS O " + + "CREATE INDEX i1 on v1(x, y, z, p, x2, y2, z2)"), + Arguments.of(AS_SELECT_SUBQUERY, "MV1", + "CREATE INDEX mv1 AS SELECT SQ1.x, SQ2.y, SQ3.z, t.p, SQ1.x2, SQ2.y2, SQ3.z2 from T AS t, " + + "(select M.x, M.x2 from t.a AS M) SQ1, (select N.y, N.y2 from t.b AS N) SQ2, " + + "(select O.z, O.z2 from t.c AS O) SQ3 " + + "order by SQ1.x, SQ2.y, SQ3.z, t.p, SQ1.x2, SQ2.y2, SQ3.z2"), + Arguments.of(AS_SELECT_PARTIQL, "MV1", + "CREATE INDEX mv1 AS SELECT M.x, N.y, O.z, t.p, M.x2, N.y2, O.z2 from T AS t, " + + "t.a AS M, t.b AS N, t.c AS O " + + "order by M.x, N.y, O.z, t.p, M.x2, N.y2, O.z2")); + } + + @ParameterizedTest(name = "{displayName} - {0}") + @MethodSource("multipleNestedRepeatedSpellings") + void createIndexOnMultipleRepeatedUsesSyntheticTable(@Nonnull final String spelling, @Nonnull final String indexName, + @Nonnull final String indexDdl) throws Exception { + // one constituent per unnested repeated field, in declaration order, all parented to the stored record + syntheticIndexIs(MULTIPLE_NESTED_REPEATED_SCHEMA + indexDdl, IndexTypes.VALUE, 3, + (parent, constituents) -> concat( + field(constituents.get(0)).nest("X"), + field(constituents.get(1)).nest("Y"), + field(constituents.get(2)).nest("Z"), + field(parent).nest("P"), + field(constituents.get(0)).nest("X2"), + field(constituents.get(1)).nest("Y2"), + field(constituents.get(2)).nest("Z2")), + (syntheticTable, metaData) -> assertUnnestedTableIs(syntheticTable, indexName, + List.of(wrappedRepeatedElements("A"), wrappedRepeatedElements("B"), wrappedRepeatedElements("C")))); + } + + @Nonnull + private static Stream mixedRepeatedSpellings() { + return Stream.of( + Arguments.of(VIEW_SUBQUERY, "I1", + "CREATE VIEW v1 AS SELECT SQ1.x, SQ2.v, SQ1.y from T AS t, " + + "(select M.x, M.y from t.a AS M) SQ1, (select v from t.s AS v) SQ2 " + + "CREATE INDEX i1 on v1(x, v, y)"), + Arguments.of(VIEW_PARTIQL, "I1", + "CREATE VIEW v1 AS SELECT M.x, v, M.y from T AS t, t.a AS M, t.s AS v " + + "CREATE INDEX i1 on v1(x, v, y)"), + Arguments.of(AS_SELECT_SUBQUERY, "MV1", + "CREATE INDEX mv1 AS SELECT SQ1.x, SQ2.v, SQ1.y from T AS t, " + + "(select M.x, M.y from t.a AS M) SQ1, (select v from t.s AS v) SQ2 " + + "order by SQ1.x, SQ2.v, SQ1.y"), + Arguments.of(AS_SELECT_PARTIQL, "MV1", + "CREATE INDEX mv1 AS SELECT M.x, v, M.y from T AS t, t.a AS M, t.s AS v " + + "order by M.x, v, M.y")); + } + + @ParameterizedTest(name = "{displayName} - {0}") + @MethodSource("mixedRepeatedSpellings") + void createIndexOnMixedRepeatedUsesSyntheticTable(@Nonnull final String spelling, @Nonnull final String indexName, + @Nonnull final String indexDdl) throws Exception { + // nested repeated element field via the constituent; scalar repeated element via a fan-out under the parent + syntheticIndexIs(MIXED_REPEATED_SCHEMA + indexDdl, IndexTypes.VALUE, 1, + (parent, constituents) -> concat( + field(constituents.get(0)).nest("X"), + field(parent).nest(wrappedRepeatedElements("S")), + field(constituents.get(0)).nest("Y")), + (syntheticTable, metaData) -> + assertUnnestedTableIs(syntheticTable, indexName, List.of(wrappedRepeatedElements("A")))); + } + + @Nonnull + private static Stream nestedPathSpellings() { + // both repeated fields carry the same element type, so the view spellings have to alias the columns apart + return Stream.of( + Arguments.of(VIEW_SUBQUERY, "I1", + "CREATE VIEW v1 AS SELECT u.a AS ua, v.a AS va, T.buzz, u.b AS ub, v.b AS vb " + + "FROM T, (SELECT a, b FROM T.fizz.alpha.x) AS u, (SELECT a, b FROM T.fizz.beta.x) AS v " + + "CREATE INDEX i1 on v1(ua, va, buzz, ub, vb)"), + Arguments.of(VIEW_PARTIQL, "I1", + "CREATE VIEW v1 AS SELECT u.a AS ua, v.a AS va, T.buzz, u.b AS ub, v.b AS vb " + + "FROM T, T.fizz.alpha.x AS u, T.fizz.beta.x AS v " + + "CREATE INDEX i1 on v1(ua, va, buzz, ub, vb)"), + Arguments.of(AS_SELECT_SUBQUERY, "MV1", + "CREATE INDEX mv1 AS SELECT u.a, v.a, T.buzz, u.b, v.b " + + "FROM T, (SELECT a, b FROM T.fizz.alpha.x) AS u, (SELECT a, b FROM T.fizz.beta.x) AS v " + + "ORDER BY u.a, v.a, T.buzz, u.b, v.b"), + Arguments.of(AS_SELECT_PARTIQL, "MV1", + "CREATE INDEX mv1 AS SELECT u.a, v.a, T.buzz, u.b, v.b " + + "FROM T, T.fizz.alpha.x AS u, T.fizz.beta.x AS v " + + "ORDER BY u.a, v.a, T.buzz, u.b, v.b")); + } + + /** + * Two repeated fields, each reached through a path of non-repeated fields, split by a column of the stored record. The + * constituent's nesting expression is relative to the record that owns the repeated field, so it has to carry every hop of + * that path -- keeping only the repeated field would look for {@code X} directly on {@code T}, where it does not exist. + */ + @ParameterizedTest(name = "{displayName} - {0}") + @MethodSource("nestedPathSpellings") + void createIndexOverNestedRepeatedUnderPathsUsesSyntheticTable(@Nonnull final String spelling, @Nonnull final String indexName, + @Nonnull final String indexDdl) throws Exception { + // distinct repeated fields under distinct paths, so distinct constituents + syntheticIndexIs(DEEP_NESTED_REPEATED_SCHEMA + indexDdl, IndexTypes.VALUE, 2, + (parent, constituents) -> concat( + field(constituents.get(0)).nest("A"), + field(constituents.get(1)).nest("A"), + field(parent).nest("BUZZ"), + field(constituents.get(0)).nest("B"), + field(constituents.get(1)).nest("B")), + (syntheticTable, metaData) -> assertUnnestedTableIs(syntheticTable, indexName, + List.of(repeatedElementsUnderPath("FIZZ", "ALPHA", "X"), + repeatedElementsUnderPath("FIZZ", "BETA", "X")))); + } + + @Test + void unnestedTableType() throws Exception { + final String stmt = DEEP_NESTED_REPEATED_SCHEMA + + "CREATE INDEX mv1 AS SELECT u.a, v.a, T.buzz, u.b, v.b " + + "FROM T, (SELECT a, b FROM T.fizz.alpha.x) AS u, (SELECT a, b FROM T.fizz.beta.x) AS v " + + "ORDER BY u.a, v.a, T.buzz, u.b, v.b"; + shouldWorkWithInjectedFactory(stmt, new AbstractMetadataOperationsFactory() { + @Nonnull + @Override + public ConstantAction getSaveSchemaTemplateConstantAction(@Nonnull final SchemaTemplate template, + @Nonnull final Options templateProperties) { + final var original = Assert.castUnchecked(template, RecordLayerSchemaTemplate.class); + final var reloaded = RecordLayerSchemaTemplate.fromRecordMetadata( + original.toRecordMetadata(), original.getName(), original.getVersion()); + final var composed = Iterables.getOnlyElement(original.getUnnestedSyntheticTables()); + final var derived = Iterables.getOnlyElement(reloaded.getUnnestedSyntheticTables()); + Assertions.assertEquals( + List.of("parent", "unnesting_0", "unnesting_1", UnnestedRecordType.POSITIONS_FIELD), + fieldNamesOf(composed.getType())); + Assertions.assertEquals(fieldNamesOf(composed.getType()), fieldNamesOf(derived.getType())); + Assertions.assertEquals(composed.getType(), derived.getType(), + "the synthetic type composed from the index definition should equal the one derived from its descriptor"); + return txn -> { + }; + } + }); + } + + @Nonnull + private static List fieldNamesOf(@Nonnull final Type.Record type) { + return type.getFields().stream() + .map(Type.Record.Field::getFieldName) + .collect(Collectors.toList()); + } + + @Nonnull + private static Stream chainedSpellings(@Nonnull final String selectList, @Nonnull final String indexColumns) { + final String subqueryForm = " FROM A AS a, (select * from a.p) as b, (select * from b.q) as c "; + final String partiqlForm = " FROM A AS a, a.p AS b, b.q AS c "; + return Stream.of( + Arguments.of(VIEW_SUBQUERY, "I1", "CREATE VIEW v1 AS SELECT " + selectList + subqueryForm + + "CREATE INDEX i1 on v1(" + indexColumns + ")"), + Arguments.of(VIEW_PARTIQL, "I1", "CREATE VIEW v1 AS SELECT " + selectList + partiqlForm + + "CREATE INDEX i1 on v1(" + indexColumns + ")"), + Arguments.of(AS_SELECT_SUBQUERY, "MV1", "CREATE INDEX mv1 AS SELECT " + selectList + subqueryForm + + "ORDER BY " + selectList), + Arguments.of(AS_SELECT_PARTIQL, "MV1", "CREATE INDEX mv1 AS SELECT " + selectList + partiqlForm + + "ORDER BY " + selectList)); + } + + /** + * The constituent tree every chained spelling has to produce: {@code q} unnested under the element of {@code p}, so + * the inner constituent hangs off the outer one rather than off the stored record, and its nesting expression is + * relative to that element. + * + * @param syntheticTable the table the index was defined on + * @param indexName the name the definition gives the index + */ + private static void assertChainedTableIs(@Nonnull final RecordLayerUnnestedSyntheticTable syntheticTable, + @Nonnull final String indexName) { + assertUnnestedTableIs(syntheticTable, "A", indexName, + List.of(wrappedRepeatedElements("P"), wrappedRepeatedElements("Q")), + List.of(syntheticTable.getAlias(), syntheticTable.getConstituents().get(0).getAlias())); + } + + @Nonnull + private static Stream chainedSplitByParentSpellings() { + return chainedSpellings("b.x, a.k, c.y", "x, k, y"); + } + + /** + * Chained unnesting split by a parent column. + */ + @ParameterizedTest(name = "{displayName} - {0}") + @MethodSource("chainedSplitByParentSpellings") + void createIndexWithChainedUnnestingSplitByParentUsesSyntheticTable(@Nonnull final String spelling, + @Nonnull final String indexName, + @Nonnull final String indexDdl) throws Exception { + syntheticIndexIs(CHAINED_NESTED_REPEATED_SCHEMA + indexDdl, IndexTypes.VALUE, 2, + (parent, constituents) -> concat( + field(constituents.get(0)).nest("X"), + field(parent).nest("K"), + field(constituents.get(1)).nest("Y")), + (syntheticTable, metaData) -> assertChainedTableIs(syntheticTable, indexName)); + } + + @Nonnull + private static Stream chainedInnerSplitSpellings() { + return chainedSpellings("c.y, a.k, c.y2", "y, k, y2"); + } + + /** + * Chained unnesting where the split is within the inner unnesting. + */ + @ParameterizedTest(name = "{displayName} - {0}") + @MethodSource("chainedInnerSplitSpellings") + void createIndexWithChainedUnnestingInnerSplitUsesSyntheticTable(@Nonnull final String spelling, + @Nonnull final String indexName, + @Nonnull final String indexDdl) throws Exception { + syntheticIndexIs(CHAINED_NESTED_REPEATED_SCHEMA + indexDdl, IndexTypes.VALUE, 2, + (parent, constituents) -> concat( + field(constituents.get(1)).nest("Y"), + field(parent).nest("K"), + field(constituents.get(1)).nest("Y2")), + (syntheticTable, metaData) -> assertChainedTableIs(syntheticTable, indexName)); + } + + /** + * Chained unnesting where the split is within the outer unnesting. + */ + @Nonnull + private static Stream chainedOuterSplitSpellings() { + return chainedSpellings("b.x, a.k, b.x2", "x, k, x2"); + } + + /** + * The inner unnesting is never read from, yet it still multiplies the rows the index is built over, so it remains a + * constituent. + */ + @ParameterizedTest(name = "{displayName} - {0}") + @MethodSource("chainedOuterSplitSpellings") + void createIndexWithChainedUnnestingOuterSplitUsesSyntheticTable(@Nonnull final String spelling, + @Nonnull final String indexName, + @Nonnull final String indexDdl) throws Exception { + syntheticIndexIs(CHAINED_NESTED_REPEATED_SCHEMA + indexDdl, IndexTypes.VALUE, 2, + (parent, constituents) -> concat( + field(constituents.get(0)).nest("X"), + field(parent).nest("K"), + field(constituents.get(0)).nest("X2")), + (syntheticTable, metaData) -> assertChainedTableIs(syntheticTable, indexName)); + } + + // ─── More nesting shapes: below the constituent, escaped names ─────────────────────────────────── + + /** + * A column reached through a non-repeated field below the constituent, so the re-rooted path is more than one + * hop on the far side of the unnesting too. + */ + @Test + void createIndexOverNestedFieldBelowConstituentUsesSyntheticTable() throws Exception { + final String stmt = "CREATE SCHEMA TEMPLATE test_template " + + "CREATE TYPE AS STRUCT S1(a string, b string) " + + "CREATE TYPE AS STRUCT S2(part S1, tag string) " + + "CREATE TABLE T(id bigint, items S2 array, buzz bigint, primary key(id)) " + + "CREATE INDEX mv1 AS SELECT u.part.a, T.buzz, u.part.b " + + "FROM T, (SELECT part FROM T.items) AS u ORDER BY u.part.a, T.buzz, u.part.b"; + syntheticIndexIs(stmt, IndexTypes.VALUE, (parent, u) -> concat( + field(u).nest(field("PART").nest("A")), + field(parent).nest("BUZZ"), + field(u).nest(field("PART").nest("B")))); + } + + /** + * A nested path whose intermediate field is not a legal protobuf identifier. Every hop of the nesting expression has + * to be the storage name, so a declared name reaching the descriptor would not resolve. + */ + @Test + void createIndexOverNestedRepeatedUnderNonProtoCompliantPathUsesSyntheticTable() throws Exception { + final String stmt = "CREATE SCHEMA TEMPLATE test_template " + + "CREATE TYPE AS STRUCT S1(a string, b string) " + + "CREATE TYPE AS STRUCT S2(\"x.y\" S1 array) " + + "CREATE TABLE T(id bigint, \"f.g\" S2, buzz bigint, primary key(id)) " + + "CREATE INDEX mv1 AS SELECT u.a, T.buzz, u.b " + + "FROM T, (SELECT a, b FROM T.\"f.g\".\"x.y\") AS u ORDER BY u.a, T.buzz, u.b"; + syntheticIndexIs(stmt, IndexTypes.VALUE, 1, + (parent, constituents) -> concat( + field(constituents.get(0)).nest("A"), + field(parent).nest("BUZZ"), + field(constituents.get(0)).nest("B")), + (syntheticTable, metaData) -> Assertions.assertEquals( + repeatedElementsUnderPath("f__2g", "x__2y"), + syntheticTable.getConstituents().get(0).getNestingExpression())); + } + + /** + * Navigates to the elements of a nullable repeated field reached through the given path of non-repeated fields, by storage name. + * + * @param path the storage names to navigate, the repeated field last + * @return the expected nesting expression + */ + @Nonnull + private static KeyExpression repeatedElementsUnderPath(@Nonnull final String... path) { + var expression = wrappedRepeatedElements(path[path.length - 1]); + for (int i = path.length - 2; i >= 0; i--) { + expression = field(path[i]).nest(expression); + } + return expression; + } +} diff --git a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/metadata/NoOpSchemaTemplateTests.java b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/metadata/NoOpSchemaTemplateTests.java index c20824299c8..43c6a3014a7 100644 --- a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/metadata/NoOpSchemaTemplateTests.java +++ b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/metadata/NoOpSchemaTemplateTests.java @@ -96,6 +96,17 @@ public void testGetViewsThrowsException() { assertEquals("NoOpSchemaTemplate doesn't have views!", exception.getMessage()); } + @Test + public void testGetSyntheticTablesThrowsException() { + final NoOpSchemaTemplate template = new NoOpSchemaTemplate("test", 1); + + final RelationalException exception = assertThrows(RelationalException.class, + template::getSyntheticTables); + + assertEquals(ErrorCode.INVALID_PARAMETER, exception.getErrorCode()); + assertEquals("NoOpSchemaTemplate doesn't have synthetic tables!", exception.getMessage()); + } + @Test public void testFindTableByNameThrowsException() { final NoOpSchemaTemplate template = new NoOpSchemaTemplate("test", 1); diff --git a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/metadata/SchemaTemplateSerDeTests.java b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/metadata/SchemaTemplateSerDeTests.java index ff5efbf31c8..378c615de4d 100644 --- a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/metadata/SchemaTemplateSerDeTests.java +++ b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/metadata/SchemaTemplateSerDeTests.java @@ -24,12 +24,15 @@ import com.apple.foundationdb.record.RecordStoreState; import com.apple.foundationdb.record.metadata.Index; import com.apple.foundationdb.record.metadata.IndexTypes; +import com.apple.foundationdb.record.metadata.UnnestedRecordType; import com.apple.foundationdb.record.metadata.Key; import com.apple.foundationdb.record.metadata.RecordTypeBuilder; import com.apple.foundationdb.record.metadata.expressions.KeyExpression; import com.apple.foundationdb.record.provider.foundationdb.IndexMaintainerFactoryRegistryImpl; import com.apple.foundationdb.record.query.plan.cascades.RawSqlFunction; +import com.apple.foundationdb.record.query.plan.cascades.typing.Type; import com.apple.foundationdb.record.query.plan.cascades.UserDefinedFunction; +import com.apple.foundationdb.record.util.ProtoUtils; import com.apple.foundationdb.record.util.pair.NonnullPair; import com.apple.foundationdb.relational.api.Options; import com.apple.foundationdb.relational.api.ddl.NoOpQueryFactory; @@ -46,6 +49,7 @@ import com.apple.foundationdb.relational.recordlayer.query.cache.NoOpMetricCollector; import com.apple.foundationdb.relational.recordlayer.query.functions.CompiledSqlFunction; import com.apple.foundationdb.relational.util.Assert; +import com.apple.test.BooleanSource; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Iterables; import com.google.protobuf.DescriptorProtos; @@ -802,6 +806,503 @@ void testRemoveViewFromSchemaTemplate() { Assertions.assertFalse(updatedTemplate.findViewByName("test_view").isPresent()); } + /** + * Navigates to an array's elements, the way the DDL layer does: a nullable array is stored wrapped as + * {@code { repeated T values; }}, a non-nullable one is a plain repeated field. + */ + @Nonnull + private static KeyExpression arrayElementsExpression(final String arrayFieldName, final boolean nullableArray) { + return nullableArray + ? Key.Expressions.field(arrayFieldName) + .nest(Key.Expressions.field("values", KeyExpression.FanType.FanOut)) + : Key.Expressions.field(arrayFieldName, KeyExpression.FanType.FanOut); + } + + @Nonnull + private static KeyExpression constituentField(final String alias, final String fieldName) { + return Key.Expressions.field(alias, KeyExpression.FanType.None).nest(fieldName); + } + + @Nonnull + private static DataType.StructType struct(final String name, final DataType.StructType.Field... fields) { + return DataType.StructType.from(name, List.of(fields), false); + } + + @Nonnull + private static DataType.StructType.Field structField(final String name, final DataType type, final int number) { + return DataType.StructType.Field.from(name, type, number); + } + + /** A table with a {@code bigint id} primary key plus the one column the unnested synthetic table unnests through. */ + @Nonnull + private static RecordLayerTable tableWithId(final String tableName, final String columnName, + final DataType columnType) { + return RecordLayerTable.newBuilder(false) + .setName(tableName) + .addColumn(RecordLayerColumn.newBuilder() + .setName("id") + .setDataType(DataType.Primitives.LONG.type()) + .build()) + .addColumn(RecordLayerColumn.newBuilder() + .setName(columnName) + .setDataType(columnType) + .build()) + .setPrimaryKey(Key.Expressions.concat(Key.Expressions.recordType(), Key.Expressions.field("id"))) + .build(); + } + + @Nonnull + private static RecordLayerUnnestedSyntheticTable syntheticTable( + final String syntheticName, final RecordLayerTable parentTable, final String indexName, + final KeyExpression keyExpression, + final RecordLayerUnnestedSyntheticTable.NestedConstituent... constituents) { + final var syntheticType = syntheticRecord(syntheticName, parentTable.getDatatype()); + final var builder = RecordLayerUnnestedSyntheticTable.newBuilder(syntheticType) + .setAlias("row") + .setParentTableType(parentTable.getType()); + for (final var constituent : constituents) { + builder.addConstituent(constituent); + } + return builder.addIndex(RecordLayerIndex.newBuilder() + .setName(indexName) + // both names come from the synthetic type, as the generator does it, so that they stay consistent + // when the declared name is not a legal protobuf identifier + .setTableType(syntheticType) + .setIndexType(IndexTypes.VALUE) + .setKeyExpression(keyExpression) + .build()) + .build(); + } + + /** + * The record type of the synthetic table itself, whose first field is the parent constituent carrying the stored + * table's type, as {@code RecordLayerUnnestedSyntheticTableGenerator} builds it. The fields for the nested + * constituents are left off: a {@link RecordLayerUnnestedSyntheticTable.NestedConstituent} carries only an alias and + * a nesting expression, not the element type the field would need, and nothing in this file reads the synthetic + * type. Once it is read -- by an accessor, by {@code equals}, or by serialization -- these tests need the real + * element types instead. + */ + @Nonnull + private static Type.Record syntheticRecord(final String syntheticName, final DataType.StructType parentType) { + return (Type.Record)DataTypeUtils.toRecordLayerType( + struct(syntheticName, structField("row", parentType, 1))); + } + + @Nonnull + private static RecordLayerSchemaTemplate templateWith(final RecordLayerTable table, + final RecordLayerUnnestedSyntheticTable syntheticTable, + final DataType.Named... auxiliaryTypes) { + final var builder = RecordLayerSchemaTemplate.newBuilder() + .setName("TestSchemaTemplate") + .setVersion(42); + for (final var auxiliaryType : auxiliaryTypes) { + builder.addAuxiliaryType(auxiliaryType); + } + return builder.addTable(table).addSyntheticTable(syntheticTable).build(); + } + + /** Serializes, asserting the synthetic type reaches the metadata rather than being silently dropped. */ + @Nonnull + private static RecordMetaData serializeWithSyntheticType(final RecordLayerSchemaTemplate template, + final String syntheticName) { + final var recordMetaData = template.toRecordMetadata(); + Assertions.assertTrue(recordMetaData.getSyntheticRecordTypes().containsKey(syntheticName), + () -> "synthetic type missing from serialized metadata, got " + + recordMetaData.getSyntheticRecordTypes().keySet()); + return recordMetaData; + } + + /** The sole non-parent constituent of a serialized synthetic type. */ + @Nonnull + private static UnnestedRecordType.NestedConstituent serializedConstituent(final RecordMetaData recordMetaData, + final String syntheticName) { + final var unnestedRecordType = (UnnestedRecordType) recordMetaData.getSyntheticRecordTypes().get(syntheticName); + return unnestedRecordType.getConstituents().stream() + .filter(candidate -> !candidate.isParent()).findFirst().orElseThrow(); + } + + @Nonnull + private static RecordLayerUnnestedSyntheticTable deserializeSyntheticTable(final RecordMetaData recordMetaData) { + final var syntheticTables = RecordLayerSchemaTemplate + .fromRecordMetadata(recordMetaData, "TestSchemaTemplate", 42) + .getUnnestedSyntheticTables(); + Assertions.assertEquals(1, syntheticTables.size()); + return syntheticTables.stream().findFirst().orElseThrow(); + } + + /** + * Round trips an unnested synthetic table over a struct array in both storage forms: a nullable array is stored wrapped as + * {@code { repeated T values; }}, a non-nullable one as a plain repeated field, and the serializer picks the + * constituent descriptor and nesting expression differently for each. + */ + @ParameterizedTest(name = "nullableArray = {0}") + @BooleanSource + void testUnnestedSyntheticTableSerializationAndDeserialization(final boolean nullableArray) { + final var syntheticName = "__unnested_employees_score_idx"; + final var scoreType = struct("score", + structField("label", DataType.Primitives.STRING.type(), 1), + structField("value", DataType.Primitives.LONG.type(), 2)); + final var table = tableWithId("employees", "scores", DataType.ArrayType.from(scoreType, nullableArray)); + final var keyExpression = Key.Expressions.concat( + constituentField("SQ", "label"), + constituentField("row", "id"), + constituentField("SQ", "value")); + final var originalTemplate = templateWith(table, + syntheticTable(syntheticName, table, "score_idx", keyExpression, + new RecordLayerUnnestedSyntheticTable.NestedConstituent("SQ", "row", + arrayElementsExpression("scores", nullableArray))), + scoreType); + + // The nesting expression says how to reach the array elements, and it depends on the storage form: a + // non-nullable array is a plain repeated field, a nullable one is wrapped in a holder message. + final var recordMetaData = serializeWithSyntheticType(originalTemplate, syntheticName); + Assertions.assertEquals(arrayElementsExpression("scores", nullableArray), + serializedConstituent(recordMetaData, syntheticName).getNestingExpression()); + + final var deserialized = deserializeSyntheticTable(recordMetaData); + Assertions.assertEquals(syntheticName, deserialized.getName()); + Assertions.assertEquals(Set.of("employees"), deserialized.getUnderlyingTableNames()); + Assertions.assertEquals("row", deserialized.getAlias()); + + final var constituent = Iterables.getOnlyElement(deserialized.getConstituents()); + Assertions.assertEquals("SQ", constituent.getAlias()); + Assertions.assertEquals("row", constituent.getParentAlias()); + Assertions.assertEquals(arrayElementsExpression("scores", nullableArray), constituent.getNestingExpression()); + + final var index = Iterables.getOnlyElement(deserialized.getIndexes()); + Assertions.assertEquals("score_idx", index.getName()); + Assertions.assertEquals(IndexTypes.VALUE, index.getIndexType()); + Assertions.assertEquals(keyExpression, KeyExpression.fromProto(index.getKeyExpression().toKeyExpression())); + // the index has to name the table it is defined on, or nothing ties the two together once reloaded + Assertions.assertEquals(deserialized.getName(), index.getTableName()); + } + + /** + * Round trips a synthetic table whose declared name is not a legal protobuf identifier. The name is composed from the + * index name, which is a user identifier, so it need not be one -- and the descriptor can only hold the escaped form. + * Every name the deserializer hands back has to be the declared one again, including the table name the index carries, + * which is what lets a reloaded table and its index still refer to each other. + */ + @Test + void testUnnestedSyntheticTableWithNonProtoCompliantNameSerializationAndDeserialization() { + final var syntheticName = "__unnested_employees_score.idx"; + final var storageName = ProtoUtils.toProtoBufCompliantName(syntheticName); + // the dot cannot survive into a proto identifier, so the two forms really do differ here + Assertions.assertNotEquals(syntheticName, storageName); + + final var scoreType = struct("score", + structField("label", DataType.Primitives.STRING.type(), 1), + structField("value", DataType.Primitives.LONG.type(), 2)); + final var table = tableWithId("employees", "scores", DataType.ArrayType.from(scoreType, true)); + final var keyExpression = Key.Expressions.concat( + constituentField("SQ", "label"), + constituentField("row", "id"), + constituentField("SQ", "value")); + final var originalTemplate = templateWith(table, + syntheticTable(syntheticName, table, "score.idx", keyExpression, + new RecordLayerUnnestedSyntheticTable.NestedConstituent("SQ", "row", + arrayElementsExpression("scores", true))), + scoreType); + + // the descriptor is keyed by the escaped form, which is what serializeWithSyntheticType looks it up by + final var recordMetaData = serializeWithSyntheticType(originalTemplate, storageName); + + final var deserialized = deserializeSyntheticTable(recordMetaData); + Assertions.assertEquals(syntheticName, deserialized.getName()); + Assertions.assertEquals(storageName, deserialized.getType().getStorageName()); + + final var index = Iterables.getOnlyElement(deserialized.getIndexes()); + Assertions.assertEquals("score.idx", index.getName()); + Assertions.assertEquals(syntheticName, index.getTableName()); + Assertions.assertEquals(storageName, index.getTableStorageName()); + } + + /** + * Indexes on an unnested synthetic table are reachable through the table-index mapping, attributed to the stored table + * they are maintained from. They were previously absent while {@code getIndexes()} listed them, so the two + * views of the same metadata disagreed. + */ + @Test + void tableIndexMappingIncludesSyntheticTableIndexes() throws RelationalException { + final var scoreType = struct("score", + structField("label", DataType.Primitives.STRING.type(), 1), + structField("value", DataType.Primitives.LONG.type(), 2)); + final var table = tableWithId("employees", "scores", DataType.ArrayType.from(scoreType, true)); + final var key = Key.Expressions.concat(constituentField("SQ", "label"), constituentField("row", "id")); + final var template = templateWith(table, + syntheticTable("__unnested_employees_score_idx", table, "score_idx", key, + new RecordLayerUnnestedSyntheticTable.NestedConstituent("SQ", "row", + arrayElementsExpression("scores", true))), + scoreType); + + // Attributed to the stored table the index is maintained from, not to the synthetic table, which is not + // itself a stored table and does not appear in getTables()/findTableByName. + final var mapping = template.getTableIndexMapping(); + Assertions.assertEquals(Set.of("score_idx"), Set.copyOf(mapping.get("employees"))); + Assertions.assertFalse(mapping.keySet().contains("__unnested_employees_score_idx"), + () -> "synthetic table leaked into a table-keyed mapping: " + mapping.keySet()); + Assertions.assertTrue(template.getIndexes().contains("score_idx")); + } + + /** + * A constituent's nesting expression has to be a path of fields, since that path is what the serializer follows to + * reach the element descriptor. Anything else describes a navigation nothing downstream can walk, and is rejected as + * the constituent is created. + */ + @Test + void constituentNestingExpressionMustBeAPathOfFields() { + final var thrown = Assertions.assertThrows(UncheckedRelationalException.class, () -> + new RecordLayerUnnestedSyntheticTable.NestedConstituent("SQ", "row", + Key.Expressions.field("scores").nest(Key.Expressions.concatenateFields("a", "b")))); + MatcherAssert.assertThat(thrown.getMessage(), Matchers.containsString("unsupported nesting expression")); + MatcherAssert.assertThat(thrown.getMessage(), Matchers.containsString("'SQ'")); + } + + /** + * Aliases name the records a synthetic record is composed of, so a duplicate -- or a parent alias naming + * nothing -- describes no well-formed type. Rejected at build time rather than left for a consumer to trip over. + */ + @Test + void duplicateConstituentAliasIsRejected() { + final var innerType = struct("inner", structField("y", DataType.Primitives.STRING.type(), 1)); + final var outerType = struct("outer", + structField("x", DataType.Primitives.STRING.type(), 1), + structField("q", DataType.ArrayType.from(innerType, true), 2)); + final var table = tableWithId("dupes", "p", DataType.ArrayType.from(outerType, true)); + final var key = Key.Expressions.concat(constituentField("SQ", "x"), constituentField("row", "id")); + // Rejected where the type is built, so no consumer downstream has to cope with a malformed one. + final var thrown = Assertions.assertThrows(UncheckedRelationalException.class, () -> + syntheticTable("__unnested_dupes_idx", table, "dupe_idx", key, + new RecordLayerUnnestedSyntheticTable.NestedConstituent("SQ", "row", + arrayElementsExpression("p", true)), + new RecordLayerUnnestedSyntheticTable.NestedConstituent("SQ", "SQ", + arrayElementsExpression("q", true)))); + Assertions.assertTrue(thrown.getMessage().contains("duplicate constituent alias"), + () -> "unexpected message: " + thrown.getMessage()); + + // A parent alias that names nothing is equally malformed. + final var danglingParent = Assertions.assertThrows(UncheckedRelationalException.class, () -> + syntheticTable("__unnested_dupes_idx", table, "dupe_idx", key, + new RecordLayerUnnestedSyntheticTable.NestedConstituent("SQ", "nobody", + arrayElementsExpression("p", true)))); + Assertions.assertTrue(danglingParent.getMessage().contains("is not a known alias"), + () -> "unexpected message: " + danglingParent.getMessage()); + + // Two constituents naming each other as parent describe a cycle, not a tree. Both aliases exist, so validating + // against the complete set would accept this; the parent has to be known when the constituent is declared. + final var cycle = Assertions.assertThrows(UncheckedRelationalException.class, () -> + syntheticTable("__unnested_dupes_idx", table, "dupe_idx", key, + new RecordLayerUnnestedSyntheticTable.NestedConstituent("SQ", "INNER", + arrayElementsExpression("p", true)), + new RecordLayerUnnestedSyntheticTable.NestedConstituent("INNER", "SQ", + arrayElementsExpression("q", true)))); + Assertions.assertTrue(cycle.getMessage().contains("is not a known alias"), + () -> "unexpected message: " + cycle.getMessage()); + + // The same shape ordered the other way is a forward reference: legal as a set, but the record layer types each + // constituent against its parent's descriptor, so the parent must come first. + final var forwardReference = Assertions.assertThrows(UncheckedRelationalException.class, () -> + syntheticTable("__unnested_dupes_idx", table, "dupe_idx", key, + new RecordLayerUnnestedSyntheticTable.NestedConstituent("INNER", "SQ", + arrayElementsExpression("q", true)), + new RecordLayerUnnestedSyntheticTable.NestedConstituent("SQ", "row", + arrayElementsExpression("p", true)))); + Assertions.assertTrue(forwardReference.getMessage().contains("is not a known alias"), + () -> "unexpected message: " + forwardReference.getMessage()); + } + + /** + * {@code toBuilder()} routes synthetic tables and views through the collection-taking builder methods, so a + * template must survive the round trip with both intact -- a synthetic table dropped here would take its + * indexes with it. + */ + @Test + void toBuilderPreservesSyntheticTablesAndViews() { + final var scoreType = struct("score", + structField("label", DataType.Primitives.STRING.type(), 1), + structField("value", DataType.Primitives.LONG.type(), 2)); + final var table = tableWithId("employees", "scores", DataType.ArrayType.from(scoreType, true)); + final var key = Key.Expressions.concat(constituentField("SQ", "label"), constituentField("row", "id")); + final var synthetic = syntheticTable("__unnested_employees_score_idx", table, "score_idx", key, + new RecordLayerUnnestedSyntheticTable.NestedConstituent("SQ", "row", + arrayElementsExpression("scores", true))); + final var original = RecordLayerSchemaTemplate.newBuilder() + .setName("TestSchemaTemplate") + .setVersion(42) + .addAuxiliaryType(scoreType) + .addTable(table) + .addSyntheticTable(synthetic) + .addView(RecordLayerView.newBuilder() + .setName("v1") + .setDescription("SELECT id FROM employees") + .setViewCompiler(ignored -> null) + .build()) + .build(); + + final var rebuilt = original.toBuilder().build(); + Assertions.assertEquals(original.getSyntheticTables(), rebuilt.getSyntheticTables()); + Assertions.assertEquals(1, rebuilt.getUnnestedSyntheticTables().size()); + Assertions.assertEquals(Set.of("v1"), + rebuilt.getViews().stream().map(RecordLayerView::getName).collect(Collectors.toSet())); + // the synthetic type's index survives too, and is still attributed to the stored table + Assertions.assertEquals(Set.of("score_idx"), + Set.copyOf(rebuilt.getTableIndexMapping().get("employees"))); + } + + /** + * Synthetic tables are held in a {@code Set}, so they need value semantics: two structurally identical + * instances must compare equal and collapse in a set, and a difference in any component must not. + */ + @Test + void unnestedSyntheticTableHasValueSemantics() { + final var scoreType = struct("score", + structField("label", DataType.Primitives.STRING.type(), 1), + structField("value", DataType.Primitives.LONG.type(), 2)); + final var table = tableWithId("employees", "scores", DataType.ArrayType.from(scoreType, true)); + final var key = Key.Expressions.concat(constituentField("SQ", "label"), constituentField("row", "id")); + final java.util.function.Supplier build = () -> + syntheticTable("__unnested_employees_score_idx", table, "score_idx", key, + new RecordLayerUnnestedSyntheticTable.NestedConstituent("SQ", "row", + arrayElementsExpression("scores", true))); + + Assertions.assertEquals(build.get(), build.get()); + Assertions.assertEquals(build.get().hashCode(), build.get().hashCode()); + // Set.copyOf collapses duplicates, unlike Set.of which rejects them. + Assertions.assertEquals(1, Set.copyOf(List.of(build.get(), build.get())).size()); + + // A differing constituent must break equality — otherwise the set would silently collapse distinct types. + final var differentConstituent = syntheticTable("__unnested_employees_score_idx", table, "score_idx", key, + new RecordLayerUnnestedSyntheticTable.NestedConstituent("OTHER", "row", + arrayElementsExpression("scores", true))); + Assertions.assertNotEquals(build.get(), differentConstituent); + + final var differentName = syntheticTable("__unnested_employees_other_idx", table, "score_idx", key, + new RecordLayerUnnestedSyntheticTable.NestedConstituent("SQ", "row", + arrayElementsExpression("scores", true))); + Assertions.assertNotEquals(build.get(), differentName); + + // Guards of equals() that no round-trip exercises: neither the table nor a constituent may equal null or an + // object of another class, or a set of them could collapse entries that are not in fact equal. The subject has + // to come first, since assertNotEquals compares via Objects.equals on its first argument. + final var constituent = new RecordLayerUnnestedSyntheticTable.NestedConstituent("SQ", "row", + arrayElementsExpression("scores", true)); + Assertions.assertNotEquals(build.get(), null); + Assertions.assertNotEquals(build.get(), "__unnested_employees_score_idx"); + Assertions.assertNotEquals(constituent, null); + Assertions.assertNotEquals(constituent, "SQ"); + } + + /** + * Both of the parent's names come from the parent's type and are kept apart: the declared name is what a schema + * template lookup uses, while the storage name is what the record layer's descriptor is keyed by. They differ + * whenever the declared name is not a legal protobuf identifier. + */ + @Test + void unnestedSyntheticTableKeepsParentDeclaredAndStorageNamesApart() { + final var parentType = struct("employee.records", structField("id", DataType.Primitives.LONG.type(), 1)); + final var table = RecordLayerUnnestedSyntheticTable.newBuilder( + syntheticRecord("__unnested_employees_score_idx", parentType)) + .setAlias("row") + .setParentTableType((Type.Record)DataTypeUtils.toRecordLayerType(parentType)) + .addConstituent(new RecordLayerUnnestedSyntheticTable.NestedConstituent("SQ", "row", + arrayElementsExpression("scores", true))) + .build(); + Assertions.assertEquals(Set.of("employee.records"), table.getUnderlyingTableNames()); + Assertions.assertEquals(ProtoUtils.toProtoBufCompliantName("employee.records"), + table.getParentTableStorageName()); + // the dot cannot survive into a proto identifier, so the storage name is not simply the table name + Assertions.assertFalse(table.getUnderlyingTableNames().contains(table.getParentTableStorageName())); + } + + /** + * Round trips an unnested synthetic table with two chained constituents, where the second unnests an array that lives on + * the element type of the first. + */ + @Test + void testChainedUnnestedSyntheticTableSerializationAndDeserialization() { + final var syntheticName = "__unnested_nested_employees_chained_idx"; + final var innerType = struct("inner", structField("y", DataType.Primitives.STRING.type(), 1)); + final var outerType = struct("outer", + structField("x", DataType.Primitives.STRING.type(), 1), + structField("q", DataType.ArrayType.from(innerType, true), 2)); + final var table = tableWithId("nested_employees", "p", DataType.ArrayType.from(outerType, true)); + final var keyExpression = Key.Expressions.concat( + constituentField("P_C", "x"), + constituentField("row", "id"), + constituentField("Q_C", "y")); + final var originalTemplate = templateWith(table, + syntheticTable(syntheticName, table, "chained_idx", keyExpression, + new RecordLayerUnnestedSyntheticTable.NestedConstituent("P_C", "row", + arrayElementsExpression("p", true)), + new RecordLayerUnnestedSyntheticTable.NestedConstituent("Q_C", "P_C", + arrayElementsExpression("q", true))), + innerType, outerType); + + final var deserialized = + deserializeSyntheticTable(serializeWithSyntheticType(originalTemplate, syntheticName)); + Assertions.assertEquals(syntheticName, deserialized.getName()); + Assertions.assertEquals(Set.of("nested_employees"), deserialized.getUnderlyingTableNames()); + Assertions.assertEquals("row", deserialized.getAlias()); + + // Both constituents must come back with the parent link and array field they went in with; the inner one + // hangs off the outer constituent, not off the parent table. + Assertions.assertEquals(List.of("P_C", "Q_C"), deserialized.getConstituents().stream() + .map(RecordLayerUnnestedSyntheticTable.NestedConstituent::getAlias) + .collect(Collectors.toList())); + Assertions.assertEquals(List.of("row", "P_C"), deserialized.getConstituents().stream() + .map(RecordLayerUnnestedSyntheticTable.NestedConstituent::getParentAlias) + .collect(Collectors.toList())); + Assertions.assertEquals(List.of(List.of("p", "values"), List.of("q", "values")), + deserialized.getConstituents().stream() + .map(RecordLayerUnnestedSyntheticTable.NestedConstituent::getFieldPath) + .collect(Collectors.toList())); + + final var index = Iterables.getOnlyElement(deserialized.getIndexes()); + Assertions.assertEquals("chained_idx", index.getName()); + Assertions.assertEquals(keyExpression, KeyExpression.fromProto(index.getKeyExpression().toKeyExpression())); + } + + /** + * Round trips a constituent whose array is reached through two meaningful hops, {@code map.entry} — the shape + * the record layer's own unnested record types use. Neither hop can be dropped and neither is the {@code values} + * wrapper, so this is only representable because the nesting expression is stored rather than a field name. + */ + @Test + void testUnnestedSyntheticTableOverTwoHopPathSerializationAndDeserialization() { + final var syntheticName = "__unnested_map_records_map_idx"; + final var entryType = struct("entryType", + structField("k", DataType.Primitives.STRING.type(), 1), + structField("v", DataType.Primitives.LONG.type(), 2)); + // A non-nullable array, so `entry` is a plain repeated field rather than a `values` wrapper. + final var mapHolderType = struct("mapHolder", + structField("entry", DataType.ArrayType.from(entryType, false), 1)); + final var table = tableWithId("map_records", "map", mapHolderType); + final var nestingExpression = Key.Expressions.field("map") + .nest(Key.Expressions.field("entry", KeyExpression.FanType.FanOut)); + final var keyExpression = Key.Expressions.concat( + constituentField("SQ", "k"), + constituentField("row", "id"), + constituentField("SQ", "v")); + final var originalTemplate = templateWith(table, + syntheticTable(syntheticName, table, "map_idx", keyExpression, + new RecordLayerUnnestedSyntheticTable.NestedConstituent("SQ", "row", + nestingExpression)), + entryType, mapHolderType); + + final var recordMetaData = serializeWithSyntheticType(originalTemplate, syntheticName); + final var nested = serializedConstituent(recordMetaData, syntheticName); + Assertions.assertEquals(nestingExpression, nested.getNestingExpression()); + // The constituent has to be the element type, which is only reachable by walking both hops. Stopping at + // `map` would yield the holder message, whose only field is `entry`. + Assertions.assertEquals(List.of("k", "v"), nested.getRecordType().getDescriptor().getFields().stream() + .map(Descriptors.FieldDescriptor::getName) + .collect(Collectors.toList())); + + final var constituent = Iterables.getOnlyElement(deserializeSyntheticTable(recordMetaData).getConstituents()); + Assertions.assertEquals(List.of("map", "entry"), constituent.getFieldPath()); + Assertions.assertEquals(nestingExpression, constituent.getNestingExpression()); + } + @Test void testViewSerializationAndDeserialization() { // Create a schema template with a view diff --git a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/DelegatingVisitorTest.java b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/DelegatingVisitorTest.java index a3571244085..372dab69353 100644 --- a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/DelegatingVisitorTest.java +++ b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/DelegatingVisitorTest.java @@ -29,7 +29,7 @@ import com.apple.foundationdb.relational.generated.RelationalParser; import com.apple.foundationdb.relational.recordlayer.ddl.NoOpMetadataOperationsFactory; import com.apple.foundationdb.relational.recordlayer.metadata.RecordLayerColumn; -import com.apple.foundationdb.relational.recordlayer.metadata.RecordLayerIndex; +import com.apple.foundationdb.relational.recordlayer.query.ddl.IndexGenerationResult; import com.apple.foundationdb.relational.recordlayer.metadata.RecordLayerSchemaTemplate; import com.apple.foundationdb.relational.recordlayer.metadata.RecordLayerTable; import com.apple.foundationdb.relational.recordlayer.query.visitors.BaseVisitor; @@ -363,7 +363,7 @@ void visitVectorIndexDefinitionTest() { generateMetadata(), NoOpQueryFactory.INSTANCE, NoOpMetadataOperationsFactory.INSTANCE, URI.create("/FDB/FRL1"), false) { @Override @SuppressWarnings({"NullableProblems", "DataFlowIssue"}) - public RecordLayerIndex visitVectorIndexDefinition(@Nonnull RelationalParser.VectorIndexDefinitionContext ctx) { + public IndexGenerationResult visitVectorIndexDefinition(@Nonnull RelationalParser.VectorIndexDefinitionContext ctx) { called.set(true); return null; } @@ -435,7 +435,7 @@ void visitIndexOnSourceDefinitionTest() { generateMetadata(), NoOpQueryFactory.INSTANCE, NoOpMetadataOperationsFactory.INSTANCE, URI.create("/FDB/FRL1"), false) { @Override @SuppressWarnings({"NullableProblems", "DataFlowIssue"}) - public RecordLayerIndex visitIndexOnSourceDefinition(@Nonnull RelationalParser.IndexOnSourceDefinitionContext ctx) { + public IndexGenerationResult visitIndexOnSourceDefinition(@Nonnull RelationalParser.IndexOnSourceDefinitionContext ctx) { called.set(true); return null; } @@ -486,7 +486,7 @@ void visitIndexAsSelectDefinitionTest() { generateMetadata(), NoOpQueryFactory.INSTANCE, NoOpMetadataOperationsFactory.INSTANCE, URI.create("/FDB/FRL1"), false) { @Override @SuppressWarnings({"NullableProblems", "DataFlowIssue"}) - public RecordLayerIndex visitIndexAsSelectDefinition(@Nonnull RelationalParser.IndexAsSelectDefinitionContext ctx) { + public IndexGenerationResult visitIndexAsSelectDefinition(@Nonnull RelationalParser.IndexAsSelectDefinitionContext ctx) { called.set(true); return null; } diff --git a/yaml-tests/src/test/java/YamlIntegrationTests.java b/yaml-tests/src/test/java/YamlIntegrationTests.java index 07bdcdd02c6..41a9cfadb36 100644 --- a/yaml-tests/src/test/java/YamlIntegrationTests.java +++ b/yaml-tests/src/test/java/YamlIntegrationTests.java @@ -482,6 +482,11 @@ public void unionEmptyTables(YamlTest.Runner runner) throws Exception { runner.runYamsql("union-empty-tables.yamsql"); } + @TestTemplate + public void unnestedRecordTypeIndexes(YamlTest.Runner runner) throws Exception { + runner.runYamsql("unnested-record-type-indexes.yamsql"); + } + @TestTemplate public void updateDeleteReturning(YamlTest.Runner runner) throws Exception { runner.runYamsql("update-delete-returning.yamsql"); diff --git a/yaml-tests/src/test/resources/array-join-at.metrics.binpb b/yaml-tests/src/test/resources/array-join-at.metrics.binpb index ef5dfc731c0..64679c4ecf1 100644 Binary files a/yaml-tests/src/test/resources/array-join-at.metrics.binpb and b/yaml-tests/src/test/resources/array-join-at.metrics.binpb differ diff --git a/yaml-tests/src/test/resources/array-join-at.metrics.yaml b/yaml-tests/src/test/resources/array-join-at.metrics.yaml index dd3181c1dd1..b9205ad1658 100644 --- a/yaml-tests/src/test/resources/array-join-at.metrics.yaml +++ b/yaml-tests/src/test/resources/array-join-at.metrics.yaml @@ -152,11 +152,11 @@ array-join-at: "at", (SELECT "val2" FROM T1 AS "OtherT1", "OtherT1"."arr1_nn" AS "val2" AT "at2" WHERE "OtherT1"."id" = T1."id" AND "val2" = "val" ) AS "subquery" ref: array-join-at.yamsql:225 - explain: SCAN([IS T1]) | FLATMAP q0 -> { EXPLODE q0.arr1 WITH ORDINALITY | FLATMAP - q1 -> { SCAN([IS T1, EQUALS q0.id]) | FLATMAP q2 -> { EXPLODE q2.arr1_nn WITH - ORDINALITY | FILTER _._0 EQUALS q1._0 AS q3 RETURN q3 } AS q3 RETURN (q3 AS - _0, q1 AS _1) } AS q5 RETURN (q0.id AS id, q5._1._1 AS at, q5._1._0 AS val, - q5._0._0 AS val2) } + explain: SCAN([IS T1]) | FLATMAP q0 -> { SCAN([IS T1, EQUALS q0.id]) | FLATMAP + q1 -> { EXPLODE q1.arr1_nn WITH ORDINALITY | FLATMAP q2 -> { EXPLODE q0.arr1 + WITH ORDINALITY | FILTER q2._0 EQUALS _._0 AS q3 RETURN (q3 AS _0, q2 AS _1) + } AS q4 RETURN (q4._0 AS _0, q4._1 AS _1) } AS q5 RETURN (q0.id AS id, q5._0._1 + AS at, q5._0._0 AS val, q5._1._0 AS val2) } task_count: 6359 task_total_time_ms: 111 transform_count: 3063 @@ -183,8 +183,8 @@ array-join-at: WHERE T1."id" IN (1, 2) ref: array-join-at.yamsql:308 explain: '[IN arrayDistinct(promote(@c21 AS ARRAY(LONG)))] | INJOIN q0 -> { SCAN([IS - T1, EQUALS q0]) | FLATMAP q1 -> { EXPLODE q1.arr1_nn WITH ORDINALITY AS q2 - RETURN (q1.id AS id, q2._0 AS val, q2._1 AS at) } }' + T1, EQUALS q0]) } | FLATMAP q1 -> { EXPLODE q1.arr1_nn WITH ORDINALITY AS + q2 RETURN (q1.id AS id, q2._0 AS val, q2._1 AS at) }' task_count: 1162 task_total_time_ms: 14 transform_count: 452 @@ -196,14 +196,14 @@ array-join-at: - query: EXPLAIN SELECT "id", "val", "at" FROM T1, T1."arr1_nn" AS "val" AT "at" WHERE "at" IN (1, 2) ref: array-join-at.yamsql:320 - explain: SCAN([IS T1]) | FLATMAP q0 -> { EXPLODE q0.arr1_nn WITH ORDINALITY | - FILTER _._1 IN @c19 AS q1 RETURN (q0.id AS id, q1._0 AS val, q1._1 AS at) - } - task_count: 1056 - task_total_time_ms: 11 - transform_count: 421 - transform_time_ms: 3 - transform_yield_count: 52 - insert_time_ms: 0 - insert_new_count: 115 + explain: EXPLODE arrayDistinct(@c19) | FLATMAP q0 -> { SCAN([IS T1]) | FLATMAP + q1 -> { EXPLODE q1.arr1_nn WITH ORDINALITY | FILTER _._1 EQUALS q0 AS q2 RETURN + (q1.id AS id, q2._0 AS val, q2._1 AS at) } AS q3 RETURN q3 } + task_count: 1045 + task_total_time_ms: 25 + transform_count: 420 + transform_time_ms: 9 + transform_yield_count: 51 + insert_time_ms: 2 + insert_new_count: 114 insert_reused_count: 6 diff --git a/yaml-tests/src/test/resources/array-join-at.yamsql b/yaml-tests/src/test/resources/array-join-at.yamsql index 21903dcaf78..88ca8f3461d 100644 --- a/yaml-tests/src/test/resources/array-join-at.yamsql +++ b/yaml-tests/src/test/resources/array-join-at.yamsql @@ -222,7 +222,7 @@ test_block: WHERE "OtherT1"."id" = T1."id" AND "val2" = "val" ) AS "subquery" - - explain: "SCAN([IS T1]) | FLATMAP q0 -> { EXPLODE q0.arr1 WITH ORDINALITY | FLATMAP q1 -> { SCAN([IS T1, EQUALS q0.id]) | FLATMAP q2 -> { EXPLODE q2.arr1_nn WITH ORDINALITY | FILTER _._0 EQUALS q1._0 AS q3 RETURN q3 } AS q3 RETURN (q3 AS _0, q1 AS _1) } AS q5 RETURN (q0.id AS id, q5._1._1 AS at, q5._1._0 AS val, q5._0._0 AS val2) }" + - explain: "SCAN([IS T1]) | FLATMAP q0 -> { SCAN([IS T1, EQUALS q0.id]) | FLATMAP q1 -> { EXPLODE q1.arr1_nn WITH ORDINALITY | FLATMAP q2 -> { EXPLODE q0.arr1 WITH ORDINALITY | FILTER q2._0 EQUALS _._0 AS q3 RETURN (q3 AS _0, q2 AS _1) } AS q4 RETURN (q4._0 AS _0, q4._1 AS _1) } AS q5 RETURN (q0.id AS id, q5._0._1 AS at, q5._0._0 AS val, q5._1._0 AS val2) }" - resultMetadata: [{id: BIGINT}, {at: INTEGER}, {val: INTEGER}, {val2: INTEGER}] - unorderedResult: [ {id: 1, at: 1, val: 101, val2: 101}, @@ -305,7 +305,7 @@ test_block: - query: SELECT "id", "val", "at" FROM T1, T1."arr1_nn" AS "val" AT "at" WHERE T1."id" IN (1, 2) - - explain: "[IN arrayDistinct(promote(@c21 AS ARRAY(LONG)))] | INJOIN q0 -> { SCAN([IS T1, EQUALS q0]) | FLATMAP q1 -> { EXPLODE q1.arr1_nn WITH ORDINALITY AS q2 RETURN (q1.id AS id, q2._0 AS val, q2._1 AS at) } }" + - explain: "[IN arrayDistinct(promote(@c21 AS ARRAY(LONG)))] | INJOIN q0 -> { SCAN([IS T1, EQUALS q0]) } | FLATMAP q1 -> { EXPLODE q1.arr1_nn WITH ORDINALITY AS q2 RETURN (q1.id AS id, q2._0 AS val, q2._1 AS at) }" - unorderedResult: [ {id: 1, val: 101, at: 1}, {id: 2, val: 201, at: 1}, @@ -317,7 +317,7 @@ test_block: - query: SELECT "id", "val", "at" FROM T1, T1."arr1_nn" AS "val" AT "at" WHERE "at" IN (1, 2) - - explain: "SCAN([IS T1]) | FLATMAP q0 -> { EXPLODE q0.arr1_nn WITH ORDINALITY | FILTER _._1 IN @c19 AS q1 RETURN (q0.id AS id, q1._0 AS val, q1._1 AS at) }" + - explain: "EXPLODE arrayDistinct(@c19) | FLATMAP q0 -> { SCAN([IS T1]) | FLATMAP q1 -> { EXPLODE q1.arr1_nn WITH ORDINALITY | FILTER _._1 EQUALS q0 AS q2 RETURN (q1.id AS id, q2._0 AS val, q2._1 AS at) } AS q3 RETURN q3 }" - resultMetadata: [{id: BIGINT}, {val: INTEGER}, {at: INTEGER}] - unorderedResult: [ {id: 1, val: 101, at: 1}, diff --git a/yaml-tests/src/test/resources/in-predicate.metrics.binpb b/yaml-tests/src/test/resources/in-predicate.metrics.binpb index 84bd3cdf1be..e1de582ff43 100644 --- a/yaml-tests/src/test/resources/in-predicate.metrics.binpb +++ b/yaml-tests/src/test/resources/in-predicate.metrics.binpb @@ -918,83 +918,59 @@ U rankDir=LR; 2 -> 4 [ color="red" style="invis" ]; } -}" +} i - unnamed-2\EXPLAIN SELECT t.id, n, idx FROM array_table AS t, t.numbers AS n AT idx WHERE n IN (10, 20)! -' (60Ҹ8s@EXPLODE arrayDistinct(promote(@c23 AS ARRAY(LONG))) | FLATMAP q0 -> { SCAN([IS ARRAY_TABLE]) | FLATMAP q1 -> { EXPLODE q1.NUMBERS WITH ORDINALITY | FILTER _._0 EQUALS q0 AS q2 RETURN (q1.ID AS ID, q2._0 AS N, q2._1 AS IDX) } AS q3 RETURN q3 }digraph G { + unnamed-2\EXPLAIN SELECT t.id, n, idx FROM array_table AS t, t.numbers AS n AT idx WHERE n IN (10, 20) ++ (508r@SCAN([IS ARRAY_TABLE]) | FLATMAP q0 -> { EXPLODE q0.NUMBERS WITH ORDINALITY | FILTER _._0 IN promote(@c23 AS ARRAY(LONG)) AS q1 RETURN (q0.ID AS ID, q1._0 AS N, q1._1 AS IDX) }digraph G { fontname=courier; rankdir=BT; splines=line; - 1 [ label=<
Nested Loop Join
FLATMAP q92
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(LONG AS ID, LONG AS N, INT AS IDX)" ]; - 2 [ label=<
Value Computation
EXPLODE arrayDistinct(promote(@c23 AS ARRAY(LONG)))
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(LONG)" ]; - 3 [ label=<
Nested Loop Join
FLATMAP (q2.ID AS ID, q4._0 AS N, q4._1 AS IDX)
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(LONG AS ID, LONG AS N, INT AS IDX)" ]; - 4 [ label=<
Scan
comparisons: [IS ARRAY_TABLE]
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(LONG AS ID, ARRAY(STRING) AS FRUITS, ARRAY(LONG) AS NUMBERS, ARRAY(STRING AS NAME, STRING AS COLOR) AS FRUIT_RECORDS)" ]; - 5 [ label=<
Predicate Filter
WHERE q4._0 EQUALS q62
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(LONG AS _0, INT AS _1)" ]; - 6 [ label=<
Primary Storage
record types: [T_CHILD, ARRAY_TABLE, T_PARENT, TA]
> color="black" shape="plain" style="filled" fillcolor="lightblue" fontname="courier" fontsize="8" tooltip="RELATION(LONG AS ID, ARRAY(STRING) AS FRUITS, ARRAY(LONG) AS NUMBERS, ARRAY(STRING AS NAME, STRING AS COLOR) AS FRUIT_RECORDS)" ]; - 7 [ label=<
Value Computation
EXPLODE q2.NUMBERS WITH ORDINALITY
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(LONG AS _0, INT AS _1)" ]; - 2 -> 1 [ label=< q62> label="q62" color="gray20" style="solid" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; - 4 -> 3 [ label=< q2> label="q2" color="gray20" style="solid" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; - 5 -> 3 [ label=< q4> label="q4" color="gray20" style="solid" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; - 6 -> 4 [ color="gray20" style="solid" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; - 7 -> 5 [ label=< q4> label="q4" color="gray20" style="bold" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; - 3 -> 1 [ label=< q92> label="q92" color="gray20" style="solid" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; + 1 [ label=<
Nested Loop Join
FLATMAP (q2.ID AS ID, q4._0 AS N, q4._1 AS IDX)
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(LONG AS ID, LONG AS N, INT AS IDX)" ]; + 2 [ label=<
Scan
comparisons: [IS ARRAY_TABLE]
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(LONG AS ID, ARRAY(STRING) AS FRUITS, ARRAY(LONG) AS NUMBERS, ARRAY(STRING AS NAME, STRING AS COLOR) AS FRUIT_RECORDS)" ]; + 3 [ label=<
Primary Storage
record types: [T_CHILD, ARRAY_TABLE, T_PARENT, TA]
> color="black" shape="plain" style="filled" fillcolor="lightblue" fontname="courier" fontsize="8" tooltip="RELATION(LONG AS ID, ARRAY(STRING) AS FRUITS, ARRAY(LONG) AS NUMBERS, ARRAY(STRING AS NAME, STRING AS COLOR) AS FRUIT_RECORDS)" ]; + 4 [ label=<
Predicate Filter
WHERE q4._0 IN promote(@c23 AS ARRAY(LONG))
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(LONG AS _0, INT AS _1)" ]; + 5 [ label=<
Value Computation
EXPLODE q2.NUMBERS WITH ORDINALITY
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(LONG AS _0, INT AS _1)" ]; + 3 -> 2 [ color="gray20" style="solid" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; + 2 -> 1 [ label=< q2> label="q2" color="gray20" style="solid" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; + 5 -> 4 [ label=< q4> label="q4" color="gray20" style="bold" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; + 4 -> 1 [ label=< q4> label="q4" color="gray20" style="solid" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; { 5 -> 1 [ color="blue" style="dotted" arrowhead="none" tailport="nw" headport="s" constraint="false" ]; } { - rank=same; - rankDir=LR; - 2 -> 3 [ color="red" style="invis" ]; - } - { - 7 -> 3 [ color="blue" style="dotted" arrowhead="none" tailport="nw" headport="s" constraint="false" ]; - } - { - 5 -> 3 [ color="blue" style="dotted" arrowhead="none" tailport="nw" headport="s" constraint="false" ]; + 4 -> 1 [ color="blue" style="dotted" arrowhead="none" tailport="nw" headport="s" constraint="false" ]; } { rank=same; rankDir=LR; - 4 -> 5 [ color="red" style="invis" ]; + 2 -> 4 [ color="red" style="invis" ]; } -}! +} i - unnamed-2\EXPLAIN SELECT t.id, n, idx FROM array_table AS t, t.numbers AS n AT idx WHERE idx IN (1, 3)! -އ  (60洊8s@EXPLODE arrayDistinct(@c23) | FLATMAP q0 -> { SCAN([IS ARRAY_TABLE]) | FLATMAP q1 -> { EXPLODE q1.NUMBERS WITH ORDINALITY | FILTER _._1 EQUALS q0 AS q2 RETURN (q1.ID AS ID, q2._0 AS N, q2._1 AS IDX) } AS q3 RETURN q3 }digraph G { + unnamed-2\EXPLAIN SELECT t.id, n, idx FROM array_table AS t, t.numbers AS n AT idx WHERE idx IN (1, 3) +Ȟ- ⰹ (508r@SCAN([IS ARRAY_TABLE]) | FLATMAP q0 -> { EXPLODE q0.NUMBERS WITH ORDINALITY | FILTER _._1 IN @c23 AS q1 RETURN (q0.ID AS ID, q1._0 AS N, q1._1 AS IDX) }digraph G { fontname=courier; rankdir=BT; splines=line; - 1 [ label=<
Nested Loop Join
FLATMAP q92
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(LONG AS ID, LONG AS N, INT AS IDX)" ]; - 2 [ label=<
Value Computation
EXPLODE arrayDistinct(@c23)
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(INT)" ]; - 3 [ label=<
Nested Loop Join
FLATMAP (q2.ID AS ID, q4._0 AS N, q4._1 AS IDX)
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(LONG AS ID, LONG AS N, INT AS IDX)" ]; - 4 [ label=<
Scan
comparisons: [IS ARRAY_TABLE]
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(LONG AS ID, ARRAY(STRING) AS FRUITS, ARRAY(LONG) AS NUMBERS, ARRAY(STRING AS NAME, STRING AS COLOR) AS FRUIT_RECORDS)" ]; - 5 [ label=<
Predicate Filter
WHERE q4._1 EQUALS q62
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(LONG AS _0, INT AS _1)" ]; - 6 [ label=<
Primary Storage
record types: [T_CHILD, ARRAY_TABLE, T_PARENT, TA]
> color="black" shape="plain" style="filled" fillcolor="lightblue" fontname="courier" fontsize="8" tooltip="RELATION(LONG AS ID, ARRAY(STRING) AS FRUITS, ARRAY(LONG) AS NUMBERS, ARRAY(STRING AS NAME, STRING AS COLOR) AS FRUIT_RECORDS)" ]; - 7 [ label=<
Value Computation
EXPLODE q2.NUMBERS WITH ORDINALITY
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(LONG AS _0, INT AS _1)" ]; - 2 -> 1 [ label=< q62> label="q62" color="gray20" style="solid" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; - 4 -> 3 [ label=< q2> label="q2" color="gray20" style="solid" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; - 5 -> 3 [ label=< q4> label="q4" color="gray20" style="solid" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; - 6 -> 4 [ color="gray20" style="solid" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; - 7 -> 5 [ label=< q4> label="q4" color="gray20" style="bold" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; - 3 -> 1 [ label=< q92> label="q92" color="gray20" style="solid" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; + 1 [ label=<
Nested Loop Join
FLATMAP (q2.ID AS ID, q4._0 AS N, q4._1 AS IDX)
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(LONG AS ID, LONG AS N, INT AS IDX)" ]; + 2 [ label=<
Scan
comparisons: [IS ARRAY_TABLE]
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(LONG AS ID, ARRAY(STRING) AS FRUITS, ARRAY(LONG) AS NUMBERS, ARRAY(STRING AS NAME, STRING AS COLOR) AS FRUIT_RECORDS)" ]; + 3 [ label=<
Primary Storage
record types: [T_CHILD, ARRAY_TABLE, T_PARENT, TA]
> color="black" shape="plain" style="filled" fillcolor="lightblue" fontname="courier" fontsize="8" tooltip="RELATION(LONG AS ID, ARRAY(STRING) AS FRUITS, ARRAY(LONG) AS NUMBERS, ARRAY(STRING AS NAME, STRING AS COLOR) AS FRUIT_RECORDS)" ]; + 4 [ label=<
Predicate Filter
WHERE q4._1 IN @c23
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(LONG AS _0, INT AS _1)" ]; + 5 [ label=<
Value Computation
EXPLODE q2.NUMBERS WITH ORDINALITY
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(LONG AS _0, INT AS _1)" ]; + 3 -> 2 [ color="gray20" style="solid" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; + 2 -> 1 [ label=< q2> label="q2" color="gray20" style="solid" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; + 5 -> 4 [ label=< q4> label="q4" color="gray20" style="bold" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; + 4 -> 1 [ label=< q4> label="q4" color="gray20" style="solid" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; { 5 -> 1 [ color="blue" style="dotted" arrowhead="none" tailport="nw" headport="s" constraint="false" ]; } { - rank=same; - rankDir=LR; - 2 -> 3 [ color="red" style="invis" ]; - } - { - 7 -> 3 [ color="blue" style="dotted" arrowhead="none" tailport="nw" headport="s" constraint="false" ]; - } - { - 5 -> 3 [ color="blue" style="dotted" arrowhead="none" tailport="nw" headport="s" constraint="false" ]; + 4 -> 1 [ color="blue" style="dotted" arrowhead="none" tailport="nw" headport="s" constraint="false" ]; } { rank=same; rankDir=LR; - 4 -> 5 [ color="red" style="invis" ]; + 2 -> 4 [ color="red" style="invis" ]; } } j @@ -1027,43 +1003,31 @@ j rankDir=LR; 2 -> 6 [ color="red" style="invis" ]; } -}" +} r - unnamed-2eEXPLAIN SELECT t.id, f, idx FROM array_table AS t, t.fruits AS f AT idx WHERE f IN ('apple', 'mango')! -% Ծ(608s@EXPLODE arrayDistinct(@c23) | FLATMAP q0 -> { SCAN([IS ARRAY_TABLE]) | FLATMAP q1 -> { EXPLODE q1.FRUITS WITH ORDINALITY | FILTER _._0 EQUALS q0 AS q2 RETURN (q1.ID AS ID, q2._0 AS F, q2._1 AS IDX) } AS q3 RETURN q3 }digraph G { + unnamed-2eEXPLAIN SELECT t.id, f, idx FROM array_table AS t, t.fruits AS f AT idx WHERE f IN ('apple', 'mango') +/ (508r@SCAN([IS ARRAY_TABLE]) | FLATMAP q0 -> { EXPLODE q0.FRUITS WITH ORDINALITY | FILTER _._0 IN @c23 AS q1 RETURN (q0.ID AS ID, q1._0 AS F, q1._1 AS IDX) }digraph G { fontname=courier; rankdir=BT; splines=line; - 1 [ label=<
Nested Loop Join
FLATMAP q92
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(LONG AS ID, STRING AS F, INT AS IDX)" ]; - 2 [ label=<
Value Computation
EXPLODE arrayDistinct(@c23)
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(STRING)" ]; - 3 [ label=<
Nested Loop Join
FLATMAP (q2.ID AS ID, q4._0 AS F, q4._1 AS IDX)
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(LONG AS ID, STRING AS F, INT AS IDX)" ]; - 4 [ label=<
Scan
comparisons: [IS ARRAY_TABLE]
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(LONG AS ID, ARRAY(STRING) AS FRUITS, ARRAY(LONG) AS NUMBERS, ARRAY(STRING AS NAME, STRING AS COLOR) AS FRUIT_RECORDS)" ]; - 5 [ label=<
Predicate Filter
WHERE q4._0 EQUALS q62
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(STRING AS _0, INT AS _1)" ]; - 6 [ label=<
Primary Storage
record types: [T_CHILD, ARRAY_TABLE, T_PARENT, TA]
> color="black" shape="plain" style="filled" fillcolor="lightblue" fontname="courier" fontsize="8" tooltip="RELATION(LONG AS ID, ARRAY(STRING) AS FRUITS, ARRAY(LONG) AS NUMBERS, ARRAY(STRING AS NAME, STRING AS COLOR) AS FRUIT_RECORDS)" ]; - 7 [ label=<
Value Computation
EXPLODE q2.FRUITS WITH ORDINALITY
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(STRING AS _0, INT AS _1)" ]; - 2 -> 1 [ label=< q62> label="q62" color="gray20" style="solid" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; - 4 -> 3 [ label=< q2> label="q2" color="gray20" style="solid" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; - 5 -> 3 [ label=< q4> label="q4" color="gray20" style="solid" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; - 6 -> 4 [ color="gray20" style="solid" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; - 7 -> 5 [ label=< q4> label="q4" color="gray20" style="bold" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; - 3 -> 1 [ label=< q92> label="q92" color="gray20" style="solid" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; - { - 7 -> 3 [ color="blue" style="dotted" arrowhead="none" tailport="nw" headport="s" constraint="false" ]; - } - { - 5 -> 3 [ color="blue" style="dotted" arrowhead="none" tailport="nw" headport="s" constraint="false" ]; - } + 1 [ label=<
Nested Loop Join
FLATMAP (q2.ID AS ID, q4._0 AS F, q4._1 AS IDX)
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(LONG AS ID, STRING AS F, INT AS IDX)" ]; + 2 [ label=<
Scan
comparisons: [IS ARRAY_TABLE]
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(LONG AS ID, ARRAY(STRING) AS FRUITS, ARRAY(LONG) AS NUMBERS, ARRAY(STRING AS NAME, STRING AS COLOR) AS FRUIT_RECORDS)" ]; + 3 [ label=<
Primary Storage
record types: [T_CHILD, ARRAY_TABLE, T_PARENT, TA]
> color="black" shape="plain" style="filled" fillcolor="lightblue" fontname="courier" fontsize="8" tooltip="RELATION(LONG AS ID, ARRAY(STRING) AS FRUITS, ARRAY(LONG) AS NUMBERS, ARRAY(STRING AS NAME, STRING AS COLOR) AS FRUIT_RECORDS)" ]; + 4 [ label=<
Predicate Filter
WHERE q4._0 IN @c23
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(STRING AS _0, INT AS _1)" ]; + 5 [ label=<
Value Computation
EXPLODE q2.FRUITS WITH ORDINALITY
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(STRING AS _0, INT AS _1)" ]; + 3 -> 2 [ color="gray20" style="solid" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; + 2 -> 1 [ label=< q2> label="q2" color="gray20" style="solid" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; + 5 -> 4 [ label=< q4> label="q4" color="gray20" style="bold" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; + 4 -> 1 [ label=< q4> label="q4" color="gray20" style="solid" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; { - rank=same; - rankDir=LR; - 4 -> 5 [ color="red" style="invis" ]; + 5 -> 1 [ color="blue" style="dotted" arrowhead="none" tailport="nw" headport="s" constraint="false" ]; } { - 5 -> 1 [ color="blue" style="dotted" arrowhead="none" tailport="nw" headport="s" constraint="false" ]; + 4 -> 1 [ color="blue" style="dotted" arrowhead="none" tailport="nw" headport="s" constraint="false" ]; } { rank=same; rankDir=LR; - 2 -> 3 [ color="red" style="invis" ]; + 2 -> 4 [ color="red" style="invis" ]; } } \ No newline at end of file diff --git a/yaml-tests/src/test/resources/in-predicate.metrics.yaml b/yaml-tests/src/test/resources/in-predicate.metrics.yaml index bfbe2cafa21..5f73ca8f0e1 100644 --- a/yaml-tests/src/test/resources/in-predicate.metrics.yaml +++ b/yaml-tests/src/test/resources/in-predicate.metrics.yaml @@ -1,6 +1,6 @@ unnamed-2: - query: EXPLAIN select a, b from ta where b in (1, 3, 5, 7) - ref: in-predicate.yamsql:69 + ref: in-predicate.yamsql:72 explain: ISCAN(F1 <,>) | FLATMAP q0 -> { EXPLODE arrayDistinct(promote(@c9 AS ARRAY(LONG))) | FILTER q0.B EQUALS _ AS q1 RETURN (q0.A AS A, q0.B AS B) } task_count: 939 @@ -12,7 +12,7 @@ unnamed-2: insert_new_count: 118 insert_reused_count: 7 - query: EXPLAIN select a, b from ta where b in (6) - ref: in-predicate.yamsql:74 + ref: in-predicate.yamsql:77 explain: ISCAN(F1 <,>) | FLATMAP q0 -> { EXPLODE arrayDistinct(promote(@c9 AS ARRAY(LONG))) | FILTER q0.B EQUALS _ AS q1 RETURN (q0.A AS A, q0.B AS B) } task_count: 939 @@ -24,7 +24,7 @@ unnamed-2: insert_new_count: 118 insert_reused_count: 7 - query: EXPLAIN select a, b from ta where b in (10, 33, 66) - ref: in-predicate.yamsql:83 + ref: in-predicate.yamsql:86 explain: ISCAN(F1 <,>) | FLATMAP q0 -> { EXPLODE arrayDistinct(promote(@c9 AS ARRAY(LONG))) | FILTER q0.B EQUALS _ AS q1 RETURN (q0.A AS A, q0.B AS B) } task_count: 939 @@ -36,7 +36,7 @@ unnamed-2: insert_new_count: 118 insert_reused_count: 7 - query: EXPLAIN select a, b from ta where b in (1 + 0, 3 + 0, 5, 7) - ref: in-predicate.yamsql:88 + ref: in-predicate.yamsql:91 explain: ISCAN(F1 <,>) | FLATMAP q0 -> { EXPLODE arrayDistinct(promote(array(@c10 + @c12, @c14 + @c12, @c18, @c20) AS ARRAY(LONG))) | FILTER q0.B EQUALS _ AS q1 RETURN (q0.A AS A, q0.B AS B) } @@ -49,7 +49,7 @@ unnamed-2: insert_new_count: 118 insert_reused_count: 7 - query: EXPLAIN select a, b from ta where b in (1, 1, 1, 1) - ref: in-predicate.yamsql:93 + ref: in-predicate.yamsql:96 explain: ISCAN(F1 <,>) | FLATMAP q0 -> { EXPLODE arrayDistinct(promote(@c9 AS ARRAY(LONG))) | FILTER q0.B EQUALS _ AS q1 RETURN (q0.A AS A, q0.B AS B) } task_count: 939 @@ -61,7 +61,7 @@ unnamed-2: insert_new_count: 118 insert_reused_count: 7 - query: EXPLAIN select a, b from ta where b in (1, 2, 1, 3) - ref: in-predicate.yamsql:102 + ref: in-predicate.yamsql:105 explain: ISCAN(F1 <,>) | FLATMAP q0 -> { EXPLODE arrayDistinct(promote(@c9 AS ARRAY(LONG))) | FILTER q0.B EQUALS _ AS q1 RETURN (q0.A AS A, q0.B AS B) } task_count: 939 @@ -73,7 +73,7 @@ unnamed-2: insert_new_count: 118 insert_reused_count: 7 - query: EXPLAIN select a, b from ta where b in (1 + 0, 1 + 0, 1 + 0, 1 + 0) - ref: in-predicate.yamsql:111 + ref: in-predicate.yamsql:114 explain: ISCAN(F1 <,>) | FLATMAP q0 -> { EXPLODE arrayDistinct(promote(array(@c10 + @c12, @c10 + @c12, @c10 + @c12, @c10 + @c12) AS ARRAY(LONG))) | FILTER q0.B EQUALS _ AS q1 RETURN (q0.A AS A, q0.B AS B) } @@ -86,7 +86,7 @@ unnamed-2: insert_new_count: 118 insert_reused_count: 7 - query: EXPLAIN select a, b from ta where b in (1 + 0, 0 + 1) - ref: in-predicate.yamsql:120 + ref: in-predicate.yamsql:123 explain: ISCAN(F1 <,>) | FLATMAP q0 -> { EXPLODE arrayDistinct(promote(array(@c10 + @c12, @c12 + @c10) AS ARRAY(LONG))) | FILTER q0.B EQUALS _ AS q1 RETURN (q0.A AS A, q0.B AS B) } @@ -99,7 +99,7 @@ unnamed-2: insert_new_count: 118 insert_reused_count: 7 - query: EXPLAIN select a, c from ta where c in (4.56, 3.45, 2.34) - ref: in-predicate.yamsql:133 + ref: in-predicate.yamsql:168 explain: ISCAN(F1 <,>) | FLATMAP q0 -> { EXPLODE arrayDistinct(@c9) | FILTER q0.C EQUALS _ AS q1 RETURN (q0.A AS A, q0.C AS C) } task_count: 939 @@ -111,7 +111,7 @@ unnamed-2: insert_new_count: 118 insert_reused_count: 7 - query: EXPLAIN select a, c from ta where c in (9.01) - ref: in-predicate.yamsql:138 + ref: in-predicate.yamsql:173 explain: ISCAN(F1 <,>) | FLATMAP q0 -> { EXPLODE arrayDistinct(@c9) | FILTER q0.C EQUALS _ AS q1 RETURN (q0.A AS A, q0.C AS C) } task_count: 939 @@ -123,7 +123,7 @@ unnamed-2: insert_new_count: 118 insert_reused_count: 7 - query: EXPLAIN select a, c from ta where c in (3.01 + 6) - ref: in-predicate.yamsql:143 + ref: in-predicate.yamsql:178 explain: ISCAN(F1 <,>) | FLATMAP q0 -> { EXPLODE arrayDistinct(array(@c10 + @c12)) | FILTER q0.C EQUALS _ AS q1 RETURN (q0.A AS A, q0.C AS C) } task_count: 939 @@ -135,7 +135,7 @@ unnamed-2: insert_new_count: 118 insert_reused_count: 7 - query: EXPLAIN select a, d from ta where d in (true, false) - ref: in-predicate.yamsql:148 + ref: in-predicate.yamsql:183 explain: ISCAN(F1 <,>) | FLATMAP q0 -> { EXPLODE arrayDistinct(@c9) | FILTER q0.D EQUALS _ AS q1 RETURN (q0.A AS A, q0.D AS D) } task_count: 939 @@ -147,7 +147,7 @@ unnamed-2: insert_new_count: 118 insert_reused_count: 7 - query: EXPLAIN select a, d from ta where d in (true) - ref: in-predicate.yamsql:154 + ref: in-predicate.yamsql:189 explain: ISCAN(F1 <,>) | FLATMAP q0 -> { EXPLODE arrayDistinct(@c9) | FILTER q0.D EQUALS _ AS q1 RETURN (q0.A AS A, q0.D AS D) } task_count: 939 @@ -159,7 +159,7 @@ unnamed-2: insert_new_count: 118 insert_reused_count: 7 - query: EXPLAIN select a, d from ta where d in (3 < 4) - ref: in-predicate.yamsql:159 + ref: in-predicate.yamsql:194 explain: ISCAN(F1 <,>) | FLATMAP q0 -> { EXPLODE arrayDistinct(array(@c10 lt @c12)) | FILTER q0.D EQUALS _ AS q1 RETURN (q0.A AS A, q0.D AS D) } task_count: 939 @@ -171,7 +171,7 @@ unnamed-2: insert_new_count: 118 insert_reused_count: 7 - query: EXPLAIN select a, e from ta where e in ('bar', 'doe') - ref: in-predicate.yamsql:164 + ref: in-predicate.yamsql:199 explain: ISCAN(F1 <,>) | FLATMAP q0 -> { EXPLODE arrayDistinct(@c9) | FILTER q0.E EQUALS _ AS q1 RETURN (q0.A AS A, q0.E AS E) } task_count: 939 @@ -183,7 +183,7 @@ unnamed-2: insert_new_count: 118 insert_reused_count: 7 - query: EXPLAIN select a, e from ta where e in ('foo') - ref: in-predicate.yamsql:169 + ref: in-predicate.yamsql:204 explain: ISCAN(F1 <,>) | FLATMAP q0 -> { EXPLODE arrayDistinct(@c9) | FILTER q0.E EQUALS _ AS q1 RETURN (q0.A AS A, q0.E AS E) } task_count: 939 @@ -195,7 +195,7 @@ unnamed-2: insert_new_count: 118 insert_reused_count: 7 - query: EXPLAIN select a, e from ta where e in ('b' + 'a' + 'r', 'doe') - ref: in-predicate.yamsql:174 + ref: in-predicate.yamsql:209 explain: ISCAN(F1 <,>) | FLATMAP q0 -> { EXPLODE arrayDistinct(array(@c10 + @c12 + @c14, @c16)) | FILTER q0.E EQUALS _ AS q1 RETURN (q0.A AS A, q0.E AS E) } @@ -208,7 +208,7 @@ unnamed-2: insert_new_count: 118 insert_reused_count: 7 - query: EXPLAIN select a, b from ta where (a, b) in ((0L, 9L), (1L, 8L)) - ref: in-predicate.yamsql:179 + ref: in-predicate.yamsql:214 explain: ISCAN(F1 <,>) | FILTER (_.A AS A, _.B AS B) IN array((@c15 AS _0, @c17 AS _1), (@c21 AS _0, @c23 AS _1)) | MAP (_.A AS A, _.B AS B) task_count: 1122 @@ -220,7 +220,7 @@ unnamed-2: insert_new_count: 144 insert_reused_count: 5 - query: EXPLAIN select a, b from ta where f in ((90L, 9L), (81L, 18L)) - ref: in-predicate.yamsql:184 + ref: in-predicate.yamsql:219 explain: EXPLODE arrayDistinct(array((@c11 AS _0, @c13 AS _1), (@c17 AS _0, @c19 AS _1))) | FLATMAP q0 -> { ISCAN(F1 [EQUALS promote(q0._0 AS LONG), EQUALS promote(q0._1 AS LONG)]) AS q1 RETURN (q1.A AS A, q1.B AS B) } @@ -234,7 +234,7 @@ unnamed-2: insert_reused_count: 8 - query: EXPLAIN select a, b from ta where f in ((90L, 9L), (81L, 18L), (81L, 18L), (90L, 9L)) - ref: in-predicate.yamsql:189 + ref: in-predicate.yamsql:224 explain: EXPLODE arrayDistinct(array((@c11 AS _0, @c13 AS _1), (@c17 AS _0, @c19 AS _1), (@c17 AS _0, @c19 AS _1), (@c11 AS _0, @c13 AS _1))) | FLATMAP q0 -> { ISCAN(F1 [EQUALS promote(q0._0 AS LONG), EQUALS promote(q0._1 AS LONG)]) @@ -249,7 +249,7 @@ unnamed-2: insert_reused_count: 8 - query: EXPLAIN select e, a from ta where e not in ('foo' , 'bar', 'doe', 'arc', 'something') - ref: in-predicate.yamsql:199 + ref: in-predicate.yamsql:234 explain: ISCAN(F1 <,>) | FILTER NOT _.E IN @c10 | MAP (_.E AS E, _.A AS A) task_count: 349 task_total_time_ms: 24 @@ -260,7 +260,7 @@ unnamed-2: insert_new_count: 36 insert_reused_count: 4 - query: EXPLAIN select a, e from ta where 1 in (2, 3) - ref: in-predicate.yamsql:215 + ref: in-predicate.yamsql:250 explain: EXPLODE arrayDistinct(@c9) | FILTER @c7 EQUALS _ | FLATMAP q0 -> { ISCAN(F1 <,>) AS q1 RETURN (q1.A AS A, q1.E AS E) } task_count: 941 @@ -272,7 +272,7 @@ unnamed-2: insert_new_count: 118 insert_reused_count: 6 - query: EXPLAIN select a from ta where 1 in (1, 2, 3) - ref: in-predicate.yamsql:220 + ref: in-predicate.yamsql:255 explain: 'EXPLODE arrayDistinct(@c7) | FLATMAP q0 -> { COVERING(F1 <,> -> [A: KEY:[3], F: [SA: KEY:[0], SB: KEY:[1]]]) | FILTER @c5 EQUALS q0 | MAP (_.A AS A) AS q1 RETURN q1 }' @@ -286,7 +286,7 @@ unnamed-2: insert_reused_count: 9 - query: EXPLAIN select id from array_table where exists (select 1 from array_table.fruits f where f = 'apple') - ref: in-predicate.yamsql:226 + ref: in-predicate.yamsql:261 explain: 'COVERING(FRUITS [EQUALS @c17] -> [ID: KEY:[2]]) | MAP (_.ID AS ID)' task_count: 457 task_total_time_ms: 25 @@ -297,7 +297,7 @@ unnamed-2: insert_new_count: 44 insert_reused_count: 0 - query: EXPLAIN select id from array_table where 'apple' in fruits - ref: in-predicate.yamsql:234 + ref: in-predicate.yamsql:269 explain: SCAN([IS ARRAY_TABLE]) | FLATMAP q0 -> { EXPLODE arrayDistinct(q0.FRUITS) | FILTER @c5 EQUALS _ AS q1 RETURN (q0.ID AS ID) } task_count: 341 @@ -309,7 +309,7 @@ unnamed-2: insert_new_count: 32 insert_reused_count: 3 - query: EXPLAIN select id from array_table where 'banana' in fruits - ref: in-predicate.yamsql:240 + ref: in-predicate.yamsql:275 explain: SCAN([IS ARRAY_TABLE]) | FLATMAP q0 -> { EXPLODE arrayDistinct(q0.FRUITS) | FILTER @c5 EQUALS _ AS q1 RETURN (q0.ID AS ID) } task_count: 341 @@ -321,7 +321,7 @@ unnamed-2: insert_new_count: 32 insert_reused_count: 3 - query: EXPLAIN select id from array_table where 'pineapple' in fruits - ref: in-predicate.yamsql:246 + ref: in-predicate.yamsql:281 explain: SCAN([IS ARRAY_TABLE]) | FLATMAP q0 -> { EXPLODE arrayDistinct(q0.FRUITS) | FILTER @c5 EQUALS _ AS q1 RETURN (q0.ID AS ID) } task_count: 341 @@ -334,7 +334,7 @@ unnamed-2: insert_reused_count: 3 - query: EXPLAIN select id from array_table where exists (select 1 from array_table.numbers n where n = 10) - ref: in-predicate.yamsql:252 + ref: in-predicate.yamsql:287 explain: SCAN([IS ARRAY_TABLE]) | FLATMAP q0 -> { EXPLODE q0.NUMBERS | FILTER _ EQUALS promote(@c17 AS LONG) | MAP (@c8 AS _0) | DEFAULT NULL | FILTER _ NOT_NULL AS q1 RETURN (q0.ID AS ID) } @@ -347,7 +347,7 @@ unnamed-2: insert_new_count: 32 insert_reused_count: 0 - query: EXPLAIN select id from array_table where 10 in numbers - ref: in-predicate.yamsql:259 + ref: in-predicate.yamsql:294 explain: SCAN([IS ARRAY_TABLE]) | FLATMAP q0 -> { EXPLODE arrayDistinct(q0.NUMBERS) | FILTER promote(@c5 AS LONG) EQUALS _ AS q1 RETURN (q0.ID AS ID) } task_count: 341 @@ -359,7 +359,7 @@ unnamed-2: insert_new_count: 32 insert_reused_count: 3 - query: EXPLAIN select id from array_table where 15 in numbers - ref: in-predicate.yamsql:265 + ref: in-predicate.yamsql:300 explain: SCAN([IS ARRAY_TABLE]) | FLATMAP q0 -> { EXPLODE arrayDistinct(q0.NUMBERS) | FILTER promote(@c5 AS LONG) EQUALS _ AS q1 RETURN (q0.ID AS ID) } task_count: 341 @@ -371,7 +371,7 @@ unnamed-2: insert_new_count: 32 insert_reused_count: 3 - query: EXPLAIN select id from array_table where ('apple', 'red') in fruit_records - ref: in-predicate.yamsql:271 + ref: in-predicate.yamsql:306 explain: SCAN([IS ARRAY_TABLE]) | FILTER (@c6 AS _0, @c8 AS _1) IN _.FRUIT_RECORDS | MAP (_.ID AS ID) task_count: 341 @@ -383,7 +383,7 @@ unnamed-2: insert_new_count: 32 insert_reused_count: 3 - query: EXPLAIN select id from array_table where ('mango', 'orange') in fruit_records - ref: in-predicate.yamsql:277 + ref: in-predicate.yamsql:312 explain: SCAN([IS ARRAY_TABLE]) | FILTER (@c6 AS _0, @c8 AS _1) IN _.FRUIT_RECORDS | MAP (_.ID AS ID) task_count: 341 @@ -395,7 +395,7 @@ unnamed-2: insert_new_count: 32 insert_reused_count: 3 - query: EXPLAIN select id from array_table where ('apple', 'blue') in fruit_records - ref: in-predicate.yamsql:283 + ref: in-predicate.yamsql:318 explain: SCAN([IS ARRAY_TABLE]) | FILTER (@c6 AS _0, @c8 AS _1) IN _.FRUIT_RECORDS | MAP (_.ID AS ID) task_count: 341 @@ -407,7 +407,7 @@ unnamed-2: insert_new_count: 32 insert_reused_count: 3 - query: EXPLAIN select id from array_table where ('grape', 'green') in fruit_records - ref: in-predicate.yamsql:289 + ref: in-predicate.yamsql:324 explain: SCAN([IS ARRAY_TABLE]) | FILTER (@c6 AS _0, @c8 AS _1) IN _.FRUIT_RECORDS | MAP (_.ID AS ID) task_count: 341 @@ -419,7 +419,7 @@ unnamed-2: insert_new_count: 32 insert_reused_count: 3 - query: EXPLAIN select id from array_table where 'apple' not in fruits - ref: in-predicate.yamsql:295 + ref: in-predicate.yamsql:330 explain: SCAN([IS ARRAY_TABLE]) | FILTER NOT @c5 IN _.FRUITS | MAP (_.ID AS ID) task_count: 208 task_total_time_ms: 11 @@ -430,7 +430,7 @@ unnamed-2: insert_new_count: 18 insert_reused_count: 2 - query: EXPLAIN select id from array_table where ('apple', 'red') not in fruit_records - ref: in-predicate.yamsql:301 + ref: in-predicate.yamsql:336 explain: SCAN([IS ARRAY_TABLE]) | FILTER NOT (@c6 AS _0, @c8 AS _1) IN _.FRUIT_RECORDS | MAP (_.ID AS ID) task_count: 208 @@ -445,7 +445,7 @@ unnamed-2: AS parent_id FROM t_parent, t_child WHERE get_key(t_child.t_link) = t_parent.id AND 'tag_a' IN t_parent.lst AND t_parent.id = 'p1' AND t_child.num IN (1, 2, 3) - ref: in-predicate.yamsql:321 + ref: in-predicate.yamsql:356 explain: SCAN([IS T_PARENT, EQUALS promote(@c44 AS STRING)]) | FLATMAP q0 -> { EXPLODE arrayDistinct(q0.LST) | FLATMAP q1 -> { [IN arrayDistinct(promote(@c50 AS ARRAY(LONG)))] | INJOIN q2 -> { SCAN([IS T_CHILD]) | FILTER _.T_LINK.NAME @@ -461,7 +461,7 @@ unnamed-2: insert_reused_count: 287 - query: EXPLAIN SELECT t.id, n FROM array_table AS t, t.numbers AS n WHERE n IN (10, 20) - ref: in-predicate.yamsql:332 + ref: in-predicate.yamsql:367 explain: EXPLODE arrayDistinct(promote(@c19 AS ARRAY(LONG))) | FLATMAP q0 -> { SCAN([IS ARRAY_TABLE]) | FLATMAP q1 -> { EXPLODE q1.NUMBERS | FILTER _ EQUALS q0 AS q2 RETURN (q1.ID AS ID, q2 AS N) } AS q3 RETURN q3 } @@ -474,7 +474,7 @@ unnamed-2: insert_new_count: 115 insert_reused_count: 6 - query: EXPLAIN SELECT t.id, n, idx FROM array_table AS t, t.numbers AS n AT idx - ref: in-predicate.yamsql:338 + ref: in-predicate.yamsql:373 explain: SCAN([IS ARRAY_TABLE]) | FLATMAP q0 -> { EXPLODE q0.NUMBERS WITH ORDINALITY AS q1 RETURN (q0.ID AS ID, q1._0 AS N, q1._1 AS IDX) } task_count: 162 @@ -487,37 +487,35 @@ unnamed-2: insert_reused_count: 0 - query: EXPLAIN SELECT t.id, n, idx FROM array_table AS t, t.numbers AS n AT idx WHERE n IN (10, 20) - ref: in-predicate.yamsql:350 - explain: EXPLODE arrayDistinct(promote(@c23 AS ARRAY(LONG))) | FLATMAP q0 -> { - SCAN([IS ARRAY_TABLE]) | FLATMAP q1 -> { EXPLODE q1.NUMBERS WITH ORDINALITY - | FILTER _._0 EQUALS q0 AS q2 RETURN (q1.ID AS ID, q2._0 AS N, q2._1 AS IDX) - } AS q3 RETURN q3 } - task_count: 1060 - task_total_time_ms: 82 - transform_count: 423 - transform_time_ms: 18 - transform_yield_count: 54 - insert_time_ms: 3 - insert_new_count: 115 + ref: in-predicate.yamsql:385 + explain: SCAN([IS ARRAY_TABLE]) | FLATMAP q0 -> { EXPLODE q0.NUMBERS WITH ORDINALITY + | FILTER _._0 IN promote(@c23 AS ARRAY(LONG)) AS q1 RETURN (q0.ID AS ID, q1._0 + AS N, q1._1 AS IDX) } + task_count: 1049 + task_total_time_ms: 90 + transform_count: 422 + transform_time_ms: 26 + transform_yield_count: 53 + insert_time_ms: 4 + insert_new_count: 114 insert_reused_count: 6 - query: EXPLAIN SELECT t.id, n, idx FROM array_table AS t, t.numbers AS n AT idx WHERE idx IN (1, 3) - ref: in-predicate.yamsql:359 - explain: EXPLODE arrayDistinct(@c23) | FLATMAP q0 -> { SCAN([IS ARRAY_TABLE]) - | FLATMAP q1 -> { EXPLODE q1.NUMBERS WITH ORDINALITY | FILTER _._1 EQUALS - q0 AS q2 RETURN (q1.ID AS ID, q2._0 AS N, q2._1 AS IDX) } AS q3 RETURN q3 + ref: in-predicate.yamsql:394 + explain: SCAN([IS ARRAY_TABLE]) | FLATMAP q0 -> { EXPLODE q0.NUMBERS WITH ORDINALITY + | FILTER _._1 IN @c23 AS q1 RETURN (q0.ID AS ID, q1._0 AS N, q1._1 AS IDX) } - task_count: 1060 - task_total_time_ms: 67 - transform_count: 423 - transform_time_ms: 14 - transform_yield_count: 54 - insert_time_ms: 2 - insert_new_count: 115 + task_count: 1049 + task_total_time_ms: 94 + transform_count: 422 + transform_time_ms: 24 + transform_yield_count: 53 + insert_time_ms: 5 + insert_new_count: 114 insert_reused_count: 6 - query: EXPLAIN SELECT t.id, n, idx FROM array_table AS t, t.numbers AS n AT idx WHERE t.id IN (2, 4) - ref: in-predicate.yamsql:370 + ref: in-predicate.yamsql:405 explain: '[IN arrayDistinct(promote(@c25 AS ARRAY(LONG)))] | INJOIN q0 -> { SCAN([IS ARRAY_TABLE, EQUALS q0]) } | FLATMAP q1 -> { EXPLODE q1.NUMBERS WITH ORDINALITY AS q2 RETURN (q1.ID AS ID, q2._0 AS N, q2._1 AS IDX) }' @@ -531,15 +529,15 @@ unnamed-2: insert_reused_count: 9 - query: EXPLAIN SELECT t.id, f, idx FROM array_table AS t, t.fruits AS f AT idx WHERE f IN ('apple', 'mango') - ref: in-predicate.yamsql:378 - explain: EXPLODE arrayDistinct(@c23) | FLATMAP q0 -> { SCAN([IS ARRAY_TABLE]) - | FLATMAP q1 -> { EXPLODE q1.FRUITS WITH ORDINALITY | FILTER _._0 EQUALS q0 - AS q2 RETURN (q1.ID AS ID, q2._0 AS F, q2._1 AS IDX) } AS q3 RETURN q3 } - task_count: 1060 - task_total_time_ms: 78 - transform_count: 423 - transform_time_ms: 16 - transform_yield_count: 54 - insert_time_ms: 2 - insert_new_count: 115 + ref: in-predicate.yamsql:413 + explain: SCAN([IS ARRAY_TABLE]) | FLATMAP q0 -> { EXPLODE q0.FRUITS WITH ORDINALITY + | FILTER _._0 IN @c23 AS q1 RETURN (q0.ID AS ID, q1._0 AS F, q1._1 AS IDX) + } + task_count: 1049 + task_total_time_ms: 99 + transform_count: 422 + transform_time_ms: 25 + transform_yield_count: 53 + insert_time_ms: 5 + insert_new_count: 114 insert_reused_count: 6 diff --git a/yaml-tests/src/test/resources/in-predicate.yamsql b/yaml-tests/src/test/resources/in-predicate.yamsql index da72a7fe4a8..b24b822af1b 100644 --- a/yaml-tests/src/test/resources/in-predicate.yamsql +++ b/yaml-tests/src/test/resources/in-predicate.yamsql @@ -382,7 +382,7 @@ test_block: # The ordinal (IDX) reflects the position in the original array, not the rank among filtered rows. - query: SELECT t.id, n, idx FROM array_table AS t, t.numbers AS n AT idx WHERE n IN (10, 20) - supported_version: 4.12.5.0 - - explain: "EXPLODE arrayDistinct(promote(@c23 AS ARRAY(LONG))) | FLATMAP q0 -> { SCAN([IS ARRAY_TABLE]) | FLATMAP q1 -> { EXPLODE q1.NUMBERS WITH ORDINALITY | FILTER _._0 EQUALS q0 AS q2 RETURN (q1.ID AS ID, q2._0 AS N, q2._1 AS IDX) } AS q3 RETURN q3 }" + - explain: "SCAN([IS ARRAY_TABLE]) | FLATMAP q0 -> { EXPLODE q0.NUMBERS WITH ORDINALITY | FILTER _._0 IN promote(@c23 AS ARRAY(LONG)) AS q1 RETURN (q0.ID AS ID, q1._0 AS N, q1._1 AS IDX) }" - unorderedResult: [ {ID: 1, N: 10, IDX: 1}, {ID: 1, N: 20, IDX: 2}, {ID: 3, N: 10, IDX: 1}, @@ -391,7 +391,7 @@ test_block: # PartiQL join AT on bigint array with IN predicate on the ordinal. - query: SELECT t.id, n, idx FROM array_table AS t, t.numbers AS n AT idx WHERE idx IN (1, 3) - supported_version: 4.12.5.0 - - explain: "EXPLODE arrayDistinct(@c23) | FLATMAP q0 -> { SCAN([IS ARRAY_TABLE]) | FLATMAP q1 -> { EXPLODE q1.NUMBERS WITH ORDINALITY | FILTER _._1 EQUALS q0 AS q2 RETURN (q1.ID AS ID, q2._0 AS N, q2._1 AS IDX) } AS q3 RETURN q3 }" + - explain: "SCAN([IS ARRAY_TABLE]) | FLATMAP q0 -> { EXPLODE q0.NUMBERS WITH ORDINALITY | FILTER _._1 IN @c23 AS q1 RETURN (q0.ID AS ID, q1._0 AS N, q1._1 AS IDX) }" - unorderedResult: [ {ID: 1, N: 10, IDX: 1}, {ID: 1, N: 30, IDX: 3}, {ID: 2, N: 40, IDX: 1}, @@ -410,7 +410,7 @@ test_block: # PartiQL join AT on string array with IN predicate on the unnested value. - query: SELECT t.id, f, idx FROM array_table AS t, t.fruits AS f AT idx WHERE f IN ('apple', 'mango') - supported_version: 4.12.5.0 - - explain: "EXPLODE arrayDistinct(@c23) | FLATMAP q0 -> { SCAN([IS ARRAY_TABLE]) | FLATMAP q1 -> { EXPLODE q1.FRUITS WITH ORDINALITY | FILTER _._0 EQUALS q0 AS q2 RETURN (q1.ID AS ID, q2._0 AS F, q2._1 AS IDX) } AS q3 RETURN q3 }" + - explain: "SCAN([IS ARRAY_TABLE]) | FLATMAP q0 -> { EXPLODE q0.FRUITS WITH ORDINALITY | FILTER _._0 IN @c23 AS q1 RETURN (q0.ID AS ID, q1._0 AS F, q1._1 AS IDX) }" - unorderedResult: [ {ID: 1, F: 'apple', IDX: 1}, {ID: 3, F: 'apple', IDX: 1}, {ID: 3, F: 'mango', IDX: 3}] diff --git a/yaml-tests/src/test/resources/subquery-tests.metrics.binpb b/yaml-tests/src/test/resources/subquery-tests.metrics.binpb index 5668031056b..b0207093987 100644 --- a/yaml-tests/src/test/resources/subquery-tests.metrics.binpb +++ b/yaml-tests/src/test/resources/subquery-tests.metrics.binpb @@ -221,7 +221,7 @@ p }  subquery-testsEXPLAIN select sq.idr, sq.z from (select * from r where idr = 1) sq, (select f from sq.nr where f > 10) sq2 where sq.z = 10 AND sq2.f is not null -뺛) (508R@ISCAN(IR [EQUALS promote(@c17 AS INT), EQUALS promote(@c31 AS INT)]) | FLATMAP q0 -> { EXPLODE q0.NR | FILTER _.F GREATER_THAN promote(@c31 AS INT) AND _.F NOT_NULL AS q1 RETURN (q0.IDR AS IDR, q0.Z AS Z) }digraph G { +B (50ɍ8S@ISCAN(IR [EQUALS promote(@c17 AS INT), EQUALS promote(@c31 AS INT)]) | FLATMAP q0 -> { EXPLODE q0.NR | FILTER _.F GREATER_THAN promote(@c31 AS INT) AND _.F NOT_NULL AS q1 RETURN (q0.IDR AS IDR, q0.Z AS Z) }digraph G { fontname=courier; rankdir=BT; splines=line; diff --git a/yaml-tests/src/test/resources/subquery-tests.metrics.yaml b/yaml-tests/src/test/resources/subquery-tests.metrics.yaml index 26b075d5388..06fcf17543e 100644 --- a/yaml-tests/src/test/resources/subquery-tests.metrics.yaml +++ b/yaml-tests/src/test/resources/subquery-tests.metrics.yaml @@ -92,10 +92,10 @@ subquery-tests: | FLATMAP q0 -> { EXPLODE q0.NR | FILTER _.F GREATER_THAN promote(@c31 AS INT) AND _.F NOT_NULL AS q1 RETURN (q0.IDR AS IDR, q0.Z AS Z) } task_count: 598 - task_total_time_ms: 86 + task_total_time_ms: 138 transform_count: 173 - transform_time_ms: 33 + transform_time_ms: 60 transform_yield_count: 53 - insert_time_ms: 3 - insert_new_count: 82 + insert_time_ms: 5 + insert_new_count: 83 insert_reused_count: 2 diff --git a/yaml-tests/src/test/resources/unnested-record-type-indexes.metrics.binpb b/yaml-tests/src/test/resources/unnested-record-type-indexes.metrics.binpb new file mode 100644 index 00000000000..25aed9d4013 Binary files /dev/null and b/yaml-tests/src/test/resources/unnested-record-type-indexes.metrics.binpb differ diff --git a/yaml-tests/src/test/resources/unnested-record-type-indexes.metrics.yaml b/yaml-tests/src/test/resources/unnested-record-type-indexes.metrics.yaml new file mode 100644 index 00000000000..85d58877030 --- /dev/null +++ b/yaml-tests/src/test/resources/unnested-record-type-indexes.metrics.yaml @@ -0,0 +1,109 @@ +unnamed-3: +- query: EXPLAIN SELECT SQ."reviewer", "restaurant"."region", SQ."rating" FROM "restaurant", + (SELECT "reviewer", "rating" FROM "restaurant"."reviews") AS SQ ORDER BY SQ."reviewer", + "restaurant"."region", SQ."rating" + ref: unnested-record-type-indexes.yamsql:55 + explain: 'COVERING(mv_split <,> -> [__positions: [unnesting_0: KEY:[5, 0]], parent: + [region: KEY:[1]], unnesting_0: [rating: KEY:[2], reviewer: KEY:[0]]]) | MAP + (_.unnesting_0.reviewer AS reviewer, _.parent.region AS region, _.unnesting_0.rating + AS rating)' + task_count: 261 + task_total_time_ms: 181 + transform_count: 78 + transform_time_ms: 111 + transform_yield_count: 24 + insert_time_ms: 13 + insert_new_count: 26 + insert_reused_count: 0 +- query: EXPLAIN SELECT SQ."reviewer", "restaurant"."region", SQ."rating" FROM "restaurant", + (SELECT "reviewer", "rating" FROM "restaurant"."reviews") AS SQ + ref: unnested-record-type-indexes.yamsql:66 + explain: 'COVERING(mv_split <,> -> [__positions: [unnesting_0: KEY:[5, 0]], parent: + [region: KEY:[1]], unnesting_0: [rating: KEY:[2], reviewer: KEY:[0]]]) | MAP + (_.unnesting_0.reviewer AS reviewer, _.parent.region AS region, _.unnesting_0.rating + AS rating)' + task_count: 290 + task_total_time_ms: 190 + transform_count: 85 + transform_time_ms: 115 + transform_yield_count: 27 + insert_time_ms: 13 + insert_new_count: 31 + insert_reused_count: 0 +- query: EXPLAIN SELECT SQ."reviewer" FROM "restaurant", (SELECT "reviewer", "rating" + FROM "restaurant"."reviews") AS SQ ORDER BY SQ."reviewer" + ref: unnested-record-type-indexes.yamsql:78 + explain: 'COVERING(mv_split <,> -> [__positions: [unnesting_0: KEY:[5, 0]], parent: + [region: KEY:[1]], unnesting_0: [rating: KEY:[2], reviewer: KEY:[0]]]) | MAP + (_.unnesting_0.reviewer AS reviewer)' + task_count: 268 + task_total_time_ms: 177 + transform_count: 82 + transform_time_ms: 113 + transform_yield_count: 25 + insert_time_ms: 13 + insert_new_count: 27 + insert_reused_count: 0 +- query: EXPLAIN SELECT "r"."reviewer", "restaurant"."region", "r"."rating" FROM + "restaurant", "restaurant"."reviews" AS "r" ORDER BY "r"."reviewer", "restaurant"."region", + "r"."rating" + ref: unnested-record-type-indexes.yamsql:91 + explain: 'COVERING(mv_split <,> -> [__positions: [unnesting_0: KEY:[5, 0]], parent: + [region: KEY:[1]], unnesting_0: [rating: KEY:[2], reviewer: KEY:[0]]]) | MAP + (_.unnesting_0.reviewer AS reviewer, _.parent.region AS region, _.unnesting_0.rating + AS rating)' + task_count: 236 + task_total_time_ms: 43 + transform_count: 73 + transform_time_ms: 22 + transform_yield_count: 22 + insert_time_ms: 0 + insert_new_count: 21 + insert_reused_count: 0 +- query: EXPLAIN SELECT SQ."reviewer", "restaurant"."region", SQ."rating" FROM "restaurant", + (SELECT "reviewer", "rating" FROM "restaurant"."reviews") AS SQ WHERE SQ."reviewer" + = 'Alice' + ref: unnested-record-type-indexes.yamsql:104 + explain: 'COVERING(mv_split [EQUALS promote(@c32 AS STRING)] -> [__positions: + [unnesting_0: KEY:[5, 0]], parent: [region: KEY:[1]], unnesting_0: [rating: + KEY:[2], reviewer: KEY:[0]]]) | MAP (_.unnesting_0.reviewer AS reviewer, _.parent.region + AS region, _.unnesting_0.rating AS rating)' + task_count: 397 + task_total_time_ms: 189 + transform_count: 129 + transform_time_ms: 110 + transform_yield_count: 35 + insert_time_ms: 7 + insert_new_count: 47 + insert_reused_count: 1 +- query: EXPLAIN SELECT SQ."reviewer", "restaurant"."region", SQ."rating" FROM "restaurant", + (SELECT "reviewer", "rating" FROM "restaurant"."reviews") AS SQ WHERE SQ."reviewer" + = 'Alice' AND "restaurant"."region" = 'north' + ref: unnested-record-type-indexes.yamsql:113 + explain: 'COVERING(mv_split [EQUALS promote(@c32 AS STRING), EQUALS promote(@c38 + AS STRING)] -> [__positions: [unnesting_0: KEY:[5, 0]], parent: [region: KEY:[1]], + unnesting_0: [rating: KEY:[2], reviewer: KEY:[0]]]) | MAP (_.unnesting_0.reviewer + AS reviewer, _.parent.region AS region, _.unnesting_0.rating AS rating)' + task_count: 467 + task_total_time_ms: 193 + transform_count: 149 + transform_time_ms: 112 + transform_yield_count: 37 + insert_time_ms: 8 + insert_new_count: 53 + insert_reused_count: 3 +- query: EXPLAIN SELECT "r"."reviewer", "restaurant"."region", "r"."rating" FROM + "restaurant", "restaurant"."reviews" AS "r" WHERE "r"."reviewer" = 'Alice' + ref: unnested-record-type-indexes.yamsql:122 + explain: 'COVERING(mv_split [EQUALS promote(@c25 AS STRING)] -> [__positions: + [unnesting_0: KEY:[5, 0]], parent: [region: KEY:[1]], unnesting_0: [rating: + KEY:[2], reviewer: KEY:[0]]]) | MAP (_.unnesting_0.reviewer AS reviewer, _.parent.region + AS region, _.unnesting_0.rating AS rating)' + task_count: 362 + task_total_time_ms: 190 + transform_count: 123 + transform_time_ms: 115 + transform_yield_count: 32 + insert_time_ms: 5 + insert_new_count: 39 + insert_reused_count: 1 diff --git a/yaml-tests/src/test/resources/unnested-record-type-indexes.yamsql b/yaml-tests/src/test/resources/unnested-record-type-indexes.yamsql new file mode 100644 index 00000000000..5fd3b868eac --- /dev/null +++ b/yaml-tests/src/test/resources/unnested-record-type-indexes.yamsql @@ -0,0 +1,126 @@ +# +# unnested-record-type-indexes.yamsql +# +# This source file is part of the FoundationDB open source project +# +# Copyright 2021-2026 Apple Inc. and the FoundationDB project authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +options: + supported_version: !current_version +--- +# End-to-end tests for an index over a single unnesting, and for predicates on top of it. +# +# An index over an unnesting only needs a synthetic record type when two or more columns reached through the same +# unnesting are not adjacent in the index key. `mv_split` is such an index: `reviewer` and `rating` both come from one +# review, but `region` (a column of the parent row) sits between them. Evaluating that key as a plain fan-out would +# produce the cross product of the reviews with themselves, so the index is instead defined over a synthetic type +# whose records pair the parent row with a single review. +schema_template: + CREATE TYPE AS STRUCT "review" ("reviewer" STRING, "rating" BIGINT) + + CREATE TABLE "restaurant" ("rest_no" BIGINT, "region" STRING, "reviews" "review" ARRAY, PRIMARY KEY ("rest_no")) + CREATE INDEX "mv_split" AS + SELECT SQ."reviewer", "restaurant"."region", SQ."rating" + FROM "restaurant", (SELECT "reviewer", "rating" FROM "restaurant"."reviews") AS SQ + ORDER BY SQ."reviewer", "restaurant"."region", SQ."rating" +--- +setup: + steps: + - query: INSERT INTO "restaurant" + VALUES (1, 'north', [('Alice', 5), ('Bob', 3)]), + (2, 'south', [('Carol', 4)]), + (3, 'north', [('Dave', 2), ('Alice', 4)]) +--- +test_block: + tests: + - + # The query the index was defined for. The index supplies the requested ordering, so no sort is needed, and + # every selected column is in the index key, so the scan is covering. + - query: SELECT SQ."reviewer", "restaurant"."region", SQ."rating" + FROM "restaurant", (SELECT "reviewer", "rating" FROM "restaurant"."reviews") AS SQ + ORDER BY SQ."reviewer", "restaurant"."region", SQ."rating" + - explain: "COVERING(mv_split <,> -> [__positions: [unnesting_0: KEY:[5, 0]], parent: [region: KEY:[1]], unnesting_0: [rating: KEY:[2], reviewer: KEY:[0]]]) | MAP (_.unnesting_0.reviewer AS reviewer, _.parent.region AS region, _.unnesting_0.rating AS rating)" + - result: [ + {reviewer: 'Alice', region: 'north', rating: 4}, + {reviewer: 'Alice', region: 'north', rating: 5}, + {reviewer: 'Bob', region: 'north', rating: 3}, + {reviewer: 'Carol', region: 'south', rating: 4}, + {reviewer: 'Dave', region: 'north', rating: 2}] + - + # The same unnesting without an ORDER BY. The index still answers it; the result order is not guaranteed. + - query: SELECT SQ."reviewer", "restaurant"."region", SQ."rating" + FROM "restaurant", (SELECT "reviewer", "rating" FROM "restaurant"."reviews") AS SQ + - explain: "COVERING(mv_split <,> -> [__positions: [unnesting_0: KEY:[5, 0]], parent: [region: KEY:[1]], unnesting_0: [rating: KEY:[2], reviewer: KEY:[0]]]) | MAP (_.unnesting_0.reviewer AS reviewer, _.parent.region AS region, _.unnesting_0.rating AS rating)" + - unorderedResult: [ + {reviewer: 'Alice', region: 'north', rating: 5}, + {reviewer: 'Bob', region: 'north', rating: 3}, + {reviewer: 'Carol', region: 'south', rating: 4}, + {reviewer: 'Dave', region: 'north', rating: 2}, + {reviewer: 'Alice', region: 'north', rating: 4}] + - + # Selecting a prefix of the key rather than all of it. + - query: SELECT SQ."reviewer" + FROM "restaurant", (SELECT "reviewer", "rating" FROM "restaurant"."reviews") AS SQ + ORDER BY SQ."reviewer" + - explain: "COVERING(mv_split <,> -> [__positions: [unnesting_0: KEY:[5, 0]], parent: [region: KEY:[1]], unnesting_0: [rating: KEY:[2], reviewer: KEY:[0]]]) | MAP (_.unnesting_0.reviewer AS reviewer)" + - result: [ + {reviewer: 'Alice'}, + {reviewer: 'Alice'}, + {reviewer: 'Bob'}, + {reviewer: 'Carol'}, + {reviewer: 'Dave'}] + - + # PartiQL-style unnesting binds the array element directly rather than packing it into a subquery tuple. The + # index is defined with the subquery spelling, but both describe the same unnesting. + - query: SELECT "r"."reviewer", "restaurant"."region", "r"."rating" + FROM "restaurant", "restaurant"."reviews" AS "r" + ORDER BY "r"."reviewer", "restaurant"."region", "r"."rating" + - explain: "COVERING(mv_split <,> -> [__positions: [unnesting_0: KEY:[5, 0]], parent: [region: KEY:[1]], unnesting_0: [rating: KEY:[2], reviewer: KEY:[0]]]) | MAP (_.unnesting_0.reviewer AS reviewer, _.parent.region AS region, _.unnesting_0.rating AS rating)" + - result: [ + {reviewer: 'Alice', region: 'north', rating: 4}, + {reviewer: 'Alice', region: 'north', rating: 5}, + {reviewer: 'Bob', region: 'north', rating: 3}, + {reviewer: 'Carol', region: 'south', rating: 4}, + {reviewer: 'Dave', region: 'north', rating: 2}] + - + # An equality on the leading key column. The index's placeholders live on the same select as the constituents' + # quantifiers, so a predicate over the unnesting can be matched against them and bind the scan. + - query: SELECT SQ."reviewer", "restaurant"."region", SQ."rating" + FROM "restaurant", (SELECT "reviewer", "rating" FROM "restaurant"."reviews") AS SQ + WHERE SQ."reviewer" = 'Alice' + - explain: "COVERING(mv_split [EQUALS promote(@c32 AS STRING)] -> [__positions: [unnesting_0: KEY:[5, 0]], parent: [region: KEY:[1]], unnesting_0: [rating: KEY:[2], reviewer: KEY:[0]]]) | MAP (_.unnesting_0.reviewer AS reviewer, _.parent.region AS region, _.unnesting_0.rating AS rating)" + - result: [ + {reviewer: 'Alice', region: 'north', rating: 4}, + {reviewer: 'Alice', region: 'north', rating: 5}] + - + # A full key prefix, spanning the unnesting and the parent row, binds both key columns. + - query: SELECT SQ."reviewer", "restaurant"."region", SQ."rating" + FROM "restaurant", (SELECT "reviewer", "rating" FROM "restaurant"."reviews") AS SQ + WHERE SQ."reviewer" = 'Alice' AND "restaurant"."region" = 'north' + - explain: "COVERING(mv_split [EQUALS promote(@c32 AS STRING), EQUALS promote(@c38 AS STRING)] -> [__positions: [unnesting_0: KEY:[5, 0]], parent: [region: KEY:[1]], unnesting_0: [rating: KEY:[2], reviewer: KEY:[0]]]) | MAP (_.unnesting_0.reviewer AS reviewer, _.parent.region AS region, _.unnesting_0.rating AS rating)" + - result: [ + {reviewer: 'Alice', region: 'north', rating: 4}, + {reviewer: 'Alice', region: 'north', rating: 5}] + - + # An equality on the unnesting reached through the PartiQL spelling of the same unnesting. + - query: SELECT "r"."reviewer", "restaurant"."region", "r"."rating" + FROM "restaurant", "restaurant"."reviews" AS "r" + WHERE "r"."reviewer" = 'Alice' + - explain: "COVERING(mv_split [EQUALS promote(@c25 AS STRING)] -> [__positions: [unnesting_0: KEY:[5, 0]], parent: [region: KEY:[1]], unnesting_0: [rating: KEY:[2], reviewer: KEY:[0]]]) | MAP (_.unnesting_0.reviewer AS reviewer, _.parent.region AS region, _.unnesting_0.rating AS rating)" + - result: [ + {reviewer: 'Alice', region: 'north', rating: 4}, + {reviewer: 'Alice', region: 'north', rating: 5}] +... diff --git a/yaml-tests/src/test/resources/valid-identifiers.metrics.binpb b/yaml-tests/src/test/resources/valid-identifiers.metrics.binpb index 2e82b0f7dfa..fa927077e2a 100644 --- a/yaml-tests/src/test/resources/valid-identifiers.metrics.binpb +++ b/yaml-tests/src/test/resources/valid-identifiers.metrics.binpb @@ -22,18 +22,18 @@ 3 [ label=<
Index
foo.table$nested.repeated.idx.field.1.2.1
> color="black" shape="plain" style="filled" fillcolor="lightblue" fontname="courier" fontsize="8" tooltip="RELATION(LONG AS ID, ARRAY(ARRAY(LONG AS level2$array.field.1, LONG AS level2$field.2) AS level1$field.1, LONG AS level2$field.1, LONG AS level2$field.2 AS level1$field.2, LONG AS level1$field.3) AS level0.field1)" ]; 3 -> 2 [ color="gray20" style="solid" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; 2 -> 1 [ label=< q102> label="q102" color="gray20" style="bold" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; -} +}  - all-testsEXPLAIN select t.id from "foo.table$nested.repeated" as t where exists (select * from t."level0.field1" as b, b."level1$field.1" where "level2$array.field.1" = 10) -0 (E0i8^@vCOVERING(foo.table$nested.repeated.idx.field.1.1.1 [EQUALS promote(@c26 AS LONG)] -> [ID: KEY:[2]]) | MAP (_.ID AS ID) digraph G { + all-testsEXPLAIN select t.id from "foo.table$nested.repeated" as t where exists (select * from t."level0.field1" as b, b."level1$field.1" where "level2$array.field.1" = 10) +1 ΀(<08I@vCOVERING(foo.table$nested.repeated.idx.field.1.1.1 [EQUALS promote(@c26 AS LONG)] -> [ID: KEY:[2]]) | MAP (_.ID AS ID) digraph G { fontname=courier; rankdir=BT; splines=line; - 1 [ label=<
Value Computation
MAP (q117.ID AS ID)
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(LONG AS ID)" ]; + 1 [ label=<
Value Computation
MAP (q116.ID AS ID)
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(LONG AS ID)" ]; 2 [ label=<
Covering Index Scan
comparisons: [EQUALS promote(@c26 AS LONG)]
> color="black" shape="plain" style="solid" fillcolor="black" fontname="courier" fontsize="8" tooltip="RELATION(LONG AS ID, ARRAY(ARRAY(LONG AS level2$array.field.1, LONG AS level2$field.2) AS level1$field.1, LONG AS level2$field.1, LONG AS level2$field.2 AS level1$field.2, LONG AS level1$field.3) AS level0.field1)" ]; 3 [ label=<
Index
foo.table$nested.repeated.idx.field.1.1.1
> color="black" shape="plain" style="filled" fillcolor="lightblue" fontname="courier" fontsize="8" tooltip="RELATION(LONG AS ID, ARRAY(ARRAY(LONG AS level2$array.field.1, LONG AS level2$field.2) AS level1$field.1, LONG AS level2$field.1, LONG AS level2$field.2 AS level1$field.2, LONG AS level1$field.3) AS level0.field1)" ]; 3 -> 2 [ color="gray20" style="solid" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; - 2 -> 1 [ label=< q117> label="q117" color="gray20" style="bold" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; + 2 -> 1 [ label=< q116> label="q116" color="gray20" style="bold" fontname="courier" fontsize="8" arrowhead="normal" arrowtail="none" dir="both" ]; } r all-testseEXPLAIN select "level0.field1"."level1$field.1"."level2$field.1" from "foo.table$nested" where id = 1 diff --git a/yaml-tests/src/test/resources/valid-identifiers.metrics.yaml b/yaml-tests/src/test/resources/valid-identifiers.metrics.yaml index a53a4601a85..c3f050867ce 100644 --- a/yaml-tests/src/test/resources/valid-identifiers.metrics.yaml +++ b/yaml-tests/src/test/resources/valid-identifiers.metrics.yaml @@ -32,18 +32,18 @@ all-tests: ref: valid-identifiers.yamsql:166 explain: 'COVERING(foo.table$nested.repeated.idx.field.1.1.1 [EQUALS promote(@c26 AS LONG)] -> [ID: KEY:[2]]) | MAP (_.ID AS ID)' - task_count: 899 - task_total_time_ms: 101 - transform_count: 320 - transform_time_ms: 38 - transform_yield_count: 69 - insert_time_ms: 1 - insert_new_count: 94 + task_count: 724 + task_total_time_ms: 103 + transform_count: 263 + transform_time_ms: 56 + transform_yield_count: 60 + insert_time_ms: 2 + insert_new_count: 73 insert_reused_count: 3 - query: EXPLAIN select "level0.field1"."level1$field.1"."level2$field.1" from "foo.table$nested" where id = 1 ref: valid-identifiers.yamsql:171 - explain: SCAN([IS foo.table$nested, EQUALS promote(@c11 AS LONG)]) | MAP (_.level0.field1.level1$field.1.level2$field.1 + explain: SCAN([IS foo__2table__1nested, EQUALS promote(@c11 AS LONG)]) | MAP (_.level0.field1.level1$field.1.level2$field.1 AS level2$field.1) task_count: 448 task_total_time_ms: 34 @@ -56,7 +56,7 @@ all-tests: - query: EXPLAIN select "foo.table$repeated".id from "foo.table$repeated" where 'foo' IN "foo.table$repeated"."field.1$array" ref: valid-identifiers.yamsql:178 - explain: SCAN([IS foo.table$repeated]) | FLATMAP q0 -> { EXPLODE arrayDistinct(q0.field.1$array) + explain: SCAN([IS foo__2table__1repeated]) | FLATMAP q0 -> { EXPLODE arrayDistinct(q0.field.1$array) | FILTER @c7 EQUALS _ AS q1 RETURN (q0.ID AS ID) } task_count: 345 task_total_time_ms: 17 @@ -82,7 +82,7 @@ all-tests: - query: EXPLAIN select t."field.1$array" from "foo.table$repeated" as t where 3 IN t."field.0$array" ref: valid-identifiers.yamsql:190 - explain: SCAN([IS foo.table$repeated]) | FLATMAP q0 -> { EXPLODE arrayDistinct(q0.field.0$array) + explain: SCAN([IS foo__2table__1repeated]) | FLATMAP q0 -> { EXPLODE arrayDistinct(q0.field.0$array) | FILTER promote(@c9 AS LONG) EQUALS _ AS q1 RETURN (q0.field.1$array AS field.1$array) } task_count: 345 @@ -155,7 +155,7 @@ all-tests: - query: EXPLAIN select * from "foo.tableA", "foo.tableB" where "foo.tableA"."foo.tableA.A1" = "foo.tableB"."foo.tableB.B1"; ref: valid-identifiers.yamsql:286 - explain: SCAN([IS foo.tableB]) | FLATMAP q0 -> { ISCAN(foo.tableA.idx [EQUALS + explain: SCAN([IS foo__2tableB]) | FLATMAP q0 -> { ISCAN(foo.tableA.idx [EQUALS q0.foo.tableB.B1]) AS q1 RETURN (q1.foo.tableA.A1 AS foo.tableA.A1, q1.foo.tableA.A2 AS foo.tableA.A2, q1.foo.tableA.A3 AS foo.tableA.A3, q0.foo.tableB.B1 AS foo.tableB.B1, q0.foo.tableB.B2 AS foo.tableB.B2, q0.foo.tableB.B3 AS foo.tableB.B3) } @@ -246,7 +246,7 @@ all-tests: - query: EXPLAIN select "βήτα__f"("f__e"."foo.tableE.E3") AS "___h.1" from "foo.tableE" as "f__e" where "alpha__f"("f__e"."foo.tableE.E3") = 5 ref: valid-identifiers.yamsql:346 - explain: SCAN([IS foo.tableE]) | FILTER _.foo.tableE.E3.S1 EQUALS promote(@c21 + explain: SCAN([IS foo__2tableE]) | FILTER _.foo.tableE.E3.S1 EQUALS promote(@c21 AS LONG) | MAP (_.foo.tableE.E3.S2 AS ___h.1) task_count: 204 task_total_time_ms: 11 @@ -259,7 +259,7 @@ all-tests: - query: EXPLAIN select "alpha__f"("f__e"."foo.tableE.E3") AS "___h.1" from "foo.tableE" as "f__e" where "βήτα__f"("f__e"."foo.tableE.E3") >= 50 ref: valid-identifiers.yamsql:350 - explain: SCAN([IS foo.tableE]) | FILTER _.foo.tableE.E3.S2 GREATER_THAN_OR_EQUALS + explain: SCAN([IS foo__2tableE]) | FILTER _.foo.tableE.E3.S2 GREATER_THAN_OR_EQUALS promote(@c22 AS LONG) | MAP (_.foo.tableE.E3.S1 AS ___h.1) task_count: 204 task_total_time_ms: 10 @@ -271,8 +271,8 @@ all-tests: insert_reused_count: 2 - query: EXPLAIN select * from "__$func3"(10, 1, 1); ref: valid-identifiers.yamsql:356 - explain: SCAN([IS my$adjacency$list, EQUALS promote(@c7 AS LONG)]) | FLATMAP - q0 -> { SCAN([IS my$adjacency$list, [LESS_THAN promote(@c5 AS LONG)]]) + explain: SCAN([IS my__1adjacency__1list, EQUALS promote(@c7 AS LONG)]) | FLATMAP + q0 -> { SCAN([IS my__1adjacency__1list, [LESS_THAN promote(@c5 AS LONG)]]) | FILTER _.my__parent EQUALS promote(@c7 AS LONG) AS q1 RETURN (q1.me AS _0, q1.my__parent AS _1, q0.me AS _2, q0.my__parent AS _3) } task_count: 1337 @@ -285,7 +285,7 @@ all-tests: insert_reused_count: 17 - query: EXPLAIN select * from "$yay"(5); ref: valid-identifiers.yamsql:360 - explain: SCAN([IS foo.tableE]) | FILTER _.foo.tableE.E3.S1 EQUALS promote(@c5 + explain: SCAN([IS foo__2tableE]) | FILTER _.foo.tableE.E3.S1 EQUALS promote(@c5 AS LONG) | MAP (_.foo.tableE.E1 AS _$x.id) task_count: 338 task_total_time_ms: 28 @@ -297,7 +297,7 @@ all-tests: insert_reused_count: 2 - query: EXPLAIN select * from "__2yay"(6); ref: valid-identifiers.yamsql:364 - explain: SCAN([IS foo.tableE]) | FILTER _.foo.tableE.E3.S1 EQUALS promote(@c5 + explain: SCAN([IS foo__2tableE]) | FILTER _.foo.tableE.E3.S1 EQUALS promote(@c5 AS LONG) | MAP (_.foo.tableE.E1 AS _$y.id) task_count: 338 task_total_time_ms: 12 @@ -309,7 +309,7 @@ all-tests: insert_reused_count: 2 - query: EXPLAIN select * from "नमस्त"(4); ref: valid-identifiers.yamsql:368 - explain: SCAN([IS foo.tableE]) | FILTER _.foo.tableE.E3.S1 EQUALS promote(@c5 + explain: SCAN([IS foo__2tableE]) | FILTER _.foo.tableE.E3.S1 EQUALS promote(@c5 AS LONG) | MAP (_.foo.tableE.E1 AS _$z.id) task_count: 338 task_total_time_ms: 24 @@ -321,7 +321,7 @@ all-tests: insert_reused_count: 2 - query: EXPLAIN select * from "$yay__view"; ref: valid-identifiers.yamsql:374 - explain: SCAN([IS foo.tableE]) | FILTER _.foo.tableE.E3.S1 EQUALS 4 | MAP (_.foo.tableE.E1 + explain: SCAN([IS foo__2tableE]) | FILTER _.foo.tableE.E3.S1 EQUALS 4 | MAP (_.foo.tableE.E1 AS _$x.id) task_count: 294 task_total_time_ms: 14 @@ -333,7 +333,7 @@ all-tests: insert_reused_count: 2 - query: EXPLAIN select * from "__2yay__view"; ref: valid-identifiers.yamsql:378 - explain: SCAN([IS foo.tableE]) | FILTER _.foo.tableE.E3.S1 EQUALS 5 | MAP (_.foo.tableE.E1 + explain: SCAN([IS foo__2tableE]) | FILTER _.foo.tableE.E3.S1 EQUALS 5 | MAP (_.foo.tableE.E1 AS _$y.id) task_count: 294 task_total_time_ms: 14 @@ -345,7 +345,7 @@ all-tests: insert_reused_count: 2 - query: EXPLAIN select * from "வணக்கம்"; ref: valid-identifiers.yamsql:382 - explain: SCAN([IS foo.tableE]) | FILTER _.foo.tableE.E3.S1 EQUALS 6 | MAP (_.foo.tableE.E1 + explain: SCAN([IS foo__2tableE]) | FILTER _.foo.tableE.E3.S1 EQUALS 6 | MAP (_.foo.tableE.E1 AS _$z.id) task_count: 294 task_total_time_ms: 15 @@ -371,9 +371,9 @@ all-tests: - query: EXPLAIN select struct "x$$" ("foo.tableA.A1", "foo.tableA.A2", "foo.tableA.A3") from "foo.tableA" ref: valid-identifiers.yamsql:392 - explain: SCAN([IS foo.tableA]) | MAP (_ AS _0) - task_count: 499 - task_total_time_ms: 50 + explain: SCAN([IS foo__2tableA]) | MAP (_ AS _0) + task_count: 497 + task_total_time_ms: 46 transform_count: 136 transform_time_ms: 10 transform_yield_count: 50 @@ -493,9 +493,9 @@ update-delete-statements: ref: valid-identifiers.yamsql:510 explain: 'COVERING(foo.tableA.idx [EQUALS promote(@c9 AS LONG)] -> [foo__2tableA__2A1: KEY:[0], foo__2tableA__2A2: KEY:[1], foo__2tableA__2A3: KEY:[2]]) | DISTINCT - BY PK | FETCH | UPDATE foo.tableA' - task_count: 709 - task_total_time_ms: 19 + BY PK | FETCH | UPDATE foo__2tableA' + task_count: 707 + task_total_time_ms: 16 transform_count: 141 transform_time_ms: 6 transform_yield_count: 57 @@ -507,9 +507,9 @@ update-delete-statements: ref: valid-identifiers.yamsql:514 explain: 'COVERING(foo.tableA.idx [[GREATER_THAN promote(@c9 AS LONG)]] -> [foo__2tableA__2A1: KEY:[0], foo__2tableA__2A2: KEY:[1], foo__2tableA__2A3: KEY:[2]]) | DISTINCT - BY PK | FETCH | UPDATE foo.tableA | MAP (_.new.foo.tableA.A1 AS foo.tableA.A1)' - task_count: 766 - task_total_time_ms: 17 + BY PK | FETCH | UPDATE foo__2tableA | MAP (_.new.foo.tableA.A1 AS foo.tableA.A1)' + task_count: 762 + task_total_time_ms: 10 transform_count: 155 transform_time_ms: 4 transform_yield_count: 57