Skip to content

Commit

Permalink
fix: attempt to fix proper scope evaluation in EntityHavingTranslator
Browse files Browse the repository at this point in the history
  • Loading branch information
@novoj
novoj committed Dec 16, 2024
1 parent 3be5c88 commit 15aa056
Show file tree
Hide file tree
Showing 4 changed files with 117 additions and 96 deletions.
2 changes: 1 addition & 1 deletion evita_api/src/main/java/io/evitadb/api/requestResponse/schema/ClassSchemaAnalyzer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1075,7 +1075,7 @@ private void defineReference(
"the value of `faceted` property is not taken into an account " +
"(and thus it doesn't make sense to set it to true)!"
);
editor.indexedInScope(
editor.facetedInScope(
Arrays.stream(scopedDefinition)
.filter(ScopeReferenceSettings::faceted)
.map(ScopeReferenceSettings::scope)
Expand Down
6 changes: 4 additions & 2 deletions ...n/java/io/evitadb/api/requestResponse/schema/builder/ReflectedReferenceSchemaBuilder.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -375,7 +375,9 @@ public ReflectedReferenceSchemaBuilder nonIndexed(@Nullable Scope... inScope) {
@Override
public ReflectedReferenceSchemaBuilder facetedInScope(@Nonnull Scope... inScope) {
final boolean reflectedReferenceAvailable = isReflectedReferenceAvailable();
if (reflectedReferenceAvailable && Arrays.stream(inScope).allMatch(this::isIndexedInScope)) {
final boolean indexedInherited = isIndexedInherited();
final boolean allInformationPresent = reflectedReferenceAvailable || !indexedInherited;
if (allInformationPresent && Arrays.stream(inScope).allMatch(this::isIndexedInScope)) {
// just update the faceted scopes
this.updatedSchemaDirty = updateMutationImpact(
this.updatedSchemaDirty,
Expand All @@ -394,7 +396,7 @@ public ReflectedReferenceSchemaBuilder facetedInScope(@Nonnull Scope... inScope)
new SetReferenceSchemaIndexedMutation(
getName(),
Arrays.stream(Scope.values())
.filter(scope -> includedScopes.contains(scope) && (reflectedReferenceAvailable || !this.isIndexedInScope(scope)))
.filter(scope -> includedScopes.contains(scope) || (allInformationPresent && this.isIndexedInScope(scope)))
.toArray(Scope[]::new)
),
new SetReferenceSchemaFacetedMutation(getName(), inScope)
Expand Down
22 changes: 14 additions & 8 deletions evita_engine/src/main/java/io/evitadb/core/query/extraResult/ExtraResultPlanningVisitor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,8 @@
import io.evitadb.dataType.Scope;
import io.evitadb.exception.GenericEvitaInternalError;
import io.evitadb.index.EntityIndex;
import io.evitadb.index.EntityIndexKey;
import io.evitadb.index.EntityIndexType;
import io.evitadb.index.GlobalEntityIndex;
import io.evitadb.utils.ArrayUtils;
import lombok.Getter;
Expand Down Expand Up @@ -391,26 +393,30 @@ public NestedContextSorter createSorter(
@Nonnull Supplier<String> stepDescriptionSupplier
) {
try {
queryContext.pushStep(
this.queryContext.pushStep(
QueryPhase.PLANNING_SORT,
stepDescriptionSupplier
);
// we have to create and trap the nested query context here to carry it along with the sorter
// otherwise the sorter will target and use the incorrectly originally queried (prefetched) entities
final QueryPlanningContext nestedQueryContext = entityCollection.createQueryContext(
queryContext,
queryContext.getEvitaRequest().deriveCopyWith(
this.queryContext,
this.queryContext.getEvitaRequest().deriveCopyWith(
entityType,
null,
new OrderBy(orderBy.getChildren()),
queryContext.getLocale()
this.queryContext.getLocale()
),
queryContext.getEvitaSession()
this.queryContext.getEvitaSession()
);

final GlobalEntityIndex entityIndex = entityCollection.getGlobalIndexIfExists().orElse(null);
final Set<Scope> scopes = getProcessingScope().getScopes();
final GlobalEntityIndex[] entityIndexes = scopes.stream()
.map(scope -> entityCollection.getIndexByKeyIfExists(new EntityIndexKey(EntityIndexType.GLOBAL, scope)))
.map(GlobalEntityIndex.class::cast)
.toArray(GlobalEntityIndex[]::new);
final Sorter sorter;
if (entityIndex == null) {
if (entityIndexes.length == 0) {
sorter = NoSorter.INSTANCE;
} else {
// create a visitor
Expand All @@ -422,7 +428,7 @@ public NestedContextSorter createSorter(
);
// now analyze the filter by in a nested context with exchanged primary entity index
sorter = orderByVisitor.executeInContext(
new EntityIndex[]{entityIndex},
entityIndexes,
entityType,
locale,
new AttributeSchemaAccessor(nestedQueryContext.getCatalogSchema(), entityCollection.getSchema()),
Expand Down
183 changes: 98 additions & 85 deletions ...c/main/java/io/evitadb/core/query/filter/translator/reference/EntityHavingTranslator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,13 +39,16 @@
import io.evitadb.core.query.algebra.deferred.DeferredFormula;
import io.evitadb.core.query.algebra.deferred.FormulaWrapper;
import io.evitadb.core.query.algebra.reference.ReferenceOwnerTranslatingFormula;
import io.evitadb.core.query.algebra.utils.FormulaFactory;
import io.evitadb.core.query.common.translator.SelfTraversingTranslator;
import io.evitadb.core.query.filter.FilterByVisitor;
import io.evitadb.core.query.filter.FilterByVisitor.ProcessingScope;
import io.evitadb.core.query.filter.translator.FilteringConstraintTranslator;
import io.evitadb.core.query.sort.attribute.translator.EntityNestedQueryComparator;
import io.evitadb.exception.EvitaInvalidUsageException;
import io.evitadb.index.EntityIndex;
import io.evitadb.index.EntityIndexKey;
import io.evitadb.index.EntityIndexType;
import io.evitadb.index.GlobalEntityIndex;
import io.evitadb.index.ReferencedTypeEntityIndex;
import io.evitadb.index.bitmap.BaseBitmap;
Expand All @@ -58,8 +61,8 @@
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.function.Function;
import java.util.function.Supplier;

Expand All @@ -86,50 +89,57 @@ public class EntityHavingTranslator implements FilteringConstraintTranslator<Ent
* filter formula resulting from planning the nested query.
*/
@Nonnull
private static GlobalIndexAndFormula planNestedQuery(
private static List<GlobalIndexAndFormula> planNestedQuery(
@Nonnull String targetEntityType,
@Nonnull FilterConstraint filter,
@Nonnull FilterByVisitor filterByVisitor,
@Nonnull Supplier<String> taskDescriptionSupplier
) {
final ProcessingScope<?> processingScope = filterByVisitor.getProcessingScope();
final EntityCollection targetEntityCollection = filterByVisitor.getEntityCollectionOrThrowException(targetEntityType, taskDescriptionSupplier);
final Optional<GlobalEntityIndex> globalIndexIfExists = targetEntityCollection.getGlobalIndexIfExists();
if (globalIndexIfExists.isEmpty()) {
return new GlobalIndexAndFormula(null, EmptyFormula.INSTANCE);
final List<GlobalEntityIndex> globalIndexes = processingScope.getScopes()
.stream()
.map(scope -> targetEntityCollection.getIndexByKeyIfExists(new EntityIndexKey(EntityIndexType.GLOBAL, scope)))
.map(GlobalEntityIndex.class::cast)
.toList();
if (globalIndexes.isEmpty()) {
return List.of(new GlobalIndexAndFormula(null, EmptyFormula.INSTANCE));
} else {
final ProcessingScope<?> processingScope = filterByVisitor.getProcessingScope();
final Function<FilterConstraint, FilterConstraint> enricher = processingScope.getNestedQueryFormulaEnricher();
final FilterConstraint enrichedConstraint = enricher.apply(filter);
final FilterBy combinedFilterBy = enrichedConstraint instanceof FilterBy fb ? fb : new FilterBy(enrichedConstraint);
final EntityNestedQueryComparator entityNestedQueryComparator = processingScope.getEntityNestedQueryComparator();

final GlobalEntityIndex globalIndex = globalIndexIfExists.get();
return new GlobalIndexAndFormula(
globalIndex,
filterByVisitor.computeOnlyOnce(
Collections.singletonList(globalIndex),
combinedFilterBy,
() -> {
final QueryPlanningContext nestedQueryContext = targetEntityCollection.createQueryContext(
filterByVisitor.getQueryContext(),
filterByVisitor.getEvitaRequest().deriveCopyWith(
targetEntityType,
combinedFilterBy,
ofNullable(entityNestedQueryComparator)
.map(EntityNestedQueryComparator::getOrderBy)
.map(it -> new OrderBy(it.getChildren()))
.orElse(null),
ofNullable(entityNestedQueryComparator)
.map(EntityNestedQueryComparator::getLocale)
.orElse(null)
),
filterByVisitor.getEvitaSession()
);
return globalIndexes
.stream()
.map(globalIndex -> new GlobalIndexAndFormula(
globalIndex,
filterByVisitor.computeOnlyOnce(
Collections.singletonList(globalIndex),
combinedFilterBy,
() -> {
final QueryPlanningContext nestedQueryContext = targetEntityCollection.createQueryContext(
filterByVisitor.getQueryContext(),
filterByVisitor.getEvitaRequest().deriveCopyWith(
targetEntityType,
combinedFilterBy,
ofNullable(entityNestedQueryComparator)
.map(EntityNestedQueryComparator::getOrderBy)
.map(it -> new OrderBy(it.getChildren()))
.orElse(null),
ofNullable(entityNestedQueryComparator)
.map(EntityNestedQueryComparator::getLocale)
.orElse(null)
),
filterByVisitor.getEvitaSession()
);

return QueryPlanner.planNestedQuery(nestedQueryContext, taskDescriptionSupplier)
.getFilter();
})
);
return QueryPlanner.planNestedQuery(nestedQueryContext, taskDescriptionSupplier)
.getFilter();
}
)
)
).toList();
}
}

Expand All @@ -156,62 +166,65 @@ public Formula translate(@Nonnull EntityHaving entityHaving, @Nonnull FilterByVi
final Supplier<String> nestedQueryDescription = () -> "filtering reference `" + referenceSchema.getName() +
"` by entity `" + referencedEntityType + "` having: " + filterConstraint;

final GlobalIndexAndFormula nestedResult = planNestedQuery(
referencedEntityType,
filterConstraint,
filterByVisitor,
nestedQueryDescription
);

if (!nestedResult.isEmpty()) {
if (ReferencedTypeEntityIndex.class.isAssignableFrom(processingScope.getIndexType())) {
return nestedResult.filter();
} else {
return filterByVisitor.computeOnlyOnce(
processingScope.getIndexes(),
filterConstraint,
() -> {
if (nestedResult.globalIndex() == null) {
return EmptyFormula.INSTANCE;
}
final ReferenceOwnerTranslatingFormula outputFormula = new ReferenceOwnerTranslatingFormula(
nestedResult.globalIndex(),
nestedResult.filter(),
it -> {
// leave the return here, so that we can easily debug it
final RoaringBitmap combinedResult = RoaringBitmap.or(
filterByVisitor.getReferencedEntityIndex(entitySchema, referenceSchema, it)
.map(EntityIndex::getAllPrimaryKeys)
.map(RoaringBitmapBackedBitmap::getRoaringBitmap)
.toArray(RoaringBitmap[]::new)
);
return combinedResult.isEmpty() ? EmptyBitmap.INSTANCE : new BaseBitmap(combinedResult);
}
);
return new DeferredFormula(
new FormulaWrapper(
outputFormula,
(executionContext, formula) -> {
try {
executionContext.pushStep(QueryPhase.EXECUTION_FILTER_NESTED_QUERY, nestedQueryDescription);
return formula.compute();
} finally {
executionContext.popStep();
}
return FormulaFactory.or(
planNestedQuery(
referencedEntityType,
filterConstraint,
filterByVisitor,
nestedQueryDescription
)
.stream()
.map(nestedResult -> {
if (ReferencedTypeEntityIndex.class.isAssignableFrom(processingScope.getIndexType())) {
return nestedResult.filter();
} else {
return filterByVisitor.computeOnlyOnce(
processingScope.getIndexes(),
filterConstraint,
() -> {
if (nestedResult.globalIndex() == null) {
return EmptyFormula.INSTANCE;
}
final ReferenceOwnerTranslatingFormula outputFormula = new ReferenceOwnerTranslatingFormula(
nestedResult.globalIndex(),
nestedResult.filter(),
it -> {
// leave the return here, so that we can easily debug it
final RoaringBitmap combinedResult = RoaringBitmap.or(
filterByVisitor.getReferencedEntityIndex(entitySchema, referenceSchema, it)
.map(EntityIndex::getAllPrimaryKeys)
.map(RoaringBitmapBackedBitmap::getRoaringBitmap)
.toArray(RoaringBitmap[]::new)
);
return combinedResult.isEmpty() ? EmptyBitmap.INSTANCE : new BaseBitmap(combinedResult);
}
);
return new DeferredFormula(
new FormulaWrapper(
outputFormula,
(executionContext, formula) -> {
try {
executionContext.pushStep(QueryPhase.EXECUTION_FILTER_NESTED_QUERY, nestedQueryDescription);
return formula.compute();
} finally {
executionContext.popStep();
}
}
)
);
},
2L,
// we need to add exact pointers to the entity schema and reference schema, which play role
// in the lambda evaluation
NumberUtils.join(
System.identityHashCode(entitySchema),
System.identityHashCode(referenceSchema)
)
);
},
2L,
// we need to add exact pointers to the entity schema and reference schema, which play role
// in the lambda evaluation
NumberUtils.join(
System.identityHashCode(entitySchema),
System.identityHashCode(referenceSchema)
)
);
}
}
}
})
.toArray(Formula[]::new)
);
}
return EmptyFormula.INSTANCE;
}
Expand Down

0 comments on commit 15aa056

Please sign in to comment.