diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/model/ExplainSimplifiedResponse.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/ExplainSimplifiedResponse.java index 2aad03b6..323ab79a 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/model/ExplainSimplifiedResponse.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/ExplainSimplifiedResponse.java @@ -34,7 +34,8 @@ public static class Total { @JsonInclude(JsonInclude.Include.NON_NULL) @JsonPropertyOrder({ "rank", "id", "title", "final_score", - "es_relevance", "internal_score", "quality_multiplier", "matched_terms", "filters" + "es_relevance", "semantic_score", "non_semantic_score", "semantic_contribute", + "internal_score", "quality_multiplier", "matched_terms", "filters" }) public static class Hit { private Integer rank; @@ -48,6 +49,18 @@ public static class Hit { @JsonProperty("es_relevance") private Double esRelevance; + /** Aggregate contribution from the semantic_text query, without ELSER token details */ + @JsonProperty("semantic_score") + private Double semanticScore; + + /** All relevance not contributed by semantic_text, including keyword and constant scores */ + @JsonProperty("non_semantic_score") + private Double nonSemanticScore; + + /** Fraction of es_relevance contributed by semantic_text */ + @JsonProperty("semantic_contribute") + private Double semanticContribute; + /** The stored summaries.score of the document */ @JsonProperty("internal_score") private Double internalScore; diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java index 0b7ceaff..31ac704a 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java @@ -40,9 +40,15 @@ public class ElasticSearch extends ElasticSearchBase implements Search { protected Map defaultElasticSetting; - // the semantic_text field on the vocabs index + // the semantic_text field for the vocabs index protected static final String SEMANTIC_CONCEPT_FIELD = "concept_semantic"; + // the semantic_text field for the record index + protected static final String SEMANTIC_DESCRIPTION_FIELD = "description_semantic"; + + // Weight of the semantic search score contribute to the final ES relevance + protected static final float SEMANTIC_DESCRIPTION_BOOST = 1.0F; + // organisation vocabs are skipped from sementic query protected static final String ORGANISATION_VOCAB_FIELD = "organisation_vocab"; @@ -426,6 +432,47 @@ public ElasticSearchBase.SearchResult searchAllCollections( return searchCollectionsByIds(null, Boolean.FALSE, sortBy); } + /** + * Wraps the existing keyword should-clauses so that the semantic query boosts ranking without + * changing which records match. The inner bool preserves the keyword recall set, while the + * semantic clause is optional and can only contribute to the score of those records. + * + * @param keywordClauses - The lexical clauses, as built for the "should" block + * @param keywords - The raw keywords as typed by the end user, quotes included + * @return - A single composite clause when semantic applies, otherwise keywordClauses as given + */ + protected List applySemanticBoost(List keywordClauses, List keywords) { + if (keywordClauses == null || keywordClauses.isEmpty() + || keywords == null || keywords.isEmpty() + || !Boolean.TRUE.equals(semanticEnabled)) { + return keywordClauses; + } + + boolean allExact = keywords.stream() + .allMatch(t -> t.startsWith("\"") && t.endsWith("\"") && t.length() > 2); + if (allExact) { + return keywordClauses; + } + + String semanticInput = keywords.stream() + .map(t -> t.startsWith("\"") && t.endsWith("\"") && t.length() > 2 + ? t.substring(1, t.length() - 1) + : t) + .collect(Collectors.joining(" ")); + + Query keywordRecall = Query.of(q -> q.bool(b -> b + .should(keywordClauses) + .minimumShouldMatch("1"))); + + Query semanticBoostQuery = Query.of(q -> q.semantic(s -> s + .field(SEMANTIC_DESCRIPTION_FIELD) + .query(semanticInput) + .boost(SEMANTIC_DESCRIPTION_BOOST))); + + return List.of(Query.of(q -> q.bool(b -> b + .must(keywordRecall) + .should(semanticBoostQuery)))); + } /** * Build SearchRequest for searchByParameters and explainByParameters @@ -484,6 +531,8 @@ protected Supplier buildParameterSearchRequestSupplier( should.add(CQLFields.getDatasetGroupTextSearchQuery(term, isExact)); } } + // Re-rank by meaning, without changing which records match + should = applySemanticBoost(should, keywords); List filters = new ArrayList<>(); CQLToElasticFilterFactory factory = new CQLToElasticFilterFactory<>(coor, CQLFields.class); @@ -592,6 +641,8 @@ public ElasticSearchBase.SearchResult searchByParameters(Li should.add(CQLFields.getDatasetGroupTextSearchQuery(term, isExact)); } } + // Re-rank by meaning, without changing which records match + should = applySemanticBoost(should, keywords); List filters = new ArrayList<>(); diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/util/ExplainSimplifier.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/util/ExplainSimplifier.java index ee415e4f..61b7f173 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/util/ExplainSimplifier.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/util/ExplainSimplifier.java @@ -48,6 +48,13 @@ public class ExplainSimplifier { protected static final String RELEVANCE_DESCRIPTION_PREFIX = "_score:"; + /** The enclosing explanation node for a semantic_text nested-document score. */ + protected static final String SEMANTIC_SCORE_PREFIX = "Score based on "; + + /** Identifies the otherwise implementation-specific ELSER sparse-feature explanation tree. */ + protected static final String SEMANTIC_EMBEDDING_MARKER = + "description_semantic.inference.chunks.embeddings field"; + protected static final Comparator BY_SCORE_DESC = Comparator.comparing( ExplainSimplifiedResponse.MatchedTerm::getScore, @@ -137,6 +144,14 @@ else if (explanation != null) { List terms = new ArrayList<>(); List filters = new ArrayList<>(); + Double semanticScore = semanticScoreOf(scored); + Double nonSemanticScore = semanticScore != null && esRelevance != null + ? esRelevance - semanticScore + : null; + Double semanticContribute = semanticScore != null && esRelevance != null && esRelevance != 0.0 + ? semanticScore / esRelevance + : null; + collectScoreParts(scored, terms, filters); terms.sort(BY_SCORE_DESC); @@ -146,6 +161,9 @@ else if (explanation != null) { .title(stringField(hit.source(), StacBasicField.Title.searchField)) .finalScore(finalScore) .esRelevance(esRelevance) + .semanticScore(semanticScore) + .nonSemanticScore(nonSemanticScore) + .semanticContribute(semanticContribute) .internalScore(doubleField(hit.source(), StacSummeries.Score.searchField)) .qualityMultiplier(qualityMultiplier) .matchedTerms(terms) @@ -192,6 +210,13 @@ protected static void collectScoreParts(List details, for (ExplanationDetail detail : details) { String description = detail.description(); + // A semantic_text score contains one leaf per sparse model feature. Those tokens are + // implementation details rather than user-readable query matches, so report the + // enclosing score once via semantic_score and keep the whole subtree out of filters. + if (isSemanticScoreNode(detail) || isSemanticEmbeddingDetail(detail)) { + continue; + } + // most nodes are idf/tf breakdowns, skip the regex for them if (description != null && description.startsWith(WEIGHT_PREFIX)) { Matcher matcher = WEIGHT_PATTERN.matcher(description); @@ -223,6 +248,56 @@ protected static void collectScoreParts(List details, } } + /** + * Add the values of semantic_text score roots. There is normally one root because the search + * builds one semantic query, but summing keeps the simplifier correct if that changes later. + * A null result distinguishes "no semantic clause" from a semantic clause that scored zero. + */ + protected static Double semanticScoreOf(List details) { + if (details == null) { + return null; + } + + Double total = null; + + for (ExplanationDetail detail : details) { + if (isSemanticScoreNode(detail)) { + total = (total == null ? 0.0 : total) + detail.value(); + continue; + } + + Double nested = semanticScoreOf(detail.details()); + if (nested != null) { + total = (total == null ? 0.0 : total) + nested; + } + } + + return total; + } + + protected static boolean isSemanticScoreNode(ExplanationDetail detail) { + return detail.description() != null + && detail.description().startsWith(SEMANTIC_SCORE_PREFIX) + && containsSemanticEmbeddingDetail(detail); + } + + protected static boolean containsSemanticEmbeddingDetail(ExplanationDetail detail) { + if (isSemanticEmbeddingDetail(detail)) { + return true; + } + + if (detail.details() == null) { + return false; + } + + return detail.details().stream().anyMatch(ExplainSimplifier::containsSemanticEmbeddingDetail); + } + + protected static boolean isSemanticEmbeddingDetail(ExplanationDetail detail) { + return detail.description() != null + && detail.description().contains(SEMANTIC_EMBEDDING_MARKER); + } + protected static Map termsClausesOf(JsonNode request) { Map byRendering = new HashMap<>(); collectTermsClauses(request, byRendering); diff --git a/server/src/main/resources/application.yaml b/server/src/main/resources/application.yaml index c11293f3..1664c7f7 100644 --- a/server/src/main/resources/application.yaml +++ b/server/src/main/resources/application.yaml @@ -22,9 +22,8 @@ elasticsearch: search_suggestions: path: search_suggestions fields: abstract_phrases, parameter_vocabs_sayt, platform_vocabs_sayt, organisation_vocabs_sayt - # Semantic suggestions: the vocabs index is searched by meaning (ELSER via semantic_text). - # The suggested concepts are not filtered against records, so a suggestion may be a concept - # no record currently carries. + # Semantic features use ELSER via semantic_text. Suggestions search the vocabs index by meaning; + # free-text record searches use description_semantic only to re-rank keyword matches. # Off by default - semantic_text needs the licensed `inference` feature. semantic: enabled: false diff --git a/server/src/test/java/au/org/aodn/ogcapi/server/core/util/ExplainSimplifierTest.java b/server/src/test/java/au/org/aodn/ogcapi/server/core/util/ExplainSimplifierTest.java index e02a132a..56f0dcab 100644 --- a/server/src/test/java/au/org/aodn/ogcapi/server/core/util/ExplainSimplifierTest.java +++ b/server/src/test/java/au/org/aodn/ogcapi/server/core/util/ExplainSimplifierTest.java @@ -1,13 +1,21 @@ package au.org.aodn.ogcapi.server.core.util; +import au.org.aodn.ogcapi.server.core.model.ExplainSimplifiedResponse; +import co.elastic.clients.elasticsearch.core.explain.ExplanationDetail; +import co.elastic.clients.elasticsearch.core.search.Hit; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; import org.junit.jupiter.api.Test; +import java.util.ArrayList; +import java.util.List; import java.util.Map; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; public class ExplainSimplifierTest { @@ -37,4 +45,77 @@ public void separateTermsValuesCommaSeparatesTheValuesOfTheClause() throws JsonP "summaries.dataset_group:(csiro csiro temperature temperature)^100.0", datasetGroupRequest())); } + + @Test + public void semanticTextDetailsAreAggregatedWithoutExposingSparseFeatures() { + ExplanationDetail semantic = semanticScoreDetail(12.833507F); + List scored = List.of( + detail("summaries.dataset_group:(imos imos wave measurements)^100.0", 100.0F), + semantic, + detail("ConstantScore(spatial query)", 1.0F)); + + assertEquals(12.833507, ExplainSimplifier.semanticScoreOf(scored), 0.000001); + + List terms = new ArrayList<>(); + List filters = new ArrayList<>(); + ExplainSimplifier.collectScoreParts(scored, terms, filters); + + assertEquals(2, filters.size()); + assertFalse(filters.stream() + .anyMatch(filter -> filter.getDescription().contains("embeddings field"))); + } + + @Test + public void simplifiedHitSeparatesSemanticFromOtherRelevance() { + ExplanationDetail score = ExplanationDetail.of(d -> d + .description("_score: ") + .value(113.833504F) + .details( + detail("summaries.dataset_group:(imos imos wave measurements)^100.0", 100.0F), + semanticScoreDetail(12.833507F), + detail("ConstantScore(spatial query)", 1.0F))); + + ObjectNode source = MAPPER.createObjectNode(); + source.put("title", "Wave buoys Observations"); + source.putObject("summaries").put("score", 144.0); + + Hit hit = Hit.of(h -> h + .index("records") + .id("wave-record") + .score(112.27414) + .source(source) + .explanation(e -> e + .description("sum of:") + .value(112.27414F) + .details(score))); + + ExplainSimplifiedResponse.Hit simplified = ExplainSimplifier.toSimplifiedHit(hit, 1); + + assertNotNull(simplified.getSemanticScore()); + assertEquals(12.833507, simplified.getSemanticScore(), 0.000001); + assertEquals(101.0, simplified.getNonSemanticScore(), 0.00001); + assertEquals(12.833507 / 113.833504, simplified.getSemanticContribute(), 0.000001); + assertEquals(2, simplified.getFilters().size()); + } + + private ExplanationDetail semanticScoreDetail(float score) { + return ExplanationDetail.of(d -> d + .description("Score based on 2 child docs in range from 25 to 85, using score mode Max") + .value(score) + .details(child -> child + .description("sum of:") + .value(score) + .details(feature -> feature + .description("Linear function on the " + + "description_semantic.inference.chunks.embeddings field " + + "for the wave feature, computed as w * S from:") + .value(score) + .details(weight -> weight + .description("w, weight of this function") + .value(2.578F))))); + } + + private ExplanationDetail detail(String description, float score) { + return ExplanationDetail.of(d -> d.description(description).value(score)); + } }