Feature/8369 add semantic query

server/src/main/java/au/org/aodn/ogcapi/server/core/model/ExplainSimplifiedResponse.java

@@ -34,7 +34,8 @@ public static class Total {
     @JsonInclude(JsonInclude.Include.NON_NULL)
     @JsonPropertyOrder({
             "rank", "id", "title", "final_score",
-            "es_relevance", "internal_score", "quality_multiplier", "matched_terms", "filters"
+            "es_relevance", "semantic_score", "non_semantic_score", "semantic_contribute",
+            "internal_score", "quality_multiplier", "matched_terms", "filters"
     })
     public static class Hit {
         private Integer rank;
@@ -48,6 +49,18 @@ public static class Hit {
         @JsonProperty("es_relevance")
         private Double esRelevance;
 
+        /** Aggregate contribution from the semantic_text query, without ELSER token details */
+        @JsonProperty("semantic_score")
+        private Double semanticScore;
+
+        /** All relevance not contributed by semantic_text, including keyword and constant scores */
+        @JsonProperty("non_semantic_score")
+        private Double nonSemanticScore;
+
+        /** Fraction of es_relevance contributed by semantic_text */
+        @JsonProperty("semantic_contribute")
+        private Double semanticContribute;
+
         /** The stored summaries.score of the document */
         @JsonProperty("internal_score")
         private Double internalScore;

server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java

@@ -40,9 +40,15 @@ public class ElasticSearch extends ElasticSearchBase implements Search {
 
     protected Map<CQLElasticSetting, String> defaultElasticSetting;
 
-    // the semantic_text field on the vocabs index
+    // the semantic_text field for the vocabs index
     protected static final String SEMANTIC_CONCEPT_FIELD = "concept_semantic";
 
+    // the semantic_text field for the record index
+    protected static final String SEMANTIC_DESCRIPTION_FIELD = "description_semantic";
+
+    // Weight of the semantic search score contribute to the final ES relevance
+    protected static final float SEMANTIC_DESCRIPTION_BOOST = 1.0F;
+
     // organisation vocabs are skipped from sementic query
     protected static final String ORGANISATION_VOCAB_FIELD = "organisation_vocab";
 
@@ -426,6 +432,47 @@ public ElasticSearchBase.SearchResult<StacCollectionModel> searchAllCollections(
         return searchCollectionsByIds(null, Boolean.FALSE, sortBy);
     }
 
+    /**
+     * Wraps the existing keyword should-clauses so that the semantic query boosts ranking without
+     * changing which records match. The inner bool preserves the keyword recall set, while the
+     * semantic clause is optional and can only contribute to the score of those records.
+     *
+     * @param keywordClauses - The lexical clauses, as built for the "should" block
+     * @param keywords - The raw keywords as typed by the end user, quotes included
+     * @return - A single composite clause when semantic applies, otherwise keywordClauses as given
+     */
+    protected List<Query> applySemanticBoost(List<Query> keywordClauses, List<String> keywords) {
+        if (keywordClauses == null || keywordClauses.isEmpty()
+                || keywords == null || keywords.isEmpty()
+                || !Boolean.TRUE.equals(semanticEnabled)) {
+            return keywordClauses;
+        }
+
+        boolean allExact = keywords.stream()
+                .allMatch(t -> t.startsWith("\"") && t.endsWith("\"") && t.length() > 2);
+        if (allExact) {
+            return keywordClauses;
+        }
+
+        String semanticInput = keywords.stream()
+                .map(t -> t.startsWith("\"") && t.endsWith("\"") && t.length() > 2
+                        ? t.substring(1, t.length() - 1)
+                        : t)
+                .collect(Collectors.joining(" "));
+
+        Query keywordRecall = Query.of(q -> q.bool(b -> b
+                .should(keywordClauses)
+                .minimumShouldMatch("1")));
+
+        Query semanticBoostQuery = Query.of(q -> q.semantic(s -> s
+                .field(SEMANTIC_DESCRIPTION_FIELD)
+                .query(semanticInput)
+                .boost(SEMANTIC_DESCRIPTION_BOOST)));
+
+        return List.of(Query.of(q -> q.bool(b -> b
+                .must(keywordRecall)
+                .should(semanticBoostQuery))));
+    }
 
     /**
      * Build SearchRequest for searchByParameters and explainByParameters
@@ -484,6 +531,8 @@ protected Supplier<SearchRequest.Builder> buildParameterSearchRequestSupplier(
                 should.add(CQLFields.getDatasetGroupTextSearchQuery(term, isExact));
             }
         }
+        // Re-rank by meaning, without changing which records match
+        should = applySemanticBoost(should, keywords);
 
         List<Query> filters = new ArrayList<>();
         CQLToElasticFilterFactory<CQLFields> factory = new CQLToElasticFilterFactory<>(coor, CQLFields.class);
@@ -592,6 +641,8 @@ public ElasticSearchBase.SearchResult<StacCollectionModel> searchByParameters(Li
                     should.add(CQLFields.getDatasetGroupTextSearchQuery(term, isExact));
                 }
             }
+            // Re-rank by meaning, without changing which records match
+            should = applySemanticBoost(should, keywords);
 
             List<Query> filters = new ArrayList<>();
 

server/src/main/java/au/org/aodn/ogcapi/server/core/util/ExplainSimplifier.java

@@ -48,6 +48,13 @@ public class ExplainSimplifier {
 
     protected static final String RELEVANCE_DESCRIPTION_PREFIX = "_score:";
 
+    /** The enclosing explanation node for a semantic_text nested-document score. */
+    protected static final String SEMANTIC_SCORE_PREFIX = "Score based on ";
+
+    /** Identifies the otherwise implementation-specific ELSER sparse-feature explanation tree. */
+    protected static final String SEMANTIC_EMBEDDING_MARKER =
+            "description_semantic.inference.chunks.embeddings field";
+
     protected static final Comparator<ExplainSimplifiedResponse.MatchedTerm> BY_SCORE_DESC =
             Comparator.comparing(
                     ExplainSimplifiedResponse.MatchedTerm::getScore,
@@ -137,6 +144,14 @@ else if (explanation != null) {
         List<ExplainSimplifiedResponse.MatchedTerm> terms = new ArrayList<>();
         List<ExplainSimplifiedResponse.MatchedFilter> filters = new ArrayList<>();
 
+        Double semanticScore = semanticScoreOf(scored);
+        Double nonSemanticScore = semanticScore != null && esRelevance != null
+                ? esRelevance - semanticScore
+                : null;
+        Double semanticContribute = semanticScore != null && esRelevance != null && esRelevance != 0.0
+                ? semanticScore / esRelevance
+                : null;
+
         collectScoreParts(scored, terms, filters);
         terms.sort(BY_SCORE_DESC);
 
@@ -146,6 +161,9 @@ else if (explanation != null) {
                 .title(stringField(hit.source(), StacBasicField.Title.searchField))
                 .finalScore(finalScore)
                 .esRelevance(esRelevance)
+                .semanticScore(semanticScore)
+                .nonSemanticScore(nonSemanticScore)
+                .semanticContribute(semanticContribute)
                 .internalScore(doubleField(hit.source(), StacSummeries.Score.searchField))
                 .qualityMultiplier(qualityMultiplier)
                 .matchedTerms(terms)
@@ -192,6 +210,13 @@ protected static void collectScoreParts(List<ExplanationDetail> details,
         for (ExplanationDetail detail : details) {
             String description = detail.description();
 
+            // A semantic_text score contains one leaf per sparse model feature. Those tokens are
+            // implementation details rather than user-readable query matches, so report the
+            // enclosing score once via semantic_score and keep the whole subtree out of filters.
+            if (isSemanticScoreNode(detail) || isSemanticEmbeddingDetail(detail)) {
+                continue;
+            }
+
             // most nodes are idf/tf breakdowns, skip the regex for them
             if (description != null && description.startsWith(WEIGHT_PREFIX)) {
                 Matcher matcher = WEIGHT_PATTERN.matcher(description);
@@ -223,6 +248,56 @@ protected static void collectScoreParts(List<ExplanationDetail> details,
         }
     }
 
+    /**
+     * Add the values of semantic_text score roots. There is normally one root because the search
+     * builds one semantic query, but summing keeps the simplifier correct if that changes later.
+     * A null result distinguishes "no semantic clause" from a semantic clause that scored zero.
+     */
+    protected static Double semanticScoreOf(List<ExplanationDetail> details) {
+        if (details == null) {
+            return null;
+        }
+
+        Double total = null;
+
+        for (ExplanationDetail detail : details) {
+            if (isSemanticScoreNode(detail)) {
+                total = (total == null ? 0.0 : total) + detail.value();
+                continue;
+            }
+
+            Double nested = semanticScoreOf(detail.details());
+            if (nested != null) {
+                total = (total == null ? 0.0 : total) + nested;
+            }
+        }
+
+        return total;
+    }
+
+    protected static boolean isSemanticScoreNode(ExplanationDetail detail) {
+        return detail.description() != null
+                && detail.description().startsWith(SEMANTIC_SCORE_PREFIX)
+                && containsSemanticEmbeddingDetail(detail);
+    }
+
+    protected static boolean containsSemanticEmbeddingDetail(ExplanationDetail detail) {
+        if (isSemanticEmbeddingDetail(detail)) {
+            return true;
+        }
+
+        if (detail.details() == null) {
+            return false;
+        }
+
+        return detail.details().stream().anyMatch(ExplainSimplifier::containsSemanticEmbeddingDetail);
+    }
+
+    protected static boolean isSemanticEmbeddingDetail(ExplanationDetail detail) {
+        return detail.description() != null
+                && detail.description().contains(SEMANTIC_EMBEDDING_MARKER);
+    }
+
     protected static Map<String, String> termsClausesOf(JsonNode request) {
         Map<String, String> byRendering = new HashMap<>();
         collectTermsClauses(request, byRendering);

server/src/main/resources/application.yaml

@@ -22,9 +22,8 @@ elasticsearch:
     search_suggestions:
       path: search_suggestions
       fields: abstract_phrases, parameter_vocabs_sayt, platform_vocabs_sayt, organisation_vocabs_sayt
-  # Semantic suggestions: the vocabs index is searched by meaning (ELSER via semantic_text).
-  # The suggested concepts are not filtered against records, so a suggestion may be a concept
-  # no record currently carries.
+  # Semantic features use ELSER via semantic_text. Suggestions search the vocabs index by meaning;
+  # free-text record searches use description_semantic only to re-rank keyword matches.
   # Off by default - semantic_text needs the licensed `inference` feature.
   semantic:
     enabled: false

server/src/test/java/au/org/aodn/ogcapi/server/core/util/ExplainSimplifierTest.java

@@ -1,13 +1,21 @@
 package au.org.aodn.ogcapi.server.core.util;
 
+import au.org.aodn.ogcapi.server.core.model.ExplainSimplifiedResponse;
+import co.elastic.clients.elasticsearch.core.explain.ExplanationDetail;
+import co.elastic.clients.elasticsearch.core.search.Hit;
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
 import org.junit.jupiter.api.Test;
 
+import java.util.ArrayList;
+import java.util.List;
 import java.util.Map;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
 
 public class ExplainSimplifierTest {
 
@@ -37,4 +45,77 @@ public void separateTermsValuesCommaSeparatesTheValuesOfTheClause() throws JsonP
                         "summaries.dataset_group:(csiro csiro temperature temperature)^100.0",
                         datasetGroupRequest()));
     }
+
+    @Test
+    public void semanticTextDetailsAreAggregatedWithoutExposingSparseFeatures() {
+        ExplanationDetail semantic = semanticScoreDetail(12.833507F);
+        List<ExplanationDetail> scored = List.of(
+                detail("summaries.dataset_group:(imos imos wave measurements)^100.0", 100.0F),
+                semantic,
+                detail("ConstantScore(spatial query)", 1.0F));
+
+        assertEquals(12.833507, ExplainSimplifier.semanticScoreOf(scored), 0.000001);
+
+        List<ExplainSimplifiedResponse.MatchedTerm> terms = new ArrayList<>();
+        List<ExplainSimplifiedResponse.MatchedFilter> filters = new ArrayList<>();
+        ExplainSimplifier.collectScoreParts(scored, terms, filters);
+
+        assertEquals(2, filters.size());
+        assertFalse(filters.stream()
+                .anyMatch(filter -> filter.getDescription().contains("embeddings field")));
+    }
+
+    @Test
+    public void simplifiedHitSeparatesSemanticFromOtherRelevance() {
+        ExplanationDetail score = ExplanationDetail.of(d -> d
+                .description("_score: ")
+                .value(113.833504F)
+                .details(
+                        detail("summaries.dataset_group:(imos imos wave measurements)^100.0", 100.0F),
+                        semanticScoreDetail(12.833507F),
+                        detail("ConstantScore(spatial query)", 1.0F)));
+
+        ObjectNode source = MAPPER.createObjectNode();
+        source.put("title", "Wave buoys Observations");
+        source.putObject("summaries").put("score", 144.0);
+
+        Hit<ObjectNode> hit = Hit.of(h -> h
+                .index("records")
+                .id("wave-record")
+                .score(112.27414)
+                .source(source)
+                .explanation(e -> e
+                        .description("sum of:")
+                        .value(112.27414F)
+                        .details(score)));
+
+        ExplainSimplifiedResponse.Hit simplified = ExplainSimplifier.toSimplifiedHit(hit, 1);
+
+        assertNotNull(simplified.getSemanticScore());
+        assertEquals(12.833507, simplified.getSemanticScore(), 0.000001);
+        assertEquals(101.0, simplified.getNonSemanticScore(), 0.00001);
+        assertEquals(12.833507 / 113.833504, simplified.getSemanticContribute(), 0.000001);
+        assertEquals(2, simplified.getFilters().size());
+    }
+
+    private ExplanationDetail semanticScoreDetail(float score) {
+        return ExplanationDetail.of(d -> d
+                .description("Score based on 2 child docs in range from 25 to 85, using score mode Max")
+                .value(score)
+                .details(child -> child
+                        .description("sum of:")
+                        .value(score)
+                        .details(feature -> feature
+                                .description("Linear function on the "
+                                        + "description_semantic.inference.chunks.embeddings field "
+                                        + "for the wave feature, computed as w * S from:")
+                                .value(score)
+                                .details(weight -> weight
+                                        .description("w, weight of this function")
+                                        .value(2.578F)))));
+    }
+
+    private ExplanationDetail detail(String description, float score) {
+        return ExplanationDetail.of(d -> d.description(description).value(score));
+    }
 }