Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
130 changes: 115 additions & 15 deletions src/Builders/PromptBuilder.php
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
use WordPress\AiClient\Providers\Models\DTO\ModelConfig;
use WordPress\AiClient\Providers\Models\DTO\ModelMetadata;
use WordPress\AiClient\Providers\Models\DTO\ModelRequirements;
use WordPress\AiClient\Providers\Models\DTO\RequiredOption;
use WordPress\AiClient\Providers\Models\Enums\CapabilityEnum;
use WordPress\AiClient\Providers\Models\ImageGeneration\Contracts\ImageGenerationModelInterface;
use WordPress\AiClient\Providers\Models\SpeechGeneration\Contracts\SpeechGenerationModelInterface;
Expand Down Expand Up @@ -1313,14 +1314,15 @@ protected function appendPartToMessages(MessagePart $part): void
/**
* Gets the model to use for generation.
*
* If a model has been explicitly set, validates it meets requirements and returns it.
* If a model has been explicitly set, it is used as-is without validating it against the prompt
* requirements; unsupported parameters surface as errors from the provider's API.
* Otherwise, finds a suitable model based on the prompt requirements.
*
* @since 0.1.0
*
* @param CapabilityEnum $capability The capability the model will be using.
* @return ModelInterface The model to use.
* @throws InvalidArgumentException If no suitable model is found or set model doesn't meet requirements.
* @throws InvalidArgumentException If no suitable model is found.
*/
private function getConfiguredModel(CapabilityEnum $capability): ModelInterface
{
Expand All @@ -1339,20 +1341,9 @@ private function getConfiguredModel(CapabilityEnum $capability): ModelInterface
$candidateMap = $this->getCandidateModelsMap($requirements);

if (empty($candidateMap)) {
$message = sprintf(
'No models found that support %s for this prompt.',
$capability->value
throw new InvalidArgumentException(
$this->getNoSuitableModelsMessage($capability, $requirements)
);

if ($this->providerIdOrClassName !== null) {
$message = sprintf(
'No models found for provider "%s" that support %s for this prompt.',
$this->providerIdOrClassName,
$capability->value
);
}

throw new InvalidArgumentException($message);
}

// Check if any preferred models match the candidates, in priority order.
Expand Down Expand Up @@ -1437,6 +1428,115 @@ private function getCandidateModelsMap(ModelRequirements $requirements): array
return $this->generateMapFromCandidates($providerId, $modelsMetadata);
}

/**
* Builds the exception message for when no models satisfy the prompt requirements.
*
* Distinguishes between no model supporting the required capability at all and models supporting
* the capability but not the required options. This avoids misleading messages where an
* unsupported option (e.g. a sampling parameter such as temperature) would otherwise be reported
* as the capability itself being unsupported.
*
* @since n.e.x.t
*
* @param CapabilityEnum $capability The capability the model must support.
* @param ModelRequirements $requirements The full requirements derived from the prompt.
* @return string The exception message.
*/
private function getNoSuitableModelsMessage(CapabilityEnum $capability, ModelRequirements $requirements): string
{
$scope = '';
if ($this->providerIdOrClassName !== null) {
$scope = sprintf(' for provider "%s"', $this->providerIdOrClassName);
}

$capabilityCandidates = [];
if (count($requirements->getRequiredOptions()) > 0) {
// Check whether models would qualify based on the required capabilities alone.
$capabilityCandidates = $this->findCandidateModelsMetadata(
new ModelRequirements($requirements->getRequiredCapabilities(), [])
);
}

if (count($capabilityCandidates) === 0) {
return sprintf(
'No models found%s that support %s for this prompt.',
$scope,
$capability->value
);
}

/*
* Models support the required capabilities, so the required options are what excluded them.
* Determine which option names are unmet by every candidate (definite blockers) and which
* are unmet by at least one candidate (relevant when only the combination is unsupported).
*/
$unmetByAll = null;
$unmetByAny = [];
foreach ($capabilityCandidates as $modelMetadata) {
$unmetNames = array_map(
static function (RequiredOption $option): string {
return $option->getName()->value;
},
$requirements->getUnmetRequiredOptions($modelMetadata)
);

$unmetByAny = array_merge($unmetByAny, $unmetNames);
$unmetByAll = $unmetByAll === null ? $unmetNames : array_intersect($unmetByAll, $unmetNames);
}
$unmetByAll = array_values(array_unique($unmetByAll));
$unmetByAny = array_values(array_unique($unmetByAny));

if (count($unmetByAll) > 0) {
$detail = sprintf(
'Models supporting %s are available, but none of them support the following required options: %s.',
$capability->value,
implode(', ', $unmetByAll)
);
} else {
$detail = sprintf(
'Models supporting %s are available, ' .
'but no single model supports all of the following required options together: %s.',
$capability->value,
implode(', ', $unmetByAny)
);
}

return sprintf(
'No models found%s that support %s with the required options for this prompt. %s',
$scope,
$capability->value,
$detail
);
}

/**
* Finds the metadata of all models that satisfy the given requirements.
*
* Honors the configured provider restriction, if any.
*
* @since n.e.x.t
*
* @param ModelRequirements $requirements The requirements to match against.
* @return list<ModelMetadata> The metadata of the matching models.
*/
private function findCandidateModelsMetadata(ModelRequirements $requirements): array
{
if ($this->providerIdOrClassName === null) {
$modelsMetadata = [];
foreach ($this->registry->findModelsMetadataForSupport($requirements) as $providerModelsMetadata) {
foreach ($providerModelsMetadata->getModels() as $modelMetadata) {
$modelsMetadata[] = $modelMetadata;
}
}
return $modelsMetadata;
}

return $this->registry->findProviderModelsMetadataForSupport(
$this->providerIdOrClassName,
$requirements
);
}

/**
* Generates a candidate map from model metadata with both provider-specific and model-only keys.
*
Expand Down
44 changes: 29 additions & 15 deletions src/Providers/Models/DTO/ModelRequirements.php
Original file line number Diff line number Diff line change
Expand Up @@ -101,17 +101,12 @@ public function getRequiredOptions(): array
*/
public function areMetBy(ModelMetadata $metadata): bool
{
// Create lookup maps for better performance (instead of nested foreach loops)
// Create lookup map for better performance (instead of nested foreach loops)
$capabilitiesMap = [];
foreach ($metadata->getSupportedCapabilities() as $capability) {
$capabilitiesMap[$capability->value] = $capability;
}

$optionsMap = [];
foreach ($metadata->getSupportedOptions() as $option) {
$optionsMap[$option->getName()->value] = $option;
}

// Check if all required capabilities are supported using map lookup
foreach ($this->requiredCapabilities as $requiredCapability) {
if (!isset($capabilitiesMap[$requiredCapability->value])) {
Expand All @@ -120,21 +115,40 @@ public function areMetBy(ModelMetadata $metadata): bool
}

// Check if all required options are supported with the specified values
return count($this->getUnmetRequiredOptions($metadata)) === 0;
}

/**
* Returns the required options that the given model metadata does not support.
*
* A required option is unmet when the model does not list it as a supported
* option at all, or when the required value is not among the option's
* supported values.
*
* @since n.e.x.t
*
* @param ModelMetadata $metadata The model metadata to check against.
* @return list<RequiredOption> The required options the model does not support.
*/
public function getUnmetRequiredOptions(ModelMetadata $metadata): array
{
// Create lookup map for better performance (instead of nested foreach loops)
$optionsMap = [];
foreach ($metadata->getSupportedOptions() as $option) {
$optionsMap[$option->getName()->value] = $option;
}

$unmetOptions = [];
foreach ($this->requiredOptions as $requiredOption) {
// Use map lookup instead of linear search
if (!isset($optionsMap[$requiredOption->getName()->value])) {
return false;
}
$supportedOption = $optionsMap[$requiredOption->getName()->value] ?? null;

$supportedOption = $optionsMap[$requiredOption->getName()->value];

// Check if the required value is supported by this option
if (!$supportedOption->isSupportedValue($requiredOption->getValue())) {
return false;
if ($supportedOption === null || !$supportedOption->isSupportedValue($requiredOption->getValue())) {
$unmetOptions[] = $requiredOption;
}
}

return true;
return $unmetOptions;
}

/**
Expand Down
129 changes: 127 additions & 2 deletions tests/unit/Builders/PromptBuilderTest.php
Original file line number Diff line number Diff line change
Expand Up @@ -3487,8 +3487,10 @@ public function testGenerateResultWithProviderClassName(): void
*/
public function testGenerateResultWithProviderNoModelsThrowsException(): void
{
// Mock the registry to return empty array when provider is specified
$this->registry->expects($this->once())
// Mock the registry to return empty array when provider is specified. Called twice:
// once with the full requirements, once with capability-only requirements while
// building the exception message.
$this->registry->expects($this->exactly(2))
->method('findProviderModelsMetadataForSupport')
->with('test-provider', $this->isInstanceOf(ModelRequirements::class))
->willReturn([]);
Expand All @@ -3504,6 +3506,129 @@ public function testGenerateResultWithProviderNoModelsThrowsException(): void
$builder->generateResult();
}

/**
* Tests generateResult names the unmet options when models support the capability itself.
*
* @return void
*/
public function testGenerateResultWithUnsupportedOptionNamesOptionInException(): void
{
$metadata = $this->createTextModelMetadataWithInputSupport('text-model');
$providerMetadata = new ProviderMetadata(
'test-provider',
'Test Provider',
ProviderTypeEnum::cloud()
);

$this->registry->expects($this->exactly(2))
->method('findModelsMetadataForSupport')
->with($this->isInstanceOf(ModelRequirements::class))
->willReturnOnConsecutiveCalls(
// No model matches the full requirements (including temperature).
[],
// A model matches the capability-only requirements.
[new ProviderModelsMetadata($providerMetadata, [$metadata])]
);

$builder = new PromptBuilder($this->registry, 'Test prompt');
$builder->usingTemperature(0.7);

$this->expectException(InvalidArgumentException::class);
$this->expectExceptionMessage(
'No models found that support text_generation with the required options for this prompt. '
. 'Models supporting text_generation are available, but none of them support the following '
. 'required options: temperature.'
);

$builder->generateTextResult();
}

/**
* Tests generateResult names the option combination when each option is only individually supported.
*
* @return void
*/
public function testGenerateResultWithUnsupportedOptionCombinationNamesOptionsInException(): void
{
$temperatureOnlyMetadata = new ModelMetadata(
'temperature-model',
'Temperature Model',
[CapabilityEnum::textGeneration()],
[
new SupportedOption(OptionEnum::inputModalities()),
new SupportedOption(OptionEnum::outputModalities()),
new SupportedOption(OptionEnum::temperature()),
]
);
$topPOnlyMetadata = new ModelMetadata(
'top-p-model',
'Top P Model',
[CapabilityEnum::textGeneration()],
[
new SupportedOption(OptionEnum::inputModalities()),
new SupportedOption(OptionEnum::outputModalities()),
new SupportedOption(OptionEnum::topP()),
]
);
$providerMetadata = new ProviderMetadata(
'test-provider',
'Test Provider',
ProviderTypeEnum::cloud()
);

$this->registry->expects($this->exactly(2))
->method('findModelsMetadataForSupport')
->with($this->isInstanceOf(ModelRequirements::class))
->willReturnOnConsecutiveCalls(
[],
[new ProviderModelsMetadata($providerMetadata, [$temperatureOnlyMetadata, $topPOnlyMetadata])]
);

$builder = new PromptBuilder($this->registry, 'Test prompt');
$builder->usingTemperature(0.7);
$builder->usingTopP(0.9);

$this->expectException(InvalidArgumentException::class);
$this->expectExceptionMessage(
'No models found that support text_generation with the required options for this prompt. '
. 'Models supporting text_generation are available, but no single model supports all of the '
. 'following required options together: topP, temperature.'
);

$builder->generateTextResult();
}

/**
* Tests generateResult names the unmet options for a provider-restricted prompt.
*
* @return void
*/
public function testGenerateResultWithProviderUnsupportedOptionNamesOptionInException(): void
{
$metadata = $this->createTextModelMetadataWithInputSupport('text-model');

$this->registry->expects($this->exactly(2))
->method('findProviderModelsMetadataForSupport')
->with('test-provider', $this->isInstanceOf(ModelRequirements::class))
->willReturnOnConsecutiveCalls(
[],
[$metadata]
);

$builder = new PromptBuilder($this->registry, 'Test prompt');
$builder->usingProvider('test-provider');
$builder->usingTemperature(0.7);

$this->expectException(InvalidArgumentException::class);
$this->expectExceptionMessage(
'No models found for provider "test-provider" that support text_generation with the required '
. 'options for this prompt. Models supporting text_generation are available, but none of them '
. 'support the following required options: temperature.'
);

$builder->generateResult();
}

/**
* Tests that provider takes precedence when both provider and model are set.
*
Expand Down
Loading
Loading