diff --git a/src/Builders/PromptBuilder.php b/src/Builders/PromptBuilder.php index 130fc574..48f83f31 100644 --- a/src/Builders/PromptBuilder.php +++ b/src/Builders/PromptBuilder.php @@ -23,6 +23,7 @@ use WordPress\AiClient\Providers\Models\DTO\ModelConfig; use WordPress\AiClient\Providers\Models\DTO\ModelMetadata; use WordPress\AiClient\Providers\Models\DTO\ModelRequirements; +use WordPress\AiClient\Providers\Models\DTO\RequiredOption; use WordPress\AiClient\Providers\Models\Enums\CapabilityEnum; use WordPress\AiClient\Providers\Models\ImageGeneration\Contracts\ImageGenerationModelInterface; use WordPress\AiClient\Providers\Models\SpeechGeneration\Contracts\SpeechGenerationModelInterface; @@ -1313,14 +1314,15 @@ protected function appendPartToMessages(MessagePart $part): void /** * Gets the model to use for generation. * - * If a model has been explicitly set, validates it meets requirements and returns it. + * If a model has been explicitly set, it is used as-is without validating it against the prompt + * requirements; unsupported parameters surface as errors from the provider's API. * Otherwise, finds a suitable model based on the prompt requirements. * * @since 0.1.0 * * @param CapabilityEnum $capability The capability the model will be using. * @return ModelInterface The model to use. - * @throws InvalidArgumentException If no suitable model is found or set model doesn't meet requirements. + * @throws InvalidArgumentException If no suitable model is found. */ private function getConfiguredModel(CapabilityEnum $capability): ModelInterface { @@ -1339,20 +1341,9 @@ private function getConfiguredModel(CapabilityEnum $capability): ModelInterface $candidateMap = $this->getCandidateModelsMap($requirements); if (empty($candidateMap)) { - $message = sprintf( - 'No models found that support %s for this prompt.', - $capability->value + throw new InvalidArgumentException( + $this->getNoSuitableModelsMessage($capability, $requirements) ); - - if ($this->providerIdOrClassName !== null) { - $message = sprintf( - 'No models found for provider "%s" that support %s for this prompt.', - $this->providerIdOrClassName, - $capability->value - ); - } - - throw new InvalidArgumentException($message); } // Check if any preferred models match the candidates, in priority order. @@ -1437,6 +1428,115 @@ private function getCandidateModelsMap(ModelRequirements $requirements): array return $this->generateMapFromCandidates($providerId, $modelsMetadata); } + /** + * Builds the exception message for when no models satisfy the prompt requirements. + * + * Distinguishes between no model supporting the required capability at all and models supporting + * the capability but not the required options. This avoids misleading messages where an + * unsupported option (e.g. a sampling parameter such as temperature) would otherwise be reported + * as the capability itself being unsupported. + * + * @since n.e.x.t + * + * @param CapabilityEnum $capability The capability the model must support. + * @param ModelRequirements $requirements The full requirements derived from the prompt. + * @return string The exception message. + */ + private function getNoSuitableModelsMessage(CapabilityEnum $capability, ModelRequirements $requirements): string + { + $scope = ''; + if ($this->providerIdOrClassName !== null) { + $scope = sprintf(' for provider "%s"', $this->providerIdOrClassName); + } + + $capabilityCandidates = []; + if (count($requirements->getRequiredOptions()) > 0) { + // Check whether models would qualify based on the required capabilities alone. + $capabilityCandidates = $this->findCandidateModelsMetadata( + new ModelRequirements($requirements->getRequiredCapabilities(), []) + ); + } + + if (count($capabilityCandidates) === 0) { + return sprintf( + 'No models found%s that support %s for this prompt.', + $scope, + $capability->value + ); + } + + /* + * Models support the required capabilities, so the required options are what excluded them. + * Determine which option names are unmet by every candidate (definite blockers) and which + * are unmet by at least one candidate (relevant when only the combination is unsupported). + */ + $unmetByAll = null; + $unmetByAny = []; + foreach ($capabilityCandidates as $modelMetadata) { + $unmetNames = array_map( + static function (RequiredOption $option): string { + return $option->getName()->value; + }, + $requirements->getUnmetRequiredOptions($modelMetadata) + ); + + $unmetByAny = array_merge($unmetByAny, $unmetNames); + $unmetByAll = $unmetByAll === null ? $unmetNames : array_intersect($unmetByAll, $unmetNames); + } + $unmetByAll = array_values(array_unique($unmetByAll)); + $unmetByAny = array_values(array_unique($unmetByAny)); + + if (count($unmetByAll) > 0) { + $detail = sprintf( + 'Models supporting %s are available, but none of them support the following required options: %s.', + $capability->value, + implode(', ', $unmetByAll) + ); + } else { + $detail = sprintf( + 'Models supporting %s are available, ' . + 'but no single model supports all of the following required options together: %s.', + $capability->value, + implode(', ', $unmetByAny) + ); + } + + return sprintf( + 'No models found%s that support %s with the required options for this prompt. %s', + $scope, + $capability->value, + $detail + ); + } + + /** + * Finds the metadata of all models that satisfy the given requirements. + * + * Honors the configured provider restriction, if any. + * + * @since n.e.x.t + * + * @param ModelRequirements $requirements The requirements to match against. + * @return list The metadata of the matching models. + */ + private function findCandidateModelsMetadata(ModelRequirements $requirements): array + { + if ($this->providerIdOrClassName === null) { + $modelsMetadata = []; + foreach ($this->registry->findModelsMetadataForSupport($requirements) as $providerModelsMetadata) { + foreach ($providerModelsMetadata->getModels() as $modelMetadata) { + $modelsMetadata[] = $modelMetadata; + } + } + return $modelsMetadata; + } + + return $this->registry->findProviderModelsMetadataForSupport( + $this->providerIdOrClassName, + $requirements + ); + } + /** * Generates a candidate map from model metadata with both provider-specific and model-only keys. * diff --git a/src/Providers/Models/DTO/ModelRequirements.php b/src/Providers/Models/DTO/ModelRequirements.php index 7b92d804..987fb69b 100644 --- a/src/Providers/Models/DTO/ModelRequirements.php +++ b/src/Providers/Models/DTO/ModelRequirements.php @@ -101,17 +101,12 @@ public function getRequiredOptions(): array */ public function areMetBy(ModelMetadata $metadata): bool { - // Create lookup maps for better performance (instead of nested foreach loops) + // Create lookup map for better performance (instead of nested foreach loops) $capabilitiesMap = []; foreach ($metadata->getSupportedCapabilities() as $capability) { $capabilitiesMap[$capability->value] = $capability; } - $optionsMap = []; - foreach ($metadata->getSupportedOptions() as $option) { - $optionsMap[$option->getName()->value] = $option; - } - // Check if all required capabilities are supported using map lookup foreach ($this->requiredCapabilities as $requiredCapability) { if (!isset($capabilitiesMap[$requiredCapability->value])) { @@ -120,21 +115,40 @@ public function areMetBy(ModelMetadata $metadata): bool } // Check if all required options are supported with the specified values + return count($this->getUnmetRequiredOptions($metadata)) === 0; + } + + /** + * Returns the required options that the given model metadata does not support. + * + * A required option is unmet when the model does not list it as a supported + * option at all, or when the required value is not among the option's + * supported values. + * + * @since n.e.x.t + * + * @param ModelMetadata $metadata The model metadata to check against. + * @return list The required options the model does not support. + */ + public function getUnmetRequiredOptions(ModelMetadata $metadata): array + { + // Create lookup map for better performance (instead of nested foreach loops) + $optionsMap = []; + foreach ($metadata->getSupportedOptions() as $option) { + $optionsMap[$option->getName()->value] = $option; + } + + $unmetOptions = []; foreach ($this->requiredOptions as $requiredOption) { // Use map lookup instead of linear search - if (!isset($optionsMap[$requiredOption->getName()->value])) { - return false; - } + $supportedOption = $optionsMap[$requiredOption->getName()->value] ?? null; - $supportedOption = $optionsMap[$requiredOption->getName()->value]; - - // Check if the required value is supported by this option - if (!$supportedOption->isSupportedValue($requiredOption->getValue())) { - return false; + if ($supportedOption === null || !$supportedOption->isSupportedValue($requiredOption->getValue())) { + $unmetOptions[] = $requiredOption; } } - return true; + return $unmetOptions; } /** diff --git a/tests/unit/Builders/PromptBuilderTest.php b/tests/unit/Builders/PromptBuilderTest.php index ce68a223..a345a95c 100644 --- a/tests/unit/Builders/PromptBuilderTest.php +++ b/tests/unit/Builders/PromptBuilderTest.php @@ -3487,8 +3487,10 @@ public function testGenerateResultWithProviderClassName(): void */ public function testGenerateResultWithProviderNoModelsThrowsException(): void { - // Mock the registry to return empty array when provider is specified - $this->registry->expects($this->once()) + // Mock the registry to return empty array when provider is specified. Called twice: + // once with the full requirements, once with capability-only requirements while + // building the exception message. + $this->registry->expects($this->exactly(2)) ->method('findProviderModelsMetadataForSupport') ->with('test-provider', $this->isInstanceOf(ModelRequirements::class)) ->willReturn([]); @@ -3504,6 +3506,129 @@ public function testGenerateResultWithProviderNoModelsThrowsException(): void $builder->generateResult(); } + /** + * Tests generateResult names the unmet options when models support the capability itself. + * + * @return void + */ + public function testGenerateResultWithUnsupportedOptionNamesOptionInException(): void + { + $metadata = $this->createTextModelMetadataWithInputSupport('text-model'); + $providerMetadata = new ProviderMetadata( + 'test-provider', + 'Test Provider', + ProviderTypeEnum::cloud() + ); + + $this->registry->expects($this->exactly(2)) + ->method('findModelsMetadataForSupport') + ->with($this->isInstanceOf(ModelRequirements::class)) + ->willReturnOnConsecutiveCalls( + // No model matches the full requirements (including temperature). + [], + // A model matches the capability-only requirements. + [new ProviderModelsMetadata($providerMetadata, [$metadata])] + ); + + $builder = new PromptBuilder($this->registry, 'Test prompt'); + $builder->usingTemperature(0.7); + + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage( + 'No models found that support text_generation with the required options for this prompt. ' + . 'Models supporting text_generation are available, but none of them support the following ' + . 'required options: temperature.' + ); + + $builder->generateTextResult(); + } + + /** + * Tests generateResult names the option combination when each option is only individually supported. + * + * @return void + */ + public function testGenerateResultWithUnsupportedOptionCombinationNamesOptionsInException(): void + { + $temperatureOnlyMetadata = new ModelMetadata( + 'temperature-model', + 'Temperature Model', + [CapabilityEnum::textGeneration()], + [ + new SupportedOption(OptionEnum::inputModalities()), + new SupportedOption(OptionEnum::outputModalities()), + new SupportedOption(OptionEnum::temperature()), + ] + ); + $topPOnlyMetadata = new ModelMetadata( + 'top-p-model', + 'Top P Model', + [CapabilityEnum::textGeneration()], + [ + new SupportedOption(OptionEnum::inputModalities()), + new SupportedOption(OptionEnum::outputModalities()), + new SupportedOption(OptionEnum::topP()), + ] + ); + $providerMetadata = new ProviderMetadata( + 'test-provider', + 'Test Provider', + ProviderTypeEnum::cloud() + ); + + $this->registry->expects($this->exactly(2)) + ->method('findModelsMetadataForSupport') + ->with($this->isInstanceOf(ModelRequirements::class)) + ->willReturnOnConsecutiveCalls( + [], + [new ProviderModelsMetadata($providerMetadata, [$temperatureOnlyMetadata, $topPOnlyMetadata])] + ); + + $builder = new PromptBuilder($this->registry, 'Test prompt'); + $builder->usingTemperature(0.7); + $builder->usingTopP(0.9); + + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage( + 'No models found that support text_generation with the required options for this prompt. ' + . 'Models supporting text_generation are available, but no single model supports all of the ' + . 'following required options together: topP, temperature.' + ); + + $builder->generateTextResult(); + } + + /** + * Tests generateResult names the unmet options for a provider-restricted prompt. + * + * @return void + */ + public function testGenerateResultWithProviderUnsupportedOptionNamesOptionInException(): void + { + $metadata = $this->createTextModelMetadataWithInputSupport('text-model'); + + $this->registry->expects($this->exactly(2)) + ->method('findProviderModelsMetadataForSupport') + ->with('test-provider', $this->isInstanceOf(ModelRequirements::class)) + ->willReturnOnConsecutiveCalls( + [], + [$metadata] + ); + + $builder = new PromptBuilder($this->registry, 'Test prompt'); + $builder->usingProvider('test-provider'); + $builder->usingTemperature(0.7); + + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage( + 'No models found for provider "test-provider" that support text_generation with the required ' + . 'options for this prompt. Models supporting text_generation are available, but none of them ' + . 'support the following required options: temperature.' + ); + + $builder->generateResult(); + } + /** * Tests that provider takes precedence when both provider and model are set. * diff --git a/tests/unit/Providers/Models/DTO/ModelRequirementsTest.php b/tests/unit/Providers/Models/DTO/ModelRequirementsTest.php index 1cb7897e..57afdc40 100644 --- a/tests/unit/Providers/Models/DTO/ModelRequirementsTest.php +++ b/tests/unit/Providers/Models/DTO/ModelRequirementsTest.php @@ -493,6 +493,60 @@ public function testAreMetByWithNoRequirements(): void $this->assertTrue($requirements->areMetBy($metadata)); } + /** + * Tests getUnmetRequiredOptions returns the options the model does not support. + * + * @return void + */ + public function testGetUnmetRequiredOptionsReturnsUnsupportedOptions(): void + { + $missingOption = new RequiredOption(OptionEnum::topP(), 0.9); + $unsupportedValueOption = new RequiredOption(OptionEnum::temperature(), 0.5); + $supportedOption = new RequiredOption(OptionEnum::maxTokens(), 1000); + + $requirements = new ModelRequirements( + [CapabilityEnum::textGeneration()], + [$missingOption, $unsupportedValueOption, $supportedOption] + ); + + $metadata = $this->createMock(ModelMetadata::class); + $metadata->method('getSupportedCapabilities')->willReturn([ + CapabilityEnum::textGeneration() + ]); + $metadata->method('getSupportedOptions')->willReturn([ + new SupportedOption(OptionEnum::temperature(), [0.1, 0.7, 1.0]), + new SupportedOption(OptionEnum::maxTokens(), [500, 1000, 2000]) + ]); + + $this->assertSame( + [$missingOption, $unsupportedValueOption], + $requirements->getUnmetRequiredOptions($metadata) + ); + } + + /** + * Tests getUnmetRequiredOptions returns an empty list when all options are supported. + * + * @return void + */ + public function testGetUnmetRequiredOptionsWithAllOptionsSupported(): void + { + $requirements = new ModelRequirements( + [CapabilityEnum::textGeneration()], + [new RequiredOption(OptionEnum::temperature(), 0.7)] + ); + + $metadata = $this->createMock(ModelMetadata::class); + $metadata->method('getSupportedCapabilities')->willReturn([ + CapabilityEnum::textGeneration() + ]); + $metadata->method('getSupportedOptions')->willReturn([ + new SupportedOption(OptionEnum::temperature(), [0.1, 0.7, 1.0]) + ]); + + $this->assertSame([], $requirements->getUnmetRequiredOptions($metadata)); + } + /** * Tests fromPromptData method with simple text generation. *