From 94869c5508f6a26cb8a79947f3748980726aa5a9 Mon Sep 17 00:00:00 2001 From: Lukas Schaefer Date: Tue, 4 Aug 2026 11:06:00 -0400 Subject: [PATCH 1/3] Add image editing support Signed-off-by: Lukas Schaefer Co-authored-by: Cursor --- lib/AppInfo/Application.php | 5 + lib/Service/OpenAiAPIService.php | 207 +++++++++++++++++- lib/Service/ServiceConfig.php | 4 + lib/TaskProcessing/ImageToImageProvider.php | 226 ++++++++++++++++++++ lib/TaskProcessing/ImageToImageTaskType.php | 77 +++++++ lib/TaskProcessing/ProviderFactory.php | 4 + tests/unit/Providers/OpenAiProviderTest.php | 73 +++++++ tests/unit/Service/MultiServiceTest.php | 149 +++++++++++++ 8 files changed, 739 insertions(+), 6 deletions(-) create mode 100644 lib/TaskProcessing/ImageToImageProvider.php create mode 100644 lib/TaskProcessing/ImageToImageTaskType.php diff --git a/lib/AppInfo/Application.php b/lib/AppInfo/Application.php index ff2983ea..c34ddfe6 100644 --- a/lib/AppInfo/Application.php +++ b/lib/AppInfo/Application.php @@ -10,6 +10,7 @@ use OCA\OpenAi\Capabilities; use OCA\OpenAi\Listener\TaskProcessingProviderListener; use OCA\OpenAi\Notification\Notifier; +use OCA\OpenAi\TaskProcessing\ImageToImageTaskType; use OCP\AppFramework\App; use OCP\AppFramework\Bootstrap\IBootContext; use OCP\AppFramework\Bootstrap\IBootstrap; @@ -103,6 +104,10 @@ public function register(IRegistrationContext $context): void { // classes. They are built per (service, model, task type) instead. $context->registerEventListener(GetTaskProcessingProvidersEvent::class, TaskProcessingProviderListener::class); + if (!class_exists('OCP\\TaskProcessing\\TaskTypes\\ImageToImage')) { + $context->registerTaskProcessingTaskType(ImageToImageTaskType::class); + } + $context->registerCapability(Capabilities::class); $context->registerNotifierService(Notifier::class); } diff --git a/lib/Service/OpenAiAPIService.php b/lib/Service/OpenAiAPIService.php index 097731c9..ecae6f31 100644 --- a/lib/Service/OpenAiAPIService.php +++ b/lib/Service/OpenAiAPIService.php @@ -936,6 +936,197 @@ public function requestImageCreation( return $apiResponse; } + /** + * @param string|null $userId + * @param string $prompt + * @param list $images + * @param string $model + * @param string $size + * @return array + * @throws Exception + * @throws UserFacingProcessingException + */ + public function requestImageEdit( + ?string $userId, + ServiceConfig $service, + string $prompt, + array $images, + string $model, + string $size = Application::DEFAULT_DEFAULT_IMAGE_SIZE, + ): array { + if ($this->isQuotaExceeded($userId, Application::QUOTA_TYPE_IMAGE, $service)) { + throw new Exception($this->l10n->t('Image generation quota exceeded'), Http::STATUS_TOO_MANY_REQUESTS); + } + + $apiModel = $this->modelParam($service, $model, Application::DEFAULT_IMAGE_MODEL_ID) ?? $model; + + if ($service->isUsingOpenAi()) { + $apiResponse = $this->requestOpenAiImageEdit($userId, $service, $prompt, $images, $apiModel, $size); + } elseif ($service->isUsingOpenRouter()) { + $apiResponse = $this->requestOpenRouterImageEdit($userId, $service, $prompt, $images, $apiModel, $size); + } elseif ($service->isUsingIonos()) { + $apiResponse = $this->requestIonosImageEdit($userId, $service, $prompt, $images, $apiModel, $size); + } else { + $apiResponse = $this->requestLocalAiImageEdit($userId, $service, $prompt, $images, $apiModel, $size); + } + + if (!isset($apiResponse['data']) || !is_array($apiResponse['data'])) { + $this->logger->warning('OpenAI image edit error', ['api_response' => $apiResponse]); + throw new Exception($this->l10n->t('Unknown image generation error'), Http::STATUS_INTERNAL_SERVER_ERROR); + } + + try { + $this->createQuotaUsage($userId ?? '', Application::QUOTA_TYPE_IMAGE, 1, $service); + } catch (DBException $e) { + $this->logger->warning('Could not create quota usage for user: ' . $userId . ' and quota type: ' . Application::QUOTA_TYPE_IMAGE . '. Error: ' . $e->getMessage(), ['app' => Application::APP_ID]); + } + + return $apiResponse; + } + + /** + * @param list $images + * @return array + * @throws Exception + */ + private function requestOpenAiImageEdit( + ?string $userId, + ServiceConfig $service, + string $prompt, + array $images, + string $model, + string $size, + ): array { + $params = [ + 'prompt' => $prompt, + 'size' => $size, + 'n' => 1, + 'model' => $model, + ]; + foreach ($images as $index => $image) { + $mimeType = $image['mimeType']; + $extension = match ($mimeType) { + 'image/jpeg' => 'jpg', + 'image/webp' => 'webp', + 'image/gif' => 'gif', + default => 'png', + }; + $name = 'image_' . ($index); + $params[$name] = [ + 'name' => 'image[]', + 'contents' => $image['content'], + 'filename' => $name . '.' . $extension, + 'headers' => [ + 'Content-Type' => $mimeType, + ], + ]; + } + + return $this->request($userId, $service, 'images/edits', $params, 'POST', 'multipart/form-data'); + } + + /** + * @param list $images + * @return array + * @throws Exception + * @throws UserFacingProcessingException + */ + private function requestIonosImageEdit( + ?string $userId, + ServiceConfig $service, + string $prompt, + array $images, + string $model, + string $size, + ): array { + if (count($images) > 1) { + throw new UserFacingProcessingException( + 'IONOS image editing supports only one input image', + 0, + null, + $this->l10n->t('Only one input image is supported.'), + ); + } + + $image = $images[0]; + $params = [ + 'prompt' => $prompt, + 'size' => $size, + 'n' => 1, + 'model' => $model, + 'url' => 'data:' . $image['mimeType'] . ';base64,' . base64_encode($image['content']), + ]; + + return $this->request($userId, $service, 'images/edits', $params, 'POST', 'multipart/form-data'); + } + + /** + * OpenRouter image edit path using the unified /images API with input_references. + * + * @param list $images + * @return array + * @throws Exception + */ + private function requestOpenRouterImageEdit( + ?string $userId, + ServiceConfig $service, + string $prompt, + array $images, + string $model, + string $size, + ): array { + $inputReferences = []; + foreach ($images as $image) { + $inputReferences[] = [ + 'type' => 'image_url', + 'image_url' => [ + 'url' => 'data:' . $image['mimeType'] . ';base64,' . base64_encode($image['content']), + ], + ]; + } + + $params = [ + 'prompt' => $prompt, + 'size' => $size, + 'n' => 1, + 'model' => $model, + 'input_references' => $inputReferences, + ]; + + return $this->request($userId, $service, 'images', $params, 'POST'); + } + + /** + * LocalAI and other OpenAI-compatible image edit path via /images/generations. + * + * @param list $images + * @return array + * @throws Exception + */ + private function requestLocalAiImageEdit( + ?string $userId, + ServiceConfig $service, + string $prompt, + array $images, + string $model, + string $size, + ): array { + $refImages = []; + foreach ($images as $image) { + $refImages[] = base64_encode($image['content']); + } + + $params = [ + 'prompt' => $prompt, + 'size' => $size, + 'n' => 1, + 'model' => $model, + 'ref_images' => $refImages, + ]; + + return $this->request($userId, $service, 'images/generations', $params, 'POST'); + } + /** * @param string|null $userId * @return array @@ -1153,12 +1344,16 @@ public function request( if ($contentType === 'multipart/form-data') { $multipart = []; foreach ($params as $key => $value) { - $part = [ - 'name' => $key, - 'contents' => $value, - ]; - if ($key === 'file') { - $part['filename'] = 'file.mp3'; + if (is_array($value) && array_key_exists('contents', $value)) { + $part = $value; + } else { + $part = [ + 'name' => $key, + 'contents' => $value, + ]; + if ($key === 'file') { + $part['filename'] = 'file.mp3'; + } } $multipart[] = $part; } diff --git a/lib/Service/ServiceConfig.php b/lib/Service/ServiceConfig.php index 62906b93..3d57d181 100644 --- a/lib/Service/ServiceConfig.php +++ b/lib/Service/ServiceConfig.php @@ -277,6 +277,10 @@ public function isUsingMistral(): bool { return str_starts_with(strtolower($this->url), 'https://api.mistral.ai'); } + public function isUsingIonos(): bool { + return (bool)preg_match('#^https://([a-zA-Z0-9-]+\.)+ionos\.com#', strtolower($this->getRequestUrl())); + } + public function getApiKey(): string { return $this->apiKey; } diff --git a/lib/TaskProcessing/ImageToImageProvider.php b/lib/TaskProcessing/ImageToImageProvider.php new file mode 100644 index 00000000..5487e624 --- /dev/null +++ b/lib/TaskProcessing/ImageToImageProvider.php @@ -0,0 +1,226 @@ +buildProviderId(); + } + + public function getName(): string { + return $this->buildProviderName(); + } + + public function getTaskTypeId(): string { + if (class_exists('OCP\\TaskProcessing\\TaskTypes\\ImageToImage')) { + return \OCP\TaskProcessing\TaskTypes\ImageToImage::ID; + } + return ImageToImageTaskType::ID; + } + + public function getExpectedRuntime(): int { + return $this->openAiAPIService->getExpImgProcessingTime($this->service); + } + + public function getInputShapeEnumValues(): array { + return []; + } + + public function getInputShapeDefaults(): array { + return []; + } + + public function getOptionalInputShape(): array { + $defaultImageSize = $this->service->getDefaultImageSize(); + return [ + 'size' => new ShapeDescriptor( + $this->l->t('Size'), + $this->l->t('Optional. The size of the generated images. Must be in 256x256 format. Default is %s', [$defaultImageSize]), + EShapeType::Text + ), + ]; + } + + public function getOptionalInputShapeEnumValues(): array { + return []; + } + + public function getOptionalInputShapeDefaults(): array { + return []; + } + + public function getOutputShapeEnumValues(): array { + return []; + } + + public function getOptionalOutputShape(): array { + return []; + } + + public function getOptionalOutputShapeEnumValues(): array { + return []; + } + + public function process( + ?string $userId, + array $input, + callable $reportProgress, + SynchronousProviderOptions $options = new SynchronousProviderOptions(), + ): array { + $startTime = time(); + $includeWatermark = $options->getIncludeWatermarks(); + + if (!isset($input['input']) || !is_array($input['input']) || $input['input'] === []) { + throw new ProcessingException('Invalid input files'); + } + + if (!isset($input['prompt']) || !is_string($input['prompt'])) { + throw new ProcessingException('Invalid prompt'); + } + $prompt = $input['prompt']; + + $images = []; + + if (count($input['input']) > 16) { + throw new UserFacingProcessingException( + 'Too many input images. Max is 16', + 0, + null, + $this->l->t('Cannot use more than 16 input images.'), + ); + } + + foreach ($input['input'] as $inputFile) { + if (!$inputFile instanceof File || !$inputFile->isReadable()) { + throw new ProcessingException('Invalid input file'); + } + if ($inputFile->getSize() > self::MAX_FILE_SIZE_BYTES) { + throw new UserFacingProcessingException( + 'Filesize of input file too large. Max is 25MB', + 0, + null, + $this->l->t('The size of the input file is too large. A maximum of 25MB is allowed.'), + ); + } + + $mimeType = $inputFile->getMimeType(); + if (!in_array($mimeType, self::VALID_IMAGE_MIME_TYPES, true)) { + throw new UserFacingProcessingException( + 'Invalid input file type for OpenAI ' . $mimeType, + 0, + null, + $this->l->t('Invalid input file type "%1$s".', [$mimeType]), + ); + } + + $images[] = [ + 'content' => $inputFile->getContent(), + 'mimeType' => $mimeType, + ]; + } + + $size = $this->service->getDefaultImageSize(); + if (isset($input['size']) && is_string($input['size']) && preg_match('/^\d+x\d+$/', $input['size'])) { + $size = trim($input['size']); + } + if (preg_match('/^\d+x\d+$/', $size) !== 1) { + $size = Application::DEFAULT_DEFAULT_IMAGE_SIZE; + } + [$x, $y] = explode('x', $size, 2); + if ((int)$x > 4096 || (int)$y > 4096) { + throw new UserFacingProcessingException('size is out of bounds', userFacingMessage: $this->l->t('Cannot generate images larger than 4096x4096')); + } + + try { + $apiResponse = $this->openAiAPIService->requestImageEdit( + $userId, + $this->service, + $prompt, + $images, + $this->model, + $size, + ); + $b64s = array_map(static function (array $result) { + return $result['b64_json'] ?? null; + }, $apiResponse['data']); + $b64s = array_values(array_filter($b64s, static function (?string $b64) { + return $b64 !== null; + })); + + $urls = array_map(static function (array $result) { + return $result['url'] ?? null; + }, $apiResponse['data']); + $urls = array_values(array_filter($urls, static function (?string $url) { + return $url !== null; + })); + + if (empty($urls) && empty($b64s)) { + $this->logger->warning('OpenAI/LocalAI\'s image to image generation failed: no image returned'); + throw new ProcessingException('OpenAI/LocalAI\'s image to image generation failed: no image returned'); + } + + $image = null; + if (!empty($urls)) { + $client = $this->clientService->newClient(); + $requestOptions = $this->openAiAPIService->getImageRequestOptions($userId, $this->service); + $imageResponse = $client->get($urls[0], $requestOptions); + $image = $imageResponse->getBody(); + } else { + $image = base64_decode($b64s[0]); + } + + $image = $includeWatermark ? $this->watermarkingService->markImage($image) : $image; + $endTime = time(); + $this->openAiAPIService->updateExpImgProcessingTime($endTime - $startTime, $this->service); + return ['output' => $image]; + } catch (UserFacingProcessingException $e) { + throw $e; + } catch (\Throwable $e) { + $this->logger->warning('OpenAI/LocalAI\'s image to image generation failed with: ' . $e->getMessage(), ['exception' => $e]); + throw new ProcessingException('OpenAI/LocalAI\'s image to image generation failed with: ' . $e->getMessage()); + } + } +} diff --git a/lib/TaskProcessing/ImageToImageTaskType.php b/lib/TaskProcessing/ImageToImageTaskType.php new file mode 100644 index 00000000..c19a8651 --- /dev/null +++ b/lib/TaskProcessing/ImageToImageTaskType.php @@ -0,0 +1,77 @@ +l->t('Edit image'); + } + + /** + * @inheritDoc + */ + public function getDescription(): string { + return $this->l->t('Edit an image based on a text description of the changes'); + } + + /** + * @return string + */ + public function getId(): string { + return self::ID; + } + + /** + * @return ShapeDescriptor[] + */ + public function getInputShape(): array { + return [ + 'input' => new ShapeDescriptor( + $this->l->t('Input images'), + $this->l->t('The images to edit'), + EShapeType::ListOfImages + ), + 'prompt' => new ShapeDescriptor( + $this->l->t('Prompt'), + $this->l->t('Describe the changes you want to make to the image'), + EShapeType::Text + ), + ]; + } + + /** + * @return ShapeDescriptor[] + */ + public function getOutputShape(): array { + return [ + 'output' => new ShapeDescriptor( + $this->l->t('Output image'), + $this->l->t('The edited image'), + EShapeType::Image + ), + ]; + } +} diff --git a/lib/TaskProcessing/ProviderFactory.php b/lib/TaskProcessing/ProviderFactory.php index 762df420..43a2792e 100644 --- a/lib/TaskProcessing/ProviderFactory.php +++ b/lib/TaskProcessing/ProviderFactory.php @@ -143,6 +143,10 @@ private function getImageProviders(ServiceConfig $service, string $model): array $this->logger, $this->l, $this->openAiAPIService, $service, ); } + $providers[] = new ImageToImageProvider( + $this->openAiAPIService, $this->l, $this->logger, $this->clientService, + $this->watermarkingService, $service, $model, + ); return $providers; } diff --git a/tests/unit/Providers/OpenAiProviderTest.php b/tests/unit/Providers/OpenAiProviderTest.php index 710ce32e..5c5f9d02 100644 --- a/tests/unit/Providers/OpenAiProviderTest.php +++ b/tests/unit/Providers/OpenAiProviderTest.php @@ -28,6 +28,7 @@ use OCA\OpenAi\TaskProcessing\ChangeToneProvider; use OCA\OpenAi\TaskProcessing\EmojiProvider; use OCA\OpenAi\TaskProcessing\HeadlineProvider; +use OCA\OpenAi\TaskProcessing\ImageToImageProvider; use OCA\OpenAi\TaskProcessing\MultimodalChatWithToolsProvider; use OCA\OpenAi\TaskProcessing\ProofreadProvider; use OCA\OpenAi\TaskProcessing\ReformatParagraphsProvider; @@ -970,6 +971,78 @@ public function testTextToImageProvider(): void { $this->quotaUsageMapper->deleteUserQuotaUsages(self::TEST_USER1); } + public function testImageToImageProvider(): void { + $imageToImageProvider = new ImageToImageProvider( + $this->openAiApiService, + $this->createMock(\OCP\IL10N::class), + $this->createMock(\Psr\Log\LoggerInterface::class), + \OCP\Server::get(IClientService::class), + \OCP\Server::get(WatermarkingService::class), + $this->service, + self::IMAGE_MODEL, + ); + + $inputImage = file_get_contents(__DIR__ . '/../../res/trees.jpg'); + if (!$inputImage) { + throw new \RuntimeException('Could not read test resource `trees.jpg`'); + } + + $file = $this->createMock(\OCP\Files\File::class); + $file->method('isReadable')->willReturn(true); + $file->method('getContent')->willReturn($inputImage); + $file->method('getSize')->willReturn(strlen($inputImage)); + $file->method('getMimeType')->willReturn('image/jpeg'); + + $prompt = 'Make the sky blue'; + $response = json_encode([ + 'data' => [ + [ + 'b64_json' => base64_encode($inputImage), + ] + ] + ]); + + $url = self::OPENAI_API_BASE . 'images/edits'; + $options = [ + 'timeout' => Application::OPENAI_DEFAULT_REQUEST_TIMEOUT, + 'headers' => [ + 'User-Agent' => Application::USER_AGENT, + 'Authorization' => self::AUTHORIZATION_HEADER, + ], + 'multipart' => [ + ['name' => 'prompt', 'contents' => $prompt], + ['name' => 'size', 'contents' => '1024x1024'], + ['name' => 'n', 'contents' => 1], + ['name' => 'model', 'contents' => Application::DEFAULT_IMAGE_MODEL_ID], + [ + 'name' => 'image[]', + 'contents' => $inputImage, + 'filename' => 'image_0.jpg', + 'headers' => ['Content-Type' => 'image/jpeg'], + ], + ], + ]; + + $iResponse = $this->createMock(\OCP\Http\Client\IResponse::class); + $iResponse->method('getHeader')->with('Content-Type')->willReturn('application/json'); + $iResponse->method('getBody')->willReturn($response); + $iResponse->method('getStatusCode')->willReturn(200); + + $this->iClient->expects($this->once())->method('post')->with($url, $options)->willReturn($iResponse); + + $result = $imageToImageProvider->process( + self::TEST_USER1, + ['input' => [$file], 'prompt' => $prompt], + fn () => true, + ); + $this->assertArrayHasKey('output', $result); + $this->assertEquals($inputImage, $result['output']); + + $usage = $this->quotaUsageMapper->getQuotaUnitsOfUser(self::TEST_USER1, Application::QUOTA_TYPE_IMAGE); + $this->assertEquals(1, $usage); + $this->quotaUsageMapper->deleteUserQuotaUsages(self::TEST_USER1); + } + public function testReformatParagraphsProvider(): void { if (!class_exists(TextToTextReformatParagraphs::class)) { $this->markTestSkipped('TextToTextReformatParagraphs task type is not available in this Nextcloud version.'); diff --git a/tests/unit/Service/MultiServiceTest.php b/tests/unit/Service/MultiServiceTest.php index 4de5d509..7077ecef 100644 --- a/tests/unit/Service/MultiServiceTest.php +++ b/tests/unit/Service/MultiServiceTest.php @@ -23,6 +23,7 @@ use OCA\OpenAi\Service\StreamingService; use OCA\OpenAi\Service\WatermarkingService; use OCA\OpenAi\TaskProcessing\AudioToTextProvider; +use OCA\OpenAi\TaskProcessing\ImageToImageProvider; use OCA\OpenAi\TaskProcessing\ProviderFactory; use OCA\OpenAi\TaskProcessing\TextToImageProvider; use OCA\OpenAi\TaskProcessing\TextToSpeechProvider; @@ -249,6 +250,154 @@ public function testTextToImageProvider(): void { $textToImageProvider->process(self::TEST_USER1, ['input' => $inputText, 'numberOfImages' => 1], fn () => null); } + public function testImageToImageProvider(): void { + $service = $this->addService([ + 'url' => self::IMAGE_BASE, + 'api_key' => self::APIKEY_IMAGE, + 'request_timeout' => self::REQUEST_TIMEOUT_IMAGE, + 'image_models' => [self::IMAGE_MODEL], + ]); + + $imageToImageProvider = new ImageToImageProvider( + $this->openAiApiService, + $this->createMock(\OCP\IL10N::class), + $this->createMock(\Psr\Log\LoggerInterface::class), + \OCP\Server::get(IClientService::class), + \OCP\Server::get(WatermarkingService::class), + $service, + self::IMAGE_MODEL, + ); + + $inputImage = file_get_contents(__DIR__ . '/../../res/trees.jpg'); + if (!$inputImage) { + throw new \RuntimeException('Could not read test resource `trees.jpg`'); + } + + $file = $this->createMock(\OCP\Files\File::class); + $file->method('isReadable')->willReturn(true); + $file->method('getContent')->willReturn($inputImage); + $file->method('getSize')->willReturn(strlen($inputImage)); + $file->method('getMimeType')->willReturn('image/jpeg'); + + $prompt = 'Make the sky blue'; + $response = json_encode([ + 'data' => [ + [ + 'b64_json' => base64_encode($inputImage), + ] + ] + ]); + + $url = self::IMAGE_BASE . '/images/generations'; + $options = [ + 'timeout' => self::REQUEST_TIMEOUT_IMAGE, + 'headers' => [ + 'User-Agent' => Application::USER_AGENT, + 'Authorization' => 'Bearer ' . self::APIKEY_IMAGE, + 'Content-Type' => 'application/json', + ], + 'nextcloud' => ['allow_local_address' => true], + 'body' => json_encode([ + 'prompt' => $prompt, + 'size' => '1024x1024', + 'n' => 1, + 'model' => self::IMAGE_MODEL, + 'ref_images' => [base64_encode($inputImage)], + ]), + ]; + + $iResponse = $this->createMock(\OCP\Http\Client\IResponse::class); + $iResponse->method('getHeader')->with('Content-Type')->willReturn('application/json'); + $iResponse->method('getBody')->willReturn($response); + $iResponse->method('getStatusCode')->willReturn(200); + + $this->iClient->expects($this->once())->method('post')->with($url, $options)->willReturn($iResponse); + + $imageToImageProvider->process( + self::TEST_USER1, + ['input' => [$file], 'prompt' => $prompt], + fn () => null, + ); + } + + public function testImageToImageProviderOpenRouter(): void { + $openRouterBase = 'https://openrouter.ai/api/v1/'; + $service = $this->addService([ + 'url' => $openRouterBase, + 'api_key' => self::APIKEY_IMAGE, + 'request_timeout' => self::REQUEST_TIMEOUT_IMAGE, + 'image_models' => [self::IMAGE_MODEL], + ]); + + $imageToImageProvider = new ImageToImageProvider( + $this->openAiApiService, + $this->createMock(\OCP\IL10N::class), + $this->createMock(\Psr\Log\LoggerInterface::class), + \OCP\Server::get(IClientService::class), + \OCP\Server::get(WatermarkingService::class), + $service, + self::IMAGE_MODEL, + ); + + $inputImage = file_get_contents(__DIR__ . '/../../res/trees.jpg'); + if (!$inputImage) { + throw new \RuntimeException('Could not read test resource `trees.jpg`'); + } + + $file = $this->createMock(\OCP\Files\File::class); + $file->method('isReadable')->willReturn(true); + $file->method('getContent')->willReturn($inputImage); + $file->method('getSize')->willReturn(strlen($inputImage)); + $file->method('getMimeType')->willReturn('image/jpeg'); + + $prompt = 'Make the sky blue'; + $response = json_encode([ + 'data' => [ + [ + 'b64_json' => base64_encode($inputImage), + ] + ] + ]); + + $url = $openRouterBase . 'images'; + $options = [ + 'timeout' => self::REQUEST_TIMEOUT_IMAGE, + 'headers' => [ + 'User-Agent' => Application::USER_AGENT, + 'Authorization' => 'Bearer ' . self::APIKEY_IMAGE, + 'Content-Type' => 'application/json', + ], + 'nextcloud' => ['allow_local_address' => true], + 'body' => json_encode([ + 'prompt' => $prompt, + 'size' => '1024x1024', + 'n' => 1, + 'model' => self::IMAGE_MODEL, + 'input_references' => [ + [ + 'type' => 'image_url', + 'image_url' => [ + 'url' => 'data:image/jpeg;base64,' . base64_encode($inputImage), + ], + ], + ], + ]), + ]; + + $iResponse = $this->createMock(\OCP\Http\Client\IResponse::class); + $iResponse->method('getHeader')->with('Content-Type')->willReturn('application/json'); + $iResponse->method('getBody')->willReturn($response); + $iResponse->method('getStatusCode')->willReturn(200); + + $this->iClient->expects($this->once())->method('post')->with($url, $options)->willReturn($iResponse); + + $imageToImageProvider->process( + self::TEST_USER1, + ['input' => [$file], 'prompt' => $prompt], + fn () => null, + ); + } + public function testAudioToTextProvider(): void { $service = $this->addService([ 'url' => self::TRANSCRIPTION_BASE, From 6dba13743171b339306dbfbaf396912e1f0e263d Mon Sep 17 00:00:00 2001 From: Lukas Schaefer Date: Tue, 15 Sep 2026 11:13:30 -0400 Subject: [PATCH 2/3] Review fixes Signed-off-by: Lukas Schaefer --- lib/Service/OpenAiAPIService.php | 74 ++++++++++++++++----- lib/Service/ServiceConfig.php | 2 +- lib/TaskProcessing/ImageToImageProvider.php | 16 +++++ lib/TaskProcessing/ProviderIdentity.php | 16 +++-- tests/unit/Service/MultiServiceTest.php | 14 +++- 5 files changed, 99 insertions(+), 23 deletions(-) diff --git a/lib/Service/OpenAiAPIService.php b/lib/Service/OpenAiAPIService.php index ecae6f31..d222efe8 100644 --- a/lib/Service/OpenAiAPIService.php +++ b/lib/Service/OpenAiAPIService.php @@ -936,6 +936,41 @@ public function requestImageCreation( return $apiResponse; } + /** + * @param ServiceConfig $service + * @return bool + */ + public function isLocalAIService(ServiceConfig $service): bool { + $serviceUrl = $service->getUrl(); + $url = rtrim($serviceUrl, '/'); + // LocalAI urls always have a v1 + if (!str_ends_with($url, '/v1')) { + return false; + } + $cacheKey = 'localai_service_' . base64_encode($url); + $wellKnownUrl = substr($url, 0, -2) . '.well-known/localai.json'; + $cache = $this->cacheFactory->createLocal(); + $result = $cache->get($cacheKey); + if ($result !== null) { + return $result; + } + $this->logger->debug('Checking if service is a LocalAI service at URL: ' . $url, ['app' => Application::APP_ID]); + try { + $wellKnownService = $this->client->get($wellKnownUrl, ['http_errors' => false, 'nextcloud' => ['allow_local_address' => true]]); + if ($wellKnownService->getStatusCode() !== 200) { + $result = false; + } else { + $jsonResponse = json_decode($wellKnownService->getBody(), true); + $result = $jsonResponse !== null; + } + } catch (Exception $e) { + $this->logger->warning('Could not check if service is a LocalAI service at URL: ' . $url . '. Error: ' . $e->getMessage(), ['app' => Application::APP_ID]); + $result = false; + } + $cache->set($cacheKey, $result); + return $result; + } + /** * @param string|null $userId * @param string $prompt @@ -958,16 +993,17 @@ public function requestImageEdit( throw new Exception($this->l10n->t('Image generation quota exceeded'), Http::STATUS_TOO_MANY_REQUESTS); } - $apiModel = $this->modelParam($service, $model, Application::DEFAULT_IMAGE_MODEL_ID) ?? $model; + $modelParam = $this->modelParam($service, $model, Application::DEFAULT_IMAGE_MODEL_ID); - if ($service->isUsingOpenAi()) { - $apiResponse = $this->requestOpenAiImageEdit($userId, $service, $prompt, $images, $apiModel, $size); - } elseif ($service->isUsingOpenRouter()) { - $apiResponse = $this->requestOpenRouterImageEdit($userId, $service, $prompt, $images, $apiModel, $size); + if ($service->isUsingOpenRouter()) { + $apiResponse = $this->requestOpenRouterImageEdit($userId, $service, $prompt, $images, $modelParam, $size); } elseif ($service->isUsingIonos()) { - $apiResponse = $this->requestIonosImageEdit($userId, $service, $prompt, $images, $apiModel, $size); + $apiResponse = $this->requestIonosImageEdit($userId, $service, $prompt, $images, $modelParam, $size); + } elseif ($this->isLocalAIService($service)) { + $apiResponse = $this->requestLocalAiImageEdit($userId, $service, $prompt, $images, $modelParam, $size); } else { - $apiResponse = $this->requestLocalAiImageEdit($userId, $service, $prompt, $images, $apiModel, $size); + // Default to OpenAI + $apiResponse = $this->requestOpenAiImageEdit($userId, $service, $prompt, $images, $modelParam, $size); } if (!isset($apiResponse['data']) || !is_array($apiResponse['data'])) { @@ -994,15 +1030,17 @@ private function requestOpenAiImageEdit( ServiceConfig $service, string $prompt, array $images, - string $model, + ?string $model, string $size, ): array { $params = [ 'prompt' => $prompt, 'size' => $size, 'n' => 1, - 'model' => $model, ]; + if ($model !== null) { + $params['model'] = $model; + } foreach ($images as $index => $image) { $mimeType = $image['mimeType']; $extension = match ($mimeType) { @@ -1036,7 +1074,7 @@ private function requestIonosImageEdit( ServiceConfig $service, string $prompt, array $images, - string $model, + ?string $model, string $size, ): array { if (count($images) > 1) { @@ -1053,9 +1091,11 @@ private function requestIonosImageEdit( 'prompt' => $prompt, 'size' => $size, 'n' => 1, - 'model' => $model, 'url' => 'data:' . $image['mimeType'] . ';base64,' . base64_encode($image['content']), ]; + if ($model !== null) { + $params['model'] = $model; + } return $this->request($userId, $service, 'images/edits', $params, 'POST', 'multipart/form-data'); } @@ -1072,7 +1112,7 @@ private function requestOpenRouterImageEdit( ServiceConfig $service, string $prompt, array $images, - string $model, + ?string $model, string $size, ): array { $inputReferences = []; @@ -1089,9 +1129,11 @@ private function requestOpenRouterImageEdit( 'prompt' => $prompt, 'size' => $size, 'n' => 1, - 'model' => $model, 'input_references' => $inputReferences, ]; + if ($model !== null) { + $params['model'] = $model; + } return $this->request($userId, $service, 'images', $params, 'POST'); } @@ -1108,7 +1150,7 @@ private function requestLocalAiImageEdit( ServiceConfig $service, string $prompt, array $images, - string $model, + ?string $model, string $size, ): array { $refImages = []; @@ -1120,9 +1162,11 @@ private function requestLocalAiImageEdit( 'prompt' => $prompt, 'size' => $size, 'n' => 1, - 'model' => $model, 'ref_images' => $refImages, ]; + if ($model !== null) { + $params['model'] = $model; + } return $this->request($userId, $service, 'images/generations', $params, 'POST'); } diff --git a/lib/Service/ServiceConfig.php b/lib/Service/ServiceConfig.php index 3d57d181..0bd8b7bf 100644 --- a/lib/Service/ServiceConfig.php +++ b/lib/Service/ServiceConfig.php @@ -278,7 +278,7 @@ public function isUsingMistral(): bool { } public function isUsingIonos(): bool { - return (bool)preg_match('#^https://([a-zA-Z0-9-]+\.)+ionos\.com#', strtolower($this->getRequestUrl())); + return str_starts_with(strtolower($this->url), 'https://openai.inference.de-txl.ionos.com'); } public function getApiKey(): string { diff --git a/lib/TaskProcessing/ImageToImageProvider.php b/lib/TaskProcessing/ImageToImageProvider.php index 5487e624..f836ce76 100644 --- a/lib/TaskProcessing/ImageToImageProvider.php +++ b/lib/TaskProcessing/ImageToImageProvider.php @@ -133,6 +133,22 @@ public function process( ); } + $fileSizeTotal = array_reduce( + $input['input'], + function ($carry, $file) { + return $carry + (method_exists($file, 'getSize') ? $file->getSize() : 0); + }, + 0 + ); + if ($fileSizeTotal > 50 * 1000 * 1000) { + throw new UserFacingProcessingException( + 'Filesize of input files too large. Max is 50MB', + 0, + null, + $this->l->t('The total size of the input files is too large. A maximum of 50MB is allowed.'), + ); + } + foreach ($input['input'] as $inputFile) { if (!$inputFile instanceof File || !$inputFile->isReadable()) { throw new ProcessingException('Invalid input file'); diff --git a/lib/TaskProcessing/ProviderIdentity.php b/lib/TaskProcessing/ProviderIdentity.php index 20fc54a4..c5f35d95 100644 --- a/lib/TaskProcessing/ProviderIdentity.php +++ b/lib/TaskProcessing/ProviderIdentity.php @@ -54,12 +54,18 @@ public static function slugifyModel(string $model): string { } /** - * The task type ID as it appears in a provider ID: the `core:` prefix of - * the task types the server ships is dropped, because every provider ID - * already says which app it belongs to. The prefix of a task type defined - * by another app is kept, so that two of them cannot collide. + * Formats the task type ID for inclusion in a provider ID. + * The `core:` prefix (for server-shipped task types) and this app's prefix are removed, + * since the provider ID already includes the app context. Prefixes from other apps are retained + * to avoid ID collisions in case of similarly named task types across apps. */ public static function slugifyTaskType(string $taskTypeId): string { - return str_starts_with($taskTypeId, 'core:') ? substr($taskTypeId, 5) : $taskTypeId; + if (str_starts_with($taskTypeId, Application::APP_ID . ':')) { + return substr($taskTypeId, strlen(Application::APP_ID . ':')); + } + if (str_starts_with($taskTypeId, 'core:')) { + return substr($taskTypeId, 5); + } + return $taskTypeId; } } diff --git a/tests/unit/Service/MultiServiceTest.php b/tests/unit/Service/MultiServiceTest.php index 7077ecef..f1cb0228 100644 --- a/tests/unit/Service/MultiServiceTest.php +++ b/tests/unit/Service/MultiServiceTest.php @@ -288,6 +288,16 @@ public function testImageToImageProvider(): void { ] ]); + $wellKnownUrl = substr(self::IMAGE_BASE, 0, -2) . '.well-known/localai.json'; + $wellKnownResponse = $this->createMock(\OCP\Http\Client\IResponse::class); + $wellKnownResponse->method('getBody')->willReturn('{"version":"1.0"}'); + $wellKnownResponse->method('getStatusCode')->willReturn(200); + + $this->iClient->expects($this->once())->method('get')->with( + $wellKnownUrl, + ['http_errors' => false, 'nextcloud' => ['allow_local_address' => true]], + )->willReturn($wellKnownResponse); + $url = self::IMAGE_BASE . '/images/generations'; $options = [ 'timeout' => self::REQUEST_TIMEOUT_IMAGE, @@ -301,8 +311,8 @@ public function testImageToImageProvider(): void { 'prompt' => $prompt, 'size' => '1024x1024', 'n' => 1, - 'model' => self::IMAGE_MODEL, 'ref_images' => [base64_encode($inputImage)], + 'model' => self::IMAGE_MODEL, ]), ]; @@ -372,7 +382,6 @@ public function testImageToImageProviderOpenRouter(): void { 'prompt' => $prompt, 'size' => '1024x1024', 'n' => 1, - 'model' => self::IMAGE_MODEL, 'input_references' => [ [ 'type' => 'image_url', @@ -381,6 +390,7 @@ public function testImageToImageProviderOpenRouter(): void { ], ], ], + 'model' => self::IMAGE_MODEL, ]), ]; From 03fc51d80d30d2feb55b021228cda498cac87006 Mon Sep 17 00:00:00 2001 From: Lukas Schaefer Date: Wed, 16 Sep 2026 10:07:28 -0400 Subject: [PATCH 3/3] Validate sizes for edit image Signed-off-by: Lukas Schaefer --- lib/TaskProcessing/ImageToImageProvider.php | 95 +++++++- .../ImageToImageProviderSizeTest.php | 221 ++++++++++++++++++ 2 files changed, 305 insertions(+), 11 deletions(-) create mode 100644 tests/unit/Providers/ImageToImageProviderSizeTest.php diff --git a/lib/TaskProcessing/ImageToImageProvider.php b/lib/TaskProcessing/ImageToImageProvider.php index f836ce76..205dd138 100644 --- a/lib/TaskProcessing/ImageToImageProvider.php +++ b/lib/TaskProcessing/ImageToImageProvider.php @@ -178,17 +178,7 @@ function ($carry, $file) { ]; } - $size = $this->service->getDefaultImageSize(); - if (isset($input['size']) && is_string($input['size']) && preg_match('/^\d+x\d+$/', $input['size'])) { - $size = trim($input['size']); - } - if (preg_match('/^\d+x\d+$/', $size) !== 1) { - $size = Application::DEFAULT_DEFAULT_IMAGE_SIZE; - } - [$x, $y] = explode('x', $size, 2); - if ((int)$x > 4096 || (int)$y > 4096) { - throw new UserFacingProcessingException('size is out of bounds', userFacingMessage: $this->l->t('Cannot generate images larger than 4096x4096')); - } + $size = $this->resolveSize($input); try { $apiResponse = $this->openAiAPIService->requestImageEdit( @@ -239,4 +229,87 @@ function ($carry, $file) { throw new ProcessingException('OpenAI/LocalAI\'s image to image generation failed with: ' . $e->getMessage()); } } + + /** + * @param array $input + */ + private function resolveSize(array $input): string { + $size = $this->service->getDefaultImageSize(); + if (isset($input['size']) && is_string($input['size']) && preg_match('/^\d+x\d+$/', $input['size']) === 1) { + $size = trim($input['size']); + } + if (preg_match('/^\d+x\d+$/', $size) !== 1) { + $size = Application::DEFAULT_DEFAULT_IMAGE_SIZE; + } + + [$widthStr, $heightStr] = explode('x', $size, 2); + $width = (int)$widthStr; + $height = (int)$heightStr; + $this->validateSize($size, $width, $height); + + return $size; + } + + private function validateSize(string $size, int $width, int $height): void { + // https://api.ionos.com/docs/inference-openai/v1/ + $ionosMinDimension = 64; + $ionosMaxDimension = 2048; + $ionosDimensionMultiple = 16; + // https://platform.openai.com/docs/api-reference/images/createEdit + $openAiImageSizes = [ + '1024x1024', + '1024x1536', + '1536x1024' + ]; + $maxDimension = 4096; + + if ($this->service->isUsingIonos()) { + $valid = $width >= $ionosMinDimension + && $height >= $ionosMinDimension + && $width <= $ionosMaxDimension + && $height <= $ionosMaxDimension + && $width % $ionosDimensionMultiple === 0 + && $height % $ionosDimensionMultiple === 0; + if (!$valid) { + throw new UserFacingProcessingException( + 'size is out of bounds', + 0, + null, + $this->l->t( + 'Image size must use dimensions that are multiples of %1$d and between %2$d and %3$d.', + [ + $ionosDimensionMultiple, + $ionosMinDimension, + $ionosMaxDimension, + ], + ), + ); + } + return; + } + + if ($this->service->isUsingOpenAi()) { + if (!in_array($size, $openAiImageSizes, true)) { + throw new UserFacingProcessingException( + 'size is out of bounds', + 0, + null, + $this->l->t( + 'Image size must be one of: %s', + [implode(', ', $openAiImageSizes)], + ), + ); + } + return; + } + + if ($width > $maxDimension || $height > $maxDimension) { + throw new UserFacingProcessingException( + 'size is out of bounds', + 0, + null, + $this->l->t('Cannot generate images larger than %1$dx%2$d', [$maxDimension, $maxDimension]), + ); + } + } } diff --git a/tests/unit/Providers/ImageToImageProviderSizeTest.php b/tests/unit/Providers/ImageToImageProviderSizeTest.php new file mode 100644 index 00000000..cc3eda50 --- /dev/null +++ b/tests/unit/Providers/ImageToImageProviderSizeTest.php @@ -0,0 +1,221 @@ +openAiAPIService = $this->createMock(OpenAiAPIService::class); + $this->l10n = $this->createMock(IL10N::class); + $this->l10n->method('t')->willReturnCallback( + static fn (string $text, array $parameters = []): string => vsprintf(str_replace('%%', '%', preg_replace('/%\d+\$/', '%', $text) ?? $text), $parameters) + ); + } + + /** + * @dataProvider gptImage1InvalidSizeProvider + */ + public function testGptImage1RejectsInvalidSize(string $size): void { + $provider = $this->createProvider( + ServiceConfig::fromArray('openai', []), + Application::DEFAULT_IMAGE_MODEL_ID, + ); + + $this->openAiAPIService->expects($this->never())->method('requestImageEdit'); + $this->expectException(UserFacingProcessingException::class); + + $provider->process('user', $this->validInput(['size' => $size]), static fn () => null); + } + + /** + * @return list + */ + public static function gptImage1InvalidSizeProvider(): array { + return [ + ['512x512'], + ['2048x2048'], + ['1024x1792'], + ['1792x1024'], + ['256x256'], + ]; + } + + /** + * @dataProvider gptImage1ValidSizeProvider + */ + public function testGptImage1AcceptsValidSize(string $size): void { + $provider = $this->createProvider( + ServiceConfig::fromArray('openai', []), + 'gpt-image-1', + ); + + $this->openAiAPIService->expects($this->once()) + ->method('requestImageEdit') + ->with( + 'user', + $this->isInstanceOf(ServiceConfig::class), + 'edit me', + $this->isType('array'), + 'gpt-image-1', + $size, + ) + ->willReturn(['data' => [['b64_json' => base64_encode('img')]]]); + + $result = $provider->process('user', $this->validInput(['size' => $size]), static fn () => null); + $this->assertSame(['output' => 'img'], $result); + } + + /** + * @return list + */ + public static function gptImage1ValidSizeProvider(): array { + return [ + ['1024x1024'], + ['1024x1536'], + ['1536x1024'], + ]; + } + + /** + * @dataProvider ionosInvalidSizeProvider + */ + public function testIonosRejectsInvalidSize(string $size): void { + $provider = $this->createProvider( + ServiceConfig::fromArray('ionos', [ + 'url' => 'https://openai.inference.de-txl.ionos.com/v1', + ]), + 'black-forest-labs/FLUX.2-Klein-4b', + ); + + $this->openAiAPIService->expects($this->never())->method('requestImageEdit'); + $this->expectException(UserFacingProcessingException::class); + + $provider->process('user', $this->validInput(['size' => $size]), static fn () => null); + } + + /** + * @return list + */ + public static function ionosInvalidSizeProvider(): array { + return [ + ['2049x1024'], + ['1024x2049'], + ['1025x1024'], + ['1000x1000'], + ['32x32'], + ['4096x4096'], + ]; + } + + /** + * @dataProvider ionosValidSizeProvider + */ + public function testIonosAcceptsValidSize(string $size): void { + $provider = $this->createProvider( + ServiceConfig::fromArray('ionos', [ + 'url' => 'https://openai.inference.de-txl.ionos.com/v1', + ]), + 'black-forest-labs/FLUX.2-Klein-4b', + ); + + $this->openAiAPIService->expects($this->once()) + ->method('requestImageEdit') + ->with( + 'user', + $this->isInstanceOf(ServiceConfig::class), + 'edit me', + $this->isType('array'), + 'black-forest-labs/FLUX.2-Klein-4b', + $size, + ) + ->willReturn(['data' => [['b64_json' => base64_encode('img')]]]); + + $result = $provider->process('user', $this->validInput(['size' => $size]), static fn () => null); + $this->assertSame(['output' => 'img'], $result); + } + + /** + * @return list + */ + public static function ionosValidSizeProvider(): array { + return [ + ['1024x1024'], + ['2048x2048'], + ['2048x1152'], + ['64x64'], + ['1536x1024'], + ]; + } + + public function testOtherServicesKeepMaxDimensionLimit(): void { + $provider = $this->createProvider( + ServiceConfig::fromArray('localai', [ + 'url' => 'http://localhost:8080/v1', + ]), + 'my-image-model', + ); + + $this->openAiAPIService->expects($this->never())->method('requestImageEdit'); + $this->expectException(UserFacingProcessingException::class); + + $provider->process('user', $this->validInput(['size' => '4097x1024']), static fn () => null); + } + + private function createProvider(ServiceConfig $service, string $model): ImageToImageProvider { + $watermarking = $this->createMock(WatermarkingService::class); + $watermarking->method('markImage')->willReturnArgument(0); + + return new ImageToImageProvider( + $this->openAiAPIService, + $this->l10n, + $this->createMock(LoggerInterface::class), + $this->createMock(IClientService::class), + $watermarking, + $service, + $model, + ); + } + + /** + * @param array $overrides + * @return array + */ + private function validInput(array $overrides = []): array { + $file = $this->createMock(File::class); + $file->method('isReadable')->willReturn(true); + $file->method('getContent')->willReturn('png-bytes'); + $file->method('getSize')->willReturn(9); + $file->method('getMimeType')->willReturn('image/png'); + + return array_merge([ + 'input' => [$file], + 'prompt' => 'edit me', + ], $overrides); + } +}