From 628d5d66cb9d0c5b108e16eec1fbd05937989dd6 Mon Sep 17 00:00:00 2001 From: Fernando Date: Fri, 30 Jan 2026 23:29:27 +0100 Subject: [PATCH 1/2] Improve LLM prompts with Ogre3D material reference and available textures - Enhanced system prompt with comprehensive Ogre3D material attributes: - Pass attributes (ambient, diffuse, specular, emissive, lighting, blending, etc.) - Texture unit attributes (texture, filtering, addressing, transforms, etc.) - Multiple examples (textured, transparent, animated materials) - Added available textures context to prompts: - Gets list of textures from Ogre's TextureManager - Filters out internal Ogre textures (RTT, Ogre/, etc.) - Instructs LLM to only use textures from the available list - Limits to 50 textures to avoid prompt bloat - Improved retry logic to include texture list on validation failures This helps the LLM generate valid material scripts that reference actual project textures rather than non-existent files. Co-Authored-By: Claude Opus 4.5 --- src/LLMManager.cpp | 202 ++++++++++++++++++++++++++++---------- src/LLMManager.h | 4 +- src/MaterialEditorQML.cpp | 5 +- 3 files changed, 159 insertions(+), 52 deletions(-) diff --git a/src/LLMManager.cpp b/src/LLMManager.cpp index 4b11b59..569c545 100644 --- a/src/LLMManager.cpp +++ b/src/LLMManager.cpp @@ -230,47 +230,110 @@ bool LLMManager::isGenerating() const QString LLMManager::getOgre3DSystemPrompt() { - return R"(You generate Ogre3D material scripts. + return R"(You are an expert Ogre3D material script generator. Output ONLY valid material scripts. + +STRUCTURE: +material NAME { technique { pass { texture_unit { } } } } + +PASS ATTRIBUTES (all colors are R G B values 0.0-1.0): +- ambient R G B [A] : Ambient color (default: 1.0 1.0 1.0) +- diffuse R G B [A] : Diffuse color (default: 1.0 1.0 1.0) +- specular R G B [A] SHINE : Specular color + shininess 1-128 (default: 0 0 0 0) +- emissive R G B [A] : Self-illumination (default: 0 0 0) +- lighting on|off : Enable dynamic lighting (default: on) +- shading flat|gouraud|phong : Shading mode (default: gouraud) +- scene_blend : Blending: add, modulate, alpha_blend, colour_blend +- scene_blend : Custom blend: one, zero, src_alpha, one_minus_src_alpha, dest_colour, etc. +- depth_check on|off : Depth buffer check (default: on) +- depth_write on|off : Write to depth buffer (default: on) +- depth_func : always_fail, always_pass, less, less_equal, equal, not_equal, greater_equal, greater +- cull_hardware clockwise|anticlockwise|none : Back-face culling (default: clockwise) +- polygon_mode solid|wireframe|points + +TEXTURE_UNIT ATTRIBUTES: +- texture [2d|cubic] : Texture file to use +- filtering none|bilinear|trilinear|anisotropic +- tex_address_mode wrap|clamp|mirror|border +- scroll : Static UV offset +- scroll_anim : Animated scrolling +- rotate : Static rotation +- rotate_anim : Animated rotation +- scale : UV scaling +- colour_op replace|add|modulate|alpha_blend +- colour_op_ex [manual_factor] [manual_color] +- env_map off|spherical|planar|cubic_reflection|cubic_normal +- alpha_rejection greater_equal 128 : Alpha testing + +COLOUR_OP_EX operations: add, subtract, modulate, modulate_x2, modulate_x4, add_signed, add_smooth, blend_diffuse_alpha, blend_texture_alpha, blend_current_alpha, blend_manual +COLOUR_OP_EX sources: src_current, src_texture, src_diffuse, src_specular, src_manual RULES: -1. Output ONLY the material script, nothing else -2. Keep the same material name if one is provided -3. All color values must be NUMBERS between 0.0 and 1.0 -4. NO WORDS in property values - only numbers +1. Output ONLY the material script - no markdown, no explanations +2. Keep the same material name if modifying an existing material +3. All color/numeric values must be NUMBERS (0.0 to 1.0 for colors) +4. If textures are needed, ONLY use textures from the provided available list +5. Ensure balanced braces - every { must have a matching } -SYNTAX: -material NAME +EXAMPLE - Textured material with glow: +material Example/GlowingTextured { technique { pass { - ambient R G B - diffuse R G B - specular R G B SHININESS - emissive R G B + ambient 0.3 0.3 0.3 + diffuse 0.8 0.8 0.8 + specular 1.0 1.0 1.0 64 + emissive 0.2 0.1 0.0 + + texture_unit + { + texture myTexture.png + filtering trilinear + } } } } -COLORS (use these exact values): -- green: ambient 0.0 0.2 0.0 diffuse 0.0 1.0 0.0 specular 0.5 0.5 0.5 32 -- red: ambient 0.2 0.0 0.0 diffuse 1.0 0.0 0.0 specular 0.5 0.5 0.5 32 -- blue: ambient 0.0 0.0 0.2 diffuse 0.0 0.0 1.0 specular 0.5 0.5 0.5 32 -- yellow: ambient 0.2 0.2 0.0 diffuse 1.0 1.0 0.0 specular 0.5 0.5 0.5 32 -- white: ambient 0.2 0.2 0.2 diffuse 1.0 1.0 1.0 specular 0.5 0.5 0.5 32 +EXAMPLE - Transparent material: +material Example/Transparent +{ + technique + { + pass + { + scene_blend alpha_blend + depth_write off + + diffuse 1.0 1.0 1.0 0.5 + + texture_unit + { + texture glass.png + } + } + } +} -EXAMPLE - green material: -material MyMaterial +EXAMPLE - Animated water: +material Example/Water { technique { pass { - ambient 0.0 0.2 0.0 - diffuse 0.0 1.0 0.0 - specular 0.5 0.5 0.5 32 - emissive 0.0 0.0 0.0 + ambient 0.1 0.2 0.4 + diffuse 0.3 0.5 0.8 + specular 1.0 1.0 1.0 128 + scene_blend alpha_blend + depth_write off + + texture_unit + { + texture water.png + scroll_anim 0.05 0.02 + wave_xform scale_x sine 1.0 0.1 0.0 0.5 + } } } })"; @@ -393,7 +456,7 @@ void LLMManager::scanForModels() emit availableModelsChanged(); } -void LLMManager::generateMaterial(const QString &prompt, const QString ¤tMaterial) +void LLMManager::generateMaterial(const QString &prompt, const QString ¤tMaterial, const QStringList &availableTextures) { if (!isModelLoaded()) { emit generationError("No model loaded. Please load a model first."); @@ -403,20 +466,70 @@ void LLMManager::generateMaterial(const QString &prompt, const QString ¤tM // Store for potential retry m_pendingPrompt = prompt; m_pendingCurrentMaterial = currentMaterial; + m_pendingAvailableTextures = availableTextures; m_retryCount = 0; + QString userPrompt = buildUserPrompt(prompt, currentMaterial, availableTextures); + + QMetaObject::invokeMethod(m_worker, [this, userPrompt]() { + m_worker->generate(getOgre3DSystemPrompt(), userPrompt); + }, Qt::QueuedConnection); +} + +QString LLMManager::buildUserPrompt(const QString &prompt, const QString ¤tMaterial, const QStringList &availableTextures) const +{ QString userPrompt; + + // Add available textures section if any exist + QString texturesSection; + if (!availableTextures.isEmpty()) { + // Filter to only include image-like textures (exclude internal Ogre textures) + QStringList filteredTextures; + for (const QString &tex : availableTextures) { + // Skip internal Ogre textures and non-image resources + if (!tex.startsWith("Ogre/") && + !tex.startsWith("RTT") && + !tex.startsWith("__") && + !tex.contains("RenderTarget") && + (tex.endsWith(".png", Qt::CaseInsensitive) || + tex.endsWith(".jpg", Qt::CaseInsensitive) || + tex.endsWith(".jpeg", Qt::CaseInsensitive) || + tex.endsWith(".dds", Qt::CaseInsensitive) || + tex.endsWith(".tga", Qt::CaseInsensitive) || + tex.endsWith(".bmp", Qt::CaseInsensitive) || + // Also include textures without extensions (might be valid) + !tex.contains("."))) { + filteredTextures.append(tex); + } + } + + if (!filteredTextures.isEmpty()) { + // Limit to reasonable number to avoid prompt bloat + int maxTextures = qMin(filteredTextures.size(), 50); + QStringList limitedTextures = filteredTextures.mid(0, maxTextures); + + texturesSection = QString("\nAVAILABLE TEXTURES (use ONLY these if you need textures):\n%1\n") + .arg(limitedTextures.join(", ")); + + if (filteredTextures.size() > maxTextures) { + texturesSection += QString("... and %1 more textures available.\n") + .arg(filteredTextures.size() - maxTextures); + } + } + } + if (!currentMaterial.isEmpty()) { - userPrompt = QString("Here is the current material script (keep the same material name):\n%1\n\nModify it to: %2\n\nOutput only the modified material script:") + userPrompt = QString("Here is the current material script (keep the same material name):\n%1\n%2\nModify it to: %3\n\nOutput only the modified material script:") .arg(currentMaterial) + .arg(texturesSection) .arg(prompt); } else { - userPrompt = QString("Create a new Ogre3D material: %1\n\nOutput only the material script:").arg(prompt); + userPrompt = QString("%1Create a new Ogre3D material: %2\n\nOutput only the material script:") + .arg(texturesSection) + .arg(prompt); } - QMetaObject::invokeMethod(m_worker, [this, userPrompt]() { - m_worker->generate(getOgre3DSystemPrompt(), userPrompt); - }, Qt::QueuedConnection); + return userPrompt; } void LLMManager::stopGeneration() @@ -600,27 +713,16 @@ void LLMManager::onWorkerGenerationCompleted(const QString &fullText) qDebug() << "LLMManager: Invalid script generated (" << errorMessage << "), retry" << m_retryCount << "of" << MAX_RETRIES; if (m_retryCount <= MAX_RETRIES) { - // Build a correction prompt - QString correctionPrompt; - if (!m_pendingCurrentMaterial.isEmpty()) { - correctionPrompt = QString( - "The previous output was invalid: %1\n\n" - "Here is the current material script (keep the same material name):\n%2\n\n" - "Modify it to: %3\n\n" - "IMPORTANT: Output ONLY a valid Ogre3D material script with proper numeric values (0.0 to 1.0 for colors).\n" - "Do not include any words in the color values, only numbers.") - .arg(errorMessage) - .arg(m_pendingCurrentMaterial) - .arg(m_pendingPrompt); - } else { - correctionPrompt = QString( - "The previous output was invalid: %1\n\n" - "Create a new Ogre3D material: %2\n\n" - "IMPORTANT: Output ONLY a valid Ogre3D material script with proper numeric values (0.0 to 1.0 for colors).\n" - "Do not include any words in the color values, only numbers.") - .arg(errorMessage) - .arg(m_pendingPrompt); - } + // Build a correction prompt with the error information + QString basePrompt = buildUserPrompt(m_pendingPrompt, m_pendingCurrentMaterial, m_pendingAvailableTextures); + QString correctionPrompt = QString( + "The previous output was invalid: %1\n\n" + "IMPORTANT: Output ONLY a valid Ogre3D material script with:\n" + "- Proper numeric values (0.0 to 1.0 for colors)\n" + "- Balanced braces (every { must have a matching })\n" + "- No markdown or explanations\n\n%2") + .arg(errorMessage) + .arg(basePrompt); QMetaObject::invokeMethod(m_worker, [this, correctionPrompt]() { m_worker->generate(getOgre3DSystemPrompt(), correctionPrompt); diff --git a/src/LLMManager.h b/src/LLMManager.h index f39913e..a2434af 100644 --- a/src/LLMManager.h +++ b/src/LLMManager.h @@ -97,7 +97,7 @@ public slots: Q_INVOKABLE void tryAutoLoadModel(); // Generation - Q_INVOKABLE void generateMaterial(const QString &prompt, const QString ¤tMaterial = QString()); + Q_INVOKABLE void generateMaterial(const QString &prompt, const QString ¤tMaterial = QString(), const QStringList &availableTextures = QStringList()); Q_INVOKABLE void stopGeneration(); // Settings @@ -146,6 +146,7 @@ public slots: void shutdownWorkerThread(); QString getDefaultModelsDirectory() const; void populateRecommendedModels(); + QString buildUserPrompt(const QString &prompt, const QString ¤tMaterial, const QStringList &availableTextures) const; private slots: void onWorkerModelLoaded(const QString &modelPath); @@ -175,6 +176,7 @@ private slots: // Retry logic for invalid scripts QString m_pendingPrompt; QString m_pendingCurrentMaterial; + QStringList m_pendingAvailableTextures; int m_retryCount = 0; static const int MAX_RETRIES = 2; }; diff --git a/src/MaterialEditorQML.cpp b/src/MaterialEditorQML.cpp index dc8043b..a86a30b 100644 --- a/src/MaterialEditorQML.cpp +++ b/src/MaterialEditorQML.cpp @@ -2617,7 +2617,10 @@ void MaterialEditorQML::generateMaterialFromPrompt(const QString &prompt) // Use local LLM m_llmGenerationProgress = 0.0f; emit llmGenerationProgressChanged(); - llmManager->generateMaterial(prompt, m_materialText); + + // Get available textures to provide context to the LLM + QStringList availableTextures = getAvailableTextures(); + llmManager->generateMaterial(prompt, m_materialText, availableTextures); } void MaterialEditorQML::stopAIGeneration() From 39d17babc1659ec11039c790aef68da3a3f84444 Mon Sep 17 00:00:00 2001 From: Fernando Date: Fri, 30 Jan 2026 23:41:32 +0100 Subject: [PATCH 2/2] Simplify LLM prompts and bump version to 2.1.1 - Simplified system prompt to focus on basic color operations - Added explicit rule: "Do NOT add texture_unit unless specifically asked" - Textures list only included when prompt contains texture-related keywords (texture, image, map, .png, .jpg, etc.) - Clearer color reference with simple RGB values - Bump version to 2.1.1 This fixes the issue where the LLM was adding unwanted textures for simple color change requests. Co-Authored-By: Claude Opus 4.5 --- CMakeLists.txt | 2 +- src/LLMManager.cpp | 147 +++++++++++++++------------------------------ 2 files changed, 50 insertions(+), 99 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 0fddd98..c4defde 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -13,7 +13,7 @@ cmake_minimum_required(VERSION 3.24.0) cmake_policy(SET CMP0005 NEW) cmake_policy(SET CMP0048 NEW) # manages project version -project(QtMeshEditor VERSION 2.1.0 LANGUAGES CXX) +project(QtMeshEditor VERSION 2.1.1 LANGUAGES CXX) message(STATUS "Building QtMeshEditor version ${PROJECT_VERSION}") set(QTMESHEDITOR_VERSION_STRING "\"${PROJECT_VERSION}\"") diff --git a/src/LLMManager.cpp b/src/LLMManager.cpp index 569c545..ce8010b 100644 --- a/src/LLMManager.cpp +++ b/src/LLMManager.cpp @@ -230,113 +230,61 @@ bool LLMManager::isGenerating() const QString LLMManager::getOgre3DSystemPrompt() { - return R"(You are an expert Ogre3D material script generator. Output ONLY valid material scripts. - -STRUCTURE: -material NAME { technique { pass { texture_unit { } } } } - -PASS ATTRIBUTES (all colors are R G B values 0.0-1.0): -- ambient R G B [A] : Ambient color (default: 1.0 1.0 1.0) -- diffuse R G B [A] : Diffuse color (default: 1.0 1.0 1.0) -- specular R G B [A] SHINE : Specular color + shininess 1-128 (default: 0 0 0 0) -- emissive R G B [A] : Self-illumination (default: 0 0 0) -- lighting on|off : Enable dynamic lighting (default: on) -- shading flat|gouraud|phong : Shading mode (default: gouraud) -- scene_blend : Blending: add, modulate, alpha_blend, colour_blend -- scene_blend : Custom blend: one, zero, src_alpha, one_minus_src_alpha, dest_colour, etc. -- depth_check on|off : Depth buffer check (default: on) -- depth_write on|off : Write to depth buffer (default: on) -- depth_func : always_fail, always_pass, less, less_equal, equal, not_equal, greater_equal, greater -- cull_hardware clockwise|anticlockwise|none : Back-face culling (default: clockwise) -- polygon_mode solid|wireframe|points - -TEXTURE_UNIT ATTRIBUTES: -- texture [2d|cubic] : Texture file to use -- filtering none|bilinear|trilinear|anisotropic -- tex_address_mode wrap|clamp|mirror|border -- scroll : Static UV offset -- scroll_anim : Animated scrolling -- rotate : Static rotation -- rotate_anim : Animated rotation -- scale : UV scaling -- colour_op replace|add|modulate|alpha_blend -- colour_op_ex [manual_factor] [manual_color] -- env_map off|spherical|planar|cubic_reflection|cubic_normal -- alpha_rejection greater_equal 128 : Alpha testing - -COLOUR_OP_EX operations: add, subtract, modulate, modulate_x2, modulate_x4, add_signed, add_smooth, blend_diffuse_alpha, blend_texture_alpha, blend_current_alpha, blend_manual -COLOUR_OP_EX sources: src_current, src_texture, src_diffuse, src_specular, src_manual + return R"(You generate Ogre3D material scripts. Output ONLY the material script, nothing else. RULES: 1. Output ONLY the material script - no markdown, no explanations 2. Keep the same material name if modifying an existing material -3. All color/numeric values must be NUMBERS (0.0 to 1.0 for colors) -4. If textures are needed, ONLY use textures from the provided available list -5. Ensure balanced braces - every { must have a matching } +3. All color values must be NUMBERS between 0.0 and 1.0 +4. Do NOT add texture_unit unless specifically asked for textures +5. For simple color changes, only modify ambient/diffuse/specular/emissive values -EXAMPLE - Textured material with glow: -material Example/GlowingTextured +BASIC STRUCTURE: +material NAME { technique { pass { - ambient 0.3 0.3 0.3 - diffuse 0.8 0.8 0.8 - specular 1.0 1.0 1.0 64 - emissive 0.2 0.1 0.0 - - texture_unit - { - texture myTexture.png - filtering trilinear - } + ambient R G B + diffuse R G B + specular R G B SHININESS + emissive R G B } } } -EXAMPLE - Transparent material: -material Example/Transparent +COMMON COLORS (R G B values): +- red: 1.0 0.0 0.0 +- green: 0.0 1.0 0.0 +- blue: 0.0 0.0 1.0 +- yellow: 1.0 1.0 0.0 +- white: 1.0 1.0 1.0 +- black: 0.0 0.0 0.0 +- orange: 1.0 0.5 0.0 +- purple: 0.5 0.0 1.0 +- cyan: 0.0 1.0 1.0 + +EXAMPLE - Green shiny material: +material MyMaterial { technique { pass { - scene_blend alpha_blend - depth_write off - - diffuse 1.0 1.0 1.0 0.5 - - texture_unit - { - texture glass.png - } + ambient 0.0 0.2 0.0 + diffuse 0.0 0.8 0.0 + specular 0.5 1.0 0.5 64 + emissive 0.0 0.0 0.0 } } } -EXAMPLE - Animated water: -material Example/Water -{ - technique - { - pass - { - ambient 0.1 0.2 0.4 - diffuse 0.3 0.5 0.8 - specular 1.0 1.0 1.0 128 - scene_blend alpha_blend - depth_write off - - texture_unit - { - texture water.png - scroll_anim 0.05 0.02 - wave_xform scale_x sine 1.0 0.1 0.0 0.5 - } - } - } -})"; +ADVANCED (only use when requested): +- scene_blend add|alpha_blend : For transparency/glow +- depth_write off : For transparent materials +- lighting off : Disable dynamic lighting +- texture_unit { texture } : Only if textures requested)"; } void LLMManager::loadModel(const QString &modelName) @@ -479,10 +427,20 @@ void LLMManager::generateMaterial(const QString &prompt, const QString ¤tM QString LLMManager::buildUserPrompt(const QString &prompt, const QString ¤tMaterial, const QStringList &availableTextures) const { QString userPrompt; + QString lowerPrompt = prompt.toLower(); + + // Only include textures if the user's prompt mentions texture-related keywords + bool needsTextures = lowerPrompt.contains("texture") || + lowerPrompt.contains("image") || + lowerPrompt.contains("picture") || + lowerPrompt.contains("map") || + lowerPrompt.contains("bitmap") || + lowerPrompt.contains(".png") || + lowerPrompt.contains(".jpg") || + lowerPrompt.contains(".dds"); - // Add available textures section if any exist QString texturesSection; - if (!availableTextures.isEmpty()) { + if (needsTextures && !availableTextures.isEmpty()) { // Filter to only include image-like textures (exclude internal Ogre textures) QStringList filteredTextures; for (const QString &tex : availableTextures) { @@ -496,35 +454,28 @@ QString LLMManager::buildUserPrompt(const QString &prompt, const QString ¤ tex.endsWith(".jpeg", Qt::CaseInsensitive) || tex.endsWith(".dds", Qt::CaseInsensitive) || tex.endsWith(".tga", Qt::CaseInsensitive) || - tex.endsWith(".bmp", Qt::CaseInsensitive) || - // Also include textures without extensions (might be valid) - !tex.contains("."))) { + tex.endsWith(".bmp", Qt::CaseInsensitive))) { filteredTextures.append(tex); } } if (!filteredTextures.isEmpty()) { // Limit to reasonable number to avoid prompt bloat - int maxTextures = qMin(filteredTextures.size(), 50); + int maxTextures = qMin(filteredTextures.size(), 30); QStringList limitedTextures = filteredTextures.mid(0, maxTextures); - texturesSection = QString("\nAVAILABLE TEXTURES (use ONLY these if you need textures):\n%1\n") + texturesSection = QString("\nAVAILABLE TEXTURES (use ONLY from this list):\n%1\n") .arg(limitedTextures.join(", ")); - - if (filteredTextures.size() > maxTextures) { - texturesSection += QString("... and %1 more textures available.\n") - .arg(filteredTextures.size() - maxTextures); - } } } if (!currentMaterial.isEmpty()) { - userPrompt = QString("Here is the current material script (keep the same material name):\n%1\n%2\nModify it to: %3\n\nOutput only the modified material script:") + userPrompt = QString("Current material (keep the same name):\n%1\n%2\nRequest: %3") .arg(currentMaterial) .arg(texturesSection) .arg(prompt); } else { - userPrompt = QString("%1Create a new Ogre3D material: %2\n\nOutput only the material script:") + userPrompt = QString("%1Create: %2") .arg(texturesSection) .arg(prompt); }