Skip to content

MilkdropPreset: add opt-in preprocessed-HLSL cache hooks (targeting 4.3) #1011

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
keeper-of-memes wants to merge 1 commit into
base: master
Choose a base branch
from
Draft

MilkdropPreset: add opt-in preprocessed-HLSL cache hooks (targeting 4.3) #1011

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
54 changes: 51 additions & 3 deletions src/api/include/projectM-4/callbacks.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,7 @@ PROJECTM_EXPORT void projectm_set_preset_switch_failed_event_callback(projectm_h
* and will delete it (via glDeleteTextures) when it is no longer needed. Do not
* delete the texture yourself or reuse the texture ID after passing it here.
*
* @since 4.2.0
* @since 4.3.0
*/
typedef struct projectm_texture_load_data {
const unsigned char* data; /**< Pointer to raw pixel data in standard OpenGL format (first row is bottom of image). Can be NULL. */
Expand Down Expand Up @@ -139,7 +139,7 @@ typedef struct projectm_texture_load_data {
* @param texture_name The name of the texture being requested, as used in the preset.
* @param[out] data Pointer to a structure where the application should place texture data.
* @param user_data A user-defined data pointer that was provided when registering the callback.
* @since 4.2.0
* @since 4.3.0
*/
typedef void (*projectm_texture_load_event)(const char* texture_name,
projectm_texture_load_data* data,
Expand All @@ -155,12 +155,60 @@ typedef void (*projectm_texture_load_event)(const char* texture_name,
* @param callback A pointer to the callback function.
* @param user_data A pointer to any data that will be sent back in the callback, e.g. context
* information.
* @since 4.2.0
* @since 4.3.0
*/
PROJECTM_EXPORT void projectm_set_texture_load_event_callback(projectm_handle instance,
projectm_texture_load_event callback,
void* user_data);

/**
* @brief Optional hooks for caching the preprocessed-HLSL text of Milkdrop preset shaders.
*
* The Milkdrop shader transpiler runs a text preprocessor (macro expansion) over each
* preset shader before parsing it. That step is pure text and deterministic for a given
* input, so its result can be cached across runs. When these hooks are registered, projectM
* computes a stable key for the preprocessor input and consults the cache before running the
* (comparatively expensive) preprocessor.
*
* The cache is fully opt-in. If no hooks are registered, behaviour is byte-identical to a
* build without this feature. projectM owns nothing referenced by the value pointers after a
* callback returns; the get sink is only valid for the duration of the sink call.
*
* All hook invocations happen on the same thread that calls the projectM rendering
* functions (the transpile runs there), so no additional synchronization is required.
*
* @since 4.3.0
*/
typedef struct projectm_preprocess_cache_hooks {
/**
* @brief Lookup hook. On a hit, invoke @p sink exactly once with the cached value (valid
* only for the duration of that call) and return true. On a miss, return false and
* do not call @p sink.
*/
bool (*get)(void* user, const char* key, size_t keylen,
void* sinkctx, void (*sink)(void* sinkctx, const char* data, size_t len));
/**
* @brief Store hook. Copy @p vallen bytes at @p value under the @p keylen-byte @p key.
* The callee owns no passed-in pointer after returning.
*/
void (*put)(void* user, const char* key, size_t keylen, const char* value, size_t vallen);
void* user; /**< User-defined pointer passed back to get/put. */
} projectm_preprocess_cache_hooks;

/**
* @brief Registers (or clears) the preprocessed-HLSL cache hooks for this instance.
*
* The hooks structure is copied by value; the caller need not keep it alive, but the @c user
* pointer and anything the callbacks touch must outlive the projectM instance (or until the
* hooks are cleared). Passing NULL clears any previously registered hooks.
*
* @param instance The projectM instance handle.
* @param hooks A pointer to the hooks structure, or NULL to clear.
* @since 4.3.0
*/
PROJECTM_EXPORT void projectm_set_preprocess_cache(projectm_handle instance,
const projectm_preprocess_cache_hooks* hooks);

#ifdef __cplusplus
} // extern "C"
#endif
2 changes: 2 additions & 0 deletions src/libprojectM/MilkdropPreset/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,8 @@ add_library(MilkdropPreset OBJECT
MilkdropPresetExceptions.hpp
MilkdropShader.cpp
MilkdropShader.hpp
MilkdropShaderPreprocess.cpp
MilkdropShaderPreprocess.hpp
MilkdropStaticShaders.cpp.in
MilkdropStaticShaders.hpp.in
MotionVectors.cpp
Expand Down
225 changes: 45 additions & 180 deletions src/libprojectM/MilkdropPreset/MilkdropShader.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
#include "MilkdropShader.hpp"

#include "MilkdropShaderPreprocess.hpp"
#include "PresetState.hpp"
#include "Utils.hpp"

Expand Down Expand Up @@ -331,182 +332,13 @@ auto MilkdropShader::Shader() -> Renderer::Shader&

void MilkdropShader::PreprocessPresetShader(std::string& program)
{
std::string shaderTypeString = "composite";
if (m_type == ShaderType::WarpShader)
{
shaderTypeString = "warp";
}

if (program.length() <= 0)
{
throw Renderer::ShaderException("[MilkdropShader] Preset " + shaderTypeString + " shader is declared, but empty.");
}

size_t found;

// Find "sampler_state" overrides and remove them first, as they're not supported by GLSL.
// The logic isn't totally fool-proof, but should work in general.
// Use a comment-stripped copy for searching so commented-out sampler_state blocks are skipped.
// StripComments preserves string length, so positions map 1:1 to the original.
std::string stripped = Utils::StripComments(program);
found = stripped.find("sampler_state");
while (found != std::string::npos)
{
// Now go backwards and find the assignment
found = stripped.rfind('=', found);
auto startPos = found;

// Find closing brace and semicolon
found = stripped.find('}', found);
found = stripped.find(';', found);

if (found != std::string::npos)
{
stripped.replace(startPos, found - startPos, "");
}
else
{
// No closing brace and semicolon.
break;
}

found = stripped.find("sampler_state");
}

// replace shader_body with entry point function
// Use the stripped copy so a commented-out shader_body is not matched.
found = stripped.find("shader_body");
if (found != std::string::npos)
{
if (m_type == ShaderType::WarpShader)
{
program.replace(int(found), 11, R"(
void PS(float4 _vDiffuse : COLOR,
float4 _uv : TEXCOORD0,
float2 _rad_ang : TEXCOORD1,
out float4 _return_value : COLOR0,
out float4 _mv_tex_coords : COLOR1)
)");
}
else
{
program.replace(int(found), 11, R"(
void PS(float4 _vDiffuse : COLOR,
float2 _uv : TEXCOORD0,
float2 _rad_ang : TEXCOORD1,
out float4 _return_value : COLOR)
)");
}
}
else
{
LOG_DEBUG("[MilkdropShader] Failed " + shaderTypeString + " shader code:\n" + program);
throw Renderer::ShaderException("[MilkdropShader] Preset " + shaderTypeString + " shader is missing \"shader_body\" entry point.");
}

// replace the "{" immediately following shader_body with some variable declarations
found = program.find('{', found);
if (found != std::string::npos)
{
std::string progMain = "{\nfloat3 ret = 0;\n";
if (m_type == ShaderType::WarpShader)
{
progMain.append("_mv_tex_coords.xy = _uv.xy;\n");
}
program.replace(int(found), 1, progMain);
}
else
{
LOG_DEBUG("[MilkdropShader] Failed " + shaderTypeString + " shader code:\n" + program);
throw Renderer::ShaderException("[MilkdropShader] Preset " + shaderTypeString + " shader has no opening braces.");
}

// replace "}" with return statement (this can probably be optimized for the GLSL conversion...)
found = program.rfind('}');
if (found != std::string::npos)
{
program.replace(int(found), 1, "_return_value = float4(ret.xyz, 1.0);\n"
"}\n");
}
else
{
LOG_DEBUG("[MilkdropShader] Failed " + shaderTypeString + " shader code:\n" + program);
throw Renderer::ShaderException("[MilkdropShader] Preset " + shaderTypeString + " shader has no closing brace.");
}

// Find matching closing brace and cut off excess text after shader's main function
int bracesOpen = 1;
size_t pos = found + 1;
for (; pos < program.length() && bracesOpen > 0; ++pos)
{
switch (program.at(pos))
{
case '/':
// Skip line comments until EoL to prevent false counting
if (pos < program.length() - 1 && program.at(pos + 1) == '/')
{
for (; pos < program.length(); ++pos)
{
if (program.at(pos) == '\n')
{
break;
}
}
}
// Skip block comments to prevent false counting
else if (pos < program.length() - 1 && program.at(pos + 1) == '*')
{
pos += 2;
for (; pos < program.length() - 1; ++pos)
{
if (program.at(pos) == '*' && program.at(pos + 1) == '/')
{
++pos; // skip past '/'
break;
}
}
}
continue;

case '{':
bracesOpen++;
continue;

case '}':
bracesOpen--;
}
}

if (pos < program.length() - 1)
{
program.resize(pos);
}

std::string fullSource; //!< Full shader source before translation, includes all uniforms etc.

// First copy the generic "header" into the shader. Includes uniforms and some defines
// to unwrap the packed 4-element uniforms into single values.
fullSource.append(MilkdropStaticShaders::Get()->GetPresetShaderHeader());

if (m_type == ShaderType::WarpShader)
{
fullSource.append("#define rad _rad_ang.x\n"
"#define ang _rad_ang.y\n"
"#define uv _uv.xy\n"
"#define uv_orig _uv.zw\n");
}
else
{
fullSource.append("#define rad _rad_ang.x\n"
"#define ang _rad_ang.y\n"
"#define uv _uv.xy\n"
"#define uv_orig _uv.xy\n"
"#define hue_shader _vDiffuse.xyz\n");
}

fullSource.append(program);

program = fullSource;
// The pure-text assembly lives in the GL-free free function
// AssembleApplyPreprocessorInput(type, body) so that external tooling (e.g. a build-time
// cache precompute step) and the runtime produce byte-identical preprocessor input. This
// wrapper preserves the original in/out contract (mutates `program` in place).
// GetReferencedSamplers still runs separately in LoadCode, so m_samplerNames population
// is unchanged.
program = AssembleApplyPreprocessorInput(m_type, program);
}

void MilkdropShader::GetReferencedSamplers(const std::string& program)
Expand Down Expand Up @@ -615,12 +447,45 @@ void MilkdropShader::TranspileHLSLShader(const PresetState& presetState, std::st
M4::HLSLTree tree(&allocator);
M4::HLSLParser parser(&allocator, &tree);

// Preprocess define macros
// Preprocess define macros. Optionally short-circuit via the app-provided preprocessed-
// HLSL cache: the preprocessor output is a pure, deterministic function of `program`, so
// a hit lets us skip ApplyPreprocessor entirely. With no hooks registered this path is
// byte-identical to the historical behaviour.
std::string sourcePreprocessed;
if (!parser.ApplyPreprocessor("", program.c_str(), program.size(), sourcePreprocessed))
const auto* preprocessCacheHooks = presetState.renderContext.preprocessCache;
bool preprocessCacheHit = false;
std::string preprocessCacheKey;
if (preprocessCacheHooks != nullptr && (preprocessCacheHooks->get != nullptr || preprocessCacheHooks->put != nullptr))
{
LOG_DEBUG("[MilkdropShader] Failed " + shaderTypeString + " shader code:\n" + program);
throw Renderer::ShaderException("Error translating HLSL " + shaderTypeString + " shader: Preprocessing failed.");
preprocessCacheKey = ComputePreprocessCacheKey(program);
}
if (preprocessCacheHooks != nullptr && preprocessCacheHooks->get != nullptr)
{
// Sink appends the cached bytes into sourcePreprocessed for the duration of the call.
auto appendSink = [](void* sinkctx, const char* data, size_t len) {
auto* out = static_cast<std::string*>(sinkctx);
out->append(data, len);
};
preprocessCacheHit = preprocessCacheHooks->get(preprocessCacheHooks->user,
preprocessCacheKey.data(), preprocessCacheKey.size(),
&sourcePreprocessed, appendSink);
}

if (!preprocessCacheHit)
{
sourcePreprocessed.clear();
if (!parser.ApplyPreprocessor("", program.c_str(), program.size(), sourcePreprocessed))
{
LOG_DEBUG("[MilkdropShader] Failed " + shaderTypeString + " shader code:\n" + program);
throw Renderer::ShaderException("Error translating HLSL " + shaderTypeString + " shader: Preprocessing failed.");
}

if (preprocessCacheHooks != nullptr && preprocessCacheHooks->put != nullptr)
{
preprocessCacheHooks->put(preprocessCacheHooks->user,
preprocessCacheKey.data(), preprocessCacheKey.size(),
sourcePreprocessed.data(), sourcePreprocessed.size());
}
}

// Remove previous shader declarations
Expand Down
Loading