Skip to content

feat: add Voyage AI provider support as embedding model #19

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
ecairol wants to merge 1 commit into
base: main
Choose a base branch
from
Open

feat: add Voyage AI provider support as embedding model #19

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
41 changes: 23 additions & 18 deletions admin/views/embeddings.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -498,31 +498,36 @@ function ( $a, $b ) {
<p class="description"><?php esc_html_e( 'Maximum number of posts to process', 'wpvdb' ); ?></p>
</div>

<?php
$wpvdb_active_provider = \WPVDB\Settings::get_active_provider();
$wpvdb_bulk_settings = \WPVDB\Settings::get_validated_settings();
$wpvdb_active_model = ! empty( $wpvdb_bulk_settings['active_model'] )
? $wpvdb_bulk_settings['active_model']
: \WPVDB\Models::get_default_model_for_provider( $wpvdb_active_provider );

$wpvdb_providers = \WPVDB\Providers::get_available_providers();
$wpvdb_provider_label = isset( $wpvdb_providers[ $wpvdb_active_provider ]['label'] )
? $wpvdb_providers[ $wpvdb_active_provider ]['label']
: $wpvdb_active_provider;

$wpvdb_active_model_data = \WPVDB\Models::get_model( $wpvdb_active_provider, $wpvdb_active_model );
$wpvdb_model_label = ( is_array( $wpvdb_active_model_data ) && ! empty( $wpvdb_active_model_data['label'] ) )
? $wpvdb_active_model_data['label']
: $wpvdb_active_model;
?>

<div class="wpvdb-form-group">
<label for="wpvdb-provider"><?php esc_html_e( 'Provider', 'wpvdb' ); ?></label>
<select id="wpvdb-provider" name="provider">
<?php
$providers = \WPVDB\Providers::get_available_providers();
foreach ( $providers as $provider_id => $provider ) {
echo '<option value="' . esc_attr( $provider_id ) . '">' . esc_html( $provider['label'] ) . '</option>';
}
?>
<select id="wpvdb-provider" name="provider" disabled>
<option value="<?php echo esc_attr( $wpvdb_active_provider ); ?>" selected><?php echo esc_html( $wpvdb_provider_label ); ?></option>
</select>
<p class="description"><?php esc_html_e( 'Uses the active provider configured under Settings. To embed with a different provider, change it there first.', 'wpvdb' ); ?></p>
</div>

<div class="wpvdb-form-group" id="wpvdb-bulk-models">
<label for="wpvdb-model"><?php esc_html_e( 'Model', 'wpvdb' ); ?></label>
<select id="wpvdb-model" name="model">
<?php
// Get models for the first provider.
$first_provider = reset( $providers );
$first_provider_id = key( $providers );
$provider_models = \WPVDB\Models::get_selectable_provider_models( $first_provider_id );

foreach ( $provider_models as $model_id => $model ) {
echo '<option value="' . esc_attr( $model_id ) . '">' . esc_html( $model['label'] ) . '</option>';
}
?>
<select id="wpvdb-model" name="model" disabled>
<option value="<?php echo esc_attr( $wpvdb_active_model ); ?>" selected><?php echo esc_html( $wpvdb_model_label ); ?></option>
</select>
</div>

Expand Down
91 changes: 91 additions & 0 deletions admin/views/settings.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,8 @@
$openai_api_version = isset( $settings['openai']['api_version'] ) ? $settings['openai']['api_version'] : '';
$automattic_api_key = isset( $settings['automattic']['api_key'] ) ? $settings['automattic']['api_key'] : '';
$automattic_endpoint = isset( $settings['automattic']['api_base'] ) ? $settings['automattic']['api_base'] : \WPVDB\Providers::get_api_base( 'automattic' );
$voyage_api_key = isset( $settings['voyage']['api_key'] ) ? $settings['voyage']['api_key'] : '';
$voyage_endpoint = isset( $settings['voyage']['api_base'] ) ? $settings['voyage']['api_base'] : \WPVDB\Providers::get_api_base( 'voyage' );
$embedding_batch_size = isset( $settings['batch_size'] ) ? $settings['batch_size'] : \WPVDB\Settings::DEFAULTS['batch_size'];
$provider_model = static function ( $provider_id ) use ( $settings, $provider, $active_model ) {
if ( $provider === $provider_id && ! empty( $active_model ) ) {
Expand All @@ -49,6 +51,7 @@
$openai_model = $provider_model( 'openai' );
$automattic_model = $provider_model( 'automattic' );
$specter_model = $provider_model( 'specter' );
$voyage_model = $provider_model( 'voyage' );
?>

<div class="wrap wpvdb-settings">
Expand Down Expand Up @@ -348,6 +351,94 @@ class="regular-text">
</td>
</tr>

<!-- Voyage AI Provider Fields -->
<tr id="voyage_api_key_field" class="api-key-field" <?php echo 'voyage' !== $provider ? 'style="display: none;"' : ''; ?>>
<th scope="row">
<label for="wpvdb_voyage_api_key"><?php esc_html_e( 'Voyage AI API Key', 'wpvdb' ); ?></label>
</th>
<td>
<input type="password"
name="wpvdb_settings[voyage][api_key]"
id="wpvdb_voyage_api_key"
value="<?php echo esc_attr( $voyage_api_key ); ?>"
class="regular-text">
<p class="description">
<?php esc_html_e( 'Enter your Voyage AI API key. You can get one from', 'wpvdb' ); ?>
<a href="https://dashboard.voyageai.com/" target="_blank">https://dashboard.voyageai.com/</a>
</p>
</td>
</tr>

<?php
$voyage_selectable = \WPVDB\Models::get_selectable_provider_models( 'voyage' );
$voyage_current_dim = \WPVDB\Models::get_configured_dimension();
$voyage_allowed_dims = \WPVDB\Models::get_provider_compatible_dimensions( 'voyage' );
?>
<tr id="voyage_model_field" class="model-field" <?php echo 'voyage' !== $provider ? 'style="display: none;"' : ''; ?>>
<th scope="row">
<label for="wpvdb_voyage_model"><?php esc_html_e( 'Voyage AI Embedding Model', 'wpvdb' ); ?></label>
</th>
<td>
<?php if ( empty( $voyage_selectable ) ) : ?>
<div class="notice notice-warning inline" style="margin: 0; padding: 8px 12px;">
<p style="margin-top: 0;">
<?php
printf(
/* translators: 1: current embedding dimension, 2: comma-separated list of supported dimensions. */
esc_html__( 'No Voyage AI models fit the current embedding dimension (%1$d). Voyage models require a dimension of %2$s.', 'wpvdb' ),
(int) $voyage_current_dim,
esc_html( implode( ', ', $voyage_allowed_dims ) )
);
?>
</p>
<p style="margin-bottom: 0;">
<?php
printf(
wp_kses(
/* translators: 1: WPVDB_DEFAULT_EMBED_DIM constant snippet, 2: rebuild-embeddings admin URL. */
__( 'To use Voyage, set <code>%1$s</code> in <code>wp-config.php</code> to a supported value (e.g. 1024), then <a href="%2$s">rebuild your embeddings</a>.', 'wpvdb' ),
array(
'code' => array(),
'a' => array( 'href' => array() ),
)
),
"define( 'WPVDB_DEFAULT_EMBED_DIM', 1024 );",
esc_url( admin_url( 'admin.php?page=wpvdb-embeddings' ) )
);
?>
</p>
</div>
<?php else : ?>
<select name="wpvdb_settings[voyage][default_model]" id="wpvdb_voyage_model">
<?php foreach ( $voyage_selectable as $model_id => $model_data ) : ?>
<option value="<?php echo esc_attr( $model_id ); ?>" <?php selected( $voyage_model, $model_id ); ?>>
<?php echo esc_html( $model_data['label'] ); ?>
</option>
<?php endforeach; ?>
</select>
<p class="description">
<?php esc_html_e( 'Select the Voyage AI model to use for generating embeddings.', 'wpvdb' ); ?>
</p>
<?php endif; ?>
</td>
</tr>

<tr class="provider-specific-field" data-provider="voyage" <?php echo 'voyage' !== $provider ? 'style="display: none;"' : ''; ?>>
<th scope="row">
<label for="wpvdb_voyage_endpoint"><?php esc_html_e( 'Voyage AI API Endpoint', 'wpvdb' ); ?></label>
</th>
<td>
<input type="text"
name="wpvdb_settings[voyage][api_base]"
id="wpvdb_voyage_endpoint"
value="<?php echo esc_attr( $voyage_endpoint ); ?>"
class="regular-text">
<p class="description">
<?php esc_html_e( 'API endpoint for Voyage AI embeddings. You probably don\'t need to change this.', 'wpvdb' ); ?>
</p>
</td>
</tr>

<tr class="provider-specific-field" data-provider="all">
<th scope="row">
<label for="wpvdb_embedding_batch_size"><?php esc_html_e( 'Embedding Batch Size', 'wpvdb' ); ?></label>
Expand Down
4 changes: 2 additions & 2 deletions assets/js/admin.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -180,7 +180,7 @@ jQuery(document).ready(function($) {
response.data.embedding_count + ' existing embeddings. Continue?')) {
console.log('WPVDB: User confirmed provider change');
// User confirmed, submit the form
$('#wpvdb-settings-form').off('submit').trigger('submit');
HTMLFormElement.prototype.submit.call($('#wpvdb-settings-form').off('submit').get(0));
} else {
console.log('WPVDB: User cancelled provider change');
// User cancelled, reset the form
Expand All @@ -190,7 +190,7 @@ jQuery(document).ready(function($) {
} else {
console.log('WPVDB: No embeddings exist, proceeding with provider change');
// No embeddings exist, just submit the form
$('#wpvdb-settings-form').off('submit').trigger('submit');
HTMLFormElement.prototype.submit.call($('#wpvdb-settings-form').off('submit').get(0));
}
} else {
console.error('WPVDB: Provider change validation error:', response.data.message);
Expand Down
125 changes: 121 additions & 4 deletions includes/class-wpvdb-admin.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -312,6 +312,12 @@ public function validate_settings( $input ) {
if ( isset( $input['specter']['api_base'] ) ) {
$input['specter']['api_base'] = sanitize_text_field( $input['specter']['api_base'] );
}
if ( isset( $input['voyage']['api_key'] ) ) {
$input['voyage']['api_key'] = Settings::encrypt_api_key( sanitize_text_field( $input['voyage']['api_key'] ) );
}
if ( isset( $input['voyage']['api_base'] ) ) {
$input['voyage']['api_base'] = sanitize_text_field( $input['voyage']['api_base'] );
}
if ( isset( $input['active_provider'] ) ) {
$input['active_provider'] = sanitize_text_field( $input['active_provider'] );
// For backwards compatibility.
Expand All @@ -332,6 +338,9 @@ public function validate_settings( $input ) {
if ( isset( $input['specter']['default_model'] ) ) {
$input['specter']['default_model'] = sanitize_text_field( $input['specter']['default_model'] );
}
if ( isset( $input['voyage']['default_model'] ) ) {
$input['voyage']['default_model'] = sanitize_text_field( $input['voyage']['default_model'] );
}

// Update individual options for backwards compatibility.
if ( isset( $input['openai']['api_key'] ) ) {
Expand Down Expand Up @@ -359,6 +368,9 @@ public function validate_settings( $input ) {
if ( ! isset( $input['specter'] ) || ! is_array( $input['specter'] ) ) {
$input['specter'] = isset( $current_settings['specter'] ) && is_array( $current_settings['specter'] ) ? $current_settings['specter'] : array();
}
if ( ! isset( $input['voyage'] ) || ! is_array( $input['voyage'] ) ) {
$input['voyage'] = isset( $current_settings['voyage'] ) && is_array( $current_settings['voyage'] ) ? $current_settings['voyage'] : array();
}

// Make sure api_key and default_model at least exist (even if empty).
if ( ! isset( $input['openai']['api_key'] ) ) {
Expand All @@ -376,6 +388,9 @@ public function validate_settings( $input ) {
if ( ! isset( $input['specter']['default_model'] ) ) {
$input['specter']['default_model'] = $this->get_default_model( 'specter' );
}
if ( ! isset( $input['voyage']['default_model'] ) ) {
$input['voyage']['default_model'] = $this->get_default_model( 'voyage' );
}

// Make sure post_types is always an array.
if ( isset( $input['post_types'] ) && ! is_array( $input['post_types'] ) ) {
Expand Down Expand Up @@ -1508,12 +1523,50 @@ public function ajax_bulk_embed() {

$queue->save()->dispatch();

// Report the real outcome instead of treating "queued" as success: the queue
// runs inline in the admin request, so chunk counts reflect what actually persisted.
$succeeded = array();
$failed = array();
foreach ( $post_ids as $post_id ) {
if ( (int) get_post_meta( $post_id, '_wpvdb_chunks_count', true ) > 0 ) {
$succeeded[] = $post_id;
} else {
$failed[] = $post_id;
}
}

if ( ! empty( $failed ) ) {
$recorded = get_transient( 'wpvdb_embedding_failures' );
$reason = '';
if ( is_array( $recorded ) && ! empty( $recorded ) ) {
$entry = end( $recorded );
$reason = isset( $entry['message'] ) ? (string) $entry['message'] : '';
}

$failed_message = sprintf(
/* translators: 1: number of posts that failed, 2: failure reason from the provider. */
_n( '%1$d post failed to embed: %2$s', '%1$d posts failed to embed: %2$s', count( $failed ), 'wpvdb' ),
count( $failed ),
$reason
);

wp_send_json_error(
array(
'message' => $failed_message,
'failed_ids' => $failed,
)
);
}
Comment on lines +1538 to +1559

$success_message = sprintf(
/* translators: %d: number of posts successfully embedded. */
_n( '%d post embedded successfully.', '%d posts embedded successfully.', count( $succeeded ), 'wpvdb' ),
count( $succeeded )
);

wp_send_json_success(
array(
'message' => sprintf(
__( 'Queued %d posts for embedding generation', 'wpvdb' ),
count( $post_ids )
),
'message' => $success_message,
'using_pending' => $using_pending,
)
);
Expand Down Expand Up @@ -2131,10 +2184,74 @@ public function handle_admin_actions() {
}
}

/**
* Human-readable label for an embedding error code.
*
* @param string $code WP_Error code from the embedding pipeline.
* @return string
*/
private static function embedding_error_label( $code ) {
switch ( $code ) {
case 'embedding_auth_error':
return __( 'authentication failed (check the API key under Settings)', 'wpvdb' );
case 'embedding_forbidden':
return __( 'access forbidden by the provider', 'wpvdb' );
case 'embedding_rate_limited':
return __( 'rate limited by the provider (HTTP 429)', 'wpvdb' );
case 'embedding_model_not_found':
return __( 'model not found', 'wpvdb' );
case 'embedding_provider_error':
return __( 'provider server error', 'wpvdb' );
default:
return __( 'embedding error', 'wpvdb' );
}
}

/**
* Display admin notices for action results
*/
public function admin_notices() {
// Surface embedding failures captured during background processing so they are never lost silently.
$embedding_failures = get_transient( 'wpvdb_embedding_failures' );
if ( is_array( $embedding_failures ) && ! empty( $embedding_failures ) ) {
delete_transient( 'wpvdb_embedding_failures' );

$counts = array();
$samples = array();
foreach ( $embedding_failures as $failure ) {
$code = isset( $failure['code'] ) ? (string) $failure['code'] : 'embedding_error';
$counts[ $code ] = isset( $counts[ $code ] ) ? $counts[ $code ] + 1 : 1;
if ( ! isset( $samples[ $code ] ) && ! empty( $failure['message'] ) ) {
$samples[ $code ] = (string) $failure['message'];
}
}

$total = count( $embedding_failures );
echo '<div class="notice notice-error is-dismissible"><p><strong>';
printf(
/* translators: %d: number of failed embeddings. */
esc_html( _n( 'WPVDB: %d embedding failed and was not saved.', 'WPVDB: %d embeddings failed and were not saved.', $total, 'wpvdb' ) ),
(int) $total
);
echo '</strong></p><ul style="list-style:disc;margin-left:20px;">';
foreach ( $counts as $code => $count ) {
echo '<li>';
printf(
/* translators: 1: number of failures, 2: human-readable reason. */
esc_html__( '%1$d × %2$s', 'wpvdb' ),
(int) $count,
esc_html( self::embedding_error_label( $code ) )
);
if ( ! empty( $samples[ $code ] ) ) {
echo ' — <code>' . esc_html( $samples[ $code ] ) . '</code>';
}
echo '</li>';
}
echo '</ul><p>';
esc_html_e( 'See the debug log for full details, then re-run “Bulk Generate Embeddings” for the affected items.', 'wpvdb' );
echo '</p></div>';
}

// Check for table recreation status.
$recreate_status = get_transient( 'wpvdb_table_recreate_status' );
if ( $recreate_status ) {
Expand Down
23 changes: 19 additions & 4 deletions includes/class-wpvdb-core.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -291,7 +291,20 @@ public static function get_embedding( $text, $model, $api_base, $api_key ) {
}

if ( 200 !== $code ) {
return new \WP_Error( 'embedding_error', 'Failed to get embedding: ' . $code . ' ' . ( is_string( $data ) ? $data : wp_json_encode( $data ) ) );
$status_code = (int) $code;
$error_code = 'embedding_error';
if ( 401 === $status_code ) {
$error_code = 'embedding_auth_error';
} elseif ( 403 === $status_code ) {
$error_code = 'embedding_forbidden';
} elseif ( 404 === $status_code ) {
$error_code = 'embedding_model_not_found';
} elseif ( 429 === $status_code ) {
$error_code = 'embedding_rate_limited';
} elseif ( $status_code >= 500 && $status_code < 600 ) {
$error_code = 'embedding_provider_error';
}
return new \WP_Error( $error_code, 'Failed to get embedding: ' . $code . ' ' . ( is_string( $data ) ? $data : wp_json_encode( $data ) ) );
}

if ( 'a8c_nomic_native' === $response_format ) {
Expand Down Expand Up @@ -446,13 +459,15 @@ private static function get_embedding_custom_options( $model, $api_base ) {
return array();
}

// Only send dimensions to models that explicitly support it.
// Only send dimensions to models that explicitly support it, under the
// parameter name the provider expects (OpenAI: dimensions, Voyage: output_dimension).
$dimension_param = Models::get_dimension_param( $model, $api_base );
if (
! isset( $options['dimensions'] ) &&
! isset( $options[ $dimension_param ] ) &&
defined( 'WPVDB_DEFAULT_EMBED_DIM' ) &&
Models::supports_dimensions( $model, $api_base )
) {
$options['dimensions'] = (int) WPVDB_DEFAULT_EMBED_DIM;
$options[ $dimension_param ] = (int) WPVDB_DEFAULT_EMBED_DIM;
}

return array_filter(
Expand Down
Loading