Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
51 changes: 34 additions & 17 deletions pkg/e2e/adhoctestimages/adhoctestimages.go
Original file line number Diff line number Diff line change
Expand Up @@ -129,19 +129,43 @@ var _ = ginkgo.Describe("Ad Hoc Test Images", ginkgo.Ordered, ginkgo.ContinueOnF
}
}

if len(allFailures) > 0 && viper.GetBool(config.LogAnalysis.EnableAnalysis) {
if len(allFailures) > 0 {
combinedErr := fmt.Errorf("failures in %s: %s", testImage, strings.Join(allFailures, "; "))
runLogAnalysisForAdHocTestImage(ctx, logger, testSuite, combinedErr, exeConfig.OutputDir)
var analysisContent string
if viper.GetBool(config.LogAnalysis.EnableAnalysis) {
analysisContent = runLogAnalysisForAdHocTestImage(ctx, logger, testSuite, combinedErr, exeConfig.OutputDir)
}
queueNotification(testSuite, analysisContent)
}
},
testImageEntries)
})

// runLogAnalysisForAdHocTestImage runs AI analysis and queues the result for
// deferred Slack delivery. The engine runs without sending notifications because
// S3 artifacts have not been uploaded yet at this point. The queued result is
// later sent by Report after S3 upload populates presigned URLs.
func runLogAnalysisForAdHocTestImage(ctx context.Context, logger logr.Logger, testSuite config.TestSuite, err error, artifactsDir string) {
// queueNotification adds a PendingNotification for deferred Slack delivery.
// Called directly when log analysis is disabled so notifications are still sent.
func queueNotification(testSuite config.TestSuite, analysisContent string) {
clusterInfo := &analysisengine.ClusterInfo{
ID: viper.GetString(config.Cluster.ID),
Name: viper.GetString(config.Cluster.Name),
Provider: viper.GetString(config.Provider),
Region: viper.GetString(config.CloudProvider.Region),
CloudProvider: viper.GetString(config.CloudProvider.CloudProviderID),
Version: viper.GetString(config.Cluster.Version),
}
pendingMu.Lock()
pendingNotifications = append(pendingNotifications, PendingNotification{
AnalysisContent: analysisContent,
TestSuite: testSuite,
ClusterInfo: clusterInfo,
Env: viper.GetString(ocmprovider.Env),
})
pendingMu.Unlock()
}

// runLogAnalysisForAdHocTestImage runs AI analysis and returns the result content.
// Returns an empty string if analysis fails. The caller is responsible for
// queuing the notification via queueNotification.
func runLogAnalysisForAdHocTestImage(ctx context.Context, logger logr.Logger, testSuite config.TestSuite, err error, artifactsDir string) string {
logger.Info("Running Log analysis for test image", "image", testSuite.Image, "slackChannel", testSuite.SlackChannel)

clusterInfo := &analysisengine.ClusterInfo{
Expand All @@ -166,24 +190,17 @@ func runLogAnalysisForAdHocTestImage(ctx context.Context, logger logr.Logger, te
engine, err := analysisengine.New(ctx, engineConfig)
if err != nil {
logger.Error(err, "Unable to create analysis engine for image", "image", testSuite.Image)
return
return ""
}

result, runErr := engine.Run(ctx)
if runErr != nil {
logger.Error(runErr, "Log analysis failed for image", "image", testSuite.Image)
return
return ""
}

logger.Info("Log analysis completed successfully", "image", testSuite.Image, "resultsDir", fmt.Sprintf("%s/%s/", artifactsDir, analysisengine.AnalysisDirName))
log.Printf("=== Log Analysis Result for %s ===\n%s", testSuite.Image, result.Content)

pendingMu.Lock()
pendingNotifications = append(pendingNotifications, PendingNotification{
AnalysisContent: result.Content,
TestSuite: testSuite,
ClusterInfo: clusterInfo,
Env: viper.GetString(ocmprovider.Env),
})
pendingMu.Unlock()
return result.Content
}
26 changes: 11 additions & 15 deletions pkg/e2e/e2e.go
Original file line number Diff line number Diff line change
Expand Up @@ -276,14 +276,15 @@ func (o *E2EOrchestrator) Report(ctx context.Context) error {
}
}

// Send failure notification (with or without LLM analysis)
if o.result.ExitCode != config.Success && viper.GetBool(config.Tests.EnableSlackNotify) {
// Drain per-suite pending notifications. If any were queued (suites ran),
// send them; otherwise fall back to global failure notification.
pending := adhoctestimages.DrainPendingNotifications()
if len(pending) > 0 {
o.sendDeferredNotifications(ctx, pending)
} else if o.result.ExitCode != config.Success && viper.GetBool(config.Tests.EnableSlackNotify) {
o.sendFailureNotification(ctx)
}

// Send deferred ad-hoc test suite notifications (with S3 URLs)
o.sendDeferredNotifications(ctx)

runner.ReportClusterInstallLogs(o.provider)
return nil
}
Expand All @@ -296,7 +297,7 @@ func (o *E2EOrchestrator) sendFailureNotification(ctx context.Context) {
reportDir := viper.GetString(config.ReportDir)
notificationConfig := slack.BuildNotificationConfig(
viper.GetString(config.LogAnalysis.SlackWebhook),
viper.GetString(config.LogAnalysis.SlackChannel),
viper.GetString(config.Tests.SlackChannel),
&slack.ClusterInfo{
ID: viper.GetString(config.Cluster.ID),
Name: viper.GetString(config.Cluster.Name),
Expand Down Expand Up @@ -342,15 +343,10 @@ func (o *E2EOrchestrator) sendFailureNotification(ctx context.Context) {
}
}

// sendDeferredNotifications delivers Slack notifications that were queued by
// adhoctestimages during test execution. Called by Report after S3 upload so
// that presigned URLs are available for inclusion in the message.
func (o *E2EOrchestrator) sendDeferredNotifications(ctx context.Context) {
pending := adhoctestimages.DrainPendingNotifications()
if len(pending) == 0 {
return
}

// sendDeferredNotifications delivers the given Slack notifications that were
// queued by adhoctestimages during test execution. Called by Report after S3
// upload so that presigned URLs are available for inclusion in the message.
func (o *E2EOrchestrator) sendDeferredNotifications(ctx context.Context, pending []adhoctestimages.PendingNotification) {
webhook := viper.GetString(config.LogAnalysis.SlackWebhook)
if webhook == "" || !viper.GetBool(config.Tests.EnableSlackNotify) {
return
Expand Down