Skip to content

Surgical scheduler logs for jobs observability (phase 1.5) #646

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
cbb330 wants to merge 1 commit into
base: main
Choose a base branch
from
+34 −1
Open

Surgical scheduler logs for jobs observability (phase 1.5) #646

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions apps/spark/src/main/java/com/linkedin/openhouse/jobs/scheduler/JobsScheduler.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -265,6 +265,19 @@ public static void main(String[] args) {
}
log.info("Shutdown hook completed.");
}));
log.info(
"OH_SCHED_START job_type={} numParallelJobs={} tasksWaitHours={} numJobsSubmitter={} "
+ "numJobsPoller={} taskPollIntervalMs={} taskQueuedTimeoutMs={} taskTimeoutMs={} "
+ "multiOperationMode={}",
operationType.getValue(),
getNumParallelJobs(cmdLine),
getTasksWaitHours(cmdLine),
getNumJobsSubmitter(cmdLine),
getNumJobsPoller(cmdLine),
getTaskPollIntervalMs(cmdLine),
getTaskQueuedTimeoutMs(cmdLine),
getTaskTimeoutMs(cmdLine),
isMultiOperationMode(cmdLine));
app.run(
operationType,
operationTaskCls.toString(),
Expand Down
10 changes: 9 additions & 1 deletion apps/spark/src/main/java/com/linkedin/openhouse/jobs/scheduler/tasks/OperationTask.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -170,16 +170,24 @@ private Optional<JobState> submitJob(Attributes typeAttributes) {

protected Optional<JobState> pollJobStatus(Attributes typeAttributes) {
long startTime = System.currentTimeMillis();
Optional<JobState> lastObservedState = Optional.empty();
try {
Optional<JobState> jobState;
do {
jobState = jobsClient.getState(jobId);
lastObservedState = jobState;
long elapsedTime = System.currentTimeMillis() - startTime;
// Exit status check if a job is queued for more than queuedTimeoutMs.
if (elapsedTime > queuedTimeoutMs) {
if (jobState.isPresent() && jobState.get().equals(JobState.QUEUED)) {
String executionId =
jobsClient.getJob(jobId).map(JobResponseBody::getExecutionId).orElse(null);
log.info(
"Exiting status check for {} for {} due to queued timeout", getType(), metadata);
"Exiting status check for {} for {} due to queued timeout lastObservedState={} executionId={}",
getType(),
metadata,
lastObservedState.orElse(null),
executionId);
break;
}
}
Expand Down
12 changes: 12 additions & 0 deletions ...park/src/main/java/com/linkedin/openhouse/jobs/scheduler/tasks/OperationTasksBuilder.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -441,6 +441,10 @@ private void buildOperationTaskListInParallelInternal(
.doAfterTerminate(
() -> {
operationTaskManager.updateDataGenerationCompletion();
log.info(
"OH_SCHED_ELIGIBLE job_type={} count={}",
jobType,
operationTaskManager.getTotalDataCount());
log.info(
"The metadata fetched count: {} for the job type: {}",
operationTaskManager.getTotalDataCount(),
Expand Down Expand Up @@ -485,6 +489,10 @@ private void buildDataLayoutOperationTaskListInParallel(
.doAfterTerminate(
() -> {
operationTaskManager.updateDataGenerationCompletion();
log.info(
"OH_SCHED_ELIGIBLE job_type={} count={}",
jobType,
operationTaskManager.getTotalDataCount());
log.info(
"The metadata fetched count: {} for the job type: {}",
operationTaskManager.getTotalDataCount(),
Expand Down Expand Up @@ -516,6 +524,10 @@ private void buildDatabaseLevelOperationTasksInParallel(
.doAfterTerminate(
() -> {
operationTaskManager.updateDataGenerationCompletion();
log.info(
"OH_SCHED_ELIGIBLE job_type={} count={}",
jobType,
operationTaskManager.getTotalDataCount());
log.info(
"The metadata fetched count: {} for the job type: {}",
operationTaskManager.getTotalDataCount(),
Expand Down
Loading