Skip to content

Commit b6093b3

Browse files
authored
GH-3914: Simplify and deprecate ContextUtil (#3916)
1 parent 087f9f3 commit b6093b3

23 files changed

Lines changed: 100 additions & 313 deletions

File tree

‎parquet-avro/src/main/java/org/apache/parquet/avro/AvroParquetInputFormat.java‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,6 @@
2121
import org.apache.avro.Schema;
2222
import org.apache.hadoop.mapreduce.Job;
2323
import org.apache.parquet.hadoop.ParquetInputFormat;
24-
import org.apache.parquet.hadoop.util.ContextUtil;
2524

2625
/**
2726
* A Hadoop {@link org.apache.hadoop.mapreduce.InputFormat} for Parquet files.
@@ -53,7 +52,7 @@ public AvroParquetInputFormat() {
5352
* @see org.apache.parquet.avro.AvroParquetOutputFormat#setSchema(org.apache.hadoop.mapreduce.Job, org.apache.avro.Schema)
5453
*/
5554
public static void setRequestedProjection(Job job, Schema requestedProjection) {
56-
AvroReadSupport.setRequestedProjection(ContextUtil.getConfiguration(job), requestedProjection);
55+
AvroReadSupport.setRequestedProjection(job.getConfiguration(), requestedProjection);
5756
}
5857

5958
/**
@@ -69,7 +68,7 @@ public static void setRequestedProjection(Job job, Schema requestedProjection) {
6968
* @see org.apache.parquet.avro.AvroParquetOutputFormat#setSchema(org.apache.hadoop.mapreduce.Job, org.apache.avro.Schema)
7069
*/
7170
public static void setAvroReadSchema(Job job, Schema avroReadSchema) {
72-
AvroReadSupport.setAvroReadSchema(ContextUtil.getConfiguration(job), avroReadSchema);
71+
AvroReadSupport.setAvroReadSchema(job.getConfiguration(), avroReadSchema);
7372
}
7473

7574
/**
@@ -81,6 +80,6 @@ public static void setAvroReadSchema(Job job, Schema avroReadSchema) {
8180
* @param supplierClass an avro data supplier class
8281
*/
8382
public static void setAvroDataSupplier(Job job, Class<? extends AvroDataSupplier> supplierClass) {
84-
AvroReadSupport.setAvroDataSupplier(ContextUtil.getConfiguration(job), supplierClass);
83+
AvroReadSupport.setAvroDataSupplier(job.getConfiguration(), supplierClass);
8584
}
8685
}

‎parquet-avro/src/main/java/org/apache/parquet/avro/AvroParquetOutputFormat.java‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,6 @@
2121
import org.apache.avro.Schema;
2222
import org.apache.hadoop.mapreduce.Job;
2323
import org.apache.parquet.hadoop.ParquetOutputFormat;
24-
import org.apache.parquet.hadoop.util.ContextUtil;
2524

2625
/**
2726
* A Hadoop {@link org.apache.hadoop.mapreduce.OutputFormat} for Parquet files.
@@ -39,7 +38,7 @@ public class AvroParquetOutputFormat<T> extends ParquetOutputFormat<T> {
3938
* @see org.apache.parquet.avro.AvroParquetInputFormat#setAvroReadSchema(org.apache.hadoop.mapreduce.Job, org.apache.avro.Schema)
4039
*/
4140
public static void setSchema(Job job, Schema schema) {
42-
AvroWriteSupport.setSchema(ContextUtil.getConfiguration(job), schema);
41+
AvroWriteSupport.setSchema(job.getConfiguration(), schema);
4342
}
4443

4544
public AvroParquetOutputFormat() {
@@ -55,6 +54,6 @@ public AvroParquetOutputFormat() {
5554
* @param supplierClass a supplier class
5655
*/
5756
public static void setAvroDataSupplier(Job job, Class<? extends AvroDataSupplier> supplierClass) {
58-
AvroWriteSupport.setAvroDataSupplier(ContextUtil.getConfiguration(job), supplierClass);
57+
AvroWriteSupport.setAvroDataSupplier(job.getConfiguration(), supplierClass);
5958
}
6059
}

‎parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetInputFormat.java‎

Lines changed: 8 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,6 @@
6464
import org.apache.parquet.hadoop.metadata.GlobalMetaData;
6565
import org.apache.parquet.hadoop.metadata.ParquetMetadata;
6666
import org.apache.parquet.hadoop.util.ConfigurationUtil;
67-
import org.apache.parquet.hadoop.util.ContextUtil;
6867
import org.apache.parquet.hadoop.util.HiddenFileFilter;
6968
import org.apache.parquet.hadoop.util.SerializationUtil;
7069
import org.apache.parquet.io.ParquetDecodingException;
@@ -173,19 +172,19 @@ public class ParquetInputFormat<T> extends FileInputFormat<Void, T> {
173172
public static final boolean HADOOP_VECTORED_IO_DEFAULT = true;
174173

175174
public static void setTaskSideMetaData(Job job, boolean taskSideMetadata) {
176-
ContextUtil.getConfiguration(job).setBoolean(TASK_SIDE_METADATA, taskSideMetadata);
175+
job.getConfiguration().setBoolean(TASK_SIDE_METADATA, taskSideMetadata);
177176
}
178177

179178
public static boolean isTaskSideMetaData(Configuration configuration) {
180179
return configuration.getBoolean(TASK_SIDE_METADATA, TRUE);
181180
}
182181

183182
public static void setReadSupportClass(Job job, Class<?> readSupportClass) {
184-
ContextUtil.getConfiguration(job).set(READ_SUPPORT_CLASS, readSupportClass.getName());
183+
job.getConfiguration().set(READ_SUPPORT_CLASS, readSupportClass.getName());
185184
}
186185

187186
public static void setUnboundRecordFilter(Job job, Class<? extends UnboundRecordFilter> filterClass) {
188-
Configuration conf = ContextUtil.getConfiguration(job);
187+
Configuration conf = job.getConfiguration();
189188
checkArgument(
190189
getFilterPredicate(conf) == null,
191190
"You cannot provide an UnboundRecordFilter after providing a FilterPredicate");
@@ -295,7 +294,7 @@ public <S extends ReadSupport<T>> ParquetInputFormat(Class<S> readSupportClass)
295294
@Override
296295
public RecordReader<Void, T> createRecordReader(InputSplit inputSplit, TaskAttemptContext taskAttemptContext)
297296
throws IOException, InterruptedException {
298-
Configuration conf = ContextUtil.getConfiguration(taskAttemptContext);
297+
Configuration conf = taskAttemptContext.getConfiguration();
299298
ReadSupport<T> readSupport = getReadSupport(conf);
300299
return new ParquetRecordReader<T>(readSupport, getFilter(conf));
301300
}
@@ -340,15 +339,15 @@ static <T> ReadSupport<T> getReadSupportInstance(Class<? extends ReadSupport<T>>
340339

341340
@Override
342341
protected boolean isSplitable(JobContext context, Path filename) {
343-
return ContextUtil.getConfiguration(context).getBoolean(SPLIT_FILES, true);
342+
return context.getConfiguration().getBoolean(SPLIT_FILES, true);
344343
}
345344

346345
/**
347346
* {@inheritDoc}
348347
*/
349348
@Override
350349
public List<InputSplit> getSplits(JobContext jobContext) throws IOException {
351-
Configuration configuration = ContextUtil.getConfiguration(jobContext);
350+
Configuration configuration = jobContext.getConfiguration();
352351
List<InputSplit> splits = new ArrayList<InputSplit>();
353352

354353
if (isTaskSideMetaData(configuration)) {
@@ -396,7 +395,7 @@ public List<ParquetInputSplit> getSplits(Configuration configuration, List<Foote
396395
*/
397396
@Override
398397
protected List<FileStatus> listStatus(JobContext jobContext) throws IOException {
399-
return getAllFileRecursively(super.listStatus(jobContext), ContextUtil.getConfiguration(jobContext));
398+
return getAllFileRecursively(super.listStatus(jobContext), jobContext.getConfiguration());
400399
}
401400

402401
private static List<FileStatus> getAllFileRecursively(List<FileStatus> files, Configuration conf)
@@ -436,7 +435,7 @@ public List<Footer> getFooters(JobContext jobContext) throws IOException {
436435
if (statuses.isEmpty()) {
437436
return Collections.emptyList();
438437
}
439-
Configuration config = ContextUtil.getConfiguration(jobContext);
438+
Configuration config = jobContext.getConfiguration();
440439
// Use LinkedHashMap to preserve the insertion order and ultimately to return the list of
441440
// footers in the same order as the list of file statuses returned from listStatus()
442441
Map<FileStatusWrapper, Footer> footersMap = new LinkedHashMap<FileStatusWrapper, Footer>();

‎parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetOutputCommitter.java‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,6 @@
2828
import org.apache.hadoop.mapreduce.TaskAttemptContext;
2929
import org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter;
3030
import org.apache.parquet.hadoop.ParquetOutputFormat.JobSummaryLevel;
31-
import org.apache.parquet.hadoop.util.ContextUtil;
3231
import org.slf4j.Logger;
3332
import org.slf4j.LoggerFactory;
3433

@@ -44,7 +43,7 @@ public ParquetOutputCommitter(Path outputPath, TaskAttemptContext context) throw
4443

4544
public void commitJob(JobContext jobContext) throws IOException {
4645
super.commitJob(jobContext);
47-
Configuration configuration = ContextUtil.getConfiguration(jobContext);
46+
Configuration configuration = jobContext.getConfiguration();
4847
writeMetaDataFile(configuration, outputPath);
4948
}
5049

‎parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetOutputFormat.java‎

Lines changed: 26 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,6 @@
2121
import static org.apache.parquet.column.ParquetProperties.DEFAULT_ADAPTIVE_BLOOM_FILTER_ENABLED;
2222
import static org.apache.parquet.column.ParquetProperties.DEFAULT_BLOOM_FILTER_ENABLED;
2323
import static org.apache.parquet.hadoop.ParquetWriter.DEFAULT_BLOCK_SIZE;
24-
import static org.apache.parquet.hadoop.util.ContextUtil.getConfiguration;
2524

2625
import java.io.IOException;
2726
import java.util.Objects;
@@ -190,7 +189,7 @@ public static JobSummaryLevel getJobSummaryLevel(Configuration conf) {
190189
}
191190

192191
public static void setWriteSupportClass(Job job, Class<?> writeSupportClass) {
193-
getConfiguration(job).set(WRITE_SUPPORT_CLASS, writeSupportClass.getName());
192+
job.getConfiguration().set(WRITE_SUPPORT_CLASS, writeSupportClass.getName());
194193
}
195194

196195
public static void setWriteSupportClass(JobConf job, Class<?> writeSupportClass) {
@@ -208,35 +207,35 @@ public static Class<?> getWriteSupportClass(Configuration configuration) {
208207
}
209208

210209
public static void setBlockSize(Job job, int blockSize) {
211-
getConfiguration(job).setInt(BLOCK_SIZE, blockSize);
210+
job.getConfiguration().setInt(BLOCK_SIZE, blockSize);
212211
}
213212

214213
public static void setPageSize(Job job, int pageSize) {
215-
getConfiguration(job).setInt(PAGE_SIZE, pageSize);
214+
job.getConfiguration().setInt(PAGE_SIZE, pageSize);
216215
}
217216

218217
public static void setDictionaryPageSize(Job job, int pageSize) {
219-
getConfiguration(job).setInt(DICTIONARY_PAGE_SIZE, pageSize);
218+
job.getConfiguration().setInt(DICTIONARY_PAGE_SIZE, pageSize);
220219
}
221220

222221
public static void setCompression(Job job, CompressionCodecName compression) {
223-
getConfiguration(job).set(COMPRESSION, compression.name());
222+
job.getConfiguration().set(COMPRESSION, compression.name());
224223
}
225224

226225
public static void setColumnCompression(Job job, String columnPath, CompressionCodecName codec) {
227-
getConfiguration(job).set(COMPRESSION + '#' + columnPath, codec.name());
226+
job.getConfiguration().set(COMPRESSION + '#' + columnPath, codec.name());
228227
}
229228

230229
public static void setColumnCompressionLevel(Job job, String columnPath, int level) {
231-
getConfiguration(job).setInt(COLUMN_COMPRESSION_LEVEL_PREFIX + '#' + columnPath, level);
230+
job.getConfiguration().setInt(COLUMN_COMPRESSION_LEVEL_PREFIX + '#' + columnPath, level);
232231
}
233232

234233
public static void setEnableDictionary(Job job, boolean enableDictionary) {
235-
getConfiguration(job).setBoolean(ENABLE_DICTIONARY, enableDictionary);
234+
job.getConfiguration().setBoolean(ENABLE_DICTIONARY, enableDictionary);
236235
}
237236

238237
public static boolean getEnableDictionary(JobContext jobContext) {
239-
return getEnableDictionary(getConfiguration(jobContext));
238+
return getEnableDictionary(jobContext.getConfiguration());
240239
}
241240

242241
public static int getBloomFilterMaxBytes(Configuration conf) {
@@ -252,31 +251,31 @@ public static boolean getAdaptiveBloomFilterEnabled(Configuration conf) {
252251
}
253252

254253
public static int getBlockSize(JobContext jobContext) {
255-
return getBlockSize(getConfiguration(jobContext));
254+
return getBlockSize(jobContext.getConfiguration());
256255
}
257256

258257
public static int getPageSize(JobContext jobContext) {
259-
return getPageSize(getConfiguration(jobContext));
258+
return getPageSize(jobContext.getConfiguration());
260259
}
261260

262261
public static int getDictionaryPageSize(JobContext jobContext) {
263-
return getDictionaryPageSize(getConfiguration(jobContext));
262+
return getDictionaryPageSize(jobContext.getConfiguration());
264263
}
265264

266265
public static CompressionCodecName getCompression(JobContext jobContext) {
267-
return getCompression(getConfiguration(jobContext));
266+
return getCompression(jobContext.getConfiguration());
268267
}
269268

270269
public static boolean isCompressionSet(JobContext jobContext) {
271-
return isCompressionSet(getConfiguration(jobContext));
270+
return isCompressionSet(jobContext.getConfiguration());
272271
}
273272

274273
public static void setValidation(JobContext jobContext, boolean validating) {
275-
setValidation(getConfiguration(jobContext), validating);
274+
setValidation(jobContext.getConfiguration(), validating);
276275
}
277276

278277
public static boolean getValidation(JobContext jobContext) {
279-
return getValidation(getConfiguration(jobContext));
278+
return getValidation(jobContext.getConfiguration());
280279
}
281280

282281
public static boolean getEnableDictionary(Configuration configuration) {
@@ -350,7 +349,7 @@ private CompressionCodecName getCodec(TaskAttemptContext taskAttemptContext) {
350349
}
351350

352351
public static void setMaxPaddingSize(JobContext jobContext, int maxPaddingSize) {
353-
setMaxPaddingSize(getConfiguration(jobContext), maxPaddingSize);
352+
setMaxPaddingSize(jobContext.getConfiguration(), maxPaddingSize);
354353
}
355354

356355
public static void setMaxPaddingSize(Configuration conf, int maxPaddingSize) {
@@ -362,7 +361,7 @@ private static int getMaxPaddingSize(Configuration conf) {
362361
}
363362

364363
public static void setColumnIndexTruncateLength(JobContext jobContext, int length) {
365-
setColumnIndexTruncateLength(getConfiguration(jobContext), length);
364+
setColumnIndexTruncateLength(jobContext.getConfiguration(), length);
366365
}
367366

368367
public static void setColumnIndexTruncateLength(Configuration conf, int length) {
@@ -374,7 +373,7 @@ private static int getColumnIndexTruncateLength(Configuration conf) {
374373
}
375374

376375
public static void setStatisticsTruncateLength(JobContext jobContext, int length) {
377-
setStatisticsTruncateLength(getConfiguration(jobContext), length);
376+
setStatisticsTruncateLength(jobContext.getConfiguration(), length);
378377
}
379378

380379
private static void setStatisticsTruncateLength(Configuration conf, int length) {
@@ -386,7 +385,7 @@ private static int getStatisticsTruncateLength(Configuration conf) {
386385
}
387386

388387
public static void setBlockRowCountLimit(JobContext jobContext, int rowCount) {
389-
setBlockRowCountLimit(getConfiguration(jobContext), rowCount);
388+
setBlockRowCountLimit(jobContext.getConfiguration(), rowCount);
390389
}
391390

392391
public static void setBlockRowCountLimit(Configuration conf, int rowCount) {
@@ -398,7 +397,7 @@ static int getBlockRowCountLimit(Configuration conf) {
398397
}
399398

400399
public static void setPageRowCountLimit(JobContext jobContext, int rowCount) {
401-
setPageRowCountLimit(getConfiguration(jobContext), rowCount);
400+
setPageRowCountLimit(jobContext.getConfiguration(), rowCount);
402401
}
403402

404403
public static void setPageRowCountLimit(Configuration conf, int rowCount) {
@@ -410,7 +409,7 @@ private static int getPageRowCountLimit(Configuration conf) {
410409
}
411410

412411
public static void setPageWriteChecksumEnabled(JobContext jobContext, boolean val) {
413-
setPageWriteChecksumEnabled(getConfiguration(jobContext), val);
412+
setPageWriteChecksumEnabled(jobContext.getConfiguration(), val);
414413
}
415414

416415
public static void setPageWriteChecksumEnabled(Configuration conf, boolean val) {
@@ -422,15 +421,15 @@ public static boolean getPageWriteChecksumEnabled(Configuration conf) {
422421
}
423422

424423
public static void setStatisticsEnabled(JobContext jobContext, boolean enabled) {
425-
getConfiguration(jobContext).setBoolean(STATISTICS_ENABLED, enabled);
424+
jobContext.getConfiguration().setBoolean(STATISTICS_ENABLED, enabled);
426425
}
427426

428427
public static boolean getStatisticsEnabled(Configuration conf) {
429428
return conf.getBoolean(STATISTICS_ENABLED, ParquetProperties.DEFAULT_STATISTICS_ENABLED);
430429
}
431430

432431
public static void setStatisticsEnabled(JobContext jobContext, String columnPath, boolean enabled) {
433-
getConfiguration(jobContext).set(STATISTICS_ENABLED + "#" + columnPath, String.valueOf(enabled));
432+
jobContext.getConfiguration().set(STATISTICS_ENABLED + "#" + columnPath, String.valueOf(enabled));
434433
}
435434

436435
public static boolean getStatisticsEnabled(Configuration conf, String columnPath) {
@@ -490,7 +489,7 @@ public RecordWriter<Void, T> getRecordWriter(TaskAttemptContext taskAttemptConte
490489
public RecordWriter<Void, T> getRecordWriter(TaskAttemptContext taskAttemptContext, Mode mode)
491490
throws IOException, InterruptedException {
492491

493-
final Configuration conf = getConfiguration(taskAttemptContext);
492+
final Configuration conf = taskAttemptContext.getConfiguration();
494493

495494
CompressionCodecName codec = getCodec(taskAttemptContext);
496495
String extension = codec.getExtension() + ".parquet";
@@ -505,7 +504,7 @@ public RecordWriter<Void, T> getRecordWriter(TaskAttemptContext taskAttemptConte
505504

506505
public RecordWriter<Void, T> getRecordWriter(TaskAttemptContext taskAttemptContext, Path file, Mode mode)
507506
throws IOException, InterruptedException {
508-
return getRecordWriter(getConfiguration(taskAttemptContext), file, getCodec(taskAttemptContext), mode);
507+
return getRecordWriter(taskAttemptContext.getConfiguration(), file, getCodec(taskAttemptContext), mode);
509508
}
510509

511510
public RecordWriter<Void, T> getRecordWriter(Configuration conf, Path file, CompressionCodecName codec)

‎parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetRecordReader.java‎

Lines changed: 2 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -43,12 +43,9 @@
4343
import org.apache.parquet.hadoop.metadata.BlockMetaData;
4444
import org.apache.parquet.hadoop.metadata.ColumnChunkMetaData;
4545
import org.apache.parquet.hadoop.metadata.FileMetaData;
46-
import org.apache.parquet.hadoop.util.ContextUtil;
4746
import org.apache.parquet.hadoop.util.HadoopInputFile;
4847
import org.apache.parquet.hadoop.util.counters.BenchmarkCounter;
4948
import org.apache.parquet.io.ParquetDecodingException;
50-
import org.slf4j.Logger;
51-
import org.slf4j.LoggerFactory;
5249

5350
/**
5451
* Reads the records from a block of a Parquet file
@@ -59,7 +56,6 @@
5956
*/
6057
public class ParquetRecordReader<T> extends RecordReader<Void, T> {
6158

62-
private static final Logger LOG = LoggerFactory.getLogger(ParquetRecordReader.class);
6359
private final InternalParquetRecordReader<T> internalReader;
6460

6561
/**
@@ -125,15 +121,9 @@ public float getProgress() throws IOException, InterruptedException {
125121
@Override
126122
public void initialize(InputSplit inputSplit, TaskAttemptContext context) throws IOException, InterruptedException {
127123

128-
if (ContextUtil.hasCounterMethod(context)) {
129-
BenchmarkCounter.initCounterFromContext(context);
130-
} else {
131-
LOG.error(String.format(
132-
"Can not initialize counter because the class '%s' does not have a '.getCounterMethod'",
133-
context.getClass().getCanonicalName()));
134-
}
124+
BenchmarkCounter.initCounterFromContext(context);
135125

136-
initializeInternalReader(toParquetSplit(inputSplit), ContextUtil.getConfiguration(context));
126+
initializeInternalReader(toParquetSplit(inputSplit), context.getConfiguration());
137127
}
138128

139129
public void initialize(InputSplit inputSplit, Configuration configuration, Reporter reporter)

‎parquet-hadoop/src/main/java/org/apache/parquet/hadoop/codec/CodecConfig.java‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,6 @@
2626
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
2727
import org.apache.parquet.hadoop.ParquetOutputFormat;
2828
import org.apache.parquet.hadoop.metadata.CompressionCodecName;
29-
import org.apache.parquet.hadoop.util.ContextUtil;
3029
import org.slf4j.Logger;
3130
import org.slf4j.LoggerFactory;
3231

@@ -141,7 +140,7 @@ public Class getHadoopOutputCompressorClass(Class defaultCodec) {
141140

142141
@Override
143142
public Configuration getConfiguration() {
144-
return ContextUtil.getConfiguration(context);
143+
return context.getConfiguration();
145144
}
146145
}
147146

0 commit comments

Comments
 (0)