@@ -152,6 +152,35 @@ private double allowedMaxMemorySizeInBytesOfTsTiles(
152152 * totalNonFloatingMemorySizeInBytes ;
153153 }
154154
155+ private boolean isEnoughForTsFileParserMemory (
156+ final long extraMemoryInBytes , final long totalNonFloatingMemorySizeInBytes ) {
157+ return (double ) usedMemorySizeInBytesOfTablets
158+ + (double ) usedMemorySizeInBytesOfTsFiles
159+ + (double ) extraMemoryInBytes
160+ < EXCEED_PROTECT_THRESHOLD
161+ * allowedMaxMemorySizeInBytesOfTabletsAndTsFiles (totalNonFloatingMemorySizeInBytes );
162+ }
163+
164+ /** Reports both global availability and the category limits that can reject a resize. */
165+ public synchronized String getResizeFailureMessage (final int retries , final long requestedSize ) {
166+ final long totalNonFloatingMemorySizeInBytes = getTotalNonFloatingMemorySizeInBytes ();
167+ final double combinedLimit =
168+ allowedMaxMemorySizeInBytesOfTabletsAndTsFiles (totalNonFloatingMemorySizeInBytes );
169+ return String .format (
170+ DataNodePipeMessages
171+ .EXCEPTION_FORCERESIZE_FAILED_AFTER_ARG_RETRIES_TOTAL_ARG_BYTES_USED_ARG_BYTES_USEDTABLETS_ARG_BYTES_USEDTSFILES_ARG_BYTES_TABLETLIMIT_ARG_BYTES_TSFILELIMIT_ARG_BYTES_TABLETANDTSFILELIMIT_ARG_BYTES_PROTECTEDPARSERLIMIT_ARG_BYTES_REQUESTED_ARG_BYTES_311C8C7E ,
172+ retries ,
173+ totalNonFloatingMemorySizeInBytes ,
174+ memoryBlock .getUsedMemoryInBytes (),
175+ usedMemorySizeInBytesOfTablets ,
176+ usedMemorySizeInBytesOfTsFiles ,
177+ Math .round (allowedMaxMemorySizeInBytesOfTablets (totalNonFloatingMemorySizeInBytes )),
178+ Math .round (allowedMaxMemorySizeInBytesOfTsTiles (totalNonFloatingMemorySizeInBytes )),
179+ Math .round (combinedLimit ),
180+ Math .round (EXCEED_PROTECT_THRESHOLD * combinedLimit ),
181+ requestedSize );
182+ }
183+
155184 private static long getTsFileParserMemorySizeInBytes () {
156185 return Math .max (
157186 PIPE_CONFIG .getTsFileParserMemory (), PIPE_CONFIG .getPipeMemoryAllocateMinSizeInBytes ());
@@ -556,6 +585,10 @@ < allowedMaxMemorySizeInBytesOfTabletsAndTsFiles(totalNonFloatingMemorySizeInByt
556585
557586 private boolean isHardEnoughForResizing (
558587 final PipeMemoryBlock block , final long extraMemoryInBytes ) {
588+ if (block instanceof PipeTsFileParserMemoryBlock ) {
589+ return isEnoughForTsFileParserMemory (
590+ extraMemoryInBytes , getTotalNonFloatingMemorySizeInBytes ());
591+ }
559592 if (!(block instanceof PipeTabletMemoryBlock ) && !(block instanceof PipeTsFileMemoryBlock )) {
560593 return true ;
561594 }
@@ -757,6 +790,39 @@ public PipeTabletMemoryBlock forceAllocateForTabletWithRetry(
757790 name , tabletSizeInBytes , PipeMemoryBlockCategory .EVENT_CHILD , null , parent );
758791 }
759792
793+ /** Allocates TsFile parser memory against the protected tablet + TsFile combined quota. */
794+ public PipeTsFileParserMemoryBlock forceAllocateForTsFileParserWithRetry (
795+ final String name , final long parserMemorySizeInBytes , final Object assigner ) {
796+ return forceAllocateForTsFileParserWithRetry (name , parserMemorySizeInBytes , assigner , null );
797+ }
798+
799+ /** Allocates TsFile parser memory as an event child against the combined quota. */
800+ public synchronized PipeTsFileParserMemoryBlock forceAllocateForTsFileParserWithRetry (
801+ final String name ,
802+ final long parserMemorySizeInBytes ,
803+ final Object assigner ,
804+ final PipeMemoryBlock parent ) {
805+ final PipeMemoryBlockCategory category =
806+ parent == null ? PipeMemoryBlockCategory .PARSER : PipeMemoryBlockCategory .EVENT_CHILD ;
807+ final PipeTsFileParserMemoryBlock block =
808+ (PipeTsFileParserMemoryBlock )
809+ registerMemoryBlock (
810+ name , 0 , PipeMemoryBlockType .TS_FILE_PARSER , category , assigner , parent );
811+ if (!PIPE_MEMORY_MANAGEMENT_ENABLED || parserMemorySizeInBytes == 0 ) {
812+ return block ;
813+ }
814+
815+ try {
816+ // Initial allocation and subsequent growth must check the same post-allocation usage
817+ // under the manager lock, including after a wait releases that lock.
818+ forceResize (block , parserMemorySizeInBytes );
819+ return block ;
820+ } catch (final RuntimeException e ) {
821+ release (block );
822+ throw e ;
823+ }
824+ }
825+
760826 public PipeTsFileMemoryBlock forceAllocateForTsFileWithRetry (
761827 final String name , final long tsFileSizeInBytes )
762828 throws PipeRuntimeOutOfMemoryCriticalException {
@@ -1001,17 +1067,14 @@ public synchronized void resize(
10011067
10021068 if (force ) {
10031069 throw new PipeRuntimeOutOfMemoryCriticalException (
1004- String .format (
1005- DataNodePipeMessages
1006- .PIPE_EXCEPTION_FORCERESIZE_FAILED_TO_ALLOCATE_MEMORY_AFTER_D_RETRIES_TOTAL_8C6948BC ,
1007- memoryAllocateMaxRetries ,
1008- getTotalNonFloatingMemorySizeInBytes (),
1009- memoryBlock .getUsedMemoryInBytes (),
1010- sizeInBytes ));
1070+ getResizeFailureMessage (memoryAllocateMaxRetries , sizeInBytes ));
10111071 }
10121072 }
10131073
10141074 private boolean tryResizeInternal (final PipeMemoryBlock block , final long targetSize ) {
1075+ if (block .isReleased ()) {
1076+ return false ;
1077+ }
10151078 // Parent blocks expose an aggregate usage. Do not let a direct resize reduce that aggregate
10161079 // below the bytes still owned by live children.
10171080 final long normalizedTargetSize =
@@ -1249,6 +1312,11 @@ private synchronized PipeMemoryBlock registerMemoryBlock(
12491312 returnedMemoryBlock =
12501313 new PipeTabletMemoryBlock (this , name , 0 , normalizedCategory , null , normalizedParent );
12511314 break ;
1315+ case TS_FILE_PARSER :
1316+ returnedMemoryBlock =
1317+ new PipeTsFileParserMemoryBlock (
1318+ this , name , 0 , normalizedCategory , null , normalizedParent );
1319+ break ;
12521320 case TS_FILE :
12531321 returnedMemoryBlock =
12541322 new PipeTsFileMemoryBlock (this , name , 0 , normalizedCategory , null , normalizedParent );
0 commit comments