Skip to content

Commit 044b558

Browse files
CaideyipiSteveYurongSu
Caideyipi
and
SteveYurongSu
committed
Pipe: Added metrics for batch & handler size & compression ratio & compression time for iotdb connector (apache#15397)
Co-authored-by: Steve Yurong Su <rong@apache.org>
1 parent 41dea10 commit 044b558

14 files changed

Lines changed: 274 additions & 39 deletions

File tree

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/connector/PipeConnectorSubtask.java‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
import org.apache.iotdb.commons.pipe.event.EnrichedEvent;
2828
import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent;
2929
import org.apache.iotdb.db.pipe.connector.protocol.thrift.async.IoTDBDataRegionAsyncConnector;
30+
import org.apache.iotdb.db.pipe.connector.protocol.thrift.sync.IoTDBDataRegionSyncConnector;
3031
import org.apache.iotdb.db.pipe.event.UserDefinedEnrichedEvent;
3132
import org.apache.iotdb.db.pipe.event.common.heartbeat.PipeHeartbeatEvent;
3233
import org.apache.iotdb.db.pipe.event.common.schema.PipeSchemaRegionWritePlanEvent;
@@ -304,6 +305,34 @@ public int getAsyncConnectorRetryEventQueueSize() {
304305
: 0;
305306
}
306307

308+
public int getPendingHandlersSize() {
309+
return outputPipeConnector instanceof IoTDBDataRegionAsyncConnector
310+
? ((IoTDBDataRegionAsyncConnector) outputPipeConnector).getPendingHandlersSize()
311+
: 0;
312+
}
313+
314+
public int getBatchSize() {
315+
if (outputPipeConnector instanceof IoTDBDataRegionAsyncConnector) {
316+
return ((IoTDBDataRegionAsyncConnector) outputPipeConnector).getBatchSize();
317+
}
318+
if (outputPipeConnector instanceof IoTDBDataRegionSyncConnector) {
319+
return ((IoTDBDataRegionSyncConnector) outputPipeConnector).getBatchSize();
320+
}
321+
return 0;
322+
}
323+
324+
public double getTotalUncompressedSize() {
325+
return outputPipeConnector instanceof IoTDBConnector
326+
? ((IoTDBConnector) outputPipeConnector).getTotalUncompressedSize()
327+
: 0;
328+
}
329+
330+
public double getTotalCompressedSize() {
331+
return outputPipeConnector instanceof IoTDBConnector
332+
? ((IoTDBConnector) outputPipeConnector).getTotalCompressedSize()
333+
: 0;
334+
}
335+
307336
//////////////////////////// Error report ////////////////////////////
308337

309338
@Override

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/connector/PipeConnectorSubtaskManager.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -104,6 +104,7 @@ public synchronized String register(
104104
connectorNum = 1;
105105
attributeSortedString = "schema_" + attributeSortedString;
106106
}
107+
environment.setAttributeSortedString(attributeSortedString);
107108

108109
if (!attributeSortedString2SubtaskLifeCycleMap.containsKey(attributeSortedString)) {
109110
final List<PipeConnectorSubtaskLifeCycle> pipeConnectorSubtaskLifeCycleList =

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/batch/PipeTabletEventPlainBatch.java‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -32,8 +32,6 @@
3232
import org.apache.tsfile.utils.Pair;
3333
import org.apache.tsfile.utils.PublicBAOS;
3434
import org.apache.tsfile.utils.ReadWriteIOUtils;
35-
import org.slf4j.Logger;
36-
import org.slf4j.LoggerFactory;
3735

3836
import java.io.DataOutputStream;
3937
import java.io.IOException;
@@ -46,8 +44,6 @@
4644

4745
public class PipeTabletEventPlainBatch extends PipeTabletEventBatch {
4846

49-
private static final Logger LOGGER = LoggerFactory.getLogger(PipeTabletEventPlainBatch.class);
50-
5147
private final List<ByteBuffer> binaryBuffers = new ArrayList<>();
5248
private final List<ByteBuffer> insertNodeBuffers = new ArrayList<>();
5349
private final List<ByteBuffer> tabletBuffers = new ArrayList<>();

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/batch/PipeTransferBatchReqBuilder.java‎

Lines changed: 18 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -36,10 +36,10 @@
3636
import java.io.IOException;
3737
import java.util.ArrayList;
3838
import java.util.Arrays;
39-
import java.util.HashMap;
4039
import java.util.List;
4140
import java.util.Map;
4241
import java.util.Objects;
42+
import java.util.concurrent.ConcurrentHashMap;
4343

4444
import static org.apache.iotdb.commons.pipe.config.constant.PipeConnectorConstant.CONNECTOR_FORMAT_HYBRID_VALUE;
4545
import static org.apache.iotdb.commons.pipe.config.constant.PipeConnectorConstant.CONNECTOR_FORMAT_KEY;
@@ -74,7 +74,8 @@ public class PipeTransferBatchReqBuilder implements AutoCloseable {
7474
// If the leader cache is enabled, the batch will be divided by the leader endpoint,
7575
// each endpoint has a batch.
7676
// This is only used in plain batch since tsfile does not return redirection info.
77-
private final Map<TEndPoint, PipeTabletEventPlainBatch> endPointToBatch = new HashMap<>();
77+
private final Map<TEndPoint, PipeTabletEventPlainBatch> endPointToBatch =
78+
new ConcurrentHashMap<>();
7879

7980
public PipeTransferBatchReqBuilder(final PipeParameters parameters) {
8081
final boolean usingTsFileBatch =
@@ -186,6 +187,21 @@ public synchronized void discardEventsOfPipe(final String pipeNameToDrop, final
186187
endPointToBatch.values().forEach(batch -> batch.discardEventsOfPipe(pipeNameToDrop, regionId));
187188
}
188189

190+
public int size() {
191+
try {
192+
return defaultBatch.events.size()
193+
+ endPointToBatch.values().stream()
194+
.map(batch -> batch.events.size())
195+
.reduce(0, Integer::sum);
196+
} catch (final Exception e) {
197+
LOGGER.warn(
198+
"Failed to get the size of PipeTransferBatchReqBuilder, return 0. Exception: {}",
199+
e.getMessage(),
200+
e);
201+
return 0;
202+
}
203+
}
204+
189205
@Override
190206
public synchronized void close() {
191207
defaultBatch.close();

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/airgap/IoTDBDataRegionAirGapConnector.java‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434
import org.apache.iotdb.db.pipe.event.common.tablet.PipeRawTabletInsertionEvent;
3535
import org.apache.iotdb.db.pipe.event.common.terminate.PipeTerminateEvent;
3636
import org.apache.iotdb.db.pipe.event.common.tsfile.PipeTsFileInsertionEvent;
37+
import org.apache.iotdb.db.pipe.metric.sink.PipeDataRegionConnectorMetrics;
3738
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertNode;
3839
import org.apache.iotdb.db.storageengine.dataregion.wal.exception.WALPipeException;
3940
import org.apache.iotdb.pipe.api.event.Event;
@@ -304,4 +305,13 @@ protected byte[] getTransferMultiFilePieceBytes(
304305
final String fileName, final long position, final byte[] payLoad) throws IOException {
305306
return PipeTransferTsFilePieceWithModReq.toTPipeTransferBytes(fileName, position, payLoad);
306307
}
308+
309+
@Override
310+
protected byte[] compressIfNeeded(final byte[] reqInBytes) throws IOException {
311+
if (Objects.isNull(compressionTimer)) {
312+
compressionTimer =
313+
PipeDataRegionConnectorMetrics.getInstance().getCompressionTimer(attributeSortedString);
314+
}
315+
return super.compressIfNeeded(reqInBytes);
316+
}
307317
}

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/async/IoTDBDataRegionAsyncConnector.java‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@
4444
import org.apache.iotdb.db.pipe.event.common.tablet.PipeRawTabletInsertionEvent;
4545
import org.apache.iotdb.db.pipe.event.common.terminate.PipeTerminateEvent;
4646
import org.apache.iotdb.db.pipe.event.common.tsfile.PipeTsFileInsertionEvent;
47+
import org.apache.iotdb.db.pipe.metric.sink.PipeDataRegionConnectorMetrics;
4748
import org.apache.iotdb.db.pipe.metric.source.PipeDataRegionEventCounter;
4849
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertNode;
4950
import org.apache.iotdb.pipe.api.PipeConnector;
@@ -413,6 +414,15 @@ private void transferBatchedEventsIfNecessary()
413414
}
414415
}
415416

417+
@Override
418+
public TPipeTransferReq compressIfNeeded(final TPipeTransferReq req) throws IOException {
419+
if (Objects.isNull(compressionTimer)) {
420+
compressionTimer =
421+
PipeDataRegionConnectorMetrics.getInstance().getCompressionTimer(attributeSortedString);
422+
}
423+
return super.compressIfNeeded(req);
424+
}
425+
416426
//////////////////////////// Leader cache update ////////////////////////////
417427

418428
public void updateLeaderCache(final String deviceId, final TEndPoint endPoint) {
@@ -682,6 +692,14 @@ public int getRetryEventQueueSize() {
682692
return retryEventQueue.size();
683693
}
684694

695+
public int getBatchSize() {
696+
return tabletBatchBuilder.size();
697+
}
698+
699+
public int getPendingHandlersSize() {
700+
return pendingHandlers.size();
701+
}
702+
685703
//////////////////////// APIs provided for PipeTransferTrackableHandler ////////////////////////
686704

687705
public boolean isClosed() {

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/async/handler/PipeTransferTabletBatchEventHandler.java‎

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,6 @@
2222
import org.apache.iotdb.common.rpc.thrift.TEndPoint;
2323
import org.apache.iotdb.common.rpc.thrift.TSStatus;
2424
import org.apache.iotdb.commons.client.async.AsyncPipeDataTransferServiceClient;
25-
import org.apache.iotdb.commons.pipe.connector.payload.thrift.request.PipeTransferCompressedReq;
2625
import org.apache.iotdb.commons.pipe.event.EnrichedEvent;
2726
import org.apache.iotdb.db.pipe.connector.payload.evolvable.batch.PipeTabletEventPlainBatch;
2827
import org.apache.iotdb.db.pipe.connector.protocol.thrift.async.IoTDBDataRegionAsyncConnector;
@@ -63,11 +62,7 @@ public PipeTransferTabletBatchEventHandler(
6362
pipeName2BytesAccumulated = batch.deepCopyPipeName2BytesAccumulated();
6463

6564
final TPipeTransferReq uncompressedReq = batch.toTPipeTransferReq();
66-
req =
67-
connector.isRpcCompressionEnabled()
68-
? PipeTransferCompressedReq.toTPipeTransferReq(
69-
uncompressedReq, connector.getCompressors())
70-
: uncompressedReq;
65+
req = connector.compressIfNeeded(uncompressedReq);
7166
reqCompressionRatio = (double) req.getBody().length / uncompressedReq.getBody().length;
7267
}
7368

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/async/handler/PipeTransferTsFileHandler.java‎

Lines changed: 8 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,6 @@
2222
import org.apache.iotdb.common.rpc.thrift.TSStatus;
2323
import org.apache.iotdb.commons.client.async.AsyncPipeDataTransferServiceClient;
2424
import org.apache.iotdb.commons.pipe.config.PipeConfig;
25-
import org.apache.iotdb.commons.pipe.connector.payload.thrift.request.PipeTransferCompressedReq;
2625
import org.apache.iotdb.commons.pipe.connector.payload.thrift.response.PipeTransferFilePieceResp;
2726
import org.apache.iotdb.commons.pipe.event.EnrichedEvent;
2827
import org.apache.iotdb.commons.utils.RetryUtils;
@@ -179,13 +178,13 @@ public void transfer(
179178
final TPipeTransferReq uncompressedReq =
180179
transferMod
181180
? PipeTransferTsFileSealWithModReq.toTPipeTransferReq(
182-
modFile.getName(), modFile.length(), tsFile.getName(), tsFile.length())
183-
: PipeTransferTsFileSealReq.toTPipeTransferReq(tsFile.getName(), tsFile.length());
184-
final TPipeTransferReq req =
185-
connector.isRpcCompressionEnabled()
186-
? PipeTransferCompressedReq.toTPipeTransferReq(
187-
uncompressedReq, connector.getCompressors())
188-
: uncompressedReq;
181+
modFile.getName(),
182+
modFile.length(),
183+
tsFile.getName(),
184+
tsFile.length())
185+
: PipeTransferTsFileSealReq.toTPipeTransferReq(
186+
tsFile.getName(), tsFile.length());
187+
final TPipeTransferReq req = connector.compressIfNeeded(uncompressedReq);
189188

190189
pipeName2WeightMap.forEach(
191190
(pipePair, weight) ->
@@ -212,11 +211,7 @@ public void transfer(
212211
currentFile.getName(), position, payload)
213212
: PipeTransferTsFilePieceReq.toTPipeTransferReq(
214213
currentFile.getName(), position, payload);
215-
final TPipeTransferReq req =
216-
connector.isRpcCompressionEnabled()
217-
? PipeTransferCompressedReq.toTPipeTransferReq(
218-
uncompressedReq, connector.getCompressors())
219-
: uncompressedReq;
214+
final TPipeTransferReq req = connector.compressIfNeeded(uncompressedReq);
220215

221216
pipeName2WeightMap.forEach(
222217
(pipePair, weight) ->

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/sync/IoTDBDataRegionSyncConnector.java‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@
4343
import org.apache.iotdb.db.pipe.event.common.tablet.PipeRawTabletInsertionEvent;
4444
import org.apache.iotdb.db.pipe.event.common.terminate.PipeTerminateEvent;
4545
import org.apache.iotdb.db.pipe.event.common.tsfile.PipeTsFileInsertionEvent;
46+
import org.apache.iotdb.db.pipe.metric.sink.PipeDataRegionConnectorMetrics;
4647
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertNode;
4748
import org.apache.iotdb.pipe.api.customizer.configuration.PipeConnectorRuntimeConfiguration;
4849
import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters;
@@ -497,11 +498,24 @@ private void doTransfer(
497498
LOGGER.info("Successfully transferred file {}.", tsFile);
498499
}
499500

501+
@Override
502+
public TPipeTransferReq compressIfNeeded(final TPipeTransferReq req) throws IOException {
503+
if (Objects.isNull(compressionTimer)) {
504+
compressionTimer =
505+
PipeDataRegionConnectorMetrics.getInstance().getCompressionTimer(attributeSortedString);
506+
}
507+
return super.compressIfNeeded(req);
508+
}
509+
500510
@Override
501511
public synchronized void discardEventsOfPipe(final String pipeNameToDrop, final int regionId) {
502512
tabletBatchBuilder.discardEventsOfPipe(pipeNameToDrop, regionId);
503513
}
504514

515+
public int getBatchSize() {
516+
return tabletBatchBuilder.size();
517+
}
518+
505519
@Override
506520
public void close() {
507521
if (tabletBatchBuilder != null) {

0 commit comments

Comments
 (0)