Pipe: Add InsertNode transmission time

apache · Caideyipi · Jun 7, 2025 · Jun 7, 2025 · Jun 7, 2025 · Jun 7, 2025
commit ed15d3b1ca75d5d8327b6c511387a898ccdcb432
diff --git a/.../src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/PipeTsFileInsertionEvent.java b/.../src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/PipeTsFileInsertionEvent.java
@@ -331,7 +331,8 @@ public boolean internallyDecreaseResourceReferenceCount(
     } finally {
       if (Objects.nonNull(pipeName)) {
         PipeDataNodeRemainingEventAndTimeMetrics.getInstance()
-            .decreaseTsFileEventCount(pipeName, creationTime);
+            .decreaseTsFileEventCount(
+                pipeName, creationTime, shouldReport, System.nanoTime() - extractTime);
       }
     }
   }

diff --git a/...va/org/apache/iotdb/db/pipe/metric/overview/PipeDataNodeRemainingEventAndTimeMetrics.java b/...va/org/apache/iotdb/db/pipe/metric/overview/PipeDataNodeRemainingEventAndTimeMetrics.java
@@ -26,7 +26,9 @@
 import org.apache.iotdb.db.pipe.extractor.dataregion.IoTDBDataRegionExtractor;
 import org.apache.iotdb.db.pipe.extractor.schemaregion.IoTDBSchemaRegionExtractor;
 import org.apache.iotdb.metrics.AbstractMetricService;
+import org.apache.iotdb.metrics.impl.DoNothingMetricManager;
 import org.apache.iotdb.metrics.metricsets.IMetricSet;
+import org.apache.iotdb.metrics.type.Histogram;
 import org.apache.iotdb.metrics.type.Timer;
 import org.apache.iotdb.metrics.utils.MetricLevel;
 import org.apache.iotdb.metrics.utils.MetricType;
@@ -39,6 +41,7 @@
 import java.util.Map;
 import java.util.Objects;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.TimeUnit;
 
 public class PipeDataNodeRemainingEventAndTimeMetrics implements IMetricSet {
 
@@ -51,11 +54,28 @@ public class PipeDataNodeRemainingEventAndTimeMetrics implements IMetricSet {
   private final Map<String, PipeDataNodeRemainingEventAndTimeOperator>
       remainingEventAndTimeOperatorMap = new ConcurrentHashMap<>();
 
+  private static Histogram PIPE_DATANODE_INSERTNODE_TRANSFER_TIME_HISTOGRAM =
+      DoNothingMetricManager.DO_NOTHING_HISTOGRAM;
+  private static Histogram PIPE_DATANODE_TSFILE_TRANSFER_TIME_HISTOGRAM =
+      DoNothingMetricManager.DO_NOTHING_HISTOGRAM;
+
   //////////////////////////// bindTo & unbindFrom (metric framework) ////////////////////////////
 
   @Override
   public void bindTo(final AbstractMetricService metricService) {
     this.metricService = metricService;
+    PIPE_DATANODE_INSERTNODE_TRANSFER_TIME_HISTOGRAM =
+        metricService.getOrCreateHistogram(
+            Metric.PIPE_DATANODE_INSERT_NODE_EVENT_TRANSFER.toString(),
+            MetricLevel.IMPORTANT,
+            Tag.NAME.toString(),
+            "insert_node");
+    PIPE_DATANODE_TSFILE_TRANSFER_TIME_HISTOGRAM =
+        metricService.getOrCreateHistogram(
+            Metric.PIPE_DATANODE_TSFILE_EVENT_TRANSFER.toString(),
+            MetricLevel.IMPORTANT,
+            Tag.NAME.toString(),
+            "tsfile");
     ImmutableSet.copyOf(remainingEventAndTimeOperatorMap.keySet()).forEach(this::createMetrics);
   }
 
@@ -126,6 +146,17 @@ public void unbindFrom(final AbstractMetricService metricService) {
       LOGGER.warn(
           "Failed to unbind from pipe remaining event and time metrics, RemainingEventAndTimeOperator map not empty");
     }
+    metricService.getOrCreateHistogram(
+        Metric.PIPE_DATANODE_INSERT_NODE_EVENT_TRANSFER.toString(),
+        MetricLevel.IMPORTANT,
+        Tag.NAME.toString(),
+        "insert_node");
+
+    metricService.getOrCreateHistogram(
+        Metric.PIPE_DATANODE_TSFILE_EVENT_TRANSFER.toString(),
+        MetricLevel.IMPORTANT,
+        Tag.NAME.toString(),
+        "tsfile");
   }
 
   private void removeMetrics(final String pipeID) {
@@ -200,11 +231,15 @@ public void decreaseInsertNodeEventCount(
       final long creationTime,
       final boolean needUpdateTimer,
       final long transferTime) {
-    remainingEventAndTimeOperatorMap
-        .computeIfAbsent(
+    PipeDataNodeRemainingEventAndTimeOperator operator =
+        remainingEventAndTimeOperatorMap.computeIfAbsent(
             pipeName + "_" + creationTime,
-            k -> new PipeDataNodeRemainingEventAndTimeOperator(pipeName, creationTime))
-        .decreaseInsertNodeEventCount(needUpdateTimer, transferTime);
+            k -> new PipeDataNodeRemainingEventAndTimeOperator(pipeName, creationTime));
+    operator.decreaseInsertNodeEventCount();
+    if (needUpdateTimer) {
+      operator.getInsertNodeTransferTimer().update(transferTime, TimeUnit.NANOSECONDS);
+      PIPE_DATANODE_INSERTNODE_TRANSFER_TIME_HISTOGRAM.update(transferTime);
+    }
   }
 
   public void increaseRawTabletEventCount(final String pipeName, final long creationTime) {
@@ -231,12 +266,21 @@ public void increaseTsFileEventCount(final String pipeName, final long creationT
         .increaseTsFileEventCount();
   }
 
-  public void decreaseTsFileEventCount(final String pipeName, final long creationTime) {
-    remainingEventAndTimeOperatorMap
-        .computeIfAbsent(
+  public void decreaseTsFileEventCount(
+      final String pipeName,
+      final long creationTime,
+      final boolean needUpdateTimer,
+      final long transferTime) {
+    final PipeDataNodeRemainingEventAndTimeOperator operator =
+        remainingEventAndTimeOperatorMap.computeIfAbsent(
             pipeName + "_" + creationTime,
-            k -> new PipeDataNodeRemainingEventAndTimeOperator(pipeName, creationTime))
-        .decreaseTsFileEventCount();
+            k -> new PipeDataNodeRemainingEventAndTimeOperator(pipeName, creationTime));
+
+    operator.decreaseTsFileEventCount();
+    if (needUpdateTimer) {
+      operator.getTsFileTransferTimer().update(transferTime, TimeUnit.NANOSECONDS);
+      PIPE_DATANODE_TSFILE_TRANSFER_TIME_HISTOGRAM.update(transferTime);
+    }
   }
 
   public void increaseHeartbeatEventCount(final String pipeName, final long creationTime) {

diff --git a/...a/org/apache/iotdb/db/pipe/metric/overview/PipeDataNodeRemainingEventAndTimeOperator.java b/...a/org/apache/iotdb/db/pipe/metric/overview/PipeDataNodeRemainingEventAndTimeOperator.java
@@ -37,7 +37,6 @@
 import java.util.Objects;
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
 
@@ -77,9 +76,8 @@ void increaseInsertNodeEventCount() {
     insertNodeEventCount.incrementAndGet();
   }
 
-  void decreaseInsertNodeEventCount(final boolean needUpdateTimer, final long transferTime) {
+  void decreaseInsertNodeEventCount() {
     insertNodeEventCount.decrementAndGet();
-    if (needUpdateTimer) insertNodeTransferTimer.update(transferTime, TimeUnit.NANOSECONDS);
   }
 
   void increaseRawTabletEventCount() {
@@ -245,6 +243,14 @@ public Timer getInsertNodeTransferTimer() {
     return insertNodeTransferTimer;
   }
 
+  public void setTsFileTransferTimer(Timer tsFileTransferTimer) {
+    this.tsfileTransferTimer = tsFileTransferTimer;
+  }
+
+  public Timer getTsFileTransferTimer() {
+    return tsfileTransferTimer;
+  }
+
   //////////////////////////// Switch ////////////////////////////
 
   // Thread-safe & Idempotent

diff --git a/...core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java b/...core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java
@@ -181,6 +181,9 @@ public enum Metric {
   PIPE_DATANODE_REMAINING_EVENT_COUNT("pipe_datanode_remaining_event_count"),
   PIPE_DATANODE_REMAINING_TIME("pipe_datanode_remaining_time"),
   PIPE_INSERT_NODE_EVENT_TRANSFER_TIME("pipe_insert_node_event_transfer_time"),
+  PIPE_DATANODE_INSERT_NODE_EVENT_TRANSFER("pipe_datanode_insert_node_event_transfer"),
+  PIPE_TSFILE_EVENT_TRANSFER_TIME("pipe_tsfile_event_transfer_time"),
+  PIPE_DATANODE_TSFILE_EVENT_TRANSFER("pipe_datanode_tsfile_event_transfer"),
   PIPE_CONFIG_LINKED_QUEUE_SIZE("pipe_config_linked_queue_size"),
   UNTRANSFERRED_CONFIG_COUNT("untransferred_config_count"),
   PIPE_CONNECTOR_CONFIG_TRANSFER("pipe_connector_config_transfer"),