Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
fix review
  • Loading branch information
Pengzna committed Mar 27, 2025
commit f7b9e40793c137e5632184323f59a4ffa51bef08
Original file line number Diff line number Diff line change
Expand Up @@ -125,8 +125,8 @@
import org.apache.iotdb.db.storageengine.dataregion.wal.utils.listener.WALFlushListener;
import org.apache.iotdb.db.storageengine.dataregion.wal.utils.listener.WALRecoverListener;
import org.apache.iotdb.db.storageengine.load.disk.ILoadDiskSelector;
import org.apache.iotdb.db.storageengine.load.disk.InheritSystemMultiDisksStrategySelector;
import org.apache.iotdb.db.storageengine.load.disk.MinIOSelector;
import org.apache.iotdb.db.storageengine.load.disk.StorageBalanceSelector;
import org.apache.iotdb.db.storageengine.load.limiter.LoadTsFileRateLimiter;
import org.apache.iotdb.db.storageengine.rescon.disk.TierManager;
import org.apache.iotdb.db.storageengine.rescon.memory.SystemInfo;
Expand Down Expand Up @@ -415,8 +415,8 @@ public DataRegion(String databaseName, String id) {

private void initDiskSelector() {
switch (ILoadDiskSelector.LoadDiskSelectorType.fromValue(config.getLoadDiskSelectStrategy())) {
Comment thread
Pengzna marked this conversation as resolved.
case DISK_STORAGE_BALANCE_FIRST:
ordinaryLoadDiskSelector = new StorageBalanceSelector();
case INHERIT_SYSTEM_MULTI_DISKS_SELECT_STRATEGY:
ordinaryLoadDiskSelector = new InheritSystemMultiDisksStrategySelector();
break;
case MIN_IO_FIRST:
default:
Expand All @@ -428,8 +428,8 @@ private void initDiskSelector() {
case MIN_IO_FIRST:
pipeAndIoTV2LoadDiskSelector = new MinIOSelector();
break;
case DISK_STORAGE_BALANCE_FIRST:
pipeAndIoTV2LoadDiskSelector = new StorageBalanceSelector();
case INHERIT_SYSTEM_MULTI_DISKS_SELECT_STRATEGY:
pipeAndIoTV2LoadDiskSelector = new InheritSystemMultiDisksStrategySelector();
break;
case INHERIT_LOAD:
default:
Expand Down Expand Up @@ -3100,9 +3100,7 @@ private boolean loadTsFileToUnSequence(
filePartitionId,
tsFileResource.getTsFile().getName());

// var used in lambda must be final
final File finalTargetFile = targetFile;
tsFileResource.setFile(finalTargetFile);
tsFileResource.setFile(targetFile);
if (tsFileManager.contains(tsFileResource, false)) {
logger.warn("The file {} has already been loaded in unsequence list", tsFileResource);
return false;
Expand All @@ -3123,13 +3121,13 @@ private boolean loadTsFileToUnSequence(
if (deleteOriginFile) {
RetryUtils.retryOnException(
() -> {
FileUtils.moveFile(tsFileToLoad, finalTargetFile);
FileUtils.moveFile(tsFileToLoad, targetFile);
return null;
});
} else {
RetryUtils.retryOnException(
() -> {
Files.copy(tsFileToLoad.toPath(), finalTargetFile.toPath());
Files.copy(tsFileToLoad.toPath(), targetFile.toPath());
return null;
});
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ File getTargetFile(

enum LoadDiskSelectorType {
MIN_IO_FIRST("MIN_IO_FIRST"),
DISK_STORAGE_BALANCE_FIRST("DISK_STORAGE_BALANCE_FIRST"),
INHERIT_SYSTEM_MULTI_DISKS_SELECT_STRATEGY("INHERIT_SYSTEM_MULTI_DISKS_SELECT_STRATEGY"),
// This type is specially designed for IoTV2 and Pipe, which means IoTV2 and Pipe will follow
// the same strategy as ordinary load.
INHERIT_LOAD("INHERIT_LOAD");
Expand All @@ -53,13 +53,13 @@ public String getValue() {
public static LoadDiskSelectorType fromValue(String value) {
if (value.equalsIgnoreCase(MIN_IO_FIRST.getValue())) {
return MIN_IO_FIRST;
} else if (value.equalsIgnoreCase(DISK_STORAGE_BALANCE_FIRST.getValue())) {
return DISK_STORAGE_BALANCE_FIRST;
} else if (value.equalsIgnoreCase(INHERIT_SYSTEM_MULTI_DISKS_SELECT_STRATEGY.getValue())) {
return INHERIT_SYSTEM_MULTI_DISKS_SELECT_STRATEGY;
} else if (value.equalsIgnoreCase(INHERIT_LOAD.getValue())) {
return INHERIT_LOAD;
}
// return DISK_STORAGE_BALANCE_FIRST by default
return DISK_STORAGE_BALANCE_FIRST;
// return MIN_IO_FIRST by default
return MIN_IO_FIRST;
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -27,11 +27,11 @@

import java.io.File;

public class StorageBalanceSelector implements ILoadDiskSelector {
public class InheritSystemMultiDisksStrategySelector implements ILoadDiskSelector {

protected final FSFactory fsFactory = FSFactoryProducer.getFSFactory();

public StorageBalanceSelector() {
public InheritSystemMultiDisksStrategySelector() {
// empty body
}

Expand All @@ -43,6 +43,7 @@ public File getTargetFile(
long filePartitionId,
String tsfileName)
throws DiskSpaceInsufficientException {
// inherit system multi-disks select strategy, see configuration `dn_multi_dir_strategy`
return fsFactory.getFile(
TierManager.getInstance().getNextFolderForTsFile(0, false),
databaseName
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@
import java.util.Objects;
import java.util.Optional;

public class MinIOSelector extends StorageBalanceSelector {
public class MinIOSelector extends InheritSystemMultiDisksStrategySelector {

private static final Logger logger = LoggerFactory.getLogger(MinIOSelector.class);
private static final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -246,8 +246,8 @@ dn_multi_dir_strategy=SequenceStrategy
# The strategy is used to choose a proper disk for loading tsfile.
# The info of the two strategies are as follows:
# 1. MIN_IO_FIRST: the system will try to choose the same disk as tsFileToLoad for loading.
# 2. DISK_STORAGE_BALANCE_FIRST: the system will make the disk choice that balance storage as much as possible.
# If this property is unset, system will use MIN_IO_FIRST as default strategy because storage-balance is ensured with higher priority.
# 2. INHERIT_SYSTEM_MULTI_DISKS_SELECT_STRATEGY: the system will choose one disk according to dn_multi_dir_strategy.
# If this property is unset, system will use MIN_IO_FIRST as default strategy.
# effectiveMode: restart
# Datatype: String
load_disk_select_strategy=MIN_IO_FIRST
Expand All @@ -256,9 +256,9 @@ load_disk_select_strategy=MIN_IO_FIRST
# The strategy is used to choose a proper disk for loading tsfile.
# The info of the three strategies are as follows:
# 1. MIN_IO_FIRST: the system will try to choose the same disk as tsFileToLoad for loading.
# 2. DISK_STORAGE_BALANCE_FIRST: the system will make the disk choice that balance storage as much as possible.
$ 3. INHERIT_LOAD: the system will use the same strategy as load_disk_select_strategy
# If this property is unset, system will use INHERIT_LOAD as default strategy because storage-balance is ensured with higher priority.
# 2. INHERIT_SYSTEM_MULTI_DISKS_SELECT_STRATEGY: the system will choose one disk according to dn_multi_dir_strategy.
$ 3. INHERIT_LOAD: the system will use the same strategy as load_disk_select_strategy.
# If this property is unset, system will use INHERIT_LOAD as default strategy.
# effectiveMode: restart
# Datatype: String
load_disk_select_strategy_for_pipe_and_iotv2=INHERIT_LOAD
Expand Down