improve hash validation for upload folder

modelscope · wangxingjun778 · Apr 13, 2026 · Apr 10, 2026 · Apr 10, 2026 · Apr 10, 2026
commit 8a69f73a8e92c2349a085f09d5f42d6e218950e5
diff --git a/modelscope/hub/api.py b/modelscope/hub/api.py
@@ -82,7 +82,7 @@
                                        DownloadChannel, DownloadMode,
                                        Frameworks, ModelFile, Tasks,
                                        VirgoDatasetConfig)
-from modelscope.utils.file_utils import (get_file_hash, get_file_size,
+from modelscope.utils.file_utils import (compute_file_hash, get_file_size,
                                          is_relative_path)
 from modelscope.utils.logger import get_logger
 from modelscope.utils.repo_utils import (DATASET_LFS_SUFFIX,
@@ -2267,7 +2267,7 @@ def upload_file(
         if buffer_size_mb <= 0:
             raise ValueError('Buffer size: `buffer_size_mb` must be greater than 0')
 
-        hash_info_d: dict = get_file_hash(
+        hash_info_d: dict = compute_file_hash(
             file_path_or_obj=path_or_fileobj,
             buffer_size_mb=buffer_size_mb,
         )
@@ -2425,7 +2425,7 @@ def upload_folder(
         def _upload_items(item_pair, **kwargs):
             file_path_in_repo, file_path = item_pair
 
-            hash_info_d: dict = get_file_hash(
+            hash_info_d: dict = compute_file_hash(
                 file_path_or_obj=file_path,
             )
             file_size: int = hash_info_d['file_size']

diff --git a/modelscope/hub/utils/aigc.py b/modelscope/hub/utils/aigc.py
@@ -186,7 +186,7 @@ def _validate_official_tags(self):
 
     def _process_model_path(self):
         """Process model_path to extract weight information"""
-        from modelscope.utils.file_utils import get_file_hash
+        from modelscope.utils.file_utils import compute_file_hash
 
         # Expand user path
         self.model_path = os.path.expanduser(self.model_path)
@@ -266,7 +266,7 @@ def _process_model_path(self):
         if target_file:
             # Calculate file hash and size for the target file
             logger.info('Computing hash and size for %s...', target_file)
-            hash_info = get_file_hash(target_file)
+            hash_info = compute_file_hash(target_file)
 
             # Store weight information
             self.weight_filename = os.path.basename(target_file)

diff --git a/modelscope/msdatasets/download/dataset_builder.py b/modelscope/msdatasets/download/dataset_builder.py
@@ -1,6 +1,8 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 
+import csv as csv_module
 import os
+import sys
 from typing import Dict, Union
 
 import datasets
@@ -58,6 +60,10 @@ def __init__(self, dataset_context_config: DatasetContextConfig):
         if DELIMITER_NAME in self.input_config_kwargs:
             self.csv_delimiter = self.input_config_kwargs[DELIMITER_NAME]
 
+        # Extract engine and chunksize before passing kwargs to parent class
+        self.csv_engine = self.input_config_kwargs.pop('engine', None)
+        self.csv_chunksize = self.input_config_kwargs.pop('chunksize', None)
+
         split = self.split or list(dataset_context_config.data_meta_config.
                                    target_dataset_structure.keys())
         sub_dir_hash = get_subdir_hash_from_split(
@@ -132,15 +138,24 @@ def _split_generators(self, dl_manager):
         return splits
 
     def _generate_tables(self, files, base_dir):
+        # Raise csv field size limit to avoid errors with large cells
+        if self.csv_engine == 'python':
+            csv_module.field_size_limit(sys.maxsize)
+
         schema = pa.schema(self.config.features.type
                            ) if self.config.features is not None else None
         dtype = {
             name: dtype.to_pandas_dtype()
             for name, dtype in zip(schema.names, schema.types)
         } if schema else None
         for file_idx, file in enumerate(files):
-            csv_file_reader = pd.read_csv(
-                file, iterator=True, dtype=dtype, delimiter=self.csv_delimiter)
+            pd_kwargs = dict(
+                iterator=True, dtype=dtype, delimiter=self.csv_delimiter)
+            if self.csv_engine is not None:
+                pd_kwargs['engine'] = self.csv_engine
+            if self.csv_chunksize is not None:
+                pd_kwargs['chunksize'] = self.csv_chunksize
+            csv_file_reader = pd.read_csv(file, **pd_kwargs)
             transform_fields = []
             for field_name in csv_file_reader._engine.names:
                 if field_name.endswith(':FILE'):
@@ -215,8 +230,10 @@ def _download_and_prepare(self, dl_manager, download_mode):
 
     def _convert_csv_to_dataset(self, split_name, csv_file_path):
 
-        df = pd.read_csv(
-            csv_file_path, iterator=False, delimiter=self.csv_delimiter)
+        pd_kwargs = dict(iterator=False, delimiter=self.csv_delimiter)
+        if self.csv_engine is not None:
+            pd_kwargs['engine'] = self.csv_engine
+        df = pd.read_csv(csv_file_path, **pd_kwargs)
 
         transform_fields = []
         for field_name in df.columns.tolist():

diff --git a/modelscope/msdatasets/ms_dataset.py b/modelscope/msdatasets/ms_dataset.py
@@ -245,6 +245,12 @@ def load(
                 f'Use trust_remote_code=True. Will invoke codes from {dataset_name}. Please make sure that '
                 'you can trust the external codes.')
 
+        # Raise csv field size limit to avoid errors with large cells
+        if config_kwargs.get('engine') == 'python':
+            import csv as csv_module
+            import sys
+            csv_module.field_size_limit(sys.maxsize)
+
         # Init context config
         dataset_context_config = DatasetContextConfig(
             dataset_name=dataset_name,

diff --git a/modelscope/utils/file_utils.py b/modelscope/utils/file_utils.py
@@ -204,10 +204,21 @@ def get_file_size(file_path_or_obj: Union[str, Path, bytes, BinaryIO]) -> int:
 
 def get_file_hash(
     file_path_or_obj: Union[str, Path, bytes, BinaryIO],
-    buffer_size_mb: Optional[int] = 1,
+    buffer_size_mb: Optional[int] = 16,
     tqdm_desc: Optional[str] = '[Calculating]',
     disable_tqdm: Optional[bool] = True,
 ) -> dict:
+    """Compute SHA256 hash for a file path, bytes, or file-like object.
+
+    Args:
+        file_path_or_obj: File path, bytes, or file-like object.
+        buffer_size_mb: Read buffer size in MB. Default 16MB.
+        tqdm_desc: Progress bar description.
+        disable_tqdm: Whether to disable progress bar.
+
+    Returns:
+        dict with keys: file_path_or_obj, file_hash, file_size.
+    """
     from tqdm.auto import tqdm
 
     file_size = get_file_size(file_path_or_obj)
@@ -222,7 +233,6 @@ def get_file_hash(
 
     buffer_size = buffer_size_mb * 1024 * 1024
     file_hash = hashlib.sha256()
-    chunk_hash_list = []
 
     progress = tqdm(
         total=file_size,
@@ -237,27 +247,21 @@ def get_file_hash(
     if isinstance(file_path_or_obj, (str, Path)):
         with open(file_path_or_obj, 'rb') as f:
             while byte_chunk := f.read(buffer_size):
-                chunk_hash_list.append(hashlib.sha256(byte_chunk).hexdigest())
                 file_hash.update(byte_chunk)
                 progress.update(len(byte_chunk))
         file_hash = file_hash.hexdigest()
-        final_chunk_size = buffer_size
 
     elif isinstance(file_path_or_obj, bytes):
         file_hash.update(file_path_or_obj)
         file_hash = file_hash.hexdigest()
-        chunk_hash_list.append(file_hash)
-        final_chunk_size = len(file_path_or_obj)
-        progress.update(final_chunk_size)
+        progress.update(len(file_path_or_obj))
 
     elif isinstance(file_path_or_obj, io.BufferedIOBase):
         file_path_or_obj.seek(0, os.SEEK_SET)
         while byte_chunk := file_path_or_obj.read(buffer_size):
-            chunk_hash_list.append(hashlib.sha256(byte_chunk).hexdigest())
             file_hash.update(byte_chunk)
             progress.update(len(byte_chunk))
         file_hash = file_hash.hexdigest()
-        final_chunk_size = buffer_size
         file_path_or_obj.seek(0, os.SEEK_SET)
 
     else:
@@ -271,12 +275,123 @@ def get_file_hash(
         'file_path_or_obj': file_path_or_obj,
         'file_hash': file_hash,
         'file_size': file_size,
-        'chunk_size': final_chunk_size,
-        'chunk_nums': len(chunk_hash_list),
-        'chunk_hash_list': chunk_hash_list,
     }
 
 
+def _get_file_hash_async(
+    file_path: Union[str, Path],
+    buffer_size_mb: int = 16,
+    tqdm_desc: Optional[str] = '[Calculating]',
+    disable_tqdm: Optional[bool] = True,
+) -> dict:
+    """Compute SHA256 with async I/O double-buffering for high-latency storage.
+
+    Uses a producer-consumer pattern: a background thread reads file chunks
+    into a bounded queue while the main thread computes the hash. This
+    overlaps I/O latency with CPU computation.
+    """
+    import queue
+    import threading
+
+    from tqdm.auto import tqdm
+
+    file_path = str(file_path)
+    file_size = os.path.getsize(file_path)
+    buffer_size = buffer_size_mb * 1024 * 1024
+
+    chunk_queue = queue.Queue(maxsize=2)  # Bounded to limit memory usage
+    read_error = [None]  # Mutable container for thread error propagation
+
+    def _producer():
+        try:
+            with open(file_path, 'rb') as f:
+                while True:
+                    chunk = f.read(buffer_size)
+                    if not chunk:
+                        break
+                    chunk_queue.put(chunk)
+        except Exception as e:
+            read_error[0] = e
+        finally:
+            chunk_queue.put(None)  # Sentinel to signal EOF
+
+    reader_thread = threading.Thread(target=_producer, daemon=True)
+    reader_thread.start()
+
+    file_hash = hashlib.sha256()
+    progress = tqdm(
+        total=file_size,
+        desc=tqdm_desc,
+        disable=disable_tqdm,
+        dynamic_ncols=True,
+        unit='B',
+        unit_scale=True,
+        unit_divisor=1024,
+    )
+
+    while True:
+        chunk = chunk_queue.get()
+        if chunk is None:
+            break
+        file_hash.update(chunk)
+        progress.update(len(chunk))
+
+    reader_thread.join()
+    progress.close()
+
+    if read_error[0] is not None:
+        raise read_error[0]
+
+    return {
+        'file_path_or_obj': file_path,
+        'file_hash': file_hash.hexdigest(),
+        'file_size': file_size,
+    }
+
+
+def compute_file_hash(
+    file_path_or_obj: Union[str, Path, bytes, BinaryIO],
+    buffer_size_mb: Optional[int] = 16,
+    async_threshold_mb: int = 32,
+    tqdm_desc: Optional[str] = '[Calculating]',
+    disable_tqdm: Optional[bool] = True,
+) -> dict:
+    """Compute SHA256 hash with automatic optimization for large files.
+
+    For file paths larger than async_threshold_mb, uses async double-buffered
+    I/O to overlap disk reads with hash computation. For smaller files or
+    non-path inputs (bytes, BinaryIO), falls back to synchronous hashing.
+
+    Args:
+        file_path_or_obj: File path, bytes, or file-like object.
+        buffer_size_mb: Read buffer size in MB. Default 16MB for NAS optimization.
+        async_threshold_mb: File size threshold for async mode. Default 32MB.
+        tqdm_desc: Progress bar description.
+        disable_tqdm: Whether to disable progress bar.
+
+    Returns:
+        dict with keys: file_path_or_obj, file_hash, file_size.
+    """
+    # Use async mode for large files accessed by path
+    if isinstance(file_path_or_obj, (str, Path)):
+        file_size = os.path.getsize(str(file_path_or_obj))
+        if file_size >= async_threshold_mb * 1024 * 1024:
+            return _get_file_hash_async(
+                file_path_or_obj,
+                buffer_size_mb=buffer_size_mb,
+                tqdm_desc=tqdm_desc,
+                disable_tqdm=disable_tqdm,
+            )
+
+    # Fallback to synchronous hashing
+    return get_file_hash(
+        file_path_or_obj,
+        buffer_size_mb=buffer_size_mb,
+        tqdm_desc=tqdm_desc,
+        disable_tqdm=disable_tqdm,
+    )
+
+
 def is_relative_path(url_or_filename: str) -> bool:
     """
     Check if a given string is a relative path.

diff --git a/modelscope/utils/repo_utils.py b/modelscope/utils/repo_utils.py
@@ -16,7 +16,7 @@
 
 from modelscope.hub.constants import DEFAULT_MODELSCOPE_DATA_ENDPOINT
 from modelscope.hub.utils.utils import convert_timestamp
-from modelscope.utils.file_utils import get_file_hash
+from modelscope.utils.file_utils import compute_file_hash
 
 T = TypeVar('T')
 # Always ignore `.git` and `.cache/modelscope` folders in commits
@@ -419,7 +419,7 @@ class UploadInfo:
 
     @classmethod
     def from_path(cls, path: str, file_hash_info: dict = None):
-        file_hash_info = file_hash_info or get_file_hash(path)
+        file_hash_info = file_hash_info or compute_file_hash(path)
         size = file_hash_info['file_size']
         sha = file_hash_info['file_hash']
         with open(path, 'rb') as f:
@@ -429,13 +429,13 @@ def from_path(cls, path: str, file_hash_info: dict = None):
 
     @classmethod
     def from_bytes(cls, data: bytes, file_hash_info: dict = None):
-        file_hash_info = file_hash_info or get_file_hash(data)
+        file_hash_info = file_hash_info or compute_file_hash(data)
         sha = file_hash_info['file_hash']
         return cls(size=len(data), sample=data[:512], sha256=sha)
 
     @classmethod
     def from_fileobj(cls, fileobj: BinaryIO, file_hash_info: dict = None):
-        file_hash_info: dict = file_hash_info or get_file_hash(fileobj)
+        file_hash_info: dict = file_hash_info or compute_file_hash(fileobj)
         fileobj.seek(0, os.SEEK_SET)
         sample = fileobj.read(512)
         fileobj.seek(0, os.SEEK_SET)