draft design

microsoft · you-n-g · Sep 30, 2021 · Mar 19, 2021 · Apr 21, 2021 · Apr 21, 2021
commit d39c8de800aa846eb313530b63c43bc066b12d74
diff --git a/qlib/backtest/__init__.py b/qlib/backtest/__init__.py
@@ -20,7 +20,7 @@
 from ..log import get_module_logger
 from ..config import C
 
-# make import more user-friendly by enable `from qlib.backtest import STH`
+# make import more user-friendly by adding `from qlib.backtest import STH`
 
 
 logger = get_module_logger("backtest caller")

diff --git a/qlib/backtest/exchange.py b/qlib/backtest/exchange.py
@@ -424,7 +424,7 @@ def get_deal_price(self, stock_id, start_time, end_time, direction: OrderDir, me
         else:
             raise NotImplementedError(f"This type of input is not supported")
         deal_price = self.quote.get_data(stock_id, start_time, end_time, field=pstr, method=method)
-        if method is not None and (np.isclose(deal_price, 0.0) or np.isnan(deal_price)):
+        if method is not None and (deal_price is None or np.isclose(deal_price, 0.0) or np.isnan(deal_price)):
             self.logger.warning(f"(stock_id:{stock_id}, trade_time:{(start_time, end_time)}, {pstr}): {deal_price}!!!")
             self.logger.warning(f"setting deal_price to close price")
             deal_price = self.get_close(stock_id, start_time, end_time, method)

diff --git a/qlib/backtest/high_performance_ds.py b/qlib/backtest/high_performance_ds.py
@@ -15,6 +15,7 @@
 from ..utils.resam import resam_ts_data, ts_data_last
 from ..log import get_module_logger
 from ..utils.time import is_single_value
+import qlib.utils.index_data as idd
 
 
 class BaseQuote:
@@ -61,7 +62,9 @@ def get_data(
 
                 this function is used for three case:
 
-                1. method is not None. It returns int/float/bool.
+                1. method is not None. It returns int/float/bool/None.
+                    - It will return None in one case, the method return None
+
                     print(get_data(stock_id="SH600000", start_time="2010-01-04", end_time="2010-01-06", field="$close", method="last"))
 
                     85.713585
@@ -87,8 +90,9 @@ def get_data(
         Return
         ----------
         Union[None, int, float, bool, IndexData]
-            None means there is no stock data from data source.
-            please refer to Example as following.
+            it will return None in following cases
+            - There is no stock data which meet the query criterion from data source.
+            - The `method` returns None
         """
 
         raise NotImplementedError(f"Please implement the `get_data` method")
@@ -112,7 +116,7 @@ def get_data(self, stock_id, start_time, end_time, field, method=None):
         elif isinstance(stock_data, (bool, np.bool_, int, float, np.number)):
             return stock_data
         elif isinstance(stock_data, pd.Series):
-            return IndexData.Series(stock_data)
+            return idd.SingleData(stock_data)
         else:
             raise ValueError(f"stock data from resam_ts_data must be a number, pd.Series or pd.DataFrame")
 
@@ -130,7 +134,8 @@ def __init__(self, quote_df: pd.DataFrame):
         super().__init__(quote_df=quote_df)
         quote_dict = {}
         for stock_id, stock_val in quote_df.groupby(level="instrument"):
-            quote_dict[stock_id] = IndexData.DataFrame(stock_val.droplevel(level="instrument"))
+            quote_dict[stock_id] = idd.MultiData(stock_val.droplevel(level="instrument"))
+            quote_dict[stock_id].sort_index()  # To support more flexible slicing, we must sort data first
         self.data = quote_dict
         self.freq = pd.Timedelta(minutes=1)
 
@@ -145,32 +150,22 @@ def get_data(self, stock_id, start_time, end_time, field, method=None):
 
         # single data
         # If it don't consider the classification of single data, it will consume a lot of time.
-        if is_single_value(start_time, end_time, self.freq):
-            now_index_map = self.data[stock_id].index_map
-            now_columns_map = self.data[stock_id].columns_map
-            if start_time not in now_index_map:
+        if is_single_value(start_time, end_time, self.freq) and method is not None:
+            # this is a very special case.
+            # skip aggregating function to speed-up the query calculation
+            try:
+                self.data[stock_id].loc[start_time, field]
+            except KeyError:
                 return None
-            else:
-                return self.data[stock_id].values[now_index_map[start_time], now_columns_map[field]]
-
-        # multi data
         else:
-            if method is None:
-                stock_data = self.data[stock_id].loc(start_time, end_time, field)
-                if stock_data.empty:
-                    return None
-                else:
-                    return stock_data
-            else:
-                stock_data = self.data[stock_id].loc(start_time, end_time, field)
-                if stock_data.empty:
-                    return None
-                elif len(stock_data) == 1:
-                    return stock_data[0]
-                else:
-                    return self._agg_data(stock_data.values, method)
-
-    def _agg_data(self, data, method):
+            data = self.data[stock_id].loc[start_time:end_time, field]
+            if data.empty:
+                return None
+            if method is not None:
+                data = self._agg_data(data, method)
+            return data
+
+    def _agg_data(self, data: IndexData, method):
         """Agg data by specific method."""
         if method == "sum":
             return np.nansum(data)
@@ -183,11 +178,11 @@ def _agg_data(self, data, method):
         elif method == "any":
             return data.any()
         elif method == ts_data_last:
-            valid_data = data[data != np.NaN]
+            valid_data = data.loc[~data.isna().data.astype(bool)]
             if len(valid_data) == 0:
                 return None
             else:
-                return valid_data[0]
+                return valid_data.iloc[-1]
         else:
             raise ValueError(f"{method} is not supported")
 
@@ -259,9 +254,6 @@ def count(self) -> int:
     def abs(self) -> "BaseSingleMetric":
         raise NotImplementedError(f"Please implement the `abs` method")
 
-    def astype(self, dtype: type) -> "BaseSingleMetric":
-        raise NotImplementedError(f"Please implement the `astype` method")
-
     @property
     def empty(self) -> bool:
         """If metric is empty, return True."""
@@ -332,7 +324,7 @@ def transfer(self, func: Callable, new_col: str = None) -> Union[None, BaseSingl
             the kwargs of func will be replaced with metric data by name in this function.
             e.g.
                 def func(pa):
-                    return (pa > 0).astype(int).sum() / pa.count()
+                    return (pa > 0).sum() / pa.count()
         new_col : str, optional
             New metric will be assigned in the data if new_col is not None, by default None.
 
@@ -513,9 +505,6 @@ def count(self):
     def abs(self):
         return self.__class__(self.metric.abs())
 
-    def astype(self, dtype):
-        return self.__class__(self.metric.astype(dtype))
-
     @property
     def empty(self):
         return self.metric.empty
@@ -552,9 +541,9 @@ def assign(self, col: str, metric: Union[dict, pd.Series]):
 
     def get_index_data(self, metric):
         if metric in self.data:
-            return IndexData.Series(self.data[metric].metric)
+            return idd.SingleData(self.data[metric].metric)
         else:
-            return IndexData.Series()
+            return idd.SingleData()
 
     def get_metric_series(self, metric: str) -> Union[pd.Series]:
         if metric in self.data:
@@ -579,21 +568,21 @@ def sum_all_indicators(order_indicator, indicators: list, metrics: Union[str, Li
 class NumpyOrderIndicator(BaseOrderIndicator):
     """
     The data structure is OrderedDict(str: SingleData).
-    Each IndexData.Series is one metric.
+    Each idd.SingleData is one metric.
     Str is the name of metric.
     """
 
     def __init__(self):
         self.data: Dict[str, SingleData] = OrderedDict()
 
     def assign(self, col: str, metric: dict):
-        self.data[col] = IndexData.Series(metric)
+        self.data[col] = idd.SingleData(metric)
 
     def get_index_data(self, metric):
         if metric in self.data:
             return self.data[metric]
         else:
-            return IndexData.Series()
+            return idd.SingleData()
 
     def get_metric_series(self, metric: str) -> Union[pd.Series]:
         return self.data[metric].to_series()
@@ -609,7 +598,7 @@ def sum_all_indicators(order_indicator, indicators: list, metrics: Union[str, Li
         if isinstance(metrics, str):
             metrics = [metrics]
         for metric in metrics:
-            tmp_metric = IndexData.Series()
+            tmp_metric = IndexData.SingleData()
             for indicator in indicators:
                 tmp_metric = tmp_metric.add(indicator.data[metric], fill_value)
             order_indicator.data[metric] = tmp_metric
diff --git a/qlib/backtest/report.py b/qlib/backtest/report.py
@@ -12,10 +12,10 @@
 from qlib.backtest.exchange import Exchange
 from qlib.backtest.order import BaseTradeDecision, Order, OrderDir
 from .high_performance_ds import PandasOrderIndicator, NumpyOrderIndicator, SingleMetric
-from ..utils.index_data import IndexData, SingleData
 from ..tests.config import CSI300_BENCH
 from ..utils.resam import get_higher_eq_freq_feature, resam_ts_data
 from .order import IdxTradeRange
+import qlib.utils.index_data as idd
 
 
 class Report:
@@ -386,8 +386,8 @@ def _get_base_vol_pri(
             return None, None
 
         if isinstance(price_s, (int, float, np.number)):
-            price_s = IndexData.Series(price_s, [trade_start_time])
-        elif isinstance(price_s, SingleData):
+            price_s = idd.SingleData(price_s, [trade_start_time])
+        elif isinstance(price_s, idd.SingleData):
             pass
         else:
             raise NotImplementedError(f"This type of input is not supported")
@@ -401,10 +401,10 @@ def _get_base_vol_pri(
         if agg == "vwap":
             volume_s = trade_exchange.get_volume(inst, trade_start_time, trade_end_time, method=None)
             if isinstance(volume_s, (int, float, np.number)):
-                volume_s = IndexData.Series(volume_s, [trade_start_time])
+                volume_s = idd.SingleData(volume_s, [trade_start_time])
             volume_s = volume_s.reindex(price_s.index)
         elif agg == "twap":
-            volume_s = IndexData.Series(1, price_s.index)
+            volume_s = idd.SingleData(1, price_s.index)
         else:
             raise NotImplementedError(f"This type of input is not supported")
 
@@ -414,7 +414,7 @@ def _get_base_vol_pri(
 
     def _agg_base_price(
         self,
-        inner_order_indicators: List[Dict[str, Union[SingleMetric, SingleData]]],
+        inner_order_indicators: List[Dict[str, Union[SingleMetric, idd.SingleData]]],
         decision_list: List[Tuple[BaseTradeDecision, pd.Timestamp, pd.Timestamp]],
         trade_exchange: Exchange,
         pa_config: dict = {},
@@ -467,12 +467,12 @@ def _agg_base_price(
                     else:
                         bp_new[inst], bv_new[inst] = pr, v
 
-                bp_new = IndexData.Series(bp_new)
-                bv_new = IndexData.Series(bv_new)
+                bp_new = idd.SingleData(bp_new)
+                bv_new = idd.SingleData(bv_new)
                 bp_all.append(bp_new)
                 bv_all.append(bv_new)
-            bp_all = IndexData.concat(bp_all, axis=1)
-            bv_all = IndexData.concat(bv_all, axis=1)
+            bp_all = idd.concat(bp_all, axis=1)
+            bv_all = idd.concat(bv_all, axis=1)
 
             base_volume = bv_all.sum(axis=1)
             self.order_indicator.assign("base_volume", base_volume.to_dict())
@@ -550,7 +550,7 @@ def func(pa, trade_value):
 
     def _cal_trade_positive_rate(self):
         def func(pa):
-            return (pa > 0).astype(int).sum() / pa.count()
+            return (pa > 0).sum() / pa.count()
 
         return self.order_indicator.transfer(func)
 

diff --git a/qlib/data/ops.py b/qlib/data/ops.py
@@ -1405,7 +1405,7 @@ def __init__(self, feature_left, feature_right, N):
         super(Corr, self).__init__(feature_left, feature_right, N, "corr")
 
     def _load_internal(self, instrument, start_index, end_index, freq):
-        res = super(Corr, self)._load_internal(instrument, start_index, end_index, freq)
+        res: pd.Series = super(Corr, self)._load_internal(instrument, start_index, end_index, freq)
 
         # NOTE: Load uses MemCache, so calling load again will not cause performance degradation
         series_left = self.feature_left.load(instrument, start_index, end_index, freq)