next

configs/experiment.yaml

@@ -101,6 +101,123 @@ feature_sets:
           persist_wins: [240, 480, 720, 1440]
           fast_span: 45
 
+
+
+  # ----------------------------------------------------------
+  # BIG NOTEBOOK-LIKE FEATURE SETS
+  # ----------------------------------------------------------
+  # These builders use full market matrices px/buy/sell, not only INDEX OHLC.
+  # They are implemented in:
+  #   src/ml_crypto_lab/features/fusion_factory.py
+  #   src/ml_crypto_lab/features/fusion_builders.py
+
+  B_big_primitive_pressure:
+    description: "large primitive pressure table: fusion_score, fusion_force, volume_pressure, index_pressure and additional pressures"
+    builders:
+      - name: advanced_fusion_params
+        params:
+          prefix: "b_big__"
+          factory:
+            symbol_candle: INDEX
+            max_symbols: 120
+            include_index_in_cross_section: false
+            vol_use_mean: true
+            price_z_wins: [1600]
+            price_breadth_wins: [500, 900]
+            vol_w_list: [120, 180]
+            vol_p_list: [1]
+            vol_roll_sets: [[1600], [2000], [2400]]
+            force_winsor_q_list: [0.15]
+            force_base_weight_list: [2.0]
+            q_weight_scale_list: [0.70]
+            force_w_price_list: [0.75]
+            use_base_imbalance_list: [true]
+            z_roll_wins: [400, 900, 1400, 1900]
+            breadth_wins: [100, 170, 240, 310, 380]
+            pct_up_norms: [50]
+            pct_down_norms: [50]
+            score_w_list: [60]
+            score_p_list: [2]
+            score_roll_sets: [[1400]]
+
+  C_big_mode_values:
+    description: "large continuous mode values derived from B_big primitive pressures"
+    builders:
+      - name: advanced_fusion_mode_values
+        params:
+          prefix: "c_big__"
+          factory:
+            symbol_candle: INDEX
+            max_symbols: 120
+            price_z_wins: [1600]
+            price_breadth_wins: [500, 900]
+            vol_w_list: [120, 180]
+            vol_roll_sets: [[1600], [2000], [2400]]
+
+  D_big_state_features:
+    description: "large discrete *_state features derived from B_big/C_big"
+    builders:
+      - name: advanced_fusion_states
+        params:
+          prefix: "d_big__"
+          factory:
+            symbol_candle: INDEX
+            max_symbols: 120
+            price_z_wins: [1600]
+            price_breadth_wins: [500, 900]
+            vol_w_list: [120, 180]
+            vol_roll_sets: [[1600], [2000], [2400]]
+
+  E_big_agreement_features:
+    description: "large agreement features by mode and by parameter"
+    builders:
+      - name: advanced_fusion_agreements
+        params:
+          prefix: "e_big__"
+          factory:
+            symbol_candle: INDEX
+            max_symbols: 120
+            price_z_wins: [1600]
+            price_breadth_wins: [500, 900]
+            vol_w_list: [120, 180]
+            vol_roll_sets: [[1600], [2000], [2400]]
+
+  F_big_long_context_features:
+    description: "large long-context feature set derived from primitive pressures"
+    builders:
+      - name: advanced_fusion_long_context
+        params:
+          prefix: "f_big__"
+          factory:
+            symbol_candle: INDEX
+            max_symbols: 120
+            price_z_wins: [1600]
+            price_breadth_wins: [500, 900]
+            vol_w_list: [120, 180]
+            vol_roll_sets: [[1600], [2000], [2400]]
+            lctx_ema_spans: [120, 240, 360, 720, 1440]
+            lctx_trend_lags: [120, 240, 360, 720]
+            lctx_cycle_wins: [360, 720, 1440]
+            lctx_persist_wins: [240, 480, 720, 1440]
+            lctx_breakout_wins: [360, 720, 1440]
+            lctx_impulse_lags: [120, 240, 360, 720]
+            lctx_max_output_cols: 10000
+
+  ALL_big_designed_features:
+    description: "full big designed table: params + modes + states + agreements + long_context"
+    builders:
+      - name: advanced_fusion_full
+        params:
+          prefix: "sig_big__"
+          factory:
+            symbol_candle: INDEX
+            max_symbols: 120
+            price_z_wins: [1600]
+            price_breadth_wins: [500, 900]
+            vol_w_list: [120, 180]
+            vol_roll_sets: [[1600], [2000], [2400]]
+            lctx_max_output_cols: 10000
+
 # ------------------------------------------------------------
 # TARGET SETS
 # ------------------------------------------------------------
@@ -173,7 +290,8 @@ models:
 # EXPERIMENT MATRIX
 # ------------------------------------------------------------
 experiment_matrix:
-  feature_sets: [A_market_basic, B_primitive_pressure, C_mode_values, D_state_features, E_agreement_features, F_long_context_features]
+  # Old compact sets are still available. For big notebook-like experiments use the *_big sets.
+  feature_sets: [A_market_basic, B_big_primitive_pressure, C_big_mode_values, D_big_state_features, E_big_agreement_features, F_big_long_context_features, ALL_big_designed_features]
   target_sets: [zz_long, future_mean, future_return]
   models: [logreg, extra_trees, hist_gb]
 

configs/experiment_advanced_fusion.yaml

@@ -0,0 +1,307 @@
+run:
+  name: baseline_feature_target_model_registry
+  seed: 42
+  output_dir: artifacts/runs
+
+# ------------------------------------------------------------
+# DATA
+# ------------------------------------------------------------
+data:
+  path: bars_bybit_1min_2026-03-01_2026-05-11.parquet
+  time_col: time
+  symbol_col: symbol
+  symbol_candle: INDEX
+  candle_rule: 1min
+  min_coverage: 0.98
+  max_symbols: 120
+  required_cols:
+    - open
+    - high
+    - low
+    - close
+    - buy_volume
+    - sell_volume
+
+# ------------------------------------------------------------
+# SPLIT
+# ------------------------------------------------------------
+split:
+  train_size: 0.70
+  test_size: 0.15
+  valid_size: 0.15
+
+# ------------------------------------------------------------
+# BACKTEST
+# ------------------------------------------------------------
+backtest:
+  fee_rate: 0.0005
+  initial_state: 1
+
+# ------------------------------------------------------------
+# FEATURE SETS
+# ------------------------------------------------------------
+feature_sets:
+  A_market_basic:
+    description: "returns, range, ATR, volatility, volume imbalance"
+    builders:
+      - name: market_basic
+        params:
+          return_lags: [1, 3, 5, 15, 30, 60, 120]
+          range_windows: [14, 30, 60, 120]
+          atr_windows: [14, 60, 120]
+          vol_windows: [30, 60, 120, 360]
+          volume_windows: [30, 60, 120]
+
+  B_primitive_pressure:
+    description: "primitive continuous pressure parameters from OHLCV"
+    builders:
+      - name: primitive_pressure
+        params:
+          price_lags: [13, 34, 55, 144, 233]
+          force_lag: 34
+          volume_norm_win: 1440
+
+  C_mode_values:
+    description: "continuous mode values: level, speed, accel, cycle, persistence"
+    builders:
+      - name: mode_values
+        params:
+          speed_ema: 45
+          speed_lag: 5
+          cycle_win: 360
+          persist_win: 120
+
+  D_state_features:
+    description: "discrete state features derived from mode values"
+    builders:
+      - name: state_features
+        params:
+          deadband: 0.05
+          cycle_deadband: 0.15
+          persist_deadband: 0.20
+
+  E_agreement_features:
+    description: "agreement features derived from state features"
+    builders:
+      - name: agreement_features
+        params:
+          state:
+            deadband: 0.05
+            cycle_deadband: 0.15
+            persist_deadband: 0.20
+
+  F_long_context_features:
+    description: "long-context features derived from primitive pressures"
+    builders:
+      - name: long_context
+        params:
+          ema_spans: [120, 240, 360, 720, 1440]
+          trend_lags: [120, 240, 360, 720]
+          cycle_wins: [360, 720, 1440]
+          persist_wins: [240, 480, 720, 1440]
+          fast_span: 45
+
+
+
+  # ----------------------------------------------------------
+  # BIG NOTEBOOK-LIKE FEATURE SETS
+  # ----------------------------------------------------------
+  # These builders use full market matrices px/buy/sell, not only INDEX OHLC.
+  # They are implemented in:
+  #   src/ml_crypto_lab/features/fusion_factory.py
+  #   src/ml_crypto_lab/features/fusion_builders.py
+
+  B_big_primitive_pressure:
+    description: "large primitive pressure table: fusion_score, fusion_force, volume_pressure, index_pressure and additional pressures"
+    builders:
+      - name: advanced_fusion_params
+        params:
+          prefix: "b_big__"
+          factory:
+            symbol_candle: INDEX
+            max_symbols: 120
+            include_index_in_cross_section: false
+            vol_use_mean: true
+            price_z_wins: [1600]
+            price_breadth_wins: [500, 900]
+            vol_w_list: [120, 180]
+            vol_p_list: [1]
+            vol_roll_sets: [[1600], [2000], [2400]]
+            force_winsor_q_list: [0.15]
+            force_base_weight_list: [2.0]
+            q_weight_scale_list: [0.70]
+            force_w_price_list: [0.75]
+            use_base_imbalance_list: [true]
+            z_roll_wins: [400, 900, 1400, 1900]
+            breadth_wins: [100, 170, 240, 310, 380]
+            pct_up_norms: [50]
+            pct_down_norms: [50]
+            score_w_list: [60]
+            score_p_list: [2]
+            score_roll_sets: [[1400]]
+
+  C_big_mode_values:
+    description: "large continuous mode values derived from B_big primitive pressures"
+    builders:
+      - name: advanced_fusion_mode_values
+        params:
+          prefix: "c_big__"
+          factory:
+            symbol_candle: INDEX
+            max_symbols: 120
+            price_z_wins: [1600]
+            price_breadth_wins: [500, 900]
+            vol_w_list: [120, 180]
+            vol_roll_sets: [[1600], [2000], [2400]]
+
+  D_big_state_features:
+    description: "large discrete *_state features derived from B_big/C_big"
+    builders:
+      - name: advanced_fusion_states
+        params:
+          prefix: "d_big__"
+          factory:
+            symbol_candle: INDEX
+            max_symbols: 120
+            price_z_wins: [1600]
+            price_breadth_wins: [500, 900]
+            vol_w_list: [120, 180]
+            vol_roll_sets: [[1600], [2000], [2400]]
+
+  E_big_agreement_features:
+    description: "large agreement features by mode and by parameter"
+    builders:
+      - name: advanced_fusion_agreements
+        params:
+          prefix: "e_big__"
+          factory:
+            symbol_candle: INDEX
+            max_symbols: 120
+            price_z_wins: [1600]
+            price_breadth_wins: [500, 900]
+            vol_w_list: [120, 180]
+            vol_roll_sets: [[1600], [2000], [2400]]
+
+  F_big_long_context_features:
+    description: "large long-context feature set derived from primitive pressures"
+    builders:
+      - name: advanced_fusion_long_context
+        params:
+          prefix: "f_big__"
+          factory:
+            symbol_candle: INDEX
+            max_symbols: 120
+            price_z_wins: [1600]
+            price_breadth_wins: [500, 900]
+            vol_w_list: [120, 180]
+            vol_roll_sets: [[1600], [2000], [2400]]
+            lctx_ema_spans: [120, 240, 360, 720, 1440]
+            lctx_trend_lags: [120, 240, 360, 720]
+            lctx_cycle_wins: [360, 720, 1440]
+            lctx_persist_wins: [240, 480, 720, 1440]
+            lctx_breakout_wins: [360, 720, 1440]
+            lctx_impulse_lags: [120, 240, 360, 720]
+            lctx_max_output_cols: 10000
+
+  ALL_big_designed_features:
+    description: "full big designed table: params + modes + states + agreements + long_context"
+    builders:
+      - name: advanced_fusion_full
+        params:
+          prefix: "sig_big__"
+          factory:
+            symbol_candle: INDEX
+            max_symbols: 120
+            price_z_wins: [1600]
+            price_breadth_wins: [500, 900]
+            vol_w_list: [120, 180]
+            vol_roll_sets: [[1600], [2000], [2400]]
+            lctx_max_output_cols: 10000
+
+# ------------------------------------------------------------
+# TARGET SETS
+# ------------------------------------------------------------
+target_sets:
+  zz_long:
+    builders:
+      - name: zigzag
+        params:
+          pct_list: [0.018, 0.022, 0.026, 0.030, 0.035, 0.040]
+          atr_mult_list: [1.5, 2.0, 2.5]
+          min_bars_list: [180, 240, 360, 480, 720]
+          atr_window: 60
+          initial_state: 1
+
+  future_mean:
+    builders:
+      - name: future_mean
+        params:
+          horizon_list: [360, 720, 1080, 1440]
+          min_move_pct_list: [0.004, 0.0065, 0.010]
+          atr_mult_list: [0.75, 1.25, 1.75]
+          fee_rate: 0.0005
+          fee_safety_rate: 0.00075
+          atr_window: 60
+          initial_state: 1
+
+  future_return:
+    builders:
+      - name: future_return
+        params:
+          horizon_list: [240, 360, 720, 1080, 1440]
+          min_move_pct_list: [0.005, 0.008, 0.012]
+          atr_mult_list: [1.0, 1.5, 2.0]
+          fee_rate: 0.0005
+          fee_safety_rate: 0.00075
+          atr_window: 60
+          initial_state: 1
+
+# ------------------------------------------------------------
+# MODELS
+# ------------------------------------------------------------
+models:
+  logreg:
+    name: sklearn_logreg
+    params:
+      C: 0.5
+      max_iter: 2000
+      class_weight: balanced
+
+  extra_trees:
+    name: sklearn_extra_trees
+    params:
+      n_estimators: 400
+      max_depth: 12
+      min_samples_leaf: 60
+      max_features: sqrt
+      n_jobs: -1
+      random_state: 42
+
+  hist_gb:
+    name: sklearn_hist_gradient_boosting
+    params:
+      max_iter: 250
+      learning_rate: 0.05
+      max_leaf_nodes: 31
+      l2_regularization: 0.01
+      random_state: 42
+
+# ------------------------------------------------------------
+# EXPERIMENT MATRIX
+# ------------------------------------------------------------
+experiment_matrix:
+  # Old compact sets are still available. For big notebook-like experiments use the *_big sets.
+  feature_sets: [B_big_primitive_pressure, C_big_mode_values, D_big_state_features, E_big_agreement_features, F_big_long_context_features, ALL_big_designed_features]
+  target_sets: [future_mean]
+  models: [logreg]
+
+training:
+  max_targets_per_set: 3
+  save_predictions: true
+  save_models: true
+
+ensemble:
+  enabled: true
+  methods:
+    - majority_vote
+    - score_average

src/ml_crypto_lab/features/fusion_builders.py

@@ -0,0 +1,93 @@
+
+from __future__ import annotations
+
+from typing import Any
+import numpy as np
+import pandas as pd
+
+from ml_crypto_lab.core.registry import FEATURE_REGISTRY
+from ml_crypto_lab.features.fusion_factory import FusionFeatureFactory, FusionFeatureBundle, make_fusion_cache_key
+from ml_crypto_lab.features.utils import clean_numeric
+
+
+def _extract_market_data(base_df: pd.DataFrame) -> tuple[pd.DataFrame, pd.DataFrame | None, pd.DataFrame | None, pd.DataFrame | None]:
+    """Read market matrices from base_df.attrs.
+
+    train.runner attaches these attrs from data.loading.build_index_ohlc_and_matrices.
+    If attrs are absent, the factory falls back to single-symbol INDEX data.
+    """
+    md = base_df.attrs.get("market_data", {}) or {}
+    ohlc = md.get("ohlc", base_df[["open", "high", "low", "close"]].copy())
+    px = md.get("px", None)
+    buy = md.get("buy", None)
+    sell = md.get("sell", None)
+    return ohlc, px, buy, sell
+
+
+def _get_bundle(base_df: pd.DataFrame, cfg: dict[str, Any]) -> FusionFeatureBundle:
+    cache_key = "advanced_fusion__" + make_fusion_cache_key(cfg)
+    cache = base_df.attrs.setdefault("feature_cache", {})
+    if cache_key in cache:
+        return cache[cache_key]
+    ohlc, px, buy, sell = _extract_market_data(base_df)
+    bundle = FusionFeatureFactory(ohlc=ohlc, px=px, buy=buy, sell=sell, cfg=cfg).build()
+    cache[cache_key] = bundle
+    return bundle
+
+
+def _add_prefix(df: pd.DataFrame, prefix: str) -> pd.DataFrame:
+    out = df.copy()
+    out.columns = [c if str(c).startswith(prefix) else f"{prefix}{c}" for c in out.columns]
+    return clean_numeric(out)
+
+
+@FEATURE_REGISTRY.register("advanced_fusion_params")
+def build_advanced_fusion_params(base_df: pd.DataFrame, cfg: dict[str, Any]) -> pd.DataFrame:
+    """Primitive pressure parameters: fusion_score, fusion_force, volume_pressure, index_pressure, etc."""
+    prefix = cfg.get("prefix", "b_big__")
+    bundle = _get_bundle(base_df, cfg.get("factory", cfg))
+    return _add_prefix(bundle.params_df.reindex(base_df.index).ffill().fillna(0.0), prefix)
+
+
+@FEATURE_REGISTRY.register("advanced_fusion_mode_values")
+def build_advanced_fusion_mode_values(base_df: pd.DataFrame, cfg: dict[str, Any]) -> pd.DataFrame:
+    """Continuous mode values: level, speed, accel, cycle, persistence, etc."""
+    prefix = cfg.get("prefix", "c_big__")
+    bundle = _get_bundle(base_df, cfg.get("factory", cfg))
+    parts = [bundle.extended_mode_raw_df, bundle.extended_mode_values_df]
+    df = pd.concat(parts, axis=1).reindex(base_df.index).ffill().fillna(0.0)
+    return _add_prefix(df, prefix)
+
+
+@FEATURE_REGISTRY.register("advanced_fusion_states")
+def build_advanced_fusion_states(base_df: pd.DataFrame, cfg: dict[str, Any]) -> pd.DataFrame:
+    """Discrete state features derived from the primitive pressure mode values."""
+    prefix = cfg.get("prefix", "d_big__")
+    bundle = _get_bundle(base_df, cfg.get("factory", cfg))
+    return _add_prefix(bundle.extended_mode_states_df.reindex(base_df.index).ffill().fillna(0.0), prefix)
+
+
+@FEATURE_REGISTRY.register("advanced_fusion_agreements")
+def build_advanced_fusion_agreements(base_df: pd.DataFrame, cfg: dict[str, Any]) -> pd.DataFrame:
+    """Agreement features across mode states and across parameters."""
+    prefix = cfg.get("prefix", "e_big__")
+    bundle = _get_bundle(base_df, cfg.get("factory", cfg))
+    df = pd.concat([bundle.agreement_by_mode_df, bundle.agreement_by_param_df], axis=1)
+    df = df.reindex(base_df.index).ffill().fillna(0.0)
+    return _add_prefix(df, prefix)
+
+
+@FEATURE_REGISTRY.register("advanced_fusion_long_context")
+def build_advanced_fusion_long_context(base_df: pd.DataFrame, cfg: dict[str, Any]) -> pd.DataFrame:
+    """Long-context features built on primitive pressure parameters."""
+    prefix = cfg.get("prefix", "f_big__")
+    bundle = _get_bundle(base_df, cfg.get("factory", cfg))
+    return _add_prefix(bundle.long_context_all_df.reindex(base_df.index).ffill().fillna(0.0), prefix)
+
+
+@FEATURE_REGISTRY.register("advanced_fusion_full")
+def build_advanced_fusion_full(base_df: pd.DataFrame, cfg: dict[str, Any]) -> pd.DataFrame:
+    """Full big designed feature table: primitive params + modes + states + agreements + long context."""
+    prefix = cfg.get("prefix", "sig_big__")
+    bundle = _get_bundle(base_df, cfg.get("factory", cfg))
+    return _add_prefix(bundle.state_details_df.reindex(base_df.index).ffill().fillna(0.0), prefix)

src/ml_crypto_lab/features/pipeline.py

@@ -9,6 +9,7 @@ from ml_crypto_lab.features.utils import clean_numeric
 
 # Import registers builders
 import ml_crypto_lab.features.builders  # noqa: F401
+import ml_crypto_lab.features.fusion_builders  # noqa: F401
 
 
 def build_feature_set(base_df: pd.DataFrame, feature_set_cfg: dict[str, Any]) -> pd.DataFrame:

src/ml_crypto_lab/train/runner.py

@@ -39,7 +39,20 @@ def build_base_frame_from_config(cfg: dict[str, Any]) -> pd.DataFrame:
     else:
         base["buy_volume"] = built["buy"].sum(axis=1)
         base["sell_volume"] = built["sell"].sum(axis=1)
-    return base.replace([np.inf, -np.inf], np.nan).ffill().dropna(subset=["open", "high", "low", "close"])
+
+    base = base.replace([np.inf, -np.inf], np.nan).ffill().dropna(subset=["open", "high", "low", "close"])
+
+    # Important for advanced cross-sectional feature engineering.
+    # The large fusion factory needs full market matrices, not only INDEX OHLC.
+    base.attrs["market_data"] = {
+        "ohlc": built["ohlc"],
+        "px": built["px"],
+        "buy": built["buy"],
+        "sell": built["sell"],
+        "raw_resampled": built.get("raw_resampled"),
+    }
+    base.attrs["feature_cache"] = {}
+    return base
 
 
 def train_one_experiment(