danil_belikov/ml_crypto_lab
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

next

Browse Source
This commit is contained in:
belikovme 2026-06-26 15:10:13 +07:00
parent 13ec1f0204
commit 992041987a
6 changed files with 1584 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

120
configs/experiment.yaml
View File

@ -101,6 +101,123 @@ feature_sets:
persist_wins: [240, 480, 720, 1440]
fast_span: 45
# ----------------------------------------------------------
# BIG NOTEBOOK-LIKE FEATURE SETS
# ----------------------------------------------------------
# These builders use full market matrices px/buy/sell, not only INDEX OHLC.
# They are implemented in:
# src/ml_crypto_lab/features/fusion_factory.py
# src/ml_crypto_lab/features/fusion_builders.py
B_big_primitive_pressure:
description: "large primitive pressure table: fusion_score, fusion_force, volume_pressure, index_pressure and additional pressures"
builders:
- name: advanced_fusion_params
params:
prefix: "b_big__"
factory:
symbol_candle: INDEX
max_symbols: 120
include_index_in_cross_section: false
vol_use_mean: true
price_z_wins: [1600]
price_breadth_wins: [500, 900]
vol_w_list: [120, 180]
vol_p_list: [1]
vol_roll_sets: [[1600], [2000], [2400]]
force_winsor_q_list: [0.15]
force_base_weight_list: [2.0]
q_weight_scale_list: [0.70]
force_w_price_list: [0.75]
use_base_imbalance_list: [true]
z_roll_wins: [400, 900, 1400, 1900]
breadth_wins: [100, 170, 240, 310, 380]
pct_up_norms: [50]
pct_down_norms: [50]
score_w_list: [60]
score_p_list: [2]
score_roll_sets: [[1400]]
C_big_mode_values:
description: "large continuous mode values derived from B_big primitive pressures"
builders:
- name: advanced_fusion_mode_values
params:
prefix: "c_big__"
factory:
symbol_candle: INDEX
max_symbols: 120
price_z_wins: [1600]
price_breadth_wins: [500, 900]
vol_w_list: [120, 180]
vol_roll_sets: [[1600], [2000], [2400]]
D_big_state_features:
description: "large discrete *_state features derived from B_big/C_big"
builders:
- name: advanced_fusion_states
params:
prefix: "d_big__"
factory:
symbol_candle: INDEX
max_symbols: 120
price_z_wins: [1600]
price_breadth_wins: [500, 900]
vol_w_list: [120, 180]
vol_roll_sets: [[1600], [2000], [2400]]
E_big_agreement_features:
description: "large agreement features by mode and by parameter"
builders:
- name: advanced_fusion_agreements
params:
prefix: "e_big__"
factory:
symbol_candle: INDEX
max_symbols: 120
price_z_wins: [1600]
price_breadth_wins: [500, 900]
vol_w_list: [120, 180]
vol_roll_sets: [[1600], [2000], [2400]]
F_big_long_context_features:
description: "large long-context feature set derived from primitive pressures"
builders:
- name: advanced_fusion_long_context
params:
prefix: "f_big__"
factory:
symbol_candle: INDEX
max_symbols: 120
price_z_wins: [1600]
price_breadth_wins: [500, 900]
vol_w_list: [120, 180]
vol_roll_sets: [[1600], [2000], [2400]]
lctx_ema_spans: [120, 240, 360, 720, 1440]
lctx_trend_lags: [120, 240, 360, 720]
lctx_cycle_wins: [360, 720, 1440]
lctx_persist_wins: [240, 480, 720, 1440]
lctx_breakout_wins: [360, 720, 1440]
lctx_impulse_lags: [120, 240, 360, 720]
lctx_max_output_cols: 10000
ALL_big_designed_features:
description: "full big designed table: params + modes + states + agreements + long_context"
builders:
- name: advanced_fusion_full
params:
prefix: "sig_big__"
factory:
symbol_candle: INDEX
max_symbols: 120
price_z_wins: [1600]
price_breadth_wins: [500, 900]
vol_w_list: [120, 180]
vol_roll_sets: [[1600], [2000], [2400]]
lctx_max_output_cols: 10000
# ------------------------------------------------------------
# TARGET SETS
# ------------------------------------------------------------
@ -173,7 +290,8 @@ models:
# EXPERIMENT MATRIX
# ------------------------------------------------------------
experiment_matrix:
feature_sets: [A_market_basic, B_primitive_pressure, C_mode_values, D_state_features, E_agreement_features, F_long_context_features]
# Old compact sets are still available. For big notebook-like experiments use the *_big sets.
feature_sets: [A_market_basic, B_big_primitive_pressure, C_big_mode_values, D_big_state_features, E_big_agreement_features, F_big_long_context_features, ALL_big_designed_features]
target_sets: [zz_long, future_mean, future_return]
models: [logreg, extra_trees, hist_gb]

307
configs/experiment_advanced_fusion.yaml Normal file
View File

@ -0,0 +1,307 @@
run:
name: baseline_feature_target_model_registry
seed: 42
output_dir: artifacts/runs
# ------------------------------------------------------------
# DATA
# ------------------------------------------------------------
data:
path: bars_bybit_1min_2026-03-01_2026-05-11.parquet
time_col: time
symbol_col: symbol
symbol_candle: INDEX
candle_rule: 1min
min_coverage: 0.98
max_symbols: 120
required_cols:
- open
- high
- low
- close
- buy_volume
- sell_volume
# ------------------------------------------------------------
# SPLIT
# ------------------------------------------------------------
split:
train_size: 0.70
test_size: 0.15
valid_size: 0.15
# ------------------------------------------------------------
# BACKTEST
# ------------------------------------------------------------
backtest:
fee_rate: 0.0005
initial_state: 1
# ------------------------------------------------------------
# FEATURE SETS
# ------------------------------------------------------------
feature_sets:
A_market_basic:
description: "returns, range, ATR, volatility, volume imbalance"
builders:
- name: market_basic
params:
return_lags: [1, 3, 5, 15, 30, 60, 120]
range_windows: [14, 30, 60, 120]
atr_windows: [14, 60, 120]
vol_windows: [30, 60, 120, 360]
volume_windows: [30, 60, 120]
B_primitive_pressure:
description: "primitive continuous pressure parameters from OHLCV"
builders:
- name: primitive_pressure
params:
price_lags: [13, 34, 55, 144, 233]
force_lag: 34
volume_norm_win: 1440
C_mode_values:
description: "continuous mode values: level, speed, accel, cycle, persistence"
builders:
- name: mode_values
params:
speed_ema: 45
speed_lag: 5
cycle_win: 360
persist_win: 120
D_state_features:
description: "discrete state features derived from mode values"
builders:
- name: state_features
params:
deadband: 0.05
cycle_deadband: 0.15
persist_deadband: 0.20
E_agreement_features:
description: "agreement features derived from state features"
builders:
- name: agreement_features
params:
state:
deadband: 0.05
cycle_deadband: 0.15
persist_deadband: 0.20
F_long_context_features:
description: "long-context features derived from primitive pressures"
builders:
- name: long_context
params:
ema_spans: [120, 240, 360, 720, 1440]
trend_lags: [120, 240, 360, 720]
cycle_wins: [360, 720, 1440]
persist_wins: [240, 480, 720, 1440]
fast_span: 45
# ----------------------------------------------------------
# BIG NOTEBOOK-LIKE FEATURE SETS
# ----------------------------------------------------------
# These builders use full market matrices px/buy/sell, not only INDEX OHLC.
# They are implemented in:
# src/ml_crypto_lab/features/fusion_factory.py
# src/ml_crypto_lab/features/fusion_builders.py
B_big_primitive_pressure:
description: "large primitive pressure table: fusion_score, fusion_force, volume_pressure, index_pressure and additional pressures"
builders:
- name: advanced_fusion_params
params:
prefix: "b_big__"
factory:
symbol_candle: INDEX
max_symbols: 120
include_index_in_cross_section: false
vol_use_mean: true
price_z_wins: [1600]
price_breadth_wins: [500, 900]
vol_w_list: [120, 180]
vol_p_list: [1]
vol_roll_sets: [[1600], [2000], [2400]]
force_winsor_q_list: [0.15]
force_base_weight_list: [2.0]
q_weight_scale_list: [0.70]
force_w_price_list: [0.75]
use_base_imbalance_list: [true]
z_roll_wins: [400, 900, 1400, 1900]
breadth_wins: [100, 170, 240, 310, 380]
pct_up_norms: [50]
pct_down_norms: [50]
score_w_list: [60]
score_p_list: [2]
score_roll_sets: [[1400]]
C_big_mode_values:
description: "large continuous mode values derived from B_big primitive pressures"
builders:
- name: advanced_fusion_mode_values
params:
prefix: "c_big__"
factory:
symbol_candle: INDEX
max_symbols: 120
price_z_wins: [1600]
price_breadth_wins: [500, 900]
vol_w_list: [120, 180]
vol_roll_sets: [[1600], [2000], [2400]]
D_big_state_features:
description: "large discrete *_state features derived from B_big/C_big"
builders:
- name: advanced_fusion_states
params:
prefix: "d_big__"
factory:
symbol_candle: INDEX
max_symbols: 120
price_z_wins: [1600]
price_breadth_wins: [500, 900]
vol_w_list: [120, 180]
vol_roll_sets: [[1600], [2000], [2400]]
E_big_agreement_features:
description: "large agreement features by mode and by parameter"
builders:
- name: advanced_fusion_agreements
params:
prefix: "e_big__"
factory:
symbol_candle: INDEX
max_symbols: 120
price_z_wins: [1600]
price_breadth_wins: [500, 900]
vol_w_list: [120, 180]
vol_roll_sets: [[1600], [2000], [2400]]
F_big_long_context_features:
description: "large long-context feature set derived from primitive pressures"
builders:
- name: advanced_fusion_long_context
params:
prefix: "f_big__"
factory:
symbol_candle: INDEX
max_symbols: 120
price_z_wins: [1600]
price_breadth_wins: [500, 900]
vol_w_list: [120, 180]
vol_roll_sets: [[1600], [2000], [2400]]
lctx_ema_spans: [120, 240, 360, 720, 1440]
lctx_trend_lags: [120, 240, 360, 720]
lctx_cycle_wins: [360, 720, 1440]
lctx_persist_wins: [240, 480, 720, 1440]
lctx_breakout_wins: [360, 720, 1440]
lctx_impulse_lags: [120, 240, 360, 720]
lctx_max_output_cols: 10000
ALL_big_designed_features:
description: "full big designed table: params + modes + states + agreements + long_context"
builders:
- name: advanced_fusion_full
params:
prefix: "sig_big__"
factory:
symbol_candle: INDEX
max_symbols: 120
price_z_wins: [1600]
price_breadth_wins: [500, 900]
vol_w_list: [120, 180]
vol_roll_sets: [[1600], [2000], [2400]]
lctx_max_output_cols: 10000
# ------------------------------------------------------------
# TARGET SETS
# ------------------------------------------------------------
target_sets:
zz_long:
builders:
- name: zigzag
params:
pct_list: [0.018, 0.022, 0.026, 0.030, 0.035, 0.040]
atr_mult_list: [1.5, 2.0, 2.5]
min_bars_list: [180, 240, 360, 480, 720]
atr_window: 60
initial_state: 1
future_mean:
builders:
- name: future_mean
params:
horizon_list: [360, 720, 1080, 1440]
min_move_pct_list: [0.004, 0.0065, 0.010]
atr_mult_list: [0.75, 1.25, 1.75]
fee_rate: 0.0005
fee_safety_rate: 0.00075
atr_window: 60
initial_state: 1
future_return:
builders:
- name: future_return
params:
horizon_list: [240, 360, 720, 1080, 1440]
min_move_pct_list: [0.005, 0.008, 0.012]
atr_mult_list: [1.0, 1.5, 2.0]
fee_rate: 0.0005
fee_safety_rate: 0.00075
atr_window: 60
initial_state: 1
# ------------------------------------------------------------
# MODELS
# ------------------------------------------------------------
models:
logreg:
name: sklearn_logreg
params:
C: 0.5
max_iter: 2000
class_weight: balanced
extra_trees:
name: sklearn_extra_trees
params:
n_estimators: 400
max_depth: 12
min_samples_leaf: 60
max_features: sqrt
n_jobs: -1
random_state: 42
hist_gb:
name: sklearn_hist_gradient_boosting
params:
max_iter: 250
learning_rate: 0.05
max_leaf_nodes: 31
l2_regularization: 0.01
random_state: 42
# ------------------------------------------------------------
# EXPERIMENT MATRIX
# ------------------------------------------------------------
experiment_matrix:
# Old compact sets are still available. For big notebook-like experiments use the *_big sets.
feature_sets: [B_big_primitive_pressure, C_big_mode_values, D_big_state_features, E_big_agreement_features, F_big_long_context_features, ALL_big_designed_features]
target_sets: [future_mean]
models: [logreg]
training:
max_targets_per_set: 3
save_predictions: true
save_models: true
ensemble:
enabled: true
methods:
- majority_vote
- score_average

93
src/ml_crypto_lab/features/fusion_builders.py Normal file
View File

@ -0,0 +1,93 @@
from __future__ import annotations
from typing import Any
import numpy as np
import pandas as pd
from ml_crypto_lab.core.registry import FEATURE_REGISTRY
from ml_crypto_lab.features.fusion_factory import FusionFeatureFactory, FusionFeatureBundle, make_fusion_cache_key
from ml_crypto_lab.features.utils import clean_numeric
def _extract_market_data(base_df: pd.DataFrame) -> tuple[pd.DataFrame, pd.DataFrame | None, pd.DataFrame | None, pd.DataFrame | None]:
"""Read market matrices from base_df.attrs.
train.runner attaches these attrs from data.loading.build_index_ohlc_and_matrices.
If attrs are absent, the factory falls back to single-symbol INDEX data.
"""
md = base_df.attrs.get("market_data", {}) or {}
ohlc = md.get("ohlc", base_df[["open", "high", "low", "close"]].copy())
px = md.get("px", None)
buy = md.get("buy", None)
sell = md.get("sell", None)
return ohlc, px, buy, sell
def _get_bundle(base_df: pd.DataFrame, cfg: dict[str, Any]) -> FusionFeatureBundle:
cache_key = "advanced_fusion__" + make_fusion_cache_key(cfg)
cache = base_df.attrs.setdefault("feature_cache", {})
if cache_key in cache:
return cache[cache_key]
ohlc, px, buy, sell = _extract_market_data(base_df)
bundle = FusionFeatureFactory(ohlc=ohlc, px=px, buy=buy, sell=sell, cfg=cfg).build()
cache[cache_key] = bundle
return bundle
def _add_prefix(df: pd.DataFrame, prefix: str) -> pd.DataFrame:
out = df.copy()
out.columns = [c if str(c).startswith(prefix) else f"{prefix}{c}" for c in out.columns]
return clean_numeric(out)
@FEATURE_REGISTRY.register("advanced_fusion_params")
def build_advanced_fusion_params(base_df: pd.DataFrame, cfg: dict[str, Any]) -> pd.DataFrame:
"""Primitive pressure parameters: fusion_score, fusion_force, volume_pressure, index_pressure, etc."""
prefix = cfg.get("prefix", "b_big__")
bundle = _get_bundle(base_df, cfg.get("factory", cfg))
return _add_prefix(bundle.params_df.reindex(base_df.index).ffill().fillna(0.0), prefix)
@FEATURE_REGISTRY.register("advanced_fusion_mode_values")
def build_advanced_fusion_mode_values(base_df: pd.DataFrame, cfg: dict[str, Any]) -> pd.DataFrame:
"""Continuous mode values: level, speed, accel, cycle, persistence, etc."""
prefix = cfg.get("prefix", "c_big__")
bundle = _get_bundle(base_df, cfg.get("factory", cfg))
parts = [bundle.extended_mode_raw_df, bundle.extended_mode_values_df]
df = pd.concat(parts, axis=1).reindex(base_df.index).ffill().fillna(0.0)
return _add_prefix(df, prefix)
@FEATURE_REGISTRY.register("advanced_fusion_states")
def build_advanced_fusion_states(base_df: pd.DataFrame, cfg: dict[str, Any]) -> pd.DataFrame:
"""Discrete state features derived from the primitive pressure mode values."""
prefix = cfg.get("prefix", "d_big__")
bundle = _get_bundle(base_df, cfg.get("factory", cfg))
return _add_prefix(bundle.extended_mode_states_df.reindex(base_df.index).ffill().fillna(0.0), prefix)
@FEATURE_REGISTRY.register("advanced_fusion_agreements")
def build_advanced_fusion_agreements(base_df: pd.DataFrame, cfg: dict[str, Any]) -> pd.DataFrame:
"""Agreement features across mode states and across parameters."""
prefix = cfg.get("prefix", "e_big__")
bundle = _get_bundle(base_df, cfg.get("factory", cfg))
df = pd.concat([bundle.agreement_by_mode_df, bundle.agreement_by_param_df], axis=1)
df = df.reindex(base_df.index).ffill().fillna(0.0)
return _add_prefix(df, prefix)
@FEATURE_REGISTRY.register("advanced_fusion_long_context")
def build_advanced_fusion_long_context(base_df: pd.DataFrame, cfg: dict[str, Any]) -> pd.DataFrame:
"""Long-context features built on primitive pressure parameters."""
prefix = cfg.get("prefix", "f_big__")
bundle = _get_bundle(base_df, cfg.get("factory", cfg))
return _add_prefix(bundle.long_context_all_df.reindex(base_df.index).ffill().fillna(0.0), prefix)
@FEATURE_REGISTRY.register("advanced_fusion_full")
def build_advanced_fusion_full(base_df: pd.DataFrame, cfg: dict[str, Any]) -> pd.DataFrame:
"""Full big designed feature table: primitive params + modes + states + agreements + long context."""
prefix = cfg.get("prefix", "sig_big__")
bundle = _get_bundle(base_df, cfg.get("factory", cfg))
return _add_prefix(bundle.state_details_df.reindex(base_df.index).ffill().fillna(0.0), prefix)

1050
src/ml_crypto_lab/features/fusion_factory.py Normal file
View File

File diff suppressed because it is too large Load Diff

1
src/ml_crypto_lab/features/pipeline.py
View File

@ -9,6 +9,7 @@ from ml_crypto_lab.features.utils import clean_numeric
# Import registers builders
import ml_crypto_lab.features.builders # noqa: F401
import ml_crypto_lab.features.fusion_builders # noqa: F401
def build_feature_set(base_df: pd.DataFrame, feature_set_cfg: dict[str, Any]) -> pd.DataFrame:

15
src/ml_crypto_lab/train/runner.py
View File

@ -39,7 +39,20 @@ def build_base_frame_from_config(cfg: dict[str, Any]) -> pd.DataFrame:
else:
base["buy_volume"] = built["buy"].sum(axis=1)
base["sell_volume"] = built["sell"].sum(axis=1)
return base.replace([np.inf, -np.inf], np.nan).ffill().dropna(subset=["open", "high", "low", "close"])
base = base.replace([np.inf, -np.inf], np.nan).ffill().dropna(subset=["open", "high", "low", "close"])
# Important for advanced cross-sectional feature engineering.
# The large fusion factory needs full market matrices, not only INDEX OHLC.
base.attrs["market_data"] = {
"ohlc": built["ohlc"],
"px": built["px"],
"buy": built["buy"],
"sell": built["sell"],
"raw_resampled": built.get("raw_resampled"),
}
base.attrs["feature_cache"] = {}
return base
def train_one_experiment(