diff --git a/Zeus_TensorFlow_1h.py b/Zeus_TensorFlow_1h.py index ec17f31..d70e4ba 100644 --- a/Zeus_TensorFlow_1h.py +++ b/Zeus_TensorFlow_1h.py @@ -13,6 +13,7 @@ import pandas as pd import numpy as np import os import json +import csv from pandas import DataFrame from typing import Optional, Union, Tuple import math @@ -80,7 +81,9 @@ from tensorflow.keras.models import Sequential from tensorflow.keras.layers import LSTM, Dense from tensorflow.keras.optimizers import Adam from sklearn.metrics import mean_absolute_error, mean_squared_error - +from sklearn.preprocessing import MinMaxScaler +import tensorflow as tf +from tensorflow.keras.callbacks import EarlyStopping, ReduceLROnPlateau, ModelCheckpoint os.environ["CUDA_VISIBLE_DEVICES"] = "-1" # désactive complètement le GPU os.environ["TF_XLA_FLAGS"] = "--tf_xla_enable_xla_devices=false" @@ -119,13 +122,32 @@ class Zeus_TensorFlow_1h(IStrategy): indicator_target = 'sma5' # Tensorflow lookback = 72 - future_steps = 12 + future_steps = 6 y_no_scale = False - epochs = 120 + epochs = 40 + batch_size = 64 scaler_X = None scaler_y = None + use_mc_dropout = True + mc_samples = 40 + minimal_pct_for_trade = 0.003 # 1.5% seuil (MAPE-based deadzone) + min_hit_ratio = 0.55 # seuil minimal historique pour activer trading + max_uncertainty_pct = 0.7 # si incertitude > 70% de predicted move => skip + base_risk_per_trade = 0.1 # 1% du capital (pour sizing) + + _tf_model = None + _scaler_X = None + _scaler_y = None + + # internal + _ps_model = None + _ps_scaler_X = None + _ps_scaler_y = None path = f"user_data/plots/" + model_path = "position_sizer_lstm.keras" + scaler_X_path = "position_sizer_scaler_X.pkl" + scaler_y_path = "position_sizer_scaler_y.pkl" # ROI table: minimal_roi = { @@ -140,7 +162,7 @@ class Zeus_TensorFlow_1h(IStrategy): # Custom stoploss use_custom_stoploss = False - trailing_stop = True + trailing_stop = False trailing_stop_positive = 0.15 trailing_stop_positive_offset = 0.20 trailing_only_offset_is_reached = True @@ -292,19 +314,19 @@ class Zeus_TensorFlow_1h(IStrategy): indicators = {'sma5', 'sma12', 'sma24', 'sma60'} indicators_percent = {'percent', 'percent3', 'percent12', 'percent24', 'percent_1d', 'percent3_1h', 'percent12_1d', 'percent24_1d'} - mises = IntParameter(1, 50, default=5, space='buy', optimize=True, load=True) + mises = IntParameter(1, 50, default=5, space='buy', optimize=False, load=True) ml_prob_buy = DecimalParameter(-0.5, 0.5, default=0.0, decimals=2, space='buy', optimize=True, load=True) - ml_prob_sell = DecimalParameter(-0.5, 0.5, default=0.0, decimals=2, space='sell', optimize=True, load=True) + # ml_prob_sell = DecimalParameter(-0.5, 0.5, default=0.0, decimals=2, space='sell', optimize=True, load=True) pct = DecimalParameter(0.005, 0.05, default=0.012, decimals=3, space='buy', optimize=True, load=True) pct_inc = DecimalParameter(0.0001, 0.003, default=0.0022, decimals=4, space='buy', optimize=True, load=True) - rsi_deb_protect = IntParameter(50, 90, default=70, space='protection', optimize=True, load=True) - rsi_end_protect = IntParameter(20, 60, default=55, space='protection', optimize=True, load=True) - - sma24_deriv1_deb_protect = DecimalParameter(-4, 4, default=-2, decimals=1, space='protection', optimize=True, load=True) - sma24_deriv1_end_protect = DecimalParameter(-4, 4, default=0, decimals=1, space='protection', optimize=True, load=True) + # rsi_deb_protect = IntParameter(50, 90, default=70, space='protection', optimize=True, load=True) + # rsi_end_protect = IntParameter(20, 60, default=55, space='protection', optimize=True, load=True) + # + # sma24_deriv1_deb_protect = DecimalParameter(-4, 4, default=-2, decimals=1, space='protection', optimize=True, load=True) + # sma24_deriv1_end_protect = DecimalParameter(-4, 4, default=0, decimals=1, space='protection', optimize=True, load=True) # ========================================================================= should_enter_trade_count = 0 @@ -455,7 +477,7 @@ class Zeus_TensorFlow_1h(IStrategy): self.log_trade( last_candle=last_candle, date=current_time, - action="🔴 CURRENT" if self.pairs[pair]['stop'] or last_candle['stop_buying'] else "🟢 CURRENT", + action="🟢 CURRENT", #🔴 CURRENT" if self.pairs[pair]['stop'] or last_candle['stop_buying'] else " dispo=dispo, pair=pair, rate=last_candle['close'], @@ -465,6 +487,9 @@ class Zeus_TensorFlow_1h(IStrategy): stake=0 ) + # if (last_candle['predicted_pct'] > 0): + # return None + pair_name = self.getShortName(pair) if last_candle['max_rsi_24'] > 85 and profit > max(5, expected_profit) and (last_candle['hapercent'] < 0) and last_candle['sma60_deriv1'] < 0.05: self.pairs[pair]['force_sell'] = False @@ -619,7 +644,7 @@ class Zeus_TensorFlow_1h(IStrategy): # Add all ta features pair = metadata['pair'] short_pair = self.getShortName(pair) - self.path = f"user_data/plots/{short_pair}/" + self.path = f"user_data/plots/{short_pair}/" + ("valide/" if not self.dp.runmode.value in ('backtest') else '') dataframe = self.populateDataframe(dataframe, timeframe='1h') @@ -629,7 +654,6 @@ class Zeus_TensorFlow_1h(IStrategy): # informative = self.calculateRegression(informative, 'mid', lookback=15) dataframe = merge_informative_pair(dataframe, informative, self.timeframe, "1d", ffill=True) - dataframe['last_price'] = dataframe['close'] dataframe['first_price'] = dataframe['close'] if self.dp: @@ -674,39 +698,70 @@ class Zeus_TensorFlow_1h(IStrategy): # dataframe['mid_smooth_5h'] # dataframe["mid_smooth_5h_deriv2"] = 100 * dataframe["mid_smooth_5h_deriv1"].diff().rolling(window=60).mean() - dataframe['stop_buying_deb'] = ((dataframe['max_rsi_24'] > self.rsi_deb_protect.value) - & (dataframe['sma24_deriv1'] < self.sma24_deriv1_deb_protect.value) - ) - dataframe['stop_buying_end'] = ((dataframe['max_rsi_24'] < self.rsi_end_protect.value) - & (dataframe['sma24_deriv1'] > self.sma24_deriv1_end_protect.value) - ) - - latched = np.zeros(len(dataframe), dtype=bool) - - for i in range(1, len(dataframe)): - if dataframe['stop_buying_deb'].iloc[i]: - latched[i] = True - elif dataframe['stop_buying_end'].iloc[i]: - latched[i] = False - else: - latched[i] = latched[i - 1] - - dataframe['stop_buying'] = latched + # dataframe['stop_buying_deb'] = ((dataframe['max_rsi_24'] > self.rsi_deb_protect.value) + # & (dataframe['sma24_deriv1'] < self.sma24_deriv1_deb_protect.value) + # ) + # dataframe['stop_buying_end'] = ((dataframe['max_rsi_24'] < self.rsi_end_protect.value) + # & (dataframe['sma24_deriv1'] > self.sma24_deriv1_end_protect.value) + # ) + # + # latched = np.zeros(len(dataframe), dtype=bool) + # + # for i in range(1, len(dataframe)): + # if dataframe['stop_buying_deb'].iloc[i]: + # latched[i] = True + # elif dataframe['stop_buying_end'].iloc[i]: + # latched[i] = False + # else: + # latched[i] = latched[i - 1] + # + # dataframe['stop_buying'] = latched dataframe = self.calculateRegression(dataframe, 'mid', lookback=10, future_steps=10, model_type="poly") dataframe = self.calculateRegression(dataframe, 'sma24', lookback=12, future_steps=12) - self.model_indicators = self.listUsableColumns(dataframe) - # TENSOR FLOW - if False and self.dp.runmode.value in ('backtest'): + if self.dp.runmode.value in ('backtest'): + self.model_indicators = self.listUsableColumns(dataframe) self.tensorFlowTrain(dataframe, future_steps = self.future_steps) - - self.tensorFlowPredict(dataframe) - - if False and self.dp.runmode.value in ('backtest'): + self.tensorFlowPredict(dataframe) self.kerasGenerateGraphs(dataframe) + # Lire les colonnes + with open(f"{self.path}/model_metadata.json", "r") as f: + metadata = json.load(f) + + self.model_indicators = metadata["feature_columns"] + self.lookback = metadata["lookback"] + self.future_steps = metadata["future_steps"] + + # ex: feature_columns correspond aux colonnes utilisées à l'entraînement + # feature_columns = [c for c in dataframe.columns if c not in [self.indicator_target, 'lstm_pred']] + preds, preds_std = self.predict_on_dataframe(dataframe, self.model_indicators) + + dataframe["lstm_pred"] = preds + dataframe["lstm_pred_std"] = preds_std + + # predicted % change relative to current price + dataframe["predicted_pct"] = (dataframe["lstm_pred"] - dataframe[self.indicator_target]) / dataframe[ + self.indicator_target] + # confidence score inversely related to std (optionnel) + dataframe["pred_confidence"] = 1 / (1 + dataframe["lstm_pred_std"]) # crude; scale to [0..1] if needed + + # # ---- Charger ou prédire ---- + # try: + # if self.dp.runmode.value in ('backtest'): + # self.train_position_sizer(dataframe, feature_columns=self.model_indicators) + # + # preds_positions = self.predict_position_fraction_on_dataframe(dataframe, feature_columns=self.model_indicators) + # # ---- Ajouter la colonne des fractions ---- + # dataframe["pos_frac"] = preds_positions # valeurs entre 0..1 + # # Exemple : valeur correspond à l’allocation conseillée du LSTM + # + # except Exception as e: + # print(f"[LSTM Position] Erreur prediction: {e}") + # dataframe["pos_frac"] = np.full(len(dataframe), np.nan) + return dataframe def listUsableColumns(self, dataframe): @@ -731,31 +786,27 @@ class Zeus_TensorFlow_1h(IStrategy): and not c.startswith('stop_buying')] # Étape 3 : remplacer inf et NaN par 0 # usable_cols = [ - # 'hapercent', 'percent', 'percent3', 'percent12', - # 'percent24', - # 'sma5_dist', 'sma5_deriv1', 'sma12_dist', 'sma12_deriv1', - # 'sma24_dist', 'sma24_deriv1', 'sma48_dist', 'sma48_deriv1', 'sma60_dist', 'sma60_deriv1', 'sma60_deriv2', - # 'mid_smooth_3_deriv1', 'mid_smooth_5_dist', - # 'mid_smooth_5_deriv1', 'mid_smooth_12_dist', - # 'mid_smooth_12_deriv1', 'mid_smooth_24_dist', - # 'mid_smooth_24_deriv1', - # 'rsi', 'max_rsi_12', 'max_rsi_24', - # 'rsi_dist', 'rsi_deriv1', - # 'min_max_60', 'bb_percent', 'bb_width', - # 'macd', 'macdsignal', 'macdhist', 'slope', - # 'slope_smooth', 'atr', 'atr_norm', 'adx', 'obv', 'vol_24', - # 'rsi_slope', 'adx_change', 'volatility_ratio', 'rsi_diff', - # 'slope_ratio', 'volume_sma_deriv', 'volume_dist', 'volume_deriv1', - # 'slope_norm', - # # 'mid_smooth_deriv1', - # # 'mid_smooth_5h_deriv1', 'mid_smooth_5h_deriv2', 'mid_future_pred_cons', - # # 'sma24_future_pred_cons' + # "obv_1d", "min60", "mid_future_pred_cons", "bb_upperband", + # "bb_lowerband", "open", "max60", "high", "volume_1d", + # "mid_smooth_5_1d", "haclose", "high_1d", "sma24_future_pred_cons", + # "volume_deriv2", "mid_smooth", "volume_deriv2_1d", "bb_middleband", + # "volume_deriv1", "sma60", "volume_dist", "open_1d", + # "haopen", "mid_1d", "min12_1d", "volume_deriv1_1d", + # "max12_1d", "mid_smooth_12", "sma24", "bb_middleband_1d", "sma12_1d", # ] dataframe[usable_cols] = dataframe[usable_cols].replace([np.inf, -np.inf], 0).fillna(0) print("Colonnes utilisables pour le modèle :") print(usable_cols) self.model_indicators = usable_cols + model_metadata = { + "feature_columns": self.model_indicators, + "lookback": self.lookback, + "future_steps": self.future_steps, + } + with open(f"{self.path}/model_metadata.json", "w") as f: + json.dump(model_metadata, f) + return self.model_indicators def populateDataframe(self, dataframe, timeframe='5m'): @@ -896,6 +947,129 @@ class Zeus_TensorFlow_1h(IStrategy): dataframe['volume_sma_deriv'] = dataframe['volume'] * dataframe['sma5_deriv1'] / (dataframe['volume'].rolling(5).mean()) self.calculeDerivees(dataframe, 'volume', timeframe=timeframe, ema_period=12) + ############################# + # NOUVEAUX + """ + Ajout des indicateurs avancés (fractals, stoch, mfi, entropy, hurst, donchian, keltner, vwap, wick features, etc.). + Ces indicateurs sont concus pour enrichir les entrees du modele TensorFlow. + """ + dataframe = dataframe.copy() + # ----------------------------------------------------------- + # 1) Fractals (Bill Williams) + # Fractal haut : point haut local centré (2-3-2) + # Fractal bas : point bas local centré (2-3-2) + # ----------------------------------------------------------- + dataframe["fractals_up"] = ( + (dataframe["high"].shift(2) < dataframe["high"].shift(1)) & + (dataframe["high"].shift(0) < dataframe["high"].shift(1)) & + (dataframe["high"].shift(3) < dataframe["high"].shift(1)) & + (dataframe["high"].shift(4) < dataframe["high"].shift(1)) + ).astype(int) + + dataframe["fractals_down"] = ( + (dataframe["low"].shift(2) > dataframe["low"].shift(1)) & + (dataframe["low"].shift(0) > dataframe["low"].shift(1)) & + (dataframe["low"].shift(3) > dataframe["low"].shift(1)) & + (dataframe["low"].shift(4) > dataframe["low"].shift(1)) + ).astype(int) + + # ----------------------------------------------------------- + # 2) Stochastic Oscillator (K, D) + # Capture l'epuisement du mouvement et les extremums de momentum + # ----------------------------------------------------------- + stoch_k = talib.STOCH(dataframe["high"], dataframe["low"], dataframe["close"])[0] + stoch_d = talib.STOCH(dataframe["high"], dataframe["low"], dataframe["close"])[1] + + dataframe["stoch_k"] = stoch_k + dataframe["stoch_d"] = stoch_d + dataframe["stoch_k_d_diff"] = stoch_k - stoch_d + + # ----------------------------------------------------------- + # 3) MFI (Money Flow Index) + # Combine prix + volume, excellent pour anticiper les retournements + # ----------------------------------------------------------- + dataframe["mfi"] = talib.MFI( + dataframe["high"], dataframe["low"], dataframe["close"], dataframe["volume"], timeperiod=14 + ) + dataframe["mfi_deriv1"] = dataframe["mfi"].diff() + + # ----------------------------------------------------------- + # 4) VWAP (Volume-Weighted Average Price) + # Zone d'equilibre du prix ; tres utile pour le sizing et l'analyse structurelle + # ----------------------------------------------------------- + typical_price = (dataframe["high"] + dataframe["low"] + dataframe["close"]) / 3 + dataframe["vwap"] = (typical_price * dataframe["volume"]).cumsum() / dataframe["volume"].replace(0, + np.nan).cumsum() + dataframe["close_vwap_dist"] = dataframe["close"] / dataframe["vwap"] - 1 + + # ----------------------------------------------------------- + # 5) Donchian Channels + # Basés sur les extremes haut/bas, utiles pour breakout et volatilite + # ----------------------------------------------------------- + dataframe["donchian_high"] = dataframe["high"].rolling(24).max() + dataframe["donchian_low"] = dataframe["low"].rolling(24).min() + dataframe["donchian_width"] = (dataframe["donchian_high"] - dataframe["donchian_low"]) / dataframe["close"] + dataframe["donchian_percent"] = (dataframe["close"] - dataframe["donchian_low"]) / ( + dataframe["donchian_high"] - dataframe["donchian_low"]) + + # ----------------------------------------------------------- + # 6) Keltner Channels + # Combine volatilite (ATR) + moyenne mobile ; tres stable et utile pour ML + # ----------------------------------------------------------- + atr = talib.ATR(dataframe["high"], dataframe["low"], dataframe["close"], timeperiod=20) + ema20 = talib.EMA(dataframe["close"], timeperiod=20) + + dataframe["kc_upper"] = ema20 + 2 * atr + dataframe["kc_lower"] = ema20 - 2 * atr + dataframe["kc_width"] = (dataframe["kc_upper"] - dataframe["kc_lower"]) / dataframe["close"] + + # ----------------------------------------------------------- + # 7) Wick Features + # Encode la forme de la bougie (haut, bas, corps) — tres utile en DL + # ----------------------------------------------------------- + dataframe["body"] = abs(dataframe["close"] - dataframe["open"]) + dataframe["range"] = dataframe["high"] - dataframe["low"] + + dataframe["body_pct"] = dataframe["body"] / dataframe["range"].replace(0, np.nan) + dataframe["upper_wick_pct"] = (dataframe["high"] - dataframe[["close", "open"]].max(axis=1)) / dataframe[ + "range"].replace(0, np.nan) + dataframe["lower_wick_pct"] = (dataframe[["close", "open"]].min(axis=1) - dataframe["low"]) / dataframe[ + "range"].replace(0, np.nan) + + # ----------------------------------------------------------- + # 8) Shannon Entropy (sur les variations de prix) + # Mesure le degre d'ordre / chaos ; excellent pour sizing adaptatif + # ----------------------------------------------------------- + def rolling_entropy(series, window): + eps = 1e-12 + roll = series.rolling(window) + return - (roll.apply(lambda x: np.sum((x / (np.sum(abs(x)) + eps)) * + np.log((x / (np.sum(abs(x)) + eps)) + eps)), raw=False)) + + dataframe["entropy_24"] = rolling_entropy(dataframe["close"].pct_change(), 24) + + # ----------------------------------------------------------- + # 9) Hurst Exponent (tendance vs mean reversion) + # Indique si le marche est trending (>0.5) ou mean-reverting (<0.5) + # ----------------------------------------------------------- + def hurst_exponent(ts): + if len(ts) < 40: + return np.nan + lags = range(2, 20) + tau = [np.sqrt(np.std(np.subtract(ts[lag:], ts[:-lag]))) for lag in lags] + poly = np.polyfit(np.log(lags), np.log(tau), 1) + return poly[0] * 2.0 + + dataframe["hurst_48"] = dataframe["close"].rolling(48).apply(hurst_exponent, raw=False) + + # Nettoyage final + dataframe.replace([np.inf, -np.inf], np.nan, inplace=True) + dataframe.fillna(method="ffill", inplace=True) + dataframe.fillna(method="bfill", inplace=True) + + # FIN NOUVEAUX + ############################ + self.setTrends(dataframe) return dataframe @@ -1013,15 +1187,36 @@ class Zeus_TensorFlow_1h(IStrategy): return self.trades def populate_buy_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: - dataframe.loc[ - ( - qtpylib.crossed_above(dataframe['lstm_pred'], dataframe['mid']) - ), ['enter_long', 'enter_tag']] = (1, f"future") + # dataframe.loc[ + # ( + # qtpylib.crossed_above(dataframe['lstm_pred'], dataframe['mid']) + # ), ['enter_long', 'enter_tag']] = (1, f"future") + # + # dataframe['test'] = np.where(dataframe['enter_long'] == 1, dataframe['close'] * 1.01, np.nan) + # + # if self.dp.runmode.value in ('backtest'): + # dataframe.to_feather(f"user_data/backtest_results/{metadata['pair'].replace('/', '_')}_df.feather") - dataframe['test'] = np.where(dataframe['enter_long'] == 1, dataframe['close'] * 1.01, np.nan) + dataframe.loc[:, "enter_long"] = False + dataframe.loc[:, "enter_short"] = False - if self.dp.runmode.value in ('backtest'): - dataframe.to_feather(f"user_data/backtest_results/{metadata['pair'].replace('/', '_')}_df.feather") + # thresholds + pct_thr = self.minimal_pct_for_trade + conf_thr = self.min_hit_ratio # you may want separate param + + # simple directional rule with deadzone and uncertainty filter + mask_up = (dataframe["predicted_pct"] > pct_thr) & (dataframe["pred_confidence"] > 0) # further filters below + mask_down = (dataframe["predicted_pct"] < -pct_thr) & (dataframe["pred_confidence"] > 0) + + # filter: ensure uncertainty isn't huge relative to predicted move + # if std > |predicted_move| * max_uncertainty_pct => skip + safe_up = mask_up & (dataframe["lstm_pred_std"] <= ( + dataframe["predicted_pct"].abs() * dataframe[self.indicator_target] * self.max_uncertainty_pct)) + safe_down = mask_down & (dataframe["lstm_pred_std"] <= ( + dataframe["predicted_pct"].abs() * dataframe[self.indicator_target] * self.max_uncertainty_pct)) + + dataframe.loc[safe_up, ['enter_long', 'enter_tag']] = (True, f"future") + # dataframe.loc[safe_down, "enter_short"] = True return dataframe @@ -1060,7 +1255,39 @@ class Zeus_TensorFlow_1h(IStrategy): return dataframe - def adjust_trade_position(self, trade: Trade, current_time: datetime, + # # Position sizing using simplified Kelly-like fraction + # def adjust_trade_positionNew(self, trade: Trade, current_time: datetime, + # current_rate: float, current_profit: float, min_stake: float, + # max_stake: float, **kwargs): + # """ + # Return fraction in (0..1] of available position to allocate. + # Uses predicted confidence and historical hit ratio. + # """ + # # idx = trade.open_dt_index if hasattr(trade, "open_dt_index") else trade.open_index + # # fallback: use latest row + # dataframe, _ = self.dp.get_analyzed_dataframe(trade.pair, self.timeframe) + # # last_candle = self.dp.get_pair_dataframe(pair).iloc[-1] + # last_candle = dataframe.iloc[-1].squeeze() + # hit = getattr(self, "historical_hit_ratio", 0.6) # you can compute this offline + # pred_conf = last_candle.get("pred_confidence", 0.5) + # predicted_pct = last_candle.get("predicted_pct", 0.0) + # + # # base fraction from hit ratio (simple linear mapping) + # if hit <= 0.5: + # base_frac = 0.01 + # else: + # base_frac = min((hit - 0.5) * 2.0, 1.0) # hit 0.6 -> 0.2 ; 0.75 -> 0.5 + # + # # scale by confidence and predicted move magnitude + # scale = pred_conf * min(abs(predicted_pct) / max(self.minimal_pct_for_trade, 1e-6), 1.0) + # + # fraction = base_frac * scale + # + # # clamp + # fraction = float(np.clip(fraction, 0.001, 1.0)) + # return fraction + + def adjust_trade_positionOld(self, trade: Trade, current_time: datetime, current_rate: float, current_profit: float, min_stake: float, max_stake: float, **kwargs): # ne rien faire si ordre deja en cours @@ -1117,7 +1344,7 @@ class Zeus_TensorFlow_1h(IStrategy): if not self.should_enter_trade(pair, last_candle, current_time): return None - condition = (last_candle['enter_long'] and last_candle['stop_buying'] == False and last_candle['hapercent'] > 0) + condition = (last_candle['enter_long'] and last_candle['hapercent'] > 0) #and last_candle['stop_buying'] == False # and last_candle['sma60_deriv1'] > 0 # or last_candle['enter_tag'] == 'pct3' \ # or last_candle['enter_tag'] == 'pct3' @@ -1218,6 +1445,38 @@ class Zeus_TensorFlow_1h(IStrategy): def getPctLastBuy(self, pair, last_candle): return round((last_candle['close'] - self.pairs[pair]['last_buy']) / self.pairs[pair]['last_buy'], 4) + # def adjust_stake_amount(self, pair: str, last_candle: DataFrame): + # base_stake_amount = self.config.get('stake_amount') + # + # # Récupérer le dataframe de la paire + # try: + # df = self.dp.get_pair_dataframe(pair) + # except Exception: + # return 0.1 # fallback safe size 10% + # + # # Doit exister car rempli dans populate_indicators + # if "pos_frac" not in df.columns: + # return 0.1 + # + # # On prend la dernière valeur non-nan + # last = df["pos_frac"].dropna() + # if last.empty: + # return 0.1 + # + # raw_frac = float(last.iloc[-1]) # dans [0..1] + # + # # --- Sécurisation --- + # # Clamp dans des limites + # raw_frac = max(0.0, min(raw_frac, 1.0)) + # + # # Conversion vers fraction réelle autorisée + # # min=0.1%, max=10% du portefeuille (change si tu veux) + # min_frac = 0.05 + # max_frac = 0.25 + # final_frac = min_frac + raw_frac * (max_frac - min_frac) + # + # return base_stake_amount * final_frac + def adjust_stake_amount(self, pair: str, last_candle: DataFrame): # Calculer le minimum des 14 derniers jours nb_pairs = len(self.dp.current_whitelist()) @@ -1227,8 +1486,8 @@ class Zeus_TensorFlow_1h(IStrategy): # factors = [1, 1.2, 1.3, 1.4] if self.pairs[pair]['count_of_buys'] == 0: factor = 1 #65 / min(65, last_candle['rsi_1d']) - if last_candle['min_max_60'] > 0.04: - factor = 2 + # if last_candle['min_max_60'] > 0.04: + # factor = 2 adjusted_stake_amount = max(base_stake_amount / 5, base_stake_amount * factor) else: @@ -1743,6 +2002,7 @@ class Zeus_TensorFlow_1h(IStrategy): # 7) Sauvegarde self.model.save(f"{self.path}/lstm_model.keras") + joblib.dump(self.scaler_X, f"{self.path}/lstm_scaler_X.pkl") joblib.dump(self.scaler_y, f"{self.path}/lstm_scaler_y.pkl") @@ -1779,11 +2039,25 @@ class Zeus_TensorFlow_1h(IStrategy): y_scaled = self.scaler_y.fit_transform(y_values) # 5) Création des fenêtres glissantes + # X_seq = [] + # y_seq = [] + # for i in range(len(X_scaled) - lookback - future_steps): + # X_seq.append(X_scaled[i:i + lookback]) + # y_seq.append(y_scaled[i + lookback + future_steps]) + X_seq = [] y_seq = [] - for i in range(len(X_scaled) - lookback - future_steps): - X_seq.append(X_scaled[i:i + lookback]) - y_seq.append(y_scaled[i + lookback + future_steps]) + + max_index = len(X_scaled) - (lookback + future_steps) + + for i in range(max_index): + # fenêtre d'entrée de longueur lookback + X_seq.append(X_scaled[i: i + lookback]) + + # target à +future_steps + y_seq.append(y_scaled[i + lookback + future_steps - 1]) + + X_seq = np.array(X_seq) y_seq = np.array(y_seq) # Vérification finale @@ -1853,6 +2127,25 @@ class Zeus_TensorFlow_1h(IStrategy): return preds + def tensorFlowPermutationImportance(self, X, y, metric=mean_absolute_error, n_rounds=3): + baseline_pred = self.model.predict(X, verbose=0) + baseline_score = metric(y, baseline_pred) + + importances = [] + + for col in range(X.shape[1]): + scores = [] + for _ in range(n_rounds): + X_permuted = X.copy() + np.random.shuffle(X_permuted[:, col]) + pred = self.model.predict(X_permuted, verbose=0) + scores.append(metric(y, pred)) + + importance = np.mean(scores) - baseline_score + importances.append(importance) + + return np.array(importances) + def generate_text_report(self, mae, rmse, mape, hit_ratio, n): txt = f""" Fiabilité du modèle à horizon {n} bougies @@ -1892,7 +2185,7 @@ class Zeus_TensorFlow_1h(IStrategy): y_pred_valid = preds_array[mask_valid] # Créer le graphique - plt.figure(figsize=(15, 5)) + plt.figure(figsize=(45, 5)) plt.plot(y_true_valid, label="Vraie valeur", color="blue") plt.plot(y_pred_valid, label="Prédiction LSTM", color="orange") plt.title(f"Prédictions LSTM vs vrai {self.indicator_target}") @@ -1932,5 +2225,408 @@ class Zeus_TensorFlow_1h(IStrategy): return mae, rmse, mape, hit_ratio + """ + Mixin utilitaire pour : + - charger un modèle Keras (Sequential) + - charger scalers (scaler_X, scaler_y) pré-sauvegardés (joblib / numpy) + - construire X aligned (lookback) depuis un DataFrame + - prédire mean+std via MC Dropout (ou simple predict if no dropout) + - retourner prédiction inverse-transformée et score de confiance + """ + def load_model_and_scalers(self): + if self._tf_model is None: + self._tf_model = load_model(f"{self.path}/lstm_model.keras", compile=False) + self._scaler_X = joblib.load(f"{self.path}/lstm_scaler_X.pkl") + self._scaler_y = joblib.load(f"{self.path}/lstm_scaler_y.pkl") + def build_X_from_dataframe(self, dataframe, feature_columns): + """ + Retourne X_seq aligné pour prédiction. + dataframe: pandas.DataFrame + feature_columns: list de colonnes à utiliser (dans l'ordre) + Résultat shape: (n_samples, lookback, n_features) + """ + values = dataframe[feature_columns].values + n = len(values) + L = self.lookback + if n < L: + return np.empty((0, L, len(feature_columns)), dtype=float) + X_seq = [] + for i in range(n - L + 1): # on veut prédiction pour chaque fenêtre disponible + seq = values[i:i+L] + X_seq.append(seq) + X_seq = np.array(X_seq) + return X_seq + def mc_dropout_predict(self, model, X, n_samples=40): + """ + Si le modèle contient du Dropout, on active training=True plusieurs fois + Retour: mean (N,1), std (N,1) + """ + if X.shape[0] == 0: + return np.array([]), np.array([]) + preds = [] + for _ in range(n_samples): + p = model(X, training=True).numpy() + preds.append(p) + preds = np.array(preds) # (n_samples, batch, output) + mean = preds.mean(axis=0).flatten() + std = preds.std(axis=0).flatten() + return mean, std + + def predict_on_dataframe(self, dataframe, feature_columns): + """ + Process complet : build X -> scale -> predict -> inverse transform -> align with dataframe + Retour: + preds_real (len = len(df)) : np.nan pour indices < lookback-1, + preds_std_real (same length) : np.nan pour indices < lookback-1 + """ + self.load_model_and_scalers() + + X_seq = self.build_X_from_dataframe(dataframe, feature_columns) # shape (N, L, f) + if getattr(self, "_scaler_X", None) is not None and X_seq.size: + # scaler expects 2D -> reshape then inverse reshape + ns, L, f = X_seq.shape + X_2d = X_seq.reshape(-1, f) + X_2d_scaled = self._scaler_X.transform(X_2d) + X_seq_scaled = X_2d_scaled.reshape(ns, L, f) + else: + X_seq_scaled = X_seq + + # prediction + if self.use_mc_dropout: + mean_scaled, std_scaled = self.mc_dropout_predict(self._tf_model, X_seq_scaled, n_samples=self.mc_samples) + else: + if X_seq_scaled.shape[0] == 0: + mean_scaled = np.array([]) + std_scaled = np.array([]) + else: + mean_scaled = self._tf_model.predict(X_seq_scaled, verbose=0).flatten() + std_scaled = np.zeros_like(mean_scaled) + + # inverse transform y if scaler_y exists + if getattr(self, "_scaler_y", None) is not None and mean_scaled.size: + # scaler expects 2D + mean_real = self._scaler_y.inverse_transform(mean_scaled.reshape(-1,1)).flatten() + std_real = self._scaler_y.inverse_transform((mean_scaled+std_scaled).reshape(-1,1)).flatten() - mean_real + std_real = np.abs(std_real) + else: + mean_real = mean_scaled + std_real = std_scaled + + # align with dataframe length + n_rows = len(dataframe) + preds = np.array([np.nan]*n_rows) + preds_std = np.array([np.nan]*n_rows) + # start = self.lookback - 1 # la première fenêtre correspond à index lookback-1 + start = self.lookback - 1 + self.future_steps + end = start + len(mean_real) - self.future_steps + if len(mean_real) > 0: + preds[start:end] = mean_real[:end-start] + preds_std[start:end] = std_real[:end-start] + + # Importance + # --- feature importance LSTM --- + # On doit découper y_true pour qu'il corresponde 1:1 aux séquences X_seq_scaled + # --- feature importance LSTM --- + if False and self.dp.runmode.value in ('backtest'): + # Y réel (non-scalé) + y_all = dataframe[self.indicator_target].values.reshape(-1, 1) + + # Scaler y + if getattr(self, "_scaler_y", None) is not None: + y_scaled_all = self.scaler_y.transform(y_all).flatten() + else: + y_scaled_all = y_all.flatten() + + # IMPORTANT : même offset que dans build_sequences() + offset = self.lookback + self.future_steps + y_true = y_scaled_all[offset - 1 - self.future_steps: offset + X_seq_scaled.shape[0]] + + print(len(X_seq_scaled)) + print(len(y_true)) + + # Vérification + if len(y_true) != X_seq_scaled.shape[0]: + raise ValueError(f"y_true ({len(y_true)}) != X_seq_scaled ({X_seq_scaled.shape[0]})") + + feature_importances = self.permutation_importance_lstm(X_seq_scaled, y_true, feature_columns) + + return preds, preds_std + + def permutation_importance_lstm(self, X_seq, y_true, feature_names, n_rounds=3): + """ + X_seq shape : (N, lookback, features) + y_true : (N,) + """ + # baseline + baseline_pred = self.model.predict(X_seq, verbose=0).flatten() + baseline_score = mean_absolute_error(y_true, baseline_pred) + + n_features = X_seq.shape[2] + importances = [] + + for f in range(n_features): + print(feature_names[f]) + scores = [] + for _ in range(n_rounds): + X_copy = X_seq.copy() + # on permute la colonne f dans TOUTES les fenêtres + for i in range(X_copy.shape[0]): + np.random.shuffle(X_copy[i, :, f]) + pred = self.model.predict(X_copy, verbose=0).flatten() + scores.append(mean_absolute_error(y_true, pred)) + + importance = np.mean(scores) - baseline_score + print(f"{f} importance indicator {feature_names[f]} = {100 * importance:.5f}%") + + importances.append(importance) + + for name, imp in sorted(zip(feature_names, importances), key=lambda x: -x[1]): + print(f"{name}: importance = {100 * imp:.5f}%") + + self.last_feature_importances = importances + # Sauvegardes + self.save_feature_importance_csv(self.last_feature_importances) + self.plot_feature_importances(self.last_feature_importances) + + print("✔ Feature importance calculée") + + return dict(zip(feature_names, importances)) + + def save_feature_importance_csv(self, importances_list): + # feature_columns = ['obv_1d', 'min60', ...] longueur = importances_list + importances_dict = dict(zip(self.model_indicators, importances_list)) + with open(f"{self.path}/feature_importances.csv", "w") as f: + f.write("feature,importance\n") + for k, v in importances_dict.items(): + f.write(f"{k},{v}\n") + + def plot_feature_importances(self, importances): + # Conversion en array + importances = np.array(importances) + feature_columns = self.model_indicators + # Tri décroissant + sorted_idx = np.argsort(importances)[::-1] + sorted_features = [feature_columns[i] for i in sorted_idx] + sorted_importances = importances[sorted_idx] + + # Plot + plt.figure(figsize=(24, 8)) + plt.bar(range(len(sorted_features)), sorted_importances) + plt.xticks(range(len(sorted_features)), sorted_features, rotation=90) + plt.title("Feature Importance (permutation)") + plt.tight_layout() + plt.savefig(f"{self.path}/Feature Importance.png") + plt.close() + + # ############################################################################################################ + # position_sizer_lstm.py + # Usage: intégrer les méthodes dans ta classe Strategy (ou comme mixin) + """ + Mixin pour entraîner / prédire une fraction de position (0..1) avec un LSTM. + - lookback : nombre de bougies en entrée + - feature_columns : liste des colonnes du dataframe utilisées comme features + - model, scalers saved under self.path (ou chemins fournis) + """ + + # CONFIG (à adapter dans ta stratégie) + # lookback = 50 + # future_steps = 1 # on prédit la prochaine bougie + # feature_columns = None # ['open','high','low','close','volume', ...] + # model_path = "position_sizer_lstm.keras" + # scaler_X_path = "position_sizer_scaler_X.pkl" + # scaler_y_path = "position_sizer_scaler_y.pkl" + + # training params + # epochs = 50 + # batch_size = 64 + + # ------------------------ + # Data preparation + # ------------------------ + def _build_sequences_for_position_sizer(self, df): + # features (N, f) + values = df[self.model_indicators].values.astype(float) + # target = realized return after next candle (or profit), here: simple return + # you can replace by realized profit if you have it (price change minus fees etc.) + prices = df[self.indicator_target].values.astype(float) + returns = (np.roll(prices, -self.future_steps) - prices) / prices # next-return + returns = returns.reshape(-1, 1) + + L = self.lookback + X_seq = [] + y_seq = [] + + max_i = len(values) - (L + self.future_steps) + 1 + if max_i <= 0: + return np.empty((0, L, values.shape[1])), np.empty((0, 1)) + + for i in range(max_i): + X_seq.append(values[i : i + L]) + # y is the *desired* fraction proxy: we use scaled positive return (could be improved) + # Here we use returns[i + L - 1 + future_steps] which is the return after the window + y_seq.append(returns[i + L - 1 + self.future_steps - 1]) # equals returns[i+L-1] + X_seq = np.array(X_seq) # (ns, L, f) + y_seq = np.array(y_seq) # (ns, 1) + return X_seq, y_seq + + # ------------------------ + # Scalers save/load + # ------------------------ + def save_scalers(self, scaler_X, scaler_y, folder=None): + import joblib + folder = folder or self.path + os.makedirs(folder, exist_ok=True) + joblib.dump(scaler_X, os.path.join(folder, self.scaler_X_path)) + joblib.dump(scaler_y, os.path.join(folder, self.scaler_y_path)) + + def load_scalers(self, folder=None): + import joblib + folder = folder or self.path + try: + self._ps_scaler_X = joblib.load(os.path.join(folder, self.scaler_X_path)) + self._ps_scaler_y = joblib.load(os.path.join(folder, self.scaler_y_path)) + except Exception: + self._ps_scaler_X = None + self._ps_scaler_y = None + + # ------------------------ + # Model definition + # ------------------------ + def build_position_sizer_model(self, n_features): + model = tf.keras.Sequential([ + tf.keras.layers.Input(shape=(self.lookback, n_features)), + tf.keras.layers.LSTM(64, return_sequences=False), + tf.keras.layers.Dense(32, activation="relu"), + tf.keras.layers.Dense(1, activation="sigmoid") # fraction in [0,1] + ]) + model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=1e-3), loss="mse") + return model + + # ------------------------ + # Training + # ------------------------ + def train_position_sizer(self, dataframe, feature_columns=None, + model_path=None, scaler_folder=None, + epochs=None, batch_size=None): + """ + Entrainer le modèle LSTM pour prédire la fraction (0..1). + dataframe : pandas DataFrame (doit contenir self.indicator_target) + """ + feature_columns = feature_columns or self.model_indicators + if feature_columns is None: + raise ValueError("feature_columns must be provided") + + X_seq, y_seq = self._build_sequences_for_position_sizer(dataframe) + if X_seq.shape[0] == 0: + raise ValueError("Pas assez de données pour former des séquences (lookback trop grand).") + + # scalers + scaler_X = MinMaxScaler() + ns, L, f = X_seq.shape + X_2d = X_seq.reshape(-1, f) + X_2d_scaled = scaler_X.fit_transform(X_2d) + X_seq_scaled = X_2d_scaled.reshape(ns, L, f) + + scaler_y = MinMaxScaler(feature_range=(0, 1)) + y_scaled = scaler_y.fit_transform(y_seq) # maps returns -> [0,1] (you may want custom transform) + + # build model + model = self.build_position_sizer_model(n_features=f) + + # callbacks + model_path = model_path or os.path.join(self.path, self.model_path) + callbacks = [ + EarlyStopping(monitor="val_loss", patience=5, restore_best_weights=True, verbose=1), + ReduceLROnPlateau(monitor="val_loss", factor=0.5, patience=3, verbose=1), + ModelCheckpoint(model_path, save_best_only=True, monitor="val_loss", verbose=0) + ] + + # train + epochs = epochs or self.epochs + batch_size = batch_size or self.batch_size + model.fit(X_seq_scaled, y_scaled, validation_split=0.1, + epochs=epochs, batch_size=batch_size, callbacks=callbacks, verbose=2) + + # save model and scalers + os.makedirs(self.path, exist_ok=True) + model.save(model_path) + self.save_scalers(scaler_X, scaler_y, folder=self.path) + + # store references + self._ps_model = model + self._ps_scaler_X = scaler_X + self._ps_scaler_y = scaler_y + + return model + + # ------------------------ + # Load model + # ------------------------ + def load_position_sizer(self, model_path=None, scaler_folder=None): + model_path = model_path or os.path.join(self.path, self.model_path) + scaler_folder = scaler_folder or self.path + if os.path.exists(model_path): + self._ps_model = load_model(model_path, compile=False) + else: + self._ps_model = None + self.load_scalers(scaler_folder) + return self._ps_model + + # ------------------------ + # Predict fraction on dataframe + # ------------------------ + def predict_position_fraction_on_dataframe(self, dataframe, feature_columns=None): + """ + Retourne un vecteur de fractions (len = len(dataframe)), np.nan pour indices non prédits. + """ + feature_columns = feature_columns or self.model_indicators + if feature_columns is None: + raise ValueError("feature_columns must be set") + + if self._ps_model is None: + self.load_position_sizer() + + # build X sequence (same as training) + X_seq, _ = self._build_sequences_for_position_sizer(dataframe) + if X_seq.shape[0] == 0: + # not enough data + return np.full(len(dataframe), np.nan) + + ns, L, f = X_seq.shape + X_2d = X_seq.reshape(-1, f) + + if self._ps_scaler_X is None: + raise ValueError("scaler_X missing (train first or load scalers).") + + X_2d_scaled = self._ps_scaler_X.transform(X_2d) + X_seq_scaled = X_2d_scaled.reshape(ns, L, f) + + # predict + preds = self._ps_model.predict(X_seq_scaled, verbose=0).flatten() # in [0,1] + # align with dataframe: first valid prediction corresponds to index = lookback - 1 + result = np.full(len(dataframe), np.nan) + start = self.lookback - 1 + end = start + len(preds) + result[start:end] = preds[:end-start] + return result + + # ------------------------ + # Adjust trade position (Freqtrade hook) + # ------------------------ + def position_fraction_to_trade_size(self, fraction, wallet_balance, current_price, + min_fraction=0.001, max_fraction=0.5): + """ + Map fraction [0,1] to a safe fraction of wallet. + Apply clamping and minimal size guard. + """ + if np.isnan(fraction): + return min_fraction + frac = float(np.clip(fraction, 0.0, 1.0)) + # scale to allowed range + scaled = min_fraction + frac * (max_fraction - min_fraction) + # optionally map to quantity units : quantity = (scaled * wallet_balance) / price + return scaled + +# End of mixin