Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 37 additions & 1 deletion include/pineforge/engine.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -1069,7 +1069,13 @@ class BacktestEngine {
// Independent opt-in KI-55 HTF warmup parity. When enabled,
// request.security series aggregate from security_range_start_ms_ instead
// of the feed start and their embedded ta.ema na-warm per TV built-in
// semantics. Default OFF; touched only through feed_security_eval_state.
// semantics. The cut is taken per evaluator on HTF-BUCKET opens, not on
// input timestamps: an input bar is dropped when the D/W/M (or intraday
// grid) bucket it belongs to opened before the range start, so the first
// HTF bar every series sees is a whole bucket that opened at/after the
// range start (security_input_precedes_range_start). Default OFF;
// consulted only by feed_security_eval_state and the historical
// lookahead projection builder.
bool security_range_start_na_warmup_ = false;
int64_t security_range_start_ms_ = 0;
// Opt-in historical-only request.security lookahead projection. TradingView
Expand Down Expand Up @@ -2220,6 +2226,14 @@ class BacktestEngine {
int security_lower_tf_sub_bar_index(int sec_id) const;
void validate_security_timeframes(const std::string& input_tf);
bool security_series_slot_is_new(int sec_id) const;
// KI-55 range-start gate for one evaluator: true when the input bar at
// `input_ts` belongs to an HTF bucket that opened before
// security_range_start_ms_ (always false while the flag is off). The
// progressive feed and the historical lookahead projection builder must
// agree on this predicate so projected child indexes line up with the
// per-state feed cursor.
bool security_input_precedes_range_start(const SecurityEvalState& state,
int64_t input_ts) const;
void feed_security_eval_state(SecurityEvalState& state, const Bar& input_bar);

virtual void configure_security_evaluators() {}
Expand Down Expand Up @@ -2906,6 +2920,28 @@ class BacktestEngine {
// would regress the crypto-on-shifted-chart case).
void set_syminfo_timezone(const std::string& tz) { syminfo_.timezone = tz; }
void set_syminfo_session(const std::string& s) { syminfo_.session = s; }
// ``syminfo.type`` ("crypto" default; "forex" / "stock" / "futures" /
// "index" / "fund" / "cfd" per TradingView). Scripts branch on it for
// instrument conventions — the canonical one being the pip size
// (``syminfo.type == "forex" ? 0.0001 : syminfo.mintick``), which on a
// 5-digit FX symbol under the crypto default computed every pip-scaled
// stop/target 10x too tight (finding 454). Empty is ignored.
void set_syminfo_type(const std::string& t) { if (!t.empty()) syminfo_.type = t; }
// Generic string-field injection for the remaining OHLCV-less syminfo
// members (ticker / tickerid / currency / basecurrency / description /
// volumetype / type). Unknown keys and empty values are ignored; returns
// true when a field was set.
bool set_syminfo_string(const std::string& key, const std::string& value) {
if (value.empty()) return false;
if (key == "type") { syminfo_.type = value; return true; }
if (key == "ticker") { syminfo_.ticker = value; return true; }
if (key == "tickerid") { syminfo_.tickerid = value; return true; }
if (key == "currency") { syminfo_.currency = value; return true; }
if (key == "basecurrency") { syminfo_.basecurrency = value; return true; }
if (key == "description") { syminfo_.description = value; return true; }
if (key == "volumetype") { syminfo_.volumetype = value; return true; }
return false;
}
// Runtime syminfo injection (by design — the engine stores no instrument
// metadata of its own; the harness supplies it per run). mintick drives the
// directional fill snap + slippage*tick economics; pointvalue is the
Expand Down
13 changes: 13 additions & 0 deletions include/pineforge/pineforge.h
Original file line number Diff line number Diff line change
Expand Up @@ -611,6 +611,19 @@ PF_API void strategy_set_syminfo_timezone(pf_strategy_t s, const char* tz);
* ignored. Call before #run_backtest*. */
PF_API void strategy_set_syminfo_session(pf_strategy_t s, const char* session);

/** Set the instrument class (``syminfo.type``: "forex", "stock", "crypto",
* "futures", "index", "fund", "cfd", ...; default "crypto"). Scripts branch
* on it for instrument conventions (e.g. the forex pip size). `NULL` /
* empty ignored. Call before #run_backtest*. */
PF_API void strategy_set_syminfo_type(pf_strategy_t s, const char* type);

/** Set one of the remaining string ``syminfo.*`` members by Pine member
* name: "ticker", "tickerid", "currency", "basecurrency", "description",
* "volumetype" (and "type"). Returns 0 when set, -1 for an unknown key,
* empty value or NULL. Call before #run_backtest*. */
PF_API int strategy_set_syminfo_string(pf_strategy_t s, const char* key,
const char* value);

/** Set the instrument tick size (``syminfo.mintick``, default 0.01). Drives the
* directional stop-entry snap and ``slippage = N*mintick`` economics. Set
* per-instrument (e.g. 0.25 for ES, 0.00001 for FX). Non-positive ignored.
Expand Down
23 changes: 23 additions & 0 deletions include/pineforge/timeframe.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -138,6 +138,18 @@ int64_t session_period_close_ms(int64_t ms, const std::string& tz,
const std::string& session,
CalendarPeriod period);

/// Exclusive close (Unix ms) of the LAST TRADED session-day of the D/W/M
/// bar that contains `ms`: DAY is session_period_close_ms; WEEK / MONTH
/// step back from the period's last session-day over weekend TRADING dates
/// (Saturday / Sunday never hold a session on exchange-calendar symbols),
/// so an equity week ends Friday 16:00 ET, a month whose last calendar day
/// is a weekend ends on its last Friday, and the forex week ends Friday
/// 17:00 ET (the Friday-open session is Saturday's trading date). Exchange
/// holidays are not modelled. CalendarPeriod::NONE returns `ms`.
int64_t session_period_last_traded_close_ms(int64_t ms, const std::string& tz,
const std::string& session,
CalendarPeriod period);

// ─── TimeframeAggregator ───────────────────────────────────────────────────────

class TimeframeAggregator {
Expand Down Expand Up @@ -173,6 +185,17 @@ class TimeframeAggregator {
/// Whether aggregation is active (non-passthrough).
bool is_active() const;

/// Open (Unix ms) of the target-TF bucket an input bar stamped `ms`
/// belongs to, on the aggregator's anchor clock (syminfo tz + session):
/// CALENDAR -> session_period_open_ms of the bar's D/W/M period (the
/// forex week opens Sunday 17:00 ET, its month on the session whose
/// close date is the 1st); RATIO -> the session-open-anchored intraday
/// grid bucket (the same key feed() splits on); PASSTHROUGH, or a
/// count-only ratio with no wall-clock width, -> `ms` itself. Pure
/// function of the configuration: it neither reads nor advances the
/// aggregation state, so callers may query it before feeding the bar.
int64_t bucket_open_ms(int64_t ms) const;

private:
enum class Mode { PASSTHROUGH, RATIO, CALENDAR };

Expand Down
2 changes: 2 additions & 0 deletions scripts/check_c_abi_runtime.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,8 @@
"strategy_set_chart_timezone",
"strategy_set_syminfo_timezone",
"strategy_set_syminfo_session",
"strategy_set_syminfo_type",
"strategy_set_syminfo_string",
"strategy_set_syminfo_mintick",
"strategy_set_syminfo_pointvalue",
"strategy_set_syminfo_metadata",
Expand Down
62 changes: 61 additions & 1 deletion scripts/run_strategy.py
Original file line number Diff line number Diff line change
Expand Up @@ -1017,6 +1017,13 @@ def _num(v):
chart_timezone=chart_tz,
syminfo_timezone=str(runtime_overrides.get("timezone") or "") or None,
syminfo_session=str(runtime_overrides.get("session") or "") or None,
syminfo_type=str(runtime_overrides.get("type") or "") or None,
syminfo_strings={
k: str(runtime_overrides.get(k))
for k in ("ticker", "tickerid", "currency", "basecurrency",
"description", "volumetype")
if runtime_overrides.get(k)
} or None,
syminfo_metadata=syminfo_metadata,
syminfo_mintick=_num(runtime_overrides.get("mintick")),
syminfo_pointvalue=_num(runtime_overrides.get("pointvalue")),
Expand Down Expand Up @@ -1125,6 +1132,16 @@ def _setup_signatures(self) -> None:
if hasattr(L, "strategy_set_syminfo_session"):
L.strategy_set_syminfo_session.argtypes = [ctypes.c_void_p, ctypes.c_char_p]
L.strategy_set_syminfo_session.restype = None
# syminfo.type ("forex"/"stock"/"crypto"/...) + the generic string
# member setter (ticker/tickerid/currency/basecurrency/...). Older
# .so builds predate these exports — hasattr-guarded like the rest.
if hasattr(L, "strategy_set_syminfo_type"):
L.strategy_set_syminfo_type.argtypes = [ctypes.c_void_p, ctypes.c_char_p]
L.strategy_set_syminfo_type.restype = None
if hasattr(L, "strategy_set_syminfo_string"):
L.strategy_set_syminfo_string.argtypes = [
ctypes.c_void_p, ctypes.c_char_p, ctypes.c_char_p]
L.strategy_set_syminfo_string.restype = ctypes.c_int
if hasattr(L, "strategy_set_syminfo_metadata"):
L.strategy_set_syminfo_metadata.argtypes = [
ctypes.c_void_p, ctypes.c_char_p, ctypes.c_double]
Expand All @@ -1151,6 +1168,8 @@ def run(self, bars_csv: Path, params: dict | None = None,
chart_timezone: str | None = None,
syminfo_timezone: str | None = None,
syminfo_session: str | None = None,
syminfo_type: str | None = None,
syminfo_strings: dict | None = None,
syminfo_metadata: dict | None = None,
syminfo_mintick: float | None = None,
syminfo_pointvalue: float | None = None,
Expand Down Expand Up @@ -1243,6 +1262,17 @@ def run(self, bars_csv: Path, params: dict | None = None,
self.lib.strategy_set_syminfo_timezone(state, str(syminfo_timezone).encode())
if syminfo_session and hasattr(self.lib, "strategy_set_syminfo_session"):
self.lib.strategy_set_syminfo_session(state, str(syminfo_session).encode())
# Instrument class (syminfo.type). Unset keeps the engine's
# "crypto" default byte-identical; per-symbol datasets pass
# TV's value ("forex" for OANDA:EURUSD, "stock" for NASDAQ:AAPL).
if syminfo_type and hasattr(self.lib, "strategy_set_syminfo_type"):
self.lib.strategy_set_syminfo_type(state, str(syminfo_type).encode())
if syminfo_strings and hasattr(self.lib, "strategy_set_syminfo_string"):
for skey, sval in syminfo_strings.items():
if sval is None or str(sval) == "":
continue
self.lib.strategy_set_syminfo_string(
state, str(skey).encode(), str(sval).encode())
if syminfo_metadata and hasattr(self.lib, "strategy_set_syminfo_metadata"):
for mkey, mval in syminfo_metadata.items():
try:
Expand Down Expand Up @@ -1515,6 +1545,32 @@ def _filter_trace_to_window(trace: list[dict], window: tuple[int, int] | None) -
]


def format_trade_qty(qty: float) -> str:
"""Lot-faithful quantity text for the TV-alignable export.

The previous ``f"{qty:g}"`` kept only 6 significant digits, so any
quantity with more digits than that was silently rewritten on the way
out: an OANDA:EURUSD all-in lot of 923941.16 units (TV export, 0.01 lot
step) printed as ``923941``, 897902.68 printed as ``897903`` (rounded UP),
92293.36 as ``92293.4``, and anything >= 1e6 as ``1e+06``. The ledger held
the exact value; only the CSV lied, which put a spurious 0.01-0.5 unit
"qty miss" on 1933/2708 entries of the EURUSD KI-52 all-in probe and on
every giua64 entry while the fills themselves were exact.

Eight decimals cover every lot step in use (1 share, 0.01 unit, 0.0001
contract, satoshi-scale 1e-8) and absorb binary noise from
floor(q/step)*step (0.30000000000000004 -> ``0.3``); trailing zeros are
trimmed the way TradingView prints its ``Size (qty)`` column, so every
value the old formatter rendered faithfully renders byte-identically.
"""
if not math.isfinite(qty):
return f"{qty:g}"
text = f"{qty:.8f}".rstrip("0").rstrip(".")
if text in ("", "-0", "-"):
return "0"
return text


def write_engine_trades_csv(trades: list[dict], path: Path) -> None:
"""Emit one row per trade *side* (exit then entry) in reverse-chronological
order — byte-for-byte alignable with TradingView's `trades.csv` export.
Expand Down Expand Up @@ -1552,7 +1608,7 @@ def write_engine_trades_csv(trades: list[dict], path: Path) -> None:
n, side,
_fmt_time_utc(t[time_key]),
f"{t[price_key]:.6f}",
f"{t['qty']:g}",
format_trade_qty(float(t["qty"])),
f"{t['pnl']:.6f}",
f"{t['pnl_pct']:.4f}",
f"{t['max_runup']:.6f}",
Expand Down Expand Up @@ -1677,6 +1733,10 @@ def _run_via_docker(strategy_dir: Path, ohlcv_path: Path, params: dict,
syminfo["timezone"] = run_kwargs["syminfo_timezone"]
if run_kwargs.get("syminfo_session"):
syminfo["session"] = run_kwargs["syminfo_session"]
if run_kwargs.get("syminfo_type"):
syminfo["type"] = run_kwargs["syminfo_type"]
for skey, sval in (run_kwargs.get("syminfo_strings") or {}).items():
syminfo[skey] = sval
if run_kwargs.get("syminfo_mintick") is not None:
syminfo["mintick"] = run_kwargs["syminfo_mintick"]
if run_kwargs.get("syminfo_pointvalue") is not None:
Expand Down
47 changes: 46 additions & 1 deletion scripts/test_run_strategy_identity.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
from pathlib import Path

from pf_release_run import report_trades_to_runstrategy_shape
from run_strategy import write_engine_trades_csv
from run_strategy import format_trade_qty, write_engine_trades_csv


class EngineEntryIdentityCsvTests(unittest.TestCase):
Expand Down Expand Up @@ -81,5 +81,50 @@ def test_release_report_mapping_preserves_incarnation(self) -> None:
self.assertEqual(mapped[0]["entry_incarnation"], 73)


class EngineTradeQtyFormatTests(unittest.TestCase):
"""The exported ``Qty`` column must carry the ledger quantity to the lot
step: ``%g``'s 6 significant digits truncated (and sometimes rounded UP)
every OANDA:EURUSD-scale lot (TV KI-52 all-in: 92293.36 units)."""

def test_large_lots_keep_every_lot_step_digit(self) -> None:
self.assertEqual(format_trade_qty(923941.16), "923941.16")
self.assertEqual(format_trade_qty(897902.68), "897902.68")
self.assertEqual(format_trade_qty(92293.36), "92293.36")
self.assertEqual(format_trade_qty(1000000.0), "1000000")
self.assertEqual(format_trade_qty(8741.59), "8741.59")

def test_binary_noise_and_trailing_zeros_are_trimmed(self) -> None:
self.assertEqual(format_trade_qty(0.30000000000000004), "0.3")
self.assertEqual(format_trade_qty(2.7051000000000001), "2.7051")
self.assertEqual(format_trade_qty(1.0), "1")
self.assertEqual(format_trade_qty(44.0), "44")
self.assertEqual(format_trade_qty(0.0001), "0.0001")
self.assertEqual(format_trade_qty(0.0), "0")

def test_values_the_old_formatter_rendered_exactly_are_unchanged(self) -> None:
for qty in (55.2872, 5.4103, 0.0196, 44.9622, 48, 7.7232, 30.3796):
self.assertEqual(format_trade_qty(float(qty)), f"{float(qty):g}")

def test_writer_uses_lot_faithful_qty(self) -> None:
trade = {
"entry_time": 1_735_689_600_000,
"exit_time": 1_735_689_660_000,
"entry_price": 1.08232,
"exit_price": 1.07836,
"pnl": -3658.81,
"pnl_pct": -0.37,
"is_long": True,
"max_runup": 0.0,
"max_drawdown": 0.0,
"qty": 923941.16,
}
with tempfile.TemporaryDirectory() as tmp:
path = Path(tmp) / "engine_trades.csv"
write_engine_trades_csv([trade], path)
with path.open(newline="", encoding="utf-8") as handle:
rows = list(csv.DictReader(handle))
self.assertEqual([row["Qty"] for row in rows], ["923941.16", "923941.16"])


if __name__ == "__main__":
unittest.main()
17 changes: 17 additions & 0 deletions src/c_abi.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -257,6 +257,23 @@ PF_API void strategy_set_syminfo_session(pf_strategy_t s, const char* session) {
static_cast<pineforge::BacktestEngine*>(s)->set_syminfo_session(std::string(session));
}

/* Plumb the instrument class (syminfo.type: "forex" / "stock" / "crypto" /
* "futures" / ...) into syminfo_. Defaults to "crypto"; NULL/empty ignored. */
PF_API void strategy_set_syminfo_type(pf_strategy_t s, const char* type) {
if (!s || !type) return;
static_cast<pineforge::BacktestEngine*>(s)->set_syminfo_type(std::string(type));
}

/* Generic string-member injection (ticker / tickerid / currency /
* basecurrency / description / volumetype / type). Returns 0 when set, -1
* for a NULL handle, unknown key or empty value. */
PF_API int strategy_set_syminfo_string(pf_strategy_t s, const char* key,
const char* value) {
if (!s || !key || !value) return -1;
return static_cast<pineforge::BacktestEngine*>(s)->set_syminfo_string(
std::string(key), std::string(value)) ? 0 : -1;
}

/* Inject the instrument tick size (syminfo.mintick). Drives the directional
* stop-entry snap (long ceil / short floor) and slippage = N*mintick economics.
* Defaults to 0.01 (crypto/equity); set per-instrument (e.g. 0.25 for ES,
Expand Down
Loading
Loading