run_systems crash with MILK, LUMBER-new

[Plex134]

Good evening,

I am running pysystemtrade with DO in a paper account and have started to add some less liquid markets such as MILK and LUMBER-new. When I add these from Barchart I am finding run_systems crashes. I also tried MILK from the shipped data and got the same error. I was wondering if anyone has encountered this issue and how they proceeded? For now I am just dropping these instruments into ignore_instruments,.

Here is the traceback:

2025-01-10 20:33:32 DEBUG base_system {'stage': 'portfolio', 'instrument_code': 'ALUMINIUM'} Following instruments will have zero weight in optimisation of instrument weights as they have no positions (possibly too expensive?) ['LUMBER-new']
Traceback (most recent call last):
  File "/home/cam/miniconda3/envs/pysystemtrade_env/lib/python3.10/site-packages/numpy/core/fromnumeric.py", line 59, in _wrapfunc
    return bound(*args, **kwds)
TypeError: '<' not supported between instances of 'int' and 'Timestamp'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/home/cam/pysystemtrade/sysproduction/linux/scripts/run.py", line 66, in <module>
    func(*args, **kwargs)
  File "/home/cam/pysystemtrade/sysproduction/run_systems.py", line 22, in run_systems
    system_process.run_process()
  File "/home/cam/pysystemtrade/syscontrol/run_process.py", line 105, in run_process
    self._main_loop_over_methods()
  File "/home/cam/pysystemtrade/syscontrol/run_process.py", line 131, in _main_loop_over_methods
    timer_class.check_and_run(**kwargs)
  File "/home/cam/pysystemtrade/syscontrol/timer_functions.py", line 122, in check_and_run
    self.run_function(**kwargs)
  File "/home/cam/pysystemtrade/syscontrol/timer_functions.py", line 294, in run_function
    self._function(**kwargs)
  File "/home/cam/pysystemtrade/sysproduction/run_systems.py", line 26, in backtest_function_to_be_renamed
    self.run_strategy_method()
  File "/home/cam/pysystemtrade/syscontrol/strategy_tools.py", line 41, in run_strategy_method
    method()
  File "/home/cam/pysystemtrade/sysproduction/strategy_code/run_system_classic.py", line 64, in run_backtest
    function_to_call_on_update(
  File "/home/cam/pysystemtrade/sysproduction/strategy_code/run_dynamic_optimised_system.py", line 118, in updated_optimal_positions_for_dynamic_system
    position_entry = construct_optimal_position_entry(
  File "/home/cam/pysystemtrade/sysproduction/strategy_code/run_dynamic_optimised_system.py", line 140, in construct_optimal_position_entry
    optimal_position = get_optimal_position_from_system(system, instrument_code)
  File "/home/cam/pysystemtrade/sysproduction/strategy_code/run_dynamic_optimised_system.py", line 157, in get_optimal_position_from_system
    optimal_position = system.portfolio.get_notional_position(instrument_code)
  File "/home/cam/pysystemtrade/systems/system_cache.py", line 726, in wrapper
    ans = system.cache.calc_or_cache(
  File "/home/cam/pysystemtrade/systems/system_cache.py", line 573, in calc_or_cache
    value = func(this_stage, *args, **kwargs)
  File "/home/cam/pysystemtrade/systems/portfolio.py", line 211, in get_notional_position
    self.get_notional_position_before_risk_scaling(instrument_code)
  File "/home/cam/pysystemtrade/systems/system_cache.py", line 726, in wrapper
    ans = system.cache.calc_or_cache(
  File "/home/cam/pysystemtrade/systems/system_cache.py", line 573, in calc_or_cache
    value = func(this_stage, *args, **kwargs)
  File "/home/cam/pysystemtrade/systems/portfolio.py", line 237, in get_notional_position_before_risk_scaling
    notional_position_without_idm = self.get_notional_position_without_idm(
  File "/home/cam/pysystemtrade/systems/system_cache.py", line 726, in wrapper
    ans = system.cache.calc_or_cache(
  File "/home/cam/pysystemtrade/systems/system_cache.py", line 573, in calc_or_cache
    value = func(this_stage, *args, **kwargs)
  File "/home/cam/pysystemtrade/systems/portfolio.py", line 252, in get_notional_position_without_idm
    instr_weights = self.get_instrument_weights()
  File "/home/cam/pysystemtrade/systems/system_cache.py", line 726, in wrapper
    ans = system.cache.calc_or_cache(
  File "/home/cam/pysystemtrade/systems/system_cache.py", line 573, in calc_or_cache
    value = func(this_stage, *args, **kwargs)
  File "/home/cam/pysystemtrade/systems/portfolio.py", line 433, in get_instrument_weights
    self.get_unsmoothed_instrument_weights_fitted_to_position_lengths()
  File "/home/cam/pysystemtrade/systems/system_cache.py", line 726, in wrapper
    ans = system.cache.calc_or_cache(
  File "/home/cam/pysystemtrade/systems/system_cache.py", line 573, in calc_or_cache
    value = func(this_stage, *args, **kwargs)
  File "/home/cam/pysystemtrade/systems/portfolio.py", line 464, in get_unsmoothed_instrument_weights_fitted_to_position_lengths
    instrument_weights = fix_weights_vs_position_or_forecast(
  File "/home/cam/pysystemtrade/syscore/pandas/strategy_functions.py", line 231, in fix_weights_vs_position_or_forecast
    adj_weights = uniquets(weights)
  File "/home/cam/pysystemtrade/syscore/pandas/pdutils.py", line 55, in uniquets
    x = x.groupby(level=0).last()
  File "/home/cam/miniconda3/envs/pysystemtrade_env/lib/python3.10/site-packages/pandas/core/groupby/groupby.py", line 3376, in last
    return self._agg_general(
  File "/home/cam/miniconda3/envs/pysystemtrade_env/lib/python3.10/site-packages/pandas/core/groupby/groupby.py", line 1839, in _agg_general
    result = self._cython_agg_general(
  File "/home/cam/miniconda3/envs/pysystemtrade_env/lib/python3.10/site-packages/pandas/core/groupby/groupby.py", line 1929, in _cython_agg_general
    new_mgr = data.grouped_reduce(array_func)
  File "/home/cam/miniconda3/envs/pysystemtrade_env/lib/python3.10/site-packages/pandas/core/internals/managers.py", line 1431, in grouped_reduce
    applied = blk.apply(func)
  File "/home/cam/miniconda3/envs/pysystemtrade_env/lib/python3.10/site-packages/pandas/core/internals/blocks.py", line 366, in apply
    result = func(self.values, **kwargs)
  File "/home/cam/miniconda3/envs/pysystemtrade_env/lib/python3.10/site-packages/pandas/core/groupby/groupby.py", line 1905, in array_func
    result = self.grouper._cython_operation(
  File "/home/cam/miniconda3/envs/pysystemtrade_env/lib/python3.10/site-packages/pandas/core/groupby/ops.py", line 811, in _cython_operation
    cy_op = WrappedCythonOp(kind=kind, how=how, has_dropped_na=self.has_dropped_na)
  File "properties.pyx", line 36, in pandas._libs.properties.CachedProperty.__get__
  File "/home/cam/miniconda3/envs/pysystemtrade_env/lib/python3.10/site-packages/pandas/core/groupby/ops.py", line 725, in has_dropped_na
    return bool((self.group_info[0] < 0).any())
  File "properties.pyx", line 36, in pandas._libs.properties.CachedProperty.__get__
  File "/home/cam/miniconda3/envs/pysystemtrade_env/lib/python3.10/site-packages/pandas/core/groupby/ops.py", line 729, in group_info
    comp_ids, obs_group_ids = self._get_compressed_codes()
  File "/home/cam/miniconda3/envs/pysystemtrade_env/lib/python3.10/site-packages/pandas/core/groupby/ops.py", line 753, in _get_compressed_codes
    return ping.codes, np.arange(len(ping.group_index), dtype=np.intp)
  File "/home/cam/miniconda3/envs/pysystemtrade_env/lib/python3.10/site-packages/pandas/core/groupby/grouper.py", line 691, in codes
    return self._codes_and_uniques[0]
  File "properties.pyx", line 36, in pandas._libs.properties.CachedProperty.__get__
  File "/home/cam/miniconda3/envs/pysystemtrade_env/lib/python3.10/site-packages/pandas/core/groupby/grouper.py", line 801, in _codes_and_uniques
    codes, uniques = algorithms.factorize(  # type: ignore[assignment]
  File "/home/cam/miniconda3/envs/pysystemtrade_env/lib/python3.10/site-packages/pandas/core/algorithms.py", line 763, in factorize
    return values.factorize(sort=sort, use_na_sentinel=use_na_sentinel)
  File "/home/cam/miniconda3/envs/pysystemtrade_env/lib/python3.10/site-packages/pandas/core/base.py", line 1195, in factorize
    codes, uniques = algorithms.factorize(
  File "/home/cam/miniconda3/envs/pysystemtrade_env/lib/python3.10/site-packages/pandas/core/algorithms.py", line 802, in factorize
    uniques, codes = safe_sort(
  File "/home/cam/miniconda3/envs/pysystemtrade_env/lib/python3.10/site-packages/pandas/core/algorithms.py", line 1593, in safe_sort
    ordered = _sort_mixed(values)
  File "/home/cam/miniconda3/envs/pysystemtrade_env/lib/python3.10/site-packages/pandas/core/algorithms.py", line 1667, in _sort_mixed
    num_argsort = np.argsort(values[num_pos])
  File "/home/cam/miniconda3/envs/pysystemtrade_env/lib/python3.10/site-packages/numpy/core/fromnumeric.py", line 1133, in argsort
    return _wrapfunc(a, 'argsort', axis=axis, kind=kind, order=order)
  File "/home/cam/miniconda3/envs/pysystemtrade_env/lib/python3.10/site-packages/numpy/core/fromnumeric.py", line 68, in _wrapfunc
    return _wrapit(obj, method, *args, **kwds)
  File "/home/cam/miniconda3/envs/pysystemtrade_env/lib/python3.10/site-packages/numpy/core/fromnumeric.py", line 45, in _wrapit
    result = getattr(asarray(obj), method)(*args, **kwds)
TypeError: '<' not supported between instances of 'int' and 'Timestamp'

Thank you, any help is appreciated.

[bug-or-feature]

I have seen this too, and have a potential fix. I'll create a PR

[Plex134]

Thank you