Problem with matplotlib date formatting and conversion from epoch time: OverlowError: int too big to convert
Question:
The time on the x-axis of my plot is in seconds since the epoch, and I want to convert them to "Year-Month-Day Hour:Minute" format. I follow the documentation here, and I get this traceback:
Traceback (most recent call last):
File "/home/andreas/src/masiri/booking_algorythm/simple_booking.py", line 277, in <module>
main()
File "/home/andreas/src/masiri/booking_algorythm/simple_booking.py", line 273, in main
accepted_list_of_bookings, accepted_bookings_a = process_booking_requests(randomized_list_of_bookings)
File "/home/andreas/src/masiri/booking_algorythm/simple_booking.py", line 257, in process_booking_requests
price = calculate_price_for_booking(trip_request, sequential_trip_requests,
File "/home/andreas/src/masiri/booking_algorythm/simple_booking.py", line 226, in calculate_price_for_booking
old_price = turn_booking_into_charging_plan(start_time, end_time, old_bookings_list)
File "/home/andreas/src/masiri/booking_algorythm/simple_booking.py", line 209, in turn_booking_into_charging_plan
plot_solution(sub_metric_v, soc_plot, charge_plot, discharge_plot, bookings_list, instability_ranges)
File "/home/andreas/src/masiri/booking_algorythm/simple_booking.py", line 195, in plot_solution
fig.autofmt_xdate()
File "/home/andreas/src/masiri/venv/lib/python3.10/site-packages/matplotlib/figure.py", line 249, in autofmt_xdate
for label in ax.get_xticklabels(which=which):
File "/home/andreas/src/masiri/venv/lib/python3.10/site-packages/matplotlib/axes/_base.py", line 73, in wrapper
return get_method(self)(*args, **kwargs)
File "/home/andreas/src/masiri/venv/lib/python3.10/site-packages/matplotlib/axis.py", line 1381, in get_ticklabels
return self.get_majorticklabels()
File "/home/andreas/src/masiri/venv/lib/python3.10/site-packages/matplotlib/axis.py", line 1345, in get_majorticklabels
self._update_ticks()
File "/home/andreas/src/masiri/venv/lib/python3.10/site-packages/matplotlib/axis.py", line 1191, in _update_ticks
major_labels = self.major.formatter.format_ticks(major_locs)
File "/home/andreas/src/masiri/venv/lib/python3.10/site-packages/matplotlib/ticker.py", line 233, in format_ticks
return [self(value, i) for i, value in enumerate(values)]
File "/home/andreas/src/masiri/venv/lib/python3.10/site-packages/matplotlib/ticker.py", line 233, in <listcomp>
return [self(value, i) for i, value in enumerate(values)]
File "/home/andreas/src/masiri/venv/lib/python3.10/site-packages/matplotlib/dates.py", line 640, in __call__
result = num2date(x, self.tz).strftime(self.fmt)
File "/home/andreas/src/masiri/venv/lib/python3.10/site-packages/matplotlib/dates.py", line 533, in num2date
return _from_ordinalf_np_vectorized(x, tz).tolist()
File "/home/andreas/src/masiri/venv/lib/python3.10/site-packages/numpy/lib/function_base.py", line 2328, in __call__
return self._vectorize_call(func=func, args=vargs)
File "/home/andreas/src/masiri/venv/lib/python3.10/site-packages/numpy/lib/function_base.py", line 2411, in _vectorize_call
outputs = ufunc(*inputs)
File "/home/andreas/src/masiri/venv/lib/python3.10/site-packages/matplotlib/dates.py", line 354, in _from_ordinalf
np.timedelta64(int(np.round(x * MUSECONDS_PER_DAY)), 'us'))
OverflowError: int too big to convert
This is my code (I will supply a minimal example, but as far as I can see, this is purely time conversion related).
def plot_solution(data, soc_plot,charge_plot, discharge_plot, bookings=None, boundaries=None):
my_fmt = mdates.DateFormatter('%Y-%m-%d %H:%M')
fig, ax_left = plt.subplots()
ax_right = ax_left.twinx()
ax_left.xaxis.set_major_formatter(my_fmt)
ax_right.xaxis.set_major_formatter(my_fmt)
time_axis = data["time"]
ax_right.plot(time_axis, soc_plot, label="State of Charge")
ax_left.plot(time_axis, data["price"], label="price (€/kWs)")
ax_left.plot(time_axis, data["price_co2"], label="price including CO_2 costs (€/kWs)")
#ax_right.plot(data["time"], charge_plot, label="charge")
#ax_right.plot(data["time"], discharge_plot, label="discharge")
bookings_flag = True
if bookings is not None:
for booking in bookings:
if bookings_flag:
ax_left.axvspan(booking["start"], booking["start"] + booking["duration"], facecolor='0.4', alpha=0.5, label="booking")
bookings_flag = False
else:
ax_left.axvspan(booking["start"], booking["start"] + booking["duration"], facecolor='0.4', alpha=0.5, label="booking")
boundaries_flag = True
if boundaries is not None:
for boundary in boundaries:
if boundaries_flag:
ax_left.axvspan(time_axis[boundary["start"]], time_axis[boundary["end"]], facecolor='0.9', alpha=0.5, label="boundary")
boundaries_flag = False
else:
ax_left.axvspan(time_axis[boundary["start"]], time_axis[boundary["end"]], facecolor='0.9', alpha=0.5)
ax_left.set_xlabel("time in seconds")
ax_left.set_ylabel("price in €/kWs", color="blue")
ax_right.set_ylabel("State of Charge (SoC), normalized", color="black")
legend_1 = ax_left.legend(loc=2, borderaxespad=1.)
legend_1.remove()
ax_right.legend(loc=1, borderaxespad=1.)
ax_right.add_artist(legend_1)
fig.autofmt_xdate()
plt.show(block=True)
Here is an example of my time_axis data:
>>> time_axis
array([1635905700, 1635906600, 1635907500, 1635908400, 1635909300,
1635910200, 1635911100, 1635912000, 1635912900, 1635913800,
Answers:
Dates in Matplotlib are stored as days since 1970 (if using Matplotlb >=3.5, the epoch was 0000 before). So if you want to use something like plot_date on straight floats you can if you convert the floats from seconds to days. However, better is probably to convert to datetime64.
import matplotlib.pyplot as plt
import numpy as np
import matplotlib as mpl
time = np.array([1635905700, 1635906600, 1635907500, 1635908400, 1635909300,
1635910200, 1635911100, 1635912000, 1635912900, 1635913800,])
y = np.arange(10)
plt.rcParams['date.converter'] = 'concise'
fig, ax = plt.subplots(2, 1, layout='constrained')
ax[0].plot(time/24/3600, y)
ax[0].xaxis_date()
ax[1].plot(time.astype('datetime64[s]'), y)
plt.show()
Note that I’ve also used the concise date converter https://matplotlib.org/stable/gallery/ticks/date_concise_formatter.html but that is just aesthetics.
xaxis_date (or ax.plot_date or fig.autofmt_xdate) are just short forms to set the units on the xaxis to datetime.
The time on the x-axis of my plot is in seconds since the epoch, and I want to convert them to "Year-Month-Day Hour:Minute" format. I follow the documentation here, and I get this traceback:
Traceback (most recent call last):
File "/home/andreas/src/masiri/booking_algorythm/simple_booking.py", line 277, in <module>
main()
File "/home/andreas/src/masiri/booking_algorythm/simple_booking.py", line 273, in main
accepted_list_of_bookings, accepted_bookings_a = process_booking_requests(randomized_list_of_bookings)
File "/home/andreas/src/masiri/booking_algorythm/simple_booking.py", line 257, in process_booking_requests
price = calculate_price_for_booking(trip_request, sequential_trip_requests,
File "/home/andreas/src/masiri/booking_algorythm/simple_booking.py", line 226, in calculate_price_for_booking
old_price = turn_booking_into_charging_plan(start_time, end_time, old_bookings_list)
File "/home/andreas/src/masiri/booking_algorythm/simple_booking.py", line 209, in turn_booking_into_charging_plan
plot_solution(sub_metric_v, soc_plot, charge_plot, discharge_plot, bookings_list, instability_ranges)
File "/home/andreas/src/masiri/booking_algorythm/simple_booking.py", line 195, in plot_solution
fig.autofmt_xdate()
File "/home/andreas/src/masiri/venv/lib/python3.10/site-packages/matplotlib/figure.py", line 249, in autofmt_xdate
for label in ax.get_xticklabels(which=which):
File "/home/andreas/src/masiri/venv/lib/python3.10/site-packages/matplotlib/axes/_base.py", line 73, in wrapper
return get_method(self)(*args, **kwargs)
File "/home/andreas/src/masiri/venv/lib/python3.10/site-packages/matplotlib/axis.py", line 1381, in get_ticklabels
return self.get_majorticklabels()
File "/home/andreas/src/masiri/venv/lib/python3.10/site-packages/matplotlib/axis.py", line 1345, in get_majorticklabels
self._update_ticks()
File "/home/andreas/src/masiri/venv/lib/python3.10/site-packages/matplotlib/axis.py", line 1191, in _update_ticks
major_labels = self.major.formatter.format_ticks(major_locs)
File "/home/andreas/src/masiri/venv/lib/python3.10/site-packages/matplotlib/ticker.py", line 233, in format_ticks
return [self(value, i) for i, value in enumerate(values)]
File "/home/andreas/src/masiri/venv/lib/python3.10/site-packages/matplotlib/ticker.py", line 233, in <listcomp>
return [self(value, i) for i, value in enumerate(values)]
File "/home/andreas/src/masiri/venv/lib/python3.10/site-packages/matplotlib/dates.py", line 640, in __call__
result = num2date(x, self.tz).strftime(self.fmt)
File "/home/andreas/src/masiri/venv/lib/python3.10/site-packages/matplotlib/dates.py", line 533, in num2date
return _from_ordinalf_np_vectorized(x, tz).tolist()
File "/home/andreas/src/masiri/venv/lib/python3.10/site-packages/numpy/lib/function_base.py", line 2328, in __call__
return self._vectorize_call(func=func, args=vargs)
File "/home/andreas/src/masiri/venv/lib/python3.10/site-packages/numpy/lib/function_base.py", line 2411, in _vectorize_call
outputs = ufunc(*inputs)
File "/home/andreas/src/masiri/venv/lib/python3.10/site-packages/matplotlib/dates.py", line 354, in _from_ordinalf
np.timedelta64(int(np.round(x * MUSECONDS_PER_DAY)), 'us'))
OverflowError: int too big to convert
This is my code (I will supply a minimal example, but as far as I can see, this is purely time conversion related).
def plot_solution(data, soc_plot,charge_plot, discharge_plot, bookings=None, boundaries=None):
my_fmt = mdates.DateFormatter('%Y-%m-%d %H:%M')
fig, ax_left = plt.subplots()
ax_right = ax_left.twinx()
ax_left.xaxis.set_major_formatter(my_fmt)
ax_right.xaxis.set_major_formatter(my_fmt)
time_axis = data["time"]
ax_right.plot(time_axis, soc_plot, label="State of Charge")
ax_left.plot(time_axis, data["price"], label="price (€/kWs)")
ax_left.plot(time_axis, data["price_co2"], label="price including CO_2 costs (€/kWs)")
#ax_right.plot(data["time"], charge_plot, label="charge")
#ax_right.plot(data["time"], discharge_plot, label="discharge")
bookings_flag = True
if bookings is not None:
for booking in bookings:
if bookings_flag:
ax_left.axvspan(booking["start"], booking["start"] + booking["duration"], facecolor='0.4', alpha=0.5, label="booking")
bookings_flag = False
else:
ax_left.axvspan(booking["start"], booking["start"] + booking["duration"], facecolor='0.4', alpha=0.5, label="booking")
boundaries_flag = True
if boundaries is not None:
for boundary in boundaries:
if boundaries_flag:
ax_left.axvspan(time_axis[boundary["start"]], time_axis[boundary["end"]], facecolor='0.9', alpha=0.5, label="boundary")
boundaries_flag = False
else:
ax_left.axvspan(time_axis[boundary["start"]], time_axis[boundary["end"]], facecolor='0.9', alpha=0.5)
ax_left.set_xlabel("time in seconds")
ax_left.set_ylabel("price in €/kWs", color="blue")
ax_right.set_ylabel("State of Charge (SoC), normalized", color="black")
legend_1 = ax_left.legend(loc=2, borderaxespad=1.)
legend_1.remove()
ax_right.legend(loc=1, borderaxespad=1.)
ax_right.add_artist(legend_1)
fig.autofmt_xdate()
plt.show(block=True)
Here is an example of my time_axis data:
>>> time_axis
array([1635905700, 1635906600, 1635907500, 1635908400, 1635909300,
1635910200, 1635911100, 1635912000, 1635912900, 1635913800,
Dates in Matplotlib are stored as days since 1970 (if using Matplotlb >=3.5, the epoch was 0000 before). So if you want to use something like plot_date on straight floats you can if you convert the floats from seconds to days. However, better is probably to convert to datetime64.
import matplotlib.pyplot as plt
import numpy as np
import matplotlib as mpl
time = np.array([1635905700, 1635906600, 1635907500, 1635908400, 1635909300,
1635910200, 1635911100, 1635912000, 1635912900, 1635913800,])
y = np.arange(10)
plt.rcParams['date.converter'] = 'concise'
fig, ax = plt.subplots(2, 1, layout='constrained')
ax[0].plot(time/24/3600, y)
ax[0].xaxis_date()
ax[1].plot(time.astype('datetime64[s]'), y)
plt.show()
Note that I’ve also used the concise date converter https://matplotlib.org/stable/gallery/ticks/date_concise_formatter.html but that is just aesthetics.
xaxis_date (or ax.plot_date or fig.autofmt_xdate) are just short forms to set the units on the xaxis to datetime.