Reading files from a file handle is not thread safe. If you try to read multiple files simultaneously (for instance using multi threading), it will usually fail. This happens because the pyobject_read_handler and pyobject_seek_handler functions get file_obj by using a global mutable variable. (I will open a PR to fix this)
from concurrent.futures import ThreadPoolExecutor
import pandas as pd
import pyreadstat
df = pd.DataFrame({"my_var": [1, 2, 3]})
num_threads = 10
paths = [f"sample{i}.sav" for i in range(num_threads)]
for path in paths:
pyreadstat.write_sav(df, path)
def read_sav_file(path):
with open(path, "rb") as fo:
df, meta = pyreadstat.read_sav(fo)
return df, meta
with ThreadPoolExecutor(max_workers=num_threads) as executor:
results = list(executor.map(read_sav_file, paths))
Traceback (most recent call last):
File "/home/ebs/pyreadstat-repro/main.py", line 20, in <module>
results = list(executor.map(read_sav_file, paths))
File "/usr/lib/python3.14/concurrent/futures/_base.py", line 645, in result_iterator
yield _result_or_cancel(fs.pop())
~~~~~~~~~~~~~~~~~^^^^^^^^^^
File "/usr/lib/python3.14/concurrent/futures/_base.py", line 312, in _result_or_cancel
return fut.result(timeout)
~~~~~~~~~~^^^^^^^^^
File "/usr/lib/python3.14/concurrent/futures/_base.py", line 447, in result
return self.__get_result()
~~~~~~~~~~~~~~~~~^^
File "/usr/lib/python3.14/concurrent/futures/_base.py", line 396, in __get_result
raise self._exception
File "/usr/lib/python3.14/concurrent/futures/thread.py", line 86, in run
result = ctx.run(self.task)
File "/usr/lib/python3.14/concurrent/futures/thread.py", line 73, in run
return fn(*args, **kwargs)
File "/home/ebs/pyreadstat-repro/main.py", line 15, in read_sav_file
df, meta = pyreadstat.read_sav(fo)
~~~~~~~~~~~~~~~~~~~^^^^
File "/home/ebs/pyreadstat-repro/.venv/lib/python3.14/site-packages/pyreadstat/pyreadstat.py", line 687, in read_sav
data_frame, metadata = parser_entry_point(
~~~~~~~~~~~~~~~~~~^
filename_path,
^^^^^^^^^^^^^^
...<14 lines>...
extra_time_formats=extra_time_formats,
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
)
^
File "pyreadstat/_readstat_parser.pyx", line 1411, in pyreadstat._readstat_parser.parser_entry_point
File "pyreadstat/_readstat_parser.pyx", line 1354, in pyreadstat._readstat_parser.run_conversion
run_readstat_parser(filename, data, file_extension, row_limit, row_offset, file_obj)
File "pyreadstat/_readstat_parser.pyx", line 1009, in pyreadstat._readstat_parser.run_readstat_parser
check_exit_status(error)
File "pyreadstat/_readstat_parser.pyx", line 913, in pyreadstat._readstat_parser.check_exit_status
raise ReadstatError(err_message)
pyreadstat._readstat_parser.ReadstatError: Unable to read from file
Reading files from a file handle is not thread safe. If you try to read multiple files simultaneously (for instance using multi threading), it will usually fail. This happens because the
pyobject_read_handlerandpyobject_seek_handlerfunctions getfile_objby using a global mutable variable. (I will open a PR to fix this)To reproduce:
Setup Information:
I have tested this on Ubuntu with both head and version 1.3.5 as installed from PyPI.