Fix load explosion: use subprocess for binary data plots to avoid thread conflict
Co-authored-by: copilot-swe-agent[bot] <198982749+copilot@users.noreply.github.com>
This commit is contained in:
@@ -17,7 +17,9 @@ import os ## operating system utilities
|
||||
|
||||
import plot_binary_data
|
||||
import AMSS_NCKU_Input as input_data
|
||||
from parallel_plot_helper import run_plot_tasks_parallel
|
||||
import subprocess
|
||||
import sys
|
||||
import multiprocessing
|
||||
|
||||
# plt.rcParams['text.usetex'] = True ## enable LaTeX fonts in plots
|
||||
|
||||
@@ -53,13 +55,40 @@ def generate_binary_data_plot( binary_outdir, figure_outdir ):
|
||||
file_list.append(x)
|
||||
print(x)
|
||||
|
||||
## Plot each file in the list (in parallel)
|
||||
plot_tasks = []
|
||||
## Plot each file in parallel using subprocesses.
|
||||
## Each subprocess is a fresh Python process where the BLAS thread-count
|
||||
## environment variables (set at the top of plot_binary_data.py) take
|
||||
## effect before numpy is imported. This avoids the thread explosion
|
||||
## that occurs when multiprocessing.Pool with 'fork' context inherits
|
||||
## already-initialized multi-threaded BLAS from the parent.
|
||||
script = os.path.join( os.path.dirname(__file__), "plot_binary_data.py" )
|
||||
max_workers = min( multiprocessing.cpu_count(), len(file_list) ) if file_list else 0
|
||||
|
||||
running = []
|
||||
failed = []
|
||||
for filename in file_list:
|
||||
print(filename)
|
||||
plot_tasks.append( ( plot_binary_data.plot_binary_data, (filename, binary_outdir, figure_outdir) ) )
|
||||
proc = subprocess.Popen(
|
||||
[sys.executable, script, filename, binary_outdir, figure_outdir],
|
||||
)
|
||||
running.append( (proc, filename) )
|
||||
## Keep at most max_workers subprocesses active at a time
|
||||
if len(running) >= max_workers:
|
||||
p, fn = running.pop(0)
|
||||
p.wait()
|
||||
if p.returncode != 0:
|
||||
failed.append(fn)
|
||||
|
||||
run_plot_tasks_parallel(plot_tasks)
|
||||
## Wait for all remaining subprocesses to finish
|
||||
for p, fn in running:
|
||||
p.wait()
|
||||
if p.returncode != 0:
|
||||
failed.append(fn)
|
||||
|
||||
if failed:
|
||||
print( " WARNING: the following binary data plots failed:" )
|
||||
for fn in failed:
|
||||
print( " ", fn )
|
||||
|
||||
print( )
|
||||
print( " Binary Data Plot Has been Finished " )
|
||||
|
||||
Reference in New Issue
Block a user