���]�N � @ s� d Z d d l Z d d l Z d d l m Z d d l Z d d l m Z d d l Z d d l m Z d d l m
Z
d d l Z d d l Z d d l
m Z d d l Z d d l Z e j � Z d a d d
� Z d Z Gd d
� d
e � Z Gd d � d � Z d d � Z Gd d � d e � Z Gd d � d e � Z Gd d � d e � Z d d � Z d d � Z d d � Z d d � Z! d d! � Z" d a# d a$ d" d# � Z% Gd$ d% � d% e& � Z' Gd& d' � d' e j( � Z) e j* e � d S)(z"Brian Quinlan (brian@sweetapp.com)� N)�_base)�Full)� SimpleQueue)�wait)�partialFc C sa d a t t j � � } x! | D] \ } } | j d � q Wx | D] \ } } | j � qC Wd S)NT)� _shutdown�list�_threads_queues�items�put�join)r
�t�q� r �,/opt/alt/python35/lib64/python3.5/process.py� _python_exitO s r � c @ s( e Z d Z d d � Z d d � Z d S)�_RemoteTracebackc C s
| | _ d S)N)�tb)�selfr r r r �__init__a s z_RemoteTraceback.__init__c C s | j S)N)r )r r r r �__str__c s z_RemoteTraceback.__str__N)�__name__�
__module__� __qualname__r r r r r r r ` s r c @ s( e Z d Z d d � Z d d � Z d S)�_ExceptionWithTracebackc C sD t j t | � | | � } d j | � } | | _ d | | _ d S)N� z
"""
%s""")� traceback�format_exception�typer �excr )r r r r r r r g s z _ExceptionWithTraceback.__init__c C s t | j | j f f S)N)� _rebuild_excr r )r r r r �
__reduce__l s z"_ExceptionWithTraceback.__reduce__N)r r r r r" r r r r r f s r c C s t | � | _ | S)N)r � __cause__)r r r r r r! o s r! c @ s e Z d Z d d � Z d S)� _WorkItemc C s( | | _ | | _ | | _ | | _ d S)N)�future�fn�args�kwargs)r r% r& r' r( r r r r t s z_WorkItem.__init__N)r r r r r r r r r$ s s r$ c @ s" e Z d Z d d d d � Z d S)� _ResultItemNc C s | | _ | | _ | | _ d S)N)�work_id� exception�result)r r* r+ r, r r r r { s z_ResultItem.__init__)r r r r r r r r r) z s r) c @ s e Z d Z d d � Z d S)� _CallItemc C s( | | _ | | _ | | _ | | _ d S)N)r* r& r' r( )r r* r& r' r( r r r r � s z_CallItem.__init__N)r r r r r r r r r- � s r- c g s> t | � } x+ t t j | | � � } | s1 d S| Vq Wd S)N)�zip�tuple� itertools�islice)� chunksize� iterables�it�chunkr r r � _get_chunks� s r6 c s � f d d � | D� S)Nc s g | ] } � | � � q Sr r )�.0r' )r& r r �
<listcomp>� s z"_process_chunk.<locals>.<listcomp>r )r& r5 r )r& r �_process_chunk� s r9 c C s� x� | j d d � } | d k r8 | j t j � � d Sy | j | j | j � } WnS t k
r� } z3 t | | j � } | j t
| j d | �� WYd d } ~ Xq X| j t
| j d | �� q Wd S)N�blockTr+ r, ) �getr �os�getpidr& r' r( �
BaseExceptionr �
__traceback__r) r* )�
call_queue� result_queue� call_item�r�er r r r �_process_worker� s
/rE c C s� x� | j � r d Sy | j d d � } Wn t j k
rD d SYq X| | } | j j � r� | j t | | j | j | j
� d d �q | | =q q Wd S)Nr: FT) �fullr; �queue�Emptyr% � set_running_or_notify_cancelr r- r& r' r( )�pending_work_items�work_idsr@ r* � work_itemr r r �_add_call_item_to_queue� s
rM c s d � � f d d � } � � f d d � } | j } x�t | | � � d d � � j � D� } t | g | � }
| |
k r� | j � } n� | � � � d k r� d � _ d � _ d � x3 | j � D]% \ } }
|
j j t
d � � ~
q� W| j � x � j � D] } | j � q
W| � d St
| t � re� j | � } | j � � s�| � d Snb | d k r�| j | j d � }
|
d k r�| j r�|
j j | j � n |
j j | j � ~
| � � | � ry | s�| � d SWn t k
rYn Xd � q9 Wd S) Nc s t p � d k p � j S)N)r �_shutdown_threadr )�executorr r �
shutting_down� s z/_queue_management_worker.<locals>.shutting_downc su t d d � � j � D� � } x$ t d | � D] } � j d � q/ W� j � x � j � D] } | j � q] Wd S)Nc s s | ] } | j � Vq d S)N)�is_alive)r7 �pr r r � <genexpr>� s zD_queue_management_worker.<locals>.shutdown_worker.<locals>.<genexpr>r )�sum�values�range�
put_nowait�closer )�nb_children_alive�irR )r@ � processesr r �shutdown_worker� s
z1_queue_management_worker.<locals>.shutdown_workerc S s g | ] } | j � q Sr )�sentinel)r7 rR r r r r8 s z,_queue_management_worker.<locals>.<listcomp>Tz^A process in the process pool was terminated abruptly while the future was running or pending.)�_readerrM rU r �recv�_brokenrN r
r% �
set_exception�BrokenProcessPool�clear� terminate�
isinstance�int�popr r* r+ �
set_resultr, r )�executor_referencer[ rJ �work_ids_queuer@ rA rP r\ �reader� sentinels�ready� result_itemr* rL rR r )r@ rO r[ r �_queue_management_worker� sb
ro c C s� t r t r t t � � d a y t j d � } Wn t t f k
rO d SYn X| d k r` d S| d k rp d Sd | a t t � � d S)NT�SC_SEM_NSEMS_MAXr � z@system provides too few semaphores (%d available, 256 necessary)���)�_system_limits_checked�_system_limited�NotImplementedErrorr< �sysconf�AttributeError�
ValueError)� nsems_maxr r r �_check_system_limitsQ s
rz c @ s e Z d Z d S)rb N)r r r r r r r rb h s rb c s� e Z d Z d d d � Z d d � Z d d � Z d d � Z e j j j e _ d
d d d � f d
d � Z
d d d � Z e j j j e _ � S)�ProcessPoolExecutorNc C s� t � | d k r+ t j � p" d | _ n! | d k rC t d � � | | _ t j | j t � | _ d | j _ t
� | _ t j � | _
d | _ i | _ d | _ t j � | _ d | _ d | _ i | _ d S)Nr r z"max_workers must be greater than 0TF)rz r< � cpu_count� _max_workersrx �multiprocessing�Queue�EXTRA_QUEUED_CALLS� _call_queue�
_ignore_epiper �
_result_queuerG � _work_ids�_queue_management_thread�
_processesrN � threading�Lock�_shutdown_lockr` � _queue_count�_pending_work_items)r � max_workersr r r r p s$
z ProcessPoolExecutor.__init__c
C s� | j d d � } | j d k r� | j � t j d t d t j | | � | j | j | j
| j | j f � | _ d | j _ | j j
� | j t | j <d S)Nc S s | j d � d S)N)r )�_r r r r �
weakref_cb� s zFProcessPoolExecutor._start_queue_management_thread.<locals>.weakref_cb�targetr' T)r� r� �_adjust_process_countr� �Threadro �weakref�refr� r� r� r� �daemon�startr )r r� r r r � _start_queue_management_thread� s
z2ProcessPoolExecutor._start_queue_management_threadc C sh xa t t | j � | j � D]D } t j d t d | j | j f � } | j � | | j | j
<q Wd S)Nr� r' ) rV �lenr� r} r~ �ProcessrE r� r� r� �pid)r r� rR r r r r� � s "
z)ProcessPoolExecutor._adjust_process_countc O s� | j �� | j r t d � � | j r4 t d � � t j � } t | | | | � } | | j | j <| j
j | j � | j d 7_ | j j d � | j
� | SWd QRXd S)NzKA child process terminated abruptly, the process pool is not usable anymorez*cannot schedule new futures after shutdownr )r� r` rb rN � RuntimeErrorr �Futurer$ r� r� r� r r� r� )r r& r' r( �f�wr r r �submit� s
zProcessPoolExecutor.submit�timeoutr2 r c sX | d k r t d � � t � j t t | � t | d | � d | �} t j j | � S)Nr zchunksize must be >= 1.r2 r� ) rx �super�mapr r9 r6 r0 �chain�
from_iterable)r r& r� r2 r3 �results)� __class__r r r� � s zProcessPoolExecutor.mapTc C sn | j � d | _ Wd QRX| j rF | j j d � | rF | j j � d | _ d | _ d | _ d | _ d S)NT)r� rN r� r� r r r� r� )r r r r r �shutdown� s
z ProcessPoolExecutor.shutdown) r r r r r� r� r� r �Executor�__doc__r� r� r r )r� r r{ o s ( r{ )+�
__author__�atexitr< �concurrent.futuresr rG r r~ r �multiprocessing.connectionr r� r� � functoolsr r0 r �WeakKeyDictionaryr r r r� � Exceptionr r r! �objectr$ r) r- r6 r9 rE rM ro rs rt rz r� rb r� r{ �registerr r r r �<module>. sD
% s �