File "pool.cpython-34.pyc"
Full Path: /home/attunedd/public_html/byp/izo/con7ext_sym404/rintoar.txt/opt/alt/python34/lib64/python3.4/multiprocessing/__pycache__/pool.cpython-34.pyc
File size: 22.61 KB
MIME-type: text/x-bytecode.python
Charset: 8 bit
e fa @ s d d g Z d d l Z d d l Z d d l Z d d l Z d d l Z d d l Z d d l Z d d l m Z d d l m
Z
m Z d Z d Z
d Z e j Z d d Z d
d Z Gd d
d
e Z Gd d d Z d d Z Gd d d e Z d f d d d d Z Gd d d e Z Gd d d e Z e Z Gd d d e Z Gd d d e Z Gd d d e Z Gd d d e Z d S)!Pool
ThreadPool N )util) get_context TimeoutError c C s t t | S)N)listmap)args r 9/opt/alt/python34/lib64/python3.4/multiprocessing/pool.pymapstar+ s r c C s t t j | d | d S)Nr r )r itertoolsstarmap)r r r r
starmapstar. s r c @ s( e Z d Z d d Z d d Z d S)RemoteTracebackc C s
| | _ d S)N)tb)selfr r r r
__init__6 s zRemoteTraceback.__init__c C s | j S)N)r )r r r r
__str__8 s zRemoteTraceback.__str__N)__name__
__module__ __qualname__r r r r r r
r 5 s r c @ s( e Z d Z d d Z d d Z d S)ExceptionWithTracebackc C sD t j t | | | } d j | } | | _ d | | _ d S)N z
"""
%s""") tracebackformat_exceptiontypejoinexcr )r r r r r r
r < s z ExceptionWithTraceback.__init__c C s t | j | j f f S)N) rebuild_excr r )r r r r
__reduce__A s z!ExceptionWithTraceback.__reduce__N)r r r r r" r r r r
r ; s r c C s t | | _ | S)N)r __cause__)r r r r r
r! D s r! c s@ e Z d Z d Z f d d Z d d Z d d Z S)MaybeEncodingErrorzVWraps possible unpickleable errors, so they can be
safely sent through the socket.c sA t | | _ t | | _ t t | j | j | j d S)N)reprr valuesuperr$ r )r r r& ) __class__r r
r P s zMaybeEncodingError.__init__c C s d | j | j f S)Nz(Error sending result: '%s'. Reason: '%s')r& r )r r r r
r U s zMaybeEncodingError.__str__c C s d t | S)Nz<MaybeEncodingError: %s>)str)r r r r
__repr__Y s zMaybeEncodingError.__repr__)r r r __doc__r r r* r r )r( r
r$ L s r$ Fc ' C s | d k s0 t | t k r* | d k s0 t | j } | j } t | d rn | j j | j j n | d k r | | n d } xx| d k s | r| | k ry
| } Wn& t t
f k
r t j d PYn X| d k rt j d Pn | \ }
} } }
} y d | |
| f } WnL t
k
r}} z, | r_t | | j } n d | f } WYd d } ~ Xn Xy | |
| | f Wnb t
k
r} zB t | | d } t j d | | |
| d | f f WYd d } ~ Xn X| d 7} q Wt j d | d S)
Nr _writerz)worker got EOFError or OSError -- exitingz worker got sentinel -- exitingTFr z0Possible encoding error while sending result: %sz worker exiting after %d tasks)r intAssertionErrorputgethasattrr, close_readerEOFErrorOSErrorr debug Exceptionr
__traceback__r$ )inqueueoutqueue initializerinitargsZmaxtasksZwrap_exceptionr/ r0 Z completedtaskjobifuncr kwdsresultewrappedr r r
worker] sD 0
!
,rE c @ s e Z d Z d Z d Z d d Z d d f d d d d Z d d Z d
d Z d d
Z d d Z
f i d d Z d d d Z d d d Z
d d d d d Z d d d Z d d d Z f i d d d d Z d d d d d Z d d d d! d" Z e d# d$ Z e d% d& Z e d' d( Z e d) d* Z d+ d, Z d- d. Z d/ d0 Z d1 d2 Z e d3 d4 Z e d5 d6 Z d7 d8 Z d9 d: Z! d S);r zS
Class which supports an async version of applying functions to arguments.
Tc O s | j j | | S)N)_ctxProcess)r r rA r r r
rG s z Pool.ProcessNc C s, | p t | _ | j t j | _ i | _ t | _ | | _ | | _
| | _ | d k ry t j
ps d } n | d k r t d n | d k r t | r t d n | | _ g | _ | j t j d t j d | f | _ d | j _ t | j _ | j j t j d t j d | j | j | j | j | j f | _ d | j _ t | j _ | j j t j d t j d | j | j | j f | _! d | j! _ t | j! _ | j! j t" j# | | j$ d | j | j% | j | j | j | j | j! | j f d d | _& d S) Nr z&Number of processes must be at least 1z initializer must be a callabletargetr TZ exitpriority )'r rF
_setup_queuesqueueQueue
_taskqueue_cacheRUN_state_maxtasksperchild _initializer _initargsos cpu_count
ValueErrorcallable TypeError
_processes_pool_repopulate_pool threadingZThreadr _handle_workers_worker_handlerdaemonstart
_handle_tasks
_quick_put _outqueue
_task_handler_handle_results
_quick_get_result_handlerr ZFinalize_terminate_pool_inqueue
_terminate)r processesr; r< Zmaxtasksperchildcontextr r r
r sT
z
Pool.__init__c C sz d } xm t t t | j D]P } | j | } | j d k r" t j d | | j d } | j | =q" q" W| S)zCleanup after any worker processes which have exited due to reaching
their specified lifetime. Returns True if any workers were cleaned up.
FNzcleaning up worker %dT)reversedrangelenrZ exitcoder r6 r )r Zcleanedr? rE r r r
_join_exited_workers s "
zPool._join_exited_workersc C s x t | j t | j D] } | j d t d | j | j | j | j | j
| j f } | j j | | j
j d d | _
d | _ | j t j d q Wd S)zBring the number of pool processes up to the specified number,
for use after reaping workers which have exited.
rH r rG Z
PoolWorkerTz added workerN)rn rY ro rZ rG rE ri rc rR rS rQ _wrap_exceptionappendnamereplacer_ r` r r6 )r r? wr r r
r[ s #
zPool._repopulate_poolc C s | j r | j n d S)zEClean up any exited workers and start replacements for them.
N)rq r[ )r r r r
_maintain_pool s zPool._maintain_poolc C sL | j j | _ | j j | _ | j j j | _ | j j j | _ d S)N)
rF Z SimpleQueueri rc r, sendrb r3 recvrf )r r r r
rJ s zPool._setup_queuesc C s. | j t k s t | j | | | j S)z6
Equivalent of `func(*args, **kwds)`.
)rP rO r. apply_asyncr0 )r r@ r rA r r r
apply s z
Pool.applyc C s | j | | t | j S)zx
Apply `func` to each element in `iterable`, collecting the results
in a list that is returned.
)
_map_asyncr r0 )r r@ iterable chunksizer r r
r
s zPool.mapc C s | j | | t | j S)z
Like `map()` method but the elements of the `iterable` are expected to
be iterables as well and will be unpacked as arguments. Hence
`func` and (a, b) becomes func(a, b).
)r| r r0 )r r@ r} r~ r r r
r s z Pool.starmapc C s | j | | t | | | S)z=
Asynchronous version of `starmap()` method.
)r| r )r r@ r} r~ callbackerror_callbackr r r
starmap_async s zPool.starmap_asyncr c s | j t k r t d n | d k rr t | j | j j f d d t | D j f S| d k s t t
j | | } t | j | j j f d d t | D j f d d D Sd S)zP
Equivalent of `map()` -- can be MUCH slower than `Pool.map()`.
zPool not runningr c 3 s0 | ]& \ } } j | | f i f Vq d S)N)_job).0r? x)r@ rB r r
<genexpr> s z Pool.imap.<locals>.<genexpr>c 3 s0 | ]& \ } } j | t | f i f Vq d S)N)r r )r r? r )rB r r
r % s c s s" | ] } | D] } | Vq
q d S)Nr )r chunkitemr r r
r ' s N) rP rO rV IMapIteratorrN rM r/ enumerate _set_lengthr. r
_get_tasks)r r@ r} r~ task_batchesr )r@ rB r
imap s z Pool.imapc s | j t k r t d n | d k rr t | j | j j f d d t | D j f S| d k s t t
j | | } t | j | j j f d d t | D j f d d D Sd S)zL
Like `imap()` method but ordering of results is arbitrary.
zPool not runningr c 3 s0 | ]& \ } } j | | f i f Vq d S)N)r )r r? r )r@ rB r r
r 1 s z&Pool.imap_unordered.<locals>.<genexpr>c 3 s0 | ]& \ } } j | t | f i f Vq d S)N)r r )r r? r )rB r r
r 8 s c s s" | ] } | D] } | Vq
q d S)Nr )r r r r r r
r : s N) rP rO rV IMapUnorderedIteratorrN rM r/ r r r. r r )r r@ r} r~ r r )r@ rB r
imap_unordered) s zPool.imap_unorderedc C sb | j t k r t d n t | j | | } | j j | j d | | | f g d f | S)z;
Asynchronous version of `apply()` method.
zPool not runningN)rP rO rV ApplyResultrN rM r/ r )r r@ r rA r r rB r r r
rz < s
+zPool.apply_asyncc C s | j | | t | | | S)z9
Asynchronous version of `map()` method.
)r| r )r r@ r} r~ r r r r r
map_asyncG s zPool.map_asyncc s | j t k r t d n t | d s< t | } n | d k r t t | t | j d \ } } | r | d 7} q n t | d k r d } n t j | | | } t
| j | t | | d | | j j
f d d t | D d f S)
zY
Helper function to implement map, starmap and their async counterparts.
zPool not running__len__N r r r c 3 s0 | ]& \ } } j | | f i f Vq d S)N)r )r r? r )mapperrB r r
r c s z"Pool._map_async.<locals>.<genexpr>)rP rO rV r1 r divmodro rZ r r MapResultrN rM r/ r ) r r@ r} r r~ r r Zextrar r )r rB r
r| O s ( zPool._map_asyncc C sr t j } xB | j t k s6 | j rP | j t k rP | j t j d q W| j j
d t j d d S)Ng?zworker handler exiting)
r\ current_threadrP rO rN TERMINATErw timesleeprM r/ r r6 )poolthreadr r r
r] g s *
zPool._handle_workersc C s> t j } xt | j d D]\ } } d } d } y x t | D] \ } } | j rm t j d Pn y | | WqG t k
r }
zN | d d \ } } y | | j | d |
f Wn t
k
r Yn XWYd d }
~
XqG XqG W| rt j d | | d n w PWq t k
r}
zx | rD| d d n d
\ } } | | k r}| | j | d d |
f n | rt j d | | d n WYd d }
~
Xq Xq Wt j d yF t j d | j d t j d x | D] } | d qWWn t k
r,t j d
Yn Xt j d d S)Nr z'task handler found thread._state != RUNr Fzdoing set_length()r ztask handler got sentinelz/task handler sending sentinel to result handlerz(task handler sending sentinel to workersz/task handler got OSError when sending sentinelsztask handler exiting)r r )
r\ r iterr0 r rP r r6 r7 _setKeyErrorr/ r5 ) taskqueuer/ r: r cacher ZtaskseqZ
set_lengthr= r? rC r> ZindZexpr r r
ra t sN
" !
(
zPool._handle_tasksc C s t j } x y
| } Wn) t t f k
rG t j d d SYn X| j rw | j t k sf t t j d Pn | d k r t j d Pn | \ } } } y | | j | | Wq t
k
r Yq Xq Wx | r| j t k ry
| } Wn) t t f k
r$t j d d SYn X| d k rDt j d q n | \ } } } y | | j | | Wq t
k
rYq Xq Wt | d rt j d y5 x. t d D] } | j
j sPn | qWWqt t f k
rYqXn t j d t | | j d S) Nz.result handler got EOFError/OSError -- exitingz,result handler found thread._state=TERMINATEzresult handler got sentinelz&result handler ignoring extra sentinelr3 z"ensuring that outqueue is not full
z7result handler exiting: len(cache)=%s, thread._state=%s)r\ r r5 r4 r r6 rP r r. r r r1 rn r3 pollro )r: r0 r r r= r> r? objr r r
re sX
zPool._handle_resultsc c sD t | } x1 t t j | | } | s1 d S| | f Vq Wd S)N)r tupler islice)r@ itsizer r r r
r s zPool._get_tasksc C s t d d S)Nz:pool objects cannot be passed between processes or pickled)NotImplementedError)r r r r
r" s zPool.__reduce__c C s8 t j d | j t k r4 t | _ t | j _ n d S)Nz closing pool)r r6 rP rO CLOSEr^ )r r r r
r2 s
z
Pool.closec C s0 t j d t | _ t | j _ | j d S)Nzterminating pool)r r6 r rP r^ rj )r r r r
terminate s
zPool.terminatec C sq t j d | j t t f k s( t | j j | j j | j j x | j
D] } | j qY Wd S)Nz joining pool) r r6 rP r r r. r^ r rd rg rZ )r r r r r
r s
z Pool.joinc C sZ t j d | j j x9 | j rU | j j rU | j j t j d q Wd S)Nz7removing tasks from inqueue until task handler finishedr )
r r6 Z_rlockacquireis_aliver3 r ry r r )r9 task_handlerr r r r
_help_stuff_finish s
zPool._help_stuff_finishc
C s t j d t | _ t | _ t j d | j | | t | | j si t | d k si t t | _ | j d t j d t j
| k r | j n | rt | d d rt j d x- | D]" } | j
d k r | j q q Wn t j d t j
| k r-| j n t j d t j
| k rY| j n | rt | d d rt j d x> | D]3 } | j rt j d
| j | j qqWn d S) Nzfinalizing poolz&helping task handler/workers to finishr zjoining worker handlerr zterminating workerszjoining task handlerzjoining result handlerzjoining pool workerszcleaning up worker %d)r r6 r rP r ro r r. r/ r\ r r r1 rp r pid)
clsr r9 r: r Zworker_handlerr Zresult_handlerr r r r r
rh
s8
$
zPool._terminate_poolc C s | S)Nr )r r r r
__enter__; s zPool.__enter__c C s | j d S)N)r )r exc_typeZexc_valZexc_tbr r r
__exit__> s z
Pool.__exit__)"r r r r+ rr rG r rq r[ rw rJ r{ r
r r r r rz r r| staticmethodr] ra re r r" r2 r r r classmethodrh r r r r r r
r sD 8
3: . c @ s^ e Z d Z d d Z d d Z d d Z d d d Z d d
d Z d d
Z d S)r c C sJ t j | _ t t | _ | | _ | | _ | | _ | | | j <d S)N) r\ ZEvent_eventnext job_counterr rN _callback_error_callback)r r r r r r r
r G s zApplyResult.__init__c C s
| j j S)N)r Zis_set)r r r r
readyO s zApplyResult.readyc C s | j s t | j S)N)r r. _success)r r r r
successfulR s zApplyResult.successfulNc C s | j j | d S)N)r wait)r timeoutr r r
r V s zApplyResult.waitc C s? | j | | j s" t n | j r2 | j S| j d S)N)r r r r _value)r r r r r
r0 Y s
zApplyResult.getc C s{ | \ | _ | _ | j r7 | j r7 | j | j n | j r] | j r] | j | j n | j j | j | j =d S)N)r r r r r setrN r )r r? r r r r
r b s
zApplyResult._set) r r r r r r r r0 r r r r r
r E s r c @ s( e Z d Z d d Z d d Z d S)r c C s t j | | | d | d | _ d g | | _ | | _ | d k rj d | _ | j j | | j =n | | t | | | _ d S)Nr Tr )
r r r r
_chunksize _number_leftr r r bool)r r r~ lengthr r r r r
r s s
zMapResult.__init__c C s | \ } } | r | | j | | j | d | j <| j d 8_ | j d k r | j rq | j | j n | j | j =| j j q nH d | _ | | _ | j r | j | j n | j | j =| j j d S)Nr r F)
r r r r rN r r r r r )r r? Zsuccess_resultsuccessrB r r r
r s %
zMapResult._setN)r r r r r r r r r
r q s
r c @ sU e Z d Z d d Z d d Z d d d Z e Z d d Z d
d Z d S) r c C sk t j t j | _ t t | _ | | _ t j | _
d | _ d | _ i | _
| | | j <d S)Nr )r\ Z ConditionZLock_condr r r rN collectionsdeque_items_index_length _unsorted)r r r r r
r s zIMapIterator.__init__c C s | S)Nr )r r r r
__iter__ s zIMapIterator.__iter__Nc C s | j j z y | j j } Wn t k
r | j | j k rN t n | j j | y | j j } Wn3 t k
r | j | j k r t n t Yn XYn XWd | j j
X| \ } } | r | S| d S)N) r r r popleft
IndexErrorr r
StopIterationr r release)r r r r r& r r r
r s&
zIMapIterator.nextc
C s | j j z | j | k r | j j | | j d 7_ xJ | j | j k r | j j | j } | j j | | j d 7_ qA W| j j n
| | j | <| j | j k r | j | j
=n Wd | j j Xd S)Nr ) r r r r rs r popnotifyr rN r r )r r? r r r r
r s
zIMapIterator._setc
C s^ | j j z< | | _ | j | j k rH | j j | j | j =n Wd | j j Xd S)N)r r r r r rN r r )r r r r r
r s
zIMapIterator._set_length) r r r r r r __next__r r r r r r
r s
r c @ s e Z d Z d d Z d S)r c
C st | j j zR | j j | | j d 7_ | j j | j | j k r^ | j | j =n Wd | j j Xd S)Nr )
r r r rs r r r rN r r )r r? r r r r
r s
zIMapUnorderedIterator._setN)r r r r r r r r
r s r c @ s[ e Z d Z d Z e d d Z d d f d d Z d d Z e d d
Z d S) r Fc O s d d l m } | | | S)Nr )rG )ZdummyrG )r rA rG r r r
rG s zThreadPool.ProcessNc C s t j | | | | d S)N)r r )r rk r; r< r r r
r s zThreadPool.__init__c C s@ t j | _ t j | _ | j j | _ | j j | _ d S)N)rK rL ri rc r/ rb r0 rf )r r r r
rJ s zThreadPool._setup_queuesc
C sW | j j z5 | j j | j j d g | | j j Wd | j j Xd S)N)Z not_emptyr rK clearextendZ
notify_allr )r9 r r r r r
r s
z ThreadPool._help_stuff_finish) r r r rr r rG r rJ r r r r r
r s
)!__all__r\ rK r r rT r r r r r r rO r r countr r r r7 r r r! r$ rE objectr r Z AsyncResultr r r r r r r r
<module>
s: , &%I