\dddgZddlZddlZddlZddlZddlZddlZddlZddlZddl Z ddl m Z ddl m Z m Z ddlmZdZd Zd Zd ZejZd Zd ZGddeZGddZdZGddeZ d%dZdZGddeZ Gdde!Z"Gdde!Z#e#Z$Gdde#Z%Gd d!e!Z&Gd"d#e&Z'Gd$de"Z(dS)&Pool ThreadPoolN)util) get_context TimeoutError)waitINITRUNCLOSE TERMINATEc.tt|SN)listmapargss %..\python\lib\multiprocessing\pool.pymapstarr/s T  c^ttj|d|dS)Nrr)r itertoolsstarmaprs r starmapstarr2s%  !$q'4733 4 44rceZdZdZdZdS)RemoteTracebackc||_dSrtb)selfrs r__init__zRemoteTraceback.__init__:s rc|jSrrr s r__str__zRemoteTraceback.__str__<s wrN)__name__ __module__ __qualname__r!r$rrrr9s2rrceZdZdZdZdS)ExceptionWithTracebackctjt|||}d|}||_d|z|_dS)Nz """ %s""") tracebackformat_exceptiontypejoinexcr)r r1rs rr!zExceptionWithTraceback.__init__@sB  'S 3 ; ; WWR[[ 2%rc,t|j|jffSr) rebuild_excr1rr#s r __reduce__z!ExceptionWithTraceback.__reduce__EsTXtw///rN)r%r&r'r!r4r(rrr*r*?s2&&& 00000rr*c.t||_|Sr)r __cause__)r1rs rr3r3Hs#B''CM Jrc.eZdZdZfdZdZdZxZS)MaybeEncodingErrorzVWraps possible unpickleable errors, so they can be safely sent through the socket.ct||_t||_tt||j|jdSr)reprr1valuesuperr8r!)r r1r; __class__s rr!zMaybeEncodingError.__init__TsH99%[[   $''004:FFFFFrc(d|jd|jdS)NzError sending result: 'z '. Reason: '')r;r1r#s rr$zMaybeEncodingError.__str__Ys"=AZZZ=AXXXG Grc(d|jjd|dS)N)r=r%r#s r__repr__zMaybeEncodingError.__repr__]s!^444ddd;;r)r%r&r'__doc__r!r$rC __classcell__r=s@rr8r8Psh''GGGGG GGG<<<<<<>QZFFFFFF  , Ca ! ! ! ! , , ,(F1I66G JI    Ca%)* + + + + + + + +  , 4877s7V7d7TDQ 7  x I,@,@8 J.:;;;;;sB& B11%CC< D D?$D::D?E F#=FF#c|)z@Pickle-able helper function for use by _guarded_task_generation.r()exs rrVrVs Hrc2eZdZdZddfd ZfdZxZS) _PoolCachez Class that implements a cache for the Pool class that will notify the pool management threads every time the cache is emptied. The notification is done by the use of a queue that is provided when instantiating the cache. NnotifiercH||_tj|i|dSr)rmr<r!)r rmrrcr=s rr!z_PoolCache.__init__s,  $'$'''''rct||s|jddSdSr)r< __delitem__rmrM)r itemr=s rrpz_PoolCache.__delitem__sK D!!! $ M  d # # # # # $ $r)r%r&r'rDr!rprErFs@rrkrksj +/((((((( $ $ $ $ $ $ $ $ $rrkceZdZdZdZedZ d)dZej e fdZ dZ d Z ed Zed Zd Zed ZedZdZdZdifdZd*dZd*dZ d+dZdZd,dZd,dZdiddfdZ d+dZ d+dZed*dZe dZ!edZ"edZ#ed Z$d!Z%d"Z&d#Z'd$Z(ed%Z)e d&Z*d'Z+d(Z,dS)-rzS Class which supports an async version of applying functions to arguments. Tc|j|i|SrProcess)ctxrrcs rruz Pool.Processss{D)D)))rNr(c&g|_t|_|p t|_|t j|_|j|_ t|j |_ ||_ ||_ ||_|tjpd}|dkrt#d|*t%|t&r|dkrt#d|t)|st+d||_ |nR#t0$rE|jD]}|j||jD]}|wxYw|}t;jt>j |j |j|j|j!|j|j|j"|j#|j |j|j |j$||j f|_%d|j%_&tN|j%_|j%(t;jt>j)|j|j*|j#|j|j f|_+d|j+_&tN|j+_|j+(t;jt>j,|j#|j-|j f|_.d|j._&tN|j._|j.(t_j0||j1|j|j"|j#|j|j |j%|j+|j.|j f d |_2tN|_dS) Nrlrz&Number of processes must be at least 1rz/maxtasksperchild must be a positive int or Nonezinitializer must be a callabletargetrT)r exitpriority)3_poolr _stater_ctx _setup_queuesqueue SimpleQueue _taskqueue_change_notifierrk_cache_maxtasksperchild _initializer _initargsos cpu_count ValueErrorrIrJcallable TypeError _processes_repopulate_poolrUexitcode terminater0_get_sentinels threadingThreadr_handle_workersru_inqueue _outqueue_wrap_exception_worker_handlerdaemonr start _handle_tasks _quick_put _task_handler_handle_results _quick_get_result_handlerrFinalize_terminate_pool _terminate)r processesrZr[maxtasksperchildcontextp sentinelss rr!z Pool.__init__s_  ,{}}  +--!% 5 5 7 7 $*?@@@ !1'!   +!I q==EFF F  '.44 T8HA8M8M !RSSS  "8K+@+@ "<== =#   ! ! # # # #   Z " ":%KKMMMZ     '')) (/'+t 4</4:t}dn#T^T5K& 43HJ   '+#&)# ""$$$'-%/4?DN*dk+ %)!$'!   """(/'.$/4;?   '+#&)# ""$$$- $&/4=$.$*')=t?Q& 5  s "D77AFc|j|krC|d|t|t|dd|jddSdSdS)Nz&unclosed running multiprocessing pool )sourcer)r}ResourceWarninggetattrrrM)r _warnr s r__del__z Pool.__del__ sy ;#   EC4CC!$ 0 0 0 0t/66B%))$/////  CBrc p|j}d|jd|jd|jdt |jd S)NrA.z state=z pool_size=rB)r=r&r'r}lenr|)r clss rrCz Pool.__repr__sYn0CN00S%50000 __000 1rcB|jjg}|jjg}g||Sr)rrQr)r task_queue_sentinelsself_notifier_sentinelss rrzPool._get_sentinelss1 $ 67#'#8#@"A@%@(?@@rcd|DS)Nc<g|]}t|d|jS)sentinel)rOr).0rgs r z.Pool._get_worker_sentinels..s8888F"6:668888rr(workerss r_get_worker_sentinelszPool._get_worker_sentinelss#88888 8rcd}ttt|D]A}||}|j0t jd|z|d}||=B|S)zCleanup after any worker processes which have exited due to reaching their specified lifetime. Returns True if any workers were cleaned up. FNcleaning up worker %dT)reversedrangerrrrTr0)poolcleanedrargs r_join_exited_workerszPool._join_exited_workers!st %D **++  A!WF* 2Q6777 Grc ||j|j|j|j|j|j|j|j|j |j Sr) _repopulate_pool_staticr~rurr|rrrrrrr#s rrzPool._repopulate_pool1sN++DIt|,0O,0J ,0NDbI I I I I I I I I I Js!' AA  Arc ||dkrLt|}|j||j|||jf|S|dkr"td|t |||}t|}|j||jt||jfd|DS)zP Equivalent of `map()` -- can be MUCH slower than `Pool.map()`. rzChunksize must be 1+, not {0:n}c3$K|] }|D]}|V dSrr(rchunkrqs r zPool.imap../??U??D???????r) r IMapIteratorrrMr_job _set_lengthrrLr _get_tasksrr rbrrrd task_batchess rimapz Pool.imaps!  >>!$''F O  11&+tXNN&    M1}} 5<<!##$$$ ??49EEL!$''F O  11&+292>@@&    @?f??? ?rc ||dkrLt|}|j||j|||jf|S|dkr"td|t |||}t|}|j||jt||jfd|DS)zL Like `imap()` method but ordering of results is arbitrary. rzChunksize must be 1+, not {0!r}c3$K|] }|D]}|V dSrr(rs rrz&Pool.imap_unordered..rr) rIMapUnorderedIteratorrrMrrrrrLrrrrs rimap_unorderedzPool.imap_unordereds  >>*400F O  11&+tXNN&    M1}} 5<@@&    @?f??? ?rc|t|||}|j|jd|||fgdf|S)z; Asynchronous version of `apply()` method. rN)r ApplyResultrrMr)r rbrrcrrrds rrzPool.apply_asyncsW T8^<< v{AtT4@A4HIII rc@|||t|||S)z9 Asynchronous version of `map()` method. )rrrs r map_asynczPool.map_asyncs' tXw 8  rc|t|dst|}|""" ;#  DK*/D '  ! % %d + + + + +  rcntjdt|_|dS)Nzterminating pool)rrTr r}rr#s rrzPool.terminates0 %&&&  rctjd|jtkrt d|jt t fvrt d|j|j |j |j D]}|dS)Nz joining poolzPool is still runningzIn unknown state) rrTr}r rr r rr0rrr|)r rs rr0z Pool.joins >""" ;#  455 5 [ 2 2 2/00 0 !!### !!! !!###  A FFHHHH  rcztjd|j|ru|jr`|jtj d|r|jZdSdSdSdS)Nz7removing tasks from inqueue until task handler finishedr) rrT_rlockacquireis_aliverQrrtimesleep)rX task_handlerr s r_help_stuff_finishzPool._help_stuff_finishs LMMM   ##%% '/*>*>*@*@  O " " " JqMMM##%% '/*>*>*@*@         rc tjdt|_|dt|_tjd|||t ||s"t | dkrtdt|_|d|dtjdtj |ur| |rJt|ddr4tjd|D]} | j | tjdtj |ur| tjd tj |ur| |rst|ddr_tjd |D]J} | r0tjd | jz| GdSdSdS) Nzfinalizing poolz&helping task handler/workers to finishrz.Cannot have cache with result_hander not alivezjoining worker handlerrzterminating workerszjoining task handlerzjoining result handlerzjoining pool workersr)rrTr r}rMr-rr)rKrr r0rOrrpid) rr rXrYrrworker_handlerr,result_handlerr rs rrzPool._terminate_poolsb $%%% !*D!!!'  ;<<< w c$ii@@@'')) BE a @BB B!*D!!! T +,,,  # % %^ ; ;    ! ! !  "GDG[11 " J, - - - " ":%KKMMM )***  # % %\ 9 9       +,,,  # % %^ ; ;    ! ! !  GDG[11  J- . . .  ::<<J6>???FFHHH       rc.||Sr)rr#s r __enter__zPool.__enter__s  rc.|dSr)r)r exc_typeexc_valexc_tbs r__exit__z Pool.__exit__s r)NNr(NNr)NNN)r)-r%r&r'rDr staticmethodrur!warningswarnr rrCrrrrrrrrrrrrrrrrrrr  classmethodrrrrr4rPrr0r-rr3r8r(rrrrsaO**\*CE04OOOOf%Ms0000111 AAA 88\8  \ BBB''\'$ 9 9\ 9666 111 "8888IIIIMMMMFJ9999 J J J@@@@:@@@@8&(b4BFKO:"""\" --[-(++++\++Z9)9)\9)v\ ,,,   \33[3jrcVeZdZdZdZdZddZddZdZe e j Z dS) rc||_tj|_t t |_|j|_||_||_ ||j|j<dSr) r|rEvent_eventnext job_counterrr _callback_error_callback)r rrrs rr!zApplyResult.__init__sS o'' %% k !-!% DIrc4|jSr)r@is_setr#s rreadyzApplyResult.readys{!!###rc||s"td||jS)Nz{0!r} not ready)rGrrL_successr#s r successfulzApplyResult.successfuls7zz|| =.55d;;<< <}rNc:|j|dSr)r@r r rs rr zApplyResult.waits !!!!!rc|||st|jr|jS|jr)r rGrrI_valuerLs rrNzApplyResult.getsD 'zz||   = ; + rc|\|_|_|jr!|jr||j|jr!|js||j|j|j|j=d|_dSr) rIrNrCrDr@setrrr|r rars rrzApplyResult._sets%(" t{ > (dm ( NN4; ' ' '   .  .   - - -  K " rr) r%r&r'r!rGrJr rNrr<types GenericAlias__class_getitem__r(rrrrs&&&$$$ """"$ E$677rrceZdZdZdZdS)rct||||d|_dg|z|_||_|dkr/d|_|j|j|j =dS||zt||zz|_dS)NrTr) rr!rIrN _chunksize _number_leftr@rPrrbool)r rrlengthrrs rr!zMapResult.__init__sT4,:  < < < fvo # >> !D  KOO    DI&&& & 1D)9K4L4L LD   rc|xjdzc_|\}}|r|jr|||j||jz|dz|jz<|jdkrP|jr||j|j|j=|jd|_ dSdS|s|jrd|_||_|jdkrP|j r| |j|j|j=|jd|_ dSdS)NrrF) rXrIrNrWrCrrr@rPr|rD)r rasuccess_resultsuccessrds rrzMapResult._set)s1 Q(  "t} "CIDK$/)1Q3*?? @ A%%>0NN4;///K * !!!! &% %t} % % $  A%%'6((555K * !!!! &%rN)r%r&r'r!rr(rrrrs5 M M M"""""rrc2eZdZdZdZddZeZdZdZdS)rc*||_tjtj|_t t |_|j|_tj |_ d|_ d|_ i|_||j|j<dS)Nr)r|r ConditionLock_condrArBrr collectionsdeque_items_index_length _unsorted)r rs rr!zIMapIterator.__init__Gst ()9)9:: %% k !'))   !% DIrc|Srr(r#s r__iter__zIMapIterator.__iter__Rs rNc|j5 |j}n#t$r|j|jkrd|_td|j| |j}n5#t$r(|j|jkrd|_tdtdwxYwYnwxYwdddn #1swxYwY|\}}|r|S|r) rbrepopleft IndexErrorrfrgr| StopIterationr r)r rrqr]r;s rrAzIMapIterator.nextUsE Z 1 1 1{**,, 1 1 1;$,..!%DJ'T1 (((1;..00DD!111{dl22%) +5&D0 1D 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1   L sJC $C AB:(BB:2B44B:7C 9B::C  C C c |j5|j|kr|j||xjdz c_|j|jvrW|j|j}|j||xjdz c_|j|jvW|jn ||j|<|j|jkr|j|j =d|_ ddddS#1swxYwYdSNr) rbrfrerrhpopnotifyrgrrr|rQs rrzIMapIterator._setmsE Z " "{a ""3''' q kT^33.,,T[99CK&&s+++KK1$KKkT^33 !!####$'q!{dl**K *!  " " " " " " " " " " " " " " " " " "sC#C88C<?C<c|j5||_|j|jkr-|j|j|j=d|_ddddS#1swxYwYdSr)rbrgrfrrrrr|)r rZs rrzIMapIterator._set_length~s Z " "!DL{dl** !!###K *!  " " " " " " " " " " " " " " " " " "sAAA!Ar) r%r&r'r!rjrA__next__rrr(rrrrEsk & & &,H"""""""""rrceZdZdZdS)rc|j5|j||xjdz c_|j|j|jkr|j|j=d|_ddddS#1swxYwYdSrp) rbrerrfrrrgrrr|rQs rrzIMapUnorderedIterator._sets Z " " K  s # # # KK1 KK J     {dl**K *!  " " " " " " " " " " " " " " " " " "sA(A==BBN)r%r&r'rr(rrrrs#"""""rrcneZdZdZedZd dZdZdZedZ ed Z d Z dS) rFcddlm}||i|S)Nrrt)dummyru)rvrrcrus rruzThreadPool.Processs)""""""w%%%%rNr(c@t||||dSr)rr!)r rrZr[s rr!zThreadPool.__init__s  dI{H=====rctj|_tj|_|jj|_|jj|_dSr)rrrrrMrrNrr#s rrzThreadPool._setup_queuess>)++ *,,-+.,rc|jjgSr)rrQr#s rrzThreadPool._get_sentinelss%-..rcgSrr(rs rrz ThreadPool._get_worker_sentinelss rc |d#tj$rYnwxYwt|D]}|ddS)NTF)block)rNrEmptyrrM)rXr,r ras rr-zThreadPool._help_stuff_finishs{  ) % ((( ){    D t  A KK      s ,,c.tj|dSr)r*r+)r rrrs rr zThreadPool._wait_for_updatess 7r)NNr() r%r&r'rr9rur!rrrr-r r(rrrrsO&&\&>>>>--- ///\\r)Nr(NF))__all__rcrrrrr*r-rRr:r,rrr connectionr r r r r countrBrrrUrr*r3r8rgrVdictrkobjectrr AsyncResultrrrrr(rrrs <     ''''''''    io 555i 00000000<<<<<<<<"GK+<+<+<+