U |e@sDdZddlZddlZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl Z ddl m Z ddlmZmZddlmZmZddlmZmZmZmZmZmZmZmZmZmZmZm Z m!Z!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+m,Z,m-Z-m.Z.m/Z/m0Z0m1Z1ej2dkr&dKd d Z3dLd d Z4ndMd d Z3dNdd Z4ej5ej6Z5Z6ddZ7ddZ8ddZ9ddZ:ddZ;ddZdd Z?d!d"Z@d#d$ZAd%d&ZBd'd(ZCd)d*ZDd+d,ZEd-d.ZFd/d0ZGd1d2ZHd3d4ZId5d6ZJd7d8ZKd9d:ZLd;d<ZMd=d>ZNd?d@ZOdAdBZPdCdDZQdEdFZRdGdHZSGdIdJdJeZTdS)Oa New, fast version of the CloudPickler. This new CloudPickler class can now extend the fast C Pickler instead of the previous Python implementation of the Pickler class. Because this functionality is only available for Python versions 3.8+, a lot of backward-compatibility code is also removed. Note that the C Pickler subclassing API is CPython-specific. Therefore, some guards present in cloudpickle.py that were written to handle PyPy specificities are not present in cloudpickle_fast.py N)Enum)ChainMap OrderedDict)picklePickler)_extract_code_globals_BUILTIN_TYPE_NAMESDEFAULT_PROTOCOL_find_imported_submodules_get_cell_contents_should_pickle_by_reference _builtin_type_get_or_create_tracker_id_make_skeleton_class_make_skeleton_enum_extract_class_dictdynamic_subimport subimport_typevar_reduce _get_bases _make_cell_make_empty_cellCellType_is_parametrized_type_hintPYPYcell_set"parametrized_type_hint_getinitargs_create_parametrized_type_hintbuiltin_code_type_make_dict_keys_make_dict_values_make_dict_items_make_function_DYNAMIC_CLASS_TRACKER_REUSINGcCst|||d|dS)Serialize obj as bytes streamed into file protocol defaults to cloudpickle.DEFAULT_PROTOCOL which is an alias to pickle.HIGHEST_PROTOCOL. This setting favors maximum communication speed between processes running the same Python version. Set protocol=pickle.DEFAULT_PROTOCOL instead if you need to ensure compatibility with older versions of Python. protocolbuffer_callbackN CloudPicklerdump)objfiler(r)r/_/var/www/website-v5/atlas_env/lib/python3.8/site-packages/numba/cloudpickle/cloudpickle_fast.pyr,.s r,c Cs@t.}t|||d}|||W5QRSQRXdS)Serialize obj as a string of bytes allocated in memory protocol defaults to cloudpickle.DEFAULT_PROTOCOL which is an alias to pickle.HIGHEST_PROTOCOL. This setting favors maximum communication speed between processes running the same Python version. Set protocol=pickle.DEFAULT_PROTOCOL instead if you need to ensure compatibility with older versions of Python. r'NioBytesIOr+r,getvalue)r-r(r)r.cpr/r/r0dumps<s  r7cCst||d|dS)r&r(Nr*)r-r.r(r/r/r0r,Os c Cs>t,}t||d}|||W5QRSQRXdS)r1r8Nr2)r-r(r.r6r/r/r0r7[s   cCsVi}d|jkr|j|d<|jdd}t|tr8||d<t||jt||t|dfS)N __slots____dict__) r:r9get isinstancepropertytype__name__rr)r- type_kwargsr:r/r/r0_class_getnewargsqs   rAcCs,dd|D}|j|j|j||jt|dfS)NcSsi|]}|j|jqSr/namevalue.0er/r/r0 sz$_enum_getnewargs..) __bases__r? __qualname__ __module__r)r-membersr/r/r0_enum_getnewargs~s rMcCs|SNr/)retvalr/r/r0_file_reconstructorsrPc sjjjjjjjjd}tj }fdd|D}jdk r\t t t jnd}t j t|||d<||d<j}||fS)N)r?rJ__annotations____kwdefaults__ __defaults__rK__doc__ __closure__cs"i|]}|jkr|j|qSr/) __globals__)rFkfuncr/r0rHs z&_function_getstate..r/_cloudpickle_submodulesrV)r?rJrQrRrSrKrTrUr__code__listmapr r itertoolschainvaluesr:)rY slotstate f_globals_ref f_globalsclosure_valuesstater/rXr0_function_getstates,  rfcCst|}|ddtt|tjr|dd|dd|dd|dd}|dkr|ddt|\}}}}dd|D|d<nd d|D|d<d |krt|jt r||jn|jD]}||dq|d d|ifS) N __weakref__ _abc_cache_abc_negative_cache_abc_negative_cache_version _abc_registry _abc_implcSsg|] }|qSr/r/)rFsubclass_weakrefr/r/r0 sz#_class_getstate..cSsg|]}|qSr/r/)rFtype_r/r/r0rnsr9r:) rpop issubclassr>abcABCMeta _get_dumpr<r9str)r-clsdictregistry_rWr/r/r0_class_getstates*          rycCsLt|\}}dd|D}dD]}||dq|D]}||q4||fS)NcSsi|]}|j|jqSr/rBrEr/r/r0rHsz"_enum_getstate..)_generate_next_value__member_names_ _member_map_ _member_type__value2member_map_)ryrp)r-rvrarLattrnamememberr/r/r0_enum_getstates  rcCst|drZ|j|j|j|j|j|j|j|j|j |j |j |j |j |j|j|j|j|jf}nHt|dr|j|j|j|j|j|j|j|j|j |j |j |j |j|j|j|jf}nt|dr|j|j|j|j|j|j|j|j|j|j|j |j |j |j|j|j|j|j|j|j|jf}nt|drb|j|j|j|j|j|j|j|j|j |j |j |j |j|j|j|jf}n@|j|j|j|j|j|j|j|j |j |j |j |j|j|j|jf}tj|fS)zcodeobject reducerco_exceptiontable co_linetableco_nmetaco_posonlyargcount)hasattr co_argcountrco_kwonlyargcount co_nlocals co_stacksizeco_flagsco_code co_constsco_names co_varnames co_filenameco_name co_qualnameco_firstlinenorr co_freevars co_cellvars co_framesizeco_ndefaultargsr co_lnotabco_exc_handlers co_jump_table co_free2reg co_cell2regtypesCodeType)r-argsr/r/r0 _code_reduces     rcCs8z |jWntk r&tdfYSXt|jffSdS)z?Cell (containing values of a function's free variables) reducerr/N) cell_contents ValueErrorrrr-r/r/r0 _cell_reduce4s  rcCs|j}t||ffSrN)__func__r>)r- orig_funcr/r/r0_classmethod_reduce>src CsFddl}t|drt|ds&td|tjkrEllipsisNotImplementedr rr rrr/r/r0 _class_reduces     rcCstt|ffSrNr r\rr/r/r0_dict_keys_reducesrcCstt|ffSrNr!r\rr/r/r0_dict_values_reducesrcCstt|ffSrNr"rrr/r/r0_dict_items_reducesrcCstt|dffSNTrrr/r/r0_odict_keys_reducesrcCstt|dffSrrrr/r/r0_odict_values_reducesrcCstt|dffSrrrr/r/r0_odict_items_reducesrc Cs|\}}|j||d}|d}|d|j|t|jd<|dk rt|D]<\}}z |j}Wntk rYqXYnXt|j ||qX| D]\}} t ||| qdS)aUpdate the state of a dynamic function. As __closure__ and __globals__ are readonly attributes of a function, we cannot rely on the native setstate routine of pickle.load_build, that calls setattr on items of the slotstate. Instead, we have to modify them inplace. rVrUrZrN) r:updaterprVr enumeraterrrrUitemssetattr) r-rera obj_globals obj_closureicellrDrWvr/r/r0_function_setstates         rcCsd|tkr |S|\}}d}|D]"\}}|dkr6|}q t|||q |dk r`|D]}||qP|S)Nrl)r$rrregister)r-rerarwrattrsubclassr/r/r0rs rc@seZdZiZeee<eeej<e ee j <e ee j <eee<eee<eee<eee<eeej<eeej<eeej<eeej<eeej<e ee!j"<e#ee$j%<e&ee'j(<e)ee'j*<e+ee'j,<e-ee.e/0<e1ee.e/2<e3ee.e/4<eee5j6<eee5j7<eee5j8<eee5j9<e:ee;j<ZddZ?ddZ@eAjBd krXdd d ZCn dd d ZCeAjBd kreDseReduce a function that is not pickleable via attribute lookup.N)_function_getnewargsrfr#r)selfrYnewargsrer/r/r0_dynamic_function_reduceBs   z%CloudPickler._dynamic_function_reducecCst|r tS||SdS)aReducer for function objects. If obj is a top-level attribute of a file-backed module, this reducer returns NotImplemented, making the CloudPickler fallback to traditional _pickle.Pickler routines to save obj. Otherwise, it reduces obj using a custom cloudpickle reducer designed specifically to handle dynamic functions. As opposed to cloudpickle.py, There no special handling for builtin pypy functions because cloudpickle_fast is CPython-specific. N)r rr)rr-r/r/r0_function_reduceIs zCloudPickler._function_reducecCs~|j}|jt|ji}|ikrDdD]}||jkr&|j|||<q&|jdkrTd}ntddtt|j D}||dd|fS)N) __package__r?__path____file__css|] }tVqdSrN)r)rFrxr/r/r0 vsz4CloudPickler._function_getnewargs..) r[ globals_ref setdefaultidrVrUtuplerangelenr)rrYcode base_globalsrWclosurer/r/r0rZs    z!CloudPickler._function_getnewargsc CsXzt||WStk rR}z&d|jdkr@d}t||nW5d}~XYnXdS)N recursionrz?Could not pickle object as excessively deep recursion required.)rr, RuntimeErrorrrr)rr-rGmsgr/r/r0r,{szCloudPickler.dumpr%NcCs2|dkr t}tj||||di|_t||_dS)Nr')r r__init__rintproto)rr.r(r)r/r/r0rszCloudPickler.__init__cCs4|dkr t}tj|||di|_t|ds0tdS)Nr8r)r rrrrAssertionError)rr.r(r/r/r0rs cCstjdddkr&t|r&tt|fSt|}zt|t}Wntk rTd}YnX|rbt|St |t j rx| |St SdS)a*Type-agnostic reducing callback for function and classes. For performance reasons, subclasses of the C _pickle.Pickler class cannot register custom reducers for functions and classes in the dispatch_table. Reducer for such types must instead implemented in the special reducer_override method. Note that method will be called for any object except a few builtin-types (int, lists, dicts etc.), which differs from reducers in the Pickler's dispatch_table, each of them being invoked for objects of a specific type only. This property comes in handy for classes: although most classes are instances of the ``type`` metaclass, some of them can be instances of other custom metaclasses (such as enum.EnumMeta for example). In particular, the metaclass will likely not be known in advance, and thus cannot be special-cased using an entry in the dispatch_table. reducer_override, among other things, allows us to register a reducer that will be called for any class, independently of its type. Notes: * reducer_override has the priority over dispatch_table-registered reducers. * reducer_override can be used to fix other limitations of cloudpickle for other types that suffered from type-specific reducers, such as Exceptions. See https://github.com/cloudpipe/cloudpickle/issues/248 NF)r version_inforrrr>rq TypeErrorrr<r FunctionTyperr)rr-t is_anyclassr/r/r0reducer_overrides    zCloudPickler.reducer_overridec Cs\|j}|j} |j||d|||d||||||| tj| tj| tjdS)N)re listitems dictitemsr-)saver save_reducerTUPLE2REDUCEPOP) rrYrrerr state_setterr-rrr/r/r0_save_reduce_pickle5s   z!CloudPickler._save_reduce_pickle5cCs|tdkr|jtd|dS|ttkr:|jttf|dS|ttkrX|jttf|dS|tkrv|jtt|f|dStjdddkrt|r|jt t ||dnL|dk rt j |||dn2t ||ds|jt|d|int j |||ddS)z Save a "global". The name of this method is somewhat misleading: all types get dispatched here. NrNrr r rCr-)r>rrrr rrrrrrr save_globalr rr)rr-rCpackr/r/r0rs.    zCloudPickler.save_globalcCsPt||drtj|||dStr6t|jtr6||S|j| |d|iSdS)z Registered with the dispatch to handle all function types. Determines what kind of function obj is (e.g. lambda, defined at interactive prompt, etc) and handles the pickling appropriately. rr-N) r rrrr<r[rsave_pypy_builtin_funcrr)rr-rCr/r/r0 save_function,s  zCloudPickler.save_functioncCs4tj|ji|j|j|jf|jf}|j|d|idS)a Save pypy equivalent of builtin functions. PyPy does not have the concept of builtin-functions. Instead, builtin-functions are simple function instances, but with a builtin-code attribute. Most of the time, builtin functions should be pickled by attribute. But PyPy has flaky support for __qualname__, so some builtin functions such as float.__new__ will be classified as dynamic. For this reason only, we created this special routine. Because builtin-functions are not expected to have closure or globals, there is no additional hack (compared the one already implemented in pickle) to protect ourselves from reference cycles. A simple (reconstructor, newargs, obj.__dict__) tuple is save_reduced. Note also that PyPy improved their support for __qualname__ in v3.6, so this routing should be removed when cloudpickle supports only PyPy 3.6 and later. r-N)rrr[r?rSrUr:r)rr-rvr/r/r0r ;sz#CloudPickler.save_pypy_builtin_func)NN)N)NNNNN)N)Pr?rKrJ_dispatch_tabler classmethodrr3 TextIOWrapperrrLoggerr RootLoggerr memoryviewrr= staticmethodrrrrrrGetSetDescriptorTyper ModuleTyperrrrrrrrtypingTypeVarr_collections_abc dict_keysr dict_valuesr dict_itemsrr>rkeysrr`rrrrabstractmethodabstractclassmethodabstractstaticmethodabstractpropertyrcopyregdispatch_tablerrrr,rHIGHEST_PROTOCOLrrdispatchrrrrstructrrr!r rr/r/r/r0r+!s`                  !    9  " r+)NN)NN)N)N)UrTr.rrr7r3r^rrr;rrr,enumr collectionsrrcompatrr cloudpicklerr r r r r rrrrrrrrrrrrrrrrrrr r!r"r#r$r9r,r7loadloadsrArMrPrfryrrrrrrrrrrrrrrrrrrrrrrrrr+r/r/r/r0sf  |     '(> /  "