Zi( ddlmZddlZddlZddlZddlZddlmZddlm Z ddl Z ddl Z ddl m Z mZmZddl mZmZmZddl mZddlmZmZmZmZdd lmZdd lmZmZmZ dd l m!Z!n #e"$rdZ!YnwxYwd Z#d Z$dZ%dZ&dZ'dZ(dZ)dZ*dZ+dZ,iZ-de.eDZ/GddeZ0GddeZ1GddeZ2GddeZ3ejj4e j5dejj6e j5d ejj7e j5d!ejj8e j5d"ejj9e ejj:eejj;eiZ#e j=d$e j>#fejj4e j=d%e j?#e j=d&e j?#fejj8e j=de j@#e j=d'e j@#fejj7e j=d(e jA#e j=d)e jA#fejj;e j=de#e j=d*e#fejj:e j=d+e#e j=d,e#fiZBejj4e j=d-e j?#e j=d&e j?#fejj7e j=d.e jA#e j=d)e jA#fiZCejj6e j=de j>#e j=d&e j>#fejj4e j=d/e j?#e j=d0e j?#fejj8e j=de j@#e j=d)e j@#fejj7e j=d1e jA#e j=d2e jA#fejj;e j=de#e j=d,e#fejj:e j=d3e#e j=d4e#fiZDd5d6d7ZEdd8ZFdd:ZGd;ZH dddIZI dddKZJ dddTZKddUZLddVZMddZZNdd^ZOGd_d`ZPGdadbZQGdcddZRdeZSdfZTdgZUdhZVddmZWdnZXddpZYddrZZddtZ[ddvZ\ddyZ]ddzZ^dd{Z_dd|Z`dd}ZaddZbddZcddZdddZeddZfddZgddZhddZiddZjddZkdS)) annotationsN)Enum)Path) float8_e4m3fnint4uint4) ModelProto TensorProtoexternal_data_helper)onnx_pb) make_graph make_model make_nodemake_tensor_value_info)ReferenceEvaluator)GraphOptimizationLevelInferenceSessionSessionOptions)to_array_extendedz onnx.quantizez0.1.0ai.onnxz com.microsoftQuantizeLinear_QuantizeLinear_InputDequantizeLinear_DequantizeLinear_Output _quantizedlci|]@}ttt|t*tt||AS) isinstancegetattrr int).0ks a/root/projects/butler/venv/lib64/python3.11/site-packages/onnxruntime/quantization/quant_utils.py r$-sAqqqq SZ[fhiSjSjloHpHpq Q''qqqc2eZdZdZdZdZedZdS)QuantizationModerc|jSNnameselfs r#__str__zQuantizationMode.__str__8 yr%cV t|S#t$rtwxYwr*)r'KeyError ValueError)modes r# from_stringzQuantizationMode.from_string;s7 #D) )   ,,   (N)__name__ __module__ __qualname__ IntegerOps QLinearOpsr/ staticmethodr5rr%r#r'r'4sHJJ\r%r'c2eZdZdZdZdZedZdS)QuantizedValueTyperr(c|jSr*r+r-s r#r/zQuantizedValueType.__str__Gr0r%cV t|S#t$rtwxYwr*)r>r2r3)vs r#r5zQuantizedValueType.from_stringJs7 %a( (   ,,  r6N)r7r8r9Input Initializerr/r<r5rr%r#r>r>CsH EK\r%r>c\eZdZdZdZdZdZdZdZdZ dZ e d Z e d Zd S) QuantTyperr(c|jSr*r+r-s r#r/zQuantType.__str__[r0r%cV t|S#t$rtwxYwr*)rEr2r3)ts r#r5zQuantType.from_string^s6 Q<    ,,  r6c|tjkr tjS|tjkr tjS|tjkr tjS|tjkr tj S|tj kr tj S|tj kr tj S|tjkr tjSt!d|d)NzUnexpected value qtype=.)rEQInt8r INT8QUInt8UINT8QUInt16UINT16QInt16INT16 QFLOAT8E4M3FN FLOAT8E4M3FNQUInt4UINT4QInt4INT4r3r-s r# tensor_typezQuantType.tensor_typees 9? " "# # 9# # #$ $ 9$ $ $% % 9# # #$ $ 9* * *+ + 9# # #$ $ 9? " "# #<4<<<===r%N)r7r8r9rPrRrXrVrTr\rZr/r<r5propertyr^rr%r#rErERs| E FM FG E F\ >>X>>>r%rEc2eZdZdZdZdZedZdS) QuantFormatrr(c|jSr*r+r-s r#r/zQuantFormat.__str__|r0r%cV t|S#t$rtwxYwr*)rar2r3)formats r#r5zQuantFormat.from_strings7 v& &   ,,  r6N)r7r8r9 QOperatorQDQr/r<r5rr%r#raraxsHI C\r%raint8uint8int16uint16dtypeiiiiiiii@ii@rGzero_point_indexcTg}t|D]\}}tjt|tjr(|tj|nEt|tjr||ntd|d|||krI|d}|j tj ks|j tj krtd|j t|dkrt|n|dS)Nzarg z is not an array: rszzero_point cannot be r(r) enumeratenumpy issubdtypetypenumberappendarrayrndarray TypeErrorrlfloat32float16lentuple)ruargsnew_argsiarAs r# _check_typersH$ C C1  DGGU\ 2 2 = OOEKNN + + + + 5= ) ) = OOA    ;1;;;;<< <   Aw%-''17em+C+C A A ABBB!(mma//5???Xa[@r%c|tvsJd|d|tjjtjjtjjtjjfvrF|dkrtd|d|jtj kr tj }n:|jtj kr tj }ntd|jdtt!t#dgdgt$jd|gdg t#d gd d ggd t+d|dt+d|dgt+d |dg}t-|}t/|d||ddSt|} t3|dd\} } |t5| |n| } |t7| |n| } tj|tj |z |z}tj|| | |t/|| S)NUnexpected data type > requested. Only INT8, UINT8, INT16, and UINT16 are supported.rz2zero_point is expected to be null for float 8 not rOzUnexpected dtype Constant zero_point)valuer)XscalerYqurr)rrF) reduce_range symmetric)out) ONNX_TYPE_TO_NP_TYPE onnx_protor rYFLOAT8E4M3FNUZ FLOAT8E5M2FLOAT8E5M2FNUZNotImplementedErrorrlrxrFLOATrFLOAT16r3rr ronnxhelper make_tensorrrrrunget_qmin_qmax_for_qTypemaxminasarrayastyperoundclip)qTypearrrrlowhigh onnx_type onnx_modelrefrlqminqmaxcliplowcliphigharr_fp32s r#quantize_nparrayrsv ( ( ( (eeee ) ( ( +-)-  ??%&j[e&j&j&jkk k 9 % %#)II Y%- ' '#+II====>> > "Bdk>U>UVbdikmpqor>s>s.0L0L0LseTT  *3 4@@*7ItDD(UD99:    !,,3774sU)C)CDDQGHHH%U+,URWXXX d$'O#dC...&*&63tT???D=#**U]";";e"C!J!J!L!Lz!YZZ 8WhH====8??511222r%Fc |dks|dkrtd|d|tj|tjd|j}tj|tjd|j}|,t ||tj||jz}|r?tjtj|tj|}| }| }||ksJd|d|tj||z tj }tj|tj tj|tj z }tj||z } | dks Jd| tj |jj kr7tjd |j} tjd|j} n|rRtjtj ||ztjd tj z |j} n3tjtj ||| z z |j} | |j} | | gS) aCalculate the scale s and zero point z for the quantization relation r = s(q-z), where r are the original values and q are the corresponding quantized values. r and z are calculated such that every value within [rmin,rmax] has an approximate representation within [qmin,qmax]. In addition, qmin <= z <= qmax is enforced. If the symmetric flag is set to True, the interval [rmin,rmax] is symmetrized to [-absmax, +absmax], where absmax = max(abs(rmin), abs(rmax)). :parameter rmin: minimum value of r :parameter rmax: maximum value of r :parameter qmin: minimum value representable by the target quantization data type :parameter qmax: maximum value representable by the target quantization data type :parameter symmetric: True if the floating-point range should be made symmetric. Defaults to False. :parameter min_real_range: Minimum floating-point range (i.e., rmax - rmin) to enforce. Defaults to None. :return: zero and scale [z, s] rBqmin and qmax must meet requirement: qmin <= 0 <= qmax while qmin:, qmmax:rkNzqmin=z > qmax=z scale issue?g@)r3rxminimumr}rlmaximumrrabsfloat64finfotinyrr) rminrmaxrrrmin_real_rangeabsmaxdrdqrrs r#compute_scale_zprsQ( axx4!88r^brrlprrsss =u{1DJ??? @ @D =u{1DJ??? @ @D!4 nDJ O O OOPPuy $@@ww 4<<<555t55<<< TD[ 6 6 6B T / / /%+d%-2X2X2X XB KR E A:::}::: u{4:&&+++ Ctz222[$*555  Y  TD[EK5=,Q,Q,QQRRZ^ZdJJU[u 1D%E%ETZXXXJ TZ((  r%cd}|tvrz|tjkrLddlm}|}dt dD}t jd|Dt j}ntd|d |t|<n|tjkrddlm}|}|td |d t j t|}t jd|}t j||z |j }||gS) arCalculate the scale s for a float8 type (E4M3FN). The function assumes the coefficient distribution and the float 8 distribution are similar to two gaussian laws. :return: zero and scale [z, s] More details in notebook `quantization_fp8.ipynb `_. Nr)rc,g|]}t|Sr)float)r!rs r# z+compute_scale_zp_float8..<s777q%((777r%cbg|],}tj|tj|*|-Sr)rxisnanisinf)r!fs r#rz+compute_scale_zp_float8..>s3TTTqek!nnTU[QR^^TTTTr%rkzQuantization to element_type=z not implemented.zUnexpected element_type rO) FLOAT8_DISTRIBUTIONSr rY ml_dtypesrrangerxr}rr3rstdrl) element_typerzp_dtyper all_valuesvaluesstd_f8zerors r#compute_scale_zp_float8r,s>H/// ;3 3 3 / / / / / /$H77E#JJ777J[TTJTTT\a\iFF\\\\\]] ]-3\** 1 1 1++++++ B<BBBCCC Y+L9 : :F ;q ) ) )D Kf CI 6 6 6E %=r%data numpy.ndarray quant_typeonnx.TensorProto.DataTyperboolrr float | None rmin_override rmax_overridereturn#tuple[numpy.ndarray, numpy.ndarray]ct|tjs tdt |d||}n%t |r|nd}||}n%t |r|nd}tj||j }tj||j }tjd|j } |tj krJ|rtdtj |} t|| \} } t| | d S|tjtjtjtjtjtjfvrit-||| \} } t |rt/||| | ||\} } ntjd| j } t| | d St1d |d) a Returns the zero_point and scale for the given data. :param data: The data for which to compute quantization parameters. :param quant_type: The quantization data type. :param symmetric: whether symmetric quantization is used or not. :parameter reduce_range: True if the quantization range should be reduced. Defaults to False. :parameter min_real_range: Minimum floating-point range (i.e., rmax - rmin) to enforce. Defaults to None. :parameter rmin_override: The value of rmin to use if not None. Otherwise, uses min(data). :parameter rmax_override: The value of rmax to use if not None. Otherwise, uses max(data). :return: zero point and scale z%Weight must be given as an array not rONgrkrz1Unsupported option reduce_range=True for float 8.rrtrz Unexpected value for quant_type=)rrxr~rrzrrrr}rlr rY RuntimeErrorrrrrQrSrWrUr]r[rrr3)rrrrrrrrrrrrrrs r#compute_data_quant_paramsrPs* dEM * *OMT MMMNNN  YY/txxzzzC  YY/txxzzzC ;t4: . . .D ;t4: . . .D K4: . . .E[---  TRSS Sioo3JDD E:uqAAAA -ZQZ[[[ d t99 : 0tT4Tb c c JQdj999J:uqAAAA E EEE F FFr%2tuple[numpy.ndarray, numpy.ndarray, numpy.ndarray]c t|||||||\}}|tjkrt||||} t | t jdzdkrxt j |} td| d| d| d| d ||| fS|tj tjtjtjtjtjfvrt||||} ||| fSt'd|d)al :param data: data to quantize :param qType: data type to quantize to. :param symmetric: whether symmetric quantization is used or not. :parameter reduce_range: True if the quantization range should be reduced. Defaults to False. :parameter min_real_range: Minimum floating-point range (i.e., rmax - rmin) to enforce. Defaults to None. :parameter rmin_override: The value of rmin to use if not None. Otherwise, uses min(data). :parameter rmax_override: The value of rmax to use if not None. Otherwise, uses max(data). :return: minimum, maximum, zero point, scale, and quantized weights To pack weights, we compute a linear transformation - when data `type == uint8` mode, from `[rmin, rmax]` -> :math:`[0, 2^{b-1}]` and - when data `type == int8`, from `[-m , m]` -> :math:`[-(2^{b-1}-1), 2^{b-1}-1]` where `m = max(abs(rmin), abs(rmax))` and add necessary intermediate nodes to transform quantized weight to full weight using the equation :math:`r = S(q-z)`, where - *r*: real original value - *q*: quantized value - *S*: scale - *z*: zero point rnz+One of the quantized value is NaN data in [z, z], quantized_data in [z].zUnexpected value for qType=rO)rr rYranyviewrxrhravelrrrrrQrSrWrUr]r[r3) rrrrrrrrrquantized_datanp_datas r# quantize_datars82  J ((()%ujII ##EK0066883>3F G G mD))GWgkkmmWWw{{}}WW&4&8&8&:&:WW>L>P>P>R>RWWW 5.00  *%ujII5.00 ;5;;; < <fvrt3d |d tAtC|}tj"#|||j|d }nmtj"$|}tj || |j}tj%&||}|S)aG Returns a quantized version of the given ONNX initializer. :param weight: The ONNX initializer to quantize. :param quant_type: The final quantized data type. :param zero_point: The zero-point value to use for quantization. :param scale: The scale value to use for quantization. :param axis: The quantization axis if quantizing per-channel. Defaults to None. :param quant_weight_name: The name of the quantized initializer. If not specified, the quantized name is generated. :return: The quantized ONNX initializer. Nr(zThe initializer of shape z! could not be created, expecting z, got z and shape=z raw=rOzQuantized weights for z. must be 8-bit before packing as 4-bit values.T)rawrk)'tensor_proto_to_arrayrrshapelistrtaker|rxrreshape concatenater,TENSOR_NAME_QUANT_SUFFIXrr rY data_typedimsextendflattencopytobytesraw_datarrstrr]r[rlrrbytespack_bytes_to_4bitrrtensor_dtype_to_np_dtype numpy_helper from_array)rrrrrr weight_data q_weight_data channel_count channel_dimsquantized_channel_data_listr channel_data channel_scalechannel_zero_pointquantized_channel_data q_weight_nameq_weight_initializercheck packed_dataquant_np_dtypes r#quantize_onnx_initializerrs((//K*.M |([5F5F5H5H%Q[\\ #)$/ K-..  T&(#}%% l lA&++At44L!!HM!+A %5L..00-AS&& " ( . .u}=S/T/T/\/\]i/j/j k k k k)*EtLL ):j%%6;@jPh@j@jMT%222#/11)3&!((555$1!(5(=(=(?(?(D(D(F(F(N(N(P(P%  (&&:;;E{k///5==??mF[F[F]F]3]3]"@ 0A@@$,,..ss3@@;@==??3B3;O@@\b\h@@ !566tt<@@@ (-t/?/EF F F  tUm 3 3u uuuvv v.}/D/D/F/FGGHH  ${66}jRXR]_jpt6uu==jII m>JJJRRSYS^__ #0;;M=YY r%c|tjjkrtdd}|rt|}n3|r|t vrt |}nt|}|std|d|\}}|dks|dkr&td|d|d|j d |d |d | |S) z Return qmin and qmax, the minimum and maximum value representable by the given qType :parameter qType: onnx.onnx_pb.TensorProto.UINT8 or onnx.onnx_pb.TensorProto.UINT8 :return: qmin, qmax z;This function is not implemented for float 8 as not needed.Nrrrrrz, dtype=z, reduce_range=z , symmetric=z, qType=) rr rYrONNX_INT_TYPE_REDUCED_RANGEgetONNX_INT_TYPE_SYMMETRIC_RANGEONNX_INT_TYPE_RANGEr3rl)rrrqrangerrs r#rrs#  &333!"_``` F0,0077 0u ===.u5$((// xvvvvwwwJD$ axx4!88 4 4 4"& 4 404  4 4KW 4 4" 4 4,1 4 4   Mr%c6t|||\}}||z S)z Helper function to get the quantization range for a type. parameter qType: quantization type. return: quantization range. r)r)rrrrrs r#get_qrange_for_qTyper%:s& ) RRRJD$ $;r%r ranktuple[bool, int]c<|dkr||zn|}|dko||k}||fS)z Helper function that tries to return a normalized axis in the range [0, rank - 1]. :parameter axis: The axis to normalize. :parameter rank: The tensor rank (number of dimensions). :return (is_valid, axis_norm) rr)rr& axis_normis_valids r#normalize_axisr+Ds7 $axxt TIA~2)d"2H Y r%src_8bitr  bytearrayc"t|}|dkrtS|dzdz}t|}d}d}||dz kr3||dzdzdz||dzz||<|dz }|dz }||dz k3||kr||dz||<|S)aB Copies a source array of 8-bit values into a destination bytearray of packed 4-bit values. Assumes that the source values are already in the appropriate int4 range. :parameter src_8bit: The 8-bit element values to pack. :return A bytearray with every two 8-bit src elements packed into a single byte. rr(rFrorH)rr-)r, num_elemsdst_sizedstsrc_idst_is r#r r PsH IA~~{{A !#H H  CEE )a-   *S0Q68E?S;PQE      )a-    ye_s*E Jr%c eZdZdZggdfdZdS)QuantizedInitializerzJ Represents a linearly quantized weight input from ONNX operators Nc ||_||_||_||_||_||_||_||_| |_dSr*) r, initializerrminsrmaxs zero_pointsscalesrrr) r.r,r7r8r9r:r;rrrs r#__init__zQuantizedInitializer.__init__ssJ &  &  , r%r7r8r9__doc__r<rr%r#r5r5ns= r%r5c"eZdZdZ ddZdS)QuantizedValuezI Represents a linearly quantized value (input\output\intializer) Nc ||_||_||_||_||_||_||_||_| |_dSr*) original_nameq_name scale_namezp_name value_typer node_type node_qtype scale_type) r.r,new_quantized_namerDzero_point_namequantized_value_typerrGrHrIs r#r<zQuantizedValue.__init__sH"( $& . "$$r%)NNNNr=rr%r#r@r@s@%%%%%%r%r@ceZdZdZdZdS)BiasToQuantizez+ Represents a bias to be quantized c0||_||_||_dSr*) bias_name input_name weight_name)r.rPrQrRs r#r<zBiasToQuantize.__init__s"$&r%Nr=rr%r#rNrNs-'''''r%rNc|jdkrtd|jd|jdkr|j}n|jdkr|j}n|jdkr|j}n|jdkr|j}n|jdkr|j}n|jd kr|j}nl|jd kr|j }nY|jd kr|j }nF|jd kr|j }n3|jd kr|j }n td|jd|jd|j|iS)z Convert attribute to kwarg format for use with onnx.helper.make_node. :parameter attribute: attribute in AttributeProto format. :return: attribute in {key: value} format. rz attribute z does not have type specified.r(rFrGrHrIrJrp rz has unsupported type rO) rzr3r,rrsrMgfloatsintsstringstensorsgraphs) attributers r#attribute_to_kwargr^sB ~TinTTTUUU~  1    1    1    1    1    1   1  ! 1  ! 2   ]in]]IN]]]^^^ NE ""r%cZfd|D}t|dkr|dndS)z Helper function to find item by name in a list. parameter item_name: name of the item. parameter item_list: list of items. return: item if found. None otherwise. c*g|]}|jk |Srr+)r!item item_names r#rz find_by_name..s% B B Bd49 +A+AT+A+A+Ar%rN)r)rb item_listitemss` r# find_by_nameres; C B B Bi B B BE5zzA~~5884/r%cdd}tt|D]}|||kr|}|S)zC Helper function to return index of an item in a node list rs)rr) elem_name elem_listelem_idxrs r#get_elem_indexrjs@H 3y>> " " Q<9 $ $H Or%cHtjd||g|S)z Helper function to create a Mul node. parameter inputs: list of input names. parameter output: output name. parameter name: name of the node. return: Mul node in NodeProto format. Mul)rrr)inputsoutputr,s r# get_mul_noderos" ; $ ? ??r%filenamer identifierr cV|j|j|z|jzS)zp Helper function to generate a identifiable filepath by concatenating the given identifier as a suffix. )parentjoinpathstemsuffix)rprqs r#generate_identified_filenamerws( ? # #HMJ$>$P Q QQr%cddl}ddlm}ddl}|j|jt dt |t dt ||||d|d| d| d | dS) Nr) thresholdz Histogram:zHistogram Edges:T)fillz Tensor valueCountszTensor value V.S. Counts) sysmatplotlib.pyplotpyplotrxset_printoptionsmaxsizeprintstairsxlabelylabeltitleshow)hist hist_edgesr|pltrxs r# apply_plotrsJJJ######LLLES[1111 , $KKK  *JJtZdJ+++JJ~JJxII()))HHJJJJJr%rOc  ddlddl}ddlddlmcmcm}ddlmcmcm}ddl m m m tjd|Gfddj}||}t#t$j|dd 5}||dddn #1swxYwYd}|d } g} t1|D]:} || } | } t7| d |t7| d |g}t=t?|}| | }| |}|!| |"| ||#| ||$| }| %|<|&| tO| | D]}| (|| )}|*| |+| ||,| }| -|| .}t#t$j|d d5}||dddn #1swxYwYt$j/dddvr|j0|d}|1}te|D]c}|3|}tj|4tj|5dt#t$j|dd 5}t1|D]} || } | } t7| d |t7| d |g}| dzt=t?|z}|||d ddddS#1swxYwYdS)z> Helper function to write calibration table to files. rN)CalibrationMethod TensorData TensorsDatazcalibration cache: c"eZdZfdZdS)*write_calibration_table..MyEncoderc\t|fr|St|jr*|t |jddSt|r|jjt |dSj ||S)Nz numpy.array)rrlCLS)rr) rto_dictr~tolistr rl __class__r7 JSONEncoderdefault)r.objrrrjsonnps r#rz2write_calibration_table..MyEncoder.default$s# K899 %{{}}$#rz** ] # s39~~m\\\#011 J"}5CIII#++D#66 6r%N)r7r8r9r)rrrrrsr# MyEncoderr#sB 7 7 7 7 7 7 7 7 7 7 7r%r)clszcalibration.jsonwihighestlowestzcalibration.flatbufferswbQUANTIZATION_DEBUG0)r(1zcalibration.cache  )6r flatbuffersrx5onnxruntime.quantization.CalTableFlatBuffers.KeyValue quantizationCalTableFlatBuffersKeyValue5onnxruntime.quantization.CalTableFlatBuffers.TrtTableTrtTable"onnxruntime.quantization.calibraterrrlogginginfordumpsopenospathjoinwriter}Buildersortedkeysrrr rar r CreateString KeyValueStartKeyValueAddKeyKeyValueAddValue KeyValueEndr|TrtTableStartDictVectorrPrependUOffsetTRelative EndVector TrtTableStartTrtTableAddDict TrtTableEndFinishOutputenvironGetRootAsTrtTable DictLengthrDictKeyValue)calibration_cachedirrrrr json_datafilerbuilderkey_value_listkeyrd_valuesrXrflat_key flat_value key_value main_dict cal_tablebufdict_lenrrrrrrs @@@@@r#write_calibration_tablers! KKKLLLLLLLLLLLLLLLLLLLLLLLL]]]]]]]]]] L:'8::;;;77777777777D$777 ,) <>## (,,y$//4466 7 7 (,,x..3355 6 6 CKK  '',,))%00 w'''222!!':666((11 i(((( $$Wc..A.ABBB#33 '' 2222!!##I 7### Wi000$$W--I NN9 ..  C bgll3 9::D A AT 3 z~~*C00H<<%77Q?? ''))x , ,A!q))I L ) ) ) L** + + + + bgll3 344c : : d+002233  C&s+F~~''Hhll9d3388::;;hll8T227799::F#ICKK 0 00E JJu    JJt                       s7)C  CCM))M-0M-&C7U++U/2U/-C6?c|dktj}|dktj}|}|j|z }|sdS|t |zt |z }|dksJd|d|d||tj}|||z| |zzz }|dkdksJ|S)a~Given a discrete distribution (may have not been normalized to 1), smooth it by replacing zeros with eps multiplied by a scaling factor and taking the corresponding amount off the non-zero values. Ref: http://web.engr.illinois.edu/~hanj/cs412/bk3/KL-divergence.pdf https://github.com//apache/incubator-mxnet/blob/master/python/mxnet/contrib/quantization.py rNrzn_zeros=z , n_nonzeros=z, eps1=)rrxrsumsizer)pepsis_zeros is_nonzerosn_zeros n_nonzeroseps1rs r#smooth_distributionrosQu}--H6//%-00KllnnG'!J t w % "3"3 3D #:::Q'QQ QQ4QQ::: 88EM " "DC(Nte{2 22D AI??   ! ! ! ! Kr% model_pathctj|d}td|jjDS)NF)load_external_datac3>K|]}tj|VdSr*)r uses_external_data)r! intializers r# z*model_has_external_data..s.mmz#6zBBmmmmmmr%)rloadas_posixrgraphr7)rmodels r#model_has_external_datarsF Ij))++ F F FE mmUZU`Ulmmm m mmr%opt_model_pathct}||_tj|_i}dg|d<t ||fddgi|}dS)z Generate model that applies graph optimization (constant folding, etc.) parameter model_path: path to the original onnx model parameter opt_model_path: path to the optimized onnx model :return: optimized onnx model ConstantSharingdisabled_optimizers providersCPUExecutionProviderN)rroptimized_model_filepathrORT_ENABLE_BASICgraph_optimization_levelr)rr sess_optionkwargs_s r#optimize_modelrsr!""K+9+B+B+D+DK(+A+RK( F%6$7F !,,.. jjH^G_jcijjAAAr%rr cddi}|jr+|jD]#}||j|ji$tj||dS)z>Tag the model that it went through quantization pre-processingonnx.quant.pre_processonnxruntime.quantNmetadata_propsupdaterrrrset_model_props)rrprops r#add_pre_process_metadatar sh.0CDN :( : :D  ! !48TZ"8 9 9 9 9K~66666r%cZ|jr#|jD]}|jdkr|jdkrdSdS)zCCheck the model whether it went through quantization pre-processingrrTFrrr)rrs r#model_has_pre_process_metadatar sG (  Dx333 FY8Y8Ytt 5r%cddi}|jr+|jD]#}||j|ji$tj||dS)N onnx.inferrr)rrrs r#add_infer_metadatarsh"$78N 4% 4 4A  ! !15!'"2 3 3 3 3K~66666r%cZ|jr#|jD]}|jdkr|jdkrdSdS)NrrTFr )rrs r#model_has_infer_metadatarsF %  Au $$4G)G)Gtt 5r%cd|jD}t|dkrtd|dj}|S)Nc6g|]}|jr |jdk|S)r)domain)r!opsets r#rz%get_opset_version..s.mmm5<mSXS_clSlSleSlSlSlr%r(z$Failed to find proper ai.onnx domainr) opset_importrr3version)rai_onnx_domain opset_versions r#get_opset_versionrsMmm);mmmN >a?@@@"1%-M r% weight_typect|}|}t|d|}|dkr0|tjjkrt jd|dd}n?|dkrt jd|dn |dkrt jd|dd}||kr/tj||}t|}|S) Nr^z$The original model opset version is z, which does not support quantization to float 8. Please update the model to opset >= 19. Automatically update the model to opset 19. Please verify the quantized model.rze, which does not support node fusions. Please update the model to opset >= 11 for better performance.z, which does not support quantization. Please update the model to opset >= 11. Automatically update the model to opset 11. Please verify the quantized model. ) rrrr rYrwarningversion_converterconvert_version&save_and_reload_model_with_shape_infer)rrrtarget_opset_versionweight_quant_types r#update_opset_versionr%s-%e,,M( ]KHHr/43C3PPP 1= 1 1 1   " "   M= M M M       1= 1 1 1   "},,&66u>RSS7u== Lr%c,t|d}tjt |t |tj|}t|||S)Nz -inferred) rwrshape_inferenceinfer_shapes_pathr rrrunlink)rinferred_model_pathrs r#load_model_with_shape_inferr+s{6z;OO**3z??C@SDEQUANT_OP_NAMErJrMODEL_SIZE_THRESHOLDrrr7r'r>rErarQrlrSrWrUrYr]r[rr}rhrgrjrir"r!rrrrrrrrrr%r+r r5r@rNr^rerjrorwrrrrrr r rrrr%r+r"rr<r?rBrErHrKrr%r#r[s  #"""""   0000000000>>>>>>>>>>&&&&&&QQQQQQQQQQQQ------PPPPPPPPPP7777777      ,$2'!qqCC 4D4Dqqq      t           #>#>#>#>#>#>#>#>L     $    V!4!4 +%+g"6"6 +%+g"6"6!;5;x#8#8' % ;5;q #D#D#DkekRU]b]hFiFiFi"j+%+d%*"E"E"E{u{SV^c^hGiGiGi!j!KEK$F$F$F  TYafamHnHnHn#o ;5;vU[#I#I#I;5;W\didoKpKpKp"q ;5;q#>#>#>  BV[@\@\@\"]+%+b"="="={u{1TX?Y?Y?Y!Z +%+d%*"E"E"E{u{SV^c^hGiGiGi!j ;5;vU[#I#I#I;5;W\didoKpKpKp"q!  ;5;q #D#D#DkekRU]b]hFiFiFi"j+%+c"D"D"DkekRT\a\fFgFgFg!h!KEK$F$F$F  TYafamHnHnHn#o ;5;vU[#I#I#I;5;W\didoKpKpKp"q ;5;q#>#>#>  AUZ@[@[@["\+%+b"="="={u{1TX?Y?Y?Y!Z )+ A A A A A 13131313h<<<<~!!!P#'"&"&;G;G;G;G;G~hl:=:=:=:=:=D$( L L L L L ^@    <>%%%%%%%%8''''''''"#"#"#J000@@@RRRR$YYYYx2nnnn k k k k 77777777!!!!H7777++++,,,,2222----3333//////s"A))A32A3