bi7ddlmZddlmZddlmZddlmZmZddl m Z m Z m Z m Z mZe r ddlZddlmZe rdd lmZe Ze resddlZejeZGd d eZGd d ejjZGddejZeddZ ddee dzfdZ!dS)) lru_cache)ACT2FN) ConversionOps)get_module_from_nameshould_convert_module)is_accelerate_availableis_fbgemm_gpu_availableis_torch_availableis_torch_xpu_availableloggingN)nn)init_empty_weightsc eZdZdZ ddeeejeejzfdej j dzdeeejffdZ dS)FbgemmFp8Quantizec||_dSN) hf_quantizer)selfrs _/root/projects/butler/venv/lib/python3.11/site-packages/transformers/integrations/fbgemm_fp8.py__init__zFbgemmFp8Quantize.__init__-s(N input_dictmodelreturnc &t|d\}}|d}ddlm}t ||\}}t ||r@|dkr|dd} | j} | d| d} t| \} } | | }|dd}| | dd| d}n|dkr|dd} | j} | d| d} t| \} } | | }|dd}| | d| dd}nPt|\}}tj | |jdd}|tj ||d|iS) Nrr)FbgemmFp8Llama4TextExperts gate_up_proj down_proj_scale)tupleitems integrationsrr isinstance transposeshapereshapequantize_fp8_per_rowtorchr Parameterview)rrrkwargs target_keyvaluermodule tensor_nametransposed_paramoriginal_shapeflattened_paramnew_value_flatweight_scale_flat new_value weight_scales rconvertzFbgemmFp8Quantize.convert0s& "*"2"2"4"455a8 Ea======25*EE f8 9 9# [n,,$)??1a#8#8 "2!7"2":":2~b?Q"R"R5I4Y4Y1 1+22>BB %//155 0889JA~^_O`aa  ++$)??1a#8#8 "2!7"2":":2~b?Q"R"R5I4Y4Y1 1+22>BB %//155 0889JN[\L]_`aa &:5&A&A #I| 8--l.?.? @RST@UWX.Y.YZZLEH..y99j;P;P;PR^__rr) __name__ __module__ __qualname__rdictstrr+TensorlistrModuler:rrrr,s))) )-2`2`elT%,-???@2`x%2` c5< 2`2`2`2`2`2`rrc2eZdZejffd ZdZxZS)FbgemmFp8Linearcxt|||||_||_tjt j||f||_tjt j|dftj |_ | dt jdgtj d|rItjt j|jtj |_ dSd|_ dS)Ndtyperinput_scale_ubF persistent)superr in_features out_featuresr+rr,zerosweightfloat32r9register_bufferfloatbias)rrMrNrTrH __class__s rrzFbgemmFp8Linear.__init__fs lD999&(h((lK5PX])^)^)^__ !H..u{L!;LTYTa/b/b/bcc -u{A3ek/R/R/R_deee  **5;8IRWR_+`+`+`aaDIIIDIIIrcg|jdddR}t|d|jd|j\}}|jtj}tratj ||j | d| |j|j}nAtjj||j ||d}|j ||jzn|}||j}||}~~|S)Nr )scale_ub)scale_ascale_b out_dtyperTTuse_fast_accum)r(r*r- contiguousrIr9tor+rQ_is_torch_xpu_available _scaled_mmrPt unsqueezerHrTopsfbgemmf8f8bf16_rowwisedevicer))rx output_shape x_quantizedx_scaleweight_scale_float32outputs rforwardzFbgemmFp8Linear.forwardtsJ*"*r**  4AFF2qwr{4K4K4V4V4X4Xcgcvwww W $033EMBB " M% ))"--,..00'Y FFY%66T['3GX\7F,09+@Vdi''fF18$$ --  r)r;r<r=r+ float8_e4m3fnrrm __classcell__rUs@rrErEesR>C>Q      rrEc2eZdZejffd ZdZxZS)rct|j|_|j|_|j|_|j|_t|j|_ tj tj |j|jd|jzftj|_tj tj |jd|jdzftj|_tj tj |j|j|jftj|_tj tj |j|jdftj|_|dtj dgtjddS)NrrGrrIFrJ)rLrnum_local_experts num_expertsintermediate_size hidden_size expert_dimr hidden_actact_fnr+rr,rOrnrrQgate_up_proj_scaler!down_proj_scalerRrS)rconfigrHrUs rrz#FbgemmFp8Llama4TextExperts.__init__s !3!'!9!-0V./ !H.. K)4+;Q=PQY^Yl m m m  #(("4"4 K)1do.AB%- X X X# # ++ K)4?DP=PQVVWY[\]]hhjjllnn+1 %$%a(221a889K9L9LMXXZZ\\^^(222666q9!<=O=P=PQVVWY[\]]hhjjllnn+1 y'88$%a(221a889L:L9LMXXZZ .q1!45H6H5HINNrSTUU``bb#' 9Y%66$%a(221a889K9L9LMXXZZ .q1!45G5H5HINNrSTUU``bb#' 7T[[...I3G S]_c_r3s3s 0 &*&:&=&=em&L&L #&  % 0'N1%//155@@BB+55b993A6;;BBBMMOOQQSS+1 !!! !& 0 A A'N1%//155@@BB#+A.33B::EEGG#' !B!! +KNN!nn]%9:: D$4555r)r;r<r=r+rQrrmrorps@rrrs^%*]ffffff0P6P6P6P6P6P6P6rrr)maxsizechtrddlm}|djStjjjS)Nr get_kernelzkernels-community/fp8-fbgemm)r_ hub_kernelsrr*r+rcrdrs rget_quantize_fp8_per_rowrs?O++++++z899NN 9  00rFmodules_to_not_convertcptad}|rinddi}|D]\}}t||sd} t d5|jjdkr2t|jd|j} t| p|j} nOt|tj r5t|j|j|jdufi|} | ddddn #1swxYwY| ||| d}|st&d|S) a A helper function to replace all `torch.nn.Linear` modules by `FbgemmFp8Linear` modules. This will enable running your models using high performance fp8 kernel from FBGEMM library. Parameters: model (`torch.nn.Module`): Input model or `torch.nn.Module` as the function is run recursively. modules_to_not_convert (`list[`str`]`, *optional*, defaults to `None`): Names of the modules to not convert. In practice we keep the `lm_head` in full precision for numerical stability reasons. quantization_config (`FbgemmFp8Config`): The quantization config object that contains the quantization parameters. pre_quantized (`book`, defaults to `False`): Whether the model is pre-quantized or not FrHNT)include_buffersLlama4TextExperts text_configzYou are loading your model using FP8 quantization but no linear modules were found in your model. Please double check your model architecture, or submit an issue on github if you think this is a bug.)rr* named_modulesrrrUr;getattrr|rr&rLinearrErMrNrTrequires_grad_ set_submoduleloggerwarning) rrquantization_config pre_quantizedtp_planhas_been_replaced module_kwargs module_namer1 new_modulers rreplace_with_fbgemm_fp8_linearr s$455'rs-  ......UUUUUUUULLL.------0022*%<*))))  H % %6`6`6`6`6` 6`6`6`r,,,,,eho,,,^i6i6i6i6i6i6i6i6X 1111tx::#'9t#3::::::r