|2g<<ddlZddlZddlmZddlZddlZddlmZddlmZddlZddlZddl m Z m Z m Z m Z ddlZddlmZddlZddlmZddlmZmZmZmZmZej.dZdde dej2d ej2fd ZGd d ej6ZGd dej2ZdZddede fdZ!ddej2de"fdZ#ddZ$ddZ%ddZ&ddZ'Gddej2Z(y)N)partial)SequenceTupleUnionCallable) trunc_normal_)Mlp PatchEmbedSwiGLUFFNFusedMemEffAttentionNestedTensorBlockdinov2fnmodulereturnc|s |r ||||jD],\}}|rdj||fn|}t||||d.|r |r ||||S)Nrname.T)rrr depth_first include_root)named_childrenjoin named_apply)rrrrr child_name child_modules )epsr4)img_size patch_sizein_chans embed_dimTrr3zusing MLP layer as FFN swiglufusedswigluzusing SwiGLU layer as FFNidentityzusing Identity layer as FFNc*tjSr")nnIdentity)argskwargss rfz)DinoVisionTransformer.__init__..fss{{}$r) dim num_heads mlp_ratioqkv_bias proj_biasffn_bias drop_path norm_layer act_layer ffn_layer init_valuesFN)$super__init__rr@ LayerNorm num_featuresr; num_tokensn_blocksrFr9 patch_embed num_patches Parametertorchzeros cls_token pos_embedlinspaceitemloggerinfor r NotImplementedErrorrangechunked_blocksappendrA ModuleListr blocksnormhead mask_token init_weights)r$r8r9r:r;depthrFrGrHrJrIdrop_path_ratedrop_path_uniformrO embed_layerrMblock_fnrN block_chunksrLrWdprr%rDi blocks_listrc chunksizep __class__s rrQzDinoVisionTransformer.__init__+sV R\\t4 -66DN "$&ZZbnwx&&22 ekk!Q &BCekk![4??5RT]&^_  $!"U*C%*^^A~u%MN1668NCN   KK0 1I - '9+@ KK3 4&I * $ KK5 6 %I% % 5\  ##!#!a&%##'   ! "&D N-I1eY/ \%%r{{}o&9KAPYM> !NN((* #AqD/#AqrE*ggbk $// ! $// !c28B--33  # #As499Q<'8#diil:KS Q Y YZ[]^`acd etyy|+R$))A,->?4  2w///33B?CXCXY[C\8\\)11!Q1=BB1b#Nyy/33A6HaPSSTbccrc|j\}}}}|j|}|Xtj|j d|j j |jj d|}tj|jj|jddd|fd}||j|||z}|S)Nrrr4r) rrVrYwhererrirrrr[expandr)r$r%masksBncrrs rrxz/DinoVisionTransformer.prepare_tokens_with_maskssgg 2q!   Q    EOOB/1C1CAGG1L1V1VWX1Y[\]A IIt~~,,QWWQZR@!D! L --aA6 6rc@t||Dcgc]\}}|j||c}}}|jD] }||} |}g}t||D]<\}}|j|}|j |dddf|ddddf||d>|Scc}}wNrr4)x_norm_clstokenx_norm_patchtokens x_prenormr)ziprxrfrgrd) r$x_list masks_listr%rr}all_xoutputx_norms rforward_features_listz+DinoVisionTransformer.forward_features_listsFI&R\F] ^(!UT + +Au 5 ^;; CAA E:. HAuYYq\F MM'-ad|*0AB-!""    ! _sBct|tr|j||S|j||}|jD] }||} |j |}|dddf|ddddf||dSr) isinstancelistrrxrfrg)r$r%rr}rs rforward_featuresz&DinoVisionTransformer.forward_featuress a --a7 7  * *1e 4;; CAA 1%ad|"(AB-   rc||j|}gt|j}}t|trt ||z |n|}t |jD]#\}}||}||vs|j|%t|t|k(s Jdt|dt|d|S)Nonly  / blocks found)rxrzrfrrrbryrd)r$r%r{rtotal_block_lenblocks_to_takerrr}s r$_get_intermediate_layers_not_chunkedz:DinoVisionTransformer._get_intermediate_layers_not_chunkeds  * *1 -"$c$++&6HRSTVYHZ2OD`a , !FAsAAN" a  !6{c.11mU3v;-s3~K^J__l3mm rc|j|}gdt|jd}}}t|trt ||z |n|}|jD].}||dD]$}||}||vr|j ||dz }&0t|t|k(s Jdt|dt|d|S)Nrrr4rrr)rxrzrfrrrbrd) r$r%r{rrrrr block_chunkr}s r _get_intermediate_layers_chunkedz6DinoVisionTransformer._get_intermediate_layers_chunkeds  * *1 -%'C B,@?HRSTVYHZ2OD`a;; K"12 F&MM!$Q    6{c.11mU3v;-s3~K^J__l3mm rr%r{rreturn_class_tokenrc H|jr|j||}n|j||}|r|Dcgc]}|j|}}|Dcgc] }|dddf }}|Dcgc] }|ddddf}}|rl|j\} } } } |Dcgc]P}|j | | |j z| |j zdjddddjR}}|rtt||St|Scc}wcc}wcc}wcc}w)Nrr4rr/r) rcrrrgrrr9r contiguoustupler) r$r%r{rrrgoutputsout class_tokensr_rrs rget_intermediate_layersz-DinoVisionTransformer.get_intermediate_layerss*   ;;AqAG??1EG 189#tyy~9G9-45cAqD 5 5)01#3q!"u:11 JAq!Q# AqDOO3Q$//5I2NVVWXZ[]^`abmmoG Wl34 4W~:51sDD,DAD) is_trainingcV|j|i|}|r|S|j|dS)Nr)rrh)r$rrBrCrets rr'zDinoVisionTransformer.forward-s6#d##T4V4 J99S!234 4r)r4Tr"r4)r4FFT)r(r)r*r r@GELUr rQr~rjrrxrrrrrYTensorrrrboolrrr' __classcell__rvs@rr,r,*s''"'pf 1 d4 & "  "#$#(   << h   !  uU\\5#667 84*/55rr,cd}||d|zz ks ||d|zzkDrtjddtj5|||z |z }|||z |z }|j d|zdz d|zdz |j |j |tjdz|j||j|||cdddS#1swYyxYw)Ncddtj|tjdz zdz S)Ng?@)rerfr)r%s rnorm_cdfz(_no_grad_trunc_normal_..norm_cdf7s(TXXa$))B-/00B66rrzjmean is more than 2 std from [a, b] in nn.init.trunc_normal_. The distribution of values may be incorrect.) stacklevelr4r)minmax) warningswarnrYno_graduniform_erfinv_mul_rradd_clamp_)tensormeanrar&rlus r_no_grad_trunc_normal_r4s7 q1s7{q1s7{ 2 E!" $  a$h#% & a$h#% & A 1q519-   C$))B-'( D  ! #+s BC&&C/ drop_probtrainingc|dk(s|s|Sd|z }|jdfd|jdz zz}|tj||j|j z}|j |j||z}|S)Nr2r4rr)rdevice)rndimrYrandrrfloor_div)r%rr keep_probr random_tensorrs rrKrKYs}BhI I WWQZMDAFFQJ/ /E 5 QQM UU9  -F Mrrct|tjrNt|jd|j *tj j|j yyy)zCViT weight initialization, original timm impl (for reproducibility)rrN)rr@Linearrweightbiasrzeros_rs rrrdsF&"))$fmm. ;; " GGNN6;; ' #%rc Ptd|ddddtttd|}|S)Nr1 attn_classr9r;rkrFrGror#r,rr r r9rCmodels r vit_smallrls< ! *G    E Lrc Ptd|ddddtttd|}|S)Nr0r1rrrr#rrs rvit_baser ys< ! *G    E Lrc Ptd|ddddtttd|}|S)Nir.rrrr#rrs r vit_larger s< ! *G    E Lrc Ptd|ddddtttd|}|S)zW Close to ViT-giant, with embed-dim 1536 and 24 heads => embed-dim per head 64 i(r rrrr#rrs r vit_giant2rs> " *G    E Lrc,eZdZfdZdZddZxZS)DINOv2Featurizerct|||_||_||_d|_t jjdd|_ y)Nzfacebookresearch/dinov2 dinov2_vits14) rPrQarchr9 feat_typen_featsrYhubloadr)r$rr9rrvs rrQzDINOv2Featurizer.__init__sA  $" YY^^$=O rc8|jj|Sr")rr')r$imgs r get_cls_tokenzDINOv2Featurizer.get_cls_tokenszz!!#&&rc|jd|jz}|jd|jz}|jj|dj d||dj ddddS)Nrr/rrrrr4)rr9rrrr)r$rr{ include_clsrrs rr'zDINOv2Featurizer.forwardsr IIaLDOO + IIaLDOO +zz**3/0DEMMbRSUVX[\ddefhiklnoppr)r4F)r(r)r*rQrr'rrs@rrrsP'qrr)TF)r2F)r)r.))rr functoolsrtimmrYtorch.nnr@loggingtypingrrrrtorch.utils.checkpoint torch.nn.initr featup.featurizers.dinov2.layersr r r r r getLoggerr_Modulerrer r,rrrrKstrrrr r rrr#rrr+s   33 'pp   8 $Hbiiceclcl H5BIIH5T!JE$("))(3(     qryyqr