From e7862a154949ff040640f3f384d416be4dfd3466 Mon Sep 17 00:00:00 2001 From: Alexandria Barghi Date: Tue, 19 Nov 2024 09:17:18 -0800 Subject: [PATCH 01/49] h --- .../cugraph/gnn/data_loading/dist_sampler.py | 20 +++++++++++++----- python/nx-cugraph/objects.inv | Bin 0 -> 51830 bytes 2 files changed, 15 insertions(+), 5 deletions(-) create mode 100644 python/nx-cugraph/objects.inv diff --git a/python/cugraph/cugraph/gnn/data_loading/dist_sampler.py b/python/cugraph/cugraph/gnn/data_loading/dist_sampler.py index 0ff38741e1a..056e714233c 100644 --- a/python/cugraph/cugraph/gnn/data_loading/dist_sampler.py +++ b/python/cugraph/cugraph/gnn/data_loading/dist_sampler.py @@ -699,6 +699,7 @@ def __init__( compress_per_hop: bool = False, with_replacement: bool = False, biased: bool = False, + heterogeneous: bool = False, ): self.__fanout = fanout self.__prior_sources_behavior = prior_sources_behavior @@ -713,11 +714,19 @@ def __init__( # change. # TODO allow func to be a call to a future remote sampling API # if the provided graph is in another process (rapidsai/cugraph#4623). - self.__func = ( - pylibcugraph.biased_neighbor_sample - if biased - else pylibcugraph.uniform_neighbor_sample - ) + if heterogeneous: + self.__func = ( + pylibcugraph.heterogeneous_biased_neighbor_sample + if biased + else pylibcugraph.heterogeneous_uniform_neighbor_sample + ) + else: + # TODO change this to the new API (rapidsai/cugraph#4773) + self.__func = ( + pylibcugraph.biased_neighbor_sample + if biased + else pylibcugraph.uniform_neighbor_sample + ) super().__init__( graph, @@ -834,4 +843,5 @@ def sample_batches( return_dict=True, ) + sampling_results_dict["fanout"] = cupy.array(self.__fanout, dtype="int32") return sampling_results_dict diff --git a/python/nx-cugraph/objects.inv b/python/nx-cugraph/objects.inv new file mode 100644 index 0000000000000000000000000000000000000000..4d38bd31eddf7dfeb33fea09937aaa535c79c203 GIT binary patch literal 51830 zcmV)kK%l=PAX9K?X>NERX>N99Zgg*Qc_4OWa&u{KZXhxWBOp+6Z)#;@bUGkTWpsCM za%)%$BOq2~a&u{KZaN?{E;24L3L_v?Xk{RBWo=<;Ze(S0Aa78b#rNMXCQiPX<{x4c-qvx*|PICwkCM*r`Qoy7u^x{D=A8%sJe4G zov!Rk=kZCOtU7lL5CkQ7bC5ag!B_PT)lXPY(f~M!6sJXUfxFrf{^)9iKX%`TF2Ck+p8gsKdB)@D<^K7fX+YBSW&e^epCv4!>`&~2vb+2_-QBV<_z(Z_ z<=b~rh4E+pQ$p^y{+Q|sOaJ8Ys|++8edjndjxxfd)Nv$M$g*4P^@3el#1fJ*?@yyB zqhwc>@r;G*xN4A5DjxA{9Z!KUPLi=?C5b{9B?(xxjza;AP!P{^n0IB%IVuDIN$U7X z9ImG-a3%<&*js6pL;#RrSoOe}Abe;1dJ`~c>DdD~D&aRtcTeIwOR~@Tguji06DpuS z*H~0SjL^`4kz_fAiarHjq~=PJ0%VLTzOP6t8G8iWW7mYSHE+8>7@?5%Ice@*tLAK( zAuX5*x^#^}Q}08xPU@XG9?8gZds;9fbT^W&th+0O36gbHX@xLC!XsyWcB?_giB{)= z3S)#Oa`TAZR0F>}-5MZcRI>hd<77o;Q$vi=$OdXnswsdmf;@CZcPRGPR-I2kAj;ah zwUkp<#)-EkOQ`5$@Og$%kRB)#RI7_p4KhM>PnfrcL&l$gXOyMssv^`x86itp7{9TV zm39LJMzyw^*nyg+UDskXKwwlz7J_==343Eny2?5QFhY^?KUUz&*kj|~ zrqIgx6Yy(`>;q$jD2pABCyZuA9kh(80WwaNM<`?um=U`CE_#EHSMul<*yp|Rw4MRAfBoReKzAB z8zg@X)LVn(tAToIR{3ehy)+0u0Eh7p@s2V#n5`G~v+W{Pa( z_R0x;n)3lQNxRg!(nXo1TI*^VLxYnqbi9nAO_48j!*nqwiB>I~ z8GwS2;?xLZ>u6d5moBF1fIx_rFRK}Znxb9hTpL1~L5k8Zhj`rjA|N!MUnlOhZiaQhL%*1eIf*hGBZcLaH6VZh^$AO9B zzJxh15nY#a9G5!WmW!O0>s^+U4oe(&CD2*9-c<>3RARX)!A{E6F3M>KC5n3z;+({D zP0nykVz?y%PDwnMDTnEj zo8^$hbjQtd#;te70UU8y?i;Z42G?~n({Y34wgEeBa9uVt9X6{yDAWEE6z>VlcY@_z z0Q(m3JPR}Y3J6|>37^8>ix2<)eQ=Evp54M9lO#?StiGy%R>!Xbt-$zuymSrO1K}5v z-Btl)e+};e3t3d=l!ZXQ#5~FZmZm?%kqo{9>YtGti#)eV&Oalck(9g3Qdd+`B4Pd_ zRV;}9OZ?7~dN$R~dN$BB^qHx+8bdSNY%}sq$VJd%3XEXka~p|EU$c zKrSMD0}+Vf82>$ITk({C9MxEmK+^i6tEx=h_Y?QjC-X>VBzFZ65b-9R5~gaYGzjhR zEYKQ~59jd`HHdr=EK|qBJao7y?sq2si83c;nLep~Z1Z&IsVLbzx;ivwzRxKaC|i_| zKHeP>l?qXLr1}{@>43zuQZbGq(V^gPJp0@r?HX(%RN&kRPm1xj^Q;6l>43aTQ*IZ; zlmr{wF~GUlhHcfa((UmyBSMK2impYTl;dfwS+_$eo)FQyE(X__$u!I~D~aNeN8+QX zEy`+KbNaOSc44{L&FShuO)C)V@^n*(^|Zz2whS=OvFDY>;XRH-OR+&AX^`z+oi;OO zQVsFUNf_~m-pXcLZ>m&G>XoXVkEpks^q5IAN;WTjK4_k8kvfuRCtjc-X#^rBXUf>wp9Nh^CQ?@~`uHY&EIR7xQQ{)+cP8x_VF&9i( zIeVx#AsS>m(O@T>ie7mv+i>h88D?6TIMrvTN-14xn>=&0W{DW+1gZff0ZT)&fy<@K zxg#m@?s1+tBunqu=`_W+F1>d2mWZxp?DKL0O0)DP-Ad(h-roCkwn_2_*-F(Z5A#6k zaEKV@5cT6WCJZpIv?sfX!N@HR1S(Z7(#`A{U|wra=aC~~xB&~dFlmr{o>h#!bFY|q z&|MjpRF6+i*xHVTn_^>Zi<0OG$>%}f@MtUHCEXn5Z=%U$A7Xg(R?gsSeZNh%_Tckb z2WE1?SSM0Vt}D+7*ooN;4oj$343dfdmYZkHX+3}TLa$7vxs)@Kr~w!94_O6yoNCnE zAX%kSWSNA2XxCZL%u-sq;ET!$+lLCJTB5jG5QbNaLulqGEe+kQs7>)L(5|tNbs3iH z(^I_qI2LHUGm@lnB;G`lut2pP8{~os);V@)%mfSHzM6q~)P5i6bybI8wz~2)Xp&Z= z&(f8LOuAb(tB@uMr_^UGiMdy-;h5?fRbheMqB^+GBQZ9szdPcc-vgB9WP$=}{Enw0 zD>^<>Bi92vuEgbq7Z>1#tb>AH!XvaxQcg9)?`Ryv2!nuLSYQg4QHs)=;%g~^=cdsvf?2dTD!5uUTgI zyIM4exMUi_GB+i5xA;zz*w36-lHB!*?WdZpxY~uMRqHd!e3d*>0L97>$IpX6v*HoL z+3K?xdHu{Z#|(|-gs-*|EGK(iUZ0MIHpdJgnU@MXc@T!PiPQlmsNQ=L|Hde6&-v0f z_u!B-GXZhqvWHf$&MHHQPw4H`Z1VXjSG0wkZf(yL?E*_=fe*-iYfGkR7g(~5@bx-3 zPU*ENoG>pJ1ODoSLoM~igQHkk(dNWTlwlTVTBLW8fC*$N#L|?g80rtzaQqF~yE95O zksk6i6_P;mLm3j52vuO^Zf=sSw7C}-W@3?1G-zqzPH0JZeb(G~Y8L@viCuDzu4K+!{qv0M#EWhOf2^Em4$TwkB60j6es^@u-7n{c zCf9BQ=~@)ZlU-5Pu0WZ7I!YJVuY+Uj%OB(=7N5+QJ zF0({`mY-q+>(;RRXZfO^W=t;nkx5#574QVruwW%cFHe8D8sWjvs-qlD)6G-9byWr> zMYlrR%g|7>Dcm3?p4PYUL;zRE1<#upamI-@eZ3Sb6*^HK-nY#)xLB)!tDVrkSZ!Z$D*#W2VOXH~_se z`)Fn=ecUanS86X0SnG`>VI0Y$g!E|a!3S&8{;pTOWggK)9o#MIH0@=4ifo2zKTg7O zSF0|qeH37YuO$0Jm$WDBrJ6K!EP5Hh`T}wf2<1fEJMANYtJB!AUD~HD9KE``JcBG? zo%&mmb?t{3>C-sSTA%b&z!fI)9Mxhmp=d90rM0oano>sN$aB;!JGzo>fCH|vyjw+^ zzApY6Z4Udm>((bZz#6k8zTZU0$K!q}BSqEMgFjoF=vKtHBC(f#v^|clgQ=@FoMf@n0z%X`L;+K%jtmk1 z=ahMmd-rgpb+KWZcH2kQE$wgq5a^jI(dVuFxJf9FSS>x$DzGCU26*`Wpn0r7)6}~_X z)XfE0r%DcE!!)J8=OoedsyfgxMWUN8&vimt7Tkvl)xk-04RDI-6Xb*fdE^&p2VoTw zRy?b~?Dw2!#0})h7bs_z4I#tKtd5n(7Sn_UDv=Yz8M}T~r^kxIlnR@Q9k+n%_nlDR5Vl2wN; zXiv@|uGyMHG5V9S1ENW2lVD0+drh(U3DS;rA_1GUZoH>w7|Ah7GCN5+&v5+mFd~pz z(1wjF=$w}&w@jp;k>dwB_qO0@e57b;Jjeh{+<6;g7N$XS6%0?_NaRH9y}Ta^D;0(~ z!0AwZa`KD^sm8=;6G_)#eSI8if{gG$6Xg}`k6f4-C4symvhZV6KXH`t4u}Vw7%YkS z^(Jywz$1-G(xhpBb6lQ24i-R58wepZkpkkfKyHM-lPiH8fv_~8I#fmV7u<=iTm)lW zoK)Q>+Zi%S4YnxG--yn#Q4&;twI$Cm&CH~zA?m7EtI!7vSZDsqG&Xh$zW5jUO4ex( zXc8!6NFrOxkbLM%i#AW?7HT}zl$+=9E=#jrMEIMWr1=_)0PIk^e$O>`g-e2l3784f zMW1Q-t}Z}MBPp=ubx0;vg6~JK!$RLnbuYP;MDudt;yvalccPEuL#blgz?Rm4o});co<(X>7V1l(7WgbxSrSLr;8R~>bs6U9duQZTr@Ci+u(sDYB?m=^~(N9wLFOEY{oT?;ea1%~h*CD~#~W)w*5&g%HkF|6G1GYagKl zE8i}`aZ-8|1$80mY)-HZTV<0K`J;VX^VA6N`NHMNL0}`oS8?s+y6HRNN$ND41+3L) zOK9OQNvqsAJ<^ zxhByho@y&aFa9fR8RA}LQwjE*Ikei+!#UT6*Ljm-tMhV$Z}+2k@I&?W`~2glYU>{j zNSb1$*d*hMBpU#99MPU+j-yhPWw+S-|FJP5PMRe-&E{BxZ&xh?-@>{cXqAL7v$P{e zo5j^(C;w*{T&r1QZ*hn}vg>IU;TrY5IZ>y2)V$W0t2Pk7B;Rb9Yo!3Js|6p9SLZ+MglxLMh2As;2mA@gg9-EsudH4 zL|uSES>uRc@(MT@bCToipM0Q!c+(pT0yJKQ2I3XtgEU3sRcHX-I4U;0ezF9Mya_p_ z;Eh4gR(P9n^hp(%r&SgBhEr_&aA77&FRdOiZQuztu*r_PuQ8R-GGKMPoZQSRkM%%t z9rXiG8>g0cig7j(bVxE9h!Flfb5i~X8hHr^U{-km`yI4F0V+cNAy-CW>v)C*C{djT zY#nV-kcuQMPXa>D8a>DWJ#TT4hgcE{unE>r8h|!esc^7rrwFRf5)9O#@-@dwqxyj% z1{QxDm!up~$O99kp*UerenykR@Rn;u0objb66zoAtz)+S^yZ|cA~&KDV>VP&* zQ6|pD@#kuLm3Q^2jY;yCOozNDin$79Q+>a&L`%}u;B!?a#N&CYLsEe}TMtXCXVs5d z#C2(aN)vXK$Fj_*G0Pb9D$FWUd4+F%ffMGd8swOl6gw^}xbloSha?I4)F84;GFN5l z^u0uEnO&CoI_uq`T6I3NOz4#N?p4KcBIj2@VOhQoU4L?Xp)BJ#xqS>CX!4GSArq}; zCTa`y49UYWiVxA2fo7i{VQ0J$75woL=alk6XsEHOeCZR_K_Az}d|z}{1l#9~o^($n z&}-?7_jtBG8OHHvCq&h%=$6=_nXk0m{i<2$Gs}cdkre=)3>6HPeN;&Guewy2Cp6~> zrKcYkVB8}Pj?1(M6s`heTDy%0;l7d7(UJ-sV2B$2h=Zl00SQzf4|>v|q6s^$#1G>4 z>2fDSfr_09=*FL~@tgA+2VpFJiQm>+){i@FTDdq?j(@yd?wBvaVF~I52dJXtPUNuP zNkt`zGDpx03sDlMnY{h1$_E%K8jt`5p4`Cp2NHJy24H1g7)M^}yprUO#NC7hD7eO9 zoP0XBTwRfhq@fK8P$6#!lpo~uvm@Uel8_EAOp98&ftVMYkTVrHZzRZ>1|0=3L`|7= zs0q@qPaAxOs(Z?EFV-dN34Mkx(US1$O55^UKgt}Ic(F)*5(I=JOPwZMt^P*dSRnp= z<>5B!n}p-)Z(J}X^aePDu?`BV9v-1ptJvaMXa5lLgrT4EBm#hyDjdwjBB(t5C%`PgRew`V1zfzr2n>;Jj)<0>_4jrz=szdo#qf&*xSY66O zwJX5!8A>|xU7c6>&eynbP`Ow`8*SFgP#T-gq8 z+6p4Jfg7}dtJ}YAT0e}oZ@ZQctKHMv>aA(>Hf`|`+q;chJJh!B3tBq#c5d5N4zrE> z>#sk5|I@E4&%W&f&K01gd0i-lL?rn9gE{GED3m0hcxRLX$Y!fmhdea1u2#7nw%K|Y zS4y+BF07C0*efca1zH!^Jtaaw616*7URE1NEf=LW)dQTfwXbinW=mh&YIR|@%q|#h z78Qi@hx+oHx+k{U#W}&TOrx5R+d}9Q(K5B_uqn+#9~?6@#(Cg5E_*6xhyuF=aCJUW zwTWhlLoD-jsv8Eh4VrnPJjkjz)9t4w$C;o3SaCZ4Q?Eu;9K zT+?~8R7ITGlT~H3=P}GT)R6;l1O-uBe17xId8s2J%ubpp51HP!y^jH&OCGVcmieOGfO0B*CRNfQjaSA(|^eF=T z)ivS}kA$N0F8(AAk35F;&c-NaCBu$JPFkGDPJSbibBpgZiTw;;=K#@6wS_Y=V*Gk@ zV=?%Z$0Csc<=7=S*F4R?5lTrSFNCkQxNAt=9+G(~@1+Pi8SaoQBlH$(Q93|0Pi;bq zyuMN19i^M8oC%DCGTEbvsXEJ0SIsa;HBa%oc+6!?=+;>!MMvD~FU1VVx>WL}hryREIj)UJk4U-v>l^y8MPQ9fuhU=ysV@42E*^oJrXn1yFc(2-EPJJDZ;*6gA{g0=3o)-5M>;E09byzPNDccv=WirX zTl19fA5l;bHb@iqWXIR%=qdCJ3{vxZ9%pi-fOXt?FYJ^OP^o|xiL|aSWJmYTBAF~W zhvAKYPnZ}dkbU4P7|cu+QvHke=AI$PO)Bo~l=)s{ISRb%r6&@|+}?O35xpt7uCX$E zbrOJRW+zK|xE+Z+r#z2Oo|4SP=mzCDy}682`XeV15#umoX$pCx&G5jN*^1I-T$>&v z@Z-kEcb1T9L^W;uBRItD7WhuxgPp2D!g`Yjp=X#YPr>~y7GrNGS9g4B5Y}awqpy9& zF}}KI{5jgRCE4tha(P6OPkni}z_0Qcrnzn8dG?B|<(JeNBt5`qt7>L=Ey|h%Gu8Qd z5I8cr24OY!Oij%~s6kc}ex|Z=0lw%6Xw7SVG&7az%51ap*gc8mZ7cdJqLN@oi9Bvg z9p(C<#Ik&lWM(q#io4OZN^CLA)z`WJN~g?TqPc4C@jJ%+!vOGHHRW0(jfvJ~n5(b2 z%0Q>SHp5(fVpNjGiB5fO2B^MVZ%-<(yE_5HnPwfvo@MebVQ;dP#1~WokE^){L95%1xC$Jy95&9zo*DsOuUw^Up(uF-uXfnO{sr}TX* z=`B(u^G5RX+RTEwC@&a7lnx7~aD-Re+Hqk4IR;es< z{$^SWy2Xl%BTXIU@{L$|^|*H79oFUg@1NT!zAr9y#R0|JRAmL%Ql+Ye$3K<=NVrCI zx=r$~Dr;AkIUdQ#771H%tbKXwa=t{J864Bh3%O zrK%$mZj^x~(_)ow9?=`}c~Bs_#frOX~z4AJ(w^P0<%{MEMtIA-UXPb729(-#Q zZ`dWO>6Yr1UF~h8N~Sd`%K^bAg{;Z9RCB_@_>FDWZZzl+6mK_o&>>x^eygE%gAPIQ z=IOnDzNxWNMZae7+LgZO>4yHdbfy0-oq5^9Vx5yWE{w$8&{4eBM=V`3+q7?zX^qP5 zTDJ$=QoUKcsVAjDhoHFFu(h%7f^>!YJlZ(LAA~5XH%;|t+Qk+mM72C>K)#9gCgD=m z`Q58-pVTb3V1nu_mISnxQDQ9a?16?Vl1D+PFPK_t&es5FQpk?^uZH(NX--B zI=d$bz^3oCshXkYng`{|pNW^C0s>z^5msI@>@*nG+Sf$|87E`tP%%rxt88je4rF>p zoZ#p5MuXf1ykQYi9!ciH-f@Y$tE1tb{5Ot2ZgYBzr-KZV16<*B0B+#t{aFfd1h(#Sd_9YUYR12cfEgl&O1rc8_EWE0LlF1cmi;u zJiLEyqP3t}uejmvRR?!?Tx1fIJ;rO;7uY45*ElNH-0CUl{WP?`fcVQJnhC$$Js)o+-C`goVwvSDdPD#P2C zT-@^7M%&7tGmn0uWv%Y4{@SRYZ<+Ru>LG;MXWKdlucmn*ZJy^Rsh3&ga{&>tE6Mm9 zbLb}xG}_QZwoLUk4m_77S0^FfAN*CH;wPeIYJZDwQHpU;=Qe4u(GQ!<^VQXf6~J#9FM;UOZs@ z7`ExJbo==*wQrik5bE6#z>Ra(f|{3yJOG}J`3MVhojrnAwv~5Epv~k?FhEV;sf`Mv z-sc%+?|T#gpU78$aQ?@bi@MKbTf&ZT$J#bidj$BCy&!iLx6@oOWqM)7XUv zD-$2pbSzK-9>d*=e5ZSs}2pVvK|rCf{@~Y>Jj2)@Cgoh5;Ws$mN!=!Pn$83 zb+y4~sH&S$jfvLFF`-e^z=3!gKqj&3u&^xs2p*6uP!y*_yf(|u#7l&fxg{nm% zg89oXgPvr01_!H3`8C8%6@@s(udY#nV%BGushhrar|fmXZ>Vu{eJidzRrl=F97Wr5C5# zkBzIVjXzIYalfF>RC{Ka(3gjy=yUx6yoRzB9F}2PF+3*MAaft12iXri!`xjOk-I~# z*F<^!P(13R_!HXR$sOBG!y~ksHLJg3U4lD~4>VeK>NP6sqEG1JA#>`>iUDt=6(06l z+EJ*du1Eeru(}_3!r(Yhn85oX$Lk%vrUJ`mEF2)4Y z0Pq|&#ob`Rr&HIl*s{6|bM&<{3#P8;tOd|h4zV<8>Y#&lr8G`Nz0ZEnaYmJc&=ZP^ zlPO5Ba~TO!MiP1h6!AJ1sD$@$>|9^2?IhvjSiP0@Pm_RCLX=7`yA^liINH;shfN8` zPnc1rHL0Ue0`gf7qO}Vys8%a}K7UZVZukL&0I>GyQrfjckCy^KW#!4A+BIjBM}AI# zcr2x&qeP-CbqMAp(K6muwv>ATo$czD4kJ{*=$>v?|8}`{o{kh6<5`|A3L_vc3#vQ1 zHqhMSTdDmXr>4s$H_kL3{ArW60U$#M^tEOp5m&gVc|+d9rmdqJ!;r-C2u?<0ciJl6%thN z>%cUKQ=vIcohH*dGe^KDjE$2xiaEg6A|TAvGHm%*y#F9T$W^_tAs$6%J-x5w!S&wQ{=$w8t>hh2@E`^~#h7pnY^sIE#T^+S9Samm`~|YEEYqn< zz%LNRGZv;gWhsIM!d}^@R$UT*fvh(Yruf&nHi6OXVefC_6KGj73Os6j)4Z(+}r zgI~~_&@;v8DBx}InSwMF>=yP+Ia&(z6FNxJUwDwQq^=7Piu4oW5*4V$8$%Cz+3F!! zBuqm&>!nyE4!_LxF)Wg%qx|)A%#kMxFkT6iXmimE zR-@0Bq_Jc*@Mp;SSsp=)-@vtD7wgeb;INcwDuP{>eDd(12QftXG3yZS)W<5r$q}bn%zS(ojwGF)WdWUp@2?ED@%oTIi*iAr8q6_5hp7 zt)UuNqC!{kufgXE(p2ncx#SvAv=#T64!K4oUB!H!JFXE&U-6#nj8n1DeQ`agOV!|) zdL&(%>VY1X#S-<@3=2JleQ|Zw6^s3ZeX%vw9!tH1eUY`*C(C_=kLXw~!Y9NfDl}Cl z%k}82L>>%_q-m@?<~c5(<7url=DIDPW9qCdW;!jOBkHXrfG$gq7l%3jng3KB8C_Hh z-}=;4p%q?@RmJUr_mvM7X@PACw!3jp-qkj~GZIjB^x=&pRVz3y`7CO6Stz#oP0(3F zG{~4GV*F^fx%@-tu002iVyF0s!CZp@h|^pAj+ytHcu;y=@>$f_dA|cXONa&;vqaQ+ ze^F1a3pI-uq3eh$bqb!*AQC)w z9oxhV;VmHGWX_GJSuA>nAp-3K3??8-$2K*>cohbOuDFRNcgL?+fsnS4pw#JVN>DEL z#cTLuT^1@JAY@|HSzPc#vMk|lo~^X92M;Pkwm_INp706tZffV6ppMf^Y-p)OJcG9A%388J8&p#mcw z_wm51Z}Cx&#w!MZS8Nn`D&2~B36mEgFia^;NSKH!b7QJ#VUEZsUAjdEQS)K239eSW zcE)O<3{f@+!E}RUU8*<|FJb7U+w#|uoLdG^M>2b&<&mwf{5!7#9YpDSAANxXo+3~x zlqpKtzQ}-!*(!h$B2}TTOw4Bt7~SH8e8w23Ta$>5=;I{SGiQFQIs`II=!8VH^&cw> zQi)QYzn+KpPlqH4`CK4=2x`)V@_cm?c_ej6bIITu1gG^vJ$(}mETQ^&J$?Q2T6J=?7%wAvt!2`=EhV1L< z3mhzNj)^+V z?)%W?*F4VCU*jOJP{|RlD1UW(Hc`U&c z(#8bJiDhe(F&rH&u$cl{lcuO@y2wCjvRi>8<$(mF_TnVI#&225-kfV3sK261AUADb z=$V3ti7d9OV(4ig!KnRQ)H?}DJ~1kiHW*9*V(FbSZ zZb!)*3)g1Uel)OzIEvJM8wXiTKQ2_(mE@ufh*)K|DY{`o43fMFe5g$6IhQtK%NxLj z%F>i*>(VM=0r^1BXT7LkNpgW!>^n5zzvnDnFQ$zLm62vKy^%EI)NzBDzB-~lW5H@Q z{jg9`?>zECzP7(M05>k8Jm)24zvI;Z4?I~d`TUv}-mq2aJPryqu<2|)`~PLo0I3fiX?0d zeL`YmFEc{rQ|)KIu50o-I4~nbaEirCXA&n^#K|n;^3A_Ga%nw^f-WLZO0mk{UtI$X z;Z4ZM?*XS|WQu=oUVZ>JKRX_;=9AW$34{k*y;EIP5;#pXEQN#EZm3c65puLYz^OrtC@Srjx z^2SA;5qUWc<)Nn{C|WX&U_<4V?c>8Z9A*8mP*G{@XO1RiBiK-Rb%qqZc}*&(DWGEC z$;VokO+VPUfE!WMYh%0PW>!5-#NF*Ra-Jy`ST>3YH*j~0LVP(-nB#JIl(9to;E-}c z7pbxD;#Ot5^UFe`XD|SgAY~M<)7(TXp9fA;T>0C2`@tmeB-p*`mHLVzGLN_)C!t=c zlOzDaZ_O!Dq^wnA_E0Rfc?fgd#NXtCPzT$4`;7YV&$WJ8WAhw}c}XD8qZO&gJ*O8E zOYb<#QpsACt`Mm2z0ZB!0m(96EXrM<(Mz!1H4({UuIt|SVu5TaqM&ilAil8=a*ZQ( zRD*hgqGA9QDCzUg5}Lb6Qrc)R38KqPqHDQK6E9hYU?)WJ2rIJ-B+#XWW zn>J>LLk*=2KPhKH`pR(19?+`CClcXFAkjE|6h}Ob!#I(y9yQg8G;HYs z9wJ*{DG+_O?X5GrwTd0%8JArx1`G03bT6@{+MZE64^|ZpAUeo!%)xaFQf{swE=jpV z0#_bThhXs-@(gPq8!r$wwIJt+d8|)+7tLSlq*)SQN%jY#xN(X(V&w%yqJ4?#!6%-` ze&|`^a*_V}vfVKJ9FZ*X;?xoBpBzzJ2X&UDk4$cz@HD2~IK>>XK1oHa=#HNOdy(fz z(^4$f6bLQJv?hfj_AVXn+A6vc{5dl3a;s_}4~!{VR78dS&~wC9OF*PA8M6iJ!CfFL zh~0GTuGs~OO2`Fb>cn2LmT?+*t+0AYm^Otq!vd)ZE5_#N8Lb7pKv;2lD7vf$>>N?r zsCr2w-QRxr%PNDOPyp(8?%Mr%&oB^DdV3=o!QgEnfv9s=Ee=7TZX!Xb zJ(kMFKr1)q_F{qPISJ#)!ys=Wfv97TQF3Emf|BxGIEV;8$>}Xapl<>}h;NL%$1z62 zw~#>8vG2=%TFRqqz_OKdqobHGIeE%ZB-A)?+7C414>&caaf+AmJ7^FIuL+^AHMB*uel0HC)keFt8Z{-;bBH&K$_Z*`kXaa$VnOehuQ4zGE zAmpiB0r}45zBKX&Mo#Plgb2x4mf&T84jM>8BItRxENBzd3FP^f@X4iPl{n4x5?J(kmX}K4_=168Ktn$w~bGk_5W@J2+uC9b*O}jS$ccNfyIF^mED1dC<3&rDDY7r@W7d*^fJ1uSvi#FIEd6W4~FE2{2IxxBGs1w~jF zda<}T@z~sx?B^kmNXxQh^d?_u&aH9O$Hwizb1muxf*4SQFYgyc<8R>&%W#<(T;7;E zG7BR;yLiPTr+l{DKp^t4sj6DUu(1KOtMA5LZrB(c$eJCGDEHJU3Me9;;f6&7Hfr!2i&bS|jMg5>+N8 ziD;5A^7S*?;|)H3S#IzQbhyDgEy`vHDtZEWxGv=@lt#7M1fFTn2->tEPi+;3jDAFb zlt)3pkkLg0NN8jQ3=KVS01>@)1BRSY;2vS}UPX2Bpv+qGp8J7pUx)is8E&`)bRetI zB6AuXy#_v5dQ6?hh1j<%I;Y4=TA&~q>OxCo6-`W#h~mmgWECyYQ5kY>6_n?yfRIdA zhplr*lV**9!^j{j%S?iaaZ=?PZnGm2vTTOxVl(X&}(e#B2HRUGTv3w1+y$CWIh!6o;tRpR+P$5Fftbv&nO~`2pkXpM42hMD*+V;V*+O!^Ah<#O6 zgR7(k3X<{2qIfErm>?02%z>w(1v)C@x3sxzbTrpJf8+g7P?DTm;h6;*ePaPKYik&`f;Q4Z})8`2pDoE{NMNXO; zbCw9~1j)vZcYDMLm`5h7FBu3@aj9@cV^ZO+vE~VG(vG49L1V7bMv2u8s_JYksK>e! z1Cq4eNOkw2SHT+V=WxwSoAPaLpm>02rd*W;Hc;EoGgEBQbJ;-c0MV%2JNb>hP8JOn z3Q`=3fOWR4(bpK-hdFIinbQyzGynngD(fJ!KkxwLMNU9uZvc+c7mfpb2SbTlcmm2V zIK%2;WQd`~YiU@GA*G245|PDv(&QNzB&NtJ$Vt)y9hH%>kLjjf27+We6|Sy%$^cG9 zvBt1r_LRTHf;mUD^PI z9JZyN|Bx{kQMUkkh;q>&53R>nzYQl?@v}?lD~mi7jGg0gQIIk^YbE~>#*no;kI}eT z`oBi>17$~&Ye?=AOkGk96Woap&|8IcMBd?<<0$3?Y^7}iN>V?pTSJBu8@%#DkQYZM6O{S(jPC#*qwkvZ?;o&27&oUN%bn>f3l)t7&* zFlSVpLoguytq4o2Q9K3r_`bXf8ADwlB~NcUvQ(bXA!X@kImU@=V=!#flnseznn$Gb(#WMK4|s&DU00BVZr_c?YUV;pl(P9DkI6MY_eYtrR}`%tAl{nNp7yo9`q z_=Sw7b>krms}FyE`E&7f7&s)KSZ zyhsd2WPL~ydINDhM=H}%Jb3w4J(%Rn-WiJ`mZlCBW37b9!@1^A_rVAW6@%x0h#=+4`N_}tiLehV$e}b3!fD$;j2wX7zP4wr}Oyv}jIldPy zdMX{x6I>zr;9)MWkoMSJUeFX}qUy}x`~v`01F3!#J(2oHQ4ePmn(jm-Q^0~|Z3-9E z7rRjNOZiK)CZ3QYd)Q!R+4)pN|0sH(FFzDL(Et5E|JTp-zcDt!mEPh+bd{t9*^Jh% z6|LZxUB#DQ%jd%EY^vEyBXXa?7?pFFFTei%`(H#oAXmixnuj4trm}XLyXAd@aJDK+ zNm+dzW}4F(8jS|)45<#=R6nQ@sL{9UMJjS}65m;p@iFba!ON^(e;=kOHaW{h9w3TG z@G1acK%c*p6Hj&G3yAU^4b!D+stpI7I0gpY#t- zT0;QYh_FO0ADM=Jj#28*MeQHH{9Qu)5~sYUx6%fXr?QC3T;8*lLP?u(N?Q3yRQu}D zD4)_-EY9In3{at*&+%I3suWX?$GFQ|xa@#K+oV>UlC6C7@?+>S<>MVA%~EJt5FrbKe@n<43uKP| zm55N%Z6dm|ZotAZqCDas4hsnnfP9Y#1?@k@cVUzGK1Nf|2hGX!?MMaP|0ti0X7V|G z*|q*$c{y_;dNots*>_{v}wtB~agN`4&1 zEG}BfKJTNIUvK!TYc4Nr87Amlj|E}ry#VXwzt>NvYQ^_B@nH6-kY)EfXq(YN6ad1#m6bm*|~N+^2`iebUDEJw;B8? z`wn+c8WGDZ2us^e96+iz;x-n`h)08M(=&$LL>Au5xty8&t+-XU ztnmoKe-(|FEO#POBq#FcDR_!qzHRwX4 z)nHs$nVKj=%`zibHFT!l)7(}2`=)@};p$0t^=t^TC__;;nMv{Zkf%Z zdBmvZ!o~OAcD{}`=lzaE8hWuVG0gUY+13KvZ{NAhIE@F0gCKTEYV1dY(VOVnkN~j4 zw+vZD{h|2!owAG#E)gz&Zm$n6)_aq@bX(hnl8sP(J33QIH$Z^QbE-;1x6#)AI|tW@W* zd%}v9kE{r3BV?(s@#I?>_ZRg?puWl8`>P+s@6faqzkI9Xlrq&G%>Pk7{H~5W z1lh`1wWZ&6)3;ew45a?c_ww7zKhy69%RoO$UqR^vHPX-%${X=-2EHacc z`rdp$XeVz7w8xY0p59PDMsvMQ&#UVx;r%#6RY7mbZ$$fG_jF8OBS4Aw??3*Mb`a#h z-wg5J7)l-Jf%Ia${O|wy|GxZ^C!&ML)GDoCl9`(wP{bny{n$W1D)c`ypX5RIQcQ@y$RmjopHSA%lQGebk)a~p6R^Se?~3o= zDg^n8TDyWCIQJy}jnQnNOjf^^?ln=9ic+pJm3rL3Nm#&0Iv}?Ev06Y_gzV@%X?fC-jszdyEEx_;h|n_Pfacm z;B`4D`?fM~m399ac@vqpK8EPANoUf8V@31VR^;zG&2DpGJ32t%qH27aj42;IHdM+P z7oLXX3ff1R#$lY?Zz963h*WF-_2=(@`t`99?z=|NnZOm|X_sRL%aUP}XH?G@!;ajyhA*+F;^?%jchdrTmxA7sVhNh9a*hmh`)@sI}Ef!<#TjvjsUdY>=@IFyKQAxObQ!gC1u#UXWpdjHC{p8OLtqDMh1V z6az~cYP1*AXj|&AmBfFk9(>oR3@2*IUp}*&>^d|n%S701GoO0QrsUfmAf@MUgvYNt z(LCJLVg>m@En5&{b>{n=^42j~ZC-Or-oyD~T@0TX48dMnYyzgFoH3V;_|vl%`N{W% z<@D$thRJ&v8y#9&Z9A5y=|<&^^W=?*3hnvMio`HoYFMUw@oX2*SV1k1Kazio<@mDo zubwo>5Bt;pNcTtEIxaP~UYEX$$Q!>BH(-wFvju_lJ_cUb;KSwMo`~hK{TapJa#23} zJ?EKXSC5*E$Af(lDSS9v6oIkIvh>plMVpoaVm(>7ESV^DXb_FE07) z<1U(9xrwoy7IWw)4SHUoYn{%Pbu@jlnCr1{aQD2JaFov6fbP7fXVdVAzelOh5~lv? zoN$~No`0t2gJ{DZ%)p1Lve9>+ZmTa&@m&T!De;2%!=y{P_(XQB2+Udu}FwXFS|>L=c6 zP4@eXVfuT|JYFzrLbuwCxGO#Jav6{49th01bg|SMjmi+Ln<#rxzAvY}1d5@DYj(RFm>u{#FYwXz>xpiSlB&u;SycX45` zLc9|*E-pP-Hg46%BVTSqS4~TQz1E~Qx23i(M`9l-Wk(D@61j^e^Mec(J6BKNd36UM19xvJkPc< z>RNdovUK1+hW+CI>RI*U5WTrQ{e>J3m!sibY!AVDRhUJqmdZ9;9`c*|j0Y@n0(Pq% zTk7|g2LI$z?@ZX(;Ap%~sm?zrPD*+geU;Qnvn0Nf?2m30LyXhW^-YhRebm01=sD$) zN2G^G9YN9f-ek8=V)RK}!#kjtuq$E#qY?ZM?xN3A+IvLC%J&=IsQD^Gnd&G^9}0&8?l5uij!9Q`QGVh znW8(sJnuAIH`UCnsFg(9B>sKnA@AfPb8j5?-hooe8GcPV);m4`0A}+=%ny?FFgp zosawV)A09zsH++ubK#x*)8sQZ;3LG`y9d`gSjTVl7Uy0b?k~25Gfb21ove!=Uxt{2 z{pGSXDZB13ma#Eu(fJd^=H|qr@43I6P8;x2lU~FaWPf5FZY9XXfSY*4d0KzhW;C~9 zB&Q?Yv*XEPr~~{)K^~YJ=@#Oq7W0d>c;24c2gACcxE>mBLRHPx?05gDf!P&Av zO1^6jYhP)er7RQuoggR=|ID*i&(OCBGf!um95Cj|6nZ2bsR92(uG`Ax@sWQt$GHd7 z@Q%d097XR6qi@ASxf`%N=zmAOPM$xQXG+c%6Bc4Pwd9##k!W~Hkd}*X9#4i?%`ag+ zd-kGq?d@0@JM0bO+SYlMYeD?rkhB^DHQfu%Uz*H^{hoLSgSx(t zGw_lZ_b}%gg--K%lk#YaUX>s=ZFBL+Jf<=Q{UBB?UBn!pggoG^$HL>;^~DI?KU0kL zvLuymAyPIr^#XsV^?YER^5JwgT>EtKFDNTC>Xod`YlDX9%x&oo`spmwVzz18dl$nm zCP!c-sZMB;Lo=60q_}R+Vbw9ra!1uYPFpfAO!jE4i(w+j2uY?qCe7ua^|J=+WpAAK zC&S(^_ii;7oewNEKU_@1JR9P@(vhySys$*wP146#|F}y%p6v(PfqfPs(Z9(?y{cLZ4j9{iQGLs?N8^Eckq$i?&`S3TBi2#j`{mZ{T<#7{M8iVB-w357@rH_z zZvVO=Tf@G8Yhwx?=G*mA=PCJl@9ylo~UUUHpS_P`y|aoKm0z_ z4?h~-=|D}wPA8^$IRr^RBPY0%fmIMkxq{qezG$Hq&#e9?k3(-{wMX`A{xU>rZbs@K z4+ln$qTpe-i|W3qX7eC#o2t)z{67Eq={W*z9E@9nExCHMF)cNN_~hnUSx znNNHBV7nK_T#N?dvHx0BRqE8Yd;H#7nRc{n*E09SJh8))g{SKJdLrmPi(@<<>x&-0 zIOR_b)-xrS^IjAY>aN^)6#Kp$*U1Zv1D5s7Dz>=H4Y>}Qx$M1Z+J}?qfO~iw(bS(7 ziN_9x=ZmHUw+!A$62_4ju=dWL9Bt-a+@nDc6vK^_zLWbSXR202JsKc1w;&`Zxfs#) z4My?y@Tx5@-3`etGIL9^gLX3MA@j_mhmwq{@TooeyEtT5(qf_qx!QwRCmV&SI3OT( z$`1`D*YK%fdmcNYty7ZsSyWe8$Wy|57APIvy?gMfO$~Cu=|z;^%0+M88mviZZbEq3 zADFjiqf4;B348l=1Iz0)TA)=Q256SJ22Z8BMO8Rl*@g0SGo27)4Ao(Tf2|GEwBw^Z+Vx+ z9%p$%mMG0$oW$4oElb&(bBzP_Hd6jw`pf#i9ZpTHe{BF?G`TqRVa{C|SJ1vc5g}8qppC>&TdIyQh<}{%DxTMQ6g6 zVed_g)ZC1en4WPwoe9eEN~-Vk$ctW_%QI>Ext<(%!kPcCGCLD?!L`qjMfUzHn5n8&leJ;kT<#oQl>=scc~k3xAQ$k`n#xv_b)gq# zpM9GhaSL~Gk#=>~W=Ld-3!L`T0U74k$smTiKONcLqu2}T<(*Vh=O2mQicQX^^Lc+c zY1qK`qbE;jX}x_y#L&w-9?vMAxN0tsyoo^9fyulUr@0ZQOI+i$x2>ab!0P#`PNzB! z@wrIT<3;2m<{R!wM;kFcrfLc>zgocIs|6gtM1bvimV0z9W%f>GKbqaMo8$9D zdPW&Z=xyiuX}4N$@|$15@4IK)@~w*pv%lw-1_{F~(d#37_hebi*T}Y==jHb%i|FfK=%o;<_ArL2a*{rS=|*nxWFe7Lw5gu^yQ&)8^j znVPxOAr`iEFwXvZ&*eM$=vzx|nU--rGUMbkOT(hfgo)vCLe-(mad|c1^R6)K;8|io zDv#PR^kh|!`z1^(zly}&d&Kl@pRM!+$Qy@3o{Fb#(Bq`Cj%RA0%){|ul69U9KgsDW z8z(Te5HQ0tPVBNbkE@s5D{yP4u7fE)JML%JemA1S=$iIfbZ{`XFJvMUMns(p%8Ns{ zqHFNkJO5{$T+znZrePkDN5jwO_fIys*2y>bqVylFvw@xns@E>FYm?BD%?6Q|23(MK zUFBkcFm>jIWqHQI^pDivBTuFYI~zmBCrT=J`kpQntMmL82YJ{|PVeYlPDeVrqiop` zi8Lv$yb7lL8!Zv$SBbFgqitA6QH7XGoGbxRS47qRj>f*``lqhjqs3}&$4aQp_D9bL zE?;e|UumFR4rh-ZJ<~wCUE=vVjmJ{IAy)HCSPx&qdi)aB({`+v139^_TuZZdm4u%M z@))$_^LZ(Tbzw(?(KGSS7(N0qbP_!)^>4SuAol5KioqcDEz~_3YQ@^{6uU*d%lUj^ zfgr^R!?u!6eEoTxg31o zSi7n{PrH-a=Yw4BO4vn=XcpgfA`ey(gG;$P>5Zqn96xla-1FHYC&Two>o;GgSuDl} zX(ooxt{A$%%Dn-sc{s%RXw%Dui6VJixq^h|u3FmEG3ZE4YMPe=J2MTw32AE_Igu>x zZk!C{{UbdwvZp39QO-uV5gwI)9FM7?O3I&3bw8hu2bWM{A5dBuPo7Pu=D&Fu3hQMP zQ#^ZG6LdbEEYH0Vd94|Uw|R_614gbdWfnOwa@KRbk1x+`%vzKe7lD4-jlx9z4(wt_wsKG?nRW^HS;fF{8c>o=6ZH5=L{>is2-#!>xcj*mrnj3M_ z3#FcI?`?L#ntN_n+P~#daXj+VSg6HpZp%!_;dt^~w*2;!M=zkP6o=3MJX3G)o5q=4 za8?(z3T$VICtl!qH12eFx1QM0jK5l50B4|$KP0TRp&oE zXKnjLjwUggP2D}-ZBxb7lF}XCN=)la#4GiPKl1|H0asCw&*9Ig}Ys1tv zlut*y+Qc~d#5|Zj%9#Y^uF2+>$$D=>iQlc%a~7J#%z5V^LbNd_hMh@#dNKi>wwC>f%u=fqCugp-gCp<-o01QrhDrws;{GU zz-Vp+;j^Q;e$*gkdwTKq@`MZn@kUYEc@3E8Rf?f_b-#C~;$r4WwCH>YjYXxWyJyvNJQKJTrT!Z$B_F@QT9FTR#bfdi6}`)B_eX4|)n%gH2X z${3E{ZscIG7MZD??0k?%%GgN9*w-U+b!M3#iB0t?*>xVAJ!M;eB7Qs3FsaZIR?zx-uKxoK6c)-=*e8ekoF)YfCmb)iMigWYw zYczBla*NB<%|%>#IvHnTEncN1ng+Jxkz4@H#+SU^7HFnHi7;fr`@jLNfase!!zuySV97Prs(xEx(_pq}O7 zTy9u;7sRMPPg4HIoT1y1FQ=0zq6cPqV+)Jst>jc+%B7qwDstSD)8S~I&+8X3r?H>a zwQcpZ`$C(X=2o1ZW!mh>#0y<-=QQ2fu*m*MiDev-@3XJnL%r_wi|vskGrhfaqGA!TVTjb+h?E$olP9kOwEYXlw6|*V#XD?He?K?u#JDiSJBjh| z{)BmqY1zR^%hODsyNtx3#5SzG?b0*e-G^TPSq4NHc(L~T%Yh;9@JpFII?Ra^j}0s* zCFaTYPfVup8Rh8iu@k#R&Y_|+e!kc!8!~zh^>u20!w{|cHMEEAXivt8Qnt7DF6MC~ zMW|x4S?9V2ax#7MY%uF$?6oesnm#;=oKDt-`KSGbnz|;s)d@Fws5Xw5w~IcF*;^{i zuTyb0$=UF%6T_cH0$i}v;y1sT|L_(3?p{{wrpF8{*JiO7>e;hNtmFQ4q2yw?HqP3& z(uLYy_|~E^_0gP7%RKJwqkUL;qOdFXOm+Gc=hJ!biPqgjSt2UN1(NXhJkEIE1=51f zBg``l=AX4pXpWxBB)T!?=|H^m<;bw_84L3u<9+v~i&oD+$@A7cdc>JVEe;bCZO_!+ zePUlMk!=VS1HtvvHGlidKO9%YxP>Hn9p^GNb6wcc@{jB=nxw#@_gh+bwdrRTrcw8^r_NZit%uMV;#Ky`Fud(%Wtr*2O-e49e9p_-_398oxQOaS+Dh zm-uan&)k5IosZ1lAN`|sz+D3`>su;DHEM)S`OoP=z9ioc$kNU}XiEA9$$@k^8r$kEwmBcQOq)UN)2wFK5?Q5xA zkyp;f{@|MCJokbqhHtVdD94*}hZE)Z&(c@pkR&0^5rQi^<%fgEF6Pz^XPqE?WFQnk1(^J)RGyOTD$1DKk-f3>-Bs z6U+6F7t3=^goEiCdpaVYU-P7>AfNYTyVl-iC(rON#KPFo{?WC|Vz$lB7A1Rso4a&b z;>D?R6`}|q5@PXn@=7~;@YjQE+2SgXDpzbk)D@%@r*kb0eOEMxG0?y~BBL9hYB`qgtwJA-Nb} z`Ee3f%Nm-$bzfSO+}xU+$h};~!7!6D1Sbra&G&iDj>E&{=FU)o^_iHrwXsC(8ckZ>ji2mVxlHD9?Qrz=`;{1*9 zsQUX@rQFQM)G%DaeEYB`#*!&KWmh?N5+hCSsiwR$c}ET3^L)cQoLstKr))LhT6Pt7 zfAG8SB+c28>`P`(V0%)CcpUw{6}V7Y|+?yXNu%S7bKB9 z+U1R@!*ui3mEQDZ*oVu-JeoH0jme^)`2RKcB}=l~SeAQzMVNJOBQnBcUaBRfD!Wpd zYQ3zi+R@-7+=DOZq>H4-pjG`t_ZQZeG)5920TRQ7zcQI#I4^*6&jCRY#Mqat)}D9x zXkIsG^olXQLHFqS8uUoxSKQWk-b>J%+RVUyqAZAp{w@fu`U(l{skh*3+2 z^vQ=mU0}+4T8uNZH8z`X&*tGC-Qi+)^9b?9uAXo7moLA3|MM5kS8=zwM>kuT>lqnk%zpzBDS74uTIMvC62WozLdK8K2w#W_<4c zJMekL&j>v|q30dI;G}5tPDiMw&q;B8P3Px5)8n2J)C#CThceH|pLTZ4RB`ikefMy) z+dcsA$T96TH0QVGe)DbJ-+l}CHyGuQH`mvITf7fPDbaq_sZC!sTHHO};D`MAZVNJI zt-pxnjL=~#@!5?1;{JJq@|&CI=lnkDjQxSi!kqo$9$gnV_jt3ob9=}F?IWt`S?HN! zQr+&*ZSi#faQ}P*y2s_hu9n5aEhew!$EO|0Q20wp0^9uMlg%s1#!JQH_WI%GCg0u` z&k&g{F&ai8U=$!oSl`Q)Y+yW;VF zlRs=89?!7aXK72N0zW-o=l3_)yZixk2-nh=rn=tU&>fE+?hBANh`9n6%@>>c9r zy9b1}B)?Wmg??a&mUI)I^wuxw=KlHa4n1B!J%ZPV?B422%|v}@SZ}5g*?dzXkI#5_ z_q^K_K`kr37wGAreoGEE)lI%D?l#-q24unqpWyL=I<|1)9hcn>mlbI}R``YP)i!Je zL!;o%Mr89XiQL{k+&t$yv;}z|PPaN@Z)3wwNBH(}bN_IScIY1X7`nHnpSG;%#G>8x z=K3DR9$rj}Q=QW|*)VI`FeeY+;=65ecTG>e0q;(p?E0m3C|h>~!*_T2<1@M|o^S6# zw$55R&Pa2(Ig7(>agCnHLv?o>-7@G%T3l?1-r_^^ zrmd31_q*F0c8WDf^G%zdeeCe5XiRG}F=@8n-t2C#pKsl~YF~U7oDC4pEfCHf5YBxJ z&i(yv`%KytPn*c&=w9AXZ&=QudDx`!@VI@v+2Usma$vNUQ_CF(`0Abx$7k~Ty?YElVv*EKuP5uZ@>RN*S z?R5C{^FjeD=-&U`1!?|Wu)QQMQQx=a_C)R)D!!teR?8Qx4?V{;|MPtNNGii^UchVy zM!VTM(mLAG!qC5DyRz-*COWha{c5}G*e9-5s8j}Qs^Tv#fqnGL1r;iR-E;nYeRG@dP?2;bU@;ejADjD0^)Afc8Tn}+a15|$ za;qBsbB}ts_+B-2>@D&y_|%s4-0IHaQ|QX(+u-D`AP$h)1|kGjUR$7> zs&NAr5e2NcvVqPfeK_FHCyEGHTEL=5m$Ej@hf&e#Bp53)|L}7`<}n9k@x^EZ9xH7J z>`4tz;75ziLF-Tin^qP;+SO=7dUGs&BBDfmTPk_v3uhhh)XN9>WX(vnJ;^%Mc)rTT z5A64MtG!mMptCNniytq)3>J$6w4RZ4#F@j1zS@iLDgQFR_*s5(nNrD>ow6%u-On_I z^vcZ_zXlfoefJ(xfgl4gxkB6a^-9i1(_xNf^DVL z(9yQ)$kaK~f^@=hAF+`kUsKd93`xG4eOdJvq+&-y-t{G7fn+ z#Er{3!+H`n|BrkZcb^A(GR`Mh&N%qu!YzT|O4&x^ioHuIb>>XdItv9CTXrLUo>Hjh zNE4}rBoH{}UPivT+q5Dl&L%=9q2OZKLF8S}rv5kMbeWWrEiN_TmN?+@hv`9QE7fL? zns{@Rl-kc&;O$;*c>wwdjGH0-%6|QH*y4h9XQQHEZ$vQh>#n$BTWD&b9UFT(qg?KA zSu~H6Qu+(3cB0vszn@9H=F^@5TC_9e!8gbPw;VWA2Cz}Yv*!Wx>XHpCV#=JL!$oJ3 zPhHtH)6mpH&syb;4wprzukwazvS+XM4W8|pVThbASH7Ml2fo_Weod%MOa?iNKJLq^ zk_V^K%J;V%Cdr&Z@x-H9_ass>^%X|#IW$v07Ns;OsXks8oJ*xLh3H}-LNdj?%P&P& ztdCQR{#m?q*>L^hS@mwVi21?tfmBuZCil`=G?P*!5jsCMvYmpnvCrTYbS=jsBJuFPs>XiA}H`tX&m z>_St1_)>hj#}dvlo%whsy^2S>Y$Z}k_1!5ytM%A*oS(PSz`<7iMm+78E>*&oQcZ`rw@px?Nc{Gpytg6Q-sdU%qaE>RJx-qMyA<4vT zPj&tRJIv_h%_4P-kW8`IS?(NF)ZojaRt6{6n&t0V(-!Bs$up-b>W6sgvf-nrn}OkR zBCP@z#z-Za?U{tn)znbDnI$U`&Y<{ga~ftoSJkJT#~>V?sV`IUwx~O0kEY_{+;@F# zLeDH(x;&s;VVviVGfy_XEHIVI?tH^d-QsXlmw33TEB`phz3k-4(HEwWWa7>@4!k@B zoO<+^_UfNSWS-ju=6OnSXKPT6&g??dHm3UKoPF(zH?v?R!Wk52d4;;}yt=v1RrNet z!aO+D=e_TvKdbgJ%2{-u^X^Z*I^;p(62LbZ!>n`k9C@Jnv#3|2q|$vllJ?Cx1an7b zr93Q7BHN#A+oC$B`+?JC(ubxlI=(EH?bLOG`(SW!CRbq3L%vJ1X|jj9m~ELabQ_o$ zw(v6(;GPZRa3@^S%nYXLfIc$E9iXDnxsLG~QUEQp>|nMdCjZoQ=fyse8k4I*nkf)M zr`#1wJAYFiF5|J^RuZpV{oD>9TswiRsBCKvlr$V+p0O7 zI&0)&Awql}=tl_!Ee;BI!u7p+Ec;paMgFiRGs{=@N+5T3z!Uk6b)$+7a8A(mW09IswG!koFy<`XNwLU{uF<*-!`Di zrr=VGxQu5{8iOT?&AvzJ4p(JPnuty3wkjd+qY8>}qxJL~;sql7!*l2l8)Al8>t zvG)S1aCboU73xrNY));4`tArSY|WWyTdSrd3iB7brFXL{{U(S-`x4z0iY19Wac27& z?%wE$OB=6~2iifE}B`AR=cGlBJT2l9Wz0-Vsc=T16832)FVc1*VN>`w=J&t znR|(g<%&y9^h(_LY)Dl$XN3~9G_t{<@m=!)`wp0)D<8Ayk?)hR>_(29b=20;IG zz_c^(Jhl3&Hz9H@TA1$P5H~hGXS~ixc{o<_ujmzguFV`4DX8p7sbQ9LX(u0ZIYSVHE&>Bx+9{;y*TsEgIbS^*7mWL=Coahx1}2gY zvf(pGH2IMeK<6P~^5iFYQf)IQTKgPK6`!mJKb6qM1?0TJxxa)`rV^m3lMgO*{7iW> zW*+1u=7c>%F{%L`DPG3_^bGnIc~vy^rIt+ zQzOIe064T{q#;?YH~s3LWxm_YhMhq}qXujQf4d>2v! zd6cv8m*Rklr}IoJOp4F=8Iw%5&6n&eFoyz4t#{@A#7~CK4&-C4N~3b-Mhct-fN9&; zhu4fx*GzZ+W=_PLZE6Nnb)Kiu_5>z;>W%`aFG;P2`X1#k2fDT@nZ`wj;-$V61x$JB zGCD7wNlK>Ji{(u33%fVjkgM35zWyttzx_ zM00hRI4NR~U$Cz`Z_mRBqeEP&#aoF5 zmIY@}Y!sME&pB`9@-0kr>snl#>H^V|GuasJcnzk|?n0R`G#cadbVrgkEvkkevd`oV zvU_gPix@~Ygak=zXuh zT3h@gh8;by36`OPDEoB=Lkv^CT?i~!4O88fB!aIp-af9K!C8K4X=hKitD5&L$sCSw z$D#+^FHvOPq|So#OkP=aNKfUApCRSyW=Woi5~(u5#GJEE$5EX%hr_8Zb1}Y^{GoL7 zHw?}=#Es2>b5=>t)lmAX$H4>mW9#nIITwH)Xw^~Tr+dwnL3)Dc9(wZKnWo&SZc6eY z(01hW4v_kQDfjl0*O6p-(?4rXcvfQ#=zvz|ebc59W5gG7jA2OxW%cQBOfHLi zSrW%%#7gYQuL7~HbUy$!sjxzXYKmmNubM5YvKAvYWSFG1*?owS4QqI;xVAYRlfNjI zyvUuU>@h$jrfo*seW>yhG@Lnb!sfHZfhr5SDa}!b&4&`oGWk%InZiNDjYmr+RA+Re z$}>iY)c1ngLsQc|858I-fXn{uSwN(Yw8hU}@Ddccvd&vf=P{BuNZM*}&6AY494lJx zu4xZqcZ(zd7z1N9i32v%=$V5HE%N|) z9U1;OCe=b6SoP4rNKJj)9A8L_yeM}9ri0UC0GHdk&H{jRkXCE+!g1(H03b)ZxVq7` z8KVHHtneGI65J?`tLxylNaAk|5s&IuILZ={la*ING+$6zi$CP$%FZ+C*a;G`Y^}g@ z3;J0=7=Kbmr+F%D$IeLJwatCnhy zq2K*}LT!RFGQ$;p4^wjtck%^CcZ7vl0%n37jshAiab!zW?<;@`WP&GYipwC5bkAb8 zBgOBhvPavB?m!p6OR^1CG!S)T6mM}*87OHLQ#%eVo-UoO)Z_i9JOG3|$sNfKz7Y)+ z|3xSA?7@X7=@e$$U3V6h)RBWy8ZDvP;#_JAH?;+nS~}bAh{->753mLGD*i;OD&X-6H#4w(l7xN;;&~ZEtLzgXgpLMpYPGzC| z%wZ&1;pIWWgCS0XIy16t-%y>^`2B!tFcmfjqM4|gB9+Kr*{Lt9j`j}^r;@{|S#UCR zLl2fXL9e3!<5hOh$e=R*!^p{rm@H-pi4=z0y4=5z`(Bl`ewA&@j_(8mr_jZlkZKN$ zQ(K`euIPp}N3`dslCp=lI-(>u)Xkkzi%deG#m+2eQoZO`P90o`5}6Op=?#(MCjO?; zZ1*8&;>`kJRSgwgI#HtwUDIfflyrKf8>`TNM@)dOy;TQ7&deSs=g<@<(yXR0r~~?t z{W6L*MM3m&s`U<;qk!f>jmkGVp!5y$!NIi{$ykpVmnXlUcmr(22Gcx;7K38LH4OFP z5y^xFT*1u4aF~T|4%BmO=ycWYio)e5G5hw|lywhK02jc;n|>Y;tD`5)`kb9VFj@hX znBd2wuV+D#YPRLr{WqA|nHhkZ7M#fKXsr=a=u0haNmZ;MI%OKy9fJ8(Frf~oG7Oe@ z$_ej)J^vHrK09S}{uIKgu{co9IGqQ^Yf?%os1yZPITv!U#S>3MdrlgVa)%Fza$7?l zsB2aL`hDBf`|2}M*3aQbA+w-(HSZX`CetznACyc3B6Ykr9ct0nlOcza)VOZ`M9-2_ z+{g^zley-&aFT5v)X8M?&{$b~qF3Su=aLs~!dZ&ur+`jdUKgBb0#lSsj(nyE)5V*n zIO#*lT};)wvICh_8RAo(1!;`{tX@o7KBIPFCU7Jusg5ob|+zH)5wrz%cdhR>(fqB4QL6RH2@No!)a&>mEl$$@^ro3z4;;dzx% zjC7!jkycnSU7!BuC_Z#aN&vX3lwJ0!y;K1@`FtG?9e>k+GoV;7n6FSffSiF4NpXn~!1QGu%CUr42hw zC$?18&~^hBWAZTe5DlhiEnVN>_8skOx^|iV&iDnS8k-Nes(Dt|Xk!8==ianY8*5NdS`W1tf*wa2 zYqu81k!orLMvGu6#K=_)we|$C`fr39h|!A(Ufm0>Di~f2O7dbC5xjb$tApXipd>GL z5zMQp`6AIz+=7r(@x;cyiY1KZ8{e;keA9_q`3;~VF@X->j-x4x@u92}P}Wh0KC4*L zv>hYQHm1uq=`JJj-?3&?@y40igNf4_q^G<`Bqao61#X<7Dh%NqPkq*vf4~qFSn_m4 zeiclO6&=uft|Kbljq&&TP=3&PL7;S}nDVL8t-+L8TMCr0j;M5ULXd3_Vml|uB5Qz^ zv}Xaq+~1n&bO52tk>nn{^mh=c&z$av#P-ar*eN5-5KGXEs_gWtSYv3@KiL*_B^2Wk zP>>xB5j^=mlk_(X%@I%C$C6%0kldGT{8qmT)UdWca2H<=B?r5}Z?yqfA6|pG^}`e^ zQg6`xfI%NH3pm$d35?K^K!JgGG+VlN6W#WmjXDe$(L+8qSaJ05Xo%)a8u+Xb(ygbY zJ$jn}mq2L0HU|&G=Y=6GT2x|1Ryj~w(4O_F_RX=Wnte$p>vT;U-Fs`r9s6u&LBag# zX%9p{2xo~Uc*@IgZhgsj?FS(z!6Z9*9ZHYkiJ z1|XruwA(el&&&opqd$ZMkuN*eMVuWlqM6WLZBhG4H|LA){a=C>z3Ia>4E02`RmzZb zixT8SG&`_pA(Y|Aoz1av&US7r^Z0@PWpL2}uUKFNywY=&*_tpK(t~T5;L9BZH6x4? z+N3(Z;1>Qq49LU5p!^bscZ-Vh)Pa-cu%f4n{RX0oJ86@F@Z9G_jLcQ zt9{jMQ8k{02zEQH4n*hg)C{7gq?NMRl`}(ai$0;{%st4{%cpi2L0vYrfh(H3P(F9` z04Pnv+T(PZrY`3Dqr_;Urwg-(bNH3*2y9;hJV8s?;!Qpeh*MD>4wy6#dQ!PC8$?gw zBDlC4hLCU}bjuCpC%`cZR$TQqNLwUv`m`nDK7$^g?`Q>hwCb4KbrKG6~ZMAlspi@P@Vc5SVsP6(_o*1CIwn z-H5ENj+ll8i^$Ot(O#IWBQ<;cz?%_M^s!B>;gp-Mrw4)33lO>ro}6SmT7qd7WnK1v zKoI0evfGdT4km?*LsJ)B_KMmg1T_^;&aE9S!8G^Hp=m#}mlHc69)hTjDB5S5Eto7l z5t#mL2wzY z0*;F<1``ZZxB@&wHE@nB!g2!V35v0Vn@kWrfp7#YzEjW{-^ju%0>p&U4+zc^NNvvq zX$KiViYAGG3G{gKqHd3tK;D^U zkrkMp%_L6d&2fg?q9HdLRTU~BRjqp)gsw+Tr2PfG;fnnGS{}}zUG=u2U5y%|{iP&< zZT|AfUY^--87a~RD^_o5wjkP{bRQ+Q+gb&-#hDp&+`g@f+p7Vbqn|65te2Pj5!4rijqWH?vvwCCHbo$Ts0NZ`8 zy8Enqdi>P69s`p*a}Jnd#)?nfo?2*4C232YvnA!&j!NE&O5cWBY(XVzKb@uZwAOY? z*K#^ryUC^1l(x;3w8dm=FTuB#&e>LSY$+vgC!M#IqkHsYKha|Pd99&Fvo!Oyeq^tQX9&|y#HJ*e`QrpeJbuDnUT&fnaq3sLW z%B@caoUn~xf*-~PF?9Q2#7y^$Sba=o0Y4`raFQp*1V2>-_q5Qt|LV8ORogUlw!qX5 zDA3iV*et>v2(J5_g3U^yCBE%8$0PqYqZ@3hU!;KQmfVPayNzLx@>13LRv)S`(6@l@ zF-2*#c+eF^6~d~lUo%vr>XYp@UG^Qh`Fr*wQpW9@ zYTLH_0OfCYBUflCCSXfkE!d62918 zcWQ>vlGj1{yqYVTJ$*MMr@_2M{yV_b7ikz^k%&8P^;c0wL73 zh(EyRc=Q&1$&K9>qJZed1i8&IZG$yA+dwu+#cB03?E7X8hS2y<%q7KRU02nqiyskk zaUm8W8A5_YHSd^pEV9~a+5;{*IJuf*e~;+;13;58Va$K5F=0kImA6eM#g0xd#xn}bx@KZhQ-R)EOLTyoLD1h|9)zE`nd>l;)oHGQ`T)^gu9{AqTdTgT+?g-*jRJ{S+5l*fk7&o z7FGOTm@J>-3C5!#ge5<2o<0+#l<{x0s~6Og#vGm2b*dpwfpGF6cQ2fnp)~QmJW*X_ zVxLmFjeSYJ+u}DA^>wFiE^3YuCc4gA0@oaB+)Xc|6)s@Q(_~d!h=AsZTO>b2X$Rl2 z>DZS>BzY_D&{HT4@vMuo``VPG&N#8t*P+SJ;pH}YQ+vD=eN%W~AyaVEKJ*YG(%gng z?gM4|wyaV6DGMkVVI5X0^B{bSawkBbDUZmql*?-%;_5>KPhLegJ*FTccBiV!C@3`l z0a#2Cl$Ri^)dwA#xQacmRzT=?pN9jc=lNvm3ctyR*xbwz0wOEPbD$J|fKq>K8+uOI zRgUQXZfu8ZZULih^&lr|Ps~7inzpG?Ix!l&dkVdn`$v6$| z#|i_?39ti?ZBu>?LA%9o^xY54S)j^MZgvsE_E;f;mlZ8~r`_5_`j($0M5jRD!xk3> z-MX}IGSXa+)Zpc7xD^X=!UfY=I=`}CKS@jwaRyxbvOjB*I9>S*&R?_P8cf#&SKJ`o z7{{Cr$*~JC&I95&=Ep9BcsA8%-5f|k>_}d;%>mJMr=vBN^a{Mp?|De9nEuq$9Vt=> znL>+q!^9dQpv8GfUM}p2(}q-kbUuJ$i`1}!&Se+^VwI#(Fk>FgcsM8U73d5?6Xduf zX3SxO3Nkp=DNz8Luh|ae@zZRkaAJ2(aTP?c%wO{Av}Hf&jI%A^>ChF=zu(Ira#wozM9a0=^+3m6JAW60A;!y}*hyf^_ z9^MFFNT=xIL6N}4QZuK~pqVG9y`)r=wEWr2U83@Ks`<=GTqtw?6W+abf77J}&^L9R zL6SS{>~>Lqk{iD*i?ZdTqNqC4&0p{-&ik}SfK{)wJCdnYd<;w6qybhHO=f8GLN@~LHlDZ<&*pQ%7BxL(Z*xB_8# z_iS-h(Sy@=O>daD!C9S&7TwyI&Q|G9hbRTEw!1VKN%8GRTmzJ-l z#1*&Mnd$2+MFoD2YTHm*C##~*STWnHcetP!{4HYllJ=Fd>US*@61;4Nij8gXo3es6c#_je zX9183^}a&2wV6n$-Jz-TQVl9-JCn+NrER|_n7!CXLe?i zp5i@c_He%E!0@J9h;Sz5=Pdj7m)&~2pzJ6QWyNRB%ZOq-1(@kQEAYzM4AY=kHSBOK zT$DZ7*WeaK`Kfq#;gNILfNT!kOFk%VR?ySizYubLQ>rXql3wm!sy#U-c1#;R>f6?`o1L1 zp#ddU%L24m=|43`DrgSmqaKbmgA*?a`ZJQ9Xpkky-YK%&p2QLbxC$b8L@nJ=4!7)gT}SUeVL%i#`9%;*w!GogqUxooFxxzo zeh!tcS3ATg>3*xgA#Ky{CLguJ)TzP5_C!Qex&AASd*ZoV2Nh`7vW6;AT z$Tqjj;K~U1BqUALi6<_5A$U!d_RPo>z;itIt&8e*s-|YvH z5A?pN*=B0-pKuGva=n{wjik>u960FiNHL^NDEgOdSGKe#XLJgzXwl6mE2W|e zeUcdIC2>EaCVrhrlrk0Z4AN9@qd?V~NM?7+&n9y8HA)<9i3HNqu+mGTUPe*$DuJ8= zcR6YAp_(4D3pSn>=2U*B8v_$)7Yp&TGu5Cocgpy?YK~J%+V8XM2G_UbMBK%sYAUqF zmG}lvu1a$@s?*F#R3008s$PLxkQ0Y|-Hl2o0co_Rcr`lsQdEMy80~J)c{eq0n#|Pq zGQsg{iF3Ht6|@G%OCfAiBSio9OfU;3T_<39x!!yMu&o^&Cx3t-YJutK^B zB~9+vmV9!v^9Z}E@7pc;f~C!v9RH=~yGTdVUw2{O_Shvggz%q=LsJ(WuvUTUE>9$Y zP&E7$r%nP(o5G9s{=^C)WKXV(&up*&3z@>p8JW5jLI^vo?{r9}WQMPt5vZCWwXm^x zdcTqKU?$RK^rdSwMgh@V)@kqE6n{80Ho?6Upl7*vOTKiCu=g%+*nawdYx?j10nzZ8 zLvr@=DwgQ3Z4OzDJ32typNTq9ySCmXbV#V}Ls@pL^ z7Ter6ptpjad6Dc!%)ti@P&a{m(6U4SN;Z$=t2kX1<{M@iUMzZ6O~EP-U4QjORBN=7 zfNWCRuvZ$Jwq#KnXRlwF?yW_1%ZopvQ_JQKs?b@%EmES3-t5f^ds~d?mUlY9ZJ9!X z-WC(MMN0H-5hL75gqX0kadj9O{Ddh|KXz9Dr+Jzn<=zuvM|heLnc^-1RSrB7=GVcr z=|1fHn!lOK4!m)`06$D00cn9x9D#m0*{}x%@gGbR7dx6EeFE+rcp%O%p;$vzEe6qDGWiBB-)ELVm%}lxWJ`CRY*kq64NvcCf_~CKdE2bj}W#3gN*P zLinxhQN<23teTF*?7B>vB55xmi#f|PdyoKnW^z-?zvSLkRcsM4Vsg@&DTMDks$RRR z`gGXhHfwenyN={*#1!Aix-p1xNrS~7kg4njM9x;VSg5j9{V%9@))(Mw74sLfy2pR^$+_wz6E(t`WBZOio+5054g*>5wP z+Z|W#;`f8q0@yVbDI4phUPk1we-T$!&_mOb?C%aPxH9elEe)&aq`7cF`_9V=Pa=GM z`Q`hczl@o*?(l85@$yFEb65ZT^JdLYRo1T=s!{dXmFt?c*xbioiRt2dVOsD(C-9WH z({j8?e8tbNgVExtd8e!B^X719>WofIq;sPjbeYo$&n)ob2Np=A@<20z1M1zr|71k& z;umpc0WE765j>gzWWQ~2E?5$XcG!^$jXvID=@F^bEZ=UV1UgijES5m0Kh0Ql0#ue) zsH4lHbGLPzhBR^jtH2jOXaP2_wb*HiHcT8Gs?o(8x2Z2jo7v59c=@V_@tAW4{&#t4 z#?=oLUj?6>Kh8YxleaH^C#BvE7IJhAs2WSH3XIqKILii*@*MOAHQ9uP>RwhV8c3MW=0Iaj~?Cg|ex{ zI;^q*1t&ZjEa>_!1nV9qyB2USgYkk{Ew0p+_v?&Em#=5#)4H<3CnbGY)e+*O758s`eJk;MclCYo712?o&wr9jVz(V(4F)g1vS)T^DSO-} zR6@{ALiXCn1&F7z$A>PV5{7Tm!UCYxyyDN4Vko>x3$;T_c|+9+rxe2-nB?#f06{%5 zC3^5RXdyzgLDC0**j3~yd6)2ENP!IM12>(!P&{O_E5A%SR;z6E(PbEM- zz<(~Q9=Ai~0R9{p@VOL20K6LbC&TE$tO)2SnZklUD<%hKMKI6J6g9!qV|HN{1^gt@ zCp6VlB(XIi(g>>{*^@J(&sD%aF$^-*(=ve1Re(Mv3^CmkqQK9Ez&;tj&?A{rzi8%8 zR9W-TMB-9CJtj8`0zsdr33j3f%iv`}AmpPqfll_!8T~A1Oz;%`g>AH)@Ch6o%ybPQ z2?Qm5>J%3iQSeV2$4&X9jUFtbz@IPj%{99c%_4v{UuR{#@ zT$3RHTn%)+F^4y50jV!0nDBLl$$?Q0P#rNr1+O0r_(nM}^}+-+L7iaqU{nNK7qCz8 zMD@Uct?C$zpbFAF`+859ZUd#C>Up1bgH5fVA`(60lRmJC5mZ64=X%xwHZue1n0(cz zd9V#m6jg@MG~$vx^R})%OJEvhiK+v8zn(QXjlMKBg|$=97Mey}s!C*`N6#J(DUo++ zB&zp(O!D~ubT}*o>O(F1tl2#G(Zb@_4qO8G_ znBsYr(c^1+RMuUVxe{Ll(PofpA-=@Ob3_YX6t4keN^1!4WmVQ?A4Q1+O!;uDH<=!{ zaby_4j105-lBsYV#f7DxQej3vCaB;$*9TLP06G_G4I|TKbPK|0N*u;ajEDKtqYM9{ zh*5w!C29>L1CUoEjG0(yUhz$fG1xsd;LQmeBKb1^8o~1Gl+jmj@4W3A!$Guur%brn zk567<+ag2P)H)Al3gED`+nyhWfzND1Jr_ieZvbQB36HgQ{R~6y{18ZiC)Y`En;#8+ znCNNiux`c`MJJ^N`E%={ICXO_p5=rzp7?>Q1?o{S7Y#+{qy_nLdqG{?+g}3cjHK0{ zOw{XD0?f(yzh28ai+sKY1f9-c_U02;m?glBO+b;Zx2HOPVJ+WaJ_?3TNeS>|Mc@2e z>sALo2opxq?8(P|ypL<@36IH1JuWE2rc80AXM!`~r3Z#GjLa_`$Hp)q57{wE!Ms>% zoeeX2aya#|_0b41VKdnMDa9440bx!kpdit0e1fUa#5xOV@h7l+beSi}oRHyMG6&2S zADTBD_o7!YbXrP4FssOK!x@s5VaWHONFLQPQ80|${M#o=LYUmMbpp(SeR~ydBJ0ZOseg03AEmy@ z3b}P4XC`{xSjA;ZWWc(#=^u;jt5-{=-IqqT_bqN`B0U2?zkGF)6wnUD# zm_}Mc##ul{S?psh;Ug@t<0}rMD*$6F;Ug=PaTU)|mDn*A259WF)@p-_r3P*zF(N>2K`v59;ub>F%F(_Ir2r!*uj7bn{1a z^22oTFLdxnbnnM>?$5gRy*u{fduA>DvVpy_P<^tCJ+e{#vGKjJmcH1)o>-`U*u`Gh zs6N<;e$Q#Ir*EHUWRIt*zcZw_GrnKX(yJHPrw7raXYJ1m?#+wp%bWM)`S;_oY1tr? ztgT4Os`BvZ)|2GneS9_`BTLU7ze(!Ajz1%ck)BUC;52DiZ=eyy%*~^3aFR5vi_ipO zWaiOfsHF_25$X8=LX2qTJxp?h<}m=r1}r_G9m2!U)N z`XXIGE&PLfCMQT=V@ZEC68suQCa~Ai3}Ytd(}$_04eQFBLl|gzbZpMk1%Ch9fNb=9 zzJm4CA)THx1QRXiE>JC5U{9z4!@$X>UsOvN*F9>7GH?v&ES;tA;wg3(Y9VDTt|QeB zWuO?)t*WID?p-ya7zq0Hwa${pw-EH6S7T5BxA6JtW*7rI&zb7ev;fbAK4;w+6u>Q} z3)=={VCdbUt)~v|*EYkL2>SJMPtv~g$Nm+6#xfj{j3me45S~FD&T|M8Gxx4$JzZo^ z{SugoW=Oxi1py*AO}+R_F#-sO_36(c4CFj|`1N$*^Y<+{CUTxL`Dcj%3gl!?zd0;` zUeN6R8G?b7`#k?yvf#=7RwN5QziIzk@|Y1O8<2se_jr>nHAL5eW%Ow|PB`uOCgN;J z$`%?&5-@KUJCp*4t0ov>>jq39Mv5LQ2(*+z!)Y@F110ytH9Z-~NAkk=am1%_&?18w zf{~N!a)Vhih$?%!@W2unz%XX0(gtMU=sj>bO&u|QX@@cs^&QRB(gzM|8ZZpxyoWfU z1~?aoH=_nOfrd7h1~y}cH6aEytwWl@1DX)SnbyI~;Gs-_flS*lX6PU$#1N)+05f>_ z5^nHvW#}?;;1Xik(mH4vJY)$sV7W3}897*)V5rh{pb~1BGIEg8G(;IPKnXHDX&;;n zADWCFm^2Sdh7C#r4M{EyNX86D#tudrha!UpBIAc4ErXDOLy%wtkjul5af6T1LyzWx z$FN~XutCS=A;-7@M~LA@>tJK>P$R@Zqji`uc#sijoNQ^7EM|-hWQ5E4_g}ziyaFC90^++2a6pAiyi|rkAQ`Ze*q0bEDb@#3_ySm zKdcWv#1B0H4LmFjJH!k+fDSpV4>-gRH^lbo8+-JF`tu=r^R0dP!9Dpvz0ym4(lI^K zVExg{z0q-f(eXXemVRjeUg-Ie3EB~h-t1QJI9J!$!4l{RQc(0I);e_=GnBSC{yn%G zWfgH%!X$pA5SkxwBG{>5jsj@hIP&_jL0~E-rxCfy(GvmaV&|Q}YT`$zc<5mXLbc4# zf(1}xIAC^7&pb(}BK8xg{=-?UR>E2J4#(L zb%je)lQL;LQkMsEuRrjMvff|x4FhEz=1u=X-sfy2eb=-H)T`)4i{5DhBf1C_n`-k5 zB`yBQ@v$e(bW8O&hI53UhHC|6cdGLRj3Jmq{8?Tq2#W{1p8B%tt_n;(EKpC!W>LF8 zpyL(2Z-;GV=|qi! z_Y*$ht6$i~|6#w6%`uqs5~!#abR!h!Q}>d9G(k16CRQNYe5ehcILDn+=XN0SOCvDx zxO4KeftjTyrKls>EYbWkU#~SBAjDHAF9VnyQU4N0gd&VQ=v4rXuD8wT^g$SD&?{dW zRh6yGIP!)OgjAC!D^m9#ni_)G6p3@2T7BuYn8e8N;n;s>_PM4(P=esp?8$Ix+Npbb z&Sf^!)EyvuDZ9QQEzLtLl@UoLHCnyt)m_J;>K_s1mZ#_b`EO4G!!%sf_ied7^)Z=Y0F(U?vw#4ewC4qk=LBrzISKIO zNj|qvIcZ+Rm-8tSXE?R`((6`c0C|E4Qk`kik0TVT^P|Wxgn_BqO`6e@o6O_bM?uFCV@sONn;2kYzOyyvyXeMz2ichKY>oLYx-lQ$h}RqM zfsOXnY5SfaoQ!#*>IHdt62T19Z_ANE?e0@ zBM?j>ISmSC8d}7bu;x6y%AN+@L}wg6Id!q zQm8U%fEwsUmB%rCY-yY$O&JE2M3Nk?92&s}n!aUF&Iqc|XLIVWwk1hPw_eM*!I|U= z4(B(udQr>EHa}IUjbt`~5hI1V(TmqBI-mk&ujn0>k=zVWoSz}UizA!P26JZ(P@JP7 zz>DLtLUr`r)j&jdfok$%)RIS@RO@k!G!W5Ipql&`;Vws2>@Cz$hRzGc7D0+NRbiMh zC=)#v;Ki|f!BrKRZ7PPGb3R)9=p4z%Xp5p5=`eznZuI0;La-|5oXTKEExC$jq<~09 ziph;p{gEO5A;MaF$nUde5k+Z4B8QKfIhL4P{zqs=S&_sLwr=!gcl?CoH^UOO{AVP` z7Na67niy&lLRMM)%9jXf#6=ga0b^(vh83&#mcGp6a%I5aT!u?$E)@x3&0h8+go%zY z+(;cq$Y{r$a5Clz5ItW{v`)-Y35-I)2jI%s<>e153>mL+uRs^eHwxH9*B< z1d6K{IVAfxnwW|r>%!6ENhzil)DZ$JK@LK{x8n+sL>Rx{0}4-6^*4bCt^JOl71}aG z7zvqLOtSQ;UdF_k{6LQ5Fu;st22%(t^|LsVnF^(_Q_Ud^WpE_BA$VjaO);$ah$}JU zssm!ELK@hr!O_e?Ia}Gr@r2D>d%gl>soy*o7cdi1hxsmMXx@(#$&Z65ytpBR5AVf^ zs(?nF=I_a1@g_Gttiiw2#%w{n^V!W z&Axf-JN%aIn~MEC;F``?k0!c?Ena*pvAiYmMVis<6&SI8ZZvpv+MP%h+@kg~x+2w) zM3;ub^u3XV#W)v>_DFOa&6mRFglm6Zwh+;5EVs zR-?h2)B6j?0C}arhz`}!;LfSx+cwF$8R9yKml;Z%=OJt!^bUtpP1nW$_0PX$Tax1C zfZErH%v%EFR1E{mwC3Wn*238GzG5WsG2clqskN(K>WR| zi$fXJGPYoeSF~nxCMY(i>G1EglHHYi()wUUKR92P5r-;!8W6&G{CT1YgFhfwAhm6L zw2&auVvSq|pWjQ+Lc~IhC^0Xj?y1PXujQc(FXC><6DjiuAxiMak;4faga#v~=rJ(` zh42Q?cw2tMWV3k1P9G)+;!Dul``sABU5s4zx{r9T&p<@;uMQ`ZgRseUI(ays7KBf$ z^Jzo*h+F@f(TVNx#O82X2~+I;RQVkIU0fw(0~4*9iPfFt=~$IH;`_>tzq6uz!|guu zQFg=@)&D|_AUUknkE%=s2GL>#mi>s9U}qJ0-_ZZV*=hjd{Uk5lS$xpNdvve;{H19N zq9}4uXAw}eBj*7Dd=IGIpQ^IU_GnA@V%ejrtRQ$V0E^XJ_6Pwwj^v^Kgy5|J$uc?^ zoO$hWbHH?~7W#`NC8fK{`j-}WFHKd16LmwD+XC8Sgh}DQJg3IFfDMFIV*tk*y_KXq zkq4qX(4B2bRG8de02X63_6Pww#JhCN5C92gOzqu02@4?JHM>4b!F~Z)yw~j!0(1;zDdomid8mUWL?5|AM=Z(vnoL4Xs~)(r)o6bD-iD!d{W4gN4N&f-tlFHZer-s?%+$-M zg#xwCi84Rr{r!Yb<_yg^mn(f8FEb;6tkfLQ|4)o{!dBkmg+CuQv zp#)Fe=*^Ebw^6+p)472%)`mGu5i^FmQ!i^w-eCNAM0ydy*bF6Kh*P^CMF473N&!?3 zpLvO^?zJ3$g_E*^iM}`87{d|OK>ScW;f88wUUgHKU2|w!+S$cMr;WA{rjF3VR^`)x zI1S1s-s#jVIhk)J9)U3J`ahtEFXK-qTbkko3ZsWo zkyk}Me2oGLGUJX1Va>xr1)RvOP4pDlvE@LhEj&;RZi}YNNV?AwO2Z6N3^Gp5p^QIz zx+O-x9!NS(U2m`CqiT0H1?1|lKm0kI=LAA5qLY`NOtw+WlBXErv(A4} zG6%|E-cXOC`Die5N}OWyWp&!Jc~Y^gR2acZHF>fsa7U-f#caD^1d`u@n$eeAi4IL& zL^D%i8|wRe-3h%`0PTS-1Qn^XUS+d6x14pC408mMT;TEl(m@2ZBx z?&++D9Xga%8TG}of+^a%wqRfGuZTvW-JsiO&ND1|y4J+(%dsNj4rNEI+X^6SO>ko3 ziXp_EtUMTq@5s7q+C4TJgQ&ZLPYx+kLfnN6J71_9WQ6Cs9d5SMhG&kSistns{zMR* z!^oqbM~gG5zC6(G)s{SwSEPa|x8&iB*?MsakJFr5-KnxUE=Ll*{0Qz-pMi*m)oAeK zGaTQx#T8xm(=_y*M~X1^IC#gB^l= z(-zq490otw33X|9W8U|zIA)pt6h|(56E&iULOdFR7>SdeVTfs93O~*6M01PDN9qe^ zJ46aNX3B&ukgy}H81giPIFV-+qRbb00(l_gB`lG{6RS7Hvd+tbZ7CB^af&6EZ0J@e ziWNFmC?9W>vZ32yOQMjfuVm|hM6^Kmfo^7-%q>NZ)0<4;iY5>h(B#dWL8#UZsm!YK zj|5r104&o(1T$4}ttSCt%ywBY~|7`%A$#ChyqQH40Pjzee2Xyo#F-tEe<}b(cq{cFp zeBNbz`C~Z836OjcG^01SJ}c=NR0mv(ZnqeIGcd_<7U0WsSntvGBx(dl9Z2xgjh@_^ zBd)VuSw*lKAjHfQdFjdI_z9Ft6Cin7w|6d)hvi7__wOfs!dJhri~mD^dor`Z7#~+# zWGB95my?HlTlQ>^EX+%+FIyvsXlG5$&aCnl)3M*UTfBfIUQfm_Uy?u!t@MJp6=mMj zS1qY0i9FP{=re?y0Y-kE%mSRbv3)8t{QJq{AX;;Bv66B$ICI$1-8pa_;$I|{6Drt+ zvtPc1B3!(hUOC9sSN^_jPRA~c08J@f)8Ecc59pQdH`4WVPXy8i4Dvq%ksISq%+9QM zlOp(rD9P0(ir?)xqOyH2yP)brrj_$0`IU=nS+U4vU-{`kzK{Cy4byfh`EuW$yaqG8 zP~|cYV_*n-lActUeZVR?M?6{aP0S%&hedTjTxWQb)v7UE#q~Qb_b;?L+acPvjU;Ra z)rM*(VIkB9i2S7~qceGeCRV)Z#t@c&l*e7xzD2SVD4dnJaAK1m$Rpan(35};^_~qhKkH3-wb?x5wO-b#e7?^yi=HFJ||cM zsqKZ}GQevM09q1At&BoRi6;+$)qtcu(m}jvE%k7g{HRV2*tQp05)~u?(bv$x~65z%o_E(@JYhwrXz2h?$pqZ_siUA7E21FV(vnQ}q+VPpq=V_MpD`;lxsN*R4 zXagb*n%NWBDed^oW*e$P=JNn3mz83GqTH{XHx+}Z@C?k(3KvMcLFvyZiW4mLRZOn~ z$+0&i47UMFdH04fJDjS%WXJ8yMp$M+QM`Q|O`2-6a0r=V1!ZE57B6avN}`UHh}7cD zPLv{h0srPgNgqBbA_)E&`R{5n1e>@rG-32^c>FO|yBrZTmtAI)w5SDPFeUl)E zr=UY`5>W2)(gm5l8lqZ+gEP3tA4=0gM98Q$im)AaQoX7{f$Vt>z*3)Ms=4hz+GY4k z4om&~(|`Qg4CBzLBYSCn{^hS`49CtPwa16eWV7Hc%tf0-lC#oKK_?obx?s6SfAg95 zM)>h_XU&K5VZAYG3hc3DY8T;9vOK@n4w9 z?7_E@W@&HzBF!%I-JIU7$}96%>&ueYL2ljWYFv){g;{>Kca6I|*yDrxH~xrT{A2ov zvSDf)4qy2y$}PK~#dW&el}#))L-0ALd}1lR2Yty>Rv2~oPv}@?*H_QadQv@1WB(r) z)TE`qxw^I`Cqa|c#4<^Ivyl2`p8RH({$?Tl%{=|hB>kN|`Q0q_-6Zk7J@Nf4@%<$6 zgFW%XEb+r6@uNNQ<1F#xB=M6y@zX5v(;ZW2lNk)(5Pv;5T3UF)-5)x2k0k^tkHE~v@Ih*@{qcH$f5 zfBesj%L%8=e-}>q{pK0n=Xdwd_I!$6w`TCQN$l$eZ=bg}*ZB?30q*~{BwtZI>!PQd z<1k;%F>jk)pS_~?n2qL$%KHc{H7@IHH2fj#2wN7JH>&dSNp2jqADY$9Ua(-LYi;1 zxi21y+nc)whyr~&98g>S!9{m9I%a6UCq&S&WK#Hj*x<)UQl@Y39wQ6Z;$iLvZ=3Yq z9`F|3ZSnPXlb|56@=L0vEN{qjTjnL*2?von6zy>~25W?hC!9Y%?Vh)H5I30~u|up? zan+Uk1G0JAB=z+8u*JnS@v~dd1Ttq!IDC4{pYYx0cJ~1B_F>a|j-(zc`D)_4rlW%f_p0(wKNgkNEy>x4GNgLo~Y7 z?YqevpKdQwmxmMGd5CY|M~td_go@{z$D5no?GC^_yT|m%SEjifFIA7*=MB1l+}_?@ zM;9#9Y#-@nhi@JqAMf$QEqEnCs(_NjW(O09-{UC-2LGFH?Em%~_|LD4o5$@_vAubY zzRe30?xi=39t-rmy?wxeP4e~TvFc-!$Vai+-akB{JGAs^q1-k7zF2@ZP(Zg)K-=4! z9lF1JxW0j?=st7Wvd&*fE>`S|jT=;lyZeXhhnt&wRD?Ex3cRQD8awF{v>*M z52Py9YBhYld)VGR=XW`R{&r$J^OwdA?+wm3XLo*cN7~nTi=S>`dkv}v70QRW&FAJ^ zJ~z+#HF*-y^)-Y}T5$P$QvOtY?d~@Z1!>;i-aJ8mu^ce!PAz##Ug<`TpY*I38q(Wb zZ_ey`cYC|dA0HkbAR1sbc}$gUTqc);+b%DUpO*#Rp;Ogg(k)-7p4~@x2;i6ZlI|?r zHy3|99e(}%)0e#b_48ubNtPjW7u(MZ;qiiPU)vn;#Yp|T4x?6VPdxD8HZeZU!+#}T zB;O6^!(LCaY#d$VI)`|h<#@-Jk8EGH89EVV_=D}+)n&IuRjHL|j z{SOLxNgyw2pY<==Wc&Fq`c%Ve`pTJ6MsN1n8n6?d{CBXD)q#AQB)wmS02MP|KbK?M7MBbkLU7j zRyfU4y3rFio20ws>DVnNm!eK^=WOch_NH$H@l!HDT#gW2)DUvf$iQwHy0{CYjmI;U0L_p_4wfX94`R*D3dUk1I(p>p(6hxLN9)lp%-6xsH@)B z7Ui!ms4e>NmxdX`N9Bkxd1d_T5D@UjsB+|~gr@M}P2y9Qe{J6B5w-MjB2UowLh^HL?Il06?V0O;;M`G0s6_HDSTLy_>|;te9Zsi2j1cMU;p`kf64z9 zV^^NtOVjqGSdv`qecQBpS|9E*a^}J+C;-G8+%^4ky;O!n#_Tc__Ae*E@>k6#d0DotHZ_L$mPYaO;;{` zbm^SwQH`XH)Unw-KCHEb?!0NTOOcE*rv}oBVg@(LOe-JaFKOqx7wuVMcj_!D&vKSj z$#`nKU`8hie6Y=jw@BekmdEFx;`A43eWU8LD;@cXvs}Fx$~5jAMGD4J{N~LICz5BSnv|DX#{FNY@#5c!k(XRI39mKzy2=h^on7i(lxa6IF3)D`|-Z)JM>fXX9x7qq)F)_ivx-g`67NMpc zE0A9lE0OfsoO&l6d=g$j1+6D^X3yTtY6Z=jLtYhhi^_W2yu0?*YOH5zor#N-T0iY? z+_u_LJE6~K7USlC`<67^y-SY;!9xm7e>tQdOA$X;=Hh1=;_*(RCu()8LmHY6nZ%j6 zNWi{}_)j-#xY|`sbM&fa#zgM)WD-V@TwJm%H;(*U(V>S+XQ?D}=gGe@WxtmA#9Zpo z(y6jD!coOu&|YML;c}N zn5Xd}%u-Ooe?Q{-3tg_!|ALMe^i%la?_YkYKV0eZyWz=KB=RB5lT5QXxL6#U>T|!| zF*+wMlKD~RiKgU#6?O;Ce$|-5+Or~4s7F<0!!~I)hBSgp1;&6D8yPH$oR7B8D)#Zc zc3YYIX_{3cau(YDlBZy`)%QPNTweU6Avd|%vuQNTpZ_Y|VLgdB^4Xd5B%v#D-r+5i zDLvm`{GCxI#?4-H)# z+LQLTJ4YprBSn_tlBQrigY=4gnU*fM_t1INT!+rN$~ts>A{Jl$OLsJ^YxberuSx(}ha$mcXwL>^ckN}qg;8FsxUQ7{q@f#1k_(23Zh-;fY_Q-V} z-H|kl%zF0d4+k=&)Sx2kKacBe){&B$$dMRJa>l4fhl;r*j#>9M%zr6BO;LtH`=Dyl7ZDuVyrZ+#bA!k3bbVm$w+ba}SV3g$k z#A)J7mY5e9^23>YK;5B|52;2}{vb69|B7eVMn|$T@hBF8p_0&p zrmR4`ih@d&E#svJT))T_(Tm#hj`igkimnIh9i2oRlOnLoY;%4&W_COU^%% z6@!Bdn3Vx|z@au~qn0Kht-sEIwV%;Y7hLUlzN!9H(TRgRk6TMA8l z$rJObBXvbu)=sZjAtA8V6beEsO`##C&g@RxoYV=6bY&ceT5^E17Zut*X{fgp1)w2u^Z2byf(Y)7e z_mlJPyCB{AY`}2o!y0rChIO|3W(@00&#{?+h4sp=0wLVbwW)x!BHQi)0ph@S6mS)x zZ7twW9Cm8~N7CvZ1OAlZcNuV`t#3CFKs@1g1FjP0`wsZ)NVVmFqlVeu0|5#WZZF`f zV!qFSzmD^7G2p0fd9ML~q37CLz)_xclL3Ea5IYMvvKCtm_;Wnxt^$r)7Pc4g*LR*B z1ss*xwifW01+k%kBWba-fIr7nn+Z7bp6x0Spa5hy0aw0aTY&(Yz}pG95?1#WSZAGT z$-ITFb?v+tQ_!{Z>rC}U^j=KC7tXJ>w873D7MjZ4ctMS!mDqD^`o=}(o+1*j zE_ab(US{s8BgyJ=7xuGd=AOh8t}%C^Z(3~bt>7%n%-z)3)|-230$XG5!dh8u?n@JL zy}1)rb>X=$S?Go5PK5Op=)ROwEI@anJza_JNj!Q5x(i9M6y1{=@X~V^qWT(iPrixP zpu6y%FGgSIo_bZah4o!zdA{t@i#YHy z*LA{^^{#$o&$ZmuLB_(mS1(OqD_z$)t4m(JI07$vU1vI3{pv*)bn)vtRecq#AJe#1 zunych|C;QKYhc#}EG~xik_E8(b)D|84AzSp@LJe)nv+GbUSyLkj9sT~S|RJj9bj$j zI?G{^tRuCvt?XQ)zOuwKR7$EPo?cS)wVt6;AlG_&(Rc3yI`6tqFA@3zP!B1|Rw=I0 zH!fEUB%XB5qK}Z-;>AEgNmnlV2%0Tn3>1`n{i2U3!$QVDS;^Kg`pB6qW?Yw&aw(pL z_`;ezU%t?b@|+m!tMdru9%DA3mp7+6FRRO*9+s++DexPv=+wYJp)D;ir{Q$yFCy{9 z4?OXJmI*W)Q2l|A9sBq4`uh4t2$@is!}oMI+qOfM!{OIMTuJsH((oOv)0_}w75YE< zlI@=4Yb!USESj9Als10w^gqPacW*nFv%$-*As>aum$ExtkrdL|KmWWLkJo`$z_ LcJ==U%D~KLlSU`D literal 0 HcmV?d00001 From 22155fc47a496142c81ee6398e85b0a0ff0c5a6f Mon Sep 17 00:00:00 2001 From: Alexandria Barghi Date: Tue, 19 Nov 2024 09:17:35 -0800 Subject: [PATCH 02/49] remove file --- python/nx-cugraph/objects.inv | Bin 51830 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 python/nx-cugraph/objects.inv diff --git a/python/nx-cugraph/objects.inv b/python/nx-cugraph/objects.inv deleted file mode 100644 index 4d38bd31eddf7dfeb33fea09937aaa535c79c203..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 51830 zcmV)kK%l=PAX9K?X>NERX>N99Zgg*Qc_4OWa&u{KZXhxWBOp+6Z)#;@bUGkTWpsCM za%)%$BOq2~a&u{KZaN?{E;24L3L_v?Xk{RBWo=<;Ze(S0Aa78b#rNMXCQiPX<{x4c-qvx*|PICwkCM*r`Qoy7u^x{D=A8%sJe4G zov!Rk=kZCOtU7lL5CkQ7bC5ag!B_PT)lXPY(f~M!6sJXUfxFrf{^)9iKX%`TF2Ck+p8gsKdB)@D<^K7fX+YBSW&e^epCv4!>`&~2vb+2_-QBV<_z(Z_ z<=b~rh4E+pQ$p^y{+Q|sOaJ8Ys|++8edjndjxxfd)Nv$M$g*4P^@3el#1fJ*?@yyB zqhwc>@r;G*xN4A5DjxA{9Z!KUPLi=?C5b{9B?(xxjza;AP!P{^n0IB%IVuDIN$U7X z9ImG-a3%<&*js6pL;#RrSoOe}Abe;1dJ`~c>DdD~D&aRtcTeIwOR~@Tguji06DpuS z*H~0SjL^`4kz_fAiarHjq~=PJ0%VLTzOP6t8G8iWW7mYSHE+8>7@?5%Ice@*tLAK( zAuX5*x^#^}Q}08xPU@XG9?8gZds;9fbT^W&th+0O36gbHX@xLC!XsyWcB?_giB{)= z3S)#Oa`TAZR0F>}-5MZcRI>hd<77o;Q$vi=$OdXnswsdmf;@CZcPRGPR-I2kAj;ah zwUkp<#)-EkOQ`5$@Og$%kRB)#RI7_p4KhM>PnfrcL&l$gXOyMssv^`x86itp7{9TV zm39LJMzyw^*nyg+UDskXKwwlz7J_==343Eny2?5QFhY^?KUUz&*kj|~ zrqIgx6Yy(`>;q$jD2pABCyZuA9kh(80WwaNM<`?um=U`CE_#EHSMul<*yp|Rw4MRAfBoReKzAB z8zg@X)LVn(tAToIR{3ehy)+0u0Eh7p@s2V#n5`G~v+W{Pa( z_R0x;n)3lQNxRg!(nXo1TI*^VLxYnqbi9nAO_48j!*nqwiB>I~ z8GwS2;?xLZ>u6d5moBF1fIx_rFRK}Znxb9hTpL1~L5k8Zhj`rjA|N!MUnlOhZiaQhL%*1eIf*hGBZcLaH6VZh^$AO9B zzJxh15nY#a9G5!WmW!O0>s^+U4oe(&CD2*9-c<>3RARX)!A{E6F3M>KC5n3z;+({D zP0nykVz?y%PDwnMDTnEj zo8^$hbjQtd#;te70UU8y?i;Z42G?~n({Y34wgEeBa9uVt9X6{yDAWEE6z>VlcY@_z z0Q(m3JPR}Y3J6|>37^8>ix2<)eQ=Evp54M9lO#?StiGy%R>!Xbt-$zuymSrO1K}5v z-Btl)e+};e3t3d=l!ZXQ#5~FZmZm?%kqo{9>YtGti#)eV&Oalck(9g3Qdd+`B4Pd_ zRV;}9OZ?7~dN$R~dN$BB^qHx+8bdSNY%}sq$VJd%3XEXka~p|EU$c zKrSMD0}+Vf82>$ITk({C9MxEmK+^i6tEx=h_Y?QjC-X>VBzFZ65b-9R5~gaYGzjhR zEYKQ~59jd`HHdr=EK|qBJao7y?sq2si83c;nLep~Z1Z&IsVLbzx;ivwzRxKaC|i_| zKHeP>l?qXLr1}{@>43zuQZbGq(V^gPJp0@r?HX(%RN&kRPm1xj^Q;6l>43aTQ*IZ; zlmr{wF~GUlhHcfa((UmyBSMK2impYTl;dfwS+_$eo)FQyE(X__$u!I~D~aNeN8+QX zEy`+KbNaOSc44{L&FShuO)C)V@^n*(^|Zz2whS=OvFDY>;XRH-OR+&AX^`z+oi;OO zQVsFUNf_~m-pXcLZ>m&G>XoXVkEpks^q5IAN;WTjK4_k8kvfuRCtjc-X#^rBXUf>wp9Nh^CQ?@~`uHY&EIR7xQQ{)+cP8x_VF&9i( zIeVx#AsS>m(O@T>ie7mv+i>h88D?6TIMrvTN-14xn>=&0W{DW+1gZff0ZT)&fy<@K zxg#m@?s1+tBunqu=`_W+F1>d2mWZxp?DKL0O0)DP-Ad(h-roCkwn_2_*-F(Z5A#6k zaEKV@5cT6WCJZpIv?sfX!N@HR1S(Z7(#`A{U|wra=aC~~xB&~dFlmr{o>h#!bFY|q z&|MjpRF6+i*xHVTn_^>Zi<0OG$>%}f@MtUHCEXn5Z=%U$A7Xg(R?gsSeZNh%_Tckb z2WE1?SSM0Vt}D+7*ooN;4oj$343dfdmYZkHX+3}TLa$7vxs)@Kr~w!94_O6yoNCnE zAX%kSWSNA2XxCZL%u-sq;ET!$+lLCJTB5jG5QbNaLulqGEe+kQs7>)L(5|tNbs3iH z(^I_qI2LHUGm@lnB;G`lut2pP8{~os);V@)%mfSHzM6q~)P5i6bybI8wz~2)Xp&Z= z&(f8LOuAb(tB@uMr_^UGiMdy-;h5?fRbheMqB^+GBQZ9szdPcc-vgB9WP$=}{Enw0 zD>^<>Bi92vuEgbq7Z>1#tb>AH!XvaxQcg9)?`Ryv2!nuLSYQg4QHs)=;%g~^=cdsvf?2dTD!5uUTgI zyIM4exMUi_GB+i5xA;zz*w36-lHB!*?WdZpxY~uMRqHd!e3d*>0L97>$IpX6v*HoL z+3K?xdHu{Z#|(|-gs-*|EGK(iUZ0MIHpdJgnU@MXc@T!PiPQlmsNQ=L|Hde6&-v0f z_u!B-GXZhqvWHf$&MHHQPw4H`Z1VXjSG0wkZf(yL?E*_=fe*-iYfGkR7g(~5@bx-3 zPU*ENoG>pJ1ODoSLoM~igQHkk(dNWTlwlTVTBLW8fC*$N#L|?g80rtzaQqF~yE95O zksk6i6_P;mLm3j52vuO^Zf=sSw7C}-W@3?1G-zqzPH0JZeb(G~Y8L@viCuDzu4K+!{qv0M#EWhOf2^Em4$TwkB60j6es^@u-7n{c zCf9BQ=~@)ZlU-5Pu0WZ7I!YJVuY+Uj%OB(=7N5+QJ zF0({`mY-q+>(;RRXZfO^W=t;nkx5#574QVruwW%cFHe8D8sWjvs-qlD)6G-9byWr> zMYlrR%g|7>Dcm3?p4PYUL;zRE1<#upamI-@eZ3Sb6*^HK-nY#)xLB)!tDVrkSZ!Z$D*#W2VOXH~_se z`)Fn=ecUanS86X0SnG`>VI0Y$g!E|a!3S&8{;pTOWggK)9o#MIH0@=4ifo2zKTg7O zSF0|qeH37YuO$0Jm$WDBrJ6K!EP5Hh`T}wf2<1fEJMANYtJB!AUD~HD9KE``JcBG? zo%&mmb?t{3>C-sSTA%b&z!fI)9Mxhmp=d90rM0oano>sN$aB;!JGzo>fCH|vyjw+^ zzApY6Z4Udm>((bZz#6k8zTZU0$K!q}BSqEMgFjoF=vKtHBC(f#v^|clgQ=@FoMf@n0z%X`L;+K%jtmk1 z=ahMmd-rgpb+KWZcH2kQE$wgq5a^jI(dVuFxJf9FSS>x$DzGCU26*`Wpn0r7)6}~_X z)XfE0r%DcE!!)J8=OoedsyfgxMWUN8&vimt7Tkvl)xk-04RDI-6Xb*fdE^&p2VoTw zRy?b~?Dw2!#0})h7bs_z4I#tKtd5n(7Sn_UDv=Yz8M}T~r^kxIlnR@Q9k+n%_nlDR5Vl2wN; zXiv@|uGyMHG5V9S1ENW2lVD0+drh(U3DS;rA_1GUZoH>w7|Ah7GCN5+&v5+mFd~pz z(1wjF=$w}&w@jp;k>dwB_qO0@e57b;Jjeh{+<6;g7N$XS6%0?_NaRH9y}Ta^D;0(~ z!0AwZa`KD^sm8=;6G_)#eSI8if{gG$6Xg}`k6f4-C4symvhZV6KXH`t4u}Vw7%YkS z^(Jywz$1-G(xhpBb6lQ24i-R58wepZkpkkfKyHM-lPiH8fv_~8I#fmV7u<=iTm)lW zoK)Q>+Zi%S4YnxG--yn#Q4&;twI$Cm&CH~zA?m7EtI!7vSZDsqG&Xh$zW5jUO4ex( zXc8!6NFrOxkbLM%i#AW?7HT}zl$+=9E=#jrMEIMWr1=_)0PIk^e$O>`g-e2l3784f zMW1Q-t}Z}MBPp=ubx0;vg6~JK!$RLnbuYP;MDudt;yvalccPEuL#blgz?Rm4o});co<(X>7V1l(7WgbxSrSLr;8R~>bs6U9duQZTr@Ci+u(sDYB?m=^~(N9wLFOEY{oT?;ea1%~h*CD~#~W)w*5&g%HkF|6G1GYagKl zE8i}`aZ-8|1$80mY)-HZTV<0K`J;VX^VA6N`NHMNL0}`oS8?s+y6HRNN$ND41+3L) zOK9OQNvqsAJ<^ zxhByho@y&aFa9fR8RA}LQwjE*Ikei+!#UT6*Ljm-tMhV$Z}+2k@I&?W`~2glYU>{j zNSb1$*d*hMBpU#99MPU+j-yhPWw+S-|FJP5PMRe-&E{BxZ&xh?-@>{cXqAL7v$P{e zo5j^(C;w*{T&r1QZ*hn}vg>IU;TrY5IZ>y2)V$W0t2Pk7B;Rb9Yo!3Js|6p9SLZ+MglxLMh2As;2mA@gg9-EsudH4 zL|uSES>uRc@(MT@bCToipM0Q!c+(pT0yJKQ2I3XtgEU3sRcHX-I4U;0ezF9Mya_p_ z;Eh4gR(P9n^hp(%r&SgBhEr_&aA77&FRdOiZQuztu*r_PuQ8R-GGKMPoZQSRkM%%t z9rXiG8>g0cig7j(bVxE9h!Flfb5i~X8hHr^U{-km`yI4F0V+cNAy-CW>v)C*C{djT zY#nV-kcuQMPXa>D8a>DWJ#TT4hgcE{unE>r8h|!esc^7rrwFRf5)9O#@-@dwqxyj% z1{QxDm!up~$O99kp*UerenykR@Rn;u0objb66zoAtz)+S^yZ|cA~&KDV>VP&* zQ6|pD@#kuLm3Q^2jY;yCOozNDin$79Q+>a&L`%}u;B!?a#N&CYLsEe}TMtXCXVs5d z#C2(aN)vXK$Fj_*G0Pb9D$FWUd4+F%ffMGd8swOl6gw^}xbloSha?I4)F84;GFN5l z^u0uEnO&CoI_uq`T6I3NOz4#N?p4KcBIj2@VOhQoU4L?Xp)BJ#xqS>CX!4GSArq}; zCTa`y49UYWiVxA2fo7i{VQ0J$75woL=alk6XsEHOeCZR_K_Az}d|z}{1l#9~o^($n z&}-?7_jtBG8OHHvCq&h%=$6=_nXk0m{i<2$Gs}cdkre=)3>6HPeN;&Guewy2Cp6~> zrKcYkVB8}Pj?1(M6s`heTDy%0;l7d7(UJ-sV2B$2h=Zl00SQzf4|>v|q6s^$#1G>4 z>2fDSfr_09=*FL~@tgA+2VpFJiQm>+){i@FTDdq?j(@yd?wBvaVF~I52dJXtPUNuP zNkt`zGDpx03sDlMnY{h1$_E%K8jt`5p4`Cp2NHJy24H1g7)M^}yprUO#NC7hD7eO9 zoP0XBTwRfhq@fK8P$6#!lpo~uvm@Uel8_EAOp98&ftVMYkTVrHZzRZ>1|0=3L`|7= zs0q@qPaAxOs(Z?EFV-dN34Mkx(US1$O55^UKgt}Ic(F)*5(I=JOPwZMt^P*dSRnp= z<>5B!n}p-)Z(J}X^aePDu?`BV9v-1ptJvaMXa5lLgrT4EBm#hyDjdwjBB(t5C%`PgRew`V1zfzr2n>;Jj)<0>_4jrz=szdo#qf&*xSY66O zwJX5!8A>|xU7c6>&eynbP`Ow`8*SFgP#T-gq8 z+6p4Jfg7}dtJ}YAT0e}oZ@ZQctKHMv>aA(>Hf`|`+q;chJJh!B3tBq#c5d5N4zrE> z>#sk5|I@E4&%W&f&K01gd0i-lL?rn9gE{GED3m0hcxRLX$Y!fmhdea1u2#7nw%K|Y zS4y+BF07C0*efca1zH!^Jtaaw616*7URE1NEf=LW)dQTfwXbinW=mh&YIR|@%q|#h z78Qi@hx+oHx+k{U#W}&TOrx5R+d}9Q(K5B_uqn+#9~?6@#(Cg5E_*6xhyuF=aCJUW zwTWhlLoD-jsv8Eh4VrnPJjkjz)9t4w$C;o3SaCZ4Q?Eu;9K zT+?~8R7ITGlT~H3=P}GT)R6;l1O-uBe17xId8s2J%ubpp51HP!y^jH&OCGVcmieOGfO0B*CRNfQjaSA(|^eF=T z)ivS}kA$N0F8(AAk35F;&c-NaCBu$JPFkGDPJSbibBpgZiTw;;=K#@6wS_Y=V*Gk@ zV=?%Z$0Csc<=7=S*F4R?5lTrSFNCkQxNAt=9+G(~@1+Pi8SaoQBlH$(Q93|0Pi;bq zyuMN19i^M8oC%DCGTEbvsXEJ0SIsa;HBa%oc+6!?=+;>!MMvD~FU1VVx>WL}hryREIj)UJk4U-v>l^y8MPQ9fuhU=ysV@42E*^oJrXn1yFc(2-EPJJDZ;*6gA{g0=3o)-5M>;E09byzPNDccv=WirX zTl19fA5l;bHb@iqWXIR%=qdCJ3{vxZ9%pi-fOXt?FYJ^OP^o|xiL|aSWJmYTBAF~W zhvAKYPnZ}dkbU4P7|cu+QvHke=AI$PO)Bo~l=)s{ISRb%r6&@|+}?O35xpt7uCX$E zbrOJRW+zK|xE+Z+r#z2Oo|4SP=mzCDy}682`XeV15#umoX$pCx&G5jN*^1I-T$>&v z@Z-kEcb1T9L^W;uBRItD7WhuxgPp2D!g`Yjp=X#YPr>~y7GrNGS9g4B5Y}awqpy9& zF}}KI{5jgRCE4tha(P6OPkni}z_0Qcrnzn8dG?B|<(JeNBt5`qt7>L=Ey|h%Gu8Qd z5I8cr24OY!Oij%~s6kc}ex|Z=0lw%6Xw7SVG&7az%51ap*gc8mZ7cdJqLN@oi9Bvg z9p(C<#Ik&lWM(q#io4OZN^CLA)z`WJN~g?TqPc4C@jJ%+!vOGHHRW0(jfvJ~n5(b2 z%0Q>SHp5(fVpNjGiB5fO2B^MVZ%-<(yE_5HnPwfvo@MebVQ;dP#1~WokE^){L95%1xC$Jy95&9zo*DsOuUw^Up(uF-uXfnO{sr}TX* z=`B(u^G5RX+RTEwC@&a7lnx7~aD-Re+Hqk4IR;es< z{$^SWy2Xl%BTXIU@{L$|^|*H79oFUg@1NT!zAr9y#R0|JRAmL%Ql+Ye$3K<=NVrCI zx=r$~Dr;AkIUdQ#771H%tbKXwa=t{J864Bh3%O zrK%$mZj^x~(_)ow9?=`}c~Bs_#frOX~z4AJ(w^P0<%{MEMtIA-UXPb729(-#Q zZ`dWO>6Yr1UF~h8N~Sd`%K^bAg{;Z9RCB_@_>FDWZZzl+6mK_o&>>x^eygE%gAPIQ z=IOnDzNxWNMZae7+LgZO>4yHdbfy0-oq5^9Vx5yWE{w$8&{4eBM=V`3+q7?zX^qP5 zTDJ$=QoUKcsVAjDhoHFFu(h%7f^>!YJlZ(LAA~5XH%;|t+Qk+mM72C>K)#9gCgD=m z`Q58-pVTb3V1nu_mISnxQDQ9a?16?Vl1D+PFPK_t&es5FQpk?^uZH(NX--B zI=d$bz^3oCshXkYng`{|pNW^C0s>z^5msI@>@*nG+Sf$|87E`tP%%rxt88je4rF>p zoZ#p5MuXf1ykQYi9!ciH-f@Y$tE1tb{5Ot2ZgYBzr-KZV16<*B0B+#t{aFfd1h(#Sd_9YUYR12cfEgl&O1rc8_EWE0LlF1cmi;u zJiLEyqP3t}uejmvRR?!?Tx1fIJ;rO;7uY45*ElNH-0CUl{WP?`fcVQJnhC$$Js)o+-C`goVwvSDdPD#P2C zT-@^7M%&7tGmn0uWv%Y4{@SRYZ<+Ru>LG;MXWKdlucmn*ZJy^Rsh3&ga{&>tE6Mm9 zbLb}xG}_QZwoLUk4m_77S0^FfAN*CH;wPeIYJZDwQHpU;=Qe4u(GQ!<^VQXf6~J#9FM;UOZs@ z7`ExJbo==*wQrik5bE6#z>Ra(f|{3yJOG}J`3MVhojrnAwv~5Epv~k?FhEV;sf`Mv z-sc%+?|T#gpU78$aQ?@bi@MKbTf&ZT$J#bidj$BCy&!iLx6@oOWqM)7XUv zD-$2pbSzK-9>d*=e5ZSs}2pVvK|rCf{@~Y>Jj2)@Cgoh5;Ws$mN!=!Pn$83 zb+y4~sH&S$jfvLFF`-e^z=3!gKqj&3u&^xs2p*6uP!y*_yf(|u#7l&fxg{nm% zg89oXgPvr01_!H3`8C8%6@@s(udY#nV%BGushhrar|fmXZ>Vu{eJidzRrl=F97Wr5C5# zkBzIVjXzIYalfF>RC{Ka(3gjy=yUx6yoRzB9F}2PF+3*MAaft12iXri!`xjOk-I~# z*F<^!P(13R_!HXR$sOBG!y~ksHLJg3U4lD~4>VeK>NP6sqEG1JA#>`>iUDt=6(06l z+EJ*du1Eeru(}_3!r(Yhn85oX$Lk%vrUJ`mEF2)4Y z0Pq|&#ob`Rr&HIl*s{6|bM&<{3#P8;tOd|h4zV<8>Y#&lr8G`Nz0ZEnaYmJc&=ZP^ zlPO5Ba~TO!MiP1h6!AJ1sD$@$>|9^2?IhvjSiP0@Pm_RCLX=7`yA^liINH;shfN8` zPnc1rHL0Ue0`gf7qO}Vys8%a}K7UZVZukL&0I>GyQrfjckCy^KW#!4A+BIjBM}AI# zcr2x&qeP-CbqMAp(K6muwv>ATo$czD4kJ{*=$>v?|8}`{o{kh6<5`|A3L_vc3#vQ1 zHqhMSTdDmXr>4s$H_kL3{ArW60U$#M^tEOp5m&gVc|+d9rmdqJ!;r-C2u?<0ciJl6%thN z>%cUKQ=vIcohH*dGe^KDjE$2xiaEg6A|TAvGHm%*y#F9T$W^_tAs$6%J-x5w!S&wQ{=$w8t>hh2@E`^~#h7pnY^sIE#T^+S9Samm`~|YEEYqn< zz%LNRGZv;gWhsIM!d}^@R$UT*fvh(Yruf&nHi6OXVefC_6KGj73Os6j)4Z(+}r zgI~~_&@;v8DBx}InSwMF>=yP+Ia&(z6FNxJUwDwQq^=7Piu4oW5*4V$8$%Cz+3F!! zBuqm&>!nyE4!_LxF)Wg%qx|)A%#kMxFkT6iXmimE zR-@0Bq_Jc*@Mp;SSsp=)-@vtD7wgeb;INcwDuP{>eDd(12QftXG3yZS)W<5r$q}bn%zS(ojwGF)WdWUp@2?ED@%oTIi*iAr8q6_5hp7 zt)UuNqC!{kufgXE(p2ncx#SvAv=#T64!K4oUB!H!JFXE&U-6#nj8n1DeQ`agOV!|) zdL&(%>VY1X#S-<@3=2JleQ|Zw6^s3ZeX%vw9!tH1eUY`*C(C_=kLXw~!Y9NfDl}Cl z%k}82L>>%_q-m@?<~c5(<7url=DIDPW9qCdW;!jOBkHXrfG$gq7l%3jng3KB8C_Hh z-}=;4p%q?@RmJUr_mvM7X@PACw!3jp-qkj~GZIjB^x=&pRVz3y`7CO6Stz#oP0(3F zG{~4GV*F^fx%@-tu002iVyF0s!CZp@h|^pAj+ytHcu;y=@>$f_dA|cXONa&;vqaQ+ ze^F1a3pI-uq3eh$bqb!*AQC)w z9oxhV;VmHGWX_GJSuA>nAp-3K3??8-$2K*>cohbOuDFRNcgL?+fsnS4pw#JVN>DEL z#cTLuT^1@JAY@|HSzPc#vMk|lo~^X92M;Pkwm_INp706tZffV6ppMf^Y-p)OJcG9A%388J8&p#mcw z_wm51Z}Cx&#w!MZS8Nn`D&2~B36mEgFia^;NSKH!b7QJ#VUEZsUAjdEQS)K239eSW zcE)O<3{f@+!E}RUU8*<|FJb7U+w#|uoLdG^M>2b&<&mwf{5!7#9YpDSAANxXo+3~x zlqpKtzQ}-!*(!h$B2}TTOw4Bt7~SH8e8w23Ta$>5=;I{SGiQFQIs`II=!8VH^&cw> zQi)QYzn+KpPlqH4`CK4=2x`)V@_cm?c_ej6bIITu1gG^vJ$(}mETQ^&J$?Q2T6J=?7%wAvt!2`=EhV1L< z3mhzNj)^+V z?)%W?*F4VCU*jOJP{|RlD1UW(Hc`U&c z(#8bJiDhe(F&rH&u$cl{lcuO@y2wCjvRi>8<$(mF_TnVI#&225-kfV3sK261AUADb z=$V3ti7d9OV(4ig!KnRQ)H?}DJ~1kiHW*9*V(FbSZ zZb!)*3)g1Uel)OzIEvJM8wXiTKQ2_(mE@ufh*)K|DY{`o43fMFe5g$6IhQtK%NxLj z%F>i*>(VM=0r^1BXT7LkNpgW!>^n5zzvnDnFQ$zLm62vKy^%EI)NzBDzB-~lW5H@Q z{jg9`?>zECzP7(M05>k8Jm)24zvI;Z4?I~d`TUv}-mq2aJPryqu<2|)`~PLo0I3fiX?0d zeL`YmFEc{rQ|)KIu50o-I4~nbaEirCXA&n^#K|n;^3A_Ga%nw^f-WLZO0mk{UtI$X z;Z4ZM?*XS|WQu=oUVZ>JKRX_;=9AW$34{k*y;EIP5;#pXEQN#EZm3c65puLYz^OrtC@Srjx z^2SA;5qUWc<)Nn{C|WX&U_<4V?c>8Z9A*8mP*G{@XO1RiBiK-Rb%qqZc}*&(DWGEC z$;VokO+VPUfE!WMYh%0PW>!5-#NF*Ra-Jy`ST>3YH*j~0LVP(-nB#JIl(9to;E-}c z7pbxD;#Ot5^UFe`XD|SgAY~M<)7(TXp9fA;T>0C2`@tmeB-p*`mHLVzGLN_)C!t=c zlOzDaZ_O!Dq^wnA_E0Rfc?fgd#NXtCPzT$4`;7YV&$WJ8WAhw}c}XD8qZO&gJ*O8E zOYb<#QpsACt`Mm2z0ZB!0m(96EXrM<(Mz!1H4({UuIt|SVu5TaqM&ilAil8=a*ZQ( zRD*hgqGA9QDCzUg5}Lb6Qrc)R38KqPqHDQK6E9hYU?)WJ2rIJ-B+#XWW zn>J>LLk*=2KPhKH`pR(19?+`CClcXFAkjE|6h}Ob!#I(y9yQg8G;HYs z9wJ*{DG+_O?X5GrwTd0%8JArx1`G03bT6@{+MZE64^|ZpAUeo!%)xaFQf{swE=jpV z0#_bThhXs-@(gPq8!r$wwIJt+d8|)+7tLSlq*)SQN%jY#xN(X(V&w%yqJ4?#!6%-` ze&|`^a*_V}vfVKJ9FZ*X;?xoBpBzzJ2X&UDk4$cz@HD2~IK>>XK1oHa=#HNOdy(fz z(^4$f6bLQJv?hfj_AVXn+A6vc{5dl3a;s_}4~!{VR78dS&~wC9OF*PA8M6iJ!CfFL zh~0GTuGs~OO2`Fb>cn2LmT?+*t+0AYm^Otq!vd)ZE5_#N8Lb7pKv;2lD7vf$>>N?r zsCr2w-QRxr%PNDOPyp(8?%Mr%&oB^DdV3=o!QgEnfv9s=Ee=7TZX!Xb zJ(kMFKr1)q_F{qPISJ#)!ys=Wfv97TQF3Emf|BxGIEV;8$>}Xapl<>}h;NL%$1z62 zw~#>8vG2=%TFRqqz_OKdqobHGIeE%ZB-A)?+7C414>&caaf+AmJ7^FIuL+^AHMB*uel0HC)keFt8Z{-;bBH&K$_Z*`kXaa$VnOehuQ4zGE zAmpiB0r}45zBKX&Mo#Plgb2x4mf&T84jM>8BItRxENBzd3FP^f@X4iPl{n4x5?J(kmX}K4_=168Ktn$w~bGk_5W@J2+uC9b*O}jS$ccNfyIF^mED1dC<3&rDDY7r@W7d*^fJ1uSvi#FIEd6W4~FE2{2IxxBGs1w~jF zda<}T@z~sx?B^kmNXxQh^d?_u&aH9O$Hwizb1muxf*4SQFYgyc<8R>&%W#<(T;7;E zG7BR;yLiPTr+l{DKp^t4sj6DUu(1KOtMA5LZrB(c$eJCGDEHJU3Me9;;f6&7Hfr!2i&bS|jMg5>+N8 ziD;5A^7S*?;|)H3S#IzQbhyDgEy`vHDtZEWxGv=@lt#7M1fFTn2->tEPi+;3jDAFb zlt)3pkkLg0NN8jQ3=KVS01>@)1BRSY;2vS}UPX2Bpv+qGp8J7pUx)is8E&`)bRetI zB6AuXy#_v5dQ6?hh1j<%I;Y4=TA&~q>OxCo6-`W#h~mmgWECyYQ5kY>6_n?yfRIdA zhplr*lV**9!^j{j%S?iaaZ=?PZnGm2vTTOxVl(X&}(e#B2HRUGTv3w1+y$CWIh!6o;tRpR+P$5Fftbv&nO~`2pkXpM42hMD*+V;V*+O!^Ah<#O6 zgR7(k3X<{2qIfErm>?02%z>w(1v)C@x3sxzbTrpJf8+g7P?DTm;h6;*ePaPKYik&`f;Q4Z})8`2pDoE{NMNXO; zbCw9~1j)vZcYDMLm`5h7FBu3@aj9@cV^ZO+vE~VG(vG49L1V7bMv2u8s_JYksK>e! z1Cq4eNOkw2SHT+V=WxwSoAPaLpm>02rd*W;Hc;EoGgEBQbJ;-c0MV%2JNb>hP8JOn z3Q`=3fOWR4(bpK-hdFIinbQyzGynngD(fJ!KkxwLMNU9uZvc+c7mfpb2SbTlcmm2V zIK%2;WQd`~YiU@GA*G245|PDv(&QNzB&NtJ$Vt)y9hH%>kLjjf27+We6|Sy%$^cG9 zvBt1r_LRTHf;mUD^PI z9JZyN|Bx{kQMUkkh;q>&53R>nzYQl?@v}?lD~mi7jGg0gQIIk^YbE~>#*no;kI}eT z`oBi>17$~&Ye?=AOkGk96Woap&|8IcMBd?<<0$3?Y^7}iN>V?pTSJBu8@%#DkQYZM6O{S(jPC#*qwkvZ?;o&27&oUN%bn>f3l)t7&* zFlSVpLoguytq4o2Q9K3r_`bXf8ADwlB~NcUvQ(bXA!X@kImU@=V=!#flnseznn$Gb(#WMK4|s&DU00BVZr_c?YUV;pl(P9DkI6MY_eYtrR}`%tAl{nNp7yo9`q z_=Sw7b>krms}FyE`E&7f7&s)KSZ zyhsd2WPL~ydINDhM=H}%Jb3w4J(%Rn-WiJ`mZlCBW37b9!@1^A_rVAW6@%x0h#=+4`N_}tiLehV$e}b3!fD$;j2wX7zP4wr}Oyv}jIldPy zdMX{x6I>zr;9)MWkoMSJUeFX}qUy}x`~v`01F3!#J(2oHQ4ePmn(jm-Q^0~|Z3-9E z7rRjNOZiK)CZ3QYd)Q!R+4)pN|0sH(FFzDL(Et5E|JTp-zcDt!mEPh+bd{t9*^Jh% z6|LZxUB#DQ%jd%EY^vEyBXXa?7?pFFFTei%`(H#oAXmixnuj4trm}XLyXAd@aJDK+ zNm+dzW}4F(8jS|)45<#=R6nQ@sL{9UMJjS}65m;p@iFba!ON^(e;=kOHaW{h9w3TG z@G1acK%c*p6Hj&G3yAU^4b!D+stpI7I0gpY#t- zT0;QYh_FO0ADM=Jj#28*MeQHH{9Qu)5~sYUx6%fXr?QC3T;8*lLP?u(N?Q3yRQu}D zD4)_-EY9In3{at*&+%I3suWX?$GFQ|xa@#K+oV>UlC6C7@?+>S<>MVA%~EJt5FrbKe@n<43uKP| zm55N%Z6dm|ZotAZqCDas4hsnnfP9Y#1?@k@cVUzGK1Nf|2hGX!?MMaP|0ti0X7V|G z*|q*$c{y_;dNots*>_{v}wtB~agN`4&1 zEG}BfKJTNIUvK!TYc4Nr87Amlj|E}ry#VXwzt>NvYQ^_B@nH6-kY)EfXq(YN6ad1#m6bm*|~N+^2`iebUDEJw;B8? z`wn+c8WGDZ2us^e96+iz;x-n`h)08M(=&$LL>Au5xty8&t+-XU ztnmoKe-(|FEO#POBq#FcDR_!qzHRwX4 z)nHs$nVKj=%`zibHFT!l)7(}2`=)@};p$0t^=t^TC__;;nMv{Zkf%Z zdBmvZ!o~OAcD{}`=lzaE8hWuVG0gUY+13KvZ{NAhIE@F0gCKTEYV1dY(VOVnkN~j4 zw+vZD{h|2!owAG#E)gz&Zm$n6)_aq@bX(hnl8sP(J33QIH$Z^QbE-;1x6#)AI|tW@W* zd%}v9kE{r3BV?(s@#I?>_ZRg?puWl8`>P+s@6faqzkI9Xlrq&G%>Pk7{H~5W z1lh`1wWZ&6)3;ew45a?c_ww7zKhy69%RoO$UqR^vHPX-%${X=-2EHacc z`rdp$XeVz7w8xY0p59PDMsvMQ&#UVx;r%#6RY7mbZ$$fG_jF8OBS4Aw??3*Mb`a#h z-wg5J7)l-Jf%Ia${O|wy|GxZ^C!&ML)GDoCl9`(wP{bny{n$W1D)c`ypX5RIQcQ@y$RmjopHSA%lQGebk)a~p6R^Se?~3o= zDg^n8TDyWCIQJy}jnQnNOjf^^?ln=9ic+pJm3rL3Nm#&0Iv}?Ev06Y_gzV@%X?fC-jszdyEEx_;h|n_Pfacm z;B`4D`?fM~m399ac@vqpK8EPANoUf8V@31VR^;zG&2DpGJ32t%qH27aj42;IHdM+P z7oLXX3ff1R#$lY?Zz963h*WF-_2=(@`t`99?z=|NnZOm|X_sRL%aUP}XH?G@!;ajyhA*+F;^?%jchdrTmxA7sVhNh9a*hmh`)@sI}Ef!<#TjvjsUdY>=@IFyKQAxObQ!gC1u#UXWpdjHC{p8OLtqDMh1V z6az~cYP1*AXj|&AmBfFk9(>oR3@2*IUp}*&>^d|n%S701GoO0QrsUfmAf@MUgvYNt z(LCJLVg>m@En5&{b>{n=^42j~ZC-Or-oyD~T@0TX48dMnYyzgFoH3V;_|vl%`N{W% z<@D$thRJ&v8y#9&Z9A5y=|<&^^W=?*3hnvMio`HoYFMUw@oX2*SV1k1Kazio<@mDo zubwo>5Bt;pNcTtEIxaP~UYEX$$Q!>BH(-wFvju_lJ_cUb;KSwMo`~hK{TapJa#23} zJ?EKXSC5*E$Af(lDSS9v6oIkIvh>plMVpoaVm(>7ESV^DXb_FE07) z<1U(9xrwoy7IWw)4SHUoYn{%Pbu@jlnCr1{aQD2JaFov6fbP7fXVdVAzelOh5~lv? zoN$~No`0t2gJ{DZ%)p1Lve9>+ZmTa&@m&T!De;2%!=y{P_(XQB2+Udu}FwXFS|>L=c6 zP4@eXVfuT|JYFzrLbuwCxGO#Jav6{49th01bg|SMjmi+Ln<#rxzAvY}1d5@DYj(RFm>u{#FYwXz>xpiSlB&u;SycX45` zLc9|*E-pP-Hg46%BVTSqS4~TQz1E~Qx23i(M`9l-Wk(D@61j^e^Mec(J6BKNd36UM19xvJkPc< z>RNdovUK1+hW+CI>RI*U5WTrQ{e>J3m!sibY!AVDRhUJqmdZ9;9`c*|j0Y@n0(Pq% zTk7|g2LI$z?@ZX(;Ap%~sm?zrPD*+geU;Qnvn0Nf?2m30LyXhW^-YhRebm01=sD$) zN2G^G9YN9f-ek8=V)RK}!#kjtuq$E#qY?ZM?xN3A+IvLC%J&=IsQD^Gnd&G^9}0&8?l5uij!9Q`QGVh znW8(sJnuAIH`UCnsFg(9B>sKnA@AfPb8j5?-hooe8GcPV);m4`0A}+=%ny?FFgp zosawV)A09zsH++ubK#x*)8sQZ;3LG`y9d`gSjTVl7Uy0b?k~25Gfb21ove!=Uxt{2 z{pGSXDZB13ma#Eu(fJd^=H|qr@43I6P8;x2lU~FaWPf5FZY9XXfSY*4d0KzhW;C~9 zB&Q?Yv*XEPr~~{)K^~YJ=@#Oq7W0d>c;24c2gACcxE>mBLRHPx?05gDf!P&Av zO1^6jYhP)er7RQuoggR=|ID*i&(OCBGf!um95Cj|6nZ2bsR92(uG`Ax@sWQt$GHd7 z@Q%d097XR6qi@ASxf`%N=zmAOPM$xQXG+c%6Bc4Pwd9##k!W~Hkd}*X9#4i?%`ag+ zd-kGq?d@0@JM0bO+SYlMYeD?rkhB^DHQfu%Uz*H^{hoLSgSx(t zGw_lZ_b}%gg--K%lk#YaUX>s=ZFBL+Jf<=Q{UBB?UBn!pggoG^$HL>;^~DI?KU0kL zvLuymAyPIr^#XsV^?YER^5JwgT>EtKFDNTC>Xod`YlDX9%x&oo`spmwVzz18dl$nm zCP!c-sZMB;Lo=60q_}R+Vbw9ra!1uYPFpfAO!jE4i(w+j2uY?qCe7ua^|J=+WpAAK zC&S(^_ii;7oewNEKU_@1JR9P@(vhySys$*wP146#|F}y%p6v(PfqfPs(Z9(?y{cLZ4j9{iQGLs?N8^Eckq$i?&`S3TBi2#j`{mZ{T<#7{M8iVB-w357@rH_z zZvVO=Tf@G8Yhwx?=G*mA=PCJl@9ylo~UUUHpS_P`y|aoKm0z_ z4?h~-=|D}wPA8^$IRr^RBPY0%fmIMkxq{qezG$Hq&#e9?k3(-{wMX`A{xU>rZbs@K z4+ln$qTpe-i|W3qX7eC#o2t)z{67Eq={W*z9E@9nExCHMF)cNN_~hnUSx znNNHBV7nK_T#N?dvHx0BRqE8Yd;H#7nRc{n*E09SJh8))g{SKJdLrmPi(@<<>x&-0 zIOR_b)-xrS^IjAY>aN^)6#Kp$*U1Zv1D5s7Dz>=H4Y>}Qx$M1Z+J}?qfO~iw(bS(7 ziN_9x=ZmHUw+!A$62_4ju=dWL9Bt-a+@nDc6vK^_zLWbSXR202JsKc1w;&`Zxfs#) z4My?y@Tx5@-3`etGIL9^gLX3MA@j_mhmwq{@TooeyEtT5(qf_qx!QwRCmV&SI3OT( z$`1`D*YK%fdmcNYty7ZsSyWe8$Wy|57APIvy?gMfO$~Cu=|z;^%0+M88mviZZbEq3 zADFjiqf4;B348l=1Iz0)TA)=Q256SJ22Z8BMO8Rl*@g0SGo27)4Ao(Tf2|GEwBw^Z+Vx+ z9%p$%mMG0$oW$4oElb&(bBzP_Hd6jw`pf#i9ZpTHe{BF?G`TqRVa{C|SJ1vc5g}8qppC>&TdIyQh<}{%DxTMQ6g6 zVed_g)ZC1en4WPwoe9eEN~-Vk$ctW_%QI>Ext<(%!kPcCGCLD?!L`qjMfUzHn5n8&leJ;kT<#oQl>=scc~k3xAQ$k`n#xv_b)gq# zpM9GhaSL~Gk#=>~W=Ld-3!L`T0U74k$smTiKONcLqu2}T<(*Vh=O2mQicQX^^Lc+c zY1qK`qbE;jX}x_y#L&w-9?vMAxN0tsyoo^9fyulUr@0ZQOI+i$x2>ab!0P#`PNzB! z@wrIT<3;2m<{R!wM;kFcrfLc>zgocIs|6gtM1bvimV0z9W%f>GKbqaMo8$9D zdPW&Z=xyiuX}4N$@|$15@4IK)@~w*pv%lw-1_{F~(d#37_hebi*T}Y==jHb%i|FfK=%o;<_ArL2a*{rS=|*nxWFe7Lw5gu^yQ&)8^j znVPxOAr`iEFwXvZ&*eM$=vzx|nU--rGUMbkOT(hfgo)vCLe-(mad|c1^R6)K;8|io zDv#PR^kh|!`z1^(zly}&d&Kl@pRM!+$Qy@3o{Fb#(Bq`Cj%RA0%){|ul69U9KgsDW z8z(Te5HQ0tPVBNbkE@s5D{yP4u7fE)JML%JemA1S=$iIfbZ{`XFJvMUMns(p%8Ns{ zqHFNkJO5{$T+znZrePkDN5jwO_fIys*2y>bqVylFvw@xns@E>FYm?BD%?6Q|23(MK zUFBkcFm>jIWqHQI^pDivBTuFYI~zmBCrT=J`kpQntMmL82YJ{|PVeYlPDeVrqiop` zi8Lv$yb7lL8!Zv$SBbFgqitA6QH7XGoGbxRS47qRj>f*``lqhjqs3}&$4aQp_D9bL zE?;e|UumFR4rh-ZJ<~wCUE=vVjmJ{IAy)HCSPx&qdi)aB({`+v139^_TuZZdm4u%M z@))$_^LZ(Tbzw(?(KGSS7(N0qbP_!)^>4SuAol5KioqcDEz~_3YQ@^{6uU*d%lUj^ zfgr^R!?u!6eEoTxg31o zSi7n{PrH-a=Yw4BO4vn=XcpgfA`ey(gG;$P>5Zqn96xla-1FHYC&Two>o;GgSuDl} zX(ooxt{A$%%Dn-sc{s%RXw%Dui6VJixq^h|u3FmEG3ZE4YMPe=J2MTw32AE_Igu>x zZk!C{{UbdwvZp39QO-uV5gwI)9FM7?O3I&3bw8hu2bWM{A5dBuPo7Pu=D&Fu3hQMP zQ#^ZG6LdbEEYH0Vd94|Uw|R_614gbdWfnOwa@KRbk1x+`%vzKe7lD4-jlx9z4(wt_wsKG?nRW^HS;fF{8c>o=6ZH5=L{>is2-#!>xcj*mrnj3M_ z3#FcI?`?L#ntN_n+P~#daXj+VSg6HpZp%!_;dt^~w*2;!M=zkP6o=3MJX3G)o5q=4 za8?(z3T$VICtl!qH12eFx1QM0jK5l50B4|$KP0TRp&oE zXKnjLjwUggP2D}-ZBxb7lF}XCN=)la#4GiPKl1|H0asCw&*9Ig}Ys1tv zlut*y+Qc~d#5|Zj%9#Y^uF2+>$$D=>iQlc%a~7J#%z5V^LbNd_hMh@#dNKi>wwC>f%u=fqCugp-gCp<-o01QrhDrws;{GU zz-Vp+;j^Q;e$*gkdwTKq@`MZn@kUYEc@3E8Rf?f_b-#C~;$r4WwCH>YjYXxWyJyvNJQKJTrT!Z$B_F@QT9FTR#bfdi6}`)B_eX4|)n%gH2X z${3E{ZscIG7MZD??0k?%%GgN9*w-U+b!M3#iB0t?*>xVAJ!M;eB7Qs3FsaZIR?zx-uKxoK6c)-=*e8ekoF)YfCmb)iMigWYw zYczBla*NB<%|%>#IvHnTEncN1ng+Jxkz4@H#+SU^7HFnHi7;fr`@jLNfase!!zuySV97Prs(xEx(_pq}O7 zTy9u;7sRMPPg4HIoT1y1FQ=0zq6cPqV+)Jst>jc+%B7qwDstSD)8S~I&+8X3r?H>a zwQcpZ`$C(X=2o1ZW!mh>#0y<-=QQ2fu*m*MiDev-@3XJnL%r_wi|vskGrhfaqGA!TVTjb+h?E$olP9kOwEYXlw6|*V#XD?He?K?u#JDiSJBjh| z{)BmqY1zR^%hODsyNtx3#5SzG?b0*e-G^TPSq4NHc(L~T%Yh;9@JpFII?Ra^j}0s* zCFaTYPfVup8Rh8iu@k#R&Y_|+e!kc!8!~zh^>u20!w{|cHMEEAXivt8Qnt7DF6MC~ zMW|x4S?9V2ax#7MY%uF$?6oesnm#;=oKDt-`KSGbnz|;s)d@Fws5Xw5w~IcF*;^{i zuTyb0$=UF%6T_cH0$i}v;y1sT|L_(3?p{{wrpF8{*JiO7>e;hNtmFQ4q2yw?HqP3& z(uLYy_|~E^_0gP7%RKJwqkUL;qOdFXOm+Gc=hJ!biPqgjSt2UN1(NXhJkEIE1=51f zBg``l=AX4pXpWxBB)T!?=|H^m<;bw_84L3u<9+v~i&oD+$@A7cdc>JVEe;bCZO_!+ zePUlMk!=VS1HtvvHGlidKO9%YxP>Hn9p^GNb6wcc@{jB=nxw#@_gh+bwdrRTrcw8^r_NZit%uMV;#Ky`Fud(%Wtr*2O-e49e9p_-_398oxQOaS+Dh zm-uan&)k5IosZ1lAN`|sz+D3`>su;DHEM)S`OoP=z9ioc$kNU}XiEA9$$@k^8r$kEwmBcQOq)UN)2wFK5?Q5xA zkyp;f{@|MCJokbqhHtVdD94*}hZE)Z&(c@pkR&0^5rQi^<%fgEF6Pz^XPqE?WFQnk1(^J)RGyOTD$1DKk-f3>-Bs z6U+6F7t3=^goEiCdpaVYU-P7>AfNYTyVl-iC(rON#KPFo{?WC|Vz$lB7A1Rso4a&b z;>D?R6`}|q5@PXn@=7~;@YjQE+2SgXDpzbk)D@%@r*kb0eOEMxG0?y~BBL9hYB`qgtwJA-Nb} z`Ee3f%Nm-$bzfSO+}xU+$h};~!7!6D1Sbra&G&iDj>E&{=FU)o^_iHrwXsC(8ckZ>ji2mVxlHD9?Qrz=`;{1*9 zsQUX@rQFQM)G%DaeEYB`#*!&KWmh?N5+hCSsiwR$c}ET3^L)cQoLstKr))LhT6Pt7 zfAG8SB+c28>`P`(V0%)CcpUw{6}V7Y|+?yXNu%S7bKB9 z+U1R@!*ui3mEQDZ*oVu-JeoH0jme^)`2RKcB}=l~SeAQzMVNJOBQnBcUaBRfD!Wpd zYQ3zi+R@-7+=DOZq>H4-pjG`t_ZQZeG)5920TRQ7zcQI#I4^*6&jCRY#Mqat)}D9x zXkIsG^olXQLHFqS8uUoxSKQWk-b>J%+RVUyqAZAp{w@fu`U(l{skh*3+2 z^vQ=mU0}+4T8uNZH8z`X&*tGC-Qi+)^9b?9uAXo7moLA3|MM5kS8=zwM>kuT>lqnk%zpzBDS74uTIMvC62WozLdK8K2w#W_<4c zJMekL&j>v|q30dI;G}5tPDiMw&q;B8P3Px5)8n2J)C#CThceH|pLTZ4RB`ikefMy) z+dcsA$T96TH0QVGe)DbJ-+l}CHyGuQH`mvITf7fPDbaq_sZC!sTHHO};D`MAZVNJI zt-pxnjL=~#@!5?1;{JJq@|&CI=lnkDjQxSi!kqo$9$gnV_jt3ob9=}F?IWt`S?HN! zQr+&*ZSi#faQ}P*y2s_hu9n5aEhew!$EO|0Q20wp0^9uMlg%s1#!JQH_WI%GCg0u` z&k&g{F&ai8U=$!oSl`Q)Y+yW;VF zlRs=89?!7aXK72N0zW-o=l3_)yZixk2-nh=rn=tU&>fE+?hBANh`9n6%@>>c9r zy9b1}B)?Wmg??a&mUI)I^wuxw=KlHa4n1B!J%ZPV?B422%|v}@SZ}5g*?dzXkI#5_ z_q^K_K`kr37wGAreoGEE)lI%D?l#-q24unqpWyL=I<|1)9hcn>mlbI}R``YP)i!Je zL!;o%Mr89XiQL{k+&t$yv;}z|PPaN@Z)3wwNBH(}bN_IScIY1X7`nHnpSG;%#G>8x z=K3DR9$rj}Q=QW|*)VI`FeeY+;=65ecTG>e0q;(p?E0m3C|h>~!*_T2<1@M|o^S6# zw$55R&Pa2(Ig7(>agCnHLv?o>-7@G%T3l?1-r_^^ zrmd31_q*F0c8WDf^G%zdeeCe5XiRG}F=@8n-t2C#pKsl~YF~U7oDC4pEfCHf5YBxJ z&i(yv`%KytPn*c&=w9AXZ&=QudDx`!@VI@v+2Usma$vNUQ_CF(`0Abx$7k~Ty?YElVv*EKuP5uZ@>RN*S z?R5C{^FjeD=-&U`1!?|Wu)QQMQQx=a_C)R)D!!teR?8Qx4?V{;|MPtNNGii^UchVy zM!VTM(mLAG!qC5DyRz-*COWha{c5}G*e9-5s8j}Qs^Tv#fqnGL1r;iR-E;nYeRG@dP?2;bU@;ejADjD0^)Afc8Tn}+a15|$ za;qBsbB}ts_+B-2>@D&y_|%s4-0IHaQ|QX(+u-D`AP$h)1|kGjUR$7> zs&NAr5e2NcvVqPfeK_FHCyEGHTEL=5m$Ej@hf&e#Bp53)|L}7`<}n9k@x^EZ9xH7J z>`4tz;75ziLF-Tin^qP;+SO=7dUGs&BBDfmTPk_v3uhhh)XN9>WX(vnJ;^%Mc)rTT z5A64MtG!mMptCNniytq)3>J$6w4RZ4#F@j1zS@iLDgQFR_*s5(nNrD>ow6%u-On_I z^vcZ_zXlfoefJ(xfgl4gxkB6a^-9i1(_xNf^DVL z(9yQ)$kaK~f^@=hAF+`kUsKd93`xG4eOdJvq+&-y-t{G7fn+ z#Er{3!+H`n|BrkZcb^A(GR`Mh&N%qu!YzT|O4&x^ioHuIb>>XdItv9CTXrLUo>Hjh zNE4}rBoH{}UPivT+q5Dl&L%=9q2OZKLF8S}rv5kMbeWWrEiN_TmN?+@hv`9QE7fL? zns{@Rl-kc&;O$;*c>wwdjGH0-%6|QH*y4h9XQQHEZ$vQh>#n$BTWD&b9UFT(qg?KA zSu~H6Qu+(3cB0vszn@9H=F^@5TC_9e!8gbPw;VWA2Cz}Yv*!Wx>XHpCV#=JL!$oJ3 zPhHtH)6mpH&syb;4wprzukwazvS+XM4W8|pVThbASH7Ml2fo_Weod%MOa?iNKJLq^ zk_V^K%J;V%Cdr&Z@x-H9_ass>^%X|#IW$v07Ns;OsXks8oJ*xLh3H}-LNdj?%P&P& ztdCQR{#m?q*>L^hS@mwVi21?tfmBuZCil`=G?P*!5jsCMvYmpnvCrTYbS=jsBJuFPs>XiA}H`tX&m z>_St1_)>hj#}dvlo%whsy^2S>Y$Z}k_1!5ytM%A*oS(PSz`<7iMm+78E>*&oQcZ`rw@px?Nc{Gpytg6Q-sdU%qaE>RJx-qMyA<4vT zPj&tRJIv_h%_4P-kW8`IS?(NF)ZojaRt6{6n&t0V(-!Bs$up-b>W6sgvf-nrn}OkR zBCP@z#z-Za?U{tn)znbDnI$U`&Y<{ga~ftoSJkJT#~>V?sV`IUwx~O0kEY_{+;@F# zLeDH(x;&s;VVviVGfy_XEHIVI?tH^d-QsXlmw33TEB`phz3k-4(HEwWWa7>@4!k@B zoO<+^_UfNSWS-ju=6OnSXKPT6&g??dHm3UKoPF(zH?v?R!Wk52d4;;}yt=v1RrNet z!aO+D=e_TvKdbgJ%2{-u^X^Z*I^;p(62LbZ!>n`k9C@Jnv#3|2q|$vllJ?Cx1an7b zr93Q7BHN#A+oC$B`+?JC(ubxlI=(EH?bLOG`(SW!CRbq3L%vJ1X|jj9m~ELabQ_o$ zw(v6(;GPZRa3@^S%nYXLfIc$E9iXDnxsLG~QUEQp>|nMdCjZoQ=fyse8k4I*nkf)M zr`#1wJAYFiF5|J^RuZpV{oD>9TswiRsBCKvlr$V+p0O7 zI&0)&Awql}=tl_!Ee;BI!u7p+Ec;paMgFiRGs{=@N+5T3z!Uk6b)$+7a8A(mW09IswG!koFy<`XNwLU{uF<*-!`Di zrr=VGxQu5{8iOT?&AvzJ4p(JPnuty3wkjd+qY8>}qxJL~;sql7!*l2l8)Al8>t zvG)S1aCboU73xrNY));4`tArSY|WWyTdSrd3iB7brFXL{{U(S-`x4z0iY19Wac27& z?%wE$OB=6~2iifE}B`AR=cGlBJT2l9Wz0-Vsc=T16832)FVc1*VN>`w=J&t znR|(g<%&y9^h(_LY)Dl$XN3~9G_t{<@m=!)`wp0)D<8Ayk?)hR>_(29b=20;IG zz_c^(Jhl3&Hz9H@TA1$P5H~hGXS~ixc{o<_ujmzguFV`4DX8p7sbQ9LX(u0ZIYSVHE&>Bx+9{;y*TsEgIbS^*7mWL=Coahx1}2gY zvf(pGH2IMeK<6P~^5iFYQf)IQTKgPK6`!mJKb6qM1?0TJxxa)`rV^m3lMgO*{7iW> zW*+1u=7c>%F{%L`DPG3_^bGnIc~vy^rIt+ zQzOIe064T{q#;?YH~s3LWxm_YhMhq}qXujQf4d>2v! zd6cv8m*Rklr}IoJOp4F=8Iw%5&6n&eFoyz4t#{@A#7~CK4&-C4N~3b-Mhct-fN9&; zhu4fx*GzZ+W=_PLZE6Nnb)Kiu_5>z;>W%`aFG;P2`X1#k2fDT@nZ`wj;-$V61x$JB zGCD7wNlK>Ji{(u33%fVjkgM35zWyttzx_ zM00hRI4NR~U$Cz`Z_mRBqeEP&#aoF5 zmIY@}Y!sME&pB`9@-0kr>snl#>H^V|GuasJcnzk|?n0R`G#cadbVrgkEvkkevd`oV zvU_gPix@~Ygak=zXuh zT3h@gh8;by36`OPDEoB=Lkv^CT?i~!4O88fB!aIp-af9K!C8K4X=hKitD5&L$sCSw z$D#+^FHvOPq|So#OkP=aNKfUApCRSyW=Woi5~(u5#GJEE$5EX%hr_8Zb1}Y^{GoL7 zHw?}=#Es2>b5=>t)lmAX$H4>mW9#nIITwH)Xw^~Tr+dwnL3)Dc9(wZKnWo&SZc6eY z(01hW4v_kQDfjl0*O6p-(?4rXcvfQ#=zvz|ebc59W5gG7jA2OxW%cQBOfHLi zSrW%%#7gYQuL7~HbUy$!sjxzXYKmmNubM5YvKAvYWSFG1*?owS4QqI;xVAYRlfNjI zyvUuU>@h$jrfo*seW>yhG@Lnb!sfHZfhr5SDa}!b&4&`oGWk%InZiNDjYmr+RA+Re z$}>iY)c1ngLsQc|858I-fXn{uSwN(Yw8hU}@Ddccvd&vf=P{BuNZM*}&6AY494lJx zu4xZqcZ(zd7z1N9i32v%=$V5HE%N|) z9U1;OCe=b6SoP4rNKJj)9A8L_yeM}9ri0UC0GHdk&H{jRkXCE+!g1(H03b)ZxVq7` z8KVHHtneGI65J?`tLxylNaAk|5s&IuILZ={la*ING+$6zi$CP$%FZ+C*a;G`Y^}g@ z3;J0=7=Kbmr+F%D$IeLJwatCnhy zq2K*}LT!RFGQ$;p4^wjtck%^CcZ7vl0%n37jshAiab!zW?<;@`WP&GYipwC5bkAb8 zBgOBhvPavB?m!p6OR^1CG!S)T6mM}*87OHLQ#%eVo-UoO)Z_i9JOG3|$sNfKz7Y)+ z|3xSA?7@X7=@e$$U3V6h)RBWy8ZDvP;#_JAH?;+nS~}bAh{->753mLGD*i;OD&X-6H#4w(l7xN;;&~ZEtLzgXgpLMpYPGzC| z%wZ&1;pIWWgCS0XIy16t-%y>^`2B!tFcmfjqM4|gB9+Kr*{Lt9j`j}^r;@{|S#UCR zLl2fXL9e3!<5hOh$e=R*!^p{rm@H-pi4=z0y4=5z`(Bl`ewA&@j_(8mr_jZlkZKN$ zQ(K`euIPp}N3`dslCp=lI-(>u)Xkkzi%deG#m+2eQoZO`P90o`5}6Op=?#(MCjO?; zZ1*8&;>`kJRSgwgI#HtwUDIfflyrKf8>`TNM@)dOy;TQ7&deSs=g<@<(yXR0r~~?t z{W6L*MM3m&s`U<;qk!f>jmkGVp!5y$!NIi{$ykpVmnXlUcmr(22Gcx;7K38LH4OFP z5y^xFT*1u4aF~T|4%BmO=ycWYio)e5G5hw|lywhK02jc;n|>Y;tD`5)`kb9VFj@hX znBd2wuV+D#YPRLr{WqA|nHhkZ7M#fKXsr=a=u0haNmZ;MI%OKy9fJ8(Frf~oG7Oe@ z$_ej)J^vHrK09S}{uIKgu{co9IGqQ^Yf?%os1yZPITv!U#S>3MdrlgVa)%Fza$7?l zsB2aL`hDBf`|2}M*3aQbA+w-(HSZX`CetznACyc3B6Ykr9ct0nlOcza)VOZ`M9-2_ z+{g^zley-&aFT5v)X8M?&{$b~qF3Su=aLs~!dZ&ur+`jdUKgBb0#lSsj(nyE)5V*n zIO#*lT};)wvICh_8RAo(1!;`{tX@o7KBIPFCU7Jusg5ob|+zH)5wrz%cdhR>(fqB4QL6RH2@No!)a&>mEl$$@^ro3z4;;dzx% zjC7!jkycnSU7!BuC_Z#aN&vX3lwJ0!y;K1@`FtG?9e>k+GoV;7n6FSffSiF4NpXn~!1QGu%CUr42hw zC$?18&~^hBWAZTe5DlhiEnVN>_8skOx^|iV&iDnS8k-Nes(Dt|Xk!8==ianY8*5NdS`W1tf*wa2 zYqu81k!orLMvGu6#K=_)we|$C`fr39h|!A(Ufm0>Di~f2O7dbC5xjb$tApXipd>GL z5zMQp`6AIz+=7r(@x;cyiY1KZ8{e;keA9_q`3;~VF@X->j-x4x@u92}P}Wh0KC4*L zv>hYQHm1uq=`JJj-?3&?@y40igNf4_q^G<`Bqao61#X<7Dh%NqPkq*vf4~qFSn_m4 zeiclO6&=uft|Kbljq&&TP=3&PL7;S}nDVL8t-+L8TMCr0j;M5ULXd3_Vml|uB5Qz^ zv}Xaq+~1n&bO52tk>nn{^mh=c&z$av#P-ar*eN5-5KGXEs_gWtSYv3@KiL*_B^2Wk zP>>xB5j^=mlk_(X%@I%C$C6%0kldGT{8qmT)UdWca2H<=B?r5}Z?yqfA6|pG^}`e^ zQg6`xfI%NH3pm$d35?K^K!JgGG+VlN6W#WmjXDe$(L+8qSaJ05Xo%)a8u+Xb(ygbY zJ$jn}mq2L0HU|&G=Y=6GT2x|1Ryj~w(4O_F_RX=Wnte$p>vT;U-Fs`r9s6u&LBag# zX%9p{2xo~Uc*@IgZhgsj?FS(z!6Z9*9ZHYkiJ z1|XruwA(el&&&opqd$ZMkuN*eMVuWlqM6WLZBhG4H|LA){a=C>z3Ia>4E02`RmzZb zixT8SG&`_pA(Y|Aoz1av&US7r^Z0@PWpL2}uUKFNywY=&*_tpK(t~T5;L9BZH6x4? z+N3(Z;1>Qq49LU5p!^bscZ-Vh)Pa-cu%f4n{RX0oJ86@F@Z9G_jLcQ zt9{jMQ8k{02zEQH4n*hg)C{7gq?NMRl`}(ai$0;{%st4{%cpi2L0vYrfh(H3P(F9` z04Pnv+T(PZrY`3Dqr_;Urwg-(bNH3*2y9;hJV8s?;!Qpeh*MD>4wy6#dQ!PC8$?gw zBDlC4hLCU}bjuCpC%`cZR$TQqNLwUv`m`nDK7$^g?`Q>hwCb4KbrKG6~ZMAlspi@P@Vc5SVsP6(_o*1CIwn z-H5ENj+ll8i^$Ot(O#IWBQ<;cz?%_M^s!B>;gp-Mrw4)33lO>ro}6SmT7qd7WnK1v zKoI0evfGdT4km?*LsJ)B_KMmg1T_^;&aE9S!8G^Hp=m#}mlHc69)hTjDB5S5Eto7l z5t#mL2wzY z0*;F<1``ZZxB@&wHE@nB!g2!V35v0Vn@kWrfp7#YzEjW{-^ju%0>p&U4+zc^NNvvq zX$KiViYAGG3G{gKqHd3tK;D^U zkrkMp%_L6d&2fg?q9HdLRTU~BRjqp)gsw+Tr2PfG;fnnGS{}}zUG=u2U5y%|{iP&< zZT|AfUY^--87a~RD^_o5wjkP{bRQ+Q+gb&-#hDp&+`g@f+p7Vbqn|65te2Pj5!4rijqWH?vvwCCHbo$Ts0NZ`8 zy8Enqdi>P69s`p*a}Jnd#)?nfo?2*4C232YvnA!&j!NE&O5cWBY(XVzKb@uZwAOY? z*K#^ryUC^1l(x;3w8dm=FTuB#&e>LSY$+vgC!M#IqkHsYKha|Pd99&Fvo!Oyeq^tQX9&|y#HJ*e`QrpeJbuDnUT&fnaq3sLW z%B@caoUn~xf*-~PF?9Q2#7y^$Sba=o0Y4`raFQp*1V2>-_q5Qt|LV8ORogUlw!qX5 zDA3iV*et>v2(J5_g3U^yCBE%8$0PqYqZ@3hU!;KQmfVPayNzLx@>13LRv)S`(6@l@ zF-2*#c+eF^6~d~lUo%vr>XYp@UG^Qh`Fr*wQpW9@ zYTLH_0OfCYBUflCCSXfkE!d62918 zcWQ>vlGj1{yqYVTJ$*MMr@_2M{yV_b7ikz^k%&8P^;c0wL73 zh(EyRc=Q&1$&K9>qJZed1i8&IZG$yA+dwu+#cB03?E7X8hS2y<%q7KRU02nqiyskk zaUm8W8A5_YHSd^pEV9~a+5;{*IJuf*e~;+;13;58Va$K5F=0kImA6eM#g0xd#xn}bx@KZhQ-R)EOLTyoLD1h|9)zE`nd>l;)oHGQ`T)^gu9{AqTdTgT+?g-*jRJ{S+5l*fk7&o z7FGOTm@J>-3C5!#ge5<2o<0+#l<{x0s~6Og#vGm2b*dpwfpGF6cQ2fnp)~QmJW*X_ zVxLmFjeSYJ+u}DA^>wFiE^3YuCc4gA0@oaB+)Xc|6)s@Q(_~d!h=AsZTO>b2X$Rl2 z>DZS>BzY_D&{HT4@vMuo``VPG&N#8t*P+SJ;pH}YQ+vD=eN%W~AyaVEKJ*YG(%gng z?gM4|wyaV6DGMkVVI5X0^B{bSawkBbDUZmql*?-%;_5>KPhLegJ*FTccBiV!C@3`l z0a#2Cl$Ri^)dwA#xQacmRzT=?pN9jc=lNvm3ctyR*xbwz0wOEPbD$J|fKq>K8+uOI zRgUQXZfu8ZZULih^&lr|Ps~7inzpG?Ix!l&dkVdn`$v6$| z#|i_?39ti?ZBu>?LA%9o^xY54S)j^MZgvsE_E;f;mlZ8~r`_5_`j($0M5jRD!xk3> z-MX}IGSXa+)Zpc7xD^X=!UfY=I=`}CKS@jwaRyxbvOjB*I9>S*&R?_P8cf#&SKJ`o z7{{Cr$*~JC&I95&=Ep9BcsA8%-5f|k>_}d;%>mJMr=vBN^a{Mp?|De9nEuq$9Vt=> znL>+q!^9dQpv8GfUM}p2(}q-kbUuJ$i`1}!&Se+^VwI#(Fk>FgcsM8U73d5?6Xduf zX3SxO3Nkp=DNz8Luh|ae@zZRkaAJ2(aTP?c%wO{Av}Hf&jI%A^>ChF=zu(Ira#wozM9a0=^+3m6JAW60A;!y}*hyf^_ z9^MFFNT=xIL6N}4QZuK~pqVG9y`)r=wEWr2U83@Ks`<=GTqtw?6W+abf77J}&^L9R zL6SS{>~>Lqk{iD*i?ZdTqNqC4&0p{-&ik}SfK{)wJCdnYd<;w6qybhHO=f8GLN@~LHlDZ<&*pQ%7BxL(Z*xB_8# z_iS-h(Sy@=O>daD!C9S&7TwyI&Q|G9hbRTEw!1VKN%8GRTmzJ-l z#1*&Mnd$2+MFoD2YTHm*C##~*STWnHcetP!{4HYllJ=Fd>US*@61;4Nij8gXo3es6c#_je zX9183^}a&2wV6n$-Jz-TQVl9-JCn+NrER|_n7!CXLe?i zp5i@c_He%E!0@J9h;Sz5=Pdj7m)&~2pzJ6QWyNRB%ZOq-1(@kQEAYzM4AY=kHSBOK zT$DZ7*WeaK`Kfq#;gNILfNT!kOFk%VR?ySizYubLQ>rXql3wm!sy#U-c1#;R>f6?`o1L1 zp#ddU%L24m=|43`DrgSmqaKbmgA*?a`ZJQ9Xpkky-YK%&p2QLbxC$b8L@nJ=4!7)gT}SUeVL%i#`9%;*w!GogqUxooFxxzo zeh!tcS3ATg>3*xgA#Ky{CLguJ)TzP5_C!Qex&AASd*ZoV2Nh`7vW6;AT z$Tqjj;K~U1BqUALi6<_5A$U!d_RPo>z;itIt&8e*s-|YvH z5A?pN*=B0-pKuGva=n{wjik>u960FiNHL^NDEgOdSGKe#XLJgzXwl6mE2W|e zeUcdIC2>EaCVrhrlrk0Z4AN9@qd?V~NM?7+&n9y8HA)<9i3HNqu+mGTUPe*$DuJ8= zcR6YAp_(4D3pSn>=2U*B8v_$)7Yp&TGu5Cocgpy?YK~J%+V8XM2G_UbMBK%sYAUqF zmG}lvu1a$@s?*F#R3008s$PLxkQ0Y|-Hl2o0co_Rcr`lsQdEMy80~J)c{eq0n#|Pq zGQsg{iF3Ht6|@G%OCfAiBSio9OfU;3T_<39x!!yMu&o^&Cx3t-YJutK^B zB~9+vmV9!v^9Z}E@7pc;f~C!v9RH=~yGTdVUw2{O_Shvggz%q=LsJ(WuvUTUE>9$Y zP&E7$r%nP(o5G9s{=^C)WKXV(&up*&3z@>p8JW5jLI^vo?{r9}WQMPt5vZCWwXm^x zdcTqKU?$RK^rdSwMgh@V)@kqE6n{80Ho?6Upl7*vOTKiCu=g%+*nawdYx?j10nzZ8 zLvr@=DwgQ3Z4OzDJ32typNTq9ySCmXbV#V}Ls@pL^ z7Ter6ptpjad6Dc!%)ti@P&a{m(6U4SN;Z$=t2kX1<{M@iUMzZ6O~EP-U4QjORBN=7 zfNWCRuvZ$Jwq#KnXRlwF?yW_1%ZopvQ_JQKs?b@%EmES3-t5f^ds~d?mUlY9ZJ9!X z-WC(MMN0H-5hL75gqX0kadj9O{Ddh|KXz9Dr+Jzn<=zuvM|heLnc^-1RSrB7=GVcr z=|1fHn!lOK4!m)`06$D00cn9x9D#m0*{}x%@gGbR7dx6EeFE+rcp%O%p;$vzEe6qDGWiBB-)ELVm%}lxWJ`CRY*kq64NvcCf_~CKdE2bj}W#3gN*P zLinxhQN<23teTF*?7B>vB55xmi#f|PdyoKnW^z-?zvSLkRcsM4Vsg@&DTMDks$RRR z`gGXhHfwenyN={*#1!Aix-p1xNrS~7kg4njM9x;VSg5j9{V%9@))(Mw74sLfy2pR^$+_wz6E(t`WBZOio+5054g*>5wP z+Z|W#;`f8q0@yVbDI4phUPk1we-T$!&_mOb?C%aPxH9elEe)&aq`7cF`_9V=Pa=GM z`Q`hczl@o*?(l85@$yFEb65ZT^JdLYRo1T=s!{dXmFt?c*xbioiRt2dVOsD(C-9WH z({j8?e8tbNgVExtd8e!B^X719>WofIq;sPjbeYo$&n)ob2Np=A@<20z1M1zr|71k& z;umpc0WE765j>gzWWQ~2E?5$XcG!^$jXvID=@F^bEZ=UV1UgijES5m0Kh0Ql0#ue) zsH4lHbGLPzhBR^jtH2jOXaP2_wb*HiHcT8Gs?o(8x2Z2jo7v59c=@V_@tAW4{&#t4 z#?=oLUj?6>Kh8YxleaH^C#BvE7IJhAs2WSH3XIqKILii*@*MOAHQ9uP>RwhV8c3MW=0Iaj~?Cg|ex{ zI;^q*1t&ZjEa>_!1nV9qyB2USgYkk{Ew0p+_v?&Em#=5#)4H<3CnbGY)e+*O758s`eJk;MclCYo712?o&wr9jVz(V(4F)g1vS)T^DSO-} zR6@{ALiXCn1&F7z$A>PV5{7Tm!UCYxyyDN4Vko>x3$;T_c|+9+rxe2-nB?#f06{%5 zC3^5RXdyzgLDC0**j3~yd6)2ENP!IM12>(!P&{O_E5A%SR;z6E(PbEM- zz<(~Q9=Ai~0R9{p@VOL20K6LbC&TE$tO)2SnZklUD<%hKMKI6J6g9!qV|HN{1^gt@ zCp6VlB(XIi(g>>{*^@J(&sD%aF$^-*(=ve1Re(Mv3^CmkqQK9Ez&;tj&?A{rzi8%8 zR9W-TMB-9CJtj8`0zsdr33j3f%iv`}AmpPqfll_!8T~A1Oz;%`g>AH)@Ch6o%ybPQ z2?Qm5>J%3iQSeV2$4&X9jUFtbz@IPj%{99c%_4v{UuR{#@ zT$3RHTn%)+F^4y50jV!0nDBLl$$?Q0P#rNr1+O0r_(nM}^}+-+L7iaqU{nNK7qCz8 zMD@Uct?C$zpbFAF`+859ZUd#C>Up1bgH5fVA`(60lRmJC5mZ64=X%xwHZue1n0(cz zd9V#m6jg@MG~$vx^R})%OJEvhiK+v8zn(QXjlMKBg|$=97Mey}s!C*`N6#J(DUo++ zB&zp(O!D~ubT}*o>O(F1tl2#G(Zb@_4qO8G_ znBsYr(c^1+RMuUVxe{Ll(PofpA-=@Ob3_YX6t4keN^1!4WmVQ?A4Q1+O!;uDH<=!{ zaby_4j105-lBsYV#f7DxQej3vCaB;$*9TLP06G_G4I|TKbPK|0N*u;ajEDKtqYM9{ zh*5w!C29>L1CUoEjG0(yUhz$fG1xsd;LQmeBKb1^8o~1Gl+jmj@4W3A!$Guur%brn zk567<+ag2P)H)Al3gED`+nyhWfzND1Jr_ieZvbQB36HgQ{R~6y{18ZiC)Y`En;#8+ znCNNiux`c`MJJ^N`E%={ICXO_p5=rzp7?>Q1?o{S7Y#+{qy_nLdqG{?+g}3cjHK0{ zOw{XD0?f(yzh28ai+sKY1f9-c_U02;m?glBO+b;Zx2HOPVJ+WaJ_?3TNeS>|Mc@2e z>sALo2opxq?8(P|ypL<@36IH1JuWE2rc80AXM!`~r3Z#GjLa_`$Hp)q57{wE!Ms>% zoeeX2aya#|_0b41VKdnMDa9440bx!kpdit0e1fUa#5xOV@h7l+beSi}oRHyMG6&2S zADTBD_o7!YbXrP4FssOK!x@s5VaWHONFLQPQ80|${M#o=LYUmMbpp(SeR~ydBJ0ZOseg03AEmy@ z3b}P4XC`{xSjA;ZWWc(#=^u;jt5-{=-IqqT_bqN`B0U2?zkGF)6wnUD# zm_}Mc##ul{S?psh;Ug@t<0}rMD*$6F;Ug=PaTU)|mDn*A259WF)@p-_r3P*zF(N>2K`v59;ub>F%F(_Ir2r!*uj7bn{1a z^22oTFLdxnbnnM>?$5gRy*u{fduA>DvVpy_P<^tCJ+e{#vGKjJmcH1)o>-`U*u`Gh zs6N<;e$Q#Ir*EHUWRIt*zcZw_GrnKX(yJHPrw7raXYJ1m?#+wp%bWM)`S;_oY1tr? ztgT4Os`BvZ)|2GneS9_`BTLU7ze(!Ajz1%ck)BUC;52DiZ=eyy%*~^3aFR5vi_ipO zWaiOfsHF_25$X8=LX2qTJxp?h<}m=r1}r_G9m2!U)N z`XXIGE&PLfCMQT=V@ZEC68suQCa~Ai3}Ytd(}$_04eQFBLl|gzbZpMk1%Ch9fNb=9 zzJm4CA)THx1QRXiE>JC5U{9z4!@$X>UsOvN*F9>7GH?v&ES;tA;wg3(Y9VDTt|QeB zWuO?)t*WID?p-ya7zq0Hwa${pw-EH6S7T5BxA6JtW*7rI&zb7ev;fbAK4;w+6u>Q} z3)=={VCdbUt)~v|*EYkL2>SJMPtv~g$Nm+6#xfj{j3me45S~FD&T|M8Gxx4$JzZo^ z{SugoW=Oxi1py*AO}+R_F#-sO_36(c4CFj|`1N$*^Y<+{CUTxL`Dcj%3gl!?zd0;` zUeN6R8G?b7`#k?yvf#=7RwN5QziIzk@|Y1O8<2se_jr>nHAL5eW%Ow|PB`uOCgN;J z$`%?&5-@KUJCp*4t0ov>>jq39Mv5LQ2(*+z!)Y@F110ytH9Z-~NAkk=am1%_&?18w zf{~N!a)Vhih$?%!@W2unz%XX0(gtMU=sj>bO&u|QX@@cs^&QRB(gzM|8ZZpxyoWfU z1~?aoH=_nOfrd7h1~y}cH6aEytwWl@1DX)SnbyI~;Gs-_flS*lX6PU$#1N)+05f>_ z5^nHvW#}?;;1Xik(mH4vJY)$sV7W3}897*)V5rh{pb~1BGIEg8G(;IPKnXHDX&;;n zADWCFm^2Sdh7C#r4M{EyNX86D#tudrha!UpBIAc4ErXDOLy%wtkjul5af6T1LyzWx z$FN~XutCS=A;-7@M~LA@>tJK>P$R@Zqji`uc#sijoNQ^7EM|-hWQ5E4_g}ziyaFC90^++2a6pAiyi|rkAQ`Ze*q0bEDb@#3_ySm zKdcWv#1B0H4LmFjJH!k+fDSpV4>-gRH^lbo8+-JF`tu=r^R0dP!9Dpvz0ym4(lI^K zVExg{z0q-f(eXXemVRjeUg-Ie3EB~h-t1QJI9J!$!4l{RQc(0I);e_=GnBSC{yn%G zWfgH%!X$pA5SkxwBG{>5jsj@hIP&_jL0~E-rxCfy(GvmaV&|Q}YT`$zc<5mXLbc4# zf(1}xIAC^7&pb(}BK8xg{=-?UR>E2J4#(L zb%je)lQL;LQkMsEuRrjMvff|x4FhEz=1u=X-sfy2eb=-H)T`)4i{5DhBf1C_n`-k5 zB`yBQ@v$e(bW8O&hI53UhHC|6cdGLRj3Jmq{8?Tq2#W{1p8B%tt_n;(EKpC!W>LF8 zpyL(2Z-;GV=|qi! z_Y*$ht6$i~|6#w6%`uqs5~!#abR!h!Q}>d9G(k16CRQNYe5ehcILDn+=XN0SOCvDx zxO4KeftjTyrKls>EYbWkU#~SBAjDHAF9VnyQU4N0gd&VQ=v4rXuD8wT^g$SD&?{dW zRh6yGIP!)OgjAC!D^m9#ni_)G6p3@2T7BuYn8e8N;n;s>_PM4(P=esp?8$Ix+Npbb z&Sf^!)EyvuDZ9QQEzLtLl@UoLHCnyt)m_J;>K_s1mZ#_b`EO4G!!%sf_ied7^)Z=Y0F(U?vw#4ewC4qk=LBrzISKIO zNj|qvIcZ+Rm-8tSXE?R`((6`c0C|E4Qk`kik0TVT^P|Wxgn_BqO`6e@o6O_bM?uFCV@sONn;2kYzOyyvyXeMz2ichKY>oLYx-lQ$h}RqM zfsOXnY5SfaoQ!#*>IHdt62T19Z_ANE?e0@ zBM?j>ISmSC8d}7bu;x6y%AN+@L}wg6Id!q zQm8U%fEwsUmB%rCY-yY$O&JE2M3Nk?92&s}n!aUF&Iqc|XLIVWwk1hPw_eM*!I|U= z4(B(udQr>EHa}IUjbt`~5hI1V(TmqBI-mk&ujn0>k=zVWoSz}UizA!P26JZ(P@JP7 zz>DLtLUr`r)j&jdfok$%)RIS@RO@k!G!W5Ipql&`;Vws2>@Cz$hRzGc7D0+NRbiMh zC=)#v;Ki|f!BrKRZ7PPGb3R)9=p4z%Xp5p5=`eznZuI0;La-|5oXTKEExC$jq<~09 ziph;p{gEO5A;MaF$nUde5k+Z4B8QKfIhL4P{zqs=S&_sLwr=!gcl?CoH^UOO{AVP` z7Na67niy&lLRMM)%9jXf#6=ga0b^(vh83&#mcGp6a%I5aT!u?$E)@x3&0h8+go%zY z+(;cq$Y{r$a5Clz5ItW{v`)-Y35-I)2jI%s<>e153>mL+uRs^eHwxH9*B< z1d6K{IVAfxnwW|r>%!6ENhzil)DZ$JK@LK{x8n+sL>Rx{0}4-6^*4bCt^JOl71}aG z7zvqLOtSQ;UdF_k{6LQ5Fu;st22%(t^|LsVnF^(_Q_Ud^WpE_BA$VjaO);$ah$}JU zssm!ELK@hr!O_e?Ia}Gr@r2D>d%gl>soy*o7cdi1hxsmMXx@(#$&Z65ytpBR5AVf^ zs(?nF=I_a1@g_Gttiiw2#%w{n^V!W z&Axf-JN%aIn~MEC;F``?k0!c?Ena*pvAiYmMVis<6&SI8ZZvpv+MP%h+@kg~x+2w) zM3;ub^u3XV#W)v>_DFOa&6mRFglm6Zwh+;5EVs zR-?h2)B6j?0C}arhz`}!;LfSx+cwF$8R9yKml;Z%=OJt!^bUtpP1nW$_0PX$Tax1C zfZErH%v%EFR1E{mwC3Wn*238GzG5WsG2clqskN(K>WR| zi$fXJGPYoeSF~nxCMY(i>G1EglHHYi()wUUKR92P5r-;!8W6&G{CT1YgFhfwAhm6L zw2&auVvSq|pWjQ+Lc~IhC^0Xj?y1PXujQc(FXC><6DjiuAxiMak;4faga#v~=rJ(` zh42Q?cw2tMWV3k1P9G)+;!Dul``sABU5s4zx{r9T&p<@;uMQ`ZgRseUI(ays7KBf$ z^Jzo*h+F@f(TVNx#O82X2~+I;RQVkIU0fw(0~4*9iPfFt=~$IH;`_>tzq6uz!|guu zQFg=@)&D|_AUUknkE%=s2GL>#mi>s9U}qJ0-_ZZV*=hjd{Uk5lS$xpNdvve;{H19N zq9}4uXAw}eBj*7Dd=IGIpQ^IU_GnA@V%ejrtRQ$V0E^XJ_6Pwwj^v^Kgy5|J$uc?^ zoO$hWbHH?~7W#`NC8fK{`j-}WFHKd16LmwD+XC8Sgh}DQJg3IFfDMFIV*tk*y_KXq zkq4qX(4B2bRG8de02X63_6Pww#JhCN5C92gOzqu02@4?JHM>4b!F~Z)yw~j!0(1;zDdomid8mUWL?5|AM=Z(vnoL4Xs~)(r)o6bD-iD!d{W4gN4N&f-tlFHZer-s?%+$-M zg#xwCi84Rr{r!Yb<_yg^mn(f8FEb;6tkfLQ|4)o{!dBkmg+CuQv zp#)Fe=*^Ebw^6+p)472%)`mGu5i^FmQ!i^w-eCNAM0ydy*bF6Kh*P^CMF473N&!?3 zpLvO^?zJ3$g_E*^iM}`87{d|OK>ScW;f88wUUgHKU2|w!+S$cMr;WA{rjF3VR^`)x zI1S1s-s#jVIhk)J9)U3J`ahtEFXK-qTbkko3ZsWo zkyk}Me2oGLGUJX1Va>xr1)RvOP4pDlvE@LhEj&;RZi}YNNV?AwO2Z6N3^Gp5p^QIz zx+O-x9!NS(U2m`CqiT0H1?1|lKm0kI=LAA5qLY`NOtw+WlBXErv(A4} zG6%|E-cXOC`Die5N}OWyWp&!Jc~Y^gR2acZHF>fsa7U-f#caD^1d`u@n$eeAi4IL& zL^D%i8|wRe-3h%`0PTS-1Qn^XUS+d6x14pC408mMT;TEl(m@2ZBx z?&++D9Xga%8TG}of+^a%wqRfGuZTvW-JsiO&ND1|y4J+(%dsNj4rNEI+X^6SO>ko3 ziXp_EtUMTq@5s7q+C4TJgQ&ZLPYx+kLfnN6J71_9WQ6Cs9d5SMhG&kSistns{zMR* z!^oqbM~gG5zC6(G)s{SwSEPa|x8&iB*?MsakJFr5-KnxUE=Ll*{0Qz-pMi*m)oAeK zGaTQx#T8xm(=_y*M~X1^IC#gB^l= z(-zq490otw33X|9W8U|zIA)pt6h|(56E&iULOdFR7>SdeVTfs93O~*6M01PDN9qe^ zJ46aNX3B&ukgy}H81giPIFV-+qRbb00(l_gB`lG{6RS7Hvd+tbZ7CB^af&6EZ0J@e ziWNFmC?9W>vZ32yOQMjfuVm|hM6^Kmfo^7-%q>NZ)0<4;iY5>h(B#dWL8#UZsm!YK zj|5r104&o(1T$4}ttSCt%ywBY~|7`%A$#ChyqQH40Pjzee2Xyo#F-tEe<}b(cq{cFp zeBNbz`C~Z836OjcG^01SJ}c=NR0mv(ZnqeIGcd_<7U0WsSntvGBx(dl9Z2xgjh@_^ zBd)VuSw*lKAjHfQdFjdI_z9Ft6Cin7w|6d)hvi7__wOfs!dJhri~mD^dor`Z7#~+# zWGB95my?HlTlQ>^EX+%+FIyvsXlG5$&aCnl)3M*UTfBfIUQfm_Uy?u!t@MJp6=mMj zS1qY0i9FP{=re?y0Y-kE%mSRbv3)8t{QJq{AX;;Bv66B$ICI$1-8pa_;$I|{6Drt+ zvtPc1B3!(hUOC9sSN^_jPRA~c08J@f)8Ecc59pQdH`4WVPXy8i4Dvq%ksISq%+9QM zlOp(rD9P0(ir?)xqOyH2yP)brrj_$0`IU=nS+U4vU-{`kzK{Cy4byfh`EuW$yaqG8 zP~|cYV_*n-lActUeZVR?M?6{aP0S%&hedTjTxWQb)v7UE#q~Qb_b;?L+acPvjU;Ra z)rM*(VIkB9i2S7~qceGeCRV)Z#t@c&l*e7xzD2SVD4dnJaAK1m$Rpan(35};^_~qhKkH3-wb?x5wO-b#e7?^yi=HFJ||cM zsqKZ}GQevM09q1At&BoRi6;+$)qtcu(m}jvE%k7g{HRV2*tQp05)~u?(bv$x~65z%o_E(@JYhwrXz2h?$pqZ_siUA7E21FV(vnQ}q+VPpq=V_MpD`;lxsN*R4 zXagb*n%NWBDed^oW*e$P=JNn3mz83GqTH{XHx+}Z@C?k(3KvMcLFvyZiW4mLRZOn~ z$+0&i47UMFdH04fJDjS%WXJ8yMp$M+QM`Q|O`2-6a0r=V1!ZE57B6avN}`UHh}7cD zPLv{h0srPgNgqBbA_)E&`R{5n1e>@rG-32^c>FO|yBrZTmtAI)w5SDPFeUl)E zr=UY`5>W2)(gm5l8lqZ+gEP3tA4=0gM98Q$im)AaQoX7{f$Vt>z*3)Ms=4hz+GY4k z4om&~(|`Qg4CBzLBYSCn{^hS`49CtPwa16eWV7Hc%tf0-lC#oKK_?obx?s6SfAg95 zM)>h_XU&K5VZAYG3hc3DY8T;9vOK@n4w9 z?7_E@W@&HzBF!%I-JIU7$}96%>&ueYL2ljWYFv){g;{>Kca6I|*yDrxH~xrT{A2ov zvSDf)4qy2y$}PK~#dW&el}#))L-0ALd}1lR2Yty>Rv2~oPv}@?*H_QadQv@1WB(r) z)TE`qxw^I`Cqa|c#4<^Ivyl2`p8RH({$?Tl%{=|hB>kN|`Q0q_-6Zk7J@Nf4@%<$6 zgFW%XEb+r6@uNNQ<1F#xB=M6y@zX5v(;ZW2lNk)(5Pv;5T3UF)-5)x2k0k^tkHE~v@Ih*@{qcH$f5 zfBesj%L%8=e-}>q{pK0n=Xdwd_I!$6w`TCQN$l$eZ=bg}*ZB?30q*~{BwtZI>!PQd z<1k;%F>jk)pS_~?n2qL$%KHc{H7@IHH2fj#2wN7JH>&dSNp2jqADY$9Ua(-LYi;1 zxi21y+nc)whyr~&98g>S!9{m9I%a6UCq&S&WK#Hj*x<)UQl@Y39wQ6Z;$iLvZ=3Yq z9`F|3ZSnPXlb|56@=L0vEN{qjTjnL*2?von6zy>~25W?hC!9Y%?Vh)H5I30~u|up? zan+Uk1G0JAB=z+8u*JnS@v~dd1Ttq!IDC4{pYYx0cJ~1B_F>a|j-(zc`D)_4rlW%f_p0(wKNgkNEy>x4GNgLo~Y7 z?YqevpKdQwmxmMGd5CY|M~td_go@{z$D5no?GC^_yT|m%SEjifFIA7*=MB1l+}_?@ zM;9#9Y#-@nhi@JqAMf$QEqEnCs(_NjW(O09-{UC-2LGFH?Em%~_|LD4o5$@_vAubY zzRe30?xi=39t-rmy?wxeP4e~TvFc-!$Vai+-akB{JGAs^q1-k7zF2@ZP(Zg)K-=4! z9lF1JxW0j?=st7Wvd&*fE>`S|jT=;lyZeXhhnt&wRD?Ex3cRQD8awF{v>*M z52Py9YBhYld)VGR=XW`R{&r$J^OwdA?+wm3XLo*cN7~nTi=S>`dkv}v70QRW&FAJ^ zJ~z+#HF*-y^)-Y}T5$P$QvOtY?d~@Z1!>;i-aJ8mu^ce!PAz##Ug<`TpY*I38q(Wb zZ_ey`cYC|dA0HkbAR1sbc}$gUTqc);+b%DUpO*#Rp;Ogg(k)-7p4~@x2;i6ZlI|?r zHy3|99e(}%)0e#b_48ubNtPjW7u(MZ;qiiPU)vn;#Yp|T4x?6VPdxD8HZeZU!+#}T zB;O6^!(LCaY#d$VI)`|h<#@-Jk8EGH89EVV_=D}+)n&IuRjHL|j z{SOLxNgyw2pY<==Wc&Fq`c%Ve`pTJ6MsN1n8n6?d{CBXD)q#AQB)wmS02MP|KbK?M7MBbkLU7j zRyfU4y3rFio20ws>DVnNm!eK^=WOch_NH$H@l!HDT#gW2)DUvf$iQwHy0{CYjmI;U0L_p_4wfX94`R*D3dUk1I(p>p(6hxLN9)lp%-6xsH@)B z7Ui!ms4e>NmxdX`N9Bkxd1d_T5D@UjsB+|~gr@M}P2y9Qe{J6B5w-MjB2UowLh^HL?Il06?V0O;;M`G0s6_HDSTLy_>|;te9Zsi2j1cMU;p`kf64z9 zV^^NtOVjqGSdv`qecQBpS|9E*a^}J+C;-G8+%^4ky;O!n#_Tc__Ae*E@>k6#d0DotHZ_L$mPYaO;;{` zbm^SwQH`XH)Unw-KCHEb?!0NTOOcE*rv}oBVg@(LOe-JaFKOqx7wuVMcj_!D&vKSj z$#`nKU`8hie6Y=jw@BekmdEFx;`A43eWU8LD;@cXvs}Fx$~5jAMGD4J{N~LICz5BSnv|DX#{FNY@#5c!k(XRI39mKzy2=h^on7i(lxa6IF3)D`|-Z)JM>fXX9x7qq)F)_ivx-g`67NMpc zE0A9lE0OfsoO&l6d=g$j1+6D^X3yTtY6Z=jLtYhhi^_W2yu0?*YOH5zor#N-T0iY? z+_u_LJE6~K7USlC`<67^y-SY;!9xm7e>tQdOA$X;=Hh1=;_*(RCu()8LmHY6nZ%j6 zNWi{}_)j-#xY|`sbM&fa#zgM)WD-V@TwJm%H;(*U(V>S+XQ?D}=gGe@WxtmA#9Zpo z(y6jD!coOu&|YML;c}N zn5Xd}%u-Ooe?Q{-3tg_!|ALMe^i%la?_YkYKV0eZyWz=KB=RB5lT5QXxL6#U>T|!| zF*+wMlKD~RiKgU#6?O;Ce$|-5+Or~4s7F<0!!~I)hBSgp1;&6D8yPH$oR7B8D)#Zc zc3YYIX_{3cau(YDlBZy`)%QPNTweU6Avd|%vuQNTpZ_Y|VLgdB^4Xd5B%v#D-r+5i zDLvm`{GCxI#?4-H)# z+LQLTJ4YprBSn_tlBQrigY=4gnU*fM_t1INT!+rN$~ts>A{Jl$OLsJ^YxberuSx(}ha$mcXwL>^ckN}qg;8FsxUQ7{q@f#1k_(23Zh-;fY_Q-V} z-H|kl%zF0d4+k=&)Sx2kKacBe){&B$$dMRJa>l4fhl;r*j#>9M%zr6BO;LtH`=Dyl7ZDuVyrZ+#bA!k3bbVm$w+ba}SV3g$k z#A)J7mY5e9^23>YK;5B|52;2}{vb69|B7eVMn|$T@hBF8p_0&p zrmR4`ih@d&E#svJT))T_(Tm#hj`igkimnIh9i2oRlOnLoY;%4&W_COU^%% z6@!Bdn3Vx|z@au~qn0Kht-sEIwV%;Y7hLUlzN!9H(TRgRk6TMA8l z$rJObBXvbu)=sZjAtA8V6beEsO`##C&g@RxoYV=6bY&ceT5^E17Zut*X{fgp1)w2u^Z2byf(Y)7e z_mlJPyCB{AY`}2o!y0rChIO|3W(@00&#{?+h4sp=0wLVbwW)x!BHQi)0ph@S6mS)x zZ7twW9Cm8~N7CvZ1OAlZcNuV`t#3CFKs@1g1FjP0`wsZ)NVVmFqlVeu0|5#WZZF`f zV!qFSzmD^7G2p0fd9ML~q37CLz)_xclL3Ea5IYMvvKCtm_;Wnxt^$r)7Pc4g*LR*B z1ss*xwifW01+k%kBWba-fIr7nn+Z7bp6x0Spa5hy0aw0aTY&(Yz}pG95?1#WSZAGT z$-ITFb?v+tQ_!{Z>rC}U^j=KC7tXJ>w873D7MjZ4ctMS!mDqD^`o=}(o+1*j zE_ab(US{s8BgyJ=7xuGd=AOh8t}%C^Z(3~bt>7%n%-z)3)|-230$XG5!dh8u?n@JL zy}1)rb>X=$S?Go5PK5Op=)ROwEI@anJza_JNj!Q5x(i9M6y1{=@X~V^qWT(iPrixP zpu6y%FGgSIo_bZah4o!zdA{t@i#YHy z*LA{^^{#$o&$ZmuLB_(mS1(OqD_z$)t4m(JI07$vU1vI3{pv*)bn)vtRecq#AJe#1 zunych|C;QKYhc#}EG~xik_E8(b)D|84AzSp@LJe)nv+GbUSyLkj9sT~S|RJj9bj$j zI?G{^tRuCvt?XQ)zOuwKR7$EPo?cS)wVt6;AlG_&(Rc3yI`6tqFA@3zP!B1|Rw=I0 zH!fEUB%XB5qK}Z-;>AEgNmnlV2%0Tn3>1`n{i2U3!$QVDS;^Kg`pB6qW?Yw&aw(pL z_`;ezU%t?b@|+m!tMdru9%DA3mp7+6FRRO*9+s++DexPv=+wYJp)D;ir{Q$yFCy{9 z4?OXJmI*W)Q2l|A9sBq4`uh4t2$@is!}oMI+qOfM!{OIMTuJsH((oOv)0_}w75YE< zlI@=4Yb!USESj9Als10w^gqPacW*nFv%$-*As>aum$ExtkrdL|KmWWLkJo`$z_ LcJ==U%D~KLlSU`D From 95c5ba23e5c087ee956f666d0a188e2c8c6fcb50 Mon Sep 17 00:00:00 2001 From: jnke2016 Date: Tue, 19 Nov 2024 14:57:08 -0800 Subject: [PATCH 03/49] homogeneous neighborhood sampling doesn't support edge renumber_map along with offsets --- .../pylibcugraph/homogeneous_biased_neighbor_sample.pyx | 6 +----- .../pylibcugraph/homogeneous_uniform_neighbor_sample.pyx | 6 +----- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/python/pylibcugraph/pylibcugraph/homogeneous_biased_neighbor_sample.pyx b/python/pylibcugraph/pylibcugraph/homogeneous_biased_neighbor_sample.pyx index e2476de1607..3c278bc06c4 100644 --- a/python/pylibcugraph/pylibcugraph/homogeneous_biased_neighbor_sample.pyx +++ b/python/pylibcugraph/pylibcugraph/homogeneous_biased_neighbor_sample.pyx @@ -383,8 +383,6 @@ def homogeneous_biased_neighbor_sample(ResourceHandle resource_handle, if renumber: cupy_renumber_map = result.get_renumber_map() cupy_renumber_map_offsets = result.get_renumber_map_offsets() - cupy_edge_renumber_map = result.get_edge_renumber_map() - cupy_edge_renumber_map_offsets = result.get_edge_renumber_map_offsets() sampling_results = { 'major_offsets': cupy_major_offsets, @@ -397,9 +395,7 @@ def homogeneous_biased_neighbor_sample(ResourceHandle resource_handle, 'label_hop_offsets': cupy_label_hop_offsets, 'hop_id': None, 'renumber_map': cupy_renumber_map, - 'renumber_map_offsets': cupy_renumber_map_offsets, - 'edge_renumber_map' : cupy_edge_renumber_map, - 'edge_renumber_map_offsets' : cupy_edge_renumber_map_offsets + 'renumber_map_offsets': cupy_renumber_map_offsets } else: diff --git a/python/pylibcugraph/pylibcugraph/homogeneous_uniform_neighbor_sample.pyx b/python/pylibcugraph/pylibcugraph/homogeneous_uniform_neighbor_sample.pyx index 3c6cdf77420..32ca1f10c10 100644 --- a/python/pylibcugraph/pylibcugraph/homogeneous_uniform_neighbor_sample.pyx +++ b/python/pylibcugraph/pylibcugraph/homogeneous_uniform_neighbor_sample.pyx @@ -378,8 +378,6 @@ def homogeneous_uniform_neighbor_sample(ResourceHandle resource_handle, if renumber: cupy_renumber_map = result.get_renumber_map() cupy_renumber_map_offsets = result.get_renumber_map_offsets() - cupy_edge_renumber_map = result.get_edge_renumber_map() - cupy_edge_renumber_map_offsets = result.get_edge_renumber_map_offsets() sampling_results = { 'major_offsets': cupy_major_offsets, @@ -392,9 +390,7 @@ def homogeneous_uniform_neighbor_sample(ResourceHandle resource_handle, 'label_hop_offsets': cupy_label_hop_offsets, 'hop_id': None, 'renumber_map': cupy_renumber_map, - 'renumber_map_offsets': cupy_renumber_map_offsets, - 'edge_renumber_map' : cupy_edge_renumber_map, - 'edge_renumber_map_offsets' : cupy_edge_renumber_map_offsets + 'renumber_map_offsets': cupy_renumber_map_offsets } else: From af465f9976f3dac7bf4bfb91eb2e0b7a5f7d4948 Mon Sep 17 00:00:00 2001 From: jnke2016 Date: Tue, 19 Nov 2024 15:01:20 -0800 Subject: [PATCH 04/49] update docstrings --- .../pylibcugraph/homogeneous_biased_neighbor_sample.pyx | 2 +- .../pylibcugraph/homogeneous_uniform_neighbor_sample.pyx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/python/pylibcugraph/pylibcugraph/homogeneous_biased_neighbor_sample.pyx b/python/pylibcugraph/pylibcugraph/homogeneous_biased_neighbor_sample.pyx index 3c278bc06c4..cbd7a5dcffb 100644 --- a/python/pylibcugraph/pylibcugraph/homogeneous_biased_neighbor_sample.pyx +++ b/python/pylibcugraph/pylibcugraph/homogeneous_biased_neighbor_sample.pyx @@ -216,7 +216,7 @@ def homogeneous_biased_neighbor_sample(ResourceHandle resource_handle, >>> sampling_results = pylibcugraph.homogeneous_biased_neighbor_sample( ... resource_handle, G, start_vertices, starting_vertex_label_offsets, ... h_fan_out, False, True) - >>> >>> sampling_results + >>> sampling_results {'majors': array([2, 2, 5, 5, 1, 1], dtype=int32), 'minors': array([1, 3, 3, 4, 3, 4], dtype=int32), 'weight': array([3.1, 4.1, 7.2, 3.2, 2.1, 1.1], dtype=float32)} diff --git a/python/pylibcugraph/pylibcugraph/homogeneous_uniform_neighbor_sample.pyx b/python/pylibcugraph/pylibcugraph/homogeneous_uniform_neighbor_sample.pyx index 32ca1f10c10..bb88ffcf6af 100644 --- a/python/pylibcugraph/pylibcugraph/homogeneous_uniform_neighbor_sample.pyx +++ b/python/pylibcugraph/pylibcugraph/homogeneous_uniform_neighbor_sample.pyx @@ -211,7 +211,7 @@ def homogeneous_uniform_neighbor_sample(ResourceHandle resource_handle, >>> sampling_results = pylibcugraph.homogeneous_uniform_neighbor_sample( ... resource_handle, G, start_vertices, starting_vertex_label_offsets, ... h_fan_out, False, True) - >>> >>> sampling_results + >>> sampling_results {'majors': array([2, 2, 5, 5, 1, 1], dtype=int32), 'minors': array([1, 3, 3, 4, 3, 4], dtype=int32), 'weight': array([3.1, 4.1, 7.2, 3.2, 2.1, 1.1], dtype=float32)} From c3861e1fb6a526d2978666d5bc23b7a356f7ee25 Mon Sep 17 00:00:00 2001 From: Alexandria Barghi Date: Wed, 20 Nov 2024 07:10:25 -0800 Subject: [PATCH 05/49] fix batch ids --- .../cugraph/gnn/data_loading/dist_sampler.py | 281 +++++------------- .../heterogeneous_biased_neighbor_sample.pyx | 1 + .../heterogeneous_uniform_neighbor_sample.pyx | 1 + .../homogeneous_biased_neighbor_sample.pyx | 1 + .../homogeneous_uniform_neighbor_sample.pyx | 1 + 5 files changed, 83 insertions(+), 202 deletions(-) diff --git a/python/cugraph/cugraph/gnn/data_loading/dist_sampler.py b/python/cugraph/cugraph/gnn/data_loading/dist_sampler.py index 056e714233c..336d5604c58 100644 --- a/python/cugraph/cugraph/gnn/data_loading/dist_sampler.py +++ b/python/cugraph/cugraph/gnn/data_loading/dist_sampler.py @@ -75,7 +75,7 @@ def __init__( def sample_batches( self, seeds: TensorType, - batch_ids: TensorType, + batch_id_offsets: TensorType, random_state: int = 0, assume_equal_input_size: bool = False, ) -> Dict[str, TensorType]: @@ -87,8 +87,10 @@ def sample_batches( ---------- seeds: TensorType Input seeds for a single call group (node ids). - batch_ids: TensorType - The batch id for each seed. + batch_id_offsets: TensorType + Offsets (start/end) of each batch. i.e. 0, 5, 10 + corresponds to 2 batches, the first from index 0-4, + inclusive, and the second from index 5-9, inclusive. random_state: int The random seed to use for sampling. assume_equal_input_size: bool @@ -102,78 +104,6 @@ def sample_batches( """ raise NotImplementedError("Must be implemented by subclass") - def get_label_list_and_output_rank( - self, local_label_list: TensorType, assume_equal_input_size: bool = False - ): - """ - Computes the label list and output rank mapping for - the list of labels (batch ids). - Subclasses may override this as needed depending on their - memory and compute constraints. - - Parameters - ---------- - local_label_list: TensorType - The list of unique labels on this rank. - assume_equal_input_size: bool - If True, assumes that all ranks have the same number of inputs (batches) - and skips some synchronization/gathering accordingly. - - Returns - ------- - label_list: TensorType - The global label list containing all labels used across ranks. - label_to_output_comm_rank: TensorType - The global mapping of labels to ranks. - """ - torch = import_optional("torch") - - world_size = torch.distributed.get_world_size() - - if assume_equal_input_size: - num_batches = len(local_label_list) * world_size - label_list = torch.empty((num_batches,), dtype=torch.int32, device="cuda") - w = torch.distributed.all_gather_into_tensor( - label_list, local_label_list, async_op=True - ) - - label_to_output_comm_rank = torch.concat( - [ - torch.full( - (len(local_label_list),), r, dtype=torch.int32, device="cuda" - ) - for r in range(world_size) - ] - ) - else: - num_batches = torch.tensor( - [len(local_label_list)], device="cuda", dtype=torch.int64 - ) - num_batches_all_ranks = torch.empty( - (world_size,), device="cuda", dtype=torch.int64 - ) - torch.distributed.all_gather_into_tensor(num_batches_all_ranks, num_batches) - - label_list = [ - torch.empty((n,), dtype=torch.int32, device="cuda") - for n in num_batches_all_ranks - ] - w = torch.distributed.all_gather( - label_list, local_label_list, async_op=True - ) - - label_to_output_comm_rank = torch.concat( - [ - torch.full((num_batches_r,), r, device="cuda", dtype=torch.int32) - for r, num_batches_r in enumerate(num_batches_all_ranks) - ] - ) - - w.wait() - if isinstance(label_list, list): - label_list = torch.concat(label_list) - return label_list, label_to_output_comm_rank - def get_start_batch_offset( self, local_num_batches: int, assume_equal_input_size: bool = False ) -> Tuple[int, bool]: @@ -240,23 +170,10 @@ def __sample_from_nodes_func( current_seeds, current_ix = current_seeds_and_ix - current_batches = torch.arange( - batch_id_start + call_id * batches_per_call, - batch_id_start - + call_id * batches_per_call - + int(ceil(len(current_seeds))) - + 1, - device="cuda", - dtype=torch.int32, - ) - - current_batches = current_batches.repeat_interleave(batch_size)[ - : len(current_seeds) - ] - # do qr division to get the number of batch_size batches and the # size of the last batch num_full, last_count = divmod(len(current_seeds), batch_size) + input_offsets = torch.concatenate( [ torch.tensor([0], device="cuda", dtype=torch.int64), @@ -267,11 +184,13 @@ def __sample_from_nodes_func( ] ).cumsum(-1) + print(current_seeds) + print(input_offsets) + minibatch_dict = self.sample_batches( seeds=current_seeds, - batch_ids=current_batches, + batch_id_offsets=input_offsets, random_state=random_state, - assume_equal_input_size=assume_equal_input_size, ) minibatch_dict["input_index"] = current_ix.cuda() minibatch_dict["input_offsets"] = input_offsets @@ -286,7 +205,15 @@ def __sample_from_nodes_func( if v is not None } - return iter([(minibatch_dict, current_batches[0], current_batches[-1])]) + return iter( + [ + ( + minibatch_dict, + batch_id_start, + batch_id_start + input_offsets.numel() - 1, + ) + ] + ) else: self.__writer.write_minibatches(minibatch_dict) return None @@ -493,21 +420,13 @@ def __sample_from_edges_func( if len(u) > 0: current_seeds = torch.concat([a[0] for a, _ in u]) current_inv = torch.concat([a[1][i] for a, i in u]) - current_batches = torch.concat( - [ - torch.full( - (a[0].numel(),), - i + batch_id_start + (call_id * batches_per_call), - device="cuda", - dtype=torch.int32, - ) - for i, (a, _) in enumerate(u) - ] + current_batch_offsets = torch.tensor( + [a[0].numel() for (a, _) in u], dtype=torch.int64 ) else: current_seeds = torch.tensor([], device="cuda", dtype=torch.int64) current_inv = torch.tensor([], device="cuda", dtype=torch.int64) - current_batches = torch.tensor([], device="cuda", dtype=torch.int32) + current_batch_offsets = torch.tensor([], device="cuda", dtype=torch.int64) del u # Join with the leftovers @@ -521,14 +440,11 @@ def __sample_from_edges_func( current_seeds = torch.concat([current_seeds, leftover_seeds]) current_inv = torch.concat([current_inv, leftover_inv]) - current_batches = torch.concat( + current_batch_offsets = torch.concat( [ - current_batches, - torch.full( - (leftover_seeds.numel(),), - (current_batches[-1] + 1) if current_batches.numel() > 0 else 0, - device="cuda", - dtype=torch.int32, + current_batch_offsets, + torch.tensor( + [leftover_seeds.numel()], device="cuda", dtype=torch.int64 ), ] ) @@ -536,11 +452,18 @@ def __sample_from_edges_func( del lz del lui + if current_batch_offsets.numel() > 0: + current_batch_offsets = torch.concat( + [ + torch.tensor([0], device="cuda", dtype=torch.int64), + current_batch_offsets, + ] + ).cumsum(-1) + minibatch_dict = self.sample_batches( seeds=current_seeds, - batch_ids=current_batches, + batch_id_offsets=current_batch_offsets, random_state=random_state, - assume_equal_input_size=assume_equal_input_size, ) minibatch_dict["input_index"] = current_ix.cuda() minibatch_dict["input_offsets"] = input_offsets @@ -558,7 +481,15 @@ def __sample_from_edges_func( if v is not None } - return iter([(minibatch_dict, current_batches[0], current_batches[-1])]) + return iter( + [ + ( + minibatch_dict, + batch_id_start, + batch_id_start + current_batch_offsets.numel() - 1, + ) + ] + ) else: self.__writer.write_minibatches(minibatch_dict) return None @@ -700,6 +631,7 @@ def __init__( with_replacement: bool = False, biased: bool = False, heterogeneous: bool = False, + num_edge_types: int = 1, ): self.__fanout = fanout self.__prior_sources_behavior = prior_sources_behavior @@ -721,13 +653,19 @@ def __init__( else pylibcugraph.heterogeneous_uniform_neighbor_sample ) else: - # TODO change this to the new API (rapidsai/cugraph#4773) self.__func = ( - pylibcugraph.biased_neighbor_sample + pylibcugraph.homogeneous_biased_neighbor_sample if biased - else pylibcugraph.uniform_neighbor_sample + else pylibcugraph.homogeneous_uniform_neighbor_sample ) + if num_edge_types > 1 and not heterogeneous: + raise ValueError( + "Heterogeneous sampling must be selected if there is > 1 edge type" + ) + + self.__num_edge_types = num_edge_types + super().__init__( graph, writer, @@ -753,95 +691,34 @@ def __calc_local_seeds_per_call(self, local_seeds_per_call: Optional[int] = None def sample_batches( self, seeds: TensorType, - batch_ids: TensorType, + batch_id_offsets: TensorType, random_state: int = 0, - assume_equal_input_size: bool = False, ) -> Dict[str, TensorType]: torch = import_optional("torch") - if self.is_multi_gpu: - rank = torch.distributed.get_rank() - - batch_ids = batch_ids.to(device="cuda", dtype=torch.int32) - local_label_list = torch.unique(batch_ids) - - label_list, label_to_output_comm_rank = self.get_label_list_and_output_rank( - local_label_list, assume_equal_input_size=assume_equal_input_size - ) - - if self._retain_original_seeds: - label_offsets = torch.concat( - [ - torch.searchsorted(batch_ids, local_label_list), - torch.tensor( - [batch_ids.shape[0]], device="cuda", dtype=torch.int64 - ), - ] - ) - else: - label_offsets = None - - sampling_results_dict = self.__func( - self._resource_handle, - self._graph, - start_list=cupy.asarray(seeds), - batch_id_list=cupy.asarray(batch_ids), - label_list=cupy.asarray(label_list), - label_to_output_comm_rank=cupy.asarray(label_to_output_comm_rank), - h_fan_out=np.array(self.__fanout, dtype="int32"), - with_replacement=self.__with_replacement, - do_expensive_check=False, - with_edge_properties=True, - random_state=random_state + rank, - prior_sources_behavior=self.__prior_sources_behavior, - deduplicate_sources=self.__deduplicate_sources, - return_hops=True, - renumber=True, - compression=self.__compression, - compress_per_hop=self.__compress_per_hop, - retain_seeds=self._retain_original_seeds, - label_offsets=None - if label_offsets is None - else cupy.asarray(label_offsets), - return_dict=True, - ) - sampling_results_dict["rank"] = rank - else: - if self._retain_original_seeds: - batch_ids = batch_ids.to(device="cuda", dtype=torch.int32) - local_label_list = torch.unique(batch_ids) - label_offsets = torch.concat( - [ - torch.searchsorted(batch_ids, local_label_list), - torch.tensor( - [batch_ids.shape[0]], device="cuda", dtype=torch.int64 - ), - ] - ) - else: - label_offsets = None - - sampling_results_dict = self.__func( - self._resource_handle, - self._graph, - start_list=cupy.asarray(seeds), - batch_id_list=cupy.asarray(batch_ids), - h_fan_out=np.array(self.__fanout, dtype="int32"), - with_replacement=self.__with_replacement, - do_expensive_check=False, - with_edge_properties=True, - random_state=random_state, - prior_sources_behavior=self.__prior_sources_behavior, - deduplicate_sources=self.__deduplicate_sources, - return_hops=True, - renumber=True, - compression=self.__compression, - compress_per_hop=self.__compress_per_hop, - retain_seeds=self._retain_original_seeds, - label_offsets=None - if label_offsets is None - else cupy.asarray(label_offsets), - return_dict=True, - ) + rank = torch.distributed.get_rank() if self.is_multi_gpu else 0 + + kwargs = { + "resource_handle": self._resource_handle, + "input_graph": self._graph, + "start_vertex_list": cupy.asarray(seeds), + "starting_vertex_label_offsets": cupy.asarray(batch_id_offsets), + "h_fan_out": np.asarray(self.__fanout, dtype="int32"), + "with_replacement": self.__with_replacement, + "renumber": True, + "return_hops": True, + "do_expensive_check": False, + "prior_sources_behavior": self.__prior_sources_behavior, + "deduplicate_sources": self.__deduplicate_sources, + "retain_seeds": self._retain_original_seeds, + "compression": self.__compression, + "compress_per_hop": self.__compress_per_hop, + "random_state": random_state + rank, + } + + if self.__num_edge_types > 1: + kwargs["num_edge_types"] = self.__num_edge_types + + sampling_results_dict = self.__func(**kwargs) sampling_results_dict["fanout"] = cupy.array(self.__fanout, dtype="int32") return sampling_results_dict diff --git a/python/pylibcugraph/pylibcugraph/heterogeneous_biased_neighbor_sample.pyx b/python/pylibcugraph/pylibcugraph/heterogeneous_biased_neighbor_sample.pyx index ecdfba3afc5..3009fba723f 100644 --- a/python/pylibcugraph/pylibcugraph/heterogeneous_biased_neighbor_sample.pyx +++ b/python/pylibcugraph/pylibcugraph/heterogeneous_biased_neighbor_sample.pyx @@ -88,6 +88,7 @@ def heterogeneous_biased_neighbor_sample(ResourceHandle resource_handle, start_vertex_list, starting_vertex_label_offsets, h_fan_out, + * num_edge_types, bool_t with_replacement, bool_t do_expensive_check, diff --git a/python/pylibcugraph/pylibcugraph/heterogeneous_uniform_neighbor_sample.pyx b/python/pylibcugraph/pylibcugraph/heterogeneous_uniform_neighbor_sample.pyx index 3fa3575e27d..996aa35e7c2 100644 --- a/python/pylibcugraph/pylibcugraph/heterogeneous_uniform_neighbor_sample.pyx +++ b/python/pylibcugraph/pylibcugraph/heterogeneous_uniform_neighbor_sample.pyx @@ -85,6 +85,7 @@ def heterogeneous_uniform_neighbor_sample(ResourceHandle resource_handle, start_vertex_list, starting_vertex_label_offsets, h_fan_out, + *, num_edge_types, bool_t with_replacement, bool_t do_expensive_check, diff --git a/python/pylibcugraph/pylibcugraph/homogeneous_biased_neighbor_sample.pyx b/python/pylibcugraph/pylibcugraph/homogeneous_biased_neighbor_sample.pyx index e2476de1607..47bded165dc 100644 --- a/python/pylibcugraph/pylibcugraph/homogeneous_biased_neighbor_sample.pyx +++ b/python/pylibcugraph/pylibcugraph/homogeneous_biased_neighbor_sample.pyx @@ -88,6 +88,7 @@ def homogeneous_biased_neighbor_sample(ResourceHandle resource_handle, start_vertex_list, starting_vertex_label_offsets, h_fan_out, + *, bool_t with_replacement, bool_t do_expensive_check, prior_sources_behavior=None, diff --git a/python/pylibcugraph/pylibcugraph/homogeneous_uniform_neighbor_sample.pyx b/python/pylibcugraph/pylibcugraph/homogeneous_uniform_neighbor_sample.pyx index 3c6cdf77420..325e24ae270 100644 --- a/python/pylibcugraph/pylibcugraph/homogeneous_uniform_neighbor_sample.pyx +++ b/python/pylibcugraph/pylibcugraph/homogeneous_uniform_neighbor_sample.pyx @@ -85,6 +85,7 @@ def homogeneous_uniform_neighbor_sample(ResourceHandle resource_handle, start_vertex_list, starting_vertex_label_offsets, h_fan_out, + *, bool_t with_replacement, bool_t do_expensive_check, prior_sources_behavior=None, From 61a0926fd61693165228a261ddf096bb507e9721 Mon Sep 17 00:00:00 2001 From: jnke2016 Date: Wed, 20 Nov 2024 13:02:10 -0800 Subject: [PATCH 06/49] fix bug in heterogeneous renumbering --- cpp/src/c_api/neighbor_sampling.cpp | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/cpp/src/c_api/neighbor_sampling.cpp b/cpp/src/c_api/neighbor_sampling.cpp index be3a44d813a..e89630dfb88 100644 --- a/cpp/src/c_api/neighbor_sampling.cpp +++ b/cpp/src/c_api/neighbor_sampling.cpp @@ -1220,13 +1220,18 @@ struct neighbor_sampling_functor : public cugraph::c_api::abstract_functor { } else { // heterogeneous renumbering + // FIXME: If no 'vertex_type_offsets' is provided, all vertices are assumed to have + // a vertex type of value 1. Update the API once 'vertex_type_offsets' is supported rmm::device_uvector vertex_type_offsets( - graph_view.local_vertex_partition_range_size(), handle_.get_stream()); + 2, handle_.get_stream()); + + cugraph::detail::stride_fill( + handle_.get_stream(), + vertex_type_offsets.begin(), + vertex_type_offsets.size(), + vertex_t{0}, + vertex_t{graph_view.local_vertex_partition_range_size()} - cugraph::detail::sequence_fill(handle_.get_stream(), - vertex_type_offsets.begin(), - vertex_type_offsets.size(), - vertex_t{0} // FIXME: Update array ); rmm::device_uvector output_majors(0, handle_.get_stream()); @@ -1240,7 +1245,7 @@ struct neighbor_sampling_functor : public cugraph::c_api::abstract_functor { edge_id, label_type_hop_offsets, // Contains information about the type and hop offsets output_renumber_map, - (*renumber_map_offsets), + renumber_map_offsets, renumbered_and_sorted_edge_id_renumber_map, renumbered_and_sorted_edge_id_renumber_map_label_type_offsets) = cugraph::heterogeneous_renumber_and_sort_sampled_edgelist( @@ -1267,7 +1272,7 @@ struct neighbor_sampling_functor : public cugraph::c_api::abstract_functor { edge_label ? (*offsets).size() - 1 : size_t{1}, hop ? fan_out_->size_ : size_t{1}, - size_t{1}, + vertex_type_offsets.size() - 1, // num_vertex_type is by default 1 if not provided num_edge_types_, src_is_major, do_expensive_check_); From 707cf3de5e4c17bac579824e928220360806826b Mon Sep 17 00:00:00 2001 From: jnke2016 Date: Wed, 20 Nov 2024 13:11:20 -0800 Subject: [PATCH 07/49] fix style --- cpp/src/c_api/neighbor_sampling.cpp | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/cpp/src/c_api/neighbor_sampling.cpp b/cpp/src/c_api/neighbor_sampling.cpp index e89630dfb88..d3ee4e93c6c 100644 --- a/cpp/src/c_api/neighbor_sampling.cpp +++ b/cpp/src/c_api/neighbor_sampling.cpp @@ -1222,15 +1222,13 @@ struct neighbor_sampling_functor : public cugraph::c_api::abstract_functor { // FIXME: If no 'vertex_type_offsets' is provided, all vertices are assumed to have // a vertex type of value 1. Update the API once 'vertex_type_offsets' is supported - rmm::device_uvector vertex_type_offsets( - 2, handle_.get_stream()); + rmm::device_uvector vertex_type_offsets(2, handle_.get_stream()); - cugraph::detail::stride_fill( - handle_.get_stream(), - vertex_type_offsets.begin(), - vertex_type_offsets.size(), - vertex_t{0}, - vertex_t{graph_view.local_vertex_partition_range_size()} + cugraph::detail::stride_fill(handle_.get_stream(), + vertex_type_offsets.begin(), + vertex_type_offsets.size(), + vertex_t{0}, + vertex_t{graph_view.local_vertex_partition_range_size()} ); @@ -1272,7 +1270,9 @@ struct neighbor_sampling_functor : public cugraph::c_api::abstract_functor { edge_label ? (*offsets).size() - 1 : size_t{1}, hop ? fan_out_->size_ : size_t{1}, - vertex_type_offsets.size() - 1, // num_vertex_type is by default 1 if not provided + + vertex_type_offsets.size() - + 1, // num_vertex_type is by default 1 if 'vertex_type_offsets' is not provided num_edge_types_, src_is_major, do_expensive_check_); From 4f427271274231b43f64c45fde1de0084f07fb1e Mon Sep 17 00:00:00 2001 From: Alexandria Barghi Date: Thu, 21 Nov 2024 09:19:37 -0800 Subject: [PATCH 08/49] dist sampling --- python/cugraph/cugraph/gnn/data_loading/dist_sampler.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python/cugraph/cugraph/gnn/data_loading/dist_sampler.py b/python/cugraph/cugraph/gnn/data_loading/dist_sampler.py index 336d5604c58..2e58e258c55 100644 --- a/python/cugraph/cugraph/gnn/data_loading/dist_sampler.py +++ b/python/cugraph/cugraph/gnn/data_loading/dist_sampler.py @@ -718,6 +718,8 @@ def sample_batches( if self.__num_edge_types > 1: kwargs["num_edge_types"] = self.__num_edge_types + print(kwargs) + sampling_results_dict = self.__func(**kwargs) sampling_results_dict["fanout"] = cupy.array(self.__fanout, dtype="int32") From ef848e9d8095ff51a6f4167c03997326ea596a69 Mon Sep 17 00:00:00 2001 From: jnke2016 Date: Thu, 21 Nov 2024 16:48:00 -0800 Subject: [PATCH 09/49] handle case where there is no sampling result prior to renumbering --- cpp/src/c_api/neighbor_sampling.cpp | 289 +++++++++++--------- cpp/src/sampling/neighbor_sampling_impl.hpp | 53 +++- 2 files changed, 197 insertions(+), 145 deletions(-) diff --git a/cpp/src/c_api/neighbor_sampling.cpp b/cpp/src/c_api/neighbor_sampling.cpp index d3ee4e93c6c..cd6cdaf2bbb 100644 --- a/cpp/src/c_api/neighbor_sampling.cpp +++ b/cpp/src/c_api/neighbor_sampling.cpp @@ -63,6 +63,7 @@ struct cugraph_sample_result_t { cugraph_type_erased_device_array_t* wgt_{nullptr}; cugraph_type_erased_device_array_t* hop_{nullptr}; cugraph_type_erased_device_array_t* label_hop_offsets_{nullptr}; + cugraph_type_erased_device_array_t* label_type_hop_offsets_{nullptr}; cugraph_type_erased_device_array_t* label_{nullptr}; cugraph_type_erased_device_array_t* renumber_map_{nullptr}; cugraph_type_erased_device_array_t* renumber_map_offsets_{nullptr}; @@ -1011,6 +1012,8 @@ struct neighbor_sampling_functor : public cugraph::c_api::abstract_functor { options_.with_replacement_}, do_expensive_check_); } else { + + raft::print_device_vector("labels", (*start_vertex_labels).data(), (*start_vertex_labels).size(), std::cout); std::tie(src, dst, wgt, edge_id, edge_type, hop, offsets) = cugraph::heterogeneous_uniform_neighbor_sample( handle_, @@ -1108,6 +1111,7 @@ struct neighbor_sampling_functor : public cugraph::c_api::abstract_functor { std::optional> major_offsets{std::nullopt}; std::optional> label_hop_offsets{std::nullopt}; + std::optional> label_type_hop_offsets{std::nullopt}; std::optional> renumber_map{std::nullopt}; std::optional> renumber_map_offsets{std::nullopt}; @@ -1125,21 +1129,127 @@ struct neighbor_sampling_functor : public cugraph::c_api::abstract_functor { } if (options_.renumber_results_) { - if (num_edge_types_ == 1) { // homogeneous renumbering - if (options_.compression_type_ == cugraph_compression_type_t::COO) { - // COO + if (src.size() > 0) { // Only renumber if there are edgelist to renumber + if (num_edge_types_ == 1) { // homogeneous renumbering + if (options_.compression_type_ == cugraph_compression_type_t::COO) { + // COO + + rmm::device_uvector output_majors(0, handle_.get_stream()); + rmm::device_uvector output_renumber_map(0, handle_.get_stream()); + std::tie(output_majors, + minors, + wgt, + edge_id, + edge_type, + label_hop_offsets, + output_renumber_map, + renumber_map_offsets) = + cugraph::renumber_and_sort_sampled_edgelist( + handle_, + std::move(src), + std::move(dst), + std::move(wgt), + std::move(edge_id), + std::move(edge_type), + std::move(hop), + options_.retain_seeds_ + ? std::make_optional(raft::device_span{ + start_vertices_->as_type(), start_vertices_->size_}) + : std::nullopt, + options_.retain_seeds_ + ? std::make_optional(raft::device_span{ + start_vertex_offsets_->as_type(), start_vertex_offsets_->size_}) + : std::nullopt, + offsets ? std::make_optional( + raft::device_span{offsets->data(), offsets->size()}) + : std::nullopt, + offsets ? (*offsets).size() - 1 : size_t{1}, + hop ? fan_out_->size_ : size_t{1}, + src_is_major, + do_expensive_check_); + + majors.emplace(std::move(output_majors)); + renumber_map.emplace(std::move(output_renumber_map)); + } else { + // (D)CSC, (D)CSR + + bool doubly_compress = + (options_.compression_type_ == cugraph_compression_type_t::DCSR) || + (options_.compression_type_ == cugraph_compression_type_t::DCSC); + + rmm::device_uvector output_major_offsets(0, handle_.get_stream()); + rmm::device_uvector output_renumber_map(0, handle_.get_stream()); + + std::tie(majors, + output_major_offsets, + minors, + wgt, + edge_id, + edge_type, + label_hop_offsets, + output_renumber_map, + renumber_map_offsets) = + cugraph::renumber_and_compress_sampled_edgelist( + handle_, + std::move(src), + std::move(dst), + std::move(wgt), + std::move(edge_id), + std::move(edge_type), + std::move(hop), + options_.retain_seeds_ + ? std::make_optional(raft::device_span{ + start_vertices_->as_type(), start_vertices_->size_}) + : std::nullopt, + options_.retain_seeds_ + ? std::make_optional(raft::device_span{ + start_vertex_offsets_->as_type(), start_vertex_offsets_->size_}) + : std::nullopt, + offsets ? std::make_optional( + raft::device_span{offsets->data(), offsets->size()}) + : std::nullopt, + edge_label ? (*offsets).size() - 1 : size_t{1}, // FIXME: update edge_label + hop ? fan_out_->size_ : size_t{1}, + src_is_major, + options_.compress_per_hop_, + doubly_compress, + do_expensive_check_); + + renumber_map.emplace(std::move(output_renumber_map)); + major_offsets.emplace(std::move(output_major_offsets)); + } + + // These are now represented by label_hop_offsets + hop.reset(); + offsets.reset(); + + } else { // heterogeneous renumbering + + // FIXME: If no 'vertex_type_offsets' is provided, all vertices are assumed to have + // a vertex type of value 1. Update the API once 'vertex_type_offsets' is supported + rmm::device_uvector vertex_type_offsets(2, handle_.get_stream()); + + cugraph::detail::stride_fill(handle_.get_stream(), + vertex_type_offsets.begin(), + vertex_type_offsets.size(), + vertex_t{0}, + vertex_t{graph_view.local_vertex_partition_range_size()} + + ); rmm::device_uvector output_majors(0, handle_.get_stream()); rmm::device_uvector output_renumber_map(0, handle_.get_stream()); + std::tie(output_majors, - minors, - wgt, - edge_id, - edge_type, - label_hop_offsets, - output_renumber_map, - renumber_map_offsets) = - cugraph::renumber_and_sort_sampled_edgelist( + minors, + wgt, + edge_id, + label_type_hop_offsets, // Contains information about the type and hop offsets + output_renumber_map, + renumber_map_offsets, + renumbered_and_sorted_edge_id_renumber_map, + renumbered_and_sorted_edge_id_renumber_map_label_type_offsets) = + cugraph::heterogeneous_renumber_and_sort_sampled_edgelist( handle_, std::move(src), std::move(dst), @@ -1158,138 +1268,32 @@ struct neighbor_sampling_functor : public cugraph::c_api::abstract_functor { offsets ? std::make_optional( raft::device_span{offsets->data(), offsets->size()}) : std::nullopt, - offsets ? (*offsets).size() - 1 : size_t{1}, - hop ? fan_out_->size_ : size_t{1}, - src_is_major, - do_expensive_check_); - - majors.emplace(std::move(output_majors)); - renumber_map.emplace(std::move(output_renumber_map)); - } else { - // (D)CSC, (D)CSR - - bool doubly_compress = - (options_.compression_type_ == cugraph_compression_type_t::DCSR) || - (options_.compression_type_ == cugraph_compression_type_t::DCSC); - - rmm::device_uvector output_major_offsets(0, handle_.get_stream()); - rmm::device_uvector output_renumber_map(0, handle_.get_stream()); + raft::device_span{vertex_type_offsets.data(), + vertex_type_offsets.size()}, - std::tie(majors, - output_major_offsets, - minors, - wgt, - edge_id, - edge_type, - label_hop_offsets, - output_renumber_map, - renumber_map_offsets) = - cugraph::renumber_and_compress_sampled_edgelist( - handle_, - std::move(src), - std::move(dst), - std::move(wgt), - std::move(edge_id), - std::move(edge_type), - std::move(hop), - options_.retain_seeds_ - ? std::make_optional(raft::device_span{ - start_vertices_->as_type(), start_vertices_->size_}) - : std::nullopt, - options_.retain_seeds_ - ? std::make_optional(raft::device_span{ - start_vertex_offsets_->as_type(), start_vertex_offsets_->size_}) - : std::nullopt, - offsets ? std::make_optional( - raft::device_span{offsets->data(), offsets->size()}) - : std::nullopt, - edge_label ? (*offsets).size() - 1 : size_t{1}, // FIXME: update edge_label + edge_label ? (*offsets).size() - 1 : size_t{1}, hop ? fan_out_->size_ : size_t{1}, + + vertex_type_offsets.size() - + 1, // num_vertex_type is by default 1 if 'vertex_type_offsets' is not provided + num_edge_types_, src_is_major, - options_.compress_per_hop_, - doubly_compress, do_expensive_check_); + if (edge_type) { + (*edge_type) + .resize(raft::device_span{(*label_type_hop_offsets).data(), + (*label_type_hop_offsets).size()} + .back() - + 1, + handle_.get_stream()); + cugraph::detail::sequence_fill( + handle_.get_stream(), (*edge_type).begin(), (*edge_type).size(), edge_type_t{0}); + } + majors.emplace(std::move(output_majors)); + // FIXME: Need to update renumber_map because default values are being passed renumber_map.emplace(std::move(output_renumber_map)); - major_offsets.emplace(std::move(output_major_offsets)); } - - // These are now represented by label_hop_offsets - hop.reset(); - offsets.reset(); - - } else { // heterogeneous renumbering - - // FIXME: If no 'vertex_type_offsets' is provided, all vertices are assumed to have - // a vertex type of value 1. Update the API once 'vertex_type_offsets' is supported - rmm::device_uvector vertex_type_offsets(2, handle_.get_stream()); - - cugraph::detail::stride_fill(handle_.get_stream(), - vertex_type_offsets.begin(), - vertex_type_offsets.size(), - vertex_t{0}, - vertex_t{graph_view.local_vertex_partition_range_size()} - - ); - - rmm::device_uvector output_majors(0, handle_.get_stream()); - rmm::device_uvector output_renumber_map(0, handle_.get_stream()); - - // extract the edge_type from label_type_hop_offsets - std::optional> label_type_hop_offsets{std::nullopt}; - std::tie(output_majors, - minors, - wgt, - edge_id, - label_type_hop_offsets, // Contains information about the type and hop offsets - output_renumber_map, - renumber_map_offsets, - renumbered_and_sorted_edge_id_renumber_map, - renumbered_and_sorted_edge_id_renumber_map_label_type_offsets) = - cugraph::heterogeneous_renumber_and_sort_sampled_edgelist( - handle_, - std::move(src), - std::move(dst), - std::move(wgt), - std::move(edge_id), - std::move(edge_type), - std::move(hop), - options_.retain_seeds_ - ? std::make_optional(raft::device_span{ - start_vertices_->as_type(), start_vertices_->size_}) - : std::nullopt, - options_.retain_seeds_ - ? std::make_optional(raft::device_span{ - start_vertex_offsets_->as_type(), start_vertex_offsets_->size_}) - : std::nullopt, - offsets ? std::make_optional( - raft::device_span{offsets->data(), offsets->size()}) - : std::nullopt, - raft::device_span{vertex_type_offsets.data(), - vertex_type_offsets.size()}, - - edge_label ? (*offsets).size() - 1 : size_t{1}, - hop ? fan_out_->size_ : size_t{1}, - - vertex_type_offsets.size() - - 1, // num_vertex_type is by default 1 if 'vertex_type_offsets' is not provided - num_edge_types_, - src_is_major, - do_expensive_check_); - if (edge_type) { - (*edge_type) - .resize(raft::device_span{(*label_type_hop_offsets).data(), - (*label_type_hop_offsets).size()} - .back() - - 1, - handle_.get_stream()); - cugraph::detail::sequence_fill( - handle_.get_stream(), (*edge_type).begin(), (*edge_type).size(), edge_type_t{0}); - } - - majors.emplace(std::move(output_majors)); - // FIXME: Need to update renumber_map because default values are being passed - renumber_map.emplace(std::move(output_renumber_map)); } } else { @@ -1344,6 +1348,9 @@ struct neighbor_sampling_functor : public cugraph::c_api::abstract_functor { (label_hop_offsets) ? new cugraph::c_api::cugraph_type_erased_device_array_t(*label_hop_offsets, SIZE_T) : nullptr, + (label_type_hop_offsets) + ? new cugraph::c_api::cugraph_type_erased_device_array_t(*label_type_hop_offsets, SIZE_T) + : nullptr, (edge_label) ? new cugraph::c_api::cugraph_type_erased_device_array_t(edge_label.value(), INT32) : nullptr, @@ -1562,6 +1569,16 @@ extern "C" cugraph_type_erased_device_array_view_t* cugraph_sample_result_get_la : NULL; } +extern "C" cugraph_type_erased_device_array_view_t* cugraph_sample_result_get_label_type_hop_offsets( + const cugraph_sample_result_t* result) +{ + auto internal_pointer = reinterpret_cast(result); + return internal_pointer->label_type_hop_offsets_ != nullptr + ? reinterpret_cast( + internal_pointer->label_type_hop_offsets_->view()) + : NULL; +} + extern "C" cugraph_type_erased_device_array_view_t* cugraph_sample_result_get_index( const cugraph_sample_result_t* result) { diff --git a/cpp/src/sampling/neighbor_sampling_impl.hpp b/cpp/src/sampling/neighbor_sampling_impl.hpp index ccca71cdf20..79451a38d43 100644 --- a/cpp/src/sampling/neighbor_sampling_impl.hpp +++ b/cpp/src/sampling/neighbor_sampling_impl.hpp @@ -104,6 +104,13 @@ neighbor_sample_impl(raft::handle_t const& handle, edge_masks_vector{}; graph_view_t modified_graph_view = graph_view; edge_masks_vector.reserve(num_edge_types); + + label_t num_labels = 0; + + if (starting_vertex_labels) { + // Initial number of labels. Will be leveraged if there is no sampling result + num_labels = starting_vertex_labels->size(); + } if (num_edge_types > 1) { for (int i = 0; i < num_edge_types; i++) { @@ -362,15 +369,43 @@ neighbor_sample_impl(raft::handle_t const& handle, level_result_label_vectors = std::nullopt; } - return detail::shuffle_and_organize_output(handle, - std::move(result_srcs), - std::move(result_dsts), - std::move(result_weights), - std::move(result_edge_ids), - std::move(result_edge_types), - std::move(result_hops), - std::move(result_labels), - label_to_output_comm_rank); + std::optional> result_offsets{std::nullopt}; + + std::tie(result_srcs, result_dsts, result_weights, result_edge_ids, + result_edge_types, result_hops, result_labels, result_offsets) + = detail::shuffle_and_organize_output(handle, + std::move(result_srcs), + std::move(result_dsts), + std::move(result_weights), + std::move(result_edge_ids), + std::move(result_edge_types), + std::move(result_hops), + std::move(result_labels), + label_to_output_comm_rank); + + if (result_srcs.size() == 0){ + // Update the 'edgelist_label_offsets' array to be proportional to the + // number of labels + result_offsets->resize(num_labels + 1 ,handle.get_stream()); + + thrust::transform( + handle.get_thrust_policy(), + thrust::make_counting_iterator(0), + thrust::make_counting_iterator(result_offsets->size()), + result_offsets->begin(), + [] __device__(auto idx) { + return 0; + }); + } + + return std::make_tuple(std::move(result_srcs), + std::move(result_dsts), + std::move(result_weights), + std::move(result_edge_ids), + std::move(result_edge_types), + std::move(result_hops), + std::move(result_labels), + std::move(result_offsets)); } } // namespace detail From 535df5d75de49f45ac2d5fdb363f5b92fbb9e985 Mon Sep 17 00:00:00 2001 From: jnke2016 Date: Thu, 21 Nov 2024 16:54:49 -0800 Subject: [PATCH 10/49] expose 'label_type_hop_offsets' --- cpp/include/cugraph_c/sampling_algorithms.h | 10 ++++++++++ .../pylibcugraph/_cugraph_c/algorithms.pxd | 5 +++++ .../heterogeneous_biased_neighbor_sample.pyx | 4 +++- .../heterogeneous_uniform_neighbor_sample.pyx | 4 ++++ .../internal_types/sampling_result.pyx | 14 ++++++++++++++ 5 files changed, 36 insertions(+), 1 deletion(-) diff --git a/cpp/include/cugraph_c/sampling_algorithms.h b/cpp/include/cugraph_c/sampling_algorithms.h index ef75e726d80..499b3131554 100644 --- a/cpp/include/cugraph_c/sampling_algorithms.h +++ b/cpp/include/cugraph_c/sampling_algorithms.h @@ -735,6 +735,16 @@ cugraph_type_erased_device_array_view_t* cugraph_sample_result_get_hop( cugraph_type_erased_device_array_view_t* cugraph_sample_result_get_label_hop_offsets( const cugraph_sample_result_t* result); +/** + * @ingroup samplingC + * @brief Get the label-type-hop offsets from the sampling algorithm result + * + * @param [in] result The result from a sampling algorithm + * @return type erased array pointing to the label-type-hop offsets + */ +cugraph_type_erased_device_array_view_t* cugraph_sample_result_get_label_type_hop_offsets( + const cugraph_sample_result_t* result); + /** * @ingroup samplingC * @brief Get the index from the sampling algorithm result diff --git a/python/pylibcugraph/pylibcugraph/_cugraph_c/algorithms.pxd b/python/pylibcugraph/pylibcugraph/_cugraph_c/algorithms.pxd index 21f5190ad5f..149d34473b2 100644 --- a/python/pylibcugraph/pylibcugraph/_cugraph_c/algorithms.pxd +++ b/python/pylibcugraph/pylibcugraph/_cugraph_c/algorithms.pxd @@ -244,6 +244,11 @@ cdef extern from "cugraph_c/algorithms.h": cugraph_sample_result_get_label_hop_offsets( const cugraph_sample_result_t* result ) + + cdef cugraph_type_erased_device_array_view_t* \ + cugraph_sample_result_get_label_type_hop_offsets( + const cugraph_sample_result_t* result + ) cdef cugraph_type_erased_device_array_view_t* \ cugraph_sample_result_get_start_labels( diff --git a/python/pylibcugraph/pylibcugraph/heterogeneous_biased_neighbor_sample.pyx b/python/pylibcugraph/pylibcugraph/heterogeneous_biased_neighbor_sample.pyx index ecdfba3afc5..0f49dec712b 100644 --- a/python/pylibcugraph/pylibcugraph/heterogeneous_biased_neighbor_sample.pyx +++ b/python/pylibcugraph/pylibcugraph/heterogeneous_biased_neighbor_sample.pyx @@ -388,7 +388,7 @@ def heterogeneous_biased_neighbor_sample(ResourceHandle resource_handle, cupy_edge_types = result.get_edge_types() cupy_batch_ids = result.get_batch_ids() cupy_label_hop_offsets = result.get_label_hop_offsets() - + cupy_label_type_hop_offsets = result.get_label_type_hop_offsets() if renumber: cupy_renumber_map = result.get_renumber_map() @@ -405,6 +405,7 @@ def heterogeneous_biased_neighbor_sample(ResourceHandle resource_handle, 'edge_type': cupy_edge_types, 'batch_id': cupy_batch_ids, 'label_hop_offsets': cupy_label_hop_offsets, + 'label_type_hop_offsets': cupy_label_type_hop_offsets, 'hop_id': None, 'renumber_map': cupy_renumber_map, 'renumber_map_offsets': cupy_renumber_map_offsets, @@ -422,6 +423,7 @@ def heterogeneous_biased_neighbor_sample(ResourceHandle resource_handle, 'edge_type': cupy_edge_types, 'batch_id': cupy_batch_ids, 'label_hop_offsets': cupy_label_hop_offsets, + 'label_type_hop_offsets': cupy_label_type_hop_offsets, } # Return everything that isn't null diff --git a/python/pylibcugraph/pylibcugraph/heterogeneous_uniform_neighbor_sample.pyx b/python/pylibcugraph/pylibcugraph/heterogeneous_uniform_neighbor_sample.pyx index 3fa3575e27d..c586fcdff8c 100644 --- a/python/pylibcugraph/pylibcugraph/heterogeneous_uniform_neighbor_sample.pyx +++ b/python/pylibcugraph/pylibcugraph/heterogeneous_uniform_neighbor_sample.pyx @@ -372,6 +372,7 @@ def heterogeneous_uniform_neighbor_sample(ResourceHandle resource_handle, # Get cupy "views" of the individual arrays to return. These each increment # the refcount on the SamplingResult instance which will keep the data alive # until all references are removed and the GC runs. + cupy_majors = result.get_majors() cupy_major_offsets = result.get_major_offsets() cupy_minors = result.get_minors() @@ -380,6 +381,7 @@ def heterogeneous_uniform_neighbor_sample(ResourceHandle resource_handle, cupy_edge_types = result.get_edge_types() cupy_batch_ids = result.get_batch_ids() cupy_label_hop_offsets = result.get_label_hop_offsets() + cupy_label_type_hop_offsets = result.get_label_type_hop_offsets() if renumber: cupy_renumber_map = result.get_renumber_map() @@ -396,6 +398,7 @@ def heterogeneous_uniform_neighbor_sample(ResourceHandle resource_handle, 'edge_type': cupy_edge_types, 'batch_id': cupy_batch_ids, 'label_hop_offsets': cupy_label_hop_offsets, + 'label_type_hop_offsets': cupy_label_type_hop_offsets, 'hop_id': None, 'renumber_map': cupy_renumber_map, 'renumber_map_offsets': cupy_renumber_map_offsets, @@ -413,6 +416,7 @@ def heterogeneous_uniform_neighbor_sample(ResourceHandle resource_handle, 'edge_type': cupy_edge_types, 'batch_id': cupy_batch_ids, 'label_hop_offsets': cupy_label_hop_offsets, + 'label_type_hop_offsets': cupy_label_type_hop_offsets, } # Return everything that isn't null diff --git a/python/pylibcugraph/pylibcugraph/internal_types/sampling_result.pyx b/python/pylibcugraph/pylibcugraph/internal_types/sampling_result.pyx index b93618d73ce..9e53d0aed99 100644 --- a/python/pylibcugraph/pylibcugraph/internal_types/sampling_result.pyx +++ b/python/pylibcugraph/pylibcugraph/internal_types/sampling_result.pyx @@ -24,6 +24,7 @@ from pylibcugraph._cugraph_c.algorithms cimport ( cugraph_sample_result_get_majors, cugraph_sample_result_get_minors, cugraph_sample_result_get_label_hop_offsets, + cugraph_sample_result_get_label_type_hop_offsets, cugraph_sample_result_get_sources, # deprecated cugraph_sample_result_get_destinations, # deprecated cugraph_sample_result_get_edge_weight, @@ -205,6 +206,19 @@ cdef class SamplingResult: return create_cupy_array_view_for_device_ptr(device_array_view_ptr, self) + + def get_label_type_hop_offsets(self): + if self.c_sample_result_ptr is NULL: + raise ValueError("pointer not set, must call set_ptr() with a " + "non-NULL value first.") + cdef cugraph_type_erased_device_array_view_t* device_array_view_ptr = ( + cugraph_sample_result_get_label_type_hop_offsets(self.c_sample_result_ptr) + ) + if device_array_view_ptr is NULL: + return None + + return create_cupy_array_view_for_device_ptr(device_array_view_ptr, + self) # Deprecated def get_offsets(self): From 3501a453c7ba1f246077abba672f8f5f602fc651 Mon Sep 17 00:00:00 2001 From: jnke2016 Date: Thu, 21 Nov 2024 16:56:15 -0800 Subject: [PATCH 11/49] fix style --- cpp/src/c_api/neighbor_sampling.cpp | 64 +++++++++---------- cpp/src/sampling/neighbor_sampling_impl.hpp | 56 ++++++++-------- .../pylibcugraph/_cugraph_c/algorithms.pxd | 2 +- .../internal_types/sampling_result.pyx | 2 +- 4 files changed, 63 insertions(+), 61 deletions(-) diff --git a/cpp/src/c_api/neighbor_sampling.cpp b/cpp/src/c_api/neighbor_sampling.cpp index cd6cdaf2bbb..c343adc228e 100644 --- a/cpp/src/c_api/neighbor_sampling.cpp +++ b/cpp/src/c_api/neighbor_sampling.cpp @@ -1012,8 +1012,8 @@ struct neighbor_sampling_functor : public cugraph::c_api::abstract_functor { options_.with_replacement_}, do_expensive_check_); } else { - - raft::print_device_vector("labels", (*start_vertex_labels).data(), (*start_vertex_labels).size(), std::cout); + raft::print_device_vector( + "labels", (*start_vertex_labels).data(), (*start_vertex_labels).size(), std::cout); std::tie(src, dst, wgt, edge_id, edge_type, hop, offsets) = cugraph::heterogeneous_uniform_neighbor_sample( handle_, @@ -1129,7 +1129,7 @@ struct neighbor_sampling_functor : public cugraph::c_api::abstract_functor { } if (options_.renumber_results_) { - if (src.size() > 0) { // Only renumber if there are edgelist to renumber + if (src.size() > 0) { // Only renumber if there are edgelist to renumber if (num_edge_types_ == 1) { // homogeneous renumbering if (options_.compression_type_ == cugraph_compression_type_t::COO) { // COO @@ -1137,13 +1137,13 @@ struct neighbor_sampling_functor : public cugraph::c_api::abstract_functor { rmm::device_uvector output_majors(0, handle_.get_stream()); rmm::device_uvector output_renumber_map(0, handle_.get_stream()); std::tie(output_majors, - minors, - wgt, - edge_id, - edge_type, - label_hop_offsets, - output_renumber_map, - renumber_map_offsets) = + minors, + wgt, + edge_id, + edge_type, + label_hop_offsets, + output_renumber_map, + renumber_map_offsets) = cugraph::renumber_and_sort_sampled_edgelist( handle_, std::move(src), @@ -1181,14 +1181,14 @@ struct neighbor_sampling_functor : public cugraph::c_api::abstract_functor { rmm::device_uvector output_renumber_map(0, handle_.get_stream()); std::tie(majors, - output_major_offsets, - minors, - wgt, - edge_id, - edge_type, - label_hop_offsets, - output_renumber_map, - renumber_map_offsets) = + output_major_offsets, + minors, + wgt, + edge_id, + edge_type, + label_hop_offsets, + output_renumber_map, + renumber_map_offsets) = cugraph::renumber_and_compress_sampled_edgelist( handle_, std::move(src), @@ -1230,10 +1230,10 @@ struct neighbor_sampling_functor : public cugraph::c_api::abstract_functor { rmm::device_uvector vertex_type_offsets(2, handle_.get_stream()); cugraph::detail::stride_fill(handle_.get_stream(), - vertex_type_offsets.begin(), - vertex_type_offsets.size(), - vertex_t{0}, - vertex_t{graph_view.local_vertex_partition_range_size()} + vertex_type_offsets.begin(), + vertex_type_offsets.size(), + vertex_t{0}, + vertex_t{graph_view.local_vertex_partition_range_size()} ); @@ -1241,14 +1241,14 @@ struct neighbor_sampling_functor : public cugraph::c_api::abstract_functor { rmm::device_uvector output_renumber_map(0, handle_.get_stream()); std::tie(output_majors, - minors, - wgt, - edge_id, - label_type_hop_offsets, // Contains information about the type and hop offsets - output_renumber_map, - renumber_map_offsets, - renumbered_and_sorted_edge_id_renumber_map, - renumbered_and_sorted_edge_id_renumber_map_label_type_offsets) = + minors, + wgt, + edge_id, + label_type_hop_offsets, // Contains information about the type and hop offsets + output_renumber_map, + renumber_map_offsets, + renumbered_and_sorted_edge_id_renumber_map, + renumbered_and_sorted_edge_id_renumber_map_label_type_offsets) = cugraph::heterogeneous_renumber_and_sort_sampled_edgelist( handle_, std::move(src), @@ -1569,8 +1569,8 @@ extern "C" cugraph_type_erased_device_array_view_t* cugraph_sample_result_get_la : NULL; } -extern "C" cugraph_type_erased_device_array_view_t* cugraph_sample_result_get_label_type_hop_offsets( - const cugraph_sample_result_t* result) +extern "C" cugraph_type_erased_device_array_view_t* +cugraph_sample_result_get_label_type_hop_offsets(const cugraph_sample_result_t* result) { auto internal_pointer = reinterpret_cast(result); return internal_pointer->label_type_hop_offsets_ != nullptr diff --git a/cpp/src/sampling/neighbor_sampling_impl.hpp b/cpp/src/sampling/neighbor_sampling_impl.hpp index 79451a38d43..0a5d65ea1eb 100644 --- a/cpp/src/sampling/neighbor_sampling_impl.hpp +++ b/cpp/src/sampling/neighbor_sampling_impl.hpp @@ -104,7 +104,7 @@ neighbor_sample_impl(raft::handle_t const& handle, edge_masks_vector{}; graph_view_t modified_graph_view = graph_view; edge_masks_vector.reserve(num_edge_types); - + label_t num_labels = 0; if (starting_vertex_labels) { @@ -370,34 +370,36 @@ neighbor_sample_impl(raft::handle_t const& handle, } std::optional> result_offsets{std::nullopt}; - - std::tie(result_srcs, result_dsts, result_weights, result_edge_ids, - result_edge_types, result_hops, result_labels, result_offsets) - = detail::shuffle_and_organize_output(handle, - std::move(result_srcs), - std::move(result_dsts), - std::move(result_weights), - std::move(result_edge_ids), - std::move(result_edge_types), - std::move(result_hops), - std::move(result_labels), - label_to_output_comm_rank); - - if (result_srcs.size() == 0){ + + std::tie(result_srcs, + result_dsts, + result_weights, + result_edge_ids, + result_edge_types, + result_hops, + result_labels, + result_offsets) = detail::shuffle_and_organize_output(handle, + std::move(result_srcs), + std::move(result_dsts), + std::move(result_weights), + std::move(result_edge_ids), + std::move(result_edge_types), + std::move(result_hops), + std::move(result_labels), + label_to_output_comm_rank); + + if (result_srcs.size() == 0) { // Update the 'edgelist_label_offsets' array to be proportional to the // number of labels - result_offsets->resize(num_labels + 1 ,handle.get_stream()); - - thrust::transform( - handle.get_thrust_policy(), - thrust::make_counting_iterator(0), - thrust::make_counting_iterator(result_offsets->size()), - result_offsets->begin(), - [] __device__(auto idx) { - return 0; - }); - } - + result_offsets->resize(num_labels + 1, handle.get_stream()); + + thrust::transform(handle.get_thrust_policy(), + thrust::make_counting_iterator(0), + thrust::make_counting_iterator(result_offsets->size()), + result_offsets->begin(), + [] __device__(auto idx) { return 0; }); + } + return std::make_tuple(std::move(result_srcs), std::move(result_dsts), std::move(result_weights), diff --git a/python/pylibcugraph/pylibcugraph/_cugraph_c/algorithms.pxd b/python/pylibcugraph/pylibcugraph/_cugraph_c/algorithms.pxd index 149d34473b2..38781614b20 100644 --- a/python/pylibcugraph/pylibcugraph/_cugraph_c/algorithms.pxd +++ b/python/pylibcugraph/pylibcugraph/_cugraph_c/algorithms.pxd @@ -244,7 +244,7 @@ cdef extern from "cugraph_c/algorithms.h": cugraph_sample_result_get_label_hop_offsets( const cugraph_sample_result_t* result ) - + cdef cugraph_type_erased_device_array_view_t* \ cugraph_sample_result_get_label_type_hop_offsets( const cugraph_sample_result_t* result diff --git a/python/pylibcugraph/pylibcugraph/internal_types/sampling_result.pyx b/python/pylibcugraph/pylibcugraph/internal_types/sampling_result.pyx index 9e53d0aed99..a2ea7cb9716 100644 --- a/python/pylibcugraph/pylibcugraph/internal_types/sampling_result.pyx +++ b/python/pylibcugraph/pylibcugraph/internal_types/sampling_result.pyx @@ -206,7 +206,7 @@ cdef class SamplingResult: return create_cupy_array_view_for_device_ptr(device_array_view_ptr, self) - + def get_label_type_hop_offsets(self): if self.c_sample_result_ptr is NULL: raise ValueError("pointer not set, must call set_ptr() with a " From 288b8fc5450b168b618475e0809b21420d703ccb Mon Sep 17 00:00:00 2001 From: jnke2016 Date: Thu, 21 Nov 2024 20:36:35 -0800 Subject: [PATCH 12/49] remove debug print and properly compute the 'result_offsets' --- cpp/src/c_api/neighbor_sampling.cpp | 2 - cpp/src/sampling/neighbor_sampling_impl.hpp | 88 ++++++++++++++------- 2 files changed, 61 insertions(+), 29 deletions(-) diff --git a/cpp/src/c_api/neighbor_sampling.cpp b/cpp/src/c_api/neighbor_sampling.cpp index c343adc228e..4fd4f346b3c 100644 --- a/cpp/src/c_api/neighbor_sampling.cpp +++ b/cpp/src/c_api/neighbor_sampling.cpp @@ -1012,8 +1012,6 @@ struct neighbor_sampling_functor : public cugraph::c_api::abstract_functor { options_.with_replacement_}, do_expensive_check_); } else { - raft::print_device_vector( - "labels", (*start_vertex_labels).data(), (*start_vertex_labels).size(), std::cout); std::tie(src, dst, wgt, edge_id, edge_type, hop, offsets) = cugraph::heterogeneous_uniform_neighbor_sample( handle_, diff --git a/cpp/src/sampling/neighbor_sampling_impl.hpp b/cpp/src/sampling/neighbor_sampling_impl.hpp index 0a5d65ea1eb..3bc2c90c57b 100644 --- a/cpp/src/sampling/neighbor_sampling_impl.hpp +++ b/cpp/src/sampling/neighbor_sampling_impl.hpp @@ -370,36 +370,70 @@ neighbor_sample_impl(raft::handle_t const& handle, } std::optional> result_offsets{std::nullopt}; + std::optional> cp_result_labels{std::nullopt}; + if (result_labels) { + cp_result_labels = rmm::device_uvector(result_labels->size(), handle.get_stream()); - std::tie(result_srcs, - result_dsts, - result_weights, - result_edge_ids, - result_edge_types, - result_hops, - result_labels, - result_offsets) = detail::shuffle_and_organize_output(handle, - std::move(result_srcs), - std::move(result_dsts), - std::move(result_weights), - std::move(result_edge_ids), - std::move(result_edge_types), - std::move(result_hops), - std::move(result_labels), - label_to_output_comm_rank); - - if (result_srcs.size() == 0) { - // Update the 'edgelist_label_offsets' array to be proportional to the - // number of labels - result_offsets->resize(num_labels + 1, handle.get_stream()); - - thrust::transform(handle.get_thrust_policy(), - thrust::make_counting_iterator(0), - thrust::make_counting_iterator(result_offsets->size()), - result_offsets->begin(), - [] __device__(auto idx) { return 0; }); + thrust::copy( + handle.get_thrust_policy(), + result_labels->begin(), + result_labels->end(), + cp_result_labels->begin()); } + // FIXME: remove the offsets computation in 'shuffle_and_organize_output' as it doesn't + // account account for missing labels + std::tie(result_srcs, result_dsts, result_weights, result_edge_ids, + result_edge_types, result_hops, result_labels, result_offsets) + = detail::shuffle_and_organize_output(handle, + std::move(result_srcs), + std::move(result_dsts), + std::move(result_weights), + std::move(result_edge_ids), + std::move(result_edge_types), + std::move(result_hops), + std::move(result_labels), + label_to_output_comm_rank); + + + if (result_labels) { + // Re-compute the result_offsets and account for missing labels + result_offsets = rmm::device_uvector(num_labels + 1, handle.get_stream()); + + // Sort labels + thrust::sort( + handle.get_thrust_policy(), + cp_result_labels->begin(), + cp_result_labels->end()); + + thrust::transform( + handle.get_thrust_policy(), + thrust::make_counting_iterator(0), + thrust::make_counting_iterator(result_offsets->size()), + result_offsets->begin() + 1, + [ + result_labels = raft::device_span( + cp_result_labels->data(), + cp_result_labels->size()) + ] __device__(auto idx) { + auto itr_lower = thrust::lower_bound( + thrust::seq, result_labels.begin(), result_labels.end(), idx); + + auto itr_upper = thrust::upper_bound( + thrust::seq, result_labels.begin(), result_labels.end(), idx); + + auto sampled_label_size = thrust::distance(itr_lower, itr_upper); + + return sampled_label_size; + }); + + // Run inclusive scan + thrust::inclusive_scan(handle.get_thrust_policy(), + result_offsets->begin() + 1, + result_offsets->end(), + result_offsets->begin() + 1); + } + return std::make_tuple(std::move(result_srcs), std::move(result_dsts), std::move(result_weights), From 9f459dedc5707f92786b7c3140461fd035b60629 Mon Sep 17 00:00:00 2001 From: jnke2016 Date: Thu, 21 Nov 2024 20:41:25 -0800 Subject: [PATCH 13/49] update docstrings --- cpp/src/sampling/neighbor_sampling_impl.hpp | 92 ++++++++++----------- 1 file changed, 44 insertions(+), 48 deletions(-) diff --git a/cpp/src/sampling/neighbor_sampling_impl.hpp b/cpp/src/sampling/neighbor_sampling_impl.hpp index 3bc2c90c57b..0e6bf50daa9 100644 --- a/cpp/src/sampling/neighbor_sampling_impl.hpp +++ b/cpp/src/sampling/neighbor_sampling_impl.hpp @@ -374,66 +374,62 @@ neighbor_sample_impl(raft::handle_t const& handle, if (result_labels) { cp_result_labels = rmm::device_uvector(result_labels->size(), handle.get_stream()); - thrust::copy( - handle.get_thrust_policy(), - result_labels->begin(), - result_labels->end(), - cp_result_labels->begin()); + thrust::copy(handle.get_thrust_policy(), + result_labels->begin(), + result_labels->end(), + cp_result_labels->begin()); } // FIXME: remove the offsets computation in 'shuffle_and_organize_output' as it doesn't - // account account for missing labels - std::tie(result_srcs, result_dsts, result_weights, result_edge_ids, - result_edge_types, result_hops, result_labels, result_offsets) - = detail::shuffle_and_organize_output(handle, - std::move(result_srcs), - std::move(result_dsts), - std::move(result_weights), - std::move(result_edge_ids), - std::move(result_edge_types), - std::move(result_hops), - std::move(result_labels), - label_to_output_comm_rank); - + // account for missing labels that are not sampled. + std::tie(result_srcs, + result_dsts, + result_weights, + result_edge_ids, + result_edge_types, + result_hops, + result_labels, + result_offsets) = detail::shuffle_and_organize_output(handle, + std::move(result_srcs), + std::move(result_dsts), + std::move(result_weights), + std::move(result_edge_ids), + std::move(result_edge_types), + std::move(result_hops), + std::move(result_labels), + label_to_output_comm_rank); if (result_labels) { // Re-compute the result_offsets and account for missing labels result_offsets = rmm::device_uvector(num_labels + 1, handle.get_stream()); // Sort labels - thrust::sort( - handle.get_thrust_policy(), - cp_result_labels->begin(), - cp_result_labels->end()); - - thrust::transform( - handle.get_thrust_policy(), - thrust::make_counting_iterator(0), - thrust::make_counting_iterator(result_offsets->size()), - result_offsets->begin() + 1, - [ - result_labels = raft::device_span( - cp_result_labels->data(), - cp_result_labels->size()) - ] __device__(auto idx) { - auto itr_lower = thrust::lower_bound( - thrust::seq, result_labels.begin(), result_labels.end(), idx); - - auto itr_upper = thrust::upper_bound( - thrust::seq, result_labels.begin(), result_labels.end(), idx); - - auto sampled_label_size = thrust::distance(itr_lower, itr_upper); - - return sampled_label_size; - }); - + thrust::sort(handle.get_thrust_policy(), cp_result_labels->begin(), cp_result_labels->end()); + + thrust::transform(handle.get_thrust_policy(), + thrust::make_counting_iterator(0), + thrust::make_counting_iterator(result_offsets->size()), + result_offsets->begin() + 1, + [result_labels = raft::device_span( + cp_result_labels->data(), cp_result_labels->size())] __device__(auto idx) { + auto itr_lower = thrust::lower_bound( + thrust::seq, result_labels.begin(), result_labels.end(), idx); + + auto itr_upper = thrust::upper_bound( + thrust::seq, result_labels.begin(), result_labels.end(), idx); + + auto sampled_label_size = thrust::distance(itr_lower, itr_upper); + + return sampled_label_size; + }); + // Run inclusive scan thrust::inclusive_scan(handle.get_thrust_policy(), - result_offsets->begin() + 1, - result_offsets->end(), - result_offsets->begin() + 1); + result_offsets->begin() + 1, + result_offsets->end(), + result_offsets->begin() + 1); } - + return std::make_tuple(std::move(result_srcs), std::move(result_dsts), std::move(result_weights), From fc0426d79bf6b307c2c24c5d243b5f41b0b33485 Mon Sep 17 00:00:00 2001 From: jnke2016 Date: Thu, 21 Nov 2024 20:43:47 -0800 Subject: [PATCH 14/49] directly return the result --- cpp/src/sampling/neighbor_sampling_impl.hpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/cpp/src/sampling/neighbor_sampling_impl.hpp b/cpp/src/sampling/neighbor_sampling_impl.hpp index 0e6bf50daa9..0b20b57d2bd 100644 --- a/cpp/src/sampling/neighbor_sampling_impl.hpp +++ b/cpp/src/sampling/neighbor_sampling_impl.hpp @@ -418,9 +418,7 @@ neighbor_sample_impl(raft::handle_t const& handle, auto itr_upper = thrust::upper_bound( thrust::seq, result_labels.begin(), result_labels.end(), idx); - auto sampled_label_size = thrust::distance(itr_lower, itr_upper); - - return sampled_label_size; + return thrust::distance(itr_lower, itr_upper); }); // Run inclusive scan From d997656f4e4c5e66fab4e96d493cc507bbb2646b Mon Sep 17 00:00:00 2001 From: jnke2016 Date: Thu, 21 Nov 2024 20:47:35 -0800 Subject: [PATCH 15/49] fix illegal memory access --- cpp/src/sampling/neighbor_sampling_impl.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/src/sampling/neighbor_sampling_impl.hpp b/cpp/src/sampling/neighbor_sampling_impl.hpp index 0b20b57d2bd..2ddc85a41bc 100644 --- a/cpp/src/sampling/neighbor_sampling_impl.hpp +++ b/cpp/src/sampling/neighbor_sampling_impl.hpp @@ -408,7 +408,7 @@ neighbor_sample_impl(raft::handle_t const& handle, thrust::transform(handle.get_thrust_policy(), thrust::make_counting_iterator(0), - thrust::make_counting_iterator(result_offsets->size()), + thrust::make_counting_iterator(result_offsets->size() - 1), result_offsets->begin() + 1, [result_labels = raft::device_span( cp_result_labels->data(), cp_result_labels->size())] __device__(auto idx) { From fc70811d69780ce9e838afbb5a591d09d4fdb4b5 Mon Sep 17 00:00:00 2001 From: jnke2016 Date: Thu, 21 Nov 2024 21:45:06 -0800 Subject: [PATCH 16/49] rename variable for consistency --- cpp/src/c_api/neighbor_sampling.cpp | 88 ++++++++++++++--------------- 1 file changed, 44 insertions(+), 44 deletions(-) diff --git a/cpp/src/c_api/neighbor_sampling.cpp b/cpp/src/c_api/neighbor_sampling.cpp index 4fd4f346b3c..51712481034 100644 --- a/cpp/src/c_api/neighbor_sampling.cpp +++ b/cpp/src/c_api/neighbor_sampling.cpp @@ -778,7 +778,7 @@ struct neighbor_sampling_functor : public cugraph::c_api::abstract_functor { cugraph::c_api::cugraph_graph_t* graph_{nullptr}; cugraph::c_api::cugraph_edge_property_view_t const* edge_biases_{nullptr}; cugraph::c_api::cugraph_type_erased_device_array_view_t const* start_vertices_{nullptr}; - cugraph::c_api::cugraph_type_erased_device_array_view_t const* start_vertex_offsets_{nullptr}; + cugraph::c_api::cugraph_type_erased_device_array_view_t const* starting_vertex_label_offsets_{nullptr}; cugraph::c_api::cugraph_type_erased_host_array_view_t const* fan_out_{nullptr}; int num_edge_types_{}; cugraph::c_api::cugraph_sampling_options_t options_{}; @@ -791,7 +791,7 @@ struct neighbor_sampling_functor : public cugraph::c_api::abstract_functor { cugraph_graph_t* graph, cugraph_edge_property_view_t const* edge_biases, cugraph_type_erased_device_array_view_t const* start_vertices, - cugraph_type_erased_device_array_view_t const* start_vertex_offsets, + cugraph_type_erased_device_array_view_t const* starting_vertex_label_offsets, cugraph_type_erased_host_array_view_t const* fan_out, int num_edge_types, cugraph::c_api::cugraph_sampling_options_t options, @@ -806,9 +806,9 @@ struct neighbor_sampling_functor : public cugraph::c_api::abstract_functor { start_vertices_( reinterpret_cast( start_vertices)), - start_vertex_offsets_( + starting_vertex_label_offsets_( reinterpret_cast( - start_vertex_offsets)), + starting_vertex_label_offsets)), fan_out_( reinterpret_cast(fan_out)), num_edge_types_(num_edge_types), @@ -880,17 +880,17 @@ struct neighbor_sampling_functor : public cugraph::c_api::abstract_functor { std::optional> renumbered_and_sorted_edge_id_renumber_map_label_type_offsets(std::nullopt); - if (start_vertex_offsets_ != nullptr) { + if (starting_vertex_label_offsets_ != nullptr) { // Retrieve the start_vertex_labels start_vertex_labels = cugraph::detail::convert_starting_vertex_label_offsets_to_labels( handle_, - raft::device_span{start_vertex_offsets_->as_type(), - start_vertex_offsets_->size_}); + raft::device_span{starting_vertex_label_offsets_->as_type(), + starting_vertex_label_offsets_->size_}); // Get the number of labels on each GPU if constexpr (multi_gpu) { - auto num_local_labels = start_vertex_offsets_->size_ - 1; + auto num_local_labels = starting_vertex_label_offsets_->size_ - 1; auto global_labels = cugraph::host_scalar_allgather( handle_.get_comms(), num_local_labels, handle_.get_stream()); @@ -898,7 +898,7 @@ struct neighbor_sampling_functor : public cugraph::c_api::abstract_functor { std::exclusive_scan( global_labels.begin(), global_labels.end(), global_labels.begin(), label_t{0}); - // Compute the global start_vertex_label_offsets + // Compute the global starting_vertex_label_offsets cugraph::detail::transform_increment_ints( raft::device_span{(*start_vertex_labels).data(), @@ -997,7 +997,7 @@ struct neighbor_sampling_functor : public cugraph::c_api::abstract_functor { (edge_types != nullptr) ? std::make_optional(edge_types->view()) : std::nullopt, (edge_biases != nullptr) ? *edge_biases : edge_weights->view(), raft::device_span{start_vertices.data(), start_vertices.size()}, - (start_vertex_offsets_ != nullptr) + (starting_vertex_label_offsets_ != nullptr) ? std::make_optional>((*start_vertex_labels).data(), (*start_vertex_labels).size()) : std::nullopt, @@ -1021,7 +1021,7 @@ struct neighbor_sampling_functor : public cugraph::c_api::abstract_functor { (edge_ids != nullptr) ? std::make_optional(edge_ids->view()) : std::nullopt, (edge_types != nullptr) ? std::make_optional(edge_types->view()) : std::nullopt, raft::device_span{start_vertices.data(), start_vertices.size()}, - (start_vertex_offsets_ != nullptr) + (starting_vertex_label_offsets_ != nullptr) ? std::make_optional>((*start_vertex_labels).data(), (*start_vertex_labels).size()) : std::nullopt, @@ -1049,7 +1049,7 @@ struct neighbor_sampling_functor : public cugraph::c_api::abstract_functor { (edge_types != nullptr) ? std::make_optional(edge_types->view()) : std::nullopt, (edge_biases != nullptr) ? *edge_biases : edge_weights->view(), raft::device_span{start_vertices.data(), start_vertices.size()}, - (start_vertex_offsets_ != nullptr) + (starting_vertex_label_offsets_ != nullptr) ? std::make_optional>((*start_vertex_labels).data(), (*start_vertex_labels).size()) : std::nullopt, @@ -1072,7 +1072,7 @@ struct neighbor_sampling_functor : public cugraph::c_api::abstract_functor { (edge_ids != nullptr) ? std::make_optional(edge_ids->view()) : std::nullopt, (edge_types != nullptr) ? std::make_optional(edge_types->view()) : std::nullopt, raft::device_span{start_vertices.data(), start_vertices.size()}, - (start_vertex_offsets_ != nullptr) + (starting_vertex_label_offsets_ != nullptr) ? std::make_optional>((*start_vertex_labels).data(), (*start_vertex_labels).size()) : std::nullopt, @@ -1156,7 +1156,7 @@ struct neighbor_sampling_functor : public cugraph::c_api::abstract_functor { : std::nullopt, options_.retain_seeds_ ? std::make_optional(raft::device_span{ - start_vertex_offsets_->as_type(), start_vertex_offsets_->size_}) + starting_vertex_label_offsets_->as_type(), starting_vertex_label_offsets_->size_}) : std::nullopt, offsets ? std::make_optional( raft::device_span{offsets->data(), offsets->size()}) @@ -1201,7 +1201,7 @@ struct neighbor_sampling_functor : public cugraph::c_api::abstract_functor { : std::nullopt, options_.retain_seeds_ ? std::make_optional(raft::device_span{ - start_vertex_offsets_->as_type(), start_vertex_offsets_->size_}) + starting_vertex_label_offsets_->as_type(), starting_vertex_label_offsets_->size_}) : std::nullopt, offsets ? std::make_optional( raft::device_span{offsets->data(), offsets->size()}) @@ -1261,7 +1261,7 @@ struct neighbor_sampling_functor : public cugraph::c_api::abstract_functor { : std::nullopt, options_.retain_seeds_ ? std::make_optional(raft::device_span{ - start_vertex_offsets_->as_type(), start_vertex_offsets_->size_}) + starting_vertex_label_offsets_->as_type(), starting_vertex_label_offsets_->size_}) : std::nullopt, offsets ? std::make_optional( raft::device_span{offsets->data(), offsets->size()}) @@ -2038,7 +2038,7 @@ cugraph_error_code_t cugraph_heterogeneous_uniform_neighbor_sample( cugraph_rng_state_t* rng_state, cugraph_graph_t* graph, const cugraph_type_erased_device_array_view_t* start_vertices, - const cugraph_type_erased_device_array_view_t* start_vertex_offsets, + const cugraph_type_erased_device_array_view_t* starting_vertex_label_offsets, const cugraph_type_erased_host_array_view_t* fan_out, int num_edge_types, const cugraph_sampling_options_t* options, @@ -2049,17 +2049,17 @@ cugraph_error_code_t cugraph_heterogeneous_uniform_neighbor_sample( auto options_cpp = *reinterpret_cast(options); // FIXME: Should we maintain this contition? - CAPI_EXPECTS((!options_cpp.retain_seeds_) || (start_vertex_offsets != nullptr), + CAPI_EXPECTS((!options_cpp.retain_seeds_) || (starting_vertex_label_offsets != nullptr), CUGRAPH_INVALID_INPUT, - "must specify start_vertex_offsets if retain_seeds is true", + "must specify starting_vertex_label_offsets if retain_seeds is true", *error); - CAPI_EXPECTS((start_vertex_offsets == nullptr) || + CAPI_EXPECTS((starting_vertex_label_offsets == nullptr) || (reinterpret_cast( - start_vertex_offsets) + starting_vertex_label_offsets) ->type_ == SIZE_T), CUGRAPH_INVALID_INPUT, - "start_vertex_offsets should be of type size_t", + "starting_vertex_label_offsets should be of type size_t", *error); CAPI_EXPECTS( @@ -2082,7 +2082,7 @@ cugraph_error_code_t cugraph_heterogeneous_uniform_neighbor_sample( graph, nullptr, start_vertices, - start_vertex_offsets, + starting_vertex_label_offsets, fan_out, num_edge_types, std::move(options_cpp), @@ -2097,7 +2097,7 @@ cugraph_error_code_t cugraph_heterogeneous_biased_neighbor_sample( cugraph_graph_t* graph, const cugraph_edge_property_view_t* edge_biases, const cugraph_type_erased_device_array_view_t* start_vertices, - const cugraph_type_erased_device_array_view_t* start_vertex_offsets, + const cugraph_type_erased_device_array_view_t* starting_vertex_label_offsets, const cugraph_type_erased_host_array_view_t* fan_out, int num_edge_types, const cugraph_sampling_options_t* options, @@ -2115,17 +2115,17 @@ cugraph_error_code_t cugraph_heterogeneous_biased_neighbor_sample( *error); // FIXME: Should we maintain this contition? - CAPI_EXPECTS((!options_cpp.retain_seeds_) || (start_vertex_offsets != nullptr), + CAPI_EXPECTS((!options_cpp.retain_seeds_) || (starting_vertex_label_offsets != nullptr), CUGRAPH_INVALID_INPUT, - "must specify start_vertex_offsets if retain_seeds is true", + "must specify starting_vertex_label_offsets if retain_seeds is true", *error); - CAPI_EXPECTS((start_vertex_offsets == nullptr) || + CAPI_EXPECTS((starting_vertex_label_offsets == nullptr) || (reinterpret_cast( - start_vertex_offsets) + starting_vertex_label_offsets) ->type_ == SIZE_T), CUGRAPH_INVALID_INPUT, - "start_vertex_offsets should be of type size_t", + "starting_vertex_label_offsets should be of type size_t", *error); CAPI_EXPECTS( @@ -2148,7 +2148,7 @@ cugraph_error_code_t cugraph_heterogeneous_biased_neighbor_sample( graph, edge_biases, start_vertices, - start_vertex_offsets, + starting_vertex_label_offsets, fan_out, num_edge_types, std::move(options_cpp), @@ -2162,7 +2162,7 @@ cugraph_error_code_t cugraph_homogeneous_uniform_neighbor_sample( cugraph_rng_state_t* rng_state, cugraph_graph_t* graph, const cugraph_type_erased_device_array_view_t* start_vertices, - const cugraph_type_erased_device_array_view_t* start_vertex_offsets, // RENAME? + const cugraph_type_erased_device_array_view_t* starting_vertex_label_offsets, // RENAME? const cugraph_type_erased_host_array_view_t* fan_out, const cugraph_sampling_options_t* options, bool_t do_expensive_check, @@ -2172,17 +2172,17 @@ cugraph_error_code_t cugraph_homogeneous_uniform_neighbor_sample( auto options_cpp = *reinterpret_cast(options); // FIXME: Should we maintain this contition? - CAPI_EXPECTS((!options_cpp.retain_seeds_) || (start_vertex_offsets != nullptr), + CAPI_EXPECTS((!options_cpp.retain_seeds_) || (starting_vertex_label_offsets != nullptr), CUGRAPH_INVALID_INPUT, - "must specify start_vertex_offsets if retain_seeds is true", + "must specify starting_vertex_label_offsets if retain_seeds is true", *error); - CAPI_EXPECTS((start_vertex_offsets == nullptr) || + CAPI_EXPECTS((starting_vertex_label_offsets == nullptr) || (reinterpret_cast( - start_vertex_offsets) + starting_vertex_label_offsets) ->type_ == SIZE_T), CUGRAPH_INVALID_INPUT, - "start_vertex_offsets should be of type size_t", + "starting_vertex_label_offsets should be of type size_t", *error); CAPI_EXPECTS( @@ -2205,7 +2205,7 @@ cugraph_error_code_t cugraph_homogeneous_uniform_neighbor_sample( graph, nullptr, start_vertices, - start_vertex_offsets, + starting_vertex_label_offsets, fan_out, 1, // num_edge_types std::move(options_cpp), @@ -2220,7 +2220,7 @@ cugraph_error_code_t cugraph_homogeneous_biased_neighbor_sample( cugraph_graph_t* graph, const cugraph_edge_property_view_t* edge_biases, const cugraph_type_erased_device_array_view_t* start_vertices, - const cugraph_type_erased_device_array_view_t* start_vertex_offsets, + const cugraph_type_erased_device_array_view_t* starting_vertex_label_offsets, const cugraph_type_erased_host_array_view_t* fan_out, const cugraph_sampling_options_t* options, bool_t do_expensive_check, @@ -2237,17 +2237,17 @@ cugraph_error_code_t cugraph_homogeneous_biased_neighbor_sample( *error); // FIXME: Should we maintain this contition? - CAPI_EXPECTS((!options_cpp.retain_seeds_) || (start_vertex_offsets != nullptr), + CAPI_EXPECTS((!options_cpp.retain_seeds_) || (starting_vertex_label_offsets != nullptr), CUGRAPH_INVALID_INPUT, - "must specify start_vertex_offsets if retain_seeds is true", + "must specify starting_vertex_label_offsets if retain_seeds is true", *error); - CAPI_EXPECTS((start_vertex_offsets == nullptr) || + CAPI_EXPECTS((starting_vertex_label_offsets == nullptr) || (reinterpret_cast( - start_vertex_offsets) + starting_vertex_label_offsets) ->type_ == SIZE_T), CUGRAPH_INVALID_INPUT, - "start_vertex_offsets should be of type size_t", + "starting_vertex_label_offsets should be of type size_t", *error); CAPI_EXPECTS( @@ -2270,7 +2270,7 @@ cugraph_error_code_t cugraph_homogeneous_biased_neighbor_sample( graph, edge_biases, start_vertices, - start_vertex_offsets, + starting_vertex_label_offsets, fan_out, 1, // num_edge_types std::move(options_cpp), From 463f12b8cb5d908fcf9595317e0c529d34f51a75 Mon Sep 17 00:00:00 2001 From: jnke2016 Date: Thu, 21 Nov 2024 21:45:42 -0800 Subject: [PATCH 17/49] fix style --- cpp/src/c_api/neighbor_sampling.cpp | 50 +++++++++++++++-------------- 1 file changed, 26 insertions(+), 24 deletions(-) diff --git a/cpp/src/c_api/neighbor_sampling.cpp b/cpp/src/c_api/neighbor_sampling.cpp index 51712481034..1cc212da5db 100644 --- a/cpp/src/c_api/neighbor_sampling.cpp +++ b/cpp/src/c_api/neighbor_sampling.cpp @@ -778,7 +778,8 @@ struct neighbor_sampling_functor : public cugraph::c_api::abstract_functor { cugraph::c_api::cugraph_graph_t* graph_{nullptr}; cugraph::c_api::cugraph_edge_property_view_t const* edge_biases_{nullptr}; cugraph::c_api::cugraph_type_erased_device_array_view_t const* start_vertices_{nullptr}; - cugraph::c_api::cugraph_type_erased_device_array_view_t const* starting_vertex_label_offsets_{nullptr}; + cugraph::c_api::cugraph_type_erased_device_array_view_t const* starting_vertex_label_offsets_{ + nullptr}; cugraph::c_api::cugraph_type_erased_host_array_view_t const* fan_out_{nullptr}; int num_edge_types_{}; cugraph::c_api::cugraph_sampling_options_t options_{}; @@ -786,17 +787,18 @@ struct neighbor_sampling_functor : public cugraph::c_api::abstract_functor { bool do_expensive_check_{false}; cugraph::c_api::cugraph_sample_result_t* result_{nullptr}; - neighbor_sampling_functor(cugraph_resource_handle_t const* handle, - cugraph_rng_state_t* rng_state, - cugraph_graph_t* graph, - cugraph_edge_property_view_t const* edge_biases, - cugraph_type_erased_device_array_view_t const* start_vertices, - cugraph_type_erased_device_array_view_t const* starting_vertex_label_offsets, - cugraph_type_erased_host_array_view_t const* fan_out, - int num_edge_types, - cugraph::c_api::cugraph_sampling_options_t options, - bool is_biased, - bool do_expensive_check) + neighbor_sampling_functor( + cugraph_resource_handle_t const* handle, + cugraph_rng_state_t* rng_state, + cugraph_graph_t* graph, + cugraph_edge_property_view_t const* edge_biases, + cugraph_type_erased_device_array_view_t const* start_vertices, + cugraph_type_erased_device_array_view_t const* starting_vertex_label_offsets, + cugraph_type_erased_host_array_view_t const* fan_out, + int num_edge_types, + cugraph::c_api::cugraph_sampling_options_t options, + bool is_biased, + bool do_expensive_check) : abstract_functor(), handle_(*reinterpret_cast(handle)->handle_), rng_state_(reinterpret_cast(rng_state)), @@ -1154,10 +1156,10 @@ struct neighbor_sampling_functor : public cugraph::c_api::abstract_functor { ? std::make_optional(raft::device_span{ start_vertices_->as_type(), start_vertices_->size_}) : std::nullopt, - options_.retain_seeds_ - ? std::make_optional(raft::device_span{ - starting_vertex_label_offsets_->as_type(), starting_vertex_label_offsets_->size_}) - : std::nullopt, + options_.retain_seeds_ ? std::make_optional(raft::device_span{ + starting_vertex_label_offsets_->as_type(), + starting_vertex_label_offsets_->size_}) + : std::nullopt, offsets ? std::make_optional( raft::device_span{offsets->data(), offsets->size()}) : std::nullopt, @@ -1199,10 +1201,10 @@ struct neighbor_sampling_functor : public cugraph::c_api::abstract_functor { ? std::make_optional(raft::device_span{ start_vertices_->as_type(), start_vertices_->size_}) : std::nullopt, - options_.retain_seeds_ - ? std::make_optional(raft::device_span{ - starting_vertex_label_offsets_->as_type(), starting_vertex_label_offsets_->size_}) - : std::nullopt, + options_.retain_seeds_ ? std::make_optional(raft::device_span{ + starting_vertex_label_offsets_->as_type(), + starting_vertex_label_offsets_->size_}) + : std::nullopt, offsets ? std::make_optional( raft::device_span{offsets->data(), offsets->size()}) : std::nullopt, @@ -1259,10 +1261,10 @@ struct neighbor_sampling_functor : public cugraph::c_api::abstract_functor { ? std::make_optional(raft::device_span{ start_vertices_->as_type(), start_vertices_->size_}) : std::nullopt, - options_.retain_seeds_ - ? std::make_optional(raft::device_span{ - starting_vertex_label_offsets_->as_type(), starting_vertex_label_offsets_->size_}) - : std::nullopt, + options_.retain_seeds_ ? std::make_optional(raft::device_span{ + starting_vertex_label_offsets_->as_type(), + starting_vertex_label_offsets_->size_}) + : std::nullopt, offsets ? std::make_optional( raft::device_span{offsets->data(), offsets->size()}) : std::nullopt, From 3767b94ecfc4de95a0b19ac746ee7c30a79a6fd3 Mon Sep 17 00:00:00 2001 From: jnke2016 Date: Thu, 21 Nov 2024 21:50:15 -0800 Subject: [PATCH 18/49] rename variable for consistency --- cpp/include/cugraph/sampling_functions.hpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/cpp/include/cugraph/sampling_functions.hpp b/cpp/include/cugraph/sampling_functions.hpp index 3d41e954416..981c42135f6 100644 --- a/cpp/include/cugraph/sampling_functions.hpp +++ b/cpp/include/cugraph/sampling_functions.hpp @@ -306,7 +306,7 @@ struct sampling_flags_t { * @param edge_type_view Optional view object holding edge types for @p graph_view. * @param starting_vertices Device span of starting vertex IDs for the sampling. * In a multi-gpu context the starting vertices should be local to this GPU. - * @param starting_vertex_label_offsets Optional device span of labels associated with each starting + * @param starting_vertex_labels Optional device span of labels associated with each starting * vertex for the sampling. * @param label_to_output_comm_rank Optional device span identifying which rank should get sampling * outputs of each vertex label. This should be the same on each rank. @@ -340,7 +340,7 @@ homogeneous_uniform_neighbor_sample( std::optional> edge_id_view, std::optional> edge_type_view, raft::device_span starting_vertices, - std::optional> starting_vertex_label_offsets, + std::optional> starting_vertex_labels, std::optional> label_to_output_comm_rank, raft::host_span fan_out, sampling_flags_t sampling_flags, @@ -385,7 +385,7 @@ homogeneous_uniform_neighbor_sample( * corresponding edge can never be selected. * @param starting_vertices Device span of starting vertex IDs for the sampling. * In a multi-gpu context the starting vertices should be local to this GPU. - * @param starting_vertex_label_offsets Optional device span of labels associated with each starting + * @param starting_vertex_labels Optional device span of labels associated with each starting * vertex for the sampling. * @param label_to_output_comm_rank Optional device span identifying which rank should get sampling * outputs of each vertex label. This should be the same on each rank. @@ -421,7 +421,7 @@ homogeneous_biased_neighbor_sample( std::optional> edge_type_view, edge_property_view_t edge_bias_view, raft::device_span starting_vertices, - std::optional> starting_vertex_label_offsets, + std::optional> starting_vertex_labels, std::optional> label_to_output_comm_rank, raft::host_span fan_out, sampling_flags_t sampling_flags, @@ -462,7 +462,7 @@ homogeneous_biased_neighbor_sample( * @param edge_type_view Optional view object holding edge types for @p graph_view. * @param starting_vertices Device span of starting vertex IDs for the sampling. * In a multi-gpu context the starting vertices should be local to this GPU. - * @param starting_vertex_label_offsets Optional device span of labels associated with each starting + * @param starting_vertex_labels Optional device span of labels associated with each starting * vertex for the sampling. * @param label_to_output_comm_rank Optional device span identifying which rank should get sampling * outputs of each vertex label. This should be the same on each rank. @@ -498,7 +498,7 @@ heterogeneous_uniform_neighbor_sample( std::optional> edge_id_view, std::optional> edge_type_view, raft::device_span starting_vertices, - std::optional> starting_vertex_label_offsets, + std::optional> starting_vertex_labels, std::optional> label_to_output_comm_rank, raft::host_span fan_out, edge_type_t num_edge_types, @@ -545,7 +545,7 @@ heterogeneous_uniform_neighbor_sample( * corresponding edge can never be selected. * @param starting_vertices Device span of starting vertex IDs for the sampling. * In a multi-gpu context the starting vertices should be local to this GPU. - * @param starting_vertex_label_offsets Optional device span of labels associated with each starting + * @param starting_vertex_labels Optional device span of labels associated with each starting * vertex for the sampling. * @param label_to_output_comm_rank Optional device span identifying which rank should get sampling * outputs of each vertex label. This should be the same on each rank. @@ -583,7 +583,7 @@ heterogeneous_biased_neighbor_sample( std::optional> edge_type_view, edge_property_view_t edge_bias_view, raft::device_span starting_vertices, - std::optional> starting_vertex_label_offsets, + std::optional> starting_vertex_labels, std::optional> label_to_output_comm_rank, raft::host_span fan_out, edge_type_t num_edge_types, From 6aeee2830bd535885571998fe5d55016c239a826 Mon Sep 17 00:00:00 2001 From: jnke2016 Date: Fri, 22 Nov 2024 12:05:28 -0800 Subject: [PATCH 19/49] add support for vertex type --- cpp/include/cugraph_c/sampling_algorithms.h | 6 ++ cpp/src/c_api/neighbor_sampling.cpp | 97 ++++++++++++--------- 2 files changed, 63 insertions(+), 40 deletions(-) diff --git a/cpp/include/cugraph_c/sampling_algorithms.h b/cpp/include/cugraph_c/sampling_algorithms.h index 499b3131554..f048d338b97 100644 --- a/cpp/include/cugraph_c/sampling_algorithms.h +++ b/cpp/include/cugraph_c/sampling_algorithms.h @@ -551,6 +551,8 @@ cugraph_error_code_t cugraph_homogeneous_biased_neighbor_sample( * @param [in] start_vertices Device array of start vertices for the sampling * @param [in] starting_vertex_label_offsets Device array of the offsets for each label in * the seed list. This parameter is only used with the retain_seeds option. + * @param [in] vertex_type_offsets Device array of the offsets for each vertex type in the + * graph. * @param [in] fan_out Host array defining the fan out at each step in the sampling * algorithm. We only support fan_out values of type INT32 * @param [in] num_edge_types Number of edge types where a value of 1 translates to homogeneous @@ -570,6 +572,7 @@ cugraph_error_code_t cugraph_heterogeneous_uniform_neighbor_sample( cugraph_graph_t* graph, const cugraph_type_erased_device_array_view_t* start_vertices, const cugraph_type_erased_device_array_view_t* starting_vertex_label_offsets, + const cugraph_type_erased_device_array_view_t* vertex_type_offsets, const cugraph_type_erased_host_array_view_t* fan_out, int num_edge_types, const cugraph_sampling_options_t* options, @@ -598,6 +601,8 @@ cugraph_error_code_t cugraph_heterogeneous_uniform_neighbor_sample( * @param [in] start_vertices Device array of start vertices for the sampling * @param [in] starting_vertex_label_offsets Device array of the offsets for each label in * the seed list. This parameter is only used with the retain_seeds option. + * @param [in] vertex_type_offsets Device array of the offsets for each vertex type in the + * graph. * @param [in] fan_out Host array defining the fan out at each step in the sampling * algorithm. We only support fan_out values of type INT32 * @param [in] num_edge_types Number of edge types where a value of 1 translates to homogeneous @@ -618,6 +623,7 @@ cugraph_error_code_t cugraph_heterogeneous_biased_neighbor_sample( const cugraph_edge_property_view_t* edge_biases, const cugraph_type_erased_device_array_view_t* start_vertices, const cugraph_type_erased_device_array_view_t* starting_vertex_label_offsets, + const cugraph_type_erased_device_array_view_t* vertex_type_offsets, const cugraph_type_erased_host_array_view_t* fan_out, int num_edge_types, const cugraph_sampling_options_t* options, diff --git a/cpp/src/c_api/neighbor_sampling.cpp b/cpp/src/c_api/neighbor_sampling.cpp index 1cc212da5db..54fec002329 100644 --- a/cpp/src/c_api/neighbor_sampling.cpp +++ b/cpp/src/c_api/neighbor_sampling.cpp @@ -778,8 +778,8 @@ struct neighbor_sampling_functor : public cugraph::c_api::abstract_functor { cugraph::c_api::cugraph_graph_t* graph_{nullptr}; cugraph::c_api::cugraph_edge_property_view_t const* edge_biases_{nullptr}; cugraph::c_api::cugraph_type_erased_device_array_view_t const* start_vertices_{nullptr}; - cugraph::c_api::cugraph_type_erased_device_array_view_t const* starting_vertex_label_offsets_{ - nullptr}; + cugraph::c_api::cugraph_type_erased_device_array_view_t const* starting_vertex_label_offsets_{nullptr}; + cugraph::c_api::cugraph_type_erased_device_array_view_t const* vertex_type_offsets_{nullptr}; cugraph::c_api::cugraph_type_erased_host_array_view_t const* fan_out_{nullptr}; int num_edge_types_{}; cugraph::c_api::cugraph_sampling_options_t options_{}; @@ -787,18 +787,18 @@ struct neighbor_sampling_functor : public cugraph::c_api::abstract_functor { bool do_expensive_check_{false}; cugraph::c_api::cugraph_sample_result_t* result_{nullptr}; - neighbor_sampling_functor( - cugraph_resource_handle_t const* handle, - cugraph_rng_state_t* rng_state, - cugraph_graph_t* graph, - cugraph_edge_property_view_t const* edge_biases, - cugraph_type_erased_device_array_view_t const* start_vertices, - cugraph_type_erased_device_array_view_t const* starting_vertex_label_offsets, - cugraph_type_erased_host_array_view_t const* fan_out, - int num_edge_types, - cugraph::c_api::cugraph_sampling_options_t options, - bool is_biased, - bool do_expensive_check) + neighbor_sampling_functor(cugraph_resource_handle_t const* handle, + cugraph_rng_state_t* rng_state, + cugraph_graph_t* graph, + cugraph_edge_property_view_t const* edge_biases, + cugraph_type_erased_device_array_view_t const* start_vertices, + cugraph_type_erased_device_array_view_t const* starting_vertex_label_offsets, + cugraph_type_erased_device_array_view_t const* vertex_type_offsets, + cugraph_type_erased_host_array_view_t const* fan_out, + int num_edge_types, + cugraph::c_api::cugraph_sampling_options_t options, + bool is_biased, + bool do_expensive_check) : abstract_functor(), handle_(*reinterpret_cast(handle)->handle_), rng_state_(reinterpret_cast(rng_state)), @@ -811,6 +811,9 @@ struct neighbor_sampling_functor : public cugraph::c_api::abstract_functor { starting_vertex_label_offsets_( reinterpret_cast( starting_vertex_label_offsets)), + vertex_type_offsets_( + reinterpret_cast( + vertex_type_offsets)), fan_out_( reinterpret_cast(fan_out)), num_edge_types_(num_edge_types), @@ -875,7 +878,7 @@ struct neighbor_sampling_functor : public cugraph::c_api::abstract_functor { std::optional> start_vertex_labels{std::nullopt}; std::optional> local_label_to_comm_rank{std::nullopt}; std::optional> label_to_comm_rank{ - std::nullopt}; // global after allgatherv + std::nullopt}; // global after allgatherv std::optional> renumbered_and_sorted_edge_id_renumber_map( std::nullopt); @@ -1156,10 +1159,10 @@ struct neighbor_sampling_functor : public cugraph::c_api::abstract_functor { ? std::make_optional(raft::device_span{ start_vertices_->as_type(), start_vertices_->size_}) : std::nullopt, - options_.retain_seeds_ ? std::make_optional(raft::device_span{ - starting_vertex_label_offsets_->as_type(), - starting_vertex_label_offsets_->size_}) - : std::nullopt, + options_.retain_seeds_ + ? std::make_optional(raft::device_span{ + starting_vertex_label_offsets_->as_type(), starting_vertex_label_offsets_->size_}) + : std::nullopt, offsets ? std::make_optional( raft::device_span{offsets->data(), offsets->size()}) : std::nullopt, @@ -1201,10 +1204,10 @@ struct neighbor_sampling_functor : public cugraph::c_api::abstract_functor { ? std::make_optional(raft::device_span{ start_vertices_->as_type(), start_vertices_->size_}) : std::nullopt, - options_.retain_seeds_ ? std::make_optional(raft::device_span{ - starting_vertex_label_offsets_->as_type(), - starting_vertex_label_offsets_->size_}) - : std::nullopt, + options_.retain_seeds_ + ? std::make_optional(raft::device_span{ + starting_vertex_label_offsets_->as_type(), starting_vertex_label_offsets_->size_}) + : std::nullopt, offsets ? std::make_optional( raft::device_span{offsets->data(), offsets->size()}) : std::nullopt, @@ -1225,17 +1228,19 @@ struct neighbor_sampling_functor : public cugraph::c_api::abstract_functor { } else { // heterogeneous renumbering - // FIXME: If no 'vertex_type_offsets' is provided, all vertices are assumed to have - // a vertex type of value 1. Update the API once 'vertex_type_offsets' is supported rmm::device_uvector vertex_type_offsets(2, handle_.get_stream()); - cugraph::detail::stride_fill(handle_.get_stream(), - vertex_type_offsets.begin(), - vertex_type_offsets.size(), - vertex_t{0}, - vertex_t{graph_view.local_vertex_partition_range_size()} + if (vertex_type_offsets_ != nullptr) { + // If no 'vertex_type_offsets' is provided, all vertices are assumed to have + // a vertex type of value 1. + cugraph::detail::stride_fill(handle_.get_stream(), + vertex_type_offsets.begin(), + vertex_type_offsets.size(), + vertex_t{0}, + vertex_t{graph_view.local_vertex_partition_range_size()} - ); + ); + } rmm::device_uvector output_majors(0, handle_.get_stream()); rmm::device_uvector output_renumber_map(0, handle_.get_stream()); @@ -1261,21 +1266,27 @@ struct neighbor_sampling_functor : public cugraph::c_api::abstract_functor { ? std::make_optional(raft::device_span{ start_vertices_->as_type(), start_vertices_->size_}) : std::nullopt, - options_.retain_seeds_ ? std::make_optional(raft::device_span{ - starting_vertex_label_offsets_->as_type(), - starting_vertex_label_offsets_->size_}) - : std::nullopt, + options_.retain_seeds_ + ? std::make_optional(raft::device_span{ + starting_vertex_label_offsets_->as_type(), starting_vertex_label_offsets_->size_}) + : std::nullopt, offsets ? std::make_optional( raft::device_span{offsets->data(), offsets->size()}) : std::nullopt, - raft::device_span{vertex_type_offsets.data(), + + (vertex_type_offsets_ != nullptr) + ? raft::device_span{vertex_type_offsets_->as_type(), vertex_type_offsets_->size_} + : raft::device_span{vertex_type_offsets.data(), vertex_type_offsets.size()}, - + edge_label ? (*offsets).size() - 1 : size_t{1}, hop ? fan_out_->size_ : size_t{1}, - vertex_type_offsets.size() - - 1, // num_vertex_type is by default 1 if 'vertex_type_offsets' is not provided + (vertex_type_offsets_ != nullptr) + ? vertex_type_offsets_->size_ - 1 + : vertex_type_offsets.size() - 1, + + // num_vertex_type is by default 1 if 'vertex_type_offsets' is not provided num_edge_types_, src_is_major, do_expensive_check_); @@ -2041,6 +2052,7 @@ cugraph_error_code_t cugraph_heterogeneous_uniform_neighbor_sample( cugraph_graph_t* graph, const cugraph_type_erased_device_array_view_t* start_vertices, const cugraph_type_erased_device_array_view_t* starting_vertex_label_offsets, + const cugraph_type_erased_device_array_view_t* vertex_type_offsets, const cugraph_type_erased_host_array_view_t* fan_out, int num_edge_types, const cugraph_sampling_options_t* options, @@ -2085,6 +2097,7 @@ cugraph_error_code_t cugraph_heterogeneous_uniform_neighbor_sample( nullptr, start_vertices, starting_vertex_label_offsets, + vertex_type_offsets, fan_out, num_edge_types, std::move(options_cpp), @@ -2100,6 +2113,7 @@ cugraph_error_code_t cugraph_heterogeneous_biased_neighbor_sample( const cugraph_edge_property_view_t* edge_biases, const cugraph_type_erased_device_array_view_t* start_vertices, const cugraph_type_erased_device_array_view_t* starting_vertex_label_offsets, + const cugraph_type_erased_device_array_view_t* vertex_type_offsets, const cugraph_type_erased_host_array_view_t* fan_out, int num_edge_types, const cugraph_sampling_options_t* options, @@ -2151,6 +2165,7 @@ cugraph_error_code_t cugraph_heterogeneous_biased_neighbor_sample( edge_biases, start_vertices, starting_vertex_label_offsets, + vertex_type_offsets, fan_out, num_edge_types, std::move(options_cpp), @@ -2164,7 +2179,7 @@ cugraph_error_code_t cugraph_homogeneous_uniform_neighbor_sample( cugraph_rng_state_t* rng_state, cugraph_graph_t* graph, const cugraph_type_erased_device_array_view_t* start_vertices, - const cugraph_type_erased_device_array_view_t* starting_vertex_label_offsets, // RENAME? + const cugraph_type_erased_device_array_view_t* starting_vertex_label_offsets, const cugraph_type_erased_host_array_view_t* fan_out, const cugraph_sampling_options_t* options, bool_t do_expensive_check, @@ -2208,6 +2223,7 @@ cugraph_error_code_t cugraph_homogeneous_uniform_neighbor_sample( nullptr, start_vertices, starting_vertex_label_offsets, + nullptr, fan_out, 1, // num_edge_types std::move(options_cpp), @@ -2273,6 +2289,7 @@ cugraph_error_code_t cugraph_homogeneous_biased_neighbor_sample( edge_biases, start_vertices, starting_vertex_label_offsets, + nullptr, fan_out, 1, // num_edge_types std::move(options_cpp), From 223a73b1383bc0aa69a0626ecd013769917bf176 Mon Sep 17 00:00:00 2001 From: jnke2016 Date: Fri, 22 Nov 2024 12:34:14 -0800 Subject: [PATCH 20/49] add support for vertex type at the plc layer --- cpp/src/c_api/neighbor_sampling.cpp | 2 +- .../heterogeneous_biased_neighbor_sample.pyx | 21 +++++++++++++++++++ .../heterogeneous_uniform_neighbor_sample.pyx | 20 ++++++++++++++++++ 3 files changed, 42 insertions(+), 1 deletion(-) diff --git a/cpp/src/c_api/neighbor_sampling.cpp b/cpp/src/c_api/neighbor_sampling.cpp index 54fec002329..a87fd0cfd89 100644 --- a/cpp/src/c_api/neighbor_sampling.cpp +++ b/cpp/src/c_api/neighbor_sampling.cpp @@ -1230,7 +1230,7 @@ struct neighbor_sampling_functor : public cugraph::c_api::abstract_functor { rmm::device_uvector vertex_type_offsets(2, handle_.get_stream()); - if (vertex_type_offsets_ != nullptr) { + if (vertex_type_offsets_ == nullptr) { // If no 'vertex_type_offsets' is provided, all vertices are assumed to have // a vertex type of value 1. cugraph::detail::stride_fill(handle_.get_stream(), diff --git a/python/pylibcugraph/pylibcugraph/heterogeneous_biased_neighbor_sample.pyx b/python/pylibcugraph/pylibcugraph/heterogeneous_biased_neighbor_sample.pyx index 0f49dec712b..e506a413ab5 100644 --- a/python/pylibcugraph/pylibcugraph/heterogeneous_biased_neighbor_sample.pyx +++ b/python/pylibcugraph/pylibcugraph/heterogeneous_biased_neighbor_sample.pyx @@ -87,6 +87,7 @@ def heterogeneous_biased_neighbor_sample(ResourceHandle resource_handle, _GPUGraph input_graph, start_vertex_list, starting_vertex_label_offsets, + vertex_type_offsets, h_fan_out, num_edge_types, bool_t with_replacement, @@ -123,6 +124,9 @@ def heterogeneous_biased_neighbor_sample(ResourceHandle resource_handle, Offsets of each label within the start vertex list. Expanding 'starting_vertex_label_offsets' must lead to an array of len(start_vertex_list) + + vertex_type_offsets: device array type (Optional) + Offsets for each vertex type in the graph. h_fan_out: numpy array type Device array containing the branching out (fan-out) degrees per @@ -247,6 +251,7 @@ def heterogeneous_biased_neighbor_sample(ResourceHandle resource_handle, assert_CAI_type(start_vertex_list, "start_vertex_list") assert_CAI_type(starting_vertex_label_offsets, "starting_vertex_label_offsets", True) + assert_CAI_type(vertex_type_offsets, "vertex_type_offsets", True) assert_AI_type(h_fan_out, "h_fan_out") @@ -276,6 +281,11 @@ def heterogeneous_biased_neighbor_sample(ResourceHandle resource_handle, if starting_vertex_label_offsets is not None: cai_starting_vertex_label_offsets_ptr = \ starting_vertex_label_offsets.__cuda_array_interface__['data'][0] + + cdef uintptr_t cai_vertex_type_offsets_ptr + if vertex_type_offsets is not None: + cai_vertex_type_offsets_ptr = \ + vertex_type_offsets.__cuda_array_interface__['data'][0] cdef cugraph_type_erased_device_array_view_t* start_vertex_list_ptr = \ @@ -293,6 +303,16 @@ def heterogeneous_biased_neighbor_sample(ResourceHandle resource_handle, len(starting_vertex_label_offsets), SIZE_T ) + + cdef cugraph_type_erased_device_array_view_t* vertex_type_offsets_ptr = NULL + if vertex_type_offsets is not None: + vertex_type_offsets_ptr = \ + cugraph_type_erased_device_array_view_create( + cai_vertex_type_offsets_ptr, + len(vertex_type_offsets), + SIZE_T + ) + cdef cugraph_type_erased_device_array_view_t* label_offsets_ptr = NULL if retain_seeds: @@ -354,6 +374,7 @@ def heterogeneous_biased_neighbor_sample(ResourceHandle resource_handle, NULL, # FIXME: Add support for biased neighbor sampling start_vertex_list_ptr, starting_vertex_label_offsets_ptr, + vertex_type_offsets_ptr, fan_out_ptr, num_edge_types, sampling_options, diff --git a/python/pylibcugraph/pylibcugraph/heterogeneous_uniform_neighbor_sample.pyx b/python/pylibcugraph/pylibcugraph/heterogeneous_uniform_neighbor_sample.pyx index c586fcdff8c..6e8ab93de62 100644 --- a/python/pylibcugraph/pylibcugraph/heterogeneous_uniform_neighbor_sample.pyx +++ b/python/pylibcugraph/pylibcugraph/heterogeneous_uniform_neighbor_sample.pyx @@ -84,6 +84,7 @@ def heterogeneous_uniform_neighbor_sample(ResourceHandle resource_handle, _GPUGraph input_graph, start_vertex_list, starting_vertex_label_offsets, + vertex_type_offsets, h_fan_out, num_edge_types, bool_t with_replacement, @@ -118,6 +119,9 @@ def heterogeneous_uniform_neighbor_sample(ResourceHandle resource_handle, Offsets of each label within the start vertex list. Expanding 'starting_vertex_label_offsets' must lead to an array of len(start_vertex_list) + + vertex_type_offsets: device array type (Optional) + Offsets for each vertex type in the graph. h_fan_out: numpy array type Device array containing the branching out (fan-out) degrees per @@ -242,6 +246,7 @@ def heterogeneous_uniform_neighbor_sample(ResourceHandle resource_handle, assert_CAI_type(start_vertex_list, "start_vertex_list") assert_CAI_type(starting_vertex_label_offsets, "starting_vertex_label_offsets", True) + assert_CAI_type(vertex_type_offsets, "vertex_type_offsets", True) assert_AI_type(h_fan_out, "h_fan_out") @@ -270,6 +275,11 @@ def heterogeneous_uniform_neighbor_sample(ResourceHandle resource_handle, if starting_vertex_label_offsets is not None: cai_starting_vertex_label_offsets_ptr = \ starting_vertex_label_offsets.__cuda_array_interface__['data'][0] + + cdef uintptr_t cai_vertex_type_offsets_ptr + if vertex_type_offsets is not None: + cai_vertex_type_offsets_ptr = \ + vertex_type_offsets.__cuda_array_interface__['data'][0] cdef cugraph_type_erased_device_array_view_t* start_vertex_list_ptr = \ @@ -288,6 +298,15 @@ def heterogeneous_uniform_neighbor_sample(ResourceHandle resource_handle, SIZE_T ) + cdef cugraph_type_erased_device_array_view_t* vertex_type_offsets_ptr = NULL + if vertex_type_offsets is not None: + vertex_type_offsets_ptr = \ + cugraph_type_erased_device_array_view_create( + cai_vertex_type_offsets_ptr, + len(vertex_type_offsets), + SIZE_T + ) + cdef cugraph_type_erased_device_array_view_t* label_offsets_ptr = NULL if retain_seeds: if starting_vertex_label_offsets is None: @@ -347,6 +366,7 @@ def heterogeneous_uniform_neighbor_sample(ResourceHandle resource_handle, c_graph_ptr, start_vertex_list_ptr, starting_vertex_label_offsets_ptr, + vertex_type_offsets_ptr, fan_out_ptr, num_edge_types, sampling_options, From 061c8ccbd0637a8a1df28c204231b840342e2fac Mon Sep 17 00:00:00 2001 From: jnke2016 Date: Fri, 22 Nov 2024 12:36:29 -0800 Subject: [PATCH 21/49] properly handle missing edge types --- cpp/src/sampling/neighbor_sampling_impl.hpp | 137 ++++++++++++-------- 1 file changed, 81 insertions(+), 56 deletions(-) diff --git a/cpp/src/sampling/neighbor_sampling_impl.hpp b/cpp/src/sampling/neighbor_sampling_impl.hpp index 2ddc85a41bc..142d8054abd 100644 --- a/cpp/src/sampling/neighbor_sampling_impl.hpp +++ b/cpp/src/sampling/neighbor_sampling_impl.hpp @@ -30,6 +30,7 @@ #include #include +#include namespace cugraph { namespace detail { @@ -104,14 +105,38 @@ neighbor_sample_impl(raft::handle_t const& handle, edge_masks_vector{}; graph_view_t modified_graph_view = graph_view; edge_masks_vector.reserve(num_edge_types); + + label_t num_unique_labels = 0; + + std::optional> cp_starting_vertex_labels{std::nullopt}; - label_t num_labels = 0; if (starting_vertex_labels) { - // Initial number of labels. Will be leveraged if there is no sampling result - num_labels = starting_vertex_labels->size(); + // Find the number of unique lables + cp_starting_vertex_labels = rmm::device_uvector(starting_vertex_labels->size(), handle.get_stream()); + + thrust::copy( + handle.get_thrust_policy(), + starting_vertex_labels->begin(), + starting_vertex_labels->end(), + cp_starting_vertex_labels->begin()); + + thrust::sort( + handle.get_thrust_policy(), + cp_starting_vertex_labels->begin(), + cp_starting_vertex_labels->end()); + + num_unique_labels = thrust::unique_count(handle.get_thrust_policy(), + cp_starting_vertex_labels->begin(), + cp_starting_vertex_labels->end()); + + } + + + + if (num_edge_types > 1) { for (int i = 0; i < num_edge_types; i++) { cugraph::edge_property_t, bool> @@ -374,60 +399,60 @@ neighbor_sample_impl(raft::handle_t const& handle, if (result_labels) { cp_result_labels = rmm::device_uvector(result_labels->size(), handle.get_stream()); - thrust::copy(handle.get_thrust_policy(), - result_labels->begin(), - result_labels->end(), - cp_result_labels->begin()); + thrust::copy( + handle.get_thrust_policy(), + result_labels->begin(), + result_labels->end(), + cp_result_labels->begin()); } - - // FIXME: remove the offsets computation in 'shuffle_and_organize_output' as it doesn't - // account for missing labels that are not sampled. - std::tie(result_srcs, - result_dsts, - result_weights, - result_edge_ids, - result_edge_types, - result_hops, - result_labels, - result_offsets) = detail::shuffle_and_organize_output(handle, - std::move(result_srcs), - std::move(result_dsts), - std::move(result_weights), - std::move(result_edge_ids), - std::move(result_edge_types), - std::move(result_hops), - std::move(result_labels), - label_to_output_comm_rank); - - if (result_labels) { - // Re-compute the result_offsets and account for missing labels - result_offsets = rmm::device_uvector(num_labels + 1, handle.get_stream()); - - // Sort labels - thrust::sort(handle.get_thrust_policy(), cp_result_labels->begin(), cp_result_labels->end()); - - thrust::transform(handle.get_thrust_policy(), - thrust::make_counting_iterator(0), - thrust::make_counting_iterator(result_offsets->size() - 1), - result_offsets->begin() + 1, - [result_labels = raft::device_span( - cp_result_labels->data(), cp_result_labels->size())] __device__(auto idx) { - auto itr_lower = thrust::lower_bound( - thrust::seq, result_labels.begin(), result_labels.end(), idx); - - auto itr_upper = thrust::upper_bound( - thrust::seq, result_labels.begin(), result_labels.end(), idx); - - return thrust::distance(itr_lower, itr_upper); - }); - - // Run inclusive scan - thrust::inclusive_scan(handle.get_thrust_policy(), - result_offsets->begin() + 1, - result_offsets->end(), - result_offsets->begin() + 1); - } - + std::tie(result_srcs, result_dsts, result_weights, result_edge_ids, + result_edge_types, result_hops, result_labels, result_offsets) + = detail::shuffle_and_organize_output(handle, + std::move(result_srcs), + std::move(result_dsts), + std::move(result_weights), + std::move(result_edge_ids), + std::move(result_edge_types), + std::move(result_hops), + std::move(result_labels), + label_to_output_comm_rank); + + if (result_labels && (result_offsets->size() != num_unique_labels + 1)) { + // There are missing labels not sampled. + result_offsets = rmm::device_uvector(num_unique_labels + 1, handle.get_stream()); + + // Sort labels + thrust::sort( + handle.get_thrust_policy(), + cp_result_labels->begin(), + cp_result_labels->end()); + + thrust::transform( + handle.get_thrust_policy(), + thrust::make_counting_iterator(0), + thrust::make_counting_iterator(result_offsets->size() - 1), + result_offsets->begin() + 1, + [ + result_labels = raft::device_span( + cp_result_labels->data(), + cp_result_labels->size()) + ] __device__(auto idx) { + auto itr_lower = thrust::lower_bound( + thrust::seq, result_labels.begin(), result_labels.end(), idx); + + auto itr_upper = thrust::upper_bound( + thrust::seq, result_labels.begin(), result_labels.end(), idx); + + auto sampled_label_size = thrust::distance(itr_lower, itr_upper); + return sampled_label_size; + }); + + // Run inclusive scan + thrust::inclusive_scan(handle.get_thrust_policy(), + result_offsets->begin() + 1, + result_offsets->end(), + result_offsets->begin() + 1); + } return std::make_tuple(std::move(result_srcs), std::move(result_dsts), std::move(result_weights), From ed4c06953ab2decf637551aa18aa360e5f3cc43d Mon Sep 17 00:00:00 2001 From: jnke2016 Date: Fri, 22 Nov 2024 12:37:57 -0800 Subject: [PATCH 22/49] fix style --- cpp/src/c_api/neighbor_sampling.cpp | 78 +++++----- cpp/src/sampling/neighbor_sampling_impl.hpp | 138 ++++++++---------- .../heterogeneous_biased_neighbor_sample.pyx | 6 +- .../heterogeneous_uniform_neighbor_sample.pyx | 4 +- 4 files changed, 109 insertions(+), 117 deletions(-) diff --git a/cpp/src/c_api/neighbor_sampling.cpp b/cpp/src/c_api/neighbor_sampling.cpp index a87fd0cfd89..600c2fe1a78 100644 --- a/cpp/src/c_api/neighbor_sampling.cpp +++ b/cpp/src/c_api/neighbor_sampling.cpp @@ -778,7 +778,8 @@ struct neighbor_sampling_functor : public cugraph::c_api::abstract_functor { cugraph::c_api::cugraph_graph_t* graph_{nullptr}; cugraph::c_api::cugraph_edge_property_view_t const* edge_biases_{nullptr}; cugraph::c_api::cugraph_type_erased_device_array_view_t const* start_vertices_{nullptr}; - cugraph::c_api::cugraph_type_erased_device_array_view_t const* starting_vertex_label_offsets_{nullptr}; + cugraph::c_api::cugraph_type_erased_device_array_view_t const* starting_vertex_label_offsets_{ + nullptr}; cugraph::c_api::cugraph_type_erased_device_array_view_t const* vertex_type_offsets_{nullptr}; cugraph::c_api::cugraph_type_erased_host_array_view_t const* fan_out_{nullptr}; int num_edge_types_{}; @@ -787,18 +788,19 @@ struct neighbor_sampling_functor : public cugraph::c_api::abstract_functor { bool do_expensive_check_{false}; cugraph::c_api::cugraph_sample_result_t* result_{nullptr}; - neighbor_sampling_functor(cugraph_resource_handle_t const* handle, - cugraph_rng_state_t* rng_state, - cugraph_graph_t* graph, - cugraph_edge_property_view_t const* edge_biases, - cugraph_type_erased_device_array_view_t const* start_vertices, - cugraph_type_erased_device_array_view_t const* starting_vertex_label_offsets, - cugraph_type_erased_device_array_view_t const* vertex_type_offsets, - cugraph_type_erased_host_array_view_t const* fan_out, - int num_edge_types, - cugraph::c_api::cugraph_sampling_options_t options, - bool is_biased, - bool do_expensive_check) + neighbor_sampling_functor( + cugraph_resource_handle_t const* handle, + cugraph_rng_state_t* rng_state, + cugraph_graph_t* graph, + cugraph_edge_property_view_t const* edge_biases, + cugraph_type_erased_device_array_view_t const* start_vertices, + cugraph_type_erased_device_array_view_t const* starting_vertex_label_offsets, + cugraph_type_erased_device_array_view_t const* vertex_type_offsets, + cugraph_type_erased_host_array_view_t const* fan_out, + int num_edge_types, + cugraph::c_api::cugraph_sampling_options_t options, + bool is_biased, + bool do_expensive_check) : abstract_functor(), handle_(*reinterpret_cast(handle)->handle_), rng_state_(reinterpret_cast(rng_state)), @@ -878,7 +880,7 @@ struct neighbor_sampling_functor : public cugraph::c_api::abstract_functor { std::optional> start_vertex_labels{std::nullopt}; std::optional> local_label_to_comm_rank{std::nullopt}; std::optional> label_to_comm_rank{ - std::nullopt}; // global after allgatherv + std::nullopt}; // global after allgatherv std::optional> renumbered_and_sorted_edge_id_renumber_map( std::nullopt); @@ -1159,10 +1161,10 @@ struct neighbor_sampling_functor : public cugraph::c_api::abstract_functor { ? std::make_optional(raft::device_span{ start_vertices_->as_type(), start_vertices_->size_}) : std::nullopt, - options_.retain_seeds_ - ? std::make_optional(raft::device_span{ - starting_vertex_label_offsets_->as_type(), starting_vertex_label_offsets_->size_}) - : std::nullopt, + options_.retain_seeds_ ? std::make_optional(raft::device_span{ + starting_vertex_label_offsets_->as_type(), + starting_vertex_label_offsets_->size_}) + : std::nullopt, offsets ? std::make_optional( raft::device_span{offsets->data(), offsets->size()}) : std::nullopt, @@ -1204,10 +1206,10 @@ struct neighbor_sampling_functor : public cugraph::c_api::abstract_functor { ? std::make_optional(raft::device_span{ start_vertices_->as_type(), start_vertices_->size_}) : std::nullopt, - options_.retain_seeds_ - ? std::make_optional(raft::device_span{ - starting_vertex_label_offsets_->as_type(), starting_vertex_label_offsets_->size_}) - : std::nullopt, + options_.retain_seeds_ ? std::make_optional(raft::device_span{ + starting_vertex_label_offsets_->as_type(), + starting_vertex_label_offsets_->size_}) + : std::nullopt, offsets ? std::make_optional( raft::device_span{offsets->data(), offsets->size()}) : std::nullopt, @@ -1234,10 +1236,10 @@ struct neighbor_sampling_functor : public cugraph::c_api::abstract_functor { // If no 'vertex_type_offsets' is provided, all vertices are assumed to have // a vertex type of value 1. cugraph::detail::stride_fill(handle_.get_stream(), - vertex_type_offsets.begin(), - vertex_type_offsets.size(), - vertex_t{0}, - vertex_t{graph_view.local_vertex_partition_range_size()} + vertex_type_offsets.begin(), + vertex_type_offsets.size(), + vertex_t{0}, + vertex_t{graph_view.local_vertex_partition_range_size()} ); } @@ -1266,27 +1268,27 @@ struct neighbor_sampling_functor : public cugraph::c_api::abstract_functor { ? std::make_optional(raft::device_span{ start_vertices_->as_type(), start_vertices_->size_}) : std::nullopt, - options_.retain_seeds_ - ? std::make_optional(raft::device_span{ - starting_vertex_label_offsets_->as_type(), starting_vertex_label_offsets_->size_}) - : std::nullopt, + options_.retain_seeds_ ? std::make_optional(raft::device_span{ + starting_vertex_label_offsets_->as_type(), + starting_vertex_label_offsets_->size_}) + : std::nullopt, offsets ? std::make_optional( raft::device_span{offsets->data(), offsets->size()}) : std::nullopt, - + (vertex_type_offsets_ != nullptr) - ? raft::device_span{vertex_type_offsets_->as_type(), vertex_type_offsets_->size_} + ? raft::device_span{vertex_type_offsets_->as_type(), + vertex_type_offsets_->size_} : raft::device_span{vertex_type_offsets.data(), - vertex_type_offsets.size()}, - + vertex_type_offsets.size()}, + edge_label ? (*offsets).size() - 1 : size_t{1}, hop ? fan_out_->size_ : size_t{1}, - (vertex_type_offsets_ != nullptr) - ? vertex_type_offsets_->size_ - 1 - : vertex_type_offsets.size() - 1, + (vertex_type_offsets_ != nullptr) ? vertex_type_offsets_->size_ - 1 + : vertex_type_offsets.size() - 1, - // num_vertex_type is by default 1 if 'vertex_type_offsets' is not provided + // num_vertex_type is by default 1 if 'vertex_type_offsets' is not provided num_edge_types_, src_is_major, do_expensive_check_); diff --git a/cpp/src/sampling/neighbor_sampling_impl.hpp b/cpp/src/sampling/neighbor_sampling_impl.hpp index 142d8054abd..a09cf757a54 100644 --- a/cpp/src/sampling/neighbor_sampling_impl.hpp +++ b/cpp/src/sampling/neighbor_sampling_impl.hpp @@ -30,6 +30,7 @@ #include #include + #include namespace cugraph { @@ -105,38 +106,30 @@ neighbor_sample_impl(raft::handle_t const& handle, edge_masks_vector{}; graph_view_t modified_graph_view = graph_view; edge_masks_vector.reserve(num_edge_types); - + label_t num_unique_labels = 0; std::optional> cp_starting_vertex_labels{std::nullopt}; - if (starting_vertex_labels) { // Find the number of unique lables - cp_starting_vertex_labels = rmm::device_uvector(starting_vertex_labels->size(), handle.get_stream()); - - thrust::copy( - handle.get_thrust_policy(), - starting_vertex_labels->begin(), - starting_vertex_labels->end(), - cp_starting_vertex_labels->begin()); - - thrust::sort( - handle.get_thrust_policy(), - cp_starting_vertex_labels->begin(), - cp_starting_vertex_labels->end()); - + cp_starting_vertex_labels = + rmm::device_uvector(starting_vertex_labels->size(), handle.get_stream()); + + thrust::copy(handle.get_thrust_policy(), + starting_vertex_labels->begin(), + starting_vertex_labels->end(), + cp_starting_vertex_labels->begin()); + + thrust::sort(handle.get_thrust_policy(), + cp_starting_vertex_labels->begin(), + cp_starting_vertex_labels->end()); + num_unique_labels = thrust::unique_count(handle.get_thrust_policy(), cp_starting_vertex_labels->begin(), cp_starting_vertex_labels->end()); - - } - - - - if (num_edge_types > 1) { for (int i = 0; i < num_edge_types; i++) { cugraph::edge_property_t, bool> @@ -399,60 +392,57 @@ neighbor_sample_impl(raft::handle_t const& handle, if (result_labels) { cp_result_labels = rmm::device_uvector(result_labels->size(), handle.get_stream()); - thrust::copy( - handle.get_thrust_policy(), - result_labels->begin(), - result_labels->end(), - cp_result_labels->begin()); + thrust::copy(handle.get_thrust_policy(), + result_labels->begin(), + result_labels->end(), + cp_result_labels->begin()); + } + std::tie(result_srcs, + result_dsts, + result_weights, + result_edge_ids, + result_edge_types, + result_hops, + result_labels, + result_offsets) = detail::shuffle_and_organize_output(handle, + std::move(result_srcs), + std::move(result_dsts), + std::move(result_weights), + std::move(result_edge_ids), + std::move(result_edge_types), + std::move(result_hops), + std::move(result_labels), + label_to_output_comm_rank); + + if (result_labels && (result_offsets->size() != num_unique_labels + 1)) { + // There are missing labels not sampled. + result_offsets = rmm::device_uvector(num_unique_labels + 1, handle.get_stream()); + + // Sort labels + thrust::sort(handle.get_thrust_policy(), cp_result_labels->begin(), cp_result_labels->end()); + + thrust::transform(handle.get_thrust_policy(), + thrust::make_counting_iterator(0), + thrust::make_counting_iterator(result_offsets->size() - 1), + result_offsets->begin() + 1, + [result_labels = raft::device_span( + cp_result_labels->data(), cp_result_labels->size())] __device__(auto idx) { + auto itr_lower = thrust::lower_bound( + thrust::seq, result_labels.begin(), result_labels.end(), idx); + + auto itr_upper = thrust::upper_bound( + thrust::seq, result_labels.begin(), result_labels.end(), idx); + + auto sampled_label_size = thrust::distance(itr_lower, itr_upper); + return sampled_label_size; + }); + + // Run inclusive scan + thrust::inclusive_scan(handle.get_thrust_policy(), + result_offsets->begin() + 1, + result_offsets->end(), + result_offsets->begin() + 1); } - std::tie(result_srcs, result_dsts, result_weights, result_edge_ids, - result_edge_types, result_hops, result_labels, result_offsets) - = detail::shuffle_and_organize_output(handle, - std::move(result_srcs), - std::move(result_dsts), - std::move(result_weights), - std::move(result_edge_ids), - std::move(result_edge_types), - std::move(result_hops), - std::move(result_labels), - label_to_output_comm_rank); - - if (result_labels && (result_offsets->size() != num_unique_labels + 1)) { - // There are missing labels not sampled. - result_offsets = rmm::device_uvector(num_unique_labels + 1, handle.get_stream()); - - // Sort labels - thrust::sort( - handle.get_thrust_policy(), - cp_result_labels->begin(), - cp_result_labels->end()); - - thrust::transform( - handle.get_thrust_policy(), - thrust::make_counting_iterator(0), - thrust::make_counting_iterator(result_offsets->size() - 1), - result_offsets->begin() + 1, - [ - result_labels = raft::device_span( - cp_result_labels->data(), - cp_result_labels->size()) - ] __device__(auto idx) { - auto itr_lower = thrust::lower_bound( - thrust::seq, result_labels.begin(), result_labels.end(), idx); - - auto itr_upper = thrust::upper_bound( - thrust::seq, result_labels.begin(), result_labels.end(), idx); - - auto sampled_label_size = thrust::distance(itr_lower, itr_upper); - return sampled_label_size; - }); - - // Run inclusive scan - thrust::inclusive_scan(handle.get_thrust_policy(), - result_offsets->begin() + 1, - result_offsets->end(), - result_offsets->begin() + 1); - } return std::make_tuple(std::move(result_srcs), std::move(result_dsts), std::move(result_weights), diff --git a/python/pylibcugraph/pylibcugraph/heterogeneous_biased_neighbor_sample.pyx b/python/pylibcugraph/pylibcugraph/heterogeneous_biased_neighbor_sample.pyx index e506a413ab5..ee0e85fa3bd 100644 --- a/python/pylibcugraph/pylibcugraph/heterogeneous_biased_neighbor_sample.pyx +++ b/python/pylibcugraph/pylibcugraph/heterogeneous_biased_neighbor_sample.pyx @@ -124,7 +124,7 @@ def heterogeneous_biased_neighbor_sample(ResourceHandle resource_handle, Offsets of each label within the start vertex list. Expanding 'starting_vertex_label_offsets' must lead to an array of len(start_vertex_list) - + vertex_type_offsets: device array type (Optional) Offsets for each vertex type in the graph. @@ -281,7 +281,7 @@ def heterogeneous_biased_neighbor_sample(ResourceHandle resource_handle, if starting_vertex_label_offsets is not None: cai_starting_vertex_label_offsets_ptr = \ starting_vertex_label_offsets.__cuda_array_interface__['data'][0] - + cdef uintptr_t cai_vertex_type_offsets_ptr if vertex_type_offsets is not None: cai_vertex_type_offsets_ptr = \ @@ -303,7 +303,7 @@ def heterogeneous_biased_neighbor_sample(ResourceHandle resource_handle, len(starting_vertex_label_offsets), SIZE_T ) - + cdef cugraph_type_erased_device_array_view_t* vertex_type_offsets_ptr = NULL if vertex_type_offsets is not None: vertex_type_offsets_ptr = \ diff --git a/python/pylibcugraph/pylibcugraph/heterogeneous_uniform_neighbor_sample.pyx b/python/pylibcugraph/pylibcugraph/heterogeneous_uniform_neighbor_sample.pyx index 6e8ab93de62..dbee65323d7 100644 --- a/python/pylibcugraph/pylibcugraph/heterogeneous_uniform_neighbor_sample.pyx +++ b/python/pylibcugraph/pylibcugraph/heterogeneous_uniform_neighbor_sample.pyx @@ -119,7 +119,7 @@ def heterogeneous_uniform_neighbor_sample(ResourceHandle resource_handle, Offsets of each label within the start vertex list. Expanding 'starting_vertex_label_offsets' must lead to an array of len(start_vertex_list) - + vertex_type_offsets: device array type (Optional) Offsets for each vertex type in the graph. @@ -275,7 +275,7 @@ def heterogeneous_uniform_neighbor_sample(ResourceHandle resource_handle, if starting_vertex_label_offsets is not None: cai_starting_vertex_label_offsets_ptr = \ starting_vertex_label_offsets.__cuda_array_interface__['data'][0] - + cdef uintptr_t cai_vertex_type_offsets_ptr if vertex_type_offsets is not None: cai_vertex_type_offsets_ptr = \ From 4d3f8c1f348c7646a98b6660a784ca84ccefbae3 Mon Sep 17 00:00:00 2001 From: jnke2016 Date: Tue, 26 Nov 2024 15:56:22 -0800 Subject: [PATCH 23/49] properly handle sampling with multiple edge types --- cpp/src/sampling/neighbor_sampling_impl.hpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cpp/src/sampling/neighbor_sampling_impl.hpp b/cpp/src/sampling/neighbor_sampling_impl.hpp index a09cf757a54..8b0b6fe5c84 100644 --- a/cpp/src/sampling/neighbor_sampling_impl.hpp +++ b/cpp/src/sampling/neighbor_sampling_impl.hpp @@ -252,8 +252,12 @@ neighbor_sample_impl(raft::handle_t const& handle, if (labels) { (*level_result_label_vectors).push_back(std::move(*labels)); } if (num_edge_types > 1) { modified_graph_view.clear_edge_mask(); } + + //vector.push_back (raft::device_span(vertex_t const)) //<- // level_result_dst_vectors } + + // Call prepare frontier for each hop. FIXME // FIXME: We should modify vertex_partition_range_lasts to return a raft::host_span // rather than making a copy. auto vertex_partition_range_lasts = modified_graph_view.vertex_partition_range_lasts(); @@ -262,7 +266,7 @@ neighbor_sample_impl(raft::handle_t const& handle, handle, starting_vertices, starting_vertex_labels, - raft::device_span{level_result_dst_vectors.back().data(), + raft::device_span{level_result_dst_vectors.back().data(), // define a vector . level_result_dst_vectors.back().size()}, frontier_vertex_labels ? std::make_optional(raft::device_span( From d88eebd8ee7bd2a81d6903e90b76e4942b4f91f7 Mon Sep 17 00:00:00 2001 From: jnke2016 Date: Tue, 26 Nov 2024 16:01:35 -0800 Subject: [PATCH 24/49] remove debug print --- cpp/src/sampling/detail/prepare_next_frontier_impl.cuh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cpp/src/sampling/detail/prepare_next_frontier_impl.cuh b/cpp/src/sampling/detail/prepare_next_frontier_impl.cuh index 5c04d628f09..bf401642e68 100644 --- a/cpp/src/sampling/detail/prepare_next_frontier_impl.cuh +++ b/cpp/src/sampling/detail/prepare_next_frontier_impl.cuh @@ -58,11 +58,12 @@ prepare_next_frontier( bool dedupe_sources, bool do_expensive_check) { + // Create a single new frontier. vertex_partition_device_view_t d_vertex_partition(vertex_partition); - size_t frontier_size = sampled_dst_vertices.size(); + size_t frontier_size = sampled_dst_vertices.size(); // std:reduce for all edge typw within hop / Reduce the size of each of the span. std::reduce for each of the size - example code that concatenate device vectors (look at that) if (prior_sources_behavior == prior_sources_behavior_t::CARRY_OVER) { - frontier_size += sampled_src_vertices.size(); + frontier_size += sampled_src_vertices.size(); // make room for all of the previous vertices. } rmm::device_uvector frontier_vertices(frontier_size, handle.get_stream()); From 77a28c24bdcbf58d19005cac57a320047958030f Mon Sep 17 00:00:00 2001 From: jnke2016 Date: Tue, 26 Nov 2024 16:45:54 -0800 Subject: [PATCH 25/49] undo changes to 'prepare_next_frontier' --- cpp/src/sampling/detail/prepare_next_frontier_impl.cuh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/cpp/src/sampling/detail/prepare_next_frontier_impl.cuh b/cpp/src/sampling/detail/prepare_next_frontier_impl.cuh index bf401642e68..5c04d628f09 100644 --- a/cpp/src/sampling/detail/prepare_next_frontier_impl.cuh +++ b/cpp/src/sampling/detail/prepare_next_frontier_impl.cuh @@ -58,12 +58,11 @@ prepare_next_frontier( bool dedupe_sources, bool do_expensive_check) { - // Create a single new frontier. vertex_partition_device_view_t d_vertex_partition(vertex_partition); - size_t frontier_size = sampled_dst_vertices.size(); // std:reduce for all edge typw within hop / Reduce the size of each of the span. std::reduce for each of the size - example code that concatenate device vectors (look at that) + size_t frontier_size = sampled_dst_vertices.size(); if (prior_sources_behavior == prior_sources_behavior_t::CARRY_OVER) { - frontier_size += sampled_src_vertices.size(); // make room for all of the previous vertices. + frontier_size += sampled_src_vertices.size(); } rmm::device_uvector frontier_vertices(frontier_size, handle.get_stream()); From b760a61aa1a0a9e05f63022b6439a6f61f51be02 Mon Sep 17 00:00:00 2001 From: jnke2016 Date: Tue, 26 Nov 2024 17:14:55 -0800 Subject: [PATCH 26/49] properly handle sampling with multiple edge types --- cpp/src/sampling/neighbor_sampling_impl.hpp | 254 +++++++++++++------- 1 file changed, 166 insertions(+), 88 deletions(-) diff --git a/cpp/src/sampling/neighbor_sampling_impl.hpp b/cpp/src/sampling/neighbor_sampling_impl.hpp index 8b0b6fe5c84..0cd7dbd35ac 100644 --- a/cpp/src/sampling/neighbor_sampling_impl.hpp +++ b/cpp/src/sampling/neighbor_sampling_impl.hpp @@ -30,7 +30,6 @@ #include #include - #include namespace cugraph { @@ -106,25 +105,26 @@ neighbor_sample_impl(raft::handle_t const& handle, edge_masks_vector{}; graph_view_t modified_graph_view = graph_view; edge_masks_vector.reserve(num_edge_types); - + label_t num_unique_labels = 0; std::optional> cp_starting_vertex_labels{std::nullopt}; if (starting_vertex_labels) { // Find the number of unique lables - cp_starting_vertex_labels = - rmm::device_uvector(starting_vertex_labels->size(), handle.get_stream()); - - thrust::copy(handle.get_thrust_policy(), - starting_vertex_labels->begin(), - starting_vertex_labels->end(), - cp_starting_vertex_labels->begin()); - - thrust::sort(handle.get_thrust_policy(), - cp_starting_vertex_labels->begin(), - cp_starting_vertex_labels->end()); - + cp_starting_vertex_labels = rmm::device_uvector(starting_vertex_labels->size(), handle.get_stream()); + + thrust::copy( + handle.get_thrust_policy(), + starting_vertex_labels->begin(), + starting_vertex_labels->end(), + cp_starting_vertex_labels->begin()); + + thrust::sort( + handle.get_thrust_policy(), + cp_starting_vertex_labels->begin(), + cp_starting_vertex_labels->end()); + num_unique_labels = thrust::unique_count(handle.get_thrust_policy(), cp_starting_vertex_labels->begin(), cp_starting_vertex_labels->end()); @@ -160,6 +160,16 @@ neighbor_sample_impl(raft::handle_t const& handle, std::vector> level_result_src_vectors{}; std::vector> level_result_dst_vectors{}; + + rmm::device_uvector level_result_src(0, handle.get_stream()); + rmm::device_uvector level_result_dst(0, handle.get_stream()); + + // Get the number of hop. If homogeneous neighbor sample, num_edge_types = 1 + auto num_hops = ((fan_out.size() % num_edge_types) == 0) + ? (fan_out.size() / num_edge_types) + : ((fan_out.size() / num_edge_types) + 1); + + auto level_result_weight_vectors = edge_weight_view ? std::make_optional(std::vector>{}) : std::nullopt; @@ -172,12 +182,26 @@ neighbor_sample_impl(raft::handle_t const& handle, starting_vertex_labels ? std::make_optional(std::vector>{}) : std::nullopt; - level_result_src_vectors.reserve(fan_out.size()); - level_result_dst_vectors.reserve(fan_out.size()); - if (level_result_weight_vectors) { (*level_result_weight_vectors).reserve(fan_out.size()); } - if (level_result_edge_id_vectors) { (*level_result_edge_id_vectors).reserve(fan_out.size()); } - if (level_result_edge_type_vectors) { (*level_result_edge_type_vectors).reserve(fan_out.size()); } - if (level_result_label_vectors) { (*level_result_label_vectors).reserve(fan_out.size()); } + level_result_src_vectors.reserve(num_hops); + level_result_dst_vectors.reserve(num_hops); + + auto level_result_weight = + edge_weight_view ? std::make_optional(rmm::device_uvector(0, handle.get_stream())) + : std::nullopt; + auto level_result_edge_id = + edge_id_view ? std::make_optional(rmm::device_uvector(0, handle.get_stream())) + : std::nullopt; + auto level_result_edge_type = + edge_type_view ? std::make_optional(rmm::device_uvector(0, handle.get_stream())) + : std::nullopt; + auto level_result_label = + starting_vertex_labels ? std::make_optional(rmm::device_uvector(0, handle.get_stream())) + : std::nullopt; + + if (level_result_weight_vectors) { (*level_result_weight_vectors).reserve(num_hops); } + if (level_result_edge_id_vectors) { (*level_result_edge_id_vectors).reserve(num_hops); } + if (level_result_edge_type_vectors) { (*level_result_edge_type_vectors).reserve(num_hops); } + if (level_result_label_vectors) { (*level_result_label_vectors).reserve(num_hops); } rmm::device_uvector frontier_vertices(0, handle.get_stream()); auto frontier_vertex_labels = @@ -198,11 +222,8 @@ neighbor_sample_impl(raft::handle_t const& handle, } std::vector level_sizes{}; + std::vector level_sizes_edge_types{}; - // Get the number of hop. If homogeneous neighbor sample, num_edge_types = 1 - auto num_hops = ((fan_out.size() % num_edge_types) == 0) - ? (fan_out.size() / num_edge_types) - : ((fan_out.size() / num_edge_types) + 1); for (auto hop = 0; hop < num_hops; hop++) { for (auto edge_type_id = 0; edge_type_id < num_edge_types; edge_type_id++) { @@ -242,22 +263,70 @@ neighbor_sample_impl(raft::handle_t const& handle, starting_vertex_labels); } - level_sizes.push_back(srcs.size()); - level_result_src_vectors.push_back(std::move(srcs)); - level_result_dst_vectors.push_back(std::move(dsts)); + level_sizes_edge_types.push_back(srcs.size()); - if (weights) { (*level_result_weight_vectors).push_back(std::move(*weights)); } - if (edge_ids) { (*level_result_edge_id_vectors).push_back(std::move(*edge_ids)); } - if (edge_types) { (*level_result_edge_type_vectors).push_back(std::move(*edge_types)); } - if (labels) { (*level_result_label_vectors).push_back(std::move(*labels)); } + level_result_src.resize(std::reduce(level_sizes_edge_types.begin(), level_sizes_edge_types.end()), handle.get_stream()); + level_result_dst.resize(std::reduce(level_sizes_edge_types.begin(), level_sizes_edge_types.end()), handle.get_stream()); - if (num_edge_types > 1) { modified_graph_view.clear_edge_mask(); } + raft::copy(level_result_src.begin() + std::reduce(level_sizes_edge_types.begin(), level_sizes_edge_types.end() - 1), + srcs.begin(), + srcs.size(), + handle.get_stream()); + + raft::copy(level_result_dst.begin() + std::reduce(level_sizes_edge_types.begin(), level_sizes_edge_types.end() - 1), + dsts.begin(), + srcs.size(), + handle.get_stream()); + + if (weights) { + (*level_result_weight).resize(std::reduce(level_sizes_edge_types.begin(), level_sizes_edge_types.end()), handle.get_stream()); + + raft::copy(level_result_weight->begin() + std::reduce(level_sizes_edge_types.begin(), level_sizes_edge_types.end() - 1), + weights->begin(), + srcs.size(), + handle.get_stream()); + } + + if (edge_ids) { + (*level_result_edge_id).resize(std::reduce(level_sizes_edge_types.begin(), level_sizes_edge_types.end()), handle.get_stream()); + raft::copy(level_result_edge_id->begin() + std::reduce(level_sizes_edge_types.begin(), level_sizes_edge_types.end() - 1), + edge_ids->begin(), + srcs.size(), + handle.get_stream()); + } + if (edge_types) { + (*level_result_edge_type).resize(std::reduce(level_sizes_edge_types.begin(), level_sizes_edge_types.end()), handle.get_stream()); + + + raft::copy(level_result_edge_type->begin() + std::reduce(level_sizes_edge_types.begin(), level_sizes_edge_types.end() - 1), + edge_types->begin(), + srcs.size(), + handle.get_stream()); + } + + if (labels) { + (*level_result_label).resize(std::reduce(level_sizes_edge_types.begin(), level_sizes_edge_types.end()), handle.get_stream()); + + raft::copy(level_result_label->begin() + std::reduce(level_sizes_edge_types.begin(), level_sizes_edge_types.end() - 1), + labels->begin(), + srcs.size(), + handle.get_stream()); + + } - //vector.push_back (raft::device_span(vertex_t const)) //<- // level_result_dst_vectors + if (num_edge_types > 1) { modified_graph_view.clear_edge_mask(); } } + level_sizes.push_back(level_result_src.size()); + level_result_src_vectors.push_back(std::move(level_result_src)); + level_result_dst_vectors.push_back(std::move(level_result_dst)); + + if (level_result_weight) { (*level_result_weight_vectors).push_back(std::move(*level_result_weight)); } + if (level_result_edge_id) { (*level_result_edge_id_vectors).push_back(std::move(*level_result_edge_id)); } + if (level_result_edge_type) { (*level_result_edge_type_vectors).push_back(std::move(*level_result_edge_type)); } + if (level_result_label) { (*level_result_label_vectors).push_back(std::move(*level_result_label)); } + - // Call prepare frontier for each hop. FIXME // FIXME: We should modify vertex_partition_range_lasts to return a raft::host_span // rather than making a copy. auto vertex_partition_range_lasts = modified_graph_view.vertex_partition_range_lasts(); @@ -266,11 +335,11 @@ neighbor_sample_impl(raft::handle_t const& handle, handle, starting_vertices, starting_vertex_labels, - raft::device_span{level_result_dst_vectors.back().data(), // define a vector . - level_result_dst_vectors.back().size()}, + raft::device_span{level_result_dst.data(), + level_result_dst.size()}, frontier_vertex_labels ? std::make_optional(raft::device_span( - level_result_label_vectors->back().data(), level_result_label_vectors->back().size())) + level_result_label->data(), level_result_label->size())) : std::nullopt, std::move(vertex_used_as_source), modified_graph_view.local_vertex_partition_view(), @@ -368,13 +437,14 @@ neighbor_sample_impl(raft::handle_t const& handle, if (return_hops) { result_hops = rmm::device_uvector(result_size, handle.get_stream()); output_offset = 0; - for (size_t i = 0; i < fan_out.size(); ++i) { + for (size_t i = 0; i < num_hops; ++i) { // FIXME: replace this by the number of hops scalar_fill( handle, result_hops->data() + output_offset, level_sizes[i], static_cast(i)); output_offset += level_sizes[i]; } } + auto result_labels = level_result_label_vectors ? std::make_optional(rmm::device_uvector(result_size, handle.get_stream())) @@ -396,57 +466,65 @@ neighbor_sample_impl(raft::handle_t const& handle, if (result_labels) { cp_result_labels = rmm::device_uvector(result_labels->size(), handle.get_stream()); - thrust::copy(handle.get_thrust_policy(), - result_labels->begin(), - result_labels->end(), - cp_result_labels->begin()); - } - std::tie(result_srcs, - result_dsts, - result_weights, - result_edge_ids, - result_edge_types, - result_hops, - result_labels, - result_offsets) = detail::shuffle_and_organize_output(handle, - std::move(result_srcs), - std::move(result_dsts), - std::move(result_weights), - std::move(result_edge_ids), - std::move(result_edge_types), - std::move(result_hops), - std::move(result_labels), - label_to_output_comm_rank); - - if (result_labels && (result_offsets->size() != num_unique_labels + 1)) { - // There are missing labels not sampled. - result_offsets = rmm::device_uvector(num_unique_labels + 1, handle.get_stream()); - - // Sort labels - thrust::sort(handle.get_thrust_policy(), cp_result_labels->begin(), cp_result_labels->end()); - - thrust::transform(handle.get_thrust_policy(), - thrust::make_counting_iterator(0), - thrust::make_counting_iterator(result_offsets->size() - 1), - result_offsets->begin() + 1, - [result_labels = raft::device_span( - cp_result_labels->data(), cp_result_labels->size())] __device__(auto idx) { - auto itr_lower = thrust::lower_bound( - thrust::seq, result_labels.begin(), result_labels.end(), idx); - - auto itr_upper = thrust::upper_bound( - thrust::seq, result_labels.begin(), result_labels.end(), idx); - - auto sampled_label_size = thrust::distance(itr_lower, itr_upper); - return sampled_label_size; - }); - - // Run inclusive scan - thrust::inclusive_scan(handle.get_thrust_policy(), - result_offsets->begin() + 1, - result_offsets->end(), - result_offsets->begin() + 1); + thrust::copy( + handle.get_thrust_policy(), + result_labels->begin(), + result_labels->end(), + cp_result_labels->begin()); } + + std::tie(result_srcs, result_dsts, result_weights, result_edge_ids, + result_edge_types, result_hops, result_labels, result_offsets) + = detail::shuffle_and_organize_output(handle, + std::move(result_srcs), + std::move(result_dsts), + std::move(result_weights), + std::move(result_edge_ids), + std::move(result_edge_types), + std::move(result_hops), + std::move(result_labels), + label_to_output_comm_rank); + + + if (result_labels && (result_offsets->size() != num_unique_labels + 1)) { + result_offsets = rmm::device_uvector(num_unique_labels + 1, handle.get_stream()); + + // Sort labels + thrust::sort( + handle.get_thrust_policy(), + cp_result_labels->begin(), + cp_result_labels->end()); + + + thrust::transform( + handle.get_thrust_policy(), + thrust::make_counting_iterator(0), + thrust::make_counting_iterator(result_offsets->size() - 1), + result_offsets->begin() + 1, + [ + result_labels = raft::device_span( + cp_result_labels->data(), + cp_result_labels->size()) + ] __device__(auto idx) { + auto itr_lower = thrust::lower_bound( + thrust::seq, result_labels.begin(), result_labels.end(), idx); + + auto itr_upper = thrust::upper_bound( + thrust::seq, result_labels.begin(), result_labels.end(), idx); + + auto sampled_label_size = thrust::distance(itr_lower, itr_upper); + + //return thrust::distance(itr_lower, itr_upper); + return sampled_label_size; + }); + + // Run inclusive scan + thrust::inclusive_scan(handle.get_thrust_policy(), + result_offsets->begin() + 1, + result_offsets->end(), + result_offsets->begin() + 1); + } + return std::make_tuple(std::move(result_srcs), std::move(result_dsts), std::move(result_weights), From 580d3e9dd7803df29047e2340d2d576b6412b2d7 Mon Sep 17 00:00:00 2001 From: jnke2016 Date: Tue, 26 Nov 2024 17:20:30 -0800 Subject: [PATCH 27/49] properly compute the number of hops for heterogeneous renumbering --- cpp/src/c_api/neighbor_sampling.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cpp/src/c_api/neighbor_sampling.cpp b/cpp/src/c_api/neighbor_sampling.cpp index 600c2fe1a78..47b1fa4c2a7 100644 --- a/cpp/src/c_api/neighbor_sampling.cpp +++ b/cpp/src/c_api/neighbor_sampling.cpp @@ -1283,8 +1283,10 @@ struct neighbor_sampling_functor : public cugraph::c_api::abstract_functor { vertex_type_offsets.size()}, edge_label ? (*offsets).size() - 1 : size_t{1}, - hop ? fan_out_->size_ : size_t{1}, - + hop ? (((fan_out_->size_ % num_edge_types_) == 0) + ? (fan_out_->size_ / num_edge_types_) + : ((fan_out_->size_ / num_edge_types_) + 1)) + : size_t{1}, (vertex_type_offsets_ != nullptr) ? vertex_type_offsets_->size_ - 1 : vertex_type_offsets.size() - 1, From 529955a17ba529da6a6469c47119d864bfa44888 Mon Sep 17 00:00:00 2001 From: jnke2016 Date: Tue, 26 Nov 2024 17:21:45 -0800 Subject: [PATCH 28/49] fix style --- cpp/src/c_api/neighbor_sampling.cpp | 4 +- cpp/src/sampling/neighbor_sampling_impl.hpp | 258 +++++++++++--------- 2 files changed, 138 insertions(+), 124 deletions(-) diff --git a/cpp/src/c_api/neighbor_sampling.cpp b/cpp/src/c_api/neighbor_sampling.cpp index 47b1fa4c2a7..1aa39fdeddd 100644 --- a/cpp/src/c_api/neighbor_sampling.cpp +++ b/cpp/src/c_api/neighbor_sampling.cpp @@ -1284,8 +1284,8 @@ struct neighbor_sampling_functor : public cugraph::c_api::abstract_functor { edge_label ? (*offsets).size() - 1 : size_t{1}, hop ? (((fan_out_->size_ % num_edge_types_) == 0) - ? (fan_out_->size_ / num_edge_types_) - : ((fan_out_->size_ / num_edge_types_) + 1)) + ? (fan_out_->size_ / num_edge_types_) + : ((fan_out_->size_ / num_edge_types_) + 1)) : size_t{1}, (vertex_type_offsets_ != nullptr) ? vertex_type_offsets_->size_ - 1 : vertex_type_offsets.size() - 1, diff --git a/cpp/src/sampling/neighbor_sampling_impl.hpp b/cpp/src/sampling/neighbor_sampling_impl.hpp index 0cd7dbd35ac..73566358692 100644 --- a/cpp/src/sampling/neighbor_sampling_impl.hpp +++ b/cpp/src/sampling/neighbor_sampling_impl.hpp @@ -30,6 +30,7 @@ #include #include + #include namespace cugraph { @@ -105,26 +106,25 @@ neighbor_sample_impl(raft::handle_t const& handle, edge_masks_vector{}; graph_view_t modified_graph_view = graph_view; edge_masks_vector.reserve(num_edge_types); - + label_t num_unique_labels = 0; std::optional> cp_starting_vertex_labels{std::nullopt}; if (starting_vertex_labels) { // Find the number of unique lables - cp_starting_vertex_labels = rmm::device_uvector(starting_vertex_labels->size(), handle.get_stream()); - - thrust::copy( - handle.get_thrust_policy(), - starting_vertex_labels->begin(), - starting_vertex_labels->end(), - cp_starting_vertex_labels->begin()); - - thrust::sort( - handle.get_thrust_policy(), - cp_starting_vertex_labels->begin(), - cp_starting_vertex_labels->end()); - + cp_starting_vertex_labels = + rmm::device_uvector(starting_vertex_labels->size(), handle.get_stream()); + + thrust::copy(handle.get_thrust_policy(), + starting_vertex_labels->begin(), + starting_vertex_labels->end(), + cp_starting_vertex_labels->begin()); + + thrust::sort(handle.get_thrust_policy(), + cp_starting_vertex_labels->begin(), + cp_starting_vertex_labels->end()); + num_unique_labels = thrust::unique_count(handle.get_thrust_policy(), cp_starting_vertex_labels->begin(), cp_starting_vertex_labels->end()); @@ -169,7 +169,6 @@ neighbor_sample_impl(raft::handle_t const& handle, ? (fan_out.size() / num_edge_types) : ((fan_out.size() / num_edge_types) + 1); - auto level_result_weight_vectors = edge_weight_view ? std::make_optional(std::vector>{}) : std::nullopt; @@ -190,13 +189,14 @@ neighbor_sample_impl(raft::handle_t const& handle, : std::nullopt; auto level_result_edge_id = edge_id_view ? std::make_optional(rmm::device_uvector(0, handle.get_stream())) - : std::nullopt; + : std::nullopt; auto level_result_edge_type = edge_type_view ? std::make_optional(rmm::device_uvector(0, handle.get_stream())) - : std::nullopt; + : std::nullopt; auto level_result_label = - starting_vertex_labels ? std::make_optional(rmm::device_uvector(0, handle.get_stream())) - : std::nullopt; + starting_vertex_labels + ? std::make_optional(rmm::device_uvector(0, handle.get_stream())) + : std::nullopt; if (level_result_weight_vectors) { (*level_result_weight_vectors).reserve(num_hops); } if (level_result_edge_id_vectors) { (*level_result_edge_id_vectors).reserve(num_hops); } @@ -224,7 +224,6 @@ neighbor_sample_impl(raft::handle_t const& handle, std::vector level_sizes{}; std::vector level_sizes_edge_types{}; - for (auto hop = 0; hop < num_hops; hop++) { for (auto edge_type_id = 0; edge_type_id < num_edge_types; edge_type_id++) { auto k_level = fan_out[(hop * num_edge_types) + edge_type_id]; @@ -265,53 +264,69 @@ neighbor_sample_impl(raft::handle_t const& handle, level_sizes_edge_types.push_back(srcs.size()); - level_result_src.resize(std::reduce(level_sizes_edge_types.begin(), level_sizes_edge_types.end()), handle.get_stream()); - level_result_dst.resize(std::reduce(level_sizes_edge_types.begin(), level_sizes_edge_types.end()), handle.get_stream()); + level_result_src.resize( + std::reduce(level_sizes_edge_types.begin(), level_sizes_edge_types.end()), + handle.get_stream()); + level_result_dst.resize( + std::reduce(level_sizes_edge_types.begin(), level_sizes_edge_types.end()), + handle.get_stream()); - raft::copy(level_result_src.begin() + std::reduce(level_sizes_edge_types.begin(), level_sizes_edge_types.end() - 1), + raft::copy(level_result_src.begin() + + std::reduce(level_sizes_edge_types.begin(), level_sizes_edge_types.end() - 1), srcs.begin(), srcs.size(), handle.get_stream()); - - raft::copy(level_result_dst.begin() + std::reduce(level_sizes_edge_types.begin(), level_sizes_edge_types.end() - 1), + + raft::copy(level_result_dst.begin() + + std::reduce(level_sizes_edge_types.begin(), level_sizes_edge_types.end() - 1), dsts.begin(), srcs.size(), handle.get_stream()); if (weights) { - (*level_result_weight).resize(std::reduce(level_sizes_edge_types.begin(), level_sizes_edge_types.end()), handle.get_stream()); - - raft::copy(level_result_weight->begin() + std::reduce(level_sizes_edge_types.begin(), level_sizes_edge_types.end() - 1), - weights->begin(), - srcs.size(), - handle.get_stream()); + (*level_result_weight) + .resize(std::reduce(level_sizes_edge_types.begin(), level_sizes_edge_types.end()), + handle.get_stream()); + + raft::copy(level_result_weight->begin() + + std::reduce(level_sizes_edge_types.begin(), level_sizes_edge_types.end() - 1), + weights->begin(), + srcs.size(), + handle.get_stream()); } - + if (edge_ids) { - (*level_result_edge_id).resize(std::reduce(level_sizes_edge_types.begin(), level_sizes_edge_types.end()), handle.get_stream()); - raft::copy(level_result_edge_id->begin() + std::reduce(level_sizes_edge_types.begin(), level_sizes_edge_types.end() - 1), - edge_ids->begin(), - srcs.size(), - handle.get_stream()); + (*level_result_edge_id) + .resize(std::reduce(level_sizes_edge_types.begin(), level_sizes_edge_types.end()), + handle.get_stream()); + raft::copy(level_result_edge_id->begin() + + std::reduce(level_sizes_edge_types.begin(), level_sizes_edge_types.end() - 1), + edge_ids->begin(), + srcs.size(), + handle.get_stream()); } if (edge_types) { - (*level_result_edge_type).resize(std::reduce(level_sizes_edge_types.begin(), level_sizes_edge_types.end()), handle.get_stream()); - - - raft::copy(level_result_edge_type->begin() + std::reduce(level_sizes_edge_types.begin(), level_sizes_edge_types.end() - 1), - edge_types->begin(), - srcs.size(), - handle.get_stream()); + (*level_result_edge_type) + .resize(std::reduce(level_sizes_edge_types.begin(), level_sizes_edge_types.end()), + handle.get_stream()); + + raft::copy(level_result_edge_type->begin() + + std::reduce(level_sizes_edge_types.begin(), level_sizes_edge_types.end() - 1), + edge_types->begin(), + srcs.size(), + handle.get_stream()); } - - if (labels) { - (*level_result_label).resize(std::reduce(level_sizes_edge_types.begin(), level_sizes_edge_types.end()), handle.get_stream()); - raft::copy(level_result_label->begin() + std::reduce(level_sizes_edge_types.begin(), level_sizes_edge_types.end() - 1), - labels->begin(), - srcs.size(), - handle.get_stream()); - + if (labels) { + (*level_result_label) + .resize(std::reduce(level_sizes_edge_types.begin(), level_sizes_edge_types.end()), + handle.get_stream()); + + raft::copy(level_result_label->begin() + + std::reduce(level_sizes_edge_types.begin(), level_sizes_edge_types.end() - 1), + labels->begin(), + srcs.size(), + handle.get_stream()); } if (num_edge_types > 1) { modified_graph_view.clear_edge_mask(); } @@ -321,11 +336,18 @@ neighbor_sample_impl(raft::handle_t const& handle, level_result_src_vectors.push_back(std::move(level_result_src)); level_result_dst_vectors.push_back(std::move(level_result_dst)); - if (level_result_weight) { (*level_result_weight_vectors).push_back(std::move(*level_result_weight)); } - if (level_result_edge_id) { (*level_result_edge_id_vectors).push_back(std::move(*level_result_edge_id)); } - if (level_result_edge_type) { (*level_result_edge_type_vectors).push_back(std::move(*level_result_edge_type)); } - if (level_result_label) { (*level_result_label_vectors).push_back(std::move(*level_result_label)); } - + if (level_result_weight) { + (*level_result_weight_vectors).push_back(std::move(*level_result_weight)); + } + if (level_result_edge_id) { + (*level_result_edge_id_vectors).push_back(std::move(*level_result_edge_id)); + } + if (level_result_edge_type) { + (*level_result_edge_type_vectors).push_back(std::move(*level_result_edge_type)); + } + if (level_result_label) { + (*level_result_label_vectors).push_back(std::move(*level_result_label)); + } // FIXME: We should modify vertex_partition_range_lasts to return a raft::host_span // rather than making a copy. @@ -335,12 +357,10 @@ neighbor_sample_impl(raft::handle_t const& handle, handle, starting_vertices, starting_vertex_labels, - raft::device_span{level_result_dst.data(), - level_result_dst.size()}, - frontier_vertex_labels - ? std::make_optional(raft::device_span( - level_result_label->data(), level_result_label->size())) - : std::nullopt, + raft::device_span{level_result_dst.data(), level_result_dst.size()}, + frontier_vertex_labels ? std::make_optional(raft::device_span( + level_result_label->data(), level_result_label->size())) + : std::nullopt, std::move(vertex_used_as_source), modified_graph_view.local_vertex_partition_view(), vertex_partition_range_lasts, @@ -437,14 +457,13 @@ neighbor_sample_impl(raft::handle_t const& handle, if (return_hops) { result_hops = rmm::device_uvector(result_size, handle.get_stream()); output_offset = 0; - for (size_t i = 0; i < num_hops; ++i) { // FIXME: replace this by the number of hops + for (size_t i = 0; i < num_hops; ++i) { // FIXME: replace this by the number of hops scalar_fill( handle, result_hops->data() + output_offset, level_sizes[i], static_cast(i)); output_offset += level_sizes[i]; } } - auto result_labels = level_result_label_vectors ? std::make_optional(rmm::device_uvector(result_size, handle.get_stream())) @@ -466,65 +485,60 @@ neighbor_sample_impl(raft::handle_t const& handle, if (result_labels) { cp_result_labels = rmm::device_uvector(result_labels->size(), handle.get_stream()); - thrust::copy( - handle.get_thrust_policy(), - result_labels->begin(), - result_labels->end(), - cp_result_labels->begin()); + thrust::copy(handle.get_thrust_policy(), + result_labels->begin(), + result_labels->end(), + cp_result_labels->begin()); + } + + std::tie(result_srcs, + result_dsts, + result_weights, + result_edge_ids, + result_edge_types, + result_hops, + result_labels, + result_offsets) = detail::shuffle_and_organize_output(handle, + std::move(result_srcs), + std::move(result_dsts), + std::move(result_weights), + std::move(result_edge_ids), + std::move(result_edge_types), + std::move(result_hops), + std::move(result_labels), + label_to_output_comm_rank); + + if (result_labels && (result_offsets->size() != num_unique_labels + 1)) { + result_offsets = rmm::device_uvector(num_unique_labels + 1, handle.get_stream()); + + // Sort labels + thrust::sort(handle.get_thrust_policy(), cp_result_labels->begin(), cp_result_labels->end()); + + thrust::transform(handle.get_thrust_policy(), + thrust::make_counting_iterator(0), + thrust::make_counting_iterator(result_offsets->size() - 1), + result_offsets->begin() + 1, + [result_labels = raft::device_span( + cp_result_labels->data(), cp_result_labels->size())] __device__(auto idx) { + auto itr_lower = thrust::lower_bound( + thrust::seq, result_labels.begin(), result_labels.end(), idx); + + auto itr_upper = thrust::upper_bound( + thrust::seq, result_labels.begin(), result_labels.end(), idx); + + auto sampled_label_size = thrust::distance(itr_lower, itr_upper); + + // return thrust::distance(itr_lower, itr_upper); + return sampled_label_size; + }); + + // Run inclusive scan + thrust::inclusive_scan(handle.get_thrust_policy(), + result_offsets->begin() + 1, + result_offsets->end(), + result_offsets->begin() + 1); } - std::tie(result_srcs, result_dsts, result_weights, result_edge_ids, - result_edge_types, result_hops, result_labels, result_offsets) - = detail::shuffle_and_organize_output(handle, - std::move(result_srcs), - std::move(result_dsts), - std::move(result_weights), - std::move(result_edge_ids), - std::move(result_edge_types), - std::move(result_hops), - std::move(result_labels), - label_to_output_comm_rank); - - - if (result_labels && (result_offsets->size() != num_unique_labels + 1)) { - result_offsets = rmm::device_uvector(num_unique_labels + 1, handle.get_stream()); - - // Sort labels - thrust::sort( - handle.get_thrust_policy(), - cp_result_labels->begin(), - cp_result_labels->end()); - - - thrust::transform( - handle.get_thrust_policy(), - thrust::make_counting_iterator(0), - thrust::make_counting_iterator(result_offsets->size() - 1), - result_offsets->begin() + 1, - [ - result_labels = raft::device_span( - cp_result_labels->data(), - cp_result_labels->size()) - ] __device__(auto idx) { - auto itr_lower = thrust::lower_bound( - thrust::seq, result_labels.begin(), result_labels.end(), idx); - - auto itr_upper = thrust::upper_bound( - thrust::seq, result_labels.begin(), result_labels.end(), idx); - - auto sampled_label_size = thrust::distance(itr_lower, itr_upper); - - //return thrust::distance(itr_lower, itr_upper); - return sampled_label_size; - }); - - // Run inclusive scan - thrust::inclusive_scan(handle.get_thrust_policy(), - result_offsets->begin() + 1, - result_offsets->end(), - result_offsets->begin() + 1); - } - return std::make_tuple(std::move(result_srcs), std::move(result_dsts), std::move(result_weights), From bf172518b8edf230ea3ab251db2fca166ab9026e Mon Sep 17 00:00:00 2001 From: jnke2016 Date: Tue, 26 Nov 2024 19:21:06 -0800 Subject: [PATCH 29/49] update docstrings --- cpp/src/c_api/neighbor_sampling.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/src/c_api/neighbor_sampling.cpp b/cpp/src/c_api/neighbor_sampling.cpp index 1aa39fdeddd..64f997a972d 100644 --- a/cpp/src/c_api/neighbor_sampling.cpp +++ b/cpp/src/c_api/neighbor_sampling.cpp @@ -1290,7 +1290,7 @@ struct neighbor_sampling_functor : public cugraph::c_api::abstract_functor { (vertex_type_offsets_ != nullptr) ? vertex_type_offsets_->size_ - 1 : vertex_type_offsets.size() - 1, - // num_vertex_type is by default 1 if 'vertex_type_offsets' is not provided + // num_vertex_type is by default 1 if 'vertex_type_offsets' is not provided. num_edge_types_, src_is_major, do_expensive_check_); From 88b35baa276429e38a6c211ee531f387117f97f6 Mon Sep 17 00:00:00 2001 From: jnke2016 Date: Tue, 26 Nov 2024 20:48:43 -0800 Subject: [PATCH 30/49] simplify code and re-order statements --- cpp/src/sampling/neighbor_sampling_impl.hpp | 262 +++++++++----------- 1 file changed, 122 insertions(+), 140 deletions(-) diff --git a/cpp/src/sampling/neighbor_sampling_impl.hpp b/cpp/src/sampling/neighbor_sampling_impl.hpp index 882cff4b45f..c097bc68d56 100644 --- a/cpp/src/sampling/neighbor_sampling_impl.hpp +++ b/cpp/src/sampling/neighbor_sampling_impl.hpp @@ -106,28 +106,30 @@ neighbor_sample_impl(raft::handle_t const& handle, edge_masks_vector{}; graph_view_t modified_graph_view = graph_view; edge_masks_vector.reserve(num_edge_types); - + label_t num_unique_labels = 0; - std::optional> cp_starting_vertex_labels{std::nullopt}; - if (starting_vertex_labels) { // Find the number of unique lables - cp_starting_vertex_labels = - rmm::device_uvector(starting_vertex_labels->size(), handle.get_stream()); - - thrust::copy(handle.get_thrust_policy(), - starting_vertex_labels->begin(), - starting_vertex_labels->end(), - cp_starting_vertex_labels->begin()); - - thrust::sort(handle.get_thrust_policy(), - cp_starting_vertex_labels->begin(), - cp_starting_vertex_labels->end()); - + std::optional> cp_starting_vertex_labels{std::nullopt}; + cp_starting_vertex_labels = rmm::device_uvector(starting_vertex_labels->size(), handle.get_stream()); + + thrust::copy( + handle.get_thrust_policy(), + starting_vertex_labels->begin(), + starting_vertex_labels->end(), + cp_starting_vertex_labels->begin()); + + thrust::sort( + handle.get_thrust_policy(), + cp_starting_vertex_labels->begin(), + cp_starting_vertex_labels->end()); + num_unique_labels = thrust::unique_count(handle.get_thrust_policy(), cp_starting_vertex_labels->begin(), cp_starting_vertex_labels->end()); + + } if (num_edge_types > 1) { @@ -169,6 +171,7 @@ neighbor_sample_impl(raft::handle_t const& handle, ? (fan_out.size() / num_edge_types) : ((fan_out.size() / num_edge_types) + 1); + auto level_result_weight_vectors = edge_weight_view ? std::make_optional(std::vector>{}) : std::nullopt; @@ -189,14 +192,13 @@ neighbor_sample_impl(raft::handle_t const& handle, : std::nullopt; auto level_result_edge_id = edge_id_view ? std::make_optional(rmm::device_uvector(0, handle.get_stream())) - : std::nullopt; + : std::nullopt; auto level_result_edge_type = edge_type_view ? std::make_optional(rmm::device_uvector(0, handle.get_stream())) - : std::nullopt; + : std::nullopt; auto level_result_label = - starting_vertex_labels - ? std::make_optional(rmm::device_uvector(0, handle.get_stream())) - : std::nullopt; + starting_vertex_labels ? std::make_optional(rmm::device_uvector(0, handle.get_stream())) + : std::nullopt; if (level_result_weight_vectors) { (*level_result_weight_vectors).reserve(num_hops); } if (level_result_edge_id_vectors) { (*level_result_edge_id_vectors).reserve(num_hops); } @@ -222,9 +224,8 @@ neighbor_sample_impl(raft::handle_t const& handle, } std::vector level_sizes{}; - std::vector level_sizes_edge_types{}; - for (size_t hop = 0; hop < num_hops; ++hop) { + for (auto hop = 0; hop < num_hops; hop++) { for (auto edge_type_id = 0; edge_type_id < num_edge_types; edge_type_id++) { auto k_level = fan_out[(hop * num_edge_types) + edge_type_id]; rmm::device_uvector srcs(0, handle.get_stream()); @@ -262,71 +263,56 @@ neighbor_sample_impl(raft::handle_t const& handle, starting_vertex_labels); } - level_sizes_edge_types.push_back(srcs.size()); + auto old_size = level_result_src.size(); + level_result_src.resize(old_size + srcs.size(), handle.get_stream()); + level_result_dst.resize(old_size + srcs.size(), handle.get_stream()); - level_result_src.resize( - std::reduce(level_sizes_edge_types.begin(), level_sizes_edge_types.end()), - handle.get_stream()); - level_result_dst.resize( - std::reduce(level_sizes_edge_types.begin(), level_sizes_edge_types.end()), - handle.get_stream()); - raft::copy(level_result_src.begin() + - std::reduce(level_sizes_edge_types.begin(), level_sizes_edge_types.end() - 1), + raft::copy(level_result_src.begin() + old_size, srcs.begin(), srcs.size(), handle.get_stream()); - - raft::copy(level_result_dst.begin() + - std::reduce(level_sizes_edge_types.begin(), level_sizes_edge_types.end() - 1), + + raft::copy(level_result_dst.begin() + old_size, dsts.begin(), srcs.size(), handle.get_stream()); if (weights) { - (*level_result_weight) - .resize(std::reduce(level_sizes_edge_types.begin(), level_sizes_edge_types.end()), - handle.get_stream()); - - raft::copy(level_result_weight->begin() + - std::reduce(level_sizes_edge_types.begin(), level_sizes_edge_types.end() - 1), - weights->begin(), - srcs.size(), - handle.get_stream()); - } + (*level_result_weight).resize(old_size + srcs.size(), handle.get_stream()); + raft::copy(level_result_weight->begin() + old_size, + weights->begin(), + srcs.size(), + handle.get_stream()); + } + + + if (edge_ids) { - (*level_result_edge_id) - .resize(std::reduce(level_sizes_edge_types.begin(), level_sizes_edge_types.end()), - handle.get_stream()); - raft::copy(level_result_edge_id->begin() + - std::reduce(level_sizes_edge_types.begin(), level_sizes_edge_types.end() - 1), - edge_ids->begin(), - srcs.size(), - handle.get_stream()); + (*level_result_edge_id).resize(old_size + srcs.size(), handle.get_stream()); + raft::copy(level_result_edge_id->begin() + old_size, + edge_ids->begin(), + srcs.size(), + handle.get_stream()); } if (edge_types) { - (*level_result_edge_type) - .resize(std::reduce(level_sizes_edge_types.begin(), level_sizes_edge_types.end()), - handle.get_stream()); - - raft::copy(level_result_edge_type->begin() + - std::reduce(level_sizes_edge_types.begin(), level_sizes_edge_types.end() - 1), - edge_types->begin(), - srcs.size(), - handle.get_stream()); - } + (*level_result_edge_type).resize(old_size + srcs.size(), handle.get_stream()); + + raft::copy(level_result_edge_type->begin() + old_size, + edge_types->begin(), + srcs.size(), + handle.get_stream()); + } + if (labels) { - (*level_result_label) - .resize(std::reduce(level_sizes_edge_types.begin(), level_sizes_edge_types.end()), - handle.get_stream()); - - raft::copy(level_result_label->begin() + - std::reduce(level_sizes_edge_types.begin(), level_sizes_edge_types.end() - 1), - labels->begin(), - srcs.size(), - handle.get_stream()); + (*level_result_label).resize(old_size + srcs.size(), handle.get_stream()); + + raft::copy(level_result_label->begin() + old_size, + labels->begin(), + srcs.size(), + handle.get_stream()); } if (num_edge_types > 1) { modified_graph_view.clear_edge_mask(); } @@ -336,18 +322,10 @@ neighbor_sample_impl(raft::handle_t const& handle, level_result_src_vectors.push_back(std::move(level_result_src)); level_result_dst_vectors.push_back(std::move(level_result_dst)); - if (level_result_weight) { - (*level_result_weight_vectors).push_back(std::move(*level_result_weight)); - } - if (level_result_edge_id) { - (*level_result_edge_id_vectors).push_back(std::move(*level_result_edge_id)); - } - if (level_result_edge_type) { - (*level_result_edge_type_vectors).push_back(std::move(*level_result_edge_type)); - } - if (level_result_label) { - (*level_result_label_vectors).push_back(std::move(*level_result_label)); - } + if (level_result_weight) { (*level_result_weight_vectors).push_back(std::move(*level_result_weight)); } + if (level_result_edge_id) { (*level_result_edge_id_vectors).push_back(std::move(*level_result_edge_id)); } + if (level_result_edge_type) { (*level_result_edge_type_vectors).push_back(std::move(*level_result_edge_type)); } + if (level_result_label) { (*level_result_label_vectors).push_back(std::move(*level_result_label)); } // FIXME: We should modify vertex_partition_range_lasts to return a raft::host_span // rather than making a copy. @@ -357,10 +335,12 @@ neighbor_sample_impl(raft::handle_t const& handle, handle, starting_vertices, starting_vertex_labels, - raft::device_span{level_result_dst.data(), level_result_dst.size()}, - frontier_vertex_labels ? std::make_optional(raft::device_span( - level_result_label->data(), level_result_label->size())) - : std::nullopt, + raft::device_span{level_result_dst_vectors.back().data(), + level_result_dst_vectors.back().size()}, + frontier_vertex_labels + ? std::make_optional(raft::device_span( + level_result_label->data(), level_result_label->size())) + : std::nullopt, std::move(vertex_used_as_source), modified_graph_view.local_vertex_partition_view(), vertex_partition_range_lasts, @@ -457,7 +437,7 @@ neighbor_sample_impl(raft::handle_t const& handle, if (return_hops) { result_hops = rmm::device_uvector(result_size, handle.get_stream()); output_offset = 0; - for (size_t i = 0; i < num_hops; ++i) { // FIXME: replace this by the number of hops + for (size_t i = 0; i < num_hops; ++i) { scalar_fill( handle, result_hops->data() + output_offset, level_sizes[i], static_cast(i)); output_offset += level_sizes[i]; @@ -485,60 +465,62 @@ neighbor_sample_impl(raft::handle_t const& handle, if (result_labels) { cp_result_labels = rmm::device_uvector(result_labels->size(), handle.get_stream()); - thrust::copy(handle.get_thrust_policy(), - result_labels->begin(), - result_labels->end(), - cp_result_labels->begin()); - } - - std::tie(result_srcs, - result_dsts, - result_weights, - result_edge_ids, - result_edge_types, - result_hops, - result_labels, - result_offsets) = detail::shuffle_and_organize_output(handle, - std::move(result_srcs), - std::move(result_dsts), - std::move(result_weights), - std::move(result_edge_ids), - std::move(result_edge_types), - std::move(result_hops), - std::move(result_labels), - label_to_output_comm_rank); - - if (result_labels && (result_offsets->size() != num_unique_labels + 1)) { - result_offsets = rmm::device_uvector(num_unique_labels + 1, handle.get_stream()); - - // Sort labels - thrust::sort(handle.get_thrust_policy(), cp_result_labels->begin(), cp_result_labels->end()); - - thrust::transform(handle.get_thrust_policy(), - thrust::make_counting_iterator(0), - thrust::make_counting_iterator(result_offsets->size() - 1), - result_offsets->begin() + 1, - [result_labels = raft::device_span( - cp_result_labels->data(), cp_result_labels->size())] __device__(auto idx) { - auto itr_lower = thrust::lower_bound( - thrust::seq, result_labels.begin(), result_labels.end(), idx); - - auto itr_upper = thrust::upper_bound( - thrust::seq, result_labels.begin(), result_labels.end(), idx); - - auto sampled_label_size = thrust::distance(itr_lower, itr_upper); - - // return thrust::distance(itr_lower, itr_upper); - return sampled_label_size; - }); - - // Run inclusive scan - thrust::inclusive_scan(handle.get_thrust_policy(), - result_offsets->begin() + 1, - result_offsets->end(), - result_offsets->begin() + 1); + thrust::copy( + handle.get_thrust_policy(), + result_labels->begin(), + result_labels->end(), + cp_result_labels->begin()); } + std::tie(result_srcs, result_dsts, result_weights, result_edge_ids, + result_edge_types, result_hops, result_labels, result_offsets) + = detail::shuffle_and_organize_output(handle, + std::move(result_srcs), + std::move(result_dsts), + std::move(result_weights), + std::move(result_edge_ids), + std::move(result_edge_types), + std::move(result_hops), + std::move(result_labels), + label_to_output_comm_rank); + + if (result_labels && (result_offsets->size() != num_unique_labels + 1)) { + result_offsets = rmm::device_uvector(num_unique_labels + 1, handle.get_stream()); + + // Sort labels + thrust::sort( + handle.get_thrust_policy(), + cp_result_labels->begin(), + cp_result_labels->end()); + + thrust::transform( + handle.get_thrust_policy(), + thrust::make_counting_iterator(0), + thrust::make_counting_iterator(result_offsets->size() - 1), + result_offsets->begin() + 1, + [ + result_labels = raft::device_span( + cp_result_labels->data(), + cp_result_labels->size()) + ] __device__(auto idx) { + auto itr_lower = thrust::lower_bound( + thrust::seq, result_labels.begin(), result_labels.end(), idx); + + auto itr_upper = thrust::upper_bound( + thrust::seq, result_labels.begin(), result_labels.end(), idx); + + auto sampled_label_size = thrust::distance(itr_lower, itr_upper); + + return sampled_label_size; + }); + + // Run inclusive scan + thrust::inclusive_scan(handle.get_thrust_policy(), + result_offsets->begin() + 1, + result_offsets->end(), + result_offsets->begin() + 1); + } + return std::make_tuple(std::move(result_srcs), std::move(result_dsts), std::move(result_weights), From f8c576a38db64657e95eaf178479a8937c1f8b8f Mon Sep 17 00:00:00 2001 From: jnke2016 Date: Tue, 26 Nov 2024 20:49:46 -0800 Subject: [PATCH 31/49] fix style --- cpp/src/sampling/neighbor_sampling_impl.hpp | 211 ++++++++++---------- 1 file changed, 102 insertions(+), 109 deletions(-) diff --git a/cpp/src/sampling/neighbor_sampling_impl.hpp b/cpp/src/sampling/neighbor_sampling_impl.hpp index c097bc68d56..cf3cb3d0bbe 100644 --- a/cpp/src/sampling/neighbor_sampling_impl.hpp +++ b/cpp/src/sampling/neighbor_sampling_impl.hpp @@ -106,30 +106,27 @@ neighbor_sample_impl(raft::handle_t const& handle, edge_masks_vector{}; graph_view_t modified_graph_view = graph_view; edge_masks_vector.reserve(num_edge_types); - + label_t num_unique_labels = 0; if (starting_vertex_labels) { // Find the number of unique lables std::optional> cp_starting_vertex_labels{std::nullopt}; - cp_starting_vertex_labels = rmm::device_uvector(starting_vertex_labels->size(), handle.get_stream()); - - thrust::copy( - handle.get_thrust_policy(), - starting_vertex_labels->begin(), - starting_vertex_labels->end(), - cp_starting_vertex_labels->begin()); - - thrust::sort( - handle.get_thrust_policy(), - cp_starting_vertex_labels->begin(), - cp_starting_vertex_labels->end()); - + cp_starting_vertex_labels = + rmm::device_uvector(starting_vertex_labels->size(), handle.get_stream()); + + thrust::copy(handle.get_thrust_policy(), + starting_vertex_labels->begin(), + starting_vertex_labels->end(), + cp_starting_vertex_labels->begin()); + + thrust::sort(handle.get_thrust_policy(), + cp_starting_vertex_labels->begin(), + cp_starting_vertex_labels->end()); + num_unique_labels = thrust::unique_count(handle.get_thrust_policy(), cp_starting_vertex_labels->begin(), cp_starting_vertex_labels->end()); - - } if (num_edge_types > 1) { @@ -171,7 +168,6 @@ neighbor_sample_impl(raft::handle_t const& handle, ? (fan_out.size() / num_edge_types) : ((fan_out.size() / num_edge_types) + 1); - auto level_result_weight_vectors = edge_weight_view ? std::make_optional(std::vector>{}) : std::nullopt; @@ -192,13 +188,14 @@ neighbor_sample_impl(raft::handle_t const& handle, : std::nullopt; auto level_result_edge_id = edge_id_view ? std::make_optional(rmm::device_uvector(0, handle.get_stream())) - : std::nullopt; + : std::nullopt; auto level_result_edge_type = edge_type_view ? std::make_optional(rmm::device_uvector(0, handle.get_stream())) - : std::nullopt; + : std::nullopt; auto level_result_label = - starting_vertex_labels ? std::make_optional(rmm::device_uvector(0, handle.get_stream())) - : std::nullopt; + starting_vertex_labels + ? std::make_optional(rmm::device_uvector(0, handle.get_stream())) + : std::nullopt; if (level_result_weight_vectors) { (*level_result_weight_vectors).reserve(num_hops); } if (level_result_edge_id_vectors) { (*level_result_edge_id_vectors).reserve(num_hops); } @@ -267,52 +264,44 @@ neighbor_sample_impl(raft::handle_t const& handle, level_result_src.resize(old_size + srcs.size(), handle.get_stream()); level_result_dst.resize(old_size + srcs.size(), handle.get_stream()); + raft::copy( + level_result_src.begin() + old_size, srcs.begin(), srcs.size(), handle.get_stream()); - raft::copy(level_result_src.begin() + old_size, - srcs.begin(), - srcs.size(), - handle.get_stream()); - - raft::copy(level_result_dst.begin() + old_size, - dsts.begin(), - srcs.size(), - handle.get_stream()); + raft::copy( + level_result_dst.begin() + old_size, dsts.begin(), srcs.size(), handle.get_stream()); if (weights) { (*level_result_weight).resize(old_size + srcs.size(), handle.get_stream()); raft::copy(level_result_weight->begin() + old_size, - weights->begin(), - srcs.size(), - handle.get_stream()); + weights->begin(), + srcs.size(), + handle.get_stream()); } - - - + if (edge_ids) { (*level_result_edge_id).resize(old_size + srcs.size(), handle.get_stream()); raft::copy(level_result_edge_id->begin() + old_size, - edge_ids->begin(), - srcs.size(), - handle.get_stream()); + edge_ids->begin(), + srcs.size(), + handle.get_stream()); } if (edge_types) { (*level_result_edge_type).resize(old_size + srcs.size(), handle.get_stream()); - raft::copy(level_result_edge_type->begin() + old_size, - edge_types->begin(), - srcs.size(), - handle.get_stream()); + edge_types->begin(), + srcs.size(), + handle.get_stream()); } - + if (labels) { (*level_result_label).resize(old_size + srcs.size(), handle.get_stream()); raft::copy(level_result_label->begin() + old_size, - labels->begin(), - srcs.size(), - handle.get_stream()); + labels->begin(), + srcs.size(), + handle.get_stream()); } if (num_edge_types > 1) { modified_graph_view.clear_edge_mask(); } @@ -322,10 +311,18 @@ neighbor_sample_impl(raft::handle_t const& handle, level_result_src_vectors.push_back(std::move(level_result_src)); level_result_dst_vectors.push_back(std::move(level_result_dst)); - if (level_result_weight) { (*level_result_weight_vectors).push_back(std::move(*level_result_weight)); } - if (level_result_edge_id) { (*level_result_edge_id_vectors).push_back(std::move(*level_result_edge_id)); } - if (level_result_edge_type) { (*level_result_edge_type_vectors).push_back(std::move(*level_result_edge_type)); } - if (level_result_label) { (*level_result_label_vectors).push_back(std::move(*level_result_label)); } + if (level_result_weight) { + (*level_result_weight_vectors).push_back(std::move(*level_result_weight)); + } + if (level_result_edge_id) { + (*level_result_edge_id_vectors).push_back(std::move(*level_result_edge_id)); + } + if (level_result_edge_type) { + (*level_result_edge_type_vectors).push_back(std::move(*level_result_edge_type)); + } + if (level_result_label) { + (*level_result_label_vectors).push_back(std::move(*level_result_label)); + } // FIXME: We should modify vertex_partition_range_lasts to return a raft::host_span // rather than making a copy. @@ -337,10 +334,9 @@ neighbor_sample_impl(raft::handle_t const& handle, starting_vertex_labels, raft::device_span{level_result_dst_vectors.back().data(), level_result_dst_vectors.back().size()}, - frontier_vertex_labels - ? std::make_optional(raft::device_span( - level_result_label->data(), level_result_label->size())) - : std::nullopt, + frontier_vertex_labels ? std::make_optional(raft::device_span( + level_result_label->data(), level_result_label->size())) + : std::nullopt, std::move(vertex_used_as_source), modified_graph_view.local_vertex_partition_view(), vertex_partition_range_lasts, @@ -465,62 +461,59 @@ neighbor_sample_impl(raft::handle_t const& handle, if (result_labels) { cp_result_labels = rmm::device_uvector(result_labels->size(), handle.get_stream()); - thrust::copy( - handle.get_thrust_policy(), - result_labels->begin(), - result_labels->end(), - cp_result_labels->begin()); + thrust::copy(handle.get_thrust_policy(), + result_labels->begin(), + result_labels->end(), + cp_result_labels->begin()); + } + + std::tie(result_srcs, + result_dsts, + result_weights, + result_edge_ids, + result_edge_types, + result_hops, + result_labels, + result_offsets) = detail::shuffle_and_organize_output(handle, + std::move(result_srcs), + std::move(result_dsts), + std::move(result_weights), + std::move(result_edge_ids), + std::move(result_edge_types), + std::move(result_hops), + std::move(result_labels), + label_to_output_comm_rank); + + if (result_labels && (result_offsets->size() != num_unique_labels + 1)) { + result_offsets = rmm::device_uvector(num_unique_labels + 1, handle.get_stream()); + + // Sort labels + thrust::sort(handle.get_thrust_policy(), cp_result_labels->begin(), cp_result_labels->end()); + + thrust::transform(handle.get_thrust_policy(), + thrust::make_counting_iterator(0), + thrust::make_counting_iterator(result_offsets->size() - 1), + result_offsets->begin() + 1, + [result_labels = raft::device_span( + cp_result_labels->data(), cp_result_labels->size())] __device__(auto idx) { + auto itr_lower = thrust::lower_bound( + thrust::seq, result_labels.begin(), result_labels.end(), idx); + + auto itr_upper = thrust::upper_bound( + thrust::seq, result_labels.begin(), result_labels.end(), idx); + + auto sampled_label_size = thrust::distance(itr_lower, itr_upper); + + return sampled_label_size; + }); + + // Run inclusive scan + thrust::inclusive_scan(handle.get_thrust_policy(), + result_offsets->begin() + 1, + result_offsets->end(), + result_offsets->begin() + 1); } - std::tie(result_srcs, result_dsts, result_weights, result_edge_ids, - result_edge_types, result_hops, result_labels, result_offsets) - = detail::shuffle_and_organize_output(handle, - std::move(result_srcs), - std::move(result_dsts), - std::move(result_weights), - std::move(result_edge_ids), - std::move(result_edge_types), - std::move(result_hops), - std::move(result_labels), - label_to_output_comm_rank); - - if (result_labels && (result_offsets->size() != num_unique_labels + 1)) { - result_offsets = rmm::device_uvector(num_unique_labels + 1, handle.get_stream()); - - // Sort labels - thrust::sort( - handle.get_thrust_policy(), - cp_result_labels->begin(), - cp_result_labels->end()); - - thrust::transform( - handle.get_thrust_policy(), - thrust::make_counting_iterator(0), - thrust::make_counting_iterator(result_offsets->size() - 1), - result_offsets->begin() + 1, - [ - result_labels = raft::device_span( - cp_result_labels->data(), - cp_result_labels->size()) - ] __device__(auto idx) { - auto itr_lower = thrust::lower_bound( - thrust::seq, result_labels.begin(), result_labels.end(), idx); - - auto itr_upper = thrust::upper_bound( - thrust::seq, result_labels.begin(), result_labels.end(), idx); - - auto sampled_label_size = thrust::distance(itr_lower, itr_upper); - - return sampled_label_size; - }); - - // Run inclusive scan - thrust::inclusive_scan(handle.get_thrust_policy(), - result_offsets->begin() + 1, - result_offsets->end(), - result_offsets->begin() + 1); - } - return std::make_tuple(std::move(result_srcs), std::move(result_dsts), std::move(result_weights), From 9648d6992156b8f5fd5ec769a157a385c806ea3e Mon Sep 17 00:00:00 2001 From: jnke2016 Date: Wed, 27 Nov 2024 09:02:16 -0800 Subject: [PATCH 32/49] update docstrings --- cpp/src/sampling/neighbor_sampling_impl.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/src/sampling/neighbor_sampling_impl.hpp b/cpp/src/sampling/neighbor_sampling_impl.hpp index cf3cb3d0bbe..dd3ae5f1d3a 100644 --- a/cpp/src/sampling/neighbor_sampling_impl.hpp +++ b/cpp/src/sampling/neighbor_sampling_impl.hpp @@ -163,7 +163,7 @@ neighbor_sample_impl(raft::handle_t const& handle, rmm::device_uvector level_result_src(0, handle.get_stream()); rmm::device_uvector level_result_dst(0, handle.get_stream()); - // Get the number of hop. If homogeneous neighbor sample, num_edge_types = 1 + // Get the number of hop. If homogeneous neighbor sample, num_edge_types = 1. auto num_hops = ((fan_out.size() % num_edge_types) == 0) ? (fan_out.size() / num_edge_types) : ((fan_out.size() / num_edge_types) + 1); From 9d84558ed393e74c430e0379ab064b00dd188c16 Mon Sep 17 00:00:00 2001 From: jnke2016 Date: Sat, 30 Nov 2024 09:00:03 -0800 Subject: [PATCH 33/49] fix bug when creating struct --- cpp/src/c_api/neighbor_sampling.cpp | 2 ++ cpp/src/sampling/neighbor_sampling_impl.hpp | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/cpp/src/c_api/neighbor_sampling.cpp b/cpp/src/c_api/neighbor_sampling.cpp index 64f997a972d..37982eab82e 100644 --- a/cpp/src/c_api/neighbor_sampling.cpp +++ b/cpp/src/c_api/neighbor_sampling.cpp @@ -404,6 +404,7 @@ struct uniform_neighbor_sampling_functor : public cugraph::c_api::abstract_funct (label_hop_offsets) ? new cugraph::c_api::cugraph_type_erased_device_array_t(*label_hop_offsets, SIZE_T) : nullptr, + nullptr, (edge_label) ? new cugraph::c_api::cugraph_type_erased_device_array_t(edge_label.value(), INT32) : nullptr, @@ -757,6 +758,7 @@ struct biased_neighbor_sampling_functor : public cugraph::c_api::abstract_functo (label_hop_offsets) ? new cugraph::c_api::cugraph_type_erased_device_array_t(*label_hop_offsets, SIZE_T) : nullptr, + nullptr, (edge_label) ? new cugraph::c_api::cugraph_type_erased_device_array_t(edge_label.value(), INT32) : nullptr, diff --git a/cpp/src/sampling/neighbor_sampling_impl.hpp b/cpp/src/sampling/neighbor_sampling_impl.hpp index dd3ae5f1d3a..b0e78ec7db3 100644 --- a/cpp/src/sampling/neighbor_sampling_impl.hpp +++ b/cpp/src/sampling/neighbor_sampling_impl.hpp @@ -335,7 +335,7 @@ neighbor_sample_impl(raft::handle_t const& handle, raft::device_span{level_result_dst_vectors.back().data(), level_result_dst_vectors.back().size()}, frontier_vertex_labels ? std::make_optional(raft::device_span( - level_result_label->data(), level_result_label->size())) + level_result_label_vectors->back().data(), level_result_label_vectors->back().size())) : std::nullopt, std::move(vertex_used_as_source), modified_graph_view.local_vertex_partition_view(), From a7a224cb2a59272014b938517a37bf75a090924a Mon Sep 17 00:00:00 2001 From: jnke2016 Date: Sat, 30 Nov 2024 09:14:08 -0800 Subject: [PATCH 34/49] fix style --- cpp/src/sampling/neighbor_sampling_impl.hpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/cpp/src/sampling/neighbor_sampling_impl.hpp b/cpp/src/sampling/neighbor_sampling_impl.hpp index b0e78ec7db3..2df150fb766 100644 --- a/cpp/src/sampling/neighbor_sampling_impl.hpp +++ b/cpp/src/sampling/neighbor_sampling_impl.hpp @@ -334,9 +334,10 @@ neighbor_sample_impl(raft::handle_t const& handle, starting_vertex_labels, raft::device_span{level_result_dst_vectors.back().data(), level_result_dst_vectors.back().size()}, - frontier_vertex_labels ? std::make_optional(raft::device_span( - level_result_label_vectors->back().data(), level_result_label_vectors->back().size())) - : std::nullopt, + frontier_vertex_labels + ? std::make_optional(raft::device_span( + level_result_label_vectors->back().data(), level_result_label_vectors->back().size())) + : std::nullopt, std::move(vertex_used_as_source), modified_graph_view.local_vertex_partition_view(), vertex_partition_range_lasts, From d64dc665ef1bd1a09d08e031860fa4cbfa8b9da5 Mon Sep 17 00:00:00 2001 From: jnke2016 Date: Sat, 30 Nov 2024 16:25:22 -0800 Subject: [PATCH 35/49] add missing arguments --- .../pylibcugraph/_cugraph_c/sampling_algorithms.pxd | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python/pylibcugraph/pylibcugraph/_cugraph_c/sampling_algorithms.pxd b/python/pylibcugraph/pylibcugraph/_cugraph_c/sampling_algorithms.pxd index 762fd37a35d..f496cc7d880 100644 --- a/python/pylibcugraph/pylibcugraph/_cugraph_c/sampling_algorithms.pxd +++ b/python/pylibcugraph/pylibcugraph/_cugraph_c/sampling_algorithms.pxd @@ -73,6 +73,7 @@ cdef extern from "cugraph_c/sampling_algorithms.h": cugraph_graph_t* graph, const cugraph_type_erased_device_array_view_t* start_vertices, const cugraph_type_erased_device_array_view_t* starting_vertex_label_offsets, + const cugraph_type_erased_device_array_view_t* vertex_tyoe_offsets, const cugraph_type_erased_host_array_view_t* fan_out, int num_edge_types, const cugraph_sampling_options_t* options, @@ -88,6 +89,7 @@ cdef extern from "cugraph_c/sampling_algorithms.h": const cugraph_edge_property_view_t* edge_biases, const cugraph_type_erased_device_array_view_t* start_vertices, const cugraph_type_erased_device_array_view_t* starting_vertex_label_offsets, + const cugraph_type_erased_device_array_view_t* vertex_tyoe_offsets, const cugraph_type_erased_host_array_view_t* fan_out, int num_edge_types, const cugraph_sampling_options_t* options, From bcfc99cdbf6769662f0633ec9ce71c7a005cb2c3 Mon Sep 17 00:00:00 2001 From: jnke2016 Date: Sun, 1 Dec 2024 14:00:50 -0800 Subject: [PATCH 36/49] update label list if some are missing from the result --- cpp/src/sampling/neighbor_sampling_impl.hpp | 19 ++++++++++++++++--- .../tests/sampling/test_bulk_sampler.py | 10 +++++++--- .../tests/sampling/test_bulk_sampler_mg.py | 10 +++++++--- 3 files changed, 30 insertions(+), 9 deletions(-) diff --git a/cpp/src/sampling/neighbor_sampling_impl.hpp b/cpp/src/sampling/neighbor_sampling_impl.hpp index 2df150fb766..11a792fe038 100644 --- a/cpp/src/sampling/neighbor_sampling_impl.hpp +++ b/cpp/src/sampling/neighbor_sampling_impl.hpp @@ -109,9 +109,10 @@ neighbor_sample_impl(raft::handle_t const& handle, label_t num_unique_labels = 0; + std::optional> cp_starting_vertex_labels{std::nullopt}; + if (starting_vertex_labels) { // Find the number of unique lables - std::optional> cp_starting_vertex_labels{std::nullopt}; cp_starting_vertex_labels = rmm::device_uvector(starting_vertex_labels->size(), handle.get_stream()); @@ -486,8 +487,19 @@ neighbor_sample_impl(raft::handle_t const& handle, label_to_output_comm_rank); if (result_labels && (result_offsets->size() != num_unique_labels + 1)) { - result_offsets = rmm::device_uvector(num_unique_labels + 1, handle.get_stream()); - + // If there are missing labels, still inlude it in the result_labels + result_labels = std::move(*cp_starting_vertex_labels); + auto unique_labels_end = + thrust::unique(handle.get_thrust_policy(), + result_labels->begin(), + result_labels->end()); + + auto num_unique_labels = thrust::distance( + result_labels->begin(), unique_labels_end); + + result_labels->resize(num_unique_labels, handle.get_stream()); + + result_offsets->resize(num_unique_labels + 1, handle.get_stream()); // Sort labels thrust::sort(handle.get_thrust_policy(), cp_result_labels->begin(), cp_result_labels->end()); @@ -507,6 +519,7 @@ neighbor_sample_impl(raft::handle_t const& handle, return sampled_label_size; }); + // Run inclusive scan thrust::inclusive_scan(handle.get_thrust_policy(), diff --git a/python/cugraph/cugraph/tests/sampling/test_bulk_sampler.py b/python/cugraph/cugraph/tests/sampling/test_bulk_sampler.py index 3c5d6428001..765c6ef8935 100644 --- a/python/cugraph/cugraph/tests/sampling/test_bulk_sampler.py +++ b/python/cugraph/cugraph/tests/sampling/test_bulk_sampler.py @@ -278,7 +278,11 @@ def test_bulk_sampler_empty_batches(scratch_dir): assert len(os.listdir(samples_path)) == 1 - df = cudf.read_parquet(os.path.join(samples_path, "batch=0-1.parquet")) + # There are 3 batches [0, 1, 2] where batch 1 has no results. In fact, seeds + # [7, 8, 9] have no outgoing edges. The previous implementation returned and + # offsets array omitting seeds with no outgoing edges from the + # edge_label_offsets which is no longer the case + df = cudf.read_parquet(os.path.join(samples_path, "batch=0-2.parquet")) assert df[ (df.batch_id == 0) & (df.hop_id == 0) @@ -289,12 +293,12 @@ def test_bulk_sampler_empty_batches(scratch_dir): ].destinations.sort_values().values_host.tolist() == [2, 3, 7, 8] assert df[ - (df.batch_id == 1) & (df.hop_id == 0) + (df.batch_id == 2) & (df.hop_id == 0) ].destinations.sort_values().values_host.tolist() == [7, 8] assert len(df[(df.batch_id == 1) & (df.hop_id == 1)]) == 0 - assert df.batch_id.max() == 1 + assert df.batch_id.max() == 2 shutil.rmtree(samples_path) diff --git a/python/cugraph/cugraph/tests/sampling/test_bulk_sampler_mg.py b/python/cugraph/cugraph/tests/sampling/test_bulk_sampler_mg.py index 3fddb8f405b..77db37d4b98 100644 --- a/python/cugraph/cugraph/tests/sampling/test_bulk_sampler_mg.py +++ b/python/cugraph/cugraph/tests/sampling/test_bulk_sampler_mg.py @@ -228,7 +228,11 @@ def test_bulk_sampler_empty_batches(dask_client, scratch_dir): assert len(os.listdir(samples_path)) == 1 - df = cudf.read_parquet(os.path.join(samples_path, "batch=0-1.parquet")) + # There are 3 batches [0, 1, 2] where batch 1 has no results. In fact, seeds + # [7, 8, 9] have no outgoing edges. The previous implementation returned and + # offsets array omitting seeds with no outgoing edges from the + # edge_label_offsets which is no longer the case + df = cudf.read_parquet(os.path.join(samples_path, "batch=0-2.parquet")) assert df[ (df.batch_id == 0) & (df.hop_id == 0) @@ -239,12 +243,12 @@ def test_bulk_sampler_empty_batches(dask_client, scratch_dir): ].destinations.sort_values().values_host.tolist() == [2, 3, 7, 8] assert df[ - (df.batch_id == 1) & (df.hop_id == 0) + (df.batch_id == 2) & (df.hop_id == 0) ].destinations.sort_values().values_host.tolist() == [7, 8] assert len(df[(df.batch_id == 1) & (df.hop_id == 1)]) == 0 - assert df.batch_id.max() == 1 + assert df.batch_id.max() == 2 shutil.rmtree(samples_path) From 19c37a810023f139567b819862530d6d0b35102e Mon Sep 17 00:00:00 2001 From: jnke2016 Date: Sun, 1 Dec 2024 14:01:49 -0800 Subject: [PATCH 37/49] fix style --- cpp/src/sampling/neighbor_sampling_impl.hpp | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/cpp/src/sampling/neighbor_sampling_impl.hpp b/cpp/src/sampling/neighbor_sampling_impl.hpp index 11a792fe038..d666d2dd282 100644 --- a/cpp/src/sampling/neighbor_sampling_impl.hpp +++ b/cpp/src/sampling/neighbor_sampling_impl.hpp @@ -490,13 +490,10 @@ neighbor_sample_impl(raft::handle_t const& handle, // If there are missing labels, still inlude it in the result_labels result_labels = std::move(*cp_starting_vertex_labels); auto unique_labels_end = - thrust::unique(handle.get_thrust_policy(), - result_labels->begin(), - result_labels->end()); - - auto num_unique_labels = thrust::distance( - result_labels->begin(), unique_labels_end); - + thrust::unique(handle.get_thrust_policy(), result_labels->begin(), result_labels->end()); + + auto num_unique_labels = thrust::distance(result_labels->begin(), unique_labels_end); + result_labels->resize(num_unique_labels, handle.get_stream()); result_offsets->resize(num_unique_labels + 1, handle.get_stream()); @@ -519,7 +516,6 @@ neighbor_sample_impl(raft::handle_t const& handle, return sampled_label_size; }); - // Run inclusive scan thrust::inclusive_scan(handle.get_thrust_policy(), From 7b7c6487b3bfbf390a670dddbda474f92ef0aa17 Mon Sep 17 00:00:00 2001 From: Alexandria Barghi Date: Mon, 2 Dec 2024 08:58:52 -0800 Subject: [PATCH 38/49] x --- .../cugraph/gnn/data_loading/dist_sampler.py | 66 ++++++++++++------- .../_cugraph_c/sampling_algorithms.pxd | 2 + 2 files changed, 46 insertions(+), 22 deletions(-) diff --git a/python/cugraph/cugraph/gnn/data_loading/dist_sampler.py b/python/cugraph/cugraph/gnn/data_loading/dist_sampler.py index 2e58e258c55..d5769ead226 100644 --- a/python/cugraph/cugraph/gnn/data_loading/dist_sampler.py +++ b/python/cugraph/cugraph/gnn/data_loading/dist_sampler.py @@ -631,14 +631,20 @@ def __init__( with_replacement: bool = False, biased: bool = False, heterogeneous: bool = False, + vertex_type_offsets: Optional[TensorType] = None, num_edge_types: int = 1, ): + self.__fanout = fanout - self.__prior_sources_behavior = prior_sources_behavior - self.__deduplicate_sources = deduplicate_sources - self.__compress_per_hop = compress_per_hop - self.__compression = compression - self.__with_replacement = with_replacement + self.__func_kwargs = { + "h_fan_out": np.asarray(fanout, dtype="int32"), + "prior_sources_behavior": prior_sources_behavior, + "retain_seeds": retain_original_seeds, + "deduplicate_sources": deduplicate_sources, + "compress_per_hop": compress_per_hop, + "compression": compression, + "with_replacement": with_replacement, + } # It is currently required that graphs are weighted for biased # sampling. So setting the function here is safe. In the future, @@ -647,11 +653,17 @@ def __init__( # TODO allow func to be a call to a future remote sampling API # if the provided graph is in another process (rapidsai/cugraph#4623). if heterogeneous: + if vertex_type_offsets is None: + raise ValueError("Heterogeneous sampling requires vertex type offsets.") self.__func = ( pylibcugraph.heterogeneous_biased_neighbor_sample if biased else pylibcugraph.heterogeneous_uniform_neighbor_sample ) + self.__func_kwargs["num_edge_types"] = num_edge_types + self.__func_kwargs["vertex_type_offsets"] = cupy.asarray( + vertex_type_offsets + ) else: self.__func = ( pylibcugraph.homogeneous_biased_neighbor_sample @@ -661,27 +673,45 @@ def __init__( if num_edge_types > 1 and not heterogeneous: raise ValueError( - "Heterogeneous sampling must be selected if there is > 1 edge type" + "Heterogeneous sampling must be selected if there is > 1 edge type." ) - self.__num_edge_types = num_edge_types - super().__init__( graph, writer, - local_seeds_per_call=self.__calc_local_seeds_per_call(local_seeds_per_call), + local_seeds_per_call=self.__calc_local_seeds_per_call( + local_seeds_per_call, + heterogeneous=heterogeneous, + num_edge_types=num_edge_types, + ), retain_original_seeds=retain_original_seeds, ) - def __calc_local_seeds_per_call(self, local_seeds_per_call: Optional[int] = None): + def __calc_local_seeds_per_call( + self, + local_seeds_per_call: Optional[int] = None, + heterogeneous: bool = False, + num_edge_types: int = 1, + ): torch = import_optional("torch") + fanout = self.__fanout + if local_seeds_per_call is None: - if len([x for x in self.__fanout if x <= 0]) > 0: + if len([x for x in fanout if x <= 0]) > 0: return NeighborSampler.UNKNOWN_VERTICES_DEFAULT + if heterogeneous: + if len(fanout) % num_edge_types != 0: + raise ValueError(f"Illegal fanout for {num_edge_types} edge types.") + num_hops = len(fanout) // num_edge_types + fanout = [ + sum([fanout[t * num_hops + h] for t in range(num_edge_types)]) + for h in range(num_hops) + ] + total_memory = torch.cuda.get_device_properties(0).total_memory - fanout_prod = reduce(lambda x, y: x * y, self.__fanout) + fanout_prod = reduce(lambda x, y: x * y, fanout) return int( NeighborSampler.BASE_VERTICES_PER_BYTE * total_memory / fanout_prod ) @@ -702,25 +732,17 @@ def sample_batches( "input_graph": self._graph, "start_vertex_list": cupy.asarray(seeds), "starting_vertex_label_offsets": cupy.asarray(batch_id_offsets), - "h_fan_out": np.asarray(self.__fanout, dtype="int32"), - "with_replacement": self.__with_replacement, "renumber": True, "return_hops": True, "do_expensive_check": False, - "prior_sources_behavior": self.__prior_sources_behavior, - "deduplicate_sources": self.__deduplicate_sources, - "retain_seeds": self._retain_original_seeds, - "compression": self.__compression, - "compress_per_hop": self.__compress_per_hop, "random_state": random_state + rank, } - - if self.__num_edge_types > 1: - kwargs["num_edge_types"] = self.__num_edge_types + kwargs.update(self.__func_kwargs) print(kwargs) sampling_results_dict = self.__func(**kwargs) + print(sampling_results_dict) sampling_results_dict["fanout"] = cupy.array(self.__fanout, dtype="int32") return sampling_results_dict diff --git a/python/pylibcugraph/pylibcugraph/_cugraph_c/sampling_algorithms.pxd b/python/pylibcugraph/pylibcugraph/_cugraph_c/sampling_algorithms.pxd index 762fd37a35d..bc57c647d26 100644 --- a/python/pylibcugraph/pylibcugraph/_cugraph_c/sampling_algorithms.pxd +++ b/python/pylibcugraph/pylibcugraph/_cugraph_c/sampling_algorithms.pxd @@ -73,6 +73,7 @@ cdef extern from "cugraph_c/sampling_algorithms.h": cugraph_graph_t* graph, const cugraph_type_erased_device_array_view_t* start_vertices, const cugraph_type_erased_device_array_view_t* starting_vertex_label_offsets, + const cugraph_type_erased_device_array_view_t* vertex_type_offsets, const cugraph_type_erased_host_array_view_t* fan_out, int num_edge_types, const cugraph_sampling_options_t* options, @@ -88,6 +89,7 @@ cdef extern from "cugraph_c/sampling_algorithms.h": const cugraph_edge_property_view_t* edge_biases, const cugraph_type_erased_device_array_view_t* start_vertices, const cugraph_type_erased_device_array_view_t* starting_vertex_label_offsets, + const cugraph_type_erased_device_array_view_t* vertex_type_offsets, const cugraph_type_erased_host_array_view_t* fan_out, int num_edge_types, const cugraph_sampling_options_t* options, From bbc32cd294b568ed11b244341c9965cccdc63dd9 Mon Sep 17 00:00:00 2001 From: Alexandria Barghi Date: Mon, 2 Dec 2024 14:17:20 -0800 Subject: [PATCH 39/49] c --- cpp/src/sampling/neighbor_sampling_impl.hpp | 14 +++++++++++++- .../cugraph/gnn/data_loading/dist_sampler.py | 1 + 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/cpp/src/sampling/neighbor_sampling_impl.hpp b/cpp/src/sampling/neighbor_sampling_impl.hpp index dd3ae5f1d3a..4f288448b1e 100644 --- a/cpp/src/sampling/neighbor_sampling_impl.hpp +++ b/cpp/src/sampling/neighbor_sampling_impl.hpp @@ -225,6 +225,7 @@ neighbor_sample_impl(raft::handle_t const& handle, for (auto hop = 0; hop < num_hops; hop++) { for (auto edge_type_id = 0; edge_type_id < num_edge_types; edge_type_id++) { auto k_level = fan_out[(hop * num_edge_types) + edge_type_id]; + rmm::device_uvector srcs(0, handle.get_stream()); rmm::device_uvector dsts(0, handle.get_stream()); std::optional> weights{std::nullopt}; @@ -249,7 +250,7 @@ neighbor_sample_impl(raft::handle_t const& handle, starting_vertex_labels, static_cast(k_level), with_replacement); - } else { + } else if (k_level < 0) { std::tie(srcs, dsts, weights, edge_ids, edge_types, labels) = gather_one_hop_edgelist(handle, modified_graph_view, @@ -258,6 +259,17 @@ neighbor_sample_impl(raft::handle_t const& handle, edge_type_view, starting_vertices, starting_vertex_labels); + } else { + if (edge_weight_view) { + weights.emplace(rmm::device_uvector(0, handle.get_stream())); + } + if (edge_id_view) { edge_ids.emplace(rmm::device_uvector(0, handle.get_stream())); } + if (edge_type_view) { + edge_types.emplace(rmm::device_uvector(0, handle.get_stream())); + } + if (starting_vertex_labels) { + labels.emplace(rmm::device_uvector(0, handle.get_stream())); + } } auto old_size = level_result_src.size(); diff --git a/python/cugraph/cugraph/gnn/data_loading/dist_sampler.py b/python/cugraph/cugraph/gnn/data_loading/dist_sampler.py index d5769ead226..9d581cd2d7c 100644 --- a/python/cugraph/cugraph/gnn/data_loading/dist_sampler.py +++ b/python/cugraph/cugraph/gnn/data_loading/dist_sampler.py @@ -664,6 +664,7 @@ def __init__( self.__func_kwargs["vertex_type_offsets"] = cupy.asarray( vertex_type_offsets ) + print("selected heterogeneous sampling") else: self.__func = ( pylibcugraph.homogeneous_biased_neighbor_sample From c94c1cb4efe4cd9f68a174ac8423de65683c86e4 Mon Sep 17 00:00:00 2001 From: Alexandria Barghi Date: Tue, 10 Dec 2024 08:46:57 -0800 Subject: [PATCH 40/49] revert cpp change --- cpp/src/sampling/neighbor_sampling_impl.hpp | 107 ++------------------ 1 file changed, 9 insertions(+), 98 deletions(-) diff --git a/cpp/src/sampling/neighbor_sampling_impl.hpp b/cpp/src/sampling/neighbor_sampling_impl.hpp index 81005ffffe3..ed77b330439 100644 --- a/cpp/src/sampling/neighbor_sampling_impl.hpp +++ b/cpp/src/sampling/neighbor_sampling_impl.hpp @@ -107,29 +107,6 @@ neighbor_sample_impl(raft::handle_t const& handle, graph_view_t modified_graph_view = graph_view; edge_masks_vector.reserve(num_edge_types); - label_t num_unique_labels = 0; - - std::optional> cp_starting_vertex_labels{std::nullopt}; - - if (starting_vertex_labels) { - // Find the number of unique lables - cp_starting_vertex_labels = - rmm::device_uvector(starting_vertex_labels->size(), handle.get_stream()); - - thrust::copy(handle.get_thrust_policy(), - starting_vertex_labels->begin(), - starting_vertex_labels->end(), - cp_starting_vertex_labels->begin()); - - thrust::sort(handle.get_thrust_policy(), - cp_starting_vertex_labels->begin(), - cp_starting_vertex_labels->end()); - - num_unique_labels = thrust::unique_count(handle.get_thrust_policy(), - cp_starting_vertex_labels->begin(), - cp_starting_vertex_labels->end()); - } - if (num_edge_types > 1) { for (int i = 0; i < num_edge_types; i++) { cugraph::edge_property_t, bool> @@ -227,7 +204,6 @@ neighbor_sample_impl(raft::handle_t const& handle, for (auto edge_type_id = 0; edge_type_id < num_edge_types; edge_type_id++) { auto k_level = fan_out[(hop * num_edge_types) + edge_type_id]; - rmm::device_uvector srcs(0, handle.get_stream()); rmm::device_uvector dsts(0, handle.get_stream()); std::optional> weights{std::nullopt}; @@ -470,80 +446,15 @@ neighbor_sample_impl(raft::handle_t const& handle, level_result_label_vectors = std::nullopt; } - std::optional> result_offsets{std::nullopt}; - std::optional> cp_result_labels{std::nullopt}; - if (result_labels) { - cp_result_labels = rmm::device_uvector(result_labels->size(), handle.get_stream()); - - thrust::copy(handle.get_thrust_policy(), - result_labels->begin(), - result_labels->end(), - cp_result_labels->begin()); - } - - std::tie(result_srcs, - result_dsts, - result_weights, - result_edge_ids, - result_edge_types, - result_hops, - result_labels, - result_offsets) = detail::shuffle_and_organize_output(handle, - std::move(result_srcs), - std::move(result_dsts), - std::move(result_weights), - std::move(result_edge_ids), - std::move(result_edge_types), - std::move(result_hops), - std::move(result_labels), - label_to_output_comm_rank); - - if (result_labels && (result_offsets->size() != num_unique_labels + 1)) { - // If there are missing labels, still inlude it in the result_labels - result_labels = std::move(*cp_starting_vertex_labels); - auto unique_labels_end = - thrust::unique(handle.get_thrust_policy(), result_labels->begin(), result_labels->end()); - - auto num_unique_labels = thrust::distance(result_labels->begin(), unique_labels_end); - - result_labels->resize(num_unique_labels, handle.get_stream()); - - result_offsets->resize(num_unique_labels + 1, handle.get_stream()); - // Sort labels - thrust::sort(handle.get_thrust_policy(), cp_result_labels->begin(), cp_result_labels->end()); - - thrust::transform(handle.get_thrust_policy(), - thrust::make_counting_iterator(0), - thrust::make_counting_iterator(result_offsets->size() - 1), - result_offsets->begin() + 1, - [result_labels = raft::device_span( - cp_result_labels->data(), cp_result_labels->size())] __device__(auto idx) { - auto itr_lower = thrust::lower_bound( - thrust::seq, result_labels.begin(), result_labels.end(), idx); - - auto itr_upper = thrust::upper_bound( - thrust::seq, result_labels.begin(), result_labels.end(), idx); - - auto sampled_label_size = thrust::distance(itr_lower, itr_upper); - - return sampled_label_size; - }); - - // Run inclusive scan - thrust::inclusive_scan(handle.get_thrust_policy(), - result_offsets->begin() + 1, - result_offsets->end(), - result_offsets->begin() + 1); - } - - return std::make_tuple(std::move(result_srcs), - std::move(result_dsts), - std::move(result_weights), - std::move(result_edge_ids), - std::move(result_edge_types), - std::move(result_hops), - std::move(result_labels), - std::move(result_offsets)); + return detail::shuffle_and_organize_output(handle, + std::move(result_srcs), + std::move(result_dsts), + std::move(result_weights), + std::move(result_edge_ids), + std::move(result_edge_types), + std::move(result_hops), + std::move(result_labels), + label_to_output_comm_rank); } } // namespace detail From 9754f9d61a74aa37127f483665958f1f3836aaef Mon Sep 17 00:00:00 2001 From: Alexandria Barghi Date: Tue, 10 Dec 2024 10:55:05 -0800 Subject: [PATCH 41/49] fix disk writing with new api --- .../gnn/data_loading/dist_io/writer.py | 71 +++++++++++-------- .../cugraph/gnn/data_loading/dist_sampler.py | 18 ++--- .../tests/sampling/test_dist_sampler.py | 49 +++++++++++-- 3 files changed, 96 insertions(+), 42 deletions(-) diff --git a/python/cugraph/cugraph/gnn/data_loading/dist_io/writer.py b/python/cugraph/cugraph/gnn/data_loading/dist_io/writer.py index f8ad4719a76..d71c2da8a3e 100644 --- a/python/cugraph/cugraph/gnn/data_loading/dist_io/writer.py +++ b/python/cugraph/cugraph/gnn/data_loading/dist_io/writer.py @@ -79,9 +79,15 @@ def get_reader( return DistSampleReader(self._directory, format=self._format, rank=rank) def __write_minibatches_coo(self, minibatch_dict): - has_edge_ids = minibatch_dict["edge_id"] is not None - has_edge_types = minibatch_dict["edge_type"] is not None - has_weights = minibatch_dict["weight"] is not None + has_edge_ids = ( + "edge_id" in minibatch_dict and minibatch_dict["edge_id"] is not None + ) + has_edge_types = ( + "edge_type" in minibatch_dict and minibatch_dict["edge_type"] is not None + ) + has_weights = ( + "weight" in minibatch_dict and minibatch_dict["weight"] is not None + ) if minibatch_dict["renumber_map"] is None: raise ValueError( @@ -92,13 +98,12 @@ def __write_minibatches_coo(self, minibatch_dict): if len(minibatch_dict["batch_id"]) == 0: return - fanout_length = (len(minibatch_dict["label_hop_offsets"]) - 1) // len( - minibatch_dict["batch_id"] - ) + fanout_length = len(minibatch_dict["fanout"]) + total_num_batches = ( + len(minibatch_dict["label_hop_offsets"]) - 1 + ) / fanout_length - for p in range( - 0, int(ceil(len(minibatch_dict["batch_id"]) / self.__batches_per_partition)) - ): + for p in range(0, int(ceil(total_num_batches / self.__batches_per_partition))): partition_start = p * (self.__batches_per_partition) partition_end = (p + 1) * (self.__batches_per_partition) @@ -106,8 +111,9 @@ def __write_minibatches_coo(self, minibatch_dict): partition_start * fanout_length : partition_end * fanout_length + 1 ] - batch_id_array_p = minibatch_dict["batch_id"][partition_start:partition_end] - start_batch_id = batch_id_array_p[0] + num_batches_p = len(label_hop_offsets_array_p) - 1 + + start_batch_id = minibatch_dict["batch_start"] input_offsets_p = minibatch_dict["input_offsets"][ partition_start : (partition_end + 1) @@ -171,7 +177,7 @@ def __write_minibatches_coo(self, minibatch_dict): } ) - end_batch_id = start_batch_id + len(batch_id_array_p) - 1 + end_batch_id = start_batch_id + num_batches_p - 1 rank = minibatch_dict["rank"] if "rank" in minibatch_dict else 0 full_output_path = os.path.join( @@ -188,9 +194,15 @@ def __write_minibatches_coo(self, minibatch_dict): ) def __write_minibatches_csr(self, minibatch_dict): - has_edge_ids = minibatch_dict["edge_id"] is not None - has_edge_types = minibatch_dict["edge_type"] is not None - has_weights = minibatch_dict["weight"] is not None + has_edge_ids = ( + "edge_id" in minibatch_dict and minibatch_dict["edge_id"] is not None + ) + has_edge_types = ( + "edge_type" in minibatch_dict and minibatch_dict["edge_type"] is not None + ) + has_weights = ( + "weight" in minibatch_dict and minibatch_dict["weight"] is not None + ) if minibatch_dict["renumber_map"] is None: raise ValueError( @@ -201,13 +213,12 @@ def __write_minibatches_csr(self, minibatch_dict): if len(minibatch_dict["batch_id"]) == 0: return - fanout_length = (len(minibatch_dict["label_hop_offsets"]) - 1) // len( - minibatch_dict["batch_id"] - ) + fanout_length = len(minibatch_dict["fanout"]) + total_num_batches = ( + len(minibatch_dict["label_hop_offsets"]) - 1 + ) / fanout_length - for p in range( - 0, int(ceil(len(minibatch_dict["batch_id"]) / self.__batches_per_partition)) - ): + for p in range(0, int(ceil(total_num_batches / self.__batches_per_partition))): partition_start = p * (self.__batches_per_partition) partition_end = (p + 1) * (self.__batches_per_partition) @@ -215,8 +226,9 @@ def __write_minibatches_csr(self, minibatch_dict): partition_start * fanout_length : partition_end * fanout_length + 1 ] - batch_id_array_p = minibatch_dict["batch_id"][partition_start:partition_end] - start_batch_id = batch_id_array_p[0] + num_batches_p = len(label_hop_offsets_array_p) - 1 + + start_batch_id = minibatch_dict["batch_start"] input_offsets_p = minibatch_dict["input_offsets"][ partition_start : (partition_end + 1) @@ -292,7 +304,7 @@ def __write_minibatches_csr(self, minibatch_dict): } ) - end_batch_id = start_batch_id + len(batch_id_array_p) - 1 + end_batch_id = start_batch_id + num_batches_p - 1 rank = minibatch_dict["rank"] if "rank" in minibatch_dict else 0 full_output_path = os.path.join( @@ -309,13 +321,14 @@ def __write_minibatches_csr(self, minibatch_dict): ) def write_minibatches(self, minibatch_dict): - if (minibatch_dict["majors"] is not None) and ( - minibatch_dict["minors"] is not None + if ("majors" in minibatch_dict and minibatch_dict["majors"] is not None) and ( + "minors" in minibatch_dict and minibatch_dict["minors"] is not None ): self.__write_minibatches_coo(minibatch_dict) - elif (minibatch_dict["major_offsets"] is not None) and ( - minibatch_dict["minors"] is not None - ): + elif ( + "major_offsets" in minibatch_dict + and minibatch_dict["major_offsets"] is not None + ) and ("minors" in minibatch_dict and minibatch_dict["minors"] is not None): self.__write_minibatches_csr(minibatch_dict) else: raise ValueError("invalid columns") diff --git a/python/cugraph/cugraph/gnn/data_loading/dist_sampler.py b/python/cugraph/cugraph/gnn/data_loading/dist_sampler.py index c9164109a2f..d633169959d 100644 --- a/python/cugraph/cugraph/gnn/data_loading/dist_sampler.py +++ b/python/cugraph/cugraph/gnn/data_loading/dist_sampler.py @@ -212,8 +212,9 @@ def __sample_from_nodes_func( ] ) else: + minibatch_dict["batch_start"] = batch_id_start self.__writer.write_minibatches(minibatch_dict) - return None + return batch_id_start + input_offsets.numel() - 1 def __get_call_groups( self, @@ -307,13 +308,13 @@ def sample_from_nodes( assume_equal_input_size=input_size_is_equal, ) - sample_args = ( + sample_args = [ batch_id_start, batch_size, batches_per_call, random_state, input_size_is_equal, - ) + ] if self.__writer is None: # Buffered sampling @@ -327,7 +328,7 @@ def sample_from_nodes( for i, current_seeds_and_ix in enumerate( zip(nodes_call_groups, index_call_groups) ): - self.__sample_from_nodes_func( + sample_args[0] = self.__sample_from_nodes_func( i, current_seeds_and_ix, *sample_args, @@ -489,8 +490,9 @@ def __sample_from_edges_func( ] ) else: + minibatch_dict["batch_start"] = batch_id_start self.__writer.write_minibatches(minibatch_dict) - return None + return batch_id_start + current_batch_offsets.numel() - 1 def sample_from_edges( self, @@ -547,13 +549,13 @@ def sample_from_edges( assume_equal_input_size=input_size_is_equal, ) - sample_args = ( + sample_args = [ batch_id_start, batch_size, batches_per_call, random_state, input_size_is_equal, - ) + ] if self.__writer is None: # Buffered sampling @@ -567,7 +569,7 @@ def sample_from_edges( for i, current_seeds_and_ix in enumerate( zip(edges_call_groups, index_call_groups) ): - self.__sample_from_edges_func( + sample_args[0] = self.__sample_from_edges_func( i, current_seeds_and_ix, *sample_args, diff --git a/python/cugraph/cugraph/tests/sampling/test_dist_sampler.py b/python/cugraph/cugraph/tests/sampling/test_dist_sampler.py index 64db0232fb1..06838be059b 100644 --- a/python/cugraph/cugraph/tests/sampling/test_dist_sampler.py +++ b/python/cugraph/cugraph/tests/sampling/test_dist_sampler.py @@ -79,13 +79,12 @@ def test_dist_sampler_simple( ) recovered_samples = cudf.read_parquet(samples_path) - print(recovered_samples) original_el = karate.get_edgelist() for b in range(len(seeds) // batch_size): el_start = recovered_samples.label_hop_offsets.iloc[b * len(fanout)] el_end = recovered_samples.label_hop_offsets.iloc[(b + 1) * len(fanout)] - print(el_start, el_end) + src = recovered_samples.majors.iloc[el_start:el_end] dst = recovered_samples.minors.iloc[el_start:el_end] edge_id = recovered_samples.edge_id.iloc[el_start:el_end] @@ -107,7 +106,7 @@ def test_dist_sampler_simple( @pytest.mark.sg @pytest.mark.skipif(isinstance(torch, MissingModule), reason="torch not available") @pytest.mark.parametrize("seeds_per_call", [4, 5, 10]) -@pytest.mark.parametrize("compression", ["COO", "CSR"]) +@pytest.mark.parametrize("compression", ["CSR", "COO"]) def test_dist_sampler_buffered_in_memory( scratch_dir: str, karate_graph: SGGraph, seeds_per_call: int, compression: str ): @@ -146,16 +145,56 @@ def test_dist_sampler_buffered_in_memory( (create_df_from_disjoint_arrays(r[0]), r[1], r[2]) for r in buffered_results ] + print([r[1] for r in unbuffered_results]) + print("\n\n") + print([r[1] for r in buffered_results]) + assert len(buffered_results) == len(unbuffered_results) for k in range(len(buffered_results)): br, bs, be = buffered_results[k] ur, us, ue = unbuffered_results[k] - assert bs == us - assert be == ue + assert (be - bs) == (ue - us) for col in ur.columns: assert (br[col].dropna() == ur[col].dropna()).all() shutil.rmtree(samples_path) + + +@pytest.mark.sg +@pytest.mark.skipif(isinstance(torch, MissingModule), reason="torch not available") +def test_dist_sampler_hetero_from_nodes(): + props = GraphProperties( + is_symmetric=False, + is_multigraph=True, + ) + + handle = ResourceHandle() + + graph = SGGraph( + handle, + props, + cupy.array([4, 5, 6, 7, 8, 9, 8, 9, 8, 7, 6, 5, 4, 5]), + cupy.array([0, 1, 2, 3, 3, 0, 4, 5, 6, 8, 7, 8, 9, 9]), + vertices_array=cupy.arange(10), + edge_id_array=cupy.array([0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 6, 7]), + edge_type_array=cupy.array( + [0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1], dtype="int32" + ), + weight_array=cupy.ones((14,), dtype="float32"), + ) + + sampler = UniformNeighborSampler( + graph, + writer=None, + compression="COO", + ) + + out = sampler.sample_from_nodes( + nodes=cupy.array([4, 5]), + input_id=cupy.array([5, 10]), + ) + + print(out) From f87578e7058be6bfed904453a8166b93aa74070d Mon Sep 17 00:00:00 2001 From: Alexandria Barghi Date: Tue, 10 Dec 2024 11:52:29 -0800 Subject: [PATCH 42/49] add tests for sg --- .../cugraph/gnn/data_loading/dist_sampler.py | 2 + .../tests/sampling/test_dist_sampler.py | 116 ++++++++++++++++-- 2 files changed, 108 insertions(+), 10 deletions(-) diff --git a/python/cugraph/cugraph/gnn/data_loading/dist_sampler.py b/python/cugraph/cugraph/gnn/data_loading/dist_sampler.py index d633169959d..9574e55d2ee 100644 --- a/python/cugraph/cugraph/gnn/data_loading/dist_sampler.py +++ b/python/cugraph/cugraph/gnn/data_loading/dist_sampler.py @@ -295,6 +295,8 @@ def sample_from_nodes( if input_id is None: input_id = torch.arange(num_seeds, dtype=torch.int64, device="cpu") + else: + input_id = torch.as_tensor(input_id, device="cpu") local_num_batches = int(ceil(num_seeds / batch_size)) batch_id_start, input_size_is_equal = self.get_start_batch_offset( diff --git a/python/cugraph/cugraph/tests/sampling/test_dist_sampler.py b/python/cugraph/cugraph/tests/sampling/test_dist_sampler.py index 06838be059b..eaa49917110 100644 --- a/python/cugraph/cugraph/tests/sampling/test_dist_sampler.py +++ b/python/cugraph/cugraph/tests/sampling/test_dist_sampler.py @@ -59,6 +59,7 @@ def karate_graph() -> SGGraph: @pytest.mark.parametrize("equal_input_size", [True, False]) @pytest.mark.parametrize("fanout", [[2, 2], [4, 4], [4, 2, 1]]) @pytest.mark.parametrize("batch_size", [1, 2, 4]) +@pytest.mark.skip(reason="bleh") @pytest.mark.skipif(isinstance(torch, MissingModule), reason="torch not available") def test_dist_sampler_simple( scratch_dir, karate_graph, batch_size, fanout, equal_input_size @@ -107,6 +108,7 @@ def test_dist_sampler_simple( @pytest.mark.skipif(isinstance(torch, MissingModule), reason="torch not available") @pytest.mark.parametrize("seeds_per_call", [4, 5, 10]) @pytest.mark.parametrize("compression", ["CSR", "COO"]) +@pytest.mark.skip(reason="bleh") def test_dist_sampler_buffered_in_memory( scratch_dir: str, karate_graph: SGGraph, seeds_per_call: int, compression: str ): @@ -145,10 +147,6 @@ def test_dist_sampler_buffered_in_memory( (create_df_from_disjoint_arrays(r[0]), r[1], r[2]) for r in buffered_results ] - print([r[1] for r in unbuffered_results]) - print("\n\n") - print([r[1] for r in buffered_results]) - assert len(buffered_results) == len(unbuffered_results) for k in range(len(buffered_results)): @@ -173,23 +171,31 @@ def test_dist_sampler_hetero_from_nodes(): handle = ResourceHandle() + srcs = cupy.array([4, 5, 6, 7, 8, 9, 8, 9, 8, 7, 6, 5, 4, 5]) + dsts = cupy.array([0, 1, 2, 3, 3, 0, 4, 5, 6, 8, 7, 8, 9, 9]) + eids = cupy.array([0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 6, 7]) + etps = cupy.array([0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1], dtype="int32") + graph = SGGraph( handle, props, - cupy.array([4, 5, 6, 7, 8, 9, 8, 9, 8, 7, 6, 5, 4, 5]), - cupy.array([0, 1, 2, 3, 3, 0, 4, 5, 6, 8, 7, 8, 9, 9]), + srcs, + dsts, vertices_array=cupy.arange(10), - edge_id_array=cupy.array([0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 6, 7]), - edge_type_array=cupy.array( - [0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1], dtype="int32" - ), + edge_id_array=eids, + edge_type_array=etps, weight_array=cupy.ones((14,), dtype="float32"), ) sampler = UniformNeighborSampler( graph, + fanout=[-1, -1, -1, -1], writer=None, compression="COO", + heterogeneous=True, + vertex_type_offsets=cupy.array([0, 4, 10]), + num_edge_types=2, + deduplicate_sources=True, ) out = sampler.sample_from_nodes( @@ -197,4 +203,94 @@ def test_dist_sampler_hetero_from_nodes(): input_id=cupy.array([5, 10]), ) + out = [z for z in out] + assert len(out) == 1 + out, _, _ = out[0] + print(out) + + lho = out["label_type_hop_offsets"] + + # Edge type 0 + emap = out["edge_renumber_map"][ + out["edge_renumber_map_offsets"][0] : out["edge_renumber_map_offsets"][1] + ] + + smap = out["map"][out["renumber_map_offsets"][1] : out["renumber_map_offsets"][2]] + + dmap = out["map"][out["renumber_map_offsets"][0] : out["renumber_map_offsets"][1]] + + # Edge type 0, hop 0 + hop_start = lho[0] + hop_end = lho[1] + + assert hop_end - hop_start == 2 + + e = out["edge_id"][hop_start:hop_end] + e = emap[e] + assert sorted(e.tolist()) == [0, 1] + + s = cupy.asarray(smap[out["majors"][hop_start:hop_end]]) + d = cupy.asarray(dmap[out["minors"][hop_start:hop_end]]) + + assert sorted(s.tolist()) == [4, 5] + assert sorted(d.tolist()) == [0, 1] + + # Edge type 0, hop 1 + hop_start = int(lho[1]) + hop_end = int(lho[2]) + + assert hop_end - hop_start == 2 + + e = out["edge_id"][hop_start:hop_end] + e = emap[e] + assert sorted(e.tolist()) == [4, 5] + + s = cupy.asarray(smap[out["majors"][hop_start:hop_end]]) + d = cupy.asarray(dmap[out["minors"][hop_start:hop_end]]) + + assert sorted(s.tolist()) == [8, 9] + assert sorted(d.tolist()) == [0, 3] + + ############################# + + # Edge type 1 + emap = out["edge_renumber_map"][ + out["edge_renumber_map_offsets"][1] : out["edge_renumber_map_offsets"][2] + ] + + smap = out["map"][out["renumber_map_offsets"][1] : out["renumber_map_offsets"][2]] + + dmap = smap + + # Edge type 1, hop 0 + hop_start = lho[2] + hop_end = lho[3] + + assert hop_end - hop_start == 3 + + e = out["edge_id"][hop_start:hop_end] + e = emap[e] + assert sorted(e.tolist()) == [5, 6, 7] + + s = cupy.asarray(smap[out["majors"][hop_start:hop_end]]) + d = cupy.asarray(dmap[out["minors"][hop_start:hop_end]]) + + assert sorted(s.tolist()) == [4, 5, 5] + assert sorted(d.tolist()) == [8, 9, 9] + + # Edge type 1, hop 1 + hop_start = lho[3] + hop_end = lho[4] + + assert hop_end - hop_start == 3 + + e = out["edge_id"][hop_start:hop_end] + e = emap[e] + assert sorted(e.tolist()) == [0, 1, 2] + + s = cupy.asarray(smap[out["majors"][hop_start:hop_end]]) + d = cupy.asarray(dmap[out["minors"][hop_start:hop_end]]) + + assert sorted(s.tolist()) == [8, 8, 9] + assert sorted(d.tolist()) == [4, 5, 6] From 42a2d4a721469803882957cd458b885294e2bc59 Mon Sep 17 00:00:00 2001 From: jnke2016 Date: Tue, 10 Dec 2024 14:24:41 -0800 Subject: [PATCH 43/49] fix style --- cpp/src/c_api/neighbor_sampling.cpp | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/cpp/src/c_api/neighbor_sampling.cpp b/cpp/src/c_api/neighbor_sampling.cpp index 37982eab82e..2cc96463099 100644 --- a/cpp/src/c_api/neighbor_sampling.cpp +++ b/cpp/src/c_api/neighbor_sampling.cpp @@ -880,7 +880,6 @@ struct neighbor_sampling_functor : public cugraph::c_api::abstract_functor { handle_.get_stream()); std::optional> start_vertex_labels{std::nullopt}; - std::optional> local_label_to_comm_rank{std::nullopt}; std::optional> label_to_comm_rank{ std::nullopt}; // global after allgatherv @@ -932,12 +931,13 @@ struct neighbor_sampling_functor : public cugraph::c_api::abstract_functor { handle_.get_stream(), raft::device_span{unique_labels.data(), unique_labels.size()}); - (*local_label_to_comm_rank).resize(num_unique_labels, handle_.get_stream()); + rmm::device_uvector local_label_to_comm_rank(num_unique_labels, + handle_.get_stream()); cugraph::detail::scalar_fill( handle_.get_stream(), - (*local_label_to_comm_rank).begin(), // This should be rename to rank - (*local_label_to_comm_rank).size(), + local_label_to_comm_rank.begin(), // This should be rename to rank + local_label_to_comm_rank.size(), label_t{handle_.get_comms().get_rank()}); // Perform allgather to get global_label_to_comm_rank_d_vector @@ -948,11 +948,13 @@ struct neighbor_sampling_functor : public cugraph::c_api::abstract_functor { std::exclusive_scan( recvcounts.begin(), recvcounts.end(), displacements.begin(), size_t{0}); - (*label_to_comm_rank) - .resize(displacements.back() + recvcounts.back(), handle_.get_stream()); + rmm::device_uvector tmp_label_to_comm_rank( + displacements.back() + recvcounts.back(), handle_.get_stream()); + + label_to_comm_rank = std::move(tmp_label_to_comm_rank); cugraph::device_allgatherv(handle_.get_comms(), - (*local_label_to_comm_rank).begin(), + local_label_to_comm_rank.begin(), (*label_to_comm_rank).begin(), recvcounts, displacements, From ba5283830a44f3a9139aa8a0651407ca0a400bca Mon Sep 17 00:00:00 2001 From: Alexandria Barghi Date: Tue, 10 Dec 2024 15:50:08 -0800 Subject: [PATCH 44/49] fix rank issue in dist sampler --- python/cugraph/cugraph/gnn/data_loading/dist_sampler.py | 1 + python/cugraph/cugraph/tests/sampling/test_dist_sampler.py | 4 ---- .../cugraph/cugraph/tests/sampling/test_dist_sampler_mg.py | 7 +++++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/python/cugraph/cugraph/gnn/data_loading/dist_sampler.py b/python/cugraph/cugraph/gnn/data_loading/dist_sampler.py index 9574e55d2ee..2891289866f 100644 --- a/python/cugraph/cugraph/gnn/data_loading/dist_sampler.py +++ b/python/cugraph/cugraph/gnn/data_loading/dist_sampler.py @@ -743,4 +743,5 @@ def sample_batches( sampling_results_dict = self.__func(**kwargs) sampling_results_dict["fanout"] = cupy.array(self.__fanout, dtype="int32") + sampling_results_dict["rank"] = rank return sampling_results_dict diff --git a/python/cugraph/cugraph/tests/sampling/test_dist_sampler.py b/python/cugraph/cugraph/tests/sampling/test_dist_sampler.py index eaa49917110..1f4df8a648d 100644 --- a/python/cugraph/cugraph/tests/sampling/test_dist_sampler.py +++ b/python/cugraph/cugraph/tests/sampling/test_dist_sampler.py @@ -59,7 +59,6 @@ def karate_graph() -> SGGraph: @pytest.mark.parametrize("equal_input_size", [True, False]) @pytest.mark.parametrize("fanout", [[2, 2], [4, 4], [4, 2, 1]]) @pytest.mark.parametrize("batch_size", [1, 2, 4]) -@pytest.mark.skip(reason="bleh") @pytest.mark.skipif(isinstance(torch, MissingModule), reason="torch not available") def test_dist_sampler_simple( scratch_dir, karate_graph, batch_size, fanout, equal_input_size @@ -108,7 +107,6 @@ def test_dist_sampler_simple( @pytest.mark.skipif(isinstance(torch, MissingModule), reason="torch not available") @pytest.mark.parametrize("seeds_per_call", [4, 5, 10]) @pytest.mark.parametrize("compression", ["CSR", "COO"]) -@pytest.mark.skip(reason="bleh") def test_dist_sampler_buffered_in_memory( scratch_dir: str, karate_graph: SGGraph, seeds_per_call: int, compression: str ): @@ -207,8 +205,6 @@ def test_dist_sampler_hetero_from_nodes(): assert len(out) == 1 out, _, _ = out[0] - print(out) - lho = out["label_type_hop_offsets"] # Edge type 0 diff --git a/python/cugraph/cugraph/tests/sampling/test_dist_sampler_mg.py b/python/cugraph/cugraph/tests/sampling/test_dist_sampler_mg.py index 5bb541d6cf3..0fcfd97c53f 100644 --- a/python/cugraph/cugraph/tests/sampling/test_dist_sampler_mg.py +++ b/python/cugraph/cugraph/tests/sampling/test_dist_sampler_mg.py @@ -59,6 +59,9 @@ def karate_mg_graph(rank, world_size): [el.src.astype("int64")], [el.dst.astype("int64")], edge_id_array=[el.eid], + vertices_array=[ + cupy.array_split(cupy.arange(34, dtype="int64"), world_size)[rank] + ], ) return G @@ -199,6 +202,7 @@ def run_test_dist_sampler_uneven( @pytest.mark.parametrize("fanout", [[4, 4], [4, 2, 1]]) @pytest.mark.parametrize("batch_size", [1, 4]) @pytest.mark.parametrize("seeds_per_call", [4, 8, 16]) +@pytest.mark.skip(reason="broken") @pytest.mark.skipif(isinstance(torch, MissingModule), reason="torch not installed") def test_dist_sampler_uneven(scratch_dir, batch_size, fanout, seeds_per_call): uid = cugraph_comms_create_unique_id() @@ -290,8 +294,7 @@ def run_test_dist_sampler_buffered_in_memory( br, bs, be = buffered_results[k] ur, us, ue = unbuffered_results[k] - assert bs == us - assert be == ue + assert be - bs == ue - us for col in ur.columns: assert (br[col].dropna() == ur[col].dropna()).all() From d934b72ad00000fb967c646e7ae4beb8513c1177 Mon Sep 17 00:00:00 2001 From: Alexandria Barghi Date: Wed, 11 Dec 2024 07:31:16 -0800 Subject: [PATCH 45/49] change empty input handling to work with new plc api --- .../cugraph/gnn/data_loading/dist_io/writer.py | 14 ++++++++++---- .../cugraph/gnn/data_loading/dist_sampler.py | 1 + .../cugraph/tests/sampling/test_dist_sampler_mg.py | 3 ++- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/python/cugraph/cugraph/gnn/data_loading/dist_io/writer.py b/python/cugraph/cugraph/gnn/data_loading/dist_io/writer.py index d71c2da8a3e..15ad7e30df3 100644 --- a/python/cugraph/cugraph/gnn/data_loading/dist_io/writer.py +++ b/python/cugraph/cugraph/gnn/data_loading/dist_io/writer.py @@ -321,14 +321,20 @@ def __write_minibatches_csr(self, minibatch_dict): ) def write_minibatches(self, minibatch_dict): - if ("majors" in minibatch_dict and minibatch_dict["majors"] is not None) and ( - "minors" in minibatch_dict and minibatch_dict["minors"] is not None - ): + if "minors" not in minibatch_dict: + raise ValueError("invalid columns") + + # PLC API specifies this behavior for empty input + # This needs to be handled here to avoid causing a hang + if len(minibatch_dict["minors"]) == 0: + return + + if "majors" in minibatch_dict and minibatch_dict["majors"] is not None: self.__write_minibatches_coo(minibatch_dict) elif ( "major_offsets" in minibatch_dict and minibatch_dict["major_offsets"] is not None - ) and ("minors" in minibatch_dict and minibatch_dict["minors"] is not None): + ): self.__write_minibatches_csr(minibatch_dict) else: raise ValueError("invalid columns") diff --git a/python/cugraph/cugraph/gnn/data_loading/dist_sampler.py b/python/cugraph/cugraph/gnn/data_loading/dist_sampler.py index 2891289866f..33874733340 100644 --- a/python/cugraph/cugraph/gnn/data_loading/dist_sampler.py +++ b/python/cugraph/cugraph/gnn/data_loading/dist_sampler.py @@ -189,6 +189,7 @@ def __sample_from_nodes_func( batch_id_offsets=input_offsets, random_state=random_state, ) + minibatch_dict["input_index"] = current_ix.cuda() minibatch_dict["input_offsets"] = input_offsets diff --git a/python/cugraph/cugraph/tests/sampling/test_dist_sampler_mg.py b/python/cugraph/cugraph/tests/sampling/test_dist_sampler_mg.py index 0fcfd97c53f..281c259c2e9 100644 --- a/python/cugraph/cugraph/tests/sampling/test_dist_sampler_mg.py +++ b/python/cugraph/cugraph/tests/sampling/test_dist_sampler_mg.py @@ -116,6 +116,7 @@ def run_test_dist_sampler_simple( @pytest.mark.parametrize("batch_size", [1, 4]) @pytest.mark.parametrize("seeds_per_rank", [8, 1]) @pytest.mark.parametrize("seeds_per_call", [4, 8]) +@pytest.mark.skip("bleh") @pytest.mark.skipif(isinstance(torch, MissingModule), reason="torch not installed") def test_dist_sampler_simple( scratch_dir, batch_size, seeds_per_rank, fanout, equal_input_size, seeds_per_call @@ -202,7 +203,6 @@ def run_test_dist_sampler_uneven( @pytest.mark.parametrize("fanout", [[4, 4], [4, 2, 1]]) @pytest.mark.parametrize("batch_size", [1, 4]) @pytest.mark.parametrize("seeds_per_call", [4, 8, 16]) -@pytest.mark.skip(reason="broken") @pytest.mark.skipif(isinstance(torch, MissingModule), reason="torch not installed") def test_dist_sampler_uneven(scratch_dir, batch_size, fanout, seeds_per_call): uid = cugraph_comms_create_unique_id() @@ -304,6 +304,7 @@ def run_test_dist_sampler_buffered_in_memory( @pytest.mark.skipif(isinstance(torch, MissingModule), reason="torch not available") @pytest.mark.parametrize("seeds_per_call", [4, 5, 10]) @pytest.mark.parametrize("compression", ["COO", "CSR"]) +@pytest.mark.skip(reason="bleh") def test_dist_sampler_buffered_in_memory(scratch_dir, seeds_per_call, compression): uid = cugraph_comms_create_unique_id() From b9437ea2cf5f763ae6f9eb0bcf52009d0b8a84cb Mon Sep 17 00:00:00 2001 From: Alexandria Barghi Date: Wed, 11 Dec 2024 07:43:19 -0800 Subject: [PATCH 46/49] fix bulk sampler tests, re-enable other tests --- python/cugraph/cugraph/tests/sampling/test_bulk_sampler.py | 6 +++--- .../cugraph/cugraph/tests/sampling/test_bulk_sampler_mg.py | 6 +++--- .../cugraph/cugraph/tests/sampling/test_dist_sampler_mg.py | 2 -- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/python/cugraph/cugraph/tests/sampling/test_bulk_sampler.py b/python/cugraph/cugraph/tests/sampling/test_bulk_sampler.py index 765c6ef8935..54e159012e2 100644 --- a/python/cugraph/cugraph/tests/sampling/test_bulk_sampler.py +++ b/python/cugraph/cugraph/tests/sampling/test_bulk_sampler.py @@ -282,7 +282,7 @@ def test_bulk_sampler_empty_batches(scratch_dir): # [7, 8, 9] have no outgoing edges. The previous implementation returned and # offsets array omitting seeds with no outgoing edges from the # edge_label_offsets which is no longer the case - df = cudf.read_parquet(os.path.join(samples_path, "batch=0-2.parquet")) + df = cudf.read_parquet(os.path.join(samples_path, "batch=0-1.parquet")) assert df[ (df.batch_id == 0) & (df.hop_id == 0) @@ -293,12 +293,12 @@ def test_bulk_sampler_empty_batches(scratch_dir): ].destinations.sort_values().values_host.tolist() == [2, 3, 7, 8] assert df[ - (df.batch_id == 2) & (df.hop_id == 0) + (df.batch_id == 1) & (df.hop_id == 0) ].destinations.sort_values().values_host.tolist() == [7, 8] assert len(df[(df.batch_id == 1) & (df.hop_id == 1)]) == 0 - assert df.batch_id.max() == 2 + assert df.batch_id.max() == 1 shutil.rmtree(samples_path) diff --git a/python/cugraph/cugraph/tests/sampling/test_bulk_sampler_mg.py b/python/cugraph/cugraph/tests/sampling/test_bulk_sampler_mg.py index 77db37d4b98..f16b78a40a1 100644 --- a/python/cugraph/cugraph/tests/sampling/test_bulk_sampler_mg.py +++ b/python/cugraph/cugraph/tests/sampling/test_bulk_sampler_mg.py @@ -232,7 +232,7 @@ def test_bulk_sampler_empty_batches(dask_client, scratch_dir): # [7, 8, 9] have no outgoing edges. The previous implementation returned and # offsets array omitting seeds with no outgoing edges from the # edge_label_offsets which is no longer the case - df = cudf.read_parquet(os.path.join(samples_path, "batch=0-2.parquet")) + df = cudf.read_parquet(os.path.join(samples_path, "batch=0-1.parquet")) assert df[ (df.batch_id == 0) & (df.hop_id == 0) @@ -243,12 +243,12 @@ def test_bulk_sampler_empty_batches(dask_client, scratch_dir): ].destinations.sort_values().values_host.tolist() == [2, 3, 7, 8] assert df[ - (df.batch_id == 2) & (df.hop_id == 0) + (df.batch_id == 1) & (df.hop_id == 0) ].destinations.sort_values().values_host.tolist() == [7, 8] assert len(df[(df.batch_id == 1) & (df.hop_id == 1)]) == 0 - assert df.batch_id.max() == 2 + assert df.batch_id.max() == 1 shutil.rmtree(samples_path) diff --git a/python/cugraph/cugraph/tests/sampling/test_dist_sampler_mg.py b/python/cugraph/cugraph/tests/sampling/test_dist_sampler_mg.py index 281c259c2e9..ee6f64ef9e4 100644 --- a/python/cugraph/cugraph/tests/sampling/test_dist_sampler_mg.py +++ b/python/cugraph/cugraph/tests/sampling/test_dist_sampler_mg.py @@ -116,7 +116,6 @@ def run_test_dist_sampler_simple( @pytest.mark.parametrize("batch_size", [1, 4]) @pytest.mark.parametrize("seeds_per_rank", [8, 1]) @pytest.mark.parametrize("seeds_per_call", [4, 8]) -@pytest.mark.skip("bleh") @pytest.mark.skipif(isinstance(torch, MissingModule), reason="torch not installed") def test_dist_sampler_simple( scratch_dir, batch_size, seeds_per_rank, fanout, equal_input_size, seeds_per_call @@ -304,7 +303,6 @@ def run_test_dist_sampler_buffered_in_memory( @pytest.mark.skipif(isinstance(torch, MissingModule), reason="torch not available") @pytest.mark.parametrize("seeds_per_call", [4, 5, 10]) @pytest.mark.parametrize("compression", ["COO", "CSR"]) -@pytest.mark.skip(reason="bleh") def test_dist_sampler_buffered_in_memory(scratch_dir, seeds_per_call, compression): uid = cugraph_comms_create_unique_id() From eff502541a8c00721310b63b8f72b6c5aaaea5bf Mon Sep 17 00:00:00 2001 From: Alexandria Barghi Date: Thu, 2 Jan 2025 09:18:16 -0800 Subject: [PATCH 47/49] update branch --- ci/build_docs.sh | 2 +- ci/release/update-version.sh | 2 +- conda/recipes/libcugraph/meta.yaml | 2 +- cpp/src/c_api/neighbor_sampling.cpp | 2 +- python/cugraph/cugraph/datasets/__init__.py | 2 +- python/cugraph/cugraph/datasets/dataset.py | 2 +- python/cugraph/cugraph/gnn/data_loading/dist_io/writer.py | 2 +- python/cugraph/cugraph/gnn/data_loading/dist_sampler.py | 2 +- python/cugraph/cugraph/testing/__init__.py | 2 +- python/cugraph/cugraph/testing/resultset.py | 2 +- python/cugraph/cugraph/tests/conftest.py | 2 +- .../tests/data_store/test_gnn_feat_storage_wholegraph.py | 2 +- python/cugraph/cugraph/tests/sampling/test_bulk_sampler.py | 2 +- python/cugraph/cugraph/tests/sampling/test_bulk_sampler_mg.py | 2 +- python/cugraph/cugraph/tests/sampling/test_dist_sampler.py | 2 +- python/cugraph/cugraph/tests/sampling/test_dist_sampler_mg.py | 2 +- python/cugraph/cugraph/tests/utils/test_dataset.py | 2 +- python/cugraph/cugraph/utilities/utils.py | 2 +- .../pylibcugraph/heterogeneous_biased_neighbor_sample.pyx | 2 +- .../pylibcugraph/heterogeneous_uniform_neighbor_sample.pyx | 2 +- .../pylibcugraph/homogeneous_biased_neighbor_sample.pyx | 2 +- .../pylibcugraph/homogeneous_uniform_neighbor_sample.pyx | 2 +- 22 files changed, 22 insertions(+), 22 deletions(-) diff --git a/ci/build_docs.sh b/ci/build_docs.sh index 46a6005b8dc..331fd0d7fe2 100755 --- a/ci/build_docs.sh +++ b/ci/build_docs.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2023-2024, NVIDIA CORPORATION. +# Copyright (c) 2023-2025, NVIDIA CORPORATION. set -euo pipefail diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh index 2890011bc7f..ac8e2182066 100755 --- a/ci/release/update-version.sh +++ b/ci/release/update-version.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2018-2024, NVIDIA CORPORATION. +# Copyright (c) 2018-2025, NVIDIA CORPORATION. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at diff --git a/conda/recipes/libcugraph/meta.yaml b/conda/recipes/libcugraph/meta.yaml index eb948e08d14..dc2ae36edce 100644 --- a/conda/recipes/libcugraph/meta.yaml +++ b/conda/recipes/libcugraph/meta.yaml @@ -1,4 +1,4 @@ -# Copyright (c) 2018-2024, NVIDIA CORPORATION. +# Copyright (c) 2018-2025, NVIDIA CORPORATION. {% set version = environ['RAPIDS_PACKAGE_VERSION'].lstrip('v') + environ.get('VERSION_SUFFIX', '') %} {% set minor_version = version.split('.')[0] + '.' + version.split('.')[1] %} diff --git a/cpp/src/c_api/neighbor_sampling.cpp b/cpp/src/c_api/neighbor_sampling.cpp index 2cc96463099..927d7d9c769 100644 --- a/cpp/src/c_api/neighbor_sampling.cpp +++ b/cpp/src/c_api/neighbor_sampling.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2024, NVIDIA CORPORATION. + * Copyright (c) 2022-2025, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/python/cugraph/cugraph/datasets/__init__.py b/python/cugraph/cugraph/datasets/__init__.py index ecf10f3c4ef..01ce9cdec78 100644 --- a/python/cugraph/cugraph/datasets/__init__.py +++ b/python/cugraph/cugraph/datasets/__init__.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022-2024, NVIDIA CORPORATION. +# Copyright (c) 2022-2025, NVIDIA CORPORATION. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at diff --git a/python/cugraph/cugraph/datasets/dataset.py b/python/cugraph/cugraph/datasets/dataset.py index 763ae8033f5..cc36ccde6da 100644 --- a/python/cugraph/cugraph/datasets/dataset.py +++ b/python/cugraph/cugraph/datasets/dataset.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022-2024, NVIDIA CORPORATION. +# Copyright (c) 2022-2025, NVIDIA CORPORATION. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at diff --git a/python/cugraph/cugraph/gnn/data_loading/dist_io/writer.py b/python/cugraph/cugraph/gnn/data_loading/dist_io/writer.py index 15ad7e30df3..fe5d169731e 100644 --- a/python/cugraph/cugraph/gnn/data_loading/dist_io/writer.py +++ b/python/cugraph/cugraph/gnn/data_loading/dist_io/writer.py @@ -1,4 +1,4 @@ -# Copyright (c) 2024, NVIDIA CORPORATION. +# Copyright (c) 2024-2025, NVIDIA CORPORATION. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at diff --git a/python/cugraph/cugraph/gnn/data_loading/dist_sampler.py b/python/cugraph/cugraph/gnn/data_loading/dist_sampler.py index 33874733340..2edafe95716 100644 --- a/python/cugraph/cugraph/gnn/data_loading/dist_sampler.py +++ b/python/cugraph/cugraph/gnn/data_loading/dist_sampler.py @@ -1,4 +1,4 @@ -# Copyright (c) 2024, NVIDIA CORPORATION. +# Copyright (c) 2024-2025, NVIDIA CORPORATION. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at diff --git a/python/cugraph/cugraph/testing/__init__.py b/python/cugraph/cugraph/testing/__init__.py index 5c89159bcff..9750eb7cd1f 100644 --- a/python/cugraph/cugraph/testing/__init__.py +++ b/python/cugraph/cugraph/testing/__init__.py @@ -1,4 +1,4 @@ -# Copyright (c) 2023-2024, NVIDIA CORPORATION. +# Copyright (c) 2023-2025, NVIDIA CORPORATION. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at diff --git a/python/cugraph/cugraph/testing/resultset.py b/python/cugraph/cugraph/testing/resultset.py index f7d2521752c..dd1507bb6f3 100644 --- a/python/cugraph/cugraph/testing/resultset.py +++ b/python/cugraph/cugraph/testing/resultset.py @@ -1,4 +1,4 @@ -# Copyright (c) 2023-2024, NVIDIA CORPORATION. +# Copyright (c) 2023-2025, NVIDIA CORPORATION. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at diff --git a/python/cugraph/cugraph/tests/conftest.py b/python/cugraph/cugraph/tests/conftest.py index 101a4e6a192..45f9e11f957 100644 --- a/python/cugraph/cugraph/tests/conftest.py +++ b/python/cugraph/cugraph/tests/conftest.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021-2024, NVIDIA CORPORATION. +# Copyright (c) 2021-2025, NVIDIA CORPORATION. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at diff --git a/python/cugraph/cugraph/tests/data_store/test_gnn_feat_storage_wholegraph.py b/python/cugraph/cugraph/tests/data_store/test_gnn_feat_storage_wholegraph.py index f760ef3e1ba..1403ee3de66 100644 --- a/python/cugraph/cugraph/tests/data_store/test_gnn_feat_storage_wholegraph.py +++ b/python/cugraph/cugraph/tests/data_store/test_gnn_feat_storage_wholegraph.py @@ -1,4 +1,4 @@ -# Copyright (c) 2023-2024, NVIDIA CORPORATION. +# Copyright (c) 2023-2025, NVIDIA CORPORATION. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at diff --git a/python/cugraph/cugraph/tests/sampling/test_bulk_sampler.py b/python/cugraph/cugraph/tests/sampling/test_bulk_sampler.py index 54e159012e2..cffa399c6ca 100644 --- a/python/cugraph/cugraph/tests/sampling/test_bulk_sampler.py +++ b/python/cugraph/cugraph/tests/sampling/test_bulk_sampler.py @@ -1,4 +1,4 @@ -# Copyright (c) 2023-2024, NVIDIA CORPORATION. +# Copyright (c) 2023-2025, NVIDIA CORPORATION. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at diff --git a/python/cugraph/cugraph/tests/sampling/test_bulk_sampler_mg.py b/python/cugraph/cugraph/tests/sampling/test_bulk_sampler_mg.py index f16b78a40a1..fa73ce95792 100644 --- a/python/cugraph/cugraph/tests/sampling/test_bulk_sampler_mg.py +++ b/python/cugraph/cugraph/tests/sampling/test_bulk_sampler_mg.py @@ -1,4 +1,4 @@ -# Copyright (c) 2023-2024, NVIDIA CORPORATION. +# Copyright (c) 2023-2025, NVIDIA CORPORATION. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at diff --git a/python/cugraph/cugraph/tests/sampling/test_dist_sampler.py b/python/cugraph/cugraph/tests/sampling/test_dist_sampler.py index 1f4df8a648d..a30a8f90b80 100644 --- a/python/cugraph/cugraph/tests/sampling/test_dist_sampler.py +++ b/python/cugraph/cugraph/tests/sampling/test_dist_sampler.py @@ -1,4 +1,4 @@ -# Copyright (c) 2024, NVIDIA CORPORATION. +# Copyright (c) 2024-2025, NVIDIA CORPORATION. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at diff --git a/python/cugraph/cugraph/tests/sampling/test_dist_sampler_mg.py b/python/cugraph/cugraph/tests/sampling/test_dist_sampler_mg.py index ee6f64ef9e4..3cc7859b5d9 100644 --- a/python/cugraph/cugraph/tests/sampling/test_dist_sampler_mg.py +++ b/python/cugraph/cugraph/tests/sampling/test_dist_sampler_mg.py @@ -1,4 +1,4 @@ -# Copyright (c) 2024, NVIDIA CORPORATION. +# Copyright (c) 2024-2025, NVIDIA CORPORATION. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at diff --git a/python/cugraph/cugraph/tests/utils/test_dataset.py b/python/cugraph/cugraph/tests/utils/test_dataset.py index 9895eb61c82..524aa526613 100644 --- a/python/cugraph/cugraph/tests/utils/test_dataset.py +++ b/python/cugraph/cugraph/tests/utils/test_dataset.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022-2024, NVIDIA CORPORATION. +# Copyright (c) 2022-2025, NVIDIA CORPORATION. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at diff --git a/python/cugraph/cugraph/utilities/utils.py b/python/cugraph/cugraph/utilities/utils.py index 0257da4ffc0..23e544eca0d 100644 --- a/python/cugraph/cugraph/utilities/utils.py +++ b/python/cugraph/cugraph/utilities/utils.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020-2024, NVIDIA CORPORATION. +# Copyright (c) 2020-2025, NVIDIA CORPORATION. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at diff --git a/python/pylibcugraph/pylibcugraph/heterogeneous_biased_neighbor_sample.pyx b/python/pylibcugraph/pylibcugraph/heterogeneous_biased_neighbor_sample.pyx index 85b070a45cf..44fe0644a64 100644 --- a/python/pylibcugraph/pylibcugraph/heterogeneous_biased_neighbor_sample.pyx +++ b/python/pylibcugraph/pylibcugraph/heterogeneous_biased_neighbor_sample.pyx @@ -1,4 +1,4 @@ -# Copyright (c) 2022-2024, NVIDIA CORPORATION. +# Copyright (c) 2022-2025, NVIDIA CORPORATION. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at diff --git a/python/pylibcugraph/pylibcugraph/heterogeneous_uniform_neighbor_sample.pyx b/python/pylibcugraph/pylibcugraph/heterogeneous_uniform_neighbor_sample.pyx index 8ffa8189aaa..4488815a6d9 100644 --- a/python/pylibcugraph/pylibcugraph/heterogeneous_uniform_neighbor_sample.pyx +++ b/python/pylibcugraph/pylibcugraph/heterogeneous_uniform_neighbor_sample.pyx @@ -1,4 +1,4 @@ -# Copyright (c) 2022-2024, NVIDIA CORPORATION. +# Copyright (c) 2022-2025, NVIDIA CORPORATION. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at diff --git a/python/pylibcugraph/pylibcugraph/homogeneous_biased_neighbor_sample.pyx b/python/pylibcugraph/pylibcugraph/homogeneous_biased_neighbor_sample.pyx index d2cd0504261..8fd88253028 100644 --- a/python/pylibcugraph/pylibcugraph/homogeneous_biased_neighbor_sample.pyx +++ b/python/pylibcugraph/pylibcugraph/homogeneous_biased_neighbor_sample.pyx @@ -1,4 +1,4 @@ -# Copyright (c) 2022-2024, NVIDIA CORPORATION. +# Copyright (c) 2022-2025, NVIDIA CORPORATION. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at diff --git a/python/pylibcugraph/pylibcugraph/homogeneous_uniform_neighbor_sample.pyx b/python/pylibcugraph/pylibcugraph/homogeneous_uniform_neighbor_sample.pyx index b56acc1ee68..14f474c5bd3 100644 --- a/python/pylibcugraph/pylibcugraph/homogeneous_uniform_neighbor_sample.pyx +++ b/python/pylibcugraph/pylibcugraph/homogeneous_uniform_neighbor_sample.pyx @@ -1,4 +1,4 @@ -# Copyright (c) 2022-2024, NVIDIA CORPORATION. +# Copyright (c) 2022-2025, NVIDIA CORPORATION. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at From 4659bd2d9fbaef494b93ebb125f58d457610af5e Mon Sep 17 00:00:00 2001 From: Alexandria Barghi Date: Thu, 2 Jan 2025 09:19:11 -0800 Subject: [PATCH 48/49] fix style --- cpp/CMakeLists.txt | 2 +- cpp/tests/CMakeLists.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index ad30b3769d7..2bdeb76a4d0 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -1,5 +1,5 @@ #============================================================================= -# Copyright (c) 2018-2024, NVIDIA CORPORATION. +# Copyright (c) 2018-2025, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/cpp/tests/CMakeLists.txt b/cpp/tests/CMakeLists.txt index 4cc9f2d94bf..4d508aae81b 100644 --- a/cpp/tests/CMakeLists.txt +++ b/cpp/tests/CMakeLists.txt @@ -1,5 +1,5 @@ #============================================================================= -# Copyright (c) 2019-2024, NVIDIA CORPORATION. +# Copyright (c) 2019-2025, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. From e1e2d34fdded8cffd7d4c61876d460ddb57db2e7 Mon Sep 17 00:00:00 2001 From: Alexandria Barghi Date: Fri, 3 Jan 2025 08:37:53 -0800 Subject: [PATCH 49/49] change copyright --- ci/build_docs.sh | 2 +- ci/release/update-version.sh | 2 +- conda/recipes/libcugraph/meta.yaml | 2 +- cpp/CMakeLists.txt | 2 +- cpp/tests/CMakeLists.txt | 2 +- python/cugraph/cugraph/datasets/__init__.py | 2 +- python/cugraph/cugraph/datasets/dataset.py | 2 +- python/cugraph/cugraph/testing/__init__.py | 2 +- python/cugraph/cugraph/testing/resultset.py | 2 +- python/cugraph/cugraph/tests/conftest.py | 2 +- .../tests/data_store/test_gnn_feat_storage_wholegraph.py | 2 +- python/cugraph/cugraph/tests/utils/test_dataset.py | 2 +- python/cugraph/cugraph/utilities/utils.py | 2 +- 13 files changed, 13 insertions(+), 13 deletions(-) diff --git a/ci/build_docs.sh b/ci/build_docs.sh index 7ca353595c2..938a2b55541 100755 --- a/ci/build_docs.sh +++ b/ci/build_docs.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2023-2025, NVIDIA CORPORATION. +# Copyright (c) 2023-2024, NVIDIA CORPORATION. set -euo pipefail diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh index ac8e2182066..2890011bc7f 100755 --- a/ci/release/update-version.sh +++ b/ci/release/update-version.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2018-2025, NVIDIA CORPORATION. +# Copyright (c) 2018-2024, NVIDIA CORPORATION. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at diff --git a/conda/recipes/libcugraph/meta.yaml b/conda/recipes/libcugraph/meta.yaml index dc2ae36edce..eb948e08d14 100644 --- a/conda/recipes/libcugraph/meta.yaml +++ b/conda/recipes/libcugraph/meta.yaml @@ -1,4 +1,4 @@ -# Copyright (c) 2018-2025, NVIDIA CORPORATION. +# Copyright (c) 2018-2024, NVIDIA CORPORATION. {% set version = environ['RAPIDS_PACKAGE_VERSION'].lstrip('v') + environ.get('VERSION_SUFFIX', '') %} {% set minor_version = version.split('.')[0] + '.' + version.split('.')[1] %} diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 2bdeb76a4d0..ad30b3769d7 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -1,5 +1,5 @@ #============================================================================= -# Copyright (c) 2018-2025, NVIDIA CORPORATION. +# Copyright (c) 2018-2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/cpp/tests/CMakeLists.txt b/cpp/tests/CMakeLists.txt index 4d508aae81b..4cc9f2d94bf 100644 --- a/cpp/tests/CMakeLists.txt +++ b/cpp/tests/CMakeLists.txt @@ -1,5 +1,5 @@ #============================================================================= -# Copyright (c) 2019-2025, NVIDIA CORPORATION. +# Copyright (c) 2019-2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/python/cugraph/cugraph/datasets/__init__.py b/python/cugraph/cugraph/datasets/__init__.py index 01ce9cdec78..ecf10f3c4ef 100644 --- a/python/cugraph/cugraph/datasets/__init__.py +++ b/python/cugraph/cugraph/datasets/__init__.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022-2025, NVIDIA CORPORATION. +# Copyright (c) 2022-2024, NVIDIA CORPORATION. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at diff --git a/python/cugraph/cugraph/datasets/dataset.py b/python/cugraph/cugraph/datasets/dataset.py index cc36ccde6da..763ae8033f5 100644 --- a/python/cugraph/cugraph/datasets/dataset.py +++ b/python/cugraph/cugraph/datasets/dataset.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022-2025, NVIDIA CORPORATION. +# Copyright (c) 2022-2024, NVIDIA CORPORATION. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at diff --git a/python/cugraph/cugraph/testing/__init__.py b/python/cugraph/cugraph/testing/__init__.py index 9750eb7cd1f..5c89159bcff 100644 --- a/python/cugraph/cugraph/testing/__init__.py +++ b/python/cugraph/cugraph/testing/__init__.py @@ -1,4 +1,4 @@ -# Copyright (c) 2023-2025, NVIDIA CORPORATION. +# Copyright (c) 2023-2024, NVIDIA CORPORATION. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at diff --git a/python/cugraph/cugraph/testing/resultset.py b/python/cugraph/cugraph/testing/resultset.py index dd1507bb6f3..f7d2521752c 100644 --- a/python/cugraph/cugraph/testing/resultset.py +++ b/python/cugraph/cugraph/testing/resultset.py @@ -1,4 +1,4 @@ -# Copyright (c) 2023-2025, NVIDIA CORPORATION. +# Copyright (c) 2023-2024, NVIDIA CORPORATION. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at diff --git a/python/cugraph/cugraph/tests/conftest.py b/python/cugraph/cugraph/tests/conftest.py index 45f9e11f957..101a4e6a192 100644 --- a/python/cugraph/cugraph/tests/conftest.py +++ b/python/cugraph/cugraph/tests/conftest.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021-2025, NVIDIA CORPORATION. +# Copyright (c) 2021-2024, NVIDIA CORPORATION. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at diff --git a/python/cugraph/cugraph/tests/data_store/test_gnn_feat_storage_wholegraph.py b/python/cugraph/cugraph/tests/data_store/test_gnn_feat_storage_wholegraph.py index 1403ee3de66..f760ef3e1ba 100644 --- a/python/cugraph/cugraph/tests/data_store/test_gnn_feat_storage_wholegraph.py +++ b/python/cugraph/cugraph/tests/data_store/test_gnn_feat_storage_wholegraph.py @@ -1,4 +1,4 @@ -# Copyright (c) 2023-2025, NVIDIA CORPORATION. +# Copyright (c) 2023-2024, NVIDIA CORPORATION. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at diff --git a/python/cugraph/cugraph/tests/utils/test_dataset.py b/python/cugraph/cugraph/tests/utils/test_dataset.py index 524aa526613..9895eb61c82 100644 --- a/python/cugraph/cugraph/tests/utils/test_dataset.py +++ b/python/cugraph/cugraph/tests/utils/test_dataset.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022-2025, NVIDIA CORPORATION. +# Copyright (c) 2022-2024, NVIDIA CORPORATION. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at diff --git a/python/cugraph/cugraph/utilities/utils.py b/python/cugraph/cugraph/utilities/utils.py index 23e544eca0d..0257da4ffc0 100644 --- a/python/cugraph/cugraph/utilities/utils.py +++ b/python/cugraph/cugraph/utilities/utils.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020-2025, NVIDIA CORPORATION. +# Copyright (c) 2020-2024, NVIDIA CORPORATION. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at