From 7cf59c94801331fc2e8428f7c95a3ad423c82921 Mon Sep 17 00:00:00 2001 From: Richard Yan Date: Fri, 7 Jun 2024 18:11:19 -0700 Subject: [PATCH] dma and demo kernels --- kernel/include/gemmini_mmio.h | 13 +- kernel/linker/vx_link32.ld | 20 +- kernel/src/vx_start.S | 1 + kernel/tohost.S | 7 + tests/regression/bad_apple/.gitignore | 6 + tests/regression/bad_apple/Makefile | 9 + tests/regression/bad_apple/bad_apple | Bin 0 -> 28448 bytes tests/regression/bad_apple/common.h | 18 ++ tests/regression/bad_apple/display.py | 103 +++++++ tests/regression/bad_apple/display.py.bak | 83 ++++++ tests/regression/bad_apple/kernel.cpp | 137 +++++++++ tests/regression/bad_apple/main.cpp | 274 ++++++++++++++++++ tests/regression/common.mk | 3 +- tests/regression/rickroll/.gitignore | 6 + tests/regression/rickroll/Makefile | 9 + tests/regression/rickroll/common.h | 18 ++ tests/regression/rickroll/display_color.py | 128 ++++++++ tests/regression/rickroll/kernel.cpp | 92 ++++++ tests/regression/rickroll/main.cpp | 274 ++++++++++++++++++ tests/regression/rickroll/rickroll | Bin 0 -> 28448 bytes tests/regression/sgemm_gemmini/kernel.cpp | 100 ++++--- tests/regression/sgemm_gemmini_dma/.gitignore | 5 + tests/regression/sgemm_gemmini_dma/Makefile | 9 + tests/regression/sgemm_gemmini_dma/common.h | 18 ++ tests/regression/sgemm_gemmini_dma/kernel.cpp | 175 +++++++++++ tests/regression/sgemm_gemmini_dma/main.cpp | 274 ++++++++++++++++++ .../sgemm_gemmini_dma/sgemm_gemmini_dma | Bin 0 -> 28448 bytes 27 files changed, 1731 insertions(+), 51 deletions(-) create mode 100644 kernel/tohost.S create mode 100644 tests/regression/bad_apple/.gitignore create mode 100644 tests/regression/bad_apple/Makefile create mode 100755 tests/regression/bad_apple/bad_apple create mode 100644 tests/regression/bad_apple/common.h create mode 100644 tests/regression/bad_apple/display.py create mode 100644 tests/regression/bad_apple/display.py.bak create mode 100644 tests/regression/bad_apple/kernel.cpp create mode 100644 tests/regression/bad_apple/main.cpp create mode 100644 tests/regression/rickroll/.gitignore create mode 100644 tests/regression/rickroll/Makefile create mode 100644 tests/regression/rickroll/common.h create mode 100644 tests/regression/rickroll/display_color.py create mode 100644 tests/regression/rickroll/kernel.cpp create mode 100644 tests/regression/rickroll/main.cpp create mode 100755 tests/regression/rickroll/rickroll create mode 100644 tests/regression/sgemm_gemmini_dma/.gitignore create mode 100644 tests/regression/sgemm_gemmini_dma/Makefile create mode 100644 tests/regression/sgemm_gemmini_dma/common.h create mode 100644 tests/regression/sgemm_gemmini_dma/kernel.cpp create mode 100644 tests/regression/sgemm_gemmini_dma/main.cpp create mode 100755 tests/regression/sgemm_gemmini_dma/sgemm_gemmini_dma diff --git a/kernel/include/gemmini_mmio.h b/kernel/include/gemmini_mmio.h index 17b18a79..9cb67ad4 100644 --- a/kernel/include/gemmini_mmio.h +++ b/kernel/include/gemmini_mmio.h @@ -5,7 +5,8 @@ #endif #define SMEM_BASE 0xff000000 -#define SMEM_SIZE 0x4000 +//#define SMEM_SIZE 0x4000 +#define SMEM_SIZE 0x10000 #define SMEM_MASK (SMEM_SIZE - 1) #define SMEM_ADDR_END (SMEM_BASE + SMEM_SIZE) @@ -31,10 +32,13 @@ // 0: k = 0, no accumulation // 1: k % 2 = 0, buffer regions 0 // 2: k % 2 = 1, buffer regions 1 -// 8, 9: memory ops -// 8: tile-sized move-in (unused) +// 8, 9, 10, 11: memory ops +// 8: tile-sized move-in stride // 9: tile-sized move-out +// 10: tile-sized move-in, buffer regions 0 +// 11: tile-sized move-in, buffer regions 1 #define GEMMINI_CISC_CMD_I(x) asm("csrwi 0xacc, "#x) +#define GEMMINI_CISC_CMD_R(x) asm("csrw 0xacc, %0" :: "r" (x)) #define GEMMINI_STATUS() ({uint32_t status; asm volatile ("csrr %0, 0xacc" : "=r" (status)); status;}) // convert normal matrix i,j into tiled smem offset @@ -62,6 +66,9 @@ /* sprintf((char *) PRINT_BUF, "%llx %llx %d\n", rs1, rs2, funct); */ \ } +#define loop_matmul_skips(skip_lda, skip_ldb, skip_ldd, skip_ex, skip_stc) \ + (((skip_lda) | ((skip_ldb) << 1) | ((skip_ldd) << 2) | ((skip_ex) << 3) | ((skip_stc) << 4)) << 3) + #define sp_tiled_matmul_full_spad_ws(A_sp_addr_start, B_sp_addr_start, D_sp_addr_start, C_dst_sp_addr_start,\ I, J, K, pad_I, pad_J, pad_K, a_transpose, b_transpose, full_C, low_D, acc, act, skips) \ gemmini_loop_ws_spad(I, J, K, pad_I, pad_J, pad_K, A_sp_addr_start, (B_sp_addr_start) + (K) * (J) * DIM, NULL, \ diff --git a/kernel/linker/vx_link32.ld b/kernel/linker/vx_link32.ld index ea5c4e56..e84c0342 100644 --- a/kernel/linker/vx_link32.ld +++ b/kernel/linker/vx_link32.ld @@ -10,8 +10,8 @@ ENTRY(_start) MEMORY { DRAM0 (rwx): ORIGIN = 0x80000000, LENGTH = 512M - DRAM1 (rwx): ORIGIN = 0xa0000000, LENGTH = 32K - DRAM2 (rwx): ORIGIN = 0xa1000000, LENGTH = 32K + DRAM1 (rwx): ORIGIN = 0xa0000000, LENGTH = 16M + DRAM2 (rwx): ORIGIN = 0xa1000000, LENGTH = 16M } SECTIONS @@ -69,6 +69,7 @@ SECTIONS /* .gnu.warning sections are handled specially by elf.em. */ *(.gnu.warning) } + .fini : { KEEP (*(SORT_NONE(.fini))) @@ -128,6 +129,7 @@ SECTIONS KEEP (*(.init_array EXCLUDE_FILE (*crtbegin.o *crtbegin?.o *crtend.o *crtend?.o ) .ctors)) PROVIDE_HIDDEN (__init_array_end = .); } + .fini_array : { PROVIDE_HIDDEN (__fini_array_start = .); @@ -135,6 +137,18 @@ SECTIONS KEEP (*(.fini_array EXCLUDE_FILE (*crtbegin.o *crtbegin?.o *crtend.o *crtend?.o ) .dtors)) PROVIDE_HIDDEN (__fini_array_end = .); } + + .htif_pad : { + . = ALIGN(0x1000); + } + + . = ALIGN(0x1000); + .tohost : { + *(.tohost) + /* . += 0x100; */ + } + + . = ALIGN(0x1000); .ctors : { /* gcc uses crtbegin.o to find the start of @@ -216,6 +230,8 @@ SECTIONS MAX(__DATA_BEGIN__ + 0x800, __BSS_END__ - 0x800)); _end = .; PROVIDE (end = .); . = DATA_SEGMENT_END (.); + + /* Stabs debugging sections. */ .stab 0 : { *(.stab) } .stabstr 0 : { *(.stabstr) } diff --git a/kernel/src/vx_start.S b/kernel/src/vx_start.S index 49e520b6..65dbb9a6 100644 --- a/kernel/src/vx_start.S +++ b/kernel/src/vx_start.S @@ -150,3 +150,4 @@ vx_wspawn_wait: .weak __dso_handle __dso_handle: .long 0 + diff --git a/kernel/tohost.S b/kernel/tohost.S new file mode 100644 index 00000000..cfb4ba25 --- /dev/null +++ b/kernel/tohost.S @@ -0,0 +1,7 @@ +.section ".tohost","aw",@progbits +.align 6 +.globl tohost +tohost: .dword 0 +.align 6 +.globl fromhost +fromhost: .dword 0 diff --git a/tests/regression/bad_apple/.gitignore b/tests/regression/bad_apple/.gitignore new file mode 100644 index 00000000..0322feda --- /dev/null +++ b/tests/regression/bad_apple/.gitignore @@ -0,0 +1,6 @@ +*.bin +*.dump +*.elf +sgemm_wg +.depend +kernel.radiance.elf diff --git a/tests/regression/bad_apple/Makefile b/tests/regression/bad_apple/Makefile new file mode 100644 index 00000000..e83a1a8e --- /dev/null +++ b/tests/regression/bad_apple/Makefile @@ -0,0 +1,9 @@ +PROJECT = bad_apple + +SRCS = main.cpp common.h + +VX_SRCS = kernel.cpp + +OPTS ?= -n16 + +include ../common.mk diff --git a/tests/regression/bad_apple/bad_apple b/tests/regression/bad_apple/bad_apple new file mode 100755 index 0000000000000000000000000000000000000000..67ade61b5a3759e630c5a613c78401d05b717369 GIT binary patch literal 28448 zcmeHwdwf*Ywg1UuBqC&jk~S)p5rZZIF#&=J?MNmMIHMCuAt+k$WHOnM)MO@|2Lyjw zqaoV4(`oV6TKlW^hppbLZT$(owL+{IKnV3#M68P91FPtXh+q&zeB^%Dew>^+%=BY> z@9*~)`}4_Jd#%0J+H0@zCP+CZkf!0?b;)p#D0k5qPM31zE|O~!a!o=` zLWl6Dgwmg6BmGqfy;kTz8U+V?DvDKZsgSe6TEQuX zD-z25z709@%U2De+Lc8jzjXDmL6loUY9|yf>1c1BGy9T`mKh!GfoS)P?ut1x=FBb$ z2TNwMbh3-`pf~@v!n|uUF1jCMrauK;3kge`n{^&T%!r|J5gFhWb3`=HD}}h9#zW zJckU5e>cWnMRPaSYc2kee|dX2;tw^}*K`B}{zh+ehhNdO z<(*PJgH7?dS;l2w`chpjdpbey)D`e z{!qZ*p|!RnFR+FD-j-F`N^g6FFeuV|9l3a->T{DI|>Hq9Rj1w##2 z_$Z^r!H#fUJL8+}543c+eTySyWxh6VNQ;DESr`V?Edp&;vp3xC(}Lj$%b3Sj*dL0xJBdHocydCEIU!Qf9t>;H>7VTmv`1>o>Q-HqS9FBk zU#%{Dji`xojn`;kWhXP<;@}$T;F+_fKZDVT7Hri*-oSFd*4i0?{Ef_uvwcY7O1@GV zMCDxxYrjg7VG6RMCZ)O21{R{(b_QGg)O45od|H@VZkeJj39M`nv}lX`;bZ*2YWhJwe z1s-=bP>^Sr%q&ep%gU0_N;Age+RT#KMyzZ!I&P-@FS{2?GTH+A5z;lfoAI79IW zkql@$2s))V;qQeQlV@ZWXk-I&Inrra;^=sE4DQ+a+T^_@+4rm{iklj|nA zUQx%>D2zjE1VJ5n&_x_BRpiHqq>c-&O{eg$kH~L=#;k%1136K--v`ACc2y>P|_|F z-Kcxe_n7EtKt|YSqK`GGilUq7=paToY@&}dsETsjL`Q=j4WhotMw;WvkYl3TB@r;s zM5q2HLxG7tK@tHSCc1e(SY)D$Ee|GDYNFHnQ-%r?Jzo+5t4#E-ndqvC{&f?*!9*`G z(U+R&XPf9vCi*!hdYg%Uu8H1dqE9l>yG`_OnCLww`gtb$4@~sQCi)r^-C?5NW1^pL zqOUd4r zPI=&z2Tpn5lm||E;J@pEBlh!7sJ$QLsBzmX^Ke|#74-b9Mz-i$>JmfWSTJR4KecQlkp+7w2GH_bp4-b_YI4$glht4-} zTF?&_{)oiY*qeGA1_Cv9NFNYzDWE6zp1ux{#y{89uB%*2T#c^S=S$SY zj9bse&tufs5%ty&iZGJ3sPXKnw6&wgRu>k4P&S~(-dAI9)Ivo)+Ia8rkw{bT@f#vn z+%lo}_`K+JeI*90vJrhPI$-bdK-8heop-^Yej3-xj_DgfkJ)~ME^*+o-s5eN=k)my z=zVTP4y8!xW+=Zd*5J&DeL1kU+J$0JxXN=(7heK*L{*Cp7#cB(4)M?+i4{^N> zCU|0B!n#f1&{x5_etKm<$9(8eW82i&pk7UeqSj7)Mh2J9LP|dk>=3(y#HO%(!`y2fCG#TLg+(%C+9w7aQ_;LGv1F^7xeCXtanFFthzv-KzYKk51)X)S=A-VCy+vY+ge=Rb>>SnEf;_f{K^n0yT#64eIqwLl#D@O>D?v)%dEK zxUMiy)@3HuWgQHM)OUk+Y;|D~h6KYR5&9t8;o|A zi>(cDWVR7nka8_yS#5f}PL2h21-cC?rH__k`YOiUoy?3npocuM4@J9G6Td6WL+vLy z>FHz4Nrg;yP_H9>!bvk2?U4R$(8x*Xjc90pbM?x7C0!^Cm5r1 zBE^HSAU?hxT54nO#`CL4Yr%Lcf>Cr6<2 zS7^isehH=bbGhAA+Aomvfrr_mHnweO9{fOYckJn*8WBfJ9l8i{`{W)N%hnTY{hM$u zqq5ygw#1#U<18IzdzXRo8cGR4HApXT(nOFj$e#{C0j_DrD-o0tn9nGKw$DXWzm$vK z!$miO495Q}857Qnxab3tGHAPBihhowBG+YH^m;D3STZJ@A7CvDWo}6sw9S{Io4M!| z5`|rG53?&#ejHIyjv5r_cCO`sr1UV$%vk$rrAE9I$rdz|~rP_*c@C2Ftp1pQh= zjqQ(&Lp5FqF@57lXe});al7qF@MSE6tYVfQT8Tty5B5U(X@39>>m5uNVc#{R5#h}v$%1{sZSW}EU??c9F-$iK?!VqbkXMk$wl(~25suLiGv^I^zg-eDt`#GQXcThq@*oZk5Tkn2AST9DBj)WQ?a zT#|X1@!B#|<#okqUc(qaWL_`LfZPO;*Ut&YotGo8P0Do_+VnOMD8tJrLrk41!&`ZV z_mN-)&+tyfPb%E2R3cuuVpHM#B)9euFI*AyyW>|oP*+c)ZeycoPhmvP`1P z$u)9T?BJO@NXZk-THC}_nJ*j7JYp)XJ{@uwh|GUWdAyHhP&ION*&qFD-){Tr6&Pr1 z6U`%aWyk8uP}MW2e;haq?b@AKkWpXxo_+O|U~$I=)i~b9@%$M`SBtl(n@?nVDo@yN z-;L~!RNM2ipP=W&4fdiPN9@-HE50nd2+8#F_l&L}*sSxp58=>6`T3-TOvCdgnxNAh z?1Y>i&{~7NNcA2lJz~#;el-q0STvxIgFE|aKM#7|1|&T+Yt$EWW|+j7TkVGiAh1Q> zOP1k9g3Qpc-SIzVP{T%Fw6A^xZ;&3Wfg;uRO)G0WvFBX9+cWgv9OSh=6UIt!_Wd5K z;kv$K(I(70C!re_nckRez3qv4kTVrWe}zg(27BT}ON_6;YkU)1AjHSOh*HcdB6pHN z&d4s}$+2%be4vMqYjPob7L_gGeEniHg>I(M_B`UXw6b&#B9_X(c9gY0hdbvLWGH53 z6-=JmL6z=_KOlPcJNDJDqm8(FZYrd%{UeIoH$@6-*!=1P?t~d|hMZxnA&m*=<;Yy$ zPfm*4E@Os}TYty2s(lCTtM7oOo|{Q$4r2CAQ5r2KlGH+>^<}_8I1e?6~VH`=i@w(+e*YdYANmfKEDoJ8BuD zdTIiAXQ!}DFeTQZO|!Sfha|1U)QUA==Du8h5| zj1Myo)mFmt=r7R83z9ndYS^kMqyjlXOQ_$28BxL+!lWPGTzml4m&m?dt=w#1JqZrQ zBL8vB4uz-!X^*EQ}E*dn2aokuk`zkMZVT&qWpj*dKiz@BccB@K*aB zAqd-V^WX{PlwP(M-p1x+<@kBPJc)8_UMSH8L$tEN;Ka+1Az~#PA$oNx;-G%U5Osoa zg?P#2KrB&%b<)KMGC*_mx<598v8T55Os(O=WMQ3E3OEC4;W*4HbutC1ser@&!{bNvRxZ}gTC;lff%3+xQ1C4An z&SQpvy70tC^f^q&7%?bUm3k@<+gD%66M1(!8wky(;G5I2r$N3s2H$*hd=@a&QVWzu zYle=ZK&&0|nF{*&4exQk907F4>!Bcj*F~d#+5Y$ZQiJh8`sM9H;g|J#HypvX6rDRk z)ML;r{gOi?2=~h(aM3H+?3Z==Ypj|^{bHKe-3#0K)JOQ`6a74(a(-RB)r0Lt?4E{@Jv-xJ{!?RLV!Ts} zx9isuy#f;YIuw-V&#}W>4gGvRJ&)$ndLb*)~?PgJl1acB++c*l)WFlAg*V^|7NKd(DCRjHB+#XFc`>L+;J{GvOtD z9_**Cl*leBI|vulVmCMYA)enhSNz3bSJVOt&epS=A||Yt>q2s5ZjOw?Kj=GAM0nV>YYEqJ3@c%9rLT1<?N8~JPBpkJ#c4iKX4x_KgK6| zFaH#NBENMa{lLG$Up~Ak_5bsBN`JYo_9UEpu~qN73Tj)ZV%ZO2W~O zh{C>Vb9f^HcU+*%b$ES|sJFwZOn3OZyZpY0zs0G{4@Ltm4#mOv!{;mWU2acZ?fHsp zt^*P)yd6=$Q>g|Ln0v$Awt4tO$R7}efnk9F)R1A~i#r&D+uv(!9>n?oq5c3G)$VD1cv=w+EI> zcZhuWM}`i3)`ah-0uhv*%y9~9#wth%mo(#BruG0nZ!7Vd;>|4X>+pL+uLgu(|YZ~gb1)hb~F3)tw zg+iB6I4{$fR$uc!s;@b{zLs$0AJyg)<>k(y1}od9Xt|C^P;`dr0y^Lr2i2QD9Cix# z@(w=NK_B>G#DEOznX);vfrCE5L#ox3;ayj4 z?ILs&-kp+~5!H9FwP@7gTBKMM$7qih{iD3R<$2R+PO50qIqIKqFdFjt9WF^ z{)+xdQ(mh*&FSqKZ73=2ss4Xr4|--Q&zl<>_c(?hjfNjnZJ@m*wMSD~TiQFd&ZKdG zD1l;gw{P*cdhzPeyj>W@TVy-Kgya?(Czv?+*xFCFue&Vc+^lat0G#$CD_$8H*#o#? z@5l)K5@H8n0bs=&BO?>D6=f;l8Z1=`FdLo@yaKC*0q_@L!Q279Zot)mI^ey4D)twf zuvWSUo2b77r1OkY(02gV5+C4Fz#JSq=){jhspH`5rsFo~*8$f6Qkckrl5yiAC8Ikp zJ()MEe8& zeqf@J&K*RDtqp)mZEjxXg4`UU@5A2*pnCz8#@xJ{b8{3G(c}25g`XR-&{O8;=KVO+ zlUwkktm<4xBHNu?^g~;1Zt3l(sks%sW9oC8vSwzD&8=|dmb!9_z*3!CP@S7sotrcN z3@Quxya4u;0V+#K7Z`-DYD1TcbX}fXm06QYlY8s-zdXD1IW@Pccg*dl{m_=k{!!MC zGc#{YnQiSEtPFobe*3W3R!{k|GSpJObyNnHFATo@G-TI1W`1r%=I)fb++0@<>f#-2 z=Q(J}%yiWymq9h@s>-@IGo`Z2nLiq#ONBqqC%djm_D3E0gV`nh@x7Fq7MzjkP35P$ zlz!rE6}h1uH+Wb}9`lPwa87rwY|qp#sO=7*-EBM_UjUDRajcxF>WbC-<+=ODW&J7# z=GB~3@h;jD#;!99raUW0^MtgyD#1zZaW?)Y!3GCl_7*%4=p6Y6@cTQM&Ywpt0i+edpMJ!-&2FQfnLkI{>Vn*=Kq|MfZ>>Loo(AZ_8r}#aBN^Xp{1-Un zv(qpB=~>F~*c~w7rE%jQIZOHVcxqtBa>w6)mhww`4}>?KX+!*_Gi^xy_{_=Rhj{iG zDD(gSfa56^*x-bEE#)Y=kAf8*g%ReDn=D1soOd;NSo!D=h56kCiFNaY%kA zPRBnKXk4OzUxKh8zYC`^lmgWyg`;1N1PQJc`N_IiD;Q`vqOeD#^Vx}_JT1m+YIhW< z%}}uVUWh3@*Yk8a#LDtg8>Aq|*spU`$q@yR^tHJ>LE7CW9EFy`g3SLsflEH@JhSqF zosqEX$naglk$7J+T-u4_6^6Hn^iKsY{UX%=j{$kFnN?w_gHx&mY!I+Xz%BuM1Y9HF zS^?JyI3VCI0rv@bSU_B+#X_Ed4gpIAtP-$6z$O8^1nd!Tjeu(fTqodwfV%|TC*WZL z<)u1#Q@9-t0ZRp}60kwQCIPzy>=AH{fLzNz{>*9OGFjEhvbZy2uA^v4b2Jc%Ix2BX zM(K>QD8tIWKeM!?6xU`Hb9%B|Dl7}VyFfNH6GImEASzPQvy`PN>DkInDd{$&ADh(U zc8b%C{+mo6qsV@qOwUnd`%k8yuE>6qOdqSr{+Uca!|2D!bc|{aGlj>qaI2ytB|TSJ zo1%WR6uYu6B|T4ZOiiVKf+ELFb7~gdG*xA$@OTzx%kq9cb7~geoNE@CDUv^5S#w1y z`qvaWKATgsl&>ptTu!DJD5=JUER6PY98KoODBonJ@OYN#{wZ^6mT{}(C^ZAoY~`bo zQ3{XIc%O;4NnRPpY2HGC-ZMm(es&NMf!t1XdASCgKY&(+zYIuq_z+=n4i^HX%9nP* zNl!y5=aWyG!xO#7LZ?BS=<>ZQ`TG!|eAhK|3YC+>UHHpL;r}NS-(~z0l^>>P52?!gF=djXy)DKIFq38h4#r?SXu62e2r>e)vjGm`73IBM7z_c{{m82hbw)r@t^z%H>&quxmO`I<6{7xD@ zy=myzLh#JYlgI5}rs4lB=wwfhFFD`egPyAW{AU_He@#PwI}M%IQ4`P(^DN~(Ci1o3 z_csBKO6C7)pp*T1K5oBkhqKf0)BZ^+J%#8O)IJ?zoV%FklsRemf0BlNb{c!0PQyP$`NE$)B40i}B6sE=X~if!4|E!T6u~c3 zCSlyjgPtx?FOohjjh=aF=(m7Q?XdJRu7Qusit=z8{{5g+zE;0|l!m_$?~7FJ!OiIC z*J9o17X`Qmbn?$(G0w>L=1-$1A^01FA0+=hY4{&WL*EH{s`tfvj34j!N^YPdi$fcY z1HxuAm;}1ROc3#kH2e*Wj`3gk0nLMjCeX>A!{Yr?hWe)PFKP7LoreBc8u}K{9ZX~L zHhVj? z792!{HE*04G0`}}fxFk}z zW8wVyi|cSFsH@sjM_Na(XsoJY7bN1ML%PS1T^7o()f5T*=0twmb+^{l z)y`x$Va_aH?hp7wI5NeJsW|ub2Yh}FCnKHFj_-c=yW&E}(QN4IM9oj7s`)vWWrig4bvVQ5J zQ!w-PnS3Tj7klE?&(Tuj#!buRuI9^j&DV#H%24}4K}CCD7u6)Inm9$mRz*va9EUrob~sw15T2q+ih8oOTN09ZWd1BL-SzQ=K8`> z_QIgw1JUnP=wO{GFy9s}FNIFhfSTvOgJ5GWIxXGCD>$ULQ(OUERb{%CH)(8;FBBRZ zv}|O zlmvnif64Mdv;;qT!ZqEIRVGq%v>pEvqrF9taEDN=eJA0J7jP65=WN ztyh?ScE#8Pq9HBh@9>g!oy)qXFibjBv-_Xv3jcL;;YN8~#x zFV|)5ppl4_m;3z^%5yxhuweE7_dumHGAS?51te@0a?*auC*k*zPG@csm*)ot6!K(0 z*(vQ8WF>(J6;aB|a|Q|JIiSp6%FFW87ZxO6jXw(VTv0-K4oUK~XK9tcA2jlb)UP;z zP;js(R%pq=+J5duvRU5Y;QSKy$btzq*g0oG!hZ*5mR~!Cr%NdLIFZ3NIY>E4dkPVA{_^}qLV2Do`K|ujCgcq_RS8Z3 zb)#Z9-J1WiAXENQUY_Sj_y89(p4R;LTIA)qkA&6^FWZk)xZf%-&I9B*(E`aRAlYgT zp8%nwZ_(|Ib0qoyoUG-Q`Ae7ug?5v?Ja1a+;kigYj*Yf|Nw-6uTrBgK=Th?jb2)@# zssB=ta?-Bff<|SM^78zu=St#MlpafgrM!gnyEL=B(!|8@!x0W00+m3@Nu17~h$i!| zYT_vlAzvyJgg}*m5?mniFO>uVrTjDtoouCGhO+JP6#i@w`FD#Sc+l?_X5mlTFUK!B kRfBuzjhmHZ(;=a_C0u%OC+0X9c<#Q*>R literal 0 HcmV?d00001 diff --git a/tests/regression/bad_apple/common.h b/tests/regression/bad_apple/common.h new file mode 100644 index 00000000..74941562 --- /dev/null +++ b/tests/regression/bad_apple/common.h @@ -0,0 +1,18 @@ +#ifndef _COMMON_H_ +#define _COMMON_H_ + +#include + +#define KERNEL_ARG_DEV_MEM_ADDR 0x7fff0000 +#define DEV_SMEM_START_ADDR 0xff000000 + +typedef struct { + uint32_t dim_m; + uint32_t dim_n; + uint32_t dim_k; + uint64_t addr_a; + uint64_t addr_b; + uint64_t addr_c; +} kernel_arg_t; + +#endif diff --git a/tests/regression/bad_apple/display.py b/tests/regression/bad_apple/display.py new file mode 100644 index 00000000..13b7885b --- /dev/null +++ b/tests/regression/bad_apple/display.py @@ -0,0 +1,103 @@ +import time +import os +import struct +from PIL import Image +from io import BytesIO +import subprocess +import numpy as np +import base64 +import cv2 + +use_fpga = False +buffer_width = 16 +buffer_depth = 1350 +frame_width = 480 +frame_height = 360 +upscale = 3 + +# buffer_width = 16 +# buffer_depth = 0x152 +# frame_width = 240 +# frame_height = 180 +# upscale = 6 + +buffer_size = buffer_depth * buffer_width +truncate = 300 if use_fpga else 0 + +def follow(filename): + frame_count = 0 + with open(filename, "r") as file: + while True: + line = file.readline() + if not line: + time.sleep(0.001) + continue + if truncate and (" 0151 " in line): + frame_count += 1 + if frame_count == truncate: + with open(filename, "w") as f: + f.truncate(0) + frame_count = 0 + yield line + +def process_frame(frame_data): + bits = np.unpackbits(np.frombuffer(frame_data, dtype=np.uint8)) + bits = bits[:frame_width * frame_height] + image_array = bits.reshape((frame_height, frame_width)) + # image_array = np.flipud(np.fliplr(image_array)) + image_array = (image_array * 255).astype(np.uint8) + filtered_image_array = cv2.fastNlMeansDenoising(image_array, None, h=72, templateWindowSize=8, searchWindowSize=8) + # filtered_image_array = cv2.GaussianBlur(image_array, (3, 3), 0) + # filtered_image_array = image_array + + filtered_image_array = np.kron(filtered_image_array, np.ones((upscale, upscale), dtype=np.uint8)) + + image = Image.fromarray(filtered_image_array, mode='L') + + return image + +def display_image(img): + with BytesIO() as output: + # img = img.resize((frame_width * upscale, frame_height * upscale), Image.NEAREST) + img.save(output, format='PNG') + + output.seek(0) + # subprocess.run(["/home/eecs/yrh/.iterm2/imgcat"], input=output.read()) + image_data = output.getvalue() + b64_image_data = base64.b64encode(image_data).decode('utf-8') + + print("\033]", end='') + print(f"1337;File=inline=1", end='') + print(f";size={len(image_data)}", end='') + print(f";name={base64.b64encode('tmp.png'.encode()).decode('utf-8')}", end='') + # print(f";width={frame_width * upscale * 4};height={frame_height * upscale * 4}", end='') + print(f":{b64_image_data}", end='') + print("\a", end='') + print('\n') + +def main(): + if not use_fpga: + filename = "/scratch/yrh/chipyard/sims/vcs/output/chipyard.harness.TestHarness.RadianceClusterConfig/kernel.radiance.out" + else: + filename = "/scratch/yrh/chipyard/sims/firesim/sim/generated-src/xilinx_alveo_u250/xilinx_alveo_u250-firesim-FireSim-FireSimRadianceClusterSynConfig-WithPrintfSynthesis_BaseXilinxAlveoU250Config/synthesized-prints.out0" + # frame_data = {} + frame_data0 = bytearray(buffer_size) + for line in follow(filename): + if not "fb0" in line: + continue + tokens = line.split() + if not len(tokens) == (5 if use_fpga else 3): + continue + offset, data = tokens[3 if use_fpga else 1:] + offset0 = int(offset, 16) + + frame_data0[offset0 * buffer_width : (offset0 + 1) * buffer_width] = bytes.fromhex(data)[::-1] + + if offset0 == buffer_depth - 1: + img = process_frame(frame_data0) + display_image(img) + frame_data0 = bytearray(buffer_size) + +if __name__ == "__main__": + main() + diff --git a/tests/regression/bad_apple/display.py.bak b/tests/regression/bad_apple/display.py.bak new file mode 100644 index 00000000..9beee99d --- /dev/null +++ b/tests/regression/bad_apple/display.py.bak @@ -0,0 +1,83 @@ +import time +import os +import struct +from PIL import Image +from io import BytesIO +import subprocess + +use_fpga = False + +def follow(filename): + with open(filename, "r") as file: + # file.seek(0, os.SEEK_END) # Move to the end of the file + while True: + line = file.readline() + if not line: + time.sleep(0.001) + continue + yield line.strip() + +def process_frame(frame_data): + # Create a byte array from the frame data + byte_array = bytearray() + # for offset, data in frame_data.items(): + # byte_array += struct.pack("> bit_index) & 1 + + return img + +def display_image(img): + with BytesIO() as output: + img = img.resize((120, 90), Image.NEAREST) + img.save(output, format='PNG') + output.seek(0) + subprocess.run(["/home/eecs/yrh/.iterm2/imgcat", "-H", "98%"], input=output.read()) + +def main(): + if not use_fpga: + filename = "/scratch/yrh/chipyard/sims/vcs/output/chipyard.harness.TestHarness.RadianceClusterConfig/kernel.radiance.out" + else: + filename = "/scratch/yrh/chipyard/sims/firesim/sim/generated-src/xilinx_alveo_u250/xilinx_alveo_u250-firesim-FireSim-FireSimRadianceClusterSynConfig-WithPrintfSynthesis_BaseXilinxAlveoU250Config/synthesized-prints.out0" + # frame_data = {} + frame_data0 = [0 for _ in range(80)] + frame_data1 = [0 for _ in range(80)] + for line in follow(filename): + if not "fb0" in line: + continue + tokens = line.split() + if not len(tokens) == 7 if use_fpga else 5: + continue + offset, data = tokens[4 if use_fpga else 2:-1] + offset = int(offset, 16) - 0xff010000 + offset0 = offset + offset1 = offset - 0x200 + + if offset0 >= 0 and offset0 < 320: + frame_data0[offset0 // 4] = int(data, 16) + + if offset1 >= 0 and offset1 < 320: + frame_data1[offset1 // 4] = int(data, 16) + + if offset0 == 0x130 and data == "ff010130": + img = process_frame(frame_data0) + frame_data0 = [0 for _ in range(80)] + display_image(img) + elif offset1 == 0x130 and data == "ff010330": + img = process_frame(frame_data1) + frame_data1 = [0 for _ in range(80)] + display_image(img) + +if __name__ == "__main__": + main() + diff --git a/tests/regression/bad_apple/kernel.cpp b/tests/regression/bad_apple/kernel.cpp new file mode 100644 index 00000000..2917f40f --- /dev/null +++ b/tests/regression/bad_apple/kernel.cpp @@ -0,0 +1,137 @@ +#include +#include +#include +#include +#include "common.h" + +#define rd_cycles(x) asm volatile ("csrr %0, mcycle" : "=r" (x)) +#define HW_TID() ({uint32_t gtid; asm volatile ("csrr %0, mhartid" : "=r" (gtid)); gtid;}) +#define PRINTF(...) sprintf((char *) (0xff010000UL), __VA_ARGS__) + +inline void threadblock_barrier(unsigned int barrier_id, unsigned int count) { + vx_fence(); + vx_barrier(barrier_id, count); +} + +void kernel_body(int task_id, kernel_arg_t *__UNIFORM__ arg) { + vx_tmc(0xff); + const volatile uint32_t *const A = (const volatile uint32_t *const) arg->addr_a; + + // vx_tmc(1); + // for (int i = 0; i < 75; i++) { + // if (task_id == i) { + // PRINTF("%d", task_id); + // } + // } + // threadblock_barrier(2, 5); + + /* + #define mark(x) \ + vx_tmc(0x80); \ + if (task_id == 79) *(((volatile uint32_t *) x)) = x; \ + vx_fence(); \ + vx_barrier(x & 1, 5); \ + vx_tmc(0xff) + + #define write_fb0(value) *(((volatile uint32_t *) 0xff010000UL) + task_id) = (value) + #define write_fb1(value) *(((volatile uint32_t *) 0xff010200UL) + task_id) = (value) + + while (true) { + uint32_t v0, v1, v2, v3; + v0 = A[task_id]; + v1 = A[1 * 75 + task_id]; + v2 = A[2 * 75 + task_id]; + v3 = A[3 * 75 + task_id]; + for (int i = 30; i < 6569; i += 4) { + write_fb1(v0); + v0 = A[(i + 0) * 75 + task_id]; + mark(0xff010130); + + write_fb0(v1); + v1 = A[(i + 1) * 75 + task_id]; + mark(0xff010330); + + write_fb1(v2); + v2 = A[(i + 2) * 75 + task_id]; + mark(0xff010130); + + write_fb0(v3); + v3 = A[(i + 3) * 75 + task_id]; + mark(0xff010330); + } + } + */ + + #define WORKERS 128 + + // #define WORDS 1350 + // #define LINES 338 + // #define ITERS 11 // = 1350 / 128 + + #define WORDS 5400 + #define LINES 1350 + #define T_ITERS 43 + + #define mark_fb0() \ + vx_tmc(0x80); if (task_id == 127) *(((volatile uint32_t *) 0xff011000UL)) = LINES; \ + vx_fence(); vx_barrier(0, 8); vx_tmc(0xff) + #define mark_fb1() \ + vx_tmc(0x80); if (task_id == 127) *(((volatile uint32_t *) 0xff011004UL)) = LINES; \ + vx_fence(); vx_barrier(1, 8); vx_tmc(0xff) + #define write_fb0(addr, value) *(((volatile uint32_t *) 0xff018000UL) + addr) = (value) + #define write_fb1(addr, value) *(((volatile uint32_t *) 0xff020000UL) + addr) = (value) + + #define CYCLES_TO_WAIT 240000 + + uint64_t cycles0, cycles1; + cycles0 = 0; + + while (true) { + volatile uint32_t v0, v1; + for (int i = 20; i < 6569; i += 1) { + v0 = A[i * WORDS + task_id]; + v1 = A[i * WORDS + WORKERS + task_id]; + int offset0 = 0 * WORKERS + task_id; + int offset1 = 1 * WORKERS + task_id; + + for (int j = 1; j < T_ITERS; j += 2) { + write_fb0(offset0, v0); + offset0 += 2 * WORKERS; + v0 = A[(i + 0) * WORDS + offset0]; + write_fb0(offset1, v1); + offset1 += 2 * WORKERS; + v1 = A[(i + 0) * WORDS + offset1]; + } + write_fb0(offset0, v0); + write_fb0(offset1, v1); + + /*offset0 += 2 * WORKERS; + v0 = A[(i + 0) * WORDS + offset0]; + write_fb0(offset0, v0);*/ + + if (task_id == 0) { + rd_cycles(cycles1); + while (cycles1 - cycles0 < CYCLES_TO_WAIT) { + rd_cycles(cycles1); + } + cycles0 = cycles1; + } + + threadblock_barrier(0, 8); + mark_fb0(); + } + } +} + +int main() { + kernel_arg_t *arg = (kernel_arg_t *)KERNEL_ARG_DEV_MEM_ADDR; + +#ifdef RADIANCE + vx_spawn_tasks_cluster(128, (vx_spawn_tasks_cb)kernel_body, arg); +#else + // NOTE: This kernel assumes contiguous thread scheduling for efficient shared + // memory allocation, and therefore does not work with original vx_spawn_tasks + vx_spawn_tasks_contiguous(8, (vx_spawn_tasks_cb)kernel_body, arg); +#endif + return 0; +} \ No newline at end of file diff --git a/tests/regression/bad_apple/main.cpp b/tests/regression/bad_apple/main.cpp new file mode 100644 index 00000000..54531062 --- /dev/null +++ b/tests/regression/bad_apple/main.cpp @@ -0,0 +1,274 @@ +#include +#include +#include +#include +#include +#include +#include "common.h" + +#define RT_CHECK(_expr) \ + do { \ + int _ret = _expr; \ + if (0 == _ret) \ + break; \ + printf("Error: '%s' returned %d!\n", #_expr, (int)_ret); \ + cleanup(); \ + exit(-1); \ + } while (false) + +/////////////////////////////////////////////////////////////////////////////// + +const char* kernel_file = "kernel.bin"; +uint32_t count = 0; + +std::vector src_a_data; +std::vector src_b_data; +std::vector ref_data; + +vx_device_h device = nullptr; +std::vector staging_buf; +kernel_arg_t kernel_arg = {}; + +static void show_usage() { + std::cout << "Vortex Test." << std::endl; + std::cout << "Usage: [-k: kernel] [-n words] [-h: help]" << std::endl; +} + +static void parse_args(int argc, char **argv) { + int c; + while ((c = getopt(argc, argv, "n:k:h?")) != -1) { + switch (c) { + case 'n': + count = atoi(optarg); + break; + case 'k': + kernel_file = optarg; + break; + case 'h': + case '?': { + show_usage(); + exit(0); + } break; + default: + show_usage(); + exit(-1); + } + } +} + +void cleanup() { + if (device) { + vx_mem_free(device, kernel_arg.addr_a); + vx_mem_free(device, kernel_arg.addr_b); + vx_mem_free(device, kernel_arg.addr_c); + vx_dev_close(device); + } +} + +void generate_source_matrix(uint32_t dim_m, uint32_t dim_n, uint32_t dim_k) { + src_a_data.resize(dim_m * dim_k); + src_b_data.resize(dim_k * dim_n); + + for (uint32_t i = 0; i < src_a_data.size(); ++i) { + src_a_data[i] = static_cast(i); + std::cout << "A: " << i << ": value=" << src_a_data[i] << std::endl; + } + for (uint32_t i = 0; i < src_b_data.size(); ++i) { + src_b_data[i] = static_cast(i); + std::cout << "B: " << i << ": value=" << src_b_data[i] << std::endl; + } +} + +void generate_reference_matmul(uint32_t dim_m, uint32_t dim_n, uint32_t dim_k) { + ref_data.resize(dim_m * dim_n); + + for (uint32_t i = 0; i < dim_m; ++i) { + for (uint32_t j = 0; j < dim_n; ++j) { + float ref = 0.0f; + for (uint32_t k = 0; k < dim_k; ++k) { + ref += src_a_data[dim_k * i + k] * src_b_data[dim_n * k + j]; + } + ref_data.at(dim_n * i + j) = ref; + } + } +} + +int run_test(const kernel_arg_t& kernel_arg, + uint32_t buf_size, + uint32_t dim_m, uint32_t dim_n) { + // start device + std::cout << "start device" << std::endl; + RT_CHECK(vx_start(device)); + + // wait for completion + std::cout << "wait for completion" << std::endl; + RT_CHECK(vx_ready_wait(device, VX_MAX_TIMEOUT)); + + // download destination buffer + std::cout << "download destination buffer" << std::endl; + RT_CHECK(vx_copy_from_dev(device, staging_buf.data(), kernel_arg.addr_c, buf_size)); + + // verify result + std::cout << "verify result" << std::endl; + { + int errors = 0; + auto buf_ptr = (float*)staging_buf.data(); + for (uint32_t i = 0; i < dim_m * dim_n; ++i) { + float ref = ref_data.at(i); + float cur = buf_ptr[i]; + if (std::abs((cur - ref) / ref) > 1e-6) { + std::cout << "error at result #" << std::dec << i + << std::hex << ": actual=" << cur << ", expected=" << ref << std::endl; + ++errors; + } + } + if (errors != 0) { + std::cout << "Found " << std::dec << errors << " errors!" << std::endl; + std::cout << "FAILED!" << std::endl; + return 1; + } + } + + return 0; +} + +int main(int argc, char *argv[]) { + // parse command arguments + parse_args(argc, argv); + + if (count == 0) { + count = 1; + } + + std::srand(50); + + // open device connection + std::cout << "open device connection" << std::endl; + RT_CHECK(vx_dev_open(&device)); + + // FIXME: hardcoded + uint32_t dim_m = 64; + uint32_t dim_n = 64; + uint32_t dim_k = 64; + + generate_source_matrix(dim_m, dim_n, dim_k); + generate_reference_matmul(dim_m, dim_n, dim_k); + + uint32_t src_a_buf_size = src_a_data.size() * sizeof(src_a_data[0]); + uint32_t src_b_buf_size = src_b_data.size() * sizeof(src_b_data[0]); + uint32_t dst_buf_size = ref_data.size() * sizeof(src_a_data[0]); + + std::cout << "buffer size: " << dst_buf_size << " bytes" << std::endl; + + // upload program + std::cout << "upload program" << std::endl; + RT_CHECK(vx_upload_kernel_file(device, kernel_file)); + + // allocate device memory + std::cout << "allocate device memory" << std::endl; + RT_CHECK(vx_mem_alloc(device, src_a_buf_size, VX_MEM_TYPE_GLOBAL, &kernel_arg.addr_a)); + RT_CHECK(vx_mem_alloc(device, src_b_buf_size, VX_MEM_TYPE_GLOBAL, &kernel_arg.addr_b)); + RT_CHECK(vx_mem_alloc(device, dst_buf_size, VX_MEM_TYPE_GLOBAL, &kernel_arg.addr_c)); + + kernel_arg.dim_m = dim_m; + kernel_arg.dim_n = dim_n; + kernel_arg.dim_k = dim_k; + + std::cout << "dev_addr_a=0x" << std::hex << kernel_arg.addr_a << std::endl; + std::cout << "dev_addr_b=0x" << std::hex << kernel_arg.addr_b << std::endl; + std::cout << "dev_addr_c=0x" << std::hex << kernel_arg.addr_c << std::endl; + + // allocate staging buffer + { + std::cout << "allocate staging buffer" << std::endl; + uint32_t staging_buf_size = std::max( + src_a_buf_size, + std::max( + src_b_buf_size, + std::max(dst_buf_size, sizeof(kernel_arg_t)))); + staging_buf.resize(staging_buf_size); + } + + // upload kernel argument + { + std::cout << "upload kernel argument" << std::endl; + auto buf_ptr = staging_buf.data(); + kernel_arg.addr_a = (uint64_t) 0x20000; + kernel_arg.addr_b = (uint64_t) 0x28000; + kernel_arg.addr_c = (uint64_t) 0xc0000000ULL; + memcpy(buf_ptr, &kernel_arg, sizeof(kernel_arg_t)); + + std::cout << "uploading argument buffer to device, device mem address=" + << std::hex << KERNEL_ARG_DEV_MEM_ADDR << ", size=" << std::dec + << sizeof(kernel_arg_t) << " bytes\n"; + std::ofstream file("args.bin", std::ios::binary | std::ios::out); + if (!file) { + std::cerr << "error: failed to open args.bin for writing\n"; + exit(EXIT_FAILURE); + } + file.write(reinterpret_cast(staging_buf.data()), + sizeof(kernel_arg_t)); + file.close(); + + RT_CHECK(vx_copy_to_dev(device, KERNEL_ARG_DEV_MEM_ADDR, staging_buf.data(), sizeof(kernel_arg_t))); + } + + // upload source buffer + { + { + auto buf_ptr = staging_buf.data(); + memcpy(buf_ptr, src_a_data.data(), src_a_data.size() * sizeof(float)); + RT_CHECK(vx_copy_to_dev(device, kernel_arg.addr_a, staging_buf.data(), + src_a_buf_size)); + + std::cout << "uploading source A matrix to device, device mem address=" + << std::hex << kernel_arg.addr_a << ", size=" << std::dec + << src_a_buf_size << " bytes\n"; + std::ofstream file("input.a.bin", std::ios::binary | std::ios::out); + if (!file) { + std::cerr << "error: failed to open args.bin for writing\n"; + exit(EXIT_FAILURE); + } + file.write(reinterpret_cast(buf_ptr), src_a_buf_size); + file.close(); + } + { + auto buf_ptr = staging_buf.data(); + memcpy(buf_ptr, src_b_data.data(), src_b_data.size() * sizeof(float)); + RT_CHECK(vx_copy_to_dev(device, kernel_arg.addr_b, staging_buf.data(), + src_b_buf_size)); + + std::cout << "uploading source B matrix to device, device mem address=" + << std::hex << kernel_arg.addr_b << ", size=" << std::dec + << src_b_buf_size << " bytes\n"; + std::ofstream file("input.b.bin", std::ios::binary | std::ios::out); + if (!file) { + std::cerr << "error: failed to open args.bin for writing\n"; + exit(EXIT_FAILURE); + } + file.write(reinterpret_cast(buf_ptr), src_b_buf_size); + file.close(); + } + } + + // clear destination buffer + { + std::cout << "clear destination buffer" << std::endl; + auto buf_ptr = (int32_t*)staging_buf.data(); + for (uint32_t i = 0; i < ref_data.size(); ++i) { + buf_ptr[i] = 0xdeadbeef; + } + RT_CHECK(vx_copy_to_dev(device, kernel_arg.addr_c, staging_buf.data(), dst_buf_size)); + } + + // run tests + std::cout << "run tests" << std::endl; + RT_CHECK(run_test(kernel_arg, dst_buf_size, kernel_arg.dim_m, kernel_arg.dim_n)); + std::cout << "PASSED!" << std::endl; + + // cleanup + std::cout << "cleanup" << std::endl; + cleanup(); + + return 0; +} diff --git a/tests/regression/common.mk b/tests/regression/common.mk index 24a871eb..09a57795 100644 --- a/tests/regression/common.mk +++ b/tests/regression/common.mk @@ -53,7 +53,8 @@ VX_CFLAGS += -mcmodel=medany -fno-rtti -fno-exceptions -nostartfiles -fdata-sect VX_CFLAGS += -I$(VORTEX_KN_PATH)/include -I$(VORTEX_KN_PATH)/../hw -I$(GEMMINI_SW_PATH) VX_CFLAGS += -DNDEBUG -DLLVM_VORTEX -VX_LDFLAGS += -Wl,-Bstatic,--gc-sections,-T,$(VORTEX_KN_PATH)/linker/vx_link$(XLEN).ld,--defsym=STARTUP_ADDR=$(STARTUP_ADDR) $(VORTEX_KN_PATH)/libvortexrt.a +# VX_LDFLAGS += -Wl,-Bstatic,--gc-sections,-T,$(VORTEX_KN_PATH)/linker/vx_link$(XLEN).ld,--defsym=STARTUP_ADDR=$(STARTUP_ADDR) $(VORTEX_KN_PATH)/libvortexrt.a +VX_LDFLAGS += -Wl,-Bstatic,-T,$(VORTEX_KN_PATH)/linker/vx_link$(XLEN).ld,--defsym=STARTUP_ADDR=$(STARTUP_ADDR) $(VORTEX_KN_PATH)/libvortexrt.a $(VORTEX_KN_PATH)/tohost.S CXXFLAGS += -std=c++17 -Wall -Wextra -pedantic -Wfatal-errors CXXFLAGS += -I$(VORTEX_RT_PATH)/include -I$(VORTEX_KN_PATH)/../hw diff --git a/tests/regression/rickroll/.gitignore b/tests/regression/rickroll/.gitignore new file mode 100644 index 00000000..0322feda --- /dev/null +++ b/tests/regression/rickroll/.gitignore @@ -0,0 +1,6 @@ +*.bin +*.dump +*.elf +sgemm_wg +.depend +kernel.radiance.elf diff --git a/tests/regression/rickroll/Makefile b/tests/regression/rickroll/Makefile new file mode 100644 index 00000000..7717878d --- /dev/null +++ b/tests/regression/rickroll/Makefile @@ -0,0 +1,9 @@ +PROJECT = rickroll + +SRCS = main.cpp common.h + +VX_SRCS = kernel.cpp + +OPTS ?= -n16 + +include ../common.mk diff --git a/tests/regression/rickroll/common.h b/tests/regression/rickroll/common.h new file mode 100644 index 00000000..74941562 --- /dev/null +++ b/tests/regression/rickroll/common.h @@ -0,0 +1,18 @@ +#ifndef _COMMON_H_ +#define _COMMON_H_ + +#include + +#define KERNEL_ARG_DEV_MEM_ADDR 0x7fff0000 +#define DEV_SMEM_START_ADDR 0xff000000 + +typedef struct { + uint32_t dim_m; + uint32_t dim_n; + uint32_t dim_k; + uint64_t addr_a; + uint64_t addr_b; + uint64_t addr_c; +} kernel_arg_t; + +#endif diff --git a/tests/regression/rickroll/display_color.py b/tests/regression/rickroll/display_color.py new file mode 100644 index 00000000..aaf6faf7 --- /dev/null +++ b/tests/regression/rickroll/display_color.py @@ -0,0 +1,128 @@ +import time +import os +import struct +from PIL import Image +from io import BytesIO +import subprocess +import numpy as np +import base64 +import cv2 + +use_fpga = False +buffer_width = 16 + +# buffer_depth = 0x152 +# frame_width = 240 +# frame_height = 180 +# upscale = 6 +buffer_depth = 1800 +frame_width = 160 +frame_height = 120 +upscale = 1 + + +buffer_size = int(buffer_depth * buffer_width * 1.5) +truncate = 0 # 300 if use_fpga else 0 + +def follow(filename): + frame_count = 0 + with open(filename, "r") as file: + while True: + current_position = file.tell() + line = file.readline() + if not line: + time.sleep(0.001) + continue + if "fb0" in line and len(line) < (61 if use_fpga else 41): + file.seek(current_position) + time.sleep(0.001) + continue + if truncate and (" 0151 " in line): + frame_count += 1 + if frame_count == truncate: + with open(filename, "w") as f: + f.truncate(0) + frame_count = 0 + yield line + +def process_frame(frame_data): + bits = np.unpackbits(np.frombuffer(frame_data, dtype=np.uint8)) + bits = bits[:frame_width * frame_height] + image_array = bits.reshape((frame_height, frame_width)) + # image_array = np.flipud(np.fliplr(image_array)) + image_array = (image_array * 255).astype(np.uint8) + + raw_array = np.frombuffer(frame_data, dtype=np.uint8) + y_size = frame_width * frame_height + c_size = y_size // 4 + y_array = raw_array[:y_size].reshape((frame_height, frame_width)) + cr_array, cb_array = raw_array[y_size : y_size + c_size].reshape((frame_height // 2, frame_width // 2)), raw_array[y_size + c_size : y_size + 2 * c_size].reshape((frame_height // 2, frame_width // 2)) + + cb_upscaled = cv2.resize(cb_array, (frame_width, frame_height), interpolation=cv2.INTER_LINEAR) + cr_upscaled = cv2.resize(cr_array, (frame_width, frame_height), interpolation=cv2.INTER_LINEAR) + + # Merge the channels back to YCrCb format + ycrcb_frame = cv2.merge((y_array, cb_upscaled, cr_upscaled)) + bgr_frame = cv2.cvtColor(ycrcb_frame, cv2.COLOR_YCrCb2BGR) + is_success, buffer = cv2.imencode(".png", bgr_frame) + io_buf = BytesIO(buffer) + + # filtered_image_array = image_array + + # filtered_image_array = cv2.fastNlMeansDenoising(image_array, None, h=72, templateWindowSize=8, searchWindowSize=8) + # filtered_image_array = cv2.GaussianBlur(image_array, (3, 3), 0) + + # filtered_image_array = cv2.blur(image_array, (2, 2)) + # filtered_image_array = cv2.fastNlMeansDenoising(filtered_image_array, None, h=72, templateWindowSize=4, searchWindowSize=4) + + # filtered_image_array = np.kron(filtered_image_array, np.ones((upscale, upscale), dtype=np.uint8)) + + # image = Image.fromarray(filtered_image_array, mode='L') + + return io_buf + +def display_image(img): + # img = img.resize((frame_width * upscale, frame_height * upscale), Image.NEAREST) + # img.save(output, format='PNG') + output = img + + output.seek(0) + # subprocess.run(["/home/eecs/yrh/.iterm2/imgcat"], input=output.read()) + image_data = output.getvalue() + b64_image_data = base64.b64encode(image_data).decode('utf-8') + + print("\033]", end='') + print(f"1337;File=inline=1", end='') + print(f";size={len(image_data)}", end='') + print(f";name={base64.b64encode('tmp.png'.encode()).decode('utf-8')}", end='') + # print(f";width={frame_width * upscale * 4};height={frame_height * upscale * 4}", end='') + print(f":{b64_image_data}", end='') + print("\a", end='') + print('\n') + +def main(): + if not use_fpga: + filename = "/scratch/yrh/chipyard/sims/vcs/output/chipyard.harness.TestHarness.RadianceClusterConfig/kernel.radiance.out" + else: + filename = "/scratch/yrh/firesim-rundir/sim_slot_0/synthesized-prints.out0" + # frame_data = {} + frame_data0 = bytearray(buffer_size) + for line in follow(filename): + if not "fb0" in line: + continue + tokens = line.split() + if not len(tokens) == (5 if use_fpga else 3): + continue + offset, data = tokens[3 if use_fpga else 1:] + offset0 = int(offset, 16) + + frame_data0[offset0 * buffer_width : (offset0 + 1) * buffer_width] = bytes.fromhex(data)[::-1] + + if offset0 == buffer_depth - 1: + img = process_frame(frame_data0) + display_image(img) + frame_data0 = bytearray(buffer_size) + +if __name__ == "__main__": + main() + diff --git a/tests/regression/rickroll/kernel.cpp b/tests/regression/rickroll/kernel.cpp new file mode 100644 index 00000000..64bc2d37 --- /dev/null +++ b/tests/regression/rickroll/kernel.cpp @@ -0,0 +1,92 @@ +#include +#include +#include +#include +#include "common.h" + +#define rd_cycles(x) asm volatile ("csrr %0, mcycle" : "=r" (x)) +#define HW_TID() ({uint32_t gtid; asm volatile ("csrr %0, mhartid" : "=r" (gtid)); gtid;}) +#define PRINTF(...) sprintf((char *) (0xff010000UL), __VA_ARGS__) + +inline void threadblock_barrier(unsigned int barrier_id, unsigned int count) { + vx_fence(); + vx_barrier(barrier_id, count); +} + +void kernel_body(int task_id, kernel_arg_t *__UNIFORM__ arg) { + vx_tmc(0xff); + const volatile uint32_t *const A = (const volatile uint32_t *const) arg->addr_a; + + #define WORKERS 128 + + // #define WORDS 1350 + // #define LINES 338 + // #define ITERS 11 // = 1350 / 128 + + #define WORDS 7200 + #define LINES 1800 + #define T_ITERS 57 + + #define mark_fb0() \ + vx_tmc(0x80); if (task_id == 127) *(((volatile uint32_t *) 0xff011000UL)) = LINES; \ + vx_fence(); vx_barrier(0, 8); vx_tmc(0xff) + #define mark_fb1() \ + vx_tmc(0x80); if (task_id == 127) *(((volatile uint32_t *) 0xff011004UL)) = LINES; \ + vx_fence(); vx_barrier(1, 8); vx_tmc(0xff) + #define write_fb0(addr, value) *(((volatile uint32_t *) 0xff018000UL) + addr) = (value) + #define write_fb1(addr, value) *(((volatile uint32_t *) 0xff020000UL) + addr) = (value) + + #define CYCLES_TO_WAIT 240000 + + uint64_t cycles0, cycles1; + cycles0 = 0; + + while (true) { + volatile uint32_t v0, v1; + for (int i = 0; i < 5301; i += 1) { + v0 = A[i * WORDS + task_id]; + v1 = A[i * WORDS + WORKERS + task_id]; + int offset0 = 0 * WORKERS + task_id; + int offset1 = 1 * WORKERS + task_id; + + for (int j = 1; j < T_ITERS; j += 2) { + write_fb0(offset0, v0); + offset0 += 2 * WORKERS; + v0 = A[(i + 0) * WORDS + offset0]; + write_fb0(offset1, v1); + offset1 += 2 * WORKERS; + v1 = A[(i + 0) * WORDS + offset1]; + } + write_fb0(offset0, v0); + write_fb0(offset1, v1); + + /*offset0 += 2 * WORKERS; + v0 = A[(i + 0) * WORDS + offset0]; + write_fb0(offset0, v0);*/ + + if (task_id == 0) { + rd_cycles(cycles1); + while (cycles1 - cycles0 < CYCLES_TO_WAIT) { + rd_cycles(cycles1); + } + cycles0 = cycles1; + } + + threadblock_barrier(0, 8); + mark_fb0(); + } + } +} + +int main() { + kernel_arg_t *arg = (kernel_arg_t *)KERNEL_ARG_DEV_MEM_ADDR; + +#ifdef RADIANCE + vx_spawn_tasks_cluster(128, (vx_spawn_tasks_cb)kernel_body, arg); +#else + // NOTE: This kernel assumes contiguous thread scheduling for efficient shared + // memory allocation, and therefore does not work with original vx_spawn_tasks + vx_spawn_tasks_contiguous(8, (vx_spawn_tasks_cb)kernel_body, arg); +#endif + return 0; +} \ No newline at end of file diff --git a/tests/regression/rickroll/main.cpp b/tests/regression/rickroll/main.cpp new file mode 100644 index 00000000..54531062 --- /dev/null +++ b/tests/regression/rickroll/main.cpp @@ -0,0 +1,274 @@ +#include +#include +#include +#include +#include +#include +#include "common.h" + +#define RT_CHECK(_expr) \ + do { \ + int _ret = _expr; \ + if (0 == _ret) \ + break; \ + printf("Error: '%s' returned %d!\n", #_expr, (int)_ret); \ + cleanup(); \ + exit(-1); \ + } while (false) + +/////////////////////////////////////////////////////////////////////////////// + +const char* kernel_file = "kernel.bin"; +uint32_t count = 0; + +std::vector src_a_data; +std::vector src_b_data; +std::vector ref_data; + +vx_device_h device = nullptr; +std::vector staging_buf; +kernel_arg_t kernel_arg = {}; + +static void show_usage() { + std::cout << "Vortex Test." << std::endl; + std::cout << "Usage: [-k: kernel] [-n words] [-h: help]" << std::endl; +} + +static void parse_args(int argc, char **argv) { + int c; + while ((c = getopt(argc, argv, "n:k:h?")) != -1) { + switch (c) { + case 'n': + count = atoi(optarg); + break; + case 'k': + kernel_file = optarg; + break; + case 'h': + case '?': { + show_usage(); + exit(0); + } break; + default: + show_usage(); + exit(-1); + } + } +} + +void cleanup() { + if (device) { + vx_mem_free(device, kernel_arg.addr_a); + vx_mem_free(device, kernel_arg.addr_b); + vx_mem_free(device, kernel_arg.addr_c); + vx_dev_close(device); + } +} + +void generate_source_matrix(uint32_t dim_m, uint32_t dim_n, uint32_t dim_k) { + src_a_data.resize(dim_m * dim_k); + src_b_data.resize(dim_k * dim_n); + + for (uint32_t i = 0; i < src_a_data.size(); ++i) { + src_a_data[i] = static_cast(i); + std::cout << "A: " << i << ": value=" << src_a_data[i] << std::endl; + } + for (uint32_t i = 0; i < src_b_data.size(); ++i) { + src_b_data[i] = static_cast(i); + std::cout << "B: " << i << ": value=" << src_b_data[i] << std::endl; + } +} + +void generate_reference_matmul(uint32_t dim_m, uint32_t dim_n, uint32_t dim_k) { + ref_data.resize(dim_m * dim_n); + + for (uint32_t i = 0; i < dim_m; ++i) { + for (uint32_t j = 0; j < dim_n; ++j) { + float ref = 0.0f; + for (uint32_t k = 0; k < dim_k; ++k) { + ref += src_a_data[dim_k * i + k] * src_b_data[dim_n * k + j]; + } + ref_data.at(dim_n * i + j) = ref; + } + } +} + +int run_test(const kernel_arg_t& kernel_arg, + uint32_t buf_size, + uint32_t dim_m, uint32_t dim_n) { + // start device + std::cout << "start device" << std::endl; + RT_CHECK(vx_start(device)); + + // wait for completion + std::cout << "wait for completion" << std::endl; + RT_CHECK(vx_ready_wait(device, VX_MAX_TIMEOUT)); + + // download destination buffer + std::cout << "download destination buffer" << std::endl; + RT_CHECK(vx_copy_from_dev(device, staging_buf.data(), kernel_arg.addr_c, buf_size)); + + // verify result + std::cout << "verify result" << std::endl; + { + int errors = 0; + auto buf_ptr = (float*)staging_buf.data(); + for (uint32_t i = 0; i < dim_m * dim_n; ++i) { + float ref = ref_data.at(i); + float cur = buf_ptr[i]; + if (std::abs((cur - ref) / ref) > 1e-6) { + std::cout << "error at result #" << std::dec << i + << std::hex << ": actual=" << cur << ", expected=" << ref << std::endl; + ++errors; + } + } + if (errors != 0) { + std::cout << "Found " << std::dec << errors << " errors!" << std::endl; + std::cout << "FAILED!" << std::endl; + return 1; + } + } + + return 0; +} + +int main(int argc, char *argv[]) { + // parse command arguments + parse_args(argc, argv); + + if (count == 0) { + count = 1; + } + + std::srand(50); + + // open device connection + std::cout << "open device connection" << std::endl; + RT_CHECK(vx_dev_open(&device)); + + // FIXME: hardcoded + uint32_t dim_m = 64; + uint32_t dim_n = 64; + uint32_t dim_k = 64; + + generate_source_matrix(dim_m, dim_n, dim_k); + generate_reference_matmul(dim_m, dim_n, dim_k); + + uint32_t src_a_buf_size = src_a_data.size() * sizeof(src_a_data[0]); + uint32_t src_b_buf_size = src_b_data.size() * sizeof(src_b_data[0]); + uint32_t dst_buf_size = ref_data.size() * sizeof(src_a_data[0]); + + std::cout << "buffer size: " << dst_buf_size << " bytes" << std::endl; + + // upload program + std::cout << "upload program" << std::endl; + RT_CHECK(vx_upload_kernel_file(device, kernel_file)); + + // allocate device memory + std::cout << "allocate device memory" << std::endl; + RT_CHECK(vx_mem_alloc(device, src_a_buf_size, VX_MEM_TYPE_GLOBAL, &kernel_arg.addr_a)); + RT_CHECK(vx_mem_alloc(device, src_b_buf_size, VX_MEM_TYPE_GLOBAL, &kernel_arg.addr_b)); + RT_CHECK(vx_mem_alloc(device, dst_buf_size, VX_MEM_TYPE_GLOBAL, &kernel_arg.addr_c)); + + kernel_arg.dim_m = dim_m; + kernel_arg.dim_n = dim_n; + kernel_arg.dim_k = dim_k; + + std::cout << "dev_addr_a=0x" << std::hex << kernel_arg.addr_a << std::endl; + std::cout << "dev_addr_b=0x" << std::hex << kernel_arg.addr_b << std::endl; + std::cout << "dev_addr_c=0x" << std::hex << kernel_arg.addr_c << std::endl; + + // allocate staging buffer + { + std::cout << "allocate staging buffer" << std::endl; + uint32_t staging_buf_size = std::max( + src_a_buf_size, + std::max( + src_b_buf_size, + std::max(dst_buf_size, sizeof(kernel_arg_t)))); + staging_buf.resize(staging_buf_size); + } + + // upload kernel argument + { + std::cout << "upload kernel argument" << std::endl; + auto buf_ptr = staging_buf.data(); + kernel_arg.addr_a = (uint64_t) 0x20000; + kernel_arg.addr_b = (uint64_t) 0x28000; + kernel_arg.addr_c = (uint64_t) 0xc0000000ULL; + memcpy(buf_ptr, &kernel_arg, sizeof(kernel_arg_t)); + + std::cout << "uploading argument buffer to device, device mem address=" + << std::hex << KERNEL_ARG_DEV_MEM_ADDR << ", size=" << std::dec + << sizeof(kernel_arg_t) << " bytes\n"; + std::ofstream file("args.bin", std::ios::binary | std::ios::out); + if (!file) { + std::cerr << "error: failed to open args.bin for writing\n"; + exit(EXIT_FAILURE); + } + file.write(reinterpret_cast(staging_buf.data()), + sizeof(kernel_arg_t)); + file.close(); + + RT_CHECK(vx_copy_to_dev(device, KERNEL_ARG_DEV_MEM_ADDR, staging_buf.data(), sizeof(kernel_arg_t))); + } + + // upload source buffer + { + { + auto buf_ptr = staging_buf.data(); + memcpy(buf_ptr, src_a_data.data(), src_a_data.size() * sizeof(float)); + RT_CHECK(vx_copy_to_dev(device, kernel_arg.addr_a, staging_buf.data(), + src_a_buf_size)); + + std::cout << "uploading source A matrix to device, device mem address=" + << std::hex << kernel_arg.addr_a << ", size=" << std::dec + << src_a_buf_size << " bytes\n"; + std::ofstream file("input.a.bin", std::ios::binary | std::ios::out); + if (!file) { + std::cerr << "error: failed to open args.bin for writing\n"; + exit(EXIT_FAILURE); + } + file.write(reinterpret_cast(buf_ptr), src_a_buf_size); + file.close(); + } + { + auto buf_ptr = staging_buf.data(); + memcpy(buf_ptr, src_b_data.data(), src_b_data.size() * sizeof(float)); + RT_CHECK(vx_copy_to_dev(device, kernel_arg.addr_b, staging_buf.data(), + src_b_buf_size)); + + std::cout << "uploading source B matrix to device, device mem address=" + << std::hex << kernel_arg.addr_b << ", size=" << std::dec + << src_b_buf_size << " bytes\n"; + std::ofstream file("input.b.bin", std::ios::binary | std::ios::out); + if (!file) { + std::cerr << "error: failed to open args.bin for writing\n"; + exit(EXIT_FAILURE); + } + file.write(reinterpret_cast(buf_ptr), src_b_buf_size); + file.close(); + } + } + + // clear destination buffer + { + std::cout << "clear destination buffer" << std::endl; + auto buf_ptr = (int32_t*)staging_buf.data(); + for (uint32_t i = 0; i < ref_data.size(); ++i) { + buf_ptr[i] = 0xdeadbeef; + } + RT_CHECK(vx_copy_to_dev(device, kernel_arg.addr_c, staging_buf.data(), dst_buf_size)); + } + + // run tests + std::cout << "run tests" << std::endl; + RT_CHECK(run_test(kernel_arg, dst_buf_size, kernel_arg.dim_m, kernel_arg.dim_n)); + std::cout << "PASSED!" << std::endl; + + // cleanup + std::cout << "cleanup" << std::endl; + cleanup(); + + return 0; +} diff --git a/tests/regression/rickroll/rickroll b/tests/regression/rickroll/rickroll new file mode 100755 index 0000000000000000000000000000000000000000..67ade61b5a3759e630c5a613c78401d05b717369 GIT binary patch literal 28448 zcmeHwdwf*Ywg1UuBqC&jk~S)p5rZZIF#&=J?MNmMIHMCuAt+k$WHOnM)MO@|2Lyjw zqaoV4(`oV6TKlW^hppbLZT$(owL+{IKnV3#M68P91FPtXh+q&zeB^%Dew>^+%=BY> z@9*~)`}4_Jd#%0J+H0@zCP+CZkf!0?b;)p#D0k5qPM31zE|O~!a!o=` zLWl6Dgwmg6BmGqfy;kTz8U+V?DvDKZsgSe6TEQuX zD-z25z709@%U2De+Lc8jzjXDmL6loUY9|yf>1c1BGy9T`mKh!GfoS)P?ut1x=FBb$ z2TNwMbh3-`pf~@v!n|uUF1jCMrauK;3kge`n{^&T%!r|J5gFhWb3`=HD}}h9#zW zJckU5e>cWnMRPaSYc2kee|dX2;tw^}*K`B}{zh+ehhNdO z<(*PJgH7?dS;l2w`chpjdpbey)D`e z{!qZ*p|!RnFR+FD-j-F`N^g6FFeuV|9l3a->T{DI|>Hq9Rj1w##2 z_$Z^r!H#fUJL8+}543c+eTySyWxh6VNQ;DESr`V?Edp&;vp3xC(}Lj$%b3Sj*dL0xJBdHocydCEIU!Qf9t>;H>7VTmv`1>o>Q-HqS9FBk zU#%{Dji`xojn`;kWhXP<;@}$T;F+_fKZDVT7Hri*-oSFd*4i0?{Ef_uvwcY7O1@GV zMCDxxYrjg7VG6RMCZ)O21{R{(b_QGg)O45od|H@VZkeJj39M`nv}lX`;bZ*2YWhJwe z1s-=bP>^Sr%q&ep%gU0_N;Age+RT#KMyzZ!I&P-@FS{2?GTH+A5z;lfoAI79IW zkql@$2s))V;qQeQlV@ZWXk-I&Inrra;^=sE4DQ+a+T^_@+4rm{iklj|nA zUQx%>D2zjE1VJ5n&_x_BRpiHqq>c-&O{eg$kH~L=#;k%1136K--v`ACc2y>P|_|F z-Kcxe_n7EtKt|YSqK`GGilUq7=paToY@&}dsETsjL`Q=j4WhotMw;WvkYl3TB@r;s zM5q2HLxG7tK@tHSCc1e(SY)D$Ee|GDYNFHnQ-%r?Jzo+5t4#E-ndqvC{&f?*!9*`G z(U+R&XPf9vCi*!hdYg%Uu8H1dqE9l>yG`_OnCLww`gtb$4@~sQCi)r^-C?5NW1^pL zqOUd4r zPI=&z2Tpn5lm||E;J@pEBlh!7sJ$QLsBzmX^Ke|#74-b9Mz-i$>JmfWSTJR4KecQlkp+7w2GH_bp4-b_YI4$glht4-} zTF?&_{)oiY*qeGA1_Cv9NFNYzDWE6zp1ux{#y{89uB%*2T#c^S=S$SY zj9bse&tufs5%ty&iZGJ3sPXKnw6&wgRu>k4P&S~(-dAI9)Ivo)+Ia8rkw{bT@f#vn z+%lo}_`K+JeI*90vJrhPI$-bdK-8heop-^Yej3-xj_DgfkJ)~ME^*+o-s5eN=k)my z=zVTP4y8!xW+=Zd*5J&DeL1kU+J$0JxXN=(7heK*L{*Cp7#cB(4)M?+i4{^N> zCU|0B!n#f1&{x5_etKm<$9(8eW82i&pk7UeqSj7)Mh2J9LP|dk>=3(y#HO%(!`y2fCG#TLg+(%C+9w7aQ_;LGv1F^7xeCXtanFFthzv-KzYKk51)X)S=A-VCy+vY+ge=Rb>>SnEf;_f{K^n0yT#64eIqwLl#D@O>D?v)%dEK zxUMiy)@3HuWgQHM)OUk+Y;|D~h6KYR5&9t8;o|A zi>(cDWVR7nka8_yS#5f}PL2h21-cC?rH__k`YOiUoy?3npocuM4@J9G6Td6WL+vLy z>FHz4Nrg;yP_H9>!bvk2?U4R$(8x*Xjc90pbM?x7C0!^Cm5r1 zBE^HSAU?hxT54nO#`CL4Yr%Lcf>Cr6<2 zS7^isehH=bbGhAA+Aomvfrr_mHnweO9{fOYckJn*8WBfJ9l8i{`{W)N%hnTY{hM$u zqq5ygw#1#U<18IzdzXRo8cGR4HApXT(nOFj$e#{C0j_DrD-o0tn9nGKw$DXWzm$vK z!$miO495Q}857Qnxab3tGHAPBihhowBG+YH^m;D3STZJ@A7CvDWo}6sw9S{Io4M!| z5`|rG53?&#ejHIyjv5r_cCO`sr1UV$%vk$rrAE9I$rdz|~rP_*c@C2Ftp1pQh= zjqQ(&Lp5FqF@57lXe});al7qF@MSE6tYVfQT8Tty5B5U(X@39>>m5uNVc#{R5#h}v$%1{sZSW}EU??c9F-$iK?!VqbkXMk$wl(~25suLiGv^I^zg-eDt`#GQXcThq@*oZk5Tkn2AST9DBj)WQ?a zT#|X1@!B#|<#okqUc(qaWL_`LfZPO;*Ut&YotGo8P0Do_+VnOMD8tJrLrk41!&`ZV z_mN-)&+tyfPb%E2R3cuuVpHM#B)9euFI*AyyW>|oP*+c)ZeycoPhmvP`1P z$u)9T?BJO@NXZk-THC}_nJ*j7JYp)XJ{@uwh|GUWdAyHhP&ION*&qFD-){Tr6&Pr1 z6U`%aWyk8uP}MW2e;haq?b@AKkWpXxo_+O|U~$I=)i~b9@%$M`SBtl(n@?nVDo@yN z-;L~!RNM2ipP=W&4fdiPN9@-HE50nd2+8#F_l&L}*sSxp58=>6`T3-TOvCdgnxNAh z?1Y>i&{~7NNcA2lJz~#;el-q0STvxIgFE|aKM#7|1|&T+Yt$EWW|+j7TkVGiAh1Q> zOP1k9g3Qpc-SIzVP{T%Fw6A^xZ;&3Wfg;uRO)G0WvFBX9+cWgv9OSh=6UIt!_Wd5K z;kv$K(I(70C!re_nckRez3qv4kTVrWe}zg(27BT}ON_6;YkU)1AjHSOh*HcdB6pHN z&d4s}$+2%be4vMqYjPob7L_gGeEniHg>I(M_B`UXw6b&#B9_X(c9gY0hdbvLWGH53 z6-=JmL6z=_KOlPcJNDJDqm8(FZYrd%{UeIoH$@6-*!=1P?t~d|hMZxnA&m*=<;Yy$ zPfm*4E@Os}TYty2s(lCTtM7oOo|{Q$4r2CAQ5r2KlGH+>^<}_8I1e?6~VH`=i@w(+e*YdYANmfKEDoJ8BuD zdTIiAXQ!}DFeTQZO|!Sfha|1U)QUA==Du8h5| zj1Myo)mFmt=r7R83z9ndYS^kMqyjlXOQ_$28BxL+!lWPGTzml4m&m?dt=w#1JqZrQ zBL8vB4uz-!X^*EQ}E*dn2aokuk`zkMZVT&qWpj*dKiz@BccB@K*aB zAqd-V^WX{PlwP(M-p1x+<@kBPJc)8_UMSH8L$tEN;Ka+1Az~#PA$oNx;-G%U5Osoa zg?P#2KrB&%b<)KMGC*_mx<598v8T55Os(O=WMQ3E3OEC4;W*4HbutC1ser@&!{bNvRxZ}gTC;lff%3+xQ1C4An z&SQpvy70tC^f^q&7%?bUm3k@<+gD%66M1(!8wky(;G5I2r$N3s2H$*hd=@a&QVWzu zYle=ZK&&0|nF{*&4exQk907F4>!Bcj*F~d#+5Y$ZQiJh8`sM9H;g|J#HypvX6rDRk z)ML;r{gOi?2=~h(aM3H+?3Z==Ypj|^{bHKe-3#0K)JOQ`6a74(a(-RB)r0Lt?4E{@Jv-xJ{!?RLV!Ts} zx9isuy#f;YIuw-V&#}W>4gGvRJ&)$ndLb*)~?PgJl1acB++c*l)WFlAg*V^|7NKd(DCRjHB+#XFc`>L+;J{GvOtD z9_**Cl*leBI|vulVmCMYA)enhSNz3bSJVOt&epS=A||Yt>q2s5ZjOw?Kj=GAM0nV>YYEqJ3@c%9rLT1<?N8~JPBpkJ#c4iKX4x_KgK6| zFaH#NBENMa{lLG$Up~Ak_5bsBN`JYo_9UEpu~qN73Tj)ZV%ZO2W~O zh{C>Vb9f^HcU+*%b$ES|sJFwZOn3OZyZpY0zs0G{4@Ltm4#mOv!{;mWU2acZ?fHsp zt^*P)yd6=$Q>g|Ln0v$Awt4tO$R7}efnk9F)R1A~i#r&D+uv(!9>n?oq5c3G)$VD1cv=w+EI> zcZhuWM}`i3)`ah-0uhv*%y9~9#wth%mo(#BruG0nZ!7Vd;>|4X>+pL+uLgu(|YZ~gb1)hb~F3)tw zg+iB6I4{$fR$uc!s;@b{zLs$0AJyg)<>k(y1}od9Xt|C^P;`dr0y^Lr2i2QD9Cix# z@(w=NK_B>G#DEOznX);vfrCE5L#ox3;ayj4 z?ILs&-kp+~5!H9FwP@7gTBKMM$7qih{iD3R<$2R+PO50qIqIKqFdFjt9WF^ z{)+xdQ(mh*&FSqKZ73=2ss4Xr4|--Q&zl<>_c(?hjfNjnZJ@m*wMSD~TiQFd&ZKdG zD1l;gw{P*cdhzPeyj>W@TVy-Kgya?(Czv?+*xFCFue&Vc+^lat0G#$CD_$8H*#o#? z@5l)K5@H8n0bs=&BO?>D6=f;l8Z1=`FdLo@yaKC*0q_@L!Q279Zot)mI^ey4D)twf zuvWSUo2b77r1OkY(02gV5+C4Fz#JSq=){jhspH`5rsFo~*8$f6Qkckrl5yiAC8Ikp zJ()MEe8& zeqf@J&K*RDtqp)mZEjxXg4`UU@5A2*pnCz8#@xJ{b8{3G(c}25g`XR-&{O8;=KVO+ zlUwkktm<4xBHNu?^g~;1Zt3l(sks%sW9oC8vSwzD&8=|dmb!9_z*3!CP@S7sotrcN z3@Quxya4u;0V+#K7Z`-DYD1TcbX}fXm06QYlY8s-zdXD1IW@Pccg*dl{m_=k{!!MC zGc#{YnQiSEtPFobe*3W3R!{k|GSpJObyNnHFATo@G-TI1W`1r%=I)fb++0@<>f#-2 z=Q(J}%yiWymq9h@s>-@IGo`Z2nLiq#ONBqqC%djm_D3E0gV`nh@x7Fq7MzjkP35P$ zlz!rE6}h1uH+Wb}9`lPwa87rwY|qp#sO=7*-EBM_UjUDRajcxF>WbC-<+=ODW&J7# z=GB~3@h;jD#;!99raUW0^MtgyD#1zZaW?)Y!3GCl_7*%4=p6Y6@cTQM&Ywpt0i+edpMJ!-&2FQfnLkI{>Vn*=Kq|MfZ>>Loo(AZ_8r}#aBN^Xp{1-Un zv(qpB=~>F~*c~w7rE%jQIZOHVcxqtBa>w6)mhww`4}>?KX+!*_Gi^xy_{_=Rhj{iG zDD(gSfa56^*x-bEE#)Y=kAf8*g%ReDn=D1soOd;NSo!D=h56kCiFNaY%kA zPRBnKXk4OzUxKh8zYC`^lmgWyg`;1N1PQJc`N_IiD;Q`vqOeD#^Vx}_JT1m+YIhW< z%}}uVUWh3@*Yk8a#LDtg8>Aq|*spU`$q@yR^tHJ>LE7CW9EFy`g3SLsflEH@JhSqF zosqEX$naglk$7J+T-u4_6^6Hn^iKsY{UX%=j{$kFnN?w_gHx&mY!I+Xz%BuM1Y9HF zS^?JyI3VCI0rv@bSU_B+#X_Ed4gpIAtP-$6z$O8^1nd!Tjeu(fTqodwfV%|TC*WZL z<)u1#Q@9-t0ZRp}60kwQCIPzy>=AH{fLzNz{>*9OGFjEhvbZy2uA^v4b2Jc%Ix2BX zM(K>QD8tIWKeM!?6xU`Hb9%B|Dl7}VyFfNH6GImEASzPQvy`PN>DkInDd{$&ADh(U zc8b%C{+mo6qsV@qOwUnd`%k8yuE>6qOdqSr{+Uca!|2D!bc|{aGlj>qaI2ytB|TSJ zo1%WR6uYu6B|T4ZOiiVKf+ELFb7~gdG*xA$@OTzx%kq9cb7~geoNE@CDUv^5S#w1y z`qvaWKATgsl&>ptTu!DJD5=JUER6PY98KoODBonJ@OYN#{wZ^6mT{}(C^ZAoY~`bo zQ3{XIc%O;4NnRPpY2HGC-ZMm(es&NMf!t1XdASCgKY&(+zYIuq_z+=n4i^HX%9nP* zNl!y5=aWyG!xO#7LZ?BS=<>ZQ`TG!|eAhK|3YC+>UHHpL;r}NS-(~z0l^>>P52?!gF=djXy)DKIFq38h4#r?SXu62e2r>e)vjGm`73IBM7z_c{{m82hbw)r@t^z%H>&quxmO`I<6{7xD@ zy=myzLh#JYlgI5}rs4lB=wwfhFFD`egPyAW{AU_He@#PwI}M%IQ4`P(^DN~(Ci1o3 z_csBKO6C7)pp*T1K5oBkhqKf0)BZ^+J%#8O)IJ?zoV%FklsRemf0BlNb{c!0PQyP$`NE$)B40i}B6sE=X~if!4|E!T6u~c3 zCSlyjgPtx?FOohjjh=aF=(m7Q?XdJRu7Qusit=z8{{5g+zE;0|l!m_$?~7FJ!OiIC z*J9o17X`Qmbn?$(G0w>L=1-$1A^01FA0+=hY4{&WL*EH{s`tfvj34j!N^YPdi$fcY z1HxuAm;}1ROc3#kH2e*Wj`3gk0nLMjCeX>A!{Yr?hWe)PFKP7LoreBc8u}K{9ZX~L zHhVj? z792!{HE*04G0`}}fxFk}z zW8wVyi|cSFsH@sjM_Na(XsoJY7bN1ML%PS1T^7o()f5T*=0twmb+^{l z)y`x$Va_aH?hp7wI5NeJsW|ub2Yh}FCnKHFj_-c=yW&E}(QN4IM9oj7s`)vWWrig4bvVQ5J zQ!w-PnS3Tj7klE?&(Tuj#!buRuI9^j&DV#H%24}4K}CCD7u6)Inm9$mRz*va9EUrob~sw15T2q+ih8oOTN09ZWd1BL-SzQ=K8`> z_QIgw1JUnP=wO{GFy9s}FNIFhfSTvOgJ5GWIxXGCD>$ULQ(OUERb{%CH)(8;FBBRZ zv}|O zlmvnif64Mdv;;qT!ZqEIRVGq%v>pEvqrF9taEDN=eJA0J7jP65=WN ztyh?ScE#8Pq9HBh@9>g!oy)qXFibjBv-_Xv3jcL;;YN8~#x zFV|)5ppl4_m;3z^%5yxhuweE7_dumHGAS?51te@0a?*auC*k*zPG@csm*)ot6!K(0 z*(vQ8WF>(J6;aB|a|Q|JIiSp6%FFW87ZxO6jXw(VTv0-K4oUK~XK9tcA2jlb)UP;z zP;js(R%pq=+J5duvRU5Y;QSKy$btzq*g0oG!hZ*5mR~!Cr%NdLIFZ3NIY>E4dkPVA{_^}qLV2Do`K|ujCgcq_RS8Z3 zb)#Z9-J1WiAXENQUY_Sj_y89(p4R;LTIA)qkA&6^FWZk)xZf%-&I9B*(E`aRAlYgT zp8%nwZ_(|Ib0qoyoUG-Q`Ae7ug?5v?Ja1a+;kigYj*Yf|Nw-6uTrBgK=Th?jb2)@# zssB=ta?-Bff<|SM^78zu=St#MlpafgrM!gnyEL=B(!|8@!x0W00+m3@Nu17~h$i!| zYT_vlAzvyJgg}*m5?mniFO>uVrTjDtoouCGhO+JP6#i@w`FD#Sc+l?_X5mlTFUK!B kRfBuzjhmHZ(;=a_C0u%OC+0X9c<#Q*>R literal 0 HcmV?d00001 diff --git a/tests/regression/sgemm_gemmini/kernel.cpp b/tests/regression/sgemm_gemmini/kernel.cpp index dcd931aa..221af0a5 100644 --- a/tests/regression/sgemm_gemmini/kernel.cpp +++ b/tests/regression/sgemm_gemmini/kernel.cpp @@ -16,23 +16,26 @@ #define NUM_CLUSTERS 1 #define NUM_THREADS_IN_CLUSTER 128 -#define SMEM_ADDR_0K ((float * const) 0xff000000) -#define SMEM_ADDR_4K ((float * const) 0xff001000) -#define SMEM_ADDR_8K ((float * const) 0xff002000) -#define SMEM_ADDR_12K ((float * const) 0xff003000) -#define SPAD_ADDR_0K 0x0 -#define SPAD_ADDR_4K 0x80 -#define SPAD_ADDR_8K 0x100 -#define SPAD_ADDR_12K 0x180 +#define SMEM_ADDR_Q0 ((float * const) 0xff000000) +#define SMEM_ADDR_Q1 ((float * const) 0xff001000) +#define SMEM_ADDR_Q2 ((float * const) 0xff002000) +#define SMEM_ADDR_Q3 ((float * const) 0xff003000) +#define SPAD_ADDR_Q0 0x0 +#define SPAD_ADDR_Q1 0x80 +#define SPAD_ADDR_Q2 0x100 +#define SPAD_ADDR_Q3 0x180 +#define SPAD_ADDR_Q4 0x200 -//#define DEBUG_PRINT -//#define EXT_ACCUMULATE #define HARDCODE #define REGBLOCK +#define OFFLOAD_ACCUMULATE +#define REMATERIALIZE #define DBUF +//#define CISC + +//#define DEBUG_PRINT //#define DETAILED_PERF //#define ACTIVATE -#define CISC #define rd_cycles_force(x) asm volatile ("csrr %0, mcycle" : "=r" (x)) #ifdef DETAILED_PERF @@ -40,7 +43,11 @@ #else #define rd_cycles(x) #endif -#define HW_TID() ({uint32_t gtid; asm volatile ("csrr %0, mhartid" : "=r" (gtid)); gtid;}) +#ifdef REMATERIALIZE + #define HW_TID() ({uint32_t gtid; asm volatile ("csrr %0, mhartid" : "=r" (gtid)); gtid;}) +#else + #define HW_TID() hw_tid +#endif #define PRINTF(...) sprintf(PRINT_BUF, __VA_ARGS__) // #define PRINTF(...) vx_printf(__VA_ARGS__) #define SWISH(beta, x) ((x) / (1 + exp(-(beta) * (x)))) @@ -58,10 +65,11 @@ void thread_block_matmul_gemmini(kernel_arg_t *__UNIFORM__ arg, const float * const B = (const float * const) arg->addr_b; float * const C = (float * const) arg->addr_c; - if (HW_TID() == 0) { - gemmini_config_ld(0); + if (tid_in_threadblock % NUM_THREADS_IN_CLUSTER == 0) { gemmini_extended_config_ex(WEIGHT_STATIONARY, 0, 0, 1, 0, 0); - gemmini_config_st(0); + // gemmini_extended_config_ex(dataflow, act & 3, 0, 1, a_transpose, b_transpose); + + // gemmini_extended_config_st(stride_C * sizeof_C, act & 3, scale); PRINTF("start\n"); } @@ -100,17 +108,26 @@ void thread_block_matmul_gemmini(kernel_arg_t *__UNIFORM__ arg, constexpr uint32_t i1_iters = (DIM * DIM * (TILE_K / DIM)) / num_threads_in_cluster; // num of iters before striding const uint32_t num_tile_rows_per_tb = num_tiles_m / NUM_CLUSTERS; + + if (HW_TID() == 0) { + gemmini_extended3_config_ld(dim_k * sizeof(elem_t), MVIN_SCALE_IDENTITY, false, 0); + gemmini_extended3_config_ld(dim_n * sizeof(elem_t), MVIN_SCALE_IDENTITY, false, 1); + // gemmini_extended3_config_ld(repeating_bias ? 0 : (stride_D * sizeof_D), D_scale_factor, low_D, 2); + gemmini_extended_config_st(dim_n * sizeof(elem_t), 0, MVIN_SCALE_IDENTITY); + // gemmini_extended_config_st(stride_C * sizeof_C, act & 3, scale); + } + for (uint32_t tile_i = num_tile_rows_per_tb * threadblock_id; tile_i < num_tile_rows_per_tb * (threadblock_id + 1); tile_i += 1) { __asm__("i_loop:"); for (int tile_j = 0; tile_j < num_tiles_n; tile_j += 1) { __asm__("j_loop:"); - float * const smem_c_tile_start = SMEM_ADDR_4K; - #ifndef EXT_ACCUMULATE - float * const smem_acc_tile_start = SMEM_ADDR_0K + HW_TID(); + float * const smem_c_tile_start = SMEM_ADDR_Q1; + #ifdef OFFLOAD_ACCUMULATE + float * const smem_acc_tile_start = SMEM_ADDR_Q0 + HW_TID(); #else - float * const smem_acc_tile_start = SMEM_ADDR_8K + hw_tid; + float * const smem_acc_tile_start = SMEM_ADDR_Q2 + hw_tid; #endif __asm__("k_loop:"); @@ -132,11 +149,11 @@ void thread_block_matmul_gemmini(kernel_arg_t *__UNIFORM__ arg, const float * const dram_a_tile_start = A + tile_i * TILE_M * dim_k + tile_k * TILE_K + runtime_const_a; const float * const dram_b_tile_start = B + tile_k * TILE_K * dim_n + tile_j * TILE_N + runtime_const_b; #ifdef DBUF - float * const smem_a_tile_start = ((tile_k & 1) ? SMEM_ADDR_4K : SMEM_ADDR_0K) + HW_TID(); - float * const smem_b_tile_start = ((tile_k & 1) ? SMEM_ADDR_12K : SMEM_ADDR_8K) + HW_TID(); + float * const smem_a_tile_start = ((tile_k & 1) ? SMEM_ADDR_Q1 : SMEM_ADDR_Q0) + HW_TID(); + float * const smem_b_tile_start = ((tile_k & 1) ? SMEM_ADDR_Q3 : SMEM_ADDR_Q2) + HW_TID(); #else - float * const smem_a_tile_start = SMEM_ADDR_0K + HW_TID(); - float * const smem_b_tile_start = SMEM_ADDR_12K + HW_TID(); + float * const smem_a_tile_start = SMEM_ADDR_Q0 + HW_TID(); + float * const smem_b_tile_start = SMEM_ADDR_Q3 + HW_TID(); #endif { @@ -175,7 +192,6 @@ void thread_block_matmul_gemmini(kernel_arg_t *__UNIFORM__ arg, smem_b_tile_start[7 * num_threads_in_cluster + hw_tid] = \ dram_b_tile_start[every_iter * 1 + every_2iters_b * 3]; #else - __asm__("load_ab:"); float v0 = dram_a_tile_start[every_iter * 0 + every_2iters_a * 0]; float v1 = dram_a_tile_start[every_iter * 1 + every_2iters_a * 0]; float v2 = dram_a_tile_start[every_iter * 0 + every_2iters_a * 1]; @@ -185,7 +201,6 @@ void thread_block_matmul_gemmini(kernel_arg_t *__UNIFORM__ arg, smem_a_tile_start[2 * num_threads_in_cluster] = v2; smem_a_tile_start[3 * num_threads_in_cluster] = v3; - __asm__("load_ab1:"); v0 = dram_b_tile_start[every_iter * 0 + every_2iters_b * 0]; v1 = dram_b_tile_start[every_iter * 1 + every_2iters_b * 0]; v2 = dram_b_tile_start[every_iter * 0 + every_2iters_b * 1]; @@ -195,7 +210,6 @@ void thread_block_matmul_gemmini(kernel_arg_t *__UNIFORM__ arg, smem_b_tile_start[2 * num_threads_in_cluster] = v2; smem_b_tile_start[3 * num_threads_in_cluster] = v3; - __asm__("load_ab2:"); v0 = dram_a_tile_start[every_iter * 0 + every_2iters_a * 2]; v1 = dram_a_tile_start[every_iter * 1 + every_2iters_a * 2]; v2 = dram_a_tile_start[every_iter * 0 + every_2iters_a * 3]; @@ -205,7 +219,6 @@ void thread_block_matmul_gemmini(kernel_arg_t *__UNIFORM__ arg, smem_a_tile_start[6 * num_threads_in_cluster] = v2; smem_a_tile_start[7 * num_threads_in_cluster] = v3; - __asm__("load_ab3:"); v0 = dram_b_tile_start[every_iter * 0 + every_2iters_b * 2]; v1 = dram_b_tile_start[every_iter * 1 + every_2iters_b * 2]; v2 = dram_b_tile_start[every_iter * 0 + every_2iters_b * 3]; @@ -214,8 +227,6 @@ void thread_block_matmul_gemmini(kernel_arg_t *__UNIFORM__ arg, smem_b_tile_start[5 * num_threads_in_cluster] = v1; smem_b_tile_start[6 * num_threads_in_cluster] = v2; smem_b_tile_start[7 * num_threads_in_cluster] = v3; - - __asm__("end_loadab:"); #endif } #else @@ -223,8 +234,8 @@ void thread_block_matmul_gemmini(kernel_arg_t *__UNIFORM__ arg, const float * const dram_a_tile_start = A + tile_i * TILE_M * dim_k + tile_k * TILE_K; const float * const dram_b_tile_start = B + tile_k * TILE_K * dim_n + tile_j * TILE_N; - float * const smem_a_tile_start = SMEM_ADDR_0K; - float * const smem_b_tile_start = SMEM_ADDR_12K; + float * const smem_a_tile_start = SMEM_ADDR_Q0; + float * const smem_b_tile_start = SMEM_ADDR_Q3; /* for (uint32_t thread_i = 0, j1 = 0, i1 = 0; thread_i < a_elems_per_thread; @@ -281,7 +292,6 @@ void thread_block_matmul_gemmini(kernel_arg_t *__UNIFORM__ arg, // cluster wide barrier to wait for A and B loads to complete threadblock_barrier(/*barrier_id=*/0, /*count=*/NUM_WARPS); rd_cycles(marker3); - __asm__("gemmini:"); if (HW_TID() == 0) { #ifdef DBUF gemmini_fence(); @@ -290,8 +300,8 @@ void thread_block_matmul_gemmini(kernel_arg_t *__UNIFORM__ arg, #ifndef DBUF #error MUST ENABLE DBUF #endif - #ifdef EXT_ACCUMULATE - #error MUST DISABLE EXT ACCUMULATE + #ifndef OFFLOAD_ACCUMULATE + #error MUST OFFLOAD ACCUMULATE #endif if (tile_k == 0) { GEMMINI_CISC_CMD_I(0); @@ -303,14 +313,14 @@ void thread_block_matmul_gemmini(kernel_arg_t *__UNIFORM__ arg, #else sp_tiled_matmul_full_spad_ws( #ifdef DBUF - (tile_k & 1) ? SPAD_ADDR_4K : SPAD_ADDR_0K, (tile_k & 1) ? SPAD_ADDR_12K : SPAD_ADDR_8K, + (tile_k & 1) ? SPAD_ADDR_Q1 : SPAD_ADDR_Q0, (tile_k & 1) ? SPAD_ADDR_Q3 : SPAD_ADDR_Q2, #else - SPAD_ADDR_0K, SPAD_ADDR_12K, + SPAD_ADDR_Q0, SPAD_ADDR_Q3, #endif - /*spad_D=*/0, /*spad_C=*/SPAD_ADDR_4K, + /*spad_D=*/0, /*spad_C=*/SPAD_ADDR_Q1, /*I=*/TILE_M / DIM, /*J=*/TILE_N / DIM, /*K=*/TILE_K / DIM, /*pad_I=*/0, /*pad_J=*/0, /*pad_K=*/0, /*a_transpose=*/0, /*b_transpose=*/0, /*full_C=*/0, /*low_D=*/0, - #ifdef EXT_ACCUMULATE + #ifndef OFFLOAD_ACCUMULATE /*acc=*/0, /*act=*/NO_ACTIVATION, /*skips=*/0x38U) #else /*acc=*/tile_k != 0, /*act=*/NO_ACTIVATION, /*skips=*/0xB8U) @@ -321,13 +331,14 @@ void thread_block_matmul_gemmini(kernel_arg_t *__UNIFORM__ arg, gemmini_fence(); #endif } - __asm__("end_gemmini:"); rd_cycles(marker4); - // threadblock_barrier(/*barrier_id=*/0, /*count=*/NUM_WARPS); + #ifndef DBUF + threadblock_barrier(/*barrier_id=*/0, /*count=*/NUM_WARPS); + #endif rd_cycles(marker5); // accumulate C matrix - #ifdef EXT_ACCUMULATE + #ifndef OFFLOAD_ACCUMULATE __asm__("accumulate:"); if (tile_k == 0) { #pragma GCC ivdep @@ -383,7 +394,7 @@ void thread_block_matmul_gemmini(kernel_arg_t *__UNIFORM__ arg, } - #ifndef EXT_ACCUMULATE + #ifdef OFFLOAD_ACCUMULATE threadblock_barrier(/*barrier_id=*/0, /*count=*/NUM_WARPS); rd_cycles(marker6); __asm__("mvout_spad_ser:"); @@ -476,13 +487,12 @@ void thread_block_matmul_gemmini(kernel_arg_t *__UNIFORM__ arg, #endif #else - float * const dram_c_tile_start = C + tile_i * TILE_M * dim_n + tile_j * TILE_N; #pragma clang loop unroll(disable) for (int thread_i = 0; thread_i < c_elems_per_thread; thread_i++) { uint32_t elem_offset = hw_tid + num_threads_in_cluster * thread_i; dram_c_tile_start[elem_offset / TILE_N * dim_n + elem_offset % TILE_N] = \ - *(SMEM_ADDR_8K + SMEM_MAT_OFFSET(elem_offset / TILE_N, elem_offset % TILE_N, TILE_N)); + *(SMEM_ADDR_Q2 + SMEM_MAT_OFFSET(elem_offset / TILE_N, elem_offset % TILE_N, TILE_N)); } #endif __asm__("end_mvout_dram:"); @@ -513,7 +523,7 @@ void thread_block_matmul_gemmini(kernel_arg_t *__UNIFORM__ arg, PRINTF("first barrier: %d\n", marker3 - marker2); PRINTF("gemmini cycles: %d\n", marker4 - marker3); PRINTF("second barrier: %d\n", marker5 - marker4); - #ifdef EXT_ACCUMULATE + #ifndef OFFLOAD_ACCUMULATE PRINTF("accumulation cycles: %d\n", marker6 - marker5); #else PRINTF("smem mvout cycles: %d %d-%d\n", marker7 - marker6, marker7, marker6); diff --git a/tests/regression/sgemm_gemmini_dma/.gitignore b/tests/regression/sgemm_gemmini_dma/.gitignore new file mode 100644 index 00000000..7c35ba59 --- /dev/null +++ b/tests/regression/sgemm_gemmini_dma/.gitignore @@ -0,0 +1,5 @@ +*.bin +*.dump +*.elf +sgemm_wg +.depend diff --git a/tests/regression/sgemm_gemmini_dma/Makefile b/tests/regression/sgemm_gemmini_dma/Makefile new file mode 100644 index 00000000..3a8ffb18 --- /dev/null +++ b/tests/regression/sgemm_gemmini_dma/Makefile @@ -0,0 +1,9 @@ +PROJECT = sgemm_gemmini_dma + +SRCS = main.cpp common.h + +VX_SRCS = kernel.cpp + +OPTS ?= -n16 + +include ../common.mk diff --git a/tests/regression/sgemm_gemmini_dma/common.h b/tests/regression/sgemm_gemmini_dma/common.h new file mode 100644 index 00000000..74941562 --- /dev/null +++ b/tests/regression/sgemm_gemmini_dma/common.h @@ -0,0 +1,18 @@ +#ifndef _COMMON_H_ +#define _COMMON_H_ + +#include + +#define KERNEL_ARG_DEV_MEM_ADDR 0x7fff0000 +#define DEV_SMEM_START_ADDR 0xff000000 + +typedef struct { + uint32_t dim_m; + uint32_t dim_n; + uint32_t dim_k; + uint64_t addr_a; + uint64_t addr_b; + uint64_t addr_c; +} kernel_arg_t; + +#endif diff --git a/tests/regression/sgemm_gemmini_dma/kernel.cpp b/tests/regression/sgemm_gemmini_dma/kernel.cpp new file mode 100644 index 00000000..128c49aa --- /dev/null +++ b/tests/regression/sgemm_gemmini_dma/kernel.cpp @@ -0,0 +1,175 @@ +#include +#include +#include +#include +#include "common.h" +#include "include/gemmini.h" +#include "gemmini_mmio.h" + +#define TILE_M 64 +#define TILE_N 64 +#define TILE_K 64 +#define SMEM_ADDR_Q0 ((float * const) 0xff000000) +#define SMEM_ADDR_Q1 ((float * const) 0xff004000) +#define SMEM_ADDR_Q2 ((float * const) 0xff008000) +#define SMEM_ADDR_Q3 ((float * const) 0xff00c000) +#define SPAD_ADDR_Q0 0x0 +#define SPAD_ADDR_Q1 0x200 +#define SPAD_ADDR_Q2 0x400 +#define SPAD_ADDR_Q3 0x600 +#define BOUND_INST 0x800080008ULL + +// #define TILE_M 32 +// #define TILE_N 32 +// #define TILE_K 32 +// #define SMEM_ADDR_Q0 ((float * const) 0xff000000) +// #define SMEM_ADDR_Q1 ((float * const) 0xff001000) +// #define SMEM_ADDR_Q2 ((float * const) 0xff002000) +// #define SMEM_ADDR_Q3 ((float * const) 0xff003000) +// #define SPAD_ADDR_Q0 0x0 +// #define SPAD_ADDR_Q1 0x80 +// #define SPAD_ADDR_Q2 0x100 +// #define SPAD_ADDR_Q3 0x180 +// #define BOUND_INST 0x400040004ULL + +#define NUM_CLUSTERS 1 +#define NUM_THREADS_IN_CLUSTER 128 + +#define rd_cycles_force(x) asm volatile ("csrr %0, mcycle" : "=r" (x)) +#define rd_cycles(x) rd_cycles_force(x) +#define HW_TID() ({uint32_t gtid; asm volatile ("csrr %0, mhartid" : "=r" (gtid)); gtid;}) +#define PRINTF(...) sprintf(PRINT_BUF, __VA_ARGS__) +// #define PRINTF(...) vx_printf(__VA_ARGS__) +#define SWISH(beta, x) ((x) / (1 + exp(-(beta) * (x)))) + +inline void threadblock_barrier(unsigned int barrier_id, unsigned int count) { + vx_fence(); + vx_barrier(barrier_id, count); +} + +void thread_block_matmul_gemmini(kernel_arg_t *__UNIFORM__ arg, + const uint32_t threadblock_id, + const uint32_t tid_in_threadblock) { + __asm__("matmul_start:"); + const float * const A = (const float * const) arg->addr_a; + const float * const B = (const float * const) arg->addr_b; + float * const C = (float * const) arg->addr_c; + + if (HW_TID() == 0) { + gemmini_extended_config_ex(WEIGHT_STATIONARY, 0, 0, 1, 0, 0); + // gemmini_extended_config_ex(dataflow, act & 3, 0, 1, a_transpose, b_transpose); + PRINTF("start\n"); + } + + vx_fence(); + + uint32_t marker0, marker1; + rd_cycles_force(marker0); + + const uint32_t dim_m = arg->dim_m; + const uint32_t dim_n = arg->dim_n; + const uint32_t dim_k = arg->dim_k; + const uint32_t num_tiles_m = dim_m / TILE_M; + const uint32_t num_tiles_n = dim_n / TILE_N; + const uint32_t num_tiles_k = dim_k / TILE_K; + constexpr uint32_t num_threads_in_cluster = NUM_THREADS_IN_CLUSTER; + + const uint32_t num_tile_rows_per_tb = num_tiles_m / NUM_CLUSTERS; + + if (HW_TID() == 0) { + gemmini_extended3_config_ld(dim_k * sizeof(elem_t), MVIN_SCALE_IDENTITY, false, 0); + gemmini_extended3_config_ld(dim_n * sizeof(elem_t), MVIN_SCALE_IDENTITY, false, 1); + // gemmini_extended3_config_ld(repeating_bias ? 0 : (stride_D * sizeof_D), D_scale_factor, low_D, 2); + gemmini_extended_config_st(dim_n * sizeof(elem_t), 0, MVIN_SCALE_IDENTITY); + // gemmini_extended_config_st(stride_C * sizeof_C, act & 3, scale); + } + + for (uint32_t tile_i = num_tile_rows_per_tb * threadblock_id; + tile_i < num_tile_rows_per_tb * (threadblock_id + 1); + tile_i += 1) { + for (int tile_j = 0; tile_j < num_tiles_n; tile_j += 1) { + if (HW_TID() == 0) { + for (int tile_k = 0; tile_k < num_tiles_k; tile_k += 1) { + ROCC_INSTRUCTION_RS1_RS2(XCUSTOM_ACC, + (uint64_t) (A + tile_i * TILE_M * dim_k + tile_k * TILE_K), + (uint64_t) (B + tile_k * TILE_K * dim_n + tile_j * TILE_N), k_LOOP_WS_CONFIG_ADDRS_AB) + GEMMINI_CISC_CMD_R((dim_n) << 16 | (dim_k << 8) | 8); + if (tile_k & 1) { + GEMMINI_CISC_CMD_I(11); + } else { + GEMMINI_CISC_CMD_I(10); + } + + if (tile_k == 0) { + gemmini_fence(); + GEMMINI_CISC_CMD_I(0); + } else if (tile_k & 1) { + gemmini_fence(); + GEMMINI_CISC_CMD_I(2); + } else { + gemmini_fence(); + GEMMINI_CISC_CMD_I(1); + } + } + + gemmini_fence(); + gemmini_fence(); + gemmini_fence(); + gemmini_fence(); + // mvout to scratchpad for activation + GEMMINI_CISC_CMD_I(9); + gemmini_fence(); + } + + threadblock_barrier(/*barrier_id=*/0, /*count=*/NUM_WARPS); + // activate + + // move out to dram + if (HW_TID() == 0) { + float * const dram_c_tile_start = C + tile_i * TILE_M * dim_n + tile_j * TILE_N; + ROCC_INSTRUCTION_RS1_RS2(XCUSTOM_ACC, 0, BOUND_INST, k_LOOP_WS_CONFIG_BOUNDS) + ROCC_INSTRUCTION_RS1_RS2(XCUSTOM_ACC, 0, (uint64_t) dram_c_tile_start, k_LOOP_WS_CONFIG_ADDRS_DC) + ROCC_INSTRUCTION_RS1_RS2(XCUSTOM_ACC, 0, dim_n, k_LOOP_WS_CONFIG_STRIDES_DC) + ROCC_INSTRUCTION_RS1_RS2(XCUSTOM_ACC, 0, loop_matmul_skips(1, 1, 1, 1, 0), k_LOOP_WS) + } + } + } + // last thread block complete + if (threadblock_id == NUM_CLUSTERS - 1) { + threadblock_barrier(/*barrier_id=*/0, /*count=*/NUM_WARPS); + rd_cycles_force(marker1); + if (HW_TID() == 0) { + PRINTF("\ncomplete\n"); + PRINTF("total cycles: %d\n", marker1 - marker0); + for (int i = 0; i < dim_m; i += 8) { + for (int j = 0; j < dim_n; j += 8) { + PRINTF("%d %d ", (int) (C[i * dim_n + j]), (int) (C[i * dim_n + j + 4])); + } + PRINTF("\n"); + } + } + } + vx_tmc(0); +} + +void kernel_body(int task_id, kernel_arg_t *__UNIFORM__ arg) { + const int threadblock_id = task_id / NUM_THREADS_IN_CLUSTER; + const int tid_in_threadblock = task_id % NUM_THREADS_IN_CLUSTER; + + thread_block_matmul_gemmini(arg, threadblock_id, tid_in_threadblock); +} + +int main() { + kernel_arg_t *arg = (kernel_arg_t *)KERNEL_ARG_DEV_MEM_ADDR; + + const uint32_t num_threads_in_cluster = vx_num_threads() * vx_num_warps() * CORES_PER_CLUSTER; + const uint32_t grid_size = num_threads_in_cluster * NUM_CLUSTERS; +#ifdef RADIANCE + vx_spawn_tasks_cluster(grid_size, (vx_spawn_tasks_cb)kernel_body, arg); +#else + // NOTE: This kernel assumes contiguous thread scheduling for efficient shared + // memory allocation, and therefore does not work with original vx_spawn_tasks + vx_spawn_tasks_contiguous(grid_size, (vx_spawn_tasks_cb)kernel_body, arg); +#endif + return 0; +} \ No newline at end of file diff --git a/tests/regression/sgemm_gemmini_dma/main.cpp b/tests/regression/sgemm_gemmini_dma/main.cpp new file mode 100644 index 00000000..54531062 --- /dev/null +++ b/tests/regression/sgemm_gemmini_dma/main.cpp @@ -0,0 +1,274 @@ +#include +#include +#include +#include +#include +#include +#include "common.h" + +#define RT_CHECK(_expr) \ + do { \ + int _ret = _expr; \ + if (0 == _ret) \ + break; \ + printf("Error: '%s' returned %d!\n", #_expr, (int)_ret); \ + cleanup(); \ + exit(-1); \ + } while (false) + +/////////////////////////////////////////////////////////////////////////////// + +const char* kernel_file = "kernel.bin"; +uint32_t count = 0; + +std::vector src_a_data; +std::vector src_b_data; +std::vector ref_data; + +vx_device_h device = nullptr; +std::vector staging_buf; +kernel_arg_t kernel_arg = {}; + +static void show_usage() { + std::cout << "Vortex Test." << std::endl; + std::cout << "Usage: [-k: kernel] [-n words] [-h: help]" << std::endl; +} + +static void parse_args(int argc, char **argv) { + int c; + while ((c = getopt(argc, argv, "n:k:h?")) != -1) { + switch (c) { + case 'n': + count = atoi(optarg); + break; + case 'k': + kernel_file = optarg; + break; + case 'h': + case '?': { + show_usage(); + exit(0); + } break; + default: + show_usage(); + exit(-1); + } + } +} + +void cleanup() { + if (device) { + vx_mem_free(device, kernel_arg.addr_a); + vx_mem_free(device, kernel_arg.addr_b); + vx_mem_free(device, kernel_arg.addr_c); + vx_dev_close(device); + } +} + +void generate_source_matrix(uint32_t dim_m, uint32_t dim_n, uint32_t dim_k) { + src_a_data.resize(dim_m * dim_k); + src_b_data.resize(dim_k * dim_n); + + for (uint32_t i = 0; i < src_a_data.size(); ++i) { + src_a_data[i] = static_cast(i); + std::cout << "A: " << i << ": value=" << src_a_data[i] << std::endl; + } + for (uint32_t i = 0; i < src_b_data.size(); ++i) { + src_b_data[i] = static_cast(i); + std::cout << "B: " << i << ": value=" << src_b_data[i] << std::endl; + } +} + +void generate_reference_matmul(uint32_t dim_m, uint32_t dim_n, uint32_t dim_k) { + ref_data.resize(dim_m * dim_n); + + for (uint32_t i = 0; i < dim_m; ++i) { + for (uint32_t j = 0; j < dim_n; ++j) { + float ref = 0.0f; + for (uint32_t k = 0; k < dim_k; ++k) { + ref += src_a_data[dim_k * i + k] * src_b_data[dim_n * k + j]; + } + ref_data.at(dim_n * i + j) = ref; + } + } +} + +int run_test(const kernel_arg_t& kernel_arg, + uint32_t buf_size, + uint32_t dim_m, uint32_t dim_n) { + // start device + std::cout << "start device" << std::endl; + RT_CHECK(vx_start(device)); + + // wait for completion + std::cout << "wait for completion" << std::endl; + RT_CHECK(vx_ready_wait(device, VX_MAX_TIMEOUT)); + + // download destination buffer + std::cout << "download destination buffer" << std::endl; + RT_CHECK(vx_copy_from_dev(device, staging_buf.data(), kernel_arg.addr_c, buf_size)); + + // verify result + std::cout << "verify result" << std::endl; + { + int errors = 0; + auto buf_ptr = (float*)staging_buf.data(); + for (uint32_t i = 0; i < dim_m * dim_n; ++i) { + float ref = ref_data.at(i); + float cur = buf_ptr[i]; + if (std::abs((cur - ref) / ref) > 1e-6) { + std::cout << "error at result #" << std::dec << i + << std::hex << ": actual=" << cur << ", expected=" << ref << std::endl; + ++errors; + } + } + if (errors != 0) { + std::cout << "Found " << std::dec << errors << " errors!" << std::endl; + std::cout << "FAILED!" << std::endl; + return 1; + } + } + + return 0; +} + +int main(int argc, char *argv[]) { + // parse command arguments + parse_args(argc, argv); + + if (count == 0) { + count = 1; + } + + std::srand(50); + + // open device connection + std::cout << "open device connection" << std::endl; + RT_CHECK(vx_dev_open(&device)); + + // FIXME: hardcoded + uint32_t dim_m = 64; + uint32_t dim_n = 64; + uint32_t dim_k = 64; + + generate_source_matrix(dim_m, dim_n, dim_k); + generate_reference_matmul(dim_m, dim_n, dim_k); + + uint32_t src_a_buf_size = src_a_data.size() * sizeof(src_a_data[0]); + uint32_t src_b_buf_size = src_b_data.size() * sizeof(src_b_data[0]); + uint32_t dst_buf_size = ref_data.size() * sizeof(src_a_data[0]); + + std::cout << "buffer size: " << dst_buf_size << " bytes" << std::endl; + + // upload program + std::cout << "upload program" << std::endl; + RT_CHECK(vx_upload_kernel_file(device, kernel_file)); + + // allocate device memory + std::cout << "allocate device memory" << std::endl; + RT_CHECK(vx_mem_alloc(device, src_a_buf_size, VX_MEM_TYPE_GLOBAL, &kernel_arg.addr_a)); + RT_CHECK(vx_mem_alloc(device, src_b_buf_size, VX_MEM_TYPE_GLOBAL, &kernel_arg.addr_b)); + RT_CHECK(vx_mem_alloc(device, dst_buf_size, VX_MEM_TYPE_GLOBAL, &kernel_arg.addr_c)); + + kernel_arg.dim_m = dim_m; + kernel_arg.dim_n = dim_n; + kernel_arg.dim_k = dim_k; + + std::cout << "dev_addr_a=0x" << std::hex << kernel_arg.addr_a << std::endl; + std::cout << "dev_addr_b=0x" << std::hex << kernel_arg.addr_b << std::endl; + std::cout << "dev_addr_c=0x" << std::hex << kernel_arg.addr_c << std::endl; + + // allocate staging buffer + { + std::cout << "allocate staging buffer" << std::endl; + uint32_t staging_buf_size = std::max( + src_a_buf_size, + std::max( + src_b_buf_size, + std::max(dst_buf_size, sizeof(kernel_arg_t)))); + staging_buf.resize(staging_buf_size); + } + + // upload kernel argument + { + std::cout << "upload kernel argument" << std::endl; + auto buf_ptr = staging_buf.data(); + kernel_arg.addr_a = (uint64_t) 0x20000; + kernel_arg.addr_b = (uint64_t) 0x28000; + kernel_arg.addr_c = (uint64_t) 0xc0000000ULL; + memcpy(buf_ptr, &kernel_arg, sizeof(kernel_arg_t)); + + std::cout << "uploading argument buffer to device, device mem address=" + << std::hex << KERNEL_ARG_DEV_MEM_ADDR << ", size=" << std::dec + << sizeof(kernel_arg_t) << " bytes\n"; + std::ofstream file("args.bin", std::ios::binary | std::ios::out); + if (!file) { + std::cerr << "error: failed to open args.bin for writing\n"; + exit(EXIT_FAILURE); + } + file.write(reinterpret_cast(staging_buf.data()), + sizeof(kernel_arg_t)); + file.close(); + + RT_CHECK(vx_copy_to_dev(device, KERNEL_ARG_DEV_MEM_ADDR, staging_buf.data(), sizeof(kernel_arg_t))); + } + + // upload source buffer + { + { + auto buf_ptr = staging_buf.data(); + memcpy(buf_ptr, src_a_data.data(), src_a_data.size() * sizeof(float)); + RT_CHECK(vx_copy_to_dev(device, kernel_arg.addr_a, staging_buf.data(), + src_a_buf_size)); + + std::cout << "uploading source A matrix to device, device mem address=" + << std::hex << kernel_arg.addr_a << ", size=" << std::dec + << src_a_buf_size << " bytes\n"; + std::ofstream file("input.a.bin", std::ios::binary | std::ios::out); + if (!file) { + std::cerr << "error: failed to open args.bin for writing\n"; + exit(EXIT_FAILURE); + } + file.write(reinterpret_cast(buf_ptr), src_a_buf_size); + file.close(); + } + { + auto buf_ptr = staging_buf.data(); + memcpy(buf_ptr, src_b_data.data(), src_b_data.size() * sizeof(float)); + RT_CHECK(vx_copy_to_dev(device, kernel_arg.addr_b, staging_buf.data(), + src_b_buf_size)); + + std::cout << "uploading source B matrix to device, device mem address=" + << std::hex << kernel_arg.addr_b << ", size=" << std::dec + << src_b_buf_size << " bytes\n"; + std::ofstream file("input.b.bin", std::ios::binary | std::ios::out); + if (!file) { + std::cerr << "error: failed to open args.bin for writing\n"; + exit(EXIT_FAILURE); + } + file.write(reinterpret_cast(buf_ptr), src_b_buf_size); + file.close(); + } + } + + // clear destination buffer + { + std::cout << "clear destination buffer" << std::endl; + auto buf_ptr = (int32_t*)staging_buf.data(); + for (uint32_t i = 0; i < ref_data.size(); ++i) { + buf_ptr[i] = 0xdeadbeef; + } + RT_CHECK(vx_copy_to_dev(device, kernel_arg.addr_c, staging_buf.data(), dst_buf_size)); + } + + // run tests + std::cout << "run tests" << std::endl; + RT_CHECK(run_test(kernel_arg, dst_buf_size, kernel_arg.dim_m, kernel_arg.dim_n)); + std::cout << "PASSED!" << std::endl; + + // cleanup + std::cout << "cleanup" << std::endl; + cleanup(); + + return 0; +} diff --git a/tests/regression/sgemm_gemmini_dma/sgemm_gemmini_dma b/tests/regression/sgemm_gemmini_dma/sgemm_gemmini_dma new file mode 100755 index 0000000000000000000000000000000000000000..67ade61b5a3759e630c5a613c78401d05b717369 GIT binary patch literal 28448 zcmeHwdwf*Ywg1UuBqC&jk~S)p5rZZIF#&=J?MNmMIHMCuAt+k$WHOnM)MO@|2Lyjw zqaoV4(`oV6TKlW^hppbLZT$(owL+{IKnV3#M68P91FPtXh+q&zeB^%Dew>^+%=BY> z@9*~)`}4_Jd#%0J+H0@zCP+CZkf!0?b;)p#D0k5qPM31zE|O~!a!o=` zLWl6Dgwmg6BmGqfy;kTz8U+V?DvDKZsgSe6TEQuX zD-z25z709@%U2De+Lc8jzjXDmL6loUY9|yf>1c1BGy9T`mKh!GfoS)P?ut1x=FBb$ z2TNwMbh3-`pf~@v!n|uUF1jCMrauK;3kge`n{^&T%!r|J5gFhWb3`=HD}}h9#zW zJckU5e>cWnMRPaSYc2kee|dX2;tw^}*K`B}{zh+ehhNdO z<(*PJgH7?dS;l2w`chpjdpbey)D`e z{!qZ*p|!RnFR+FD-j-F`N^g6FFeuV|9l3a->T{DI|>Hq9Rj1w##2 z_$Z^r!H#fUJL8+}543c+eTySyWxh6VNQ;DESr`V?Edp&;vp3xC(}Lj$%b3Sj*dL0xJBdHocydCEIU!Qf9t>;H>7VTmv`1>o>Q-HqS9FBk zU#%{Dji`xojn`;kWhXP<;@}$T;F+_fKZDVT7Hri*-oSFd*4i0?{Ef_uvwcY7O1@GV zMCDxxYrjg7VG6RMCZ)O21{R{(b_QGg)O45od|H@VZkeJj39M`nv}lX`;bZ*2YWhJwe z1s-=bP>^Sr%q&ep%gU0_N;Age+RT#KMyzZ!I&P-@FS{2?GTH+A5z;lfoAI79IW zkql@$2s))V;qQeQlV@ZWXk-I&Inrra;^=sE4DQ+a+T^_@+4rm{iklj|nA zUQx%>D2zjE1VJ5n&_x_BRpiHqq>c-&O{eg$kH~L=#;k%1136K--v`ACc2y>P|_|F z-Kcxe_n7EtKt|YSqK`GGilUq7=paToY@&}dsETsjL`Q=j4WhotMw;WvkYl3TB@r;s zM5q2HLxG7tK@tHSCc1e(SY)D$Ee|GDYNFHnQ-%r?Jzo+5t4#E-ndqvC{&f?*!9*`G z(U+R&XPf9vCi*!hdYg%Uu8H1dqE9l>yG`_OnCLww`gtb$4@~sQCi)r^-C?5NW1^pL zqOUd4r zPI=&z2Tpn5lm||E;J@pEBlh!7sJ$QLsBzmX^Ke|#74-b9Mz-i$>JmfWSTJR4KecQlkp+7w2GH_bp4-b_YI4$glht4-} zTF?&_{)oiY*qeGA1_Cv9NFNYzDWE6zp1ux{#y{89uB%*2T#c^S=S$SY zj9bse&tufs5%ty&iZGJ3sPXKnw6&wgRu>k4P&S~(-dAI9)Ivo)+Ia8rkw{bT@f#vn z+%lo}_`K+JeI*90vJrhPI$-bdK-8heop-^Yej3-xj_DgfkJ)~ME^*+o-s5eN=k)my z=zVTP4y8!xW+=Zd*5J&DeL1kU+J$0JxXN=(7heK*L{*Cp7#cB(4)M?+i4{^N> zCU|0B!n#f1&{x5_etKm<$9(8eW82i&pk7UeqSj7)Mh2J9LP|dk>=3(y#HO%(!`y2fCG#TLg+(%C+9w7aQ_;LGv1F^7xeCXtanFFthzv-KzYKk51)X)S=A-VCy+vY+ge=Rb>>SnEf;_f{K^n0yT#64eIqwLl#D@O>D?v)%dEK zxUMiy)@3HuWgQHM)OUk+Y;|D~h6KYR5&9t8;o|A zi>(cDWVR7nka8_yS#5f}PL2h21-cC?rH__k`YOiUoy?3npocuM4@J9G6Td6WL+vLy z>FHz4Nrg;yP_H9>!bvk2?U4R$(8x*Xjc90pbM?x7C0!^Cm5r1 zBE^HSAU?hxT54nO#`CL4Yr%Lcf>Cr6<2 zS7^isehH=bbGhAA+Aomvfrr_mHnweO9{fOYckJn*8WBfJ9l8i{`{W)N%hnTY{hM$u zqq5ygw#1#U<18IzdzXRo8cGR4HApXT(nOFj$e#{C0j_DrD-o0tn9nGKw$DXWzm$vK z!$miO495Q}857Qnxab3tGHAPBihhowBG+YH^m;D3STZJ@A7CvDWo}6sw9S{Io4M!| z5`|rG53?&#ejHIyjv5r_cCO`sr1UV$%vk$rrAE9I$rdz|~rP_*c@C2Ftp1pQh= zjqQ(&Lp5FqF@57lXe});al7qF@MSE6tYVfQT8Tty5B5U(X@39>>m5uNVc#{R5#h}v$%1{sZSW}EU??c9F-$iK?!VqbkXMk$wl(~25suLiGv^I^zg-eDt`#GQXcThq@*oZk5Tkn2AST9DBj)WQ?a zT#|X1@!B#|<#okqUc(qaWL_`LfZPO;*Ut&YotGo8P0Do_+VnOMD8tJrLrk41!&`ZV z_mN-)&+tyfPb%E2R3cuuVpHM#B)9euFI*AyyW>|oP*+c)ZeycoPhmvP`1P z$u)9T?BJO@NXZk-THC}_nJ*j7JYp)XJ{@uwh|GUWdAyHhP&ION*&qFD-){Tr6&Pr1 z6U`%aWyk8uP}MW2e;haq?b@AKkWpXxo_+O|U~$I=)i~b9@%$M`SBtl(n@?nVDo@yN z-;L~!RNM2ipP=W&4fdiPN9@-HE50nd2+8#F_l&L}*sSxp58=>6`T3-TOvCdgnxNAh z?1Y>i&{~7NNcA2lJz~#;el-q0STvxIgFE|aKM#7|1|&T+Yt$EWW|+j7TkVGiAh1Q> zOP1k9g3Qpc-SIzVP{T%Fw6A^xZ;&3Wfg;uRO)G0WvFBX9+cWgv9OSh=6UIt!_Wd5K z;kv$K(I(70C!re_nckRez3qv4kTVrWe}zg(27BT}ON_6;YkU)1AjHSOh*HcdB6pHN z&d4s}$+2%be4vMqYjPob7L_gGeEniHg>I(M_B`UXw6b&#B9_X(c9gY0hdbvLWGH53 z6-=JmL6z=_KOlPcJNDJDqm8(FZYrd%{UeIoH$@6-*!=1P?t~d|hMZxnA&m*=<;Yy$ zPfm*4E@Os}TYty2s(lCTtM7oOo|{Q$4r2CAQ5r2KlGH+>^<}_8I1e?6~VH`=i@w(+e*YdYANmfKEDoJ8BuD zdTIiAXQ!}DFeTQZO|!Sfha|1U)QUA==Du8h5| zj1Myo)mFmt=r7R83z9ndYS^kMqyjlXOQ_$28BxL+!lWPGTzml4m&m?dt=w#1JqZrQ zBL8vB4uz-!X^*EQ}E*dn2aokuk`zkMZVT&qWpj*dKiz@BccB@K*aB zAqd-V^WX{PlwP(M-p1x+<@kBPJc)8_UMSH8L$tEN;Ka+1Az~#PA$oNx;-G%U5Osoa zg?P#2KrB&%b<)KMGC*_mx<598v8T55Os(O=WMQ3E3OEC4;W*4HbutC1ser@&!{bNvRxZ}gTC;lff%3+xQ1C4An z&SQpvy70tC^f^q&7%?bUm3k@<+gD%66M1(!8wky(;G5I2r$N3s2H$*hd=@a&QVWzu zYle=ZK&&0|nF{*&4exQk907F4>!Bcj*F~d#+5Y$ZQiJh8`sM9H;g|J#HypvX6rDRk z)ML;r{gOi?2=~h(aM3H+?3Z==Ypj|^{bHKe-3#0K)JOQ`6a74(a(-RB)r0Lt?4E{@Jv-xJ{!?RLV!Ts} zx9isuy#f;YIuw-V&#}W>4gGvRJ&)$ndLb*)~?PgJl1acB++c*l)WFlAg*V^|7NKd(DCRjHB+#XFc`>L+;J{GvOtD z9_**Cl*leBI|vulVmCMYA)enhSNz3bSJVOt&epS=A||Yt>q2s5ZjOw?Kj=GAM0nV>YYEqJ3@c%9rLT1<?N8~JPBpkJ#c4iKX4x_KgK6| zFaH#NBENMa{lLG$Up~Ak_5bsBN`JYo_9UEpu~qN73Tj)ZV%ZO2W~O zh{C>Vb9f^HcU+*%b$ES|sJFwZOn3OZyZpY0zs0G{4@Ltm4#mOv!{;mWU2acZ?fHsp zt^*P)yd6=$Q>g|Ln0v$Awt4tO$R7}efnk9F)R1A~i#r&D+uv(!9>n?oq5c3G)$VD1cv=w+EI> zcZhuWM}`i3)`ah-0uhv*%y9~9#wth%mo(#BruG0nZ!7Vd;>|4X>+pL+uLgu(|YZ~gb1)hb~F3)tw zg+iB6I4{$fR$uc!s;@b{zLs$0AJyg)<>k(y1}od9Xt|C^P;`dr0y^Lr2i2QD9Cix# z@(w=NK_B>G#DEOznX);vfrCE5L#ox3;ayj4 z?ILs&-kp+~5!H9FwP@7gTBKMM$7qih{iD3R<$2R+PO50qIqIKqFdFjt9WF^ z{)+xdQ(mh*&FSqKZ73=2ss4Xr4|--Q&zl<>_c(?hjfNjnZJ@m*wMSD~TiQFd&ZKdG zD1l;gw{P*cdhzPeyj>W@TVy-Kgya?(Czv?+*xFCFue&Vc+^lat0G#$CD_$8H*#o#? z@5l)K5@H8n0bs=&BO?>D6=f;l8Z1=`FdLo@yaKC*0q_@L!Q279Zot)mI^ey4D)twf zuvWSUo2b77r1OkY(02gV5+C4Fz#JSq=){jhspH`5rsFo~*8$f6Qkckrl5yiAC8Ikp zJ()MEe8& zeqf@J&K*RDtqp)mZEjxXg4`UU@5A2*pnCz8#@xJ{b8{3G(c}25g`XR-&{O8;=KVO+ zlUwkktm<4xBHNu?^g~;1Zt3l(sks%sW9oC8vSwzD&8=|dmb!9_z*3!CP@S7sotrcN z3@Quxya4u;0V+#K7Z`-DYD1TcbX}fXm06QYlY8s-zdXD1IW@Pccg*dl{m_=k{!!MC zGc#{YnQiSEtPFobe*3W3R!{k|GSpJObyNnHFATo@G-TI1W`1r%=I)fb++0@<>f#-2 z=Q(J}%yiWymq9h@s>-@IGo`Z2nLiq#ONBqqC%djm_D3E0gV`nh@x7Fq7MzjkP35P$ zlz!rE6}h1uH+Wb}9`lPwa87rwY|qp#sO=7*-EBM_UjUDRajcxF>WbC-<+=ODW&J7# z=GB~3@h;jD#;!99raUW0^MtgyD#1zZaW?)Y!3GCl_7*%4=p6Y6@cTQM&Ywpt0i+edpMJ!-&2FQfnLkI{>Vn*=Kq|MfZ>>Loo(AZ_8r}#aBN^Xp{1-Un zv(qpB=~>F~*c~w7rE%jQIZOHVcxqtBa>w6)mhww`4}>?KX+!*_Gi^xy_{_=Rhj{iG zDD(gSfa56^*x-bEE#)Y=kAf8*g%ReDn=D1soOd;NSo!D=h56kCiFNaY%kA zPRBnKXk4OzUxKh8zYC`^lmgWyg`;1N1PQJc`N_IiD;Q`vqOeD#^Vx}_JT1m+YIhW< z%}}uVUWh3@*Yk8a#LDtg8>Aq|*spU`$q@yR^tHJ>LE7CW9EFy`g3SLsflEH@JhSqF zosqEX$naglk$7J+T-u4_6^6Hn^iKsY{UX%=j{$kFnN?w_gHx&mY!I+Xz%BuM1Y9HF zS^?JyI3VCI0rv@bSU_B+#X_Ed4gpIAtP-$6z$O8^1nd!Tjeu(fTqodwfV%|TC*WZL z<)u1#Q@9-t0ZRp}60kwQCIPzy>=AH{fLzNz{>*9OGFjEhvbZy2uA^v4b2Jc%Ix2BX zM(K>QD8tIWKeM!?6xU`Hb9%B|Dl7}VyFfNH6GImEASzPQvy`PN>DkInDd{$&ADh(U zc8b%C{+mo6qsV@qOwUnd`%k8yuE>6qOdqSr{+Uca!|2D!bc|{aGlj>qaI2ytB|TSJ zo1%WR6uYu6B|T4ZOiiVKf+ELFb7~gdG*xA$@OTzx%kq9cb7~geoNE@CDUv^5S#w1y z`qvaWKATgsl&>ptTu!DJD5=JUER6PY98KoODBonJ@OYN#{wZ^6mT{}(C^ZAoY~`bo zQ3{XIc%O;4NnRPpY2HGC-ZMm(es&NMf!t1XdASCgKY&(+zYIuq_z+=n4i^HX%9nP* zNl!y5=aWyG!xO#7LZ?BS=<>ZQ`TG!|eAhK|3YC+>UHHpL;r}NS-(~z0l^>>P52?!gF=djXy)DKIFq38h4#r?SXu62e2r>e)vjGm`73IBM7z_c{{m82hbw)r@t^z%H>&quxmO`I<6{7xD@ zy=myzLh#JYlgI5}rs4lB=wwfhFFD`egPyAW{AU_He@#PwI}M%IQ4`P(^DN~(Ci1o3 z_csBKO6C7)pp*T1K5oBkhqKf0)BZ^+J%#8O)IJ?zoV%FklsRemf0BlNb{c!0PQyP$`NE$)B40i}B6sE=X~if!4|E!T6u~c3 zCSlyjgPtx?FOohjjh=aF=(m7Q?XdJRu7Qusit=z8{{5g+zE;0|l!m_$?~7FJ!OiIC z*J9o17X`Qmbn?$(G0w>L=1-$1A^01FA0+=hY4{&WL*EH{s`tfvj34j!N^YPdi$fcY z1HxuAm;}1ROc3#kH2e*Wj`3gk0nLMjCeX>A!{Yr?hWe)PFKP7LoreBc8u}K{9ZX~L zHhVj? z792!{HE*04G0`}}fxFk}z zW8wVyi|cSFsH@sjM_Na(XsoJY7bN1ML%PS1T^7o()f5T*=0twmb+^{l z)y`x$Va_aH?hp7wI5NeJsW|ub2Yh}FCnKHFj_-c=yW&E}(QN4IM9oj7s`)vWWrig4bvVQ5J zQ!w-PnS3Tj7klE?&(Tuj#!buRuI9^j&DV#H%24}4K}CCD7u6)Inm9$mRz*va9EUrob~sw15T2q+ih8oOTN09ZWd1BL-SzQ=K8`> z_QIgw1JUnP=wO{GFy9s}FNIFhfSTvOgJ5GWIxXGCD>$ULQ(OUERb{%CH)(8;FBBRZ zv}|O zlmvnif64Mdv;;qT!ZqEIRVGq%v>pEvqrF9taEDN=eJA0J7jP65=WN ztyh?ScE#8Pq9HBh@9>g!oy)qXFibjBv-_Xv3jcL;;YN8~#x zFV|)5ppl4_m;3z^%5yxhuweE7_dumHGAS?51te@0a?*auC*k*zPG@csm*)ot6!K(0 z*(vQ8WF>(J6;aB|a|Q|JIiSp6%FFW87ZxO6jXw(VTv0-K4oUK~XK9tcA2jlb)UP;z zP;js(R%pq=+J5duvRU5Y;QSKy$btzq*g0oG!hZ*5mR~!Cr%NdLIFZ3NIY>E4dkPVA{_^}qLV2Do`K|ujCgcq_RS8Z3 zb)#Z9-J1WiAXENQUY_Sj_y89(p4R;LTIA)qkA&6^FWZk)xZf%-&I9B*(E`aRAlYgT zp8%nwZ_(|Ib0qoyoUG-Q`Ae7ug?5v?Ja1a+;kigYj*Yf|Nw-6uTrBgK=Th?jb2)@# zssB=ta?-Bff<|SM^78zu=St#MlpafgrM!gnyEL=B(!|8@!x0W00+m3@Nu17~h$i!| zYT_vlAzvyJgg}*m5?mniFO>uVrTjDtoouCGhO+JP6#i@w`FD#Sc+l?_X5mlTFUK!B kRfBuzjhmHZ(;=a_C0u%OC+0X9c<#Q*>R literal 0 HcmV?d00001