diff --git a/benchmarks/opencl/sgemm/Makefile b/benchmarks/opencl/sgemm/Makefile index 1bd573fd..3469fcdf 100644 --- a/benchmarks/opencl/sgemm/Makefile +++ b/benchmarks/opencl/sgemm/Makefile @@ -33,7 +33,7 @@ QEMU_LIBS = -Wl,--whole-archive lib$(PROJECT).a -Wl,--no-whole-archive $(POCL_LI PROJECT = sgemm - SRCS = main.cc +SRCS = main.cc all: $(PROJECT).dump $(PROJECT).hex diff --git a/benchmarks/opencl/spmv/1138_bus.mtx b/benchmarks/opencl/spmv/1138_bus.mtx new file mode 100755 index 00000000..09591c70 --- /dev/null +++ b/benchmarks/opencl/spmv/1138_bus.mtx @@ -0,0 +1,2610 @@ +%%MatrixMarket matrix coordinate real symmetric +%------------------------------------------------------------------------------- +% UF Sparse Matrix Collection, Tim Davis +% http://www.cise.ufl.edu/research/sparse/matrices/HB/1138_bus +% name: HB/1138_bus +% [S ADMITTANCE MATRIX 1138 BUS POWER SYSTEM, D.J.TYLAVSKY, JULY 1985.] +% id: 1 +% date: 1985 +% author: D. Tylavsky +% ed: I. Duff, R. Grimes, J. Lewis +% fields: title A name id date author ed kind +% kind: power network problem +%------------------------------------------------------------------------------- +1138 1138 2596 +1 1 1474.779 +5 1 -9.017133 +563 1 -5.730659 +2 2 9.136654 +10 2 -3.405995 +563 2 -5.730659 +3 3 69.61468 +11 3 -8.810573 +34 3 -31.15265 +35 3 -16.06684 +104 3 -4.86926 +475 3 -8.715357 +4 4 68.60106 +7 4 -34.62025 +27 4 -.4755112 +101 4 -28.66497 +102 4 -.7463244 +103 4 -4.093998 +5 5 13.88805 +9 5 -4.870921 +6 6 116.8288 +7 6 -10.96124 +37 6 -56.81818 +98 6 -10.88139 +103 6 -38.16794 +7 7 52.76726 +37 7 -4.773726 +101 7 -.7772424 +102 7 -1.072973 +103 7 -.5618293 +8 8 30.25788 +26 8 -3.752486 +35 8 -.4975867 +724 8 -26.0078 +9 9 7.174451 +10 9 -.905633 +104 9 -1.397898 +10 10 5.709526 +104 10 -1.397898 +11 11 27.18584 +12 11 -1.238697 +38 11 -7.418397 +566 11 -9.718173 +12 12 1.238697 +13 13 7.142646 +34 13 -1.85192 +104 13 -5.290726 +14 14 12.95337 +413 14 -12.95337 +15 15 1696.756 +16 15 -30.67485 +17 15 -30.4878 +18 15 -30.39514 +19 15 -30.39514 +411 15 -1574.803 +16 16 30.67485 +17 17 30.4878 +18 18 30.39514 +19 19 30.39514 +20 20 5746.122 +21 20 -5714.287 +37 20 -18.24817 +102 20 -13.58696 +21 21 5735.022 +22 21 -5.205622 +23 21 -5.186722 +24 21 -5.181347 +25 21 -5.162622 +22 22 5.205622 +23 23 5.186722 +24 24 5.181347 +25 25 5.162622 +26 26 4.371336 +35 26 -.6188502 +27 27 9.123474 +28 27 -2.373605 +29 27 -2.439024 +30 27 -1.190334 +101 27 -2.644999 +28 28 2.373605 +29 29 2.439024 +30 30 1.190334 +31 31 1.688214 +32 31 -.6254691 +100 31 -1.062744 +32 32 1.688214 +100 32 -1.062744 +33 33 .6581979 +100 33 -.6581979 +34 34 56.31459 +104 34 -10.13479 +553 34 -13.17523 +35 35 10018.3 +104 35 -1.112756 +710 35 -10000 +36 36 10000 +711 36 -10000 +37 37 80.33353 +102 37 -.4934373 +38 38 74.99123 +39 38 -15.67398 +98 38 -46.2963 +412 38 -5.602555 +39 39 16.41742 +99 39 -.7434391 +40 40 245.9335 +41 40 -126.5823 +43 40 -55.24862 +45 40 -64.10257 +41 41 988.9011 +42 41 -28.98551 +44 41 -833.3333 +42 42 28.98551 +43 43 4403.075 +146 43 -4347.826 +44 44 10833.33 +486 44 -10000 +45 45 118.7474 +49 45 -54.64481 +46 46 10000 +48 46 -10000 +47 47 104.9318 +48 47 -104.9318 +48 48 20183.36 +54 48 -78.43137 +506 48 -10000 +49 49 288.754 +50 49 -126.5823 +53 49 -107.5269 +50 50 988.9011 +51 50 -28.98551 +52 50 -833.3333 +51 51 28.98551 +52 52 874.3169 +125 52 -40.98361 +53 53 1586.098 +54 53 -1428.572 +55 53 -50 +54 54 1957.633 +425 54 -119.0476 +445 54 -184.5238 +447 54 -147.0588 +55 55 50 +56 56 29.32551 +63 56 -29.32551 +57 57 112.3596 +66 57 -112.3596 +58 58 15.64945 +70 58 -15.64945 +59 59 24.3309 +67 59 -24.3309 +60 60 104.1667 +68 60 -104.1667 +61 61 125 +130 61 -125 +62 62 50.76142 +129 62 -50.76142 +63 63 10059.54 +64 63 -30.21148 +65 63 -10000 +64 64 159.7102 +71 64 -40.98361 +131 64 -16.05136 +226 64 -72.46377 +65 65 10015.92 +1095 65 -15.92357 +66 66 442.0023 +67 66 -97.08738 +68 66 -24.27184 +74 66 -32.65326 +183 66 -147.0588 +226 66 -28.57143 +67 67 167.4071 +68 67 -30.67485 +69 67 -15.31394 +68 68 289.2932 +74 68 -47.5921 +90 68 -32.33649 +201 68 -50.25126 +69 69 190.7525 +70 69 -175.4386 +70 70 225.5077 +182 70 -4.955401 +196 70 -5.170631 +1063 70 -4.623209 +1065 70 -16.47446 +1074 70 -3.195909 +71 71 96.53917 +72 71 -55.55556 +72 72 84.92064 +73 72 -5.555555 +121 72 -23.80952 +73 73 5.555555 +74 74 384.9024 +75 74 -40.32258 +76 74 -25.44529 +78 74 -34.3871 +80 74 -15.82279 +119 74 -188.6792 +75 75 693.1041 +204 75 -645.1613 +810 75 -4.608295 +918 75 -3.012048 +76 76 147.3945 +77 76 -30.03003 +78 76 -36.36364 +127 76 -55.55556 +77 77 5030.03 +1050 77 -5000 +78 78 262.67 +79 78 -90.90909 +452 78 -101.0101 +79 79 564.4214 +416 79 -48.07692 +417 79 -48.07692 +454 79 -377.3585 +80 80 657.1666 +81 80 -67.1141 +447 80 -476.1905 +452 80 -98.03922 +81 81 420.0095 +448 81 -158.7302 +450 81 -175.4386 +464 81 -18.72659 +82 82 4.975124 +94 82 -4.975124 +83 83 17.24138 +94 83 -17.24138 +84 84 17.57469 +95 84 -17.57469 +85 85 439.0805 +87 85 -172.4138 +126 85 -266.6667 +86 86 6807.496 +87 86 -5000 +241 86 -46.51163 +251 86 -71.94245 +253 86 -74.62687 +267 86 -94.78674 +269 86 -277.7778 +283 86 -138.8889 +291 86 -49.01961 +293 86 -57.14286 +294 86 -57.14286 +302 86 -689.6552 +315 86 -250 +87 87 5319.472 +88 87 -147.0588 +88 88 147.0588 +89 89 65.3412 +105 89 -38.31417 +115 89 -27.02703 +90 90 64.83059 +91 90 -32.4941 +91 91 645.056 +92 91 -163.6515 +108 91 -67.56757 +115 91 -24.19992 +116 91 -357.1429 +92 92 258.8896 +93 92 -95.2381 +93 93 612.1849 +94 93 -10.90512 +252 93 -131.5789 +264 93 -12.67427 +271 93 -66.0066 +288 93 -65.35947 +292 93 -31.84714 +293 93 -38.31417 +294 93 -38.31417 +322 93 -32.15434 +323 93 -34.8432 +324 93 -24.03846 +325 93 -14.40922 +326 93 -16.50165 +94 94 464.0852 +96 94 -4.800768 +264 94 -239.4475 +318 94 -110.6696 +325 94 -76.04562 +95 95 76.57204 +97 95 -8.285004 +262 95 -9.67118 +268 95 -7.142857 +300 95 -33.8983 +96 96 10022.9 +145 96 -7.24113 +704 96 -10.85776 +705 96 -10000 +97 97 12.12083 +268 97 -3.835827 +98 98 57.17769 +99 99 3.134068 +413 99 -2.390629 +100 100 40.73998 +725 100 -35.58719 +732 100 -2.369107 +101 101 34.2926 +102 101 -.754091 +103 101 -1.451295 +102 102 47.04131 +103 102 -30.38753 +103 103 10074.66 +478 103 -10000 +104 104 24.20333 +105 105 420.7618 +106 105 -12.07729 +109 105 -370.3704 +106 106 12.07729 +107 107 32.15434 +108 107 -32.15434 +108 108 1008.813 +110 108 -909.0909 +109 109 10370.37 +1029 109 -10000 +110 110 10974.79 +111 110 -41.84558 +328 110 -10000 +367 110 -10.77006 +370 110 -13.08045 +111 111 10056.95 +367 111 -7.27802 +368 111 -7.824726 +701 111 -10000 +112 112 301.7592 +113 112 -7.420491 +370 112 -8.624407 +686 112 -285.7143 +113 113 10040.49 +114 113 -7.624856 +115 113 -25.44529 +690 113 -10000 +114 114 10007.63 +1009 114 -10000 +115 115 76.67225 +116 116 436.123 +117 116 -10.9529 +126 116 -68.02721 +117 117 10.9529 +118 118 5.521811 +119 118 -5.521811 +119 119 228.3307 +123 119 -34.12969 +120 120 126.5823 +121 120 -126.5823 +121 121 272.2888 +122 121 -70.87661 +504 121 -51.02041 +122 122 152.6848 +123 122 -40.48583 +124 122 -41.32232 +123 123 111.929 +125 123 -37.31343 +124 124 97.81158 +125 124 -32.67974 +127 124 -23.80952 +125 125 363.2857 +126 125 -81.38689 +129 125 -70.92198 +546 125 -100 +126 126 478.1926 +129 126 -62.1118 +127 127 84.92064 +128 127 -5.555555 +128 128 5.555555 +129 129 183.7952 +130 130 356.9032 +133 130 -136.9863 +173 130 -51.81347 +213 130 -43.10345 +131 131 896.646 +132 131 -75.18797 +133 131 -769.2308 +136 131 -36.17595 +132 132 101.0104 +144 132 -4.409171 +742 132 -21.41328 +133 133 918.7799 +134 133 -12.56281 +134 134 12.56281 +135 135 10030.19 +136 135 -25.44529 +139 135 -4.748338 +740 135 -10000 +136 136 99.82595 +141 136 -38.20471 +137 137 108.031 +139 137 -72.46377 +141 137 -27.39726 +761 137 -8.169934 +138 138 217.898 +141 138 -28.98551 +877 138 -59.88025 +882 138 -129.0323 +139 139 90.06557 +140 139 -12.85347 +140 140 45.52996 +829 140 -18.62891 +830 140 -14.04757 +141 141 94.58748 +142 142 36.70324 +258 142 -21.55172 +366 142 -15.15152 +143 143 20014.59 +144 143 -12.5 +743 143 -2.090738 +745 143 -10000 +826 143 -10000 +144 144 10016.91 +827 144 -10000 +145 145 10022.01 +318 145 -14.77105 +703 145 -10000 +146 146 7289.003 +147 146 -2941.177 +147 147 12941.18 +148 147 -10000 +148 148 10000 +149 149 5.205622 +166 149 -5.205622 +150 150 73.52941 +173 150 -73.52941 +151 151 13.29787 +175 151 -13.29787 +152 152 10.40583 +180 152 -10.40583 +153 153 34.48276 +183 153 -34.48276 +154 154 13.71742 +207 154 -13.71742 +155 155 28.98551 +198 155 -28.98551 +156 156 144.9275 +213 156 -144.9275 +157 157 38.46154 +219 157 -38.46154 +158 158 28.0112 +226 158 -28.0112 +159 159 9.64196 +182 159 -4.730369 +205 159 -4.911591 +160 160 21.37365 +172 160 -10.01001 +233 160 -11.36364 +161 161 10.90321 +222 161 -4.730369 +743 161 -6.17284 +162 162 87.51172 +163 162 -46.08295 +169 162 -11.53403 +209 162 -7.451564 +223 162 -12.61034 +232 162 -9.832842 +163 163 145.9383 +186 163 -46.51163 +201 163 -13.6612 +227 163 -39.68254 +164 164 5.205622 +166 164 -5.205622 +165 165 117.6942 +166 165 -60.60606 +167 165 -17.33102 +175 165 -11.18568 +194 165 -28.57143 +166 166 341.3369 +192 166 -80 +201 166 -14.88095 +218 166 -175.4386 +167 167 34.66204 +175 167 -17.33102 +168 168 69.63064 +180 168 -9.389671 +209 168 -60.24096 +169 169 25.6384 +187 169 -14.10437 +170 170 30.28655 +188 170 -22.77904 +194 170 -7.507507 +171 171 52.2535 +178 171 -39.21569 +217 171 -13.03781 +172 172 66.85269 +173 172 -42.55319 +222 172 -4.873294 +755 172 -9.416196 +173 173 206.2103 +184 173 -38.31417 +174 174 50.63389 +211 174 -12.7551 +216 174 -37.87879 +175 175 53.63491 +179 175 -11.82033 +176 176 860.8028 +177 176 -172.4138 +178 176 -20.98079 +207 176 -667.4082 +177 177 276.5804 +207 177 -104.1667 +178 178 88.17661 +214 178 -15.06024 +217 178 -12.9199 +179 179 23.64066 +203 179 -11.82033 +180 180 28.49115 +209 180 -8.695652 +181 181 23.14815 +183 181 -23.14815 +182 182 61.50294 +183 182 -29.94012 +205 182 -16.72241 +221 182 -5.154639 +183 183 398.0187 +184 183 -91.74312 +201 183 -35.14939 +226 183 -36.49635 +184 184 192.1349 +193 184 -41.40787 +213 184 -20.6697 +185 185 8.064516 +187 185 -8.064516 +186 186 126.2011 +187 186 -41.66667 +218 186 -38.02281 +187 187 107.524 +188 187 -29.67359 +206 187 -8.695652 +209 187 -5.319149 +188 188 52.45263 +189 189 41.49792 +211 189 -11.46789 +216 189 -30.03003 +190 190 33.0033 +199 190 -33.0033 +191 191 1149.633 +195 191 -10.71811 +197 191 -14.74926 +198 191 -1111.111 +211 191 -13.05483 +192 192 824.6421 +199 192 -666.6667 +218 192 -63.29114 +220 192 -14.68429 +193 193 247.007 +198 193 -60.60606 +199 193 -106.383 +208 193 -38.61004 +194 194 78.9974 +216 194 -42.91846 +195 195 36.89613 +211 195 -26.17801 +196 196 5.170631 +197 197 36.48839 +212 197 -21.73913 +198 198 1200.703 +199 199 806.053 +200 200 60.0913 +201 200 -43.87481 +202 200 -16.21649 +201 201 172.6985 +218 201 -14.88095 +202 202 20.9357 +203 202 -4.719207 +203 203 16.53954 +204 204 645.1613 +205 205 28.7262 +221 205 -7.092198 +206 206 18.89453 +216 206 -10.19888 +207 207 881.1484 +208 207 -84.38818 +224 207 -11.46789 +208 208 122.9982 +209 209 81.70734 +210 210 1796.925 +215 210 -11.21076 +217 210 -1666.667 +218 210 -119.0476 +211 211 84.37634 +212 211 -20.9205 +212 212 42.65963 +213 213 282.2301 +504 213 -73.52941 +214 214 65.82166 +217 214 -50.76142 +215 215 11.21076 +216 216 148.1264 +218 216 -27.10027 +217 217 1743.386 +218 218 437.7814 +219 219 150.9842 +220 219 -86.20689 +230 219 -26.31579 +220 220 100.8912 +221 221 18.99606 +222 221 -3.15856 +225 221 -3.590664 +222 222 91.18269 +225 222 -26.24672 +233 222 -49.50495 +743 222 -2.668802 +223 223 54.62715 +232 223 -42.01681 +224 224 18.73533 +1026 224 -7.267442 +225 225 29.83738 +226 226 165.5428 +227 227 85.76549 +228 227 -46.08295 +228 228 97.42727 +229 228 -27.62431 +231 228 -12.7551 +232 228 -10.96491 +229 229 27.62431 +230 230 26.31579 +231 231 31.20529 +232 231 -18.45018 +232 232 81.26475 +233 233 60.86859 +234 234 29.17592 +236 234 -14.55604 +307 234 -14.61988 +235 235 2117.081 +236 235 -67.56757 +243 235 -14.59854 +270 235 -2000 +272 235 -9.208103 +298 235 -12.85347 +299 235 -12.85347 +236 236 90.2339 +286 236 -8.1103 +237 237 123.4571 +261 237 -2.818489 +287 237 -18.97533 +309 237 -69.68641 +366 237 -4.004806 +702 237 -27.97203 +238 238 188.1504 +239 238 -41.25455 +270 238 -146.8959 +239 239 59.24802 +270 239 -15.33742 +281 239 -2.656042 +240 240 7.917656 +259 240 -7.917656 +241 241 2888.018 +242 241 -185.1852 +246 241 -89.28572 +253 241 -123.4568 +257 241 -92.59259 +266 241 -123.4568 +275 241 -13.44086 +279 241 -31.74603 +289 241 -30.03003 +291 241 -833.3333 +292 241 -833.3333 +293 241 -138.8889 +294 241 -138.8889 +299 241 -13.28021 +310 241 -16.80672 +321 241 -88.49557 +327 241 -89.28572 +242 242 314.7918 +244 242 -57.14286 +245 242 -72.46377 +243 243 14.59854 +244 244 118.4926 +272 244 -61.34969 +245 245 95.93795 +298 245 -23.47418 +246 246 113.1521 +310 246 -23.86635 +247 247 582.7703 +248 247 -270.2703 +255 247 -312.5 +248 248 502.8284 +255 248 -232.5581 +249 249 1666.667 +282 249 -1666.667 +250 250 1123.714 +257 250 -92.59259 +263 250 -78.74016 +289 250 -119.0476 +317 250 -833.3333 +251 251 210.8313 +267 251 -138.8889 +252 252 288.2195 +310 252 -21.50537 +313 252 -135.1351 +253 253 198.0837 +254 254 383.7719 +255 254 -208.3333 +285 254 -175.4386 +255 255 910.2564 +258 255 -31.44654 +260 255 -71.94245 +263 255 -53.47594 +256 256 10133 +290 256 -52.35602 +317 256 -10000 +321 256 -80.64516 +257 257 185.1852 +258 258 261.8486 +261 258 -128.2051 +327 258 -80.64516 +259 259 20.85427 +278 259 -12.93661 +260 260 190.9901 +289 260 -119.0476 +261 261 131.0236 +262 262 183.095 +268 262 -26.17801 +276 262 -62.5 +314 262 -84.74576 +263 263 132.2161 +264 264 383.7008 +311 264 -131.5789 +265 265 1017.575 +276 265 -999.9999 +300 265 -17.57469 +266 266 246.9136 +307 266 -123.4568 +267 267 566.0127 +282 267 -144.9275 +284 267 -77.51938 +302 267 -109.8901 +268 268 158.6071 +270 268 -53.88283 +301 268 -67.56757 +269 269 486.1111 +284 269 -208.3333 +270 270 2317.201 +275 270 -12.77139 +279 270 -8.257638 +280 270 -29.32551 +318 270 -34.36426 +320 270 -16.36661 +271 271 66.0066 +272 272 70.5578 +273 273 1295.094 +274 273 -303.0303 +303 273 -714.2858 +315 273 -277.7778 +274 274 606.0606 +312 274 -303.0303 +275 275 26.21225 +276 276 1062.5 +277 277 5086.957 +285 277 -86.95652 +289 277 -5000 +278 278 27.55649 +309 278 -14.61988 +279 279 40.00367 +280 280 29.32551 +281 281 2.656042 +282 282 1811.594 +283 283 214.6465 +292 283 -75.75758 +284 284 285.8527 +285 285 262.3951 +286 286 8.1103 +287 287 18.97533 +288 288 127.4713 +291 288 -62.1118 +289 289 5268.125 +290 290 52.35602 +291 291 944.4647 +292 292 940.938 +293 293 234.3459 +294 294 234.3459 +295 295 38.31417 +298 295 -38.31417 +296 296 101.0101 +297 296 -101.0101 +297 297 118.4317 +299 297 -17.4216 +298 298 74.64183 +299 299 43.55529 +300 300 122.395 +696 300 -70.92198 +301 301 153.7745 +314 301 -86.20689 +302 302 799.5453 +303 303 1428.572 +312 303 -714.2858 +304 304 1380.582 +305 304 -263.1579 +315 304 -208.3333 +329 304 -909.0909 +305 305 459.2363 +315 305 -196.0784 +306 306 37.87879 +322 306 -37.87879 +307 307 138.0767 +308 308 81.30082 +323 308 -81.30082 +309 309 84.30629 +310 310 62.17845 +311 311 200.5445 +326 311 -68.96552 +312 312 1295.094 +315 312 -277.7778 +313 313 135.1351 +314 314 170.9527 +315 315 1688.798 +316 315 -322.5806 +319 315 -125 +323 315 -31.25 +316 316 360.1746 +322 316 -37.59399 +317 317 10833.33 +318 318 169.825 +704 318 -10.02004 +319 319 1236.111 +329 319 -1111.111 +320 320 16.36661 +321 321 169.1407 +322 322 107.6271 +323 323 147.394 +324 324 51.89362 +325 324 -27.85515 +325 325 118.31 +326 326 85.46717 +327 327 169.9309 +328 328 10000 +329 329 2020.202 +330 330 2.409639 +335 330 -2.409639 +331 331 .8810573 +336 331 -.8810573 +332 332 1.666667 +337 332 -1.666667 +333 333 2.608922 +350 333 -2.608922 +334 334 1.446132 +350 334 -1.446132 +335 335 30.26479 +337 335 -27.85515 +336 336 218.2724 +337 336 -217.3913 +337 337 260.9173 +338 337 -3.840246 +339 337 -3.955696 +340 337 -6.208189 +338 338 19.08415 +340 338 -15.2439 +339 339 19.1996 +340 339 -15.2439 +340 340 59.70203 +341 340 -8.1103 +342 340 -14.89573 +341 341 20.88169 +342 341 -12.77139 +342 342 66.7209 +343 342 -5.675369 +344 342 -10.71237 +345 342 -4.215851 +370 342 -18.45018 +343 343 15.14507 +346 343 -9.469697 +344 344 28.10368 +346 344 -17.3913 +345 345 26.63738 +346 345 -22.42153 +346 346 102.5212 +347 346 -14.08451 +349 346 -20.70393 +367 346 -18.45018 +347 347 54.57034 +348 347 -40.48583 +348 348 79.09587 +349 348 -38.61004 +349 349 83.58582 +350 349 -24.27184 +350 350 28.3269 +351 351 7.374631 +366 351 -7.374631 +352 352 14.91256 +353 352 -5.289326 +408 352 -2.934273 +409 352 -6.688963 +353 353 17.04893 +363 353 -3.816794 +382 353 -7.942812 +354 354 28.97867 +382 354 -6.199628 +383 354 -22.77904 +355 355 2.164971 +375 355 -2.164971 +356 356 13.59556 +383 356 -5.344736 +398 356 -8.250825 +357 357 1.307306 +394 357 -.5700604 +395 357 -.7372456 +358 358 46.94836 +479 358 -46.94836 +359 359 18.55447 +474 359 -18.55447 +360 360 8.05153 +715 360 -8.05153 +361 361 509.4518 +479 361 -9.451796 +480 361 -500 +362 362 22.4661 +403 362 -11.03753 +714 362 -11.42857 +363 363 96.66204 +364 363 -9.149131 +365 363 -9.149131 +373 363 -9.149131 +374 363 -9.149131 +388 363 -34.96503 +395 363 -4.926108 +407 363 -10.75847 +474 363 -5.599104 +364 364 9.149131 +365 365 9.149131 +366 366 41.86838 +699 366 -15.33742 +367 367 307.3316 +368 367 -104.1667 +372 367 -166.6667 +368 368 111.9914 +369 369 72.9927 +371 369 -72.9927 +370 370 40.15504 +371 371 239.6594 +372 371 -166.6667 +372 372 333.3333 +373 373 9.149131 +374 374 9.149131 +375 375 122.031 +376 375 -2.019386 +377 375 -2.019386 +378 375 -2.019386 +379 375 -1.959632 +380 375 -1.959632 +381 375 -1.959632 +389 375 -105.8201 +404 375 -2.108815 +376 376 2.019386 +377 377 2.019386 +378 378 2.019386 +379 379 1.959632 +380 380 1.959632 +381 381 1.959632 +382 382 14.14244 +383 383 84.26632 +384 383 -8.143323 +385 383 -8.032128 +386 383 -16.50165 +387 383 -16.50165 +392 383 -6.963788 +384 384 8.143323 +385 385 8.032128 +386 386 16.50165 +387 387 16.50165 +388 388 94.03941 +389 388 -22.98851 +390 388 -7.077141 +391 388 -7.077141 +395 388 -4.739336 +401 388 -9.416196 +405 388 -7.77605 +389 389 128.8086 +390 390 7.077141 +391 391 7.077141 +392 392 10.41564 +395 392 -3.451847 +393 393 32.25806 +474 393 -16.12903 +479 393 -16.12903 +394 394 20.16254 +395 394 -19.59248 +395 395 785.9075 +398 395 -76.33588 +399 395 -77.51938 +405 395 -10.71811 +406 395 -473.7875 +407 395 -89.28572 +474 395 -24.8139 +396 396 97.03087 +406 396 -86.43042 +716 396 -5.336179 +717 396 -5.264266 +397 397 690.4763 +716 397 -357.1429 +717 397 -333.3333 +398 398 84.5867 +399 399 77.51938 +400 400 10000 +473 400 -10000 +401 401 29.10979 +402 401 -8.038586 +714 401 -11.65501 +402 402 8.038586 +403 403 11.03753 +404 404 5.644883 +409 404 -3.536068 +405 405 18.49416 +406 406 10591.18 +479 406 -23.97024 +483 406 -10000 +716 406 -3.519887 +717 406 -3.472463 +407 407 100.0442 +408 408 2.934273 +409 409 10.22503 +410 410 10155.49 +411 410 -10000 +412 410 -80.19246 +731 410 -75.30121 +411 411 11741.83 +476 411 -48.89976 +477 411 -68.37394 +486 411 -49.75124 +412 412 111.7064 +413 412 -9.425071 +478 412 -3.253196 +709 412 -.6465378 +724 412 -12.58653 +413 413 24.76907 +414 414 319.0159 +431 414 -227.2727 +433 414 -91.74312 +415 415 319.0159 +432 415 -227.2727 +433 415 -91.74312 +416 416 362.0423 +433 416 -91.74312 +462 416 -222.2222 +417 417 362.0423 +433 417 -91.74312 +462 417 -222.2222 +418 418 39.25825 +419 418 -9.389671 +545 418 -29.86858 +419 419 9.389671 +420 420 1716.987 +421 420 -38.75969 +422 420 -11.56069 +470 420 -1666.667 +421 421 38.75969 +422 422 11.56069 +423 423 606.9609 +425 423 -526.3158 +516 423 -80.64516 +424 424 293.4111 +466 424 -212.766 +516 424 -80.64516 +425 425 905.9952 +426 425 -73.52941 +427 425 -35.58719 +466 425 -151.5152 +426 426 536.205 +437 426 -454.5455 +519 426 -8.130081 +427 427 35.58719 +428 428 305.9395 +429 428 -25 +430 428 -37.03704 +431 428 -121.9512 +432 428 -121.9512 +429 429 25 +430 430 37.03704 +431 431 349.224 +432 432 349.224 +433 433 602.0284 +434 433 -32.05128 +510 433 -181.8182 +564 433 -21.18644 +434 434 32.05128 +435 435 83.68201 +510 435 -83.68201 +436 436 21.45923 +534 436 -21.45923 +437 437 548.2252 +438 437 -45.6621 +439 437 -6.222775 +441 437 -12.12121 +458 437 -29.67359 +438 438 45.6621 +439 439 22.0206 +440 439 -11.16819 +460 439 -4.62963 +440 440 11.16819 +441 441 45.41016 +442 441 -33.28895 +442 442 33.28895 +443 443 1625.394 +444 443 -73.52941 +470 443 -1111.111 +506 443 -155.0388 +516 443 -285.7143 +444 444 1323.529 +458 444 -1250 +445 445 298.3669 +446 445 -46.72897 +447 445 -67.1141 +446 446 46.72897 +447 447 1251.698 +449 447 -73.52941 +537 447 -487.8049 +448 448 178.454 +464 448 -19.72387 +449 449 2073.529 +450 449 -2000 +450 450 2196.806 +451 450 -21.36752 +451 451 21.36752 +452 452 237.3635 +453 452 -38.31417 +453 453 38.31417 +454 454 413.3297 +455 454 -35.97123 +455 455 35.97123 +456 456 105.5727 +457 456 -24.27184 +564 456 -81.30082 +457 457 24.27184 +458 458 1323.682 +459 458 -18.69159 +460 458 -25.31645 +459 459 18.69159 +460 460 51.68521 +461 460 -21.73913 +461 461 21.73913 +462 462 492.9882 +463 462 -48.54369 +463 463 48.54369 +464 464 86.29654 +465 464 -26.88172 +545 464 -20.96436 +465 465 92.6712 +968 465 -65.78947 +466 466 414.0324 +467 466 -49.75124 +467 467 49.75124 +468 468 10000 +544 468 -10000 +469 469 10000 +543 469 -10000 +470 470 2777.778 +471 471 71.30248 +472 471 -24.67105 +478 471 -11.01079 +481 471 -2.51756 +492 471 -2.24341 +707 471 -5.08647 +708 471 -25.7732 +472 472 24.67105 +473 473 10004.09 +474 473 -4.095004 +474 474 69.19151 +475 475 31.94728 +710 475 -12.28501 +724 475 -10.94691 +476 476 81.26222 +482 476 -32.36246 +477 477 228.7589 +478 477 -160.3849 +478 478 10195.93 +708 478 -20.67495 +709 478 -.6028454 +479 479 118.9729 +480 479 -9.02527 +714 479 -8.410429 +726 479 -5.037783 +480 480 509.0252 +481 481 26.89223 +484 481 -4.597701 +485 481 -4.425562 +491 481 -2.892765 +492 481 -3.441512 +733 481 -9.017133 +482 482 111.1026 +483 482 -78.74016 +483 483 10078.74 +484 484 22.31696 +709 484 -10.70664 +733 484 -7.012623 +485 485 12.73121 +491 485 -8.305648 +486 486 10101.3 +731 486 -51.54639 +487 487 5.743825 +491 487 -5.743825 +488 488 5.743825 +491 488 -5.743825 +489 489 6.968641 +492 489 -6.968641 +490 490 7.077141 +492 490 -7.077141 +491 491 41.56859 +492 491 -16.91332 +734 491 -1.969202 +492 492 10036.64 +707 492 -10000 +493 493 11.23596 +707 493 -11.23596 +494 494 121.9512 +505 494 -121.9512 +495 495 46.2963 +513 495 -46.2963 +496 496 303.0303 +521 496 -303.0303 +497 497 104.1667 +522 497 -104.1667 +498 498 20.08032 +523 498 -20.08032 +499 499 87.33171 +500 499 -39.37008 +819 499 -23.98081 +914 499 -23.98081 +500 500 39.37008 +501 501 227.9827 +502 501 -33.19632 +505 501 -18.58736 +507 501 -58.13953 +521 501 -32.67974 +531 501 -42.55319 +536 501 -42.82655 +502 502 120.8391 +607 502 -33.33334 +609 502 -11.02536 +613 502 -27.3224 +937 502 -15.96169 +503 503 121.5155 +507 503 -42.91846 +513 503 -55.55556 +553 503 -23.04147 +504 504 195.9784 +505 504 -71.42857 +505 505 292.5258 +521 505 -41.49377 +526 505 -17.60563 +578 505 -21.45923 +506 506 10155.04 +507 507 295.7078 +508 507 -12.03369 +509 507 -53.0504 +515 507 -20.4918 +521 507 -39.0625 +522 507 -40.16064 +547 507 -29.85075 +508 508 265.0443 +619 508 -175.4386 +639 508 -56.49718 +781 508 -21.07482 +509 509 912.198 +511 509 -12.0919 +515 509 -82.64463 +520 509 -238.0952 +534 509 -526.3158 +510 510 2646.453 +520 510 -2380.952 +511 511 29.78859 +914 511 -6.779661 +926 511 -10.91703 +512 512 99.0099 +520 512 -99.0099 +513 513 185.3068 +514 513 -9.380863 +540 513 -74.07407 +514 514 28.32451 +790 514 -11.28668 +795 514 -7.656968 +515 515 688.1429 +516 515 -416.6667 +517 515 -10.34126 +518 515 -18.18182 +536 515 -80.64516 +547 515 -59.1716 +516 516 863.6713 +517 517 27.53012 +993 517 -12.95337 +995 517 -4.235493 +518 518 18.18182 +519 519 93.04359 +957 519 -16.42036 +967 519 -68.49315 +520 520 2718.058 +521 521 520.433 +522 521 -104.1667 +522 522 260.4988 +523 522 -12.0048 +523 523 247.5269 +582 523 -196.0784 +770 523 -3.675795 +779 523 -11.60093 +802 523 -4.086637 +524 524 220.5431 +533 524 -158.7302 +985 524 -43.85965 +1001 524 -17.95332 +525 525 78.74016 +568 525 -78.74016 +526 526 71.7446 +527 526 -21.45923 +531 526 -32.67974 +527 527 149.7379 +921 527 -4.553734 +933 527 -5.621135 +1068 527 -35.27337 +1073 527 -30.67485 +1074 527 -25.83979 +1124 527 -26.31579 +528 528 336.2918 +529 528 -196.0784 +809 528 -56.17978 +819 528 -84.03362 +529 529 196.0784 +530 530 261.7177 +578 530 -9.708738 +583 530 -9.259259 +780 530 -5.390836 +797 530 -4.800768 +803 530 -232.5581 +531 531 204.1382 +532 531 -53.71729 +534 531 -75.18797 +532 532 527.8732 +554 532 -35.97123 +647 532 -113.6364 +652 532 -91.3242 +663 532 -45.87156 +668 532 -63.29114 +672 532 -12.97017 +673 532 -32.78688 +677 532 -28.24859 +908 532 -13.75516 +912 532 -18.48429 +918 532 -6.426735 +1047 532 -11.38952 +533 533 158.7302 +534 534 740.9194 +535 534 -21.45923 +536 534 -56.49718 +542 534 -40 +535 535 37.48487 +885 535 -16.02564 +536 536 698.459 +537 536 -416.6667 +538 536 -21.14165 +539 536 -18.18182 +542 536 -62.5 +537 537 904.4716 +538 538 105.9105 +809 538 -33.78378 +914 538 -7.002801 +948 538 -11.17318 +953 538 -10.48768 +964 538 -22.32143 +539 539 18.18182 +540 540 95.5333 +800 540 -21.45923 +541 541 13.9958 +543 541 -13.9958 +542 542 256.9678 +543 542 -42.91846 +545 542 -21.45923 +546 542 -90.0901 +543 543 10075.1 +544 543 -18.18182 +544 544 10018.18 +545 545 72.29217 +546 546 190.0901 +547 547 89.02235 +548 548 7.942812 +551 548 -7.942812 +549 549 8.532423 +563 549 -8.532423 +550 550 35.33569 +566 550 -35.33569 +551 551 65.62718 +552 551 -17.51313 +555 551 -15.47988 +566 551 -24.69136 +552 552 35.10244 +558 552 -3.012048 +560 552 -14.57726 +553 553 67.98992 +561 553 -26.73797 +710 553 -5.035247 +554 554 351.6514 +574 554 -93.45794 +909 554 -222.2222 +555 555 58.73003 +556 555 -16.26016 +557 555 -8.591065 +564 555 -10.9529 +566 555 -7.446017 +556 556 150.6284 +559 556 -64.93507 +579 556 -7.886436 +621 556 -45.04505 +903 556 -16.50165 +557 557 25.14736 +558 557 -16.55629 +558 558 103.3103 +979 558 -4.599816 +980 558 -12.40695 +986 558 -12.0919 +989 558 -14.4405 +991 558 -13.86001 +998 558 -8.061266 +1000 558 -18.28154 +559 559 73.91173 +579 559 -8.976661 +560 560 20.04772 +980 560 -5.470459 +561 561 200.6328 +562 561 -135.1351 +566 561 -38.75969 +562 562 143.5954 +563 562 -8.460238 +563 563 31.84037 +567 563 -3.386387 +564 564 113.4402 +565 565 5.017561 +567 565 -5.017561 +566 566 120.1828 +567 566 -4.231908 +567 567 12.63586 +568 568 164.0549 +773 568 -6.574622 +795 568 -78.74016 +569 569 20.08032 +785 569 -20.08032 +570 570 14.70588 +579 570 -14.70588 +571 571 11.42857 +776 571 -11.42857 +572 572 14.85884 +783 572 -14.85884 +573 573 18.51852 +796 573 -18.51852 +574 574 93.45794 +575 575 400 +581 575 -400 +576 576 23.74311 +582 576 -13.31558 +784 576 -10.42753 +577 577 897.2149 +786 577 -769.2308 +802 577 -8.93655 +1090 577 -119.0476 +578 578 45.98278 +780 578 -14.81481 +579 579 84.46193 +580 579 -2.915452 +782 579 -20.4918 +783 579 -7.85546 +895 579 -18.21494 +927 579 -3.4153 +580 580 16.74671 +587 580 -13.83126 +581 581 427.649 +776 581 -9.65717 +796 581 -4.690432 +797 581 -7.42942 +799 581 -5.87199 +582 582 209.394 +583 583 29.98347 +780 583 -13.03781 +803 583 -7.686395 +584 584 52.25406 +585 584 -45.6621 +1068 584 -6.591958 +585 585 121.8822 +607 585 -38.91051 +613 585 -34.0136 +659 585 -3.295979 +586 586 2.538071 +597 586 -2.538071 +587 587 66.82627 +598 587 -14.08451 +614 587 -38.91051 +588 588 10.00413 +595 588 -5.065856 +597 588 -4.938272 +589 589 6.075334 +591 589 -6.075334 +590 590 11.77992 +595 590 -4.405286 +616 590 -7.374631 +591 591 15.86007 +601 591 -9.784736 +592 592 7.564296 +611 592 -7.564296 +593 593 10.53741 +604 593 -10.53741 +594 594 63.69427 +600 594 -63.69427 +595 595 9.471143 +596 596 22.07258 +597 596 -4.837929 +603 596 -6.493506 +607 596 -10.74114 +597 597 12.31427 +598 598 14.08451 +599 599 6.038647 +1105 599 -6.038647 +600 600 346.1801 +601 600 -4.708098 +782 600 -277.7778 +601 601 23.91828 +602 601 -5.24659 +633 601 -4.178855 +602 602 10.19709 +640 602 -4.950495 +603 603 49.17807 +604 603 -4.837929 +605 603 -10.96491 +1092 603 -26.88172 +604 604 30.79998 +612 604 -7.843137 +615 604 -7.581501 +605 605 31.29012 +607 605 -20.3252 +606 606 3.743916 +610 606 -3.743916 +607 607 103.3102 +608 608 44.10909 +610 608 -41.15226 +1101 608 -2.95683 +609 609 20.31041 +1068 609 -9.285051 +610 610 44.89618 +611 611 19.35675 +615 611 -11.79245 +612 612 7.843137 +613 613 61.33601 +614 614 38.91051 +615 615 19.37395 +616 616 7.374631 +617 617 51.02041 +1123 617 -51.02041 +618 618 27.57334 +629 618 -10.09082 +639 618 -17.48252 +619 619 180.4386 +620 619 -5 +620 620 7.089864 +632 620 -2.089864 +621 621 137.2607 +624 621 -87.7193 +1061 621 -4.496403 +622 622 12.46672 +625 622 -10.67236 +641 622 -1.794366 +623 623 18.4185 +639 623 -10.37344 +642 623 -8.045053 +624 624 110.7608 +638 624 -23.04147 +625 625 17.12814 +635 625 -6.455778 +626 626 8.504332 +634 626 -5.627462 +635 626 -2.87687 +627 627 15.82279 +639 627 -15.82279 +628 628 43.10402 +631 628 -33.22259 +642 628 -9.881423 +629 629 5125.033 +637 629 -5000 +643 629 -114.9425 +630 630 27.86917 +631 630 -5.931198 +635 630 -4.347826 +796 630 -17.59015 +631 631 39.15379 +632 632 10.68662 +634 632 -6.830601 +641 632 -1.76616 +633 633 4.178855 +634 634 12.45806 +635 635 18.68799 +636 635 -5.007511 +636 636 8.039652 +1094 636 -3.03214 +637 637 5000 +638 638 23.04147 +639 639 100.1759 +640 640 4.950495 +641 641 3.560526 +642 642 17.92648 +643 643 114.9425 +644 644 87.81362 +652 644 -55.55556 +662 644 -32.25806 +645 645 32.23264 +650 645 -26.80965 +1060 645 -5.422993 +646 646 35.23614 +664 646 -30.4878 +1051 646 -4.748338 +647 647 132.4334 +1041 647 -18.79699 +648 648 87.5595 +678 648 -64.93507 +1042 648 -22.62444 +649 649 89.50495 +672 649 -40 +1055 649 -49.50495 +650 650 43.7588 +653 650 -16.94915 +651 651 35.83253 +670 651 -15.12859 +1045 651 -20.70393 +652 652 4092.357 +653 652 -4.440497 +663 652 -3333.333 +666 652 -97.08738 +668 652 -158.7302 +676 652 -18.55288 +1054 652 -333.3333 +653 653 31.9941 +1048 653 -10.60445 +654 654 184.7682 +677 654 -39.84064 +1057 654 -144.9275 +655 655 330.6451 +657 655 -80.64516 +665 655 -250 +656 656 4.040404 +678 656 -4.040404 +657 657 188.172 +1055 657 -107.5269 +658 658 298.2456 +671 658 -131.5789 +672 658 -166.6667 +659 659 65.79597 +677 659 -62.5 +660 660 36.60525 +1050 660 -17.03578 +1058 660 -19.56947 +661 661 500 +677 661 -500 +662 662 90.3976 +672 662 -58.13953 +663 663 3379.205 +664 664 60.0736 +1056 664 -29.5858 +665 665 340.0901 +682 665 -90.0901 +666 666 623.4032 +681 666 -526.3158 +667 667 5.494505 +1051 667 -5.494505 +668 668 222.0213 +669 669 400 +679 669 -66.66667 +1058 669 -333.3333 +670 670 69.18265 +1121 670 -54.05405 +671 671 184.7704 +679 671 -53.19149 +672 672 619.955 +673 672 -24.27184 +682 672 -117.647 +1049 672 -40 +1050 672 -16.80672 +1052 672 -18.34863 +1058 672 -19.84127 +1121 672 -105.2632 +673 673 57.05873 +674 674 340.5986 +675 674 -322.5806 +1046 674 -18.01802 +675 675 353.4448 +1050 675 -30.8642 +676 676 172.399 +1043 676 -153.8461 +677 677 945.5062 +678 677 -4.347826 +1041 677 -66.66667 +1059 677 -243.9024 +678 678 73.3233 +679 679 119.8582 +680 680 103.5542 +1047 680 -46.08295 +1057 680 -57.47126 +681 681 1637.427 +1053 681 -1111.111 +682 682 207.7372 +683 683 1.579031 +689 683 -1.579031 +684 684 1.923077 +689 684 -1.923077 +685 685 1.764913 +691 685 -1.764913 +686 686 328.5128 +687 686 -18.34863 +688 686 -24.44988 +687 687 18.34863 +688 688 71.2075 +689 688 -4.409431 +694 688 -8.312551 +699 688 -3.82995 +700 688 -7.374631 +702 688 -22.83105 +689 689 9.846897 +693 689 -1.935359 +690 690 10178.92 +691 690 -4.99002 +694 690 -161.2903 +697 690 -12.64223 +691 691 10040.88 +692 691 -10000 +695 691 -34.12969 +692 692 10000 +693 693 36.06505 +695 693 -34.12969 +694 694 187.9515 +695 694 -18.34863 +695 695 86.60801 +696 696 70.92198 +697 697 23.45304 +698 697 -10.81081 +698 698 10.81081 +699 699 27.1355 +700 699 -7.968128 +700 700 15.34276 +701 701 10000 +702 702 50.80308 +703 703 10000 +704 704 20.8778 +705 705 10000 +706 706 11.04972 +707 706 -11.04972 +707 707 10067.62 +726 707 -40.24145 +708 708 78.90948 +709 708 -32.46134 +709 709 44.41736 +710 710 10017.32 +711 711 10024.15 +724 711 -11.7096 +734 711 -12.43781 +712 712 10.78749 +714 712 -10.78749 +713 713 10.92896 +714 713 -10.92896 +714 714 102.2441 +715 714 -25.08151 +734 714 -23.9521 +715 715 33.13305 +716 716 370.5403 +717 716 -4.541326 +717 717 346.6114 +718 718 14.51379 +724 718 -14.51379 +719 719 14.51379 +724 719 -14.51379 +720 720 14.51379 +724 720 -14.51379 +721 721 14.51379 +724 721 -14.51379 +722 722 14.51379 +724 722 -14.51379 +723 723 14.51379 +724 723 -14.51379 +724 724 156.8111 +725 724 -2.262444 +734 724 -6.215041 +725 725 37.84963 +726 726 45.27923 +727 727 22.77734 +728 727 -1.437174 +731 727 -21.34016 +728 728 22.77734 +731 728 -21.34016 +729 729 22.54354 +730 729 -.5025378 +731 729 -22.041 +730 730 22.63133 +731 730 -22.12879 +731 731 213.6977 +732 732 2.369107 +733 733 16.02976 +734 734 44.57415 +735 735 3.749531 +743 735 -3.749531 +736 736 5.060729 +742 736 -5.060729 +737 737 10.41667 +743 737 -10.41667 +738 738 25.25253 +742 738 -25.25253 +739 739 6.666667 +744 739 -6.666667 +740 740 10119.13 +741 740 -29.06977 +742 740 -9.416196 +744 740 -80.64516 +741 741 66.38967 +749 741 -30.12048 +759 741 -7.199424 +742 742 97.15186 +743 742 -36.00912 +743 743 65.14973 +746 743 -4.042037 +744 744 120.6452 +756 744 -33.33334 +745 745 10004.53 +746 745 -4.531038 +746 746 8.573075 +747 747 83.11967 +748 747 -15.5521 +758 747 -67.56757 +748 748 35.75412 +757 748 -20.20202 +749 749 109.4856 +753 749 -79.36508 +750 750 36.15381 +751 750 -9.199632 +759 750 -26.95418 +751 751 9.199632 +752 752 14.23959 +757 752 -10.20408 +762 752 -4.035512 +753 753 79.36508 +754 754 101.7248 +757 754 -42.55319 +765 754 -59.1716 +755 755 34.41619 +764 755 -25 +756 756 61.14975 +757 756 -27.81641 +757 757 100.7757 +758 758 67.56757 +759 759 34.1536 +760 760 6.238303 +761 760 -6.238303 +761 761 27.13861 +766 761 -4.730369 +767 761 -8 +762 762 8.519817 +763 762 -4.484305 +763 763 4.484305 +764 764 25 +765 765 59.1716 +766 766 7.055951 +768 766 -2.325581 +767 767 8 +768 768 8.877011 +769 768 -3.030303 +772 768 -3.521127 +769 769 3.030303 +770 770 10003.68 +823 770 -10000 +771 771 4.694836 +772 771 -4.694836 +772 772 8.215962 +773 773 41.97304 +774 773 -9.191176 +795 773 -6.067961 +801 773 -5.646527 +802 773 -14.49275 +774 774 9.191176 +775 775 87.97093 +776 775 -79.36508 +781 775 -8.605852 +776 776 145.414 +781 776 -7.067138 +796 776 -2.511932 +797 776 -3.435246 +1127 776 -31.94888 +777 777 24.34617 +793 777 -3.907777 +795 777 -5.797101 +800 777 -14.64129 +778 778 27.51031 +779 778 -27.51031 +779 779 52.63487 +780 779 -3.543586 +787 779 -9.980041 +780 780 53.90431 +797 780 -2.540005 +949 780 -14.57726 +781 781 36.74781 +782 782 311.0084 +783 782 -12.73885 +783 783 35.45316 +784 784 25.00479 +802 784 -14.57726 +785 785 85.04397 +786 785 -26.66667 +788 785 -16.80672 +796 785 -5.941771 +797 785 -11.00715 +802 785 -4.541326 +786 786 1295.897 +1128 786 -500 +787 787 18.79838 +949 787 -8.818342 +788 788 201.3442 +789 788 -2.932551 +796 788 -9.191176 +1132 788 -172.4138 +789 789 6.317792 +1094 789 -3.38524 +790 790 67.99094 +791 790 -32.89474 +795 790 -23.80952 +791 791 32.89474 +792 792 18.23483 +799 792 -10.1833 +1110 792 -8.05153 +793 793 10007.82 +794 793 -3.907777 +825 793 -10000 +794 794 24.34617 +795 794 -5.797101 +800 794 -14.64129 +795 795 127.8688 +796 796 80.27805 +799 796 -21.83406 +797 797 107.3739 +798 797 -4.818348 +803 797 -4.5106 +805 797 -18.83239 +1119 797 -50 +798 798 10.82075 +1114 798 -6.002401 +799 799 37.88935 +800 800 50.74181 +801 801 5.646527 +802 802 10046.63 +1137 802 -10000 +803 803 290.2097 +804 803 -45.45454 +804 804 45.45454 +805 805 136.4794 +1138 805 -117.647 +806 806 4.975124 +821 806 -4.975124 +807 807 4.065041 +887 807 -4.065041 +808 808 13.33333 +897 808 -13.33333 +809 809 202.3231 +920 809 -112.3596 +810 810 39.62731 +811 810 -26.31579 +918 810 -8.70322 +811 811 26.31579 +812 812 500 +909 812 -500 +813 813 805.5555 +885 813 -11.13586 +940 813 -25.18892 +1122 813 -769.2308 +814 814 149.0395 +815 814 -112.3596 +909 814 -23.14815 +918 814 -13.5318 +815 815 112.3596 +816 816 67.98565 +817 816 -47.61905 +927 816 -20.3666 +817 817 139.7763 +818 817 -52.63158 +907 817 -39.52569 +818 818 52.63158 +819 819 340.5726 +820 819 -232.5581 +820 820 232.5581 +821 821 148.7724 +888 821 -22.72727 +897 821 -114.9425 +921 821 -6.127451 +822 822 133.5555 +880 822 -77.51938 +885 822 -25.64103 +935 822 -30.39514 +823 823 10000 +824 824 10000 +825 824 -10000 +825 825 20000 +826 826 10000 +827 827 10000 +828 828 12.59221 +837 828 -3.399953 +838 828 -4.568296 +874 828 -4.623957 +829 829 38.16908 +833 829 -9.785693 +839 829 -9.75447 +830 830 35.75804 +833 830 -12.15067 +840 830 -9.559799 +831 831 21.56662 +841 831 -10.30197 +869 831 -11.26464 +832 832 24.83065 +842 832 -6.55914 +869 832 -18.27151 +833 833 25.06136 +843 833 -3.125 +834 834 65.83865 +844 834 -9.816236 +874 834 -56.02241 +835 835 97.96513 +845 835 -1.626016 +862 835 -96.33911 +836 836 13.04241 +846 836 -6.806048 +869 836 -6.236358 +837 837 3.399953 +838 838 11.84409 +859 838 -6.666667 +860 838 -.6091247 +839 839 9.75447 +840 840 9.559799 +841 841 10.30197 +842 842 6.55914 +843 843 3.125 +844 844 9.816236 +845 845 1.626016 +846 846 6.806048 +847 847 6.595976 +848 847 -6.595976 +848 848 14.46535 +873 848 -7.869369 +849 849 7.853637 +854 849 -3.149802 +863 849 -2.352 +864 849 -2.351835 +850 850 9.019895 +855 850 -6.506006 +873 850 -2.513889 +851 851 1.965569 +856 851 -.805153 +858 851 -1.160416 +852 852 2.941501 +857 852 -1.692047 +869 852 -1.249453 +853 853 5.118024 +876 853 -5.118024 +854 854 3.149802 +855 855 6.506006 +856 856 .805153 +857 857 1.692047 +858 858 3.252203 +873 858 -2.091788 +859 859 6.666667 +860 860 1.422133 +861 860 -.8130081 +861 861 .8130081 +862 862 101.7126 +863 862 -5.373455 +863 863 10.56927 +869 863 -2.843818 +864 864 26.18864 +865 864 -20.9205 +869 864 -2.916302 +865 865 20.9205 +866 866 71.85119 +869 866 -67.1141 +877 866 -4.737092 +867 867 5.135919 +878 867 -5.135919 +868 868 79.89591 +869 868 -68.02721 +871 868 -7.077141 +877 868 -4.791567 +869 869 245.0375 +870 869 -67.1141 +870 870 78.94842 +871 870 -7.097232 +877 870 -4.737092 +871 871 14.17437 +872 872 3.439506 +882 872 -3.439506 +873 873 19.16737 +882 873 -6.692322 +874 874 65.74321 +878 874 -5.09684 +875 875 268.8182 +876 875 -227.2727 +882 875 -41.54549 +876 876 232.3908 +877 877 144.0989 +878 877 -54.4514 +883 877 -15.50147 +878 878 64.68417 +879 879 13.23977 +882 879 -13.23977 +880 880 77.51938 +881 881 15.55936 +882 881 -15.55936 +882 882 209.5087 +883 883 15.50147 +884 884 208.6253 +937 884 -114.2857 +1123 884 -94.33962 +885 885 67.07021 +914 885 -9.425071 +918 885 -4.842615 +886 886 34.37894 +895 886 -13.2626 +927 886 -6.146282 +937 886 -14.97006 +887 887 30.75514 +896 887 -15.94896 +897 887 -10.74114 +888 888 34.51973 +924 888 -11.79245 +889 889 526.3158 +917 889 -526.3158 +890 890 90.17906 +891 890 -55.55556 +908 890 -11.79245 +917 890 -22.83105 +891 891 55.55556 +892 892 185.8299 +893 892 -78.74016 +899 892 -68.02721 +927 892 -39.0625 +893 893 78.74016 +894 894 10037.21 +921 894 -24.93765 +936 894 -12.26994 +1126 894 -10000 +895 895 486.023 +1125 895 -454.5455 +896 896 48.8437 +897 896 -32.89474 +897 897 242.6641 +913 897 -54.05405 +917 897 -3.943218 +944 897 -12.7551 +898 898 119.8733 +914 898 -21.83406 +926 898 -98.03922 +899 899 208.295 +900 899 -36.10108 +908 899 -104.1667 +900 900 36.10108 +901 901 10010.86 +913 901 -10.85776 +1129 901 -10000 +902 902 400 +906 902 -400 +903 903 27.64992 +987 903 -11.14827 +904 904 479.299 +905 904 -256.4103 +939 904 -192.3077 +944 904 -30.58104 +905 905 256.4103 +906 906 613.7151 +922 906 -17.63668 +939 906 -196.0784 +907 907 58.35809 +923 907 -18.83239 +908 908 200.9013 +912 908 -43.47826 +917 908 -7.668712 +927 908 -20.04008 +909 909 941.4488 +910 909 -196.0784 +910 910 196.0784 +911 911 131.5789 +912 911 -131.5789 +912 912 193.5415 +913 913 64.91182 +914 914 181.033 +935 914 -29.23977 +938 914 -78.74016 +963 914 -4.030633 +915 915 103.0928 +926 915 -103.0928 +916 916 81.96722 +991 916 -81.96722 +917 917 754.9636 +918 917 -121.2121 +930 917 -72.9927 +918 918 232.832 +919 918 -52.63158 +1130 918 -22.47191 +919 919 69.87296 +940 919 -17.24138 +920 920 112.3596 +921 921 123.4495 +923 921 -22.98851 +924 921 -29.06977 +927 921 -5.208333 +933 921 -23.98081 +937 921 -6.583279 +922 922 10033.82 +930 922 -16.18123 +1133 922 -10000 +923 923 41.8209 +924 924 79.77273 +925 924 -38.91051 +925 925 38.91051 +926 926 212.049 +927 927 138.2073 +928 927 -37.17472 +937 927 -6.793478 +928 928 1045.494 +929 928 -999.9999 +937 928 -8.319468 +929 929 999.9999 +930 930 185.3278 +931 930 -96.15385 +931 931 96.15385 +932 932 78.74016 +949 932 -78.74016 +933 933 187.4276 +934 933 -131.5789 +942 933 -26.24672 +934 934 131.5789 +935 935 94.47811 +947 935 -34.8432 +936 936 10045.17 +937 936 -32.89474 +1134 936 -10000 +937 937 199.8084 +938 938 78.74016 +939 939 388.3861 +940 940 141.4402 +954 940 -99.0099 +941 941 400 +954 941 -400 +942 942 26.24672 +943 943 80.64516 +944 943 -80.64516 +944 944 123.9813 +945 945 3.323363 +961 945 -3.323363 +946 946 10.52632 +965 946 -10.52632 +947 947 34.8432 +948 948 110.8715 +952 948 -12.90323 +955 948 -10.88454 +957 948 -18.50139 +958 948 -35.08772 +964 948 -22.32143 +949 949 102.1358 +950 950 59.13499 +951 950 -4.399472 +955 950 -35.77818 +956 950 -18.95735 +951 951 26.76585 +965 951 -4.962779 +969 951 -10.17294 +975 951 -7.230658 +952 952 78.2627 +960 952 -65.35947 +953 953 32.95959 +955 953 -22.47191 +954 954 528.2497 +959 954 -29.23977 +955 955 69.13462 +956 956 18.95735 +957 957 153.211 +958 957 -17.85714 +967 957 -21.69197 +973 957 -78.74016 +958 958 52.94486 +959 959 29.23977 +960 960 465.3595 +973 960 -400 +961 961 17.72833 +962 961 -6.060606 +963 961 -2.007226 +968 961 -6.337135 +962 962 42.96098 +966 962 -36.90037 +963 963 10.88517 +972 963 -4.84731 +964 964 44.64286 +965 965 31.12432 +968 965 -5.945303 +969 965 -9.689922 +966 966 135.9103 +971 966 -99.0099 +967 967 90.18512 +968 968 78.07191 +969 969 26.28135 +970 969 -6.418485 +970 970 6.418485 +971 971 110.4385 +972 971 -11.42857 +972 972 16.27588 +973 973 478.7402 +974 974 999.9999 +983 974 -999.9999 +975 975 7.230658 +976 976 131.2277 +977 976 -72.9927 +989 976 -11.28668 +1001 976 -46.94836 +977 977 72.9927 +978 978 11.99041 +989 978 -11.99041 +979 979 212.9332 +983 979 -208.3333 +980 980 17.87741 +981 981 214.386 +982 981 -192.3077 +984 981 -12.04819 +998 981 -10.03009 +982 982 192.3077 +983 983 1216.031 +986 983 -7.69823 +984 984 12.04819 +985 985 73.44217 +987 985 -24.27184 +989 985 -5.310675 +986 986 19.79013 +987 987 74.94581 +999 987 -39.52569 +988 988 25.27583 +993 988 -12.42236 +995 988 -12.85347 +989 989 48.93494 +990 989 -5.906674 +990 990 5.906674 +991 991 98.75978 +992 991 -2.932551 +992 992 2.932551 +993 993 29.40311 +994 993 -4.027386 +994 994 4.027386 +995 995 28.39029 +997 995 -7.267442 +998 995 -4.033885 +996 996 78.74016 +997 996 -78.74016 +997 997 95.07378 +998 997 -9.066183 +998 998 31.19142 +999 999 39.52569 +1000 1000 18.28154 +1001 1001 64.90168 +1002 1002 8.319468 +1027 1002 -8.319468 +1003 1003 4.084967 +1035 1003 -4.084967 +1004 1004 3.15856 +1032 1004 -3.15856 +1005 1005 10.41667 +1039 1005 -10.41667 +1006 1006 125.0263 +1007 1006 -4.149378 +1013 1006 -19.26782 +1019 1006 -10.48218 +1021 1006 -55.24862 +1030 1006 -14.32665 +1035 1006 -21.55167 +1007 1007 4.149378 +1008 1008 24.42002 +1023 1008 -24.42002 +1009 1009 10093.68 +1010 1009 -1.567398 +1018 1009 -21.18644 +1019 1009 -6.297229 +1021 1009 -4.244482 +1022 1009 -57.14286 +1040 1009 -3.239391 +1010 1010 1.567398 +1011 1011 41.15226 +1040 1011 -41.15226 +1012 1012 5.87199 +1040 1012 -5.87199 +1013 1013 748.2476 +1023 1013 -15.07159 +1029 1013 -454.4287 +1030 1013 -32.57329 +1037 1013 -118.3432 +1038 1013 -102.5641 +1039 1013 -5.9988 +1014 1014 75.75758 +1039 1014 -75.75758 +1015 1015 34.60208 +1029 1015 -34.60208 +1016 1016 4.102139 +1024 1016 -2.283626 +1034 1016 -1.818512 +1017 1017 73.80074 +1029 1017 -73.80074 +1018 1018 21.18644 +1019 1019 60.25767 +1033 1019 -43.47826 +1020 1020 34.96503 +1021 1020 -34.96503 +1021 1021 94.45814 +1022 1022 57.14286 +1023 1023 66.38087 +1024 1023 -6.973501 +1025 1023 -7.352941 +1027 1023 -12.56281 +1024 1024 11.00568 +1034 1024 -1.748557 +1025 1025 11.7895 +1028 1025 -4.436557 +1026 1026 16.88283 +1027 1026 -9.615384 +1027 1027 38.77231 +1028 1027 -3.372681 +1035 1027 -4.901961 +1028 1028 7.809239 +1029 1029 10562.83 +1030 1030 56.26324 +1031 1030 -9.363297 +1031 1031 9.363297 +1032 1032 19.43661 +1034 1032 -5.455537 +1036 1032 -10.82251 +1033 1033 43.47826 +1034 1034 9.022608 +1035 1035 34.63528 +1036 1035 -4.096682 +1036 1036 14.91919 +1037 1037 118.3432 +1038 1038 102.5641 +1039 1039 92.17304 +1040 1040 50.26365 +1041 1041 85.46367 +1042 1042 22.62444 +1043 1043 179.8878 +1050 1043 -26.04167 +1044 1044 51.6919 +1050 1044 -33.8983 +1053 1044 -17.79359 +1045 1045 69.24763 +1047 1045 -48.54369 +1046 1046 71.20951 +1058 1046 -53.19149 +1047 1047 106.0162 +1048 1048 10.60445 +1049 1049 66.17801 +1054 1049 -26.17801 +1050 1050 5329.354 +1051 1050 -4.708098 +1052 1050 -200 +1051 1051 18.55844 +1056 1051 -3.607503 +1052 1052 218.3486 +1053 1053 1128.905 +1054 1054 359.5114 +1055 1055 267.5334 +1056 1055 -4.118616 +1058 1055 -106.383 +1056 1056 37.31192 +1057 1057 202.3988 +1058 1058 532.3185 +1059 1059 295.9858 +1061 1059 -52.08333 +1060 1060 5.422993 +1061 1061 56.57973 +1062 1062 9.573448 +1072 1062 -6.301197 +1085 1062 -3.272251 +1063 1063 14.97518 +1074 1063 -10.35197 +1064 1064 15.77324 +1075 1064 -11.49425 +1087 1064 -4.27899 +1065 1065 56.4111 +1077 1065 -7.256895 +1080 1065 -32.67974 +1066 1066 14.0056 +1068 1066 -14.0056 +1067 1067 2.654632 +1071 1067 -2.654632 +1068 1068 96.37038 +1069 1068 -21.64502 +1092 1068 -9.569378 +1069 1069 49.50018 +1086 1069 -27.85515 +1070 1070 27.63299 +1077 1070 -21.09705 +1113 1070 -6.535948 +1071 1071 6.953859 +1088 1071 -4.299226 +1072 1072 10.51705 +1075 1072 -4.215851 +1073 1073 194.6093 +1074 1073 -163.9344 +1074 1074 315.5717 +1075 1074 -5.040322 +1081 1074 -85.47009 +1086 1074 -21.73913 +1075 1075 25.04781 +1082 1075 -4.297379 +1076 1076 18.69159 +1077 1076 -18.69159 +1077 1077 82.05888 +1078 1077 -9.416196 +1080 1077 -9.328359 +1088 1077 -11.54734 +1095 1077 -4.721436 +1078 1078 30.82096 +1084 1078 -17.66784 +1085 1078 -3.736921 +1079 1079 3.007519 +1117 1079 -3.007519 +1080 1080 42.00809 +1081 1081 85.47009 +1082 1082 4.297379 +1083 1083 500 +1087 1083 -500 +1084 1084 17.66784 +1085 1085 7.009172 +1086 1086 49.59428 +1087 1087 504.279 +1088 1088 15.84657 +1089 1089 12.21374 +1106 1089 -9.225092 +1112 1089 -2.988643 +1090 1090 123.7557 +1091 1090 -4.708098 +1091 1091 14.65977 +1093 1091 -5.681818 +1108 1091 -4.269855 +1092 1092 36.4511 +1093 1093 14.98414 +1098 1093 -9.302325 +1094 1094 6.417381 +1095 1095 136.3807 +1096 1095 -4.440497 +1100 1095 -7.342144 +1105 1095 -16.23377 +1113 1095 -87.7193 +1096 1096 16.50748 +1103 1096 -7.581501 +1107 1096 -2.338634 +1112 1096 -2.146844 +1097 1097 99.0099 +1100 1097 -99.0099 +1098 1098 15.8597 +1099 1098 -6.557377 +1099 1099 6.557377 +1100 1100 110.9308 +1101 1100 -4.578754 +1101 1101 13.03916 +1120 1101 -5.503577 +1102 1102 4.524887 +1112 1102 -4.524887 +1103 1103 7.581501 +1104 1104 46.18962 +1118 1104 -23.14815 +1119 1104 -23.04147 +1105 1105 22.27241 +1106 1106 15.2311 +1109 1106 -6.006006 +1107 1107 2.338634 +1108 1108 4.269855 +1109 1109 6.006006 +1110 1110 10.67965 +1114 1110 -2.628121 +1111 1111 7.380074 +1119 1111 -7.380074 +1112 1112 9.660375 +1113 1113 94.25525 +1114 1114 15.25571 +1115 1114 -4.384042 +1116 1114 -2.241148 +1115 1115 4.384042 +1116 1116 2.241148 +1117 1117 7.461862 +1120 1117 -4.454343 +1118 1118 23.14815 +1119 1119 80.42155 +1120 1120 9.95792 +1121 1121 159.3172 +1122 1122 791.0173 +1136 1122 -21.78649 +1123 1123 145.36 +1124 1124 37.05693 +1135 1124 -10.74114 +1125 1125 454.5455 +1126 1126 10000 +1127 1127 31.94888 +1128 1128 500 +1129 1129 10015.82 +1135 1129 -15.82279 +1130 1130 22.47191 +1131 1131 24.39024 +1136 1131 -24.39024 +1132 1132 172.4138 +1133 1133 10000 +1134 1134 10000 +1135 1135 26.56392 +1136 1136 46.17674 +1137 1137 10000 +1138 1138 117.647 diff --git a/benchmarks/opencl/spmv/DESCRIPTION b/benchmarks/opencl/spmv/DESCRIPTION new file mode 100755 index 00000000..f67c6cb4 --- /dev/null +++ b/benchmarks/opencl/spmv/DESCRIPTION @@ -0,0 +1,3 @@ +Inputs: 1138_bus.mtx vector.bin + +This is a very small phantom data set. diff --git a/benchmarks/opencl/spmv/Makefile b/benchmarks/opencl/spmv/Makefile new file mode 100644 index 00000000..14376ff0 --- /dev/null +++ b/benchmarks/opencl/spmv/Makefile @@ -0,0 +1,72 @@ +RISCV_TOOL_PATH = $(wildcard ~/dev/riscv-gnu-toolchain/drops) +POCL_CC_PATH = $(wildcard ~/dev/pocl/drops_riscv_cc) +POCL_INC_PATH = $(wildcard ../include) +POCL_LIB_PATH = $(wildcard ../lib) +VX_RT_PATH = $(wildcard ../../../runtime) +VX_SIMX_PATH = $(wildcard ../../../simX/obj_dir) + +CC = $(RISCV_TOOL_PATH)/bin/riscv32-unknown-elf-gcc +CXX = $(RISCV_TOOL_PATH)/bin/riscv32-unknown-elf-g++ +DMP = $(RISCV_TOOL_PATH)/bin/riscv32-unknown-elf-objdump +HEX = $(RISCV_TOOL_PATH)/bin/riscv32-unknown-elf-objcopy +GDB = $(RISCV_TOOL_PATH)/bin/riscv32-unknown-elf-gdb + +VX_SRCS = $(VX_RT_PATH)/newlib/newlib.c +VX_SRCS += $(VX_RT_PATH)/startup/vx_start.s +VX_SRCS += $(VX_RT_PATH)/intrinsics/vx_intrinsics.s +VX_SRCS += $(VX_RT_PATH)/io/vx_io.s $(VX_RT_PATH)/io/vx_io.c +VX_SRCS += $(VX_RT_PATH)/fileio/fileio.s +VX_SRCS += $(VX_RT_PATH)/tests/tests.c +VX_SRCS += $(VX_RT_PATH)/vx_api/vx_api.c +VX_SRCS += $(VX_STR) $(VX_FIO) $(VX_NEWLIB) $(VX_INT) $(VX_IO) $(VX_API) $(VX_TEST) + +VX_CFLAGS = -nostartfiles -Wl,-Bstatic,-T,$(VX_RT_PATH)/mains/vortex_link.ld + +CXXFLAGS = -g -O0 -march=rv32im -mabi=ilp32 +CXXFLAGS += -ffreestanding # program may not begin at main() +CXXFLAGS += -Wl,--gc-sections # enable garbage collection of unused input sections +CXXFLAGS += -fno-rtti -fno-non-call-exceptions # disable RTTI and exceptions +CXXFLAGS += -I$(POCL_INC_PATH) -I. + +VX_LIBS = -Wl,--whole-archive lib$(PROJECT).a -Wl,--no-whole-archive $(POCL_LIB_PATH)/libOpenCL.a +QEMU_LIBS = -Wl,--whole-archive lib$(PROJECT).a -Wl,--no-whole-archive $(POCL_LIB_PATH)/qemu/libOpenCL.a + +PROJECT = spmv + +SRCS = main.cc parboil_opencl.c args.c gpu_info.c file.c convert_dataset.c mmio.c ocl.c +#stub.cc + +all: $(PROJECT).dump $(PROJECT).hex + +#parboil_opencl.o : parboil_opencl.c +# $(CC) $(CXXFLAGS) -c $^ -o $@ + +lib$(PROJECT).a: kernel.cl + POCL_DEBUG=all POCL_DEBUG_LLVM_PASSES=1 LD_LIBRARY_PATH=$(RISCV_TOOL_PATH)/lib:$(POCL_CC_PATH)/lib $(POCL_CC_PATH)/bin/poclcc -o lib$(PROJECT).a kernel.cl + +$(PROJECT).elf: $(SRCS) lib$(PROJECT).a + $(CXX) $(CXXFLAGS) $(VX_CFLAGS) $(VX_SRCS) $(SRCS) $(VX_LIBS) -o $(PROJECT).elf + +$(PROJECT).qemu: $(SRCS) lib$(PROJECT).a + $(CXX) $(CXXFLAGS) $(SRCS) $(QEMU_LIBS) -o $(PROJECT).qemu + +$(PROJECT).hex: $(PROJECT).elf + $(HEX) -O ihex $(PROJECT).elf $(PROJECT).hex + +$(PROJECT).dump: $(PROJECT).elf + $(DMP) -D $(PROJECT).elf > $(PROJECT).dump + +run: $(PROJECT).hex + POCL_DEBUG=all $(VX_SIMX_PATH)/Vcache_simX -E -a rv32i --core $(PROJECT).hex -s -b 1> emulator.debug + +qemu: $(PROJECT).qemu + POCL_DEBUG=all $(RISCV_TOOL_PATH)/bin/qemu-riscv32 -d in_asm -D debug.log $(PROJECT).qemu + +gdb-s: $(PROJECT).qemu + POCL_DEBUG=all $(RISCV_TOOL_PATH)/bin/qemu-riscv32 -g 1234 -d in_asm -D debug.log $(PROJECT).qemu + +gdb-c: $(PROJECT).qemu + $(GDB) $(PROJECT).qemu + +clean: + rm -rf *.o *.elf *.dump *.hex *.qemu *.log *.debug \ No newline at end of file diff --git a/benchmarks/opencl/spmv/args.c b/benchmarks/opencl/spmv/args.c new file mode 100644 index 00000000..9d751e29 --- /dev/null +++ b/benchmarks/opencl/spmv/args.c @@ -0,0 +1,617 @@ + +#include +#include +#include +#include +#include +#include + +/*****************************************************************************/ +/* Memory management routines */ + +/* Free an array of owned strings. */ +void +pb_FreeStringArray(char **string_array) +{ + char **p; + + if (!string_array) return; + for (p = string_array; *p; p++) free(*p); + free(string_array); +} + +struct pb_PlatformParam * +pb_PlatformParam(char *name, char *version) +{ + if (name == NULL) { + fprintf(stderr, "pb_PlatformParam: Invalid argument\n"); + exit(-1); + } + + struct pb_PlatformParam *ret = + (struct pb_PlatformParam *)malloc(sizeof (struct pb_PlatformParam)); + + ret->name = name; + ret->version = version; + return ret; +} + +void +pb_FreePlatformParam(struct pb_PlatformParam *p) +{ + if (p == NULL) return; + + free(p->name); + free(p->version); + free(p); +} + +struct pb_DeviceParam * +pb_DeviceParam_index(int index) +{ + struct pb_DeviceParam *ret = + (struct pb_DeviceParam *)malloc(sizeof (struct pb_DeviceParam)); + ret->criterion = pb_Device_INDEX; + ret->index = index; + return ret; +} + +struct pb_DeviceParam * +pb_DeviceParam_cpu(void) +{ + struct pb_DeviceParam *ret = + (struct pb_DeviceParam *)malloc(sizeof (struct pb_DeviceParam)); + ret->criterion = pb_Device_CPU; + return ret; +} + +struct pb_DeviceParam * +pb_DeviceParam_gpu(void) +{ + struct pb_DeviceParam *ret = + (struct pb_DeviceParam *)malloc(sizeof (struct pb_DeviceParam)); + ret->criterion = pb_Device_GPU; + return ret; +} + +struct pb_DeviceParam * +pb_DeviceParam_accelerator(void) +{ + struct pb_DeviceParam *ret = + (struct pb_DeviceParam *)malloc(sizeof (struct pb_DeviceParam)); + ret->criterion = pb_Device_ACCELERATOR; + return ret; +} + +struct pb_DeviceParam * +pb_DeviceParam_name(char *name) +{ + struct pb_DeviceParam *ret = + (struct pb_DeviceParam *)malloc(sizeof (struct pb_DeviceParam)); + ret->criterion = pb_Device_NAME; + ret->name = name; + return ret; +} + +void +pb_FreeDeviceParam(struct pb_DeviceParam *p) +{ + if (p == NULL) return; + + switch(p->criterion) { + case pb_Device_NAME: + free(p->name); + break; + case pb_Device_INDEX: + case pb_Device_CPU: + case pb_Device_ACCELERATOR: + break; + default: + fprintf(stderr, "pb_FreeDeviceParam: Invalid argument\n"); + exit(-1); + } +} + +void +pb_FreeParameters(struct pb_Parameters *p) +{ + free(p->outFile); + pb_FreeStringArray(p->inpFiles); + pb_FreePlatformParam(p->platform); + pb_FreeDeviceParam(p->device); + free(p); +} + +/*****************************************************************************/ + +/* Parse a comma-delimited list of strings into an + * array of strings. */ +static char ** +read_string_array(char *in) +{ + char **ret; + int i; + int count; /* Number of items in the input */ + char *substring; /* Current substring within 'in' */ + + /* Count the number of items in the string */ + count = 1; + for (i = 0; in[i]; i++) if (in[i] == ',') count++; + + /* Allocate storage */ + ret = (char **)malloc((count + 1) * sizeof(char *)); + + /* Create copies of the strings from the list */ + substring = in; + for (i = 0; i < count; i++) { + char *substring_end; + int substring_length; + + /* Find length of substring */ + for (substring_end = substring; + (*substring_end != ',') && (*substring_end != 0); + substring_end++); + + substring_length = substring_end - substring; + + /* Allocate memory and copy the substring */ + ret[i] = (char *)malloc(substring_length + 1); + memcpy(ret[i], substring, substring_length); + ret[i][substring_length] = 0; + + /* go to next substring */ + substring = substring_end + 1; + } + ret[i] = NULL; /* Write the sentinel value */ + + return ret; +} + +static void +report_parse_error(const char *str) +{ + fputs(str, stderr); +} + +/* Interpret a string as a 'pb_DeviceParam' value. + * Return a pointer to a new value, or NULL on failure. + */ +static struct pb_DeviceParam * +read_device_param(char *str) +{ + /* Try different ways of interpreting 'device_string' until one works */ + + /* If argument is an integer, then interpret it as a device index */ + errno = 0; + char *end; + long device_int = strtol(str, &end, 10); + if (!errno) { + /* Negative numbers are not valid */ + if (device_int < 0 || device_int > INT_MAX) return NULL; + + return pb_DeviceParam_index(device_int); + } + + /* Match against predefined strings */ + if (strcmp(str, "CPU") == 0) + return pb_DeviceParam_cpu(); + if (strcmp(str, "GPU") == 0) + return pb_DeviceParam_gpu(); + if (strcmp(str, "ACCELERATOR") == 0) + return pb_DeviceParam_accelerator(); + + /* Assume any other string is a device name */ + return pb_DeviceParam_name(strdup(str)); +} + +/* Interpret a string as a 'pb_PlatformParam' value. + * Return a pointer to a new value, or NULL on failure. + */ +static struct pb_PlatformParam * +read_platform_param(char *str) +{ + int separator_index; /* Index of the '-' character separating + * name and version number. It's -1 if + * there's no '-' character. */ + + /* Find the last occurrence of '-' in 'str' */ + { + char *cur; + separator_index = -1; + for (cur = str; *cur; cur++) { + if (*cur == '-') separator_index = cur - str; + } + } + + /* The platform name is either the entire string, or all characters before + * the separator */ + int name_length = separator_index == -1 ? strlen(str) : separator_index; + char *name_str = (char *)malloc(name_length + 1); + memcpy(name_str, str, name_length); + name_str[name_length] = 0; + + /* The version is either NULL, or all characters after the separator */ + char *version_str; + if (separator_index == -1) { + version_str = NULL; + } + else { + const char *version_input_str = str + separator_index + 1; + int version_length = strlen(version_input_str); + + version_str = (char *)malloc(version_length + 1); + memcpy(version_str, version_input_str, version_length); + version_str[version_length] = 0; + } + + /* Create output structure */ + return pb_PlatformParam(name_str, version_str); +} + +/****************************************************************************/ +/* Argument parsing state */ + +/* Argument parsing state. + * + * Arguments that are interpreted by the argument parser are removed from + * the list. Variables 'argc' and 'argn' do not count arguments that have + * been removed. + * + * During argument parsing, the array of arguments is compacted, overwriting + * the erased arguments. Variable 'argv_put' points to the array element + * where the next argument will be written. Variable 'argv_get' points to + * the array element where the next argument will be read from. + */ +struct argparse { + int argc; /* Number of arguments. Mutable. */ + int argn; /* Current argument index. */ + char **argv_get; /* Argument value being read. */ + char **argv_put; /* Argument value being written. + * argv_put <= argv_get. */ +}; + +static void +initialize_argparse(struct argparse *ap, int argc, char **argv) +{ + ap->argc = argc; + ap->argn = 0; + ap->argv_get = ap->argv_put = argv; +} + +/* Finish argument parsing, without processing the remaining arguments. + * Write new argument count into _argc. */ +static void +finalize_argparse(struct argparse *ap, int *_argc, char **argv) +{ + /* Move the remaining arguments */ + for(; ap->argn < ap->argc; ap->argn++) + *ap->argv_put++ = *ap->argv_get++; + + /* Update the argument count */ + *_argc = ap->argc; + + /* Insert a terminating NULL */ + argv[ap->argc] = NULL; +} + +/* Delete the current argument. The argument will not be visible + * when argument parsing is done. */ +static void +delete_argument(struct argparse *ap) +{ + if (ap->argn >= ap->argc) { + fprintf(stderr, "delete_argument\n"); + } + ap->argc--; + ap->argv_get++; +} + +/* Go to the next argument. Also, move the current argument to its + * final location in argv. */ +static void +next_argument(struct argparse *ap) +{ + if (ap->argn >= ap->argc) { + fprintf(stderr, "next_argument\n"); + } + /* Move argument to its new location. */ + *ap->argv_put++ = *ap->argv_get++; + ap->argn++; +} + +static int +is_end_of_arguments(struct argparse *ap) +{ + return ap->argn == ap->argc; +} + +/* Get the current argument */ +static char * +get_argument(struct argparse *ap) +{ + return *ap->argv_get; +} + +/* Get the current argument, and also delete it */ +static char * +consume_argument(struct argparse *ap) +{ + char *ret = get_argument(ap); + delete_argument(ap); + return ret; +} + +/****************************************************************************/ + +/* The result of parsing a command-line argument */ +typedef enum { + ARGPARSE_OK, /* Success */ + ARGPARSE_ERROR, /* Error */ + ARGPARSE_DONE /* Success, and do not continue parsing */ +} result; + +typedef result parse_action(struct argparse *ap, struct pb_Parameters *params); + + +/* A command-line option */ +struct option { + char short_name; /* If not 0, the one-character + * name of this option */ + const char *long_name; /* If not NULL, the long name of this option */ + parse_action *action; /* What to do when this option occurs. + * Sentinel value is NULL. + */ +}; + +/* Output file + * + * -o FILE + */ +static result +parse_output_file(struct argparse *ap, struct pb_Parameters *params) +{ + if (is_end_of_arguments(ap)) + { + report_parse_error("Expecting file name after '-o'\n"); + return ARGPARSE_ERROR; + } + + /* Replace the output file name */ + free(params->outFile); + params->outFile = strdup(consume_argument(ap)); + + return ARGPARSE_OK; +} + +/* Input files + * + * -i FILE,FILE,... + */ +static result +parse_input_files(struct argparse *ap, struct pb_Parameters *params) +{ + if (is_end_of_arguments(ap)) + { + report_parse_error("Expecting file name after '-i'\n"); + return ARGPARSE_ERROR; + } + + /* Replace the input file list */ + pb_FreeStringArray(params->inpFiles); + params->inpFiles = read_string_array(consume_argument(ap)); + return ARGPARSE_OK; +} + +/* End of options + * + * -- + */ + +static result +parse_end_options(struct argparse *ap, struct pb_Parameters *params) +{ + return ARGPARSE_DONE; +} + +/* OpenCL device + * + * --device X + */ + +static result +parse_device(struct argparse *ap, struct pb_Parameters *params) +{ + /* Read the next argument, which specifies a device */ + + if (is_end_of_arguments(ap)) + { + report_parse_error("Expecting device specification after '--device'\n"); + return ARGPARSE_ERROR; + } + + char *device_string = consume_argument(ap); + struct pb_DeviceParam *device_param = read_device_param(device_string); + + if (!device_param) { + report_parse_error("Unrecognized device specification format on command line\n"); + return ARGPARSE_ERROR; + } + + /* Save the result */ + pb_FreeDeviceParam(params->device); + params->device = device_param; + + return ARGPARSE_OK; +} + +static result +parse_platform(struct argparse *ap, struct pb_Parameters *params) +{ + /* Read the next argument, which specifies a platform */ + + if (is_end_of_arguments(ap)) + { + report_parse_error("Expecting device specification after '--platform'\n"); + return ARGPARSE_ERROR; + } + + char *platform_string = consume_argument(ap); + struct pb_PlatformParam *platform_param = read_platform_param(platform_string); + + if (!platform_param) { + report_parse_error("Unrecognized platform specification format on command line\n"); + return ARGPARSE_ERROR; + } + + /* Save the result */ + pb_FreePlatformParam(params->platform); + params->platform = platform_param; + + return ARGPARSE_OK; +} + + +static struct option options[] = { + { 'o', NULL, &parse_output_file }, + { 'i', NULL, &parse_input_files }, + { '-', NULL, &parse_end_options }, + { 0, "device", &parse_device }, + { 0, "platform", &parse_platform }, + { 0, NULL, NULL } +}; + +static int +is_last_option(struct option *op) +{ + return op->action == NULL; +} + +/****************************************************************************/ + +/* Parse command-line parameters. + * Return zero on error, nonzero otherwise. + * On error, the other outputs may be invalid. + * + * The information collected from parameters is used to update + * 'ret'. 'ret' should be initialized. + * + * '_argc' and 'argv' are updated to contain only the unprocessed arguments. + */ +static int +pb_ParseParameters (struct pb_Parameters *ret, int *_argc, char **argv) +{ + char *err_message; + struct argparse ap; + + /* Each argument */ + initialize_argparse(&ap, *_argc, argv); + while(!is_end_of_arguments(&ap)) { + result arg_result; /* Result of parsing this option */ + char *arg = get_argument(&ap); + + /* Process this argument */ + if (arg[0] == '-') { + /* Single-character flag */ + if ((arg[1] != 0) && (arg[2] == 0)) { + delete_argument(&ap); /* This argument is consumed here */ + + /* Find a matching short option */ + struct option *op; + for (op = options; !is_last_option(op); op++) { + if (op->short_name == arg[1]) { + arg_result = (*op->action)(&ap, ret); + goto option_was_processed; + } + } + + /* No option matches */ + report_parse_error("Unexpected command-line parameter\n"); + arg_result = ARGPARSE_ERROR; + goto option_was_processed; + } + + /* Long flag */ + if (arg[1] == '-') { + delete_argument(&ap); /* This argument is consumed here */ + + /* Find a matching long option */ + struct option *op; + for (op = options; !is_last_option(op); op++) { + if (op->long_name && strcmp(&arg[2], op->long_name) == 0) { + arg_result = (*op->action)(&ap, ret); + goto option_was_processed; + } + } + + /* No option matches */ + report_parse_error("Unexpected command-line parameter\n"); + arg_result = ARGPARSE_ERROR; + goto option_was_processed; + } + } + else { + /* Other arguments are ignored */ + next_argument(&ap); + arg_result = ARGPARSE_OK; + goto option_was_processed; + } + + option_was_processed: + /* Decide what to do next based on 'arg_result' */ + switch(arg_result) { + case ARGPARSE_OK: + /* Continue processing */ + break; + + case ARGPARSE_ERROR: + /* Error exit from the function */ + return 0; + + case ARGPARSE_DONE: + /* Normal exit from the argument parsing loop */ + goto end_of_options; + } + } /* end for each argument */ + + /* If all arguments were processed, then normal exit from the loop */ + + end_of_options: + finalize_argparse(&ap, _argc, argv); + return 1; +} + +/*****************************************************************************/ +/* Other exported functions */ + +struct pb_Parameters * +pb_ReadParameters(int *_argc, char **argv) +{ + struct pb_Parameters *ret = + (struct pb_Parameters *)malloc(sizeof(struct pb_Parameters)); + + /* Initialize the parameters structure */ + ret->outFile = NULL; + ret->inpFiles = (char **)malloc(sizeof(char *)); + ret->inpFiles[0] = NULL; + ret->platform = NULL; + ret->device = NULL; + + /* Read parameters and update _argc, argv */ + if (!pb_ParseParameters(ret, _argc, argv)) { + /* Parse error */ + pb_FreeParameters(ret); + return NULL; + } + + return ret; +} + +int +pb_Parameters_CountInputs(struct pb_Parameters *p) +{ + int n; + + for (n = 0; p->inpFiles[n]; n++); + return n; +} + diff --git a/benchmarks/opencl/spmv/convert_dataset.c b/benchmarks/opencl/spmv/convert_dataset.c new file mode 100644 index 00000000..122d8819 --- /dev/null +++ b/benchmarks/opencl/spmv/convert_dataset.c @@ -0,0 +1,356 @@ +/* +* NOTES: +* +* 1) Matrix Market files are always 1-based, i.e. the index of the first +* element of a matrix is (1,1), not (0,0) as in C. ADJUST THESE +* OFFSETS ACCORDINGLY when reading and writing +* to files. +* +* 2) ANSI C requires one to use the "l" format modifier when reading +* double precision floating point numbers in scanf() and +* its variants. For example, use "%lf", "%lg", or "%le" +* when reading doubles, otherwise errors will occur. +*/ + +#include "convert_dataset.h" +#include "mmio.h" +#include +#include +#include + +typedef struct _mat_entry { + int row, col; /* i,j */ + float val; +} mat_entry; + +typedef struct _row_stats { // stats on each row + int index; + int size; + int start; + int padding; +} row_stats; + +int sort_rows(const void *a, const void *b) { + return (((mat_entry *)a)->row - ((mat_entry *)b)->row); +} +int sort_cols(const void *a, const void *b) { + return (((mat_entry *)a)->col - ((mat_entry *)b)->col); +} +/* sorts largest by size first */ +int sort_stats(const void *a, const void *b) { + return (((row_stats *)b)->size - ((row_stats *)a)->size); +} + +/* + * COO to JDS matrix conversion. + * + * Needs to output both column and row major JDS formats + * with the minor unit padded to a multiple of `pad_minor` + * and the major unit arranged into groups of `group_size` + * + * Major unit is col, minor is row. Each block is either a scalar or vec4 + * + * Inputs: + * mtx_filename - the file in COO format + * pad_rows - multiple of packed groups to pad each row to + * warp_size - each group of `warp_size` cols is padded to the same amount + * pack_size - number of items to pack + * mirrored - is the input mtx file a symmetric matrix? The other half will be + * filled in if this is =1 + * binary - does the sparse matrix file have values in the format "%d %d" + * or "%d %d %lg"? + * debug_level - 0 for no output, 1 for simple JDS data, 2 for visual grid + * Outputs: + * data - the raw data, padded and grouped as requested + * data_row_ptr - pointer offset into the `data` output, referenced + * by the current row loop index + * nz_count - number of non-zero entries in each row + * indexed by col / warp_size + * data_col_index - corresponds to the col that the same + * array index in `data` is at + * data_row_map - JDS row to real row + * data_cols - number of columns the output JDS matrix has + * dim - dimensions of the input matrix + * data_ptr_len - size of data_row_ptr (maps to original `depth` var) + */ +int coo_to_jds(char *mtx_filename, int pad_rows, int warp_size, int pack_size, + int mirrored, int binary, int debug_level, float **data, + int **data_row_ptr, int **nz_count, int **data_col_index, + int **data_row_map, int *data_cols, int *dim, int *len, + int *nz_count_len, int *data_ptr_len) { + int ret_code; + MM_typecode matcode; + FILE *f; + int nz; + int i; + float *val; + mat_entry *entries; + row_stats *stats; + int rows, cols; + + if ((f = fopen(mtx_filename, "r")) == NULL) + exit(1); + + printf("OK**\n"); + + if (mm_read_banner(f, &matcode) != 0) { + printf("Could not process Matrix Market banner.\n"); + exit(1); + } + + printf("OK**\n"); + + /* This is how one can screen matrix types if their application */ + /* only supports a subset of the Matrix Market data types. */ + + if (mm_is_complex(matcode) && mm_is_matrix(matcode) && + mm_is_sparse(matcode)) { + printf("Sorry, this application does not support "); + printf("Market Market type: [%s]\n", mm_typecode_to_str(matcode)); + exit(1); + } + + /* find out size of sparse matrix .... */ + + if ((ret_code = mm_read_mtx_crd_size(f, &rows, &cols, &nz)) != 0) + exit(1); + *dim = rows; + + if (mirrored) { + // max possible size, might be less because diagonal values aren't doubled + entries = (mat_entry *)malloc(2 * nz * sizeof(mat_entry)); + } else { + entries = (mat_entry *)malloc(nz * sizeof(mat_entry)); + } + + /* NOTE: when reading in doubles, ANSI C requires the use of the "l" */ + /* specifier as in "%lg", "%lf", "%le", otherwise errors will occur */ + /* (ANSI C X3.159-1989, Sec. 4.9.6.2, p. 136 lines 13-15) */ + int cur_i = 0; // to account for mirrored diagonal entries + + for (i = 0; i < nz; i++, cur_i++) { + if (!binary) { + fscanf(f, "%d %d %f\n", &entries[cur_i].row, &entries[cur_i].col, + &entries[cur_i].val); + } else { + fscanf(f, "%d %d\n", &entries[cur_i].row, &entries[cur_i].col); + entries[cur_i].val = 1.0; + } + entries[cur_i].row--; + entries[cur_i].col--; + // printf("%d,%d = %f\n", entries[cur_i].row, entries[cur_i].col, + // entries[cur_i].val); + if (mirrored) { + // fill in mirrored diagonal + if (entries[cur_i].row != entries[cur_i].col) { // not a diagonal value + cur_i++; + entries[cur_i].val = entries[cur_i - 1].val; + entries[cur_i].col = entries[cur_i - 1].row; + entries[cur_i].row = entries[cur_i - 1].col; + // printf("%d,%d = %f\n", entries[cur_i].row, entries[cur_i].col, + // entries[cur_i].val); + } + } + } + // set new non-zero count + nz = cur_i; + if (debug_level >= 1) { + printf("Converting COO to JDS format (%dx%d)\n%d matrix entries, warp size " + "= %d, " + "row padding align = %d, pack size = %d\n\n", + rows, cols, nz, warp_size, pad_rows, pack_size); + } + if (f != stdin) + fclose(f); + + /* + * Now we have an array of values in entries + * Transform to padded JDS format - sort by rows, then fubini + */ + + int irow, icol = 0, istart = 0; + int total_size = 0; + + /* Loop through each entry to figure out padding, grouping that determine + * final data array size + * + * First calculate stats for each row + * + * Collect stats using the major_stats typedef + */ + + qsort(entries, nz, sizeof(mat_entry), sort_rows); // sort by row number + rows = entries[nz - 1].row + 1; // last item is greatest row (zero indexed) + if (rows % warp_size) { // pad group number to warp_size here + rows += warp_size - rows % warp_size; + } + stats = (row_stats *)calloc(rows, sizeof(row_stats)); // set to 0 + *data_row_map = (int *)calloc(rows, sizeof(int)); + irow = entries[0].row; // set first row + + // printf("First row %d\n", irow); + for (i = 0; i < nz; i++) { // loop through each sorted entry + if (entries[i].row != irow || i == nz - 1) { // new row + // printf("%d != %d\n", entries[i].row, irow); + if (i == nz - 1) { + // last item, add it to current row + // printf("Last item i=%d, row=%d, irow=%d\n", i, entries[i].row, irow); + icol++; + } + // hit a new row, record stats for the last row (i-1) + stats[irow].size = icol; // record # cols in previous row + stats[irow].index = entries[i - 1].row; // row # for previous stat item + // printf("Row %d, i=%d, irow=%d\n", entries[i].row, i, irow); + stats[irow].start = istart; // starting location in entries array + // set stats for the next row until this break again + icol = 0; // reset row items + irow = entries[i].row; + istart = i; + } + icol++; // keep track of number of items in this row + } + + *nz_count_len = rows / warp_size + rows % warp_size; + *nz_count = + (int *)malloc(*nz_count_len * sizeof(int)); // only one value per group + + /* sort based upon row size, greatest first */ + qsort(stats, rows, sizeof(row_stats), sort_stats); + /* figure out padding and grouping */ + if (debug_level >= 1) { + printf("Padding data....%d rows, %d groups\n", rows, *nz_count_len); + } + int pad_to, total_padding = 0, pack_to; + pad_rows *= pack_size; // change padding to account for packed items + for (i = 0; i < rows; i++) { + // record JDS to real row number + (*data_row_map)[i] = stats[i].index; + if (i < rows - 1) { + // (*data_row_map)[i]--; // ???? no idea why this is off by 1 + } + // each row is padded so the number of packed groups % pad_rows == 0 + if (i % warp_size == + 0) { // on a group boundary with the largest number of items + // find padding in individual items + if (stats[i].size % pad_rows) { + stats[i].padding = + pad_rows - (stats[i].size % pad_rows); // find padding + } else { + stats[i].padding = 0; // no padding necessary, already at pad multiple + } + if (stats[i].size % pack_size) { + pack_to = ceil((float)stats[i].size / pack_size); + } else { + pack_to = stats[i].size / pack_size; + } + // pack_to = stats[i].size + (!stats[i].size%pack_size) ? 0 : (pack_size - + // stats[i].size%pack_size); + pad_to = stats[i].size + + stats[i].padding; // total size of this row, with padding + // TODO: change this to reflect the real number of nonzero packed items, + // not the padded + // value + (*nz_count)[i / warp_size] = + pack_to; // number of packed items in this group + total_size += pad_to * warp_size; // allocate size for this padded group + if (debug_level >= 2) + printf("Padding warp group %d to %d items, zn = %d\n", i / warp_size, + pad_to, pack_to); + } else { + stats[i].padding = pad_to - stats[i].size; + } + total_padding += stats[i].padding; + // if (debug_level >= 2) + // printf("Row %d, %d items, %d padding\n", stats[i].index, + // stats[i].size, stats[i].padding); + } + + /* allocate data and data_row_index */ + if (debug_level >= 1) + printf("Allocating data space: %d entries (%f%% padding)\n", total_size, + (float)100 * total_padding / total_size); + *data = (float *)calloc(total_size, + sizeof(float)); // set to 0 so padded values are set + *data_col_index = + (int *)calloc(total_size, sizeof(int)); // any unset indexes point to 0 + *data_row_ptr = (int *)calloc(rows, sizeof(int)); + *len = total_size; + i = 0; // data index, including padding + + /* + * Keep looping through each row, writing data a group at a time + * to the output array. Increment `irow` each time, and use it as + * an index into entries along with stats.start to get the next + * data item + */ + irow = 0; // keep track of which row we are in inside the fubini-ed array + int idata = 0; // position within final data array + int entry_index, j; + int ipack; // used in internal loop for writing packed values + mat_entry entry; + while (1) { + /* record data_row_ptr */ + (*data_row_ptr)[irow] = idata; + + /* End condtion: the size of the greatest row is smaller than the current + Fubini-ed row */ + if (stats[0].size + stats[0].padding <= irow * pack_size) + break; + + // printf("Data row pointer for row %d is %d\n", irow, idata); + for (i = 0; i < rows; i++) { + /* take one packed group from each original row */ + // printf("Output irow %d icol %d (real %d,%d size %d)\n", irow, i, + // entry.col, i, stats[i].size); + /* Watch out for little vs big endian, and how opencl interprets vector + * casting from pointers */ + for (ipack = 0; ipack < pack_size; ipack++) { + if (stats[i].size > irow * pack_size + ipack) { + // copy value + entry_index = stats[i].start + irow * pack_size + ipack; + entry = entries[entry_index]; + /* record index and value */ + (*data)[idata] = entry.val; + /* each data item will get its row index from the thread, col from + * here */ + (*data_col_index)[idata] = entry.col; + + if (debug_level >= 2) { + if (i < 3) { + // first row debugging + printf("[%d row%d=%.3f]", ipack + 1, i, entry.val); + } else { + printf("%d", ipack + 1); + } + } + } else if (stats[i].size + stats[i].padding > + irow * pack_size + ipack) { + /* add padding to the end of each row here - this assumes padding is + * factored into allocated size */ + if (debug_level >= 2) + printf("0"); + (*data_col_index)[idata] = -1; + } else { + goto endwrite; // no data written this pass, so don't increment idata + } + idata += 1; + } + } + endwrite: + if (debug_level >= 2) { + printf("\n"); + } + irow += 1; + } + + if (debug_level >= 1) + printf("Finished converting.\nJDS format has %d columns, %d rows.\n", rows, + irow); + free(entries); + free(stats); + printf("nz_count_len = %d\n", *nz_count_len); + + *data_cols = rows; + *data_ptr_len = irow + 1; + return 0; +} diff --git a/benchmarks/opencl/spmv/convert_dataset.h b/benchmarks/opencl/spmv/convert_dataset.h new file mode 100644 index 00000000..a495ffa6 --- /dev/null +++ b/benchmarks/opencl/spmv/convert_dataset.h @@ -0,0 +1,17 @@ +#ifndef _CONVERT_DATASET_H +#define _CONVERT_DATASET_H + +#ifdef __cplusplus +extern "C" { +#endif +int coo_to_jds(char* mtx_filename, int pad_rows, int warp_size, int pack_size, + int mirrored, int binary, int debug_level, + float** data, int** data_row_ptr, int** nz_count, int** data_col_index, + int** data_row_map, int* data_cols, int* dim, int* len, int* nz_count_len, + int* data_ptr_len); + +#ifdef __cplusplus +} +#endif + +#endif \ No newline at end of file diff --git a/benchmarks/opencl/spmv/file.c b/benchmarks/opencl/spmv/file.c new file mode 100644 index 00000000..a98451e5 --- /dev/null +++ b/benchmarks/opencl/spmv/file.c @@ -0,0 +1,78 @@ +/*************************************************************************** + *cr + *cr (C) Copyright 2007 The Board of Trustees of the + *cr University of Illinois + *cr All Rights Reserved + *cr + ***************************************************************************/ + +//#include +#include +#include +#include +#include + +#if __BYTE_ORDER != __LITTLE_ENDIAN +# error "File I/O is not implemented for this system: wrong endianness." +#endif + +void inputData(char* fName, int* len, int* depth, int* dim,int *nzcnt_len,int *pad, + float** h_data, int** h_indices, int** h_ptr, + int** h_perm, int** h_nzcnt) +{ + FILE* fid = fopen(fName, "rb"); + + if (fid == NULL) + { + fprintf(stderr, "Cannot open input file\n"); + exit(-1); + } + + fscanf(fid, "%d %d %d %d %d\n",len,depth,nzcnt_len,dim,pad); + int _len=len[0]; + int _depth=depth[0]; + int _dim=dim[0]; + int _pad=pad[0]; + int _nzcnt_len=nzcnt_len[0]; + + *h_data = (float *) malloc(_len * sizeof (float)); + fread (*h_data, sizeof (float), _len, fid); + + *h_indices = (int *) malloc(_len * sizeof (int)); + fread (*h_indices, sizeof (int), _len, fid); + + *h_ptr = (int *) malloc(_depth * sizeof (int)); + fread (*h_ptr, sizeof (int), _depth, fid); + + *h_perm = (int *) malloc(_dim * sizeof (int)); + fread (*h_perm, sizeof (int), _dim, fid); + + *h_nzcnt = (int *) malloc(_nzcnt_len * sizeof (int)); + fread (*h_nzcnt, sizeof (int), _nzcnt_len, fid); + + fclose (fid); +} + +void input_vec(char *fName,float *h_vec,int dim) +{ + FILE* fid = fopen(fName, "rb"); + fread (h_vec, sizeof (float), dim, fid); + fclose(fid); + +} + +void outputData(char* fName, float *h_Ax_vector,int dim) +{ + FILE* fid = fopen(fName, "w"); + uint32_t tmp32; + if (fid == NULL) + { + fprintf(stderr, "Cannot open output file\n"); + exit(-1); + } + tmp32 = dim; + fwrite(&tmp32, sizeof(uint32_t), 1, fid); + fwrite(h_Ax_vector, sizeof(float), dim, fid); + + fclose (fid); +} diff --git a/benchmarks/opencl/spmv/file.h b/benchmarks/opencl/spmv/file.h new file mode 100644 index 00000000..65f80135 --- /dev/null +++ b/benchmarks/opencl/spmv/file.h @@ -0,0 +1,18 @@ +/*************************************************************************** + *cr + *cr (C) Copyright 2007 The Board of Trustees of the + *cr University of Illinois + *cr All Rights Reserved + *cr + ***************************************************************************/ +#ifndef __FILEH__ +#define __FILEH__ + +void inputData(char* fName, int* len, int* depth, int* dim,int *nzcnt_len,int *pad, + float** h_data, int** h_indices, int** h_ptr, + int** h_perm, int** h_nzcnt); + +void input_vec(char* fNanme, float *h_vec,int dim); +void outputData(char* fName, float *h_Ax_vector,int dim); + +#endif \ No newline at end of file diff --git a/benchmarks/opencl/spmv/gpu_info.c b/benchmarks/opencl/spmv/gpu_info.c new file mode 100644 index 00000000..4d641f81 --- /dev/null +++ b/benchmarks/opencl/spmv/gpu_info.c @@ -0,0 +1,55 @@ +/*************************************************************************** + *cr + *cr (C) Copyright 2010 The Board of Trustees of the + *cr University of Illinois + *cr All Rights Reserved + *cr + ***************************************************************************/ +//#include +#include +#include +#include +#include + +#include "gpu_info.h" + +void compute_active_thread(size_t *thread, + size_t *grid, + int task, + int pad, + int major, + int minor, + int sm) +{ + int max_thread; + int max_block=8; + if(major==1) + { + if(minor>=2) + max_thread=1024; + else + max_thread=768; + } + else if(major==2) + max_thread=1536; + else + //newer GPU //keep using 2.0 + max_thread=1536; + + int _grid; + int _thread; + + if(task*pad>sm*max_thread) + { + _thread=max_thread/max_block; + _grid = ((task*pad+_thread-1)/_thread)*_thread; + } + else + { + _thread=pad; + _grid=task*pad; + } + + thread[0]=_thread; + grid[0]=_grid; +} diff --git a/benchmarks/opencl/spmv/gpu_info.h b/benchmarks/opencl/spmv/gpu_info.h new file mode 100644 index 00000000..4219cda9 --- /dev/null +++ b/benchmarks/opencl/spmv/gpu_info.h @@ -0,0 +1,20 @@ +/*************************************************************************** + *cr + *cr (C) Copyright 2010 The Board of Trustees of the + *cr University of Illinois + *cr All Rights Reserved + *cr + ***************************************************************************/ + +#ifndef __GPUINFOH__ +#define __GPUINFOH__ + +void compute_active_thread(size_t *thread, + size_t *grid, + int task, + int pad, + int major, + int minor, + int sm); + +#endif diff --git a/benchmarks/opencl/spmv/input/1138_bus.mtx b/benchmarks/opencl/spmv/input/1138_bus.mtx new file mode 100755 index 00000000..09591c70 --- /dev/null +++ b/benchmarks/opencl/spmv/input/1138_bus.mtx @@ -0,0 +1,2610 @@ +%%MatrixMarket matrix coordinate real symmetric +%------------------------------------------------------------------------------- +% UF Sparse Matrix Collection, Tim Davis +% http://www.cise.ufl.edu/research/sparse/matrices/HB/1138_bus +% name: HB/1138_bus +% [S ADMITTANCE MATRIX 1138 BUS POWER SYSTEM, D.J.TYLAVSKY, JULY 1985.] +% id: 1 +% date: 1985 +% author: D. Tylavsky +% ed: I. Duff, R. Grimes, J. Lewis +% fields: title A name id date author ed kind +% kind: power network problem +%------------------------------------------------------------------------------- +1138 1138 2596 +1 1 1474.779 +5 1 -9.017133 +563 1 -5.730659 +2 2 9.136654 +10 2 -3.405995 +563 2 -5.730659 +3 3 69.61468 +11 3 -8.810573 +34 3 -31.15265 +35 3 -16.06684 +104 3 -4.86926 +475 3 -8.715357 +4 4 68.60106 +7 4 -34.62025 +27 4 -.4755112 +101 4 -28.66497 +102 4 -.7463244 +103 4 -4.093998 +5 5 13.88805 +9 5 -4.870921 +6 6 116.8288 +7 6 -10.96124 +37 6 -56.81818 +98 6 -10.88139 +103 6 -38.16794 +7 7 52.76726 +37 7 -4.773726 +101 7 -.7772424 +102 7 -1.072973 +103 7 -.5618293 +8 8 30.25788 +26 8 -3.752486 +35 8 -.4975867 +724 8 -26.0078 +9 9 7.174451 +10 9 -.905633 +104 9 -1.397898 +10 10 5.709526 +104 10 -1.397898 +11 11 27.18584 +12 11 -1.238697 +38 11 -7.418397 +566 11 -9.718173 +12 12 1.238697 +13 13 7.142646 +34 13 -1.85192 +104 13 -5.290726 +14 14 12.95337 +413 14 -12.95337 +15 15 1696.756 +16 15 -30.67485 +17 15 -30.4878 +18 15 -30.39514 +19 15 -30.39514 +411 15 -1574.803 +16 16 30.67485 +17 17 30.4878 +18 18 30.39514 +19 19 30.39514 +20 20 5746.122 +21 20 -5714.287 +37 20 -18.24817 +102 20 -13.58696 +21 21 5735.022 +22 21 -5.205622 +23 21 -5.186722 +24 21 -5.181347 +25 21 -5.162622 +22 22 5.205622 +23 23 5.186722 +24 24 5.181347 +25 25 5.162622 +26 26 4.371336 +35 26 -.6188502 +27 27 9.123474 +28 27 -2.373605 +29 27 -2.439024 +30 27 -1.190334 +101 27 -2.644999 +28 28 2.373605 +29 29 2.439024 +30 30 1.190334 +31 31 1.688214 +32 31 -.6254691 +100 31 -1.062744 +32 32 1.688214 +100 32 -1.062744 +33 33 .6581979 +100 33 -.6581979 +34 34 56.31459 +104 34 -10.13479 +553 34 -13.17523 +35 35 10018.3 +104 35 -1.112756 +710 35 -10000 +36 36 10000 +711 36 -10000 +37 37 80.33353 +102 37 -.4934373 +38 38 74.99123 +39 38 -15.67398 +98 38 -46.2963 +412 38 -5.602555 +39 39 16.41742 +99 39 -.7434391 +40 40 245.9335 +41 40 -126.5823 +43 40 -55.24862 +45 40 -64.10257 +41 41 988.9011 +42 41 -28.98551 +44 41 -833.3333 +42 42 28.98551 +43 43 4403.075 +146 43 -4347.826 +44 44 10833.33 +486 44 -10000 +45 45 118.7474 +49 45 -54.64481 +46 46 10000 +48 46 -10000 +47 47 104.9318 +48 47 -104.9318 +48 48 20183.36 +54 48 -78.43137 +506 48 -10000 +49 49 288.754 +50 49 -126.5823 +53 49 -107.5269 +50 50 988.9011 +51 50 -28.98551 +52 50 -833.3333 +51 51 28.98551 +52 52 874.3169 +125 52 -40.98361 +53 53 1586.098 +54 53 -1428.572 +55 53 -50 +54 54 1957.633 +425 54 -119.0476 +445 54 -184.5238 +447 54 -147.0588 +55 55 50 +56 56 29.32551 +63 56 -29.32551 +57 57 112.3596 +66 57 -112.3596 +58 58 15.64945 +70 58 -15.64945 +59 59 24.3309 +67 59 -24.3309 +60 60 104.1667 +68 60 -104.1667 +61 61 125 +130 61 -125 +62 62 50.76142 +129 62 -50.76142 +63 63 10059.54 +64 63 -30.21148 +65 63 -10000 +64 64 159.7102 +71 64 -40.98361 +131 64 -16.05136 +226 64 -72.46377 +65 65 10015.92 +1095 65 -15.92357 +66 66 442.0023 +67 66 -97.08738 +68 66 -24.27184 +74 66 -32.65326 +183 66 -147.0588 +226 66 -28.57143 +67 67 167.4071 +68 67 -30.67485 +69 67 -15.31394 +68 68 289.2932 +74 68 -47.5921 +90 68 -32.33649 +201 68 -50.25126 +69 69 190.7525 +70 69 -175.4386 +70 70 225.5077 +182 70 -4.955401 +196 70 -5.170631 +1063 70 -4.623209 +1065 70 -16.47446 +1074 70 -3.195909 +71 71 96.53917 +72 71 -55.55556 +72 72 84.92064 +73 72 -5.555555 +121 72 -23.80952 +73 73 5.555555 +74 74 384.9024 +75 74 -40.32258 +76 74 -25.44529 +78 74 -34.3871 +80 74 -15.82279 +119 74 -188.6792 +75 75 693.1041 +204 75 -645.1613 +810 75 -4.608295 +918 75 -3.012048 +76 76 147.3945 +77 76 -30.03003 +78 76 -36.36364 +127 76 -55.55556 +77 77 5030.03 +1050 77 -5000 +78 78 262.67 +79 78 -90.90909 +452 78 -101.0101 +79 79 564.4214 +416 79 -48.07692 +417 79 -48.07692 +454 79 -377.3585 +80 80 657.1666 +81 80 -67.1141 +447 80 -476.1905 +452 80 -98.03922 +81 81 420.0095 +448 81 -158.7302 +450 81 -175.4386 +464 81 -18.72659 +82 82 4.975124 +94 82 -4.975124 +83 83 17.24138 +94 83 -17.24138 +84 84 17.57469 +95 84 -17.57469 +85 85 439.0805 +87 85 -172.4138 +126 85 -266.6667 +86 86 6807.496 +87 86 -5000 +241 86 -46.51163 +251 86 -71.94245 +253 86 -74.62687 +267 86 -94.78674 +269 86 -277.7778 +283 86 -138.8889 +291 86 -49.01961 +293 86 -57.14286 +294 86 -57.14286 +302 86 -689.6552 +315 86 -250 +87 87 5319.472 +88 87 -147.0588 +88 88 147.0588 +89 89 65.3412 +105 89 -38.31417 +115 89 -27.02703 +90 90 64.83059 +91 90 -32.4941 +91 91 645.056 +92 91 -163.6515 +108 91 -67.56757 +115 91 -24.19992 +116 91 -357.1429 +92 92 258.8896 +93 92 -95.2381 +93 93 612.1849 +94 93 -10.90512 +252 93 -131.5789 +264 93 -12.67427 +271 93 -66.0066 +288 93 -65.35947 +292 93 -31.84714 +293 93 -38.31417 +294 93 -38.31417 +322 93 -32.15434 +323 93 -34.8432 +324 93 -24.03846 +325 93 -14.40922 +326 93 -16.50165 +94 94 464.0852 +96 94 -4.800768 +264 94 -239.4475 +318 94 -110.6696 +325 94 -76.04562 +95 95 76.57204 +97 95 -8.285004 +262 95 -9.67118 +268 95 -7.142857 +300 95 -33.8983 +96 96 10022.9 +145 96 -7.24113 +704 96 -10.85776 +705 96 -10000 +97 97 12.12083 +268 97 -3.835827 +98 98 57.17769 +99 99 3.134068 +413 99 -2.390629 +100 100 40.73998 +725 100 -35.58719 +732 100 -2.369107 +101 101 34.2926 +102 101 -.754091 +103 101 -1.451295 +102 102 47.04131 +103 102 -30.38753 +103 103 10074.66 +478 103 -10000 +104 104 24.20333 +105 105 420.7618 +106 105 -12.07729 +109 105 -370.3704 +106 106 12.07729 +107 107 32.15434 +108 107 -32.15434 +108 108 1008.813 +110 108 -909.0909 +109 109 10370.37 +1029 109 -10000 +110 110 10974.79 +111 110 -41.84558 +328 110 -10000 +367 110 -10.77006 +370 110 -13.08045 +111 111 10056.95 +367 111 -7.27802 +368 111 -7.824726 +701 111 -10000 +112 112 301.7592 +113 112 -7.420491 +370 112 -8.624407 +686 112 -285.7143 +113 113 10040.49 +114 113 -7.624856 +115 113 -25.44529 +690 113 -10000 +114 114 10007.63 +1009 114 -10000 +115 115 76.67225 +116 116 436.123 +117 116 -10.9529 +126 116 -68.02721 +117 117 10.9529 +118 118 5.521811 +119 118 -5.521811 +119 119 228.3307 +123 119 -34.12969 +120 120 126.5823 +121 120 -126.5823 +121 121 272.2888 +122 121 -70.87661 +504 121 -51.02041 +122 122 152.6848 +123 122 -40.48583 +124 122 -41.32232 +123 123 111.929 +125 123 -37.31343 +124 124 97.81158 +125 124 -32.67974 +127 124 -23.80952 +125 125 363.2857 +126 125 -81.38689 +129 125 -70.92198 +546 125 -100 +126 126 478.1926 +129 126 -62.1118 +127 127 84.92064 +128 127 -5.555555 +128 128 5.555555 +129 129 183.7952 +130 130 356.9032 +133 130 -136.9863 +173 130 -51.81347 +213 130 -43.10345 +131 131 896.646 +132 131 -75.18797 +133 131 -769.2308 +136 131 -36.17595 +132 132 101.0104 +144 132 -4.409171 +742 132 -21.41328 +133 133 918.7799 +134 133 -12.56281 +134 134 12.56281 +135 135 10030.19 +136 135 -25.44529 +139 135 -4.748338 +740 135 -10000 +136 136 99.82595 +141 136 -38.20471 +137 137 108.031 +139 137 -72.46377 +141 137 -27.39726 +761 137 -8.169934 +138 138 217.898 +141 138 -28.98551 +877 138 -59.88025 +882 138 -129.0323 +139 139 90.06557 +140 139 -12.85347 +140 140 45.52996 +829 140 -18.62891 +830 140 -14.04757 +141 141 94.58748 +142 142 36.70324 +258 142 -21.55172 +366 142 -15.15152 +143 143 20014.59 +144 143 -12.5 +743 143 -2.090738 +745 143 -10000 +826 143 -10000 +144 144 10016.91 +827 144 -10000 +145 145 10022.01 +318 145 -14.77105 +703 145 -10000 +146 146 7289.003 +147 146 -2941.177 +147 147 12941.18 +148 147 -10000 +148 148 10000 +149 149 5.205622 +166 149 -5.205622 +150 150 73.52941 +173 150 -73.52941 +151 151 13.29787 +175 151 -13.29787 +152 152 10.40583 +180 152 -10.40583 +153 153 34.48276 +183 153 -34.48276 +154 154 13.71742 +207 154 -13.71742 +155 155 28.98551 +198 155 -28.98551 +156 156 144.9275 +213 156 -144.9275 +157 157 38.46154 +219 157 -38.46154 +158 158 28.0112 +226 158 -28.0112 +159 159 9.64196 +182 159 -4.730369 +205 159 -4.911591 +160 160 21.37365 +172 160 -10.01001 +233 160 -11.36364 +161 161 10.90321 +222 161 -4.730369 +743 161 -6.17284 +162 162 87.51172 +163 162 -46.08295 +169 162 -11.53403 +209 162 -7.451564 +223 162 -12.61034 +232 162 -9.832842 +163 163 145.9383 +186 163 -46.51163 +201 163 -13.6612 +227 163 -39.68254 +164 164 5.205622 +166 164 -5.205622 +165 165 117.6942 +166 165 -60.60606 +167 165 -17.33102 +175 165 -11.18568 +194 165 -28.57143 +166 166 341.3369 +192 166 -80 +201 166 -14.88095 +218 166 -175.4386 +167 167 34.66204 +175 167 -17.33102 +168 168 69.63064 +180 168 -9.389671 +209 168 -60.24096 +169 169 25.6384 +187 169 -14.10437 +170 170 30.28655 +188 170 -22.77904 +194 170 -7.507507 +171 171 52.2535 +178 171 -39.21569 +217 171 -13.03781 +172 172 66.85269 +173 172 -42.55319 +222 172 -4.873294 +755 172 -9.416196 +173 173 206.2103 +184 173 -38.31417 +174 174 50.63389 +211 174 -12.7551 +216 174 -37.87879 +175 175 53.63491 +179 175 -11.82033 +176 176 860.8028 +177 176 -172.4138 +178 176 -20.98079 +207 176 -667.4082 +177 177 276.5804 +207 177 -104.1667 +178 178 88.17661 +214 178 -15.06024 +217 178 -12.9199 +179 179 23.64066 +203 179 -11.82033 +180 180 28.49115 +209 180 -8.695652 +181 181 23.14815 +183 181 -23.14815 +182 182 61.50294 +183 182 -29.94012 +205 182 -16.72241 +221 182 -5.154639 +183 183 398.0187 +184 183 -91.74312 +201 183 -35.14939 +226 183 -36.49635 +184 184 192.1349 +193 184 -41.40787 +213 184 -20.6697 +185 185 8.064516 +187 185 -8.064516 +186 186 126.2011 +187 186 -41.66667 +218 186 -38.02281 +187 187 107.524 +188 187 -29.67359 +206 187 -8.695652 +209 187 -5.319149 +188 188 52.45263 +189 189 41.49792 +211 189 -11.46789 +216 189 -30.03003 +190 190 33.0033 +199 190 -33.0033 +191 191 1149.633 +195 191 -10.71811 +197 191 -14.74926 +198 191 -1111.111 +211 191 -13.05483 +192 192 824.6421 +199 192 -666.6667 +218 192 -63.29114 +220 192 -14.68429 +193 193 247.007 +198 193 -60.60606 +199 193 -106.383 +208 193 -38.61004 +194 194 78.9974 +216 194 -42.91846 +195 195 36.89613 +211 195 -26.17801 +196 196 5.170631 +197 197 36.48839 +212 197 -21.73913 +198 198 1200.703 +199 199 806.053 +200 200 60.0913 +201 200 -43.87481 +202 200 -16.21649 +201 201 172.6985 +218 201 -14.88095 +202 202 20.9357 +203 202 -4.719207 +203 203 16.53954 +204 204 645.1613 +205 205 28.7262 +221 205 -7.092198 +206 206 18.89453 +216 206 -10.19888 +207 207 881.1484 +208 207 -84.38818 +224 207 -11.46789 +208 208 122.9982 +209 209 81.70734 +210 210 1796.925 +215 210 -11.21076 +217 210 -1666.667 +218 210 -119.0476 +211 211 84.37634 +212 211 -20.9205 +212 212 42.65963 +213 213 282.2301 +504 213 -73.52941 +214 214 65.82166 +217 214 -50.76142 +215 215 11.21076 +216 216 148.1264 +218 216 -27.10027 +217 217 1743.386 +218 218 437.7814 +219 219 150.9842 +220 219 -86.20689 +230 219 -26.31579 +220 220 100.8912 +221 221 18.99606 +222 221 -3.15856 +225 221 -3.590664 +222 222 91.18269 +225 222 -26.24672 +233 222 -49.50495 +743 222 -2.668802 +223 223 54.62715 +232 223 -42.01681 +224 224 18.73533 +1026 224 -7.267442 +225 225 29.83738 +226 226 165.5428 +227 227 85.76549 +228 227 -46.08295 +228 228 97.42727 +229 228 -27.62431 +231 228 -12.7551 +232 228 -10.96491 +229 229 27.62431 +230 230 26.31579 +231 231 31.20529 +232 231 -18.45018 +232 232 81.26475 +233 233 60.86859 +234 234 29.17592 +236 234 -14.55604 +307 234 -14.61988 +235 235 2117.081 +236 235 -67.56757 +243 235 -14.59854 +270 235 -2000 +272 235 -9.208103 +298 235 -12.85347 +299 235 -12.85347 +236 236 90.2339 +286 236 -8.1103 +237 237 123.4571 +261 237 -2.818489 +287 237 -18.97533 +309 237 -69.68641 +366 237 -4.004806 +702 237 -27.97203 +238 238 188.1504 +239 238 -41.25455 +270 238 -146.8959 +239 239 59.24802 +270 239 -15.33742 +281 239 -2.656042 +240 240 7.917656 +259 240 -7.917656 +241 241 2888.018 +242 241 -185.1852 +246 241 -89.28572 +253 241 -123.4568 +257 241 -92.59259 +266 241 -123.4568 +275 241 -13.44086 +279 241 -31.74603 +289 241 -30.03003 +291 241 -833.3333 +292 241 -833.3333 +293 241 -138.8889 +294 241 -138.8889 +299 241 -13.28021 +310 241 -16.80672 +321 241 -88.49557 +327 241 -89.28572 +242 242 314.7918 +244 242 -57.14286 +245 242 -72.46377 +243 243 14.59854 +244 244 118.4926 +272 244 -61.34969 +245 245 95.93795 +298 245 -23.47418 +246 246 113.1521 +310 246 -23.86635 +247 247 582.7703 +248 247 -270.2703 +255 247 -312.5 +248 248 502.8284 +255 248 -232.5581 +249 249 1666.667 +282 249 -1666.667 +250 250 1123.714 +257 250 -92.59259 +263 250 -78.74016 +289 250 -119.0476 +317 250 -833.3333 +251 251 210.8313 +267 251 -138.8889 +252 252 288.2195 +310 252 -21.50537 +313 252 -135.1351 +253 253 198.0837 +254 254 383.7719 +255 254 -208.3333 +285 254 -175.4386 +255 255 910.2564 +258 255 -31.44654 +260 255 -71.94245 +263 255 -53.47594 +256 256 10133 +290 256 -52.35602 +317 256 -10000 +321 256 -80.64516 +257 257 185.1852 +258 258 261.8486 +261 258 -128.2051 +327 258 -80.64516 +259 259 20.85427 +278 259 -12.93661 +260 260 190.9901 +289 260 -119.0476 +261 261 131.0236 +262 262 183.095 +268 262 -26.17801 +276 262 -62.5 +314 262 -84.74576 +263 263 132.2161 +264 264 383.7008 +311 264 -131.5789 +265 265 1017.575 +276 265 -999.9999 +300 265 -17.57469 +266 266 246.9136 +307 266 -123.4568 +267 267 566.0127 +282 267 -144.9275 +284 267 -77.51938 +302 267 -109.8901 +268 268 158.6071 +270 268 -53.88283 +301 268 -67.56757 +269 269 486.1111 +284 269 -208.3333 +270 270 2317.201 +275 270 -12.77139 +279 270 -8.257638 +280 270 -29.32551 +318 270 -34.36426 +320 270 -16.36661 +271 271 66.0066 +272 272 70.5578 +273 273 1295.094 +274 273 -303.0303 +303 273 -714.2858 +315 273 -277.7778 +274 274 606.0606 +312 274 -303.0303 +275 275 26.21225 +276 276 1062.5 +277 277 5086.957 +285 277 -86.95652 +289 277 -5000 +278 278 27.55649 +309 278 -14.61988 +279 279 40.00367 +280 280 29.32551 +281 281 2.656042 +282 282 1811.594 +283 283 214.6465 +292 283 -75.75758 +284 284 285.8527 +285 285 262.3951 +286 286 8.1103 +287 287 18.97533 +288 288 127.4713 +291 288 -62.1118 +289 289 5268.125 +290 290 52.35602 +291 291 944.4647 +292 292 940.938 +293 293 234.3459 +294 294 234.3459 +295 295 38.31417 +298 295 -38.31417 +296 296 101.0101 +297 296 -101.0101 +297 297 118.4317 +299 297 -17.4216 +298 298 74.64183 +299 299 43.55529 +300 300 122.395 +696 300 -70.92198 +301 301 153.7745 +314 301 -86.20689 +302 302 799.5453 +303 303 1428.572 +312 303 -714.2858 +304 304 1380.582 +305 304 -263.1579 +315 304 -208.3333 +329 304 -909.0909 +305 305 459.2363 +315 305 -196.0784 +306 306 37.87879 +322 306 -37.87879 +307 307 138.0767 +308 308 81.30082 +323 308 -81.30082 +309 309 84.30629 +310 310 62.17845 +311 311 200.5445 +326 311 -68.96552 +312 312 1295.094 +315 312 -277.7778 +313 313 135.1351 +314 314 170.9527 +315 315 1688.798 +316 315 -322.5806 +319 315 -125 +323 315 -31.25 +316 316 360.1746 +322 316 -37.59399 +317 317 10833.33 +318 318 169.825 +704 318 -10.02004 +319 319 1236.111 +329 319 -1111.111 +320 320 16.36661 +321 321 169.1407 +322 322 107.6271 +323 323 147.394 +324 324 51.89362 +325 324 -27.85515 +325 325 118.31 +326 326 85.46717 +327 327 169.9309 +328 328 10000 +329 329 2020.202 +330 330 2.409639 +335 330 -2.409639 +331 331 .8810573 +336 331 -.8810573 +332 332 1.666667 +337 332 -1.666667 +333 333 2.608922 +350 333 -2.608922 +334 334 1.446132 +350 334 -1.446132 +335 335 30.26479 +337 335 -27.85515 +336 336 218.2724 +337 336 -217.3913 +337 337 260.9173 +338 337 -3.840246 +339 337 -3.955696 +340 337 -6.208189 +338 338 19.08415 +340 338 -15.2439 +339 339 19.1996 +340 339 -15.2439 +340 340 59.70203 +341 340 -8.1103 +342 340 -14.89573 +341 341 20.88169 +342 341 -12.77139 +342 342 66.7209 +343 342 -5.675369 +344 342 -10.71237 +345 342 -4.215851 +370 342 -18.45018 +343 343 15.14507 +346 343 -9.469697 +344 344 28.10368 +346 344 -17.3913 +345 345 26.63738 +346 345 -22.42153 +346 346 102.5212 +347 346 -14.08451 +349 346 -20.70393 +367 346 -18.45018 +347 347 54.57034 +348 347 -40.48583 +348 348 79.09587 +349 348 -38.61004 +349 349 83.58582 +350 349 -24.27184 +350 350 28.3269 +351 351 7.374631 +366 351 -7.374631 +352 352 14.91256 +353 352 -5.289326 +408 352 -2.934273 +409 352 -6.688963 +353 353 17.04893 +363 353 -3.816794 +382 353 -7.942812 +354 354 28.97867 +382 354 -6.199628 +383 354 -22.77904 +355 355 2.164971 +375 355 -2.164971 +356 356 13.59556 +383 356 -5.344736 +398 356 -8.250825 +357 357 1.307306 +394 357 -.5700604 +395 357 -.7372456 +358 358 46.94836 +479 358 -46.94836 +359 359 18.55447 +474 359 -18.55447 +360 360 8.05153 +715 360 -8.05153 +361 361 509.4518 +479 361 -9.451796 +480 361 -500 +362 362 22.4661 +403 362 -11.03753 +714 362 -11.42857 +363 363 96.66204 +364 363 -9.149131 +365 363 -9.149131 +373 363 -9.149131 +374 363 -9.149131 +388 363 -34.96503 +395 363 -4.926108 +407 363 -10.75847 +474 363 -5.599104 +364 364 9.149131 +365 365 9.149131 +366 366 41.86838 +699 366 -15.33742 +367 367 307.3316 +368 367 -104.1667 +372 367 -166.6667 +368 368 111.9914 +369 369 72.9927 +371 369 -72.9927 +370 370 40.15504 +371 371 239.6594 +372 371 -166.6667 +372 372 333.3333 +373 373 9.149131 +374 374 9.149131 +375 375 122.031 +376 375 -2.019386 +377 375 -2.019386 +378 375 -2.019386 +379 375 -1.959632 +380 375 -1.959632 +381 375 -1.959632 +389 375 -105.8201 +404 375 -2.108815 +376 376 2.019386 +377 377 2.019386 +378 378 2.019386 +379 379 1.959632 +380 380 1.959632 +381 381 1.959632 +382 382 14.14244 +383 383 84.26632 +384 383 -8.143323 +385 383 -8.032128 +386 383 -16.50165 +387 383 -16.50165 +392 383 -6.963788 +384 384 8.143323 +385 385 8.032128 +386 386 16.50165 +387 387 16.50165 +388 388 94.03941 +389 388 -22.98851 +390 388 -7.077141 +391 388 -7.077141 +395 388 -4.739336 +401 388 -9.416196 +405 388 -7.77605 +389 389 128.8086 +390 390 7.077141 +391 391 7.077141 +392 392 10.41564 +395 392 -3.451847 +393 393 32.25806 +474 393 -16.12903 +479 393 -16.12903 +394 394 20.16254 +395 394 -19.59248 +395 395 785.9075 +398 395 -76.33588 +399 395 -77.51938 +405 395 -10.71811 +406 395 -473.7875 +407 395 -89.28572 +474 395 -24.8139 +396 396 97.03087 +406 396 -86.43042 +716 396 -5.336179 +717 396 -5.264266 +397 397 690.4763 +716 397 -357.1429 +717 397 -333.3333 +398 398 84.5867 +399 399 77.51938 +400 400 10000 +473 400 -10000 +401 401 29.10979 +402 401 -8.038586 +714 401 -11.65501 +402 402 8.038586 +403 403 11.03753 +404 404 5.644883 +409 404 -3.536068 +405 405 18.49416 +406 406 10591.18 +479 406 -23.97024 +483 406 -10000 +716 406 -3.519887 +717 406 -3.472463 +407 407 100.0442 +408 408 2.934273 +409 409 10.22503 +410 410 10155.49 +411 410 -10000 +412 410 -80.19246 +731 410 -75.30121 +411 411 11741.83 +476 411 -48.89976 +477 411 -68.37394 +486 411 -49.75124 +412 412 111.7064 +413 412 -9.425071 +478 412 -3.253196 +709 412 -.6465378 +724 412 -12.58653 +413 413 24.76907 +414 414 319.0159 +431 414 -227.2727 +433 414 -91.74312 +415 415 319.0159 +432 415 -227.2727 +433 415 -91.74312 +416 416 362.0423 +433 416 -91.74312 +462 416 -222.2222 +417 417 362.0423 +433 417 -91.74312 +462 417 -222.2222 +418 418 39.25825 +419 418 -9.389671 +545 418 -29.86858 +419 419 9.389671 +420 420 1716.987 +421 420 -38.75969 +422 420 -11.56069 +470 420 -1666.667 +421 421 38.75969 +422 422 11.56069 +423 423 606.9609 +425 423 -526.3158 +516 423 -80.64516 +424 424 293.4111 +466 424 -212.766 +516 424 -80.64516 +425 425 905.9952 +426 425 -73.52941 +427 425 -35.58719 +466 425 -151.5152 +426 426 536.205 +437 426 -454.5455 +519 426 -8.130081 +427 427 35.58719 +428 428 305.9395 +429 428 -25 +430 428 -37.03704 +431 428 -121.9512 +432 428 -121.9512 +429 429 25 +430 430 37.03704 +431 431 349.224 +432 432 349.224 +433 433 602.0284 +434 433 -32.05128 +510 433 -181.8182 +564 433 -21.18644 +434 434 32.05128 +435 435 83.68201 +510 435 -83.68201 +436 436 21.45923 +534 436 -21.45923 +437 437 548.2252 +438 437 -45.6621 +439 437 -6.222775 +441 437 -12.12121 +458 437 -29.67359 +438 438 45.6621 +439 439 22.0206 +440 439 -11.16819 +460 439 -4.62963 +440 440 11.16819 +441 441 45.41016 +442 441 -33.28895 +442 442 33.28895 +443 443 1625.394 +444 443 -73.52941 +470 443 -1111.111 +506 443 -155.0388 +516 443 -285.7143 +444 444 1323.529 +458 444 -1250 +445 445 298.3669 +446 445 -46.72897 +447 445 -67.1141 +446 446 46.72897 +447 447 1251.698 +449 447 -73.52941 +537 447 -487.8049 +448 448 178.454 +464 448 -19.72387 +449 449 2073.529 +450 449 -2000 +450 450 2196.806 +451 450 -21.36752 +451 451 21.36752 +452 452 237.3635 +453 452 -38.31417 +453 453 38.31417 +454 454 413.3297 +455 454 -35.97123 +455 455 35.97123 +456 456 105.5727 +457 456 -24.27184 +564 456 -81.30082 +457 457 24.27184 +458 458 1323.682 +459 458 -18.69159 +460 458 -25.31645 +459 459 18.69159 +460 460 51.68521 +461 460 -21.73913 +461 461 21.73913 +462 462 492.9882 +463 462 -48.54369 +463 463 48.54369 +464 464 86.29654 +465 464 -26.88172 +545 464 -20.96436 +465 465 92.6712 +968 465 -65.78947 +466 466 414.0324 +467 466 -49.75124 +467 467 49.75124 +468 468 10000 +544 468 -10000 +469 469 10000 +543 469 -10000 +470 470 2777.778 +471 471 71.30248 +472 471 -24.67105 +478 471 -11.01079 +481 471 -2.51756 +492 471 -2.24341 +707 471 -5.08647 +708 471 -25.7732 +472 472 24.67105 +473 473 10004.09 +474 473 -4.095004 +474 474 69.19151 +475 475 31.94728 +710 475 -12.28501 +724 475 -10.94691 +476 476 81.26222 +482 476 -32.36246 +477 477 228.7589 +478 477 -160.3849 +478 478 10195.93 +708 478 -20.67495 +709 478 -.6028454 +479 479 118.9729 +480 479 -9.02527 +714 479 -8.410429 +726 479 -5.037783 +480 480 509.0252 +481 481 26.89223 +484 481 -4.597701 +485 481 -4.425562 +491 481 -2.892765 +492 481 -3.441512 +733 481 -9.017133 +482 482 111.1026 +483 482 -78.74016 +483 483 10078.74 +484 484 22.31696 +709 484 -10.70664 +733 484 -7.012623 +485 485 12.73121 +491 485 -8.305648 +486 486 10101.3 +731 486 -51.54639 +487 487 5.743825 +491 487 -5.743825 +488 488 5.743825 +491 488 -5.743825 +489 489 6.968641 +492 489 -6.968641 +490 490 7.077141 +492 490 -7.077141 +491 491 41.56859 +492 491 -16.91332 +734 491 -1.969202 +492 492 10036.64 +707 492 -10000 +493 493 11.23596 +707 493 -11.23596 +494 494 121.9512 +505 494 -121.9512 +495 495 46.2963 +513 495 -46.2963 +496 496 303.0303 +521 496 -303.0303 +497 497 104.1667 +522 497 -104.1667 +498 498 20.08032 +523 498 -20.08032 +499 499 87.33171 +500 499 -39.37008 +819 499 -23.98081 +914 499 -23.98081 +500 500 39.37008 +501 501 227.9827 +502 501 -33.19632 +505 501 -18.58736 +507 501 -58.13953 +521 501 -32.67974 +531 501 -42.55319 +536 501 -42.82655 +502 502 120.8391 +607 502 -33.33334 +609 502 -11.02536 +613 502 -27.3224 +937 502 -15.96169 +503 503 121.5155 +507 503 -42.91846 +513 503 -55.55556 +553 503 -23.04147 +504 504 195.9784 +505 504 -71.42857 +505 505 292.5258 +521 505 -41.49377 +526 505 -17.60563 +578 505 -21.45923 +506 506 10155.04 +507 507 295.7078 +508 507 -12.03369 +509 507 -53.0504 +515 507 -20.4918 +521 507 -39.0625 +522 507 -40.16064 +547 507 -29.85075 +508 508 265.0443 +619 508 -175.4386 +639 508 -56.49718 +781 508 -21.07482 +509 509 912.198 +511 509 -12.0919 +515 509 -82.64463 +520 509 -238.0952 +534 509 -526.3158 +510 510 2646.453 +520 510 -2380.952 +511 511 29.78859 +914 511 -6.779661 +926 511 -10.91703 +512 512 99.0099 +520 512 -99.0099 +513 513 185.3068 +514 513 -9.380863 +540 513 -74.07407 +514 514 28.32451 +790 514 -11.28668 +795 514 -7.656968 +515 515 688.1429 +516 515 -416.6667 +517 515 -10.34126 +518 515 -18.18182 +536 515 -80.64516 +547 515 -59.1716 +516 516 863.6713 +517 517 27.53012 +993 517 -12.95337 +995 517 -4.235493 +518 518 18.18182 +519 519 93.04359 +957 519 -16.42036 +967 519 -68.49315 +520 520 2718.058 +521 521 520.433 +522 521 -104.1667 +522 522 260.4988 +523 522 -12.0048 +523 523 247.5269 +582 523 -196.0784 +770 523 -3.675795 +779 523 -11.60093 +802 523 -4.086637 +524 524 220.5431 +533 524 -158.7302 +985 524 -43.85965 +1001 524 -17.95332 +525 525 78.74016 +568 525 -78.74016 +526 526 71.7446 +527 526 -21.45923 +531 526 -32.67974 +527 527 149.7379 +921 527 -4.553734 +933 527 -5.621135 +1068 527 -35.27337 +1073 527 -30.67485 +1074 527 -25.83979 +1124 527 -26.31579 +528 528 336.2918 +529 528 -196.0784 +809 528 -56.17978 +819 528 -84.03362 +529 529 196.0784 +530 530 261.7177 +578 530 -9.708738 +583 530 -9.259259 +780 530 -5.390836 +797 530 -4.800768 +803 530 -232.5581 +531 531 204.1382 +532 531 -53.71729 +534 531 -75.18797 +532 532 527.8732 +554 532 -35.97123 +647 532 -113.6364 +652 532 -91.3242 +663 532 -45.87156 +668 532 -63.29114 +672 532 -12.97017 +673 532 -32.78688 +677 532 -28.24859 +908 532 -13.75516 +912 532 -18.48429 +918 532 -6.426735 +1047 532 -11.38952 +533 533 158.7302 +534 534 740.9194 +535 534 -21.45923 +536 534 -56.49718 +542 534 -40 +535 535 37.48487 +885 535 -16.02564 +536 536 698.459 +537 536 -416.6667 +538 536 -21.14165 +539 536 -18.18182 +542 536 -62.5 +537 537 904.4716 +538 538 105.9105 +809 538 -33.78378 +914 538 -7.002801 +948 538 -11.17318 +953 538 -10.48768 +964 538 -22.32143 +539 539 18.18182 +540 540 95.5333 +800 540 -21.45923 +541 541 13.9958 +543 541 -13.9958 +542 542 256.9678 +543 542 -42.91846 +545 542 -21.45923 +546 542 -90.0901 +543 543 10075.1 +544 543 -18.18182 +544 544 10018.18 +545 545 72.29217 +546 546 190.0901 +547 547 89.02235 +548 548 7.942812 +551 548 -7.942812 +549 549 8.532423 +563 549 -8.532423 +550 550 35.33569 +566 550 -35.33569 +551 551 65.62718 +552 551 -17.51313 +555 551 -15.47988 +566 551 -24.69136 +552 552 35.10244 +558 552 -3.012048 +560 552 -14.57726 +553 553 67.98992 +561 553 -26.73797 +710 553 -5.035247 +554 554 351.6514 +574 554 -93.45794 +909 554 -222.2222 +555 555 58.73003 +556 555 -16.26016 +557 555 -8.591065 +564 555 -10.9529 +566 555 -7.446017 +556 556 150.6284 +559 556 -64.93507 +579 556 -7.886436 +621 556 -45.04505 +903 556 -16.50165 +557 557 25.14736 +558 557 -16.55629 +558 558 103.3103 +979 558 -4.599816 +980 558 -12.40695 +986 558 -12.0919 +989 558 -14.4405 +991 558 -13.86001 +998 558 -8.061266 +1000 558 -18.28154 +559 559 73.91173 +579 559 -8.976661 +560 560 20.04772 +980 560 -5.470459 +561 561 200.6328 +562 561 -135.1351 +566 561 -38.75969 +562 562 143.5954 +563 562 -8.460238 +563 563 31.84037 +567 563 -3.386387 +564 564 113.4402 +565 565 5.017561 +567 565 -5.017561 +566 566 120.1828 +567 566 -4.231908 +567 567 12.63586 +568 568 164.0549 +773 568 -6.574622 +795 568 -78.74016 +569 569 20.08032 +785 569 -20.08032 +570 570 14.70588 +579 570 -14.70588 +571 571 11.42857 +776 571 -11.42857 +572 572 14.85884 +783 572 -14.85884 +573 573 18.51852 +796 573 -18.51852 +574 574 93.45794 +575 575 400 +581 575 -400 +576 576 23.74311 +582 576 -13.31558 +784 576 -10.42753 +577 577 897.2149 +786 577 -769.2308 +802 577 -8.93655 +1090 577 -119.0476 +578 578 45.98278 +780 578 -14.81481 +579 579 84.46193 +580 579 -2.915452 +782 579 -20.4918 +783 579 -7.85546 +895 579 -18.21494 +927 579 -3.4153 +580 580 16.74671 +587 580 -13.83126 +581 581 427.649 +776 581 -9.65717 +796 581 -4.690432 +797 581 -7.42942 +799 581 -5.87199 +582 582 209.394 +583 583 29.98347 +780 583 -13.03781 +803 583 -7.686395 +584 584 52.25406 +585 584 -45.6621 +1068 584 -6.591958 +585 585 121.8822 +607 585 -38.91051 +613 585 -34.0136 +659 585 -3.295979 +586 586 2.538071 +597 586 -2.538071 +587 587 66.82627 +598 587 -14.08451 +614 587 -38.91051 +588 588 10.00413 +595 588 -5.065856 +597 588 -4.938272 +589 589 6.075334 +591 589 -6.075334 +590 590 11.77992 +595 590 -4.405286 +616 590 -7.374631 +591 591 15.86007 +601 591 -9.784736 +592 592 7.564296 +611 592 -7.564296 +593 593 10.53741 +604 593 -10.53741 +594 594 63.69427 +600 594 -63.69427 +595 595 9.471143 +596 596 22.07258 +597 596 -4.837929 +603 596 -6.493506 +607 596 -10.74114 +597 597 12.31427 +598 598 14.08451 +599 599 6.038647 +1105 599 -6.038647 +600 600 346.1801 +601 600 -4.708098 +782 600 -277.7778 +601 601 23.91828 +602 601 -5.24659 +633 601 -4.178855 +602 602 10.19709 +640 602 -4.950495 +603 603 49.17807 +604 603 -4.837929 +605 603 -10.96491 +1092 603 -26.88172 +604 604 30.79998 +612 604 -7.843137 +615 604 -7.581501 +605 605 31.29012 +607 605 -20.3252 +606 606 3.743916 +610 606 -3.743916 +607 607 103.3102 +608 608 44.10909 +610 608 -41.15226 +1101 608 -2.95683 +609 609 20.31041 +1068 609 -9.285051 +610 610 44.89618 +611 611 19.35675 +615 611 -11.79245 +612 612 7.843137 +613 613 61.33601 +614 614 38.91051 +615 615 19.37395 +616 616 7.374631 +617 617 51.02041 +1123 617 -51.02041 +618 618 27.57334 +629 618 -10.09082 +639 618 -17.48252 +619 619 180.4386 +620 619 -5 +620 620 7.089864 +632 620 -2.089864 +621 621 137.2607 +624 621 -87.7193 +1061 621 -4.496403 +622 622 12.46672 +625 622 -10.67236 +641 622 -1.794366 +623 623 18.4185 +639 623 -10.37344 +642 623 -8.045053 +624 624 110.7608 +638 624 -23.04147 +625 625 17.12814 +635 625 -6.455778 +626 626 8.504332 +634 626 -5.627462 +635 626 -2.87687 +627 627 15.82279 +639 627 -15.82279 +628 628 43.10402 +631 628 -33.22259 +642 628 -9.881423 +629 629 5125.033 +637 629 -5000 +643 629 -114.9425 +630 630 27.86917 +631 630 -5.931198 +635 630 -4.347826 +796 630 -17.59015 +631 631 39.15379 +632 632 10.68662 +634 632 -6.830601 +641 632 -1.76616 +633 633 4.178855 +634 634 12.45806 +635 635 18.68799 +636 635 -5.007511 +636 636 8.039652 +1094 636 -3.03214 +637 637 5000 +638 638 23.04147 +639 639 100.1759 +640 640 4.950495 +641 641 3.560526 +642 642 17.92648 +643 643 114.9425 +644 644 87.81362 +652 644 -55.55556 +662 644 -32.25806 +645 645 32.23264 +650 645 -26.80965 +1060 645 -5.422993 +646 646 35.23614 +664 646 -30.4878 +1051 646 -4.748338 +647 647 132.4334 +1041 647 -18.79699 +648 648 87.5595 +678 648 -64.93507 +1042 648 -22.62444 +649 649 89.50495 +672 649 -40 +1055 649 -49.50495 +650 650 43.7588 +653 650 -16.94915 +651 651 35.83253 +670 651 -15.12859 +1045 651 -20.70393 +652 652 4092.357 +653 652 -4.440497 +663 652 -3333.333 +666 652 -97.08738 +668 652 -158.7302 +676 652 -18.55288 +1054 652 -333.3333 +653 653 31.9941 +1048 653 -10.60445 +654 654 184.7682 +677 654 -39.84064 +1057 654 -144.9275 +655 655 330.6451 +657 655 -80.64516 +665 655 -250 +656 656 4.040404 +678 656 -4.040404 +657 657 188.172 +1055 657 -107.5269 +658 658 298.2456 +671 658 -131.5789 +672 658 -166.6667 +659 659 65.79597 +677 659 -62.5 +660 660 36.60525 +1050 660 -17.03578 +1058 660 -19.56947 +661 661 500 +677 661 -500 +662 662 90.3976 +672 662 -58.13953 +663 663 3379.205 +664 664 60.0736 +1056 664 -29.5858 +665 665 340.0901 +682 665 -90.0901 +666 666 623.4032 +681 666 -526.3158 +667 667 5.494505 +1051 667 -5.494505 +668 668 222.0213 +669 669 400 +679 669 -66.66667 +1058 669 -333.3333 +670 670 69.18265 +1121 670 -54.05405 +671 671 184.7704 +679 671 -53.19149 +672 672 619.955 +673 672 -24.27184 +682 672 -117.647 +1049 672 -40 +1050 672 -16.80672 +1052 672 -18.34863 +1058 672 -19.84127 +1121 672 -105.2632 +673 673 57.05873 +674 674 340.5986 +675 674 -322.5806 +1046 674 -18.01802 +675 675 353.4448 +1050 675 -30.8642 +676 676 172.399 +1043 676 -153.8461 +677 677 945.5062 +678 677 -4.347826 +1041 677 -66.66667 +1059 677 -243.9024 +678 678 73.3233 +679 679 119.8582 +680 680 103.5542 +1047 680 -46.08295 +1057 680 -57.47126 +681 681 1637.427 +1053 681 -1111.111 +682 682 207.7372 +683 683 1.579031 +689 683 -1.579031 +684 684 1.923077 +689 684 -1.923077 +685 685 1.764913 +691 685 -1.764913 +686 686 328.5128 +687 686 -18.34863 +688 686 -24.44988 +687 687 18.34863 +688 688 71.2075 +689 688 -4.409431 +694 688 -8.312551 +699 688 -3.82995 +700 688 -7.374631 +702 688 -22.83105 +689 689 9.846897 +693 689 -1.935359 +690 690 10178.92 +691 690 -4.99002 +694 690 -161.2903 +697 690 -12.64223 +691 691 10040.88 +692 691 -10000 +695 691 -34.12969 +692 692 10000 +693 693 36.06505 +695 693 -34.12969 +694 694 187.9515 +695 694 -18.34863 +695 695 86.60801 +696 696 70.92198 +697 697 23.45304 +698 697 -10.81081 +698 698 10.81081 +699 699 27.1355 +700 699 -7.968128 +700 700 15.34276 +701 701 10000 +702 702 50.80308 +703 703 10000 +704 704 20.8778 +705 705 10000 +706 706 11.04972 +707 706 -11.04972 +707 707 10067.62 +726 707 -40.24145 +708 708 78.90948 +709 708 -32.46134 +709 709 44.41736 +710 710 10017.32 +711 711 10024.15 +724 711 -11.7096 +734 711 -12.43781 +712 712 10.78749 +714 712 -10.78749 +713 713 10.92896 +714 713 -10.92896 +714 714 102.2441 +715 714 -25.08151 +734 714 -23.9521 +715 715 33.13305 +716 716 370.5403 +717 716 -4.541326 +717 717 346.6114 +718 718 14.51379 +724 718 -14.51379 +719 719 14.51379 +724 719 -14.51379 +720 720 14.51379 +724 720 -14.51379 +721 721 14.51379 +724 721 -14.51379 +722 722 14.51379 +724 722 -14.51379 +723 723 14.51379 +724 723 -14.51379 +724 724 156.8111 +725 724 -2.262444 +734 724 -6.215041 +725 725 37.84963 +726 726 45.27923 +727 727 22.77734 +728 727 -1.437174 +731 727 -21.34016 +728 728 22.77734 +731 728 -21.34016 +729 729 22.54354 +730 729 -.5025378 +731 729 -22.041 +730 730 22.63133 +731 730 -22.12879 +731 731 213.6977 +732 732 2.369107 +733 733 16.02976 +734 734 44.57415 +735 735 3.749531 +743 735 -3.749531 +736 736 5.060729 +742 736 -5.060729 +737 737 10.41667 +743 737 -10.41667 +738 738 25.25253 +742 738 -25.25253 +739 739 6.666667 +744 739 -6.666667 +740 740 10119.13 +741 740 -29.06977 +742 740 -9.416196 +744 740 -80.64516 +741 741 66.38967 +749 741 -30.12048 +759 741 -7.199424 +742 742 97.15186 +743 742 -36.00912 +743 743 65.14973 +746 743 -4.042037 +744 744 120.6452 +756 744 -33.33334 +745 745 10004.53 +746 745 -4.531038 +746 746 8.573075 +747 747 83.11967 +748 747 -15.5521 +758 747 -67.56757 +748 748 35.75412 +757 748 -20.20202 +749 749 109.4856 +753 749 -79.36508 +750 750 36.15381 +751 750 -9.199632 +759 750 -26.95418 +751 751 9.199632 +752 752 14.23959 +757 752 -10.20408 +762 752 -4.035512 +753 753 79.36508 +754 754 101.7248 +757 754 -42.55319 +765 754 -59.1716 +755 755 34.41619 +764 755 -25 +756 756 61.14975 +757 756 -27.81641 +757 757 100.7757 +758 758 67.56757 +759 759 34.1536 +760 760 6.238303 +761 760 -6.238303 +761 761 27.13861 +766 761 -4.730369 +767 761 -8 +762 762 8.519817 +763 762 -4.484305 +763 763 4.484305 +764 764 25 +765 765 59.1716 +766 766 7.055951 +768 766 -2.325581 +767 767 8 +768 768 8.877011 +769 768 -3.030303 +772 768 -3.521127 +769 769 3.030303 +770 770 10003.68 +823 770 -10000 +771 771 4.694836 +772 771 -4.694836 +772 772 8.215962 +773 773 41.97304 +774 773 -9.191176 +795 773 -6.067961 +801 773 -5.646527 +802 773 -14.49275 +774 774 9.191176 +775 775 87.97093 +776 775 -79.36508 +781 775 -8.605852 +776 776 145.414 +781 776 -7.067138 +796 776 -2.511932 +797 776 -3.435246 +1127 776 -31.94888 +777 777 24.34617 +793 777 -3.907777 +795 777 -5.797101 +800 777 -14.64129 +778 778 27.51031 +779 778 -27.51031 +779 779 52.63487 +780 779 -3.543586 +787 779 -9.980041 +780 780 53.90431 +797 780 -2.540005 +949 780 -14.57726 +781 781 36.74781 +782 782 311.0084 +783 782 -12.73885 +783 783 35.45316 +784 784 25.00479 +802 784 -14.57726 +785 785 85.04397 +786 785 -26.66667 +788 785 -16.80672 +796 785 -5.941771 +797 785 -11.00715 +802 785 -4.541326 +786 786 1295.897 +1128 786 -500 +787 787 18.79838 +949 787 -8.818342 +788 788 201.3442 +789 788 -2.932551 +796 788 -9.191176 +1132 788 -172.4138 +789 789 6.317792 +1094 789 -3.38524 +790 790 67.99094 +791 790 -32.89474 +795 790 -23.80952 +791 791 32.89474 +792 792 18.23483 +799 792 -10.1833 +1110 792 -8.05153 +793 793 10007.82 +794 793 -3.907777 +825 793 -10000 +794 794 24.34617 +795 794 -5.797101 +800 794 -14.64129 +795 795 127.8688 +796 796 80.27805 +799 796 -21.83406 +797 797 107.3739 +798 797 -4.818348 +803 797 -4.5106 +805 797 -18.83239 +1119 797 -50 +798 798 10.82075 +1114 798 -6.002401 +799 799 37.88935 +800 800 50.74181 +801 801 5.646527 +802 802 10046.63 +1137 802 -10000 +803 803 290.2097 +804 803 -45.45454 +804 804 45.45454 +805 805 136.4794 +1138 805 -117.647 +806 806 4.975124 +821 806 -4.975124 +807 807 4.065041 +887 807 -4.065041 +808 808 13.33333 +897 808 -13.33333 +809 809 202.3231 +920 809 -112.3596 +810 810 39.62731 +811 810 -26.31579 +918 810 -8.70322 +811 811 26.31579 +812 812 500 +909 812 -500 +813 813 805.5555 +885 813 -11.13586 +940 813 -25.18892 +1122 813 -769.2308 +814 814 149.0395 +815 814 -112.3596 +909 814 -23.14815 +918 814 -13.5318 +815 815 112.3596 +816 816 67.98565 +817 816 -47.61905 +927 816 -20.3666 +817 817 139.7763 +818 817 -52.63158 +907 817 -39.52569 +818 818 52.63158 +819 819 340.5726 +820 819 -232.5581 +820 820 232.5581 +821 821 148.7724 +888 821 -22.72727 +897 821 -114.9425 +921 821 -6.127451 +822 822 133.5555 +880 822 -77.51938 +885 822 -25.64103 +935 822 -30.39514 +823 823 10000 +824 824 10000 +825 824 -10000 +825 825 20000 +826 826 10000 +827 827 10000 +828 828 12.59221 +837 828 -3.399953 +838 828 -4.568296 +874 828 -4.623957 +829 829 38.16908 +833 829 -9.785693 +839 829 -9.75447 +830 830 35.75804 +833 830 -12.15067 +840 830 -9.559799 +831 831 21.56662 +841 831 -10.30197 +869 831 -11.26464 +832 832 24.83065 +842 832 -6.55914 +869 832 -18.27151 +833 833 25.06136 +843 833 -3.125 +834 834 65.83865 +844 834 -9.816236 +874 834 -56.02241 +835 835 97.96513 +845 835 -1.626016 +862 835 -96.33911 +836 836 13.04241 +846 836 -6.806048 +869 836 -6.236358 +837 837 3.399953 +838 838 11.84409 +859 838 -6.666667 +860 838 -.6091247 +839 839 9.75447 +840 840 9.559799 +841 841 10.30197 +842 842 6.55914 +843 843 3.125 +844 844 9.816236 +845 845 1.626016 +846 846 6.806048 +847 847 6.595976 +848 847 -6.595976 +848 848 14.46535 +873 848 -7.869369 +849 849 7.853637 +854 849 -3.149802 +863 849 -2.352 +864 849 -2.351835 +850 850 9.019895 +855 850 -6.506006 +873 850 -2.513889 +851 851 1.965569 +856 851 -.805153 +858 851 -1.160416 +852 852 2.941501 +857 852 -1.692047 +869 852 -1.249453 +853 853 5.118024 +876 853 -5.118024 +854 854 3.149802 +855 855 6.506006 +856 856 .805153 +857 857 1.692047 +858 858 3.252203 +873 858 -2.091788 +859 859 6.666667 +860 860 1.422133 +861 860 -.8130081 +861 861 .8130081 +862 862 101.7126 +863 862 -5.373455 +863 863 10.56927 +869 863 -2.843818 +864 864 26.18864 +865 864 -20.9205 +869 864 -2.916302 +865 865 20.9205 +866 866 71.85119 +869 866 -67.1141 +877 866 -4.737092 +867 867 5.135919 +878 867 -5.135919 +868 868 79.89591 +869 868 -68.02721 +871 868 -7.077141 +877 868 -4.791567 +869 869 245.0375 +870 869 -67.1141 +870 870 78.94842 +871 870 -7.097232 +877 870 -4.737092 +871 871 14.17437 +872 872 3.439506 +882 872 -3.439506 +873 873 19.16737 +882 873 -6.692322 +874 874 65.74321 +878 874 -5.09684 +875 875 268.8182 +876 875 -227.2727 +882 875 -41.54549 +876 876 232.3908 +877 877 144.0989 +878 877 -54.4514 +883 877 -15.50147 +878 878 64.68417 +879 879 13.23977 +882 879 -13.23977 +880 880 77.51938 +881 881 15.55936 +882 881 -15.55936 +882 882 209.5087 +883 883 15.50147 +884 884 208.6253 +937 884 -114.2857 +1123 884 -94.33962 +885 885 67.07021 +914 885 -9.425071 +918 885 -4.842615 +886 886 34.37894 +895 886 -13.2626 +927 886 -6.146282 +937 886 -14.97006 +887 887 30.75514 +896 887 -15.94896 +897 887 -10.74114 +888 888 34.51973 +924 888 -11.79245 +889 889 526.3158 +917 889 -526.3158 +890 890 90.17906 +891 890 -55.55556 +908 890 -11.79245 +917 890 -22.83105 +891 891 55.55556 +892 892 185.8299 +893 892 -78.74016 +899 892 -68.02721 +927 892 -39.0625 +893 893 78.74016 +894 894 10037.21 +921 894 -24.93765 +936 894 -12.26994 +1126 894 -10000 +895 895 486.023 +1125 895 -454.5455 +896 896 48.8437 +897 896 -32.89474 +897 897 242.6641 +913 897 -54.05405 +917 897 -3.943218 +944 897 -12.7551 +898 898 119.8733 +914 898 -21.83406 +926 898 -98.03922 +899 899 208.295 +900 899 -36.10108 +908 899 -104.1667 +900 900 36.10108 +901 901 10010.86 +913 901 -10.85776 +1129 901 -10000 +902 902 400 +906 902 -400 +903 903 27.64992 +987 903 -11.14827 +904 904 479.299 +905 904 -256.4103 +939 904 -192.3077 +944 904 -30.58104 +905 905 256.4103 +906 906 613.7151 +922 906 -17.63668 +939 906 -196.0784 +907 907 58.35809 +923 907 -18.83239 +908 908 200.9013 +912 908 -43.47826 +917 908 -7.668712 +927 908 -20.04008 +909 909 941.4488 +910 909 -196.0784 +910 910 196.0784 +911 911 131.5789 +912 911 -131.5789 +912 912 193.5415 +913 913 64.91182 +914 914 181.033 +935 914 -29.23977 +938 914 -78.74016 +963 914 -4.030633 +915 915 103.0928 +926 915 -103.0928 +916 916 81.96722 +991 916 -81.96722 +917 917 754.9636 +918 917 -121.2121 +930 917 -72.9927 +918 918 232.832 +919 918 -52.63158 +1130 918 -22.47191 +919 919 69.87296 +940 919 -17.24138 +920 920 112.3596 +921 921 123.4495 +923 921 -22.98851 +924 921 -29.06977 +927 921 -5.208333 +933 921 -23.98081 +937 921 -6.583279 +922 922 10033.82 +930 922 -16.18123 +1133 922 -10000 +923 923 41.8209 +924 924 79.77273 +925 924 -38.91051 +925 925 38.91051 +926 926 212.049 +927 927 138.2073 +928 927 -37.17472 +937 927 -6.793478 +928 928 1045.494 +929 928 -999.9999 +937 928 -8.319468 +929 929 999.9999 +930 930 185.3278 +931 930 -96.15385 +931 931 96.15385 +932 932 78.74016 +949 932 -78.74016 +933 933 187.4276 +934 933 -131.5789 +942 933 -26.24672 +934 934 131.5789 +935 935 94.47811 +947 935 -34.8432 +936 936 10045.17 +937 936 -32.89474 +1134 936 -10000 +937 937 199.8084 +938 938 78.74016 +939 939 388.3861 +940 940 141.4402 +954 940 -99.0099 +941 941 400 +954 941 -400 +942 942 26.24672 +943 943 80.64516 +944 943 -80.64516 +944 944 123.9813 +945 945 3.323363 +961 945 -3.323363 +946 946 10.52632 +965 946 -10.52632 +947 947 34.8432 +948 948 110.8715 +952 948 -12.90323 +955 948 -10.88454 +957 948 -18.50139 +958 948 -35.08772 +964 948 -22.32143 +949 949 102.1358 +950 950 59.13499 +951 950 -4.399472 +955 950 -35.77818 +956 950 -18.95735 +951 951 26.76585 +965 951 -4.962779 +969 951 -10.17294 +975 951 -7.230658 +952 952 78.2627 +960 952 -65.35947 +953 953 32.95959 +955 953 -22.47191 +954 954 528.2497 +959 954 -29.23977 +955 955 69.13462 +956 956 18.95735 +957 957 153.211 +958 957 -17.85714 +967 957 -21.69197 +973 957 -78.74016 +958 958 52.94486 +959 959 29.23977 +960 960 465.3595 +973 960 -400 +961 961 17.72833 +962 961 -6.060606 +963 961 -2.007226 +968 961 -6.337135 +962 962 42.96098 +966 962 -36.90037 +963 963 10.88517 +972 963 -4.84731 +964 964 44.64286 +965 965 31.12432 +968 965 -5.945303 +969 965 -9.689922 +966 966 135.9103 +971 966 -99.0099 +967 967 90.18512 +968 968 78.07191 +969 969 26.28135 +970 969 -6.418485 +970 970 6.418485 +971 971 110.4385 +972 971 -11.42857 +972 972 16.27588 +973 973 478.7402 +974 974 999.9999 +983 974 -999.9999 +975 975 7.230658 +976 976 131.2277 +977 976 -72.9927 +989 976 -11.28668 +1001 976 -46.94836 +977 977 72.9927 +978 978 11.99041 +989 978 -11.99041 +979 979 212.9332 +983 979 -208.3333 +980 980 17.87741 +981 981 214.386 +982 981 -192.3077 +984 981 -12.04819 +998 981 -10.03009 +982 982 192.3077 +983 983 1216.031 +986 983 -7.69823 +984 984 12.04819 +985 985 73.44217 +987 985 -24.27184 +989 985 -5.310675 +986 986 19.79013 +987 987 74.94581 +999 987 -39.52569 +988 988 25.27583 +993 988 -12.42236 +995 988 -12.85347 +989 989 48.93494 +990 989 -5.906674 +990 990 5.906674 +991 991 98.75978 +992 991 -2.932551 +992 992 2.932551 +993 993 29.40311 +994 993 -4.027386 +994 994 4.027386 +995 995 28.39029 +997 995 -7.267442 +998 995 -4.033885 +996 996 78.74016 +997 996 -78.74016 +997 997 95.07378 +998 997 -9.066183 +998 998 31.19142 +999 999 39.52569 +1000 1000 18.28154 +1001 1001 64.90168 +1002 1002 8.319468 +1027 1002 -8.319468 +1003 1003 4.084967 +1035 1003 -4.084967 +1004 1004 3.15856 +1032 1004 -3.15856 +1005 1005 10.41667 +1039 1005 -10.41667 +1006 1006 125.0263 +1007 1006 -4.149378 +1013 1006 -19.26782 +1019 1006 -10.48218 +1021 1006 -55.24862 +1030 1006 -14.32665 +1035 1006 -21.55167 +1007 1007 4.149378 +1008 1008 24.42002 +1023 1008 -24.42002 +1009 1009 10093.68 +1010 1009 -1.567398 +1018 1009 -21.18644 +1019 1009 -6.297229 +1021 1009 -4.244482 +1022 1009 -57.14286 +1040 1009 -3.239391 +1010 1010 1.567398 +1011 1011 41.15226 +1040 1011 -41.15226 +1012 1012 5.87199 +1040 1012 -5.87199 +1013 1013 748.2476 +1023 1013 -15.07159 +1029 1013 -454.4287 +1030 1013 -32.57329 +1037 1013 -118.3432 +1038 1013 -102.5641 +1039 1013 -5.9988 +1014 1014 75.75758 +1039 1014 -75.75758 +1015 1015 34.60208 +1029 1015 -34.60208 +1016 1016 4.102139 +1024 1016 -2.283626 +1034 1016 -1.818512 +1017 1017 73.80074 +1029 1017 -73.80074 +1018 1018 21.18644 +1019 1019 60.25767 +1033 1019 -43.47826 +1020 1020 34.96503 +1021 1020 -34.96503 +1021 1021 94.45814 +1022 1022 57.14286 +1023 1023 66.38087 +1024 1023 -6.973501 +1025 1023 -7.352941 +1027 1023 -12.56281 +1024 1024 11.00568 +1034 1024 -1.748557 +1025 1025 11.7895 +1028 1025 -4.436557 +1026 1026 16.88283 +1027 1026 -9.615384 +1027 1027 38.77231 +1028 1027 -3.372681 +1035 1027 -4.901961 +1028 1028 7.809239 +1029 1029 10562.83 +1030 1030 56.26324 +1031 1030 -9.363297 +1031 1031 9.363297 +1032 1032 19.43661 +1034 1032 -5.455537 +1036 1032 -10.82251 +1033 1033 43.47826 +1034 1034 9.022608 +1035 1035 34.63528 +1036 1035 -4.096682 +1036 1036 14.91919 +1037 1037 118.3432 +1038 1038 102.5641 +1039 1039 92.17304 +1040 1040 50.26365 +1041 1041 85.46367 +1042 1042 22.62444 +1043 1043 179.8878 +1050 1043 -26.04167 +1044 1044 51.6919 +1050 1044 -33.8983 +1053 1044 -17.79359 +1045 1045 69.24763 +1047 1045 -48.54369 +1046 1046 71.20951 +1058 1046 -53.19149 +1047 1047 106.0162 +1048 1048 10.60445 +1049 1049 66.17801 +1054 1049 -26.17801 +1050 1050 5329.354 +1051 1050 -4.708098 +1052 1050 -200 +1051 1051 18.55844 +1056 1051 -3.607503 +1052 1052 218.3486 +1053 1053 1128.905 +1054 1054 359.5114 +1055 1055 267.5334 +1056 1055 -4.118616 +1058 1055 -106.383 +1056 1056 37.31192 +1057 1057 202.3988 +1058 1058 532.3185 +1059 1059 295.9858 +1061 1059 -52.08333 +1060 1060 5.422993 +1061 1061 56.57973 +1062 1062 9.573448 +1072 1062 -6.301197 +1085 1062 -3.272251 +1063 1063 14.97518 +1074 1063 -10.35197 +1064 1064 15.77324 +1075 1064 -11.49425 +1087 1064 -4.27899 +1065 1065 56.4111 +1077 1065 -7.256895 +1080 1065 -32.67974 +1066 1066 14.0056 +1068 1066 -14.0056 +1067 1067 2.654632 +1071 1067 -2.654632 +1068 1068 96.37038 +1069 1068 -21.64502 +1092 1068 -9.569378 +1069 1069 49.50018 +1086 1069 -27.85515 +1070 1070 27.63299 +1077 1070 -21.09705 +1113 1070 -6.535948 +1071 1071 6.953859 +1088 1071 -4.299226 +1072 1072 10.51705 +1075 1072 -4.215851 +1073 1073 194.6093 +1074 1073 -163.9344 +1074 1074 315.5717 +1075 1074 -5.040322 +1081 1074 -85.47009 +1086 1074 -21.73913 +1075 1075 25.04781 +1082 1075 -4.297379 +1076 1076 18.69159 +1077 1076 -18.69159 +1077 1077 82.05888 +1078 1077 -9.416196 +1080 1077 -9.328359 +1088 1077 -11.54734 +1095 1077 -4.721436 +1078 1078 30.82096 +1084 1078 -17.66784 +1085 1078 -3.736921 +1079 1079 3.007519 +1117 1079 -3.007519 +1080 1080 42.00809 +1081 1081 85.47009 +1082 1082 4.297379 +1083 1083 500 +1087 1083 -500 +1084 1084 17.66784 +1085 1085 7.009172 +1086 1086 49.59428 +1087 1087 504.279 +1088 1088 15.84657 +1089 1089 12.21374 +1106 1089 -9.225092 +1112 1089 -2.988643 +1090 1090 123.7557 +1091 1090 -4.708098 +1091 1091 14.65977 +1093 1091 -5.681818 +1108 1091 -4.269855 +1092 1092 36.4511 +1093 1093 14.98414 +1098 1093 -9.302325 +1094 1094 6.417381 +1095 1095 136.3807 +1096 1095 -4.440497 +1100 1095 -7.342144 +1105 1095 -16.23377 +1113 1095 -87.7193 +1096 1096 16.50748 +1103 1096 -7.581501 +1107 1096 -2.338634 +1112 1096 -2.146844 +1097 1097 99.0099 +1100 1097 -99.0099 +1098 1098 15.8597 +1099 1098 -6.557377 +1099 1099 6.557377 +1100 1100 110.9308 +1101 1100 -4.578754 +1101 1101 13.03916 +1120 1101 -5.503577 +1102 1102 4.524887 +1112 1102 -4.524887 +1103 1103 7.581501 +1104 1104 46.18962 +1118 1104 -23.14815 +1119 1104 -23.04147 +1105 1105 22.27241 +1106 1106 15.2311 +1109 1106 -6.006006 +1107 1107 2.338634 +1108 1108 4.269855 +1109 1109 6.006006 +1110 1110 10.67965 +1114 1110 -2.628121 +1111 1111 7.380074 +1119 1111 -7.380074 +1112 1112 9.660375 +1113 1113 94.25525 +1114 1114 15.25571 +1115 1114 -4.384042 +1116 1114 -2.241148 +1115 1115 4.384042 +1116 1116 2.241148 +1117 1117 7.461862 +1120 1117 -4.454343 +1118 1118 23.14815 +1119 1119 80.42155 +1120 1120 9.95792 +1121 1121 159.3172 +1122 1122 791.0173 +1136 1122 -21.78649 +1123 1123 145.36 +1124 1124 37.05693 +1135 1124 -10.74114 +1125 1125 454.5455 +1126 1126 10000 +1127 1127 31.94888 +1128 1128 500 +1129 1129 10015.82 +1135 1129 -15.82279 +1130 1130 22.47191 +1131 1131 24.39024 +1136 1131 -24.39024 +1132 1132 172.4138 +1133 1133 10000 +1134 1134 10000 +1135 1135 26.56392 +1136 1136 46.17674 +1137 1137 10000 +1138 1138 117.647 diff --git a/benchmarks/opencl/spmv/input/1138_bus.mtx.bin b/benchmarks/opencl/spmv/input/1138_bus.mtx.bin new file mode 100755 index 00000000..e939aba5 Binary files /dev/null and b/benchmarks/opencl/spmv/input/1138_bus.mtx.bin differ diff --git a/benchmarks/opencl/spmv/input/DESCRIPTION b/benchmarks/opencl/spmv/input/DESCRIPTION new file mode 100755 index 00000000..f67c6cb4 --- /dev/null +++ b/benchmarks/opencl/spmv/input/DESCRIPTION @@ -0,0 +1,3 @@ +Inputs: 1138_bus.mtx vector.bin + +This is a very small phantom data set. diff --git a/benchmarks/opencl/spmv/input/vector.bin b/benchmarks/opencl/spmv/input/vector.bin new file mode 100644 index 00000000..94aa7258 Binary files /dev/null and b/benchmarks/opencl/spmv/input/vector.bin differ diff --git a/benchmarks/opencl/spmv/kernel.cl b/benchmarks/opencl/spmv/kernel.cl new file mode 100644 index 00000000..3f44205e --- /dev/null +++ b/benchmarks/opencl/spmv/kernel.cl @@ -0,0 +1,36 @@ +/*************************************************************************** + *cr + *cr (C) Copyright 2010 The Board of Trustees of the + *cr University of Illinois + *cr All Rights Reserved + *cr + ***************************************************************************/ + +__kernel void spmv_jds_naive(__global float *dst_vector, __global float *d_data, + __global int *d_index, __global int *d_perm, + __global float *x_vec, const int dim, + __constant int *jds_ptr_int, + __constant int *sh_zcnt_int) +{ + int ix = get_global_id(0); + + if (ix < dim) { + float sum = 0.0f; + // 32 is warp size + int bound=sh_zcnt_int[ix/32]; + + for(int k=0;k +#include +#include +#include +#include + +#include "convert_dataset.h" +#include "file.h" +#include "gpu_info.h" +#include "ocl.h" + + +static int generate_vector(float *x_vector, int dim) { + srand(54321); + int i; + for (i = 0; i < dim; i++) { + x_vector[i] = (rand() / (float)RAND_MAX); + } + return 0; +} + +int main(int argc, char **argv) { + struct pb_TimerSet timers; + struct pb_Parameters *parameters; + + printf("CUDA accelerated sparse matrix vector multiplication****\n"); + printf("Original version by Li-Wen Chang and " + "Shengzhao Wu\n"); + printf("This version maintained by Chris Rodrigues ***********\n"); + // parameters = pb_ReadParameters(&argc, argv); + parameters->inpFiles = (char **)malloc(sizeof(char *) * 3); + parameters->inpFiles[0] = (char *)malloc(100); + parameters->inpFiles[1] = (char *)malloc(100); + parameters->inpFiles[2] = NULL; + strncpy(parameters->inpFiles[0], "1138_bus.mtx", 100); + strncpy(parameters->inpFiles[1], "vector.bin", 100); + + printf("OK\n"); + + if ((parameters->inpFiles[0] == NULL) || (parameters->inpFiles[1] == NULL)) { + fprintf(stderr, "Expecting one input filename\n"); + exit(-1); + } + + pb_InitializeTimerSet(&timers); + + pb_SwitchToTimer(&timers, pb_TimerID_COMPUTE); + + // parameters declaration + cl_int clStatus; + pb_Context *pb_context; + pb_context = pb_InitOpenCLContext(parameters); + if (pb_context == NULL) { + fprintf(stderr, "Error: No OpenCL platform/device can be found."); + return -1; + } + + printf("OK\n"); + + cl_device_id clDevice = (cl_device_id)pb_context->clDeviceId; + cl_platform_id clPlatform = (cl_platform_id)pb_context->clPlatformId; + cl_context clContext = (cl_context)pb_context->clContext; + cl_command_queue clCommandQueue = clCreateCommandQueue( + clContext, clDevice, CL_QUEUE_PROFILING_ENABLE, &clStatus); + CHECK_ERROR("clCreateCommandQueue") + + printf("OK\n"); + + pb_SetOpenCL(&clContext, &clCommandQueue); + + //const char *clSource[] = {readFile("src/opencl_base/kernel.cl")}; + // cl_program clProgram = + // clCreateProgramWithSource(clContext,1,clSource,NULL,&clStatus); + cl_program clProgram = clCreateProgramWithBuiltInKernels( + clContext, 1, &clDevice, "spmv_jds_naive", &clStatus); + CHECK_ERROR("clCreateProgramWithSource") + + printf("OK\n"); + + char clOptions[50]; + sprintf(clOptions, ""); + clStatus = clBuildProgram(clProgram, 1, &clDevice, clOptions, NULL, NULL); + CHECK_ERROR("clBuildProgram") + + cl_kernel clKernel = clCreateKernel(clProgram, "spmv_jds_naive", &clStatus); + CHECK_ERROR("clCreateKernel") + + printf("OK\n"); + + int len; + int depth; + int dim; + int pad = 32; + int nzcnt_len; + + // host memory allocation + // matrix + float *h_data; + int *h_indices; + int *h_ptr; + int *h_perm; + int *h_nzcnt; + // vector + float *h_Ax_vector; + float *h_x_vector; + + // device memory allocation + // matrix + cl_mem d_data; + cl_mem d_indices; + cl_mem d_ptr; + cl_mem d_perm; + cl_mem d_nzcnt; + + // vector + cl_mem d_Ax_vector; + cl_mem d_x_vector; + + cl_mem jds_ptr_int; + cl_mem sh_zcnt_int; + + // load matrix from files + pb_SwitchToTimer(&timers, pb_TimerID_IO); + // inputData(parameters->inpFiles[0], &len, &depth, &dim,&nzcnt_len,&pad, + // &h_data, &h_indices, &h_ptr, + // &h_perm, &h_nzcnt); + int col_count; + printf("OK--\n"); + coo_to_jds(parameters->inpFiles[0], // bcsstk32.mtx, fidapm05.mtx, jgl009.mtx + 1, // row padding + pad, // warp size + 1, // pack size + 1, // is mirrored? + 0, // binary matrix + 1, // debug level [0:2] + &h_data, &h_ptr, &h_nzcnt, &h_indices, &h_perm, &col_count, &dim, + &len, &nzcnt_len, &depth); + + printf("OK++\n"); + + // pb_SwitchToTimer(&timers, pb_TimerID_COMPUTE); + h_Ax_vector = (float *)malloc(sizeof(float) * dim); + h_x_vector = (float *)malloc(sizeof(float) * dim); + + input_vec(parameters->inpFiles[1], h_x_vector, dim); + + pb_SwitchToTimer(&timers, pb_TimerID_COMPUTE); + OpenCLDeviceProp clDeviceProp; + // clStatus = + //clGetDeviceInfo(clDevice,CL_DEVICE_COMPUTE_CAPABILITY_MAJOR_NV,sizeof(cl_uint),&(clDeviceProp.major),NULL); + // CHECK_ERROR("clGetDeviceInfo") + // clStatus = + //clGetDeviceInfo(clDevice,CL_DEVICE_COMPUTE_CAPABILITY_MINOR_NV,sizeof(cl_uint),&(clDeviceProp.minor),NULL); + // CHECK_ERROR("clGetDeviceInfo") + clStatus = + clGetDeviceInfo(clDevice, CL_DEVICE_MAX_COMPUTE_UNITS, sizeof(cl_uint), + &(clDeviceProp.multiProcessorCount), NULL); + CHECK_ERROR("clGetDeviceInfo") + + pb_SwitchToTimer(&timers, pb_TimerID_COPY); + // memory allocation + d_data = clCreateBuffer(clContext, CL_MEM_READ_ONLY, len * sizeof(float), + NULL, &clStatus); + CHECK_ERROR("clCreateBuffer") + d_indices = clCreateBuffer(clContext, CL_MEM_READ_ONLY, len * sizeof(int), + NULL, &clStatus); + CHECK_ERROR("clCreateBuffer") + d_perm = clCreateBuffer(clContext, CL_MEM_READ_ONLY, dim * sizeof(int), NULL, + &clStatus); + CHECK_ERROR("clCreateBuffer") + d_x_vector = clCreateBuffer(clContext, CL_MEM_READ_ONLY, dim * sizeof(float), + NULL, &clStatus); + CHECK_ERROR("clCreateBuffer") + d_Ax_vector = clCreateBuffer(clContext, CL_MEM_WRITE_ONLY, + dim * sizeof(float), NULL, &clStatus); + CHECK_ERROR("clCreateBuffer") + + jds_ptr_int = clCreateBuffer(clContext, CL_MEM_READ_ONLY, 5000 * sizeof(int), + NULL, &clStatus); + CHECK_ERROR("clCreateBuffer") + sh_zcnt_int = clCreateBuffer(clContext, CL_MEM_READ_ONLY, 5000 * sizeof(int), + NULL, &clStatus); + CHECK_ERROR("clCreateBuffer") + + clMemSet(clCommandQueue, d_Ax_vector, 0, dim * sizeof(float)); + + // memory copy + clStatus = clEnqueueWriteBuffer(clCommandQueue, d_data, CL_FALSE, 0, + len * sizeof(float), h_data, 0, NULL, NULL); + CHECK_ERROR("clEnqueueWriteBuffer") + clStatus = clEnqueueWriteBuffer(clCommandQueue, d_indices, CL_FALSE, 0, + len * sizeof(int), h_indices, 0, NULL, NULL); + CHECK_ERROR("clEnqueueWriteBuffer") + clStatus = clEnqueueWriteBuffer(clCommandQueue, d_perm, CL_FALSE, 0, + dim * sizeof(int), h_perm, 0, NULL, NULL); + CHECK_ERROR("clEnqueueWriteBuffer") + clStatus = clEnqueueWriteBuffer(clCommandQueue, d_x_vector, CL_FALSE, 0, + dim * sizeof(int), h_x_vector, 0, NULL, NULL); + CHECK_ERROR("clEnqueueWriteBuffer") + + clStatus = clEnqueueWriteBuffer(clCommandQueue, jds_ptr_int, CL_FALSE, 0, + depth * sizeof(int), h_ptr, 0, NULL, NULL); + CHECK_ERROR("clEnqueueWriteBuffer") + clStatus = + clEnqueueWriteBuffer(clCommandQueue, sh_zcnt_int, CL_TRUE, 0, + nzcnt_len * sizeof(int), h_nzcnt, 0, NULL, NULL); + CHECK_ERROR("clEnqueueWriteBuffer") + + pb_SwitchToTimer(&timers, pb_TimerID_COMPUTE); + + size_t grid; + size_t block; + + compute_active_thread(&block, &grid, nzcnt_len, pad, clDeviceProp.major, + clDeviceProp.minor, clDeviceProp.multiProcessorCount); + // printf("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!grid is %d and block is + // %d=\n",grid,block); + // printf("!!! dim is %d\n",dim); + + clStatus = clSetKernelArg(clKernel, 0, sizeof(cl_mem), &d_Ax_vector); + CHECK_ERROR("clSetKernelArg") + clStatus = clSetKernelArg(clKernel, 1, sizeof(cl_mem), &d_data); + CHECK_ERROR("clSetKernelArg") + clStatus = clSetKernelArg(clKernel, 2, sizeof(cl_mem), &d_indices); + CHECK_ERROR("clSetKernelArg") + clStatus = clSetKernelArg(clKernel, 3, sizeof(cl_mem), &d_perm); + CHECK_ERROR("clSetKernelArg") + clStatus = clSetKernelArg(clKernel, 4, sizeof(cl_mem), &d_x_vector); + CHECK_ERROR("clSetKernelArg") + clStatus = clSetKernelArg(clKernel, 5, sizeof(int), &dim); + CHECK_ERROR("clSetKernelArg") + + clStatus = clSetKernelArg(clKernel, 6, sizeof(cl_mem), &jds_ptr_int); + CHECK_ERROR("clSetKernelArg") + clStatus = clSetKernelArg(clKernel, 7, sizeof(cl_mem), &sh_zcnt_int); + CHECK_ERROR("clSetKernelArg") + + // main execution + pb_SwitchToTimer(&timers, pb_TimerID_KERNEL); + + int i; + for (i = 0; i < 50; i++) { + clStatus = clEnqueueNDRangeKernel(clCommandQueue, clKernel, 1, NULL, &grid, + &block, 0, NULL, NULL); + CHECK_ERROR("clEnqueueNDRangeKernel") + } + + clStatus = clFinish(clCommandQueue); + CHECK_ERROR("clFinish") + + pb_SwitchToTimer(&timers, pb_TimerID_COPY); + // HtoD memory copy + clStatus = + clEnqueueReadBuffer(clCommandQueue, d_Ax_vector, CL_TRUE, 0, + dim * sizeof(float), h_Ax_vector, 0, NULL, NULL); + CHECK_ERROR("clEnqueueReadBuffer") + + clStatus = clReleaseKernel(clKernel); + clStatus = clReleaseProgram(clProgram); + + clStatus = clReleaseMemObject(d_data); + clStatus = clReleaseMemObject(d_indices); + clStatus = clReleaseMemObject(d_perm); + clStatus = clReleaseMemObject(d_x_vector); + clStatus = clReleaseMemObject(d_Ax_vector); + CHECK_ERROR("clReleaseMemObject") + + clStatus = clReleaseCommandQueue(clCommandQueue); + clStatus = clReleaseContext(clContext); + + if (parameters->outFile) { + pb_SwitchToTimer(&timers, pb_TimerID_IO); + outputData(parameters->outFile, h_Ax_vector, dim); + } + + pb_SwitchToTimer(&timers, pb_TimerID_COMPUTE); + + //free((void *)clSource[0]); + + free(h_data); + free(h_indices); + free(h_ptr); + free(h_perm); + free(h_nzcnt); + free(h_Ax_vector); + free(h_x_vector); + pb_SwitchToTimer(&timers, pb_TimerID_NONE); + + pb_PrintTimerSet(&timers); + pb_FreeParameters(parameters); + + return 0; +} diff --git a/benchmarks/opencl/spmv/mmio.c b/benchmarks/opencl/spmv/mmio.c new file mode 100644 index 00000000..a3564718 --- /dev/null +++ b/benchmarks/opencl/spmv/mmio.c @@ -0,0 +1,509 @@ +/* +* Matrix Market I/O library for ANSI C +* +* See http://math.nist.gov/MatrixMarket for details. +* +* +*/ + +#include +#include +#include +#include +#include "mmio.h" + +int mm_read_unsymmetric_sparse(const char *fname, int *M_, int *N_, int *nz_, + double **val_, int **I_, int **J_) +{ + FILE *f; + MM_typecode matcode; + int M, N, nz; + int i; + double *val; + int *I, *J; + + if ((f = fopen(fname, "r")) == NULL) + return -1; + + + if (mm_read_banner(f, &matcode) != 0) + { + printf("mm_read_unsymetric: Could not process Matrix Market banner "); + printf(" in file [%s]\n", fname); + return -1; + } + + + + if ( !(mm_is_real(matcode) && mm_is_matrix(matcode) && + mm_is_sparse(matcode))) + { + fprintf(stderr, "Sorry, this application does not support "); + fprintf(stderr, "Market Market type: [%s]\n", + mm_typecode_to_str(matcode)); + return -1; + } + + /* find out size of sparse matrix: M, N, nz .... */ + + if (mm_read_mtx_crd_size(f, &M, &N, &nz) !=0) + { + fprintf(stderr, "read_unsymmetric_sparse(): could not parse matrix size.\n"); + return -1; + } + + *M_ = M; + *N_ = N; + *nz_ = nz; + + /* reseve memory for matrices */ + + I = (int *) malloc(nz * sizeof(int)); + J = (int *) malloc(nz * sizeof(int)); + val = (double *) malloc(nz * sizeof(double)); + + *val_ = val; + *I_ = I; + *J_ = J; + + /* NOTE: when reading in doubles, ANSI C requires the use of the "l" */ + /* specifier as in "%lg", "%lf", "%le", otherwise errors will occur */ + /* (ANSI C X3.159-1989, Sec. 4.9.6.2, p. 136 lines 13-15) */ + + for (i=0; i + +#define MM_MAX_LINE_LENGTH 1025 +#define MatrixMarketBanner "%%MatrixMarket" +#define MM_MAX_TOKEN_LENGTH 64 + +typedef char MM_typecode[4]; + +char *mm_typecode_to_str(MM_typecode matcode); + +int mm_read_banner(FILE *f, MM_typecode *matcode); +int mm_read_mtx_crd_size(FILE *f, int *M, int *N, int *nz); +int mm_read_mtx_array_size(FILE *f, int *M, int *N); + +int mm_write_banner(FILE *f, MM_typecode matcode); +int mm_write_mtx_crd_size(FILE *f, int M, int N, int nz); +int mm_write_mtx_array_size(FILE *f, int M, int N); + + +/********************* MM_typecode query fucntions ***************************/ + +#define mm_is_matrix(typecode) ((typecode)[0]=='M') + +#define mm_is_sparse(typecode) ((typecode)[1]=='C') +#define mm_is_coordinate(typecode)((typecode)[1]=='C') +#define mm_is_dense(typecode) ((typecode)[1]=='A') +#define mm_is_array(typecode) ((typecode)[1]=='A') + +#define mm_is_complex(typecode) ((typecode)[2]=='C') +#define mm_is_real(typecode) ((typecode)[2]=='R') +#define mm_is_pattern(typecode) ((typecode)[2]=='P') +#define mm_is_integer(typecode) ((typecode)[2]=='I') + +#define mm_is_symmetric(typecode)((typecode)[3]=='S') +#define mm_is_general(typecode) ((typecode)[3]=='G') +#define mm_is_skew(typecode) ((typecode)[3]=='K') +#define mm_is_hermitian(typecode)((typecode)[3]=='H') + +int mm_is_valid(MM_typecode matcode); /* too complex for a macro */ + + +/********************* MM_typecode modify fucntions ***************************/ + +#define mm_set_matrix(typecode) ((*typecode)[0]='M') +#define mm_set_coordinate(typecode) ((*typecode)[1]='C') +#define mm_set_array(typecode) ((*typecode)[1]='A') +#define mm_set_dense(typecode) mm_set_array(typecode) +#define mm_set_sparse(typecode) mm_set_coordinate(typecode) + +#define mm_set_complex(typecode)((*typecode)[2]='C') +#define mm_set_real(typecode) ((*typecode)[2]='R') +#define mm_set_pattern(typecode)((*typecode)[2]='P') +#define mm_set_integer(typecode)((*typecode)[2]='I') + + +#define mm_set_symmetric(typecode)((*typecode)[3]='S') +#define mm_set_general(typecode)((*typecode)[3]='G') +#define mm_set_skew(typecode) ((*typecode)[3]='K') +#define mm_set_hermitian(typecode)((*typecode)[3]='H') + +#define mm_clear_typecode(typecode) ((*typecode)[0]=(*typecode)[1]= \ + (*typecode)[2]=' ',(*typecode)[3]='G') + +#define mm_initialize_typecode(typecode) mm_clear_typecode(typecode) + + +/********************* Matrix Market error codes ***************************/ + + +#define MM_COULD_NOT_READ_FILE 11 +#define MM_PREMATURE_EOF 12 +#define MM_NOT_MTX 13 +#define MM_NO_HEADER 14 +#define MM_UNSUPPORTED_TYPE 15 +#define MM_LINE_TOO_LONG 16 +#define MM_COULD_NOT_WRITE_FILE 17 + + +/******************** Matrix Market internal definitions ******************** + + MM_matrix_typecode: 4-character sequence + + ojbect sparse/ data storage + dense type scheme + + string position: [0] [1] [2] [3] + + Matrix typecode: M(atrix) C(oord) R(eal) G(eneral) + A(array) C(omplex) H(ermitian) + P(attern) S(ymmetric) + I(nteger) K(kew) + + ***********************************************************************/ + +#define MM_MTX_STR "matrix" +#define MM_ARRAY_STR "array" +#define MM_DENSE_STR "array" +#define MM_COORDINATE_STR "coordinate" +#define MM_SPARSE_STR "coordinate" +#define MM_COMPLEX_STR "complex" +#define MM_REAL_STR "real" +#define MM_INT_STR "integer" +#define MM_GENERAL_STR "general" +#define MM_SYMM_STR "symmetric" +#define MM_HERM_STR "hermitian" +#define MM_SKEW_STR "skew-symmetric" +#define MM_PATTERN_STR "pattern" + + +/* high level routines */ + +int mm_write_mtx_crd(char fname[], int M, int N, int nz, int I[], int J[], + double val[], MM_typecode matcode); +int mm_read_mtx_crd_data(FILE *f, int M, int N, int nz, int I[], int J[], + double val[], MM_typecode matcode); +int mm_read_mtx_crd_entry(FILE *f, int *I, int *J, double *real, double *img, + MM_typecode matcode); + +int mm_read_unsymmetric_sparse(const char *fname, int *M_, int *N_, int *nz_, + double **val_, int **I_, int **J_); + + + +#endif diff --git a/benchmarks/opencl/spmv/ocl.c b/benchmarks/opencl/spmv/ocl.c new file mode 100644 index 00000000..9ce9a2f5 --- /dev/null +++ b/benchmarks/opencl/spmv/ocl.c @@ -0,0 +1,50 @@ +#include +#include +#include +#include +#include "ocl.h" + +char* readFile(const char* fileName) +{ + FILE* fp; + fp = fopen(fileName,"r"); + if(fp == NULL) + { + printf("Error 1!\n"); + exit(1); + } + + fseek(fp,0,SEEK_END); + long size = ftell(fp); + rewind(fp); + + char* buffer = (char*)malloc(sizeof(char)*(size+1)); + if(buffer == NULL) + { + printf("Error 2!\n"); + fclose(fp); + exit(1); + } + + size_t res = fread(buffer,1,size,fp); + if(res != size) + { + printf("Error 3!\n"); + fclose(fp); + exit(1); + } + + buffer[size] = 0; + fclose(fp); + return buffer; +} + +void clMemSet(cl_command_queue clCommandQueue, cl_mem buf, int val, size_t size) +{ + cl_int clStatus; + char* temp = (char*)malloc(size); + memset(temp,val,size); + clStatus = clEnqueueWriteBuffer(clCommandQueue,buf,CL_TRUE,0,size,temp,0,NULL,NULL); + CHECK_ERROR("clEnqueueWriteBuffer") + free(temp); +} diff --git a/benchmarks/opencl/spmv/ocl.h b/benchmarks/opencl/spmv/ocl.h new file mode 100644 index 00000000..8840a868 --- /dev/null +++ b/benchmarks/opencl/spmv/ocl.h @@ -0,0 +1,21 @@ +#ifndef __OCLH__ +#define __OCLH__ + +typedef struct { + cl_uint major; + cl_uint minor; + cl_uint multiProcessorCount; +} OpenCLDeviceProp; + +void clMemSet(cl_command_queue, cl_mem, int, size_t); +char* readFile(const char*); + +#define CHECK_ERROR(errorMessage) \ + if(clStatus != CL_SUCCESS) \ + { \ + printf("Error: %s!\n",errorMessage); \ + printf("Line: %d\n",__LINE__); \ + exit(1); \ + } + +#endif diff --git a/benchmarks/opencl/spmv/parboil.c b/benchmarks/opencl/spmv/parboil.c new file mode 100644 index 00000000..54fca9d0 --- /dev/null +++ b/benchmarks/opencl/spmv/parboil.c @@ -0,0 +1,427 @@ +/* + * (c) 2007 The Board of Trustees of the University of Illinois. + */ + +#include +#include +#include +#include + +#if _POSIX_VERSION >= 200112L +# include +#endif + + +/*****************************************************************************/ +/* Timer routines */ + +static void +accumulate_time(pb_Timestamp *accum, + pb_Timestamp start, + pb_Timestamp end) +{ +#if _POSIX_VERSION >= 200112L + *accum += end - start; +#else +# error "Timestamps not implemented for this system" +#endif +} + +#if _POSIX_VERSION >= 200112L +static pb_Timestamp get_time() +{ + struct timeval tv; + gettimeofday(&tv, NULL); + return (pb_Timestamp) (tv.tv_sec * 1000000LL + tv.tv_usec); +} +#else +# error "no supported time libraries are available on this platform" +#endif + +void +pb_ResetTimer(struct pb_Timer *timer) +{ + timer->state = pb_Timer_STOPPED; + +#if _POSIX_VERSION >= 200112L + timer->elapsed = 0; +#else +# error "pb_ResetTimer: not implemented for this system" +#endif +} + +void +pb_StartTimer(struct pb_Timer *timer) +{ + if (timer->state != pb_Timer_STOPPED) { + fputs("Ignoring attempt to start a running timer\n", stderr); + return; + } + + timer->state = pb_Timer_RUNNING; + +#if _POSIX_VERSION >= 200112L + { + struct timeval tv; + gettimeofday(&tv, NULL); + timer->init = tv.tv_sec * 1000000LL + tv.tv_usec; + } +#else +# error "pb_StartTimer: not implemented for this system" +#endif +} + +void +pb_StartTimerAndSubTimer(struct pb_Timer *timer, struct pb_Timer *subtimer) +{ + unsigned int numNotStopped = 0x3; // 11 + if (timer->state != pb_Timer_STOPPED) { + fputs("Warning: Timer was not stopped\n", stderr); + numNotStopped &= 0x1; // Zero out 2^1 + } + if (subtimer->state != pb_Timer_STOPPED) { + fputs("Warning: Subtimer was not stopped\n", stderr); + numNotStopped &= 0x2; // Zero out 2^0 + } + if (numNotStopped == 0x0) { + fputs("Ignoring attempt to start running timer and subtimer\n", stderr); + return; + } + + timer->state = pb_Timer_RUNNING; + subtimer->state = pb_Timer_RUNNING; + +#if _POSIX_VERSION >= 200112L + { + struct timeval tv; + gettimeofday(&tv, NULL); + + if (numNotStopped & 0x2) { + timer->init = tv.tv_sec * 1000000LL + tv.tv_usec; + } + + if (numNotStopped & 0x1) { + subtimer->init = tv.tv_sec * 1000000LL + tv.tv_usec; + } + } +#else +# error "pb_StartTimer: not implemented for this system" +#endif + +} + +void +pb_StopTimer(struct pb_Timer *timer) +{ + + pb_Timestamp fini; + + if (timer->state != pb_Timer_RUNNING) { + fputs("Ignoring attempt to stop a stopped timer\n", stderr); + return; + } + + timer->state = pb_Timer_STOPPED; + +#if _POSIX_VERSION >= 200112L + { + struct timeval tv; + gettimeofday(&tv, NULL); + fini = tv.tv_sec * 1000000LL + tv.tv_usec; + } +#else +# error "pb_StopTimer: not implemented for this system" +#endif + + accumulate_time(&timer->elapsed, timer->init, fini); + timer->init = fini; + +} + +void pb_StopTimerAndSubTimer(struct pb_Timer *timer, struct pb_Timer *subtimer) { + + pb_Timestamp fini; + + unsigned int numNotRunning = 0x3; // 0b11 + if (timer->state != pb_Timer_RUNNING) { + fputs("Warning: Timer was not running\n", stderr); + numNotRunning &= 0x1; // Zero out 2^1 + } + if (subtimer->state != pb_Timer_RUNNING) { + fputs("Warning: Subtimer was not running\n", stderr); + numNotRunning &= 0x2; // Zero out 2^0 + } + if (numNotRunning == 0x0) { + fputs("Ignoring attempt to stop stopped timer and subtimer\n", stderr); + return; + } + + + timer->state = pb_Timer_STOPPED; + subtimer->state = pb_Timer_STOPPED; + +#if _POSIX_VERSION >= 200112L + { + struct timeval tv; + gettimeofday(&tv, NULL); + fini = tv.tv_sec * 1000000LL + tv.tv_usec; + } +#else +# error "pb_StopTimer: not implemented for this system" +#endif + + if (numNotRunning & 0x2) { + accumulate_time(&timer->elapsed, timer->init, fini); + timer->init = fini; + } + + if (numNotRunning & 0x1) { + accumulate_time(&subtimer->elapsed, subtimer->init, fini); + subtimer->init = fini; + } + +} + +/* Get the elapsed time in seconds. */ +double +pb_GetElapsedTime(struct pb_Timer *timer) +{ + double ret; + + if (timer->state != pb_Timer_STOPPED) { + fputs("Elapsed time from a running timer is inaccurate\n", stderr); + } + +#if _POSIX_VERSION >= 200112L + ret = timer->elapsed / 1e6; +#else +# error "pb_GetElapsedTime: not implemented for this system" +#endif + return ret; +} + +void +pb_InitializeTimerSet(struct pb_TimerSet *timers) +{ + int n; + + timers->wall_begin = get_time(); + + timers->current = pb_TimerID_NONE; + + timers->async_markers = NULL; + + + for (n = 0; n < pb_TimerID_LAST; n++) { + pb_ResetTimer(&timers->timers[n]); + timers->sub_timer_list[n] = NULL; // free first? + } +} + +void +pb_AddSubTimer(struct pb_TimerSet *timers, char *label, enum pb_TimerID pb_Category) { + + struct pb_SubTimer *subtimer = (struct pb_SubTimer *) malloc + (sizeof(struct pb_SubTimer)); + + int len = strlen(label); + + subtimer->label = (char *) malloc (sizeof(char)*(len+1)); + sprintf(subtimer->label, "%s\0", label); + + pb_ResetTimer(&subtimer->timer); + subtimer->next = NULL; + + struct pb_SubTimerList *subtimerlist = timers->sub_timer_list[pb_Category]; + if (subtimerlist == NULL) { + subtimerlist = (struct pb_SubTimerList *) malloc + (sizeof(struct pb_SubTimerList)); + subtimerlist->subtimer_list = subtimer; + timers->sub_timer_list[pb_Category] = subtimerlist; + } else { + // Append to list + struct pb_SubTimer *element = subtimerlist->subtimer_list; + while (element->next != NULL) { + element = element->next; + } + element->next = subtimer; + } + +} + +void +pb_SwitchToSubTimer(struct pb_TimerSet *timers, char *label, enum pb_TimerID category) +{ + +// switchToSub( NULL, NONE +// switchToSub( NULL, some +// switchToSub( some, some +// switchToSub( some, NONE -- tries to find "some" in NONE's sublist, which won't be printed + + struct pb_Timer *topLevelToStop = NULL; + if (timers->current != category && timers->current != pb_TimerID_NONE) { + // Switching to subtimer in a different category needs to stop the top-level current, different categoried timer. + // NONE shouldn't have a timer associated with it, so exclude from branch + topLevelToStop = &timers->timers[timers->current]; + } + + struct pb_SubTimerList *subtimerlist = timers->sub_timer_list[timers->current]; + struct pb_SubTimer *curr = (subtimerlist == NULL) ? NULL : subtimerlist->current; + + if (timers->current != pb_TimerID_NONE) { + if (curr != NULL && topLevelToStop != NULL) { + pb_StopTimerAndSubTimer(topLevelToStop, &curr->timer); + } else if (curr != NULL) { + pb_StopTimer(&curr->timer); + } else { + pb_StopTimer(topLevelToStop); + } + } + + subtimerlist = timers->sub_timer_list[category]; + struct pb_SubTimer *subtimer = NULL; + + if (label != NULL) { + subtimer = subtimerlist->subtimer_list; + while (subtimer != NULL) { + if (strcmp(subtimer->label, label) == 0) { + break; + } else { + subtimer = subtimer->next; + } + } + } + + if (category != pb_TimerID_NONE) { + + if (subtimerlist != NULL) { + subtimerlist->current = subtimer; + } + + if (category != timers->current && subtimer != NULL) { + pb_StartTimerAndSubTimer(&timers->timers[category], &subtimer->timer); + } else if (subtimer != NULL) { + // Same category, different non-NULL subtimer + pb_StartTimer(&subtimer->timer); + } else{ + // Different category, but no subtimer (not found or specified as NULL) -- unprefered way of setting topLevel timer + pb_StartTimer(&timers->timers[category]); + } + } + + timers->current = category; + +} + +void +pb_SwitchToTimer(struct pb_TimerSet *timers, enum pb_TimerID timer) +{ + /* Stop the currently running timer */ + /*if (timers->current != pb_TimerID_NONE) { + struct pb_SubTimer *currSubTimer = NULL; + struct pb_SubTimerList *subtimerlist = timers->sub_timer_list[timers->current]; + + if ( subtimerlist != NULL) { + currSubTimer = timers->sub_timer_list[timers->current]->current; + } + if ( currSubTimer!= NULL) { + pb_StopTimerAndSubTimer(&timers->timers[timers->current], &currSubTimer->timer); + } else { + pb_StopTimer(&timers->timers[timers->current]); + } + } + + timers->current = timer; + + if (timer != pb_TimerID_NONE) { + pb_StartTimer(&timers->timers[timer]); + }*/ +} + +void +pb_PrintTimerSet(struct pb_TimerSet *timers) +{ + + pb_Timestamp wall_end = get_time(); + + struct pb_Timer *t = timers->timers; + struct pb_SubTimer* sub = NULL; + + int maxSubLength; + + const char *categories[] = { + "IO", "Kernel", "Copy", "Driver", "Copy Async", "Compute" + }; + + const int maxCategoryLength = 10; + + int i; + for(i = 1; i < pb_TimerID_LAST-1; ++i) { // exclude NONE and OVRELAP from this format + if(pb_GetElapsedTime(&t[i]) != 0) { + + // Print Category Timer + printf("%-*s: %f\n", maxCategoryLength, categories[i-1], pb_GetElapsedTime(&t[i])); + + if (timers->sub_timer_list[i] != NULL) { + sub = timers->sub_timer_list[i]->subtimer_list; + maxSubLength = 0; + while (sub != NULL) { + // Find longest SubTimer label + if (strlen(sub->label) > maxSubLength) { + maxSubLength = strlen(sub->label); + } + sub = sub->next; + } + + // Fit to Categories + if (maxSubLength <= maxCategoryLength) { + maxSubLength = maxCategoryLength; + } + + sub = timers->sub_timer_list[i]->subtimer_list; + + // Print SubTimers + while (sub != NULL) { + printf(" -%-*s: %f\n", maxSubLength, sub->label, pb_GetElapsedTime(&sub->timer)); + sub = sub->next; + } + } + } + } + + if(pb_GetElapsedTime(&t[pb_TimerID_OVERLAP]) != 0) + printf("CPU/Kernel Overlap: %f\n", pb_GetElapsedTime(&t[pb_TimerID_OVERLAP])); + + float walltime = (wall_end - timers->wall_begin)/ 1e6; + printf("Timer Wall Time: %f\n", walltime); + +} + +void pb_DestroyTimerSet(struct pb_TimerSet * timers) +{ + /* clean up all of the async event markers */ + struct pb_async_time_marker_list ** event = &(timers->async_markers); + while( *event != NULL) { + struct pb_async_time_marker_list ** next = &((*event)->next); + free(*event); + (*event) = NULL; + event = next; + } + + int i = 0; + for(i = 0; i < pb_TimerID_LAST; ++i) { + if (timers->sub_timer_list[i] != NULL) { + struct pb_SubTimer *subtimer = timers->sub_timer_list[i]->subtimer_list; + struct pb_SubTimer *prev = NULL; + while (subtimer != NULL) { + free(subtimer->label); + prev = subtimer; + subtimer = subtimer->next; + free(prev); + } + free(timers->sub_timer_list[i]); + } + } +} + + diff --git a/benchmarks/opencl/spmv/parboil.h b/benchmarks/opencl/spmv/parboil.h new file mode 100644 index 00000000..4c9a8b5e --- /dev/null +++ b/benchmarks/opencl/spmv/parboil.h @@ -0,0 +1,348 @@ +/* + * (c) 2010 The Board of Trustees of the University of Illinois. + */ +#ifndef PARBOIL_HEADER +#define PARBOIL_HEADER + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +#include + +/* A platform as specified by the user on the command line */ +struct pb_PlatformParam { + char *name; /* The platform name. This string is owned. */ + char *version; /* The platform version; may be NULL. + * This string is owned. */ +}; + +/* Create a PlatformParam from the given strings. + * 'name' must not be NULL. 'version' may be NULL. + * If not NULL, the strings should have been allocated by malloc(), + * and they will be owned by the returned object. + */ +struct pb_PlatformParam * +pb_PlatformParam(char *name, char *version); + +void +pb_FreePlatformParam(struct pb_PlatformParam *); + +/* A criterion for how to select a device */ +enum pb_DeviceSelectionCriterion { + pb_Device_INDEX, /* Enumerate the devices and select one + * by its number */ + pb_Device_CPU, /* Select a CPU device */ + pb_Device_GPU, /* Select a GPU device */ + pb_Device_ACCELERATOR, /* Select an accelerator device */ + pb_Device_NAME /* Select a device by name */ +}; + +/* A device as specified by the user on the command line */ +struct pb_DeviceParam { + enum pb_DeviceSelectionCriterion criterion; + union { + int index; /* If criterion == pb_Device_INDEX, + * the index of the device */ + char *name; /* If criterion == pb_Device_NAME, + * the name of the device. + * This string is owned. */ + }; +}; + +struct pb_DeviceParam * +pb_DeviceParam_index(int index); + +struct pb_DeviceParam * +pb_DeviceParam_cpu(void); + +struct pb_DeviceParam * +pb_DeviceParam_gpu(void); + +struct pb_DeviceParam * +pb_DeviceParam_accelerator(void); + +/* Create a by-name device selection criterion. + * The string should have been allocated by malloc(), and it will will be + * owned by the returned object. + */ +struct pb_DeviceParam * +pb_DeviceParam_name(char *name); + +void +pb_FreeDeviceParam(struct pb_DeviceParam *); + +/* Command line parameters for benchmarks */ +struct pb_Parameters { + char *outFile; /* If not NULL, the raw output of the + * computation should be saved to this + * file. The string is owned. */ + char **inpFiles; /* A NULL-terminated array of strings + * holding the input file(s) for the + * computation. The array and strings + * are owned. */ + struct pb_PlatformParam *platform; /* If not NULL, the platform + * specified on the command line. */ + struct pb_DeviceParam *device; /* If not NULL, the device + * specified on the command line. */ +}; + +/* Read command-line parameters. + * + * The argc and argv parameters to main are read, and any parameters + * interpreted by this function are removed from the argument list. + * + * A new instance of struct pb_Parameters is returned. + * If there is an error, then an error message is printed on stderr + * and NULL is returned. + */ +struct pb_Parameters * +pb_ReadParameters(int *_argc, char **argv); + +/* Free an instance of struct pb_Parameters. + */ +void +pb_FreeParameters(struct pb_Parameters *p); + +void +pb_FreeStringArray(char **); + +/* Count the number of input files in a pb_Parameters instance. + */ +int +pb_Parameters_CountInputs(struct pb_Parameters *p); + +/* A time or duration. */ +//#if _POSIX_VERSION >= 200112L +typedef unsigned long long pb_Timestamp; /* time in microseconds */ +//#else +//# error "Timestamps not implemented" +//#endif + +enum pb_TimerState { + pb_Timer_STOPPED, + pb_Timer_RUNNING, +}; + +struct pb_Timer { + enum pb_TimerState state; + pb_Timestamp elapsed; /* Amount of time elapsed so far */ + pb_Timestamp init; /* Beginning of the current time interval, + * if state is RUNNING. End of the last + * recorded time interfal otherwise. */ +}; + +/* Reset a timer. + * Use this to initialize a timer or to clear + * its elapsed time. The reset timer is stopped. + */ +void +pb_ResetTimer(struct pb_Timer *timer); + +/* Start a timer. The timer is set to RUNNING mode and + * time elapsed while the timer is running is added to + * the timer. + * The timer should not already be running. + */ +void +pb_StartTimer(struct pb_Timer *timer); + +/* Stop a timer. + * This stops adding elapsed time to the timer. + * The timer should not already be stopped. + */ +void +pb_StopTimer(struct pb_Timer *timer); + +/* Get the elapsed time in seconds. */ +double +pb_GetElapsedTime(struct pb_Timer *timer); + +/* Execution time is assigned to one of these categories. */ +enum pb_TimerID { + pb_TimerID_NONE = 0, + pb_TimerID_IO, /* Time spent in input/output */ + pb_TimerID_KERNEL, /* Time spent computing on the device, + * recorded asynchronously */ + pb_TimerID_COPY, /* Time spent synchronously moving data + * to/from device and allocating/freeing + * memory on the device */ + pb_TimerID_DRIVER, /* Time spent in the host interacting with the + * driver, primarily for recording the time + * spent queueing asynchronous operations */ + pb_TimerID_COPY_ASYNC, /* Time spent in asynchronous transfers */ + pb_TimerID_COMPUTE, /* Time for all program execution other + * than parsing command line arguments, + * I/O, kernel, and copy */ + pb_TimerID_OVERLAP, /* Time double-counted in asynchronous and + * host activity: automatically filled in, + * not intended for direct usage */ + pb_TimerID_LAST /* Number of timer IDs */ +}; + +/* Dynamic list of asynchronously tracked times between events */ +struct pb_async_time_marker_list { + char *label; // actually just a pointer to a string + enum pb_TimerID timerID; /* The ID to which the interval beginning + * with this marker should be attributed */ + void * marker; + //cudaEvent_t marker; /* The driver event for this marker */ + struct pb_async_time_marker_list *next; +}; + +struct pb_SubTimer { + char *label; + struct pb_Timer timer; + struct pb_SubTimer *next; +}; + +struct pb_SubTimerList { + struct pb_SubTimer *current; + struct pb_SubTimer *subtimer_list; +}; + +/* A set of timers for recording execution times. */ +struct pb_TimerSet { + enum pb_TimerID current; + struct pb_async_time_marker_list* async_markers; + pb_Timestamp async_begin; + pb_Timestamp wall_begin; + struct pb_Timer timers[pb_TimerID_LAST]; + struct pb_SubTimerList *sub_timer_list[pb_TimerID_LAST]; +}; + +/* Reset all timers in the set. */ +void +pb_InitializeTimerSet(struct pb_TimerSet *timers); + +void +pb_AddSubTimer(struct pb_TimerSet *timers, char *label, enum pb_TimerID pb_Category); + +/* Select which timer the next interval of time should be accounted + * to. The selected timer is started and other timers are stopped. + * Using pb_TimerID_NONE stops all timers. */ +void +pb_SwitchToTimer(struct pb_TimerSet *timers, enum pb_TimerID timer); + +void +pb_SwitchToSubTimer(struct pb_TimerSet *timers, char *label, enum pb_TimerID category); + +/* Print timer values to standard output. */ +void +pb_PrintTimerSet(struct pb_TimerSet *timers); + +/* Release timer resources */ +void +pb_DestroyTimerSet(struct pb_TimerSet * timers); + +void +pb_SetOpenCL(void *clContextPtr, void *clCommandQueuePtr); + + +typedef struct pb_Device_tag { + char* name; + void* clDevice; + int id; + unsigned int in_use; + unsigned int available; +} pb_Device; + +struct pb_Context_tag; +typedef struct pb_Context_tag pb_Context; + +typedef struct pb_Platform_tag { + char* name; + char* version; + void* clPlatform; + unsigned int in_use; + pb_Context** contexts; + pb_Device** devices; +} pb_Platform; + +struct pb_Context_tag { + void* clPlatformId; + void* clContext; + void* clDeviceId; + pb_Platform* pb_platform; + pb_Device* pb_device; +}; + +// verbosely print out list of platforms and their devices to the console. +pb_Platform** +pb_GetPlatforms(); + +// Choose a platform according to the given platform specification +pb_Platform* +pb_GetPlatform(struct pb_PlatformParam *platform); + +// choose a platform: by name, name & version +pb_Platform* +pb_GetPlatformByName(const char* name); + +pb_Platform* +pb_GetPlatformByNameAndVersion(const char* name, const char* version); + +// Choose a device according to the given device specification +pb_Device* +pb_GetDevice(pb_Platform* pb_platform, struct pb_DeviceParam *device); + +pb_Device** +pb_GetDevices(pb_Platform* pb_platform); + +// choose a device by name. +pb_Device* +pb_GetDeviceByName(pb_Platform* pb_platform, const char* name); + +pb_Platform* +pb_GetPlatformByEnvVars(); + +pb_Context* +pb_InitOpenCLContext(struct pb_Parameters* parameters); + +void +pb_ReleasePlatforms(); + +void +pb_ReleaseContext(pb_Context* c); + +void +pb_PrintPlatformInfo(pb_Context* c); + +void +perf_init(); + +//#define MEASURE_KERNEL_TIME + +#include + +#ifdef MEASURE_KERNEL_TIME +#define clEnqueueNDRangeKernel(q,k,d,o,dg,db,a,b,c) pb_clEnqueueNDRangeKernel((q), (k), (d), (o), (dg), (db), (a), (b), (c)) +cl_int +pb_clEnqueueNDRangeKernel(cl_command_queue /* command_queue */, + cl_kernel /* kernel */, + cl_uint /* work_dim */, + const size_t * /* global_work_offset */, + const size_t * /* global_work_size */, + const size_t * /* local_work_size */, + cl_uint /* num_events_in_wait_list */, + const cl_event * /* event_wait_list */, + cl_event * /* event */); +#endif + +enum { T_FLOAT, T_DOUBLE, T_SHORT, T_INT, T_UCHAR }; +void pb_sig_float(char*, float*, int); +void pb_sig_double(char*, double*, int); +void pb_sig_short(char*, short*, int); +void pb_sig_int(char*, int*, int); +void pb_sig_uchar(char*, unsigned char*, unsigned int); +void pb_sig_clmem(char*, cl_command_queue, cl_mem, int); + +#ifdef __cplusplus +} +#endif + +#endif //PARBOIL_HEADER + diff --git a/benchmarks/opencl/spmv/parboil_opencl.c b/benchmarks/opencl/spmv/parboil_opencl.c new file mode 100644 index 00000000..4936815c --- /dev/null +++ b/benchmarks/opencl/spmv/parboil_opencl.c @@ -0,0 +1,1390 @@ +/* + * (c) 2007 The Board of Trustees of the University of Illinois. + */ + +#include +#include +#include +#include +#include +#include + +#if _POSIX_VERSION >= 200112L +# include +#endif + +#include "perfmon.h" + +cl_context *clContextPtr; +cl_command_queue *clCommandQueuePtr; + +// #define DISABLE_PARBOIL_TIMER + +/*****************************************************************************/ +/* Timer routines */ + +static int is_async(enum pb_TimerID timer) +{ + return (timer == pb_TimerID_KERNEL) || + (timer == pb_TimerID_COPY_ASYNC); +} + +static int is_blocking(enum pb_TimerID timer) +{ + return (timer == pb_TimerID_COPY) || (timer == pb_TimerID_NONE); +} + +#define INVALID_TIMERID pb_TimerID_LAST + +static int asyncs_outstanding(struct pb_TimerSet* timers) +{ + return (timers->async_markers != NULL) && + (timers->async_markers->timerID != INVALID_TIMERID); +} + +static struct pb_async_time_marker_list * +get_last_async(struct pb_TimerSet* timers) +{ + /* Find the last event recorded thus far */ + struct pb_async_time_marker_list * last_event = timers->async_markers; + if(last_event != NULL && last_event->timerID != INVALID_TIMERID) { + while(last_event->next != NULL && + last_event->next->timerID != INVALID_TIMERID) + last_event = last_event->next; + return last_event; + } else + return NULL; +} + +static void insert_marker(struct pb_TimerSet* tset, enum pb_TimerID timer) +{ + cl_int ciErrNum = CL_SUCCESS; + struct pb_async_time_marker_list ** new_event = &(tset->async_markers); + + while(*new_event != NULL && (*new_event)->timerID != INVALID_TIMERID) { + new_event = &((*new_event)->next); + } + + if(*new_event == NULL) { + *new_event = (struct pb_async_time_marker_list *) + malloc(sizeof(struct pb_async_time_marker_list)); + (*new_event)->marker = calloc(1, sizeof(cl_event)); + /* + // I don't think this is needed at all. I believe clEnqueueMarker 'creates' the event +#if ( __OPENCL_VERSION__ >= CL_VERSION_1_1 ) +fprintf(stderr, "Creating Marker [%d]\n", timer); + *((cl_event *)((*new_event)->marker)) = clCreateUserEvent(*clContextPtr, &ciErrNum); + if (ciErrNum != CL_SUCCESS) { + fprintf(stderr, "Error Creating User Event Object!\n"); + } + ciErrNum = clSetUserEventStatus(*((cl_event *)((*new_event)->marker)), CL_QUEUED); + if (ciErrNum != CL_SUCCESS) { + fprintf(stderr, "Error Setting User Event Status!\n"); + } +#endif +*/ + (*new_event)->next = NULL; + } + + /* valid event handle now aquired: insert the event record */ + (*new_event)->label = NULL; + (*new_event)->timerID = timer; + ciErrNum = clEnqueueMarker(*clCommandQueuePtr, (cl_event *)(*new_event)->marker); + if (ciErrNum != CL_SUCCESS) { + fprintf(stderr, "Error Enqueueing Marker!\n"); + } + +} + +static void insert_submarker(struct pb_TimerSet* tset, char *label, enum pb_TimerID timer) +{ + cl_int ciErrNum = CL_SUCCESS; + struct pb_async_time_marker_list ** new_event = &(tset->async_markers); + + while(*new_event != NULL && (*new_event)->timerID != INVALID_TIMERID) { + new_event = &((*new_event)->next); + } + + if(*new_event == NULL) { + *new_event = (struct pb_async_time_marker_list *) + malloc(sizeof(struct pb_async_time_marker_list)); + (*new_event)->marker = calloc(1, sizeof(cl_event)); + /* +#if ( __OPENCL_VERSION__ >= CL_VERSION_1_1 ) +fprintf(stderr, "Creating SubMarker %s[%d]\n", label, timer); + *((cl_event *)((*new_event)->marker)) = clCreateUserEvent(*clContextPtr, &ciErrNum); + if (ciErrNum != CL_SUCCESS) { + fprintf(stderr, "Error Creating User Event Object!\n"); + } + ciErrNum = clSetUserEventStatus(*((cl_event *)((*new_event)->marker)), CL_QUEUED); + if (ciErrNum != CL_SUCCESS) { + fprintf(stderr, "Error Setting User Event Status!\n"); + } +#endif +*/ + (*new_event)->next = NULL; + } + + /* valid event handle now aquired: insert the event record */ + (*new_event)->label = label; + (*new_event)->timerID = timer; + ciErrNum = clEnqueueMarker(*clCommandQueuePtr, (cl_event *)(*new_event)->marker); + if (ciErrNum != CL_SUCCESS) { + fprintf(stderr, "Error Enqueueing Marker!\n"); + } + +} + + +/* Assumes that all recorded events have completed */ +static pb_Timestamp record_async_times(struct pb_TimerSet* tset) +{ + struct pb_async_time_marker_list * next_interval = NULL; + struct pb_async_time_marker_list * last_marker = get_last_async(tset); + pb_Timestamp total_async_time = 0; + enum pb_TimerID timer; + + for(next_interval = tset->async_markers; next_interval != last_marker; + next_interval = next_interval->next) { + cl_ulong command_start=0, command_end=0; + cl_int ciErrNum = CL_SUCCESS; + + ciErrNum = clGetEventProfilingInfo(*((cl_event *)next_interval->marker), CL_PROFILING_COMMAND_END, sizeof(cl_ulong), &command_start, NULL); + if (ciErrNum != CL_SUCCESS) { + fprintf(stderr, "Error getting first EventProfilingInfo: %d\n", ciErrNum); + } + + ciErrNum = clGetEventProfilingInfo(*((cl_event *)next_interval->next->marker), CL_PROFILING_COMMAND_END, sizeof(cl_ulong), &command_end, NULL); + if (ciErrNum != CL_SUCCESS) { + fprintf(stderr, "Error getting second EventProfilingInfo: %d\n", ciErrNum); + } + + pb_Timestamp interval = (pb_Timestamp) (((double)(command_end - command_start)) / 1e3); + tset->timers[next_interval->timerID].elapsed += interval; + if (next_interval->label != NULL) { + struct pb_SubTimer *subtimer = tset->sub_timer_list[next_interval->timerID]->subtimer_list; + while (subtimer != NULL) { + if ( strcmp(subtimer->label, next_interval->label) == 0) { + subtimer->timer.elapsed += interval; + break; + } + subtimer = subtimer->next; + } + } + total_async_time += interval; + next_interval->timerID = INVALID_TIMERID; + } + + if(next_interval != NULL) + next_interval->timerID = INVALID_TIMERID; + + return total_async_time; +} + +static void +accumulate_time(pb_Timestamp *accum, + pb_Timestamp start, + pb_Timestamp end) +{ +//#if _POSIX_VERSION >= 200112L + *accum += end - start; +//#else +//# error "Timestamps not implemented for this system" +//#endif +} + +//#if _POSIX_VERSION >= 200112L +static pb_Timestamp get_time() +{ + //struct timeval tv; + //gettimeofday(&tv, NULL); + //return (pb_Timestamp) (tv.tv_sec * 1000000LL + tv.tv_usec); + return 0; +} +//#else +//# error "no supported time libraries are available on this platform" +//#endif + +void +pb_ResetTimer(struct pb_Timer *timer) +{ +//#ifndef DISABLE_PARBOIL_TIMER + timer->state = pb_Timer_STOPPED; + +//#if _POSIX_VERSION >= 200112L + timer->elapsed = 0; +//#else +//# error "pb_ResetTimer: not implemented for this system" +//#endif +//#endif +} + +void +pb_StartTimer(struct pb_Timer *timer) +{ +/*#ifndef DISABLE_PARBOIL_TIMER + if (timer->state != pb_Timer_STOPPED) { + fputs("Ignoring attempt to start a running timer\n", stderr); + return; + } + + timer->state = pb_Timer_RUNNING; + +#if _POSIX_VERSION >= 200112L + { + struct timeval tv; + gettimeofday(&tv, NULL); + timer->init = tv.tv_sec * 1000000LL + tv.tv_usec; + } +#else +# error "pb_StartTimer: not implemented for this system" +#endif +#endif*/ +} + +void +pb_StartTimerAndSubTimer(struct pb_Timer *timer, struct pb_Timer *subtimer) +{ +/*#ifndef DISABLE_PARBOIL_TIMER + + unsigned int numNotStopped = 0x3; // 11 + if (timer->state != pb_Timer_STOPPED) { + fputs("Warning: Timer was not stopped\n", stderr); + numNotStopped &= 0x1; // Zero out 2^1 + } + if (subtimer->state != pb_Timer_STOPPED) { + fputs("Warning: Subtimer was not stopped\n", stderr); + numNotStopped &= 0x2; // Zero out 2^0 + } + if (numNotStopped == 0x0) { + fputs("Ignoring attempt to start running timer and subtimer\n", stderr); + return; + } + + timer->state = pb_Timer_RUNNING; + subtimer->state = pb_Timer_RUNNING; + +#if _POSIX_VERSION >= 200112L + { + struct timeval tv; + gettimeofday(&tv, NULL); + + if (numNotStopped & 0x2) { + timer->init = tv.tv_sec * 1000000LL + tv.tv_usec; + } + + if (numNotStopped & 0x1) { + subtimer->init = tv.tv_sec * 1000000LL + tv.tv_usec; + } + } +#else +# error "pb_StartTimer: not implemented for this system" +#endif + +#endif*/ +} + +void +pb_StopTimer(struct pb_Timer *timer) +{ +/*#ifndef DISABLE_PARBOIL_TIMER + + pb_Timestamp fini; + + if (timer->state != pb_Timer_RUNNING) { + fputs("Ignoring attempt to stop a stopped timer\n", stderr); + return; + } + + timer->state = pb_Timer_STOPPED; + +#if _POSIX_VERSION >= 200112L + { + struct timeval tv; + gettimeofday(&tv, NULL); + fini = tv.tv_sec * 1000000LL + tv.tv_usec; + } +#else +# error "pb_StopTimer: not implemented for this system" +#endif + + accumulate_time(&timer->elapsed, timer->init, fini); + timer->init = fini; + +#endif*/ +} + +void pb_StopTimerAndSubTimer(struct pb_Timer *timer, struct pb_Timer *subtimer) { +/*#ifndef DISABLE_PARBOIL_TIMER + + pb_Timestamp fini; + + unsigned int numNotRunning = 0x3; // 11 + if (timer->state != pb_Timer_RUNNING) { + fputs("Warning: Timer was not running\n", stderr); + numNotRunning &= 0x1; // Zero out 2^1 + } + if (subtimer->state != pb_Timer_RUNNING) { + fputs("Warning: Subtimer was not running\n", stderr); + numNotRunning &= 0x2; // Zero out 2^0 + } + if (numNotRunning == 0x0) { + fputs("Ignoring attempt to stop stopped timer and subtimer\n", stderr); + return; + } + + + timer->state = pb_Timer_STOPPED; + subtimer->state = pb_Timer_STOPPED; + +#if _POSIX_VERSION >= 200112L + { + struct timeval tv; + gettimeofday(&tv, NULL); + fini = tv.tv_sec * 1000000LL + tv.tv_usec; + } +#else +# error "pb_StopTimer: not implemented for this system" +#endif + + if (numNotRunning & 0x2) { + accumulate_time(&timer->elapsed, timer->init, fini); + timer->init = fini; + } + + if (numNotRunning & 0x1) { + accumulate_time(&subtimer->elapsed, subtimer->init, fini); + subtimer->init = fini; + } + +#endif*/ +} + +/* Get the elapsed time in seconds. */ +double +pb_GetElapsedTime(struct pb_Timer *timer) +{ + /*double ret; +#ifndef DISABLE_PARBOIL_TIMER + + if (timer->state != pb_Timer_STOPPED) { + fputs("Elapsed time from a running timer is inaccurate\n", stderr); + } + +#if _POSIX_VERSION >= 200112L + ret = timer->elapsed / 1e6; +#else +# error "pb_GetElapsedTime: not implemented for this system" +#endif +#endif + return ret;*/ + return 0; +} + +void +pb_InitializeTimerSet(struct pb_TimerSet *timers) +{ +/*#ifndef DISABLE_PARBOIL_TIMER + int n; + + timers->wall_begin = 0; //get_time(); + timers->current = pb_TimerID_NONE; + + timers->async_markers = NULL; + + for (n = 0; n < pb_TimerID_LAST; n++) { + pb_ResetTimer(&timers->timers[n]); + timers->sub_timer_list[n] = NULL; + } +#endif*/ +} + +void pb_SetOpenCL(void *p_clContextPtr, void *p_clCommandQueuePtr) { + clContextPtr = ((cl_context *)p_clContextPtr); + clCommandQueuePtr = ((cl_command_queue *)p_clCommandQueuePtr); +} + +void +pb_AddSubTimer(struct pb_TimerSet *timers, char *label, enum pb_TimerID pb_Category) { +/*#ifndef DISABLE_PARBOIL_TIMER + + struct pb_SubTimer *subtimer = (struct pb_SubTimer *) malloc + (sizeof(struct pb_SubTimer)); + + int len = strlen(label); + + subtimer->label = (char *) malloc (sizeof(char)*(len+1)); + sprintf(subtimer->label, "%s\0", label); + + pb_ResetTimer(&subtimer->timer); + subtimer->next = NULL; + + struct pb_SubTimerList *subtimerlist = timers->sub_timer_list[pb_Category]; + if (subtimerlist == NULL) { + subtimerlist = (struct pb_SubTimerList *) calloc + (1, sizeof(struct pb_SubTimerList)); + subtimerlist->subtimer_list = subtimer; + timers->sub_timer_list[pb_Category] = subtimerlist; + } else { + // Append to list + struct pb_SubTimer *element = subtimerlist->subtimer_list; + while (element->next != NULL) { + element = element->next; + } + element->next = subtimer; + } + +#endif*/ +} + +void +pb_SwitchToTimer(struct pb_TimerSet *timers, enum pb_TimerID timer) +{ +#if 0 +#ifndef DISABLE_PARBOIL_TIMER + + /* Stop the currently running timer */ + if (timers->current != pb_TimerID_NONE) { + struct pb_SubTimerList *subtimerlist = timers->sub_timer_list[timers->current]; + struct pb_SubTimer *currSubTimer = (subtimerlist != NULL) ? subtimerlist->current : NULL; + + if (!is_async(timers->current) ) { + if (timers->current != timer) { + if (currSubTimer != NULL) { + pb_StopTimerAndSubTimer(&timers->timers[timers->current], &currSubTimer->timer); + } else { + pb_StopTimer(&timers->timers[timers->current]); + } + } else { + if (currSubTimer != NULL) { + pb_StopTimer(&currSubTimer->timer); + } + } + } else { + insert_marker(timers, timer); + if (!is_async(timer)) { // if switching to async too, keep driver going + pb_StopTimer(&timers->timers[pb_TimerID_DRIVER]); + } + } + } + + pb_Timestamp currentTime = 0; //get_time(); + + /* The only cases we check for asynchronous task completion is + * when an overlapping CPU operation completes, or the next + * segment blocks on completion of previous async operations */ + if( asyncs_outstanding(timers) && + (!is_async(timers->current) || is_blocking(timer) ) ) { + + struct pb_async_time_marker_list * last_event = get_last_async(timers); + /* CL_COMPLETE if completed */ + + cl_int ciErrNum = CL_SUCCESS; + cl_int async_done = CL_COMPLETE; + + ciErrNum = clGetEventInfo(*((cl_event *)last_event->marker), CL_EVENT_COMMAND_EXECUTION_STATUS, sizeof(cl_int), &async_done, NULL); + if (ciErrNum != CL_SUCCESS) { + fprintf(stderr, "Error Querying EventInfo!\n"); + } + + + if(is_blocking(timer)) { + /* Async operations completed after previous CPU operations: + * overlapped time is the total CPU time since this set of async + * operations were first issued */ + + // timer to switch to is COPY or NONE + if(async_done != CL_COMPLETE) { + accumulate_time(&(timers->timers[pb_TimerID_OVERLAP].elapsed), + timers->async_begin,currentTime); + } + + /* Wait on async operation completion */ + ciErrNum = clWaitForEvents(1, (cl_event *)last_event->marker); + if (ciErrNum != CL_SUCCESS) { + fprintf(stderr, "Error Waiting for Events!\n"); + } + + pb_Timestamp total_async_time = record_async_times(timers); + + /* Async operations completed before previous CPU operations: + * overlapped time is the total async time */ + if(async_done == CL_COMPLETE) { + //fprintf(stderr, "Async_done: total_async_type = %lld\n", total_async_time); + timers->timers[pb_TimerID_OVERLAP].elapsed += total_async_time; + } + + } else + /* implies (!is_async(timers->current) && asyncs_outstanding(timers)) */ + // i.e. Current Not Async (not KERNEL/COPY_ASYNC) but there are outstanding + // so something is deeper in stack + if(async_done == CL_COMPLETE ) { + /* Async operations completed before previous CPU operations: + * overlapped time is the total async time */ + timers->timers[pb_TimerID_OVERLAP].elapsed += record_async_times(timers); + } + } + + /* Start the new timer */ + if (timer != pb_TimerID_NONE) { + if(!is_async(timer)) { + pb_StartTimer(&timers->timers[timer]); + } else { + // toSwitchTo Is Async (KERNEL/COPY_ASYNC) + if (!asyncs_outstanding(timers)) { + /* No asyncs outstanding, insert a fresh async marker */ + + insert_marker(timers, timer); + timers->async_begin = currentTime; + } else if(!is_async(timers->current)) { + /* Previous asyncs still in flight, but a previous SwitchTo + * already marked the end of the most recent async operation, + * so we can rename that marker as the beginning of this async + * operation */ + + struct pb_async_time_marker_list * last_event = get_last_async(timers); + last_event->label = NULL; + last_event->timerID = timer; + } + if (!is_async(timers->current)) { + pb_StartTimer(&timers->timers[pb_TimerID_DRIVER]); + } + } + } + timers->current = timer; + +#endif +#endif +} + +void +pb_SwitchToSubTimer(struct pb_TimerSet *timers, char *label, enum pb_TimerID category) +{ +#ifndef DISABLE_PARBOIL_TIMER + struct pb_SubTimerList *subtimerlist = timers->sub_timer_list[timers->current]; + struct pb_SubTimer *curr = (subtimerlist != NULL) ? subtimerlist->current : NULL; + + if (timers->current != pb_TimerID_NONE) { + if (!is_async(timers->current) ) { + if (timers->current != category) { + if (curr != NULL) { + pb_StopTimerAndSubTimer(&timers->timers[timers->current], &curr->timer); + } else { + pb_StopTimer(&timers->timers[timers->current]); + } + } else { + if (curr != NULL) { + pb_StopTimer(&curr->timer); + } + } + } else { + insert_submarker(timers, label, category); + if (!is_async(category)) { // if switching to async too, keep driver going + pb_StopTimer(&timers->timers[pb_TimerID_DRIVER]); + } + } + } + + pb_Timestamp currentTime = 0; //get_time(); + + /* The only cases we check for asynchronous task completion is + * when an overlapping CPU operation completes, or the next + * segment blocks on completion of previous async operations */ + if( asyncs_outstanding(timers) && + (!is_async(timers->current) || is_blocking(category) ) ) { + + struct pb_async_time_marker_list * last_event = get_last_async(timers); + /* CL_COMPLETE if completed */ + + cl_int ciErrNum = CL_SUCCESS; + cl_int async_done = CL_COMPLETE; + + ciErrNum = clGetEventInfo(*((cl_event *)last_event->marker), CL_EVENT_COMMAND_EXECUTION_STATUS, sizeof(cl_int), &async_done, NULL); + if (ciErrNum != CL_SUCCESS) { + fprintf(stderr, "Error Querying EventInfo!\n"); + } + + if(is_blocking(category)) { + /* Async operations completed after previous CPU operations: + * overlapped time is the total CPU time since this set of async + * operations were first issued */ + + // timer to switch to is COPY or NONE + // if it hasn't already finished, then just take now and use that as the elapsed time in OVERLAP + // anything happening after now isn't OVERLAP because everything is being stopped to wait for synchronization + // it seems that the extra sync wall time isn't being recorded anywhere + if(async_done != CL_COMPLETE) + accumulate_time(&(timers->timers[pb_TimerID_OVERLAP].elapsed), + timers->async_begin,currentTime); + + /* Wait on async operation completion */ + ciErrNum = clWaitForEvents(1, (cl_event *)last_event->marker); + if (ciErrNum != CL_SUCCESS) { + fprintf(stderr, "Error Waiting for Events!\n"); + } + pb_Timestamp total_async_time = record_async_times(timers); + + /* Async operations completed before previous CPU operations: + * overlapped time is the total async time */ + // If it did finish, then accumulate all the async time that did happen into OVERLAP + // the immediately preceding EventSynchronize theoretically didn't have any effect since it was already completed. + if(async_done == CL_COMPLETE /*cudaSuccess*/) + timers->timers[pb_TimerID_OVERLAP].elapsed += total_async_time; + + } else + /* implies (!is_async(timers->current) && asyncs_outstanding(timers)) */ + // i.e. Current Not Async (not KERNEL/COPY_ASYNC) but there are outstanding + // so something is deeper in stack + if(async_done == CL_COMPLETE /*cudaSuccess*/) { + /* Async operations completed before previous CPU operations: + * overlapped time is the total async time */ + timers->timers[pb_TimerID_OVERLAP].elapsed += record_async_times(timers); + } + // else, this isn't blocking, so just check the next time around + } + + subtimerlist = timers->sub_timer_list[category]; + struct pb_SubTimer *subtimer = NULL; + + if (label != NULL) { + subtimer = subtimerlist->subtimer_list; + while (subtimer != NULL) { + if (strcmp(subtimer->label, label) == 0) { + break; + } else { + subtimer = subtimer->next; + } + } + } + + /* Start the new timer */ + if (category != pb_TimerID_NONE) { + if(!is_async(category)) { + if (subtimerlist != NULL) { + subtimerlist->current = subtimer; + } + + if (category != timers->current && subtimer != NULL) { + pb_StartTimerAndSubTimer(&timers->timers[category], &subtimer->timer); + } else if (subtimer != NULL) { + pb_StartTimer(&subtimer->timer); + } else { + pb_StartTimer(&timers->timers[category]); + } + } else { + if (subtimerlist != NULL) { + subtimerlist->current = subtimer; + } + + // toSwitchTo Is Async (KERNEL/COPY_ASYNC) + if (!asyncs_outstanding(timers)) { + /* No asyncs outstanding, insert a fresh async marker */ + insert_submarker(timers, label, category); + timers->async_begin = currentTime; + } else if(!is_async(timers->current)) { + /* Previous asyncs still in flight, but a previous SwitchTo + * already marked the end of the most recent async operation, + * so we can rename that marker as the beginning of this async + * operation */ + + struct pb_async_time_marker_list * last_event = get_last_async(timers); + last_event->timerID = category; + last_event->label = label; + } // else, marker for switchToThis was already inserted + + //toSwitchto is already asynchronous, but if current/prev state is async too, then DRIVER is already running + if (!is_async(timers->current)) { + pb_StartTimer(&timers->timers[pb_TimerID_DRIVER]); + } + } + } + + timers->current = category; +#endif +} + +void +pb_PrintTimerSet(struct pb_TimerSet *timers) +{ +#ifndef DISABLE_PARBOIL_TIMER + pb_Timestamp wall_end = 0; //get_time(); + + struct pb_Timer *t = timers->timers; + struct pb_SubTimer* sub = NULL; + + int maxSubLength; + + const char *categories[] = { + "IO", "Kernel", "Copy", "Driver", "Copy Async", "Compute" + }; + + const int maxCategoryLength = 10; + + int i; + for(i = 1; i < pb_TimerID_LAST-1; ++i) { // exclude NONE and OVRELAP from this format + if(pb_GetElapsedTime(&t[i]) != 0) { + + // Print Category Timer + printf("%-*s: %f\n", maxCategoryLength, categories[i-1], pb_GetElapsedTime(&t[i])); + + if (timers->sub_timer_list[i] != NULL) { + sub = timers->sub_timer_list[i]->subtimer_list; + maxSubLength = 0; + while (sub != NULL) { + // Find longest SubTimer label + if (strlen(sub->label) > maxSubLength) { + maxSubLength = strlen(sub->label); + } + sub = sub->next; + } + + // Fit to Categories + if (maxSubLength <= maxCategoryLength) { + maxSubLength = maxCategoryLength; + } + + sub = timers->sub_timer_list[i]->subtimer_list; + + // Print SubTimers + while (sub != NULL) { + printf(" -%-*s: %f\n", maxSubLength, sub->label, pb_GetElapsedTime(&sub->timer)); + sub = sub->next; + } + } + } + } + + if(pb_GetElapsedTime(&t[pb_TimerID_OVERLAP]) != 0) + printf("CPU/Kernel Overlap: %f\n", pb_GetElapsedTime(&t[pb_TimerID_OVERLAP])); + + float walltime = (wall_end - timers->wall_begin)/ 1e6; + printf("Timer Wall Time: %f\n", walltime); + +#endif +} + +void pb_DestroyTimerSet(struct pb_TimerSet * timers) +{ +#ifndef DISABLE_PARBOIL_TIMER + /* clean up all of the async event markers */ + struct pb_async_time_marker_list* event = timers->async_markers; + while(event != NULL) { + + cl_int ciErrNum = CL_SUCCESS; + ciErrNum = clWaitForEvents(1, (cl_event *)(event)->marker); + if (ciErrNum != CL_SUCCESS) { + //fprintf(stderr, "Error Waiting for Events!\n"); + } + + ciErrNum = clReleaseEvent( *((cl_event *)(event)->marker) ); + if (ciErrNum != CL_SUCCESS) { + fprintf(stderr, "Error Release Events!\n"); + } + + free((event)->marker); + struct pb_async_time_marker_list* next = ((event)->next); + + free(event); + + // (*event) = NULL; + event = next; + } + + int i = 0; + for(i = 0; i < pb_TimerID_LAST; ++i) { + if (timers->sub_timer_list[i] != NULL) { + struct pb_SubTimer *subtimer = timers->sub_timer_list[i]->subtimer_list; + struct pb_SubTimer *prev = NULL; + while (subtimer != NULL) { + free(subtimer->label); + prev = subtimer; + subtimer = subtimer->next; + free(prev); + } + free(timers->sub_timer_list[i]); + } + } +#endif +} + +static pb_Platform** ptr = NULL; + +// verbosely print out list of platforms and their devices to the console. +pb_Platform** +pb_GetPlatforms() { + if (ptr == NULL) { + cl_uint num_platforms; + clGetPlatformIDs(0, NULL, &num_platforms); + if (num_platforms == 0) return NULL; + + ptr = (pb_Platform **) malloc(sizeof(pb_Platform *) * (num_platforms + 1)); + cl_platform_id* ids = (cl_platform_id *) malloc(num_platforms * sizeof(cl_platform_id)); + clGetPlatformIDs(num_platforms, ids, NULL); + + unsigned int i; + for (i = 0; i < num_platforms; i++) { + ptr[i] = (pb_Platform *) malloc(sizeof(pb_Platform)); + ptr[i]->clPlatform = ids[i]; + ptr[i]->contexts = NULL; + ptr[i]->in_use = 0; + ptr[i]->devices = NULL; + + size_t sz; + clGetPlatformInfo(ids[i], CL_PLATFORM_NAME, 0, NULL, &sz); + char* name = (char *) malloc(sz + 1); + clGetPlatformInfo(ids[i], CL_PLATFORM_NAME, sz, name, NULL); + name[sz] = '\0'; + ptr[i]->name = name; + + clGetPlatformInfo(ids[i], CL_PLATFORM_VERSION, 0, NULL, &sz); + char* version = (char *) malloc(sz + 1); + clGetPlatformInfo(ids[i], CL_PLATFORM_VERSION, sz, version, NULL); + version[sz] = '\0'; + ptr[i]->version = version; + } + ptr[i] = NULL; + + free(ids); + } + + return (pb_Platform**) ptr; +} + +pb_Context* +createContext(pb_Platform* pb_platform, pb_Device* pb_device) { + pb_Context* c = (pb_Context*) malloc(sizeof(pb_Context)); + cl_int clStatus; + cl_context_properties clCps[3] = { + CL_CONTEXT_PLATFORM, (cl_context_properties)(pb_platform->clPlatform), 0 + }; + c->clContext = + clCreateContext(clCps, 1, (cl_device_id*)&pb_device->clDevice, NULL, NULL, &clStatus); + c->clPlatformId = pb_platform->clPlatform; + c->clDeviceId = pb_device->clDevice; + c->pb_platform = pb_platform; + c->pb_device = pb_device; + pb_platform->in_use = 1; + pb_device->in_use = 1; + unsigned int i = 0; + if (pb_platform->contexts == NULL) { + pb_platform->contexts = (pb_Context**) malloc(2*sizeof(pb_Context*)); + } else { + for (i = 0; pb_platform->contexts[i] != NULL; i++) {}; + pb_platform->contexts = (pb_Context**) realloc(pb_platform->contexts, + (i+1)*sizeof(pb_Context*)); + } + pb_platform->contexts[i+1] = NULL; + pb_platform->contexts[i] = c; + return c; +} + +// choose a platform by name. +pb_Platform* +pb_GetPlatformByName(const char* name) { + pb_Platform** ps = (pb_Platform **) pb_GetPlatforms(); + if (ps == NULL) return NULL; + if (name == NULL) { + return *ps; + } + + while (*ps) { + if (strstr((*ps)->name, name)) break; + ps++; + } + return (pb_Platform*) *ps; +} + +pb_Device** +pb_GetDevices(pb_Platform* pb_platform) { + if (pb_platform->devices == NULL) { + cl_uint num_devs; + cl_device_id* dev_ids; + clGetDeviceIDs((cl_platform_id) pb_platform->clPlatform, + CL_DEVICE_TYPE_ALL, 0, NULL, &num_devs); + if (num_devs == 0) return NULL; + + pb_platform->devices = + (pb_Device **) malloc((num_devs + 1) * sizeof(pb_Device *)); + dev_ids = (cl_device_id *) malloc(sizeof(cl_device_id) * num_devs); + clGetDeviceIDs((cl_platform_id) pb_platform->clPlatform, + CL_DEVICE_TYPE_ALL, num_devs, dev_ids, NULL); + + unsigned int i; + for (i = 0; i < num_devs; i++) { + pb_platform->devices[i] = (pb_Device *) malloc(sizeof(pb_Device)); + + pb_platform->devices[i]->clDevice = dev_ids[i]; + pb_platform->devices[i]->id = i; + + size_t sz; + clGetDeviceInfo(dev_ids[i], CL_DEVICE_NAME, 0, NULL, &sz); + char* name = (char *) malloc(sz + 1); + clGetDeviceInfo(dev_ids[i], CL_DEVICE_NAME, sz, name, NULL); + name[sz] = '\0'; + pb_platform->devices[i]->name = (char *) name; + + cl_bool available; + clGetDeviceInfo(dev_ids[i], CL_DEVICE_AVAILABLE, sizeof(cl_bool), &available, NULL); + pb_platform->devices[i]->available = (int) available; + + pb_platform->devices[i]->in_use = 0; + } + pb_platform->devices[i] = NULL; + } + return (pb_Device **) pb_platform->devices; +} + +// choose a device by name. +static pb_Device* +pb_SelectDeviceByName(pb_Device **ds, const char* name) { + if (ds == NULL) return NULL; + if (name == NULL) return *ds; + while (*ds) { + if (strstr((*ds)->name, name)) break; + ds++; + } + + return *ds; +} + +// choose a device by name and set the device's 'in_use' flag. +pb_Device* +pb_GetDeviceByName(pb_Platform* pb_platform, const char* name) { + pb_Device** ds = (pb_Device **) pb_GetDevices(pb_platform); + pb_Device *d = pb_SelectDeviceByName(ds, name); + + if (d) d->in_use = 1; + + return d; +} + +void +pb_ReleasePlatforms() { + if (!ptr) return; + pb_Platform** cur_ptr = ptr; + while (*cur_ptr) { + pb_Platform* pfptr = *cur_ptr++; + if (pfptr->devices) { + pb_Device** dvptr = pfptr->devices; + while (*dvptr) { + pb_Device* d = *dvptr++; + free(d->name); + free(d); + } + free(pfptr->devices); + } + if (pfptr->contexts) { + pb_Context** cptr = pfptr->contexts; + while (*cptr) { + free(*cptr++); + } + free(pfptr->contexts); + } + free(pfptr->name); + free(pfptr); + } + free(ptr); + ptr = NULL; +} + +pb_Platform* +pb_GetPlatformByNameAndVersion(const char* name, const char* version) { + pb_Platform** ps = (pb_Platform **) pb_GetPlatforms(); + if (ps == NULL) return NULL; + if (name == NULL) return *ps; + while (*ps) { + if (strstr((*ps)->name, name) && strstr((*ps)->version, version)) break; + ps++; + } + return (pb_Platform*) *ps; +} + +/* Return a pointer to the device at the specified index, or NULL. + * Used by pb_GetDevice. */ +static pb_Device * +select_device_by_index(pb_Device** ds, int id) +{ + int i = 0; + pb_Device** p = ds; + while (*p && (i < id)) { p++; i++; } + return *p; +} + +/* Return a pointer to the device with the specified type, or NULL. + * Used by pb_GetDevice. */ +static pb_Device * +select_device_by_type(pb_Device** ds, + enum pb_DeviceSelectionCriterion criterion) +{ + cl_device_type sought_type; + + /* Determine the OpenCL device type to search for */ + switch(criterion) { + case pb_Device_CPU: + sought_type = CL_DEVICE_TYPE_CPU; + break; + case pb_Device_GPU: + sought_type = CL_DEVICE_TYPE_GPU; + break; + case pb_Device_ACCELERATOR: + sought_type = CL_DEVICE_TYPE_ACCELERATOR; + break; + default: + fprintf(stderr, "pb_GetDevice: Invalid device type"); + exit(-1); + } + + /* Find the device */ + { + pb_Device** p = ds; + cl_device_type type; + while (*p) { + clGetDeviceInfo(((cl_device_id) ((*p)->clDevice)), CL_DEVICE_TYPE, + sizeof(cl_device_type), &type, NULL); + if (type == sought_type) break; + } + + return *p; + } +} + +pb_Device* +pb_GetDevice(pb_Platform* pb_platform, struct pb_DeviceParam *device) +{ + pb_Device** ds = (pb_Device **) pb_GetDevices(pb_platform); + + // The list of devices must be nonempty + if (ds == NULL || *ds == NULL) { + fprintf(stderr, "Error: No device is found in platform: name = %s, version = %s\n.", pb_platform->name, pb_platform->version); + exit(-1); + } + + pb_Device *selected_device = NULL; + + if (device != NULL) { + /* Use 'device' to select and return a device. + * If unable to select a device, fall + * back on the default selection mechanism. */ + switch(device->criterion) { + case pb_Device_INDEX: + selected_device = select_device_by_index(ds, device->index); + break; + case pb_Device_GPU: + case pb_Device_CPU: + case pb_Device_ACCELERATOR: + selected_device = select_device_by_type(ds, device->criterion); + break; + case pb_Device_NAME: + selected_device = pb_SelectDeviceByName(ds, device->name); + break; + default: + fprintf(stderr, "pb_GetDevice: Invalid argument"); + exit(-1); + } + } + + /* By default or if user-specified selection failed, + * select the first device */ + if (selected_device == NULL) + selected_device = *ds; + + /* Set the in_use flag */ + selected_device->in_use = 1; + + return selected_device; +} + +pb_Device* +pb_GetDeviceByEnvVars(pb_Platform* pb_platform) { + + /* Convert environment variables to a 'pb_DeviceParam' */ + struct pb_DeviceParam *param = NULL; + + char* device_num = getenv("PARBOIL_DEVICE_NUMBER"); + if (device_num && strcmp(device_num, "")) { + int id = atoi(device_num); + param = pb_DeviceParam_index(id); + } + else { + char* device_name = getenv("PARBOIL_DEVICE_NAME"); + if (device_name && strcmp(device_name, "")) { + param = pb_DeviceParam_name(strdup(device_name)); + } + else { + char* device_type = getenv("PARBOIL_DEVICE_TYPE"); + if (device_type && strcmp(device_type, "")) { + if (strcmp(device_type, "CPU") == 0) + param = pb_DeviceParam_cpu(); + else if (strcmp(device_type, "GPU") == 0) + param = pb_DeviceParam_gpu(); + else if (strcmp(device_type, "ACCELERATOR") == 0) + param = pb_DeviceParam_accelerator(); + } + } + } + + /* Get a device */ + pb_Device *d = pb_GetDevice(pb_platform, param); + pb_FreeDeviceParam(param); + + return d; +} + +pb_Platform* +pb_GetPlatformByEnvVars() { + char* name = getenv("PARBOIL_PLATFORM_NAME"); + char* version = getenv("PARBOIL_PLATFORM_VERSION"); + + /* Create a pb_PlatformParam object (or NULL) representing the data from the + * environment variables */ + struct pb_PlatformParam *platform; + + if (name) { + if (version) { + platform = pb_PlatformParam(strdup(name), strdup(version)); + } + else { + platform = pb_PlatformParam(strdup(name), NULL); + } + } + else { + platform = NULL; + } + + /* Convert to a platform */ + pb_Platform *p = pb_GetPlatform(platform); + pb_FreePlatformParam(platform); + + return p; +} + +/* Choose an OpenCL platform based on the given command-line parameters. + * If NULL, use the default OpenCL platform. */ +pb_Platform* +pb_GetPlatform(struct pb_PlatformParam *platform) { + if (platform != NULL) { + /* Try to use command-line parameters to choose platform */ + char *name = platform->name; + char *version = platform->version; + + if (!name) { + fprintf(stderr, "Internal error: NULL pointer"); + exit(-1); + } + + if (version) { + pb_Platform* p = pb_GetPlatformByNameAndVersion(name, version); + if (p) return p; + } + + pb_Platform* p = pb_GetPlatformByName(name); + if (p) return p; + } + + pb_Platform* p = pb_GetPlatformByName(NULL); + if (p == NULL) { + fprintf(stderr, "Error: No OpenCL platform in this system. Exiting."); + exit(-1); + } + return p; +} + +//extern void perf_init(); +//extern void mxpa_scheduler_init(); + +pb_Context* +pb_InitOpenCLContext(struct pb_Parameters* parameters) { +#if 0 + pb_Platform* ps = pb_GetPlatform(parameters->platform); + if (!ps) return NULL; + pb_Device* ds = pb_GetDevice(ps, parameters->device); + if (!ds) return NULL; + + /* HERE INITIALIZE TIMER */ + //perf_init(); + //mxpa_scheduler_init(); + + pb_Context* c = createContext(ps, ds); + pb_PrintPlatformInfo(c); + return c; +#endif + cl_int _err; + cl_platform_id platform_id; + cl_device_id device_id; + cl_context context; + clGetPlatformIDs(1, &platform_id, NULL); + clGetDeviceIDs(platform_id, CL_DEVICE_TYPE_DEFAULT, 1, &device_id, NULL); + context = clCreateContext(NULL, 1, &device_id, NULL, NULL, &_err); + + pb_Context* c = (pb_Context*)malloc(sizeof(pb_Context)); + c->clContext = context; + c->clDeviceId = device_id; + c->clPlatformId = platform_id; + c->pb_platform = (pb_Platform*)malloc(sizeof(pb_Platform)); + c->pb_device = (pb_Device*)malloc(sizeof(pb_Device)); + c->pb_platform->devices = (pb_Device**)malloc(sizeof(pb_Device*) * 2); + c->pb_platform->devices[0] = c->pb_device; + c->pb_platform->devices[1] = NULL; + c->pb_platform->contexts = (pb_Context**)malloc(sizeof(pb_Context*) * 2); + c->pb_platform->contexts[0] = c; + c->pb_platform->contexts[1] = NULL; + c->pb_platform->in_use = 1; + c->pb_device->in_use = 1; + return c; +} + +void +pb_ReleaseOpenCLContext(pb_Context* c) { + pb_ReleasePlatforms(); +} + +void +pb_PrintPlatformInfo(pb_Context* c) { + /*pb_Platform** ps = pb_GetPlatforms(); + if (!ps) { + fprintf (stderr, "No platform found"); + return; + } + + printf ("********************************************************\n"); + printf ("DETECTED OPENCL PLATFORMS AND DEVICES:\n"); + printf ("--------------------------------------------------------\n"); + + while (*ps) { + printf ("PLATFORM = %s, %s", (*ps)->name, (*ps)->version); + if (c->pb_platform == *ps) printf (" (SELECTED)"); + printf ("\n"); + + pb_Device** ds = (pb_Device **) pb_GetDevices((*ps)); + if (ds == NULL) { + printf (" + (No devices)\n"); + } else { + while (*ds) { + printf (" + %d: %s", (*ds)->id, (*ds)->name); + if (c->pb_device == *ds) printf (" (SELECTED)"); + printf ("\n"); + ds++; + } + } + + ps++; + } + printf ("********************************************************\n");*/ +} + +#ifdef MEASURE_KERNEL_TIME + +#undef clEnqueueNDRangeKernel + +//extern void pin_trace_enable(char*); +//extern void pin_trace_disable(char*); + +cl_int +pb_clEnqueueNDRangeKernel(cl_command_queue q/* command_queue */, + cl_kernel k/* kernel */, + cl_uint d/* work_dim */, + const size_t * o/* global_work_offset */, + const size_t * gws/* global_work_size */, + const size_t * lws/* local_work_size */, + cl_uint n/* num_events_in_wait_list */, + const cl_event * w/* event_wait_list */, + cl_event * e/* event */) { + + char buf[128]; + struct timeval begin, end; + clGetKernelInfo(k, CL_KERNEL_FUNCTION_NAME, 128, buf, NULL); + +#if 0 + int i; + for (i = 0; i < d; i++) { + printf ("%s: %d: %d / %d\n", buf, i, gws[i], (lws == NULL ? 0 : lws[i])); + } +#endif + + clFinish(q); clFlush(q); + //pin_trace_enable(buf); + //gettimeofday(&begin, NULL); + cl_int result = clEnqueueNDRangeKernel(q, k, d, o, gws, lws, n, w, e); + clFinish(q); clFlush(q); + //gettimeofday(&end, NULL); + //pin_trace_disable(buf); + //float t = (float)(end.tv_sec - begin.tv_sec) + (end.tv_usec - begin.tv_usec) / 1000000.0f; + fflush(stdout); + fflush(stderr); + //printf ("PBTIMER: %s: %f\n", buf, t); + return result; +} + +#endif + +void +pb_sig_float(char* c, float* p, int sz) { + int i; + double s = 0.0; + for (i = 0; i < sz; i++) s += p[i] * (float)(i+1); + printf ("[Signature] %s = %lf\n", c, s); +} + +void +pb_sig_double(char* c, double* p, int sz) { + int i; + double s = 0.0; + for (i = 0; i < sz; i++) s += p[i]; + printf ("[Signature] %s = %lf\n", c, s); +} + +void +pb_sig_short(char* c, short* p, int sz) { + int i; + long long int s = 0; + for (i = 0; i < sz; i++) s += p[i]; + printf ("[Signature] %s = %lld\n", c, s); +} + +void +pb_sig_int(char* c, int* p, int sz) { + int i; + long long int s = 0; + for (i = 0; i < sz; i++) s += p[i]; + printf ("[Signature] %s = %lld\n", c, s); +} + +void +pb_sig_uchar(char* c, unsigned char* p, unsigned int sz) { + int i; + unsigned long long int s = 0; + for (i = 0; i < sz; i++) s += p[i]; + printf ("[Signature] %s = %lld\n", c, s); +} + +void pb_sig_clmem(char* s, cl_command_queue command_queue, cl_mem memobj, int ty) { + size_t sz; + if (clGetMemObjectInfo(memobj, CL_MEM_SIZE, sizeof(size_t), &sz, NULL) != CL_SUCCESS) { + printf ("Something wrong.\n"); + assert(0); + } else { + printf ("size = %d\n", sz); + } + char* hp; // = (char*) malloc(sz); + //posix_memalign((void**)&hp, 64, sz); + hp = (char*)malloc(sz); + + clEnqueueReadBuffer (command_queue, + memobj, + CL_TRUE, + 0, + sz, + hp, + 0, + NULL, + NULL); + + if (ty == T_FLOAT) pb_sig_float(s, (float*)hp, sz/sizeof(float)); + if (ty == T_DOUBLE) pb_sig_double(s, (double*)hp, sz/sizeof(double)); + if (ty == T_INT) pb_sig_int(s, (int*)hp, sz/sizeof(int)); + if (ty == T_SHORT) pb_sig_short(s, (short*)hp, sz/sizeof(short)); + if (ty == T_UCHAR) pb_sig_uchar(s, (unsigned char*)hp, sz/sizeof(char)); + + free(hp); +} + diff --git a/benchmarks/opencl/spmv/perf_util.c b/benchmarks/opencl/spmv/perf_util.c new file mode 100644 index 00000000..d3e5f64b --- /dev/null +++ b/benchmarks/opencl/spmv/perf_util.c @@ -0,0 +1,670 @@ +/* + * perf_util.c - helper functions for perf_events + * + * Copyright (c) 2009 Google, Inc + * Contributed by Stephane Eranian + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A + * PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE + * OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +#include +#include +#include +#include +#include +#include +#include + +#include +#include "perf_util.h" + +/* the **fd parameter must point to a null pointer on the first call + * max_fds and num_fds must both point to a zero value on the first call + * The return value is success (0) vs. failure (non-zero) + */ +int +perf_setup_argv_events(const char **argv, perf_event_desc_t **fds, int *num_fds) +{ + perf_event_desc_t *fd; + pfm_perf_encode_arg_t arg; + int new_max, ret, num, max_fds; + int group_leader; + + if (!(argv && fds && num_fds)) + return -1; + + fd = *fds; + if (fd) { + max_fds = fd[0].max_fds; + if (max_fds < 2) + return -1; + num = *num_fds; + } else { + max_fds = num = 0; /* bootstrap */ + } + group_leader = num; + + while(*argv) { + if (num == max_fds) { + if (max_fds == 0) + new_max = 2; + else + new_max = max_fds << 1; + + if (new_max < max_fds) { + warn("too many entries"); + goto error; + } + fd = realloc(fd, new_max * sizeof(*fd)); + if (!fd) { + warn("cannot allocate memory"); + goto error; + } + /* reset newly allocated chunk */ + memset(fd + max_fds, 0, (new_max - max_fds) * sizeof(*fd)); + max_fds = new_max; + + /* update max size */ + fd[0].max_fds = max_fds; + } + /* ABI compatibility, set before calling libpfm */ + fd[num].hw.size = sizeof(fd[num].hw); + + memset(&arg, 0, sizeof(arg)); + arg.attr = &fd[num].hw; + arg.fstr = &fd[num].fstr; /* fd[].fstr is NULL */ + + ret = pfm_get_os_event_encoding(*argv, PFM_PLM0|PFM_PLM3, PFM_OS_PERF_EVENT_EXT, &arg); + if (ret != PFM_SUCCESS) { + warnx("event %s: %s", *argv, pfm_strerror(ret)); + goto error; + } + + fd[num].name = strdup(*argv); + fd[num].group_leader = group_leader; + fd[num].idx = arg.idx; + + num++; + argv++; + } + *num_fds = num; + *fds = fd; + return 0; +error: + perf_free_fds(fd, num); + return -1; +} + +int +perf_setup_list_events(const char *ev, perf_event_desc_t **fd, int *num_fds) +{ + const char **argv; + char *p, *q, *events; + int i, ret, num = 0; + + if (!(ev && fd && num_fds)) + return -1; + + events = strdup(ev); + if (!events) + return -1; + + q = events; + while((p = strchr(q, ','))) { + num++; + q = p + 1; + } + num++; + num++; /* terminator */ + + argv = malloc(num * sizeof(char *)); + if (!argv) { + free(events); + return -1; + } + + for(i=0, q = events; i < num-2; i++, q = p + 1) { + p = strchr(q, ','); + *p = '\0'; + argv[i] = q; + } + argv[i++] = q; + argv[i] = NULL; + ret = perf_setup_argv_events(argv, fd, num_fds); + free(argv); + free(events); /* strdup in perf_setup_argv_events() */ + return ret; +} + +void +perf_free_fds(perf_event_desc_t *fds, int num_fds) +{ + int i; + + for (i = 0 ; i < num_fds; i++) { + free(fds[i].name); + free(fds[i].fstr); + } + free(fds); +} + +int +perf_get_group_nevents(perf_event_desc_t *fds, int num, int idx) +{ + int leader; + int i; + + if (idx < 0 || idx >= num) + return 0; + + leader = fds[idx].group_leader; + + for (i = leader + 1; i < num; i++) { + if (fds[i].group_leader != leader) { + /* This is a new group leader, so the previous + * event was the final event of the preceding + * group. + */ + return i - leader; + } + } + return i - leader; +} + +int +perf_read_buffer(perf_event_desc_t *hw, void *buf, size_t sz) +{ + struct perf_event_mmap_page *hdr = hw->buf; + size_t pgmsk = hw->pgmsk; + void *data; + unsigned long tail; + size_t avail_sz, m, c; + + /* + * data points to beginning of buffer payload + */ + data = ((void *)hdr)+sysconf(_SC_PAGESIZE); + + /* + * position of tail within the buffer payload + */ + tail = hdr->data_tail & pgmsk; + + /* + * size of what is available + * + * data_head, data_tail never wrap around + */ + avail_sz = hdr->data_head - hdr->data_tail; + if (sz > avail_sz) + return -1; + + /* + * sz <= avail_sz, we can satisfy the request + */ + + /* + * c = size till end of buffer + * + * buffer payload size is necessarily + * a power of two, so we can do: + */ + c = pgmsk + 1 - tail; + + /* + * min with requested size + */ + m = c < sz ? c : sz; + + /* copy beginning */ + memcpy(buf, data+tail, m); + + /* + * copy wrapped around leftover + */ + if ((sz - m) > 0) + memcpy(buf+m, data, sz - m); + + //printf("\nhead=%lx tail=%lx new_tail=%lx sz=%zu\n", hdr->data_head, hdr->data_tail, hdr->data_tail+sz, sz); + hdr->data_tail += sz; + + return 0; +} + +void +perf_skip_buffer(perf_event_desc_t *hw, size_t sz) +{ + struct perf_event_mmap_page *hdr = hw->buf; + + if ((hdr->data_tail + sz) > hdr->data_head) + sz = hdr->data_head - hdr->data_tail; + + hdr->data_tail += sz; +} + +static size_t +__perf_handle_raw(perf_event_desc_t *hw) +{ + size_t sz = 0; + uint32_t raw_sz, i; + char *buf; + int ret; + + ret = perf_read_buffer_32(hw, &raw_sz); + if (ret) { + warnx("cannot read raw size"); + return -1; + } + + sz += sizeof(raw_sz); + + printf("\n\tRAWSZ:%u\n", raw_sz); + + buf = malloc(raw_sz); + if (!buf) { + warn("cannot allocate raw buffer"); + return -1; + } + + + ret = perf_read_buffer(hw, buf, raw_sz); + if (ret) { + warnx("cannot read raw data"); + free(buf); + return -1; + } + + if (raw_sz) + putchar('\t'); + + for(i=0; i < raw_sz; i++) { + printf("0x%02x ", buf[i] & 0xff ); + if (((i+1) % 16) == 0) + printf("\n\t"); + } + if (raw_sz) + putchar('\n'); + + free(buf); + + return sz + raw_sz; +} + + +int +perf_display_sample(perf_event_desc_t *fds, int num_fds, int idx, struct perf_event_header *ehdr, FILE *fp) +{ + perf_event_desc_t *hw; + struct { uint32_t pid, tid; } pid; + struct { uint64_t value, id; } grp; + uint64_t time_enabled, time_running; + size_t sz; + uint64_t type, fmt; + uint64_t val64; + const char *str; + int ret, e; + + if (!fds || !fp || !ehdr || num_fds < 0 || idx < 0 || idx >= num_fds) + return -1; + + sz = ehdr->size - sizeof(*ehdr); + + hw = fds+idx; + + type = hw->hw.sample_type; + fmt = hw->hw.read_format; + + /* + * the sample_type information is laid down + * based on the PERF_RECORD_SAMPLE format specified + * in the perf_event.h header file. + * That order is different from the enum perf_event_sample_format + */ + if (type & PERF_SAMPLE_IP) { + const char *xtra = " "; + ret = perf_read_buffer_64(hw, &val64); + if (ret) { + warnx("cannot read IP"); + return -1; + } + + /* + * MISC_EXACT_IP indicates that kernel is returning + * th IIP of an instruction which caused the event, i.e., + * no skid + */ + if (hw->hw.precise_ip && (ehdr->misc & PERF_RECORD_MISC_EXACT_IP)) + xtra = " (exact) "; + + fprintf(fp, "IIP:%#016"PRIx64"%s", val64, xtra); + sz -= sizeof(val64); + } + + if (type & PERF_SAMPLE_TID) { + ret = perf_read_buffer(hw, &pid, sizeof(pid)); + if (ret) { + warnx( "cannot read PID"); + return -1; + } + + fprintf(fp, "PID:%d TID:%d ", pid.pid, pid.tid); + sz -= sizeof(pid); + } + + if (type & PERF_SAMPLE_TIME) { + ret = perf_read_buffer_64(hw, &val64); + if (ret) { + warnx( "cannot read time"); + return -1; + } + + fprintf(fp, "TIME:%'"PRIu64" ", val64); + sz -= sizeof(val64); + } + + if (type & PERF_SAMPLE_ADDR) { + ret = perf_read_buffer_64(hw, &val64); + if (ret) { + warnx( "cannot read addr"); + return -1; + } + + fprintf(fp, "ADDR:%#016"PRIx64" ", val64); + sz -= sizeof(val64); + } + + if (type & PERF_SAMPLE_ID) { + ret = perf_read_buffer_64(hw, &val64); + if (ret) { + warnx( "cannot read id"); + return -1; + } + + fprintf(fp, "ID:%"PRIu64" ", val64); + sz -= sizeof(val64); + } + + if (type & PERF_SAMPLE_STREAM_ID) { + ret = perf_read_buffer_64(hw, &val64); + if (ret) { + warnx( "cannot read stream_id"); + return -1; + } + fprintf(fp, "STREAM_ID:%"PRIu64" ", val64); + sz -= sizeof(val64); + } + + if (type & PERF_SAMPLE_CPU) { + struct { uint32_t cpu, reserved; } cpu; + ret = perf_read_buffer(hw, &cpu, sizeof(cpu)); + if (ret) { + warnx( "cannot read cpu"); + return -1; + } + fprintf(fp, "CPU:%u ", cpu.cpu); + sz -= sizeof(cpu); + } + + if (type & PERF_SAMPLE_PERIOD) { + ret = perf_read_buffer_64(hw, &val64); + if (ret) { + warnx( "cannot read period"); + return -1; + } + fprintf(fp, "PERIOD:%'"PRIu64" ", val64); + sz -= sizeof(val64); + } + + /* struct read_format { + * { u64 value; + * { u64 time_enabled; } && PERF_FORMAT_ENABLED + * { u64 time_running; } && PERF_FORMAT_RUNNING + * { u64 id; } && PERF_FORMAT_ID + * } && !PERF_FORMAT_GROUP + * + * { u64 nr; + * { u64 time_enabled; } && PERF_FORMAT_ENABLED + * { u64 time_running; } && PERF_FORMAT_RUNNING + * { u64 value; + * { u64 id; } && PERF_FORMAT_ID + * } cntr[nr]; + * } && PERF_FORMAT_GROUP + * }; + */ + if (type & PERF_SAMPLE_READ) { + uint64_t values[3]; + uint64_t nr; + + if (fmt & PERF_FORMAT_GROUP) { + ret = perf_read_buffer_64(hw, &nr); + if (ret) { + warnx( "cannot read nr"); + return -1; + } + + sz -= sizeof(nr); + + time_enabled = time_running = 1; + + if (fmt & PERF_FORMAT_TOTAL_TIME_ENABLED) { + ret = perf_read_buffer_64(hw, &time_enabled); + if (ret) { + warnx( "cannot read timing info"); + return -1; + } + sz -= sizeof(time_enabled); + } + + if (fmt & PERF_FORMAT_TOTAL_TIME_RUNNING) { + ret = perf_read_buffer_64(hw, &time_running); + if (ret) { + warnx( "cannot read timing info"); + return -1; + } + sz -= sizeof(time_running); + } + + fprintf(fp, "ENA=%'"PRIu64" RUN=%'"PRIu64" NR=%"PRIu64"\n", time_enabled, time_running, nr); + + values[1] = time_enabled; + values[2] = time_running; + while(nr--) { + grp.id = -1; + ret = perf_read_buffer_64(hw, &grp.value); + if (ret) { + warnx( "cannot read group value"); + return -1; + } + sz -= sizeof(grp.value); + + if (fmt & PERF_FORMAT_ID) { + ret = perf_read_buffer_64(hw, &grp.id); + if (ret) { + warnx( "cannot read leader id"); + return -1; + } + sz -= sizeof(grp.id); + } + + e = perf_id2event(fds, num_fds, grp.id); + if (e == -1) + str = "unknown sample event"; + else + str = fds[e].name; + + values[0] = grp.value; + grp.value = perf_scale(values); + + fprintf(fp, "\t%'"PRIu64" %s (%"PRIu64"%s)\n", + grp.value, str, + grp.id, + time_running != time_enabled ? ", scaled":""); + + } + } else { + /* + * this program does not use FORMAT_GROUP when there is only one event + */ + ret = perf_read_buffer_64(hw, &val64); + if (ret) { + warnx( "cannot read value"); + return -1; + } + sz -= sizeof(val64); + + if (fmt & PERF_FORMAT_TOTAL_TIME_ENABLED) { + ret = perf_read_buffer_64(hw, &time_enabled); + if (ret) { + warnx( "cannot read timing info"); + return -1; + } + sz -= sizeof(time_enabled); + } + + if (fmt & PERF_FORMAT_TOTAL_TIME_RUNNING) { + ret = perf_read_buffer_64(hw, &time_running); + if (ret) { + warnx( "cannot read timing info"); + return -1; + } + sz -= sizeof(time_running); + } + if (fmt & PERF_FORMAT_ID) { + ret = perf_read_buffer_64(hw, &val64); + if (ret) { + warnx( "cannot read leader id"); + return -1; + } + sz -= sizeof(val64); + } + + fprintf(fp, "ENA=%'"PRIu64" RUN=%'"PRIu64"\n", time_enabled, time_running); + + values[0] = val64; + values[1] = time_enabled; + values[2] = time_running; + val64 = perf_scale(values); + + fprintf(fp, "\t%'"PRIu64" %s %s\n", + val64, fds[0].name, + time_running != time_enabled ? ", scaled":""); + } + } + + if (type & PERF_SAMPLE_CALLCHAIN) { + uint64_t nr, ip; + + ret = perf_read_buffer_64(hw, &nr); + if (ret) { + warnx( "cannot read callchain nr"); + return -1; + } + sz -= sizeof(nr); + + while(nr--) { + ret = perf_read_buffer_64(hw, &ip); + if (ret) { + warnx( "cannot read ip"); + return -1; + } + + sz -= sizeof(ip); + + fprintf(fp, "\t0x%"PRIx64"\n", ip); + } + } + + if (type & PERF_SAMPLE_RAW) { + ret = __perf_handle_raw(hw); + if (ret == -1) + return -1; + sz -= ret; + } + + /* + * if we have some data left, it is because there is more + * than what we know about. In fact, it is more complicated + * because we may have the right size but wrong layout. But + * that's the best we can do. + */ + if (sz) { + warnx("did not correctly parse sample leftover=%zu", sz); + perf_skip_buffer(hw, sz); + } + + fputc('\n',fp); + return 0; +} + +uint64_t +display_lost(perf_event_desc_t *hw, perf_event_desc_t *fds, int num_fds, FILE *fp) +{ + struct { uint64_t id, lost; } lost; + const char *str; + int e, ret; + + ret = perf_read_buffer(hw, &lost, sizeof(lost)); + if (ret) { + warnx("cannot read lost info"); + return 0; + } + + e = perf_id2event(fds, num_fds, lost.id); + if (e == -1) + str = "unknown lost event"; + else + str = fds[e].name; + + fprintf(fp, "<<>>\n", + lost.lost, + str); + + return lost.lost; +} + +void +display_exit(perf_event_desc_t *hw, FILE *fp) +{ + struct { pid_t pid, ppid, tid, ptid; } grp; + int ret; + + ret = perf_read_buffer(hw, &grp, sizeof(grp)); + if (ret) { + warnx("cannot read exit info"); + return; + } + + fprintf(fp,"[%d] exited\n", grp.pid); +} + +void +display_freq(int mode, perf_event_desc_t *hw, FILE *fp) +{ + struct { uint64_t time, id, stream_id; } thr; + int ret; + + ret = perf_read_buffer(hw, &thr, sizeof(thr)); + if (ret) { + warnx("cannot read throttling info"); + return; + } + + fprintf(fp, "%s value=%"PRIu64" event ID=%"PRIu64"\n", + mode ? "Throttled" : "Unthrottled", + thr.id, + thr.stream_id); +} diff --git a/benchmarks/opencl/spmv/perf_util.h b/benchmarks/opencl/spmv/perf_util.h new file mode 100644 index 00000000..6f8f6286 --- /dev/null +++ b/benchmarks/opencl/spmv/perf_util.h @@ -0,0 +1,184 @@ +/* + * perf_util.h - helper functions for perf_events + * + * Copyright (c) 2009 Google, Inc + * Contributed by Stephane Eranian + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A + * PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE + * OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +#ifndef __PERF_UTIL_H__ +#define __PERF_UTIL_H__ + +#include +#include +//#include +//#include + +typedef struct { + //struct perf_event_attr hw; + uint64_t values[3]; + uint64_t prev_values[3]; + char *name; + uint64_t id; /* event id kernel */ + void *buf; + size_t pgmsk; + int group_leader; + int fd; + int max_fds; + int idx; /* opaque libpfm event identifier */ + char *fstr; /* fstr from library, must be freed */ +} perf_event_desc_t; + +/* handy shortcut */ +#define PERF_FORMAT_SCALE (PERF_FORMAT_TOTAL_TIME_ENABLED|PERF_FORMAT_TOTAL_TIME_RUNNING) + +#ifdef __cplusplus +extern "C" { +#endif +extern int perf_setup_argv_events(const char **argv, perf_event_desc_t **fd, int *num_fds); +extern int perf_setup_list_events(const char *events, perf_event_desc_t **fd, int *num_fds); +extern int perf_read_buffer(perf_event_desc_t *hw, void *buf, size_t sz); +extern void perf_free_fds(perf_event_desc_t *fds, int num_fds); +extern void perf_skip_buffer(perf_event_desc_t *hw, size_t sz); + +extern int perf_get_group_nevents(perf_event_desc_t *fds, int num, int leader); +extern int perf_display_sample(perf_event_desc_t *fds, int num_fds, int idx, struct perf_event_header *ehdr, FILE *fp); +extern uint64_t display_lost(perf_event_desc_t *hw, perf_event_desc_t *fds, int num_fds, FILE *fp); +extern void display_exit(perf_event_desc_t *hw, FILE *fp); +extern void display_freq(int mode, perf_event_desc_t *hw, FILE *fp); +#ifdef __cplusplus +}; +#endif + +static inline int +perf_read_buffer_32(perf_event_desc_t *hw, void *buf) +{ + return perf_read_buffer(hw, buf, sizeof(uint32_t)); +} + +static inline int +perf_read_buffer_64(perf_event_desc_t *hw, void *buf) +{ + return perf_read_buffer(hw, buf, sizeof(uint64_t)); +} + +/* + * values[0] = raw count + * values[1] = TIME_ENABLED + * values[2] = TIME_RUNNING + */ +static inline uint64_t +perf_scale(uint64_t *values) +{ + uint64_t res = 0; + + if (!values[2] && !values[1] && values[0]) { + //warnx("WARNING: time_running = 0 = time_enabled, raw count not zero\n"); + } + + if (values[2] > values[1]) { + //warnx("WARNING: time_running > time_enabled: %llu vs. %llu\n", values[2], values[1]); + } + + if (values[2]) + res = (uint64_t)((double)values[0] * values[1]/values[2]); + return res; +} + +static inline uint64_t +perf_scale_valid(uint64_t *values, int* valid) +{ + uint64_t res = 0; + + if (!values[2] && !values[1] && values[0]) { + //warnx("WARNING: time_running = 0 = time_enabled, raw count not zero\n"); + *valid = 0; + } + + if (values[2] > values[1]) { + //warnx("WARNING: time_running > time_enabled: %llu vs. %llu\n", values[2], values[1]); + *valid = 0; + } else { + *valid = 1; + } + + if (values[2]) + res = (uint64_t)((double)values[0] * values[1]/values[2]); + return res; +} + +static inline uint64_t +perf_scale_delta(uint64_t *values, uint64_t *prev_values) +{ + uint64_t res = 0; + + if (!values[2] && !values[1] && values[0]) + //warnx("WARNING: time_running = 0 = time_enabled, raw count not zero\n"); + + if (values[2] > values[1]) + //warnx("WARNING: time_running > time_enabled\n"); + + if (values[2] - prev_values[2]) + res = (uint64_t)((double)((values[0] - prev_values[0]) * (values[1] - prev_values[1])/ (values[2] - prev_values[2]))); + return res; +} + + +/* + * TIME_RUNNING/TIME_ENABLED + */ +static inline double +perf_scale_ratio(uint64_t *values) +{ + if (!values[1]) + return 0.0; + + return values[2]*1.0/values[1]; +} + +static inline int +perf_fd2event(perf_event_desc_t *fds, int num_events, int fd) +{ + int i; + + for(i=0; i < num_events; i++) + if (fds[i].fd == fd) + return i; + return -1; +} + +/* + * id = PERF_FORMAT_ID + */ +static inline int +perf_id2event(perf_event_desc_t *fds, int num_events, uint64_t id) +{ + int j; + for(j=0; j < num_events; j++) + if (fds[j].id == id) + return j; + return -1; +} + +static inline int +perf_is_group_leader(perf_event_desc_t *fds, int idx) +{ + return fds[idx].group_leader == idx; +} + +#endif diff --git a/benchmarks/opencl/spmv/perfmon.c b/benchmarks/opencl/spmv/perfmon.c new file mode 100644 index 00000000..4404861a --- /dev/null +++ b/benchmarks/opencl/spmv/perfmon.c @@ -0,0 +1,281 @@ +#define __STDC_FORMAT_MACROS +#include + +#include +#include +#include +#include +#include +#include +#include +#include "perfmon.h" + +static const char* mxpa_profile_log = "mxpa_profile_%d.log"; +static int enabled = 0; + +// quartet's CPU supports these. Check check_events at libpfm4 +// to make this adaptable. +static const char *gen_events_all[]={ + "snb_ep::L3_LAT_CACHE:MISS", + "snb_ep::L3_LAT_CACHE:REFERENCE", + + "snb_ep::L2_RQSTS:ALL_DEMAND_DATA_RD", + "snb_ep::L2_RQSTS:ALL_DEMAND_RD_HIT", + + "perf::PERF_COUNT_HW_CACHE_L1D:ACCESS", + "perf::PERF_COUNT_HW_CACHE_L1D:MISS", + + "perf::PERF_COUNT_HW_CACHE_L1D:PREFETCH", + "perf::L1-DCACHE-PREFETCH-MISSES", + + "perf::PERF_COUNT_HW_CACHE_L1I:READ", + "perf::PERF_COUNT_HW_CACHE_L1I:MISS", + + "perf::ITLB-LOADS", + "perf::ITLB-LOAD-MISSES", + "perf::DTLB-LOADS", + "perf::DTLB-LOAD-MISSES", + "perf::CONTEXT-SWITCHES", + "perf::CPU-MIGRATIONS", + "perf::CYCLES", + "snb_ep::RESOURCE_STALLS:ANY", + + "perf::INSTRUCTIONS", + "perf::BRANCH-INSTRUCTIONS", + "perf::BRANCHES", + "perf::BRANCH-MISSES", + NULL +}; + +#define NUM_MAX_THREAD 256 + +static perf_event_desc_t *g_fds[NUM_MAX_THREAD]; +static int g_nthreads; +static int num_fds = 0; + +/* note: unsafe for multithreading */ +static uint64_t* begins; + +static void +fetch_counts(perf_event_desc_t *fds, int num_fds) +{ + if (begins == 0) { + begins = (uint64_t*) malloc(num_fds * sizeof(uint64_t)); + memset(begins, 0, num_fds * sizeof(uint64_t)); + } + + uint64_t val; + uint64_t values[3]; + double ratio; + int i; + ssize_t ret; + + /* + * now read the results. We use pfp_event_count because + * libpfm guarantees that counters for the events always + * come first. + */ + memset(values, 0, sizeof(values)); + + for (i = 0; i < num_fds; i++) { + ret = read(fds[i].fd, values, sizeof(values)); + if (ret < (ssize_t)sizeof(values)) { + if (ret == -1) + fprintf(stderr, "cannot read results: %s", strerror(errno)); + else + warnx("could not read event%d", i); + } + /* + * scaling is systematic because we may be sharing the PMU and + * thus may be multiplexed + */ + int valid = 0; + val = perf_scale_valid(values, &valid); + if (valid == 0) printf ("@i=%d, v0=%llu, v1=%llu, v2=%llu, val=%llu\n", i, values[0], values[1], values[2], val); + ratio = perf_scale_ratio(values); + + begins[i] = val; + } +} + +static void +print_counts(perf_event_desc_t *fds, int num_fds, const char *msg, FILE* fp) +{ + uint64_t val; + uint64_t values[3]; + double ratio; + int i; + ssize_t ret; + +#if 0 + fprintf(fp, "%s ------------------------------------\n", msg); +#else + fprintf(fp, "method=%s", msg); +#endif + + /* + * now read the results. We use pfp_event_count because + * libpfm guarantees that counters for the events always + * come first. + */ + memset(values, 0, sizeof(values)); + + for (i = 0; i < num_fds; i++) { + + ret = read(fds[i].fd, values, sizeof(values)); + if (ret < (ssize_t)sizeof(values)) { + if (ret == -1) + fprintf(stderr, "cannot read results: %s", strerror(errno)); + else + warnx("could not read event%d", i); + } + /* + * scaling is systematic because we may be sharing the PMU and + * thus may be multiplexed + */ + int valid; + val = perf_scale_valid(values, &valid); + if (valid == 0) printf ("!i=%d, v0=%llu, v1=%llu, v2=%llu, val=%llu\n", i, values[0], values[1], values[2], val); + ratio = perf_scale_ratio(values); + +#if 0 + fprintf(fp, "%s %'20"PRIu64" %s (%.2f%% scaling, raw=%'"PRIu64", ena=%'"PRIu64", run=%'"PRIu64")\n", + "-", // msg, + val, + fds[i].name, + (1.0-ratio)*100.0, + values[0], + values[1], + values[2]); +#else + fprintf (fp, " %s=%llu", fds[i].name, val); // valid ? val : 0); +#endif + } + fprintf (fp, "\n"); +} + +FILE* open_log_file(char* fname) { + FILE* fp; + if (fp = fopen(fname, "r")) { + fclose(fp); + return fopen(fname, "a"); + } + fp = fopen(fname, "a"); + return fp; +} + +void perf_init() { + static int init = 0; + + if (init) return; + init = 1; + + char* prof_envvar = getenv("MXPA_PROFILE"); + if (prof_envvar) { + enabled = 1; + } else { + return; + } + + pfm_initialize(); +} + +static void get_tids(int* tids, int* number) { + char path[32]; + int pid = getpid(); + sprintf (path, "/proc/%d/task", pid); + struct dirent *de=NULL; + DIR *d=NULL; + d=opendir(path); + assert(d != NULL && "Null for opendir"); + // Loop while not NULL + char pid_str[8]; + char last[8]; + sprintf (pid_str, "%d", pid); + int n = 0; + while(de = readdir(d)) { + if (!strcmp(de->d_name, ".")) continue; + if (!strcmp(de->d_name, "..")) continue; + if (!strcmp(de->d_name, pid_str)) continue; + *tids++ = atoi(de->d_name); + n++; + } + *number = n; + // printf ("Sampling thread %d\n", tid); + closedir(d); +} + +void perf_start(const char* kname) { + if (!enabled) return; + + char* prof_envvar = getenv("MXPA_PROFILE"); + + int tids[32]; + int ntid; + get_tids(tids, &ntid); + g_nthreads = ntid; + + int n; + for (n = 0; n < ntid; n++) { + int ret; + ret = perf_setup_list_events(prof_envvar, &(g_fds[n]), &num_fds); + perf_event_desc_t *fds = g_fds[n]; + int cpu = -1; + int group_fd = -1; + int pid = tids[n]; + fds[0].fd = -1; + int i; + for(i=0; i < num_fds; i++) { + fds[i].hw.read_format = PERF_FORMAT_SCALE; + fds[i].hw.disabled = 1; /* do not start now */ + fds[i].hw.inherit = 1; /* XXX child process will inherit, when forked only? */ + + /* each event is in an independent group (multiplexing likely) */ + fds[i].fd = perf_event_open(&fds[i].hw, pid, cpu, group_fd, 0); + if (fds[i].fd == -1) { + fprintf(stderr, "cannot open event %d\n", i); + exit(2); + } + } + } + prctl(PR_TASK_PERF_EVENTS_ENABLE); +} + +void perf_end(const char* kname) { + if (!enabled) return; + int i, n; + prctl(PR_TASK_PERF_EVENTS_DISABLE); + + static int first_time = 1; + if (first_time) { + first_time = 0; + char name[128]; + for (n = 0; n < g_nthreads; n++) { + sprintf (name, mxpa_profile_log, n); + FILE* fp = fopen(name, "w"); + fclose(fp); + } + } + + char name[128]; + for (n = 0; n < g_nthreads; n++) { + sprintf (name, mxpa_profile_log, n); + FILE* fp = open_log_file(name); + perf_event_desc_t *fds = g_fds[n]; + print_counts(fds, num_fds, kname, fp); + for (i = 0; i < num_fds; i++) close(fds[i].fd); + perf_free_fds(fds, num_fds); + g_fds[n] = fds = NULL; + fclose(fp); + } +} + +void pin_trace_enable(char* n) { + perf_start((const char*)n); +} + +void pin_trace_disable(char* n) { + perf_end((const char*)n); +} + + diff --git a/benchmarks/opencl/spmv/perfmon.h b/benchmarks/opencl/spmv/perfmon.h new file mode 100644 index 00000000..a9fef3c2 --- /dev/null +++ b/benchmarks/opencl/spmv/perfmon.h @@ -0,0 +1,20 @@ +#ifndef MXPA_RUNTIME_PERF_MONITOR +#define MXPA_RUNTIME_PERF_MONITOR + +#include "perf_util.h" + +#ifdef __cplusplus +extern "C" { +#endif + +void perf_init(); +void perf_start(const char* kname); +void perf_end(const char* kname); +void perf_fini(); + +#ifdef __cplusplus +}; +#endif + +#endif + diff --git a/benchmarks/opencl/spmv/stub.cc b/benchmarks/opencl/spmv/stub.cc new file mode 100644 index 00000000..50ebfd5e --- /dev/null +++ b/benchmarks/opencl/spmv/stub.cc @@ -0,0 +1,46 @@ +#ifdef __MXPA__ + +#include +#include +#include +#include +#include + +namespace { +tbb::task_scheduler_init init; + +class Foo { +public: + Foo() {} + void operator() (const tbb::blocked_range& r) const { + for (size_t i = r.begin(); i != r.end(); i++) { + printf (""); + } + } +}; + +} + +extern "C" +void +mxpa_scheduler_init() { + tbb::parallel_for(tbb::blocked_range(0, 100), Foo()); +#if 0 + char cmd[32]; + int pid = getpid(); + printf("----------\n"); + sprintf(cmd, "ls -1 /proc/%d/task", pid); + system(cmd); + printf("----------\n"); +#endif +} + +#else + +extern "C" +void +mxpa_scheduler_init() { +} + +#endif + diff --git a/benchmarks/opencl/spmv/vector.bin b/benchmarks/opencl/spmv/vector.bin new file mode 100644 index 00000000..94aa7258 Binary files /dev/null and b/benchmarks/opencl/spmv/vector.bin differ