File size: 176,471 Bytes
2015d6b |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 1674 1675 1676 1677 1678 1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766 1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786 1787 1788 1789 1790 1791 1792 1793 1794 1795 1796 1797 1798 1799 1800 1801 1802 1803 1804 1805 1806 1807 1808 1809 1810 1811 1812 1813 1814 1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878 1879 1880 1881 1882 1883 1884 1885 1886 1887 1888 1889 1890 1891 1892 1893 1894 1895 1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 1938 1939 1940 1941 1942 1943 1944 1945 1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971 1972 1973 1974 1975 1976 1977 1978 1979 1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020 2021 2022 2023 2024 2025 2026 2027 2028 2029 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 2040 2041 2042 2043 2044 2045 2046 2047 2048 2049 2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064 2065 2066 2067 2068 2069 2070 2071 2072 2073 2074 2075 2076 2077 2078 2079 2080 2081 2082 2083 2084 2085 2086 2087 2088 2089 2090 2091 2092 2093 2094 2095 2096 2097 2098 2099 2100 2101 2102 2103 2104 2105 2106 2107 2108 2109 2110 2111 2112 2113 2114 2115 2116 2117 2118 2119 2120 2121 2122 2123 2124 2125 2126 2127 2128 2129 2130 2131 2132 2133 2134 2135 2136 2137 2138 2139 2140 2141 2142 2143 2144 2145 2146 2147 2148 2149 2150 2151 2152 2153 2154 2155 2156 2157 2158 2159 2160 2161 2162 2163 2164 2165 2166 2167 2168 2169 2170 2171 2172 2173 2174 2175 2176 2177 2178 2179 2180 2181 2182 2183 2184 2185 2186 2187 2188 2189 2190 2191 2192 2193 2194 2195 2196 2197 2198 2199 2200 2201 2202 2203 2204 2205 2206 2207 2208 2209 2210 2211 2212 2213 2214 2215 2216 2217 2218 2219 2220 2221 2222 2223 2224 2225 2226 2227 2228 2229 2230 2231 2232 2233 2234 2235 2236 2237 2238 2239 2240 2241 2242 2243 2244 2245 2246 2247 2248 2249 2250 2251 2252 2253 2254 2255 2256 2257 2258 2259 2260 2261 2262 2263 2264 2265 2266 2267 2268 2269 2270 2271 2272 2273 2274 2275 2276 2277 2278 2279 2280 2281 2282 2283 2284 2285 2286 2287 2288 2289 2290 2291 2292 2293 2294 2295 2296 2297 2298 2299 2300 2301 2302 2303 2304 2305 2306 2307 2308 2309 2310 2311 2312 2313 2314 2315 2316 2317 2318 2319 2320 2321 2322 2323 2324 2325 2326 2327 2328 2329 2330 2331 2332 2333 2334 2335 2336 2337 2338 2339 2340 2341 2342 2343 2344 2345 2346 2347 2348 2349 2350 2351 2352 2353 2354 2355 2356 2357 2358 2359 2360 2361 2362 2363 2364 2365 2366 2367 2368 2369 2370 2371 2372 2373 2374 2375 2376 2377 2378 2379 2380 2381 2382 2383 2384 2385 2386 2387 2388 2389 2390 2391 2392 2393 2394 2395 2396 2397 2398 2399 |
7767517
2396 2947
MemoryData model_info 0 1 model_info 0=5 21=1
Input input_0 0 1 token 0=1 1=1 2=1
Input input_1 0 1 state_0_in 0=1024
Input input_2 0 1 state_1_in 0=64 1=64 2=16
Input input_3 0 1 state_2_in 0=1024
Input input_4 0 1 state_3_in 0=1024
Input input_5 0 1 state_4_in 0=64 1=64 2=16
Input input_6 0 1 state_5_in 0=1024
Input input_7 0 1 state_6_in 0=1024
Input input_8 0 1 state_7_in 0=64 1=64 2=16
Input input_9 0 1 state_8_in 0=1024
Input input_10 0 1 state_9_in 0=1024
Input input_11 0 1 state_10_in 0=64 1=64 2=16
Input input_12 0 1 state_11_in 0=1024
Input input_13 0 1 state_12_in 0=1024
Input input_14 0 1 state_13_in 0=64 1=64 2=16
Input input_15 0 1 state_14_in 0=1024
Input input_16 0 1 state_15_in 0=1024
Input input_17 0 1 state_16_in 0=64 1=64 2=16
Input input_18 0 1 state_17_in 0=1024
Input input_19 0 1 state_18_in 0=1024
Input input_20 0 1 state_19_in 0=64 1=64 2=16
Input input_21 0 1 state_20_in 0=1024
Input input_22 0 1 state_21_in 0=1024
Input input_23 0 1 state_22_in 0=64 1=64 2=16
Input input_24 0 1 state_23_in 0=1024
Input input_25 0 1 state_24_in 0=1024
Input input_26 0 1 state_25_in 0=64 1=64 2=16
Input input_27 0 1 state_26_in 0=1024
Input input_28 0 1 state_27_in 0=1024
Input input_29 0 1 state_28_in 0=64 1=64 2=16
Input input_30 0 1 state_29_in 0=1024
Input input_31 0 1 state_30_in 0=1024
Input input_32 0 1 state_31_in 0=64 1=64 2=16
Input input_33 0 1 state_32_in 0=1024
Input input_34 0 1 state_33_in 0=1024
Input input_35 0 1 state_34_in 0=64 1=64 2=16
Input input_36 0 1 state_35_in 0=1024
Input input_37 0 1 state_36_in 0=1024
Input input_38 0 1 state_37_in 0=64 1=64 2=16
Input input_39 0 1 state_38_in 0=1024
Input input_40 0 1 state_39_in 0=1024
Input input_41 0 1 state_40_in 0=64 1=64 2=16
Input input_42 0 1 state_41_in 0=1024
Input input_43 0 1 state_42_in 0=1024
Input input_44 0 1 state_43_in 0=64 1=64 2=16
Input input_45 0 1 state_44_in 0=1024
Input input_46 0 1 state_45_in 0=1024
Input input_47 0 1 state_46_in 0=64 1=64 2=16
Input input_48 0 1 state_47_in 0=1024
Input input_49 0 1 state_48_in 0=1024
Input input_50 0 1 state_49_in 0=64 1=64 2=16
Input input_51 0 1 state_50_in 0=1024
Input input_52 0 1 state_51_in 0=1024
Input input_53 0 1 state_52_in 0=64 1=64 2=16
Input input_54 0 1 state_53_in 0=1024
Input input_55 0 1 state_54_in 0=1024
Input input_56 0 1 state_55_in 0=64 1=64 2=16
Input input_57 0 1 state_56_in 0=1024
Input input_58 0 1 state_57_in 0=1024
Input input_59 0 1 state_58_in 0=64 1=64 2=16
Input input_60 0 1 state_59_in 0=1024
Input input_61 0 1 state_60_in 0=1024
Input input_62 0 1 state_61_in 0=64 1=64 2=16
Input input_63 0 1 state_62_in 0=1024
Input input_64 0 1 state_63_in 0=1024
Input input_65 0 1 state_64_in 0=64 1=64 2=16
Input input_66 0 1 state_65_in 0=1024
Input input_67 0 1 state_66_in 0=1024
Input input_68 0 1 state_67_in 0=64 1=64 2=16
Input input_69 0 1 state_68_in 0=1024
Input input_70 0 1 state_69_in 0=1024
Input input_71 0 1 state_70_in 0=64 1=64 2=16
Input input_72 0 1 state_71_in 0=1024
Embed embedding 1 1 token emb 0=1024 1=65536 3=67108864
Split split_emb 1 2 emb att_0_x_last att_0_x
LayerNorm layernorm_0 1 1 att_0_x att_0_xx 0=1024 1=0.00001 2=1
Split split_att_0_xx 1 3 att_0_xx att_0_xx_0 att_0_xx_1 state_0_out
BinaryOp sub_0 2 1 state_0_in att_0_xx_0 att_0_sx 0=1
MemoryData data_att_0_maa 0 1 att_0_maa 0=1024 1=6 21=0
BinaryOp mul_0 2 1 att_0_sx att_0_maa att_0_maa_x 0=2
BinaryOp add_0 2 1 att_0_xx_1 att_0_maa_x att_0_maa_rwkvag 0=0
Slice slice_att_0_maa_rwkvag 1 6 att_0_maa_rwkvag att_0_mr att_0_mw att_0_mk att_0_mv att_0_ma att_0_mg -23300=6,-233,-233,-233,-233,-233,-233 1=0
Split split_att_0_mv 1 2 att_0_mv att_0_mv_0 att_0_mv_1
Gemm gemm_att_0_key 1 1 att_0_mk att_0_key 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_0_value 1 1 att_0_mv_0 att_0_value 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_0_receptance 1 1 att_0_mr att_0_receptance 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_0_ma_lora_0 1 1 att_0_ma att_0_ma_lora_0 4=0 5=1 6=0 7=0 8=64 9=1024 10=-1
Gemm gemm_att_0_ma_lora_1 1 1 att_0_ma_lora_0 att_0_ma_lora_1 4=0 5=1 6=0 7=0 8=1024 9=64 10=-1
MemoryData data_att_0_a0 0 1 att_0_a0 0=1024 21=0
BinaryOp add_1 2 1 att_0_ma_lora_1 att_0_a0 att_0_ma_lora_2 0=0
Sigmoid sigmoid_0 1 1 att_0_ma_lora_2 att_0_a
Split split_att_0_a 1 2 att_0_a att_0_a_0 att_0_a_1
Gemm gemm_att_0_mg_lora 1 1 att_0_mg att_0_mg_lora 4=0 5=1 6=0 7=0 8=128 9=1024 10=-1
Sigmoid sigmoid_1 1 1 att_0_mg_lora att_0_mg_lora_sigmoid
Gemm gemm_att_0_gate 1 1 att_0_mg_lora_sigmoid att_0_gate 4=0 5=1 6=0 7=0 8=1024 9=128 10=-1
MemoryData data_att_0_k_k 0 1 att_0_k_k 0=1024 21=0
MemoryData data_att_0_k_a 0 1 att_0_k_a 0=1024 21=0
MemoryData data_att_0_r_k 0 1 att_0_r_k 0=1024 21=0
Split split_att_0_key 1 2 att_0_key att_0_key_0 att_0_key_1
BinaryOp mul_1 2 1 att_0_key_0 att_0_k_k att_0_key_k 0=2
Reshape reshape_0 1 1 att_0_key_k att_0_key_k_reshape 0=64 1=1 2=16
Normalize l2norm_0 1 1 att_0_key_k_reshape att_0_key_k_norm 0=1 1=1 2=0.0000001 3=1 4=0 9=1
Reshape reshape_1 1 1 att_0_key_k_norm att_0_key_k_norm_reshape 0=1024
BinaryOp add_2 1 1 att_0_a_0 att_0_a_0_minus 0=0 1=1 2=-1.0
BinaryOp mul_2 2 1 att_0_a_0_minus att_0_k_a att_0_a_minus1_mulka 0=2
BinaryOp add_3 1 1 att_0_a_minus1_mulka att_0_a_minus1_mulka_plus1 0=0 1=1 2=1.0
BinaryOp mul_3 2 1 att_0_a_minus1_mulka_plus1 att_0_key_1 att_0_key_a 0=2
Split split_att_0_value 1 2 att_0_value att_0_value_final v_first_0
Gemm gemm_att_0_mw_lora 1 1 att_0_mw att_0_mw_lora 4=0 5=1 6=0 7=0 8=64 9=1024 10=-1
TanH tanh_att_0_mw_lora 1 1 att_0_mw_lora att_0_mw_lora_tanh
Gemm gemm_att_0_mw_lora_tanh_linear 1 1 att_0_mw_lora_tanh att_0_mw_lora_tanh_linear 4=0 5=1 6=0 7=0 8=1024 9=64 10=-1
MemoryData data_att_0_td 0 1 att_0_td 0=1024 21=0
BinaryOp add_4 2 1 att_0_mw_lora_tanh_linear att_0_td att_0_time_decay_pre 0=0
Sigmoid sigmoid_2 1 1 att_0_time_decay_pre att_0_time_decay_sigmoid
Exp exp_0 1 1 att_0_time_decay_sigmoid att_0_time_decay 1=-0.606531
Reshape reshape_2 1 1 att_0_time_decay att_0_time_decay_reshape 0=64 1=1 2=16
Split split_att_0_key_a 1 2 att_0_key_a att_0_key_a_0 att_0_key_a_1
Reshape reshape_3 1 1 att_0_key_a_0 att_0_key_a_reshape 0=64 1=1 2=16
Split split_att_0_value_final 1 3 att_0_value_final att_0_value_final_0 att_0_value_final_1 att_0_value_final_2
Reshape reshape_4 1 1 att_0_value_final_0 att_0_value_final_0_reshape 0=64 1=16
Reshape reshape_5 1 1 att_0_value_final_1 att_0_value_final_1_reshape 0=1 1=64 2=16
MatMul matmul_att_0_vk 2 1 att_0_value_final_1_reshape att_0_key_a_reshape att_0_vk 0=0
Split split_att_0_key_k_norm_reshape 1 2 att_0_key_k_norm_reshape att_0_kk_0 att_0_kk_1
BinaryOp mul_4 1 1 att_0_kk_0 att_0_kk_0_neg 0=2 1=1 2=-1.0
Reshape reshape_6 1 1 att_0_kk_0_neg att_0_kk_0_neg_reshape 0=1 1=64 2=16
BinaryOp mul_5 2 1 att_0_kk_1 att_0_a_1 att_0_kk_a 0=2
Reshape reshape_7 1 1 att_0_kk_a att_0_kk_a_reshape 0=64 1=1 2=16
MatMul matmul_att_0_ab 2 1 att_0_kk_0_neg_reshape att_0_kk_a_reshape att_0_ab 0=0
Split split_state_1_in 1 2 state_1_in att_0_state_prev_0 att_0_state_prev_1
BinaryOp mul_6 2 1 att_0_state_prev_0 att_0_time_decay_reshape att_0_state_td 0=2
MatMul matmul_att_0_sab 2 1 att_0_state_prev_1 att_0_ab att_0_sab 0=0
BinaryOp add_5 2 1 att_0_sab att_0_vk att_0_sab_vk 0=0
BinaryOp add_6 2 1 att_0_sab_vk att_0_state_td att_0_state_new 0=0
Split split_att_0_state_new 1 2 att_0_state_new att_0_state_new_0 state_1_out
Split split_att_0_receptance 1 2 att_0_receptance att_0_receptance_0 att_0_receptance_1
Reshape reshape_8 1 1 att_0_receptance_0 att_0_receptance_reshape 0=1 1=64 2=16
MatMul matmul_att_0_wkv_out 2 1 att_0_state_new_0 att_0_receptance_reshape att_0_wkv_out 0=0
Reshape reshape_9 1 1 att_0_wkv_out att_0_wkv_out_flatten 0=1024
GroupNorm groupnorm_0 1 1 att_0_wkv_out_flatten att_0_x_gn 0=16 1=1024 2=0.00064 3=1
BinaryOp mul_7 2 1 att_0_receptance_1 att_0_key_a_1 att_0_receptance_key_a 0=2
BinaryOp mul_8 2 1 att_0_receptance_key_a att_0_r_k att_0_rk 0=2
Reshape reshape_10 1 1 att_0_rk att_0_rk_reshape 0=64 1=16
Reduction sum_0 1 1 att_0_rk_reshape att_0_rk_sum 1=0 -23303=1,-1 4=1 5=1
BinaryOp mul_9 2 1 att_0_value_final_0_reshape att_0_rk_sum att_0_value_final_0_rk 0=2
Reshape reshape_11 1 1 att_0_value_final_0_rk att_0_value_final_0_rk_reshape 0=1024
BinaryOp add_7 2 1 att_0_value_final_0_rk_reshape att_0_x_gn att_0_x_gn_rkv 0=0
BinaryOp mul_10 2 1 att_0_x_gn_rkv att_0_gate att_0_x_gate 0=2
Gemm gemm_att_0_x_out 1 1 att_0_x_gate att_0_x_out 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
BinaryOp add_8 2 1 att_0_x_out att_0_x_last time_mixing_0_out 0=0
Split split_time_mixing_0_out 1 2 time_mixing_0_out ffn_0_x_last ffn_0_x
LayerNorm layernorm_1 1 1 ffn_0_x ffn_0_xx 0=1024 1=0.00001 2=1
Split split_ffn_0_xx 1 3 ffn_0_xx ffn_0_xx_0 ffn_0_xx_1 state_2_out
BinaryOp sub_1 2 1 state_2_in ffn_0_xx_0 ffn_0_sx 0=1
Split split_ffn_0_sx 1 1 ffn_0_sx ffn_0_sx_0
MemoryData data_ffn_0_x_k 0 1 ffn_0_x_k 0=1024 21=0
BinaryOp mul_11 2 1 ffn_0_sx_0 ffn_0_x_k ffn_0_xk 0=2
BinaryOp add_9 2 1 ffn_0_xk ffn_0_xx_1 ffn_0_xxk 0=0
Gemm gemm_ffn_0_key 1 1 ffn_0_xxk ffn_0_key 4=0 5=1 6=0 7=0 8=4096 9=1024 10=-1
ReLU relu_0 1 1 ffn_0_key ffn_0_key_relu
UnaryOp square_ffn_0_key_relu 1 1 ffn_0_key_relu ffn_0_key_relu_square 0=4
Gemm gemm_ffn_0_value 1 1 ffn_0_key_relu_square ffn_0_value 4=0 5=1 6=0 7=0 8=1024 9=4096 10=-1
BinaryOp add_10 2 1 ffn_0_value ffn_0_x_last channel_mixing_0_out 0=0
Split split_channel_mixing_0_out 1 2 channel_mixing_0_out att_1_x_last att_1_x
LayerNorm layernorm_2 1 1 att_1_x att_1_xx 0=1024 1=0.00001 2=1
Split split_att_1_xx 1 3 att_1_xx att_1_xx_0 att_1_xx_1 state_3_out
BinaryOp sub_2 2 1 state_3_in att_1_xx_0 att_1_sx 0=1
MemoryData data_att_1_maa 0 1 att_1_maa 0=1024 1=6 21=0
BinaryOp mul_12 2 1 att_1_sx att_1_maa att_1_maa_x 0=2
BinaryOp add_11 2 1 att_1_xx_1 att_1_maa_x att_1_maa_rwkvag 0=0
Slice slice_att_1_maa_rwkvag 1 6 att_1_maa_rwkvag att_1_mr att_1_mw att_1_mk att_1_mv att_1_ma att_1_mg -23300=6,-233,-233,-233,-233,-233,-233 1=0
Split split_att_1_mv 1 2 att_1_mv att_1_mv_0 att_1_mv_1
Gemm gemm_att_1_key 1 1 att_1_mk att_1_key 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_1_value 1 1 att_1_mv_0 att_1_value 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_1_receptance 1 1 att_1_mr att_1_receptance 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_1_ma_lora_0 1 1 att_1_ma att_1_ma_lora_0 4=0 5=1 6=0 7=0 8=64 9=1024 10=-1
Gemm gemm_att_1_ma_lora_1 1 1 att_1_ma_lora_0 att_1_ma_lora_1 4=0 5=1 6=0 7=0 8=1024 9=64 10=-1
MemoryData data_att_1_a0 0 1 att_1_a0 0=1024 21=0
BinaryOp add_12 2 1 att_1_ma_lora_1 att_1_a0 att_1_ma_lora_2 0=0
Sigmoid sigmoid_3 1 1 att_1_ma_lora_2 att_1_a
Split split_att_1_a 1 2 att_1_a att_1_a_0 att_1_a_1
Gemm gemm_att_1_mg_lora 1 1 att_1_mg att_1_mg_lora 4=0 5=1 6=0 7=0 8=128 9=1024 10=-1
Sigmoid sigmoid_4 1 1 att_1_mg_lora att_1_mg_lora_sigmoid
Gemm gemm_att_1_gate 1 1 att_1_mg_lora_sigmoid att_1_gate 4=0 5=1 6=0 7=0 8=1024 9=128 10=-1
MemoryData data_att_1_k_k 0 1 att_1_k_k 0=1024 21=0
MemoryData data_att_1_k_a 0 1 att_1_k_a 0=1024 21=0
MemoryData data_att_1_r_k 0 1 att_1_r_k 0=1024 21=0
Split split_att_1_key 1 2 att_1_key att_1_key_0 att_1_key_1
BinaryOp mul_13 2 1 att_1_key_0 att_1_k_k att_1_key_k 0=2
Reshape reshape_12 1 1 att_1_key_k att_1_key_k_reshape 0=64 1=1 2=16
Normalize l2norm_1 1 1 att_1_key_k_reshape att_1_key_k_norm 0=1 1=1 2=0.0000001 3=1 4=0 9=1
Reshape reshape_13 1 1 att_1_key_k_norm att_1_key_k_norm_reshape 0=1024
BinaryOp add_13 1 1 att_1_a_0 att_1_a_0_minus 0=0 1=1 2=-1.0
BinaryOp mul_14 2 1 att_1_a_0_minus att_1_k_a att_1_a_minus1_mulka 0=2
BinaryOp add_14 1 1 att_1_a_minus1_mulka att_1_a_minus1_mulka_plus1 0=0 1=1 2=1.0
BinaryOp mul_15 2 1 att_1_a_minus1_mulka_plus1 att_1_key_1 att_1_key_a 0=2
Split split_att_1_value 1 2 att_1_value att_1_value_0 att_1_value_1
Split split_v_first_0 1 2 v_first_0 att_1_v_first v_first_1
BinaryOp minus_0 2 1 att_1_v_first att_1_value_1 att_1_vfirst_minus_value 0=1
Gemm gemm_att_1_mv_lora_0 1 1 att_1_mv_1 att_1_mv_lora_0 4=0 5=1 6=0 7=0 8=32 9=1024 10=-1
Gemm gemm_att_1_mv_lora_1 1 1 att_1_mv_lora_0 att_1_mv_lora_1 4=0 5=1 6=0 7=0 8=1024 9=32 10=-1
MemoryData data_att_1_v0 0 1 att_1_v0 0=1024 21=0
BinaryOp add_15 2 1 att_1_mv_lora_1 att_1_v0 att_1_mv_lora_2 0=0
Sigmoid sigmoid_5 1 1 att_1_mv_lora_2 att_1_mv_lora_sigmoid
BinaryOp mul_16 2 1 att_1_mv_lora_sigmoid att_1_vfirst_minus_value att_1_vfirst_minus_value_mul 0=2
BinaryOp add_16 2 1 att_1_value_0 att_1_vfirst_minus_value_mul att_1_value_final 0=0
Gemm gemm_att_1_mw_lora 1 1 att_1_mw att_1_mw_lora 4=0 5=1 6=0 7=0 8=64 9=1024 10=-1
TanH tanh_att_1_mw_lora 1 1 att_1_mw_lora att_1_mw_lora_tanh
Gemm gemm_att_1_mw_lora_tanh_linear 1 1 att_1_mw_lora_tanh att_1_mw_lora_tanh_linear 4=0 5=1 6=0 7=0 8=1024 9=64 10=-1
MemoryData data_att_1_td 0 1 att_1_td 0=1024 21=0
BinaryOp add_17 2 1 att_1_mw_lora_tanh_linear att_1_td att_1_time_decay_pre 0=0
Sigmoid sigmoid_6 1 1 att_1_time_decay_pre att_1_time_decay_sigmoid
Exp exp_1 1 1 att_1_time_decay_sigmoid att_1_time_decay 1=-0.606531
Reshape reshape_14 1 1 att_1_time_decay att_1_time_decay_reshape 0=64 1=1 2=16
Split split_att_1_key_a 1 2 att_1_key_a att_1_key_a_0 att_1_key_a_1
Reshape reshape_15 1 1 att_1_key_a_0 att_1_key_a_reshape 0=64 1=1 2=16
Split split_att_1_value_final 1 3 att_1_value_final att_1_value_final_0 att_1_value_final_1 att_1_value_final_2
Reshape reshape_16 1 1 att_1_value_final_0 att_1_value_final_0_reshape 0=64 1=16
Reshape reshape_17 1 1 att_1_value_final_1 att_1_value_final_1_reshape 0=1 1=64 2=16
MatMul matmul_att_1_vk 2 1 att_1_value_final_1_reshape att_1_key_a_reshape att_1_vk 0=0
Split split_att_1_key_k_norm_reshape 1 2 att_1_key_k_norm_reshape att_1_kk_0 att_1_kk_1
BinaryOp mul_17 1 1 att_1_kk_0 att_1_kk_0_neg 0=2 1=1 2=-1.0
Reshape reshape_18 1 1 att_1_kk_0_neg att_1_kk_0_neg_reshape 0=1 1=64 2=16
BinaryOp mul_18 2 1 att_1_kk_1 att_1_a_1 att_1_kk_a 0=2
Reshape reshape_19 1 1 att_1_kk_a att_1_kk_a_reshape 0=64 1=1 2=16
MatMul matmul_att_1_ab 2 1 att_1_kk_0_neg_reshape att_1_kk_a_reshape att_1_ab 0=0
Split split_state_4_in 1 2 state_4_in att_1_state_prev_0 att_1_state_prev_1
BinaryOp mul_19 2 1 att_1_state_prev_0 att_1_time_decay_reshape att_1_state_td 0=2
MatMul matmul_att_1_sab 2 1 att_1_state_prev_1 att_1_ab att_1_sab 0=0
BinaryOp add_18 2 1 att_1_sab att_1_vk att_1_sab_vk 0=0
BinaryOp add_19 2 1 att_1_sab_vk att_1_state_td att_1_state_new 0=0
Split split_att_1_state_new 1 2 att_1_state_new att_1_state_new_0 state_4_out
Split split_att_1_receptance 1 2 att_1_receptance att_1_receptance_0 att_1_receptance_1
Reshape reshape_20 1 1 att_1_receptance_0 att_1_receptance_reshape 0=1 1=64 2=16
MatMul matmul_att_1_wkv_out 2 1 att_1_state_new_0 att_1_receptance_reshape att_1_wkv_out 0=0
Reshape reshape_21 1 1 att_1_wkv_out att_1_wkv_out_flatten 0=1024
GroupNorm groupnorm_1 1 1 att_1_wkv_out_flatten att_1_x_gn 0=16 1=1024 2=0.00064 3=1
BinaryOp mul_20 2 1 att_1_receptance_1 att_1_key_a_1 att_1_receptance_key_a 0=2
BinaryOp mul_21 2 1 att_1_receptance_key_a att_1_r_k att_1_rk 0=2
Reshape reshape_22 1 1 att_1_rk att_1_rk_reshape 0=64 1=16
Reduction sum_1 1 1 att_1_rk_reshape att_1_rk_sum 1=0 -23303=1,-1 4=1 5=1
BinaryOp mul_22 2 1 att_1_value_final_0_reshape att_1_rk_sum att_1_value_final_0_rk 0=2
Reshape reshape_23 1 1 att_1_value_final_0_rk att_1_value_final_0_rk_reshape 0=1024
BinaryOp add_20 2 1 att_1_value_final_0_rk_reshape att_1_x_gn att_1_x_gn_rkv 0=0
BinaryOp mul_23 2 1 att_1_x_gn_rkv att_1_gate att_1_x_gate 0=2
Gemm gemm_att_1_x_out 1 1 att_1_x_gate att_1_x_out 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
BinaryOp add_21 2 1 att_1_x_out att_1_x_last time_mixing_1_out 0=0
Split split_time_mixing_1_out 1 2 time_mixing_1_out ffn_1_x_last ffn_1_x
LayerNorm layernorm_3 1 1 ffn_1_x ffn_1_xx 0=1024 1=0.00001 2=1
Split split_ffn_1_xx 1 3 ffn_1_xx ffn_1_xx_0 ffn_1_xx_1 state_5_out
BinaryOp sub_3 2 1 state_5_in ffn_1_xx_0 ffn_1_sx 0=1
Split split_ffn_1_sx 1 1 ffn_1_sx ffn_1_sx_0
MemoryData data_ffn_1_x_k 0 1 ffn_1_x_k 0=1024 21=0
BinaryOp mul_24 2 1 ffn_1_sx_0 ffn_1_x_k ffn_1_xk 0=2
BinaryOp add_22 2 1 ffn_1_xk ffn_1_xx_1 ffn_1_xxk 0=0
Gemm gemm_ffn_1_key 1 1 ffn_1_xxk ffn_1_key 4=0 5=1 6=0 7=0 8=4096 9=1024 10=-1
ReLU relu_1 1 1 ffn_1_key ffn_1_key_relu
UnaryOp square_ffn_1_key_relu 1 1 ffn_1_key_relu ffn_1_key_relu_square 0=4
Gemm gemm_ffn_1_value 1 1 ffn_1_key_relu_square ffn_1_value 4=0 5=1 6=0 7=0 8=1024 9=4096 10=-1
BinaryOp add_23 2 1 ffn_1_value ffn_1_x_last channel_mixing_1_out 0=0
Split split_channel_mixing_1_out 1 2 channel_mixing_1_out att_2_x_last att_2_x
LayerNorm layernorm_4 1 1 att_2_x att_2_xx 0=1024 1=0.00001 2=1
Split split_att_2_xx 1 3 att_2_xx att_2_xx_0 att_2_xx_1 state_6_out
BinaryOp sub_4 2 1 state_6_in att_2_xx_0 att_2_sx 0=1
MemoryData data_att_2_maa 0 1 att_2_maa 0=1024 1=6 21=0
BinaryOp mul_25 2 1 att_2_sx att_2_maa att_2_maa_x 0=2
BinaryOp add_24 2 1 att_2_xx_1 att_2_maa_x att_2_maa_rwkvag 0=0
Slice slice_att_2_maa_rwkvag 1 6 att_2_maa_rwkvag att_2_mr att_2_mw att_2_mk att_2_mv att_2_ma att_2_mg -23300=6,-233,-233,-233,-233,-233,-233 1=0
Split split_att_2_mv 1 2 att_2_mv att_2_mv_0 att_2_mv_1
Gemm gemm_att_2_key 1 1 att_2_mk att_2_key 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_2_value 1 1 att_2_mv_0 att_2_value 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_2_receptance 1 1 att_2_mr att_2_receptance 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_2_ma_lora_0 1 1 att_2_ma att_2_ma_lora_0 4=0 5=1 6=0 7=0 8=64 9=1024 10=-1
Gemm gemm_att_2_ma_lora_1 1 1 att_2_ma_lora_0 att_2_ma_lora_1 4=0 5=1 6=0 7=0 8=1024 9=64 10=-1
MemoryData data_att_2_a0 0 1 att_2_a0 0=1024 21=0
BinaryOp add_25 2 1 att_2_ma_lora_1 att_2_a0 att_2_ma_lora_2 0=0
Sigmoid sigmoid_7 1 1 att_2_ma_lora_2 att_2_a
Split split_att_2_a 1 2 att_2_a att_2_a_0 att_2_a_1
Gemm gemm_att_2_mg_lora 1 1 att_2_mg att_2_mg_lora 4=0 5=1 6=0 7=0 8=128 9=1024 10=-1
Sigmoid sigmoid_8 1 1 att_2_mg_lora att_2_mg_lora_sigmoid
Gemm gemm_att_2_gate 1 1 att_2_mg_lora_sigmoid att_2_gate 4=0 5=1 6=0 7=0 8=1024 9=128 10=-1
MemoryData data_att_2_k_k 0 1 att_2_k_k 0=1024 21=0
MemoryData data_att_2_k_a 0 1 att_2_k_a 0=1024 21=0
MemoryData data_att_2_r_k 0 1 att_2_r_k 0=1024 21=0
Split split_att_2_key 1 2 att_2_key att_2_key_0 att_2_key_1
BinaryOp mul_26 2 1 att_2_key_0 att_2_k_k att_2_key_k 0=2
Reshape reshape_24 1 1 att_2_key_k att_2_key_k_reshape 0=64 1=1 2=16
Normalize l2norm_2 1 1 att_2_key_k_reshape att_2_key_k_norm 0=1 1=1 2=0.0000001 3=1 4=0 9=1
Reshape reshape_25 1 1 att_2_key_k_norm att_2_key_k_norm_reshape 0=1024
BinaryOp add_26 1 1 att_2_a_0 att_2_a_0_minus 0=0 1=1 2=-1.0
BinaryOp mul_27 2 1 att_2_a_0_minus att_2_k_a att_2_a_minus1_mulka 0=2
BinaryOp add_27 1 1 att_2_a_minus1_mulka att_2_a_minus1_mulka_plus1 0=0 1=1 2=1.0
BinaryOp mul_28 2 1 att_2_a_minus1_mulka_plus1 att_2_key_1 att_2_key_a 0=2
Split split_att_2_value 1 2 att_2_value att_2_value_0 att_2_value_1
Split split_v_first_1 1 2 v_first_1 att_2_v_first v_first_2
BinaryOp minus_1 2 1 att_2_v_first att_2_value_1 att_2_vfirst_minus_value 0=1
Gemm gemm_att_2_mv_lora_0 1 1 att_2_mv_1 att_2_mv_lora_0 4=0 5=1 6=0 7=0 8=32 9=1024 10=-1
Gemm gemm_att_2_mv_lora_1 1 1 att_2_mv_lora_0 att_2_mv_lora_1 4=0 5=1 6=0 7=0 8=1024 9=32 10=-1
MemoryData data_att_2_v0 0 1 att_2_v0 0=1024 21=0
BinaryOp add_28 2 1 att_2_mv_lora_1 att_2_v0 att_2_mv_lora_2 0=0
Sigmoid sigmoid_9 1 1 att_2_mv_lora_2 att_2_mv_lora_sigmoid
BinaryOp mul_29 2 1 att_2_mv_lora_sigmoid att_2_vfirst_minus_value att_2_vfirst_minus_value_mul 0=2
BinaryOp add_29 2 1 att_2_value_0 att_2_vfirst_minus_value_mul att_2_value_final 0=0
Gemm gemm_att_2_mw_lora 1 1 att_2_mw att_2_mw_lora 4=0 5=1 6=0 7=0 8=64 9=1024 10=-1
TanH tanh_att_2_mw_lora 1 1 att_2_mw_lora att_2_mw_lora_tanh
Gemm gemm_att_2_mw_lora_tanh_linear 1 1 att_2_mw_lora_tanh att_2_mw_lora_tanh_linear 4=0 5=1 6=0 7=0 8=1024 9=64 10=-1
MemoryData data_att_2_td 0 1 att_2_td 0=1024 21=0
BinaryOp add_30 2 1 att_2_mw_lora_tanh_linear att_2_td att_2_time_decay_pre 0=0
Sigmoid sigmoid_10 1 1 att_2_time_decay_pre att_2_time_decay_sigmoid
Exp exp_2 1 1 att_2_time_decay_sigmoid att_2_time_decay 1=-0.606531
Reshape reshape_26 1 1 att_2_time_decay att_2_time_decay_reshape 0=64 1=1 2=16
Split split_att_2_key_a 1 2 att_2_key_a att_2_key_a_0 att_2_key_a_1
Reshape reshape_27 1 1 att_2_key_a_0 att_2_key_a_reshape 0=64 1=1 2=16
Split split_att_2_value_final 1 3 att_2_value_final att_2_value_final_0 att_2_value_final_1 att_2_value_final_2
Reshape reshape_28 1 1 att_2_value_final_0 att_2_value_final_0_reshape 0=64 1=16
Reshape reshape_29 1 1 att_2_value_final_1 att_2_value_final_1_reshape 0=1 1=64 2=16
MatMul matmul_att_2_vk 2 1 att_2_value_final_1_reshape att_2_key_a_reshape att_2_vk 0=0
Split split_att_2_key_k_norm_reshape 1 2 att_2_key_k_norm_reshape att_2_kk_0 att_2_kk_1
BinaryOp mul_30 1 1 att_2_kk_0 att_2_kk_0_neg 0=2 1=1 2=-1.0
Reshape reshape_30 1 1 att_2_kk_0_neg att_2_kk_0_neg_reshape 0=1 1=64 2=16
BinaryOp mul_31 2 1 att_2_kk_1 att_2_a_1 att_2_kk_a 0=2
Reshape reshape_31 1 1 att_2_kk_a att_2_kk_a_reshape 0=64 1=1 2=16
MatMul matmul_att_2_ab 2 1 att_2_kk_0_neg_reshape att_2_kk_a_reshape att_2_ab 0=0
Split split_state_7_in 1 2 state_7_in att_2_state_prev_0 att_2_state_prev_1
BinaryOp mul_32 2 1 att_2_state_prev_0 att_2_time_decay_reshape att_2_state_td 0=2
MatMul matmul_att_2_sab 2 1 att_2_state_prev_1 att_2_ab att_2_sab 0=0
BinaryOp add_31 2 1 att_2_sab att_2_vk att_2_sab_vk 0=0
BinaryOp add_32 2 1 att_2_sab_vk att_2_state_td att_2_state_new 0=0
Split split_att_2_state_new 1 2 att_2_state_new att_2_state_new_0 state_7_out
Split split_att_2_receptance 1 2 att_2_receptance att_2_receptance_0 att_2_receptance_1
Reshape reshape_32 1 1 att_2_receptance_0 att_2_receptance_reshape 0=1 1=64 2=16
MatMul matmul_att_2_wkv_out 2 1 att_2_state_new_0 att_2_receptance_reshape att_2_wkv_out 0=0
Reshape reshape_33 1 1 att_2_wkv_out att_2_wkv_out_flatten 0=1024
GroupNorm groupnorm_2 1 1 att_2_wkv_out_flatten att_2_x_gn 0=16 1=1024 2=0.00064 3=1
BinaryOp mul_33 2 1 att_2_receptance_1 att_2_key_a_1 att_2_receptance_key_a 0=2
BinaryOp mul_34 2 1 att_2_receptance_key_a att_2_r_k att_2_rk 0=2
Reshape reshape_34 1 1 att_2_rk att_2_rk_reshape 0=64 1=16
Reduction sum_2 1 1 att_2_rk_reshape att_2_rk_sum 1=0 -23303=1,-1 4=1 5=1
BinaryOp mul_35 2 1 att_2_value_final_0_reshape att_2_rk_sum att_2_value_final_0_rk 0=2
Reshape reshape_35 1 1 att_2_value_final_0_rk att_2_value_final_0_rk_reshape 0=1024
BinaryOp add_33 2 1 att_2_value_final_0_rk_reshape att_2_x_gn att_2_x_gn_rkv 0=0
BinaryOp mul_36 2 1 att_2_x_gn_rkv att_2_gate att_2_x_gate 0=2
Gemm gemm_att_2_x_out 1 1 att_2_x_gate att_2_x_out 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
BinaryOp add_34 2 1 att_2_x_out att_2_x_last time_mixing_2_out 0=0
Split split_time_mixing_2_out 1 2 time_mixing_2_out ffn_2_x_last ffn_2_x
LayerNorm layernorm_5 1 1 ffn_2_x ffn_2_xx 0=1024 1=0.00001 2=1
Split split_ffn_2_xx 1 3 ffn_2_xx ffn_2_xx_0 ffn_2_xx_1 state_8_out
BinaryOp sub_5 2 1 state_8_in ffn_2_xx_0 ffn_2_sx 0=1
Split split_ffn_2_sx 1 1 ffn_2_sx ffn_2_sx_0
MemoryData data_ffn_2_x_k 0 1 ffn_2_x_k 0=1024 21=0
BinaryOp mul_37 2 1 ffn_2_sx_0 ffn_2_x_k ffn_2_xk 0=2
BinaryOp add_35 2 1 ffn_2_xk ffn_2_xx_1 ffn_2_xxk 0=0
Gemm gemm_ffn_2_key 1 1 ffn_2_xxk ffn_2_key 4=0 5=1 6=0 7=0 8=4096 9=1024 10=-1
ReLU relu_2 1 1 ffn_2_key ffn_2_key_relu
UnaryOp square_ffn_2_key_relu 1 1 ffn_2_key_relu ffn_2_key_relu_square 0=4
Gemm gemm_ffn_2_value 1 1 ffn_2_key_relu_square ffn_2_value 4=0 5=1 6=0 7=0 8=1024 9=4096 10=-1
BinaryOp add_36 2 1 ffn_2_value ffn_2_x_last channel_mixing_2_out 0=0
Split split_channel_mixing_2_out 1 2 channel_mixing_2_out att_3_x_last att_3_x
LayerNorm layernorm_6 1 1 att_3_x att_3_xx 0=1024 1=0.00001 2=1
Split split_att_3_xx 1 3 att_3_xx att_3_xx_0 att_3_xx_1 state_9_out
BinaryOp sub_6 2 1 state_9_in att_3_xx_0 att_3_sx 0=1
MemoryData data_att_3_maa 0 1 att_3_maa 0=1024 1=6 21=0
BinaryOp mul_38 2 1 att_3_sx att_3_maa att_3_maa_x 0=2
BinaryOp add_37 2 1 att_3_xx_1 att_3_maa_x att_3_maa_rwkvag 0=0
Slice slice_att_3_maa_rwkvag 1 6 att_3_maa_rwkvag att_3_mr att_3_mw att_3_mk att_3_mv att_3_ma att_3_mg -23300=6,-233,-233,-233,-233,-233,-233 1=0
Split split_att_3_mv 1 2 att_3_mv att_3_mv_0 att_3_mv_1
Gemm gemm_att_3_key 1 1 att_3_mk att_3_key 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_3_value 1 1 att_3_mv_0 att_3_value 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_3_receptance 1 1 att_3_mr att_3_receptance 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_3_ma_lora_0 1 1 att_3_ma att_3_ma_lora_0 4=0 5=1 6=0 7=0 8=64 9=1024 10=-1
Gemm gemm_att_3_ma_lora_1 1 1 att_3_ma_lora_0 att_3_ma_lora_1 4=0 5=1 6=0 7=0 8=1024 9=64 10=-1
MemoryData data_att_3_a0 0 1 att_3_a0 0=1024 21=0
BinaryOp add_38 2 1 att_3_ma_lora_1 att_3_a0 att_3_ma_lora_2 0=0
Sigmoid sigmoid_11 1 1 att_3_ma_lora_2 att_3_a
Split split_att_3_a 1 2 att_3_a att_3_a_0 att_3_a_1
Gemm gemm_att_3_mg_lora 1 1 att_3_mg att_3_mg_lora 4=0 5=1 6=0 7=0 8=128 9=1024 10=-1
Sigmoid sigmoid_12 1 1 att_3_mg_lora att_3_mg_lora_sigmoid
Gemm gemm_att_3_gate 1 1 att_3_mg_lora_sigmoid att_3_gate 4=0 5=1 6=0 7=0 8=1024 9=128 10=-1
MemoryData data_att_3_k_k 0 1 att_3_k_k 0=1024 21=0
MemoryData data_att_3_k_a 0 1 att_3_k_a 0=1024 21=0
MemoryData data_att_3_r_k 0 1 att_3_r_k 0=1024 21=0
Split split_att_3_key 1 2 att_3_key att_3_key_0 att_3_key_1
BinaryOp mul_39 2 1 att_3_key_0 att_3_k_k att_3_key_k 0=2
Reshape reshape_36 1 1 att_3_key_k att_3_key_k_reshape 0=64 1=1 2=16
Normalize l2norm_3 1 1 att_3_key_k_reshape att_3_key_k_norm 0=1 1=1 2=0.0000001 3=1 4=0 9=1
Reshape reshape_37 1 1 att_3_key_k_norm att_3_key_k_norm_reshape 0=1024
BinaryOp add_39 1 1 att_3_a_0 att_3_a_0_minus 0=0 1=1 2=-1.0
BinaryOp mul_40 2 1 att_3_a_0_minus att_3_k_a att_3_a_minus1_mulka 0=2
BinaryOp add_40 1 1 att_3_a_minus1_mulka att_3_a_minus1_mulka_plus1 0=0 1=1 2=1.0
BinaryOp mul_41 2 1 att_3_a_minus1_mulka_plus1 att_3_key_1 att_3_key_a 0=2
Split split_att_3_value 1 2 att_3_value att_3_value_0 att_3_value_1
Split split_v_first_2 1 2 v_first_2 att_3_v_first v_first_3
BinaryOp minus_2 2 1 att_3_v_first att_3_value_1 att_3_vfirst_minus_value 0=1
Gemm gemm_att_3_mv_lora_0 1 1 att_3_mv_1 att_3_mv_lora_0 4=0 5=1 6=0 7=0 8=32 9=1024 10=-1
Gemm gemm_att_3_mv_lora_1 1 1 att_3_mv_lora_0 att_3_mv_lora_1 4=0 5=1 6=0 7=0 8=1024 9=32 10=-1
MemoryData data_att_3_v0 0 1 att_3_v0 0=1024 21=0
BinaryOp add_41 2 1 att_3_mv_lora_1 att_3_v0 att_3_mv_lora_2 0=0
Sigmoid sigmoid_13 1 1 att_3_mv_lora_2 att_3_mv_lora_sigmoid
BinaryOp mul_42 2 1 att_3_mv_lora_sigmoid att_3_vfirst_minus_value att_3_vfirst_minus_value_mul 0=2
BinaryOp add_42 2 1 att_3_value_0 att_3_vfirst_minus_value_mul att_3_value_final 0=0
Gemm gemm_att_3_mw_lora 1 1 att_3_mw att_3_mw_lora 4=0 5=1 6=0 7=0 8=64 9=1024 10=-1
TanH tanh_att_3_mw_lora 1 1 att_3_mw_lora att_3_mw_lora_tanh
Gemm gemm_att_3_mw_lora_tanh_linear 1 1 att_3_mw_lora_tanh att_3_mw_lora_tanh_linear 4=0 5=1 6=0 7=0 8=1024 9=64 10=-1
MemoryData data_att_3_td 0 1 att_3_td 0=1024 21=0
BinaryOp add_43 2 1 att_3_mw_lora_tanh_linear att_3_td att_3_time_decay_pre 0=0
Sigmoid sigmoid_14 1 1 att_3_time_decay_pre att_3_time_decay_sigmoid
Exp exp_3 1 1 att_3_time_decay_sigmoid att_3_time_decay 1=-0.606531
Reshape reshape_38 1 1 att_3_time_decay att_3_time_decay_reshape 0=64 1=1 2=16
Split split_att_3_key_a 1 2 att_3_key_a att_3_key_a_0 att_3_key_a_1
Reshape reshape_39 1 1 att_3_key_a_0 att_3_key_a_reshape 0=64 1=1 2=16
Split split_att_3_value_final 1 3 att_3_value_final att_3_value_final_0 att_3_value_final_1 att_3_value_final_2
Reshape reshape_40 1 1 att_3_value_final_0 att_3_value_final_0_reshape 0=64 1=16
Reshape reshape_41 1 1 att_3_value_final_1 att_3_value_final_1_reshape 0=1 1=64 2=16
MatMul matmul_att_3_vk 2 1 att_3_value_final_1_reshape att_3_key_a_reshape att_3_vk 0=0
Split split_att_3_key_k_norm_reshape 1 2 att_3_key_k_norm_reshape att_3_kk_0 att_3_kk_1
BinaryOp mul_43 1 1 att_3_kk_0 att_3_kk_0_neg 0=2 1=1 2=-1.0
Reshape reshape_42 1 1 att_3_kk_0_neg att_3_kk_0_neg_reshape 0=1 1=64 2=16
BinaryOp mul_44 2 1 att_3_kk_1 att_3_a_1 att_3_kk_a 0=2
Reshape reshape_43 1 1 att_3_kk_a att_3_kk_a_reshape 0=64 1=1 2=16
MatMul matmul_att_3_ab 2 1 att_3_kk_0_neg_reshape att_3_kk_a_reshape att_3_ab 0=0
Split split_state_10_in 1 2 state_10_in att_3_state_prev_0 att_3_state_prev_1
BinaryOp mul_45 2 1 att_3_state_prev_0 att_3_time_decay_reshape att_3_state_td 0=2
MatMul matmul_att_3_sab 2 1 att_3_state_prev_1 att_3_ab att_3_sab 0=0
BinaryOp add_44 2 1 att_3_sab att_3_vk att_3_sab_vk 0=0
BinaryOp add_45 2 1 att_3_sab_vk att_3_state_td att_3_state_new 0=0
Split split_att_3_state_new 1 2 att_3_state_new att_3_state_new_0 state_10_out
Split split_att_3_receptance 1 2 att_3_receptance att_3_receptance_0 att_3_receptance_1
Reshape reshape_44 1 1 att_3_receptance_0 att_3_receptance_reshape 0=1 1=64 2=16
MatMul matmul_att_3_wkv_out 2 1 att_3_state_new_0 att_3_receptance_reshape att_3_wkv_out 0=0
Reshape reshape_45 1 1 att_3_wkv_out att_3_wkv_out_flatten 0=1024
GroupNorm groupnorm_3 1 1 att_3_wkv_out_flatten att_3_x_gn 0=16 1=1024 2=0.00064 3=1
BinaryOp mul_46 2 1 att_3_receptance_1 att_3_key_a_1 att_3_receptance_key_a 0=2
BinaryOp mul_47 2 1 att_3_receptance_key_a att_3_r_k att_3_rk 0=2
Reshape reshape_46 1 1 att_3_rk att_3_rk_reshape 0=64 1=16
Reduction sum_3 1 1 att_3_rk_reshape att_3_rk_sum 1=0 -23303=1,-1 4=1 5=1
BinaryOp mul_48 2 1 att_3_value_final_0_reshape att_3_rk_sum att_3_value_final_0_rk 0=2
Reshape reshape_47 1 1 att_3_value_final_0_rk att_3_value_final_0_rk_reshape 0=1024
BinaryOp add_46 2 1 att_3_value_final_0_rk_reshape att_3_x_gn att_3_x_gn_rkv 0=0
BinaryOp mul_49 2 1 att_3_x_gn_rkv att_3_gate att_3_x_gate 0=2
Gemm gemm_att_3_x_out 1 1 att_3_x_gate att_3_x_out 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
BinaryOp add_47 2 1 att_3_x_out att_3_x_last time_mixing_3_out 0=0
Split split_time_mixing_3_out 1 2 time_mixing_3_out ffn_3_x_last ffn_3_x
LayerNorm layernorm_7 1 1 ffn_3_x ffn_3_xx 0=1024 1=0.00001 2=1
Split split_ffn_3_xx 1 3 ffn_3_xx ffn_3_xx_0 ffn_3_xx_1 state_11_out
BinaryOp sub_7 2 1 state_11_in ffn_3_xx_0 ffn_3_sx 0=1
Split split_ffn_3_sx 1 1 ffn_3_sx ffn_3_sx_0
MemoryData data_ffn_3_x_k 0 1 ffn_3_x_k 0=1024 21=0
BinaryOp mul_50 2 1 ffn_3_sx_0 ffn_3_x_k ffn_3_xk 0=2
BinaryOp add_48 2 1 ffn_3_xk ffn_3_xx_1 ffn_3_xxk 0=0
Gemm gemm_ffn_3_key 1 1 ffn_3_xxk ffn_3_key 4=0 5=1 6=0 7=0 8=4096 9=1024 10=-1
ReLU relu_3 1 1 ffn_3_key ffn_3_key_relu
UnaryOp square_ffn_3_key_relu 1 1 ffn_3_key_relu ffn_3_key_relu_square 0=4
Gemm gemm_ffn_3_value 1 1 ffn_3_key_relu_square ffn_3_value 4=0 5=1 6=0 7=0 8=1024 9=4096 10=-1
BinaryOp add_49 2 1 ffn_3_value ffn_3_x_last channel_mixing_3_out 0=0
Split split_channel_mixing_3_out 1 2 channel_mixing_3_out att_4_x_last att_4_x
LayerNorm layernorm_8 1 1 att_4_x att_4_xx 0=1024 1=0.00001 2=1
Split split_att_4_xx 1 3 att_4_xx att_4_xx_0 att_4_xx_1 state_12_out
BinaryOp sub_8 2 1 state_12_in att_4_xx_0 att_4_sx 0=1
MemoryData data_att_4_maa 0 1 att_4_maa 0=1024 1=6 21=0
BinaryOp mul_51 2 1 att_4_sx att_4_maa att_4_maa_x 0=2
BinaryOp add_50 2 1 att_4_xx_1 att_4_maa_x att_4_maa_rwkvag 0=0
Slice slice_att_4_maa_rwkvag 1 6 att_4_maa_rwkvag att_4_mr att_4_mw att_4_mk att_4_mv att_4_ma att_4_mg -23300=6,-233,-233,-233,-233,-233,-233 1=0
Split split_att_4_mv 1 2 att_4_mv att_4_mv_0 att_4_mv_1
Gemm gemm_att_4_key 1 1 att_4_mk att_4_key 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_4_value 1 1 att_4_mv_0 att_4_value 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_4_receptance 1 1 att_4_mr att_4_receptance 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_4_ma_lora_0 1 1 att_4_ma att_4_ma_lora_0 4=0 5=1 6=0 7=0 8=64 9=1024 10=-1
Gemm gemm_att_4_ma_lora_1 1 1 att_4_ma_lora_0 att_4_ma_lora_1 4=0 5=1 6=0 7=0 8=1024 9=64 10=-1
MemoryData data_att_4_a0 0 1 att_4_a0 0=1024 21=0
BinaryOp add_51 2 1 att_4_ma_lora_1 att_4_a0 att_4_ma_lora_2 0=0
Sigmoid sigmoid_15 1 1 att_4_ma_lora_2 att_4_a
Split split_att_4_a 1 2 att_4_a att_4_a_0 att_4_a_1
Gemm gemm_att_4_mg_lora 1 1 att_4_mg att_4_mg_lora 4=0 5=1 6=0 7=0 8=128 9=1024 10=-1
Sigmoid sigmoid_16 1 1 att_4_mg_lora att_4_mg_lora_sigmoid
Gemm gemm_att_4_gate 1 1 att_4_mg_lora_sigmoid att_4_gate 4=0 5=1 6=0 7=0 8=1024 9=128 10=-1
MemoryData data_att_4_k_k 0 1 att_4_k_k 0=1024 21=0
MemoryData data_att_4_k_a 0 1 att_4_k_a 0=1024 21=0
MemoryData data_att_4_r_k 0 1 att_4_r_k 0=1024 21=0
Split split_att_4_key 1 2 att_4_key att_4_key_0 att_4_key_1
BinaryOp mul_52 2 1 att_4_key_0 att_4_k_k att_4_key_k 0=2
Reshape reshape_48 1 1 att_4_key_k att_4_key_k_reshape 0=64 1=1 2=16
Normalize l2norm_4 1 1 att_4_key_k_reshape att_4_key_k_norm 0=1 1=1 2=0.0000001 3=1 4=0 9=1
Reshape reshape_49 1 1 att_4_key_k_norm att_4_key_k_norm_reshape 0=1024
BinaryOp add_52 1 1 att_4_a_0 att_4_a_0_minus 0=0 1=1 2=-1.0
BinaryOp mul_53 2 1 att_4_a_0_minus att_4_k_a att_4_a_minus1_mulka 0=2
BinaryOp add_53 1 1 att_4_a_minus1_mulka att_4_a_minus1_mulka_plus1 0=0 1=1 2=1.0
BinaryOp mul_54 2 1 att_4_a_minus1_mulka_plus1 att_4_key_1 att_4_key_a 0=2
Split split_att_4_value 1 2 att_4_value att_4_value_0 att_4_value_1
Split split_v_first_3 1 2 v_first_3 att_4_v_first v_first_4
BinaryOp minus_3 2 1 att_4_v_first att_4_value_1 att_4_vfirst_minus_value 0=1
Gemm gemm_att_4_mv_lora_0 1 1 att_4_mv_1 att_4_mv_lora_0 4=0 5=1 6=0 7=0 8=32 9=1024 10=-1
Gemm gemm_att_4_mv_lora_1 1 1 att_4_mv_lora_0 att_4_mv_lora_1 4=0 5=1 6=0 7=0 8=1024 9=32 10=-1
MemoryData data_att_4_v0 0 1 att_4_v0 0=1024 21=0
BinaryOp add_54 2 1 att_4_mv_lora_1 att_4_v0 att_4_mv_lora_2 0=0
Sigmoid sigmoid_17 1 1 att_4_mv_lora_2 att_4_mv_lora_sigmoid
BinaryOp mul_55 2 1 att_4_mv_lora_sigmoid att_4_vfirst_minus_value att_4_vfirst_minus_value_mul 0=2
BinaryOp add_55 2 1 att_4_value_0 att_4_vfirst_minus_value_mul att_4_value_final 0=0
Gemm gemm_att_4_mw_lora 1 1 att_4_mw att_4_mw_lora 4=0 5=1 6=0 7=0 8=64 9=1024 10=-1
TanH tanh_att_4_mw_lora 1 1 att_4_mw_lora att_4_mw_lora_tanh
Gemm gemm_att_4_mw_lora_tanh_linear 1 1 att_4_mw_lora_tanh att_4_mw_lora_tanh_linear 4=0 5=1 6=0 7=0 8=1024 9=64 10=-1
MemoryData data_att_4_td 0 1 att_4_td 0=1024 21=0
BinaryOp add_56 2 1 att_4_mw_lora_tanh_linear att_4_td att_4_time_decay_pre 0=0
Sigmoid sigmoid_18 1 1 att_4_time_decay_pre att_4_time_decay_sigmoid
Exp exp_4 1 1 att_4_time_decay_sigmoid att_4_time_decay 1=-0.606531
Reshape reshape_50 1 1 att_4_time_decay att_4_time_decay_reshape 0=64 1=1 2=16
Split split_att_4_key_a 1 2 att_4_key_a att_4_key_a_0 att_4_key_a_1
Reshape reshape_51 1 1 att_4_key_a_0 att_4_key_a_reshape 0=64 1=1 2=16
Split split_att_4_value_final 1 3 att_4_value_final att_4_value_final_0 att_4_value_final_1 att_4_value_final_2
Reshape reshape_52 1 1 att_4_value_final_0 att_4_value_final_0_reshape 0=64 1=16
Reshape reshape_53 1 1 att_4_value_final_1 att_4_value_final_1_reshape 0=1 1=64 2=16
MatMul matmul_att_4_vk 2 1 att_4_value_final_1_reshape att_4_key_a_reshape att_4_vk 0=0
Split split_att_4_key_k_norm_reshape 1 2 att_4_key_k_norm_reshape att_4_kk_0 att_4_kk_1
BinaryOp mul_56 1 1 att_4_kk_0 att_4_kk_0_neg 0=2 1=1 2=-1.0
Reshape reshape_54 1 1 att_4_kk_0_neg att_4_kk_0_neg_reshape 0=1 1=64 2=16
BinaryOp mul_57 2 1 att_4_kk_1 att_4_a_1 att_4_kk_a 0=2
Reshape reshape_55 1 1 att_4_kk_a att_4_kk_a_reshape 0=64 1=1 2=16
MatMul matmul_att_4_ab 2 1 att_4_kk_0_neg_reshape att_4_kk_a_reshape att_4_ab 0=0
Split split_state_13_in 1 2 state_13_in att_4_state_prev_0 att_4_state_prev_1
BinaryOp mul_58 2 1 att_4_state_prev_0 att_4_time_decay_reshape att_4_state_td 0=2
MatMul matmul_att_4_sab 2 1 att_4_state_prev_1 att_4_ab att_4_sab 0=0
BinaryOp add_57 2 1 att_4_sab att_4_vk att_4_sab_vk 0=0
BinaryOp add_58 2 1 att_4_sab_vk att_4_state_td att_4_state_new 0=0
Split split_att_4_state_new 1 2 att_4_state_new att_4_state_new_0 state_13_out
Split split_att_4_receptance 1 2 att_4_receptance att_4_receptance_0 att_4_receptance_1
Reshape reshape_56 1 1 att_4_receptance_0 att_4_receptance_reshape 0=1 1=64 2=16
MatMul matmul_att_4_wkv_out 2 1 att_4_state_new_0 att_4_receptance_reshape att_4_wkv_out 0=0
Reshape reshape_57 1 1 att_4_wkv_out att_4_wkv_out_flatten 0=1024
GroupNorm groupnorm_4 1 1 att_4_wkv_out_flatten att_4_x_gn 0=16 1=1024 2=0.00064 3=1
BinaryOp mul_59 2 1 att_4_receptance_1 att_4_key_a_1 att_4_receptance_key_a 0=2
BinaryOp mul_60 2 1 att_4_receptance_key_a att_4_r_k att_4_rk 0=2
Reshape reshape_58 1 1 att_4_rk att_4_rk_reshape 0=64 1=16
Reduction sum_4 1 1 att_4_rk_reshape att_4_rk_sum 1=0 -23303=1,-1 4=1 5=1
BinaryOp mul_61 2 1 att_4_value_final_0_reshape att_4_rk_sum att_4_value_final_0_rk 0=2
Reshape reshape_59 1 1 att_4_value_final_0_rk att_4_value_final_0_rk_reshape 0=1024
BinaryOp add_59 2 1 att_4_value_final_0_rk_reshape att_4_x_gn att_4_x_gn_rkv 0=0
BinaryOp mul_62 2 1 att_4_x_gn_rkv att_4_gate att_4_x_gate 0=2
Gemm gemm_att_4_x_out 1 1 att_4_x_gate att_4_x_out 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
BinaryOp add_60 2 1 att_4_x_out att_4_x_last time_mixing_4_out 0=0
Split split_time_mixing_4_out 1 2 time_mixing_4_out ffn_4_x_last ffn_4_x
LayerNorm layernorm_9 1 1 ffn_4_x ffn_4_xx 0=1024 1=0.00001 2=1
Split split_ffn_4_xx 1 3 ffn_4_xx ffn_4_xx_0 ffn_4_xx_1 state_14_out
BinaryOp sub_9 2 1 state_14_in ffn_4_xx_0 ffn_4_sx 0=1
Split split_ffn_4_sx 1 1 ffn_4_sx ffn_4_sx_0
MemoryData data_ffn_4_x_k 0 1 ffn_4_x_k 0=1024 21=0
BinaryOp mul_63 2 1 ffn_4_sx_0 ffn_4_x_k ffn_4_xk 0=2
BinaryOp add_61 2 1 ffn_4_xk ffn_4_xx_1 ffn_4_xxk 0=0
Gemm gemm_ffn_4_key 1 1 ffn_4_xxk ffn_4_key 4=0 5=1 6=0 7=0 8=4096 9=1024 10=-1
ReLU relu_4 1 1 ffn_4_key ffn_4_key_relu
UnaryOp square_ffn_4_key_relu 1 1 ffn_4_key_relu ffn_4_key_relu_square 0=4
Gemm gemm_ffn_4_value 1 1 ffn_4_key_relu_square ffn_4_value 4=0 5=1 6=0 7=0 8=1024 9=4096 10=-1
BinaryOp add_62 2 1 ffn_4_value ffn_4_x_last channel_mixing_4_out 0=0
Split split_channel_mixing_4_out 1 2 channel_mixing_4_out att_5_x_last att_5_x
LayerNorm layernorm_10 1 1 att_5_x att_5_xx 0=1024 1=0.00001 2=1
Split split_att_5_xx 1 3 att_5_xx att_5_xx_0 att_5_xx_1 state_15_out
BinaryOp sub_10 2 1 state_15_in att_5_xx_0 att_5_sx 0=1
MemoryData data_att_5_maa 0 1 att_5_maa 0=1024 1=6 21=0
BinaryOp mul_64 2 1 att_5_sx att_5_maa att_5_maa_x 0=2
BinaryOp add_63 2 1 att_5_xx_1 att_5_maa_x att_5_maa_rwkvag 0=0
Slice slice_att_5_maa_rwkvag 1 6 att_5_maa_rwkvag att_5_mr att_5_mw att_5_mk att_5_mv att_5_ma att_5_mg -23300=6,-233,-233,-233,-233,-233,-233 1=0
Split split_att_5_mv 1 2 att_5_mv att_5_mv_0 att_5_mv_1
Gemm gemm_att_5_key 1 1 att_5_mk att_5_key 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_5_value 1 1 att_5_mv_0 att_5_value 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_5_receptance 1 1 att_5_mr att_5_receptance 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_5_ma_lora_0 1 1 att_5_ma att_5_ma_lora_0 4=0 5=1 6=0 7=0 8=64 9=1024 10=-1
Gemm gemm_att_5_ma_lora_1 1 1 att_5_ma_lora_0 att_5_ma_lora_1 4=0 5=1 6=0 7=0 8=1024 9=64 10=-1
MemoryData data_att_5_a0 0 1 att_5_a0 0=1024 21=0
BinaryOp add_64 2 1 att_5_ma_lora_1 att_5_a0 att_5_ma_lora_2 0=0
Sigmoid sigmoid_19 1 1 att_5_ma_lora_2 att_5_a
Split split_att_5_a 1 2 att_5_a att_5_a_0 att_5_a_1
Gemm gemm_att_5_mg_lora 1 1 att_5_mg att_5_mg_lora 4=0 5=1 6=0 7=0 8=128 9=1024 10=-1
Sigmoid sigmoid_20 1 1 att_5_mg_lora att_5_mg_lora_sigmoid
Gemm gemm_att_5_gate 1 1 att_5_mg_lora_sigmoid att_5_gate 4=0 5=1 6=0 7=0 8=1024 9=128 10=-1
MemoryData data_att_5_k_k 0 1 att_5_k_k 0=1024 21=0
MemoryData data_att_5_k_a 0 1 att_5_k_a 0=1024 21=0
MemoryData data_att_5_r_k 0 1 att_5_r_k 0=1024 21=0
Split split_att_5_key 1 2 att_5_key att_5_key_0 att_5_key_1
BinaryOp mul_65 2 1 att_5_key_0 att_5_k_k att_5_key_k 0=2
Reshape reshape_60 1 1 att_5_key_k att_5_key_k_reshape 0=64 1=1 2=16
Normalize l2norm_5 1 1 att_5_key_k_reshape att_5_key_k_norm 0=1 1=1 2=0.0000001 3=1 4=0 9=1
Reshape reshape_61 1 1 att_5_key_k_norm att_5_key_k_norm_reshape 0=1024
BinaryOp add_65 1 1 att_5_a_0 att_5_a_0_minus 0=0 1=1 2=-1.0
BinaryOp mul_66 2 1 att_5_a_0_minus att_5_k_a att_5_a_minus1_mulka 0=2
BinaryOp add_66 1 1 att_5_a_minus1_mulka att_5_a_minus1_mulka_plus1 0=0 1=1 2=1.0
BinaryOp mul_67 2 1 att_5_a_minus1_mulka_plus1 att_5_key_1 att_5_key_a 0=2
Split split_att_5_value 1 2 att_5_value att_5_value_0 att_5_value_1
Split split_v_first_4 1 2 v_first_4 att_5_v_first v_first_5
BinaryOp minus_4 2 1 att_5_v_first att_5_value_1 att_5_vfirst_minus_value 0=1
Gemm gemm_att_5_mv_lora_0 1 1 att_5_mv_1 att_5_mv_lora_0 4=0 5=1 6=0 7=0 8=32 9=1024 10=-1
Gemm gemm_att_5_mv_lora_1 1 1 att_5_mv_lora_0 att_5_mv_lora_1 4=0 5=1 6=0 7=0 8=1024 9=32 10=-1
MemoryData data_att_5_v0 0 1 att_5_v0 0=1024 21=0
BinaryOp add_67 2 1 att_5_mv_lora_1 att_5_v0 att_5_mv_lora_2 0=0
Sigmoid sigmoid_21 1 1 att_5_mv_lora_2 att_5_mv_lora_sigmoid
BinaryOp mul_68 2 1 att_5_mv_lora_sigmoid att_5_vfirst_minus_value att_5_vfirst_minus_value_mul 0=2
BinaryOp add_68 2 1 att_5_value_0 att_5_vfirst_minus_value_mul att_5_value_final 0=0
Gemm gemm_att_5_mw_lora 1 1 att_5_mw att_5_mw_lora 4=0 5=1 6=0 7=0 8=64 9=1024 10=-1
TanH tanh_att_5_mw_lora 1 1 att_5_mw_lora att_5_mw_lora_tanh
Gemm gemm_att_5_mw_lora_tanh_linear 1 1 att_5_mw_lora_tanh att_5_mw_lora_tanh_linear 4=0 5=1 6=0 7=0 8=1024 9=64 10=-1
MemoryData data_att_5_td 0 1 att_5_td 0=1024 21=0
BinaryOp add_69 2 1 att_5_mw_lora_tanh_linear att_5_td att_5_time_decay_pre 0=0
Sigmoid sigmoid_22 1 1 att_5_time_decay_pre att_5_time_decay_sigmoid
Exp exp_5 1 1 att_5_time_decay_sigmoid att_5_time_decay 1=-0.606531
Reshape reshape_62 1 1 att_5_time_decay att_5_time_decay_reshape 0=64 1=1 2=16
Split split_att_5_key_a 1 2 att_5_key_a att_5_key_a_0 att_5_key_a_1
Reshape reshape_63 1 1 att_5_key_a_0 att_5_key_a_reshape 0=64 1=1 2=16
Split split_att_5_value_final 1 3 att_5_value_final att_5_value_final_0 att_5_value_final_1 att_5_value_final_2
Reshape reshape_64 1 1 att_5_value_final_0 att_5_value_final_0_reshape 0=64 1=16
Reshape reshape_65 1 1 att_5_value_final_1 att_5_value_final_1_reshape 0=1 1=64 2=16
MatMul matmul_att_5_vk 2 1 att_5_value_final_1_reshape att_5_key_a_reshape att_5_vk 0=0
Split split_att_5_key_k_norm_reshape 1 2 att_5_key_k_norm_reshape att_5_kk_0 att_5_kk_1
BinaryOp mul_69 1 1 att_5_kk_0 att_5_kk_0_neg 0=2 1=1 2=-1.0
Reshape reshape_66 1 1 att_5_kk_0_neg att_5_kk_0_neg_reshape 0=1 1=64 2=16
BinaryOp mul_70 2 1 att_5_kk_1 att_5_a_1 att_5_kk_a 0=2
Reshape reshape_67 1 1 att_5_kk_a att_5_kk_a_reshape 0=64 1=1 2=16
MatMul matmul_att_5_ab 2 1 att_5_kk_0_neg_reshape att_5_kk_a_reshape att_5_ab 0=0
Split split_state_16_in 1 2 state_16_in att_5_state_prev_0 att_5_state_prev_1
BinaryOp mul_71 2 1 att_5_state_prev_0 att_5_time_decay_reshape att_5_state_td 0=2
MatMul matmul_att_5_sab 2 1 att_5_state_prev_1 att_5_ab att_5_sab 0=0
BinaryOp add_70 2 1 att_5_sab att_5_vk att_5_sab_vk 0=0
BinaryOp add_71 2 1 att_5_sab_vk att_5_state_td att_5_state_new 0=0
Split split_att_5_state_new 1 2 att_5_state_new att_5_state_new_0 state_16_out
Split split_att_5_receptance 1 2 att_5_receptance att_5_receptance_0 att_5_receptance_1
Reshape reshape_68 1 1 att_5_receptance_0 att_5_receptance_reshape 0=1 1=64 2=16
MatMul matmul_att_5_wkv_out 2 1 att_5_state_new_0 att_5_receptance_reshape att_5_wkv_out 0=0
Reshape reshape_69 1 1 att_5_wkv_out att_5_wkv_out_flatten 0=1024
GroupNorm groupnorm_5 1 1 att_5_wkv_out_flatten att_5_x_gn 0=16 1=1024 2=0.00064 3=1
BinaryOp mul_72 2 1 att_5_receptance_1 att_5_key_a_1 att_5_receptance_key_a 0=2
BinaryOp mul_73 2 1 att_5_receptance_key_a att_5_r_k att_5_rk 0=2
Reshape reshape_70 1 1 att_5_rk att_5_rk_reshape 0=64 1=16
Reduction sum_5 1 1 att_5_rk_reshape att_5_rk_sum 1=0 -23303=1,-1 4=1 5=1
BinaryOp mul_74 2 1 att_5_value_final_0_reshape att_5_rk_sum att_5_value_final_0_rk 0=2
Reshape reshape_71 1 1 att_5_value_final_0_rk att_5_value_final_0_rk_reshape 0=1024
BinaryOp add_72 2 1 att_5_value_final_0_rk_reshape att_5_x_gn att_5_x_gn_rkv 0=0
BinaryOp mul_75 2 1 att_5_x_gn_rkv att_5_gate att_5_x_gate 0=2
Gemm gemm_att_5_x_out 1 1 att_5_x_gate att_5_x_out 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
BinaryOp add_73 2 1 att_5_x_out att_5_x_last time_mixing_5_out 0=0
Split split_time_mixing_5_out 1 2 time_mixing_5_out ffn_5_x_last ffn_5_x
LayerNorm layernorm_11 1 1 ffn_5_x ffn_5_xx 0=1024 1=0.00001 2=1
Split split_ffn_5_xx 1 3 ffn_5_xx ffn_5_xx_0 ffn_5_xx_1 state_17_out
BinaryOp sub_11 2 1 state_17_in ffn_5_xx_0 ffn_5_sx 0=1
Split split_ffn_5_sx 1 1 ffn_5_sx ffn_5_sx_0
MemoryData data_ffn_5_x_k 0 1 ffn_5_x_k 0=1024 21=0
BinaryOp mul_76 2 1 ffn_5_sx_0 ffn_5_x_k ffn_5_xk 0=2
BinaryOp add_74 2 1 ffn_5_xk ffn_5_xx_1 ffn_5_xxk 0=0
Gemm gemm_ffn_5_key 1 1 ffn_5_xxk ffn_5_key 4=0 5=1 6=0 7=0 8=4096 9=1024 10=-1
ReLU relu_5 1 1 ffn_5_key ffn_5_key_relu
UnaryOp square_ffn_5_key_relu 1 1 ffn_5_key_relu ffn_5_key_relu_square 0=4
Gemm gemm_ffn_5_value 1 1 ffn_5_key_relu_square ffn_5_value 4=0 5=1 6=0 7=0 8=1024 9=4096 10=-1
BinaryOp add_75 2 1 ffn_5_value ffn_5_x_last channel_mixing_5_out 0=0
Split split_channel_mixing_5_out 1 2 channel_mixing_5_out att_6_x_last att_6_x
LayerNorm layernorm_12 1 1 att_6_x att_6_xx 0=1024 1=0.00001 2=1
Split split_att_6_xx 1 3 att_6_xx att_6_xx_0 att_6_xx_1 state_18_out
BinaryOp sub_12 2 1 state_18_in att_6_xx_0 att_6_sx 0=1
MemoryData data_att_6_maa 0 1 att_6_maa 0=1024 1=6 21=0
BinaryOp mul_77 2 1 att_6_sx att_6_maa att_6_maa_x 0=2
BinaryOp add_76 2 1 att_6_xx_1 att_6_maa_x att_6_maa_rwkvag 0=0
Slice slice_att_6_maa_rwkvag 1 6 att_6_maa_rwkvag att_6_mr att_6_mw att_6_mk att_6_mv att_6_ma att_6_mg -23300=6,-233,-233,-233,-233,-233,-233 1=0
Split split_att_6_mv 1 2 att_6_mv att_6_mv_0 att_6_mv_1
Gemm gemm_att_6_key 1 1 att_6_mk att_6_key 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_6_value 1 1 att_6_mv_0 att_6_value 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_6_receptance 1 1 att_6_mr att_6_receptance 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_6_ma_lora_0 1 1 att_6_ma att_6_ma_lora_0 4=0 5=1 6=0 7=0 8=64 9=1024 10=-1
Gemm gemm_att_6_ma_lora_1 1 1 att_6_ma_lora_0 att_6_ma_lora_1 4=0 5=1 6=0 7=0 8=1024 9=64 10=-1
MemoryData data_att_6_a0 0 1 att_6_a0 0=1024 21=0
BinaryOp add_77 2 1 att_6_ma_lora_1 att_6_a0 att_6_ma_lora_2 0=0
Sigmoid sigmoid_23 1 1 att_6_ma_lora_2 att_6_a
Split split_att_6_a 1 2 att_6_a att_6_a_0 att_6_a_1
Gemm gemm_att_6_mg_lora 1 1 att_6_mg att_6_mg_lora 4=0 5=1 6=0 7=0 8=128 9=1024 10=-1
Sigmoid sigmoid_24 1 1 att_6_mg_lora att_6_mg_lora_sigmoid
Gemm gemm_att_6_gate 1 1 att_6_mg_lora_sigmoid att_6_gate 4=0 5=1 6=0 7=0 8=1024 9=128 10=-1
MemoryData data_att_6_k_k 0 1 att_6_k_k 0=1024 21=0
MemoryData data_att_6_k_a 0 1 att_6_k_a 0=1024 21=0
MemoryData data_att_6_r_k 0 1 att_6_r_k 0=1024 21=0
Split split_att_6_key 1 2 att_6_key att_6_key_0 att_6_key_1
BinaryOp mul_78 2 1 att_6_key_0 att_6_k_k att_6_key_k 0=2
Reshape reshape_72 1 1 att_6_key_k att_6_key_k_reshape 0=64 1=1 2=16
Normalize l2norm_6 1 1 att_6_key_k_reshape att_6_key_k_norm 0=1 1=1 2=0.0000001 3=1 4=0 9=1
Reshape reshape_73 1 1 att_6_key_k_norm att_6_key_k_norm_reshape 0=1024
BinaryOp add_78 1 1 att_6_a_0 att_6_a_0_minus 0=0 1=1 2=-1.0
BinaryOp mul_79 2 1 att_6_a_0_minus att_6_k_a att_6_a_minus1_mulka 0=2
BinaryOp add_79 1 1 att_6_a_minus1_mulka att_6_a_minus1_mulka_plus1 0=0 1=1 2=1.0
BinaryOp mul_80 2 1 att_6_a_minus1_mulka_plus1 att_6_key_1 att_6_key_a 0=2
Split split_att_6_value 1 2 att_6_value att_6_value_0 att_6_value_1
Split split_v_first_5 1 2 v_first_5 att_6_v_first v_first_6
BinaryOp minus_5 2 1 att_6_v_first att_6_value_1 att_6_vfirst_minus_value 0=1
Gemm gemm_att_6_mv_lora_0 1 1 att_6_mv_1 att_6_mv_lora_0 4=0 5=1 6=0 7=0 8=32 9=1024 10=-1
Gemm gemm_att_6_mv_lora_1 1 1 att_6_mv_lora_0 att_6_mv_lora_1 4=0 5=1 6=0 7=0 8=1024 9=32 10=-1
MemoryData data_att_6_v0 0 1 att_6_v0 0=1024 21=0
BinaryOp add_80 2 1 att_6_mv_lora_1 att_6_v0 att_6_mv_lora_2 0=0
Sigmoid sigmoid_25 1 1 att_6_mv_lora_2 att_6_mv_lora_sigmoid
BinaryOp mul_81 2 1 att_6_mv_lora_sigmoid att_6_vfirst_minus_value att_6_vfirst_minus_value_mul 0=2
BinaryOp add_81 2 1 att_6_value_0 att_6_vfirst_minus_value_mul att_6_value_final 0=0
Gemm gemm_att_6_mw_lora 1 1 att_6_mw att_6_mw_lora 4=0 5=1 6=0 7=0 8=64 9=1024 10=-1
TanH tanh_att_6_mw_lora 1 1 att_6_mw_lora att_6_mw_lora_tanh
Gemm gemm_att_6_mw_lora_tanh_linear 1 1 att_6_mw_lora_tanh att_6_mw_lora_tanh_linear 4=0 5=1 6=0 7=0 8=1024 9=64 10=-1
MemoryData data_att_6_td 0 1 att_6_td 0=1024 21=0
BinaryOp add_82 2 1 att_6_mw_lora_tanh_linear att_6_td att_6_time_decay_pre 0=0
Sigmoid sigmoid_26 1 1 att_6_time_decay_pre att_6_time_decay_sigmoid
Exp exp_6 1 1 att_6_time_decay_sigmoid att_6_time_decay 1=-0.606531
Reshape reshape_74 1 1 att_6_time_decay att_6_time_decay_reshape 0=64 1=1 2=16
Split split_att_6_key_a 1 2 att_6_key_a att_6_key_a_0 att_6_key_a_1
Reshape reshape_75 1 1 att_6_key_a_0 att_6_key_a_reshape 0=64 1=1 2=16
Split split_att_6_value_final 1 3 att_6_value_final att_6_value_final_0 att_6_value_final_1 att_6_value_final_2
Reshape reshape_76 1 1 att_6_value_final_0 att_6_value_final_0_reshape 0=64 1=16
Reshape reshape_77 1 1 att_6_value_final_1 att_6_value_final_1_reshape 0=1 1=64 2=16
MatMul matmul_att_6_vk 2 1 att_6_value_final_1_reshape att_6_key_a_reshape att_6_vk 0=0
Split split_att_6_key_k_norm_reshape 1 2 att_6_key_k_norm_reshape att_6_kk_0 att_6_kk_1
BinaryOp mul_82 1 1 att_6_kk_0 att_6_kk_0_neg 0=2 1=1 2=-1.0
Reshape reshape_78 1 1 att_6_kk_0_neg att_6_kk_0_neg_reshape 0=1 1=64 2=16
BinaryOp mul_83 2 1 att_6_kk_1 att_6_a_1 att_6_kk_a 0=2
Reshape reshape_79 1 1 att_6_kk_a att_6_kk_a_reshape 0=64 1=1 2=16
MatMul matmul_att_6_ab 2 1 att_6_kk_0_neg_reshape att_6_kk_a_reshape att_6_ab 0=0
Split split_state_19_in 1 2 state_19_in att_6_state_prev_0 att_6_state_prev_1
BinaryOp mul_84 2 1 att_6_state_prev_0 att_6_time_decay_reshape att_6_state_td 0=2
MatMul matmul_att_6_sab 2 1 att_6_state_prev_1 att_6_ab att_6_sab 0=0
BinaryOp add_83 2 1 att_6_sab att_6_vk att_6_sab_vk 0=0
BinaryOp add_84 2 1 att_6_sab_vk att_6_state_td att_6_state_new 0=0
Split split_att_6_state_new 1 2 att_6_state_new att_6_state_new_0 state_19_out
Split split_att_6_receptance 1 2 att_6_receptance att_6_receptance_0 att_6_receptance_1
Reshape reshape_80 1 1 att_6_receptance_0 att_6_receptance_reshape 0=1 1=64 2=16
MatMul matmul_att_6_wkv_out 2 1 att_6_state_new_0 att_6_receptance_reshape att_6_wkv_out 0=0
Reshape reshape_81 1 1 att_6_wkv_out att_6_wkv_out_flatten 0=1024
GroupNorm groupnorm_6 1 1 att_6_wkv_out_flatten att_6_x_gn 0=16 1=1024 2=0.00064 3=1
BinaryOp mul_85 2 1 att_6_receptance_1 att_6_key_a_1 att_6_receptance_key_a 0=2
BinaryOp mul_86 2 1 att_6_receptance_key_a att_6_r_k att_6_rk 0=2
Reshape reshape_82 1 1 att_6_rk att_6_rk_reshape 0=64 1=16
Reduction sum_6 1 1 att_6_rk_reshape att_6_rk_sum 1=0 -23303=1,-1 4=1 5=1
BinaryOp mul_87 2 1 att_6_value_final_0_reshape att_6_rk_sum att_6_value_final_0_rk 0=2
Reshape reshape_83 1 1 att_6_value_final_0_rk att_6_value_final_0_rk_reshape 0=1024
BinaryOp add_85 2 1 att_6_value_final_0_rk_reshape att_6_x_gn att_6_x_gn_rkv 0=0
BinaryOp mul_88 2 1 att_6_x_gn_rkv att_6_gate att_6_x_gate 0=2
Gemm gemm_att_6_x_out 1 1 att_6_x_gate att_6_x_out 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
BinaryOp add_86 2 1 att_6_x_out att_6_x_last time_mixing_6_out 0=0
Split split_time_mixing_6_out 1 2 time_mixing_6_out ffn_6_x_last ffn_6_x
LayerNorm layernorm_13 1 1 ffn_6_x ffn_6_xx 0=1024 1=0.00001 2=1
Split split_ffn_6_xx 1 3 ffn_6_xx ffn_6_xx_0 ffn_6_xx_1 state_20_out
BinaryOp sub_13 2 1 state_20_in ffn_6_xx_0 ffn_6_sx 0=1
Split split_ffn_6_sx 1 1 ffn_6_sx ffn_6_sx_0
MemoryData data_ffn_6_x_k 0 1 ffn_6_x_k 0=1024 21=0
BinaryOp mul_89 2 1 ffn_6_sx_0 ffn_6_x_k ffn_6_xk 0=2
BinaryOp add_87 2 1 ffn_6_xk ffn_6_xx_1 ffn_6_xxk 0=0
Gemm gemm_ffn_6_key 1 1 ffn_6_xxk ffn_6_key 4=0 5=1 6=0 7=0 8=4096 9=1024 10=-1
ReLU relu_6 1 1 ffn_6_key ffn_6_key_relu
UnaryOp square_ffn_6_key_relu 1 1 ffn_6_key_relu ffn_6_key_relu_square 0=4
Gemm gemm_ffn_6_value 1 1 ffn_6_key_relu_square ffn_6_value 4=0 5=1 6=0 7=0 8=1024 9=4096 10=-1
BinaryOp add_88 2 1 ffn_6_value ffn_6_x_last channel_mixing_6_out 0=0
Split split_channel_mixing_6_out 1 2 channel_mixing_6_out att_7_x_last att_7_x
LayerNorm layernorm_14 1 1 att_7_x att_7_xx 0=1024 1=0.00001 2=1
Split split_att_7_xx 1 3 att_7_xx att_7_xx_0 att_7_xx_1 state_21_out
BinaryOp sub_14 2 1 state_21_in att_7_xx_0 att_7_sx 0=1
MemoryData data_att_7_maa 0 1 att_7_maa 0=1024 1=6 21=0
BinaryOp mul_90 2 1 att_7_sx att_7_maa att_7_maa_x 0=2
BinaryOp add_89 2 1 att_7_xx_1 att_7_maa_x att_7_maa_rwkvag 0=0
Slice slice_att_7_maa_rwkvag 1 6 att_7_maa_rwkvag att_7_mr att_7_mw att_7_mk att_7_mv att_7_ma att_7_mg -23300=6,-233,-233,-233,-233,-233,-233 1=0
Split split_att_7_mv 1 2 att_7_mv att_7_mv_0 att_7_mv_1
Gemm gemm_att_7_key 1 1 att_7_mk att_7_key 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_7_value 1 1 att_7_mv_0 att_7_value 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_7_receptance 1 1 att_7_mr att_7_receptance 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_7_ma_lora_0 1 1 att_7_ma att_7_ma_lora_0 4=0 5=1 6=0 7=0 8=64 9=1024 10=-1
Gemm gemm_att_7_ma_lora_1 1 1 att_7_ma_lora_0 att_7_ma_lora_1 4=0 5=1 6=0 7=0 8=1024 9=64 10=-1
MemoryData data_att_7_a0 0 1 att_7_a0 0=1024 21=0
BinaryOp add_90 2 1 att_7_ma_lora_1 att_7_a0 att_7_ma_lora_2 0=0
Sigmoid sigmoid_27 1 1 att_7_ma_lora_2 att_7_a
Split split_att_7_a 1 2 att_7_a att_7_a_0 att_7_a_1
Gemm gemm_att_7_mg_lora 1 1 att_7_mg att_7_mg_lora 4=0 5=1 6=0 7=0 8=128 9=1024 10=-1
Sigmoid sigmoid_28 1 1 att_7_mg_lora att_7_mg_lora_sigmoid
Gemm gemm_att_7_gate 1 1 att_7_mg_lora_sigmoid att_7_gate 4=0 5=1 6=0 7=0 8=1024 9=128 10=-1
MemoryData data_att_7_k_k 0 1 att_7_k_k 0=1024 21=0
MemoryData data_att_7_k_a 0 1 att_7_k_a 0=1024 21=0
MemoryData data_att_7_r_k 0 1 att_7_r_k 0=1024 21=0
Split split_att_7_key 1 2 att_7_key att_7_key_0 att_7_key_1
BinaryOp mul_91 2 1 att_7_key_0 att_7_k_k att_7_key_k 0=2
Reshape reshape_84 1 1 att_7_key_k att_7_key_k_reshape 0=64 1=1 2=16
Normalize l2norm_7 1 1 att_7_key_k_reshape att_7_key_k_norm 0=1 1=1 2=0.0000001 3=1 4=0 9=1
Reshape reshape_85 1 1 att_7_key_k_norm att_7_key_k_norm_reshape 0=1024
BinaryOp add_91 1 1 att_7_a_0 att_7_a_0_minus 0=0 1=1 2=-1.0
BinaryOp mul_92 2 1 att_7_a_0_minus att_7_k_a att_7_a_minus1_mulka 0=2
BinaryOp add_92 1 1 att_7_a_minus1_mulka att_7_a_minus1_mulka_plus1 0=0 1=1 2=1.0
BinaryOp mul_93 2 1 att_7_a_minus1_mulka_plus1 att_7_key_1 att_7_key_a 0=2
Split split_att_7_value 1 2 att_7_value att_7_value_0 att_7_value_1
Split split_v_first_6 1 2 v_first_6 att_7_v_first v_first_7
BinaryOp minus_6 2 1 att_7_v_first att_7_value_1 att_7_vfirst_minus_value 0=1
Gemm gemm_att_7_mv_lora_0 1 1 att_7_mv_1 att_7_mv_lora_0 4=0 5=1 6=0 7=0 8=32 9=1024 10=-1
Gemm gemm_att_7_mv_lora_1 1 1 att_7_mv_lora_0 att_7_mv_lora_1 4=0 5=1 6=0 7=0 8=1024 9=32 10=-1
MemoryData data_att_7_v0 0 1 att_7_v0 0=1024 21=0
BinaryOp add_93 2 1 att_7_mv_lora_1 att_7_v0 att_7_mv_lora_2 0=0
Sigmoid sigmoid_29 1 1 att_7_mv_lora_2 att_7_mv_lora_sigmoid
BinaryOp mul_94 2 1 att_7_mv_lora_sigmoid att_7_vfirst_minus_value att_7_vfirst_minus_value_mul 0=2
BinaryOp add_94 2 1 att_7_value_0 att_7_vfirst_minus_value_mul att_7_value_final 0=0
Gemm gemm_att_7_mw_lora 1 1 att_7_mw att_7_mw_lora 4=0 5=1 6=0 7=0 8=64 9=1024 10=-1
TanH tanh_att_7_mw_lora 1 1 att_7_mw_lora att_7_mw_lora_tanh
Gemm gemm_att_7_mw_lora_tanh_linear 1 1 att_7_mw_lora_tanh att_7_mw_lora_tanh_linear 4=0 5=1 6=0 7=0 8=1024 9=64 10=-1
MemoryData data_att_7_td 0 1 att_7_td 0=1024 21=0
BinaryOp add_95 2 1 att_7_mw_lora_tanh_linear att_7_td att_7_time_decay_pre 0=0
Sigmoid sigmoid_30 1 1 att_7_time_decay_pre att_7_time_decay_sigmoid
Exp exp_7 1 1 att_7_time_decay_sigmoid att_7_time_decay 1=-0.606531
Reshape reshape_86 1 1 att_7_time_decay att_7_time_decay_reshape 0=64 1=1 2=16
Split split_att_7_key_a 1 2 att_7_key_a att_7_key_a_0 att_7_key_a_1
Reshape reshape_87 1 1 att_7_key_a_0 att_7_key_a_reshape 0=64 1=1 2=16
Split split_att_7_value_final 1 3 att_7_value_final att_7_value_final_0 att_7_value_final_1 att_7_value_final_2
Reshape reshape_88 1 1 att_7_value_final_0 att_7_value_final_0_reshape 0=64 1=16
Reshape reshape_89 1 1 att_7_value_final_1 att_7_value_final_1_reshape 0=1 1=64 2=16
MatMul matmul_att_7_vk 2 1 att_7_value_final_1_reshape att_7_key_a_reshape att_7_vk 0=0
Split split_att_7_key_k_norm_reshape 1 2 att_7_key_k_norm_reshape att_7_kk_0 att_7_kk_1
BinaryOp mul_95 1 1 att_7_kk_0 att_7_kk_0_neg 0=2 1=1 2=-1.0
Reshape reshape_90 1 1 att_7_kk_0_neg att_7_kk_0_neg_reshape 0=1 1=64 2=16
BinaryOp mul_96 2 1 att_7_kk_1 att_7_a_1 att_7_kk_a 0=2
Reshape reshape_91 1 1 att_7_kk_a att_7_kk_a_reshape 0=64 1=1 2=16
MatMul matmul_att_7_ab 2 1 att_7_kk_0_neg_reshape att_7_kk_a_reshape att_7_ab 0=0
Split split_state_22_in 1 2 state_22_in att_7_state_prev_0 att_7_state_prev_1
BinaryOp mul_97 2 1 att_7_state_prev_0 att_7_time_decay_reshape att_7_state_td 0=2
MatMul matmul_att_7_sab 2 1 att_7_state_prev_1 att_7_ab att_7_sab 0=0
BinaryOp add_96 2 1 att_7_sab att_7_vk att_7_sab_vk 0=0
BinaryOp add_97 2 1 att_7_sab_vk att_7_state_td att_7_state_new 0=0
Split split_att_7_state_new 1 2 att_7_state_new att_7_state_new_0 state_22_out
Split split_att_7_receptance 1 2 att_7_receptance att_7_receptance_0 att_7_receptance_1
Reshape reshape_92 1 1 att_7_receptance_0 att_7_receptance_reshape 0=1 1=64 2=16
MatMul matmul_att_7_wkv_out 2 1 att_7_state_new_0 att_7_receptance_reshape att_7_wkv_out 0=0
Reshape reshape_93 1 1 att_7_wkv_out att_7_wkv_out_flatten 0=1024
GroupNorm groupnorm_7 1 1 att_7_wkv_out_flatten att_7_x_gn 0=16 1=1024 2=0.00064 3=1
BinaryOp mul_98 2 1 att_7_receptance_1 att_7_key_a_1 att_7_receptance_key_a 0=2
BinaryOp mul_99 2 1 att_7_receptance_key_a att_7_r_k att_7_rk 0=2
Reshape reshape_94 1 1 att_7_rk att_7_rk_reshape 0=64 1=16
Reduction sum_7 1 1 att_7_rk_reshape att_7_rk_sum 1=0 -23303=1,-1 4=1 5=1
BinaryOp mul_100 2 1 att_7_value_final_0_reshape att_7_rk_sum att_7_value_final_0_rk 0=2
Reshape reshape_95 1 1 att_7_value_final_0_rk att_7_value_final_0_rk_reshape 0=1024
BinaryOp add_98 2 1 att_7_value_final_0_rk_reshape att_7_x_gn att_7_x_gn_rkv 0=0
BinaryOp mul_101 2 1 att_7_x_gn_rkv att_7_gate att_7_x_gate 0=2
Gemm gemm_att_7_x_out 1 1 att_7_x_gate att_7_x_out 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
BinaryOp add_99 2 1 att_7_x_out att_7_x_last time_mixing_7_out 0=0
Split split_time_mixing_7_out 1 2 time_mixing_7_out ffn_7_x_last ffn_7_x
LayerNorm layernorm_15 1 1 ffn_7_x ffn_7_xx 0=1024 1=0.00001 2=1
Split split_ffn_7_xx 1 3 ffn_7_xx ffn_7_xx_0 ffn_7_xx_1 state_23_out
BinaryOp sub_15 2 1 state_23_in ffn_7_xx_0 ffn_7_sx 0=1
Split split_ffn_7_sx 1 1 ffn_7_sx ffn_7_sx_0
MemoryData data_ffn_7_x_k 0 1 ffn_7_x_k 0=1024 21=0
BinaryOp mul_102 2 1 ffn_7_sx_0 ffn_7_x_k ffn_7_xk 0=2
BinaryOp add_100 2 1 ffn_7_xk ffn_7_xx_1 ffn_7_xxk 0=0
Gemm gemm_ffn_7_key 1 1 ffn_7_xxk ffn_7_key 4=0 5=1 6=0 7=0 8=4096 9=1024 10=-1
ReLU relu_7 1 1 ffn_7_key ffn_7_key_relu
UnaryOp square_ffn_7_key_relu 1 1 ffn_7_key_relu ffn_7_key_relu_square 0=4
Gemm gemm_ffn_7_value 1 1 ffn_7_key_relu_square ffn_7_value 4=0 5=1 6=0 7=0 8=1024 9=4096 10=-1
BinaryOp add_101 2 1 ffn_7_value ffn_7_x_last channel_mixing_7_out 0=0
Split split_channel_mixing_7_out 1 2 channel_mixing_7_out att_8_x_last att_8_x
LayerNorm layernorm_16 1 1 att_8_x att_8_xx 0=1024 1=0.00001 2=1
Split split_att_8_xx 1 3 att_8_xx att_8_xx_0 att_8_xx_1 state_24_out
BinaryOp sub_16 2 1 state_24_in att_8_xx_0 att_8_sx 0=1
MemoryData data_att_8_maa 0 1 att_8_maa 0=1024 1=6 21=0
BinaryOp mul_103 2 1 att_8_sx att_8_maa att_8_maa_x 0=2
BinaryOp add_102 2 1 att_8_xx_1 att_8_maa_x att_8_maa_rwkvag 0=0
Slice slice_att_8_maa_rwkvag 1 6 att_8_maa_rwkvag att_8_mr att_8_mw att_8_mk att_8_mv att_8_ma att_8_mg -23300=6,-233,-233,-233,-233,-233,-233 1=0
Split split_att_8_mv 1 2 att_8_mv att_8_mv_0 att_8_mv_1
Gemm gemm_att_8_key 1 1 att_8_mk att_8_key 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_8_value 1 1 att_8_mv_0 att_8_value 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_8_receptance 1 1 att_8_mr att_8_receptance 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_8_ma_lora_0 1 1 att_8_ma att_8_ma_lora_0 4=0 5=1 6=0 7=0 8=64 9=1024 10=-1
Gemm gemm_att_8_ma_lora_1 1 1 att_8_ma_lora_0 att_8_ma_lora_1 4=0 5=1 6=0 7=0 8=1024 9=64 10=-1
MemoryData data_att_8_a0 0 1 att_8_a0 0=1024 21=0
BinaryOp add_103 2 1 att_8_ma_lora_1 att_8_a0 att_8_ma_lora_2 0=0
Sigmoid sigmoid_31 1 1 att_8_ma_lora_2 att_8_a
Split split_att_8_a 1 2 att_8_a att_8_a_0 att_8_a_1
Gemm gemm_att_8_mg_lora 1 1 att_8_mg att_8_mg_lora 4=0 5=1 6=0 7=0 8=128 9=1024 10=-1
Sigmoid sigmoid_32 1 1 att_8_mg_lora att_8_mg_lora_sigmoid
Gemm gemm_att_8_gate 1 1 att_8_mg_lora_sigmoid att_8_gate 4=0 5=1 6=0 7=0 8=1024 9=128 10=-1
MemoryData data_att_8_k_k 0 1 att_8_k_k 0=1024 21=0
MemoryData data_att_8_k_a 0 1 att_8_k_a 0=1024 21=0
MemoryData data_att_8_r_k 0 1 att_8_r_k 0=1024 21=0
Split split_att_8_key 1 2 att_8_key att_8_key_0 att_8_key_1
BinaryOp mul_104 2 1 att_8_key_0 att_8_k_k att_8_key_k 0=2
Reshape reshape_96 1 1 att_8_key_k att_8_key_k_reshape 0=64 1=1 2=16
Normalize l2norm_8 1 1 att_8_key_k_reshape att_8_key_k_norm 0=1 1=1 2=0.0000001 3=1 4=0 9=1
Reshape reshape_97 1 1 att_8_key_k_norm att_8_key_k_norm_reshape 0=1024
BinaryOp add_104 1 1 att_8_a_0 att_8_a_0_minus 0=0 1=1 2=-1.0
BinaryOp mul_105 2 1 att_8_a_0_minus att_8_k_a att_8_a_minus1_mulka 0=2
BinaryOp add_105 1 1 att_8_a_minus1_mulka att_8_a_minus1_mulka_plus1 0=0 1=1 2=1.0
BinaryOp mul_106 2 1 att_8_a_minus1_mulka_plus1 att_8_key_1 att_8_key_a 0=2
Split split_att_8_value 1 2 att_8_value att_8_value_0 att_8_value_1
Split split_v_first_7 1 2 v_first_7 att_8_v_first v_first_8
BinaryOp minus_7 2 1 att_8_v_first att_8_value_1 att_8_vfirst_minus_value 0=1
Gemm gemm_att_8_mv_lora_0 1 1 att_8_mv_1 att_8_mv_lora_0 4=0 5=1 6=0 7=0 8=32 9=1024 10=-1
Gemm gemm_att_8_mv_lora_1 1 1 att_8_mv_lora_0 att_8_mv_lora_1 4=0 5=1 6=0 7=0 8=1024 9=32 10=-1
MemoryData data_att_8_v0 0 1 att_8_v0 0=1024 21=0
BinaryOp add_106 2 1 att_8_mv_lora_1 att_8_v0 att_8_mv_lora_2 0=0
Sigmoid sigmoid_33 1 1 att_8_mv_lora_2 att_8_mv_lora_sigmoid
BinaryOp mul_107 2 1 att_8_mv_lora_sigmoid att_8_vfirst_minus_value att_8_vfirst_minus_value_mul 0=2
BinaryOp add_107 2 1 att_8_value_0 att_8_vfirst_minus_value_mul att_8_value_final 0=0
Gemm gemm_att_8_mw_lora 1 1 att_8_mw att_8_mw_lora 4=0 5=1 6=0 7=0 8=64 9=1024 10=-1
TanH tanh_att_8_mw_lora 1 1 att_8_mw_lora att_8_mw_lora_tanh
Gemm gemm_att_8_mw_lora_tanh_linear 1 1 att_8_mw_lora_tanh att_8_mw_lora_tanh_linear 4=0 5=1 6=0 7=0 8=1024 9=64 10=-1
MemoryData data_att_8_td 0 1 att_8_td 0=1024 21=0
BinaryOp add_108 2 1 att_8_mw_lora_tanh_linear att_8_td att_8_time_decay_pre 0=0
Sigmoid sigmoid_34 1 1 att_8_time_decay_pre att_8_time_decay_sigmoid
Exp exp_8 1 1 att_8_time_decay_sigmoid att_8_time_decay 1=-0.606531
Reshape reshape_98 1 1 att_8_time_decay att_8_time_decay_reshape 0=64 1=1 2=16
Split split_att_8_key_a 1 2 att_8_key_a att_8_key_a_0 att_8_key_a_1
Reshape reshape_99 1 1 att_8_key_a_0 att_8_key_a_reshape 0=64 1=1 2=16
Split split_att_8_value_final 1 3 att_8_value_final att_8_value_final_0 att_8_value_final_1 att_8_value_final_2
Reshape reshape_100 1 1 att_8_value_final_0 att_8_value_final_0_reshape 0=64 1=16
Reshape reshape_101 1 1 att_8_value_final_1 att_8_value_final_1_reshape 0=1 1=64 2=16
MatMul matmul_att_8_vk 2 1 att_8_value_final_1_reshape att_8_key_a_reshape att_8_vk 0=0
Split split_att_8_key_k_norm_reshape 1 2 att_8_key_k_norm_reshape att_8_kk_0 att_8_kk_1
BinaryOp mul_108 1 1 att_8_kk_0 att_8_kk_0_neg 0=2 1=1 2=-1.0
Reshape reshape_102 1 1 att_8_kk_0_neg att_8_kk_0_neg_reshape 0=1 1=64 2=16
BinaryOp mul_109 2 1 att_8_kk_1 att_8_a_1 att_8_kk_a 0=2
Reshape reshape_103 1 1 att_8_kk_a att_8_kk_a_reshape 0=64 1=1 2=16
MatMul matmul_att_8_ab 2 1 att_8_kk_0_neg_reshape att_8_kk_a_reshape att_8_ab 0=0
Split split_state_25_in 1 2 state_25_in att_8_state_prev_0 att_8_state_prev_1
BinaryOp mul_110 2 1 att_8_state_prev_0 att_8_time_decay_reshape att_8_state_td 0=2
MatMul matmul_att_8_sab 2 1 att_8_state_prev_1 att_8_ab att_8_sab 0=0
BinaryOp add_109 2 1 att_8_sab att_8_vk att_8_sab_vk 0=0
BinaryOp add_110 2 1 att_8_sab_vk att_8_state_td att_8_state_new 0=0
Split split_att_8_state_new 1 2 att_8_state_new att_8_state_new_0 state_25_out
Split split_att_8_receptance 1 2 att_8_receptance att_8_receptance_0 att_8_receptance_1
Reshape reshape_104 1 1 att_8_receptance_0 att_8_receptance_reshape 0=1 1=64 2=16
MatMul matmul_att_8_wkv_out 2 1 att_8_state_new_0 att_8_receptance_reshape att_8_wkv_out 0=0
Reshape reshape_105 1 1 att_8_wkv_out att_8_wkv_out_flatten 0=1024
GroupNorm groupnorm_8 1 1 att_8_wkv_out_flatten att_8_x_gn 0=16 1=1024 2=0.00064 3=1
BinaryOp mul_111 2 1 att_8_receptance_1 att_8_key_a_1 att_8_receptance_key_a 0=2
BinaryOp mul_112 2 1 att_8_receptance_key_a att_8_r_k att_8_rk 0=2
Reshape reshape_106 1 1 att_8_rk att_8_rk_reshape 0=64 1=16
Reduction sum_8 1 1 att_8_rk_reshape att_8_rk_sum 1=0 -23303=1,-1 4=1 5=1
BinaryOp mul_113 2 1 att_8_value_final_0_reshape att_8_rk_sum att_8_value_final_0_rk 0=2
Reshape reshape_107 1 1 att_8_value_final_0_rk att_8_value_final_0_rk_reshape 0=1024
BinaryOp add_111 2 1 att_8_value_final_0_rk_reshape att_8_x_gn att_8_x_gn_rkv 0=0
BinaryOp mul_114 2 1 att_8_x_gn_rkv att_8_gate att_8_x_gate 0=2
Gemm gemm_att_8_x_out 1 1 att_8_x_gate att_8_x_out 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
BinaryOp add_112 2 1 att_8_x_out att_8_x_last time_mixing_8_out 0=0
Split split_time_mixing_8_out 1 2 time_mixing_8_out ffn_8_x_last ffn_8_x
LayerNorm layernorm_17 1 1 ffn_8_x ffn_8_xx 0=1024 1=0.00001 2=1
Split split_ffn_8_xx 1 3 ffn_8_xx ffn_8_xx_0 ffn_8_xx_1 state_26_out
BinaryOp sub_17 2 1 state_26_in ffn_8_xx_0 ffn_8_sx 0=1
Split split_ffn_8_sx 1 1 ffn_8_sx ffn_8_sx_0
MemoryData data_ffn_8_x_k 0 1 ffn_8_x_k 0=1024 21=0
BinaryOp mul_115 2 1 ffn_8_sx_0 ffn_8_x_k ffn_8_xk 0=2
BinaryOp add_113 2 1 ffn_8_xk ffn_8_xx_1 ffn_8_xxk 0=0
Gemm gemm_ffn_8_key 1 1 ffn_8_xxk ffn_8_key 4=0 5=1 6=0 7=0 8=4096 9=1024 10=-1
ReLU relu_8 1 1 ffn_8_key ffn_8_key_relu
UnaryOp square_ffn_8_key_relu 1 1 ffn_8_key_relu ffn_8_key_relu_square 0=4
Gemm gemm_ffn_8_value 1 1 ffn_8_key_relu_square ffn_8_value 4=0 5=1 6=0 7=0 8=1024 9=4096 10=-1
BinaryOp add_114 2 1 ffn_8_value ffn_8_x_last channel_mixing_8_out 0=0
Split split_channel_mixing_8_out 1 2 channel_mixing_8_out att_9_x_last att_9_x
LayerNorm layernorm_18 1 1 att_9_x att_9_xx 0=1024 1=0.00001 2=1
Split split_att_9_xx 1 3 att_9_xx att_9_xx_0 att_9_xx_1 state_27_out
BinaryOp sub_18 2 1 state_27_in att_9_xx_0 att_9_sx 0=1
MemoryData data_att_9_maa 0 1 att_9_maa 0=1024 1=6 21=0
BinaryOp mul_116 2 1 att_9_sx att_9_maa att_9_maa_x 0=2
BinaryOp add_115 2 1 att_9_xx_1 att_9_maa_x att_9_maa_rwkvag 0=0
Slice slice_att_9_maa_rwkvag 1 6 att_9_maa_rwkvag att_9_mr att_9_mw att_9_mk att_9_mv att_9_ma att_9_mg -23300=6,-233,-233,-233,-233,-233,-233 1=0
Split split_att_9_mv 1 2 att_9_mv att_9_mv_0 att_9_mv_1
Gemm gemm_att_9_key 1 1 att_9_mk att_9_key 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_9_value 1 1 att_9_mv_0 att_9_value 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_9_receptance 1 1 att_9_mr att_9_receptance 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_9_ma_lora_0 1 1 att_9_ma att_9_ma_lora_0 4=0 5=1 6=0 7=0 8=64 9=1024 10=-1
Gemm gemm_att_9_ma_lora_1 1 1 att_9_ma_lora_0 att_9_ma_lora_1 4=0 5=1 6=0 7=0 8=1024 9=64 10=-1
MemoryData data_att_9_a0 0 1 att_9_a0 0=1024 21=0
BinaryOp add_116 2 1 att_9_ma_lora_1 att_9_a0 att_9_ma_lora_2 0=0
Sigmoid sigmoid_35 1 1 att_9_ma_lora_2 att_9_a
Split split_att_9_a 1 2 att_9_a att_9_a_0 att_9_a_1
Gemm gemm_att_9_mg_lora 1 1 att_9_mg att_9_mg_lora 4=0 5=1 6=0 7=0 8=128 9=1024 10=-1
Sigmoid sigmoid_36 1 1 att_9_mg_lora att_9_mg_lora_sigmoid
Gemm gemm_att_9_gate 1 1 att_9_mg_lora_sigmoid att_9_gate 4=0 5=1 6=0 7=0 8=1024 9=128 10=-1
MemoryData data_att_9_k_k 0 1 att_9_k_k 0=1024 21=0
MemoryData data_att_9_k_a 0 1 att_9_k_a 0=1024 21=0
MemoryData data_att_9_r_k 0 1 att_9_r_k 0=1024 21=0
Split split_att_9_key 1 2 att_9_key att_9_key_0 att_9_key_1
BinaryOp mul_117 2 1 att_9_key_0 att_9_k_k att_9_key_k 0=2
Reshape reshape_108 1 1 att_9_key_k att_9_key_k_reshape 0=64 1=1 2=16
Normalize l2norm_9 1 1 att_9_key_k_reshape att_9_key_k_norm 0=1 1=1 2=0.0000001 3=1 4=0 9=1
Reshape reshape_109 1 1 att_9_key_k_norm att_9_key_k_norm_reshape 0=1024
BinaryOp add_117 1 1 att_9_a_0 att_9_a_0_minus 0=0 1=1 2=-1.0
BinaryOp mul_118 2 1 att_9_a_0_minus att_9_k_a att_9_a_minus1_mulka 0=2
BinaryOp add_118 1 1 att_9_a_minus1_mulka att_9_a_minus1_mulka_plus1 0=0 1=1 2=1.0
BinaryOp mul_119 2 1 att_9_a_minus1_mulka_plus1 att_9_key_1 att_9_key_a 0=2
Split split_att_9_value 1 2 att_9_value att_9_value_0 att_9_value_1
Split split_v_first_8 1 2 v_first_8 att_9_v_first v_first_9
BinaryOp minus_8 2 1 att_9_v_first att_9_value_1 att_9_vfirst_minus_value 0=1
Gemm gemm_att_9_mv_lora_0 1 1 att_9_mv_1 att_9_mv_lora_0 4=0 5=1 6=0 7=0 8=32 9=1024 10=-1
Gemm gemm_att_9_mv_lora_1 1 1 att_9_mv_lora_0 att_9_mv_lora_1 4=0 5=1 6=0 7=0 8=1024 9=32 10=-1
MemoryData data_att_9_v0 0 1 att_9_v0 0=1024 21=0
BinaryOp add_119 2 1 att_9_mv_lora_1 att_9_v0 att_9_mv_lora_2 0=0
Sigmoid sigmoid_37 1 1 att_9_mv_lora_2 att_9_mv_lora_sigmoid
BinaryOp mul_120 2 1 att_9_mv_lora_sigmoid att_9_vfirst_minus_value att_9_vfirst_minus_value_mul 0=2
BinaryOp add_120 2 1 att_9_value_0 att_9_vfirst_minus_value_mul att_9_value_final 0=0
Gemm gemm_att_9_mw_lora 1 1 att_9_mw att_9_mw_lora 4=0 5=1 6=0 7=0 8=64 9=1024 10=-1
TanH tanh_att_9_mw_lora 1 1 att_9_mw_lora att_9_mw_lora_tanh
Gemm gemm_att_9_mw_lora_tanh_linear 1 1 att_9_mw_lora_tanh att_9_mw_lora_tanh_linear 4=0 5=1 6=0 7=0 8=1024 9=64 10=-1
MemoryData data_att_9_td 0 1 att_9_td 0=1024 21=0
BinaryOp add_121 2 1 att_9_mw_lora_tanh_linear att_9_td att_9_time_decay_pre 0=0
Sigmoid sigmoid_38 1 1 att_9_time_decay_pre att_9_time_decay_sigmoid
Exp exp_9 1 1 att_9_time_decay_sigmoid att_9_time_decay 1=-0.606531
Reshape reshape_110 1 1 att_9_time_decay att_9_time_decay_reshape 0=64 1=1 2=16
Split split_att_9_key_a 1 2 att_9_key_a att_9_key_a_0 att_9_key_a_1
Reshape reshape_111 1 1 att_9_key_a_0 att_9_key_a_reshape 0=64 1=1 2=16
Split split_att_9_value_final 1 3 att_9_value_final att_9_value_final_0 att_9_value_final_1 att_9_value_final_2
Reshape reshape_112 1 1 att_9_value_final_0 att_9_value_final_0_reshape 0=64 1=16
Reshape reshape_113 1 1 att_9_value_final_1 att_9_value_final_1_reshape 0=1 1=64 2=16
MatMul matmul_att_9_vk 2 1 att_9_value_final_1_reshape att_9_key_a_reshape att_9_vk 0=0
Split split_att_9_key_k_norm_reshape 1 2 att_9_key_k_norm_reshape att_9_kk_0 att_9_kk_1
BinaryOp mul_121 1 1 att_9_kk_0 att_9_kk_0_neg 0=2 1=1 2=-1.0
Reshape reshape_114 1 1 att_9_kk_0_neg att_9_kk_0_neg_reshape 0=1 1=64 2=16
BinaryOp mul_122 2 1 att_9_kk_1 att_9_a_1 att_9_kk_a 0=2
Reshape reshape_115 1 1 att_9_kk_a att_9_kk_a_reshape 0=64 1=1 2=16
MatMul matmul_att_9_ab 2 1 att_9_kk_0_neg_reshape att_9_kk_a_reshape att_9_ab 0=0
Split split_state_28_in 1 2 state_28_in att_9_state_prev_0 att_9_state_prev_1
BinaryOp mul_123 2 1 att_9_state_prev_0 att_9_time_decay_reshape att_9_state_td 0=2
MatMul matmul_att_9_sab 2 1 att_9_state_prev_1 att_9_ab att_9_sab 0=0
BinaryOp add_122 2 1 att_9_sab att_9_vk att_9_sab_vk 0=0
BinaryOp add_123 2 1 att_9_sab_vk att_9_state_td att_9_state_new 0=0
Split split_att_9_state_new 1 2 att_9_state_new att_9_state_new_0 state_28_out
Split split_att_9_receptance 1 2 att_9_receptance att_9_receptance_0 att_9_receptance_1
Reshape reshape_116 1 1 att_9_receptance_0 att_9_receptance_reshape 0=1 1=64 2=16
MatMul matmul_att_9_wkv_out 2 1 att_9_state_new_0 att_9_receptance_reshape att_9_wkv_out 0=0
Reshape reshape_117 1 1 att_9_wkv_out att_9_wkv_out_flatten 0=1024
GroupNorm groupnorm_9 1 1 att_9_wkv_out_flatten att_9_x_gn 0=16 1=1024 2=0.00064 3=1
BinaryOp mul_124 2 1 att_9_receptance_1 att_9_key_a_1 att_9_receptance_key_a 0=2
BinaryOp mul_125 2 1 att_9_receptance_key_a att_9_r_k att_9_rk 0=2
Reshape reshape_118 1 1 att_9_rk att_9_rk_reshape 0=64 1=16
Reduction sum_9 1 1 att_9_rk_reshape att_9_rk_sum 1=0 -23303=1,-1 4=1 5=1
BinaryOp mul_126 2 1 att_9_value_final_0_reshape att_9_rk_sum att_9_value_final_0_rk 0=2
Reshape reshape_119 1 1 att_9_value_final_0_rk att_9_value_final_0_rk_reshape 0=1024
BinaryOp add_124 2 1 att_9_value_final_0_rk_reshape att_9_x_gn att_9_x_gn_rkv 0=0
BinaryOp mul_127 2 1 att_9_x_gn_rkv att_9_gate att_9_x_gate 0=2
Gemm gemm_att_9_x_out 1 1 att_9_x_gate att_9_x_out 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
BinaryOp add_125 2 1 att_9_x_out att_9_x_last time_mixing_9_out 0=0
Split split_time_mixing_9_out 1 2 time_mixing_9_out ffn_9_x_last ffn_9_x
LayerNorm layernorm_19 1 1 ffn_9_x ffn_9_xx 0=1024 1=0.00001 2=1
Split split_ffn_9_xx 1 3 ffn_9_xx ffn_9_xx_0 ffn_9_xx_1 state_29_out
BinaryOp sub_19 2 1 state_29_in ffn_9_xx_0 ffn_9_sx 0=1
Split split_ffn_9_sx 1 1 ffn_9_sx ffn_9_sx_0
MemoryData data_ffn_9_x_k 0 1 ffn_9_x_k 0=1024 21=0
BinaryOp mul_128 2 1 ffn_9_sx_0 ffn_9_x_k ffn_9_xk 0=2
BinaryOp add_126 2 1 ffn_9_xk ffn_9_xx_1 ffn_9_xxk 0=0
Gemm gemm_ffn_9_key 1 1 ffn_9_xxk ffn_9_key 4=0 5=1 6=0 7=0 8=4096 9=1024 10=-1
ReLU relu_9 1 1 ffn_9_key ffn_9_key_relu
UnaryOp square_ffn_9_key_relu 1 1 ffn_9_key_relu ffn_9_key_relu_square 0=4
Gemm gemm_ffn_9_value 1 1 ffn_9_key_relu_square ffn_9_value 4=0 5=1 6=0 7=0 8=1024 9=4096 10=-1
BinaryOp add_127 2 1 ffn_9_value ffn_9_x_last channel_mixing_9_out 0=0
Split split_channel_mixing_9_out 1 2 channel_mixing_9_out att_10_x_last att_10_x
LayerNorm layernorm_20 1 1 att_10_x att_10_xx 0=1024 1=0.00001 2=1
Split split_att_10_xx 1 3 att_10_xx att_10_xx_0 att_10_xx_1 state_30_out
BinaryOp sub_20 2 1 state_30_in att_10_xx_0 att_10_sx 0=1
MemoryData data_att_10_maa 0 1 att_10_maa 0=1024 1=6 21=0
BinaryOp mul_129 2 1 att_10_sx att_10_maa att_10_maa_x 0=2
BinaryOp add_128 2 1 att_10_xx_1 att_10_maa_x att_10_maa_rwkvag 0=0
Slice slice_att_10_maa_rwkvag 1 6 att_10_maa_rwkvag att_10_mr att_10_mw att_10_mk att_10_mv att_10_ma att_10_mg -23300=6,-233,-233,-233,-233,-233,-233 1=0
Split split_att_10_mv 1 2 att_10_mv att_10_mv_0 att_10_mv_1
Gemm gemm_att_10_key 1 1 att_10_mk att_10_key 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_10_value 1 1 att_10_mv_0 att_10_value 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_10_receptance 1 1 att_10_mr att_10_receptance 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_10_ma_lora_0 1 1 att_10_ma att_10_ma_lora_0 4=0 5=1 6=0 7=0 8=64 9=1024 10=-1
Gemm gemm_att_10_ma_lora_1 1 1 att_10_ma_lora_0 att_10_ma_lora_1 4=0 5=1 6=0 7=0 8=1024 9=64 10=-1
MemoryData data_att_10_a0 0 1 att_10_a0 0=1024 21=0
BinaryOp add_129 2 1 att_10_ma_lora_1 att_10_a0 att_10_ma_lora_2 0=0
Sigmoid sigmoid_39 1 1 att_10_ma_lora_2 att_10_a
Split split_att_10_a 1 2 att_10_a att_10_a_0 att_10_a_1
Gemm gemm_att_10_mg_lora 1 1 att_10_mg att_10_mg_lora 4=0 5=1 6=0 7=0 8=128 9=1024 10=-1
Sigmoid sigmoid_40 1 1 att_10_mg_lora att_10_mg_lora_sigmoid
Gemm gemm_att_10_gate 1 1 att_10_mg_lora_sigmoid att_10_gate 4=0 5=1 6=0 7=0 8=1024 9=128 10=-1
MemoryData data_att_10_k_k 0 1 att_10_k_k 0=1024 21=0
MemoryData data_att_10_k_a 0 1 att_10_k_a 0=1024 21=0
MemoryData data_att_10_r_k 0 1 att_10_r_k 0=1024 21=0
Split split_att_10_key 1 2 att_10_key att_10_key_0 att_10_key_1
BinaryOp mul_130 2 1 att_10_key_0 att_10_k_k att_10_key_k 0=2
Reshape reshape_120 1 1 att_10_key_k att_10_key_k_reshape 0=64 1=1 2=16
Normalize l2norm_10 1 1 att_10_key_k_reshape att_10_key_k_norm 0=1 1=1 2=0.0000001 3=1 4=0 9=1
Reshape reshape_121 1 1 att_10_key_k_norm att_10_key_k_norm_reshape 0=1024
BinaryOp add_130 1 1 att_10_a_0 att_10_a_0_minus 0=0 1=1 2=-1.0
BinaryOp mul_131 2 1 att_10_a_0_minus att_10_k_a att_10_a_minus1_mulka 0=2
BinaryOp add_131 1 1 att_10_a_minus1_mulka att_10_a_minus1_mulka_plus1 0=0 1=1 2=1.0
BinaryOp mul_132 2 1 att_10_a_minus1_mulka_plus1 att_10_key_1 att_10_key_a 0=2
Split split_att_10_value 1 2 att_10_value att_10_value_0 att_10_value_1
Split split_v_first_9 1 2 v_first_9 att_10_v_first v_first_10
BinaryOp minus_9 2 1 att_10_v_first att_10_value_1 att_10_vfirst_minus_value 0=1
Gemm gemm_att_10_mv_lora_0 1 1 att_10_mv_1 att_10_mv_lora_0 4=0 5=1 6=0 7=0 8=32 9=1024 10=-1
Gemm gemm_att_10_mv_lora_1 1 1 att_10_mv_lora_0 att_10_mv_lora_1 4=0 5=1 6=0 7=0 8=1024 9=32 10=-1
MemoryData data_att_10_v0 0 1 att_10_v0 0=1024 21=0
BinaryOp add_132 2 1 att_10_mv_lora_1 att_10_v0 att_10_mv_lora_2 0=0
Sigmoid sigmoid_41 1 1 att_10_mv_lora_2 att_10_mv_lora_sigmoid
BinaryOp mul_133 2 1 att_10_mv_lora_sigmoid att_10_vfirst_minus_value att_10_vfirst_minus_value_mul 0=2
BinaryOp add_133 2 1 att_10_value_0 att_10_vfirst_minus_value_mul att_10_value_final 0=0
Gemm gemm_att_10_mw_lora 1 1 att_10_mw att_10_mw_lora 4=0 5=1 6=0 7=0 8=64 9=1024 10=-1
TanH tanh_att_10_mw_lora 1 1 att_10_mw_lora att_10_mw_lora_tanh
Gemm gemm_att_10_mw_lora_tanh_linear 1 1 att_10_mw_lora_tanh att_10_mw_lora_tanh_linear 4=0 5=1 6=0 7=0 8=1024 9=64 10=-1
MemoryData data_att_10_td 0 1 att_10_td 0=1024 21=0
BinaryOp add_134 2 1 att_10_mw_lora_tanh_linear att_10_td att_10_time_decay_pre 0=0
Sigmoid sigmoid_42 1 1 att_10_time_decay_pre att_10_time_decay_sigmoid
Exp exp_10 1 1 att_10_time_decay_sigmoid att_10_time_decay 1=-0.606531
Reshape reshape_122 1 1 att_10_time_decay att_10_time_decay_reshape 0=64 1=1 2=16
Split split_att_10_key_a 1 2 att_10_key_a att_10_key_a_0 att_10_key_a_1
Reshape reshape_123 1 1 att_10_key_a_0 att_10_key_a_reshape 0=64 1=1 2=16
Split split_att_10_value_final 1 3 att_10_value_final att_10_value_final_0 att_10_value_final_1 att_10_value_final_2
Reshape reshape_124 1 1 att_10_value_final_0 att_10_value_final_0_reshape 0=64 1=16
Reshape reshape_125 1 1 att_10_value_final_1 att_10_value_final_1_reshape 0=1 1=64 2=16
MatMul matmul_att_10_vk 2 1 att_10_value_final_1_reshape att_10_key_a_reshape att_10_vk 0=0
Split split_att_10_key_k_norm_reshape 1 2 att_10_key_k_norm_reshape att_10_kk_0 att_10_kk_1
BinaryOp mul_134 1 1 att_10_kk_0 att_10_kk_0_neg 0=2 1=1 2=-1.0
Reshape reshape_126 1 1 att_10_kk_0_neg att_10_kk_0_neg_reshape 0=1 1=64 2=16
BinaryOp mul_135 2 1 att_10_kk_1 att_10_a_1 att_10_kk_a 0=2
Reshape reshape_127 1 1 att_10_kk_a att_10_kk_a_reshape 0=64 1=1 2=16
MatMul matmul_att_10_ab 2 1 att_10_kk_0_neg_reshape att_10_kk_a_reshape att_10_ab 0=0
Split split_state_31_in 1 2 state_31_in att_10_state_prev_0 att_10_state_prev_1
BinaryOp mul_136 2 1 att_10_state_prev_0 att_10_time_decay_reshape att_10_state_td 0=2
MatMul matmul_att_10_sab 2 1 att_10_state_prev_1 att_10_ab att_10_sab 0=0
BinaryOp add_135 2 1 att_10_sab att_10_vk att_10_sab_vk 0=0
BinaryOp add_136 2 1 att_10_sab_vk att_10_state_td att_10_state_new 0=0
Split split_att_10_state_new 1 2 att_10_state_new att_10_state_new_0 state_31_out
Split split_att_10_receptance 1 2 att_10_receptance att_10_receptance_0 att_10_receptance_1
Reshape reshape_128 1 1 att_10_receptance_0 att_10_receptance_reshape 0=1 1=64 2=16
MatMul matmul_att_10_wkv_out 2 1 att_10_state_new_0 att_10_receptance_reshape att_10_wkv_out 0=0
Reshape reshape_129 1 1 att_10_wkv_out att_10_wkv_out_flatten 0=1024
GroupNorm groupnorm_10 1 1 att_10_wkv_out_flatten att_10_x_gn 0=16 1=1024 2=0.00064 3=1
BinaryOp mul_137 2 1 att_10_receptance_1 att_10_key_a_1 att_10_receptance_key_a 0=2
BinaryOp mul_138 2 1 att_10_receptance_key_a att_10_r_k att_10_rk 0=2
Reshape reshape_130 1 1 att_10_rk att_10_rk_reshape 0=64 1=16
Reduction sum_10 1 1 att_10_rk_reshape att_10_rk_sum 1=0 -23303=1,-1 4=1 5=1
BinaryOp mul_139 2 1 att_10_value_final_0_reshape att_10_rk_sum att_10_value_final_0_rk 0=2
Reshape reshape_131 1 1 att_10_value_final_0_rk att_10_value_final_0_rk_reshape 0=1024
BinaryOp add_137 2 1 att_10_value_final_0_rk_reshape att_10_x_gn att_10_x_gn_rkv 0=0
BinaryOp mul_140 2 1 att_10_x_gn_rkv att_10_gate att_10_x_gate 0=2
Gemm gemm_att_10_x_out 1 1 att_10_x_gate att_10_x_out 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
BinaryOp add_138 2 1 att_10_x_out att_10_x_last time_mixing_10_out 0=0
Split split_time_mixing_10_out 1 2 time_mixing_10_out ffn_10_x_last ffn_10_x
LayerNorm layernorm_21 1 1 ffn_10_x ffn_10_xx 0=1024 1=0.00001 2=1
Split split_ffn_10_xx 1 3 ffn_10_xx ffn_10_xx_0 ffn_10_xx_1 state_32_out
BinaryOp sub_21 2 1 state_32_in ffn_10_xx_0 ffn_10_sx 0=1
Split split_ffn_10_sx 1 1 ffn_10_sx ffn_10_sx_0
MemoryData data_ffn_10_x_k 0 1 ffn_10_x_k 0=1024 21=0
BinaryOp mul_141 2 1 ffn_10_sx_0 ffn_10_x_k ffn_10_xk 0=2
BinaryOp add_139 2 1 ffn_10_xk ffn_10_xx_1 ffn_10_xxk 0=0
Gemm gemm_ffn_10_key 1 1 ffn_10_xxk ffn_10_key 4=0 5=1 6=0 7=0 8=4096 9=1024 10=-1
ReLU relu_10 1 1 ffn_10_key ffn_10_key_relu
UnaryOp square_ffn_10_key_relu 1 1 ffn_10_key_relu ffn_10_key_relu_square 0=4
Gemm gemm_ffn_10_value 1 1 ffn_10_key_relu_square ffn_10_value 4=0 5=1 6=0 7=0 8=1024 9=4096 10=-1
BinaryOp add_140 2 1 ffn_10_value ffn_10_x_last channel_mixing_10_out 0=0
Split split_channel_mixing_10_out 1 2 channel_mixing_10_out att_11_x_last att_11_x
LayerNorm layernorm_22 1 1 att_11_x att_11_xx 0=1024 1=0.00001 2=1
Split split_att_11_xx 1 3 att_11_xx att_11_xx_0 att_11_xx_1 state_33_out
BinaryOp sub_22 2 1 state_33_in att_11_xx_0 att_11_sx 0=1
MemoryData data_att_11_maa 0 1 att_11_maa 0=1024 1=6 21=0
BinaryOp mul_142 2 1 att_11_sx att_11_maa att_11_maa_x 0=2
BinaryOp add_141 2 1 att_11_xx_1 att_11_maa_x att_11_maa_rwkvag 0=0
Slice slice_att_11_maa_rwkvag 1 6 att_11_maa_rwkvag att_11_mr att_11_mw att_11_mk att_11_mv att_11_ma att_11_mg -23300=6,-233,-233,-233,-233,-233,-233 1=0
Split split_att_11_mv 1 2 att_11_mv att_11_mv_0 att_11_mv_1
Gemm gemm_att_11_key 1 1 att_11_mk att_11_key 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_11_value 1 1 att_11_mv_0 att_11_value 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_11_receptance 1 1 att_11_mr att_11_receptance 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_11_ma_lora_0 1 1 att_11_ma att_11_ma_lora_0 4=0 5=1 6=0 7=0 8=64 9=1024 10=-1
Gemm gemm_att_11_ma_lora_1 1 1 att_11_ma_lora_0 att_11_ma_lora_1 4=0 5=1 6=0 7=0 8=1024 9=64 10=-1
MemoryData data_att_11_a0 0 1 att_11_a0 0=1024 21=0
BinaryOp add_142 2 1 att_11_ma_lora_1 att_11_a0 att_11_ma_lora_2 0=0
Sigmoid sigmoid_43 1 1 att_11_ma_lora_2 att_11_a
Split split_att_11_a 1 2 att_11_a att_11_a_0 att_11_a_1
Gemm gemm_att_11_mg_lora 1 1 att_11_mg att_11_mg_lora 4=0 5=1 6=0 7=0 8=128 9=1024 10=-1
Sigmoid sigmoid_44 1 1 att_11_mg_lora att_11_mg_lora_sigmoid
Gemm gemm_att_11_gate 1 1 att_11_mg_lora_sigmoid att_11_gate 4=0 5=1 6=0 7=0 8=1024 9=128 10=-1
MemoryData data_att_11_k_k 0 1 att_11_k_k 0=1024 21=0
MemoryData data_att_11_k_a 0 1 att_11_k_a 0=1024 21=0
MemoryData data_att_11_r_k 0 1 att_11_r_k 0=1024 21=0
Split split_att_11_key 1 2 att_11_key att_11_key_0 att_11_key_1
BinaryOp mul_143 2 1 att_11_key_0 att_11_k_k att_11_key_k 0=2
Reshape reshape_132 1 1 att_11_key_k att_11_key_k_reshape 0=64 1=1 2=16
Normalize l2norm_11 1 1 att_11_key_k_reshape att_11_key_k_norm 0=1 1=1 2=0.0000001 3=1 4=0 9=1
Reshape reshape_133 1 1 att_11_key_k_norm att_11_key_k_norm_reshape 0=1024
BinaryOp add_143 1 1 att_11_a_0 att_11_a_0_minus 0=0 1=1 2=-1.0
BinaryOp mul_144 2 1 att_11_a_0_minus att_11_k_a att_11_a_minus1_mulka 0=2
BinaryOp add_144 1 1 att_11_a_minus1_mulka att_11_a_minus1_mulka_plus1 0=0 1=1 2=1.0
BinaryOp mul_145 2 1 att_11_a_minus1_mulka_plus1 att_11_key_1 att_11_key_a 0=2
Split split_att_11_value 1 2 att_11_value att_11_value_0 att_11_value_1
Split split_v_first_10 1 2 v_first_10 att_11_v_first v_first_11
BinaryOp minus_10 2 1 att_11_v_first att_11_value_1 att_11_vfirst_minus_value 0=1
Gemm gemm_att_11_mv_lora_0 1 1 att_11_mv_1 att_11_mv_lora_0 4=0 5=1 6=0 7=0 8=32 9=1024 10=-1
Gemm gemm_att_11_mv_lora_1 1 1 att_11_mv_lora_0 att_11_mv_lora_1 4=0 5=1 6=0 7=0 8=1024 9=32 10=-1
MemoryData data_att_11_v0 0 1 att_11_v0 0=1024 21=0
BinaryOp add_145 2 1 att_11_mv_lora_1 att_11_v0 att_11_mv_lora_2 0=0
Sigmoid sigmoid_45 1 1 att_11_mv_lora_2 att_11_mv_lora_sigmoid
BinaryOp mul_146 2 1 att_11_mv_lora_sigmoid att_11_vfirst_minus_value att_11_vfirst_minus_value_mul 0=2
BinaryOp add_146 2 1 att_11_value_0 att_11_vfirst_minus_value_mul att_11_value_final 0=0
Gemm gemm_att_11_mw_lora 1 1 att_11_mw att_11_mw_lora 4=0 5=1 6=0 7=0 8=64 9=1024 10=-1
TanH tanh_att_11_mw_lora 1 1 att_11_mw_lora att_11_mw_lora_tanh
Gemm gemm_att_11_mw_lora_tanh_linear 1 1 att_11_mw_lora_tanh att_11_mw_lora_tanh_linear 4=0 5=1 6=0 7=0 8=1024 9=64 10=-1
MemoryData data_att_11_td 0 1 att_11_td 0=1024 21=0
BinaryOp add_147 2 1 att_11_mw_lora_tanh_linear att_11_td att_11_time_decay_pre 0=0
Sigmoid sigmoid_46 1 1 att_11_time_decay_pre att_11_time_decay_sigmoid
Exp exp_11 1 1 att_11_time_decay_sigmoid att_11_time_decay 1=-0.606531
Reshape reshape_134 1 1 att_11_time_decay att_11_time_decay_reshape 0=64 1=1 2=16
Split split_att_11_key_a 1 2 att_11_key_a att_11_key_a_0 att_11_key_a_1
Reshape reshape_135 1 1 att_11_key_a_0 att_11_key_a_reshape 0=64 1=1 2=16
Split split_att_11_value_final 1 3 att_11_value_final att_11_value_final_0 att_11_value_final_1 att_11_value_final_2
Reshape reshape_136 1 1 att_11_value_final_0 att_11_value_final_0_reshape 0=64 1=16
Reshape reshape_137 1 1 att_11_value_final_1 att_11_value_final_1_reshape 0=1 1=64 2=16
MatMul matmul_att_11_vk 2 1 att_11_value_final_1_reshape att_11_key_a_reshape att_11_vk 0=0
Split split_att_11_key_k_norm_reshape 1 2 att_11_key_k_norm_reshape att_11_kk_0 att_11_kk_1
BinaryOp mul_147 1 1 att_11_kk_0 att_11_kk_0_neg 0=2 1=1 2=-1.0
Reshape reshape_138 1 1 att_11_kk_0_neg att_11_kk_0_neg_reshape 0=1 1=64 2=16
BinaryOp mul_148 2 1 att_11_kk_1 att_11_a_1 att_11_kk_a 0=2
Reshape reshape_139 1 1 att_11_kk_a att_11_kk_a_reshape 0=64 1=1 2=16
MatMul matmul_att_11_ab 2 1 att_11_kk_0_neg_reshape att_11_kk_a_reshape att_11_ab 0=0
Split split_state_34_in 1 2 state_34_in att_11_state_prev_0 att_11_state_prev_1
BinaryOp mul_149 2 1 att_11_state_prev_0 att_11_time_decay_reshape att_11_state_td 0=2
MatMul matmul_att_11_sab 2 1 att_11_state_prev_1 att_11_ab att_11_sab 0=0
BinaryOp add_148 2 1 att_11_sab att_11_vk att_11_sab_vk 0=0
BinaryOp add_149 2 1 att_11_sab_vk att_11_state_td att_11_state_new 0=0
Split split_att_11_state_new 1 2 att_11_state_new att_11_state_new_0 state_34_out
Split split_att_11_receptance 1 2 att_11_receptance att_11_receptance_0 att_11_receptance_1
Reshape reshape_140 1 1 att_11_receptance_0 att_11_receptance_reshape 0=1 1=64 2=16
MatMul matmul_att_11_wkv_out 2 1 att_11_state_new_0 att_11_receptance_reshape att_11_wkv_out 0=0
Reshape reshape_141 1 1 att_11_wkv_out att_11_wkv_out_flatten 0=1024
GroupNorm groupnorm_11 1 1 att_11_wkv_out_flatten att_11_x_gn 0=16 1=1024 2=0.00064 3=1
BinaryOp mul_150 2 1 att_11_receptance_1 att_11_key_a_1 att_11_receptance_key_a 0=2
BinaryOp mul_151 2 1 att_11_receptance_key_a att_11_r_k att_11_rk 0=2
Reshape reshape_142 1 1 att_11_rk att_11_rk_reshape 0=64 1=16
Reduction sum_11 1 1 att_11_rk_reshape att_11_rk_sum 1=0 -23303=1,-1 4=1 5=1
BinaryOp mul_152 2 1 att_11_value_final_0_reshape att_11_rk_sum att_11_value_final_0_rk 0=2
Reshape reshape_143 1 1 att_11_value_final_0_rk att_11_value_final_0_rk_reshape 0=1024
BinaryOp add_150 2 1 att_11_value_final_0_rk_reshape att_11_x_gn att_11_x_gn_rkv 0=0
BinaryOp mul_153 2 1 att_11_x_gn_rkv att_11_gate att_11_x_gate 0=2
Gemm gemm_att_11_x_out 1 1 att_11_x_gate att_11_x_out 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
BinaryOp add_151 2 1 att_11_x_out att_11_x_last time_mixing_11_out 0=0
Split split_time_mixing_11_out 1 2 time_mixing_11_out ffn_11_x_last ffn_11_x
LayerNorm layernorm_23 1 1 ffn_11_x ffn_11_xx 0=1024 1=0.00001 2=1
Split split_ffn_11_xx 1 3 ffn_11_xx ffn_11_xx_0 ffn_11_xx_1 state_35_out
BinaryOp sub_23 2 1 state_35_in ffn_11_xx_0 ffn_11_sx 0=1
Split split_ffn_11_sx 1 1 ffn_11_sx ffn_11_sx_0
MemoryData data_ffn_11_x_k 0 1 ffn_11_x_k 0=1024 21=0
BinaryOp mul_154 2 1 ffn_11_sx_0 ffn_11_x_k ffn_11_xk 0=2
BinaryOp add_152 2 1 ffn_11_xk ffn_11_xx_1 ffn_11_xxk 0=0
Gemm gemm_ffn_11_key 1 1 ffn_11_xxk ffn_11_key 4=0 5=1 6=0 7=0 8=4096 9=1024 10=-1
ReLU relu_11 1 1 ffn_11_key ffn_11_key_relu
UnaryOp square_ffn_11_key_relu 1 1 ffn_11_key_relu ffn_11_key_relu_square 0=4
Gemm gemm_ffn_11_value 1 1 ffn_11_key_relu_square ffn_11_value 4=0 5=1 6=0 7=0 8=1024 9=4096 10=-1
BinaryOp add_153 2 1 ffn_11_value ffn_11_x_last channel_mixing_11_out 0=0
Split split_channel_mixing_11_out 1 2 channel_mixing_11_out att_12_x_last att_12_x
LayerNorm layernorm_24 1 1 att_12_x att_12_xx 0=1024 1=0.00001 2=1
Split split_att_12_xx 1 3 att_12_xx att_12_xx_0 att_12_xx_1 state_36_out
BinaryOp sub_24 2 1 state_36_in att_12_xx_0 att_12_sx 0=1
MemoryData data_att_12_maa 0 1 att_12_maa 0=1024 1=6 21=0
BinaryOp mul_155 2 1 att_12_sx att_12_maa att_12_maa_x 0=2
BinaryOp add_154 2 1 att_12_xx_1 att_12_maa_x att_12_maa_rwkvag 0=0
Slice slice_att_12_maa_rwkvag 1 6 att_12_maa_rwkvag att_12_mr att_12_mw att_12_mk att_12_mv att_12_ma att_12_mg -23300=6,-233,-233,-233,-233,-233,-233 1=0
Split split_att_12_mv 1 2 att_12_mv att_12_mv_0 att_12_mv_1
Gemm gemm_att_12_key 1 1 att_12_mk att_12_key 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_12_value 1 1 att_12_mv_0 att_12_value 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_12_receptance 1 1 att_12_mr att_12_receptance 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_12_ma_lora_0 1 1 att_12_ma att_12_ma_lora_0 4=0 5=1 6=0 7=0 8=64 9=1024 10=-1
Gemm gemm_att_12_ma_lora_1 1 1 att_12_ma_lora_0 att_12_ma_lora_1 4=0 5=1 6=0 7=0 8=1024 9=64 10=-1
MemoryData data_att_12_a0 0 1 att_12_a0 0=1024 21=0
BinaryOp add_155 2 1 att_12_ma_lora_1 att_12_a0 att_12_ma_lora_2 0=0
Sigmoid sigmoid_47 1 1 att_12_ma_lora_2 att_12_a
Split split_att_12_a 1 2 att_12_a att_12_a_0 att_12_a_1
Gemm gemm_att_12_mg_lora 1 1 att_12_mg att_12_mg_lora 4=0 5=1 6=0 7=0 8=128 9=1024 10=-1
Sigmoid sigmoid_48 1 1 att_12_mg_lora att_12_mg_lora_sigmoid
Gemm gemm_att_12_gate 1 1 att_12_mg_lora_sigmoid att_12_gate 4=0 5=1 6=0 7=0 8=1024 9=128 10=-1
MemoryData data_att_12_k_k 0 1 att_12_k_k 0=1024 21=0
MemoryData data_att_12_k_a 0 1 att_12_k_a 0=1024 21=0
MemoryData data_att_12_r_k 0 1 att_12_r_k 0=1024 21=0
Split split_att_12_key 1 2 att_12_key att_12_key_0 att_12_key_1
BinaryOp mul_156 2 1 att_12_key_0 att_12_k_k att_12_key_k 0=2
Reshape reshape_144 1 1 att_12_key_k att_12_key_k_reshape 0=64 1=1 2=16
Normalize l2norm_12 1 1 att_12_key_k_reshape att_12_key_k_norm 0=1 1=1 2=0.0000001 3=1 4=0 9=1
Reshape reshape_145 1 1 att_12_key_k_norm att_12_key_k_norm_reshape 0=1024
BinaryOp add_156 1 1 att_12_a_0 att_12_a_0_minus 0=0 1=1 2=-1.0
BinaryOp mul_157 2 1 att_12_a_0_minus att_12_k_a att_12_a_minus1_mulka 0=2
BinaryOp add_157 1 1 att_12_a_minus1_mulka att_12_a_minus1_mulka_plus1 0=0 1=1 2=1.0
BinaryOp mul_158 2 1 att_12_a_minus1_mulka_plus1 att_12_key_1 att_12_key_a 0=2
Split split_att_12_value 1 2 att_12_value att_12_value_0 att_12_value_1
Split split_v_first_11 1 2 v_first_11 att_12_v_first v_first_12
BinaryOp minus_11 2 1 att_12_v_first att_12_value_1 att_12_vfirst_minus_value 0=1
Gemm gemm_att_12_mv_lora_0 1 1 att_12_mv_1 att_12_mv_lora_0 4=0 5=1 6=0 7=0 8=32 9=1024 10=-1
Gemm gemm_att_12_mv_lora_1 1 1 att_12_mv_lora_0 att_12_mv_lora_1 4=0 5=1 6=0 7=0 8=1024 9=32 10=-1
MemoryData data_att_12_v0 0 1 att_12_v0 0=1024 21=0
BinaryOp add_158 2 1 att_12_mv_lora_1 att_12_v0 att_12_mv_lora_2 0=0
Sigmoid sigmoid_49 1 1 att_12_mv_lora_2 att_12_mv_lora_sigmoid
BinaryOp mul_159 2 1 att_12_mv_lora_sigmoid att_12_vfirst_minus_value att_12_vfirst_minus_value_mul 0=2
BinaryOp add_159 2 1 att_12_value_0 att_12_vfirst_minus_value_mul att_12_value_final 0=0
Gemm gemm_att_12_mw_lora 1 1 att_12_mw att_12_mw_lora 4=0 5=1 6=0 7=0 8=64 9=1024 10=-1
TanH tanh_att_12_mw_lora 1 1 att_12_mw_lora att_12_mw_lora_tanh
Gemm gemm_att_12_mw_lora_tanh_linear 1 1 att_12_mw_lora_tanh att_12_mw_lora_tanh_linear 4=0 5=1 6=0 7=0 8=1024 9=64 10=-1
MemoryData data_att_12_td 0 1 att_12_td 0=1024 21=0
BinaryOp add_160 2 1 att_12_mw_lora_tanh_linear att_12_td att_12_time_decay_pre 0=0
Sigmoid sigmoid_50 1 1 att_12_time_decay_pre att_12_time_decay_sigmoid
Exp exp_12 1 1 att_12_time_decay_sigmoid att_12_time_decay 1=-0.606531
Reshape reshape_146 1 1 att_12_time_decay att_12_time_decay_reshape 0=64 1=1 2=16
Split split_att_12_key_a 1 2 att_12_key_a att_12_key_a_0 att_12_key_a_1
Reshape reshape_147 1 1 att_12_key_a_0 att_12_key_a_reshape 0=64 1=1 2=16
Split split_att_12_value_final 1 3 att_12_value_final att_12_value_final_0 att_12_value_final_1 att_12_value_final_2
Reshape reshape_148 1 1 att_12_value_final_0 att_12_value_final_0_reshape 0=64 1=16
Reshape reshape_149 1 1 att_12_value_final_1 att_12_value_final_1_reshape 0=1 1=64 2=16
MatMul matmul_att_12_vk 2 1 att_12_value_final_1_reshape att_12_key_a_reshape att_12_vk 0=0
Split split_att_12_key_k_norm_reshape 1 2 att_12_key_k_norm_reshape att_12_kk_0 att_12_kk_1
BinaryOp mul_160 1 1 att_12_kk_0 att_12_kk_0_neg 0=2 1=1 2=-1.0
Reshape reshape_150 1 1 att_12_kk_0_neg att_12_kk_0_neg_reshape 0=1 1=64 2=16
BinaryOp mul_161 2 1 att_12_kk_1 att_12_a_1 att_12_kk_a 0=2
Reshape reshape_151 1 1 att_12_kk_a att_12_kk_a_reshape 0=64 1=1 2=16
MatMul matmul_att_12_ab 2 1 att_12_kk_0_neg_reshape att_12_kk_a_reshape att_12_ab 0=0
Split split_state_37_in 1 2 state_37_in att_12_state_prev_0 att_12_state_prev_1
BinaryOp mul_162 2 1 att_12_state_prev_0 att_12_time_decay_reshape att_12_state_td 0=2
MatMul matmul_att_12_sab 2 1 att_12_state_prev_1 att_12_ab att_12_sab 0=0
BinaryOp add_161 2 1 att_12_sab att_12_vk att_12_sab_vk 0=0
BinaryOp add_162 2 1 att_12_sab_vk att_12_state_td att_12_state_new 0=0
Split split_att_12_state_new 1 2 att_12_state_new att_12_state_new_0 state_37_out
Split split_att_12_receptance 1 2 att_12_receptance att_12_receptance_0 att_12_receptance_1
Reshape reshape_152 1 1 att_12_receptance_0 att_12_receptance_reshape 0=1 1=64 2=16
MatMul matmul_att_12_wkv_out 2 1 att_12_state_new_0 att_12_receptance_reshape att_12_wkv_out 0=0
Reshape reshape_153 1 1 att_12_wkv_out att_12_wkv_out_flatten 0=1024
GroupNorm groupnorm_12 1 1 att_12_wkv_out_flatten att_12_x_gn 0=16 1=1024 2=0.00064 3=1
BinaryOp mul_163 2 1 att_12_receptance_1 att_12_key_a_1 att_12_receptance_key_a 0=2
BinaryOp mul_164 2 1 att_12_receptance_key_a att_12_r_k att_12_rk 0=2
Reshape reshape_154 1 1 att_12_rk att_12_rk_reshape 0=64 1=16
Reduction sum_12 1 1 att_12_rk_reshape att_12_rk_sum 1=0 -23303=1,-1 4=1 5=1
BinaryOp mul_165 2 1 att_12_value_final_0_reshape att_12_rk_sum att_12_value_final_0_rk 0=2
Reshape reshape_155 1 1 att_12_value_final_0_rk att_12_value_final_0_rk_reshape 0=1024
BinaryOp add_163 2 1 att_12_value_final_0_rk_reshape att_12_x_gn att_12_x_gn_rkv 0=0
BinaryOp mul_166 2 1 att_12_x_gn_rkv att_12_gate att_12_x_gate 0=2
Gemm gemm_att_12_x_out 1 1 att_12_x_gate att_12_x_out 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
BinaryOp add_164 2 1 att_12_x_out att_12_x_last time_mixing_12_out 0=0
Split split_time_mixing_12_out 1 2 time_mixing_12_out ffn_12_x_last ffn_12_x
LayerNorm layernorm_25 1 1 ffn_12_x ffn_12_xx 0=1024 1=0.00001 2=1
Split split_ffn_12_xx 1 3 ffn_12_xx ffn_12_xx_0 ffn_12_xx_1 state_38_out
BinaryOp sub_25 2 1 state_38_in ffn_12_xx_0 ffn_12_sx 0=1
Split split_ffn_12_sx 1 1 ffn_12_sx ffn_12_sx_0
MemoryData data_ffn_12_x_k 0 1 ffn_12_x_k 0=1024 21=0
BinaryOp mul_167 2 1 ffn_12_sx_0 ffn_12_x_k ffn_12_xk 0=2
BinaryOp add_165 2 1 ffn_12_xk ffn_12_xx_1 ffn_12_xxk 0=0
Gemm gemm_ffn_12_key 1 1 ffn_12_xxk ffn_12_key 4=0 5=1 6=0 7=0 8=4096 9=1024 10=-1
ReLU relu_12 1 1 ffn_12_key ffn_12_key_relu
UnaryOp square_ffn_12_key_relu 1 1 ffn_12_key_relu ffn_12_key_relu_square 0=4
Gemm gemm_ffn_12_value 1 1 ffn_12_key_relu_square ffn_12_value 4=0 5=1 6=0 7=0 8=1024 9=4096 10=-1
BinaryOp add_166 2 1 ffn_12_value ffn_12_x_last channel_mixing_12_out 0=0
Split split_channel_mixing_12_out 1 2 channel_mixing_12_out att_13_x_last att_13_x
LayerNorm layernorm_26 1 1 att_13_x att_13_xx 0=1024 1=0.00001 2=1
Split split_att_13_xx 1 3 att_13_xx att_13_xx_0 att_13_xx_1 state_39_out
BinaryOp sub_26 2 1 state_39_in att_13_xx_0 att_13_sx 0=1
MemoryData data_att_13_maa 0 1 att_13_maa 0=1024 1=6 21=0
BinaryOp mul_168 2 1 att_13_sx att_13_maa att_13_maa_x 0=2
BinaryOp add_167 2 1 att_13_xx_1 att_13_maa_x att_13_maa_rwkvag 0=0
Slice slice_att_13_maa_rwkvag 1 6 att_13_maa_rwkvag att_13_mr att_13_mw att_13_mk att_13_mv att_13_ma att_13_mg -23300=6,-233,-233,-233,-233,-233,-233 1=0
Split split_att_13_mv 1 2 att_13_mv att_13_mv_0 att_13_mv_1
Gemm gemm_att_13_key 1 1 att_13_mk att_13_key 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_13_value 1 1 att_13_mv_0 att_13_value 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_13_receptance 1 1 att_13_mr att_13_receptance 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_13_ma_lora_0 1 1 att_13_ma att_13_ma_lora_0 4=0 5=1 6=0 7=0 8=64 9=1024 10=-1
Gemm gemm_att_13_ma_lora_1 1 1 att_13_ma_lora_0 att_13_ma_lora_1 4=0 5=1 6=0 7=0 8=1024 9=64 10=-1
MemoryData data_att_13_a0 0 1 att_13_a0 0=1024 21=0
BinaryOp add_168 2 1 att_13_ma_lora_1 att_13_a0 att_13_ma_lora_2 0=0
Sigmoid sigmoid_51 1 1 att_13_ma_lora_2 att_13_a
Split split_att_13_a 1 2 att_13_a att_13_a_0 att_13_a_1
Gemm gemm_att_13_mg_lora 1 1 att_13_mg att_13_mg_lora 4=0 5=1 6=0 7=0 8=128 9=1024 10=-1
Sigmoid sigmoid_52 1 1 att_13_mg_lora att_13_mg_lora_sigmoid
Gemm gemm_att_13_gate 1 1 att_13_mg_lora_sigmoid att_13_gate 4=0 5=1 6=0 7=0 8=1024 9=128 10=-1
MemoryData data_att_13_k_k 0 1 att_13_k_k 0=1024 21=0
MemoryData data_att_13_k_a 0 1 att_13_k_a 0=1024 21=0
MemoryData data_att_13_r_k 0 1 att_13_r_k 0=1024 21=0
Split split_att_13_key 1 2 att_13_key att_13_key_0 att_13_key_1
BinaryOp mul_169 2 1 att_13_key_0 att_13_k_k att_13_key_k 0=2
Reshape reshape_156 1 1 att_13_key_k att_13_key_k_reshape 0=64 1=1 2=16
Normalize l2norm_13 1 1 att_13_key_k_reshape att_13_key_k_norm 0=1 1=1 2=0.0000001 3=1 4=0 9=1
Reshape reshape_157 1 1 att_13_key_k_norm att_13_key_k_norm_reshape 0=1024
BinaryOp add_169 1 1 att_13_a_0 att_13_a_0_minus 0=0 1=1 2=-1.0
BinaryOp mul_170 2 1 att_13_a_0_minus att_13_k_a att_13_a_minus1_mulka 0=2
BinaryOp add_170 1 1 att_13_a_minus1_mulka att_13_a_minus1_mulka_plus1 0=0 1=1 2=1.0
BinaryOp mul_171 2 1 att_13_a_minus1_mulka_plus1 att_13_key_1 att_13_key_a 0=2
Split split_att_13_value 1 2 att_13_value att_13_value_0 att_13_value_1
Split split_v_first_12 1 2 v_first_12 att_13_v_first v_first_13
BinaryOp minus_12 2 1 att_13_v_first att_13_value_1 att_13_vfirst_minus_value 0=1
Gemm gemm_att_13_mv_lora_0 1 1 att_13_mv_1 att_13_mv_lora_0 4=0 5=1 6=0 7=0 8=32 9=1024 10=-1
Gemm gemm_att_13_mv_lora_1 1 1 att_13_mv_lora_0 att_13_mv_lora_1 4=0 5=1 6=0 7=0 8=1024 9=32 10=-1
MemoryData data_att_13_v0 0 1 att_13_v0 0=1024 21=0
BinaryOp add_171 2 1 att_13_mv_lora_1 att_13_v0 att_13_mv_lora_2 0=0
Sigmoid sigmoid_53 1 1 att_13_mv_lora_2 att_13_mv_lora_sigmoid
BinaryOp mul_172 2 1 att_13_mv_lora_sigmoid att_13_vfirst_minus_value att_13_vfirst_minus_value_mul 0=2
BinaryOp add_172 2 1 att_13_value_0 att_13_vfirst_minus_value_mul att_13_value_final 0=0
Gemm gemm_att_13_mw_lora 1 1 att_13_mw att_13_mw_lora 4=0 5=1 6=0 7=0 8=64 9=1024 10=-1
TanH tanh_att_13_mw_lora 1 1 att_13_mw_lora att_13_mw_lora_tanh
Gemm gemm_att_13_mw_lora_tanh_linear 1 1 att_13_mw_lora_tanh att_13_mw_lora_tanh_linear 4=0 5=1 6=0 7=0 8=1024 9=64 10=-1
MemoryData data_att_13_td 0 1 att_13_td 0=1024 21=0
BinaryOp add_173 2 1 att_13_mw_lora_tanh_linear att_13_td att_13_time_decay_pre 0=0
Sigmoid sigmoid_54 1 1 att_13_time_decay_pre att_13_time_decay_sigmoid
Exp exp_13 1 1 att_13_time_decay_sigmoid att_13_time_decay 1=-0.606531
Reshape reshape_158 1 1 att_13_time_decay att_13_time_decay_reshape 0=64 1=1 2=16
Split split_att_13_key_a 1 2 att_13_key_a att_13_key_a_0 att_13_key_a_1
Reshape reshape_159 1 1 att_13_key_a_0 att_13_key_a_reshape 0=64 1=1 2=16
Split split_att_13_value_final 1 3 att_13_value_final att_13_value_final_0 att_13_value_final_1 att_13_value_final_2
Reshape reshape_160 1 1 att_13_value_final_0 att_13_value_final_0_reshape 0=64 1=16
Reshape reshape_161 1 1 att_13_value_final_1 att_13_value_final_1_reshape 0=1 1=64 2=16
MatMul matmul_att_13_vk 2 1 att_13_value_final_1_reshape att_13_key_a_reshape att_13_vk 0=0
Split split_att_13_key_k_norm_reshape 1 2 att_13_key_k_norm_reshape att_13_kk_0 att_13_kk_1
BinaryOp mul_173 1 1 att_13_kk_0 att_13_kk_0_neg 0=2 1=1 2=-1.0
Reshape reshape_162 1 1 att_13_kk_0_neg att_13_kk_0_neg_reshape 0=1 1=64 2=16
BinaryOp mul_174 2 1 att_13_kk_1 att_13_a_1 att_13_kk_a 0=2
Reshape reshape_163 1 1 att_13_kk_a att_13_kk_a_reshape 0=64 1=1 2=16
MatMul matmul_att_13_ab 2 1 att_13_kk_0_neg_reshape att_13_kk_a_reshape att_13_ab 0=0
Split split_state_40_in 1 2 state_40_in att_13_state_prev_0 att_13_state_prev_1
BinaryOp mul_175 2 1 att_13_state_prev_0 att_13_time_decay_reshape att_13_state_td 0=2
MatMul matmul_att_13_sab 2 1 att_13_state_prev_1 att_13_ab att_13_sab 0=0
BinaryOp add_174 2 1 att_13_sab att_13_vk att_13_sab_vk 0=0
BinaryOp add_175 2 1 att_13_sab_vk att_13_state_td att_13_state_new 0=0
Split split_att_13_state_new 1 2 att_13_state_new att_13_state_new_0 state_40_out
Split split_att_13_receptance 1 2 att_13_receptance att_13_receptance_0 att_13_receptance_1
Reshape reshape_164 1 1 att_13_receptance_0 att_13_receptance_reshape 0=1 1=64 2=16
MatMul matmul_att_13_wkv_out 2 1 att_13_state_new_0 att_13_receptance_reshape att_13_wkv_out 0=0
Reshape reshape_165 1 1 att_13_wkv_out att_13_wkv_out_flatten 0=1024
GroupNorm groupnorm_13 1 1 att_13_wkv_out_flatten att_13_x_gn 0=16 1=1024 2=0.00064 3=1
BinaryOp mul_176 2 1 att_13_receptance_1 att_13_key_a_1 att_13_receptance_key_a 0=2
BinaryOp mul_177 2 1 att_13_receptance_key_a att_13_r_k att_13_rk 0=2
Reshape reshape_166 1 1 att_13_rk att_13_rk_reshape 0=64 1=16
Reduction sum_13 1 1 att_13_rk_reshape att_13_rk_sum 1=0 -23303=1,-1 4=1 5=1
BinaryOp mul_178 2 1 att_13_value_final_0_reshape att_13_rk_sum att_13_value_final_0_rk 0=2
Reshape reshape_167 1 1 att_13_value_final_0_rk att_13_value_final_0_rk_reshape 0=1024
BinaryOp add_176 2 1 att_13_value_final_0_rk_reshape att_13_x_gn att_13_x_gn_rkv 0=0
BinaryOp mul_179 2 1 att_13_x_gn_rkv att_13_gate att_13_x_gate 0=2
Gemm gemm_att_13_x_out 1 1 att_13_x_gate att_13_x_out 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
BinaryOp add_177 2 1 att_13_x_out att_13_x_last time_mixing_13_out 0=0
Split split_time_mixing_13_out 1 2 time_mixing_13_out ffn_13_x_last ffn_13_x
LayerNorm layernorm_27 1 1 ffn_13_x ffn_13_xx 0=1024 1=0.00001 2=1
Split split_ffn_13_xx 1 3 ffn_13_xx ffn_13_xx_0 ffn_13_xx_1 state_41_out
BinaryOp sub_27 2 1 state_41_in ffn_13_xx_0 ffn_13_sx 0=1
Split split_ffn_13_sx 1 1 ffn_13_sx ffn_13_sx_0
MemoryData data_ffn_13_x_k 0 1 ffn_13_x_k 0=1024 21=0
BinaryOp mul_180 2 1 ffn_13_sx_0 ffn_13_x_k ffn_13_xk 0=2
BinaryOp add_178 2 1 ffn_13_xk ffn_13_xx_1 ffn_13_xxk 0=0
Gemm gemm_ffn_13_key 1 1 ffn_13_xxk ffn_13_key 4=0 5=1 6=0 7=0 8=4096 9=1024 10=-1
ReLU relu_13 1 1 ffn_13_key ffn_13_key_relu
UnaryOp square_ffn_13_key_relu 1 1 ffn_13_key_relu ffn_13_key_relu_square 0=4
Gemm gemm_ffn_13_value 1 1 ffn_13_key_relu_square ffn_13_value 4=0 5=1 6=0 7=0 8=1024 9=4096 10=-1
BinaryOp add_179 2 1 ffn_13_value ffn_13_x_last channel_mixing_13_out 0=0
Split split_channel_mixing_13_out 1 2 channel_mixing_13_out att_14_x_last att_14_x
LayerNorm layernorm_28 1 1 att_14_x att_14_xx 0=1024 1=0.00001 2=1
Split split_att_14_xx 1 3 att_14_xx att_14_xx_0 att_14_xx_1 state_42_out
BinaryOp sub_28 2 1 state_42_in att_14_xx_0 att_14_sx 0=1
MemoryData data_att_14_maa 0 1 att_14_maa 0=1024 1=6 21=0
BinaryOp mul_181 2 1 att_14_sx att_14_maa att_14_maa_x 0=2
BinaryOp add_180 2 1 att_14_xx_1 att_14_maa_x att_14_maa_rwkvag 0=0
Slice slice_att_14_maa_rwkvag 1 6 att_14_maa_rwkvag att_14_mr att_14_mw att_14_mk att_14_mv att_14_ma att_14_mg -23300=6,-233,-233,-233,-233,-233,-233 1=0
Split split_att_14_mv 1 2 att_14_mv att_14_mv_0 att_14_mv_1
Gemm gemm_att_14_key 1 1 att_14_mk att_14_key 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_14_value 1 1 att_14_mv_0 att_14_value 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_14_receptance 1 1 att_14_mr att_14_receptance 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_14_ma_lora_0 1 1 att_14_ma att_14_ma_lora_0 4=0 5=1 6=0 7=0 8=64 9=1024 10=-1
Gemm gemm_att_14_ma_lora_1 1 1 att_14_ma_lora_0 att_14_ma_lora_1 4=0 5=1 6=0 7=0 8=1024 9=64 10=-1
MemoryData data_att_14_a0 0 1 att_14_a0 0=1024 21=0
BinaryOp add_181 2 1 att_14_ma_lora_1 att_14_a0 att_14_ma_lora_2 0=0
Sigmoid sigmoid_55 1 1 att_14_ma_lora_2 att_14_a
Split split_att_14_a 1 2 att_14_a att_14_a_0 att_14_a_1
Gemm gemm_att_14_mg_lora 1 1 att_14_mg att_14_mg_lora 4=0 5=1 6=0 7=0 8=128 9=1024 10=-1
Sigmoid sigmoid_56 1 1 att_14_mg_lora att_14_mg_lora_sigmoid
Gemm gemm_att_14_gate 1 1 att_14_mg_lora_sigmoid att_14_gate 4=0 5=1 6=0 7=0 8=1024 9=128 10=-1
MemoryData data_att_14_k_k 0 1 att_14_k_k 0=1024 21=0
MemoryData data_att_14_k_a 0 1 att_14_k_a 0=1024 21=0
MemoryData data_att_14_r_k 0 1 att_14_r_k 0=1024 21=0
Split split_att_14_key 1 2 att_14_key att_14_key_0 att_14_key_1
BinaryOp mul_182 2 1 att_14_key_0 att_14_k_k att_14_key_k 0=2
Reshape reshape_168 1 1 att_14_key_k att_14_key_k_reshape 0=64 1=1 2=16
Normalize l2norm_14 1 1 att_14_key_k_reshape att_14_key_k_norm 0=1 1=1 2=0.0000001 3=1 4=0 9=1
Reshape reshape_169 1 1 att_14_key_k_norm att_14_key_k_norm_reshape 0=1024
BinaryOp add_182 1 1 att_14_a_0 att_14_a_0_minus 0=0 1=1 2=-1.0
BinaryOp mul_183 2 1 att_14_a_0_minus att_14_k_a att_14_a_minus1_mulka 0=2
BinaryOp add_183 1 1 att_14_a_minus1_mulka att_14_a_minus1_mulka_plus1 0=0 1=1 2=1.0
BinaryOp mul_184 2 1 att_14_a_minus1_mulka_plus1 att_14_key_1 att_14_key_a 0=2
Split split_att_14_value 1 2 att_14_value att_14_value_0 att_14_value_1
Split split_v_first_13 1 2 v_first_13 att_14_v_first v_first_14
BinaryOp minus_13 2 1 att_14_v_first att_14_value_1 att_14_vfirst_minus_value 0=1
Gemm gemm_att_14_mv_lora_0 1 1 att_14_mv_1 att_14_mv_lora_0 4=0 5=1 6=0 7=0 8=32 9=1024 10=-1
Gemm gemm_att_14_mv_lora_1 1 1 att_14_mv_lora_0 att_14_mv_lora_1 4=0 5=1 6=0 7=0 8=1024 9=32 10=-1
MemoryData data_att_14_v0 0 1 att_14_v0 0=1024 21=0
BinaryOp add_184 2 1 att_14_mv_lora_1 att_14_v0 att_14_mv_lora_2 0=0
Sigmoid sigmoid_57 1 1 att_14_mv_lora_2 att_14_mv_lora_sigmoid
BinaryOp mul_185 2 1 att_14_mv_lora_sigmoid att_14_vfirst_minus_value att_14_vfirst_minus_value_mul 0=2
BinaryOp add_185 2 1 att_14_value_0 att_14_vfirst_minus_value_mul att_14_value_final 0=0
Gemm gemm_att_14_mw_lora 1 1 att_14_mw att_14_mw_lora 4=0 5=1 6=0 7=0 8=64 9=1024 10=-1
TanH tanh_att_14_mw_lora 1 1 att_14_mw_lora att_14_mw_lora_tanh
Gemm gemm_att_14_mw_lora_tanh_linear 1 1 att_14_mw_lora_tanh att_14_mw_lora_tanh_linear 4=0 5=1 6=0 7=0 8=1024 9=64 10=-1
MemoryData data_att_14_td 0 1 att_14_td 0=1024 21=0
BinaryOp add_186 2 1 att_14_mw_lora_tanh_linear att_14_td att_14_time_decay_pre 0=0
Sigmoid sigmoid_58 1 1 att_14_time_decay_pre att_14_time_decay_sigmoid
Exp exp_14 1 1 att_14_time_decay_sigmoid att_14_time_decay 1=-0.606531
Reshape reshape_170 1 1 att_14_time_decay att_14_time_decay_reshape 0=64 1=1 2=16
Split split_att_14_key_a 1 2 att_14_key_a att_14_key_a_0 att_14_key_a_1
Reshape reshape_171 1 1 att_14_key_a_0 att_14_key_a_reshape 0=64 1=1 2=16
Split split_att_14_value_final 1 3 att_14_value_final att_14_value_final_0 att_14_value_final_1 att_14_value_final_2
Reshape reshape_172 1 1 att_14_value_final_0 att_14_value_final_0_reshape 0=64 1=16
Reshape reshape_173 1 1 att_14_value_final_1 att_14_value_final_1_reshape 0=1 1=64 2=16
MatMul matmul_att_14_vk 2 1 att_14_value_final_1_reshape att_14_key_a_reshape att_14_vk 0=0
Split split_att_14_key_k_norm_reshape 1 2 att_14_key_k_norm_reshape att_14_kk_0 att_14_kk_1
BinaryOp mul_186 1 1 att_14_kk_0 att_14_kk_0_neg 0=2 1=1 2=-1.0
Reshape reshape_174 1 1 att_14_kk_0_neg att_14_kk_0_neg_reshape 0=1 1=64 2=16
BinaryOp mul_187 2 1 att_14_kk_1 att_14_a_1 att_14_kk_a 0=2
Reshape reshape_175 1 1 att_14_kk_a att_14_kk_a_reshape 0=64 1=1 2=16
MatMul matmul_att_14_ab 2 1 att_14_kk_0_neg_reshape att_14_kk_a_reshape att_14_ab 0=0
Split split_state_43_in 1 2 state_43_in att_14_state_prev_0 att_14_state_prev_1
BinaryOp mul_188 2 1 att_14_state_prev_0 att_14_time_decay_reshape att_14_state_td 0=2
MatMul matmul_att_14_sab 2 1 att_14_state_prev_1 att_14_ab att_14_sab 0=0
BinaryOp add_187 2 1 att_14_sab att_14_vk att_14_sab_vk 0=0
BinaryOp add_188 2 1 att_14_sab_vk att_14_state_td att_14_state_new 0=0
Split split_att_14_state_new 1 2 att_14_state_new att_14_state_new_0 state_43_out
Split split_att_14_receptance 1 2 att_14_receptance att_14_receptance_0 att_14_receptance_1
Reshape reshape_176 1 1 att_14_receptance_0 att_14_receptance_reshape 0=1 1=64 2=16
MatMul matmul_att_14_wkv_out 2 1 att_14_state_new_0 att_14_receptance_reshape att_14_wkv_out 0=0
Reshape reshape_177 1 1 att_14_wkv_out att_14_wkv_out_flatten 0=1024
GroupNorm groupnorm_14 1 1 att_14_wkv_out_flatten att_14_x_gn 0=16 1=1024 2=0.00064 3=1
BinaryOp mul_189 2 1 att_14_receptance_1 att_14_key_a_1 att_14_receptance_key_a 0=2
BinaryOp mul_190 2 1 att_14_receptance_key_a att_14_r_k att_14_rk 0=2
Reshape reshape_178 1 1 att_14_rk att_14_rk_reshape 0=64 1=16
Reduction sum_14 1 1 att_14_rk_reshape att_14_rk_sum 1=0 -23303=1,-1 4=1 5=1
BinaryOp mul_191 2 1 att_14_value_final_0_reshape att_14_rk_sum att_14_value_final_0_rk 0=2
Reshape reshape_179 1 1 att_14_value_final_0_rk att_14_value_final_0_rk_reshape 0=1024
BinaryOp add_189 2 1 att_14_value_final_0_rk_reshape att_14_x_gn att_14_x_gn_rkv 0=0
BinaryOp mul_192 2 1 att_14_x_gn_rkv att_14_gate att_14_x_gate 0=2
Gemm gemm_att_14_x_out 1 1 att_14_x_gate att_14_x_out 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
BinaryOp add_190 2 1 att_14_x_out att_14_x_last time_mixing_14_out 0=0
Split split_time_mixing_14_out 1 2 time_mixing_14_out ffn_14_x_last ffn_14_x
LayerNorm layernorm_29 1 1 ffn_14_x ffn_14_xx 0=1024 1=0.00001 2=1
Split split_ffn_14_xx 1 3 ffn_14_xx ffn_14_xx_0 ffn_14_xx_1 state_44_out
BinaryOp sub_29 2 1 state_44_in ffn_14_xx_0 ffn_14_sx 0=1
Split split_ffn_14_sx 1 1 ffn_14_sx ffn_14_sx_0
MemoryData data_ffn_14_x_k 0 1 ffn_14_x_k 0=1024 21=0
BinaryOp mul_193 2 1 ffn_14_sx_0 ffn_14_x_k ffn_14_xk 0=2
BinaryOp add_191 2 1 ffn_14_xk ffn_14_xx_1 ffn_14_xxk 0=0
Gemm gemm_ffn_14_key 1 1 ffn_14_xxk ffn_14_key 4=0 5=1 6=0 7=0 8=4096 9=1024 10=-1
ReLU relu_14 1 1 ffn_14_key ffn_14_key_relu
UnaryOp square_ffn_14_key_relu 1 1 ffn_14_key_relu ffn_14_key_relu_square 0=4
Gemm gemm_ffn_14_value 1 1 ffn_14_key_relu_square ffn_14_value 4=0 5=1 6=0 7=0 8=1024 9=4096 10=-1
BinaryOp add_192 2 1 ffn_14_value ffn_14_x_last channel_mixing_14_out 0=0
Split split_channel_mixing_14_out 1 2 channel_mixing_14_out att_15_x_last att_15_x
LayerNorm layernorm_30 1 1 att_15_x att_15_xx 0=1024 1=0.00001 2=1
Split split_att_15_xx 1 3 att_15_xx att_15_xx_0 att_15_xx_1 state_45_out
BinaryOp sub_30 2 1 state_45_in att_15_xx_0 att_15_sx 0=1
MemoryData data_att_15_maa 0 1 att_15_maa 0=1024 1=6 21=0
BinaryOp mul_194 2 1 att_15_sx att_15_maa att_15_maa_x 0=2
BinaryOp add_193 2 1 att_15_xx_1 att_15_maa_x att_15_maa_rwkvag 0=0
Slice slice_att_15_maa_rwkvag 1 6 att_15_maa_rwkvag att_15_mr att_15_mw att_15_mk att_15_mv att_15_ma att_15_mg -23300=6,-233,-233,-233,-233,-233,-233 1=0
Split split_att_15_mv 1 2 att_15_mv att_15_mv_0 att_15_mv_1
Gemm gemm_att_15_key 1 1 att_15_mk att_15_key 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_15_value 1 1 att_15_mv_0 att_15_value 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_15_receptance 1 1 att_15_mr att_15_receptance 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_15_ma_lora_0 1 1 att_15_ma att_15_ma_lora_0 4=0 5=1 6=0 7=0 8=64 9=1024 10=-1
Gemm gemm_att_15_ma_lora_1 1 1 att_15_ma_lora_0 att_15_ma_lora_1 4=0 5=1 6=0 7=0 8=1024 9=64 10=-1
MemoryData data_att_15_a0 0 1 att_15_a0 0=1024 21=0
BinaryOp add_194 2 1 att_15_ma_lora_1 att_15_a0 att_15_ma_lora_2 0=0
Sigmoid sigmoid_59 1 1 att_15_ma_lora_2 att_15_a
Split split_att_15_a 1 2 att_15_a att_15_a_0 att_15_a_1
Gemm gemm_att_15_mg_lora 1 1 att_15_mg att_15_mg_lora 4=0 5=1 6=0 7=0 8=128 9=1024 10=-1
Sigmoid sigmoid_60 1 1 att_15_mg_lora att_15_mg_lora_sigmoid
Gemm gemm_att_15_gate 1 1 att_15_mg_lora_sigmoid att_15_gate 4=0 5=1 6=0 7=0 8=1024 9=128 10=-1
MemoryData data_att_15_k_k 0 1 att_15_k_k 0=1024 21=0
MemoryData data_att_15_k_a 0 1 att_15_k_a 0=1024 21=0
MemoryData data_att_15_r_k 0 1 att_15_r_k 0=1024 21=0
Split split_att_15_key 1 2 att_15_key att_15_key_0 att_15_key_1
BinaryOp mul_195 2 1 att_15_key_0 att_15_k_k att_15_key_k 0=2
Reshape reshape_180 1 1 att_15_key_k att_15_key_k_reshape 0=64 1=1 2=16
Normalize l2norm_15 1 1 att_15_key_k_reshape att_15_key_k_norm 0=1 1=1 2=0.0000001 3=1 4=0 9=1
Reshape reshape_181 1 1 att_15_key_k_norm att_15_key_k_norm_reshape 0=1024
BinaryOp add_195 1 1 att_15_a_0 att_15_a_0_minus 0=0 1=1 2=-1.0
BinaryOp mul_196 2 1 att_15_a_0_minus att_15_k_a att_15_a_minus1_mulka 0=2
BinaryOp add_196 1 1 att_15_a_minus1_mulka att_15_a_minus1_mulka_plus1 0=0 1=1 2=1.0
BinaryOp mul_197 2 1 att_15_a_minus1_mulka_plus1 att_15_key_1 att_15_key_a 0=2
Split split_att_15_value 1 2 att_15_value att_15_value_0 att_15_value_1
Split split_v_first_14 1 2 v_first_14 att_15_v_first v_first_15
BinaryOp minus_14 2 1 att_15_v_first att_15_value_1 att_15_vfirst_minus_value 0=1
Gemm gemm_att_15_mv_lora_0 1 1 att_15_mv_1 att_15_mv_lora_0 4=0 5=1 6=0 7=0 8=32 9=1024 10=-1
Gemm gemm_att_15_mv_lora_1 1 1 att_15_mv_lora_0 att_15_mv_lora_1 4=0 5=1 6=0 7=0 8=1024 9=32 10=-1
MemoryData data_att_15_v0 0 1 att_15_v0 0=1024 21=0
BinaryOp add_197 2 1 att_15_mv_lora_1 att_15_v0 att_15_mv_lora_2 0=0
Sigmoid sigmoid_61 1 1 att_15_mv_lora_2 att_15_mv_lora_sigmoid
BinaryOp mul_198 2 1 att_15_mv_lora_sigmoid att_15_vfirst_minus_value att_15_vfirst_minus_value_mul 0=2
BinaryOp add_198 2 1 att_15_value_0 att_15_vfirst_minus_value_mul att_15_value_final 0=0
Gemm gemm_att_15_mw_lora 1 1 att_15_mw att_15_mw_lora 4=0 5=1 6=0 7=0 8=64 9=1024 10=-1
TanH tanh_att_15_mw_lora 1 1 att_15_mw_lora att_15_mw_lora_tanh
Gemm gemm_att_15_mw_lora_tanh_linear 1 1 att_15_mw_lora_tanh att_15_mw_lora_tanh_linear 4=0 5=1 6=0 7=0 8=1024 9=64 10=-1
MemoryData data_att_15_td 0 1 att_15_td 0=1024 21=0
BinaryOp add_199 2 1 att_15_mw_lora_tanh_linear att_15_td att_15_time_decay_pre 0=0
Sigmoid sigmoid_62 1 1 att_15_time_decay_pre att_15_time_decay_sigmoid
Exp exp_15 1 1 att_15_time_decay_sigmoid att_15_time_decay 1=-0.606531
Reshape reshape_182 1 1 att_15_time_decay att_15_time_decay_reshape 0=64 1=1 2=16
Split split_att_15_key_a 1 2 att_15_key_a att_15_key_a_0 att_15_key_a_1
Reshape reshape_183 1 1 att_15_key_a_0 att_15_key_a_reshape 0=64 1=1 2=16
Split split_att_15_value_final 1 3 att_15_value_final att_15_value_final_0 att_15_value_final_1 att_15_value_final_2
Reshape reshape_184 1 1 att_15_value_final_0 att_15_value_final_0_reshape 0=64 1=16
Reshape reshape_185 1 1 att_15_value_final_1 att_15_value_final_1_reshape 0=1 1=64 2=16
MatMul matmul_att_15_vk 2 1 att_15_value_final_1_reshape att_15_key_a_reshape att_15_vk 0=0
Split split_att_15_key_k_norm_reshape 1 2 att_15_key_k_norm_reshape att_15_kk_0 att_15_kk_1
BinaryOp mul_199 1 1 att_15_kk_0 att_15_kk_0_neg 0=2 1=1 2=-1.0
Reshape reshape_186 1 1 att_15_kk_0_neg att_15_kk_0_neg_reshape 0=1 1=64 2=16
BinaryOp mul_200 2 1 att_15_kk_1 att_15_a_1 att_15_kk_a 0=2
Reshape reshape_187 1 1 att_15_kk_a att_15_kk_a_reshape 0=64 1=1 2=16
MatMul matmul_att_15_ab 2 1 att_15_kk_0_neg_reshape att_15_kk_a_reshape att_15_ab 0=0
Split split_state_46_in 1 2 state_46_in att_15_state_prev_0 att_15_state_prev_1
BinaryOp mul_201 2 1 att_15_state_prev_0 att_15_time_decay_reshape att_15_state_td 0=2
MatMul matmul_att_15_sab 2 1 att_15_state_prev_1 att_15_ab att_15_sab 0=0
BinaryOp add_200 2 1 att_15_sab att_15_vk att_15_sab_vk 0=0
BinaryOp add_201 2 1 att_15_sab_vk att_15_state_td att_15_state_new 0=0
Split split_att_15_state_new 1 2 att_15_state_new att_15_state_new_0 state_46_out
Split split_att_15_receptance 1 2 att_15_receptance att_15_receptance_0 att_15_receptance_1
Reshape reshape_188 1 1 att_15_receptance_0 att_15_receptance_reshape 0=1 1=64 2=16
MatMul matmul_att_15_wkv_out 2 1 att_15_state_new_0 att_15_receptance_reshape att_15_wkv_out 0=0
Reshape reshape_189 1 1 att_15_wkv_out att_15_wkv_out_flatten 0=1024
GroupNorm groupnorm_15 1 1 att_15_wkv_out_flatten att_15_x_gn 0=16 1=1024 2=0.00064 3=1
BinaryOp mul_202 2 1 att_15_receptance_1 att_15_key_a_1 att_15_receptance_key_a 0=2
BinaryOp mul_203 2 1 att_15_receptance_key_a att_15_r_k att_15_rk 0=2
Reshape reshape_190 1 1 att_15_rk att_15_rk_reshape 0=64 1=16
Reduction sum_15 1 1 att_15_rk_reshape att_15_rk_sum 1=0 -23303=1,-1 4=1 5=1
BinaryOp mul_204 2 1 att_15_value_final_0_reshape att_15_rk_sum att_15_value_final_0_rk 0=2
Reshape reshape_191 1 1 att_15_value_final_0_rk att_15_value_final_0_rk_reshape 0=1024
BinaryOp add_202 2 1 att_15_value_final_0_rk_reshape att_15_x_gn att_15_x_gn_rkv 0=0
BinaryOp mul_205 2 1 att_15_x_gn_rkv att_15_gate att_15_x_gate 0=2
Gemm gemm_att_15_x_out 1 1 att_15_x_gate att_15_x_out 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
BinaryOp add_203 2 1 att_15_x_out att_15_x_last time_mixing_15_out 0=0
Split split_time_mixing_15_out 1 2 time_mixing_15_out ffn_15_x_last ffn_15_x
LayerNorm layernorm_31 1 1 ffn_15_x ffn_15_xx 0=1024 1=0.00001 2=1
Split split_ffn_15_xx 1 3 ffn_15_xx ffn_15_xx_0 ffn_15_xx_1 state_47_out
BinaryOp sub_31 2 1 state_47_in ffn_15_xx_0 ffn_15_sx 0=1
Split split_ffn_15_sx 1 1 ffn_15_sx ffn_15_sx_0
MemoryData data_ffn_15_x_k 0 1 ffn_15_x_k 0=1024 21=0
BinaryOp mul_206 2 1 ffn_15_sx_0 ffn_15_x_k ffn_15_xk 0=2
BinaryOp add_204 2 1 ffn_15_xk ffn_15_xx_1 ffn_15_xxk 0=0
Gemm gemm_ffn_15_key 1 1 ffn_15_xxk ffn_15_key 4=0 5=1 6=0 7=0 8=4096 9=1024 10=-1
ReLU relu_15 1 1 ffn_15_key ffn_15_key_relu
UnaryOp square_ffn_15_key_relu 1 1 ffn_15_key_relu ffn_15_key_relu_square 0=4
Gemm gemm_ffn_15_value 1 1 ffn_15_key_relu_square ffn_15_value 4=0 5=1 6=0 7=0 8=1024 9=4096 10=-1
BinaryOp add_205 2 1 ffn_15_value ffn_15_x_last channel_mixing_15_out 0=0
Split split_channel_mixing_15_out 1 2 channel_mixing_15_out att_16_x_last att_16_x
LayerNorm layernorm_32 1 1 att_16_x att_16_xx 0=1024 1=0.00001 2=1
Split split_att_16_xx 1 3 att_16_xx att_16_xx_0 att_16_xx_1 state_48_out
BinaryOp sub_32 2 1 state_48_in att_16_xx_0 att_16_sx 0=1
MemoryData data_att_16_maa 0 1 att_16_maa 0=1024 1=6 21=0
BinaryOp mul_207 2 1 att_16_sx att_16_maa att_16_maa_x 0=2
BinaryOp add_206 2 1 att_16_xx_1 att_16_maa_x att_16_maa_rwkvag 0=0
Slice slice_att_16_maa_rwkvag 1 6 att_16_maa_rwkvag att_16_mr att_16_mw att_16_mk att_16_mv att_16_ma att_16_mg -23300=6,-233,-233,-233,-233,-233,-233 1=0
Split split_att_16_mv 1 2 att_16_mv att_16_mv_0 att_16_mv_1
Gemm gemm_att_16_key 1 1 att_16_mk att_16_key 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_16_value 1 1 att_16_mv_0 att_16_value 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_16_receptance 1 1 att_16_mr att_16_receptance 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_16_ma_lora_0 1 1 att_16_ma att_16_ma_lora_0 4=0 5=1 6=0 7=0 8=64 9=1024 10=-1
Gemm gemm_att_16_ma_lora_1 1 1 att_16_ma_lora_0 att_16_ma_lora_1 4=0 5=1 6=0 7=0 8=1024 9=64 10=-1
MemoryData data_att_16_a0 0 1 att_16_a0 0=1024 21=0
BinaryOp add_207 2 1 att_16_ma_lora_1 att_16_a0 att_16_ma_lora_2 0=0
Sigmoid sigmoid_63 1 1 att_16_ma_lora_2 att_16_a
Split split_att_16_a 1 2 att_16_a att_16_a_0 att_16_a_1
Gemm gemm_att_16_mg_lora 1 1 att_16_mg att_16_mg_lora 4=0 5=1 6=0 7=0 8=128 9=1024 10=-1
Sigmoid sigmoid_64 1 1 att_16_mg_lora att_16_mg_lora_sigmoid
Gemm gemm_att_16_gate 1 1 att_16_mg_lora_sigmoid att_16_gate 4=0 5=1 6=0 7=0 8=1024 9=128 10=-1
MemoryData data_att_16_k_k 0 1 att_16_k_k 0=1024 21=0
MemoryData data_att_16_k_a 0 1 att_16_k_a 0=1024 21=0
MemoryData data_att_16_r_k 0 1 att_16_r_k 0=1024 21=0
Split split_att_16_key 1 2 att_16_key att_16_key_0 att_16_key_1
BinaryOp mul_208 2 1 att_16_key_0 att_16_k_k att_16_key_k 0=2
Reshape reshape_192 1 1 att_16_key_k att_16_key_k_reshape 0=64 1=1 2=16
Normalize l2norm_16 1 1 att_16_key_k_reshape att_16_key_k_norm 0=1 1=1 2=0.0000001 3=1 4=0 9=1
Reshape reshape_193 1 1 att_16_key_k_norm att_16_key_k_norm_reshape 0=1024
BinaryOp add_208 1 1 att_16_a_0 att_16_a_0_minus 0=0 1=1 2=-1.0
BinaryOp mul_209 2 1 att_16_a_0_minus att_16_k_a att_16_a_minus1_mulka 0=2
BinaryOp add_209 1 1 att_16_a_minus1_mulka att_16_a_minus1_mulka_plus1 0=0 1=1 2=1.0
BinaryOp mul_210 2 1 att_16_a_minus1_mulka_plus1 att_16_key_1 att_16_key_a 0=2
Split split_att_16_value 1 2 att_16_value att_16_value_0 att_16_value_1
Split split_v_first_15 1 2 v_first_15 att_16_v_first v_first_16
BinaryOp minus_15 2 1 att_16_v_first att_16_value_1 att_16_vfirst_minus_value 0=1
Gemm gemm_att_16_mv_lora_0 1 1 att_16_mv_1 att_16_mv_lora_0 4=0 5=1 6=0 7=0 8=32 9=1024 10=-1
Gemm gemm_att_16_mv_lora_1 1 1 att_16_mv_lora_0 att_16_mv_lora_1 4=0 5=1 6=0 7=0 8=1024 9=32 10=-1
MemoryData data_att_16_v0 0 1 att_16_v0 0=1024 21=0
BinaryOp add_210 2 1 att_16_mv_lora_1 att_16_v0 att_16_mv_lora_2 0=0
Sigmoid sigmoid_65 1 1 att_16_mv_lora_2 att_16_mv_lora_sigmoid
BinaryOp mul_211 2 1 att_16_mv_lora_sigmoid att_16_vfirst_minus_value att_16_vfirst_minus_value_mul 0=2
BinaryOp add_211 2 1 att_16_value_0 att_16_vfirst_minus_value_mul att_16_value_final 0=0
Gemm gemm_att_16_mw_lora 1 1 att_16_mw att_16_mw_lora 4=0 5=1 6=0 7=0 8=64 9=1024 10=-1
TanH tanh_att_16_mw_lora 1 1 att_16_mw_lora att_16_mw_lora_tanh
Gemm gemm_att_16_mw_lora_tanh_linear 1 1 att_16_mw_lora_tanh att_16_mw_lora_tanh_linear 4=0 5=1 6=0 7=0 8=1024 9=64 10=-1
MemoryData data_att_16_td 0 1 att_16_td 0=1024 21=0
BinaryOp add_212 2 1 att_16_mw_lora_tanh_linear att_16_td att_16_time_decay_pre 0=0
Sigmoid sigmoid_66 1 1 att_16_time_decay_pre att_16_time_decay_sigmoid
Exp exp_16 1 1 att_16_time_decay_sigmoid att_16_time_decay 1=-0.606531
Reshape reshape_194 1 1 att_16_time_decay att_16_time_decay_reshape 0=64 1=1 2=16
Split split_att_16_key_a 1 2 att_16_key_a att_16_key_a_0 att_16_key_a_1
Reshape reshape_195 1 1 att_16_key_a_0 att_16_key_a_reshape 0=64 1=1 2=16
Split split_att_16_value_final 1 3 att_16_value_final att_16_value_final_0 att_16_value_final_1 att_16_value_final_2
Reshape reshape_196 1 1 att_16_value_final_0 att_16_value_final_0_reshape 0=64 1=16
Reshape reshape_197 1 1 att_16_value_final_1 att_16_value_final_1_reshape 0=1 1=64 2=16
MatMul matmul_att_16_vk 2 1 att_16_value_final_1_reshape att_16_key_a_reshape att_16_vk 0=0
Split split_att_16_key_k_norm_reshape 1 2 att_16_key_k_norm_reshape att_16_kk_0 att_16_kk_1
BinaryOp mul_212 1 1 att_16_kk_0 att_16_kk_0_neg 0=2 1=1 2=-1.0
Reshape reshape_198 1 1 att_16_kk_0_neg att_16_kk_0_neg_reshape 0=1 1=64 2=16
BinaryOp mul_213 2 1 att_16_kk_1 att_16_a_1 att_16_kk_a 0=2
Reshape reshape_199 1 1 att_16_kk_a att_16_kk_a_reshape 0=64 1=1 2=16
MatMul matmul_att_16_ab 2 1 att_16_kk_0_neg_reshape att_16_kk_a_reshape att_16_ab 0=0
Split split_state_49_in 1 2 state_49_in att_16_state_prev_0 att_16_state_prev_1
BinaryOp mul_214 2 1 att_16_state_prev_0 att_16_time_decay_reshape att_16_state_td 0=2
MatMul matmul_att_16_sab 2 1 att_16_state_prev_1 att_16_ab att_16_sab 0=0
BinaryOp add_213 2 1 att_16_sab att_16_vk att_16_sab_vk 0=0
BinaryOp add_214 2 1 att_16_sab_vk att_16_state_td att_16_state_new 0=0
Split split_att_16_state_new 1 2 att_16_state_new att_16_state_new_0 state_49_out
Split split_att_16_receptance 1 2 att_16_receptance att_16_receptance_0 att_16_receptance_1
Reshape reshape_200 1 1 att_16_receptance_0 att_16_receptance_reshape 0=1 1=64 2=16
MatMul matmul_att_16_wkv_out 2 1 att_16_state_new_0 att_16_receptance_reshape att_16_wkv_out 0=0
Reshape reshape_201 1 1 att_16_wkv_out att_16_wkv_out_flatten 0=1024
GroupNorm groupnorm_16 1 1 att_16_wkv_out_flatten att_16_x_gn 0=16 1=1024 2=0.00064 3=1
BinaryOp mul_215 2 1 att_16_receptance_1 att_16_key_a_1 att_16_receptance_key_a 0=2
BinaryOp mul_216 2 1 att_16_receptance_key_a att_16_r_k att_16_rk 0=2
Reshape reshape_202 1 1 att_16_rk att_16_rk_reshape 0=64 1=16
Reduction sum_16 1 1 att_16_rk_reshape att_16_rk_sum 1=0 -23303=1,-1 4=1 5=1
BinaryOp mul_217 2 1 att_16_value_final_0_reshape att_16_rk_sum att_16_value_final_0_rk 0=2
Reshape reshape_203 1 1 att_16_value_final_0_rk att_16_value_final_0_rk_reshape 0=1024
BinaryOp add_215 2 1 att_16_value_final_0_rk_reshape att_16_x_gn att_16_x_gn_rkv 0=0
BinaryOp mul_218 2 1 att_16_x_gn_rkv att_16_gate att_16_x_gate 0=2
Gemm gemm_att_16_x_out 1 1 att_16_x_gate att_16_x_out 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
BinaryOp add_216 2 1 att_16_x_out att_16_x_last time_mixing_16_out 0=0
Split split_time_mixing_16_out 1 2 time_mixing_16_out ffn_16_x_last ffn_16_x
LayerNorm layernorm_33 1 1 ffn_16_x ffn_16_xx 0=1024 1=0.00001 2=1
Split split_ffn_16_xx 1 3 ffn_16_xx ffn_16_xx_0 ffn_16_xx_1 state_50_out
BinaryOp sub_33 2 1 state_50_in ffn_16_xx_0 ffn_16_sx 0=1
Split split_ffn_16_sx 1 1 ffn_16_sx ffn_16_sx_0
MemoryData data_ffn_16_x_k 0 1 ffn_16_x_k 0=1024 21=0
BinaryOp mul_219 2 1 ffn_16_sx_0 ffn_16_x_k ffn_16_xk 0=2
BinaryOp add_217 2 1 ffn_16_xk ffn_16_xx_1 ffn_16_xxk 0=0
Gemm gemm_ffn_16_key 1 1 ffn_16_xxk ffn_16_key 4=0 5=1 6=0 7=0 8=4096 9=1024 10=-1
ReLU relu_16 1 1 ffn_16_key ffn_16_key_relu
UnaryOp square_ffn_16_key_relu 1 1 ffn_16_key_relu ffn_16_key_relu_square 0=4
Gemm gemm_ffn_16_value 1 1 ffn_16_key_relu_square ffn_16_value 4=0 5=1 6=0 7=0 8=1024 9=4096 10=-1
BinaryOp add_218 2 1 ffn_16_value ffn_16_x_last channel_mixing_16_out 0=0
Split split_channel_mixing_16_out 1 2 channel_mixing_16_out att_17_x_last att_17_x
LayerNorm layernorm_34 1 1 att_17_x att_17_xx 0=1024 1=0.00001 2=1
Split split_att_17_xx 1 3 att_17_xx att_17_xx_0 att_17_xx_1 state_51_out
BinaryOp sub_34 2 1 state_51_in att_17_xx_0 att_17_sx 0=1
MemoryData data_att_17_maa 0 1 att_17_maa 0=1024 1=6 21=0
BinaryOp mul_220 2 1 att_17_sx att_17_maa att_17_maa_x 0=2
BinaryOp add_219 2 1 att_17_xx_1 att_17_maa_x att_17_maa_rwkvag 0=0
Slice slice_att_17_maa_rwkvag 1 6 att_17_maa_rwkvag att_17_mr att_17_mw att_17_mk att_17_mv att_17_ma att_17_mg -23300=6,-233,-233,-233,-233,-233,-233 1=0
Split split_att_17_mv 1 2 att_17_mv att_17_mv_0 att_17_mv_1
Gemm gemm_att_17_key 1 1 att_17_mk att_17_key 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_17_value 1 1 att_17_mv_0 att_17_value 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_17_receptance 1 1 att_17_mr att_17_receptance 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_17_ma_lora_0 1 1 att_17_ma att_17_ma_lora_0 4=0 5=1 6=0 7=0 8=64 9=1024 10=-1
Gemm gemm_att_17_ma_lora_1 1 1 att_17_ma_lora_0 att_17_ma_lora_1 4=0 5=1 6=0 7=0 8=1024 9=64 10=-1
MemoryData data_att_17_a0 0 1 att_17_a0 0=1024 21=0
BinaryOp add_220 2 1 att_17_ma_lora_1 att_17_a0 att_17_ma_lora_2 0=0
Sigmoid sigmoid_67 1 1 att_17_ma_lora_2 att_17_a
Split split_att_17_a 1 2 att_17_a att_17_a_0 att_17_a_1
Gemm gemm_att_17_mg_lora 1 1 att_17_mg att_17_mg_lora 4=0 5=1 6=0 7=0 8=128 9=1024 10=-1
Sigmoid sigmoid_68 1 1 att_17_mg_lora att_17_mg_lora_sigmoid
Gemm gemm_att_17_gate 1 1 att_17_mg_lora_sigmoid att_17_gate 4=0 5=1 6=0 7=0 8=1024 9=128 10=-1
MemoryData data_att_17_k_k 0 1 att_17_k_k 0=1024 21=0
MemoryData data_att_17_k_a 0 1 att_17_k_a 0=1024 21=0
MemoryData data_att_17_r_k 0 1 att_17_r_k 0=1024 21=0
Split split_att_17_key 1 2 att_17_key att_17_key_0 att_17_key_1
BinaryOp mul_221 2 1 att_17_key_0 att_17_k_k att_17_key_k 0=2
Reshape reshape_204 1 1 att_17_key_k att_17_key_k_reshape 0=64 1=1 2=16
Normalize l2norm_17 1 1 att_17_key_k_reshape att_17_key_k_norm 0=1 1=1 2=0.0000001 3=1 4=0 9=1
Reshape reshape_205 1 1 att_17_key_k_norm att_17_key_k_norm_reshape 0=1024
BinaryOp add_221 1 1 att_17_a_0 att_17_a_0_minus 0=0 1=1 2=-1.0
BinaryOp mul_222 2 1 att_17_a_0_minus att_17_k_a att_17_a_minus1_mulka 0=2
BinaryOp add_222 1 1 att_17_a_minus1_mulka att_17_a_minus1_mulka_plus1 0=0 1=1 2=1.0
BinaryOp mul_223 2 1 att_17_a_minus1_mulka_plus1 att_17_key_1 att_17_key_a 0=2
Split split_att_17_value 1 2 att_17_value att_17_value_0 att_17_value_1
Split split_v_first_16 1 2 v_first_16 att_17_v_first v_first_17
BinaryOp minus_16 2 1 att_17_v_first att_17_value_1 att_17_vfirst_minus_value 0=1
Gemm gemm_att_17_mv_lora_0 1 1 att_17_mv_1 att_17_mv_lora_0 4=0 5=1 6=0 7=0 8=32 9=1024 10=-1
Gemm gemm_att_17_mv_lora_1 1 1 att_17_mv_lora_0 att_17_mv_lora_1 4=0 5=1 6=0 7=0 8=1024 9=32 10=-1
MemoryData data_att_17_v0 0 1 att_17_v0 0=1024 21=0
BinaryOp add_223 2 1 att_17_mv_lora_1 att_17_v0 att_17_mv_lora_2 0=0
Sigmoid sigmoid_69 1 1 att_17_mv_lora_2 att_17_mv_lora_sigmoid
BinaryOp mul_224 2 1 att_17_mv_lora_sigmoid att_17_vfirst_minus_value att_17_vfirst_minus_value_mul 0=2
BinaryOp add_224 2 1 att_17_value_0 att_17_vfirst_minus_value_mul att_17_value_final 0=0
Gemm gemm_att_17_mw_lora 1 1 att_17_mw att_17_mw_lora 4=0 5=1 6=0 7=0 8=64 9=1024 10=-1
TanH tanh_att_17_mw_lora 1 1 att_17_mw_lora att_17_mw_lora_tanh
Gemm gemm_att_17_mw_lora_tanh_linear 1 1 att_17_mw_lora_tanh att_17_mw_lora_tanh_linear 4=0 5=1 6=0 7=0 8=1024 9=64 10=-1
MemoryData data_att_17_td 0 1 att_17_td 0=1024 21=0
BinaryOp add_225 2 1 att_17_mw_lora_tanh_linear att_17_td att_17_time_decay_pre 0=0
Sigmoid sigmoid_70 1 1 att_17_time_decay_pre att_17_time_decay_sigmoid
Exp exp_17 1 1 att_17_time_decay_sigmoid att_17_time_decay 1=-0.606531
Reshape reshape_206 1 1 att_17_time_decay att_17_time_decay_reshape 0=64 1=1 2=16
Split split_att_17_key_a 1 2 att_17_key_a att_17_key_a_0 att_17_key_a_1
Reshape reshape_207 1 1 att_17_key_a_0 att_17_key_a_reshape 0=64 1=1 2=16
Split split_att_17_value_final 1 3 att_17_value_final att_17_value_final_0 att_17_value_final_1 att_17_value_final_2
Reshape reshape_208 1 1 att_17_value_final_0 att_17_value_final_0_reshape 0=64 1=16
Reshape reshape_209 1 1 att_17_value_final_1 att_17_value_final_1_reshape 0=1 1=64 2=16
MatMul matmul_att_17_vk 2 1 att_17_value_final_1_reshape att_17_key_a_reshape att_17_vk 0=0
Split split_att_17_key_k_norm_reshape 1 2 att_17_key_k_norm_reshape att_17_kk_0 att_17_kk_1
BinaryOp mul_225 1 1 att_17_kk_0 att_17_kk_0_neg 0=2 1=1 2=-1.0
Reshape reshape_210 1 1 att_17_kk_0_neg att_17_kk_0_neg_reshape 0=1 1=64 2=16
BinaryOp mul_226 2 1 att_17_kk_1 att_17_a_1 att_17_kk_a 0=2
Reshape reshape_211 1 1 att_17_kk_a att_17_kk_a_reshape 0=64 1=1 2=16
MatMul matmul_att_17_ab 2 1 att_17_kk_0_neg_reshape att_17_kk_a_reshape att_17_ab 0=0
Split split_state_52_in 1 2 state_52_in att_17_state_prev_0 att_17_state_prev_1
BinaryOp mul_227 2 1 att_17_state_prev_0 att_17_time_decay_reshape att_17_state_td 0=2
MatMul matmul_att_17_sab 2 1 att_17_state_prev_1 att_17_ab att_17_sab 0=0
BinaryOp add_226 2 1 att_17_sab att_17_vk att_17_sab_vk 0=0
BinaryOp add_227 2 1 att_17_sab_vk att_17_state_td att_17_state_new 0=0
Split split_att_17_state_new 1 2 att_17_state_new att_17_state_new_0 state_52_out
Split split_att_17_receptance 1 2 att_17_receptance att_17_receptance_0 att_17_receptance_1
Reshape reshape_212 1 1 att_17_receptance_0 att_17_receptance_reshape 0=1 1=64 2=16
MatMul matmul_att_17_wkv_out 2 1 att_17_state_new_0 att_17_receptance_reshape att_17_wkv_out 0=0
Reshape reshape_213 1 1 att_17_wkv_out att_17_wkv_out_flatten 0=1024
GroupNorm groupnorm_17 1 1 att_17_wkv_out_flatten att_17_x_gn 0=16 1=1024 2=0.00064 3=1
BinaryOp mul_228 2 1 att_17_receptance_1 att_17_key_a_1 att_17_receptance_key_a 0=2
BinaryOp mul_229 2 1 att_17_receptance_key_a att_17_r_k att_17_rk 0=2
Reshape reshape_214 1 1 att_17_rk att_17_rk_reshape 0=64 1=16
Reduction sum_17 1 1 att_17_rk_reshape att_17_rk_sum 1=0 -23303=1,-1 4=1 5=1
BinaryOp mul_230 2 1 att_17_value_final_0_reshape att_17_rk_sum att_17_value_final_0_rk 0=2
Reshape reshape_215 1 1 att_17_value_final_0_rk att_17_value_final_0_rk_reshape 0=1024
BinaryOp add_228 2 1 att_17_value_final_0_rk_reshape att_17_x_gn att_17_x_gn_rkv 0=0
BinaryOp mul_231 2 1 att_17_x_gn_rkv att_17_gate att_17_x_gate 0=2
Gemm gemm_att_17_x_out 1 1 att_17_x_gate att_17_x_out 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
BinaryOp add_229 2 1 att_17_x_out att_17_x_last time_mixing_17_out 0=0
Split split_time_mixing_17_out 1 2 time_mixing_17_out ffn_17_x_last ffn_17_x
LayerNorm layernorm_35 1 1 ffn_17_x ffn_17_xx 0=1024 1=0.00001 2=1
Split split_ffn_17_xx 1 3 ffn_17_xx ffn_17_xx_0 ffn_17_xx_1 state_53_out
BinaryOp sub_35 2 1 state_53_in ffn_17_xx_0 ffn_17_sx 0=1
Split split_ffn_17_sx 1 1 ffn_17_sx ffn_17_sx_0
MemoryData data_ffn_17_x_k 0 1 ffn_17_x_k 0=1024 21=0
BinaryOp mul_232 2 1 ffn_17_sx_0 ffn_17_x_k ffn_17_xk 0=2
BinaryOp add_230 2 1 ffn_17_xk ffn_17_xx_1 ffn_17_xxk 0=0
Gemm gemm_ffn_17_key 1 1 ffn_17_xxk ffn_17_key 4=0 5=1 6=0 7=0 8=4096 9=1024 10=-1
ReLU relu_17 1 1 ffn_17_key ffn_17_key_relu
UnaryOp square_ffn_17_key_relu 1 1 ffn_17_key_relu ffn_17_key_relu_square 0=4
Gemm gemm_ffn_17_value 1 1 ffn_17_key_relu_square ffn_17_value 4=0 5=1 6=0 7=0 8=1024 9=4096 10=-1
BinaryOp add_231 2 1 ffn_17_value ffn_17_x_last channel_mixing_17_out 0=0
Split split_channel_mixing_17_out 1 2 channel_mixing_17_out att_18_x_last att_18_x
LayerNorm layernorm_36 1 1 att_18_x att_18_xx 0=1024 1=0.00001 2=1
Split split_att_18_xx 1 3 att_18_xx att_18_xx_0 att_18_xx_1 state_54_out
BinaryOp sub_36 2 1 state_54_in att_18_xx_0 att_18_sx 0=1
MemoryData data_att_18_maa 0 1 att_18_maa 0=1024 1=6 21=0
BinaryOp mul_233 2 1 att_18_sx att_18_maa att_18_maa_x 0=2
BinaryOp add_232 2 1 att_18_xx_1 att_18_maa_x att_18_maa_rwkvag 0=0
Slice slice_att_18_maa_rwkvag 1 6 att_18_maa_rwkvag att_18_mr att_18_mw att_18_mk att_18_mv att_18_ma att_18_mg -23300=6,-233,-233,-233,-233,-233,-233 1=0
Split split_att_18_mv 1 2 att_18_mv att_18_mv_0 att_18_mv_1
Gemm gemm_att_18_key 1 1 att_18_mk att_18_key 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_18_value 1 1 att_18_mv_0 att_18_value 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_18_receptance 1 1 att_18_mr att_18_receptance 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_18_ma_lora_0 1 1 att_18_ma att_18_ma_lora_0 4=0 5=1 6=0 7=0 8=64 9=1024 10=-1
Gemm gemm_att_18_ma_lora_1 1 1 att_18_ma_lora_0 att_18_ma_lora_1 4=0 5=1 6=0 7=0 8=1024 9=64 10=-1
MemoryData data_att_18_a0 0 1 att_18_a0 0=1024 21=0
BinaryOp add_233 2 1 att_18_ma_lora_1 att_18_a0 att_18_ma_lora_2 0=0
Sigmoid sigmoid_71 1 1 att_18_ma_lora_2 att_18_a
Split split_att_18_a 1 2 att_18_a att_18_a_0 att_18_a_1
Gemm gemm_att_18_mg_lora 1 1 att_18_mg att_18_mg_lora 4=0 5=1 6=0 7=0 8=128 9=1024 10=-1
Sigmoid sigmoid_72 1 1 att_18_mg_lora att_18_mg_lora_sigmoid
Gemm gemm_att_18_gate 1 1 att_18_mg_lora_sigmoid att_18_gate 4=0 5=1 6=0 7=0 8=1024 9=128 10=-1
MemoryData data_att_18_k_k 0 1 att_18_k_k 0=1024 21=0
MemoryData data_att_18_k_a 0 1 att_18_k_a 0=1024 21=0
MemoryData data_att_18_r_k 0 1 att_18_r_k 0=1024 21=0
Split split_att_18_key 1 2 att_18_key att_18_key_0 att_18_key_1
BinaryOp mul_234 2 1 att_18_key_0 att_18_k_k att_18_key_k 0=2
Reshape reshape_216 1 1 att_18_key_k att_18_key_k_reshape 0=64 1=1 2=16
Normalize l2norm_18 1 1 att_18_key_k_reshape att_18_key_k_norm 0=1 1=1 2=0.0000001 3=1 4=0 9=1
Reshape reshape_217 1 1 att_18_key_k_norm att_18_key_k_norm_reshape 0=1024
BinaryOp add_234 1 1 att_18_a_0 att_18_a_0_minus 0=0 1=1 2=-1.0
BinaryOp mul_235 2 1 att_18_a_0_minus att_18_k_a att_18_a_minus1_mulka 0=2
BinaryOp add_235 1 1 att_18_a_minus1_mulka att_18_a_minus1_mulka_plus1 0=0 1=1 2=1.0
BinaryOp mul_236 2 1 att_18_a_minus1_mulka_plus1 att_18_key_1 att_18_key_a 0=2
Split split_att_18_value 1 2 att_18_value att_18_value_0 att_18_value_1
Split split_v_first_17 1 2 v_first_17 att_18_v_first v_first_18
BinaryOp minus_17 2 1 att_18_v_first att_18_value_1 att_18_vfirst_minus_value 0=1
Gemm gemm_att_18_mv_lora_0 1 1 att_18_mv_1 att_18_mv_lora_0 4=0 5=1 6=0 7=0 8=32 9=1024 10=-1
Gemm gemm_att_18_mv_lora_1 1 1 att_18_mv_lora_0 att_18_mv_lora_1 4=0 5=1 6=0 7=0 8=1024 9=32 10=-1
MemoryData data_att_18_v0 0 1 att_18_v0 0=1024 21=0
BinaryOp add_236 2 1 att_18_mv_lora_1 att_18_v0 att_18_mv_lora_2 0=0
Sigmoid sigmoid_73 1 1 att_18_mv_lora_2 att_18_mv_lora_sigmoid
BinaryOp mul_237 2 1 att_18_mv_lora_sigmoid att_18_vfirst_minus_value att_18_vfirst_minus_value_mul 0=2
BinaryOp add_237 2 1 att_18_value_0 att_18_vfirst_minus_value_mul att_18_value_final 0=0
Gemm gemm_att_18_mw_lora 1 1 att_18_mw att_18_mw_lora 4=0 5=1 6=0 7=0 8=64 9=1024 10=-1
TanH tanh_att_18_mw_lora 1 1 att_18_mw_lora att_18_mw_lora_tanh
Gemm gemm_att_18_mw_lora_tanh_linear 1 1 att_18_mw_lora_tanh att_18_mw_lora_tanh_linear 4=0 5=1 6=0 7=0 8=1024 9=64 10=-1
MemoryData data_att_18_td 0 1 att_18_td 0=1024 21=0
BinaryOp add_238 2 1 att_18_mw_lora_tanh_linear att_18_td att_18_time_decay_pre 0=0
Sigmoid sigmoid_74 1 1 att_18_time_decay_pre att_18_time_decay_sigmoid
Exp exp_18 1 1 att_18_time_decay_sigmoid att_18_time_decay 1=-0.606531
Reshape reshape_218 1 1 att_18_time_decay att_18_time_decay_reshape 0=64 1=1 2=16
Split split_att_18_key_a 1 2 att_18_key_a att_18_key_a_0 att_18_key_a_1
Reshape reshape_219 1 1 att_18_key_a_0 att_18_key_a_reshape 0=64 1=1 2=16
Split split_att_18_value_final 1 3 att_18_value_final att_18_value_final_0 att_18_value_final_1 att_18_value_final_2
Reshape reshape_220 1 1 att_18_value_final_0 att_18_value_final_0_reshape 0=64 1=16
Reshape reshape_221 1 1 att_18_value_final_1 att_18_value_final_1_reshape 0=1 1=64 2=16
MatMul matmul_att_18_vk 2 1 att_18_value_final_1_reshape att_18_key_a_reshape att_18_vk 0=0
Split split_att_18_key_k_norm_reshape 1 2 att_18_key_k_norm_reshape att_18_kk_0 att_18_kk_1
BinaryOp mul_238 1 1 att_18_kk_0 att_18_kk_0_neg 0=2 1=1 2=-1.0
Reshape reshape_222 1 1 att_18_kk_0_neg att_18_kk_0_neg_reshape 0=1 1=64 2=16
BinaryOp mul_239 2 1 att_18_kk_1 att_18_a_1 att_18_kk_a 0=2
Reshape reshape_223 1 1 att_18_kk_a att_18_kk_a_reshape 0=64 1=1 2=16
MatMul matmul_att_18_ab 2 1 att_18_kk_0_neg_reshape att_18_kk_a_reshape att_18_ab 0=0
Split split_state_55_in 1 2 state_55_in att_18_state_prev_0 att_18_state_prev_1
BinaryOp mul_240 2 1 att_18_state_prev_0 att_18_time_decay_reshape att_18_state_td 0=2
MatMul matmul_att_18_sab 2 1 att_18_state_prev_1 att_18_ab att_18_sab 0=0
BinaryOp add_239 2 1 att_18_sab att_18_vk att_18_sab_vk 0=0
BinaryOp add_240 2 1 att_18_sab_vk att_18_state_td att_18_state_new 0=0
Split split_att_18_state_new 1 2 att_18_state_new att_18_state_new_0 state_55_out
Split split_att_18_receptance 1 2 att_18_receptance att_18_receptance_0 att_18_receptance_1
Reshape reshape_224 1 1 att_18_receptance_0 att_18_receptance_reshape 0=1 1=64 2=16
MatMul matmul_att_18_wkv_out 2 1 att_18_state_new_0 att_18_receptance_reshape att_18_wkv_out 0=0
Reshape reshape_225 1 1 att_18_wkv_out att_18_wkv_out_flatten 0=1024
GroupNorm groupnorm_18 1 1 att_18_wkv_out_flatten att_18_x_gn 0=16 1=1024 2=0.00064 3=1
BinaryOp mul_241 2 1 att_18_receptance_1 att_18_key_a_1 att_18_receptance_key_a 0=2
BinaryOp mul_242 2 1 att_18_receptance_key_a att_18_r_k att_18_rk 0=2
Reshape reshape_226 1 1 att_18_rk att_18_rk_reshape 0=64 1=16
Reduction sum_18 1 1 att_18_rk_reshape att_18_rk_sum 1=0 -23303=1,-1 4=1 5=1
BinaryOp mul_243 2 1 att_18_value_final_0_reshape att_18_rk_sum att_18_value_final_0_rk 0=2
Reshape reshape_227 1 1 att_18_value_final_0_rk att_18_value_final_0_rk_reshape 0=1024
BinaryOp add_241 2 1 att_18_value_final_0_rk_reshape att_18_x_gn att_18_x_gn_rkv 0=0
BinaryOp mul_244 2 1 att_18_x_gn_rkv att_18_gate att_18_x_gate 0=2
Gemm gemm_att_18_x_out 1 1 att_18_x_gate att_18_x_out 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
BinaryOp add_242 2 1 att_18_x_out att_18_x_last time_mixing_18_out 0=0
Split split_time_mixing_18_out 1 2 time_mixing_18_out ffn_18_x_last ffn_18_x
LayerNorm layernorm_37 1 1 ffn_18_x ffn_18_xx 0=1024 1=0.00001 2=1
Split split_ffn_18_xx 1 3 ffn_18_xx ffn_18_xx_0 ffn_18_xx_1 state_56_out
BinaryOp sub_37 2 1 state_56_in ffn_18_xx_0 ffn_18_sx 0=1
Split split_ffn_18_sx 1 1 ffn_18_sx ffn_18_sx_0
MemoryData data_ffn_18_x_k 0 1 ffn_18_x_k 0=1024 21=0
BinaryOp mul_245 2 1 ffn_18_sx_0 ffn_18_x_k ffn_18_xk 0=2
BinaryOp add_243 2 1 ffn_18_xk ffn_18_xx_1 ffn_18_xxk 0=0
Gemm gemm_ffn_18_key 1 1 ffn_18_xxk ffn_18_key 4=0 5=1 6=0 7=0 8=4096 9=1024 10=-1
ReLU relu_18 1 1 ffn_18_key ffn_18_key_relu
UnaryOp square_ffn_18_key_relu 1 1 ffn_18_key_relu ffn_18_key_relu_square 0=4
Gemm gemm_ffn_18_value 1 1 ffn_18_key_relu_square ffn_18_value 4=0 5=1 6=0 7=0 8=1024 9=4096 10=-1
BinaryOp add_244 2 1 ffn_18_value ffn_18_x_last channel_mixing_18_out 0=0
Split split_channel_mixing_18_out 1 2 channel_mixing_18_out att_19_x_last att_19_x
LayerNorm layernorm_38 1 1 att_19_x att_19_xx 0=1024 1=0.00001 2=1
Split split_att_19_xx 1 3 att_19_xx att_19_xx_0 att_19_xx_1 state_57_out
BinaryOp sub_38 2 1 state_57_in att_19_xx_0 att_19_sx 0=1
MemoryData data_att_19_maa 0 1 att_19_maa 0=1024 1=6 21=0
BinaryOp mul_246 2 1 att_19_sx att_19_maa att_19_maa_x 0=2
BinaryOp add_245 2 1 att_19_xx_1 att_19_maa_x att_19_maa_rwkvag 0=0
Slice slice_att_19_maa_rwkvag 1 6 att_19_maa_rwkvag att_19_mr att_19_mw att_19_mk att_19_mv att_19_ma att_19_mg -23300=6,-233,-233,-233,-233,-233,-233 1=0
Split split_att_19_mv 1 2 att_19_mv att_19_mv_0 att_19_mv_1
Gemm gemm_att_19_key 1 1 att_19_mk att_19_key 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_19_value 1 1 att_19_mv_0 att_19_value 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_19_receptance 1 1 att_19_mr att_19_receptance 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_19_ma_lora_0 1 1 att_19_ma att_19_ma_lora_0 4=0 5=1 6=0 7=0 8=64 9=1024 10=-1
Gemm gemm_att_19_ma_lora_1 1 1 att_19_ma_lora_0 att_19_ma_lora_1 4=0 5=1 6=0 7=0 8=1024 9=64 10=-1
MemoryData data_att_19_a0 0 1 att_19_a0 0=1024 21=0
BinaryOp add_246 2 1 att_19_ma_lora_1 att_19_a0 att_19_ma_lora_2 0=0
Sigmoid sigmoid_75 1 1 att_19_ma_lora_2 att_19_a
Split split_att_19_a 1 2 att_19_a att_19_a_0 att_19_a_1
Gemm gemm_att_19_mg_lora 1 1 att_19_mg att_19_mg_lora 4=0 5=1 6=0 7=0 8=128 9=1024 10=-1
Sigmoid sigmoid_76 1 1 att_19_mg_lora att_19_mg_lora_sigmoid
Gemm gemm_att_19_gate 1 1 att_19_mg_lora_sigmoid att_19_gate 4=0 5=1 6=0 7=0 8=1024 9=128 10=-1
MemoryData data_att_19_k_k 0 1 att_19_k_k 0=1024 21=0
MemoryData data_att_19_k_a 0 1 att_19_k_a 0=1024 21=0
MemoryData data_att_19_r_k 0 1 att_19_r_k 0=1024 21=0
Split split_att_19_key 1 2 att_19_key att_19_key_0 att_19_key_1
BinaryOp mul_247 2 1 att_19_key_0 att_19_k_k att_19_key_k 0=2
Reshape reshape_228 1 1 att_19_key_k att_19_key_k_reshape 0=64 1=1 2=16
Normalize l2norm_19 1 1 att_19_key_k_reshape att_19_key_k_norm 0=1 1=1 2=0.0000001 3=1 4=0 9=1
Reshape reshape_229 1 1 att_19_key_k_norm att_19_key_k_norm_reshape 0=1024
BinaryOp add_247 1 1 att_19_a_0 att_19_a_0_minus 0=0 1=1 2=-1.0
BinaryOp mul_248 2 1 att_19_a_0_minus att_19_k_a att_19_a_minus1_mulka 0=2
BinaryOp add_248 1 1 att_19_a_minus1_mulka att_19_a_minus1_mulka_plus1 0=0 1=1 2=1.0
BinaryOp mul_249 2 1 att_19_a_minus1_mulka_plus1 att_19_key_1 att_19_key_a 0=2
Split split_att_19_value 1 2 att_19_value att_19_value_0 att_19_value_1
Split split_v_first_18 1 2 v_first_18 att_19_v_first v_first_19
BinaryOp minus_18 2 1 att_19_v_first att_19_value_1 att_19_vfirst_minus_value 0=1
Gemm gemm_att_19_mv_lora_0 1 1 att_19_mv_1 att_19_mv_lora_0 4=0 5=1 6=0 7=0 8=32 9=1024 10=-1
Gemm gemm_att_19_mv_lora_1 1 1 att_19_mv_lora_0 att_19_mv_lora_1 4=0 5=1 6=0 7=0 8=1024 9=32 10=-1
MemoryData data_att_19_v0 0 1 att_19_v0 0=1024 21=0
BinaryOp add_249 2 1 att_19_mv_lora_1 att_19_v0 att_19_mv_lora_2 0=0
Sigmoid sigmoid_77 1 1 att_19_mv_lora_2 att_19_mv_lora_sigmoid
BinaryOp mul_250 2 1 att_19_mv_lora_sigmoid att_19_vfirst_minus_value att_19_vfirst_minus_value_mul 0=2
BinaryOp add_250 2 1 att_19_value_0 att_19_vfirst_minus_value_mul att_19_value_final 0=0
Gemm gemm_att_19_mw_lora 1 1 att_19_mw att_19_mw_lora 4=0 5=1 6=0 7=0 8=64 9=1024 10=-1
TanH tanh_att_19_mw_lora 1 1 att_19_mw_lora att_19_mw_lora_tanh
Gemm gemm_att_19_mw_lora_tanh_linear 1 1 att_19_mw_lora_tanh att_19_mw_lora_tanh_linear 4=0 5=1 6=0 7=0 8=1024 9=64 10=-1
MemoryData data_att_19_td 0 1 att_19_td 0=1024 21=0
BinaryOp add_251 2 1 att_19_mw_lora_tanh_linear att_19_td att_19_time_decay_pre 0=0
Sigmoid sigmoid_78 1 1 att_19_time_decay_pre att_19_time_decay_sigmoid
Exp exp_19 1 1 att_19_time_decay_sigmoid att_19_time_decay 1=-0.606531
Reshape reshape_230 1 1 att_19_time_decay att_19_time_decay_reshape 0=64 1=1 2=16
Split split_att_19_key_a 1 2 att_19_key_a att_19_key_a_0 att_19_key_a_1
Reshape reshape_231 1 1 att_19_key_a_0 att_19_key_a_reshape 0=64 1=1 2=16
Split split_att_19_value_final 1 3 att_19_value_final att_19_value_final_0 att_19_value_final_1 att_19_value_final_2
Reshape reshape_232 1 1 att_19_value_final_0 att_19_value_final_0_reshape 0=64 1=16
Reshape reshape_233 1 1 att_19_value_final_1 att_19_value_final_1_reshape 0=1 1=64 2=16
MatMul matmul_att_19_vk 2 1 att_19_value_final_1_reshape att_19_key_a_reshape att_19_vk 0=0
Split split_att_19_key_k_norm_reshape 1 2 att_19_key_k_norm_reshape att_19_kk_0 att_19_kk_1
BinaryOp mul_251 1 1 att_19_kk_0 att_19_kk_0_neg 0=2 1=1 2=-1.0
Reshape reshape_234 1 1 att_19_kk_0_neg att_19_kk_0_neg_reshape 0=1 1=64 2=16
BinaryOp mul_252 2 1 att_19_kk_1 att_19_a_1 att_19_kk_a 0=2
Reshape reshape_235 1 1 att_19_kk_a att_19_kk_a_reshape 0=64 1=1 2=16
MatMul matmul_att_19_ab 2 1 att_19_kk_0_neg_reshape att_19_kk_a_reshape att_19_ab 0=0
Split split_state_58_in 1 2 state_58_in att_19_state_prev_0 att_19_state_prev_1
BinaryOp mul_253 2 1 att_19_state_prev_0 att_19_time_decay_reshape att_19_state_td 0=2
MatMul matmul_att_19_sab 2 1 att_19_state_prev_1 att_19_ab att_19_sab 0=0
BinaryOp add_252 2 1 att_19_sab att_19_vk att_19_sab_vk 0=0
BinaryOp add_253 2 1 att_19_sab_vk att_19_state_td att_19_state_new 0=0
Split split_att_19_state_new 1 2 att_19_state_new att_19_state_new_0 state_58_out
Split split_att_19_receptance 1 2 att_19_receptance att_19_receptance_0 att_19_receptance_1
Reshape reshape_236 1 1 att_19_receptance_0 att_19_receptance_reshape 0=1 1=64 2=16
MatMul matmul_att_19_wkv_out 2 1 att_19_state_new_0 att_19_receptance_reshape att_19_wkv_out 0=0
Reshape reshape_237 1 1 att_19_wkv_out att_19_wkv_out_flatten 0=1024
GroupNorm groupnorm_19 1 1 att_19_wkv_out_flatten att_19_x_gn 0=16 1=1024 2=0.00064 3=1
BinaryOp mul_254 2 1 att_19_receptance_1 att_19_key_a_1 att_19_receptance_key_a 0=2
BinaryOp mul_255 2 1 att_19_receptance_key_a att_19_r_k att_19_rk 0=2
Reshape reshape_238 1 1 att_19_rk att_19_rk_reshape 0=64 1=16
Reduction sum_19 1 1 att_19_rk_reshape att_19_rk_sum 1=0 -23303=1,-1 4=1 5=1
BinaryOp mul_256 2 1 att_19_value_final_0_reshape att_19_rk_sum att_19_value_final_0_rk 0=2
Reshape reshape_239 1 1 att_19_value_final_0_rk att_19_value_final_0_rk_reshape 0=1024
BinaryOp add_254 2 1 att_19_value_final_0_rk_reshape att_19_x_gn att_19_x_gn_rkv 0=0
BinaryOp mul_257 2 1 att_19_x_gn_rkv att_19_gate att_19_x_gate 0=2
Gemm gemm_att_19_x_out 1 1 att_19_x_gate att_19_x_out 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
BinaryOp add_255 2 1 att_19_x_out att_19_x_last time_mixing_19_out 0=0
Split split_time_mixing_19_out 1 2 time_mixing_19_out ffn_19_x_last ffn_19_x
LayerNorm layernorm_39 1 1 ffn_19_x ffn_19_xx 0=1024 1=0.00001 2=1
Split split_ffn_19_xx 1 3 ffn_19_xx ffn_19_xx_0 ffn_19_xx_1 state_59_out
BinaryOp sub_39 2 1 state_59_in ffn_19_xx_0 ffn_19_sx 0=1
Split split_ffn_19_sx 1 1 ffn_19_sx ffn_19_sx_0
MemoryData data_ffn_19_x_k 0 1 ffn_19_x_k 0=1024 21=0
BinaryOp mul_258 2 1 ffn_19_sx_0 ffn_19_x_k ffn_19_xk 0=2
BinaryOp add_256 2 1 ffn_19_xk ffn_19_xx_1 ffn_19_xxk 0=0
Gemm gemm_ffn_19_key 1 1 ffn_19_xxk ffn_19_key 4=0 5=1 6=0 7=0 8=4096 9=1024 10=-1
ReLU relu_19 1 1 ffn_19_key ffn_19_key_relu
UnaryOp square_ffn_19_key_relu 1 1 ffn_19_key_relu ffn_19_key_relu_square 0=4
Gemm gemm_ffn_19_value 1 1 ffn_19_key_relu_square ffn_19_value 4=0 5=1 6=0 7=0 8=1024 9=4096 10=-1
BinaryOp add_257 2 1 ffn_19_value ffn_19_x_last channel_mixing_19_out 0=0
Split split_channel_mixing_19_out 1 2 channel_mixing_19_out att_20_x_last att_20_x
LayerNorm layernorm_40 1 1 att_20_x att_20_xx 0=1024 1=0.00001 2=1
Split split_att_20_xx 1 3 att_20_xx att_20_xx_0 att_20_xx_1 state_60_out
BinaryOp sub_40 2 1 state_60_in att_20_xx_0 att_20_sx 0=1
MemoryData data_att_20_maa 0 1 att_20_maa 0=1024 1=6 21=0
BinaryOp mul_259 2 1 att_20_sx att_20_maa att_20_maa_x 0=2
BinaryOp add_258 2 1 att_20_xx_1 att_20_maa_x att_20_maa_rwkvag 0=0
Slice slice_att_20_maa_rwkvag 1 6 att_20_maa_rwkvag att_20_mr att_20_mw att_20_mk att_20_mv att_20_ma att_20_mg -23300=6,-233,-233,-233,-233,-233,-233 1=0
Split split_att_20_mv 1 2 att_20_mv att_20_mv_0 att_20_mv_1
Gemm gemm_att_20_key 1 1 att_20_mk att_20_key 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_20_value 1 1 att_20_mv_0 att_20_value 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_20_receptance 1 1 att_20_mr att_20_receptance 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_20_ma_lora_0 1 1 att_20_ma att_20_ma_lora_0 4=0 5=1 6=0 7=0 8=64 9=1024 10=-1
Gemm gemm_att_20_ma_lora_1 1 1 att_20_ma_lora_0 att_20_ma_lora_1 4=0 5=1 6=0 7=0 8=1024 9=64 10=-1
MemoryData data_att_20_a0 0 1 att_20_a0 0=1024 21=0
BinaryOp add_259 2 1 att_20_ma_lora_1 att_20_a0 att_20_ma_lora_2 0=0
Sigmoid sigmoid_79 1 1 att_20_ma_lora_2 att_20_a
Split split_att_20_a 1 2 att_20_a att_20_a_0 att_20_a_1
Gemm gemm_att_20_mg_lora 1 1 att_20_mg att_20_mg_lora 4=0 5=1 6=0 7=0 8=128 9=1024 10=-1
Sigmoid sigmoid_80 1 1 att_20_mg_lora att_20_mg_lora_sigmoid
Gemm gemm_att_20_gate 1 1 att_20_mg_lora_sigmoid att_20_gate 4=0 5=1 6=0 7=0 8=1024 9=128 10=-1
MemoryData data_att_20_k_k 0 1 att_20_k_k 0=1024 21=0
MemoryData data_att_20_k_a 0 1 att_20_k_a 0=1024 21=0
MemoryData data_att_20_r_k 0 1 att_20_r_k 0=1024 21=0
Split split_att_20_key 1 2 att_20_key att_20_key_0 att_20_key_1
BinaryOp mul_260 2 1 att_20_key_0 att_20_k_k att_20_key_k 0=2
Reshape reshape_240 1 1 att_20_key_k att_20_key_k_reshape 0=64 1=1 2=16
Normalize l2norm_20 1 1 att_20_key_k_reshape att_20_key_k_norm 0=1 1=1 2=0.0000001 3=1 4=0 9=1
Reshape reshape_241 1 1 att_20_key_k_norm att_20_key_k_norm_reshape 0=1024
BinaryOp add_260 1 1 att_20_a_0 att_20_a_0_minus 0=0 1=1 2=-1.0
BinaryOp mul_261 2 1 att_20_a_0_minus att_20_k_a att_20_a_minus1_mulka 0=2
BinaryOp add_261 1 1 att_20_a_minus1_mulka att_20_a_minus1_mulka_plus1 0=0 1=1 2=1.0
BinaryOp mul_262 2 1 att_20_a_minus1_mulka_plus1 att_20_key_1 att_20_key_a 0=2
Split split_att_20_value 1 2 att_20_value att_20_value_0 att_20_value_1
Split split_v_first_19 1 2 v_first_19 att_20_v_first v_first_20
BinaryOp minus_19 2 1 att_20_v_first att_20_value_1 att_20_vfirst_minus_value 0=1
Gemm gemm_att_20_mv_lora_0 1 1 att_20_mv_1 att_20_mv_lora_0 4=0 5=1 6=0 7=0 8=32 9=1024 10=-1
Gemm gemm_att_20_mv_lora_1 1 1 att_20_mv_lora_0 att_20_mv_lora_1 4=0 5=1 6=0 7=0 8=1024 9=32 10=-1
MemoryData data_att_20_v0 0 1 att_20_v0 0=1024 21=0
BinaryOp add_262 2 1 att_20_mv_lora_1 att_20_v0 att_20_mv_lora_2 0=0
Sigmoid sigmoid_81 1 1 att_20_mv_lora_2 att_20_mv_lora_sigmoid
BinaryOp mul_263 2 1 att_20_mv_lora_sigmoid att_20_vfirst_minus_value att_20_vfirst_minus_value_mul 0=2
BinaryOp add_263 2 1 att_20_value_0 att_20_vfirst_minus_value_mul att_20_value_final 0=0
Gemm gemm_att_20_mw_lora 1 1 att_20_mw att_20_mw_lora 4=0 5=1 6=0 7=0 8=64 9=1024 10=-1
TanH tanh_att_20_mw_lora 1 1 att_20_mw_lora att_20_mw_lora_tanh
Gemm gemm_att_20_mw_lora_tanh_linear 1 1 att_20_mw_lora_tanh att_20_mw_lora_tanh_linear 4=0 5=1 6=0 7=0 8=1024 9=64 10=-1
MemoryData data_att_20_td 0 1 att_20_td 0=1024 21=0
BinaryOp add_264 2 1 att_20_mw_lora_tanh_linear att_20_td att_20_time_decay_pre 0=0
Sigmoid sigmoid_82 1 1 att_20_time_decay_pre att_20_time_decay_sigmoid
Exp exp_20 1 1 att_20_time_decay_sigmoid att_20_time_decay 1=-0.606531
Reshape reshape_242 1 1 att_20_time_decay att_20_time_decay_reshape 0=64 1=1 2=16
Split split_att_20_key_a 1 2 att_20_key_a att_20_key_a_0 att_20_key_a_1
Reshape reshape_243 1 1 att_20_key_a_0 att_20_key_a_reshape 0=64 1=1 2=16
Split split_att_20_value_final 1 3 att_20_value_final att_20_value_final_0 att_20_value_final_1 att_20_value_final_2
Reshape reshape_244 1 1 att_20_value_final_0 att_20_value_final_0_reshape 0=64 1=16
Reshape reshape_245 1 1 att_20_value_final_1 att_20_value_final_1_reshape 0=1 1=64 2=16
MatMul matmul_att_20_vk 2 1 att_20_value_final_1_reshape att_20_key_a_reshape att_20_vk 0=0
Split split_att_20_key_k_norm_reshape 1 2 att_20_key_k_norm_reshape att_20_kk_0 att_20_kk_1
BinaryOp mul_264 1 1 att_20_kk_0 att_20_kk_0_neg 0=2 1=1 2=-1.0
Reshape reshape_246 1 1 att_20_kk_0_neg att_20_kk_0_neg_reshape 0=1 1=64 2=16
BinaryOp mul_265 2 1 att_20_kk_1 att_20_a_1 att_20_kk_a 0=2
Reshape reshape_247 1 1 att_20_kk_a att_20_kk_a_reshape 0=64 1=1 2=16
MatMul matmul_att_20_ab 2 1 att_20_kk_0_neg_reshape att_20_kk_a_reshape att_20_ab 0=0
Split split_state_61_in 1 2 state_61_in att_20_state_prev_0 att_20_state_prev_1
BinaryOp mul_266 2 1 att_20_state_prev_0 att_20_time_decay_reshape att_20_state_td 0=2
MatMul matmul_att_20_sab 2 1 att_20_state_prev_1 att_20_ab att_20_sab 0=0
BinaryOp add_265 2 1 att_20_sab att_20_vk att_20_sab_vk 0=0
BinaryOp add_266 2 1 att_20_sab_vk att_20_state_td att_20_state_new 0=0
Split split_att_20_state_new 1 2 att_20_state_new att_20_state_new_0 state_61_out
Split split_att_20_receptance 1 2 att_20_receptance att_20_receptance_0 att_20_receptance_1
Reshape reshape_248 1 1 att_20_receptance_0 att_20_receptance_reshape 0=1 1=64 2=16
MatMul matmul_att_20_wkv_out 2 1 att_20_state_new_0 att_20_receptance_reshape att_20_wkv_out 0=0
Reshape reshape_249 1 1 att_20_wkv_out att_20_wkv_out_flatten 0=1024
GroupNorm groupnorm_20 1 1 att_20_wkv_out_flatten att_20_x_gn 0=16 1=1024 2=0.00064 3=1
BinaryOp mul_267 2 1 att_20_receptance_1 att_20_key_a_1 att_20_receptance_key_a 0=2
BinaryOp mul_268 2 1 att_20_receptance_key_a att_20_r_k att_20_rk 0=2
Reshape reshape_250 1 1 att_20_rk att_20_rk_reshape 0=64 1=16
Reduction sum_20 1 1 att_20_rk_reshape att_20_rk_sum 1=0 -23303=1,-1 4=1 5=1
BinaryOp mul_269 2 1 att_20_value_final_0_reshape att_20_rk_sum att_20_value_final_0_rk 0=2
Reshape reshape_251 1 1 att_20_value_final_0_rk att_20_value_final_0_rk_reshape 0=1024
BinaryOp add_267 2 1 att_20_value_final_0_rk_reshape att_20_x_gn att_20_x_gn_rkv 0=0
BinaryOp mul_270 2 1 att_20_x_gn_rkv att_20_gate att_20_x_gate 0=2
Gemm gemm_att_20_x_out 1 1 att_20_x_gate att_20_x_out 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
BinaryOp add_268 2 1 att_20_x_out att_20_x_last time_mixing_20_out 0=0
Split split_time_mixing_20_out 1 2 time_mixing_20_out ffn_20_x_last ffn_20_x
LayerNorm layernorm_41 1 1 ffn_20_x ffn_20_xx 0=1024 1=0.00001 2=1
Split split_ffn_20_xx 1 3 ffn_20_xx ffn_20_xx_0 ffn_20_xx_1 state_62_out
BinaryOp sub_41 2 1 state_62_in ffn_20_xx_0 ffn_20_sx 0=1
Split split_ffn_20_sx 1 1 ffn_20_sx ffn_20_sx_0
MemoryData data_ffn_20_x_k 0 1 ffn_20_x_k 0=1024 21=0
BinaryOp mul_271 2 1 ffn_20_sx_0 ffn_20_x_k ffn_20_xk 0=2
BinaryOp add_269 2 1 ffn_20_xk ffn_20_xx_1 ffn_20_xxk 0=0
Gemm gemm_ffn_20_key 1 1 ffn_20_xxk ffn_20_key 4=0 5=1 6=0 7=0 8=4096 9=1024 10=-1
ReLU relu_20 1 1 ffn_20_key ffn_20_key_relu
UnaryOp square_ffn_20_key_relu 1 1 ffn_20_key_relu ffn_20_key_relu_square 0=4
Gemm gemm_ffn_20_value 1 1 ffn_20_key_relu_square ffn_20_value 4=0 5=1 6=0 7=0 8=1024 9=4096 10=-1
BinaryOp add_270 2 1 ffn_20_value ffn_20_x_last channel_mixing_20_out 0=0
Split split_channel_mixing_20_out 1 2 channel_mixing_20_out att_21_x_last att_21_x
LayerNorm layernorm_42 1 1 att_21_x att_21_xx 0=1024 1=0.00001 2=1
Split split_att_21_xx 1 3 att_21_xx att_21_xx_0 att_21_xx_1 state_63_out
BinaryOp sub_42 2 1 state_63_in att_21_xx_0 att_21_sx 0=1
MemoryData data_att_21_maa 0 1 att_21_maa 0=1024 1=6 21=0
BinaryOp mul_272 2 1 att_21_sx att_21_maa att_21_maa_x 0=2
BinaryOp add_271 2 1 att_21_xx_1 att_21_maa_x att_21_maa_rwkvag 0=0
Slice slice_att_21_maa_rwkvag 1 6 att_21_maa_rwkvag att_21_mr att_21_mw att_21_mk att_21_mv att_21_ma att_21_mg -23300=6,-233,-233,-233,-233,-233,-233 1=0
Split split_att_21_mv 1 2 att_21_mv att_21_mv_0 att_21_mv_1
Gemm gemm_att_21_key 1 1 att_21_mk att_21_key 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_21_value 1 1 att_21_mv_0 att_21_value 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_21_receptance 1 1 att_21_mr att_21_receptance 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_21_ma_lora_0 1 1 att_21_ma att_21_ma_lora_0 4=0 5=1 6=0 7=0 8=64 9=1024 10=-1
Gemm gemm_att_21_ma_lora_1 1 1 att_21_ma_lora_0 att_21_ma_lora_1 4=0 5=1 6=0 7=0 8=1024 9=64 10=-1
MemoryData data_att_21_a0 0 1 att_21_a0 0=1024 21=0
BinaryOp add_272 2 1 att_21_ma_lora_1 att_21_a0 att_21_ma_lora_2 0=0
Sigmoid sigmoid_83 1 1 att_21_ma_lora_2 att_21_a
Split split_att_21_a 1 2 att_21_a att_21_a_0 att_21_a_1
Gemm gemm_att_21_mg_lora 1 1 att_21_mg att_21_mg_lora 4=0 5=1 6=0 7=0 8=128 9=1024 10=-1
Sigmoid sigmoid_84 1 1 att_21_mg_lora att_21_mg_lora_sigmoid
Gemm gemm_att_21_gate 1 1 att_21_mg_lora_sigmoid att_21_gate 4=0 5=1 6=0 7=0 8=1024 9=128 10=-1
MemoryData data_att_21_k_k 0 1 att_21_k_k 0=1024 21=0
MemoryData data_att_21_k_a 0 1 att_21_k_a 0=1024 21=0
MemoryData data_att_21_r_k 0 1 att_21_r_k 0=1024 21=0
Split split_att_21_key 1 2 att_21_key att_21_key_0 att_21_key_1
BinaryOp mul_273 2 1 att_21_key_0 att_21_k_k att_21_key_k 0=2
Reshape reshape_252 1 1 att_21_key_k att_21_key_k_reshape 0=64 1=1 2=16
Normalize l2norm_21 1 1 att_21_key_k_reshape att_21_key_k_norm 0=1 1=1 2=0.0000001 3=1 4=0 9=1
Reshape reshape_253 1 1 att_21_key_k_norm att_21_key_k_norm_reshape 0=1024
BinaryOp add_273 1 1 att_21_a_0 att_21_a_0_minus 0=0 1=1 2=-1.0
BinaryOp mul_274 2 1 att_21_a_0_minus att_21_k_a att_21_a_minus1_mulka 0=2
BinaryOp add_274 1 1 att_21_a_minus1_mulka att_21_a_minus1_mulka_plus1 0=0 1=1 2=1.0
BinaryOp mul_275 2 1 att_21_a_minus1_mulka_plus1 att_21_key_1 att_21_key_a 0=2
Split split_att_21_value 1 2 att_21_value att_21_value_0 att_21_value_1
Split split_v_first_20 1 2 v_first_20 att_21_v_first v_first_21
BinaryOp minus_20 2 1 att_21_v_first att_21_value_1 att_21_vfirst_minus_value 0=1
Gemm gemm_att_21_mv_lora_0 1 1 att_21_mv_1 att_21_mv_lora_0 4=0 5=1 6=0 7=0 8=32 9=1024 10=-1
Gemm gemm_att_21_mv_lora_1 1 1 att_21_mv_lora_0 att_21_mv_lora_1 4=0 5=1 6=0 7=0 8=1024 9=32 10=-1
MemoryData data_att_21_v0 0 1 att_21_v0 0=1024 21=0
BinaryOp add_275 2 1 att_21_mv_lora_1 att_21_v0 att_21_mv_lora_2 0=0
Sigmoid sigmoid_85 1 1 att_21_mv_lora_2 att_21_mv_lora_sigmoid
BinaryOp mul_276 2 1 att_21_mv_lora_sigmoid att_21_vfirst_minus_value att_21_vfirst_minus_value_mul 0=2
BinaryOp add_276 2 1 att_21_value_0 att_21_vfirst_minus_value_mul att_21_value_final 0=0
Gemm gemm_att_21_mw_lora 1 1 att_21_mw att_21_mw_lora 4=0 5=1 6=0 7=0 8=64 9=1024 10=-1
TanH tanh_att_21_mw_lora 1 1 att_21_mw_lora att_21_mw_lora_tanh
Gemm gemm_att_21_mw_lora_tanh_linear 1 1 att_21_mw_lora_tanh att_21_mw_lora_tanh_linear 4=0 5=1 6=0 7=0 8=1024 9=64 10=-1
MemoryData data_att_21_td 0 1 att_21_td 0=1024 21=0
BinaryOp add_277 2 1 att_21_mw_lora_tanh_linear att_21_td att_21_time_decay_pre 0=0
Sigmoid sigmoid_86 1 1 att_21_time_decay_pre att_21_time_decay_sigmoid
Exp exp_21 1 1 att_21_time_decay_sigmoid att_21_time_decay 1=-0.606531
Reshape reshape_254 1 1 att_21_time_decay att_21_time_decay_reshape 0=64 1=1 2=16
Split split_att_21_key_a 1 2 att_21_key_a att_21_key_a_0 att_21_key_a_1
Reshape reshape_255 1 1 att_21_key_a_0 att_21_key_a_reshape 0=64 1=1 2=16
Split split_att_21_value_final 1 3 att_21_value_final att_21_value_final_0 att_21_value_final_1 att_21_value_final_2
Reshape reshape_256 1 1 att_21_value_final_0 att_21_value_final_0_reshape 0=64 1=16
Reshape reshape_257 1 1 att_21_value_final_1 att_21_value_final_1_reshape 0=1 1=64 2=16
MatMul matmul_att_21_vk 2 1 att_21_value_final_1_reshape att_21_key_a_reshape att_21_vk 0=0
Split split_att_21_key_k_norm_reshape 1 2 att_21_key_k_norm_reshape att_21_kk_0 att_21_kk_1
BinaryOp mul_277 1 1 att_21_kk_0 att_21_kk_0_neg 0=2 1=1 2=-1.0
Reshape reshape_258 1 1 att_21_kk_0_neg att_21_kk_0_neg_reshape 0=1 1=64 2=16
BinaryOp mul_278 2 1 att_21_kk_1 att_21_a_1 att_21_kk_a 0=2
Reshape reshape_259 1 1 att_21_kk_a att_21_kk_a_reshape 0=64 1=1 2=16
MatMul matmul_att_21_ab 2 1 att_21_kk_0_neg_reshape att_21_kk_a_reshape att_21_ab 0=0
Split split_state_64_in 1 2 state_64_in att_21_state_prev_0 att_21_state_prev_1
BinaryOp mul_279 2 1 att_21_state_prev_0 att_21_time_decay_reshape att_21_state_td 0=2
MatMul matmul_att_21_sab 2 1 att_21_state_prev_1 att_21_ab att_21_sab 0=0
BinaryOp add_278 2 1 att_21_sab att_21_vk att_21_sab_vk 0=0
BinaryOp add_279 2 1 att_21_sab_vk att_21_state_td att_21_state_new 0=0
Split split_att_21_state_new 1 2 att_21_state_new att_21_state_new_0 state_64_out
Split split_att_21_receptance 1 2 att_21_receptance att_21_receptance_0 att_21_receptance_1
Reshape reshape_260 1 1 att_21_receptance_0 att_21_receptance_reshape 0=1 1=64 2=16
MatMul matmul_att_21_wkv_out 2 1 att_21_state_new_0 att_21_receptance_reshape att_21_wkv_out 0=0
Reshape reshape_261 1 1 att_21_wkv_out att_21_wkv_out_flatten 0=1024
GroupNorm groupnorm_21 1 1 att_21_wkv_out_flatten att_21_x_gn 0=16 1=1024 2=0.00064 3=1
BinaryOp mul_280 2 1 att_21_receptance_1 att_21_key_a_1 att_21_receptance_key_a 0=2
BinaryOp mul_281 2 1 att_21_receptance_key_a att_21_r_k att_21_rk 0=2
Reshape reshape_262 1 1 att_21_rk att_21_rk_reshape 0=64 1=16
Reduction sum_21 1 1 att_21_rk_reshape att_21_rk_sum 1=0 -23303=1,-1 4=1 5=1
BinaryOp mul_282 2 1 att_21_value_final_0_reshape att_21_rk_sum att_21_value_final_0_rk 0=2
Reshape reshape_263 1 1 att_21_value_final_0_rk att_21_value_final_0_rk_reshape 0=1024
BinaryOp add_280 2 1 att_21_value_final_0_rk_reshape att_21_x_gn att_21_x_gn_rkv 0=0
BinaryOp mul_283 2 1 att_21_x_gn_rkv att_21_gate att_21_x_gate 0=2
Gemm gemm_att_21_x_out 1 1 att_21_x_gate att_21_x_out 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
BinaryOp add_281 2 1 att_21_x_out att_21_x_last time_mixing_21_out 0=0
Split split_time_mixing_21_out 1 2 time_mixing_21_out ffn_21_x_last ffn_21_x
LayerNorm layernorm_43 1 1 ffn_21_x ffn_21_xx 0=1024 1=0.00001 2=1
Split split_ffn_21_xx 1 3 ffn_21_xx ffn_21_xx_0 ffn_21_xx_1 state_65_out
BinaryOp sub_43 2 1 state_65_in ffn_21_xx_0 ffn_21_sx 0=1
Split split_ffn_21_sx 1 1 ffn_21_sx ffn_21_sx_0
MemoryData data_ffn_21_x_k 0 1 ffn_21_x_k 0=1024 21=0
BinaryOp mul_284 2 1 ffn_21_sx_0 ffn_21_x_k ffn_21_xk 0=2
BinaryOp add_282 2 1 ffn_21_xk ffn_21_xx_1 ffn_21_xxk 0=0
Gemm gemm_ffn_21_key 1 1 ffn_21_xxk ffn_21_key 4=0 5=1 6=0 7=0 8=4096 9=1024 10=-1
ReLU relu_21 1 1 ffn_21_key ffn_21_key_relu
UnaryOp square_ffn_21_key_relu 1 1 ffn_21_key_relu ffn_21_key_relu_square 0=4
Gemm gemm_ffn_21_value 1 1 ffn_21_key_relu_square ffn_21_value 4=0 5=1 6=0 7=0 8=1024 9=4096 10=-1
BinaryOp add_283 2 1 ffn_21_value ffn_21_x_last channel_mixing_21_out 0=0
Split split_channel_mixing_21_out 1 2 channel_mixing_21_out att_22_x_last att_22_x
LayerNorm layernorm_44 1 1 att_22_x att_22_xx 0=1024 1=0.00001 2=1
Split split_att_22_xx 1 3 att_22_xx att_22_xx_0 att_22_xx_1 state_66_out
BinaryOp sub_44 2 1 state_66_in att_22_xx_0 att_22_sx 0=1
MemoryData data_att_22_maa 0 1 att_22_maa 0=1024 1=6 21=0
BinaryOp mul_285 2 1 att_22_sx att_22_maa att_22_maa_x 0=2
BinaryOp add_284 2 1 att_22_xx_1 att_22_maa_x att_22_maa_rwkvag 0=0
Slice slice_att_22_maa_rwkvag 1 6 att_22_maa_rwkvag att_22_mr att_22_mw att_22_mk att_22_mv att_22_ma att_22_mg -23300=6,-233,-233,-233,-233,-233,-233 1=0
Split split_att_22_mv 1 2 att_22_mv att_22_mv_0 att_22_mv_1
Gemm gemm_att_22_key 1 1 att_22_mk att_22_key 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_22_value 1 1 att_22_mv_0 att_22_value 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_22_receptance 1 1 att_22_mr att_22_receptance 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_22_ma_lora_0 1 1 att_22_ma att_22_ma_lora_0 4=0 5=1 6=0 7=0 8=64 9=1024 10=-1
Gemm gemm_att_22_ma_lora_1 1 1 att_22_ma_lora_0 att_22_ma_lora_1 4=0 5=1 6=0 7=0 8=1024 9=64 10=-1
MemoryData data_att_22_a0 0 1 att_22_a0 0=1024 21=0
BinaryOp add_285 2 1 att_22_ma_lora_1 att_22_a0 att_22_ma_lora_2 0=0
Sigmoid sigmoid_87 1 1 att_22_ma_lora_2 att_22_a
Split split_att_22_a 1 2 att_22_a att_22_a_0 att_22_a_1
Gemm gemm_att_22_mg_lora 1 1 att_22_mg att_22_mg_lora 4=0 5=1 6=0 7=0 8=128 9=1024 10=-1
Sigmoid sigmoid_88 1 1 att_22_mg_lora att_22_mg_lora_sigmoid
Gemm gemm_att_22_gate 1 1 att_22_mg_lora_sigmoid att_22_gate 4=0 5=1 6=0 7=0 8=1024 9=128 10=-1
MemoryData data_att_22_k_k 0 1 att_22_k_k 0=1024 21=0
MemoryData data_att_22_k_a 0 1 att_22_k_a 0=1024 21=0
MemoryData data_att_22_r_k 0 1 att_22_r_k 0=1024 21=0
Split split_att_22_key 1 2 att_22_key att_22_key_0 att_22_key_1
BinaryOp mul_286 2 1 att_22_key_0 att_22_k_k att_22_key_k 0=2
Reshape reshape_264 1 1 att_22_key_k att_22_key_k_reshape 0=64 1=1 2=16
Normalize l2norm_22 1 1 att_22_key_k_reshape att_22_key_k_norm 0=1 1=1 2=0.0000001 3=1 4=0 9=1
Reshape reshape_265 1 1 att_22_key_k_norm att_22_key_k_norm_reshape 0=1024
BinaryOp add_286 1 1 att_22_a_0 att_22_a_0_minus 0=0 1=1 2=-1.0
BinaryOp mul_287 2 1 att_22_a_0_minus att_22_k_a att_22_a_minus1_mulka 0=2
BinaryOp add_287 1 1 att_22_a_minus1_mulka att_22_a_minus1_mulka_plus1 0=0 1=1 2=1.0
BinaryOp mul_288 2 1 att_22_a_minus1_mulka_plus1 att_22_key_1 att_22_key_a 0=2
Split split_att_22_value 1 2 att_22_value att_22_value_0 att_22_value_1
Split split_v_first_21 1 2 v_first_21 att_22_v_first v_first_22
BinaryOp minus_21 2 1 att_22_v_first att_22_value_1 att_22_vfirst_minus_value 0=1
Gemm gemm_att_22_mv_lora_0 1 1 att_22_mv_1 att_22_mv_lora_0 4=0 5=1 6=0 7=0 8=32 9=1024 10=-1
Gemm gemm_att_22_mv_lora_1 1 1 att_22_mv_lora_0 att_22_mv_lora_1 4=0 5=1 6=0 7=0 8=1024 9=32 10=-1
MemoryData data_att_22_v0 0 1 att_22_v0 0=1024 21=0
BinaryOp add_288 2 1 att_22_mv_lora_1 att_22_v0 att_22_mv_lora_2 0=0
Sigmoid sigmoid_89 1 1 att_22_mv_lora_2 att_22_mv_lora_sigmoid
BinaryOp mul_289 2 1 att_22_mv_lora_sigmoid att_22_vfirst_minus_value att_22_vfirst_minus_value_mul 0=2
BinaryOp add_289 2 1 att_22_value_0 att_22_vfirst_minus_value_mul att_22_value_final 0=0
Gemm gemm_att_22_mw_lora 1 1 att_22_mw att_22_mw_lora 4=0 5=1 6=0 7=0 8=64 9=1024 10=-1
TanH tanh_att_22_mw_lora 1 1 att_22_mw_lora att_22_mw_lora_tanh
Gemm gemm_att_22_mw_lora_tanh_linear 1 1 att_22_mw_lora_tanh att_22_mw_lora_tanh_linear 4=0 5=1 6=0 7=0 8=1024 9=64 10=-1
MemoryData data_att_22_td 0 1 att_22_td 0=1024 21=0
BinaryOp add_290 2 1 att_22_mw_lora_tanh_linear att_22_td att_22_time_decay_pre 0=0
Sigmoid sigmoid_90 1 1 att_22_time_decay_pre att_22_time_decay_sigmoid
Exp exp_22 1 1 att_22_time_decay_sigmoid att_22_time_decay 1=-0.606531
Reshape reshape_266 1 1 att_22_time_decay att_22_time_decay_reshape 0=64 1=1 2=16
Split split_att_22_key_a 1 2 att_22_key_a att_22_key_a_0 att_22_key_a_1
Reshape reshape_267 1 1 att_22_key_a_0 att_22_key_a_reshape 0=64 1=1 2=16
Split split_att_22_value_final 1 3 att_22_value_final att_22_value_final_0 att_22_value_final_1 att_22_value_final_2
Reshape reshape_268 1 1 att_22_value_final_0 att_22_value_final_0_reshape 0=64 1=16
Reshape reshape_269 1 1 att_22_value_final_1 att_22_value_final_1_reshape 0=1 1=64 2=16
MatMul matmul_att_22_vk 2 1 att_22_value_final_1_reshape att_22_key_a_reshape att_22_vk 0=0
Split split_att_22_key_k_norm_reshape 1 2 att_22_key_k_norm_reshape att_22_kk_0 att_22_kk_1
BinaryOp mul_290 1 1 att_22_kk_0 att_22_kk_0_neg 0=2 1=1 2=-1.0
Reshape reshape_270 1 1 att_22_kk_0_neg att_22_kk_0_neg_reshape 0=1 1=64 2=16
BinaryOp mul_291 2 1 att_22_kk_1 att_22_a_1 att_22_kk_a 0=2
Reshape reshape_271 1 1 att_22_kk_a att_22_kk_a_reshape 0=64 1=1 2=16
MatMul matmul_att_22_ab 2 1 att_22_kk_0_neg_reshape att_22_kk_a_reshape att_22_ab 0=0
Split split_state_67_in 1 2 state_67_in att_22_state_prev_0 att_22_state_prev_1
BinaryOp mul_292 2 1 att_22_state_prev_0 att_22_time_decay_reshape att_22_state_td 0=2
MatMul matmul_att_22_sab 2 1 att_22_state_prev_1 att_22_ab att_22_sab 0=0
BinaryOp add_291 2 1 att_22_sab att_22_vk att_22_sab_vk 0=0
BinaryOp add_292 2 1 att_22_sab_vk att_22_state_td att_22_state_new 0=0
Split split_att_22_state_new 1 2 att_22_state_new att_22_state_new_0 state_67_out
Split split_att_22_receptance 1 2 att_22_receptance att_22_receptance_0 att_22_receptance_1
Reshape reshape_272 1 1 att_22_receptance_0 att_22_receptance_reshape 0=1 1=64 2=16
MatMul matmul_att_22_wkv_out 2 1 att_22_state_new_0 att_22_receptance_reshape att_22_wkv_out 0=0
Reshape reshape_273 1 1 att_22_wkv_out att_22_wkv_out_flatten 0=1024
GroupNorm groupnorm_22 1 1 att_22_wkv_out_flatten att_22_x_gn 0=16 1=1024 2=0.00064 3=1
BinaryOp mul_293 2 1 att_22_receptance_1 att_22_key_a_1 att_22_receptance_key_a 0=2
BinaryOp mul_294 2 1 att_22_receptance_key_a att_22_r_k att_22_rk 0=2
Reshape reshape_274 1 1 att_22_rk att_22_rk_reshape 0=64 1=16
Reduction sum_22 1 1 att_22_rk_reshape att_22_rk_sum 1=0 -23303=1,-1 4=1 5=1
BinaryOp mul_295 2 1 att_22_value_final_0_reshape att_22_rk_sum att_22_value_final_0_rk 0=2
Reshape reshape_275 1 1 att_22_value_final_0_rk att_22_value_final_0_rk_reshape 0=1024
BinaryOp add_293 2 1 att_22_value_final_0_rk_reshape att_22_x_gn att_22_x_gn_rkv 0=0
BinaryOp mul_296 2 1 att_22_x_gn_rkv att_22_gate att_22_x_gate 0=2
Gemm gemm_att_22_x_out 1 1 att_22_x_gate att_22_x_out 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
BinaryOp add_294 2 1 att_22_x_out att_22_x_last time_mixing_22_out 0=0
Split split_time_mixing_22_out 1 2 time_mixing_22_out ffn_22_x_last ffn_22_x
LayerNorm layernorm_45 1 1 ffn_22_x ffn_22_xx 0=1024 1=0.00001 2=1
Split split_ffn_22_xx 1 3 ffn_22_xx ffn_22_xx_0 ffn_22_xx_1 state_68_out
BinaryOp sub_45 2 1 state_68_in ffn_22_xx_0 ffn_22_sx 0=1
Split split_ffn_22_sx 1 1 ffn_22_sx ffn_22_sx_0
MemoryData data_ffn_22_x_k 0 1 ffn_22_x_k 0=1024 21=0
BinaryOp mul_297 2 1 ffn_22_sx_0 ffn_22_x_k ffn_22_xk 0=2
BinaryOp add_295 2 1 ffn_22_xk ffn_22_xx_1 ffn_22_xxk 0=0
Gemm gemm_ffn_22_key 1 1 ffn_22_xxk ffn_22_key 4=0 5=1 6=0 7=0 8=4096 9=1024 10=-1
ReLU relu_22 1 1 ffn_22_key ffn_22_key_relu
UnaryOp square_ffn_22_key_relu 1 1 ffn_22_key_relu ffn_22_key_relu_square 0=4
Gemm gemm_ffn_22_value 1 1 ffn_22_key_relu_square ffn_22_value 4=0 5=1 6=0 7=0 8=1024 9=4096 10=-1
BinaryOp add_296 2 1 ffn_22_value ffn_22_x_last channel_mixing_22_out 0=0
Split split_channel_mixing_22_out 1 2 channel_mixing_22_out att_23_x_last att_23_x
LayerNorm layernorm_46 1 1 att_23_x att_23_xx 0=1024 1=0.00001 2=1
Split split_att_23_xx 1 3 att_23_xx att_23_xx_0 att_23_xx_1 state_69_out
BinaryOp sub_46 2 1 state_69_in att_23_xx_0 att_23_sx 0=1
MemoryData data_att_23_maa 0 1 att_23_maa 0=1024 1=6 21=0
BinaryOp mul_298 2 1 att_23_sx att_23_maa att_23_maa_x 0=2
BinaryOp add_297 2 1 att_23_xx_1 att_23_maa_x att_23_maa_rwkvag 0=0
Slice slice_att_23_maa_rwkvag 1 6 att_23_maa_rwkvag att_23_mr att_23_mw att_23_mk att_23_mv att_23_ma att_23_mg -23300=6,-233,-233,-233,-233,-233,-233 1=0
Split split_att_23_mv 1 2 att_23_mv att_23_mv_0 att_23_mv_1
Gemm gemm_att_23_key 1 1 att_23_mk att_23_key 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_23_value 1 1 att_23_mv_0 att_23_value 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_23_receptance 1 1 att_23_mr att_23_receptance 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
Gemm gemm_att_23_ma_lora_0 1 1 att_23_ma att_23_ma_lora_0 4=0 5=1 6=0 7=0 8=64 9=1024 10=-1
Gemm gemm_att_23_ma_lora_1 1 1 att_23_ma_lora_0 att_23_ma_lora_1 4=0 5=1 6=0 7=0 8=1024 9=64 10=-1
MemoryData data_att_23_a0 0 1 att_23_a0 0=1024 21=0
BinaryOp add_298 2 1 att_23_ma_lora_1 att_23_a0 att_23_ma_lora_2 0=0
Sigmoid sigmoid_91 1 1 att_23_ma_lora_2 att_23_a
Split split_att_23_a 1 2 att_23_a att_23_a_0 att_23_a_1
Gemm gemm_att_23_mg_lora 1 1 att_23_mg att_23_mg_lora 4=0 5=1 6=0 7=0 8=128 9=1024 10=-1
Sigmoid sigmoid_92 1 1 att_23_mg_lora att_23_mg_lora_sigmoid
Gemm gemm_att_23_gate 1 1 att_23_mg_lora_sigmoid att_23_gate 4=0 5=1 6=0 7=0 8=1024 9=128 10=-1
MemoryData data_att_23_k_k 0 1 att_23_k_k 0=1024 21=0
MemoryData data_att_23_k_a 0 1 att_23_k_a 0=1024 21=0
MemoryData data_att_23_r_k 0 1 att_23_r_k 0=1024 21=0
Split split_att_23_key 1 2 att_23_key att_23_key_0 att_23_key_1
BinaryOp mul_299 2 1 att_23_key_0 att_23_k_k att_23_key_k 0=2
Reshape reshape_276 1 1 att_23_key_k att_23_key_k_reshape 0=64 1=1 2=16
Normalize l2norm_23 1 1 att_23_key_k_reshape att_23_key_k_norm 0=1 1=1 2=0.0000001 3=1 4=0 9=1
Reshape reshape_277 1 1 att_23_key_k_norm att_23_key_k_norm_reshape 0=1024
BinaryOp add_299 1 1 att_23_a_0 att_23_a_0_minus 0=0 1=1 2=-1.0
BinaryOp mul_300 2 1 att_23_a_0_minus att_23_k_a att_23_a_minus1_mulka 0=2
BinaryOp add_300 1 1 att_23_a_minus1_mulka att_23_a_minus1_mulka_plus1 0=0 1=1 2=1.0
BinaryOp mul_301 2 1 att_23_a_minus1_mulka_plus1 att_23_key_1 att_23_key_a 0=2
Split split_att_23_value 1 2 att_23_value att_23_value_0 att_23_value_1
Split split_v_first_22 1 2 v_first_22 att_23_v_first v_first_23
BinaryOp minus_22 2 1 att_23_v_first att_23_value_1 att_23_vfirst_minus_value 0=1
Gemm gemm_att_23_mv_lora_0 1 1 att_23_mv_1 att_23_mv_lora_0 4=0 5=1 6=0 7=0 8=32 9=1024 10=-1
Gemm gemm_att_23_mv_lora_1 1 1 att_23_mv_lora_0 att_23_mv_lora_1 4=0 5=1 6=0 7=0 8=1024 9=32 10=-1
MemoryData data_att_23_v0 0 1 att_23_v0 0=1024 21=0
BinaryOp add_301 2 1 att_23_mv_lora_1 att_23_v0 att_23_mv_lora_2 0=0
Sigmoid sigmoid_93 1 1 att_23_mv_lora_2 att_23_mv_lora_sigmoid
BinaryOp mul_302 2 1 att_23_mv_lora_sigmoid att_23_vfirst_minus_value att_23_vfirst_minus_value_mul 0=2
BinaryOp add_302 2 1 att_23_value_0 att_23_vfirst_minus_value_mul att_23_value_final 0=0
Gemm gemm_att_23_mw_lora 1 1 att_23_mw att_23_mw_lora 4=0 5=1 6=0 7=0 8=64 9=1024 10=-1
TanH tanh_att_23_mw_lora 1 1 att_23_mw_lora att_23_mw_lora_tanh
Gemm gemm_att_23_mw_lora_tanh_linear 1 1 att_23_mw_lora_tanh att_23_mw_lora_tanh_linear 4=0 5=1 6=0 7=0 8=1024 9=64 10=-1
MemoryData data_att_23_td 0 1 att_23_td 0=1024 21=0
BinaryOp add_303 2 1 att_23_mw_lora_tanh_linear att_23_td att_23_time_decay_pre 0=0
Sigmoid sigmoid_94 1 1 att_23_time_decay_pre att_23_time_decay_sigmoid
Exp exp_23 1 1 att_23_time_decay_sigmoid att_23_time_decay 1=-0.606531
Reshape reshape_278 1 1 att_23_time_decay att_23_time_decay_reshape 0=64 1=1 2=16
Split split_att_23_key_a 1 2 att_23_key_a att_23_key_a_0 att_23_key_a_1
Reshape reshape_279 1 1 att_23_key_a_0 att_23_key_a_reshape 0=64 1=1 2=16
Split split_att_23_value_final 1 3 att_23_value_final att_23_value_final_0 att_23_value_final_1 att_23_value_final_2
Reshape reshape_280 1 1 att_23_value_final_0 att_23_value_final_0_reshape 0=64 1=16
Reshape reshape_281 1 1 att_23_value_final_1 att_23_value_final_1_reshape 0=1 1=64 2=16
MatMul matmul_att_23_vk 2 1 att_23_value_final_1_reshape att_23_key_a_reshape att_23_vk 0=0
Split split_att_23_key_k_norm_reshape 1 2 att_23_key_k_norm_reshape att_23_kk_0 att_23_kk_1
BinaryOp mul_303 1 1 att_23_kk_0 att_23_kk_0_neg 0=2 1=1 2=-1.0
Reshape reshape_282 1 1 att_23_kk_0_neg att_23_kk_0_neg_reshape 0=1 1=64 2=16
BinaryOp mul_304 2 1 att_23_kk_1 att_23_a_1 att_23_kk_a 0=2
Reshape reshape_283 1 1 att_23_kk_a att_23_kk_a_reshape 0=64 1=1 2=16
MatMul matmul_att_23_ab 2 1 att_23_kk_0_neg_reshape att_23_kk_a_reshape att_23_ab 0=0
Split split_state_70_in 1 2 state_70_in att_23_state_prev_0 att_23_state_prev_1
BinaryOp mul_305 2 1 att_23_state_prev_0 att_23_time_decay_reshape att_23_state_td 0=2
MatMul matmul_att_23_sab 2 1 att_23_state_prev_1 att_23_ab att_23_sab 0=0
BinaryOp add_304 2 1 att_23_sab att_23_vk att_23_sab_vk 0=0
BinaryOp add_305 2 1 att_23_sab_vk att_23_state_td att_23_state_new 0=0
Split split_att_23_state_new 1 2 att_23_state_new att_23_state_new_0 state_70_out
Split split_att_23_receptance 1 2 att_23_receptance att_23_receptance_0 att_23_receptance_1
Reshape reshape_284 1 1 att_23_receptance_0 att_23_receptance_reshape 0=1 1=64 2=16
MatMul matmul_att_23_wkv_out 2 1 att_23_state_new_0 att_23_receptance_reshape att_23_wkv_out 0=0
Reshape reshape_285 1 1 att_23_wkv_out att_23_wkv_out_flatten 0=1024
GroupNorm groupnorm_23 1 1 att_23_wkv_out_flatten att_23_x_gn 0=16 1=1024 2=0.00064 3=1
BinaryOp mul_306 2 1 att_23_receptance_1 att_23_key_a_1 att_23_receptance_key_a 0=2
BinaryOp mul_307 2 1 att_23_receptance_key_a att_23_r_k att_23_rk 0=2
Reshape reshape_286 1 1 att_23_rk att_23_rk_reshape 0=64 1=16
Reduction sum_23 1 1 att_23_rk_reshape att_23_rk_sum 1=0 -23303=1,-1 4=1 5=1
BinaryOp mul_308 2 1 att_23_value_final_0_reshape att_23_rk_sum att_23_value_final_0_rk 0=2
Reshape reshape_287 1 1 att_23_value_final_0_rk att_23_value_final_0_rk_reshape 0=1024
BinaryOp add_306 2 1 att_23_value_final_0_rk_reshape att_23_x_gn att_23_x_gn_rkv 0=0
BinaryOp mul_309 2 1 att_23_x_gn_rkv att_23_gate att_23_x_gate 0=2
Gemm gemm_att_23_x_out 1 1 att_23_x_gate att_23_x_out 4=0 5=1 6=0 7=0 8=1024 9=1024 10=-1
BinaryOp add_307 2 1 att_23_x_out att_23_x_last time_mixing_23_out 0=0
Split split_time_mixing_23_out 1 2 time_mixing_23_out ffn_23_x_last ffn_23_x
LayerNorm layernorm_47 1 1 ffn_23_x ffn_23_xx 0=1024 1=0.00001 2=1
Split split_ffn_23_xx 1 3 ffn_23_xx ffn_23_xx_0 ffn_23_xx_1 state_71_out
BinaryOp sub_47 2 1 state_71_in ffn_23_xx_0 ffn_23_sx 0=1
Split split_ffn_23_sx 1 1 ffn_23_sx ffn_23_sx_0
MemoryData data_ffn_23_x_k 0 1 ffn_23_x_k 0=1024 21=0
BinaryOp mul_310 2 1 ffn_23_sx_0 ffn_23_x_k ffn_23_xk 0=2
BinaryOp add_308 2 1 ffn_23_xk ffn_23_xx_1 ffn_23_xxk 0=0
Gemm gemm_ffn_23_key 1 1 ffn_23_xxk ffn_23_key 4=0 5=1 6=0 7=0 8=4096 9=1024 10=-1
ReLU relu_23 1 1 ffn_23_key ffn_23_key_relu
UnaryOp square_ffn_23_key_relu 1 1 ffn_23_key_relu ffn_23_key_relu_square 0=4
Gemm gemm_ffn_23_value 1 1 ffn_23_key_relu_square ffn_23_value 4=0 5=1 6=0 7=0 8=1024 9=4096 10=-1
BinaryOp add_309 2 1 ffn_23_value ffn_23_x_last channel_mixing_23_out 0=0
LayerNorm layernorm_48 1 1 channel_mixing_23_out norm_head 0=1024 1=0.00001 2=1
Gemm gemm_logits 1 1 norm_head logits 4=0 5=1 6=0 7=0 8=65536 9=1024 10=-1
|