{"id":568,"date":"2024-11-09T12:31:18","date_gmt":"2024-11-09T04:31:18","guid":{"rendered":"https:\/\/eve2333.top\/?p=568"},"modified":"2024-11-11T19:46:17","modified_gmt":"2024-11-11T11:46:17","slug":"%e5%bc%ba%e5%8c%96%e5%ad%a6%e4%b9%a0","status":"publish","type":"post","link":"https:\/\/eve2333.top\/?p=568","title":{"rendered":"\u5f3a\u5316\u5b66\u4e60"},"content":{"rendered":"\n<h1 class=\"wp-block-heading\">\u57fa\u672c\u6982\u5ff5<\/h1>\n\n\n\n<h2 class=\"wp-block-heading\" id=\"_1-\u57fa\u672c\u672f\u8bed\">1.\u57fa\u672c\u672f\u8bed<\/h2>\n\n\n\n<p>Attention<\/p>\n\n\n\n<ul class=\"wp-block-list\">\n<li><strong>\u5927\u5199\u5b57\u6bcd\uff1a<\/strong>&nbsp;\u5177\u6709\u968f\u673a\u6027\u7684\u53d8\u91cf<\/li>\n\n\n\n<li><strong>\u5c0f\u5199\u5b57\u6bcd\uff1a<\/strong>&nbsp;\u5b9e\u5b9e\u5728\u5728\u7684\u6570\u503c<br><\/li>\n\n\n\n<li><strong>\u4ee3\u7406\uff08agent\uff09<\/strong>\uff1a\u9700\u8981\u63a7\u5236\u7684\u89d2\u8272<\/li>\n\n\n\n<li><strong>\u73af\u5883\uff08environment\uff09<\/strong>\uff1a\u89d2\u8272\u6240\u5728\u7684\u6e38\u620f<\/li>\n\n\n\n<li><strong>\u72b6\u6001\uff08state\uff09<\/strong>\uff1a\u89d2\u8272\u6240\u5904\u7684\u67d0\u4e00\u65f6\u523b\u7684\u6e38\u620f\u573a\u666f<\/li>\n\n\n\n<li><strong>\u52a8\u4f5c\uff08action\uff09<\/strong>\uff1a\u80fd\u5bf9\u89d2\u8272\u80fd\u8fdb\u884c\u7684\u64cd\u4f5c<\/li>\n\n\n\n<li><strong>\u7b56\u7565\uff08policy\uff09<\/strong>\uff1a\u63cf\u8ff0\u89d2\u8272\u5728\u67d0\u4e00\u72b6\u6001\u4ea7\u751f\u67d0\u4e2a\u52a8\u4f5c\u7684\u6982\u7387\uff0c<strong>\u63a7\u5236\u89d2\u8272\u5728\u5f53\u524d\u72b6\u6001\u4e0b\u6240\u6709\u52a8\u4f5c\u7684\u6982\u7387\uff0c\u4e0d\u5c31\u662f\u4e00\u79cd\u884c\u52a8\u7684\u51b3\u7b56<\/strong>\u3002\u5728\u73af\u5883<code>s<\/code>\uff0c\u89d2\u8272\u4ea7\u751f\u52a8\u4f5c<code>a<\/code>\u7684\u6982\u7387\uff1a\u03c0(a\u2223s)=P(A=a\u2223S=s)&nbsp;<\/li>\n\n\n\n<li><strong>\u5956\u52b1\uff08reward\uff09<\/strong>\uff1a\u5728\u67d0\u4e00\u72b6\u6001\u4e0b\uff0c\u89d2\u8272\u91c7\u53d6\u67d0\u4e00\u52a8\u4f5c\u540e\uff0c\u5bf9\u8be5\u52a8\u4f5c\u7684\u8bc4\u4ef7<\/li>\n\n\n\n<li><strong>\u72b6\u6001\u8f6c\u79fb\uff08state transition\uff09<\/strong>\uff1a\u4ece\u65e7\u72b6\u6001\u5230\u65b0\u72b6\u6001\uff1b<strong>\u72b6\u6001\u8f6c\u79fb\u662f\u968f\u673a\u7684\uff0c\u56e0\u4e3a\u89d2\u8272\u662f\u4e0d\u6e05\u695a\u6e38\u620f\u573a\u666f\u5c06\u5982\u4f55\u53d8\u5316<\/strong>\u3002\u89d2\u8272\u5728\u65e7\u73af\u5883<code>s<\/code>\uff0c\u53d1\u751f\u52a8\u4f5c<code>a<\/code>\u540e\uff0c\u6e38\u620f\u73af\u5883\u53d8\u4e3a\u65b0\u72b6\u6001<code>s'<\/code>\u7684\u6982\u7387\uff1ap(s\u2032\u2223s,a)=P(S\u2032=s\u2032\u2223S=s,A=a)&nbsp;<\/li>\n<\/ul>\n\n\n\n<h2 class=\"wp-block-heading\" id=\"_2-\u89d2\u8272\u4e0e\u73af\u5883\u7684\u4ea4\u4e92\">2.\u89d2\u8272\u4e0e\u73af\u5883\u7684\u4ea4\u4e92<\/h2>\n\n\n\n<h3 class=\"wp-block-heading\" id=\"_21-\u4ea4\u4e92\u6d41\u7a0b\">2.1\u4ea4\u4e92\u6d41\u7a0b<\/h3>\n\n\n\n<ol class=\"wp-block-list\">\n<li>\u89d2\u8272\u6839\u636e\u300c\u51b3\u7b56\u51fd\u6570&nbsp;\u03c0(a\u2223s) \u300d\u5728\u5f53\u524d\u73af\u5883\u72b6\u6001\u4e0b\u9009\u62e9\u4e00\u4e2a\u52a8\u4f5c<img loading=\"lazy\" decoding=\"async\" height=\"15\" width=\"15\" src=\"blob:https:\/\/eve2333.top\/d05f88e9-8da6-4178-828d-a44fa37b67cd\"><img loading=\"lazy\" decoding=\"async\" width=\"500\" height=\"201\" class=\"wp-image-573\" style=\"width: 500px;\" src=\"https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731123107-\u5c4f\u5e55\u622a\u56fe-2024-11-09-113119.png\" alt=\"\" srcset=\"https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731123107-\u5c4f\u5e55\u622a\u56fe-2024-11-09-113119.png 1061w, https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731123107-\u5c4f\u5e55\u622a\u56fe-2024-11-09-113119-300x121.png 300w, https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731123107-\u5c4f\u5e55\u622a\u56fe-2024-11-09-113119-1024x412.png 1024w, https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731123107-\u5c4f\u5e55\u622a\u56fe-2024-11-09-113119-768x309.png 768w\" sizes=\"auto, (max-width: 500px) 100vw, 500px\" \/><\/li>\n\n\n\n<li>\u73af\u5883\u53d7\u5230\u89d2\u8272\u52a8\u4f5c\u7684\u5f71\u54cd\u8fdb\u5165\u4e0b\u4e00\u4e2a\u72b6\u6001\uff0c\u5e76\u5bf9\u89d2\u8272\u7684\u52a8\u4f5c\u7ed9\u51fa\u5956\u52b1\u3002\u73af\u5883\u4e5f\u6839\u636e\u300c\u72b6\u6001\u8f6c\u79fb\u51fd\u6570&nbsp;p(s\u2032\u2223s,z) \u8fdb\u5165\u4e0b\u4e00\u4e2a\u72b6\u6001<br><img loading=\"lazy\" decoding=\"async\" width=\"500\" height=\"190\" class=\"wp-image-572\" style=\"width: 500px;\" src=\"https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731123106-\u5c4f\u5e55\u622a\u56fe-2024-11-09-113123.png\" alt=\"\" srcset=\"https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731123106-\u5c4f\u5e55\u622a\u56fe-2024-11-09-113123.png 1062w, https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731123106-\u5c4f\u5e55\u622a\u56fe-2024-11-09-113123-300x114.png 300w, https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731123106-\u5c4f\u5e55\u622a\u56fe-2024-11-09-113123-1024x390.png 1024w, https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731123106-\u5c4f\u5e55\u622a\u56fe-2024-11-09-113123-768x292.png 768w\" sizes=\"auto, (max-width: 500px) 100vw, 500px\" \/><\/li>\n\n\n\n<li>\u91cd\u590d\u4e0a\u8ff0\u6b65\u9aa4<\/li>\n<\/ol>\n\n\n\n<h3 class=\"wp-block-heading\" id=\"_22-\u968f\u673a\u6027\">2.2\u968f\u673a\u6027<\/h3>\n\n\n\n<ul class=\"wp-block-list\">\n<li><strong>\u52a8\u4f5c\u7684\u968f\u673a\u6027<\/strong>\uff1a\u89d2\u8272\u5728\u5f53\u524d\u72b6\u6001\u4e0b\uff0c\u4f1a\u91c7\u53d6\u4ec0\u4e48\u52a8\u4f5c\u662f\u968f\u673a\u7684<\/li>\n<\/ul>\n\n\n\n<figure class=\"wp-block-image size-full is-resized\"><img loading=\"lazy\" decoding=\"async\" width=\"956\" height=\"521\" src=\"https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731123106-\u5c4f\u5e55\u622a\u56fe-2024-11-09-113127.png\" alt=\"\" class=\"wp-image-571\" style=\"width:602px;height:auto\" srcset=\"https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731123106-\u5c4f\u5e55\u622a\u56fe-2024-11-09-113127.png 956w, https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731123106-\u5c4f\u5e55\u622a\u56fe-2024-11-09-113127-300x163.png 300w, https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731123106-\u5c4f\u5e55\u622a\u56fe-2024-11-09-113127-768x419.png 768w\" sizes=\"auto, (max-width: 956px) 100vw, 956px\" \/><\/figure>\n\n\n\n<ul class=\"wp-block-list\">\n<li><strong>\u72b6\u6001\u8f6c\u79fb\u7684\u968f\u673a\u6027<\/strong>\uff1a\u89d2\u8272\u505a\u51fa\u4e00\u4e2a\u52a8\u4f5c\u540e\uff0c\u73af\u5883\u72b6\u6001\u7684\u53d8\u5316\u662f\u968f\u673a\u7684<\/li>\n<\/ul>\n\n\n\n<figure class=\"wp-block-image size-full is-resized\"><img loading=\"lazy\" decoding=\"async\" width=\"942\" height=\"698\" src=\"https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731123105-\u5c4f\u5e55\u622a\u56fe-2024-11-09-113131.png\" alt=\"\" class=\"wp-image-570\" style=\"width:521px;height:auto\" srcset=\"https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731123105-\u5c4f\u5e55\u622a\u56fe-2024-11-09-113131.png 942w, https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731123105-\u5c4f\u5e55\u622a\u56fe-2024-11-09-113131-300x222.png 300w, https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731123105-\u5c4f\u5e55\u622a\u56fe-2024-11-09-113131-768x569.png 768w\" sizes=\"auto, (max-width: 942px) 100vw, 942px\" \/><\/figure>\n\n\n\n<h2 class=\"wp-block-heading\" id=\"_3-\u5956\u52b1\u3001\u56de\u62a5\u3001\u4ef7\u503c\u51fd\u6570\">3.\u5956\u52b1\uff0c\u56de\u62a5\uff0c\u4ef7\u503c\u51fd\u6570<\/h2>\n\n\n\n<h3 class=\"wp-block-heading\" id=\"_31-\u5956\u52b1\">3.1\u5956\u52b1<\/h3>\n\n\n\n<p><strong>\u5956\u52b1\uff08reward\uff09\uff1a<\/strong>&nbsp;\u5bf9\u5f53\u524d\u72b6\u6001\u4e0b\u7684\u67d0\u4e00\u52a8\u4f5c\u8fdb\u884c\u8bc4\u4ef7\uff1b<strong>\u300c\u5956\u52b1\u300d\u8d8a\u5927\uff0c\u8bf4\u660e\u79bb\u300c\u76ee\u6807\u300d\u8d8a\u63a5\u8fd1\uff0c\u4f8b\u5982\u8d62\u5f97\u672c\u5c40\u6e38\u620f\u3002<\/strong><\/p>\n\n\n\n<h3 class=\"wp-block-heading\" id=\"_32-\u56de\u62a5\">3.2\u56de\u62a5<\/h3>\n\n\n\n<p><strong>\u56de\u62a5\uff08return\uff09\uff1a<\/strong>&nbsp;\u672a\u6765\u7684\u7d2f\u8ba1\u5956\u52b1\uff0c\u5373\u4ece\u5f53\u524d\u65f6\u523b\u5230\u7ed3\u675f\u7684\u5956\u52b1\u4e4b\u540e\u3002<\/p>\n\n\n\n<figure class=\"wp-block-image\"><img decoding=\"async\" src=\"https:\/\/latex.csdn.net\/eq?U_t%3DR_t+R_%7Bt+1%7D+R_%7Bt+2%7D+R_%7Bt+3%7D+.....+R_%7Bt+n%7D\" alt=\"U_t=R_t+R_{t+1}+R_{t+2}+R_{t+3}+.....+R_{t+n}\"\/><\/figure>\n\n\n\n<p><strong>\u6298\u6263\u56de\u62a5\uff08discounted return\uff09\uff1a<\/strong>&nbsp;\u7531\u4e8e\u5f53\u524d\u5956\u52b1\u662f\u9a6c\u4e0a\u80fd\u5f97\u5230\u7684\uff0c\u800c\u672a\u6765\u7684\u5956\u52b1\u53d8\u6570\u8f83\u5927\uff0c\u6240\u4ee5\u9700\u8981\u7ed9\u672a\u6765\u7684\u5956\u52b1\u964d\u4f4e\u6bd4\u91cd\u3002<strong>\u03b3\u03b3&nbsp;\u5c31\u4e3a\u6298\u6263\u7387<\/strong><\/p>\n\n\n\n<figure class=\"wp-block-image\"><img decoding=\"async\" src=\"https:\/\/latex.csdn.net\/eq?U_t%3DR_t+%5Cgamma%20R_%7Bt+1%7D+%5Cgamma%5E%7B2%7DR_%7Bt+2%7D+.....+%5Cgamma%5E%7Bn%7DR_%7Bt+n%7D\" alt=\"U_t=R_t+\\gamma R_{t+1}+\\gamma^{2}R_{t+2}+.....+\\gamma^{n}R_{t+n}\"\/><\/figure>\n\n\n\n<p>Tip<\/p>\n\n\n\n<p>\u56de\u62a5\uff08return\uff09\u662f\u5177\u6709\u300c\u968f\u673a\u6027\u300d\u7684\u3002\u56e0\u4e3a&nbsp;UtUt\u200b&nbsp;\u53d6\u51b3\u4e8e&nbsp;Rt+iRt+i\u200b&nbsp;\uff0c\u800c&nbsp;Rt+iRt+i\u200b&nbsp;\u53c8\u4e0e\u5c06\u6765\u7684\u300c\u72b6\u6001\u300d\u548c\u300c\u52a8\u4f5c\u300d\u6709\u5173\u3002<\/p>\n\n\n\n<h3 class=\"wp-block-heading\" id=\"_33-\u4ef7\u503c\u51fd\u6570\">3.3\u4ef7\u503c\u51fd\u6570<\/h3>\n\n\n\n<ul class=\"wp-block-list\">\n<li><strong>\u52a8\u4f5c\u4ef7\u503c\u51fd\u6570\uff08action-value function\uff09<\/strong>\uff1a\u7531\u4e8e\u300c\u56de\u62a5\u00a0Ut\u300d\u662f\u5177\u6709\u968f\u673a\u6027\u7684\uff0c\u5e76\u4e0e\u5c06\u6765\u7684\u72b6\u6001\u548c\u52a8\u4f5c\u6709\u5173\uff0c\u4f46\u662f\u89d2\u8272\u5e76\u4e0d\u80fd\u77e5\u9053\u672a\u6765\u4f1a\u53d1\u751f\u4ec0\u4e48\uff0c\u6240\u4ee5\u89d2\u8272\u5728\u6e38\u620f\u8fc7\u7a0b\u4e2d\u65e0\u6cd5\u4f7f\u7528\u300c\u56de\u62a5\u300d\u6765\u8bc4\u4f30\u5f53\u524d\u7684\u52a8\u4f5c\u3002<strong>\u4e3a\u4e86\u5bf9\u5f53\u524d\u52a8\u4f5c\u7684\u4ef7\u503c\u8fdb\u884c\u8bc4\u4f30\uff0c\u5c31\u5bf9\u00a0UtUt\u200b\u00a0\u6c42\u89e3\u671f\u671b\uff0c\u8fc7\u6ee4\u6389\u968f\u673a\u6027\u3002<\/strong> \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0<img decoding=\"async\" src=\"https:\/\/latex.csdn.net\/eq?Q_%5CPi%20%28s_t%2Ca_t%29%3DE%5BU_t%7CS_t%3Ds_t%2CA_t%3Da_t%5D\" alt=\"Q_\\Pi (s_t,a_t)=E[U_t|S_t=s_t,A_t=a_t]\"> \u5176\u4e2d\uff0cQ\u03c0(st,at)\u00a0\u4e3a\u52a8\u4f5c\u4ef7\u503c\u51fd\u6570\uff0c\u6839\u636e\u52a8\u4f5c\u6982\u7387\u5206\u5e03\u51fd\u6570\u00a0\u03c0(at\u2223st)\u00a0\u8fdb\u884c\u8ba1\u7b97\uff0c\u6240\u4ee5\u5c31\u4e0d\u9700\u8981\u5c06\u6765\u7684\u00a0Rt+i\u00a0\u503c\u53c2\u4e0e\u8ba1\u7b97\u3002\u00a0st,at\u00a0\u5747\u4e3a\u5c0f\u5199\u5b57\u6bcd\uff0c\u5747\u662f\u5b9e\u5b9e\u5728\u5728\u7684\u6570\u503c\uff0c\u6240\u4ee5\u00a0Q\u03c0(st,at)\u00a0\u662f\u4e00\u4e2a\u6570\u503c<\/li>\n\n\n\n<li><strong>\u6700\u4f18\u52a8\u4f5c\u4ef7\u503c\u51fd\u6570\uff08Optimal action-value function\uff09<\/strong>\uff1a\u7531\u4e8e\u300c\u52a8\u4f5c\u4ef7\u503c\u51fd\u6570\u00a0Q\u03c0\u200b\u00a0\u300d\u4e0e\u300c\u7b56\u7565\u51fd\u6570\u00a0\u03c0\u300d\u6709\u5173\uff0c\u4e0d\u540c\u7684\u300c\u7b56\u7565\u51fd\u6570\u300d\u5c31\u4f1a\u5bf9\u5e94\u4e0d\u540c\u7684\u300c\u52a8\u4f5c\u4ef7\u503c\u51fd\u6570\u300d\u3002<strong>\u4e3a\u4e86\u5bf9\u300c\u52a8\u4f5c\u4ef7\u503c\u51fd\u6570\u300d\u8fdb\u884c\u7edf\u4e00\uff0c\u5c31\u9009\u5219\u4f7f\u7528\u300c\u6700\u4f18\u7684\u7b56\u7565\u51fd\u6570\u300d\uff0c\u56e0\u6b64\u300c\u6700\u4f18\u52a8\u4f5c\u4ef7\u503c\u51fd\u6570\u300d\u4e3b\u8981\u7528\u6765\u8bc4\u4ef7\u300c\u52a8\u4f5c\u300d\u7684\u597d\u574f<\/strong> \u00a0<img decoding=\"async\" src=\"https:\/\/latex.csdn.net\/eq?Q_%5Cpi%5E*%3DmaxQ_%5Cpi%20%28s_t%2Ca_t%29\" alt=\"Q_\\pi^*=maxQ_\\pi (s_t,a_t)\"><\/li>\n\n\n\n<li><strong>\u72b6\u6001\u4ef7\u503c\u51fd\u6570\uff08state-value function\uff09\uff1a<\/strong>\u00a0\u5bf9\u52a8\u4f5c\u4ef7\u503c\u51fd\u6570\u00a0Q\u03c0(st,A)\u00a0\u5173\u4e8e\u81ea\u53d8\u91cf\u00a0A\u00a0\u8fdb\u884c\u671f\u671b\u8ba1\u7b97\u3002\u00a0\u7531\u4e8e\u00a0AA\u00a0\u662f\u5927\u5199\u5b57\u6bcd\uff0c\u662f\u4e00\u4e2a\u5177\u6709\u968f\u673a\u6027\u7684\u53d8\u91cf\uff0cQ\u03c0(st,A)\u00a0\u5219\u4e3a\u4e00\u4e2a\u5173\u4e8e\u00a0A\u00a0\u7684\u51fd\u6570\u00a0\u3002<strong>V\u03c0(st)\u00a0\u53ef\u4ee5\u7528\u6765\u8bc4\u4ef7\u89d2\u8272\u5f53\u524d\u6240\u5904\u72b6\u6001\u7684\u597d\u574f<\/strong>\n<ul class=\"wp-block-list\">\n<li><strong>\u52a8\u4f5c\u79bb\u6563<\/strong>\uff1a &nbsp;<img decoding=\"async\" src=\"https:\/\/latex.csdn.net\/eq?V_%5Cpi%28s_t%29%3DE_A%5BQ_%5Cpi%28s_t%2CA%29%5D%3D%5Csum%20%5Cpi%28a%7Cs_t%29Q_%5Cpi%28s_t%2Ca%29\" alt=\"V_\\pi(s_t)=E_A[Q_\\pi(s_t,A)]=\\sum \\pi(a|s_t)Q_\\pi(s_t,a)\"><\/li>\n\n\n\n<li><strong>\u52a8\u4f5c\u8fde\u7eed<\/strong>\uff1a &nbsp;<img decoding=\"async\" src=\"https:\/\/latex.csdn.net\/eq?V_%5Cpi%28s_t%29%3DE_A%5BQ_%5Cpi%28s_t%2CA%29%5D%3D%5Cint%20%5Cpi%28a%7Cs_t%29Q_%5Cpi%28s_t%2Ca%29da\" alt=\"V_\\pi(s_t)=E_A[Q_\\pi(s_t,A)]=\\int \\pi(a|s_t)Q_\\pi(s_t,a)da\"><\/li>\n<\/ul>\n<\/li>\n<\/ul>\n\n\n\n<p>Tip<\/p>\n\n\n\n<ul class=\"wp-block-list\">\n<li><strong>\u52a8\u4f5c\u4ef7\u503c\u51fd\u6570\uff1a<\/strong>&nbsp;\u8bc4\u4ef7\u89d2\u8272\u5728\u72b6\u6001<code>s<\/code>\u6240\u4f5c\u51fa\u7684\u52a8\u4f5c<code>a<\/code>\u7684\u597d\u574f<\/li>\n\n\n\n<li><strong>\u72b6\u6001\u4ef7\u503c\u51fd\u6570\uff1a<\/strong>&nbsp;\u8bc4\u4ef7\u89d2\u8272\u5728\u72b6\u6001<code>s<\/code>\u6240\u5904\u5c40\u52bf\u7684\u597d\u574f\uff0c\u4f8b\u5982\u5f53\u524d\u4e0d\u8bba\u505a\u4ec0\u4e48\u52a8\u4f5c\uff0c\u90fd\u8f93\u5b9a\u4e86<\/li>\n<\/ul>\n\n\n\n<h2 class=\"wp-block-heading\" id=\"_4-\u89d2\u8272\u7684\u63a7\u5236\">4.\u89d2\u8272\u7684\u63a7\u5236<\/h2>\n\n\n\n<ul class=\"wp-block-list\">\n<li><strong>\u57fa\u4e8e\u51b3\u7b56\u51fd\u6570\u65b9\u6848<\/strong>\uff1a\u7ed9\u89d2\u8272\u5b9a\u5236\u4e86\u4e00\u4e2a\u975e\u5e38\u68d2\u7684\u52a8\u4f5c\u4ea7\u751f\u7b56\u7565\n<ol class=\"wp-block-list\">\n<li>\u89c2\u5bdf\u6e38\u620f\u73af\u5883\u72b6\u6001&nbsp; st<\/li>\n\n\n\n<li>\u6839\u636e\u51b3\u7b56\u51fd\u6570&nbsp;\u03c0(a\u2223st)&nbsp; \u83b7\u53d6\u4e00\u4e2a\u52a8\u4f5c&nbsp; at<\/li>\n<\/ol>\n<\/li>\n\n\n\n<li><strong>\u57fa\u4e8e\u6700\u4f18\u52a8\u4f5c\u4ef7\u503c\u51fd\u6570\u65b9\u6848<\/strong>\uff1a\u4ece\u6240\u6709\u53ef\u80fd\u7684\u51b3\u7b56\u4e2d\uff0c\u9009\u62e9\u6700\u4f18\u7684\u52a8\u4f5c\n<ol class=\"wp-block-list\">\n<li>\u89c2\u5bdf\u6e38\u620f\u73af\u5883\u72b6\u6001&nbsp;st<\/li>\n\n\n\n<li>\u6839\u636e\u6700\u4f18\u52a8\u4f5c\u4ef7\u503c\u51fd\u6570&nbsp;&nbsp;<img decoding=\"async\" src=\"https:\/\/latex.csdn.net\/eq?Q%5E*%3Aa_t%3Dargmax_aQ%5E*%28s_t%2Ca%29\" alt=\"Q^*:a_t=argmax_aQ^*(s_t,a)\"><\/li>\n<\/ol>\n<\/li>\n\n\n\n<li><strong>\u5b66\u4e60\u7684\u8fc7\u7a0b<\/strong>\uff1a\u4e0d\u65ad\u7684\u91cd\u590d&nbsp;(si,ai,ri)(si\u200b,ai\u200b,ri\u200b)&nbsp;\u7684\u8fc7\u7a0b\uff0c\u6700\u7ec8\u5b9e\u73b0\u5956\u52b1\u7684\u6700\u5927\u5316\u3002 <img loading=\"lazy\" decoding=\"async\" height=\"15\" width=\"15\" src=\"blob:https:\/\/eve2333.top\/208eb77c-3803-4c23-8279-560e25f7bf33\"><\/li>\n<\/ul>\n\n\n\n<figure class=\"wp-block-image size-full\"><img loading=\"lazy\" decoding=\"async\" width=\"1009\" height=\"176\" src=\"https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731123105-\u5c4f\u5e55\u622a\u56fe-2024-11-09-113137.png\" alt=\"\" class=\"wp-image-569\" srcset=\"https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731123105-\u5c4f\u5e55\u622a\u56fe-2024-11-09-113137.png 1009w, https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731123105-\u5c4f\u5e55\u622a\u56fe-2024-11-09-113137-300x52.png 300w, https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731123105-\u5c4f\u5e55\u622a\u56fe-2024-11-09-113137-768x134.png 768w\" sizes=\"auto, (max-width: 1009px) 100vw, 1009px\" \/><\/figure>\n\n\n\n<h1 class=\"wp-block-heading\">\u4ef7\u503c\u5b66\u4e60<\/h1>\n\n\n\n<h2 class=\"wp-block-heading\" id=\"_1-deep-q-network\uff08dqn\uff09\">1.Deep Q-Network\uff08DQN\uff09<\/h2>\n\n\n\n<h3 class=\"wp-block-heading\" id=\"_11-\u7b97\u6cd5\u601d\u8def\">1.1\u7b97\u6cd5\u601d\u8def<\/h3>\n\n\n\n<ul class=\"wp-block-list\">\n<li><strong>\u76ee\u6807<\/strong>\uff1a\u8d62\u5f97\u8fd9\u573a\u6e38\u620f\uff0c\u5373\u5b9e\u73b0\u300c\u56de\u62a5\u300d\u6700\u5927\u3002<\/li>\n\n\n\n<li><strong>\u7b56\u7565<\/strong>\uff1a\u5f97\u5230\u300c\u6700\u4f18\u4ef7\u503c\u51fd\u6570&nbsp;Q^\u2217(s,a) \u300d\uff0c\u8fd9\u6837\u5c31\u80fd\u901a\u8fc7\u5e26\u5165\u72b6\u6001&nbsp;s\uff0c\u89e3\u7b97\u51fa\u6700\u4f18\u52a8\u4f5c&nbsp;&nbsp;<br>&nbsp;<\/li>\n\n\n\n<li><strong>\u95ee\u9898<\/strong>\uff1a&nbsp;Q^\u2217(s,a) \u53ea\u80fd\u901a\u8fc7\u4e0a\u5e1d\u89c6\u89d2\u5f97\u5230\uff0c\u89d2\u8272\u662f\u65e0\u6cd5\u8ba4\u77e5\u3002<\/li>\n\n\n\n<li><strong>\u89e3\u51b3<\/strong>\uff1a\u5229\u7528\u4e00\u4e2a\u795e\u7ecf\u7f51\u7edc&nbsp;Q(s,a;w) \u6765\u8fd1\u4f3c\u62df\u5408\u51fa&nbsp;Q^\u2217(s,a)\n<ul class=\"wp-block-list\">\n<li><code>s<\/code>\uff1a\u795e\u7ecf\u7f51\u7edc\u7684\u8f93\u5165<\/li>\n\n\n\n<li><code>a<\/code>\uff1a\u795e\u7ecf\u7f51\u7edc\u7684\u8f93\u51fa<\/li>\n\n\n\n<li><code>w<\/code>\uff1a\u795e\u7ecf\u7f51\u7edc\u7684\u53c2\u6570<\/li>\n<\/ul>\n<\/li>\n<\/ul>\n\n\n\n<h3 class=\"wp-block-heading\" id=\"_12-dqn\u795e\u7ecf\u7f51\u7edc\u6784\u9020\">1.2DQN\u795e\u7ecf\u7f51\u7edc\u6784\u9020<\/h3>\n\n\n\n<figure class=\"wp-block-image size-full is-resized\"><img loading=\"lazy\" decoding=\"async\" width=\"1017\" height=\"277\" src=\"https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731125601-\u5c4f\u5e55\u622a\u56fe-2024-11-09-121156.png\" alt=\"\" class=\"wp-image-574\" style=\"width:701px;height:auto\" srcset=\"https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731125601-\u5c4f\u5e55\u622a\u56fe-2024-11-09-121156.png 1017w, https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731125601-\u5c4f\u5e55\u622a\u56fe-2024-11-09-121156-300x82.png 300w, https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731125601-\u5c4f\u5e55\u622a\u56fe-2024-11-09-121156-768x209.png 768w\" sizes=\"auto, (max-width: 1017px) 100vw, 1017px\" \/><\/figure>\n\n\n\n<ol class=\"wp-block-list\">\n<li>\u4e00\u5f20\u6e38\u620f\u753b\u9762\u5f53\u4f5c\u4e00\u4e2a\u300c\u72b6\u6001\u300d\uff0c\u72b6\u6001\u901a\u8fc7\u300c\u5377\u79ef\u5c42\u300d\u5b9e\u73b0\u7279\u5f81\u63d0\u53d6<\/li>\n\n\n\n<li>\u7279\u5f81\u8f93\u5165\u7ed9\u300c\u6743\u8fde\u63a5\u5c42\u300d\u8fdb\u884c&nbsp;Q^\u2217(s,a)\u62df\u5408<\/li>\n\n\n\n<li>\u300c\u6743\u8fde\u63a5\u5c42\u300d\u8f93\u51fa\u6bcf\u4e2a\u52a8\u4f5c\u7684&nbsp;Q^\u2217(s,a)\u7ed3\u679c<\/li>\n<\/ol>\n\n\n\n<h3 class=\"wp-block-heading\" id=\"_13-dqn\u8fd0\u884c\u6d41\u7a0b\">1.3DQN\u8fd0\u884c\u6d41\u7a0b<img loading=\"lazy\" decoding=\"async\" height=\"15\" width=\"15\" src=\"blob:https:\/\/eve2333.top\/d29fbd19-521d-4b34-807d-07572c241b5f\"><\/h3>\n\n\n\n<figure class=\"wp-block-image size-full is-resized\"><img loading=\"lazy\" decoding=\"async\" width=\"1024\" height=\"361\" src=\"https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731125613-\u5c4f\u5e55\u622a\u56fe-2024-11-09-121201.png\" alt=\"\" class=\"wp-image-575\" style=\"width:831px;height:auto\" srcset=\"https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731125613-\u5c4f\u5e55\u622a\u56fe-2024-11-09-121201.png 1024w, https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731125613-\u5c4f\u5e55\u622a\u56fe-2024-11-09-121201-300x106.png 300w, https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731125613-\u5c4f\u5e55\u622a\u56fe-2024-11-09-121201-768x271.png 768w\" sizes=\"auto, (max-width: 1024px) 100vw, 1024px\" \/><\/figure>\n\n\n\n<ol class=\"wp-block-list\">\n<li>\u901a\u8fc7\u5f53\u524d\u7684\u72b6\u6001&nbsp;stst\u200b\uff0c\u5229\u7528\u795e\u7ecf\u7f51\u7edc\u8ba1\u7b97\u51fa\u5f53\u524d\u6700\u4f18\u7684\u52a8\u4f5c&nbsp;<img decoding=\"async\" src=\"https:\/\/latex.csdn.net\/eq?a_t%3Dargmax_aQ%28s_t%2Ca%3Bw%29\" alt=\"a_t=argmax_aQ(s_t,a;w)\"><\/li>\n\n\n\n<li>\u6267\u884c\u52a8\u4f5c<img decoding=\"async\" src=\"https:\/\/latex.csdn.net\/eq?a_t\" alt=\"a_t\">\u540e\uff0c\u7ed9\u51fa\u5956\u52b1<img decoding=\"async\" src=\"https:\/\/latex.csdn.net\/eq?r_t\" alt=\"r_t\"><\/li>\n\n\n\n<li>\u73af\u5883\u72b6\u6001\u8f6c\u79fb\uff0c\u5f97\u5230\u4e0b\u4e00\u4e2a\u72b6\u6001&nbsp;<img decoding=\"async\" src=\"https:\/\/latex.csdn.net\/eq?s_%7Bt+1%7D\" alt=\"s_{t+1}\"><\/li>\n\n\n\n<li>\u91cd\u590d\u4e0a\u8ff0\u6b65\u9aa4<\/li>\n<\/ol>\n\n\n\n<h2 class=\"wp-block-heading\" id=\"_2-dqn\u8bad\u7ec3\">2.DQN\u8bad\u7ec3<\/h2>\n\n\n\n<h3 class=\"wp-block-heading\" id=\"_21-\u65f6\u95f4\u5dee\u5206\u7b97\u6cd5\uff08temporal-different\uff09\">2.1\u65f6\u95f4\u5dee\u5206\u7b97\u6cd5\uff08Tenporal Different\uff09<\/h3>\n\n\n\n<h4 class=\"wp-block-heading\">1. \u95ee\u9898\u6a21\u578b<\/h4>\n\n\n<div class=\"wp-block-image\">\n<figure class=\"aligncenter size-full is-resized\"><img loading=\"lazy\" decoding=\"async\" width=\"332\" height=\"494\" src=\"https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731125633-\u5c4f\u5e55\u622a\u56fe-2024-11-09-121205.png\" alt=\"\" class=\"wp-image-576\" style=\"width:235px;height:auto\" srcset=\"https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731125633-\u5c4f\u5e55\u622a\u56fe-2024-11-09-121205.png 332w, https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731125633-\u5c4f\u5e55\u622a\u56fe-2024-11-09-121205-202x300.png 202w\" sizes=\"auto, (max-width: 332px) 100vw, 332px\" \/><\/figure>\n<\/div>\n\n\n<p><strong>\u95ee\u9898\uff1a<\/strong>&nbsp;\u8bbe\u8ba1\u4e00\u4e2a\u795e\u7ecf\u7f51\u7edc&nbsp;Q(w)Q(w)\uff0c\u5bf9\u4ece<code>NYC<\/code>\u9a7e\u9a76\u5230<code>atlanta<\/code>\u6240\u8981\u82b1\u8d39\u7684\u65f6\u95f4\u8fdb\u884c\u9884\u6d4b\u3002<\/p>\n\n\n\n<h4 class=\"wp-block-heading\">2. \u68af\u5ea6\u4e0b\u964d\u7b97\u6cd5<img loading=\"lazy\" decoding=\"async\" height=\"15\" width=\"15\" src=\"blob:https:\/\/eve2333.top\/405e0c43-5f7d-423b-9bd3-b67b2eec8dd5\"><\/h4>\n\n\n<div class=\"wp-block-image\">\n<figure class=\"aligncenter size-full is-resized\"><img loading=\"lazy\" decoding=\"async\" width=\"288\" height=\"483\" src=\"https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731125648-\u5c4f\u5e55\u622a\u56fe-2024-11-09-121210.png\" alt=\"\" class=\"wp-image-577\" style=\"width:177px;height:auto\" srcset=\"https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731125648-\u5c4f\u5e55\u622a\u56fe-2024-11-09-121210.png 288w, https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731125648-\u5c4f\u5e55\u622a\u56fe-2024-11-09-121210-179x300.png 179w\" sizes=\"auto, (max-width: 288px) 100vw, 288px\" \/><\/figure>\n<\/div>\n\n\n<ol class=\"wp-block-list\">\n<li>\u6a21\u578b\u4f30\u8ba1\u65f6\u95f4\uff1a&nbsp;q=Q(w)=1000min<\/li>\n\n\n\n<li>\u771f\u5b9e\u9a7e\u9a76\u65f6\u95f4\uff1a&nbsp;y=860min<\/li>\n\n\n\n<li>\u8ba1\u7b97\u635f\u5931\u503c\uff1aL=12(q\u2212y)2<\/li>\n\n\n\n<li>\u8ba1\u7b97\u5173\u4e8e\u7cfb\u6570&nbsp;w&nbsp;\u7684\u68af\u5ea6\uff1a\u2202L\u2202w=\u2202L\u2202q\u2202q\u2202w=(q\u2212y)\u2202Q(w)\u2202w\u2202w\u2202L\u200b=\u2202q\u2202L\u200b\u2202w\u2202q\u200b=(q\u2212y)\u2202w\u2202Q(w)\u200b<\/li>\n\n\n\n<li>\u66f4\u65b0\u7cfb\u6570\uff1aw=w\u2212\u03b1\u2202L\u2202w<\/li>\n<\/ol>\n\n\n\n<p>Tip<\/p>\n\n\n\n<p>\u68af\u5ea6\u4e0b\u964d\u6cd5\u53ea\u80fd\u5229\u7528\u4ece<code>NYC<\/code>\u9a7e\u9a76\u5230<code>atlanta<\/code>\u8dd1\u5b8c\u5168\u7a0b\u7684\u65f6\u95f4\u4f5c\u4e3a\u771f\u5b9e\u65f6\u95f4&nbsp;yy&nbsp;\u8fdb\u884c\u8ba1\u7b97\uff0c\u4e0d\u80fd\u5229\u7528\u5176\u4e2d\u4e00\u6bb5\u7684\u65f6\u95f4\u3002\u800cDT\u7b97\u6cd5\u5219\u5bf9\u8fd9\u4e2a\u60c5\u51b5\u8fdb\u884c\u6539\u8fdb\u3002<\/p>\n\n\n\n<h4 class=\"wp-block-heading\">3. \u65f6\u95f4\u5dee\u5206\u7b97\u6cd5<img loading=\"lazy\" decoding=\"async\" height=\"15\" width=\"15\" src=\"blob:https:\/\/eve2333.top\/56f324ef-5687-4b56-bdff-6b041d906f1b\"><\/h4>\n\n\n<div class=\"wp-block-image\">\n<figure class=\"aligncenter size-full is-resized\"><img loading=\"lazy\" decoding=\"async\" width=\"333\" height=\"548\" src=\"https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731125670-\u5c4f\u5e55\u622a\u56fe-2024-11-09-121216.png\" alt=\"\" class=\"wp-image-578\" style=\"width:182px;height:auto\" srcset=\"https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731125670-\u5c4f\u5e55\u622a\u56fe-2024-11-09-121216.png 333w, https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731125670-\u5c4f\u5e55\u622a\u56fe-2024-11-09-121216-182x300.png 182w\" sizes=\"auto, (max-width: 333px) 100vw, 333px\" \/><\/figure>\n<\/div>\n\n\n<ol class=\"wp-block-list\">\n<li>\u6a21\u578b\u4f30\u8ba1\u65f6\u95f4\uff1a&nbsp;q=Q(w)=1000min\uff1b<\/li>\n\n\n\n<li>\u771f\u5b9e\u9a7e\u9a76\u65f6\u95f4\u53ea\u6709\u4ece<code>NYC<\/code>\u5230<code>DC<\/code>\u7684 300 min\uff0c\u5269\u4f59\u7684\u8def\u7a0b\u53ef\u4ee5\u901a\u8fc7\u6a21\u578b\u4f30\u8ba1\uff1ay=300+600=900miny=300+600=900min<\/li>\n\n\n\n<li>\u8ba1\u7b97\u635f\u5931\u503c\uff1aL=12(q\u2212y)2L=21\u200b(q\u2212y)2<\/li>\n\n\n\n<li>\u8ba1\u7b97\u5173\u4e8e\u7cfb\u6570&nbsp;ww&nbsp;\u7684\u68af\u5ea6\uff1a\u2202L\u2202w=\u2202L\u2202q\u2202q\u2202w=(q\u2212y)\u2202Q(w)\u2202w\u2202w\u2202L\u200b=\u2202q\u2202L\u200b\u2202w\u2202q\u200b=(q\u2212y)\u2202w\u2202Q(w)\u200b<\/li>\n\n\n\n<li>\u66f4\u65b0\u7cfb\u6570\uff1aw=w\u2212\u03b1\u2202L\u2202ww=w\u2212\u03b1\u2202w\u2202L\u200b<\/li>\n<\/ol>\n\n\n\n<p>Tip<\/p>\n\n\n\n<ul class=\"wp-block-list\">\n<li><strong>TD\u76ee\u6807<\/strong>\uff1aTD\u7b97\u6cd5\u4e2d\u7684<code>y<\/code><\/li>\n\n\n\n<li><strong>TD\u8bef\u5dee<\/strong>\uff1aD\u7b97\u6cd5\u4e2d\u7684<code>q - y<\/code><\/li>\n<\/ul>\n\n\n\n<h3 class=\"wp-block-heading\" id=\"_22-td\u8bad\u7ec3dqn\">2.2 TD\u8bad\u7ec3DQN<\/h3>\n\n\n\n<p><strong>\u601d\u8def\uff1a<\/strong>&nbsp;\u7531\u4e8eDQN\u7b97\u6cd5\u7684\u76ee\u6807\u662f\u5b9e\u73b0&nbsp;UtUt\u200b&nbsp;\u7684\u6700\u5927\u5316<\/p>\n\n\n\n<p>Ut=Rt+\u03b3Rt+1+\u03b32Rt+2+\u03b33Rt+3+\u22efUt+1=Rt+1+\u03b3Rt+2+\u03b32Rt+3+\u22efUt\u200bUt+1\u200b\u200b=Rt\u200b+\u03b3Rt+1\u200b+\u03b32Rt+2\u200b+\u03b33Rt+3\u200b+\u22ef=Rt+1\u200b+\u03b3Rt+2\u200b+\u03b32Rt+3\u200b+\u22ef\u200b<\/p>\n\n\n\n<p>\u5bf9\u516c\u5f0f\u8fdb\u884c\u7b80\u5316<\/p>\n\n\n\n<p>Ut=Rt+\u03b3Ut+1Ut\u200b\u200b=Rt\u200b+\u03b3Ut+1\u200b\u200b<\/p>\n\n\n\n<p><strong>\u5c06\u4e0a\u8ff0\u516c\u5f0f\u4e0eDT\u7b97\u6cd5\u8fdb\u884c\u5bf9\u7167\uff1a<\/strong><\/p>\n\n\n\n<ol class=\"wp-block-list\">\n<li>UtUt\u200b&nbsp;\u4e0e&nbsp;Ut+1Ut+1\u200b&nbsp;\u5c31\u662f&nbsp;Q(s,a;w)Q(s,a;w)&nbsp;\u6a21\u578b\u7684\u9884\u6d4b\u503c\uff1aQ(st,at;w)Q(st\u200b,at\u200b;w)&nbsp;\u3001Q(st+1,at+1;w)Q(st+1\u200b,at+1\u200b;w)<\/li>\n\n\n\n<li>RtRt\u200b\u5c31\u662f\u4e00\u6b21\u5de5\u51b5\u4e0b\u7684\u90e8\u5206\u771f\u5b9e\u6d4b\u91cf\u503c<\/li>\n\n\n\n<li>TD\u76ee\u6807\u5c31\u4e3a\uff1ayt=Rt+\u03b3Q(st+1,at+1;w)=Rt+\u03b3&nbsp;max&nbsp;Q(st+1,a;w)yt\u200b==\u200bRt\u200b+\u03b3Q(st+1\u200b,at+1\u200b;w)Rt\u200b+\u03b3&nbsp;max&nbsp;Q(st+1\u200b,a;w)\u200b<\/li>\n\n\n\n<li>TD\u8bef\u5dee\u5c31\u4e3a\uff1aQ(st,at;w)\u2212[Rt+\u03b3Q(st+1,at+1;w)]Q(st\u200b,at\u200b;w)\u2212[Rt\u200b+\u03b3Q(st+1\u200b,at+1\u200b;w)]<\/li>\n<\/ol>\n\n\n\n<h1 class=\"wp-block-heading\">\u7b56\u7565\u5b66\u4e60<\/h1>\n\n\n\n<h2 class=\"wp-block-heading\" id=\"_1-\u7b97\u6cd5\u601d\u8def\">1.\u7b97\u6cd5\u601d\u8def<\/h2>\n\n\n\n<ul class=\"wp-block-list\">\n<li><strong>\u76ee\u6807<\/strong>\uff1a\u63a7\u5236\u52a8\u4f5c\u6267\u884c\u7684\u6982\u7387\uff0c\u4f7f\u5f97\u6700\u7ec8\u7684\u300c\u72b6\u6001\u6fc0\u6d3b\u51fd\u6570\u300d\u7684\u7ed3\u679c\u6700\u5927\uff0c\u5373\u52a8\u4f5c\u6267\u884c\u540e\uff0c\u4f7f\u5f97\u5e73\u5747\u5c40\u52bf\u6700\u4f18\uff08&nbsp;ES[V\u03c0(s)]ES\u200b[V\u03c0\u200b(s)]&nbsp;\u503c\u6700\u5927 \uff09\u3002<\/li>\n\n\n\n<li><strong>\u7b56\u7565<\/strong>\uff1a\u5f97\u5230\u300c\u7b56\u7565\u51fd\u6570&nbsp;\u03c0(a\u2223s)\u03c0(a\u2223s)\u300d\uff0c\u8fd9\u6837\u5c31\u80fd\u901a\u8fc7\u5e26\u5165\u72b6\u6001&nbsp;ss\uff0c<strong>\u6839\u636e\u52a8\u4f5c\u7684\u53d1\u751f\u7684\u6982\u7387\u968f\u673a\u9009\u62e9\u52a8\u4f5c<\/strong>\u3002<\/li>\n\n\n\n<li><strong>\u95ee\u9898<\/strong>\uff1a\u8981\u4f7f\u5f97\u5c40\u52bf\u6700\u4f18\uff0c\u5c31\u8981\u77e5\u9053\u6bcf\u79cd\u72b6\u6001\u4e0b\u6700\u4f18\u7684&nbsp;\u03c0(a\u2223s)\u03c0(a\u2223s)&nbsp;\uff0c\u4f46\u662f\u89d2\u8272\u65e0\u6cd5\u8ba4\u77e5\u3002<\/li>\n\n\n\n<li><strong>\u89e3\u51b3<\/strong>\uff1a\u5229\u7528\u4e00\u4e2a\u795e\u7ecf\u7f51\u7edc&nbsp;\u03c0(a\u2223s;\u03b8)\u03c0(a\u2223s;\u03b8)&nbsp;\u6765\u8fd1\u4f3c\u62df\u5408\u51fa \u6bcf\u79cd\u72b6\u6001\u4e0b\u5e94\u5f53\u91c7\u53d6\u7684&nbsp;\u03c0(a\u2223s)\u03c0(a\u2223s)\n<ul class=\"wp-block-list\">\n<li><code>s<\/code>\uff1a\u795e\u7ecf\u7f51\u7edc\u7684\u8f93\u5165<\/li>\n\n\n\n<li><code>a<\/code>\uff1a\u795e\u7ecf\u7f51\u7edc\u7684\u8f93\u51fa<\/li>\n\n\n\n<li>\u03b8\u03b8\uff1a\u795e\u7ecf\u7f51\u7edc\u7684\u53c2\u6570<\/li>\n<\/ul>\n<\/li>\n<\/ul>\n\n\n\n<h2 class=\"wp-block-heading\" id=\"_2-\u795e\u7ecf\u7f51\u7edc\u7684\u6784\u9020\">2.\u795e\u7ecf\u7f51\u7edc\u7684\u6784\u9020<img loading=\"lazy\" decoding=\"async\" height=\"15\" width=\"15\" src=\"blob:https:\/\/eve2333.top\/234f6200-d805-40eb-b11d-f836e6cb536a\"><\/h2>\n\n\n\n<figure class=\"wp-block-image size-full\"><img loading=\"lazy\" decoding=\"async\" width=\"1019\" height=\"236\" src=\"https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731125689-\u5c4f\u5e55\u622a\u56fe-2024-11-09-121225.png\" alt=\"\" class=\"wp-image-579\" srcset=\"https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731125689-\u5c4f\u5e55\u622a\u56fe-2024-11-09-121225.png 1019w, https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731125689-\u5c4f\u5e55\u622a\u56fe-2024-11-09-121225-300x69.png 300w, https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731125689-\u5c4f\u5e55\u622a\u56fe-2024-11-09-121225-768x178.png 768w\" sizes=\"auto, (max-width: 1019px) 100vw, 1019px\" \/><\/figure>\n\n\n\n<ol class=\"wp-block-list\">\n<li>\u4e00\u5f20\u6e38\u620f\u753b\u9762\u5f53\u4f5c\u4e00\u4e2a\u300c\u72b6\u6001\u300d\uff0c\u72b6\u6001\u901a\u8fc7\u300c\u5377\u79ef\u5c42\u300d\u5b9e\u73b0\u7279\u5f81\u63d0\u53d6<\/li>\n\n\n\n<li>\u7279\u5f81\u8f93\u5165\u7ed9\u300c\u6743\u8fde\u63a5\u5c42\u300d\u8fdb\u884c&nbsp;\u03c0(a\u2223s)\u03c0(a\u2223s)&nbsp;\u62df\u5408<\/li>\n\n\n\n<li>\u5c06\u300c\u6743\u8fde\u63a5\u5c42\u300d\u7684\u8f93\u51fa\u901a\u8fc7<code>softmax<\/code>\u5c42\u8f6c\u5316\u4e3a\u6982\u7387<\/li>\n\n\n\n<li>\u8f93\u51fa\u6bcf\u4e2a\u52a8\u4f5c\u5728\u5f53\u524d\u72b6\u6001\u4e0b\u4ea7\u751f\u597d\u7ed3\u679c\u7684\u6982\u7387<\/li>\n<\/ol>\n\n\n\n<h2 class=\"wp-block-heading\" id=\"_3-\u7b56\u7565\u68af\u5ea6\u8bad\u7ec3\">3.\u7b56\u7565\u68af\u5ea6\u8bad\u7ec3<\/h2>\n\n\n\n<ol class=\"wp-block-list\">\n<li><strong>\u76ee\u6807\u51fd\u6570<\/strong>\uff1a V(st;\u03b8)=EA[Q\u03c0(st,A)]=\u2211a\u03c0(a\u2223st;\u03b8)Q\u03c0(st,a)J(\u03b8)=ES[V(S;\u03b8)]V(st\u200b;\u03b8)J(\u03b8)\u200b=EA\u200b[Q\u03c0\u200b(st\u200b,A)]=a\u2211\u200b\u03c0(a\u2223st\u200b;\u03b8)Q\u03c0\u200b(st\u200b,a)=ES\u200b[V(S;\u03b8)]\u200b<\/li>\n\n\n\n<li><strong>\u53c2\u6570\u66f4\u65b0<\/strong>\uff1a&nbsp;\u7531\u4e8e&nbsp;ES[V(S;\u03b8)]ES\u200b[V(S;\u03b8)]&nbsp;\u4e0d\u597d\u8ba1\u7b97\uff0c\u6240\u4ee5\u4f7f\u7528&nbsp;V(S;\u03b8)V(S;\u03b8)&nbsp;\u6765\u8fd1\u4f3c\uff1b\u7531\u4e8e\u6c42\u89e3\u7684\u662f&nbsp;J(\u03b8)J(\u03b8)&nbsp;\u6700\u5927\uff0c\u6240\u4ee5\u4f7f\u7528\u7684\u662f<code>+<\/code> \u03b8=\u03b8+\u03b2\u2202V(s;\u03b8)\u2202\u03b8\u03b8=\u03b8+\u03b2\u2202\u03b8\u2202V(s;\u03b8)\u200b<\/li>\n<\/ol>\n\n\n\n<h2 class=\"wp-block-heading\" id=\"_4-\u7b56\u7565\u68af\u5ea6\u7b97\u6cd5\">4.\u7b56\u7565\u68af\u5ea6\u7b97\u6cd5<\/h2>\n\n\n\n<p><strong>\u5b9a\u4e49\uff1a<\/strong>&nbsp;\u2202V(s;\u03b8)\u2202\u03b8\u2202\u03b8\u2202V(s;\u03b8)\u200b&nbsp;\u5c31\u662f\u7b56\u7565\u68af\u5ea6\u3002<\/p>\n\n\n\n<p><strong>\u8ba1\u7b97\uff1a<\/strong>&nbsp;\u5c06\u7b56\u51fd\u6570\u5173\u4e8e\u03b8\u03b8\u6c42\u5bfc\u5c55\u5f00<\/p>\n\n\n\n<p>\u2202V(s;\u03b8)\u2202\u03b8=\u2202\u2211a\u03c0(a\u2223s;\u03b8)Q\u03c0(s,a)\u2202\u03b8=\u2211a(\u2202\u03c0(a\u2223s;\u03b8)\u2202\u03b8Q\u03c0(s,a)+\u2202Q\u03c0(s,a)\u2202\u03b8\u03c0(a\u2223s;\u03b8))\u2202\u03b8\u2202V(s;\u03b8)\u200b\u200b=\u2202\u03b8\u2202a\u2211\u200b\u03c0(a\u2223s;\u03b8)Q\u03c0\u200b(s,a)\u200b=a\u2211\u200b(\u2202\u03b8\u2202\u03c0(a\u2223s;\u03b8)\u200bQ\u03c0\u200b(s,a)+\u2202\u03b8\u2202Q\u03c0\u200b(s,a)\u200b\u03c0(a\u2223s;\u03b8))\u200b<\/p>\n\n\n\n<p>\u7531\u4e8e&nbsp;Q\u03c0(s,a)Q\u03c0\u200b(s,a)&nbsp;\u6c42\u89e3\u5173\u4e8e&nbsp;\u03b8\u03b8&nbsp;\u7684\u5bfc\u6570\u4e5f\u56f0\u96be\uff0c\u6240\u4ee5\u5ffd\u7565\u6389<\/p>\n\n\n\n<p>\u2202V(s;\u03b8)\u2202\u03b8=\u2211a\u2202\u03c0(a\u2223s;\u03b8)\u2202\u03b8Q\u03c0(s,a)\u2202\u03b8\u2202V(s;\u03b8)\u200b=a\u2211\u200b\u2202\u03b8\u2202\u03c0(a\u2223s;\u03b8)\u200bQ\u03c0\u200b(s,a)<\/p>\n\n\n\n<p>\u5bf9\u4e8e\u4e0a\u8ff0\u516c\u5f0f\u53ea\u80fd\u6c42\u89e3\u79bb\u6563\u5f62\u5f0f\uff0c\u8fde\u7eed\u5f62\u5f0f\u53ef\u4ee5\u53d8\u5f62\u4e3a\uff1a<\/p>\n\n\n\n<p>\u2202V(s;\u03b8)\u2202\u03b8=\u222ba\u03c0(a\u2223s;\u03b8)\u2202ln\u2061\u03c0(a\u2223s;\u03b8)\u2202\u03b8Q\u03c0(s,a)&nbsp;da=EA[\u2202ln\u2061\u03c0(a\u2223s;\u03b8)\u2202\u03b8Q\u03c0(s,a)]\u2202\u03b8\u2202V(s;\u03b8)\u200b\u200b=\u222ba\u200b\u03c0(a\u2223s;\u03b8)\u2202\u03b8\u2202ln\u03c0(a\u2223s;\u03b8)\u200bQ\u03c0\u200b(s,a)&nbsp;da=EA\u200b[\u2202\u03b8\u2202ln\u03c0(a\u2223s;\u03b8)\u200bQ\u03c0\u200b(s,a)]\u200b<\/p>\n\n\n\n<p>\u53ef\u4ee5\u5229\u7528&nbsp;\u8499\u7279\u5361\u6d1b\u79ef\u5206&nbsp;\u5bf9\u4e0a\u8ff0\u671f\u671b\u8fdb\u884c\u8fd1\u4f3c\u8ba1\u7b97\u3002<\/p>\n\n\n\n<ol class=\"wp-block-list\">\n<li>\u6839\u636e\u7b56\u7565\u51fd\u6570&nbsp;\u03c0(a\u2223s;\u03b8)\u03c0(a\u2223s;\u03b8)&nbsp;\u968f\u673a\u83b7\u53d6\u52a8\u4f5c&nbsp;a^a^<\/li>\n\n\n\n<li>\u5e26\u5165&nbsp;a^a^&nbsp;\u8ba1\u7b97\uff1ag(a^,\u03b8)=\u2202ln\u2061\u03c0(a^\u2223s;\u03b8)\u2202\u03b8Q\u03c0(s,a^)g(a^,\u03b8)=\u2202\u03b8\u2202ln\u03c0(a^\u2223s;\u03b8)\u200bQ\u03c0\u200b(s,a^)<\/li>\n\n\n\n<li>\u7136\u540e\u901a\u8fc7&nbsp;g(a^,\u03b8)g(a^,\u03b8)&nbsp;\u6765\u8fd1\u4f3c\u8ba1\u7b97\uff1a\u2202V(s;\u03b8)\u2202\u03b8=EA[g(A,\u03b8)]\u2202\u03b8\u2202V(s;\u03b8)\u200b=EA\u200b[g(A,\u03b8)]<\/li>\n<\/ol>\n\n\n\n<h2 class=\"wp-block-heading\" id=\"_5-\u4ef7\u503c\u51fd\u6570\u8ba1\u7b97\">5.\u4ef7\u503c\u51fd\u6570\u8ba1\u7b97<\/h2>\n\n\n\n<ul class=\"wp-block-list\">\n<li><strong>\u65b9\u6cd5\u4e00\uff1a<\/strong>&nbsp;\u89d2\u8272\u6e38\u73a9\u4e00\u5c40\u6e38\u620f\u540e\uff0c\u518d\u8ba1\u7b97\u4e00\u6b21&nbsp;Q\u03c0(st,at)Q\u03c0\u200b(st\u200b,at\u200b)&nbsp;\uff0c\u7136\u540e\u7528\u8be5\u503c\u6765\u8fd1\u4f3c\u4ee3\u66ff\uff0c\u8fd9\u6837\u5c31\u5bfc\u81f4\u4e00\u5c40\u6e38\u620f\u624d\u80fd\u66f4\u65b0\u4e00\u6b21\u03b8\u03b8\u7cfb\u6570\u3002<\/li>\n\n\n\n<li><strong>\u65b9\u6cd5\u4e8c\uff1a<\/strong>&nbsp;\u5bf9&nbsp;Q\u03c0(st,at)Q\u03c0\u200b(st\u200b,at\u200b)&nbsp;\u518d\u5efa\u7acb\u4e00\u4e2a\u795e\u7ecf\u7f51\u7edc<\/li>\n<\/ul>\n\n\n\n<h1 class=\"wp-block-heading\">TD\u7b97\u6cd5<\/h1>\n\n\n\n<h2 class=\"wp-block-heading\" id=\"_1-sarsa\">1.sarsa<\/h2>\n\n\n\n<h3 class=\"wp-block-heading\" id=\"_11-td\u76ee\u6807\">1.1 TD\u76ee\u6807<\/h3>\n\n\n\n<ul class=\"wp-block-list\">\n<li><strong>\u6298\u6263\u56de\u62a5\uff1a<\/strong>&nbsp;Ut=Rt+\u03b3Ut+1Ut\u200b=Rt\u200b+\u03b3Ut+1\u200b<\/li>\n\n\n\n<li><strong>\u52a8\u4f5c\u4ef7\u503c\uff1a<\/strong>Q\u03c0(st,at)=E[Ut\u2223st,at]=E[Rt+\u03b3Ut+1\u2223st,at]=E[Rt]+\u03b3E[Ut+1\u2223st,at]=E[Rt]+\u03b3E[Q\u03c0(St+1,At+1)\u2223st,at]=E[Rt+\u03b3Q\u03c0(St+1,At+1)]Q\u03c0\u200b(st\u200b,at\u200b)\u200b=E[Ut\u200b\u2223st\u200b,at\u200b]=E[Rt\u200b+\u03b3Ut+1\u200b\u2223st\u200b,at\u200b]=E[Rt\u200b]+\u03b3E[Ut+1\u200b\u2223st\u200b,at\u200b]=E[Rt\u200b]+\u03b3E[Q\u03c0\u200b(St+1\u200b,At+1\u200b)\u2223st\u200b,at\u200b]=E[Rt\u200b+\u03b3Q\u03c0\u200b(St+1\u200b,At+1\u200b)]\u200b<\/li>\n\n\n\n<li><strong>TD\u76ee\u6807\uff1a<\/strong>&nbsp;\u5bf9\u300c\u52a8\u4f5c\u4ef7\u503c\u300d\u8fdb\u884c\u8499\u7279\u5361\u6d1b\u8fd1\u4f3c\uff0c\u5f97&nbsp;yt=rt+\u03b3Q\u03c0(st+1,at+1)yt\u200b=rt\u200b+\u03b3Q\u03c0\u200b(st+1\u200b,at+1\u200b)<\/li>\n<\/ul>\n\n\n\n<h3 class=\"wp-block-heading\" id=\"_12-\u8868\u683c\u5f62\u5f0f\">1.2.\u8868\u683c\u5f62\u5f0f<\/h3>\n\n\n\n<p><strong>\u5f62\u5f0f<\/strong>\uff1a\u5229\u7528\u4e00\u4e2a\u8868\u683c\u6765\u8bb0\u5f55&nbsp;Q\u03c0(s,a)Q\u03c0\u200b(s,a)\uff0c\u884c\u4e3a\u72b6\u6001&nbsp;ss\uff0c\u5217\u4e3a\u52a8\u4f5c&nbsp;aa\u3002<img loading=\"lazy\" decoding=\"async\" height=\"15\" width=\"15\" src=\"blob:https:\/\/eve2333.top\/6c862f69-c179-45b9-98fb-24fcec489778\"><\/p>\n\n\n<div class=\"wp-block-image\">\n<figure class=\"aligncenter size-large is-resized\"><img loading=\"lazy\" decoding=\"async\" width=\"1024\" height=\"292\" src=\"https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731125704-\u5c4f\u5e55\u622a\u56fe-2024-11-09-121231-1024x292.png\" alt=\"\" class=\"wp-image-580\" style=\"width:540px;height:auto\" srcset=\"https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731125704-\u5c4f\u5e55\u622a\u56fe-2024-11-09-121231-1024x292.png 1024w, https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731125704-\u5c4f\u5e55\u622a\u56fe-2024-11-09-121231-300x86.png 300w, https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731125704-\u5c4f\u5e55\u622a\u56fe-2024-11-09-121231-768x219.png 768w, https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731125704-\u5c4f\u5e55\u622a\u56fe-2024-11-09-121231.png 1031w\" sizes=\"auto, (max-width: 1024px) 100vw, 1024px\" \/><\/figure>\n<\/div>\n\n\n<p><strong>\u7b97\u6cd5\uff1a<\/strong><\/p>\n\n\n\n<ol class=\"wp-block-list\">\n<li>\u89c2\u6d4b\u4e00\u6b21\u72b6\u6001\uff1a(st,at,rt,st+1)(st\u200b,at\u200b,rt\u200b,st+1\u200b)<\/li>\n\n\n\n<li>\u6839\u636e\u72b6\u6001&nbsp;st+1st+1\u200b\uff0c\u901a\u8fc7\u7b56\u7565\u51fd\u6570&nbsp;\u03c0(a\u2223s)\u03c0(a\u2223s)&nbsp;\u9884\u6d4b\u4e0b\u4e00\u4e2a\u52a8\u4f5c&nbsp;at+1at+1\u200b<\/li>\n\n\n\n<li>TD\u76ee\u6807\uff1ayt=rt+\u03b3Q\u03c0(st+1,at+1)yt\u200b=rt\u200b+\u03b3Q\u03c0\u200b(st+1\u200b,at+1\u200b)<\/li>\n\n\n\n<li>TD\u8bef\u5dee\uff1a\u03b4t=Q\u03c0(st,at)\u2212yt\u03b4t\u200b=Q\u03c0\u200b(st\u200b,at\u200b)\u2212yt\u200b<\/li>\n\n\n\n<li>\u66f4\u65b0&nbsp;Q\u03c0Q\u03c0\u200b\uff1aQ\u03c0(st,at)=Q\u03c0(st,at)\u2212\u03b1\u03b4tQ\u03c0\u200b(st\u200b,at\u200b)=Q\u03c0\u200b(st\u200b,at\u200b)\u2212\u03b1\u03b4t\u200b<\/li>\n<\/ol>\n\n\n\n<ul class=\"wp-block-list\">\n<li>Sarsa\u6848\u4f8b<\/li>\n<\/ul>\n\n\n\n<h3 class=\"wp-block-heading\" id=\"_13-sarsac194a9eg\">1.3 Sarsa\uff08\u03bb\uff09<\/h3>\n\n\n\n<h4 class=\"wp-block-heading\" id=\"_131-sarsa-\u8868\u683c\u7684\u95ee\u9898\">1.3.1 Sarsa\u8868\u683c\u7684\u95ee\u9898<\/h4>\n\n\n\n<ul class=\"wp-block-list\">\n<li><strong>\u95ee\u9898\uff1a<\/strong>\n<ol class=\"wp-block-list\">\n<li>Sarsa \u7b97\u6cd5\u5bf9\u4e8e&nbsp;Q\u03c0Q\u03c0\u200b&nbsp;\u66f4\u65b0\uff0c\u4e3a\u5355\u6b65\u66f4\u65b0\uff0c\u89d2\u8272\u52a8\u4e00\u6b21\u4fbf\u66f4\u65b0\u4e00\u6b21&nbsp;Q\u03c0Q\u03c0\u200b\uff0c\u8fd9\u5c31\u5bfc\u81f4\u4e00\u56de\u5408\u6e38\u620f\u7ed3\u675f\u540e\uff0c\u56de\u987e\u5bfb\u5b9d\u300c\u8f68\u8ff9\u300d\uff0c\u53d1\u73b0\u5728\u5b9d\u85cf\u9644\u8fd1\u7684\u72b6\u6001\u7684&nbsp;Q\u03c0Q\u03c0\u200b&nbsp;\u624d\u4e0e\u5b9d\u85cf\u5173\u7cfb\u5bc6\u5207\uff0c\u800c\u524d\u5f80\u5b9d\u85cf\u4e4b\u524d\u7684\u8f68\u8ff9\u72b6\u6001\u5219\u8ba4\u4e3a\u4e0e\u5b9d\u85cf\u65e0\u5173\u3002\u8fd9\u5c31\u4f1a\u5bfc\u81f4 Sarsa \u8981\u5728\u540c\u6837\u7684\u8f68\u8ff9\u4e0a\u91cd\u590d\u591a\u6b21\uff0c\u8fdb\u884c&nbsp;Q\u03c0Q\u03c0\u200b&nbsp;\u7684\u66f4\u65b0\uff0c\u624d\u80fd\u5f97\u5230\u8f83\u597d\u7684&nbsp;Q\u03c0Q\u03c0\u200b\uff0c<strong>\u524d\u63d0\u662f\u627e\u5230\u5b9d\u85cf\u7684\u60c5\u51b5\u4e0b\uff0c\u524d\u671f\u6ca1\u6709\u627e\u5230\u65f6\uff0c\u51b3\u7b56\u7684\u968f\u673a\u6570\u5f97\u5927\u4e00\u4e9b<\/strong><\/li>\n\n\n\n<li>\u89d2\u8272\u53ef\u80fd\u5728\u524d\u8fdb\u7684\u8f68\u8ff9\u4e0a\u8f6c\u5708\uff08\u5728\u67d0\u51e0\u4e2a\u72b6\u6001\u4e0a\u6765\u56de\u5207\u6362\uff09<\/li>\n\n\n\n<li> <img loading=\"lazy\" decoding=\"async\" height=\"15\" width=\"15\" src=\"blob:https:\/\/eve2333.top\/60641968-b5cd-4c22-9326-f2465ee3bb41\"><img loading=\"lazy\" decoding=\"async\" width=\"500\" height=\"177\" class=\"wp-image-581\" style=\"width: 500px;\" src=\"https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731125734-\u5c4f\u5e55\u622a\u56fe-2024-11-09-121244.png\" alt=\"\" srcset=\"https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731125734-\u5c4f\u5e55\u622a\u56fe-2024-11-09-121244.png 867w, https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731125734-\u5c4f\u5e55\u622a\u56fe-2024-11-09-121244-300x106.png 300w, https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731125734-\u5c4f\u5e55\u622a\u56fe-2024-11-09-121244-768x272.png 768w\" sizes=\"auto, (max-width: 500px) 100vw, 500px\" \/><\/li>\n<\/ol>\n<\/li>\n\n\n\n<li><strong>\u89e3\u51b3\u601d\u8def\uff1a<\/strong>\n<ol class=\"wp-block-list\">\n<li>\u5229\u7528\u591a\u7ec4\u72b6\u6001\u5bf9&nbsp;Q\u03c0Q\u03c0\u200b&nbsp;\u8fdb\u884c\u66f4\u65b0\uff0c\u589e\u5f3a\u8f68\u8ff9\u4e0a\u5404\u4e2a\u72b6\u6001\u4e0e\u5b9d\u85cf\u7684\u5173\u8054\u6027<\/li>\n\n\n\n<li>\u5bf9\u8f68\u8ff9\u4e0a\u7684\u6bcf\u4e2a\u72b6\u6001\u8fdb\u884c\u300c\u6807\u8bb0\u300d\uff0c\u6807\u8bb0\u503c\u8d8a\u5927\u7684\u72b6\u6001\uff0c\u8bf4\u660e\u8be5\u72b6\u6001\u5bf9\u627e\u5230\u5b9d\u85cf\u7684\u8d21\u732e\u8d8a\u5927\uff0c\u5c3d\u91cf\u907f\u514d\u6ca1\u5fc5\u8981\u7684\u8f6c\u5708<\/li>\n<\/ol>\n<\/li>\n<\/ul>\n\n\n\n<h4 class=\"wp-block-heading\" id=\"_132-sarsac194a9eg-\u7684\u5b9e\u73b0\">1.3.2 Sarsa\uff08\u03bb\uff09\u7684\u5b9e\u73b0<\/h4>\n\n\n<div class=\"wp-block-image\">\n<figure class=\"aligncenter size-large is-resized\"><img loading=\"lazy\" decoding=\"async\" width=\"1024\" height=\"512\" src=\"https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731125767-\u5c4f\u5e55\u622a\u56fe-2024-11-09-121255-1024x512.png\" alt=\"\" class=\"wp-image-582\" style=\"width:488px;height:auto\" srcset=\"https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731125767-\u5c4f\u5e55\u622a\u56fe-2024-11-09-121255-1024x512.png 1024w, https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731125767-\u5c4f\u5e55\u622a\u56fe-2024-11-09-121255-300x150.png 300w, https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731125767-\u5c4f\u5e55\u622a\u56fe-2024-11-09-121255-768x384.png 768w, https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731125767-\u5c4f\u5e55\u622a\u56fe-2024-11-09-121255.png 1029w\" sizes=\"auto, (max-width: 1024px) 100vw, 1024px\" \/><\/figure>\n<\/div>\n\n\n<ul class=\"wp-block-list\">\n<li><strong>\u72b6\u6001\u4e2a\u6570\uff1a<\/strong>&nbsp;\u5229\u7528\u591a\u5c11\u4e2a\u72b6\u6001\u5bf9&nbsp;Q\u03c0Q\u03c0\u200b&nbsp;\u8fdb\u884c\u66f4\u65b0\uff0c\u8fd9\u91cc\u7684\u53d6\u503c\u8303\u56f4\u4e3a&nbsp;<code>1 ~ n<\/code>\uff08n\uff1a\u8868\u793a\u4e00\u56de\u5408\u7ed3\u675f\u65f6\uff0c\u8f68\u8ff9\u6240\u5305\u542b\u72b6\u6001\u7684\u4e2a\u6570\uff09\uff0c\u4f46\u662f\u7531\u4e8e\u6211\u4eec\u4e0d\u4f1a\u771f\u7684\u4e00\u56de\u5408\u624d\u8fdb\u884c\u66f4\u65b0\uff0c\u6240\u4ee5\u5e76\u4e0d\u77e5\u9053\u4e00\u56de\u5408\u4f1a\u6709\u591a\u5c11\u72b6\u6001\u3002<strong>\u56e0\u6b64\uff0c\u53ef\u4ee5\u5c06\u72b6\u6001\u4e2a\u6570\u6620\u5c04\u5230 [0,1]\uff0c0\uff1a\u8868\u793a<code>1<\/code>\u4e2a\u72b6\u6001\uff1b1\uff1a\u8868\u793a<code>n<\/code>\u4e2a\u72b6\u6001\uff0c\u800c\u6211\u4eec\u7528\u4e8eSarsa\u66f4\u65b0\uff0c\u4f7f\u7528\u7684\u72b6\u6001\u4e2a\u6570\u4e3a&nbsp;\u03bb\u2208[0,1]\u03bb\u2208[0,1]\uff0c\u6240\u4ee5\u8be5\u7b97\u6cd5\u5c31\u79f0\u4e4b\u4e3a Sarsa(\u03bb\u03bb)<\/strong><\/li>\n\n\n\n<li><strong>\u72b6\u6001\u6807\u8bb0\uff1a<\/strong>\n<ol class=\"wp-block-list\">\n<li>\u4e00\u56de\u5408\u4e2d\uff0c\u6bcf\u4e2a\u88ab\u7ecf\u8fc7\u7684\u72b6\u6001\u7684\u6807\u8bb0\u503c\u90fd\u4f1a\u88ab\u8bb0\u5f55\uff0c\u653e\u5230\u4e00\u4e2a<code>eligibility_trace<\/code>\u8868\u4e2d<\/li>\n\n\n\n<li>\u6bcf\u4e2a\u72b6\u6001\u88ab\u7ecf\u8fc7\u4e00\u6b21\uff0c\u5c06\u4f1a\u88ab\u6807\u8bb0\uff0c\u6807\u8bb0\u4f1a\u968f\u65f6\u95f4\u8870\u51cf <img loading=\"lazy\" decoding=\"async\" width=\"400\" height=\"231\" class=\"wp-image-583\" style=\"width: 400px;\" src=\"https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731125960-\u5c4f\u5e55\u622a\u56fe-2024-11-09-121714.png\" alt=\"\" srcset=\"https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731125960-\u5c4f\u5e55\u622a\u56fe-2024-11-09-121714.png 640w, https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731125960-\u5c4f\u5e55\u622a\u56fe-2024-11-09-121714-300x173.png 300w\" sizes=\"auto, (max-width: 400px) 100vw, 400px\" \/><img loading=\"lazy\" decoding=\"async\" height=\"15\" width=\"15\" src=\"blob:https:\/\/eve2333.top\/2d8becc1-bcb6-4813-8a74-ead774f93fe2\"><\/li>\n\n\n\n<li>\u5f53\u4e00\u4e2a\u72b6\u6001\u88ab\u591a\u6b21\u7ecf\u8fc7\u65f6\uff0c\u5bf9\u4e8e\u6807\u8bb0\u503c\u7684\u5904\u7406\u65b9\u5f0f\u6709\u4e24\u79cd\n<ul class=\"wp-block-list\">\n<li>\u6807\u8bb0\u503c\u76f4\u63a5\u53e0\u52a0<\/li>\n\n\n\n<li>\u6807\u8bb0\u503c\u9650\u5e45<br> <img loading=\"lazy\" decoding=\"async\" width=\"400\" height=\"203\" class=\"wp-image-584\" style=\"width: 400px;\" src=\"https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731125987-\u5c4f\u5e55\u622a\u56fe-2024-11-09-121718.png\" alt=\"\" srcset=\"https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731125987-\u5c4f\u5e55\u622a\u56fe-2024-11-09-121718.png 836w, https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731125987-\u5c4f\u5e55\u622a\u56fe-2024-11-09-121718-300x153.png 300w, https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731125987-\u5c4f\u5e55\u622a\u56fe-2024-11-09-121718-768x390.png 768w\" sizes=\"auto, (max-width: 400px) 100vw, 400px\" \/><img loading=\"lazy\" decoding=\"async\" height=\"15\" width=\"15\" src=\"blob:https:\/\/eve2333.top\/e1c82603-86c3-463b-8ba6-fe90a9ce9ed9\"><\/li>\n<\/ul>\n<\/li>\n<\/ol>\n<\/li>\n\n\n\n<li><strong>\u7b97\u6cd5\u6d41\u7a0b<\/strong>\uff1a\u6bcf\u4e00\u8f6e\u6e38\u620f\u7684\u5f00\u59cb\u524d\uff0c\u90fd\u4f1a\u5c06\u8f68\u8ff9\u8868\u5f52\u96f6\uff0cE=0E=0\n<ol class=\"wp-block-list\">\n<li>\u89c2\u6d4b\u4e00\u6b21\u72b6\u6001\uff1a(st,at,rt,st+1)(st\u200b,at\u200b,rt\u200b,st+1\u200b)<\/li>\n\n\n\n<li>\u6839\u636e\u72b6\u6001&nbsp;st+1st+1\u200b\uff0c\u901a\u8fc7\u7b56\u7565\u51fd\u6570&nbsp;\u03c0(a\u2223s)\u03c0(a\u2223s)&nbsp;\u9884\u6d4b\u4e0b\u4e00\u4e2a\u52a8\u4f5c&nbsp;at+1at+1\u200b<\/li>\n\n\n\n<li>TD\u76ee\u6807\uff1ayt=rt+\u03b3Q\u03c0(st+1,at+1)yt\u200b=rt\u200b+\u03b3Q\u03c0\u200b(st+1\u200b,at+1\u200b)<\/li>\n\n\n\n<li>TD\u8bef\u5dee\uff1a\u03b4t=Q\u03c0(st,at)\u2212yt\u03b4t\u200b=Q\u03c0\u200b(st\u200b,at\u200b)\u2212yt\u200b<\/li>\n\n\n\n<li>\u66f4\u65b0\u7684\u8f68\u8ff9\u8868EE\uff1a\n<ul class=\"wp-block-list\">\n<li>\u65b9\u5f0f\u4e00\uff1aE(st,at)=E(st,at)+1E(st\u200b,at\u200b)=E(st\u200b,at\u200b)+1<\/li>\n\n\n\n<li>\u65b9\u5f0f\u4e8c\uff1aE(st,:)=0;&nbsp;E(st,at)=1E(st\u200b,:)=0;&nbsp;E(st\u200b,at\u200b)=1<\/li>\n<\/ul>\n<\/li>\n\n\n\n<li>\u66f4\u65b0&nbsp;Q\u03c0Q\u03c0\u200b&nbsp;\u5168\u8868\uff1aQ\u03c0=Q\u03c0\u2212\u03b1\u03b4tEQ\u03c0\u200b=Q\u03c0\u200b\u2212\u03b1\u03b4t\u200bE<\/li>\n\n\n\n<li>\u5b9e\u73b0&nbsp;EE&nbsp;\u5168\u8868\u4e2d\u6807\u8bb0\u503c\u7684\u8870\u51cf\uff1aE=\u03b3\u03bbEE=\u03b3\u03bbE<\/li>\n<\/ol>\n<\/li>\n\n\n\n<li>Sarsa\uff08\u03bb\uff09\u6848\u4f8b<\/li>\n<\/ul>\n\n\n\n<h4 class=\"wp-block-heading\" id=\"_133-sarsac194a9eg-\u89e3\u91ca\">1.3.3 Sarsa\uff08\u03bb\uff09\u7684\u89e3\u91ca<img loading=\"lazy\" decoding=\"async\" height=\"15\" width=\"15\" src=\"blob:https:\/\/eve2333.top\/01e6e4e8-95f4-4536-bfb9-95f776b4ec62\"><\/h4>\n\n\n<div class=\"wp-block-image\">\n<figure class=\"aligncenter size-full is-resized\"><img loading=\"lazy\" decoding=\"async\" width=\"989\" height=\"234\" src=\"https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731126016-\u5c4f\u5e55\u622a\u56fe-2024-11-09-121722.png\" alt=\"\" class=\"wp-image-585\" style=\"width:604px;height:auto\" srcset=\"https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731126016-\u5c4f\u5e55\u622a\u56fe-2024-11-09-121722.png 989w, https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731126016-\u5c4f\u5e55\u622a\u56fe-2024-11-09-121722-300x71.png 300w, https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731126016-\u5c4f\u5e55\u622a\u56fe-2024-11-09-121722-768x182.png 768w\" sizes=\"auto, (max-width: 989px) 100vw, 989px\" \/><\/figure>\n<\/div>\n\n\n<ul class=\"wp-block-list\">\n<li><strong><code>e4<\/code>\u65f6\u523b\u7684Q\u03c0Q\u03c0\u200b&nbsp;\u66f4\u65b0\uff1a<\/strong> \u5728<code>e4<\/code>\u65f6\u523b\uff0c\u5bf9\u4e8e&nbsp;Q(e4)Q(e4)&nbsp;\u7684TD\u8bef\u5dee\u4e3a \u03b4q4=Q(e4)\u2212r4\u2212\u03b3Q(e5)\u03b4q4\u200b=Q(e4)\u2212r4\u200b\u2212\u03b3Q(e5) \u5e76\u5bf9&nbsp;Q(e4)Q(e4)&nbsp;\u8fdb\u884c\u4fee\u6b63 Q(e4)\u2032=Q(e4)\u2212\u03b1\u03b4q4Q(e4)\u2032=Q(e4)\u2212\u03b1\u03b4q4\u200b \u5728<code>e3<\/code>\u65f6\u523b\uff0c\u5bf9\u4e8e&nbsp;Q(e3)Q(e3)&nbsp;\u7684TD\u8bef\u5dee\u4e3a \u03b4q3=Q(e3)\u2212r3\u2212\u03b3Q(e4)\u03b4q3\u200b=Q(e3)\u2212r3\u200b\u2212\u03b3Q(e4) \u5728<code>e3<\/code>\u65f6\uff0c\u6211\u4eec\u5e76\u4e0d\u77e5\u9053<code>Q(e4)<\/code>\u5177\u6709\u8bef\u5dee\u7684\u60c5\u51b5\u4e0b\uff0c\u5bf9Q(e3)Q(e3)\u8fdb\u884c\u4e86\u4fee\u6b63 Q(e3)\u2032=Q(e3)\u2212\u03b1\u03b4q3Q(e3)\u2032=Q(e3)\u2212\u03b1\u03b4q3\u200b \u5728<code>e4<\/code>\u65f6\u523b\uff0c\u6211\u4eec\u5f97\u5230\u4e86\u66f4\u51c6\u786e\u7684&nbsp;Q(e4)\u2032Q(e4)\u2032&nbsp;\uff0c\u56de\u4ee3\u4fee\u6b63&nbsp;Q(e3)Q(e3) \u03b4q3\u2032=Q(e3)\u2212r3\u2212\u03b3Q(e4)\u2032=Q(e3)\u2212r3\u2212\u03b3[Q(e4)\u2212\u03b1\u03b4q4]=Q(e3)\u2212r3\u2212\u03b3Q(e4)+\u03b3\u03b1\u03b4q4=\u03b4q3+\u03b3\u03b4q4\u03b4q3\u2032\u200b\u200b=Q(e3)\u2212r3\u200b\u2212\u03b3Q(e4)\u2032=Q(e3)\u2212r3\u200b\u2212\u03b3[Q(e4)\u2212\u03b1\u03b4q4\u200b]=Q(e3)\u2212r3\u200b\u2212\u03b3Q(e4)+\u03b3\u03b1\u03b4q4\u200b=\u03b4q3\u200b+\u03b3\u03b4q4\u200b\u200b \u91cd\u65b0\u5bf9Q(e3)Q(e3)\u8fdb\u884c\u4fee\u6b63 Q(e3)\u2032\u2032=Q(e3)\u2212\u03b1\u03b4q3\u2032=Q(e3)\u2212\u03b1\u03b4q3\u2212\u03b1\u03b3\u03b4q4=Q(e3)\u2032\u2212\u03b1\u03b4q4\u03b3Q(e3)\u2032\u2032\u200b=Q(e3)\u2212\u03b1\u03b4q3\u2032\u200b=Q(e3)\u2212\u03b1\u03b4q3\u200b\u2212\u03b1\u03b3\u03b4q4\u200b=Q(e3)\u2032\u2212\u03b1\u03b4q4\u200b\u03b3\u200b \u6700\u7ec8\u5f97\u5230\u7684\u516c\u5f0f\u5c31\u4e0e\u8ba1\u7b97\u6d41\u7a0b\u7684\u66f4\u65b0\u516c\u5f0f\u6240\u5bf9\u5e94\uff1a Q(e3)\u2032\u2032=Q(e3)\u2032\u2212\u03b1\u03b4q4\u03b3Q\u03c0=Q\u03c0\u2212\u03b1\u03b4tEQ(e3)\u2032\u2032Q\u03c0\u200b\u200b=Q(e3)\u2032\u2212\u03b1\u03b4q4\u200b\u03b3=Q\u03c0\u200b\u2212\u03b1\u03b4t\u200bE\u200b \u5176\u4ed6\u72b6\u6001\u540c\u7406\uff0c\u5c31\u662f\u4e00\u5c42\u4e00\u5c42\u7684\u5957\u5a03\u4e0b\u53bb\u3002<\/li>\n\n\n\n<li><strong>Q\u03c0Q\u03c0\u200b&nbsp;\u66f4\u65b0\uff1a<\/strong>&nbsp;\u5bf9\u4e8e\u4e0a\u8ff0\u7684 Sarsa(\u03bb\u03bb) \u7b97\u6cd5\uff0c\u5e76\u975e\u662f\u4e00\u8f6e\u6e38\u620f\u7ed3\u675f\u540e\uff0c\u624d\u5bf9&nbsp;Q\u03c0Q\u03c0\u200b&nbsp;\u8868\u8fdb\u66f4\u65b0\u3002\u4ece\u7b97\u6cd5\u7684\u8ba1\u7b97\u6d41\u7a0b\u53ef\u4ee5\u770b\u51fa&nbsp;Q\u03c0Q\u03c0\u200b&nbsp;\u8868\u5728\u6bcf\u4e00\u6b21\u52a8\u4f5c\u540e\uff0c\u90fd\u4f1a\u66f4\u65b0\u4e00\u6b21\uff0c<strong>\u4e5f\u5c31\u662f\u8bf4\uff0c\u89d2\u8272\u6bcf\u8f6c\u79fb\u4e00\u6b21\u72b6\u6001\uff0c\u5c31\u4f1a\u505c\u4e0b\u6765\uff0c\u5bf9\u4e4b\u524d\u7684\u8f68\u8ff9\u8fdb\u884c\u4e00\u6b21\u56de\u987e\uff0c\u4fee\u6b63\u4e4b\u524d\u6240\u7ecf\u5386\u7684\u72b6\u6001\u5230\u5f53\u524d\u72b6\u6001&nbsp;stst\u200b&nbsp;\u7684\u5173\u8054\u6027\u3002<\/strong><\/li>\n\n\n\n<li><strong>\u72b6\u6001\u4e2a\u6570\uff1a<\/strong>&nbsp;\u5728<code>e4<\/code>\u65f6\u523b\uff0c\u5bf9\u4e8e\u8f68\u8ff9\u72b6\u6001<code>e1<\/code>\u7684\u52a8\u4f5c\u4ef7\u503c&nbsp;Q\u03c0Q\u03c0\u200b&nbsp;\u66f4\u65b0\uff0c\u6d89\u53ca\u7684\u72b6\u6001\u5c31\u6709<code>e2 - e4<\/code>\uff1b\u5bf9\u4e8e\u8f68\u8ff9\u72b6\u6001<code>e2<\/code>\u800c\u8a00\uff0c\u5c31\u662f<code>e3 - e4<\/code><\/li>\n\n\n\n<li><strong>\u03bb\u03bb&nbsp;\u5b9e\u73b0\u72b6\u6001\u4e2a\u6570\u63a7\u5236\uff1a<\/strong>\n<ul class=\"wp-block-list\">\n<li>\u03bb=0\u03bb=0\uff1a\u8fd9\u5c31\u5bfc\u81f4\u4e00\u4e0b\u6b21\u72b6\u6001\u7684\u5f00\u59cb\uff0cE=0E=0\uff0c\u7136\u540e\u8f68\u8ff9\u8868\u53ea\u6709\u5f53\u524d\u72b6\u6001\u624d\u6709\u8bb0\u5f55\uff1aE(st,at)=1E(st\u200b,at\u200b)=1\uff0c\u56e0\u6b64&nbsp;Q\u03c0=Q\u03c0\u2212\u03b1\u03b4tEQ\u03c0\u200b=Q\u03c0\u200b\u2212\u03b1\u03b4t\u200bE&nbsp;\u5b9e\u9645\u53ea\u66f4\u65b0\u4e86\u5f53\u524d\u72b6\u6001\u7684&nbsp;Q\u03c0(st,at)Q\u03c0\u200b(st\u200b,at\u200b)\u3002<\/li>\n\n\n\n<li>\u03bb=1\u03bb=1\uff1aQ\u03c0=Q\u03c0\u2212\u03b1\u03b4tEQ\u03c0\u200b=Q\u03c0\u200b\u2212\u03b1\u03b4t\u200bE&nbsp;\u66f4\u65b0\u4e86&nbsp;<strong>\u4ece\u56de\u5408\u5f00\u59cb\u72b6\u6001&nbsp;s0s0\u200b&nbsp;\u5230\u5f53\u524d\u72b6\u6001&nbsp;stst\u200b&nbsp;\u5bf9\u5e94\u7684\u6240\u6709&nbsp;Q\u03c0Q\u03c0\u200b\u503c<\/strong><\/li>\n\n\n\n<li>\u03bb\u2208(0,1)\u03bb\u2208(0,1)\uff1a\u7531\u4e8e&nbsp;E=\u03b3\u03bbEE=\u03b3\u03bbE&nbsp;\u7684\u53cd\u590d\u8fed\u4ee3\uff0c\u5b9e\u73b0\u4e86\u8f68\u8ff9\u6807\u8bb0\u503c\u7684\u8870\u51cf\uff0c\u79bb\u5f53\u524d\u72b6\u6001\u8d8a\u8fd1\u7684\u72b6\u6001\uff0c\u5176\u6807\u8bb0\u503c\u8d8a\u5927\uff0c\u4f8b\u5982\u5230\u8fbe<code>e4<\/code>\u65f6\u523b\u65f6\uff0c&nbsp;<code>e4 = 1<\/code>\uff1b<code>e3 = 0.8<\/code>\uff1b<code>e2 = 0.64<\/code>\uff0c\u65f6\u95f4\u8d8a\u9760\u524d\uff0c<code>e<\/code>&nbsp;\u503c\u5c31\u8d8a\u5c0f\uff0c\u5bf9<code>e<\/code>\u503c\u518d\u4e58\u4ee5\u4e00\u4e2a\u500d\u6570\u503c&nbsp;\u03bb\u03bb\uff0c\u5c31\u53ef\u4ee5\u5c06\u539f\u6765\u5f53\u524d\u65f6\u523b\u7684<code>e<\/code>\u503c\u53d8\u5f97\u66f4\u5c0f\uff0c\u51e0\u4e4e\u5c31\u7b49\u4e8e<code>0<\/code>\u4e86\uff0c\u8fd9\u6837\u5c31\u5b9e\u73b0\u4e86\u7c7b\u4f3c\u4e8e&nbsp;\u03bb=0\u03bb=0&nbsp;\u65f6\uff0c\u72b6\u6001\u7684&nbsp;Q\u03c0Q\u03c0\u200b&nbsp;\u4e0d\u66f4\u65b0\u3002<strong>\u5373&nbsp;\u03bb\u03bb&nbsp;\u4f7f\u5f97\u6bd4\u8f83\u65e7\u7684&nbsp;QpiQpi\u200b&nbsp;\u4e0d\u518d\u53c2\u4e0e\u5230\u66f4\u65b0\uff0c\u4e5f\u5c31\u63a7\u5236\u4e86\u300c\u72b6\u6001\u4e2a\u6570\u300d<\/strong><\/li>\n<\/ul>\n<\/li>\n<\/ul>\n\n\n\n<h3 class=\"wp-block-heading\" id=\"_14-\u795e\u7ecf\u7f51\u7edc\u5f62\u5f0f\">1.4\u795e\u7ecf\u7f51\u7edc\u4ef7\u503c<\/h3>\n\n\n\n<p><strong>\u601d\u8def\uff1a<\/strong>&nbsp;\u5229\u7528\u795e\u7ecf\u7f51\u7edc&nbsp;q(s,a;w)q(s,a;w)&nbsp;\u6765\u8fd1\u4f3c\u52a8\u4f5c\u4ef7\u503c\u51fd\u6570&nbsp;Q\u03c0(s,a)Q\u03c0\u200b(s,a)\u3002&nbsp;\u8f93\u5165\u4e3a\u72b6\u6001\uff0c\u8f93\u51fa\u4e3a\u5404\u4e2a\u52a8\u4f5c\u5bf9\u5e94\u7684\u4ef7\u503c<img loading=\"lazy\" decoding=\"async\" height=\"15\" width=\"15\" src=\"blob:https:\/\/eve2333.top\/1e053277-c46f-4bc9-84ad-431330d03d25\"><\/p>\n\n\n<div class=\"wp-block-image\">\n<figure class=\"aligncenter size-large is-resized\"><img loading=\"lazy\" decoding=\"async\" width=\"1024\" height=\"300\" src=\"https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731126034-\u5c4f\u5e55\u622a\u56fe-2024-11-09-121729-1024x300.png\" alt=\"\" class=\"wp-image-586\" style=\"width:567px;height:auto\" srcset=\"https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731126034-\u5c4f\u5e55\u622a\u56fe-2024-11-09-121729-1024x300.png 1024w, https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731126034-\u5c4f\u5e55\u622a\u56fe-2024-11-09-121729-300x88.png 300w, https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731126034-\u5c4f\u5e55\u622a\u56fe-2024-11-09-121729-768x225.png 768w, https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731126034-\u5c4f\u5e55\u622a\u56fe-2024-11-09-121729.png 1028w\" sizes=\"auto, (max-width: 1024px) 100vw, 1024px\" \/><\/figure>\n<\/div>\n\n\n<p><strong>\u7b97\u6cd5\uff1a<\/strong><\/p>\n\n\n\n<ol class=\"wp-block-list\">\n<li>\u89c2\u6d4b\u4e00\u6b21\u72b6\u6001\uff1a(st,at,rt,st+1)(st\u200b,at\u200b,rt\u200b,st+1\u200b)<\/li>\n\n\n\n<li>\u6839\u636e\u72b6\u6001&nbsp;st+1st+1\u200b\uff0c\u901a\u8fc7\u7b56\u7565\u51fd\u6570&nbsp;\u03c0(a\u2223s)\u03c0(a\u2223s)&nbsp;\u9884\u6d4b\u4e0b\u4e00\u4e2a\u52a8\u4f5c&nbsp;at+1at+1\u200b<\/li>\n\n\n\n<li>TD\u76ee\u6807\uff1ayt=rt+\u03b3q(st+1,at+1;w)yt\u200b=rt\u200b+\u03b3q(st+1\u200b,at+1\u200b;w)<\/li>\n\n\n\n<li>TD\u8bef\u5dee\uff1a\u03b4t=q(st,at;w)\u2212yt\u03b4t\u200b=q(st\u200b,at\u200b;w)\u2212yt\u200b<\/li>\n\n\n\n<li>\u635f\u5931\u51fd\u6570\uff1aL=12\u03b4t2L=21\u200b\u03b4t2\u200b<\/li>\n\n\n\n<li>\u66f4\u65b0\u7cfb\u6570\uff1aw=w\u2212\u03b1\u03b4t\u2202q(st,at;w)\u2202ww=w\u2212\u03b1\u03b4t\u200b\u2202w\u2202q(st\u200b,at\u200b;w)\u200b<\/li>\n<\/ol>\n\n\n\n<h2 class=\"wp-block-heading\" id=\"_2-q-learning\">2.Q Learning<\/h2>\n\n\n\n<h3 class=\"wp-block-heading\" id=\"_21-td\u76ee\u6807\">2.1 TD\u76ee\u6807<\/h3>\n\n\n\n<ul class=\"wp-block-list\">\n<li><strong>\u6700\u4f18\u52a8\u4f5c\u4ef7\u503c\uff1a<\/strong>Q\u03c0(st,at)=E[Rt+\u03b3Q\u03c0(St+1,At+1)]Q\u2217(st,at)=E[Rt+\u03b3Q\u2217(St+1,At+1)]=E[Rt+\u03b3max\u2061aQ\u2217(St+1,a)]Q\u03c0\u200b(st\u200b,at\u200b)Q\u2217(st\u200b,at\u200b)\u200b=E[Rt\u200b+\u03b3Q\u03c0\u200b(St+1\u200b,At+1\u200b)]=E[Rt\u200b+\u03b3Q\u2217(St+1\u200b,At+1\u200b)]=E[Rt\u200b+\u03b3amax\u200bQ\u2217(St+1\u200b,a)]\u200b<\/li>\n\n\n\n<li><strong>TD\u76ee\u6807\uff1a<\/strong>&nbsp;\u5bf9\u300c\u6700\u4f18\u52a8\u4f5c\u4ef7\u503c\u300d\u8fdb\u884c\u8499\u7279\u5361\u6d1b\u8fd1\u4f3c\uff0c\u5f97&nbsp;yt=rt+\u03b3max\u2061aQ\u2217(st+1,a)yt\u200b=rt\u200b+\u03b3amax\u200bQ\u2217(st+1\u200b,a)<\/li>\n<\/ul>\n\n\n\n<h3 class=\"wp-block-heading\">2.2\u8868\u683c\u5f62\u5f0f<\/h3>\n\n\n\n<p><strong>\u5f62\u5f0f<\/strong>\uff1a\u5229\u7528\u4e00\u4e2a\u8868\u683c\u6765\u8bb0\u5f55&nbsp;Q\u2217(s,a)Q\u2217(s,a)\uff0c\u884c\u4e3a\u72b6\u6001&nbsp;ss\uff0c\u5217\u4e3a\u52a8\u4f5c&nbsp;aa\u3002<img loading=\"lazy\" decoding=\"async\" height=\"15\" width=\"15\" src=\"blob:https:\/\/eve2333.top\/60bb66ff-576f-4cc4-b943-782d7f6aa9e0\"><\/p>\n\n\n\n<figure class=\"wp-block-image size-large\"><img loading=\"lazy\" decoding=\"async\" width=\"1024\" height=\"304\" src=\"https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731126056-\u5c4f\u5e55\u622a\u56fe-2024-11-09-121737-1024x304.png\" alt=\"\" class=\"wp-image-587\" srcset=\"https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731126056-\u5c4f\u5e55\u622a\u56fe-2024-11-09-121737-1024x304.png 1024w, https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731126056-\u5c4f\u5e55\u622a\u56fe-2024-11-09-121737-300x89.png 300w, https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731126056-\u5c4f\u5e55\u622a\u56fe-2024-11-09-121737-768x228.png 768w, https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731126056-\u5c4f\u5e55\u622a\u56fe-2024-11-09-121737.png 1052w\" sizes=\"auto, (max-width: 1024px) 100vw, 1024px\" \/><\/figure>\n\n\n\n<ol class=\"wp-block-list\">\n<li>\u89c2\u6d4b\u4e00\u6b21\u72b6\u6001\uff1a(st,at,rt,st+1)(st\u200b,at\u200b,rt\u200b,st+1\u200b)<\/li>\n\n\n\n<li>TD\u76ee\u6807\uff1ayt=rt+\u03b3max\u2061aQ\u2217(st+1,a)yt\u200b=rt\u200b+\u03b3amax\u200bQ\u2217(st+1\u200b,a)<\/li>\n\n\n\n<li>TD\u8bef\u5dee\uff1a\u03b4t=Q\u2217(st,at)\u2212yt\u03b4t\u200b=Q\u2217(st\u200b,at\u200b)\u2212yt\u200b<\/li>\n\n\n\n<li>\u66f4\u65b0&nbsp;Q\u03c0Q\u03c0\u200b\uff1aQ\u2217(st,at)=Q\u2217(st,at)\u2212\u03b1\u03b4tQ\u2217(st\u200b,at\u200b)=Q\u2217(st\u200b,at\u200b)\u2212\u03b1\u03b4t\u200b<\/li>\n<\/ol>\n\n\n\n<ul class=\"wp-block-list\">\n<li>Q learning \u6848\u4f8b<\/li>\n<\/ul>\n\n\n\n<h3 class=\"wp-block-heading\" id=\"_23-\u795e\u7ecf\u7f51\u7edc\u5f62\u5f0f\">2.3\u795e\u7ecf\u7f51\u7edc\u5f62\u5f0f<\/h3>\n\n\n\n<p><strong>\u601d\u8def\uff1a<\/strong>&nbsp;\u5229\u7528\u795e\u7ecf\u7f51\u7edc&nbsp;Q(s,a;w)Q(s,a;w)&nbsp;\u6765\u8fd1\u4f3c\u52a8\u4f5c\u4ef7\u503c\u51fd\u6570&nbsp;Q\u2217(s,a)Q\u2217(s,a)\u3002<img loading=\"lazy\" decoding=\"async\" height=\"15\" width=\"15\" src=\"blob:https:\/\/eve2333.top\/59eaa00f-75c2-42ef-ba61-0665e4d56c94\"><\/p>\n\n\n<div class=\"wp-block-image\">\n<figure class=\"aligncenter size-large is-resized\"><img loading=\"lazy\" decoding=\"async\" width=\"1024\" height=\"317\" src=\"https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731126182-\u5c4f\u5e55\u622a\u56fe-2024-11-09-121741-1024x317.png\" alt=\"\" class=\"wp-image-588\" style=\"width:517px;height:auto\" srcset=\"https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731126182-\u5c4f\u5e55\u622a\u56fe-2024-11-09-121741-1024x317.png 1024w, https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731126182-\u5c4f\u5e55\u622a\u56fe-2024-11-09-121741-300x93.png 300w, https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731126182-\u5c4f\u5e55\u622a\u56fe-2024-11-09-121741-768x238.png 768w, https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731126182-\u5c4f\u5e55\u622a\u56fe-2024-11-09-121741.png 1027w\" sizes=\"auto, (max-width: 1024px) 100vw, 1024px\" \/><\/figure>\n<\/div>\n\n\n<p><strong>\u7b97\u6cd5\uff1a<\/strong><\/p>\n\n\n\n<ol class=\"wp-block-list\">\n<li>\u89c2\u6d4b\u4e00\u6b21\u72b6\u6001\uff1a(st,at,rt,st+1)(st\u200b,at\u200b,rt\u200b,st+1\u200b)<\/li>\n\n\n\n<li>TD\u76ee\u6807\uff1ayt=rt+\u03b3max\u2061aQ(st+1,a;w)yt\u200b=rt\u200b+\u03b3amax\u200bQ(st+1\u200b,a;w)<\/li>\n\n\n\n<li>TD\u8bef\u5dee\uff1a\u03b4t=Q(st,at;w)\u2212yt\u03b4t\u200b=Q(st\u200b,at\u200b;w)\u2212yt\u200b<\/li>\n\n\n\n<li>\u635f\u5931\u51fd\u6570\uff1aL=12\u03b4t2L=21\u200b\u03b4t2\u200b<\/li>\n\n\n\n<li>\u66f4\u65b0\u7cfb\u6570\uff1aw=w\u2212\u03b1\u03b4t\u2202q(st,at;w)\u2202ww=w\u2212\u03b1\u03b4t\u200b\u2202w\u2202q(st\u200b,at\u200b;w)\u200b<\/li>\n<\/ol>\n\n\n\n<p>Note<\/p>\n\n\n\n<ul class=\"wp-block-list\">\n<li><strong>sarsa\uff1a<\/strong>&nbsp;\u8fd1\u4f3c\u7684\u662f\u300c\u4ef7\u503c\u51fd\u6570&nbsp;Q\u03c0(s,a)Q\u03c0\u200b(s,a)\u300d<\/li>\n\n\n\n<li><strong>Q Learning\uff1a<\/strong>&nbsp;\u8fd1\u4f3c\u7684\u662f\u300c\u6700\u4f18\u4ef7\u503c\u51fd\u6570&nbsp;Q\u2217(s,a)Q\u2217(s,a)\u300d<\/li>\n\n\n\n<li><strong>\u795e\u7ecf\u7f51\u7edc\u7248\u7684 Q Learning \u5c31\u662f DQN \u7b97\u6cd5<\/strong><\/li>\n<\/ul>\n\n\n\n<h2 class=\"wp-block-heading\" id=\"_3-multi-step-td\">3.multi-step TD<\/h2>\n\n\n\n<ul class=\"wp-block-list\">\n<li><strong>\u601d\u8def\uff1a<\/strong>&nbsp;\u4e4b\u524d\u7684\u7b97\u6cd5\u90fd\u662f\u5229\u7528\u4e00\u6b21&nbsp;rr&nbsp;\u8fdb\u884c\u7b97\u6cd5\u66f4\u65b0\uff0c\u8fd8\u53ef\u4ee5\u91c7\u7528\u591a\u6b21&nbsp;rr&nbsp;\u6765\u66f4\u65b0\u7b97\u6cd5\u3002<\/li>\n\n\n\n<li><strong>multi-step TD \u76ee\u6807\uff1a<\/strong>&nbsp;\u5229\u7528\u591a\u6b21&nbsp;rr&nbsp;\u6784\u9020\u7684TD\u76ee\u6807<\/li>\n\n\n\n<li><strong>\u56de\u62a5\u5c55\u5f00\uff1a<\/strong> Ut=Rt+\u03b3Ut+1=Rt+\u03b3Rt+1+\u03b32Ut+2=Rt+\u03b3Rt+1+\u03b32Rt+2+\u03b33Ut+3=\u2211i=0m\u22121\u03b3iRt+i+\u03b3mUt+mUt\u200b\u200b=Rt\u200b+\u03b3Ut+1\u200b=Rt\u200b+\u03b3Rt+1\u200b+\u03b32Ut+2\u200b=Rt\u200b+\u03b3Rt+1\u200b+\u03b32Rt+2\u200b+\u03b33Ut+3\u200b=i=0\u2211m\u22121\u200b\u03b3iRt+i\u200b+\u03b3mUt+m\u200b\u200b<\/li>\n\n\n\n<li><strong>sarsa multi-step TD \u76ee\u6807:<\/strong> yt=\u2211i=0m\u22121\u03b3irt+i+\u03b3mQ\u03c0(st+m,at+m)yt\u200b=i=0\u2211m\u22121\u200b\u03b3irt+i\u200b+\u03b3mQ\u03c0\u200b(st+m\u200b,at+m\u200b)<\/li>\n\n\n\n<li><strong>sarsa multi-step TD \u76ee\u6807:<\/strong> yt=\u2211i=0m\u22121\u03b3irt+i+\u03b3mmax\u2061aQ\u2217(st+m,a)yt\u200b=i=0\u2211m\u22121\u200b\u03b3irt+i\u200b+\u03b3mamax\u200bQ\u2217(st+m\u200b,a)<\/li>\n<\/ul>\n\n\n\n<h1 class=\"wp-block-heading\">Actor-Critic\u7b97\u6cd5<\/h1>\n\n\n\n<h2 class=\"wp-block-heading\" id=\"_1-\u7b97\u6cd5\u6982\u5ff5\">1.\u7b97\u6cd5\u6982\u5ff5<\/h2>\n\n\n\n<p><strong>\u601d\u8def\uff1a<\/strong>&nbsp;\u5728&nbsp;\u7b56\u7565\u5b66\u4e60 \u5bf9&nbsp;Q\u03c0(st,at)Q\u03c0\u200b(st\u200b,at\u200b)&nbsp;\u7684\u8ba1\u7b97\u662f\u7528\u4e00\u6b21\u56de\u5408\u540e\u5f97\u5230\u7684&nbsp;UU&nbsp;\u8fdb\u884c\u8fd1\u4f3c\u3002\u540c\u6837\u5bf9\u4e8e&nbsp;Q\u03c0(st,at)Q\u03c0\u200b(st\u200b,at\u200b)&nbsp;\u4e5f\u53ef\u4ee5\u518d\u5efa\u7acb\u4e00\u4e2a\u795e\u7ecf\u7f51\u7edc\u8fdb\u884c\u62df\u5408\u3002<\/p>\n\n\n\n<p><strong>\u7b56\u7565\u7f51\u7edc&nbsp;\u03c0(a\u2223s;\u03b8)\u03c0(a\u2223s;\u03b8)&nbsp;\uff1a<\/strong>&nbsp;\u7528\u4e8e\u5bf9\u7b56\u7565\u51fd\u6570\u7684\u8fd1\u4f3c\uff0c\u5373<code>Actor<\/code>\uff0c\u63a7\u5236\u89d2\u8272\u7684\u52a8\u4f5c<\/p>\n\n\n\n<p><strong>\u4ef7\u503c\u7f51\u7edc&nbsp;q(s,a;w)q(s,a;w)&nbsp;\uff1a<\/strong>&nbsp;\u7528\u4e8e\u5bf9\u4ef7\u503c\u51fd\u6570\u7684\u8fd1\u4f3c\uff0c\u5373<code>Critic<\/code>\uff0c\u5bf9\u89d2\u8272\u52a8\u4f5c\u8fdb\u884c\u6253\u5206<\/p>\n\n\n\n<p><strong>\u603b\u76ee\u6807\uff1a<\/strong>&nbsp;\u4f7f\u5f97\u6700\u7ec8\u7684\u72b6\u6001\u4ef7\u503c\u51fd\u6570\u7684\u503c\u6700\u5927\uff0cV(s;\u03b8,w)=\u2211a\u03c0(a\u2223s;\u03b8)q(s,a;w)V(s;\u03b8,w)=a\u2211\u200b\u03c0(a\u2223s;\u03b8)q(s,a;w)<\/p>\n\n\n\n<h2 class=\"wp-block-heading\" id=\"_2-\u7f51\u7edc\u6a21\u578b\">2.\u7f51\u683c\u6a21\u578b<\/h2>\n\n\n\n<p><strong>\u7b56\u7565\u7f51\u7edc\uff1a<\/strong><img loading=\"lazy\" decoding=\"async\" height=\"15\" width=\"15\" src=\"blob:https:\/\/eve2333.top\/7be28f04-66b9-4d04-9bba-337b50e7bc0e\"><\/p>\n\n\n<div class=\"wp-block-image\">\n<figure class=\"aligncenter size-large is-resized\"><img loading=\"lazy\" decoding=\"async\" width=\"1024\" height=\"235\" src=\"https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731126200-\u5c4f\u5e55\u622a\u56fe-2024-11-09-121747-1024x235.png\" alt=\"\" class=\"wp-image-589\" style=\"width:623px;height:auto\" srcset=\"https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731126200-\u5c4f\u5e55\u622a\u56fe-2024-11-09-121747-1024x235.png 1024w, https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731126200-\u5c4f\u5e55\u622a\u56fe-2024-11-09-121747-300x69.png 300w, https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731126200-\u5c4f\u5e55\u622a\u56fe-2024-11-09-121747-768x176.png 768w, https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731126200-\u5c4f\u5e55\u622a\u56fe-2024-11-09-121747.png 1027w\" sizes=\"auto, (max-width: 1024px) 100vw, 1024px\" \/><\/figure>\n<\/div>\n\n\n<ol class=\"wp-block-list\">\n<li>\u4e00\u5f20\u6e38\u620f\u753b\u9762\u5f53\u4f5c\u4e00\u4e2a\u300c\u72b6\u6001\u300d\uff0c\u72b6\u6001\u901a\u8fc7\u300c\u5377\u79ef\u5c42\u300d\u5b9e\u73b0\u7279\u5f81\u63d0\u53d6<\/li>\n\n\n\n<li>\u7279\u5f81\u8f93\u5165\u7ed9\u300c\u6743\u8fde\u63a5\u5c42\u300d\u8fdb\u884c&nbsp;\u03c0(a\u2223s)\u03c0(a\u2223s)&nbsp;\u62df\u5408<\/li>\n\n\n\n<li>\u5c06\u300c\u6743\u8fde\u63a5\u5c42\u300d\u7684\u8f93\u51fa\u901a\u8fc7<code>softmax<\/code>\u5c42\u8f6c\u5316\u4e3a\u6982\u7387<\/li>\n\n\n\n<li>\u8f93\u51fa\u6bcf\u4e2a\u52a8\u4f5c\u5728\u5f53\u524d\u72b6\u6001\u4e0b\u4ea7\u751f\u597d\u7ed3\u679c\u7684\u6982\u7387<\/li>\n<\/ol>\n\n\n\n<p><strong>\u4ef7\u503c\u7f51\u7edc<\/strong><img loading=\"lazy\" decoding=\"async\" height=\"15\" width=\"15\" src=\"blob:https:\/\/eve2333.top\/8c48c387-0136-4384-9eef-0c3326d48772\"><\/p>\n\n\n<div class=\"wp-block-image\">\n<figure class=\"aligncenter size-large is-resized\"><img loading=\"lazy\" decoding=\"async\" width=\"1024\" height=\"375\" src=\"https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731126214-\u5c4f\u5e55\u622a\u56fe-2024-11-09-121754-1024x375.png\" alt=\"\" class=\"wp-image-590\" style=\"width:593px;height:auto\" srcset=\"https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731126214-\u5c4f\u5e55\u622a\u56fe-2024-11-09-121754-1024x375.png 1024w, https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731126214-\u5c4f\u5e55\u622a\u56fe-2024-11-09-121754-300x110.png 300w, https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731126214-\u5c4f\u5e55\u622a\u56fe-2024-11-09-121754-768x281.png 768w, https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731126214-\u5c4f\u5e55\u622a\u56fe-2024-11-09-121754.png 1054w\" sizes=\"auto, (max-width: 1024px) 100vw, 1024px\" \/><\/figure>\n<\/div>\n\n\n<ol class=\"wp-block-list\">\n<li>\u4e00\u5f20\u6e38\u620f\u753b\u9762\u5f53\u4f5c\u4e00\u4e2a\u300c\u72b6\u6001\u300d\uff0c\u72b6\u6001\u901a\u8fc7\u300c\u5377\u79ef\u5c42\u300d\u5b9e\u73b0\u7279\u5f81\u63d0\u53d6<\/li>\n\n\n\n<li>\u8981\u8bc4\u4ef7\u7684\u4e00\u4e2a\u300c\u52a8\u4f5c\u300d\u901a\u8fc7\u6743\u8fde\u63a5\u5c42\u8fdb\u884c\u7279\u5f81\u63d0\u53d6<\/li>\n\n\n\n<li>\u5c06\u300c\u72b6\u6001\u300d\u4e0e\u300c\u52a8\u4f5c\u300d\u7279\u5f81\u8fdb\u884c\u7ec4\u5408\uff0c\u6700\u540e\u8f93\u5165\u4e00\u4e2a\u6743\u8fde\u63a5\u5c42\uff0c\u5f97\u5230\u300c\u8bc4\u5206\u300d<\/li>\n<\/ol>\n\n\n\n<h2 class=\"wp-block-heading\" id=\"_3-\u7f51\u7edc\u8bad\u7ec3\">3.\u7f51\u683c\u8bad\u7ec3<\/h2>\n\n\n\n<p><strong>\u76ee\u6807\uff1a<\/strong><\/p>\n\n\n\n<ul class=\"wp-block-list\">\n<li><strong>\u7b56\u7565\u7f51\u7edc\uff1a<\/strong>&nbsp;\u4ef7\u503c\u51fd\u6570&nbsp;V(s)V(s)&nbsp;\u7684\u503c\u6700\u5927\u5316<\/li>\n\n\n\n<li><strong>\u4ef7\u503c\u7f51\u7edc\uff1a<\/strong>&nbsp;\u5bf9\u52a8\u4f5c\u7684\u6253\u5206&nbsp;q(s,a)q(s,a)&nbsp;\u66f4\u63a5\u8fd1\u771f\u5b9e\u503c<\/li>\n<\/ul>\n\n\n\n<p><strong>\u53c2\u6570\u66f4\u65b0\uff1a<\/strong><\/p>\n\n\n\n<ol class=\"wp-block-list\">\n<li>\u89c2\u6d4b\u5230\u73af\u5883\u72b6\u6001&nbsp;stst\u200b<\/li>\n\n\n\n<li>\u6839\u636e&nbsp;\u03c0(a\u2223s;\u03b8)\u03c0(a\u2223s;\u03b8)&nbsp;\u5f97\u5230\u52a8\u4f5c&nbsp;atat\u200b<\/li>\n\n\n\n<li>\u89d2\u8272\u6267\u884c\u52a8\u4f5c&nbsp;atat\u200b\uff0c\u7136\u540e\u83b7\u53d6\u72b6\u6001&nbsp;st+1st+1\u200b&nbsp;\u4e0e\u5956\u52b1&nbsp;rtrt\u200b<\/li>\n\n\n\n<li>\u6839\u636eTD\u7b97\u6cd5\u66f4\u65b0\u7cfb\u6570&nbsp;ww<\/li>\n\n\n\n<li>\u6839\u636e\u7b56\u7565\u68af\u5ea6\u7b97\u6cd5\u66f4\u65b0\u7cfb\u6570&nbsp;\u03b8\u03b8<\/li>\n<\/ol>\n\n\n\n<p><strong>\u7cfb\u6570&nbsp;ww&nbsp;\u66f4\u65b0\uff1a<\/strong><\/p>\n\n\n\n<ol class=\"wp-block-list\">\n<li>\u8ba1\u7b97&nbsp;q(st,at;wt)q(st\u200b,at\u200b;wt\u200b)<\/li>\n\n\n\n<li>\u6839\u636e\u72b6\u6001&nbsp;st+1st+1\u200b\uff0c\u901a\u8fc7&nbsp;\u03c0(a\u2223s;\u03b8)\u03c0(a\u2223s;\u03b8)&nbsp;\u9884\u6d4b\u52a8\u4f5c&nbsp;atat\u200b&nbsp;\uff0c\u7136\u540e\u8ba1\u7b97&nbsp;q(st+1,at+1;wt)q(st+1\u200b,at+1\u200b;wt\u200b)<\/li>\n\n\n\n<li>\u8ba1\u7b97TD\u76ee\u6807\uff1ayt=rt+\u03b3q(st+1,at+1;wt)yt\u200b=rt\u200b+\u03b3q(st+1\u200b,at+1\u200b;wt\u200b)<\/li>\n\n\n\n<li>\u635f\u5931\u51fd\u6570\uff1aL(w)=12[q(st,at;wt)\u2212yt]2L(w)=21\u200b[q(st\u200b,at\u200b;wt\u200b)\u2212yt\u200b]2<\/li>\n\n\n\n<li>\u66f4\u65b0\u53c2\u6570\uff1awt+1=wt\u2212\u03b1\u2202L(w)\u2202wwt+1\u200b=wt\u200b\u2212\u03b1\u2202w\u2202L(w)\u200b<\/li>\n<\/ol>\n\n\n\n<p><strong>\u7cfb\u6570&nbsp;\u03b8\u03b8&nbsp;\u66f4\u65b0\uff1a<\/strong><\/p>\n\n\n\n<ol class=\"wp-block-list\">\n<li>\u6839\u636e\u7b56\u7565\u51fd\u6570&nbsp;\u03c0(a\u2223st;\u03b8t)\u03c0(a\u2223st\u200b;\u03b8t\u200b)&nbsp;\u968f\u673a\u83b7\u53d6\u52a8\u4f5c&nbsp;a^a^<\/li>\n\n\n\n<li>\u5e26\u5165&nbsp;a^a^&nbsp;\u8ba1\u7b97\uff1ag(a^,\u03b8t)=\u2202ln\u2061\u03c0(a^\u2223s;\u03b8t)\u2202\u03b8q(st,a^;w)g(a^,\u03b8t\u200b)=\u2202\u03b8\u2202ln\u03c0(a^\u2223s;\u03b8t\u200b)\u200bq(st\u200b,a^;w)<\/li>\n\n\n\n<li>\u7136\u540e\u901a\u8fc7&nbsp;g(a^,\u03b8t)g(a^,\u03b8t\u200b)&nbsp;\u6765\u8fd1\u4f3c\u8ba1\u7b97\uff1a\u2202V(s;\u03b8t)\u2202\u03b8t=EA[g(A,\u03b8t)]\u2202\u03b8t\u200b\u2202V(s;\u03b8t\u200b)\u200b=EA\u200b[g(A,\u03b8t\u200b)]<\/li>\n\n\n\n<li>\u66f4\u65b0\u53c2\u6570\uff1a\u03b8t+1=\u03b8t+\u03b2g(a,\u03b8t)\u03b8t+1\u200b=\u03b8t\u200b+\u03b2g(a,\u03b8t\u200b)<\/li>\n<\/ol>\n\n\n\n<p>Note<\/p>\n\n\n\n<p>\u518d\u8ba1\u7b97&nbsp;g(a,\u03b8)g(a,\u03b8)&nbsp;\u65f6\uff0c\u4e00\u822c\u4f1a\u4f7f\u7528TD error\uff1a<\/p>\n\n\n\n<p>\u03b4t=q(st,at;wt)\u2212[rt+\u03b3q(st+1,at+1;wt)]\u03b4t\u200b=q(st\u200b,at\u200b;wt\u200b)\u2212[rt\u200b+\u03b3q(st+1\u200b,at+1\u200b;wt\u200b)]<\/p>\n\n\n\n<p>\u6765\u4ee3\u66ff&nbsp;q(st,a;w)q(st\u200b,a;w)\uff0c\u8fd9\u6837\u53ef\u4ee5\u4f7f\u7b97\u6cd5\u6536\u655b\u52a0\u5feb\u3002<\/p>\n\n\n\n<h1 class=\"wp-block-heading\">DQN\u4f18\u5316<\/h1>\n\n\n\n<h2 class=\"wp-block-heading\" id=\"_1-\u7ecf\u9a8c\u56de\u653e\">1.\u7ecf\u9a8c\u56de\u653e<\/h2>\n\n\n\n<h3 class=\"wp-block-heading\" id=\"_11-\u6982\u5ff5\">1.1\u6982\u5ff5<\/h3>\n\n\n\n<p><strong>\u4f20\u7edfDQN\u7684\u7f3a\u9677\uff1a<\/strong><\/p>\n\n\n\n<ol class=\"wp-block-list\">\n<li>\u4f20\u7edf\u7684DQN\uff0c\u4e00\u6b21&nbsp;(st,at,rt,st+1)(st\u200b,at\u200b,rt\u200b,st+1\u200b)&nbsp;\u53ea\u7528\u4e8e\u4e00\u6b21\u8ba1\u7b97\uff0c\u5341\u5206\u6d6a\u8d39<\/li>\n\n\n\n<li>\u7528\u4e8e\u8bad\u7ec3\u7684&nbsp;(st,at,rt,st+1)(st\u200b,at\u200b,rt\u200b,st+1\u200b)&nbsp;\u5177\u6709\u8f83\u5f3a\u7684\u76f8\u5173\u6027\uff08stst\u200b\u8bad\u7ec3\u5b8c\uff0c\u5c31\u4f7f\u7528&nbsp;st+1st+1\u200b\uff09\uff0c\u8fd9\u79cd\u76f8\u5173\u6027\u662f\u6709\u5bb3\u7684<\/li>\n<\/ol>\n\n\n\n<p><strong>\u7ecf\u9a8c\u56de\u653e\uff08experience replay\uff09\uff1a<\/strong><img loading=\"lazy\" decoding=\"async\" height=\"15\" width=\"15\" src=\"blob:https:\/\/eve2333.top\/81a5a23e-0ad7-413a-b15b-8f7f3e8651b1\"><\/p>\n\n\n<div class=\"wp-block-image\">\n<figure class=\"aligncenter size-full is-resized\"><img loading=\"lazy\" decoding=\"async\" width=\"1019\" height=\"424\" src=\"https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731126239-\u5c4f\u5e55\u622a\u56fe-2024-11-09-121759.png\" alt=\"\" class=\"wp-image-591\" style=\"width:476px;height:auto\" srcset=\"https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731126239-\u5c4f\u5e55\u622a\u56fe-2024-11-09-121759.png 1019w, https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731126239-\u5c4f\u5e55\u622a\u56fe-2024-11-09-121759-300x125.png 300w, https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731126239-\u5c4f\u5e55\u622a\u56fe-2024-11-09-121759-768x320.png 768w\" sizes=\"auto, (max-width: 1019px) 100vw, 1019px\" \/><\/figure>\n<\/div>\n\n\n<ol class=\"wp-block-list\">\n<li>\u5c06\u6bcf\u4e00\u6b21&nbsp;(st,at,rt,st+1)(st\u200b,at\u200b,rt\u200b,st+1\u200b)&nbsp;\u90fd\u653e\u5165\u4e00\u4e2a\u300c\u56de\u653e\u7f13\u51b2\u533a<code>replay buffer<\/code>\u300d\u4e2d\u3002\u7f13\u51b2\u533a\u5927\u5c0f\u4e00\u5b9a\uff0c\u5b58\u6ee1\u540e\uff0c\u5c31\u8fdb\u884c\u5165\u961f\u548c\u51fa\u961f<\/li>\n\n\n\n<li>\u5229\u7528\u300c\u968f\u673a\u68af\u5ea6<code>SGD<\/code>\u300d\u8fdb\u884c\u6a21\u578b\u7684\u66f4\u65b0\uff1a&nbsp;\u4ece\u56de\u653e\u7f13\u51b2\u533a\u4e2d\uff0c\u968f\u673a\u62bd\u53d6\u4e00\u9879\u8fdb\u884c\u6a21\u578b\u66f4\u65b0\u8ba1\u7b97<\/li>\n<\/ol>\n\n\n\n<h3 class=\"wp-block-heading\" id=\"_12-\u4f18\u5148\u7ecf\u9a8c\u56de\u653e\">1.2\u4f18\u5148\u7ecf\u9a8c\u56de\u653e<\/h3>\n\n\n\n<p><strong>\u539f\u56e0\uff1a<\/strong>&nbsp;\u89d2\u8272\u5bf9\u4e8e\u5e38\u89c4\u7684\u6e38\u620f\u73af\u5883\u4f1a\u5728\u300c\u56de\u653e\u7f13\u51b2\u533a\u300d\u4e2d\u5b58\u653e\u5927\u91cf\u7684\u7ecf\u9a8c\uff0c\u800c\u5bf9\u4e8eBOSS\u5173\u5361\u7684\u7ecf\u9a8c\u5374\u5f88\u5c11\u3002\u5bf9\u4e8e\u7ecf\u9a8c\u56de\u653e\uff0c\u82e5\u91c7\u7528\u5747\u5300\u62bd\u6837\uff0c\u5c31\u4f1a\u5bfc\u81f4\u5bf9BOSS\u5173\u5361\u7684\u5b66\u4e60\u673a\u4f1a\u53d8\u5c11\u3002<img loading=\"lazy\" decoding=\"async\" height=\"15\" width=\"15\" src=\"blob:https:\/\/eve2333.top\/8f606adf-4769-4c71-bb80-64624021ee4d\"><\/p>\n\n\n<div class=\"wp-block-image\">\n<figure class=\"aligncenter size-full is-resized\"><img loading=\"lazy\" decoding=\"async\" width=\"1017\" height=\"366\" src=\"https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731126252-\u5c4f\u5e55\u622a\u56fe-2024-11-09-121821.png\" alt=\"\" class=\"wp-image-592\" style=\"width:822px;height:auto\" srcset=\"https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731126252-\u5c4f\u5e55\u622a\u56fe-2024-11-09-121821.png 1017w, https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731126252-\u5c4f\u5e55\u622a\u56fe-2024-11-09-121821-300x108.png 300w, https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731126252-\u5c4f\u5e55\u622a\u56fe-2024-11-09-121821-768x276.png 768w\" sizes=\"auto, (max-width: 1017px) 100vw, 1017px\" \/><\/figure>\n<\/div>\n\n\n<p><strong>\u4f18\u5148\u7ecf\u9a8c\u56de\u653e\uff1a<\/strong>&nbsp;\u5bf9\u4e8e\u7279\u6b8a\u72b6\u6001\uff08\u4f8b\u5982BOSS\u5173\u5361\uff09\uff0c\u589e\u52a0\u5176\u88ab\u62bd\u4e2d\u5b66\u4e60\u7684\u6982\u7387\u3002<\/p>\n\n\n\n<ol class=\"wp-block-list\">\n<li><strong>\u5b9e\u73b0\u65b9\u6cd5\uff1a<\/strong>&nbsp;\u2223\u03b4t\u2223\u2223\u03b4t\u200b\u2223&nbsp;\u8d8a\u5927\uff0c\u88ab\u62bd\u4e2d\u7684\u6982\u7387\u8d8a\u5927\n<ul class=\"wp-block-list\">\n<li><strong>\u65b9\u6cd5\u4e00\uff1a<\/strong>&nbsp;pt\u221d\u2223\u03b4t\u2223+\u03f5pt\u200b\u221d\u2223\u03b4t\u200b\u2223+\u03f5\uff0c\u5373\u88ab\u62bd\u6837\u7684\u6982\u7387&nbsp;ptpt\u200b&nbsp;\u6b63\u6bd4\u4e8eTD error\u7edd\u5bf9\u503c&nbsp;\u2223\u03b4t\u2223\u2223\u03b4t\u200b\u2223<\/li>\n\n\n\n<li><strong>\u65b9\u6cd5\u4e8c\uff1a<\/strong>&nbsp;pt\u221d1ipt\u200b\u221di1\u200b\uff0c\u5176\u4e2d&nbsp;<code>i<\/code>&nbsp;\u662f\u72b6\u6001\u6839\u636e&nbsp;\u2223\u03b4t\u2223\u2223\u03b4t\u200b\u2223&nbsp;\u964d\u5e8f\u6392\u5217\u7684\u7d22\u5f15\u3002<\/li>\n<\/ul>\n<\/li>\n\n\n\n<li><strong>\u5b66\u4e60\u7387\uff1a<\/strong>&nbsp;(npt)\u2212\u03b2\u03b1,\u03b2\u2208(0,1)(npt\u200b)\u2212\u03b2\u03b1,\u03b2\u2208(0,1)\uff0c\u5373\u88ab\u62bd\u4e2d\u7684\u6982\u7387\u8d8a\u5927\uff0c\u5b66\u4e60\u7387\u5e94\u5f53\u8d8a\u5c0f\u3002&nbsp;\u7528\u4e8e\u6d88\u9664\u5927\u6982\u7387\u62bd\u6837\uff0c\u9020\u6210\u7684\u9884\u6d4b\u504f\u5dee<\/li>\n\n\n\n<li><strong>\u66f4\u65b0&nbsp;\u2223\u03b4t\u2223\u2223\u03b4t\u200b\u2223\uff1a<\/strong>\n<ul class=\"wp-block-list\">\n<li>\u521d\u59cb\u503c\u8bbe\u7f6e\u4e3a\u6700\u5927\u503c\u3002\u56e0\u4e3a\u72b6\u6001\u53ea\u6709\u8fdb\u884c\u4e00\u6b21\u5b66\u4e60\u8ba1\u7b97\u540e\u624d\u80fd\u77e5\u9053\u7ed3\u679c<\/li>\n\n\n\n<li>\u6bcf\u4e00\u6b21\u5b66\u4e60\u540e\uff0c\u66f4\u65b0\u5bf9\u4e8e\u72b6\u6001\u7684&nbsp;\u2223\u03b4t\u2223\u2223\u03b4t\u200b\u2223<img loading=\"lazy\" decoding=\"async\" height=\"15\" width=\"15\" src=\"blob:https:\/\/eve2333.top\/b8bb2f82-a741-426b-8505-f3bb5b62badf\"><\/li>\n<\/ul>\n<\/li>\n<\/ol>\n\n\n<div class=\"wp-block-image\">\n<figure class=\"aligncenter size-large is-resized\"><img loading=\"lazy\" decoding=\"async\" width=\"1024\" height=\"436\" src=\"https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731126271-\u5c4f\u5e55\u622a\u56fe-2024-11-09-121826-1024x436.png\" alt=\"\" class=\"wp-image-593\" style=\"width:582px;height:auto\" srcset=\"https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731126271-\u5c4f\u5e55\u622a\u56fe-2024-11-09-121826-1024x436.png 1024w, https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731126271-\u5c4f\u5e55\u622a\u56fe-2024-11-09-121826-300x128.png 300w, https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731126271-\u5c4f\u5e55\u622a\u56fe-2024-11-09-121826-768x327.png 768w, https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731126271-\u5c4f\u5e55\u622a\u56fe-2024-11-09-121826.png 1059w\" sizes=\"auto, (max-width: 1024px) 100vw, 1024px\" \/><\/figure>\n<\/div>\n\n\n<h2 class=\"wp-block-heading\" id=\"_2-\u9ad8\u4f30\u95ee\u9898\">2.\u9ad8\u4f30\u95ee\u9898<\/h2>\n\n\n\n<h3 class=\"wp-block-heading\" id=\"_21-\u9ad8\u4f30\u7684\u539f\u56e0\">2.1 \u9ad8\u4f30\u7684\u95ee\u9898<\/h3>\n\n\n\n<p>1. \u6700\u5927\u5316<\/p>\n\n\n\n<p><strong>\u5b9a\u7406\uff1a<\/strong>&nbsp;x1,x2,\u22ef\u2009,xnx1\u200b,x2\u200b,\u22ef,xn\u200b\u662f\u771f\u5b9e\u5e8f\u5217\uff1bQ1,Q2,\u22ef\u2009,QnQ1\u200b,Q2\u200b,\u22ef,Qn\u200b\u662f\u771f\u5b9e\u503c\u5e26\u6709\u5747\u503c\u4e3a&nbsp;00&nbsp;\u566a\u58f0\u7684\u89c2\u6d4b\u503c\u3002\u5c31\u5b58\u5728<\/p>\n\n\n\n<p>E[mean(Qi)]=mean(xi)E[max(Qi)]\u2265max(xi)E[min(Qi)]\u2264min(xi)E[mean(Qi\u200b)]E[max(Qi\u200b)]E[min(Qi\u200b)]\u200b=mean(xi\u200b)\u2265max(xi\u200b)\u2264min(xi\u200b)\u200b<\/p>\n\n\n\n<p><strong>\u9884\u6d4b\u503c\u9ad8\u4f30\uff1a<\/strong>&nbsp;TD\u76ee\u6807\u4e3a&nbsp;yt=rt+\u03b3max\u2061aQ(st+1,a;w)yt\u200b=rt\u200b+\u03b3amax\u200bQ(st+1\u200b,a;w)&nbsp;\uff0c\u6839\u636e\u4e0a\u8ff0\u5b9a\u7406\u53ef\u77e5\uff0cmax\u2061aQ(st+1,a;w)amax\u200bQ(st+1\u200b,a;w)&nbsp;\u7684\u503c\u76f8\u5bf9\u4e8e\u771f\u5b9e\u503c\uff0c\u5176\u5b9e\u88ab\u653e\u5927\u4e86\uff0c\u8fdb\u4e00\u6b65\u5bfc\u81f4&nbsp;ytyt\u200b&nbsp;\u7684\u503c\u88ab\u9ad8\u4f30\u3002\u7531\u4e8e&nbsp;ytyt\u200b&nbsp;\u662f\u76ee\u6807\uff0c\u8fd9\u5c31\u4f7f\u5f97DQN\u6a21\u578b\u7684\u9884\u6d4b\u503c\u88ab\u9ad8\u4f30\u3002<\/p>\n\n\n\n<p>2. \u81ea\u4e3e<\/p>\n\n\n\n<p>\u5728TD\u76ee\u6807\u4e2d&nbsp;yt=rt+\u03b3max\u2061aQ(st+1,a;w)yt\u200b=rt\u200b+\u03b3amax\u200bQ(st+1\u200b,a;w)\uff0cQ(st+1,a;w)Q(st+1\u200b,a;w)&nbsp;\u5176\u5b9e\u5df2\u7ecf\u88ab\u6a21\u578b\u9ad8\u4f30\u8ba1\u7b97\u4e86\uff0c\u8fd9\u4e2a\u9ad8\u4f30\u7684\u503c\u53c8\u88ab\u56de\u4ee3\u7528\u6765\u66f4\u65b0&nbsp;Q(st,a;w)Q(st\u200b,a;w)&nbsp;\u7684\u503c\uff0c\u8fd9\u5c31\u4f7f\u5f97\u9ad8\u4f30\u503c\u88ab\u8fed\u4ee3\u653e\u5927\u3002<\/p>\n\n\n\n<h3 class=\"wp-block-heading\" id=\"_22-\u9ad8\u4f30\u7684\u5371\u5bb3\">2.2\u9ad8\u4f30\u7684\u5371\u5bb3<\/h3>\n\n\n\n<p>\u7531\u4e8eDQN\u5bf9\u4e8e&nbsp;Q(s,a;w)Q(s,a;w)&nbsp;\u7684\u9ad8\u4f30\u5e76\u975e\u5747\u5300\u7684\uff0c\u8fd9\u5c31\u5bfc\u81f4\u6700\u540e\u7684\u9884\u6d4b\u7ed3\u679c\u5927\u5c0f\u5173\u7cfb\u7684\u6539\u53d8\u3002\u8fd9\u5c31\u5bfc\u81f4\u300c\u52a8\u4f5c\u300d\u7684\u9009\u62e9\uff0c\u53ef\u80fd\u662f\u9519\u8bef\u7684\u3002<img loading=\"lazy\" decoding=\"async\" height=\"15\" width=\"15\" src=\"blob:https:\/\/eve2333.top\/16e17302-71c4-412c-b675-469f24830a35\"><\/p>\n\n\n\n<figure class=\"wp-block-image size-large\"><img loading=\"lazy\" decoding=\"async\" width=\"1024\" height=\"138\" src=\"https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731126294-\u5c4f\u5e55\u622a\u56fe-2024-11-09-121832-1024x138.png\" alt=\"\" class=\"wp-image-594\" srcset=\"https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731126294-\u5c4f\u5e55\u622a\u56fe-2024-11-09-121832-1024x138.png 1024w, https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731126294-\u5c4f\u5e55\u622a\u56fe-2024-11-09-121832-300x41.png 300w, https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731126294-\u5c4f\u5e55\u622a\u56fe-2024-11-09-121832-768x104.png 768w, https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731126294-\u5c4f\u5e55\u622a\u56fe-2024-11-09-121832.png 1029w\" sizes=\"auto, (max-width: 1024px) 100vw, 1024px\" \/><\/figure>\n\n\n\n<h3 class=\"wp-block-heading\" id=\"_23-\u76ee\u6807\u7f51\u7edc\">2.3\u76ee\u6807\u7f51\u7edc<\/h3>\n\n\n\n<p><strong>\u7b97\u6cd5\uff1a<\/strong><\/p>\n\n\n\n<ol class=\"wp-block-list\">\n<li>\u89c2\u6d4b\u4e00\u6b21\u72b6\u6001\uff1a(st,at,rt,st+1)(st\u200b,at\u200b,rt\u200b,st+1\u200b)<\/li>\n\n\n\n<li>TD\u76ee\u6807\uff1ayt=rt+\u03b3max\u2061aQ(st+1,a;w\u2212)yt\u200b=rt\u200b+\u03b3amax\u200bQ(st+1\u200b,a;w\u2212)<\/li>\n\n\n\n<li>TD\u8bef\u5dee\uff1a\u03b4t=Q(st,at;w)\u2212yt\u03b4t\u200b=Q(st\u200b,at\u200b;w)\u2212yt\u200b<\/li>\n\n\n\n<li>\u635f\u5931\u51fd\u6570\uff1aL=12\u03b4t2L=21\u200b\u03b4t2\u200b<\/li>\n\n\n\n<li>\u66f4\u65b0&nbsp;Q(st,at;w)Q(st\u200b,at\u200b;w)&nbsp;\u7cfb\u6570\uff1aw=w\u2212\u03b1\u03b4t\u2202Q(s,a;w)\u2202ww=w\u2212\u03b1\u03b4t\u200b\u2202w\u2202Q(s,a;w)\u200b<\/li>\n\n\n\n<li>\u66f4\u65b0&nbsp;Q(s,a;w\u2212)Q(s,a;w\u2212)&nbsp;\u7cfb\u6570\uff1a&nbsp;\u8981\u4e00\u6bb5\u65f6\u95f4\u624d\u66f4\u65b0\u4e00\u6b21\n<ul class=\"wp-block-list\">\n<li><strong>\u65b9\u6cd5\u4e00\uff1a<\/strong>&nbsp;w\u2212=ww\u2212=w<\/li>\n\n\n\n<li><strong>\u65b9\u6cd5\u4e8c\uff1a<\/strong>&nbsp;w\u2212=\u03c4w+(1\u2212\u03c4)w\u2212w\u2212=\u03c4w+(1\u2212\u03c4)w\u2212<\/li>\n<\/ul>\n<\/li>\n<\/ol>\n\n\n\n<p>Note<\/p>\n\n\n\n<p>\u300c\u76ee\u6807\u7f51\u7edc<code>targe network<\/code>\u300d\u65b9\u6cd5\u662f\u5229\u7528\u300c\u76ee\u6807\u7f51\u7edc\uff08Q(st+1,a;w\u2212)Q(st+1\u200b,a;w\u2212)\uff09\u300d\u6765\u4ee3\u66ff\u300cTD\u76ee\u6807\u300d\u4e2d\u9884\u6d4b\u4e0b\u4e00\u52a8\u4f5c\u7684\u52a8\u4f5c\u4ef7\u503c\u7684\u7f51\u7edc&nbsp;Q(st+1,a;w)Q(st+1\u200b,a;w)\u3002\u8be5\u65b9\u6cd5\u907f\u514d\u81ea\u4e3e\u3002<\/p>\n\n\n\n<h3 class=\"wp-block-heading\" id=\"_24-double-dqn\">2.4 Double DQN<\/h3>\n\n\n\n<p><strong>\u7b97\u6cd5\uff1a<\/strong><\/p>\n\n\n\n<ol class=\"wp-block-list\">\n<li>\u89c2\u6d4b\u4e00\u6b21\u72b6\u6001\uff1a(st,at,rt,st+1)(st\u200b,at\u200b,rt\u200b,st+1\u200b)<\/li>\n\n\n\n<li>\u9009\u62e9\u4e0b\u4e00\u52a8\u4f5c\uff1aa\u2217=max\u2061aQ(st+1,a;w)a\u2217=amax\u200bQ(st+1\u200b,a;w)<\/li>\n\n\n\n<li>TD\u76ee\u6807\uff1ayt=rt+\u03b3Q(st+1,a\u2217;w\u2212)yt\u200b=rt\u200b+\u03b3Q(st+1\u200b,a\u2217;w\u2212)<\/li>\n\n\n\n<li>TD\u8bef\u5dee\uff1a\u03b4t=Q(st,at;w)\u2212yt\u03b4t\u200b=Q(st\u200b,at\u200b;w)\u2212yt\u200b<\/li>\n\n\n\n<li>\u635f\u5931\u51fd\u6570\uff1aL=12\u03b4t2L=21\u200b\u03b4t2\u200b<\/li>\n\n\n\n<li>\u66f4\u65b0&nbsp;Q(st,at;w)Q(st\u200b,at\u200b;w)&nbsp;\u7cfb\u6570\uff1aw=w\u2212\u03b1\u03b4t\u2202Q(s,a;w)\u2202ww=w\u2212\u03b1\u03b4t\u200b\u2202w\u2202Q(s,a;w)\u200b<\/li>\n\n\n\n<li>\u66f4\u65b0&nbsp;Q(s,a;w\u2212)Q(s,a;w\u2212)&nbsp;\u7cfb\u6570\uff1a&nbsp;\u8981\u4e00\u6bb5\u65f6\u95f4\u624d\u66f4\u65b0\u4e00\u6b21\n<ul class=\"wp-block-list\">\n<li><strong>\u65b9\u6cd5\u4e00\uff1a<\/strong>&nbsp;w\u2212=ww\u2212=w<\/li>\n\n\n\n<li><strong>\u65b9\u6cd5\u4e8c\uff1a<\/strong>&nbsp;w\u2212=\u03c4w+(1\u2212\u03c4)w\u2212w\u2212=\u03c4w+(1\u2212\u03c4)w\u2212<\/li>\n<\/ul>\n<\/li>\n<\/ol>\n\n\n\n<blockquote class=\"wp-block-quote is-layout-flow wp-block-quote-is-layout-flow\">\n<p>[note]&nbsp;<code>double DQN<\/code>&nbsp;\u7b97\u6cd5\u4e2d\uff0c\u5bf9\u4e8e\u8ba1\u7b97TD\u76ee\u6807\uff0c\u5229\u7528&nbsp;Q(st+1,a;w)Q(st+1\u200b,a;w)\u63d0\u4f9b\u9884\u6d4b\u52a8\u4f5c&nbsp;a\u2217a\u2217\uff0c\u7136\u540e\u5229\u7528\u300c\u76ee\u6807\u7f51\u7edc\u300d&nbsp;Q(st+1,a\u2217;w\u2212)Q(st+1\u200b,a\u2217;w\u2212)&nbsp;\u8fdb\u884c\u8ba1\u7b97\u3002<\/p>\n<\/blockquote>\n\n\n\n<h2 class=\"wp-block-heading\" id=\"_3-dueling-network\">3. Dueling Network<\/h2>\n\n\n\n<h3 class=\"wp-block-heading\" id=\"_31-\u6982\u5ff5\">3.1\u6982\u5ff5<\/h3>\n\n\n\n<p><strong>\u6700\u4f18\u52a8\u4f5c\u4ef7\u503c\uff1a<\/strong>&nbsp;Q\u2217(s,a)=max\u2061\u03c0Q\u03c0(s,a)Q\u2217(s,a)=\u03c0max\u200bQ\u03c0\u200b(s,a)<\/p>\n\n\n\n<p><strong>\u6700\u4f18\u72b6\u6001\u4ef7\u503c\uff1a<\/strong>&nbsp;V\u2217(s)=max\u2061\u03c0V\u03c0(s)V\u2217(s)=\u03c0max\u200bV\u03c0\u200b(s)<\/p>\n\n\n\n<p><strong>\u4f18\u52bf\u51fd\u6570\uff08optimal advantage function\uff09\uff1a<\/strong>&nbsp;\u63cf\u8ff0\u7684\u662f\u6700\u4f18\u52a8\u4f5c&nbsp;aa&nbsp;\u7684\u4f18\u52bf\u3002<\/p>\n\n\n\n<p>A\u2217(s,a)=Q\u2217(s,a)\u2212V\u2217(s)A\u2217(s,a)=Q\u2217(s,a)\u2212V\u2217(s)<\/p>\n\n\n\n<p><strong>\u5b9a\u7406\uff1a<\/strong><\/p>\n\n\n\n<p>V\u2217(s)=max\u2061aQ\u2217(s,a)V\u2217(s)=amax\u200bQ\u2217(s,a)<\/p>\n\n\n\n<h3 class=\"wp-block-heading\" id=\"_32-\u76ee\u6807\">3.2\u76ee\u6807<\/h3>\n\n\n\n<p>\u5c06\u4f18\u52bf\u51fd\u6570\u4e24\u8fb9\u6700\u5927\u5316\uff1a<\/p>\n\n\n\n<p>max\u2061aA\u2217(s,a)=max\u2061aQ\u2217(s,a)\u2212V\u2217(s)=V\u2217(s)\u2212V\u2217(s)=0amax\u200bA\u2217(s,a)\u200b=amax\u200bQ\u2217(s,a)\u2212V\u2217(s)=V\u2217(s)\u2212V\u2217(s)=0\u200b<\/p>\n\n\n\n<p>\u6545\u4f18\u52bf\u51fd\u6570\u53ef\u4ee5\u53d8\u5f62\u4e3a\uff1a<\/p>\n\n\n\n<p>Q\u2217(s,a)=A\u2217(s,a)+V\u2217(s)\u2212max\u2061aA\u2217(s,a)Q\u2217(s,a)=A\u2217(s,a)+V\u2217(s)\u2212amax\u200bA\u2217(s,a)<\/p>\n\n\n\n<p>Note<\/p>\n\n\n\n<p><code>dueling network<\/code>\u5c31\u662f\u5bf9\u4e0a\u5f0f\u76ee\u6807&nbsp;Q\u2217(s,a)Q\u2217(s,a)&nbsp;\u8fdb\u884c\u8fd1\u4f3c<\/p>\n\n\n\n<h2 class=\"wp-block-heading\" id=\"_4-\u7f51\u7edc\u7ed3\u6784\">4.\u7f51\u7edc\u7ed3\u6784<img loading=\"lazy\" decoding=\"async\" height=\"15\" width=\"15\" src=\"blob:https:\/\/eve2333.top\/00a4bcb8-f726-4642-b72d-8241f96ff10d\"><\/h2>\n\n\n\n<figure class=\"wp-block-image size-full\"><img loading=\"lazy\" decoding=\"async\" width=\"1015\" height=\"287\" src=\"https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731126306-\u5c4f\u5e55\u622a\u56fe-2024-11-09-121846.png\" alt=\"\" class=\"wp-image-595\" srcset=\"https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731126306-\u5c4f\u5e55\u622a\u56fe-2024-11-09-121846.png 1015w, https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731126306-\u5c4f\u5e55\u622a\u56fe-2024-11-09-121846-300x85.png 300w, https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731126306-\u5c4f\u5e55\u622a\u56fe-2024-11-09-121846-768x217.png 768w\" sizes=\"auto, (max-width: 1015px) 100vw, 1015px\" \/><\/figure>\n\n\n\n<ol class=\"wp-block-list\">\n<li>\u5bf9&nbsp;A\u2217(s,a)A\u2217(s,a)&nbsp;\u4e0e&nbsp;V\u2217(s)V\u2217(s)&nbsp;\u5206\u522b\u91c7\u7528\u795e\u7ecf\u7f51\u7edc\u8fdb\u884c\u8fd1\u4f3c\n<ul class=\"wp-block-list\">\n<li><code>Dense1<\/code>\uff1a\u8fd1\u4f3c&nbsp;A\u2217(s,a)A\u2217(s,a)\uff0c\u5373&nbsp;A(s,a;wA)A(s,a;wA)<\/li>\n\n\n\n<li><code>Dense2<\/code>\uff1a\u8fd1\u4f3c&nbsp;V\u2217(s)V\u2217(s)\uff0c\u5373&nbsp;V(s;wV)V(s;wV)<\/li>\n<\/ul>\n<\/li>\n\n\n\n<li>\u5c06\u7cfb\u6570\u5408\u5e76\uff1aw=[wA,wV]w=[wA,wV]<\/li>\n\n\n\n<li>\u76ee\u6807\u5c31\u4e3a\uff1aQ(s,a;w)=A(s,a;wA)+V(s;wV)\u2212max\u2061aA(s,a;wA)Q(s,a;w)=A(s,a;wA)+V(s;wV)\u2212amax\u200bA(s,a;wA)<\/li>\n\n\n\n<li>\u7cfb\u6570\u66f4\u65b0\u65b9\u6cd5\u4e0e DQN \u4e00\u6837<\/li>\n<\/ol>\n\n\n\n<p>Tip<\/p>\n\n\n\n<p>meanaA(s,a;wA)meana\u200bA(s,a;wA)&nbsp;\u7684\u6548\u679c\u53ef\u80fd\u6bd4&nbsp;maxaA(s,a;wA)maxa\u200bA(s,a;wA)&nbsp;\u597d<\/p>\n\n\n\n<h1 class=\"wp-block-heading\">\u8499\u7279\u5361\u6d1b&nbsp;<\/h1>\n\n\n\n<h2 class=\"wp-block-heading\" id=\"_1-\u8ba1\u7b97c194a9eg\">1.1\u8ba1\u7b97\u03c0<\/h2>\n\n\n\n<h3 class=\"wp-block-heading\" id=\"_11-\u91c7\u6837\u6cd5\">1.1\u91c7\u6837\u6cd5<img loading=\"lazy\" decoding=\"async\" height=\"15\" width=\"15\" src=\"blob:https:\/\/eve2333.top\/54fd2e17-5cc5-4c9a-bb42-dbc56f341fff\"><\/h3>\n\n\n<div class=\"wp-block-image\">\n<figure class=\"aligncenter size-full is-resized\"><img loading=\"lazy\" decoding=\"async\" width=\"844\" height=\"814\" src=\"https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731126317-\u5c4f\u5e55\u622a\u56fe-2024-11-09-121854.png\" alt=\"\" class=\"wp-image-596\" style=\"width:253px;height:auto\" srcset=\"https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731126317-\u5c4f\u5e55\u622a\u56fe-2024-11-09-121854.png 844w, https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731126317-\u5c4f\u5e55\u622a\u56fe-2024-11-09-121854-300x289.png 300w, https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731126317-\u5c4f\u5e55\u622a\u56fe-2024-11-09-121854-768x741.png 768w\" sizes=\"auto, (max-width: 844px) 100vw, 844px\" \/><\/figure>\n<\/div>\n\n\n<p>\u4ece\u56fe\u4e2d\u53ef\u77e5\uff1a\u77e9\u5f62\u7684\u9762\u79ef\u4e3a&nbsp;Ar=4Ar\u200b=4\uff0c\u5706\u7684\u9762\u79ef\u4e3a&nbsp;Ac=\u03c0Ac\u200b=\u03c0\u3002\u73b0\u5728\u5728\u77e9\u5f62\u6846\u5185\u968f\u673a\u7684\u62bd\u53d6\u91c7\u6837\u70b9\uff08<strong>\u5747\u5300\u62bd\u6837<\/strong>\uff09\uff0c\u90a3\u4e48\u91c7\u6837\u70b9\u4f4d\u4e8e\u5706\u5185\u7684\u6982\u7387\u5c31\u4e3a&nbsp;p=AcAr=\u03c04p=Ar\u200bAc\u200b\u200b=4\u03c0\u200b\u3002<\/p>\n\n\n\n<p><strong>\u6b65\u9aa4\uff1a<\/strong><\/p>\n\n\n\n<ul class=\"wp-block-list\">\n<li>\u91c7\u6837<code>m<\/code>\u4e2a\u70b9\uff0c\u6709<code>n<\/code>\u4e2a\u70b9\u5728\u5706\u5185<\/li>\n\n\n\n<li>\u03c0\u03c0\u503c\u5c31\u662f\uff1a\u03c0\u22484nm\u03c0\u2248m4n\u200b<\/li>\n\n\n\n<li>\u8bef\u5dee\u7cbe\u5ea6\u4e3a\uff1a\u22234nm\u2212\u03c0\u2223=O(1m)\u2223m4n\u200b\u2212\u03c0\u2223=O(m\u200b1\u200b)<\/li>\n<\/ul>\n\n\n\n<h3 class=\"wp-block-heading\" id=\"_12-buffon\u9488\u95ee\u9898\">1.2 buffon \u9488\u95ee\u9898<img loading=\"lazy\" decoding=\"async\" height=\"15\" width=\"15\" src=\"blob:https:\/\/eve2333.top\/7c406617-fab5-4e45-ad04-d8d81ff36da1\"><\/h3>\n\n\n<div class=\"wp-block-image\">\n<figure class=\"aligncenter size-full is-resized\"><img loading=\"lazy\" decoding=\"async\" width=\"947\" height=\"1006\" src=\"https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731126331-\u5c4f\u5e55\u622a\u56fe-2024-11-09-121901.png\" alt=\"\" class=\"wp-image-597\" style=\"width:301px;height:auto\" srcset=\"https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731126331-\u5c4f\u5e55\u622a\u56fe-2024-11-09-121901.png 947w, https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731126331-\u5c4f\u5e55\u622a\u56fe-2024-11-09-121901-282x300.png 282w, https:\/\/eve2333.top\/wp-content\/uploads\/2024\/11\/1731126331-\u5c4f\u5e55\u622a\u56fe-2024-11-09-121901-768x816.png 768w\" sizes=\"auto, (max-width: 947px) 100vw, 947px\" \/><\/figure>\n<\/div>\n\n\n<p>\u6240\u6709\u7684\u9488\u90fd\u4e00\u6478\u4e00\u6837\uff0c\u9488\u4e0e\u7eb8\u76f8\u4ea4\u7684\u6982\u7387\u5c31\u4e3a&nbsp;P=2l\u03c0dP=\u03c0d2l\u200b\u3002\u8fd9\u6837\u6839\u636e<code>monte carlo<\/code>\u5c31\u8ba1\u7b97&nbsp;\u03c0\u03c0\u3002<\/p>\n\n\n\n<h2 class=\"wp-block-heading\" id=\"_2-\u8ba1\u7b97\u5b9a\u79ef\u5206\">2.\u8ba1\u7b97\u5b9a\u79ef\u5206<\/h2>\n\n\n\n<p><strong>\u95ee\u9898\uff1a<\/strong>&nbsp;\u8ba1\u7b97\u4e0b\u5217\u5b9a\u79ef\u5206\u7684\u503c<\/p>\n\n\n\n<p>I=\u222babf(x)&nbsp;dxI=\u222bab\u200bf(x)&nbsp;dx<\/p>\n\n\n\n<p><strong>\u5b9a\u79ef\u5206\u7684\u5b9a\u4e49<\/strong>\uff1a<\/p>\n\n\n\n<p>I=lim\u2061n\u2192\u221e\u2211i=0nf(a+i\u2217b\u2212an)b\u2212anI=n\u2192\u221elim\u200bi=0\u2211n\u200bf(a+i\u2217nb\u2212a\u200b)nb\u2212a\u200b<\/p>\n\n\n\n<p><strong>\u8fd1\u4f3c\u8ba1\u7b97\uff1a<\/strong><\/p>\n\n\n\n<ol class=\"wp-block-list\">\n<li>\u5728&nbsp;[a,b][a,b]&nbsp;\u5185\u968f\u673a\u91c7\u6837&nbsp;x1,x2,\u22ef\u2009,xnx1\u200b,x2\u200b,\u22ef,xn\u200b<\/li>\n\n\n\n<li>\u6839\u636e\u5b9a\u79ef\u5206\u5b9a\u4e49\u8fdb\u884c\u8fd1\u4f3c\u8ba1\u7b97\uff1aI\u2248(b\u2212a)1n\u2211i=1nf(xi)I\u2248(b\u2212a)n1\u200bi=1\u2211n\u200bf(xi\u200b)<\/li>\n<\/ol>\n\n\n\n<h2 class=\"wp-block-heading\" id=\"_3-\u8ba1\u7b97\u671f\u671b\">3.\u8ba1\u7b97\u671f\u671b<\/h2>\n\n\n\n<p><strong>\u671f\u671b\u5b9a\u4e49\uff1a<\/strong>&nbsp;\u8bd5\u9a8c\u4e2d\u6bcf\u6b21\u53ef\u80fd\u7ed3\u679c\u7684\u6982\u7387\u4e58\u4ee5\u5176\u7ed3\u679c\u7684\u603b\u548c\uff0c\u53cd\u6620\u968f\u673a\u53d8\u91cf\u5e73\u5747\u53d6\u503c\u7684\u5927\u5c0f\u3002p(x)p(x)\u4ee3\u8868\u6982\u7387\u5bc6\u5ea6\u51fd\u6570\u3002<\/p>\n\n\n\n<p>E(f(X))=\u222bf(x)p(x)&nbsp;dxE(f(X))=\u222bf(x)p(x)&nbsp;dx<\/p>\n\n\n\n<p><strong>\u8fd1\u4f3c\u8ba1\u7b97\uff1a<\/strong><\/p>\n\n\n\n<ol class=\"wp-block-list\">\n<li>\u6839\u636e\u6982\u7387\u5bc6\u5ea6\u51fd\u6570&nbsp;p(x)p(x)&nbsp;\u968f\u673a\u91c7\u6837&nbsp;x1,x2,\u22ef\u2009,xnx1\u200b,x2\u200b,\u22ef,xn\u200b<\/li>\n\n\n\n<li>\u6839\u636e\u5b9a\u79ef\u5206\u5b9a\u4e49\u8fdb\u884c\u8fd1\u4f3c\u8ba1\u7b97\uff1aE(f(X))\u22481n\u2211i=1nf(xi)E(f(X))\u2248n1\u200bi=1\u2211n\u200bf(xi\u200b)<\/li>\n<\/ol>\n\n\n\n<p>\u200b<\/p>\n\n\n\n<p><\/p>\n","protected":false},"excerpt":{"rendered":"<p>\u57fa\u672c\u6982\u5ff5 1.\u57fa\u672c\u672f\u8bed Attention 2.\u89d2\u8272\u4e0e\u73af\u5883\u7684\u4ea4\u4e92 2.1\u4ea4\u4e92\u6d41\u7a0b 2.2\u968f\u673a\u6027 3.\u5956\u52b1\uff0c\u56de\u62a5\uff0c\u4ef7\u503c\u51fd\u6570 3.1\u5956 &#8230;<\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"emotion":"","emotion_color":"","title_style":"","license":"","footnotes":""},"categories":[2],"tags":[9,18],"class_list":["post-568","post","type-post","status-publish","format-standard","hentry","category-2","tag-9","tag-18"],"_links":{"self":[{"href":"https:\/\/eve2333.top\/index.php?rest_route=\/wp\/v2\/posts\/568","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/eve2333.top\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/eve2333.top\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/eve2333.top\/index.php?rest_route=\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/eve2333.top\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=568"}],"version-history":[{"count":0,"href":"https:\/\/eve2333.top\/index.php?rest_route=\/wp\/v2\/posts\/568\/revisions"}],"wp:attachment":[{"href":"https:\/\/eve2333.top\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=568"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/eve2333.top\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=568"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/eve2333.top\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=568"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}