1859 lines
46 KiB
JSON
1859 lines
46 KiB
JSON
[
|
|
{
|
|
"iteration": 1,
|
|
"steps": 1428,
|
|
"total_steps": 1428,
|
|
"win_rate": 0.0,
|
|
"avg_reward": -0.4146875000000002,
|
|
"policy_loss": 0.04653922698344104,
|
|
"value_loss": 0.07008047727867961,
|
|
"entropy": 1.7925717582305272,
|
|
"approx_kl": 0.07808350942408045,
|
|
"clip_fraction": 0.5683417804539204,
|
|
"lr": 0.0003,
|
|
"iter_time_s": 9.8,
|
|
"eval": {
|
|
"heuristic": {
|
|
"win_rate": 0.02,
|
|
"wins": 1,
|
|
"losses": 10,
|
|
"draws": 0,
|
|
"avg_reward": -0.28180000000000016,
|
|
"avg_steps": 22.68
|
|
},
|
|
"random": {
|
|
"win_rate": 0.06,
|
|
"wins": 3,
|
|
"losses": 1,
|
|
"draws": 0,
|
|
"avg_reward": 0.11650000000000013,
|
|
"avg_steps": 23.7
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"iteration": 2,
|
|
"steps": 1443,
|
|
"total_steps": 2871,
|
|
"win_rate": 0.015625,
|
|
"avg_reward": -0.30757812500000015,
|
|
"policy_loss": 0.006764344444187979,
|
|
"value_loss": 0.040692444425076246,
|
|
"entropy": 1.769464299082756,
|
|
"approx_kl": 0.03438184542270998,
|
|
"clip_fraction": 0.41109486669301987,
|
|
"lr": 0.0003,
|
|
"iter_time_s": 3.7,
|
|
"eval": null
|
|
},
|
|
{
|
|
"iteration": 3,
|
|
"steps": 1430,
|
|
"total_steps": 4301,
|
|
"win_rate": 0.09375,
|
|
"avg_reward": -0.17890625000000027,
|
|
"policy_loss": -0.00123176541334639,
|
|
"value_loss": 0.08456645626574755,
|
|
"entropy": 1.7230913986762364,
|
|
"approx_kl": 0.031960216661294304,
|
|
"clip_fraction": 0.41680121670166653,
|
|
"lr": 0.0003,
|
|
"iter_time_s": 3.6,
|
|
"eval": null
|
|
},
|
|
{
|
|
"iteration": 4,
|
|
"steps": 1479,
|
|
"total_steps": 5780,
|
|
"win_rate": 0.109375,
|
|
"avg_reward": 0.06453125000000014,
|
|
"policy_loss": -0.0036898852170755467,
|
|
"value_loss": 0.0673996505017082,
|
|
"entropy": 1.7115723937749863,
|
|
"approx_kl": 0.02811373712029308,
|
|
"clip_fraction": 0.34127096955974895,
|
|
"lr": 0.0003,
|
|
"iter_time_s": 3.5,
|
|
"eval": null
|
|
},
|
|
{
|
|
"iteration": 5,
|
|
"steps": 1383,
|
|
"total_steps": 7163,
|
|
"win_rate": 0.203125,
|
|
"avg_reward": 0.006796875000000249,
|
|
"policy_loss": -0.011933704256080091,
|
|
"value_loss": 0.11815625460197528,
|
|
"entropy": 1.6385686198870342,
|
|
"approx_kl": 0.02477367059327662,
|
|
"clip_fraction": 0.3284900262951851,
|
|
"lr": 0.0003,
|
|
"iter_time_s": 8.0,
|
|
"eval": {
|
|
"heuristic": {
|
|
"win_rate": 0.5,
|
|
"wins": 25,
|
|
"losses": 6,
|
|
"draws": 0,
|
|
"avg_reward": 0.5910000000000022,
|
|
"avg_steps": 21.22
|
|
},
|
|
"random": {
|
|
"win_rate": 0.42,
|
|
"wins": 21,
|
|
"losses": 0,
|
|
"draws": 0,
|
|
"avg_reward": 0.7169000000000006,
|
|
"avg_steps": 22.46
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"iteration": 6,
|
|
"steps": 1397,
|
|
"total_steps": 8560,
|
|
"win_rate": 0.421875,
|
|
"avg_reward": 0.567109375000001,
|
|
"policy_loss": 0.003435632116937389,
|
|
"value_loss": 0.0969200615460674,
|
|
"entropy": 1.5593399306138356,
|
|
"approx_kl": 0.019223477769022185,
|
|
"clip_fraction": 0.2801412834475438,
|
|
"lr": 0.0003,
|
|
"iter_time_s": 3.0,
|
|
"eval": null
|
|
},
|
|
{
|
|
"iteration": 7,
|
|
"steps": 1380,
|
|
"total_steps": 9940,
|
|
"win_rate": 0.5,
|
|
"avg_reward": 0.6142968750000017,
|
|
"policy_loss": -0.004964525656153758,
|
|
"value_loss": 0.09841936267912388,
|
|
"entropy": 1.5256123791138332,
|
|
"approx_kl": 0.023497344576753676,
|
|
"clip_fraction": 0.2969531236837308,
|
|
"lr": 0.0003,
|
|
"iter_time_s": 3.2,
|
|
"eval": null
|
|
},
|
|
{
|
|
"iteration": 8,
|
|
"steps": 1316,
|
|
"total_steps": 11256,
|
|
"win_rate": 0.5625,
|
|
"avg_reward": 0.7199218750000017,
|
|
"policy_loss": -0.0005439401041561117,
|
|
"value_loss": 0.08654300433893998,
|
|
"entropy": 1.3986669232447941,
|
|
"approx_kl": 0.03016990632750094,
|
|
"clip_fraction": 0.38545283551017445,
|
|
"lr": 0.0003,
|
|
"iter_time_s": 2.9,
|
|
"eval": null
|
|
},
|
|
{
|
|
"iteration": 9,
|
|
"steps": 1270,
|
|
"total_steps": 12526,
|
|
"win_rate": 0.65625,
|
|
"avg_reward": 0.8654687500000018,
|
|
"policy_loss": -0.0034353737719357014,
|
|
"value_loss": 0.08132061827927828,
|
|
"entropy": 1.4026022613048554,
|
|
"approx_kl": 0.020757770398631692,
|
|
"clip_fraction": 0.27876968756318093,
|
|
"lr": 0.0003,
|
|
"iter_time_s": 3.2,
|
|
"eval": null
|
|
},
|
|
{
|
|
"iteration": 10,
|
|
"steps": 1222,
|
|
"total_steps": 13748,
|
|
"win_rate": 0.703125,
|
|
"avg_reward": 0.9479687500000019,
|
|
"policy_loss": -0.002473038248717785,
|
|
"value_loss": 0.06709853410720826,
|
|
"entropy": 1.30654935836792,
|
|
"approx_kl": 0.02281575556844473,
|
|
"clip_fraction": 0.29250512942671775,
|
|
"lr": 0.0003,
|
|
"iter_time_s": 6.6,
|
|
"eval": {
|
|
"heuristic": {
|
|
"win_rate": 0.8,
|
|
"wins": 40,
|
|
"losses": 0,
|
|
"draws": 0,
|
|
"avg_reward": 1.2044000000000028,
|
|
"avg_steps": 18.62
|
|
},
|
|
"random": {
|
|
"win_rate": 0.9,
|
|
"wins": 45,
|
|
"losses": 0,
|
|
"draws": 0,
|
|
"avg_reward": 1.2917000000000016,
|
|
"avg_steps": 17.18
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"iteration": 11,
|
|
"steps": 1103,
|
|
"total_steps": 14851,
|
|
"win_rate": 0.8125,
|
|
"avg_reward": 1.1564062500000005,
|
|
"policy_loss": -0.010461012786254286,
|
|
"value_loss": 0.04935916438698769,
|
|
"entropy": 1.2921877086162568,
|
|
"approx_kl": 0.026952635822817685,
|
|
"clip_fraction": 0.3000247240066528,
|
|
"lr": 0.0003,
|
|
"iter_time_s": 2.6,
|
|
"eval": null
|
|
},
|
|
{
|
|
"iteration": 12,
|
|
"steps": 1207,
|
|
"total_steps": 16058,
|
|
"win_rate": 0.6875,
|
|
"avg_reward": 0.9566406250000017,
|
|
"policy_loss": -0.00560942068696022,
|
|
"value_loss": 0.05903085861355066,
|
|
"entropy": 1.2191595494747163,
|
|
"approx_kl": 0.027132337307557465,
|
|
"clip_fraction": 0.30670465975999833,
|
|
"lr": 0.0003,
|
|
"iter_time_s": 2.7,
|
|
"eval": null
|
|
},
|
|
{
|
|
"iteration": 13,
|
|
"steps": 1208,
|
|
"total_steps": 17266,
|
|
"win_rate": 0.671875,
|
|
"avg_reward": 0.9802343750000024,
|
|
"policy_loss": 0.0034835866652429106,
|
|
"value_loss": 0.06818591970950365,
|
|
"entropy": 1.173526006937027,
|
|
"approx_kl": 0.030236439034342765,
|
|
"clip_fraction": 0.35459409207105635,
|
|
"lr": 0.0003,
|
|
"iter_time_s": 2.6,
|
|
"eval": null
|
|
},
|
|
{
|
|
"iteration": 14,
|
|
"steps": 1081,
|
|
"total_steps": 18347,
|
|
"win_rate": 0.859375,
|
|
"avg_reward": 1.2310937499999997,
|
|
"policy_loss": 0.0038359984289854767,
|
|
"value_loss": 0.04460644479840994,
|
|
"entropy": 1.1494916677474976,
|
|
"approx_kl": 0.0338701521512121,
|
|
"clip_fraction": 0.3515248090028763,
|
|
"lr": 0.0003,
|
|
"iter_time_s": 2.3,
|
|
"eval": null
|
|
},
|
|
{
|
|
"iteration": 15,
|
|
"steps": 1147,
|
|
"total_steps": 19494,
|
|
"win_rate": 0.75,
|
|
"avg_reward": 1.0788281250000011,
|
|
"policy_loss": -0.0037996288854628803,
|
|
"value_loss": 0.047803055867552756,
|
|
"entropy": 1.180998581647873,
|
|
"approx_kl": 0.024847476603463294,
|
|
"clip_fraction": 0.30751397013664244,
|
|
"lr": 0.0003,
|
|
"iter_time_s": 6.1,
|
|
"eval": {
|
|
"heuristic": {
|
|
"win_rate": 0.76,
|
|
"wins": 38,
|
|
"losses": 0,
|
|
"draws": 0,
|
|
"avg_reward": 1.1164000000000032,
|
|
"avg_steps": 17.84
|
|
},
|
|
"random": {
|
|
"win_rate": 0.8,
|
|
"wins": 40,
|
|
"losses": 0,
|
|
"draws": 0,
|
|
"avg_reward": 1.2531000000000039,
|
|
"avg_steps": 15.72
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"iteration": 16,
|
|
"steps": 1110,
|
|
"total_steps": 20604,
|
|
"win_rate": 0.71875,
|
|
"avg_reward": 1.0292968750000024,
|
|
"policy_loss": -0.006785877654328943,
|
|
"value_loss": 0.07791313901543617,
|
|
"entropy": 1.131992471218109,
|
|
"approx_kl": 0.022502006916329264,
|
|
"clip_fraction": 0.26751907765865324,
|
|
"lr": 0.0003,
|
|
"iter_time_s": 2.7,
|
|
"eval": null
|
|
},
|
|
{
|
|
"iteration": 17,
|
|
"steps": 1228,
|
|
"total_steps": 21832,
|
|
"win_rate": 0.65625,
|
|
"avg_reward": 1.0609375000000012,
|
|
"policy_loss": -0.00714027569629252,
|
|
"value_loss": 0.021760402247309685,
|
|
"entropy": 1.1914505660533905,
|
|
"approx_kl": 0.026522979838773608,
|
|
"clip_fraction": 0.27935815155506133,
|
|
"lr": 0.0003,
|
|
"iter_time_s": 2.8,
|
|
"eval": null
|
|
},
|
|
{
|
|
"iteration": 18,
|
|
"steps": 1092,
|
|
"total_steps": 22924,
|
|
"win_rate": 0.765625,
|
|
"avg_reward": 1.149531250000002,
|
|
"policy_loss": -0.010855858493596316,
|
|
"value_loss": 0.04301616968587041,
|
|
"entropy": 1.1977117359638214,
|
|
"approx_kl": 0.019330517528578638,
|
|
"clip_fraction": 0.23288143426179886,
|
|
"lr": 0.0003,
|
|
"iter_time_s": 2.6,
|
|
"eval": null
|
|
},
|
|
{
|
|
"iteration": 19,
|
|
"steps": 1096,
|
|
"total_steps": 24020,
|
|
"win_rate": 0.78125,
|
|
"avg_reward": 1.1958593749999988,
|
|
"policy_loss": -0.005429750611074269,
|
|
"value_loss": 0.012403596052899956,
|
|
"entropy": 1.1629622042179109,
|
|
"approx_kl": 0.02170905836392194,
|
|
"clip_fraction": 0.28246527910232544,
|
|
"lr": 0.0003,
|
|
"iter_time_s": 2.5,
|
|
"eval": null
|
|
},
|
|
{
|
|
"iteration": 20,
|
|
"steps": 1207,
|
|
"total_steps": 25227,
|
|
"win_rate": 0.671875,
|
|
"avg_reward": 1.0866406250000022,
|
|
"policy_loss": -0.0011286713182926179,
|
|
"value_loss": 0.009118139976635575,
|
|
"entropy": 1.145670509338379,
|
|
"approx_kl": 0.025272513227537274,
|
|
"clip_fraction": 0.3109993994235992,
|
|
"lr": 0.0003,
|
|
"iter_time_s": 6.5,
|
|
"eval": {
|
|
"heuristic": {
|
|
"win_rate": 0.7,
|
|
"wins": 35,
|
|
"losses": 3,
|
|
"draws": 0,
|
|
"avg_reward": 1.0435000000000019,
|
|
"avg_steps": 18.18
|
|
},
|
|
"random": {
|
|
"win_rate": 0.78,
|
|
"wins": 39,
|
|
"losses": 0,
|
|
"draws": 0,
|
|
"avg_reward": 1.2905000000000042,
|
|
"avg_steps": 16.34
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"iteration": 21,
|
|
"steps": 1125,
|
|
"total_steps": 26352,
|
|
"win_rate": 0.703125,
|
|
"avg_reward": 1.1465625000000004,
|
|
"policy_loss": 0.005485015735030174,
|
|
"value_loss": 0.0299223143607378,
|
|
"entropy": 1.0930719196796417,
|
|
"approx_kl": 0.024188996152952315,
|
|
"clip_fraction": 0.29817257076501846,
|
|
"lr": 0.0003,
|
|
"iter_time_s": 2.8,
|
|
"eval": null
|
|
},
|
|
{
|
|
"iteration": 22,
|
|
"steps": 1220,
|
|
"total_steps": 27572,
|
|
"win_rate": 0.609375,
|
|
"avg_reward": 1.0085937500000024,
|
|
"policy_loss": 0.00360870910808444,
|
|
"value_loss": 0.012875276012346148,
|
|
"entropy": 1.0397337973117828,
|
|
"approx_kl": 0.01926320211496204,
|
|
"clip_fraction": 0.21935985311865808,
|
|
"lr": 0.0003,
|
|
"iter_time_s": 2.8,
|
|
"eval": null
|
|
},
|
|
{
|
|
"iteration": 23,
|
|
"steps": 1157,
|
|
"total_steps": 28729,
|
|
"win_rate": 0.65625,
|
|
"avg_reward": 1.0800781250000009,
|
|
"policy_loss": 0.00030471058562397957,
|
|
"value_loss": 0.037745835771784184,
|
|
"entropy": 1.007828888297081,
|
|
"approx_kl": 0.023771162237972022,
|
|
"clip_fraction": 0.2697603389620781,
|
|
"lr": 0.0003,
|
|
"iter_time_s": 2.5,
|
|
"eval": null
|
|
},
|
|
{
|
|
"iteration": 24,
|
|
"steps": 1153,
|
|
"total_steps": 29882,
|
|
"win_rate": 0.6875,
|
|
"avg_reward": 1.1290625,
|
|
"policy_loss": 0.006432641763240099,
|
|
"value_loss": 0.016368746757507324,
|
|
"entropy": 1.0947044909000396,
|
|
"approx_kl": 0.02725688274949789,
|
|
"clip_fraction": 0.27419755458831785,
|
|
"lr": 0.0003,
|
|
"iter_time_s": 2.5,
|
|
"eval": null
|
|
},
|
|
{
|
|
"iteration": 25,
|
|
"steps": 1117,
|
|
"total_steps": 30999,
|
|
"win_rate": 0.734375,
|
|
"avg_reward": 1.170546875000002,
|
|
"policy_loss": 0.007711798511445522,
|
|
"value_loss": 0.02989666103385389,
|
|
"entropy": 1.1145389318466186,
|
|
"approx_kl": 0.03383852019906044,
|
|
"clip_fraction": 0.34988029301166534,
|
|
"lr": 0.0003,
|
|
"iter_time_s": 6.2,
|
|
"eval": {
|
|
"heuristic": {
|
|
"win_rate": 0.64,
|
|
"wins": 32,
|
|
"losses": 0,
|
|
"draws": 0,
|
|
"avg_reward": 1.101100000000003,
|
|
"avg_steps": 17.78
|
|
},
|
|
"random": {
|
|
"win_rate": 0.78,
|
|
"wins": 39,
|
|
"losses": 0,
|
|
"draws": 0,
|
|
"avg_reward": 1.299800000000004,
|
|
"avg_steps": 16.2
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"iteration": 26,
|
|
"steps": 1252,
|
|
"total_steps": 32251,
|
|
"win_rate": 0.5,
|
|
"avg_reward": 0.9272656250000015,
|
|
"policy_loss": 0.007523976406082511,
|
|
"value_loss": 0.031222257157787682,
|
|
"entropy": 1.0343789517879487,
|
|
"approx_kl": 0.021396257262676955,
|
|
"clip_fraction": 0.25609237924218176,
|
|
"lr": 0.0003,
|
|
"iter_time_s": 3.0,
|
|
"eval": null
|
|
},
|
|
{
|
|
"iteration": 27,
|
|
"steps": 1244,
|
|
"total_steps": 33495,
|
|
"win_rate": 0.609375,
|
|
"avg_reward": 1.0660156250000041,
|
|
"policy_loss": 0.005919251055456698,
|
|
"value_loss": 0.015957068512216212,
|
|
"entropy": 1.083721935749054,
|
|
"approx_kl": 0.01973970588296652,
|
|
"clip_fraction": 0.2247159078717232,
|
|
"lr": 0.0003,
|
|
"iter_time_s": 2.7,
|
|
"eval": null
|
|
},
|
|
{
|
|
"iteration": 28,
|
|
"steps": 1149,
|
|
"total_steps": 34644,
|
|
"win_rate": 0.625,
|
|
"avg_reward": 1.0646093750000043,
|
|
"policy_loss": 0.004431715095415711,
|
|
"value_loss": 0.03190975962206721,
|
|
"entropy": 1.0571655869483947,
|
|
"approx_kl": 0.018917062785476446,
|
|
"clip_fraction": 0.23465312644839287,
|
|
"lr": 0.0003,
|
|
"iter_time_s": 2.5,
|
|
"eval": null
|
|
},
|
|
{
|
|
"iteration": 29,
|
|
"steps": 1034,
|
|
"total_steps": 35678,
|
|
"win_rate": 0.75,
|
|
"avg_reward": 1.2274999999999974,
|
|
"policy_loss": 0.01185380546376109,
|
|
"value_loss": 0.016039191838353874,
|
|
"entropy": 1.0714577436447144,
|
|
"approx_kl": 0.026410212204791606,
|
|
"clip_fraction": 0.2848828133195639,
|
|
"lr": 0.0003,
|
|
"iter_time_s": 2.4,
|
|
"eval": null
|
|
},
|
|
{
|
|
"iteration": 30,
|
|
"steps": 1148,
|
|
"total_steps": 36826,
|
|
"win_rate": 0.625,
|
|
"avg_reward": 1.072812500000002,
|
|
"policy_loss": 0.0006800146773457527,
|
|
"value_loss": 0.029333262145519255,
|
|
"entropy": 0.9994152963161469,
|
|
"approx_kl": 0.02694560317322612,
|
|
"clip_fraction": 0.2918409749865532,
|
|
"lr": 0.0003,
|
|
"iter_time_s": 6.3,
|
|
"eval": {
|
|
"heuristic": {
|
|
"win_rate": 0.64,
|
|
"wins": 32,
|
|
"losses": 0,
|
|
"draws": 0,
|
|
"avg_reward": 1.1374000000000037,
|
|
"avg_steps": 18.98
|
|
},
|
|
"random": {
|
|
"win_rate": 0.74,
|
|
"wins": 37,
|
|
"losses": 0,
|
|
"draws": 0,
|
|
"avg_reward": 1.3004000000000024,
|
|
"avg_steps": 15.8
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"iteration": 31,
|
|
"steps": 1199,
|
|
"total_steps": 38025,
|
|
"win_rate": 0.65625,
|
|
"avg_reward": 1.1076562500000016,
|
|
"policy_loss": 0.020057800551876427,
|
|
"value_loss": 0.0327716386411339,
|
|
"entropy": 1.060115024447441,
|
|
"approx_kl": 0.031923200190067294,
|
|
"clip_fraction": 0.3257354900240898,
|
|
"lr": 0.0003,
|
|
"iter_time_s": 2.7,
|
|
"eval": null
|
|
},
|
|
{
|
|
"iteration": 32,
|
|
"steps": 1192,
|
|
"total_steps": 39217,
|
|
"win_rate": 0.515625,
|
|
"avg_reward": 0.9032812500000021,
|
|
"policy_loss": 0.003908977296669036,
|
|
"value_loss": 0.04722748249769211,
|
|
"entropy": 1.0282223045825958,
|
|
"approx_kl": 0.02074765325523913,
|
|
"clip_fraction": 0.24758184552192689,
|
|
"lr": 0.0003,
|
|
"iter_time_s": 2.6,
|
|
"eval": null
|
|
},
|
|
{
|
|
"iteration": 33,
|
|
"steps": 1161,
|
|
"total_steps": 40378,
|
|
"win_rate": 0.578125,
|
|
"avg_reward": 1.0092187500000016,
|
|
"policy_loss": 0.003133119270205498,
|
|
"value_loss": 0.028510933835059406,
|
|
"entropy": 0.9864079296588898,
|
|
"approx_kl": 0.02380186119116843,
|
|
"clip_fraction": 0.24819656684994698,
|
|
"lr": 0.0003,
|
|
"iter_time_s": 2.6,
|
|
"eval": null
|
|
},
|
|
{
|
|
"iteration": 34,
|
|
"steps": 1195,
|
|
"total_steps": 41573,
|
|
"win_rate": 0.53125,
|
|
"avg_reward": 0.9696875000000015,
|
|
"policy_loss": -0.002012246660888195,
|
|
"value_loss": 0.04197124373167753,
|
|
"entropy": 1.01185884475708,
|
|
"approx_kl": 0.024393753707408906,
|
|
"clip_fraction": 0.26160224750638006,
|
|
"lr": 0.0003,
|
|
"iter_time_s": 2.5,
|
|
"eval": null
|
|
},
|
|
{
|
|
"iteration": 35,
|
|
"steps": 1199,
|
|
"total_steps": 42772,
|
|
"win_rate": 0.59375,
|
|
"avg_reward": 0.9597656250000028,
|
|
"policy_loss": 0.002296004630625248,
|
|
"value_loss": 0.07190144527703524,
|
|
"entropy": 1.0509933680295944,
|
|
"approx_kl": 0.01931018950417638,
|
|
"clip_fraction": 0.2356049098074436,
|
|
"lr": 0.0003,
|
|
"iter_time_s": 6.5,
|
|
"eval": {
|
|
"heuristic": {
|
|
"win_rate": 0.54,
|
|
"wins": 27,
|
|
"losses": 1,
|
|
"draws": 0,
|
|
"avg_reward": 0.9665000000000021,
|
|
"avg_steps": 19.84
|
|
},
|
|
"random": {
|
|
"win_rate": 0.68,
|
|
"wins": 34,
|
|
"losses": 0,
|
|
"draws": 0,
|
|
"avg_reward": 1.2469000000000028,
|
|
"avg_steps": 17.86
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"iteration": 36,
|
|
"steps": 1237,
|
|
"total_steps": 44009,
|
|
"win_rate": 0.453125,
|
|
"avg_reward": 0.8427343750000016,
|
|
"policy_loss": 0.004424499906599521,
|
|
"value_loss": 0.06590159330517054,
|
|
"entropy": 1.0472909182310104,
|
|
"approx_kl": 0.022483129985630512,
|
|
"clip_fraction": 0.27134683132171633,
|
|
"lr": 0.0003,
|
|
"iter_time_s": 3.1,
|
|
"eval": null
|
|
},
|
|
{
|
|
"iteration": 37,
|
|
"steps": 1166,
|
|
"total_steps": 45175,
|
|
"win_rate": 0.6875,
|
|
"avg_reward": 1.2154687500000012,
|
|
"policy_loss": -0.0011765243485569953,
|
|
"value_loss": 0.010787395993247628,
|
|
"entropy": 0.9558039128780365,
|
|
"approx_kl": 0.02246966273523867,
|
|
"clip_fraction": 0.27775362953543664,
|
|
"lr": 0.0003,
|
|
"iter_time_s": 2.6,
|
|
"eval": null
|
|
},
|
|
{
|
|
"iteration": 38,
|
|
"steps": 1173,
|
|
"total_steps": 46348,
|
|
"win_rate": 0.65625,
|
|
"avg_reward": 1.1479687500000002,
|
|
"policy_loss": 0.009084806125611068,
|
|
"value_loss": 0.01519847046583891,
|
|
"entropy": 0.8809868097305298,
|
|
"approx_kl": 0.023764792596921325,
|
|
"clip_fraction": 0.24463087394833566,
|
|
"lr": 0.0003,
|
|
"iter_time_s": 2.7,
|
|
"eval": null
|
|
},
|
|
{
|
|
"iteration": 39,
|
|
"steps": 1139,
|
|
"total_steps": 47487,
|
|
"win_rate": 0.71875,
|
|
"avg_reward": 1.2199218749999998,
|
|
"policy_loss": 0.01850648527033627,
|
|
"value_loss": 0.02945706658065319,
|
|
"entropy": 0.9269103765487671,
|
|
"approx_kl": 0.021095982659608124,
|
|
"clip_fraction": 0.2479160964488983,
|
|
"lr": 0.0003,
|
|
"iter_time_s": 3.0,
|
|
"eval": null
|
|
},
|
|
{
|
|
"iteration": 40,
|
|
"steps": 1176,
|
|
"total_steps": 48663,
|
|
"win_rate": 0.734375,
|
|
"avg_reward": 1.2590624999999998,
|
|
"policy_loss": 0.004166148672811687,
|
|
"value_loss": 0.008350796531885862,
|
|
"entropy": 0.8470023930072784,
|
|
"approx_kl": 0.03418059768155217,
|
|
"clip_fraction": 0.3083778776228428,
|
|
"lr": 0.0003,
|
|
"iter_time_s": 6.3,
|
|
"eval": {
|
|
"heuristic": {
|
|
"win_rate": 0.56,
|
|
"wins": 28,
|
|
"losses": 0,
|
|
"draws": 0,
|
|
"avg_reward": 1.059700000000003,
|
|
"avg_steps": 19.0
|
|
},
|
|
"random": {
|
|
"win_rate": 0.58,
|
|
"wins": 29,
|
|
"losses": 0,
|
|
"draws": 0,
|
|
"avg_reward": 1.1845000000000028,
|
|
"avg_steps": 18.5
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"iteration": 41,
|
|
"steps": 1235,
|
|
"total_steps": 49898,
|
|
"win_rate": 0.484375,
|
|
"avg_reward": 0.9888281250000027,
|
|
"policy_loss": 0.013636557012796402,
|
|
"value_loss": 0.010378800705075264,
|
|
"entropy": 0.7575703978538513,
|
|
"approx_kl": 0.027355002472177148,
|
|
"clip_fraction": 0.258442872390151,
|
|
"lr": 0.0003,
|
|
"iter_time_s": 2.7,
|
|
"eval": null
|
|
},
|
|
{
|
|
"iteration": 42,
|
|
"steps": 1310,
|
|
"total_steps": 51208,
|
|
"win_rate": 0.46875,
|
|
"avg_reward": 0.9570312500000016,
|
|
"policy_loss": -0.002983899631847938,
|
|
"value_loss": 0.01536702981684357,
|
|
"entropy": 0.7948847934603691,
|
|
"approx_kl": 0.02986454094449679,
|
|
"clip_fraction": 0.2483615471671025,
|
|
"lr": 0.0003,
|
|
"iter_time_s": 3.1,
|
|
"eval": null
|
|
},
|
|
{
|
|
"iteration": 43,
|
|
"steps": 1303,
|
|
"total_steps": 52511,
|
|
"win_rate": 0.40625,
|
|
"avg_reward": 0.8580468750000019,
|
|
"policy_loss": 0.014353306110327443,
|
|
"value_loss": 0.02863587177125737,
|
|
"entropy": 0.7559991478919983,
|
|
"approx_kl": 0.02775152454463144,
|
|
"clip_fraction": 0.25815925064186257,
|
|
"lr": 0.0003,
|
|
"iter_time_s": 2.9,
|
|
"eval": null
|
|
},
|
|
{
|
|
"iteration": 44,
|
|
"steps": 1325,
|
|
"total_steps": 53836,
|
|
"win_rate": 0.390625,
|
|
"avg_reward": 0.8401562500000009,
|
|
"policy_loss": 0.018518641125410795,
|
|
"value_loss": 0.010280470092160007,
|
|
"entropy": 0.7594694073001543,
|
|
"approx_kl": 0.02607024977138887,
|
|
"clip_fraction": 0.24557653504113355,
|
|
"lr": 0.0003,
|
|
"iter_time_s": 2.8,
|
|
"eval": null
|
|
},
|
|
{
|
|
"iteration": 45,
|
|
"steps": 1376,
|
|
"total_steps": 55212,
|
|
"win_rate": 0.328125,
|
|
"avg_reward": 0.7929687499999993,
|
|
"policy_loss": 0.015496008951837817,
|
|
"value_loss": 0.026855531226222713,
|
|
"entropy": 0.7538019220034281,
|
|
"approx_kl": 0.039623169034409024,
|
|
"clip_fraction": 0.2917209217945735,
|
|
"lr": 0.0003,
|
|
"iter_time_s": 7.3,
|
|
"eval": {
|
|
"heuristic": {
|
|
"win_rate": 0.34,
|
|
"wins": 17,
|
|
"losses": 1,
|
|
"draws": 0,
|
|
"avg_reward": 0.8111999999999988,
|
|
"avg_steps": 21.42
|
|
},
|
|
"random": {
|
|
"win_rate": 0.6,
|
|
"wins": 30,
|
|
"losses": 0,
|
|
"draws": 0,
|
|
"avg_reward": 1.1957000000000009,
|
|
"avg_steps": 18.66
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"iteration": 46,
|
|
"steps": 1331,
|
|
"total_steps": 56543,
|
|
"win_rate": 0.484375,
|
|
"avg_reward": 0.9676562500000008,
|
|
"policy_loss": 0.007094377962251504,
|
|
"value_loss": 0.014082322634446124,
|
|
"entropy": 0.7922350019216537,
|
|
"approx_kl": 0.02245047929075857,
|
|
"clip_fraction": 0.23407820301751295,
|
|
"lr": 0.0003,
|
|
"iter_time_s": 3.0,
|
|
"eval": null
|
|
},
|
|
{
|
|
"iteration": 47,
|
|
"steps": 1276,
|
|
"total_steps": 57819,
|
|
"win_rate": 0.515625,
|
|
"avg_reward": 1.039453125000001,
|
|
"policy_loss": 0.016592893470078706,
|
|
"value_loss": 0.01359582571312785,
|
|
"entropy": 0.7457897812128067,
|
|
"approx_kl": 0.054725961107760665,
|
|
"clip_fraction": 0.3385230705142021,
|
|
"lr": 0.0003,
|
|
"iter_time_s": 2.6,
|
|
"eval": null
|
|
},
|
|
{
|
|
"iteration": 48,
|
|
"steps": 1386,
|
|
"total_steps": 59205,
|
|
"win_rate": 0.390625,
|
|
"avg_reward": 0.9171875000000009,
|
|
"policy_loss": 0.013037714214685062,
|
|
"value_loss": 0.01347229816019535,
|
|
"entropy": 0.8071639612317085,
|
|
"approx_kl": 0.03561922296648845,
|
|
"clip_fraction": 0.3028787101308505,
|
|
"lr": 0.0003,
|
|
"iter_time_s": 2.8,
|
|
"eval": null
|
|
},
|
|
{
|
|
"iteration": 49,
|
|
"steps": 1356,
|
|
"total_steps": 60561,
|
|
"win_rate": 0.421875,
|
|
"avg_reward": 0.9194531250000028,
|
|
"policy_loss": 0.006827692423636715,
|
|
"value_loss": 0.014162155721957484,
|
|
"entropy": 0.7698061466217041,
|
|
"approx_kl": 0.03132808728454014,
|
|
"clip_fraction": 0.26936849020421505,
|
|
"lr": 0.0003,
|
|
"iter_time_s": 2.7,
|
|
"eval": null
|
|
},
|
|
{
|
|
"iteration": 50,
|
|
"steps": 1309,
|
|
"total_steps": 61870,
|
|
"win_rate": 0.5,
|
|
"avg_reward": 1.0132031250000026,
|
|
"policy_loss": 0.023103898856788874,
|
|
"value_loss": 0.015820173546671867,
|
|
"entropy": 0.8345763012766838,
|
|
"approx_kl": 0.0221639977923284,
|
|
"clip_fraction": 0.24321457371115685,
|
|
"lr": 0.0003,
|
|
"iter_time_s": 6.8,
|
|
"eval": {
|
|
"heuristic": {
|
|
"win_rate": 0.54,
|
|
"wins": 27,
|
|
"losses": 1,
|
|
"draws": 0,
|
|
"avg_reward": 1.050800000000001,
|
|
"avg_steps": 19.86
|
|
},
|
|
"random": {
|
|
"win_rate": 0.6,
|
|
"wins": 30,
|
|
"losses": 0,
|
|
"draws": 0,
|
|
"avg_reward": 1.2008000000000019,
|
|
"avg_steps": 20.02
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"iteration": 51,
|
|
"steps": 1341,
|
|
"total_steps": 63211,
|
|
"win_rate": 0.53125,
|
|
"avg_reward": 1.0278906250000022,
|
|
"policy_loss": 0.0034147894475609064,
|
|
"value_loss": 0.012333790674650421,
|
|
"entropy": 0.8348900377750397,
|
|
"approx_kl": 0.028805966100965936,
|
|
"clip_fraction": 0.2779280847559373,
|
|
"lr": 0.0003,
|
|
"iter_time_s": 3.0,
|
|
"eval": null
|
|
},
|
|
{
|
|
"iteration": 52,
|
|
"steps": 1351,
|
|
"total_steps": 64562,
|
|
"win_rate": 0.484375,
|
|
"avg_reward": 0.996796875,
|
|
"policy_loss": 0.010866860393434763,
|
|
"value_loss": 0.007469336230618258,
|
|
"entropy": 0.9120712007085482,
|
|
"approx_kl": 0.02744163790096839,
|
|
"clip_fraction": 0.2810115882505973,
|
|
"lr": 0.0003,
|
|
"iter_time_s": 3.1,
|
|
"eval": null
|
|
},
|
|
{
|
|
"iteration": 53,
|
|
"steps": 1334,
|
|
"total_steps": 65896,
|
|
"win_rate": 0.40625,
|
|
"avg_reward": 0.854453125000001,
|
|
"policy_loss": -0.010075381025671959,
|
|
"value_loss": 0.04857333633117378,
|
|
"entropy": 0.8342286249001821,
|
|
"approx_kl": 0.028230270914112527,
|
|
"clip_fraction": 0.2692720393339793,
|
|
"lr": 0.0003,
|
|
"iter_time_s": 3.5,
|
|
"eval": null
|
|
},
|
|
{
|
|
"iteration": 54,
|
|
"steps": 1355,
|
|
"total_steps": 67251,
|
|
"win_rate": 0.390625,
|
|
"avg_reward": 0.8151562500000018,
|
|
"policy_loss": -0.006614457195003827,
|
|
"value_loss": 0.06160727143287659,
|
|
"entropy": 0.8476736123363177,
|
|
"approx_kl": 0.022517472117518384,
|
|
"clip_fraction": 0.2495117193708817,
|
|
"lr": 0.0003,
|
|
"iter_time_s": 3.2,
|
|
"eval": null
|
|
},
|
|
{
|
|
"iteration": 55,
|
|
"steps": 1289,
|
|
"total_steps": 68540,
|
|
"win_rate": 0.5625,
|
|
"avg_reward": 1.1075000000000008,
|
|
"policy_loss": -0.00904684750518451,
|
|
"value_loss": 0.01066452645075818,
|
|
"entropy": 0.7499511614441872,
|
|
"approx_kl": 0.034616756330554686,
|
|
"clip_fraction": 0.30258969962596893,
|
|
"lr": 0.0003,
|
|
"iter_time_s": 7.0,
|
|
"eval": {
|
|
"heuristic": {
|
|
"win_rate": 0.48,
|
|
"wins": 24,
|
|
"losses": 0,
|
|
"draws": 0,
|
|
"avg_reward": 0.9893000000000002,
|
|
"avg_steps": 20.82
|
|
},
|
|
"random": {
|
|
"win_rate": 0.68,
|
|
"wins": 34,
|
|
"losses": 0,
|
|
"draws": 0,
|
|
"avg_reward": 1.3218000000000032,
|
|
"avg_steps": 18.86
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"iteration": 56,
|
|
"steps": 1347,
|
|
"total_steps": 69887,
|
|
"win_rate": 0.484375,
|
|
"avg_reward": 0.9713281250000005,
|
|
"policy_loss": 0.012595745579650005,
|
|
"value_loss": 0.030805843494211633,
|
|
"entropy": 0.8057657579580942,
|
|
"approx_kl": 0.05202590821621319,
|
|
"clip_fraction": 0.338998365526398,
|
|
"lr": 0.0003,
|
|
"iter_time_s": 2.9,
|
|
"eval": null
|
|
},
|
|
{
|
|
"iteration": 57,
|
|
"steps": 1311,
|
|
"total_steps": 71198,
|
|
"win_rate": 0.453125,
|
|
"avg_reward": 0.9245312500000025,
|
|
"policy_loss": 0.0012456322729121894,
|
|
"value_loss": 0.01383860552838693,
|
|
"entropy": 0.8018676713109016,
|
|
"approx_kl": 0.027424307928110164,
|
|
"clip_fraction": 0.2673313580453396,
|
|
"lr": 0.0003,
|
|
"iter_time_s": 2.9,
|
|
"eval": null
|
|
},
|
|
{
|
|
"iteration": 58,
|
|
"steps": 1236,
|
|
"total_steps": 72434,
|
|
"win_rate": 0.625,
|
|
"avg_reward": 1.1692187500000002,
|
|
"policy_loss": 0.010585575504228473,
|
|
"value_loss": 0.013045155070722103,
|
|
"entropy": 0.7218925058841705,
|
|
"approx_kl": 0.02843028982169926,
|
|
"clip_fraction": 0.26753021851181985,
|
|
"lr": 0.0003,
|
|
"iter_time_s": 2.9,
|
|
"eval": null
|
|
},
|
|
{
|
|
"iteration": 59,
|
|
"steps": 1315,
|
|
"total_steps": 73749,
|
|
"win_rate": 0.421875,
|
|
"avg_reward": 0.9501562500000008,
|
|
"policy_loss": -0.0014484194883455832,
|
|
"value_loss": 0.013150460241983334,
|
|
"entropy": 0.7294349918762842,
|
|
"approx_kl": 0.019765107601415366,
|
|
"clip_fraction": 0.22568824514746666,
|
|
"lr": 0.0003,
|
|
"iter_time_s": 3.1,
|
|
"eval": null
|
|
},
|
|
{
|
|
"iteration": 60,
|
|
"steps": 1270,
|
|
"total_steps": 75019,
|
|
"win_rate": 0.515625,
|
|
"avg_reward": 1.0507031250000005,
|
|
"policy_loss": 0.004877969576045871,
|
|
"value_loss": 0.013925072271376848,
|
|
"entropy": 0.6912031382322311,
|
|
"approx_kl": 0.02262979713268578,
|
|
"clip_fraction": 0.23088795617222785,
|
|
"lr": 0.0003,
|
|
"iter_time_s": 6.8,
|
|
"eval": {
|
|
"heuristic": {
|
|
"win_rate": 0.52,
|
|
"wins": 26,
|
|
"losses": 0,
|
|
"draws": 0,
|
|
"avg_reward": 1.0845000000000034,
|
|
"avg_steps": 20.18
|
|
},
|
|
"random": {
|
|
"win_rate": 0.72,
|
|
"wins": 36,
|
|
"losses": 0,
|
|
"draws": 0,
|
|
"avg_reward": 1.341400000000002,
|
|
"avg_steps": 18.06
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"iteration": 61,
|
|
"steps": 1318,
|
|
"total_steps": 76337,
|
|
"win_rate": 0.59375,
|
|
"avg_reward": 1.1479687500000018,
|
|
"policy_loss": -0.00020949790875116983,
|
|
"value_loss": 0.01354568082994471,
|
|
"entropy": 0.5942522138357162,
|
|
"approx_kl": 0.028875736248058576,
|
|
"clip_fraction": 0.23885519554217657,
|
|
"lr": 0.0003,
|
|
"iter_time_s": 3.1,
|
|
"eval": null
|
|
},
|
|
{
|
|
"iteration": 62,
|
|
"steps": 1388,
|
|
"total_steps": 77725,
|
|
"win_rate": 0.3125,
|
|
"avg_reward": 0.8124218749999996,
|
|
"policy_loss": 0.011281685748447975,
|
|
"value_loss": 0.008082649127269784,
|
|
"entropy": 0.6261685689290365,
|
|
"approx_kl": 0.01669160396947215,
|
|
"clip_fraction": 0.16309799378116926,
|
|
"lr": 0.0003,
|
|
"iter_time_s": 3.0,
|
|
"eval": null
|
|
},
|
|
{
|
|
"iteration": 63,
|
|
"steps": 1356,
|
|
"total_steps": 79081,
|
|
"win_rate": 0.390625,
|
|
"avg_reward": 0.9098437500000028,
|
|
"policy_loss": 0.01536564288350443,
|
|
"value_loss": 0.013435593883817395,
|
|
"entropy": 0.5593125397960345,
|
|
"approx_kl": 0.02736877358984202,
|
|
"clip_fraction": 0.21733655408024788,
|
|
"lr": 0.0003,
|
|
"iter_time_s": 2.9,
|
|
"eval": null
|
|
},
|
|
{
|
|
"iteration": 64,
|
|
"steps": 1423,
|
|
"total_steps": 80504,
|
|
"win_rate": 0.265625,
|
|
"avg_reward": 0.7265625000000006,
|
|
"policy_loss": 0.010744929362166053,
|
|
"value_loss": 0.016777138303344447,
|
|
"entropy": 0.607121412952741,
|
|
"approx_kl": 0.024137049137304228,
|
|
"clip_fraction": 0.1956334679077069,
|
|
"lr": 0.0003,
|
|
"iter_time_s": 3.1,
|
|
"eval": null
|
|
},
|
|
{
|
|
"iteration": 65,
|
|
"steps": 1449,
|
|
"total_steps": 81953,
|
|
"win_rate": 0.1875,
|
|
"avg_reward": 0.6361718749999996,
|
|
"policy_loss": 0.010331181498865286,
|
|
"value_loss": 0.012753426368969182,
|
|
"entropy": 0.6716435725490252,
|
|
"approx_kl": 0.03430399093000839,
|
|
"clip_fraction": 0.27085066959261894,
|
|
"lr": 0.0003,
|
|
"iter_time_s": 7.9,
|
|
"eval": {
|
|
"heuristic": {
|
|
"win_rate": 0.28,
|
|
"wins": 14,
|
|
"losses": 0,
|
|
"draws": 0,
|
|
"avg_reward": 0.7197999999999999,
|
|
"avg_steps": 22.52
|
|
},
|
|
"random": {
|
|
"win_rate": 0.4,
|
|
"wins": 20,
|
|
"losses": 0,
|
|
"draws": 0,
|
|
"avg_reward": 0.976300000000001,
|
|
"avg_steps": 21.42
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"iteration": 66,
|
|
"steps": 1369,
|
|
"total_steps": 83322,
|
|
"win_rate": 0.328125,
|
|
"avg_reward": 0.864375000000001,
|
|
"policy_loss": 0.0001025744325791796,
|
|
"value_loss": 0.017112249741330743,
|
|
"entropy": 0.6215351969003677,
|
|
"approx_kl": 0.019021677668206394,
|
|
"clip_fraction": 0.20165649894624949,
|
|
"lr": 0.0003,
|
|
"iter_time_s": 3.1,
|
|
"eval": null
|
|
},
|
|
{
|
|
"iteration": 67,
|
|
"steps": 1451,
|
|
"total_steps": 84773,
|
|
"win_rate": 0.265625,
|
|
"avg_reward": 0.7439062499999999,
|
|
"policy_loss": 0.003350305176960925,
|
|
"value_loss": 0.01410901042011877,
|
|
"entropy": 0.6218760733803114,
|
|
"approx_kl": 0.0242579133870701,
|
|
"clip_fraction": 0.2009462956339121,
|
|
"lr": 0.0003,
|
|
"iter_time_s": 3.3,
|
|
"eval": null
|
|
},
|
|
{
|
|
"iteration": 68,
|
|
"steps": 1332,
|
|
"total_steps": 86105,
|
|
"win_rate": 0.453125,
|
|
"avg_reward": 1.0317968750000026,
|
|
"policy_loss": 0.02045266512626161,
|
|
"value_loss": 0.014924520859494805,
|
|
"entropy": 0.5806841254234314,
|
|
"approx_kl": 0.039885272504761815,
|
|
"clip_fraction": 0.2675405666232109,
|
|
"lr": 0.0003,
|
|
"iter_time_s": 3.1,
|
|
"eval": null
|
|
},
|
|
{
|
|
"iteration": 69,
|
|
"steps": 1398,
|
|
"total_steps": 87503,
|
|
"win_rate": 0.296875,
|
|
"avg_reward": 0.7963281249999992,
|
|
"policy_loss": 0.00918305964053919,
|
|
"value_loss": 0.016637851988586288,
|
|
"entropy": 0.5592945193250974,
|
|
"approx_kl": 0.021566734416410327,
|
|
"clip_fraction": 0.1969290692359209,
|
|
"lr": 0.0003,
|
|
"iter_time_s": 3.1,
|
|
"eval": null
|
|
},
|
|
{
|
|
"iteration": 70,
|
|
"steps": 1428,
|
|
"total_steps": 88931,
|
|
"win_rate": 0.296875,
|
|
"avg_reward": 0.7833593750000003,
|
|
"policy_loss": 0.02045505199445567,
|
|
"value_loss": 0.04037536564283073,
|
|
"entropy": 0.5396706946194172,
|
|
"approx_kl": 0.0499255652151381,
|
|
"clip_fraction": 0.2165153439467152,
|
|
"lr": 0.0003,
|
|
"iter_time_s": 7.8,
|
|
"eval": {
|
|
"heuristic": {
|
|
"win_rate": 0.14,
|
|
"wins": 7,
|
|
"losses": 0,
|
|
"draws": 0,
|
|
"avg_reward": 0.5441999999999987,
|
|
"avg_steps": 22.88
|
|
},
|
|
"random": {
|
|
"win_rate": 0.34,
|
|
"wins": 17,
|
|
"losses": 0,
|
|
"draws": 0,
|
|
"avg_reward": 0.8910000000000019,
|
|
"avg_steps": 20.4
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"iteration": 71,
|
|
"steps": 1350,
|
|
"total_steps": 90281,
|
|
"win_rate": 0.328125,
|
|
"avg_reward": 0.8021093750000003,
|
|
"policy_loss": 0.008878157784541449,
|
|
"value_loss": 0.0193236213332663,
|
|
"entropy": 0.5871320739388466,
|
|
"approx_kl": 0.02288884933417042,
|
|
"clip_fraction": 0.19782366200039783,
|
|
"lr": 0.0003,
|
|
"iter_time_s": 3.2,
|
|
"eval": null
|
|
},
|
|
{
|
|
"iteration": 72,
|
|
"steps": 1364,
|
|
"total_steps": 91645,
|
|
"win_rate": 0.34375,
|
|
"avg_reward": 0.8377343750000016,
|
|
"policy_loss": 0.009660454893795153,
|
|
"value_loss": 0.016862792739023764,
|
|
"entropy": 0.5705930987993876,
|
|
"approx_kl": 0.022315170111445088,
|
|
"clip_fraction": 0.21855623833835125,
|
|
"lr": 0.0003,
|
|
"iter_time_s": 3.0,
|
|
"eval": null
|
|
},
|
|
{
|
|
"iteration": 73,
|
|
"steps": 1416,
|
|
"total_steps": 93061,
|
|
"win_rate": 0.234375,
|
|
"avg_reward": 0.6853906249999995,
|
|
"policy_loss": 0.007733553337554137,
|
|
"value_loss": 0.014263311789060632,
|
|
"entropy": 0.5929666409889857,
|
|
"approx_kl": 0.01741509622661397,
|
|
"clip_fraction": 0.16493374668061733,
|
|
"lr": 0.0003,
|
|
"iter_time_s": 3.2,
|
|
"eval": null
|
|
},
|
|
{
|
|
"iteration": 74,
|
|
"steps": 1397,
|
|
"total_steps": 94458,
|
|
"win_rate": 0.28125,
|
|
"avg_reward": 0.7521093749999993,
|
|
"policy_loss": 0.016447271045763046,
|
|
"value_loss": 0.014958841803794106,
|
|
"entropy": 0.5728374073902766,
|
|
"approx_kl": 0.026959338147814076,
|
|
"clip_fraction": 0.22699931201835474,
|
|
"lr": 0.0003,
|
|
"iter_time_s": 3.2,
|
|
"eval": null
|
|
},
|
|
{
|
|
"iteration": 75,
|
|
"steps": 1462,
|
|
"total_steps": 95920,
|
|
"win_rate": 0.203125,
|
|
"avg_reward": 0.6914843750000002,
|
|
"policy_loss": 0.012679687623555461,
|
|
"value_loss": 0.00775904538265119,
|
|
"entropy": 0.6515007962783178,
|
|
"approx_kl": 0.03668231831397861,
|
|
"clip_fraction": 0.2550008601198594,
|
|
"lr": 0.0003,
|
|
"iter_time_s": 7.9,
|
|
"eval": {
|
|
"heuristic": {
|
|
"win_rate": 0.44,
|
|
"wins": 22,
|
|
"losses": 0,
|
|
"draws": 0,
|
|
"avg_reward": 1.0242000000000004,
|
|
"avg_steps": 19.98
|
|
},
|
|
"random": {
|
|
"win_rate": 0.44,
|
|
"wins": 22,
|
|
"losses": 0,
|
|
"draws": 0,
|
|
"avg_reward": 1.0380000000000016,
|
|
"avg_steps": 20.92
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"iteration": 76,
|
|
"steps": 1465,
|
|
"total_steps": 97385,
|
|
"win_rate": 0.296875,
|
|
"avg_reward": 0.8106250000000006,
|
|
"policy_loss": 0.003089776805912455,
|
|
"value_loss": 0.014444326089384655,
|
|
"entropy": 0.6166619087258974,
|
|
"approx_kl": 0.018641260724204283,
|
|
"clip_fraction": 0.19305233067522445,
|
|
"lr": 0.0003,
|
|
"iter_time_s": 3.1,
|
|
"eval": null
|
|
},
|
|
{
|
|
"iteration": 77,
|
|
"steps": 1415,
|
|
"total_steps": 98800,
|
|
"win_rate": 0.3125,
|
|
"avg_reward": 0.8193750000000026,
|
|
"policy_loss": 0.003528349606009821,
|
|
"value_loss": 0.009929708127553264,
|
|
"entropy": 0.5725346008936564,
|
|
"approx_kl": 0.026436600465482723,
|
|
"clip_fraction": 0.2133222414801518,
|
|
"lr": 0.0003,
|
|
"iter_time_s": 3.3,
|
|
"eval": null
|
|
},
|
|
{
|
|
"iteration": 78,
|
|
"steps": 1382,
|
|
"total_steps": 100182,
|
|
"win_rate": 0.421875,
|
|
"avg_reward": 0.9722656250000004,
|
|
"policy_loss": 0.007570690358988941,
|
|
"value_loss": 0.015509422325218717,
|
|
"entropy": 0.6115185245871544,
|
|
"approx_kl": 0.02167838648892939,
|
|
"clip_fraction": 0.20064210643370947,
|
|
"lr": 0.0003,
|
|
"iter_time_s": 3.2,
|
|
"eval": null
|
|
},
|
|
{
|
|
"iteration": 79,
|
|
"steps": 1430,
|
|
"total_steps": 101612,
|
|
"win_rate": 0.34375,
|
|
"avg_reward": 0.8603124999999997,
|
|
"policy_loss": 0.015373940443775306,
|
|
"value_loss": 0.008587169577367604,
|
|
"entropy": 0.5425322167575359,
|
|
"approx_kl": 0.028265317901968956,
|
|
"clip_fraction": 0.22034505320092043,
|
|
"lr": 0.0003,
|
|
"iter_time_s": 3.2,
|
|
"eval": null
|
|
},
|
|
{
|
|
"iteration": 80,
|
|
"steps": 1365,
|
|
"total_steps": 102977,
|
|
"win_rate": 0.453125,
|
|
"avg_reward": 1.0218750000000014,
|
|
"policy_loss": 0.020345127132410806,
|
|
"value_loss": 0.01751556635523836,
|
|
"entropy": 0.5783474495013555,
|
|
"approx_kl": 0.03003823314793408,
|
|
"clip_fraction": 0.21791896596550941,
|
|
"lr": 0.0003,
|
|
"iter_time_s": 6.3,
|
|
"eval": {
|
|
"heuristic": {
|
|
"win_rate": 0.5,
|
|
"wins": 25,
|
|
"losses": 1,
|
|
"draws": 0,
|
|
"avg_reward": 1.0299000000000027,
|
|
"avg_steps": 20.08
|
|
},
|
|
"random": {
|
|
"win_rate": 0.66,
|
|
"wins": 33,
|
|
"losses": 0,
|
|
"draws": 0,
|
|
"avg_reward": 1.342400000000002,
|
|
"avg_steps": 19.22
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"iteration": 81,
|
|
"steps": 1279,
|
|
"total_steps": 104256,
|
|
"win_rate": 0.59375,
|
|
"avg_reward": 1.2046875,
|
|
"policy_loss": -0.001347577106207609,
|
|
"value_loss": 0.03214636510238052,
|
|
"entropy": 0.5191598325967789,
|
|
"approx_kl": 0.03263326198793948,
|
|
"clip_fraction": 0.2007529132068157,
|
|
"lr": 0.0003,
|
|
"iter_time_s": 2.6,
|
|
"eval": null
|
|
},
|
|
{
|
|
"iteration": 82,
|
|
"steps": 1394,
|
|
"total_steps": 105650,
|
|
"win_rate": 0.296875,
|
|
"avg_reward": 0.8210156249999991,
|
|
"policy_loss": 0.009702384510698417,
|
|
"value_loss": 0.010336152277886868,
|
|
"entropy": 0.5633179657161236,
|
|
"approx_kl": 0.024753623428599287,
|
|
"clip_fraction": 0.20269097263614336,
|
|
"lr": 0.0003,
|
|
"iter_time_s": 2.9,
|
|
"eval": null
|
|
},
|
|
{
|
|
"iteration": 83,
|
|
"steps": 1416,
|
|
"total_steps": 107066,
|
|
"win_rate": 0.3125,
|
|
"avg_reward": 0.7882812500000003,
|
|
"policy_loss": 0.00959018689657872,
|
|
"value_loss": 0.030197725320855778,
|
|
"entropy": 0.5529410193363825,
|
|
"approx_kl": 0.023030537956704695,
|
|
"clip_fraction": 0.19835707793633142,
|
|
"lr": 0.0003,
|
|
"iter_time_s": 2.8,
|
|
"eval": null
|
|
},
|
|
{
|
|
"iteration": 84,
|
|
"steps": 1447,
|
|
"total_steps": 108513,
|
|
"win_rate": 0.171875,
|
|
"avg_reward": 0.6148437499999977,
|
|
"policy_loss": 0.010928499046713114,
|
|
"value_loss": 0.015084931820941469,
|
|
"entropy": 0.5940684551994005,
|
|
"approx_kl": 0.017563192717110116,
|
|
"clip_fraction": 0.17302017162243524,
|
|
"lr": 0.0003,
|
|
"iter_time_s": 2.9,
|
|
"eval": null
|
|
},
|
|
{
|
|
"iteration": 85,
|
|
"steps": 1363,
|
|
"total_steps": 109876,
|
|
"win_rate": 0.296875,
|
|
"avg_reward": 0.8173437500000014,
|
|
"policy_loss": 0.011203233346653482,
|
|
"value_loss": 0.016516485367901623,
|
|
"entropy": 0.6131842931111654,
|
|
"approx_kl": 0.038998976116999984,
|
|
"clip_fraction": 0.2717334156235059,
|
|
"lr": 0.0003,
|
|
"iter_time_s": 6.9,
|
|
"eval": {
|
|
"heuristic": {
|
|
"win_rate": 0.22,
|
|
"wins": 11,
|
|
"losses": 0,
|
|
"draws": 0,
|
|
"avg_reward": 0.6381999999999985,
|
|
"avg_steps": 22.9
|
|
},
|
|
"random": {
|
|
"win_rate": 0.4,
|
|
"wins": 20,
|
|
"losses": 0,
|
|
"draws": 0,
|
|
"avg_reward": 1.005,
|
|
"avg_steps": 21.52
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"iteration": 86,
|
|
"steps": 1406,
|
|
"total_steps": 111282,
|
|
"win_rate": 0.328125,
|
|
"avg_reward": 0.8557812500000004,
|
|
"policy_loss": 0.008094639129315814,
|
|
"value_loss": 0.01672106721283247,
|
|
"entropy": 0.6154211312532425,
|
|
"approx_kl": 0.02725695368523399,
|
|
"clip_fraction": 0.20764354057610035,
|
|
"lr": 0.0003,
|
|
"iter_time_s": 2.9,
|
|
"eval": null
|
|
},
|
|
{
|
|
"iteration": 87,
|
|
"steps": 1415,
|
|
"total_steps": 112697,
|
|
"win_rate": 0.328125,
|
|
"avg_reward": 0.8129687500000008,
|
|
"policy_loss": 0.004469660809263587,
|
|
"value_loss": 0.03336539499772092,
|
|
"entropy": 0.6228352511922518,
|
|
"approx_kl": 0.02133028859194989,
|
|
"clip_fraction": 0.22343629350264868,
|
|
"lr": 0.0003,
|
|
"iter_time_s": 3.4,
|
|
"eval": null
|
|
},
|
|
{
|
|
"iteration": 88,
|
|
"steps": 1449,
|
|
"total_steps": 114146,
|
|
"win_rate": 0.234375,
|
|
"avg_reward": 0.6709374999999993,
|
|
"policy_loss": 0.007134772138670087,
|
|
"value_loss": 0.013724158867262304,
|
|
"entropy": 0.6901099781195322,
|
|
"approx_kl": 0.0231881175423041,
|
|
"clip_fraction": 0.20616447863479456,
|
|
"lr": 0.0003,
|
|
"iter_time_s": 3.7,
|
|
"eval": null
|
|
},
|
|
{
|
|
"iteration": 89,
|
|
"steps": 1433,
|
|
"total_steps": 115579,
|
|
"win_rate": 0.1875,
|
|
"avg_reward": 0.6476562499999993,
|
|
"policy_loss": 0.006302334717474878,
|
|
"value_loss": 0.014584237166369954,
|
|
"entropy": 0.6675993700822195,
|
|
"approx_kl": 0.01631183367377768,
|
|
"clip_fraction": 0.1842064963032802,
|
|
"lr": 0.0003,
|
|
"iter_time_s": 3.5,
|
|
"eval": null
|
|
},
|
|
{
|
|
"iteration": 90,
|
|
"steps": 1427,
|
|
"total_steps": 117006,
|
|
"win_rate": 0.296875,
|
|
"avg_reward": 0.7750000000000004,
|
|
"policy_loss": 0.009162008801164726,
|
|
"value_loss": 0.04259434365667403,
|
|
"entropy": 0.6236325626571974,
|
|
"approx_kl": 0.022182197426445782,
|
|
"clip_fraction": 0.20754721139868101,
|
|
"lr": 0.0003,
|
|
"iter_time_s": 8.3,
|
|
"eval": {
|
|
"heuristic": {
|
|
"win_rate": 0.34,
|
|
"wins": 17,
|
|
"losses": 2,
|
|
"draws": 0,
|
|
"avg_reward": 0.7706000000000012,
|
|
"avg_steps": 20.82
|
|
},
|
|
"random": {
|
|
"win_rate": 0.46,
|
|
"wins": 23,
|
|
"losses": 0,
|
|
"draws": 0,
|
|
"avg_reward": 1.104000000000001,
|
|
"avg_steps": 20.34
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"iteration": 91,
|
|
"steps": 1389,
|
|
"total_steps": 118395,
|
|
"win_rate": 0.40625,
|
|
"avg_reward": 0.9350781250000013,
|
|
"policy_loss": 0.010848548457336923,
|
|
"value_loss": 0.017817497563858826,
|
|
"entropy": 0.5778475652138392,
|
|
"approx_kl": 0.019782479930048186,
|
|
"clip_fraction": 0.18402976728975773,
|
|
"lr": 0.0003,
|
|
"iter_time_s": 3.8,
|
|
"eval": null
|
|
},
|
|
{
|
|
"iteration": 92,
|
|
"steps": 1408,
|
|
"total_steps": 119803,
|
|
"win_rate": 0.296875,
|
|
"avg_reward": 0.7824218750000008,
|
|
"policy_loss": 0.0075511088750014705,
|
|
"value_loss": 0.013155260045702258,
|
|
"entropy": 0.667034017542998,
|
|
"approx_kl": 0.01883021815835188,
|
|
"clip_fraction": 0.16617838541666666,
|
|
"lr": 0.0003,
|
|
"iter_time_s": 3.4,
|
|
"eval": null
|
|
},
|
|
{
|
|
"iteration": 93,
|
|
"steps": 1355,
|
|
"total_steps": 121158,
|
|
"win_rate": 0.5,
|
|
"avg_reward": 1.0386718750000015,
|
|
"policy_loss": 0.0020229387058255575,
|
|
"value_loss": 0.008497213323911032,
|
|
"entropy": 0.6026754081249237,
|
|
"approx_kl": 0.022409609829386074,
|
|
"clip_fraction": 0.18872395964960256,
|
|
"lr": 0.0003,
|
|
"iter_time_s": 3.1,
|
|
"eval": null
|
|
},
|
|
{
|
|
"iteration": 94,
|
|
"steps": 1355,
|
|
"total_steps": 122513,
|
|
"win_rate": 0.359375,
|
|
"avg_reward": 0.8957031249999996,
|
|
"policy_loss": 0.008220238300661245,
|
|
"value_loss": 0.011737800339081636,
|
|
"entropy": 0.6350580255190531,
|
|
"approx_kl": 0.029467108795264114,
|
|
"clip_fraction": 0.24638020930190882,
|
|
"lr": 0.0003,
|
|
"iter_time_s": 3.0,
|
|
"eval": null
|
|
},
|
|
{
|
|
"iteration": 95,
|
|
"steps": 1400,
|
|
"total_steps": 123913,
|
|
"win_rate": 0.3125,
|
|
"avg_reward": 0.8500781250000009,
|
|
"policy_loss": 0.012007383028200517,
|
|
"value_loss": 0.017120347862752777,
|
|
"entropy": 0.5946093201637268,
|
|
"approx_kl": 0.02164389714986707,
|
|
"clip_fraction": 0.1799804698675871,
|
|
"lr": 0.0003,
|
|
"iter_time_s": 7.9,
|
|
"eval": {
|
|
"heuristic": {
|
|
"win_rate": 0.48,
|
|
"wins": 24,
|
|
"losses": 0,
|
|
"draws": 0,
|
|
"avg_reward": 1.0792000000000028,
|
|
"avg_steps": 20.64
|
|
},
|
|
"random": {
|
|
"win_rate": 0.5,
|
|
"wins": 25,
|
|
"losses": 0,
|
|
"draws": 0,
|
|
"avg_reward": 1.151700000000003,
|
|
"avg_steps": 20.14
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"iteration": 96,
|
|
"steps": 1369,
|
|
"total_steps": 125282,
|
|
"win_rate": 0.328125,
|
|
"avg_reward": 0.8985156250000019,
|
|
"policy_loss": -0.00018501221590364972,
|
|
"value_loss": 0.01656651108836134,
|
|
"entropy": 0.6120344822605451,
|
|
"approx_kl": 0.021352123313893873,
|
|
"clip_fraction": 0.18503481956819692,
|
|
"lr": 0.0003,
|
|
"iter_time_s": 3.4,
|
|
"eval": null
|
|
},
|
|
{
|
|
"iteration": 97,
|
|
"steps": 1336,
|
|
"total_steps": 126618,
|
|
"win_rate": 0.5,
|
|
"avg_reward": 1.0882031250000004,
|
|
"policy_loss": -0.004709509861034651,
|
|
"value_loss": 0.019488867799130578,
|
|
"entropy": 0.5576261033614477,
|
|
"approx_kl": 0.020887282269541174,
|
|
"clip_fraction": 0.17699032897750536,
|
|
"lr": 0.0003,
|
|
"iter_time_s": 3.2,
|
|
"eval": null
|
|
},
|
|
{
|
|
"iteration": 98,
|
|
"steps": 1340,
|
|
"total_steps": 127958,
|
|
"win_rate": 0.421875,
|
|
"avg_reward": 0.980156250000002,
|
|
"policy_loss": 0.023723983283465106,
|
|
"value_loss": 0.009692060159674535,
|
|
"entropy": 0.5237270258367062,
|
|
"approx_kl": 0.03357523250936841,
|
|
"clip_fraction": 0.2157118065903584,
|
|
"lr": 0.0003,
|
|
"iter_time_s": 3.4,
|
|
"eval": null
|
|
},
|
|
{
|
|
"iteration": 99,
|
|
"steps": 1406,
|
|
"total_steps": 129364,
|
|
"win_rate": 0.421875,
|
|
"avg_reward": 0.9917968750000035,
|
|
"policy_loss": 0.012965027514534691,
|
|
"value_loss": 0.011839689066012701,
|
|
"entropy": 0.5783841535449028,
|
|
"approx_kl": 0.02461761930802216,
|
|
"clip_fraction": 0.1909257210791111,
|
|
"lr": 0.0003,
|
|
"iter_time_s": 3.4,
|
|
"eval": null
|
|
},
|
|
{
|
|
"iteration": 100,
|
|
"steps": 1452,
|
|
"total_steps": 130816,
|
|
"win_rate": 0.203125,
|
|
"avg_reward": 0.7160937500000014,
|
|
"policy_loss": 0.010753739159554243,
|
|
"value_loss": 0.015761558703767758,
|
|
"entropy": 0.5649129363397757,
|
|
"approx_kl": 0.03351005217215667,
|
|
"clip_fraction": 0.22037381927172342,
|
|
"lr": 0.0003,
|
|
"iter_time_s": 8.0,
|
|
"eval": {
|
|
"heuristic": {
|
|
"win_rate": 0.2,
|
|
"wins": 10,
|
|
"losses": 0,
|
|
"draws": 0,
|
|
"avg_reward": 0.7156999999999991,
|
|
"avg_steps": 22.58
|
|
},
|
|
"random": {
|
|
"win_rate": 0.5,
|
|
"wins": 25,
|
|
"losses": 0,
|
|
"draws": 0,
|
|
"avg_reward": 1.1457000000000002,
|
|
"avg_steps": 20.22
|
|
}
|
|
}
|
|
}
|
|
] |