| 493 | |
| 494 | |
| 495 | def _add_regularization_args(parser): |
| 496 | group = parser.add_argument_group(title="regularization") |
| 497 | |
| 498 | group.add_argument( |
| 499 | "--attention-dropout", |
| 500 | type=float, |
| 501 | default=0.1, |
| 502 | help="Post attention dropout probability.", |
| 503 | ) |
| 504 | group.add_argument( |
| 505 | "--hidden-dropout", |
| 506 | type=float, |
| 507 | default=0.1, |
| 508 | help="Dropout probability for hidden state transformer.", |
| 509 | ) |
| 510 | group.add_argument( |
| 511 | "--weight-decay", |
| 512 | type=float, |
| 513 | default=0.01, |
| 514 | help="Weight decay coefficient for L2 regularization.", |
| 515 | ) |
| 516 | group.add_argument( |
| 517 | "--tempering", |
| 518 | type=float, |
| 519 | default=None, |
| 520 | help="Tempering coefficient for the model.", |
| 521 | ) |
| 522 | group.add_argument( |
| 523 | "--gold", |
| 524 | action="store_true", |
| 525 | help="If set, use gold regularization.", |
| 526 | ) |
| 527 | group.add_argument( |
| 528 | "--gold-beta", |
| 529 | type=float, |
| 530 | default=0.05, |
| 531 | help="Beta for GOLD tempering.", |
| 532 | ) |
| 533 | group.add_argument( |
| 534 | "--play-tau", |
| 535 | type=float, |
| 536 | default=2.0 |
| 537 | ) |
| 538 | group.add_argument( |
| 539 | "--clip-grad", |
| 540 | type=float, |
| 541 | default=1.0, |
| 542 | help="Gradient clipping based on global L2 norm.", |
| 543 | ) |
| 544 | group.add_argument( |
| 545 | "--adam-beta1", |
| 546 | type=float, |
| 547 | default=0.9, |
| 548 | help="First coefficient for computing running averages " |
| 549 | "of gradient and its square", |
| 550 | ) |
| 551 | group.add_argument( |
| 552 | "--adam-beta2", |