{
  "device": "NVIDIA A10G",
  "case1": {
    "case": 1,
    "title": "Compute straggler localization",
    "n_ranks": 8,
    "straggler": 5,
    "onset_step": 15,
    "iter_ms": [
      15.664977259829431,
      15.82563471578891,
      16.186661159498552,
      15.658043459126194,
      15.776530110492292,
      16.067448478946822,
      15.080445433933082,
      16.147568994976766,
      15.58996279123419,
      15.814430235682444,
      15.585981248051551,
      16.258411172483356,
      15.784093544673482,
      15.432473838100055,
      15.618993113560425,
      253.76660031305977,
      256.3939797959979,
      263.58804403366776,
      258.43958481723274,
      244.58336844424704,
      233.96825653486144,
      254.49374854924258,
      261.4154588748323,
      265.6515882854605,
      248.41757477836273,
      231.4881219835869,
      250.14425913794835,
      232.84927761016823,
      260.45000483623886,
      257.604174897873,
      242.3140134414445,
      258.0111767546848,
      267.0784467371099,
      249.80022482115635,
      257.3426792843759,
      258.76712995948463,
      246.1574879827184,
      242.5154492372744,
      245.75242008691635,
      239.85423053410008
    ],
    "l1_change_point": {
      "t": 15,
      "ratio": 15.935058563370104,
      "mu_left": 15.766110370425173,
      "mu_right": 251.23389206928184
    },
    "l1_detected": true,
    "l2": {
      "self_attention_cv": 2.119589582493219,
      "mlp_cv": 1.982874528524489,
      "self_attention_means_ms": [
        5.918040239326188,
        6.0211699911078,
        5.8653321230325774,
        5.900957414421043,
        5.986649195821151,
        148.01483870289053,
        5.916563312290962,
        5.965455713937493
      ],
      "mlp_means_ms": [
        5.051364128351942,
        4.9776376939216425,
        5.003218330826214,
        5.0246729663455945,
        5.066834197366083,
        99.2334357073439,
        5.046776052212535,
        4.982191159970459
      ],
      "self_attention_z": [
        -0.35397380123858724,
        -0.3519207063020223,
        -0.3550231082019885,
        -0.3543138841022525,
        -0.3526079422236146,
        2.474872505357945,
        -0.3540032037278332,
        -0.35302985956164584
      ],
      "mlp_z": [
        -0.3526660519493205,
        -0.35487947045065693,
        -0.3541114873596792,
        -0.3534673752774257,
        -0.3522016088057256,
        2.474872555308562,
        -0.3528037953881694,
        -0.3547427660775846
      ],
      "flagged_ranks": [
        5
      ]
    },
    "l2_detected": true,
    "heatmap_attn_ms": [
      5.918040239326188,
      6.0211699911078,
      5.8653321230325774,
      5.900957414421043,
      5.986649195821151,
      148.01483870289053,
      5.916563312290962,
      5.965455713937493
    ],
    "heatmap_mlp_ms": [
      5.051364128351942,
      4.9776376939216425,
      5.003218330826214,
      5.0246729663455945,
      5.066834197366083,
      99.2334357073439,
      5.046776052212535,
      4.982191159970459
    ]
  },
  "case2": {
    "case": 2,
    "title": "Silent communication link degradation",
    "n_ranks": 8,
    "degraded_ranks": [
      4,
      5,
      6,
      7
    ],
    "observed_iter_ms_median": 27.926844591962546,
    "iter_ms": [
      27.659790258768965,
      28.18087195998671,
      27.70609047236641,
      27.622828264944314,
      27.89650858051991,
      28.40835887429764,
      27.454626032572882,
      27.68760301362933,
      27.49253603283527,
      27.638580956009076,
      27.95543172699075,
      27.60406335114259,
      28.093113256019098,
      28.031745924465167,
      28.21666144351693,
      28.576010630413695,
      27.703457908278125,
      27.855718988284362,
      27.81870861661575,
      27.864350162122733,
      28.12100871494447,
      28.179951306993143,
      27.848211847161938,
      28.399511122772317,
      27.918508799058795,
      27.965233666321282,
      27.73302630778205,
      28.20039260066274,
      28.22371348674258,
      28.00084101082223,
      27.35409705917833,
      27.926844591962546,
      28.177982217955314,
      27.874814393208545,
      27.97098158474208,
      28.153766857947346,
      27.454182380115554,
      28.238411851126763,
      27.431890457655026,
      28.02866192812728
    ],
    "l1_jitter_hits": 0,
    "l1_change_point": null,
    "l1_silent": true,
    "l3_p50_ms": {
      "AllReduce": [
        1.1940545709213606,
        1.1945938745201155,
        1.2068295187747913,
        1.180032963717522,
        9.585603284455772,
        9.613287237806619,
        9.531734302249859,
        9.52714895258882
      ],
      "AllGather": [
        1.9960636208857943,
        2.0341437144789367,
        2.014165609372383,
        1.999447437935789,
        23.990903775344623,
        24.1567315984202,
        23.70216529208328,
        23.814446809739994
      ],
      "ReduceScatter": [
        2.42443740327199,
        2.3801654964540537,
        2.3887106228022237,
        2.398624260951683,
        47.28388086068609,
        47.50860929726771,
        47.99287218842885,
        48.05131706909705
      ]
    },
    "l3_flagged_by_kernel": {
      "AllReduce": [
        4,
        5,
        6,
        7
      ],
      "AllGather": [
        4,
        5,
        6,
        7
      ],
      "ReduceScatter": [
        4,
        5,
        6,
        7
      ]
    },
    "l3_flagged_union": [
      4,
      5,
      6,
      7
    ],
    "l3_detected": true,
    "w1_allreduce_matrix": [
      [
        0.0,
        0.00838110095719547,
        0.013683811225256682,
        0.024335291768492133,
        8.399028481442986,
        8.432036770592005,
        8.346794355188557,
        8.344549592059973
      ],
      [
        0.00838110095719547,
        0.0,
        0.012168828226263449,
        0.018058380992151905,
        8.398040459384923,
        8.431048748533941,
        8.345806333130493,
        8.34356157000191
      ],
      [
        0.013683811225256682,
        0.012168828226263449,
        0.0,
        0.027520472769607472,
        8.38587163115866,
        8.418879920307678,
        8.333637504904232,
        8.331392741775648
      ],
      [
        0.024335291768492133,
        0.018058380992151905,
        0.027520472769607472,
        0.0,
        8.411665491595059,
        8.444673780744077,
        8.359431365340631,
        8.357186602212046
      ],
      [
        8.399028481442986,
        8.398040459384923,
        8.38587163115866,
        8.411665491595059,
        0.0,
        0.0940276309788436,
        0.05603591370137733,
        0.08574314737644699
      ],
      [
        8.432036770592005,
        8.431048748533941,
        8.418879920307678,
        8.444673780744077,
        0.0940276309788436,
        0.0,
        0.09451719415304016,
        0.08748926997875689
      ],
      [
        8.346794355188557,
        8.345806333130493,
        8.333637504904232,
        8.359431365340631,
        0.05603591370137733,
        0.09451719415304016,
        0.0,
        0.0404712034682173
      ],
      [
        8.344549592059973,
        8.34356157000191,
        8.331392741775648,
        8.357186602212046,
        0.08574314737644699,
        0.08748926997875689,
        0.0404712034682173,
        0.0
      ]
    ],
    "w1_intra_mean": 0.04686935379963745,
    "w1_inter_mean": 8.3802253342733,
    "w1_inter_intra_ratio": 178.7996772922891
  },
  "case3": {
    "case": 3,
    "title": "Pipeline bubble amplification / masking",
    "pp_stages": 4,
    "straggler_stage": 3,
    "backward_compute_means_ms": [
      90.89418935096074,
      89.59368790058518,
      91.084155529665,
      171.5906157858056
    ],
    "backward_ratio_vs_peers": 1.8955259939160862,
    "bubble_means_ms": [
      231.1546918195448,
      228.51013674348195,
      228.60243881594658,
      160.33621728339028
    ],
    "fwd_bwd_total_means_ms": [
      11178.045269349963,
      11178.045269349963,
      11178.045269349963,
      11178.045269349963
    ],
    "l1_l2_on_totals": {
      "cv": 0.0,
      "z": [
        0.0,
        0.0,
        0.0,
        0.0
      ],
      "flagged": []
    },
    "l1_l2_silent_on_totals": true,
    "semantics_bwd": {
      "cv": 0.36590371142846645,
      "z": [
        -0.4908017694016493,
        -0.5228822501815938,
        -0.48611572591972824,
        1.4997997455029717
      ],
      "flagged": [
        3
      ]
    },
    "manual_semantics_detected": true
  },
  "case4": {
    "case": 4,
    "title": "Intermittent host/JIT stall",
    "device": "NVIDIA A10G",
    "n_steps": 40,
    "stall_steps": [
      12,
      27
    ],
    "injected_stall_ms": 80.0,
    "wall_ms": [
      1.83002500011753,
      1.83397499995408,
      1.8256850000852864,
      1.8256939999901078,
      1.8233150001378817,
      1.824714999884236,
      1.8465460000243183,
      1.8240950000745215,
      1.8294250000963075,
      1.8260150000060094,
      1.829705000091053,
      1.8330049999804032,
      81.85067600000366,
      1.829645000043456,
      1.8475150000085705,
      1.828574999990451,
      1.8330149998746492,
      1.8269950001013058,
      1.8245349999688187,
      1.8265250000695232,
      1.8287640000380634,
      1.8274139999903127,
      1.8301150000752386,
      1.8276050000167743,
      1.827634999926886,
      1.831295000101818,
      1.8245349999688187,
      81.84363499988649,
      1.8267150001065602,
      1.846616000193535,
      1.8286150000221824,
      1.8277149999903486,
      1.8273850000696257,
      1.827774000048521,
      1.828695000085645,
      1.8407950001346762,
      1.8282949999957054,
      1.8256139999266452,
      1.829664999831948,
      1.828635000038048
    ],
    "gpu_ms": [
      1.807360053062439,
      1.8114559650421143,
      1.8053120374679565,
      1.8042880296707153,
      1.802240014076233,
      1.8032640218734741,
      1.8257919549942017,
      1.802240014076233,
      1.807360053062439,
      1.8042880296707153,
      1.8083839416503906,
      1.8124799728393555,
      1.8237440586090088,
      1.808351993560791,
      1.8257919549942017,
      1.807360053062439,
      1.8124799728393555,
      1.8053120374679565,
      1.8032640218734741,
      1.8053120374679565,
      1.8083839416503906,
      1.8053120374679565,
      1.8094079494476318,
      1.8063360452651978,
      1.8063360452651978,
      1.810431957244873,
      1.8032640218734741,
      1.8165760040283203,
      1.8063360452651978,
      1.8114559650421143,
      1.807360053062439,
      1.806399941444397,
      1.8063360452651978,
      1.8063360452651978,
      1.8083839416503906,
      1.8083839416503906,
      1.807360053062439,
      1.8042880296707153,
      1.8094079494476318,
      1.807360053062439
    ],
    "spike_ratio_vs_normal": 44.76200101195253,
    "l1_jitter_hits": [
      {
        "start": 7,
        "end": 12,
        "ratio": 44.871937040921516
      },
      {
        "start": 8,
        "end": 13,
        "ratio": 44.82475554676949
      },
      {
        "start": 9,
        "end": 14,
        "ratio": 44.82475554676949
      },
      {
        "start": 10,
        "end": 15,
        "ratio": 44.76200101195253
      },
      {
        "start": 11,
        "end": 16,
        "ratio": 44.76200101195253
      },
      {
        "start": 12,
        "end": 17,
        "ratio": 44.80071154845256
      },
      {
        "start": 22,
        "end": 27,
        "ratio": 44.857256781199155
      },
      {
        "start": 23,
        "end": 28,
        "ratio": 44.857256781199155
      },
      {
        "start": 24,
        "end": 29,
        "ratio": 44.857256781199155
      },
      {
        "start": 25,
        "end": 30,
        "ratio": 44.857256781199155
      },
      {
        "start": 26,
        "end": 31,
        "ratio": 44.857256781199155
      },
      {
        "start": 27,
        "end": 32,
        "ratio": 44.80372416885622
      }
    ],
    "l1_detected": true,
    "l2_l3_diluted": true,
    "normal_wall_median_ms": 1.828574999990451,
    "spike_wall_ms": [
      81.85067600000366,
      81.84363499988649
    ],
    "gpu_median_ms": 1.807360053062439
  }
}