{
  "revision": "b34f32853b1c1260109e5e193a1cab0c15a0d0e4",
  "reviewed": "2026-10-28",
  "conditions": "240 MHz \u00b7 1 core \u00b7 device-timed median \u00b7 README reports 1,000 inferences",
  "caveat": "A review found test-set selection in some CDT runs and sizing studies. Fixes and affected re-measurements are ongoing. These are repository-reported results, not independently reproduced website measurements.",
  "records": [
    {
      "id": "cifar-s3",
      "device": "ESP32-S3",
      "workload": "cifar",
      "title": "CIFAR-10 classification",
      "architecture": "ResNet-8 \u00b7 16\u201332\u201364 \u00b7 2\u20132\u20132 blocks",
      "baseline": "Official MLPerf Tiny ResNet-8 \u00b7 TFLite Micro INT8",
      "dataset": "CIFAR-10",
      "score": "Accuracy",
      "seeds": "SYMBEX: 3 training seeds; INT8: official reference",
      "status": "Reported \u00b7 validation review",
      "source": "benchmarks/conv2d/cnn_cifar_resnet8/README.md",
      "metrics": {
        "latency": [
          40.7,
          52.0
        ],
        "flash": [
          24.3,
          96.2
        ],
        "ram": [
          20,
          80.5
        ],
        "score": [
          71.28,
          87.0
        ],
        "exactness": [
          100,
          98.1
        ]
      },
      "deviations": [
        0.81,
        null
      ],
      "note": "Same ResNet-8 shape. Binary working RAM is about 20 KB (20.2 KB in the detailed sizing table); INT8 uses an 80.5 KB tensor arena on S3. Flash values here are reported model weight footprints, not full firmware sizes."
    },
    {
      "id": "cifar-esp32",
      "device": "ESP32",
      "workload": "cifar",
      "title": "CIFAR-10 classification",
      "architecture": "ResNet-8 \u00b7 16\u201332\u201364 \u00b7 2\u20132\u20132 blocks",
      "baseline": "Official MLPerf Tiny ResNet-8 \u00b7 TFLite Micro INT8",
      "dataset": "CIFAR-10",
      "score": "Accuracy",
      "seeds": "SYMBEX: 3 training seeds; INT8: official reference",
      "status": "Reported \u00b7 validation review",
      "source": "benchmarks/conv2d/cnn_cifar_resnet8/README.md",
      "metrics": {
        "latency": [
          59.7,
          717.8
        ],
        "flash": [
          24.3,
          96.2
        ],
        "ram": [
          103,
          53
        ],
        "score": [
          71.28,
          87.0
        ],
        "exactness": [
          100,
          98.1
        ]
      },
      "deviations": [
        0.81,
        null
      ],
      "note": "59.7 ms uses lut16 and weights in RAM: reported total RAM is 103 KB (20 KB base + 64 KB counting table + resident weights). With weights in flash latency is 64.8 ms and RAM is 84 KB; without the table it is 72.6 ms and 20 KB. INT8 arena is 53 KB. Flash figures are model weight footprints, not full firmware."
    },
    {
      "id": "fomo-s3",
      "device": "ESP32-S3",
      "workload": "fomo",
      "title": "Person-center detection",
      "architecture": "FOMO \u00b7 32\u201332\u2013128 \u00b7 1\u20132\u20132 blocks (resized)",
      "baseline": "Same architecture and training recipe \u00b7 TFLite Micro INT8",
      "dataset": "Pascal VOC persons \u00b7 96 \u00d7 96 RGB \u00b7 12 \u00d7 12 output grid",
      "score": "F1",
      "seeds": "3 seeds per model: 42, 43, 44",
      "status": "Reported \u00b7 sizing selection caveat",
      "source": "benchmarks/conv2d/fomo_voc_person/README.md",
      "metrics": {
        "latency": [
          126.3,
          256.9
        ],
        "flash": [
          41.2,
          236.1
        ],
        "ram": [
          51,
          222.6
        ],
        "score": [
          31.2,
          30.1
        ],
        "exactness": [
          100,
          88
        ]
      },
      "deviations": [
        0.7,
        0.2
      ],
      "note": "Resized matched architecture. Shape selection used single-seed test F1; the next study uses a held-out validation split. RAM on INT8 is its tensor arena. The modest F1 reflects a challenging small-model task."
    },
    {
      "id": "fomo-esp32",
      "device": "ESP32",
      "workload": "fomo",
      "title": "Person-center detection",
      "architecture": "FOMO \u00b7 32\u201364\u2013128 \u00b7 1\u20132\u20132 blocks (original)",
      "baseline": "Same original architecture \u00b7 INT8 arena does not fit",
      "dataset": "Pascal VOC persons \u00b7 96 \u00d7 96 RGB \u00b7 12 \u00d7 12 output grid",
      "score": "F1",
      "seeds": "Original shape: 1 seed",
      "status": "Reported \u00b7 baseline cannot fit",
      "source": "benchmarks/conv2d/fomo_voc_person/README.md",
      "metrics": {
        "latency": [
          296.2,
          null
        ],
        "flash": [
          52.3,
          315.7
        ],
        "ram": [
          53,
          223.3
        ],
        "score": [
          31.0,
          30.2
        ],
        "exactness": [
          100,
          null
        ]
      },
      "deviations": [
        null,
        null
      ],
      "note": "Original shape, not the resized S3 result. INT8 arena overflows classic ESP32 RAM by 97 KB, so there is no INT8 device latency. F1, model footprints and reference-agreement figures are model-level values reported for this original shape; INT8 was evaluated on S3, not on classic ESP32."
    },
    {
      "id": "fashion-esp32",
      "device": "ESP32",
      "workload": "fashion",
      "title": "Fashion-MNIST dense network",
      "architecture": "Fully binary MLP \u00b7 twice the width",
      "baseline": "Conventional MLP \u00b7 quantized INT8",
      "dataset": "Fashion-MNIST",
      "score": "Accuracy",
      "seeds": "SYMBEX: 3 seeds",
      "status": "Being re-measured",
      "source": "benchmarks/mlp/mlp_fashion/README.md",
      "metrics": {
        "latency": [
          16.7,
          41.97
        ],
        "flash": [
          null,
          null
        ],
        "ram": [
          null,
          null
        ],
        "score": [
          86.8,
          87.2
        ],
        "exactness": [
          null,
          null
        ]
      },
      "deviations": [
        0.36,
        null
      ],
      "note": "Provisional legacy figures are being re-measured with the current engine and corrected validation split. Weights: 69 KB binary vs 242 KB INT8; these are weight payloads, not total flash. Model widths differ. No current RAM or full-flash measurements are supplied here."
    }
  ]
}