{"crawledAt":"2026-09-18T16:32:41.740Z","competitions":[{"id":"assetopsbench","name":"AssetOpsBench","tagline":"A unified, open framework for building, orchestrating, and evaluating domain-specific AI agents in Industry 4.0.","description":"AssetOpsBench is an evaluation framework and benchmark for AI agents in industrial asset operations and maintenance. It provides reproducible scenarios, agent tooling, and evaluation pipelines for multi-step workflows in simulated industrial environments. The benchmark powers public competitions co-located with major AI conferences like IJCAI and CODS.","url":"https://github.com/IBM/AssetOpsBench","host":"github","status":"active","tags":["industrial-iot","ai-agents","mcp","predictive-maintenance","industry-4-0"],"domain":"other","images":["https://github.com/user-attachments/assets/615059be-e296-40d3-90ec-97ee6cb00412"],"organizer":{"name":"IBM Research","url":"https://research.ibm.com","type":"company"},"participation":{"howToSubmit":"Participate via the active competition platforms (e.g., Codabench, Kaggle) or contribute new scenarios by submitting a Pull Request.","submissionFormat":"Agent trajectories or predictions depending on the specific competition track","requirements":["Python 3.12+","Docker","uv"],"compute":"unknown","rounds":"CODS 2025, IJCAI 2026","cost":"free"},"quickstart":"git clone https://github.com/IBM/AssetOpsBench.git\ncd AssetOpsBench\npip install -e .\npython -m assetopsbench.run --scenario \"List all sensors of Chiller 6 in MAIN site\"","links":[{"label":"Paper","url":"https://arxiv.org/pdf/2506.03828"},{"label":"Dataset","url":"https://huggingface.co/datasets/ibm-research/AssetOpsBench"},{"label":"Playground","url":"https://huggingface.co/spaces/ibm-research/AssetOps-Bench"},{"label":"Kaggle","url":"https://www.kaggle.com/benchmarks/ibm-research/asset-ops-bench"},{"label":"Codabench (CODS 2025)","url":"https://www.codabench.org/competitions/10206/"},{"label":"IJCAI 2026 Challenge","url":"https://sites.google.com/view/ai-industrial-challenge-ijcai/home"}],"problems":[],"repo":{"owner":"IBM","name":"AssetOpsBench","branch":"main"},"participants":[],"stats":{"totalRecords":0,"uniqueParticipants":0,"recordsLast30d":0,"recordsLast90d":0},"agentStats":[],"lastUpdated":"2026-09-18T16:32:55.531Z"},{"id":"cogdl","name":"CogDL","tagline":"A comprehensive graph deep learning benchmark for node classification, link prediction, and graph classification.","description":"CogDL is a graph deep learning toolkit and benchmark that allows researchers to train and compare baseline or customized models. It features leaderboards for node classification, link prediction, and graph classification across numerous standard datasets. Submissions are made by implementing models in the toolkit and opening pull requests.","url":"https://github.com/THUDM/CogDL","host":"github","status":"ended","tags":["graph-neural-networks","node-classification","link-prediction","graph-classification","benchmark","pytorch"],"domain":"ml-research","images":[],"organizer":{"name":"Tsinghua, ZJU, DAMO Academy, and ZHIPU.AI","url":"https://cogdl.ai/about/","type":"academic"},"participation":{"howToSubmit":"Open a pull request implementing your model under `cogdl/models` and adding a unit test under `tests/tasks/`.","submissionFormat":"Python code implementing the model, along with a unit test and updated results in results.md.","requirements":["Python 3.7+","PyTorch 1.7.1+","Docker"],"compute":"consumer-gpu","computeDetails":"Supports training on GPUs, with fast operators like GE-SpMM and dgNN to reduce memory footprint.","verification":"CI"},"quickstart":"from cogdl import experiment\n\n# Run GCN on Cora dataset\nexperiment(dataset=\"cora\", model=\"gcn\")","links":[{"label":"Homepage","url":"https://cogdl.ai"},{"label":"Paper","url":"https://arxiv.org/abs/2103.00959"},{"label":"Documentation","url":"https://cogdl.readthedocs.io"},{"label":"Discussion Forum","url":"https://discuss.cogdl.ai"}],"problems":[{"id":"unsupervised-node-classification-ppi","name":"Unsupervised Multi-label Node Classification - PPI","description":"Unsupervised multi-label node classification on the PPI dataset, evaluated using Micro-F1 score with 90% labels as training data in L2 normalization logistic regression.","metricName":"Micro-F1","metricUnit":"%","metricDirection":"maximize","records":[{"value":23.73,"contributor":"NetMF","contributorKind":"method"},{"value":24.6,"contributor":"ProNE","contributorKind":"method","isCurrentBest":true},{"value":23.88,"contributor":"NetSMF","contributorKind":"method"},{"value":20.67,"contributor":"Node2vec","contributorKind":"method"},{"value":21.82,"contributor":"LINE","contributorKind":"method"},{"value":20.74,"contributor":"DeepWalk","contributorKind":"method"},{"value":22.48,"contributor":"Spectral","contributorKind":"method"},{"value":21.43,"contributor":"Hope","contributorKind":"method"},{"value":20.6,"contributor":"GraRep","contributorKind":"method"}],"isOpen":false,"contested":true},{"id":"unsupervised-node-classification-wikipedia","name":"Unsupervised Multi-label Node Classification - Wikipedia","description":"Unsupervised multi-label node classification on the Wikipedia dataset, evaluated using Micro-F1 score with 90% labels as training data in L2 normalization logistic regression.","metricName":"Micro-F1","metricUnit":"%","metricDirection":"maximize","records":[{"value":57.42,"contributor":"NetMF","contributorKind":"method","isCurrentBest":true},{"value":56.06,"contributor":"ProNE","contributorKind":"method"},{"value":53.81,"contributor":"NetSMF","contributorKind":"method"},{"value":54.59,"contributor":"Node2vec","contributorKind":"method"},{"value":52.46,"contributor":"LINE","contributorKind":"method"},{"value":49.53,"contributor":"DeepWalk","contributorKind":"method"},{"value":49.35,"contributor":"Spectral","contributorKind":"method"},{"value":54.04,"contributor":"Hope","contributorKind":"method"},{"value":54.37,"contributor":"GraRep","contributorKind":"method"}],"isOpen":false,"contested":true},{"id":"unsupervised-node-classification-blogcatalog","name":"Unsupervised Multi-label Node Classification - Blogcatalog","description":"Unsupervised multi-label node classification on the Blogcatalog dataset, evaluated using Micro-F1 score with 90% labels as training data in L2 normalization logistic regression.","metricName":"Micro-F1","metricUnit":"%","metricDirection":"maximize","records":[{"value":42.47,"contributor":"NetMF","contributorKind":"method","isCurrentBest":true},{"value":41.14,"contributor":"ProNE","contributorKind":"method"},{"value":40.62,"contributor":"NetSMF","contributorKind":"method"},{"value":40.16,"contributor":"Node2vec","contributorKind":"method"},{"value":38.06,"contributor":"LINE","contributorKind":"method"},{"value":40.48,"contributor":"DeepWalk","contributorKind":"method"},{"value":41.41,"contributor":"Spectral","contributorKind":"method"},{"value":33.99,"contributor":"Hope","contributorKind":"method"},{"value":33.48,"contributor":"GraRep","contributorKind":"method"}],"isOpen":false,"contested":true},{"id":"unsupervised-node-classification-dblp","name":"Unsupervised Multi-label Node Classification - DBLP","description":"Unsupervised multi-label node classification on the DBLP dataset, evaluated using Micro-F1 score with 90% labels as training data in L2 normalization logistic regression.","metricName":"Micro-F1","metricUnit":"%","metricDirection":"maximize","records":[{"value":56.72,"contributor":"NetMF","contributorKind":"method"},{"value":56.85,"contributor":"ProNE","contributorKind":"method"},{"value":59.76,"contributor":"NetSMF","contributorKind":"method","isCurrentBest":true},{"value":57.36,"contributor":"Node2vec","contributorKind":"method"},{"value":49.78,"contributor":"LINE","contributorKind":"method"},{"value":57.54,"contributor":"DeepWalk","contributorKind":"method"},{"value":43.68,"contributor":"Spectral","contributorKind":"method"},{"value":56.15,"contributor":"Hope","contributorKind":"method"},{"value":52.76,"contributor":"GraRep","contributorKind":"method"}],"isOpen":false,"contested":true},{"id":"unsupervised-node-classification-flickr","name":"Unsupervised Multi-label Node Classification - Flickr","description":"Unsupervised multi-label node classification on the Flickr dataset, evaluated using Micro-F1 score with 90% labels as training data in L2 normalization logistic regression.","metricName":"Micro-F1","metricUnit":"%","metricDirection":"maximize","records":[{"value":36.27,"contributor":"NetMF","contributorKind":"method"},{"value":36.56,"contributor":"ProNE","contributorKind":"method","isCurrentBest":true},{"value":35.49,"contributor":"NetSMF","contributorKind":"method"},{"value":36.13,"contributor":"Node2vec","contributorKind":"method"},{"value":31.61,"contributor":"LINE","contributorKind":"method"},{"value":36.09,"contributor":"DeepWalk","contributorKind":"method"},{"value":33.09,"contributor":"Spectral","contributorKind":"method"},{"value":28.97,"contributor":"Hope","contributorKind":"method"},{"value":31.83,"contributor":"GraRep","contributorKind":"method"}],"isOpen":false,"contested":true},{"id":"semi-supervised-node-classification-cora","name":"Semi-Supervised Node Classification - Cora","description":"Semi-supervised node classification with attributes under a transductive setting on the Cora dataset.","metricName":"Accuracy","metricUnit":"%","metricDirection":"maximize","records":[{"value":84.8,"contributor":"Grand","contributorKind":"method"},{"value":85.1,"contributor":"GCNII","contributorKind":"method","isCurrentBest":true},{"value":83.6,"contributor":"DR-GAT","contributorKind":"method"},{"value":83.6,"contributor":"MVGRL","contributorKind":"method"},{"value":84.3,"contributor":"APPNP","contributorKind":"method"},{"value":83.3,"contributor":"Graph U-Net","contributorKind":"method"},{"value":82.9,"contributor":"GAT","contributorKind":"method"},{"value":82.5,"contributor":"GDC_GCN","contributorKind":"method"},{"value":82.1,"contributor":"DropEdge","contributorKind":"method","agent":{"family":"other-ai","model":"DropEdge","evidence":"Adding DropEdge GCN model (#88)","confidence":"high","role":"subject"}},{"value":82.3,"contributor":"GCN","contributorKind":"method"},{"value":82,"contributor":"DGI","contributorKind":"method"},{"value":81.8,"contributor":"JK-net","contributorKind":"method","agent":{"family":"other-ai","model":"JK-net","evidence":"Implement JKNet and its cora dataset parsing class (#74)","confidence":"high","role":"subject"}},{"value":80.1,"contributor":"GraphSAGE","contributorKind":"method"},{"value":78.2,"contributor":"GraphSAGE(unsup)","contributorKind":"method"},{"value":79,"contributor":"Chebyshev","contributorKind":"method"},{"value":81.9,"contributor":"MixHop","contributorKind":"method"}],"isOpen":false,"contested":true},{"id":"semi-supervised-node-classification-citeseer","name":"Semi-Supervised Node Classification - Citeseer","description":"Semi-supervised node classification with attributes under a transductive setting on the Citeseer dataset.","metricName":"Accuracy","metricUnit":"%","metricDirection":"maximize","records":[{"value":75.1,"contributor":"Grand","contributorKind":"method","isCurrentBest":true},{"value":71.3,"contributor":"GCNII","contributorKind":"method"},{"value":72.8,"contributor":"DR-GAT","contributorKind":"method"},{"value":73,"contributor":"MVGRL","contributorKind":"method"},{"value":72,"contributor":"APPNP","contributorKind":"method"},{"value":71.2,"contributor":"Graph U-Net","contributorKind":"method"},{"value":71,"contributor":"GAT","contributorKind":"method"},{"value":71.2,"contributor":"GDC_GCN","contributorKind":"method"},{"value":72.1,"contributor":"DropEdge","contributorKind":"method","agent":{"family":"other-ai","model":"DropEdge","evidence":"Adding DropEdge GCN model (#88)","confidence":"high","role":"subject"}},{"value":71.4,"contributor":"GCN","contributorKind":"method"},{"value":71.2,"contributor":"DGI","contributorKind":"method"},{"value":69.5,"contributor":"JK-net","contributorKind":"method","agent":{"family":"other-ai","model":"JK-net","evidence":"Implement JKNet and its cora dataset parsing class (#74)","confidence":"high","role":"subject"}},{"value":66.2,"contributor":"GraphSAGE","contributorKind":"method"},{"value":65.8,"contributor":"GraphSAGE(unsup)","contributorKind":"method"},{"value":69.8,"contributor":"Chebyshev","contributorKind":"method"},{"value":71.4,"contributor":"MixHop","contributorKind":"method"}],"isOpen":false,"contested":true},{"id":"semi-supervised-node-classification-pubmed","name":"Semi-Supervised Node Classification - Pubmed","description":"Semi-supervised node classification with attributes under a transductive setting on the Pubmed dataset.","metricName":"Accuracy","metricUnit":"%","metricDirection":"maximize","records":[{"value":82.4,"contributor":"Grand","contributorKind":"method","isCurrentBest":true},{"value":80.2,"contributor":"GCNII","contributorKind":"method"},{"value":79.1,"contributor":"DR-GAT","contributorKind":"method"},{"value":80.1,"contributor":"MVGRL","contributorKind":"method"},{"value":80,"contributor":"APPNP","contributorKind":"method"},{"value":79,"contributor":"Graph U-Net","contributorKind":"method"},{"value":78.9,"contributor":"GAT","contributorKind":"method"},{"value":79.8,"contributor":"GDC_GCN","contributorKind":"method"},{"value":79.7,"contributor":"DropEdge","contributorKind":"method","agent":{"family":"other-ai","model":"DropEdge","evidence":"Adding DropEdge GCN model (#88)","confidence":"high","role":"subject"}},{"value":79.5,"contributor":"GCN","contributorKind":"method"},{"value":76.5,"contributor":"DGI","contributorKind":"method"},{"value":77.7,"contributor":"JK-net","contributorKind":"method","agent":{"family":"other-ai","model":"JK-net","evidence":"Implement JKNet and its cora dataset parsing class (#74)","confidence":"high","role":"subject"}},{"value":77.2,"contributor":"GraphSAGE","contributorKind":"method"},{"value":78.2,"contributor":"GraphSAGE(unsup)","contributorKind":"method"},{"value":68.6,"contributor":"Chebyshev","contributorKind":"method"},{"value":80.8,"contributor":"MixHop","contributorKind":"method"}],"isOpen":false,"contested":true},{"id":"multiplex-node-classification-dblp","name":"Multiplex Node Classification - DBLP","description":"Multiplex node classification evaluated using Macro-F1 on the DBLP dataset under the setting of GTN.","metricName":"Macro-F1","metricUnit":"%","metricDirection":"maximize","records":[{"value":95.09,"contributor":"Simple-HGN","contributorKind":"method","isCurrentBest":true,"agent":{"family":"other-ai","model":"Simple-HGN","evidence":"upload Simple-HGN to cogdl example and leaderboard (#233)","confidence":"high","role":"subject"}},{"value":92.03,"contributor":"GTN","contributorKind":"method"},{"value":91.21,"contributor":"HAN","contributorKind":"method"},{"value":79.42,"contributor":"GCC","contributorKind":"method","agent":{"family":"other-ai","model":"GCC","evidence":"Update GCC result (#409)","confidence":"high","role":"subject"}},{"value":78.65,"contributor":"PTE","contributorKind":"method"},{"value":75.18,"contributor":"Metapath2vec","contributorKind":"method"},{"value":74.31,"contributor":"Hin2vec","contributorKind":"method"}],"isOpen":false,"contested":true},{"id":"multiplex-node-classification-acm","name":"Multiplex Node Classification - ACM","description":"Multiplex node classification evaluated using Macro-F1 on the ACM dataset under the setting of GTN.","metricName":"Macro-F1","metricUnit":"%","metricDirection":"maximize","records":[{"value":92.57,"contributor":"Simple-HGN","contributorKind":"method","isCurrentBest":true,"agent":{"family":"other-ai","model":"Simple-HGN","evidence":"upload Simple-HGN to cogdl example and leaderboard (#233)","confidence":"high","role":"subject"}},{"value":90.85,"contributor":"GTN","contributorKind":"method"},{"value":87.25,"contributor":"HAN","contributorKind":"method"},{"value":86.82,"contributor":"GCC","contributorKind":"method","agent":{"family":"other-ai","model":"GCC","evidence":"Update GCC result (#409)","confidence":"high","role":"subject"}},{"value":87.44,"contributor":"PTE","contributorKind":"method"},{"value":88.79,"contributor":"Metapath2vec","contributorKind":"method"},{"value":84.66,"contributor":"Hin2vec","contributorKind":"method"}],"isOpen":false,"contested":true},{"id":"multiplex-node-classification-imdb","name":"Multiplex Node Classification - IMDB","description":"Multiplex node classification evaluated using Macro-F1 on the IMDB dataset under the setting of GTN.","metricName":"Macro-F1","metricUnit":"%","metricDirection":"maximize","records":[{"value":58.61,"contributor":"Simple-HGN","contributorKind":"method","isCurrentBest":true,"agent":{"family":"other-ai","model":"Simple-HGN","evidence":"upload Simple-HGN to cogdl example and leaderboard (#233)","confidence":"high","role":"subject"}},{"value":57.53,"contributor":"GTN","contributorKind":"method"},{"value":53.94,"contributor":"HAN","contributorKind":"method"},{"value":55.86,"contributor":"GCC","contributorKind":"method","agent":{"family":"other-ai","model":"GCC","evidence":"Update GCC result (#409)","confidence":"high","role":"subject"}},{"value":48.91,"contributor":"PTE","contributorKind":"method"},{"value":43.1,"contributor":"Metapath2vec","contributorKind":"method"},{"value":44.04,"contributor":"Hin2vec","contributorKind":"method"}],"isOpen":false,"contested":true},{"id":"link-prediction-ppi","name":"Link Prediction - PPI","description":"Link prediction evaluated using ROC AUC on the PPI dataset after removing 10% of edges.","metricName":"ROC AUC","metricUnit":"%","metricDirection":"maximize","records":[{"value":79.93,"contributor":"ProNE","contributorKind":"method"},{"value":79.04,"contributor":"NetMF","contributorKind":"method"},{"value":80.21,"contributor":"Hope","contributorKind":"method","isCurrentBest":true},{"value":73.75,"contributor":"LINE","contributorKind":"method"},{"value":70.19,"contributor":"Node2vec","contributorKind":"method"},{"value":68.64,"contributor":"NetSMF","contributorKind":"method"},{"value":69.65,"contributor":"DeepWalk","contributorKind":"method"},{"value":54.87,"contributor":"SDNE","contributorKind":"method"}],"isOpen":false,"contested":true},{"id":"link-prediction-wikipedia","name":"Link Prediction - Wikipedia","description":"Link prediction evaluated using ROC AUC on the Wikipedia dataset after removing 10% of edges.","metricName":"ROC AUC","metricUnit":"%","metricDirection":"maximize","records":[{"value":82.74,"contributor":"ProNE","contributorKind":"method","isCurrentBest":true},{"value":73.24,"contributor":"NetMF","contributorKind":"method"},{"value":68.89,"contributor":"Hope","contributorKind":"method"},{"value":66.51,"contributor":"LINE","contributorKind":"method"},{"value":66.6,"contributor":"Node2vec","contributorKind":"method"},{"value":67.52,"contributor":"NetSMF","contributorKind":"method"},{"value":65.93,"contributor":"DeepWalk","contributorKind":"method"},{"value":60.72,"contributor":"SDNE","contributorKind":"method"}],"isOpen":false,"contested":true},{"id":"multiplex-link-prediction-amazon","name":"Multiplex Link Prediction - Amazon","description":"Multiplex link prediction evaluated using ROC AUC on the Amazon dataset after removing 15% of edges.","metricName":"ROC AUC","metricUnit":"%","metricDirection":"maximize","records":[{"value":97.44,"contributor":"GATNE","contributorKind":"method"},{"value":97.72,"contributor":"NetMF","contributorKind":"method","isCurrentBest":true},{"value":96.51,"contributor":"ProNE","contributorKind":"method"},{"value":86.86,"contributor":"Node2vec","contributorKind":"method"},{"value":92.54,"contributor":"DeepWalk","contributorKind":"method"},{"value":92.56,"contributor":"LINE","contributorKind":"method"},{"value":94.39,"contributor":"Hope","contributorKind":"method"},{"value":83.88,"contributor":"GraRep","contributorKind":"method"}],"isOpen":false,"contested":true},{"id":"multiplex-link-prediction-youtube","name":"Multiplex Link Prediction - YouTube","description":"Multiplex link prediction evaluated using ROC AUC on the YouTube dataset after removing 15% of edges.","metricName":"ROC AUC","metricUnit":"%","metricDirection":"maximize","records":[{"value":84.61,"contributor":"GATNE","contributorKind":"method","isCurrentBest":true},{"value":82.53,"contributor":"NetMF","contributorKind":"method"},{"value":78.96,"contributor":"ProNE","contributorKind":"method"},{"value":74.01,"contributor":"Node2vec","contributorKind":"method"},{"value":74.31,"contributor":"DeepWalk","contributorKind":"method"},{"value":73.4,"contributor":"LINE","contributorKind":"method"},{"value":74.66,"contributor":"Hope","contributorKind":"method"},{"value":71.37,"contributor":"GraRep","contributorKind":"method"}],"isOpen":false,"contested":true},{"id":"multiplex-link-prediction-twitter","name":"Multiplex Link Prediction - Twitter","description":"Multiplex link prediction evaluated using ROC AUC on the Twitter dataset after removing 15% of edges.","metricName":"ROC AUC","metricUnit":"%","metricDirection":"maximize","records":[{"value":92.3,"contributor":"GATNE","contributorKind":"method","isCurrentBest":true},{"value":73.75,"contributor":"NetMF","contributorKind":"method"},{"value":81.32,"contributor":"ProNE","contributorKind":"method"},{"value":78.3,"contributor":"Node2vec","contributorKind":"method"},{"value":60.29,"contributor":"DeepWalk","contributorKind":"method"},{"value":60.36,"contributor":"LINE","contributorKind":"method"},{"value":70.61,"contributor":"Hope","contributorKind":"method"},{"value":49.64,"contributor":"GraRep","contributorKind":"method"}],"isOpen":false,"contested":true},{"id":"knowledge-graph-completion-fb15k-237","name":"Knowledge Graph Completion - FB15k-237","description":"Knowledge graph completion evaluated using Mean Reciprocal Rank (MRR) on the FB15k-237 dataset.","metricName":"MRR","metricUnit":"%","metricDirection":"maximize","records":[{"value":31.1,"contributor":"RotatE","contributorKind":"method","isCurrentBest":true},{"value":28.85,"contributor":"ComplEx","contributorKind":"method"},{"value":30.5,"contributor":"TransE","contributorKind":"method"},{"value":24.93,"contributor":"DistMult","contributorKind":"method"},{"value":21.94,"contributor":"CompGCN","contributorKind":"method"}],"isOpen":false,"contested":true},{"id":"knowledge-graph-completion-wn18","name":"Knowledge Graph Completion - WN18","description":"Knowledge graph completion evaluated using Mean Reciprocal Rank (MRR) on the WN18 dataset.","metricName":"MRR","metricUnit":"%","metricDirection":"maximize","records":[{"value":93.99,"contributor":"RotatE","contributorKind":"method","isCurrentBest":true},{"value":86.4,"contributor":"ComplEx","contributorKind":"method"},{"value":71.55,"contributor":"TransE","contributorKind":"method"},{"value":78.77,"contributor":"DistMult","contributorKind":"method"},{"value":39.48,"contributor":"CompGCN","contributorKind":"method"}],"isOpen":false,"contested":true},{"id":"knowledge-graph-completion-wn18rr","name":"Knowledge Graph Completion - WN18RR","description":"Knowledge graph completion evaluated using Mean Reciprocal Rank (MRR) on the WN18RR dataset.","metricName":"MRR","metricUnit":"%","metricDirection":"maximize","records":[{"value":46.05,"contributor":"RotatE","contributorKind":"method","isCurrentBest":true},{"value":40.77,"contributor":"ComplEx","contributorKind":"method"},{"value":21.85,"contributor":"TransE","contributorKind":"method"},{"value":41.64,"contributor":"DistMult","contributorKind":"method"},{"value":44.8,"contributor":"CompGCN","contributorKind":"method"}],"isOpen":false,"contested":true},{"id":"graph-classification-mutag","name":"Graph Classification - MUTAG","description":"Graph classification performance on the MUTAG dataset.","metricName":"Accuracy","metricUnit":"%","metricDirection":"maximize","records":[{"value":92.06,"contributor":"GIN","contributorKind":"method","isCurrentBest":true},{"value":88.95,"contributor":"Infograph","contributorKind":"method"},{"value":85.18,"contributor":"DiffPool","contributorKind":"method"},{"value":87.25,"contributor":"SortPool","contributorKind":"method"},{"value":83.68,"contributor":"Graph2Vec","contributorKind":"method"},{"value":86.12,"contributor":"PATCH_SAN","contributorKind":"method"},{"value":81.93,"contributor":"HGP-SL","contributorKind":"method"},{"value":83.33,"contributor":"DGCNN","contributorKind":"method"},{"value":71.73,"contributor":"SAGPool","contributorKind":"method"},{"value":85.58,"contributor":"DGK","contributorKind":"method"}],"isOpen":false,"contested":true},{"id":"graph-classification-imdb-b","name":"Graph Classification - IMDB-B","description":"Graph classification performance on the IMDB-B dataset.","metricName":"Accuracy","metricUnit":"%","metricDirection":"maximize","records":[{"value":76.1,"contributor":"GIN","contributorKind":"method","isCurrentBest":true},{"value":74.5,"contributor":"Infograph","contributorKind":"method"},{"value":72.5,"contributor":"DiffPool","contributorKind":"method"},{"value":75.4,"contributor":"SortPool","contributorKind":"method"},{"value":73.9,"contributor":"Graph2Vec","contributorKind":"method"},{"value":76,"contributor":"PATCH_SAN","contributorKind":"method"},{"value":74,"contributor":"HGP-SL","contributorKind":"method"},{"value":71.6,"contributor":"DGCNN","contributorKind":"method"},{"value":74.8,"contributor":"SAGPool","contributorKind":"method"},{"value":55,"contributor":"DGK","contributorKind":"method"}],"isOpen":false,"contested":true},{"id":"graph-classification-imdb-m","name":"Graph Classification - IMDB-M","description":"Graph classification performance on the IMDB-M dataset.","metricName":"Accuracy","metricUnit":"%","metricDirection":"maximize","records":[{"value":51.8,"contributor":"GIN","contributorKind":"method"},{"value":51.33,"contributor":"Infograph","contributorKind":"method"},{"value":50.5,"contributor":"DiffPool","contributorKind":"method"},{"value":50.47,"contributor":"SortPool","contributorKind":"method"},{"value":52.27,"contributor":"Graph2Vec","contributorKind":"method","isCurrentBest":true},{"value":46.4,"contributor":"PATCH_SAN","contributorKind":"method"},{"value":49.53,"contributor":"HGP-SL","contributorKind":"method"},{"value":49.2,"contributor":"DGCNN","contributorKind":"method"},{"value":51.33,"contributor":"SAGPool","contributorKind":"method"},{"value":40.4,"contributor":"DGK","contributorKind":"method"}],"isOpen":false,"contested":true},{"id":"graph-classification-proteins","name":"Graph Classification - PROTEINS","description":"Graph classification performance on the PROTEINS dataset.","metricName":"Accuracy","metricUnit":"%","metricDirection":"maximize","records":[{"value":75.19,"contributor":"GIN","contributorKind":"method"},{"value":73.93,"contributor":"Infograph","contributorKind":"method"},{"value":75.3,"contributor":"DiffPool","contributorKind":"method"},{"value":74.48,"contributor":"SortPool","contributorKind":"method"},{"value":73.3,"contributor":"Graph2Vec","contributorKind":"method"},{"value":75.38,"contributor":"PATCH_SAN","contributorKind":"method","isCurrentBest":true},{"value":73.94,"contributor":"HGP-SL","contributorKind":"method"},{"value":66.75,"contributor":"DGCNN","contributorKind":"method"},{"value":74.03,"contributor":"SAGPool","contributorKind":"method"},{"value":72.59,"contributor":"DGK","contributorKind":"method"}],"isOpen":false,"contested":true},{"id":"graph-classification-collab","name":"Graph Classification - COLLAB","description":"Graph classification performance on the COLLAB dataset.","metricName":"Accuracy","metricUnit":"%","metricDirection":"maximize","records":[{"value":79.52,"contributor":"GIN","contributorKind":"method"},{"value":79.4,"contributor":"Infograph","contributorKind":"method"},{"value":79.27,"contributor":"DiffPool","contributorKind":"method"},{"value":80.07,"contributor":"SortPool","contributorKind":"method"},{"value":85.58,"contributor":"Graph2Vec","contributorKind":"method","isCurrentBest":true},{"value":74.34,"contributor":"PATCH_SAN","contributorKind":"method"},{"value":82.08,"contributor":"HGP-SL","contributorKind":"method"},{"value":77.45,"contributor":"DGCNN","contributorKind":"method"}],"isOpen":false,"contested":true},{"id":"graph-classification-ptc","name":"Graph Classification - PTC","description":"Graph classification performance on the PTC dataset.","metricName":"Accuracy","metricUnit":"%","metricDirection":"maximize","records":[{"value":67.82,"contributor":"GIN","contributorKind":"method","isCurrentBest":true},{"value":60.74,"contributor":"Infograph","contributorKind":"method"},{"value":58,"contributor":"DiffPool","contributorKind":"method"},{"value":62.04,"contributor":"SortPool","contributorKind":"method"},{"value":54.76,"contributor":"Graph2Vec","contributorKind":"method"},{"value":61.6,"contributor":"PATCH_SAN","contributorKind":"method"},{"value":56.62,"contributor":"DGCNN","contributorKind":"method"},{"value":59.92,"contributor":"SAGPool","contributorKind":"method"}],"isOpen":false,"contested":true},{"id":"graph-classification-nci1","name":"Graph Classification - NCI1","description":"Graph classification performance on the NCI1 dataset.","metricName":"Accuracy","metricUnit":"%","metricDirection":"maximize","records":[{"value":81.66,"contributor":"GIN","contributorKind":"method","isCurrentBest":true},{"value":76.64,"contributor":"Infograph","contributorKind":"method"},{"value":69.09,"contributor":"DiffPool","contributorKind":"method"},{"value":73.99,"contributor":"SortPool","contributorKind":"method"},{"value":71.85,"contributor":"Graph2Vec","contributorKind":"method"},{"value":69.82,"contributor":"PATCH_SAN","contributorKind":"method"},{"value":65.96,"contributor":"DGCNN","contributorKind":"method"},{"value":72.87,"contributor":"SAGPool","contributorKind":"method"}],"isOpen":false,"contested":true},{"id":"graph-classification-reddit-b","name":"Graph Classification - REDDIT-B","description":"Graph classification performance on the REDDIT-B dataset.","metricName":"Accuracy","metricUnit":"%","metricDirection":"maximize","records":[{"value":83.1,"contributor":"GIN","contributorKind":"method"},{"value":76.55,"contributor":"Infograph","contributorKind":"method"},{"value":81.2,"contributor":"DiffPool","contributorKind":"method"},{"value":78.15,"contributor":"SortPool","contributorKind":"method"},{"value":91.77,"contributor":"Graph2Vec","contributorKind":"method","isCurrentBest":true},{"value":60.61,"contributor":"PATCH_SAN","contributorKind":"method"},{"value":86.2,"contributor":"DGCNN","contributorKind":"method"},{"value":89.21,"contributor":"SAGPool","contributorKind":"method"}],"isOpen":false,"contested":true}],"repo":{"owner":"THUDM","name":"CogDL","branch":"master"},"participants":[],"stats":{"totalRecords":241,"uniqueParticipants":0,"recordsLast30d":0,"recordsLast90d":0},"agentStats":[],"lastUpdated":"2026-09-18T16:32:55.536Z"},{"id":"vbench","name":"VBench","tagline":"A comprehensive benchmark suite for evaluating video generative models across multiple fine-grained dimensions.","description":"VBench decomposes video generation quality into hierarchical dimensions to facilitate fine-grained and objective evaluation. It supports text-to-video, image-to-video, and long video evaluation, aligning automatic metrics with human preferences.","url":"https://github.com/Vchitect/VBench","host":"github","status":"active","tags":["video-generation","text-to-video","image-to-video","benchmark","evaluation","computer-vision"],"domain":"ml-research","images":["https://raw.githubusercontent.com/Vchitect/VBench/master/asset/fig_paper_teaser_vbench.jpg","https://raw.githubusercontent.com/Vchitect/VBench/master/asset/fig_extention_teaser.jpg","https://raw.githubusercontent.com/Vchitect/VBench/master/VBench-2.0/asset/fig_paper_teaser.jpg"],"organizer":{"name":"Vchitect","url":"https://github.com/Vchitect","type":"community"},"participation":{"howToSubmit":"Generate videos using the standard prompt suites and submit the results via the provided Google Form or Hugging Face Leaderboard.","submissionFormat":"A zip file containing the evaluation results (JSON) or generated video frames in PNG format following the specified folder structure.","requirements":["Python","PyTorch","CUDA","FFmpeg","detectron2"],"compute":"consumer-gpu","computeDetails":"Evaluation can be run on a single or multiple GPUs (e.g., CUDA-enabled GPUs)."},"quickstart":"from vbench import VBench\nmy_VBench = VBench(device, \"vbench/VBench_full_info.json\", \"evaluation_results\")\nmy_VBench.evaluate(\n    videos_path = \"sampled_videos/lavie/human_action\",\n    name = \"lavie_human_action\",\n    dimension_list = [\"human_action\"],\n)","links":[{"label":"Hugging Face Leaderboard","url":"https://huggingface.co/spaces/Vchitect/VBench_Leaderboard"},{"label":"VBench Arena","url":"https://huggingface.co/spaces/Vchitect/VBench_Video_Arena"},{"label":"VBench-2.0 Arena","url":"https://huggingface.co/spaces/Vchitect/VBench2.0_Video_Arena"},{"label":"VBench Paper","url":"https://arxiv.org/abs/2311.17982"},{"label":"VBench++ Paper","url":"https://arxiv.org/abs/2411.13503"},{"label":"VBench-2.0 Paper","url":"https://arxiv.org/abs/2503.21755"}],"problems":[{"id":"short-videos","name":"Short Videos Track","description":"Evaluation of short video generation (1.6s - 4.0s, 8-24 FPS) across 16 dimensions including temporal quality, frame-wise quality, and text alignment.","metricName":"Total Score","metricDirection":"maximize","records":[],"isOpen":true,"contested":false},{"id":"long-videos","name":"Long Videos Track","description":"Evaluation of long video generation (10.0s - 40.0s, 8-24 FPS) focusing on temporal consistency and long-term subject consistency.","metricName":"Total Score","metricDirection":"maximize","records":[],"isOpen":true,"contested":false},{"id":"vbench-2-0","name":"VBench-2.0 (Intrinsic Faithfulness)","description":"Evaluation of advanced capabilities including commonsense reasoning, physics-based realism, human motion, and creative composition across 18 dimensions.","metricName":"Total Score","metricDirection":"maximize","records":[],"isOpen":true,"contested":false}],"repo":{"owner":"Vchitect","name":"VBench","branch":"master"},"participants":[],"stats":{"totalRecords":0,"uniqueParticipants":0,"recordsLast30d":0,"recordsLast90d":0},"agentStats":[],"lastUpdated":"2026-09-18T16:32:55.537Z"},{"id":"mle-bench","name":"MLE-bench","tagline":"Evaluating machine learning agents on machine learning engineering tasks.","description":"MLE-bench is a benchmark of 75 Kaggle competitions designed to evaluate the machine learning engineering capabilities of AI agents. It measures whether agents can autonomously prepare data, train models, and optimize submissions. Run by OpenAI, it establishes a standardized framework for testing autonomous ML engineering.","url":"https://github.com/openai/mle-bench","host":"github","status":"ended","tags":["ml-engineering","agents","kaggle","benchmark","llm-eval"],"domain":"coding-agents","images":[],"organizer":{"name":"OpenAI","url":"https://openai.com","type":"company"},"participation":{"howToSubmit":"Open a PR to main adding grading reports under runs/ and updating the README leaderboard table.","submissionFormat":"Grading reports in JSON/CSV format under runs/ and updating the README leaderboard.","requirements":["Python","Docker","Kaggle API"],"compute":"datacenter-gpu","computeDetails":"Canonical setup recommends 36 vCPUs, 440GB RAM, and one 24GB A10 GPU per agent run.","deadline":"rolling","prizes":"recognition only","cost":"free","verification":"CI and manual review of grading reports and logs"},"quickstart":"pip install -e .\nmlebench prepare --lite\nmlebench grade-sample <PATH_TO_SUBMISSION> spaceship-titanic","links":[{"label":"Paper","url":"https://arxiv.org/abs/2410.07095"},{"label":"GitHub","url":"https://github.com/openai/mle-bench"}],"problems":[{"id":"all-competitions","name":"All Competitions","description":"Percentage of competitions where the agent achieved any medal across all 75 competitions.","metricName":"Any Medal Rate","metricUnit":"%","metricDirection":"maximize","records":[{"date":"2026-02-23","value":64.44,"contributor":"Famou-Agent 2.0","contributorKind":"method","contributorUrl":"https://github.com/baidubce/FM-Agent","agent":{"family":"google","model":"Gemini-3-Pro-Preview","confidence":"high","role":"author"}},{"date":"2026-03-06","value":63.11,"contributor":"AIBuildAI","contributorKind":"method","contributorUrl":"https://github.com/aibuildai/AI-Build-AI","agent":{"family":"anthropic","model":"Claude-Opus-4.6","confidence":"high","role":"author"}},{"date":"2026-02-17","value":62.67,"contributor":"CAIR MARS+","contributorKind":"method","contributorUrl":"https://arxiv.org/pdf/2602.02660","agent":{"family":"google","model":"Gemini-3-Pro-Preview","confidence":"high","role":"author"}},{"date":"2026-02-14","value":61.33,"contributor":"MLEvolve","contributorKind":"method","contributorUrl":"https://github.com/InternScience/MLEvolve","agent":{"family":"google","model":"Gemini-3-Pro-Preview","confidence":"high","role":"author"}},{"date":"2026-01-05","value":61.33,"contributor":"PiEvolve (24h)","contributorKind":"method","contributorUrl":"https://github.com/FractalAIResearchLabs/PiEvolve","agent":{"family":"google","model":"Gemini-3-Pro-Preview","confidence":"high","role":"author"}},{"date":"2025-12-27","value":59.56,"contributor":"Famou-Agent 2.0 (Gemini-2.5-Pro)","contributorKind":"method","contributorUrl":"https://github.com/baidubce/FM-Agent","agent":{"family":"google","model":"Gemini-2.5-Pro","confidence":"high","role":"subject"}},{"date":"2025-12-16","value":56.44,"contributor":"ML-Master 2.0","contributorKind":"method","contributorUrl":"https://github.com/sjtu-sai-agents/ML-Master","agent":{"family":"deepseek","model":"Deepseek-V3.2-Speciale","confidence":"high","role":"author"}},{"date":"2026-01-25","value":56,"contributor":"CAIR MARS","contributorKind":"method","contributorUrl":"https://arxiv.org/pdf/2602.02660","agent":{"family":"google","model":"Gemini-3-Pro-Preview","confidence":"high","role":"author"}},{"date":"2026-01-05","value":52,"contributor":"PiEvolve (12h)","contributorKind":"method","contributorUrl":"https://github.com/FractalAIResearchLabs/PiEvolve","agent":{"family":"google","model":"Gemini-3-Pro-Preview","confidence":"high","role":"author"}},{"date":"2025-12-07","value":50.67,"contributor":"Leeroo","contributorKind":"method","contributorUrl":"https://github.com/Leeroo-AI/kapso","agent":{"family":"google","model":"Gemini-3-Pro-Preview","confidence":"high","role":"author"}},{"date":"2025-11-10","value":48.44,"contributor":"Thesis","contributorKind":"method","contributorUrl":"https://thesislabs.ai","agent":{"family":"openai","model":"gpt-5-codex","confidence":"high","role":"author"}},{"date":"2025-11-25","value":44,"contributor":"CAIR MLE-STAR-Pro-1.5","contributorKind":"method","contributorUrl":"https://research.google/teams/cloud-ai-research/","agent":{"family":"google","model":"Gemini-2.5-Pro","confidence":"high","role":"author"}},{"date":"2025-10-10","value":43.56,"contributor":"Famou-Agent","contributorKind":"method","contributorUrl":"https://github.com/baidubce/FM-Agent","agent":{"family":"google","model":"Gemini-2.5-Pro","confidence":"high","role":"author"}},{"date":"2025-10-06","value":39.56,"contributor":"Operand ensemble","contributorKind":"method","contributorUrl":"https://operand.com","agent":{"family":"openai","model":"gpt-5","confidence":"high","role":"author"}},{"date":"2025-11-03","value":38.67,"contributor":"CAIR MLE-STAR-Pro-1.0","contributorKind":"method","contributorUrl":"https://research.google/teams/cloud-ai-research/","agent":{"family":"google","model":"Gemini-2.5-Pro","confidence":"high","role":"author"}},{"date":"2025-09-12","value":36.44,"contributor":"InternAgent","contributorKind":"method","contributorUrl":"https://github.com/Alpha-Innovator/InternAgent/","agent":{"family":"deepseek","model":"deepseek-r1","confidence":"high","role":"author"}},{"date":"2025-09-26","value":35.11,"contributor":"R&D-Agent (gpt-5)","contributorKind":"method","contributorUrl":"https://github.com/microsoft/RD-Agent","agent":{"family":"openai","model":"gpt-5","confidence":"high","role":"subject"}},{"date":"2025-07-28","value":34.22,"contributor":"Neo multi-agent","contributorKind":"method","contributorUrl":"https://heyneo.so/","agent":{"family":"other-ai","tool":"NEO","evidence":"NEO, a novel ML Engineering and Data science Agent","confidence":"high","role":"author"}},{"date":"2025-05-15","value":31.6,"contributor":"AIRA-dojo","contributorKind":"method","contributorUrl":"https://github.com/facebookresearch/aira-dojo/","agent":{"family":"openai","model":"o3","confidence":"high","role":"author"}},{"date":"2025-08-15","value":30.22,"contributor":"R&D-Agent (o3 + GPT-4.1)","contributorKind":"method","contributorUrl":"https://github.com/microsoft/RD-Agent","agent":{"family":"openai","model":"o3 + GPT-4.1","confidence":"high","role":"subject"}},{"date":"2025-06-17","value":29.33,"contributor":"ML-Master","contributorKind":"method","contributorUrl":"https://github.com/zeroxleo/ML-Master","agent":{"family":"deepseek","model":"deepseek-r1","confidence":"high","role":"author"}},{"date":"2025-05-14","value":22.4,"contributor":"R&D-Agent (o1-preview)","contributorKind":"method","contributorUrl":"https://github.com/microsoft/RD-Agent","agent":{"family":"openai","model":"o1-preview","confidence":"high","role":"subject"}},{"date":"2024-10-08","value":17.12,"contributor":"AIDE (o1-preview)","contributorKind":"method","contributorUrl":"https://github.com/wecoai/aideml","agent":{"family":"openai","model":"o1-preview","confidence":"high","role":"subject"}},{"date":"2024-10-08","value":8.63,"contributor":"AIDE (gpt-4o)","contributorKind":"method","contributorUrl":"https://github.com/wecoai/aideml","agent":{"family":"openai","model":"gpt-4o-2024-08-06","confidence":"high","role":"subject"}},{"date":"2024-10-08","value":7.56,"contributor":"AIDE (claude-3-5-sonnet)","contributorKind":"method","contributorUrl":"https://github.com/wecoai/aideml","agent":{"family":"anthropic","model":"claude-3-5-sonnet-20240620","confidence":"high","role":"subject"}},{"date":"2024-10-08","value":4.89,"contributor":"OpenHands","contributorKind":"method","agent":{"family":"openai","model":"gpt-4o-2024-08-06","confidence":"high","role":"author"}},{"date":"2024-10-08","value":3.33,"contributor":"AIDE (llama-3.1-405b)","contributorKind":"method","contributorUrl":"https://github.com/wecoai/aideml","agent":{"family":"meta","model":"llama-3.1-405b-instruct","confidence":"high","role":"subject"}},{"date":"2024-10-08","value":1.6,"contributor":"MLAB","contributorKind":"method","agent":{"family":"openai","model":"gpt-4o-2024-08-06","confidence":"high","role":"author"}},{"date":"2026-02-03","value":77.78,"contributor":"Disarray","contributorKind":"method","contributorUrl":"https://disarray.ai","isCurrentBest":true,"agent":{"family":"other-ai","model":"Ensemble","confidence":"high","role":"author"}},{"date":"2026-02-09","value":62.66,"contributor":"LoongFlow","contributorKind":"method","contributorUrl":"https://github.com/baidu-baige/LoongFlow","agent":{"family":"google","model":"Gemini-3-Flash-Preview","confidence":"high","role":"author"}}],"isOpen":false,"contested":true},{"id":"low-complexity","name":"Low Complexity (Lite)","description":"Percentage of competitions where the agent achieved any medal across the 22 low-complexity competitions.","metricName":"Any Medal Rate","metricUnit":"%","metricDirection":"maximize","records":[{"date":"2026-02-23","value":80.3,"contributor":"Famou-Agent 2.0","contributorKind":"method","contributorUrl":"https://github.com/baidubce/FM-Agent","agent":{"family":"google","model":"Gemini-3-Pro-Preview","confidence":"high","role":"author"}},{"date":"2026-03-06","value":77.27,"contributor":"AIBuildAI","contributorKind":"method","contributorUrl":"https://github.com/aibuildai/AI-Build-AI","agent":{"family":"anthropic","model":"Claude-Opus-4.6","confidence":"high","role":"author"}},{"date":"2026-02-17","value":78.79,"contributor":"CAIR MARS+","contributorKind":"method","contributorUrl":"https://arxiv.org/pdf/2602.02660","agent":{"family":"google","model":"Gemini-3-Pro-Preview","confidence":"high","role":"author"}},{"date":"2026-02-14","value":80.3,"contributor":"MLEvolve","contributorKind":"method","contributorUrl":"https://github.com/InternScience/MLEvolve","agent":{"family":"google","model":"Gemini-3-Pro-Preview","confidence":"high","role":"author"}},{"date":"2026-01-05","value":80.3,"contributor":"PiEvolve (24h)","contributorKind":"method","contributorUrl":"https://github.com/FractalAIResearchLabs/PiEvolve","agent":{"family":"google","model":"Gemini-3-Pro-Preview","confidence":"high","role":"author"}},{"date":"2025-12-27","value":75.76,"contributor":"Famou-Agent 2.0 (Gemini-2.5-Pro)","contributorKind":"method","contributorUrl":"https://github.com/baidubce/FM-Agent","agent":{"family":"google","model":"Gemini-2.5-Pro","confidence":"high","role":"subject"}},{"date":"2025-12-16","value":75.76,"contributor":"ML-Master 2.0","contributorKind":"method","contributorUrl":"https://github.com/sjtu-sai-agents/ML-Master","agent":{"family":"deepseek","model":"Deepseek-V3.2-Speciale","confidence":"high","role":"author"}},{"date":"2026-01-25","value":74.24,"contributor":"CAIR MARS","contributorKind":"method","contributorUrl":"https://arxiv.org/pdf/2602.02660","agent":{"family":"google","model":"Gemini-3-Pro-Preview","confidence":"high","role":"author"}},{"date":"2026-01-05","value":74.24,"contributor":"PiEvolve (12h)","contributorKind":"method","contributorUrl":"https://github.com/FractalAIResearchLabs/PiEvolve","agent":{"family":"google","model":"Gemini-3-Pro-Preview","confidence":"high","role":"author"}},{"date":"2025-12-07","value":68.18,"contributor":"Leeroo","contributorKind":"method","contributorUrl":"https://github.com/Leeroo-AI/kapso","agent":{"family":"google","model":"Gemini-3-Pro-Preview","confidence":"high","role":"author"}},{"date":"2025-11-10","value":65.15,"contributor":"Thesis","contributorKind":"method","contributorUrl":"https://thesislabs.ai","agent":{"family":"openai","model":"gpt-5-codex","confidence":"high","role":"author"}},{"date":"2025-11-25","value":68.18,"contributor":"CAIR MLE-STAR-Pro-1.5","contributorKind":"method","contributorUrl":"https://research.google/teams/cloud-ai-research/","agent":{"family":"google","model":"Gemini-2.5-Pro","confidence":"high","role":"author"}},{"date":"2025-10-10","value":62.12,"contributor":"Famou-Agent","contributorKind":"method","contributorUrl":"https://github.com/baidubce/FM-Agent","agent":{"family":"google","model":"Gemini-2.5-Pro","confidence":"high","role":"author"}},{"date":"2025-10-06","value":63.64,"contributor":"Operand ensemble","contributorKind":"method","contributorUrl":"https://operand.com","agent":{"family":"openai","model":"gpt-5","confidence":"high","role":"author"}},{"date":"2025-11-03","value":66.67,"contributor":"CAIR MLE-STAR-Pro-1.0","contributorKind":"method","contributorUrl":"https://research.google/teams/cloud-ai-research/","agent":{"family":"google","model":"Gemini-2.5-Pro","confidence":"high","role":"author"}},{"date":"2025-09-12","value":62.12,"contributor":"InternAgent","contributorKind":"method","contributorUrl":"https://github.com/Alpha-Innovator/InternAgent/","agent":{"family":"deepseek","model":"deepseek-r1","confidence":"high","role":"author"}},{"date":"2025-09-26","value":68.18,"contributor":"R&D-Agent (gpt-5)","contributorKind":"method","contributorUrl":"https://github.com/microsoft/RD-Agent","agent":{"family":"openai","model":"gpt-5","confidence":"high","role":"subject"}},{"date":"2025-07-28","value":48.48,"contributor":"Neo multi-agent","contributorKind":"method","contributorUrl":"https://heyneo.so/","agent":{"family":"other-ai","tool":"NEO","evidence":"NEO, a novel ML Engineering and Data science Agent","confidence":"high","role":"author"}},{"date":"2025-05-15","value":55,"contributor":"AIRA-dojo","contributorKind":"method","contributorUrl":"https://github.com/facebookresearch/aira-dojo/","agent":{"family":"openai","model":"o3","confidence":"high","role":"author"}},{"date":"2025-08-15","value":51.52,"contributor":"R&D-Agent (o3 + GPT-4.1)","contributorKind":"method","contributorUrl":"https://github.com/microsoft/RD-Agent","agent":{"family":"openai","model":"o3 + GPT-4.1","confidence":"high","role":"subject"}},{"date":"2025-06-17","value":48.48,"contributor":"ML-Master","contributorKind":"method","contributorUrl":"https://github.com/zeroxleo/ML-Master","agent":{"family":"deepseek","model":"deepseek-r1","confidence":"high","role":"author"}},{"date":"2025-05-14","value":48.18,"contributor":"R&D-Agent (o1-preview)","contributorKind":"method","contributorUrl":"https://github.com/microsoft/RD-Agent","agent":{"family":"openai","model":"o1-preview","confidence":"high","role":"subject"}},{"date":"2024-10-08","value":35.91,"contributor":"AIDE (o1-preview)","contributorKind":"method","contributorUrl":"https://github.com/wecoai/aideml","agent":{"family":"openai","model":"o1-preview","confidence":"high","role":"subject"}},{"date":"2024-10-08","value":18.55,"contributor":"AIDE (gpt-4o)","contributorKind":"method","contributorUrl":"https://github.com/wecoai/aideml","agent":{"family":"openai","model":"gpt-4o-2024-08-06","confidence":"high","role":"subject"}},{"date":"2024-10-08","value":19.7,"contributor":"AIDE (claude-3-5-sonnet)","contributorKind":"method","contributorUrl":"https://github.com/wecoai/aideml","agent":{"family":"anthropic","model":"claude-3-5-sonnet-20240620","confidence":"high","role":"subject"}},{"date":"2024-10-08","value":12.12,"contributor":"OpenHands","contributorKind":"method","agent":{"family":"openai","model":"gpt-4o-2024-08-06","confidence":"high","role":"author"}},{"date":"2024-10-08","value":10.23,"contributor":"AIDE (llama-3.1-405b)","contributorKind":"method","contributorUrl":"https://github.com/wecoai/aideml","agent":{"family":"meta","model":"llama-3.1-405b-instruct","confidence":"high","role":"subject"}},{"date":"2024-10-08","value":4.55,"contributor":"MLAB","contributorKind":"method","agent":{"family":"openai","model":"gpt-4o-2024-08-06","confidence":"high","role":"author"}},{"date":"2026-02-03","value":90.91,"contributor":"Disarray","contributorKind":"method","contributorUrl":"https://disarray.ai","isCurrentBest":true,"agent":{"family":"other-ai","model":"Ensemble","confidence":"high","role":"author"}},{"date":"2026-02-09","value":77.27,"contributor":"LoongFlow","contributorKind":"method","contributorUrl":"https://github.com/baidu-baige/LoongFlow","agent":{"family":"google","model":"Gemini-3-Flash-Preview","confidence":"high","role":"author"}}],"isOpen":false,"contested":true},{"id":"medium-complexity","name":"Medium Complexity","description":"Percentage of competitions where the agent achieved any medal across the medium-complexity competitions.","metricName":"Any Medal Rate","metricUnit":"%","metricDirection":"maximize","records":[{"date":"2026-02-23","value":64.04,"contributor":"Famou-Agent 2.0","contributorKind":"method","contributorUrl":"https://github.com/baidubce/FM-Agent","agent":{"family":"google","model":"Gemini-3-Pro-Preview","confidence":"high","role":"author"}},{"date":"2026-03-06","value":61.4,"contributor":"AIBuildAI","contributorKind":"method","contributorUrl":"https://github.com/aibuildai/AI-Build-AI","agent":{"family":"anthropic","model":"Claude-Opus-4.6","confidence":"high","role":"author"}},{"date":"2026-02-17","value":60.53,"contributor":"CAIR MARS+","contributorKind":"method","contributorUrl":"https://arxiv.org/pdf/2602.02660","agent":{"family":"google","model":"Gemini-3-Pro-Preview","confidence":"high","role":"author"}},{"date":"2026-02-14","value":57.89,"contributor":"MLEvolve","contributorKind":"method","contributorUrl":"https://github.com/InternScience/MLEvolve","agent":{"family":"google","model":"Gemini-3-Pro-Preview","confidence":"high","role":"author"}},{"date":"2026-01-05","value":58.77,"contributor":"PiEvolve (24h)","contributorKind":"method","contributorUrl":"https://github.com/FractalAIResearchLabs/PiEvolve","agent":{"family":"google","model":"Gemini-3-Pro-Preview","confidence":"high","role":"author"}},{"date":"2025-12-27","value":57.89,"contributor":"Famou-Agent 2.0 (Gemini-2.5-Pro)","contributorKind":"method","contributorUrl":"https://github.com/baidubce/FM-Agent","agent":{"family":"google","model":"Gemini-2.5-Pro","confidence":"high","role":"subject"}},{"date":"2025-12-16","value":50.88,"contributor":"ML-Master 2.0","contributorKind":"method","contributorUrl":"https://github.com/sjtu-sai-agents/ML-Master","agent":{"family":"deepseek","model":"Deepseek-V3.2-Speciale","confidence":"high","role":"author"}},{"date":"2026-01-25","value":52.63,"contributor":"CAIR MARS","contributorKind":"method","contributorUrl":"https://arxiv.org/pdf/2602.02660","agent":{"family":"google","model":"Gemini-3-Pro-Preview","confidence":"high","role":"author"}},{"date":"2026-01-05","value":45.61,"contributor":"PiEvolve (12h)","contributorKind":"method","contributorUrl":"https://github.com/FractalAIResearchLabs/PiEvolve","agent":{"family":"google","model":"Gemini-3-Pro-Preview","confidence":"high","role":"author"}},{"date":"2025-12-07","value":44.74,"contributor":"Leeroo","contributorKind":"method","contributorUrl":"https://github.com/Leeroo-AI/kapso","agent":{"family":"google","model":"Gemini-3-Pro-Preview","confidence":"high","role":"author"}},{"date":"2025-11-10","value":45.61,"contributor":"Thesis","contributorKind":"method","contributorUrl":"https://thesislabs.ai","agent":{"family":"openai","model":"gpt-5-codex","confidence":"high","role":"author"}},{"date":"2025-11-25","value":34.21,"contributor":"CAIR MLE-STAR-Pro-1.5","contributorKind":"method","contributorUrl":"https://research.google/teams/cloud-ai-research/","agent":{"family":"google","model":"Gemini-2.5-Pro","confidence":"high","role":"author"}},{"date":"2025-10-10","value":36.84,"contributor":"Famou-Agent","contributorKind":"method","contributorUrl":"https://github.com/baidubce/FM-Agent","agent":{"family":"google","model":"Gemini-2.5-Pro","confidence":"high","role":"author"}},{"date":"2025-10-06","value":33.33,"contributor":"Operand ensemble","contributorKind":"method","contributorUrl":"https://operand.com","agent":{"family":"openai","model":"gpt-5","confidence":"high","role":"author"}},{"date":"2025-11-03","value":25.44,"contributor":"CAIR MLE-STAR-Pro-1.0","contributorKind":"method","contributorUrl":"https://research.google/teams/cloud-ai-research/","agent":{"family":"google","model":"Gemini-2.5-Pro","confidence":"high","role":"author"}},{"date":"2025-09-12","value":26.32,"contributor":"InternAgent","contributorKind":"method","contributorUrl":"https://github.com/Alpha-Innovator/InternAgent/","agent":{"family":"deepseek","model":"deepseek-r1","confidence":"high","role":"author"}},{"date":"2025-09-26","value":21.05,"contributor":"R&D-Agent (gpt-5)","contributorKind":"method","contributorUrl":"https://github.com/microsoft/RD-Agent","agent":{"family":"openai","model":"gpt-5","confidence":"high","role":"subject"}},{"date":"2025-07-28","value":29.82,"contributor":"Neo multi-agent","contributorKind":"method","contributorUrl":"https://heyneo.so/","agent":{"family":"other-ai","tool":"NEO","evidence":"NEO, a novel ML Engineering and Data science Agent","confidence":"high","role":"author"}},{"date":"2025-05-15","value":21.97,"contributor":"AIRA-dojo","contributorKind":"method","contributorUrl":"https://github.com/facebookresearch/aira-dojo/","agent":{"family":"openai","model":"o3","confidence":"high","role":"author"}},{"date":"2025-08-15","value":19.3,"contributor":"R&D-Agent (o3 + GPT-4.1)","contributorKind":"method","contributorUrl":"https://github.com/microsoft/RD-Agent","agent":{"family":"openai","model":"o3 + GPT-4.1","confidence":"high","role":"subject"}},{"date":"2025-06-17","value":20.18,"contributor":"ML-Master","contributorKind":"method","contributorUrl":"https://github.com/zeroxleo/ML-Master","agent":{"family":"deepseek","model":"deepseek-r1","confidence":"high","role":"author"}},{"date":"2025-05-14","value":8.95,"contributor":"R&D-Agent (o1-preview)","contributorKind":"method","contributorUrl":"https://github.com/microsoft/RD-Agent","agent":{"family":"openai","model":"o1-preview","confidence":"high","role":"subject"}},{"date":"2024-10-08","value":8.45,"contributor":"AIDE (o1-preview)","contributorKind":"method","contributorUrl":"https://github.com/wecoai/aideml","agent":{"family":"openai","model":"o1-preview","confidence":"high","role":"subject"}},{"date":"2024-10-08","value":3.06,"contributor":"AIDE (gpt-4o)","contributorKind":"method","contributorUrl":"https://github.com/wecoai/aideml","agent":{"family":"openai","model":"gpt-4o-2024-08-06","confidence":"high","role":"subject"}},{"date":"2024-10-08","value":2.63,"contributor":"AIDE (claude-3-5-sonnet)","contributorKind":"method","contributorUrl":"https://github.com/wecoai/aideml","agent":{"family":"anthropic","model":"claude-3-5-sonnet-20240620","confidence":"high","role":"subject"}},{"date":"2024-10-08","value":1.75,"contributor":"OpenHands","contributorKind":"method","agent":{"family":"openai","model":"gpt-4o-2024-08-06","confidence":"high","role":"author"}},{"date":"2024-10-08","value":0.66,"contributor":"AIDE (llama-3.1-405b)","contributorKind":"method","contributorUrl":"https://github.com/wecoai/aideml","agent":{"family":"meta","model":"llama-3.1-405b-instruct","confidence":"high","role":"subject"}},{"date":"2024-10-08","value":0,"contributor":"MLAB","contributorKind":"method","agent":{"family":"openai","model":"gpt-4o-2024-08-06","confidence":"high","role":"author"}},{"date":"2026-02-03","value":72.81,"contributor":"Disarray","contributorKind":"method","contributorUrl":"https://disarray.ai","isCurrentBest":true,"agent":{"family":"other-ai","model":"Ensemble","confidence":"high","role":"author"}},{"date":"2026-02-09","value":63.15,"contributor":"LoongFlow","contributorKind":"method","contributorUrl":"https://github.com/baidu-baige/LoongFlow","agent":{"family":"google","model":"Gemini-3-Flash-Preview","confidence":"high","role":"author"}}],"isOpen":false,"contested":true},{"id":"high-complexity","name":"High Complexity","description":"Percentage of competitions where the agent achieved any medal across the high-complexity competitions.","metricName":"Any Medal Rate","metricUnit":"%","metricDirection":"maximize","records":[{"date":"2026-02-23","value":42.22,"contributor":"Famou-Agent 2.0","contributorKind":"method","contributorUrl":"https://github.com/baidubce/FM-Agent","agent":{"family":"google","model":"Gemini-3-Pro-Preview","confidence":"high","role":"author"}},{"date":"2026-03-06","value":46.67,"contributor":"AIBuildAI","contributorKind":"method","contributorUrl":"https://github.com/aibuildai/AI-Build-AI","agent":{"family":"anthropic","model":"Claude-Opus-4.6","confidence":"high","role":"author"}},{"date":"2026-02-17","value":44.44,"contributor":"CAIR MARS+","contributorKind":"method","contributorUrl":"https://arxiv.org/pdf/2602.02660","agent":{"family":"google","model":"Gemini-3-Pro-Preview","confidence":"high","role":"author"}},{"date":"2026-02-14","value":42.22,"contributor":"MLEvolve","contributorKind":"method","contributorUrl":"https://github.com/InternScience/MLEvolve","agent":{"family":"google","model":"Gemini-3-Pro-Preview","confidence":"high","role":"author"}},{"date":"2026-01-05","value":40,"contributor":"PiEvolve (24h)","contributorKind":"method","contributorUrl":"https://github.com/FractalAIResearchLabs/PiEvolve","agent":{"family":"google","model":"Gemini-3-Pro-Preview","confidence":"high","role":"author"}},{"date":"2025-12-27","value":40,"contributor":"Famou-Agent 2.0 (Gemini-2.5-Pro)","contributorKind":"method","contributorUrl":"https://github.com/baidubce/FM-Agent","agent":{"family":"google","model":"Gemini-2.5-Pro","confidence":"high","role":"subject"}},{"date":"2025-12-16","value":42.22,"contributor":"ML-Master 2.0","contributorKind":"method","contributorUrl":"https://github.com/sjtu-sai-agents/ML-Master","agent":{"family":"deepseek","model":"Deepseek-V3.2-Speciale","confidence":"high","role":"author"}},{"date":"2026-01-25","value":37.78,"contributor":"CAIR MARS","contributorKind":"method","contributorUrl":"https://arxiv.org/pdf/2602.02660","agent":{"family":"google","model":"Gemini-3-Pro-Preview","confidence":"high","role":"author"}},{"date":"2026-01-05","value":35.55,"contributor":"PiEvolve (12h)","contributorKind":"method","contributorUrl":"https://github.com/FractalAIResearchLabs/PiEvolve","agent":{"family":"google","model":"Gemini-3-Pro-Preview","confidence":"high","role":"author"}},{"date":"2025-12-07","value":40,"contributor":"Leeroo","contributorKind":"method","contributorUrl":"https://github.com/Leeroo-AI/kapso","agent":{"family":"google","model":"Gemini-3-Pro-Preview","confidence":"high","role":"author"}},{"date":"2025-11-10","value":31.11,"contributor":"Thesis","contributorKind":"method","contributorUrl":"https://thesislabs.ai","agent":{"family":"openai","model":"gpt-5-codex","confidence":"high","role":"author"}},{"date":"2025-11-25","value":33.33,"contributor":"CAIR MLE-STAR-Pro-1.5","contributorKind":"method","contributorUrl":"https://research.google/teams/cloud-ai-research/","agent":{"family":"google","model":"Gemini-2.5-Pro","confidence":"high","role":"author"}},{"date":"2025-10-10","value":33.33,"contributor":"Famou-Agent","contributorKind":"method","contributorUrl":"https://github.com/baidubce/FM-Agent","agent":{"family":"google","model":"Gemini-2.5-Pro","confidence":"high","role":"author"}},{"date":"2025-10-06","value":20,"contributor":"Operand ensemble","contributorKind":"method","contributorUrl":"https://operand.com","agent":{"family":"openai","model":"gpt-5","confidence":"high","role":"author"}},{"date":"2025-11-03","value":31.11,"contributor":"CAIR MLE-STAR-Pro-1.0","contributorKind":"method","contributorUrl":"https://research.google/teams/cloud-ai-research/","agent":{"family":"google","model":"Gemini-2.5-Pro","confidence":"high","role":"author"}},{"date":"2025-09-12","value":24.44,"contributor":"InternAgent","contributorKind":"method","contributorUrl":"https://github.com/Alpha-Innovator/InternAgent/","agent":{"family":"deepseek","model":"deepseek-r1","confidence":"high","role":"author"}},{"date":"2025-09-26","value":22.22,"contributor":"R&D-Agent (gpt-5)","contributorKind":"method","contributorUrl":"https://github.com/microsoft/RD-Agent","agent":{"family":"openai","model":"gpt-5","confidence":"high","role":"subject"}},{"date":"2025-07-28","value":24.44,"contributor":"Neo multi-agent","contributorKind":"method","contributorUrl":"https://heyneo.so/","agent":{"family":"other-ai","tool":"NEO","evidence":"NEO, a novel ML Engineering and Data science Agent","confidence":"high","role":"author"}},{"date":"2025-05-15","value":21.67,"contributor":"AIRA-dojo","contributorKind":"method","contributorUrl":"https://github.com/facebookresearch/aira-dojo/","agent":{"family":"openai","model":"o3","confidence":"high","role":"author"}},{"date":"2025-08-15","value":26.67,"contributor":"R&D-Agent (o3 + GPT-4.1)","contributorKind":"method","contributorUrl":"https://github.com/microsoft/RD-Agent","agent":{"family":"openai","model":"o3 + GPT-4.1","confidence":"high","role":"subject"}},{"date":"2025-06-17","value":24.44,"contributor":"ML-Master","contributorKind":"method","contributorUrl":"https://github.com/zeroxleo/ML-Master","agent":{"family":"deepseek","model":"deepseek-r1","confidence":"high","role":"author"}},{"date":"2025-05-14","value":18.67,"contributor":"R&D-Agent (o1-preview)","contributorKind":"method","contributorUrl":"https://github.com/microsoft/RD-Agent","agent":{"family":"openai","model":"o1-preview","confidence":"high","role":"subject"}},{"date":"2024-10-08","value":11.67,"contributor":"AIDE (o1-preview)","contributorKind":"method","contributorUrl":"https://github.com/wecoai/aideml","agent":{"family":"openai","model":"o1-preview","confidence":"high","role":"subject"}},{"date":"2024-10-08","value":8.15,"contributor":"AIDE (gpt-4o)","contributorKind":"method","contributorUrl":"https://github.com/wecoai/aideml","agent":{"family":"openai","model":"gpt-4o-2024-08-06","confidence":"high","role":"subject"}},{"date":"2024-10-08","value":2.22,"contributor":"AIDE (claude-3-5-sonnet)","contributorKind":"method","contributorUrl":"https://github.com/wecoai/aideml","agent":{"family":"anthropic","model":"claude-3-5-sonnet-20240620","confidence":"high","role":"subject"}},{"date":"2024-10-08","value":2.22,"contributor":"OpenHands","contributorKind":"method","agent":{"family":"openai","model":"gpt-4o-2024-08-06","confidence":"high","role":"author"}},{"date":"2024-10-08","value":0,"contributor":"AIDE (llama-3.1-405b)","contributorKind":"method","contributorUrl":"https://github.com/wecoai/aideml","agent":{"family":"meta","model":"llama-3.1-405b-instruct","confidence":"high","role":"subject"}},{"date":"2024-10-08","value":0,"contributor":"MLAB","contributorKind":"method","agent":{"family":"openai","model":"gpt-4o-2024-08-06","confidence":"high","role":"author"}},{"date":"2026-02-03","value":71.11,"contributor":"Disarray","contributorKind":"method","contributorUrl":"https://disarray.ai","isCurrentBest":true,"agent":{"family":"other-ai","model":"Ensemble","confidence":"high","role":"author"}},{"date":"2026-02-09","value":40,"contributor":"LoongFlow","contributorKind":"method","contributorUrl":"https://github.com/baidu-baige/LoongFlow","agent":{"family":"google","model":"Gemini-3-Flash-Preview","confidence":"high","role":"author"}}],"isOpen":false,"contested":true}],"repo":{"owner":"openai","name":"mle-bench","branch":"main"},"participants":[],"stats":{"totalRecords":120,"uniqueParticipants":0,"firstSubmission":"2024-10-08","lastSubmission":"2026-03-06","recordsLast30d":0,"recordsLast90d":0},"agentStats":[{"family":"other-ai","records":8,"currentBests":4,"models":["Ensemble"],"contributors":["Neo multi-agent","Disarray"]},{"family":"google","records":44,"currentBests":0,"models":["Gemini-3-Pro-Preview","Gemini-2.5-Pro","Gemini-3-Flash-Preview"],"contributors":["Famou-Agent 2.0","CAIR MARS+","MLEvolve","PiEvolve (24h)","CAIR MARS","PiEvolve (12h)","Leeroo","CAIR MLE-STAR-Pro-1.5","Famou-Agent","CAIR MLE-STAR-Pro-1.0","LoongFlow"]},{"family":"openai","records":20,"currentBests":0,"models":["gpt-5-codex","gpt-5","o3","gpt-4o-2024-08-06"],"contributors":["Thesis","Operand ensemble","AIRA-dojo","OpenHands","MLAB"]},{"family":"deepseek","records":12,"currentBests":0,"models":["Deepseek-V3.2-Speciale","deepseek-r1"],"contributors":["ML-Master 2.0","InternAgent","ML-Master"]},{"family":"anthropic","records":4,"currentBests":0,"models":["Claude-Opus-4.6"],"contributors":["AIBuildAI"]}],"lastUpdated":"2026-09-18T16:32:55.537Z"},{"id":"llm-colosseum","name":"LLM Colosseum","tagline":"Evaluate and rank LLMs in real-time Street Fighter III battles using text or vision inputs.","description":"LLM Colosseum is a benchmark that pits LLMs against each other in real-time Street Fighter III matches. Models act as agents, receiving either text descriptions of the game state or raw screenshots, and must output valid game moves. Performance is tracked using an ELO rating system based on hundreds of fights.","url":"https://github.com/OpenGenerativeAI/llm-colosseum","host":"github","status":"active","tags":["llm-eval","street-fighter","multimodal","game-playing","elo-rating","agent"],"domain":"llm-eval","images":["https://github.com/OpenGenerativeAI/llm-colosseum/assets/78322686/3a212601-f54c-490d-aeb9-6f7c2401ebe6","https://raw.githubusercontent.com/OpenGenerativeAI/llm-colosseum/main/notebooks/result_matrix.png"],"organizer":{"name":"phospho","url":"https://phospho.ai","type":"company"},"participation":{"howToSubmit":"Create a new class inherited from Robot in agent/robot.py with your custom prompt or logic, and open a pull request to the main branch.","submissionFormat":"Python code (PR adding or modifying a Robot class)","requirements":["Python","Docker","DIAMBRA","Street Fighter III ROM"],"compute":"consumer-gpu","computeDetails":"Can be run locally using Ollama with consumer GPUs, or via commercial LLM API endpoints.","deadline":"rolling","prizes":"recognition only","cost":"free","verification":"human review"},"quickstart":"pip install -r requirements.txt\n# Follow instructions to install DIAMBRA and download the ROM to ~/.diambra/roms\nmake local","links":[{"label":"Hugging Face Leaderboard","url":"https://huggingface.co/spaces/junior-labs/llm-colosseum"},{"label":"Web Demo","url":"https://llm-colosseum.phospho.ai"},{"label":"DIAMBRA Installation Guide","url":"https://docs.diambra.ai/#installation"}],"problems":[{"id":"sf3-elo","name":"Street Fighter III ELO Ranking","description":"LLM agents compete in 1v1 matches in Street Fighter III. ELO ratings are calculated based on match outcomes.","metricName":"Rating","metricUnit":"ELO","metricDirection":"maximize","records":[{"value":1912.5,"contributor":"openai:gpt-4o:text","contributorKind":"model","isCurrentBest":true,"agent":{"family":"openai","model":"gpt-4o","confidence":"high","role":"subject"}},{"value":1835.27,"contributor":"openai:gpt-4o-mini:vision","contributorKind":"model","agent":{"family":"openai","model":"gpt-4o-mini","confidence":"high","role":"subject"}},{"value":1670.89,"contributor":"openai:gpt-4o-mini:text","contributorKind":"model","agent":{"family":"openai","model":"gpt-4o-mini","confidence":"high","role":"subject"}},{"value":1656.93,"contributor":"openai:gpt-4o:vision","contributorKind":"model","agent":{"family":"openai","model":"gpt-4o","confidence":"high","role":"subject"}},{"value":1654.61,"contributor":"mistral:pixtral-large-latest:vision","contributorKind":"model","agent":{"family":"mistral","model":"pixtral-large-latest","confidence":"high","role":"subject"}},{"value":1590.77,"contributor":"mistral:pixtral-12b-2409:vision","contributorKind":"model","agent":{"family":"mistral","model":"pixtral-12b-2409","confidence":"high","role":"subject"}},{"value":1569.03,"contributor":"mistral:pixtral-12b-2409:text","contributorKind":"model","agent":{"family":"mistral","model":"pixtral-12b-2409","confidence":"high","role":"subject"}},{"value":1441.45,"contributor":"together:meta-llama/Llama-3.2-90B-Vision-Instruct-Turbo:text","contributorKind":"model","agent":{"family":"meta","model":"Llama-3.2-90B-Vision-Instruct-Turbo","confidence":"high","role":"subject"}},{"value":1364.87,"contributor":"anthropic:claude-3-haiku-20240307:vision","contributorKind":"model","agent":{"family":"anthropic","model":"claude-3-haiku-20240307","confidence":"high","role":"subject"}},{"value":1356.32,"contributor":"mistral:pixtral-large-latest:text","contributorKind":"model","agent":{"family":"mistral","model":"pixtral-large-latest","confidence":"high","role":"subject"}},{"value":1333.6,"contributor":"anthropic:claude-3-haiku-20240307:text","contributorKind":"model","agent":{"family":"anthropic","model":"claude-3-haiku-20240307","confidence":"high","role":"subject"}},{"value":1314.61,"contributor":"anthropic:claude-3-sonnet-20240229:vision","contributorKind":"model","agent":{"family":"anthropic","model":"claude-3-sonnet-20240229","confidence":"high","role":"subject"}},{"value":1269.84,"contributor":"together:meta-llama/Llama-3.2-90B-Vision-Instruct-Turbo:vision","contributorKind":"model","agent":{"family":"meta","model":"Llama-3.2-90B-Vision-Instruct-Turbo","confidence":"high","role":"subject"}},{"value":1029.31,"contributor":"anthropic:claude-3-sonnet-20240229:text","contributorKind":"model","agent":{"family":"anthropic","model":"claude-3-sonnet-20240229","confidence":"high","role":"subject"}}],"isOpen":false,"contested":true}],"repo":{"owner":"OpenGenerativeAI","name":"llm-colosseum","branch":"main"},"participants":[],"stats":{"totalRecords":14,"uniqueParticipants":0,"recordsLast30d":0,"recordsLast90d":0},"agentStats":[],"lastUpdated":"2026-09-18T16:32:55.537Z"},{"id":"skab","name":"Skoltech Anomaly Benchmark","tagline":"Evaluate outlier and changepoint detection algorithms on multivariate time series data from an industrial testbed.","description":"SKAB is a benchmark designed for evaluating anomaly detection algorithms on multivariate time series data. It features 34 datasets with collective and point anomalies collected from a physical testbed. Submissions are evaluated on outlier detection (as a binary classification task) and changepoint detection (using Numenta Anomaly Benchmark metrics).","url":"https://github.com/waico/SKAB","host":"github","status":"active","tags":["anomaly-detection","time-series","changepoint-detection","outlier-detection","multivariate"],"domain":"ml-research","images":["https://raw.githubusercontent.com/waico/SKAB/master/docs/pictures/skab.png"],"organizer":{"name":"Skoltech","url":"https://www.skoltech.ru/en","type":"academic"},"participation":{"howToSubmit":"Evaluate your algorithm using the SKAB dataset and present/evaluate it on the Kaggle benchmark page or by submitting results to Papers with Code.","submissionFormat":"Jupyter Notebook or Kaggle submission format","requirements":["Python 3.10+","Poetry","LightGBM"],"compute":"consumer-gpu","computeDetails":"Standard deep learning models (LSTMs, Autoencoders) can be trained on consumer GPUs or CPUs.","verification":"Kaggle platform or manual reproduction via provided notebooks"},"quickstart":"git clone https://github.com/waico/SKAB.git\ncd SKAB\npoetry install\n# Run the evaluation notebooks under the notebooks/ directory","links":[{"label":"Kaggle Dataset & Benchmark","url":"https://www.kaggle.com/yuriykatser/skoltech-anomaly-benchmark-skab"},{"label":"Papers with Code CPD Leaderboard","url":"https://paperswithcode.com/sota/change-point-detection-on-skab"},{"label":"Evaluation Details","url":"https://github.com/waico/tsad/blob/main/examples/Evaluating.ipynb"}],"problems":[{"id":"outlier-detection","name":"Outlier Detection","description":"Anomalies are considered and marked up as single-point anomalies, evaluated as a binary classification problem.","metricName":"F1","metricDirection":"maximize","baseline":0,"records":[{"value":1,"contributor":"Perfect detector","contributorKind":"method","description":"Theoretical upper bound","isCurrentBest":true},{"value":0.78,"contributor":"Conv-AE","contributorKind":"method","description":"Convolutional Autoencoder"},{"value":0.78,"contributor":"MSET","contributorKind":"method","description":"Multivariate State Estimation Technique"},{"value":0.76,"contributor":"T-squared+Q (PCA-based)","contributorKind":"method","description":"Hotelling's T-squared + Q statistics based on PCA"},{"value":0.74,"contributor":"LSTM-AE","contributorKind":"method","description":"LSTM Autoencoder"},{"value":0.66,"contributor":"T-squared","contributorKind":"method","description":"Hotelling's T-squared statistics"},{"value":0.56,"contributor":"LSTM-VAE","contributorKind":"method","description":"LSTM Variational Autoencoder"},{"value":0.54,"contributor":"Vanilla LSTM","contributorKind":"method","description":"Neural Network with LSTM layer"},{"value":0.36,"contributor":"MSCRED","contributorKind":"method","description":"Multi-Scale Convolutional Recurrent Encoder-Decoder"},{"value":0.39,"contributor":"Vanilla AE","contributorKind":"method","description":"Feed-Forward Autoencoder"},{"value":0.29,"contributor":"Isolation forest","contributorKind":"method","description":"Outlier detection based on Random Forest"},{"value":0,"contributor":"Null detector","contributorKind":"method","description":"Predicts no anomalies","isBaseline":true}],"isOpen":false,"contested":true},{"id":"changepoint-detection","name":"Changepoint Detection","description":"Anomalies are considered and marked up as collective anomalies, evaluated using the Numenta Anomaly Benchmark (NAB) metric with a 60-second window.","metricName":"NAB (standard)","metricDirection":"maximize","baseline":0,"records":[{"value":100,"contributor":"Perfect detector","contributorKind":"method","description":"Theoretical upper bound","isCurrentBest":true},{"value":32.42,"contributor":"MSCRED","contributorKind":"method","description":"Multi-Scale Convolutional Recurrent Encoder-Decoder"},{"value":26.16,"contributor":"Isolation forest","contributorKind":"method","description":"Outlier detection based on Random Forest"},{"value":25.35,"contributor":"T-squared+Q (PCA-based)","contributorKind":"method","description":"Hotelling's T-squared + Q statistics based on PCA"},{"value":23.61,"contributor":"Conv-AE","contributorKind":"method","description":"Convolutional Autoencoder"},{"value":23.51,"contributor":"LSTM-AE","contributorKind":"method","description":"LSTM Autoencoder"},{"value":19.54,"contributor":"T-squared","contributorKind":"method","description":"Hotelling's T-squared statistics"},{"value":13.84,"contributor":"MSET","contributorKind":"method","description":"Multivariate State Estimation Technique"},{"value":11.41,"contributor":"Vanilla AE","contributorKind":"method","description":"Feed-Forward Autoencoder"},{"value":11.31,"contributor":"Vanilla LSTM","contributorKind":"method","description":"Neural Network with LSTM layer"},{"value":-0.09,"contributor":"ArimaFD","contributorKind":"method","description":"ARIMA-based fault detection algorithm"},{"value":0,"contributor":"Null detector","contributorKind":"method","description":"Predicts no changepoints","isBaseline":true}],"isOpen":false,"contested":true}],"repo":{"owner":"waico","name":"SKAB","branch":"master"},"participants":[],"stats":{"totalRecords":24,"uniqueParticipants":0,"recordsLast30d":0,"recordsLast90d":0},"agentStats":[],"lastUpdated":"2026-09-18T16:32:55.537Z"},{"id":"putnambench","name":"PutnamBench","tagline":"A multilingual benchmark for evaluating theorem-proving algorithms on Putnam Mathematical Competition problems.","description":"PutnamBench evaluates neural theorem-provers on competition mathematics problems from the William Lowell Putnam Mathematical Competition (1962-2025). It supports Lean 4, Isabelle, and Coq, featuring 1724 manually-crafted formalizations. Submissions are evaluated on proving correctness, with or without providing numerical answers.","url":"https://github.com/trishullab/PutnamBench","host":"github","status":"active","tags":["theorem-proving","formal-methods","lean4","isabelle","coq","mathematics","llm-eval"],"domain":"formal-methods","images":[],"organizer":{"name":"Trishul Lab","url":"https://trishullab.github.io/PutnamBench/","type":"academic"},"participation":{"howToSubmit":"Reach out privately at george.tsoukalas@utexas.edu with evaluation results accompanied by a preprint or publication.","submissionFormat":"Evaluation results accompanied by a preprint or publication. Do not include proofs as confirmation in any public setting.","requirements":["Lean 4","Isabelle","Coq"],"compute":"unknown","deadline":"rolling","prizes":"recognition only","cost":"free","verification":"human review"},"links":[{"label":"Website","url":"https://trishullab.github.io/PutnamBench/"},{"label":"Leaderboard","url":"https://trishullab.github.io/PutnamBench/leaderboard.html"},{"label":"Paper","url":"https://arxiv.org/abs/2407.11214"}],"problems":[],"repo":{"owner":"trishullab","name":"PutnamBench","branch":"main"},"participants":[],"stats":{"totalRecords":0,"uniqueParticipants":0,"recordsLast30d":0,"recordsLast90d":0},"agentStats":[],"lastUpdated":"2026-09-18T16:32:55.537Z"},{"id":"lora-speedrun","name":"LoRA Speedrun","tagline":"Optimize LoRA fine-tuning speed on a single L40S GPU to hit target accuracy benchmarks.","description":"A public leaderboard of wall-clock records for LoRA fine-tuning on frozen tasks and hardware. Participants optimize training speed on a single L40S GPU while meeting target accuracy thresholds. Submissions are verified by running 3x with fresh seeds on a network-blocked Modal sandbox.","url":"https://github.com/Saivineeth147/lora-speedrun","host":"github","status":"active","tags":["lora","fine-tuning","llm","speedrun","modal","optimization"],"domain":"systems-perf","images":["https://raw.githubusercontent.com/Saivineeth147/lora-speedrun/main/docs/leaderboard.png"],"organizer":{"name":"Saivineeth147","url":"https://github.com/Saivineeth147","type":"individual"},"participation":{"howToSubmit":"Copy submissions/TEMPLATE/ to submissions/NNN-<your-github-handle>/, add train.py, config.yaml, and NOTES.md, then open a PR.","submissionFormat":"A directory containing train.py, config.yaml, and NOTES.md.","requirements":["Python","PyTorch","Modal","CUDA"],"compute":"datacenter-gpu","computeDetails":"1× NVIDIA L40S (48 GB) on Modal network-blocked sandbox","deadline":"rolling","cost":"free","verification":"Automated CI validation, Claude security screen, and 3x sandboxed runs with fresh seeds on Modal L40S."},"quickstart":"git clone https://github.com/Saivineeth147/lora-speedrun && cd lora-speedrun\npip install modal pyyaml && modal setup\npython harness/modal_verify.py --prefetch\npython harness/modal_verify.py --submission submissions/000-baseline --runs 1","links":[{"label":"Hugging Face Space","url":"https://huggingface.co/spaces/vineeth98/lora-speedrun"}],"problems":[{"id":"t1","name":"Track 1 — GSM8K · Qwen2.5-1.5B · target ≥ 57.0% · 1× L40S","description":"Fine-tune Qwen2.5-1.5B on GSM8K train split to target accuracy ≥ 57.0% exact-match.","metricName":"time","metricUnit":"s","metricDirection":"minimize","baseline":716.9,"records":[{"date":"2026-07-18","value":716.9,"contributor":"Saivineeth147","contributorKind":"person","contributorUrl":"https://github.com/Saivineeth147","description":"Baseline: plain LoRA r=16 on all linear layers, 3 epochs, cosine LR. No tricks.","isBaseline":true,"isCurrentBest":false,"agent":{"family":"anthropic","model":"Claude Fable 5","tool":"Claude Code","evidence":"Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>","confidence":"medium","role":"author"}},{"date":"2026-07-18","value":364.9,"contributor":"Saivineeth147","contributorKind":"person","contributorUrl":"https://github.com/Saivineeth147","description":"Sequence packing + completion-only loss masking, 2 epochs. Same LoRA config as #0; ~2x faster at higher accuracy.","isBaseline":false,"isCurrentBest":false,"agent":{"family":"anthropic","model":"Claude Fable 5","tool":"Claude Code","evidence":"Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>","confidence":"medium","role":"author"}},{"date":"2026-07-20","value":113,"contributor":"stared","contributorKind":"person","contributorUrl":"https://github.com/stared","description":"One 4e-4 epoch over 3,000 examples with 2x optimizer updates (integrated-LR), 512-token packs. First outside record.","isBaseline":false,"isCurrentBest":false},{"date":"2026-07-20","value":104,"contributor":"stared","contributorKind":"person","contributorUrl":"https://github.com/stared","description":"Shortest-4k data pruning, 1 aggressive-LR epoch, <<...>> annotations stripped, custom GPU-resident packed loop, chunked completion-only CE (no full logits).","isBaseline":false,"isCurrentBest":false},{"date":"2026-07-25","value":43.9,"contributor":"slippylolo","contributorKind":"person","contributorUrl":"https://github.com/slippylolo","description":"0.75-epoch six-target BF16 LoRA: shortest-4k pruning with down_proj dropped (13.76M params), deterministic best-fit packing, per-layer fused base+LoRA-A q/k/v & gate/up GEMMs, chunked completion-only CE.","isBaseline":false,"isCurrentBest":false},{"date":"2026-08-03","value":21.6,"contributor":"slippylolo","contributorKind":"person","contributorUrl":"https://github.com/slippylolo","description":"Direct-from-safetensors Qwen load with a warmed page cache, then 12 optimizer updates: 8 full-network, then 4 through only the top decoder layer over a cached BF16 frozen-prefix boundary.","isBaseline":false,"isCurrentBest":true}],"isOpen":false,"contested":true},{"id":"t2","name":"Track 2 — SQuAD v1.1 · SmolLM2-1.7B · target ≥ 75.5% · 1× L40S","description":"Fine-tune SmolLM2-1.7B on SQuAD v1.1 train split to target accuracy ≥ 75.5% EM.","metricName":"time","metricUnit":"s","metricDirection":"minimize","baseline":525.4,"records":[{"date":"2026-07-20","value":525.4,"contributor":"Saivineeth147","contributorKind":"person","contributorUrl":"https://github.com/Saivineeth147","description":"Track 2 baseline: plain LoRA r=16 on SQuAD, first 20k examples, 1 epoch, full-sequence loss. No tricks.","isBaseline":true,"isCurrentBest":false,"agent":{"family":"anthropic","model":"Claude Fable 5","tool":"Claude Code","evidence":"Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>","confidence":"medium","role":"author"}},{"date":"2026-07-27","value":49.5,"contributor":"abacon","contributorKind":"person","contributorUrl":"https://github.com/abacon","description":"Track 2 port of the Track 1 stack: shortest-4k pruning, best-fit packing, fused base+LoRA-A GEMMs, chunked completion-only CE, 0.75 epoch on SmolLM2-1.7B.","isBaseline":false,"isCurrentBest":true}],"isOpen":false,"contested":false}],"repo":{"owner":"Saivineeth147","name":"lora-speedrun","branch":"main"},"participants":[{"name":"slippylolo","url":"https://github.com/slippylolo","submissions":2,"problems":["t1"],"lastActive":"2026-08-03","bestRank":1},{"name":"Saivineeth147","url":"https://github.com/Saivineeth147","submissions":3,"problems":["t1","t2"],"lastActive":"2026-07-20"},{"name":"stared","url":"https://github.com/stared","submissions":2,"problems":["t1"],"lastActive":"2026-07-20"},{"name":"abacon","url":"https://github.com/abacon","submissions":1,"problems":["t2"],"lastActive":"2026-07-27"}],"stats":{"totalRecords":8,"uniqueParticipants":4,"firstSubmission":"2026-07-18","lastSubmission":"2026-08-03","recordsLast30d":0,"recordsLast90d":8},"agentStats":[{"family":"anthropic","records":1,"currentBests":0,"models":["Claude Fable 5"],"contributors":["Saivineeth147"]}],"lastUpdated":"2026-09-18T16:32:55.537Z"},{"id":"sokoban-speedrun","name":"Sokoban Speedrun","tagline":"Optimize reinforcement learning recipes to solve Sokoban puzzles as fast as possible on fixed hardware.","description":"A benchmark for finding the fastest training recipes to solve Sokoban puzzles. It features an LLM Track for fine-tuning Qwen3-4B on 8xH100 GPUs and a Non-LLM Track for training from-scratch agents on a single H100 GPU.","url":"https://github.com/JeanKaddour/sokoban_speedrun","host":"github","status":"active","tags":["reinforcement-learning","sokoban","speedrun","llm-finetuning","ppo","grpo"],"domain":"ml-research","images":["https://raw.githubusercontent.com/JeanKaddour/sokoban_speedrun/main/assets/llm_records.png","https://raw.githubusercontent.com/JeanKaddour/sokoban_speedrun/main/assets/llm_train_solve_rate.png","https://raw.githubusercontent.com/JeanKaddour/sokoban_speedrun/main/assets/non_llm_records.png","https://raw.githubusercontent.com/JeanKaddour/sokoban_speedrun/main/assets/non_llm_train_solve_rate.png"],"organizer":{"name":"Jean Kaddour","url":"https://github.com/JeanKaddour","type":"individual"},"participation":{"howToSubmit":"Train and evaluate your model, assemble the record using the provided script, and open a PR with the record directory and updated leaderboard.","submissionFormat":"A record directory containing logs, eval JSON, source snapshot, and a README.md describing the idea.","requirements":["Python","PyTorch","uv","CUDA"],"compute":"datacenter-gpu","computeDetails":"LLM Track requires an 8xH100 node; Non-LLM Track requires a single H100.","deadline":"rolling","verification":"Rerun with a second seed; both runs must score above the target. The score column reports the worse of the two runs."},"quickstart":"# LLM Track\ncd llm && uv sync\nNODE_GPUS=8 uv run torchrun --standalone --nproc_per_node=3 -m speedrun\n\n# Non-LLM Track\ncd non_llm && uv sync\nav run python speedrun.py","links":[{"label":"Play Sokoban","url":"https://www.jeankaddour.com/sokoban"},{"label":"PufferLib","url":"https://github.com/pufferai/pufferlib"},{"label":"DeepMind Boxoban","url":"https://github.com/google-deepmind/boxoban-levels"}],"problems":[{"id":"llm-track","name":"LLM Track","description":"RL-fine-tune Qwen3-4B-Instruct-2507 from 57% to >80% held-out pass@1 on one 8xH100 node.","metricName":"time","metricUnit":"s","metricDirection":"minimize","baseline":2933,"records":[{"date":"2026-06-29","value":2933,"contributor":"JeanKaddour","contributorKind":"person","description":"GRPO, LR 1.6e-6 annealed, 75 steps","isBaseline":true,"contributorUrl":"https://github.com/JeanKaddour"},{"date":"2026-06-29","value":2213,"contributor":"dexhunter","contributorKind":"person","description":"steps + LR-decay horizon 75 → 60","agent":{"family":"openai","tool":"Codex","evidence":"dexhunter | 2026-06-29 | codex/llm-60step-record | Add 36:53 LLM-track GRPO record","confidence":"medium","role":"author"},"contributorUrl":"https://github.com/dexhunter"},{"date":"2026-07-02","value":2129,"contributor":"dexhunter","contributorKind":"person","description":"earlier stop: 54 steps","agent":{"family":"openai","tool":"Codex","evidence":"dexhunter | 2026-07-02 | codex/weco-54step-record | Add 35:29 GRPO 54-step LLM record","confidence":"medium","role":"author"},"contributorUrl":"https://github.com/dexhunter"},{"date":"2026-07-02","value":2020,"contributor":"dexhunter","contributorKind":"person","description":"Weco advantage shaping, 52 steps","agent":{"family":"openai","tool":"Codex","evidence":"dexhunter | 2026-07-03 | codex/weco-step8-record | Add 33:40 Weco advantage-shaping GRPO record (52 steps)","confidence":"medium","role":"author"},"contributorUrl":"https://github.com/dexhunter"},{"date":"2026-07-02","value":1587,"contributor":"lorenzflow","contributorKind":"person","description":"rollout budget 5632 → 4800 tokens, 48 steps","contributorUrl":"https://github.com/lorenzflow"},{"date":"2026-07-14","value":1551,"contributor":"lorenzflow","contributorKind":"person","description":"GRPO → CISPO, same 48-step recipe","contributorUrl":"https://github.com/lorenzflow"},{"date":"2026-07-15","value":1160,"contributor":"lorenzflow","contributorKind":"person","description":"earlier stop: 35 CISPO steps","isCurrentBest":true,"contributorUrl":"https://github.com/lorenzflow"}],"isOpen":false,"contested":true},{"id":"non-llm-track","name":"Non-LLM Track","description":"Train a from-scratch agent on a single H100 to >70% held-out solve rate on DeepMind Boxoban.","metricName":"time","metricUnit":"s","metricDirection":"minimize","baseline":1344,"records":[{"date":"2026-06-21","value":1344,"contributor":"JeanKaddour","contributorKind":"person","description":"cnn-mingru h256","isBaseline":true,"contributorUrl":"https://github.com/JeanKaddour"},{"date":"2026-06-29","value":1260,"contributor":"JeanKaddour","contributorKind":"person","description":"same recipe as #1, earliest clearing checkpoint","contributorUrl":"https://github.com/JeanKaddour"},{"date":"2026-06-30","value":955,"contributor":"JeanKaddour","contributorKind":"person","description":"torch.compile + steps-matched-anneal","contributorUrl":"https://github.com/JeanKaddour"},{"date":"2026-07-02","value":882,"contributor":"JeanKaddour","contributorKind":"person","description":"anneal horizon tuned 1300→1200 steps","contributorUrl":"https://github.com/JeanKaddour"},{"date":"2026-07-02","value":758,"contributor":"srijanpatel","contributorKind":"person","description":"conv-free shift + pooled-global encoder (sgpm2), 950-step anneal","isCurrentBest":true,"agent":{"family":"anthropic","model":"Claude Opus 4.8 (1M context)","evidence":"Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>","confidence":"high","role":"author"},"contributorUrl":"https://github.com/srijanpatel"}],"isOpen":false,"contested":true}],"repo":{"owner":"JeanKaddour","name":"sokoban_speedrun","branch":"main"},"participants":[{"name":"lorenzflow","url":"https://github.com/lorenzflow","submissions":3,"problems":["llm-track"],"lastActive":"2026-07-15","bestRank":1},{"name":"srijanpatel","url":"https://github.com/srijanpatel","submissions":1,"problems":["non-llm-track"],"bestRank":1,"lastActive":"2026-07-02"},{"name":"JeanKaddour","url":"https://github.com/JeanKaddour","submissions":5,"problems":["llm-track","non-llm-track"],"lastActive":"2026-07-02"},{"name":"dexhunter","url":"https://github.com/dexhunter","submissions":3,"problems":["llm-track"],"lastActive":"2026-07-02"}],"stats":{"totalRecords":12,"uniqueParticipants":4,"firstSubmission":"2026-06-21","lastSubmission":"2026-07-15","recordsLast30d":0,"recordsLast90d":12},"agentStats":[{"family":"anthropic","records":1,"currentBests":1,"models":["Claude Opus 4.8 (1M context)"],"contributors":["srijanpatel"]},{"family":"openai","records":3,"currentBests":0,"models":[],"contributors":["dexhunter"]}],"lastUpdated":"2026-09-18T16:32:55.537Z"},{"id":"lean-eval","name":"LeanEval","tagline":"Prove open formalization problems in Lean 4; models and submitters are ranked by problems solved.","description":"LeanEval is the Lean FRO's comparator-based benchmark for formal mathematics. Trusted theorem statements live in leanprover/lean-eval; a problem counts as solved when the comparator accepts a submitted proof. Results are sticky per (user, model, problem) and published at lean-lang.org/eval. 301 visible problems, 69 models, 54 submitters, 78 active problems still unsolved.","url":"https://lean-lang.org/eval/","host":"website","status":"active","tags":["lean","theorem-proving","formal-methods","mathlib","benchmark"],"domain":"formal-methods","images":[],"organizer":{"name":"Lean FRO","url":"https://lean-lang.org","type":"foundation"},"participation":{"howToSubmit":"Prove a problem's theorem in your own GitHub repository and submit the ref at lean-lang.org/eval/submit; the comparator verifies the proof and the result is appended to results/<login>.json in leanprover/lean-eval-submissions.","submissionFormat":"GitHub repo + ref containing the completed Lean proof, plus the declared model label used to produce it.","requirements":["Lean 4","Mathlib","GitHub account"],"compute":"none","computeDetails":"Scoring is a comparator check of the proof; whatever compute the prover needs is on the submitter's side.","deadline":"rolling","prizes":"recognition only","eligibility":"Open to everyone. Successes are sticky: an accepted (user, model, problem) tuple is never removed.","cost":"free","verification":"Comparator acceptance in CI, replay by maintainers, public audit log per user."},"venue":{"type":"online","location":"GitHub / lean-lang.org"},"quickstart":"git clone https://github.com/leanprover/lean-eval && cd lean-eval\nlake exe cache get && lake build\nlake exe lean-eval --help   # pick a problem under generated/, replace sorry, submit at lean-lang.org/eval/submit","links":[{"label":"Leaderboard","url":"https://lean-lang.org/eval/"},{"label":"Submit a solution","url":"https://lean-lang.org/eval/submit/"},{"label":"Benchmark problems (leanprover/lean-eval)","url":"https://github.com/leanprover/lean-eval"},{"label":"Results store (lean-eval-submissions)","url":"https://github.com/leanprover/lean-eval-submissions"}],"problems":[{"id":"open-annals_absolute_profinite_rigidity","name":"Absolute profinite rigidity and hyperbolic geometry","description":"Unsolved. Group: formalization-evaluation. Source: M. R. Bridson, D. B. McReynolds, A. W. Reid, and R. Spitler, `Absolute profinite rigidity and hyperbolic geometry`, Annals of Math, 192 (3) 2020. Statement take Problem by Thomas Browning.","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"id":"open-annals_bose_gases","name":"The energy of dilute Bose gases","description":"Unsolved. Group: formalization-evaluation. Source: S. Fournais and J. P. Solovej, `The energy of dilute Bose gases`, Annals of Math, 192 (3) 2020. Statement taken from https://github.com/ImperialCollegeLondon/An Problem by David Ledvinka.","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"id":"open-annals_bounded_multiplicative_functions","name":"Higher uniformity of bounded multiplicative functions in short intervals on average","description":"Unsolved. Group: formalization-evaluation. Source: K. Matomäki, M. Radziwiłł, T. Tao, J. Teräväinen, and T. Ziegler, `Higher uniformity of bounded multiplicative functions in short intervals on average`, Annals  Problem by David Ledvinka.","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"id":"open-annals_chowla_and_twin_prime_over_fq_t","name":"On the Chowla and twin primes conjectures over 𝔽_q[T]","description":"Unsolved. Group: formalization-evaluation. Source: W. Sawin and M. Shusterman, `On the Chowla and twin primes conjectures over 𝔽_q[T]`, Annals of Math, 196 (2) 2022. Statement taken from https://github.com/Impe Problem by Thomas Browning.","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"id":"open-annals_dirichlet_weyl_bound","name":"The Weyl bound for Dirichlet L-functions of cube-free conductor","description":"Unsolved. Group: formalization-evaluation. Source: I. Petrow and M. P. Young, `The Weyl bound for Dirichlet L-functions of cube-free conductor`, Annals of Math, 192 (2) 2020. Statement taken from https://github. Problem by Thomas Browning.","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"id":"open-annals_erdos_faber_lovasz_conjecture","name":"A proof of the Erdős–Faber–Lovász conjecture","description":"Unsolved. Group: formalization-evaluation. Source: D. Y. Kang, T. Kelly, D. Kühn, A. Methuku, and D. Osthus, `A proof of the Erdős–Faber–Lovász conjecture`, Annals of Math, 198 (2) 2023. Statement taken from htt Problem by David Ledvinka.","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"id":"open-annals_erdos_supersingular_primes","name":"A conjecture of Erdős, supersingular primes and short character sums","description":"Unsolved. Group: formalization-evaluation. Source: M. Bennett and S. Siksek, `A conjecture of Erdős, supersingular primes and short character sums`, Annals of Math, 191 (2) 2020. Statement taken from https://git Problem by Justus Springer.","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"id":"open-annals_finite_time_singularity","name":"Finite-time singularity formation for C^{1,α} solutions to the incompressible Euler equations on ℝ³","description":"Unsolved. Group: formalization-evaluation. Source: T. M. Elgindi, `Finite-time singularity formation for C^{1,α} solutions to the incompressible Euler equations on ℝ³`, Annals of Math, 194 (3) 2021. Statement ta Problem by David Ledvinka.","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"id":"open-annals_good_lt_codes","name":"Good Locally Testable Codes","description":"Unsolved. Group: formalization-evaluation. Source: I. Dinur, S. Evra, R. Livne, A. Lubotzky, and S. Mozes, `Good Locally Testable Codes`, Annals of Math, 203 (2) 2026. Statement taken from https://github.com/Imp Problem by Thomas Browning, Katerina Hristova.","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"id":"open-annals_hasse_principle_random_fano","name":"The Hasse principle for random Fano hypersurfaces","description":"Unsolved. Group: formalization-evaluation. Source: T. Browning, P. L. Boudec, and W. Sawin, `The Hasse principle for random Fano hypersurfaces`, Annals of Math, 197 (3) 2023. Statement taken from https://github. Problem by Justus Springer.","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"id":"open-annals_integer_multiplication","name":"Integer multiplication in time O(n log n)","description":"Unsolved. Group: formalization-evaluation. Source: D. Harvey and J. van der Hoeven, `Integer multiplication in time O(n log n)`, Annals of Math, 193 (2) 2021. Statement taken from https://github.com/ImperialColl Problem by Justus Springer.","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"id":"open-annals_mckay_conjecture","name":"The McKay Conjecture on character degrees","description":"Unsolved. Group: formalization-evaluation. Source: M. Cabanes and B. Späth, `The McKay Conjecture on character degrees`, Annals of Math, 203 (3) 2026. Statement taken from https://github.com/ImperialCollegeLondo Problem by Thomas Browning.","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"id":"open-annals_motivic_invariants","name":"Motivic invariants of birational maps","description":"Unsolved. Group: formalization-evaluation. Source: H.-Y. Lin and E. Shinder, `Motivic invariants of birational maps`, Annals of Math, 199 (1) 2024. Statement taken from https://github.com/ImperialCollegeLondon/A Problem by Justus Springer.","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"id":"open-annals_on_coherence_of_one_relator_groups","name":"On the coherence of one-relator groups and their group algebras","description":"Unsolved. Group: formalization-evaluation. Source: A. Jaikin-Zapirain and M. Linton, `On the coherence of one-relator groups and their group algebras`, Annals of Math, 201 (3) 2025. Statement taken from https:// Problem by Katerina Hristova.","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"id":"open-annals_on_property_t","name":"On property (T) for Aut(F_n) and SL_n(Z)","description":"Unsolved. Group: formalization-evaluation. Source: M. Kaluba, D. Kielak, and P. W. Nowak, `On property (T) for Aut(F_n) and SL_n(Z)`, Annals of Math, 193 (2) 2021. Statement taken from https://github.com/Imperia Problem by Thomas Browning.","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"id":"open-annals_pointwise_ergodic_theorems","name":"Pointwise ergodic theorems for non-conventional bilinear polynomial averages","description":"Unsolved. Group: formalization-evaluation. Source: B. Krause, M. Mirek, and T. Tao, `Pointwise ergodic theorems for non-conventional bilinear polynomial averages`, Annals of Math, 195 (3) 2022. Statement taken f Problem by David Ledvinka.","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"id":"open-annals_rectangular_peg_problem","name":"The rectangular peg problem","description":"Unsolved. Group: formalization-evaluation. Source: J. E. Greene and A. Lobb, `The rectangular peg problem`, Annals of Math, 194 (2) 2021. Statement taken from https://github.com/ImperialCollegeLondon/AnnalsChall Problem by Katerina Hristova.","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"id":"open-annals_simplicity_conjecture","name":"Proof of the simplicity conjecture","description":"Unsolved. Group: formalization-evaluation. Source: D. Cristofaro-Gardiner, V. Humilière, and S. Seyfaddini, `Proof of the simplicity conjecture`, Annals of Math, 199 (1) 2024. Statement taken from https://github Problem by Thomas Browning.","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"id":"open-annals_spread_of_a_finite_group","name":"The spread of a finite group","description":"Unsolved. Group: formalization-evaluation. Source: T. C. Burness, R. M. Guralnick, and S. Harper, `The spread of a finite group`, Annals of Math, 193 (2) 2021. Statement taken from https://github.com/ImperialCol Problem by Katerina Hristova.","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"id":"open-annals_symplectic_monodromy","name":"Symplectic monodromy at radius zero and equimultiplicity of μ-constant families","description":"Unsolved. Group: formalization-evaluation. Source: J. Fernández de Bobadilla and T. Pełka, `Symplectic monodromy at radius zero and equimultiplicity of μ-constant families`, Annals of Math, 200 (1) 2024. Stateme Problem by Justus Springer.","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"id":"open-annals_uniform_mordell_lang","name":"Uniformity in Mordell–Lang for curves","description":"Unsolved. Group: formalization-evaluation. Source: V. Dimitrov, Z. Gao, and P. Habegger, `Uniformity in Mordell–Lang for curves`, Annals of Math, 194 (1) 2021. Statement taken from https://github.com/ImperialCol Problem by Thomas Browning, Christian Merten.","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"id":"open-annals_wilkies_conjecture","name":"Wilkie's conjecture for Pfaffian structures","description":"Unsolved. Group: formalization-evaluation. Source: G. Binyamini, D. Novikov, and B. Zak, `Wilkie's conjecture for Pfaffian structures`, Annals of Math, 199 (2) 2024. Statement taken from https://github.com/Imper Problem by Justus Springer, Mathias Stout.","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"id":"open-annulus_theorem_dim_four","name":"The Annulus Theorem in dimension 4 (Quinn)","description":"Unsolved. Group: formalization-evaluation. Source: F. Quinn, *Ends of maps III: dimensions 4 and 5*, J. Differential Geom. 17 (1982), building on M. H. Freedman, *The topology of four-dimensional manifolds*, J.  Problem by Kim Morrison.","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"id":"open-annulus_theorem_high_dim","name":"The Annulus Theorem in dimension ≥ 5 (Kirby)","description":"Unsolved. Group: formalization-evaluation. Source: R. C. Kirby, *Stable homeomorphisms and the annulus conjecture*, Ann. of Math. 89 (1969). Problem by Kim Morrison.","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"id":"open-aspherical_integer_homology_four_sphere","name":"Existence of an aspherical integer homology 4-sphere","description":"Unsolved. Group: formalization-evaluation. Source: J. G. Ratcliffe and S. T. Tschantz, *On the Davis hyperbolic 4-manifold*, Topology Appl. 111 (2001); see also the discussion of [Kir97, Problem 4.17] in the K3  Problem by Kim Morrison.","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"id":"open-bakerWustholz_linearForms_logs","name":"Baker-Wüstholz theorem on linear forms in logarithms","description":"Unsolved. Group: formalization-evaluation. Source: A. Baker, G. Wüstholz, Logarithmic forms and group varieties, J. reine angew. Math. 442 (1993), 19-62. Problem by Ralf Stephan.","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"id":"open-bourgain_polynomial_ergodic","name":"Bourgain's polynomial ergodic theorem","description":"Unsolved. Group: formalization-evaluation. Source: J. Bourgain, *Pointwise ergodic theorems for arithmetic sets*, Publ. Math. IHÉS 69 (1989). Knill, *Some fundamental theorems in mathematics*, §173. Problem by Kim Morrison.","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"id":"open-budney_gabai_knotted_three_spheres","name":"Budney--Gabai knotted three-spheres in S¹ × S³","description":"Unsolved. Group: formalization-evaluation. Source: R. Budney and D. Gabai, 'Knotted 3-balls in S⁴', Corollary 8.6, arXiv:1912.09029v3 (2021), https://arxiv.org/abs/1912.09029. Problem by Vasily Ilin.","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"id":"open-cdt_linearIndependent","name":"Linear independence results of Calegari–Dimitrov–Tang","description":"Unsolved. Group: formalization-evaluation. Source: https://arxiv.org/abs/2408.15403 Problem by Junyan Xu.","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"id":"open-cerf_gamma_four","name":"Cerf's theorem: every self-diffeomorphism of S3 is smoothly isotopic to a linear isometry","description":"Unsolved. Group: formalization-evaluation. Source: J. Cerf, Sur les diffeomorphismes de la sphere de dimension trois, Lecture Notes in Mathematics 53, Springer (1968). Problem by Kim Morrison.","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"id":"open-ckmrv_fourier_interpolation","name":"Fourier interpolation in dimensions 8 and 24","description":"Unsolved. Group: formalization-evaluation. Source: H. Cohn, A. Kumar, S. D. Miller, D. Radchenko, and M. Viazovska, 'Universal optimality of the E8 and Leech lattices and interpolation formulas', Ann. of Math. 1 Problem by Kim Morrison.","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"id":"open-conway_knot_not_smoothly_slice","name":"The Conway knot is not smoothly slice","description":"Unsolved. Group: formalization-evaluation. Source: https://arxiv.org/abs/1808.02923 Problem by Kim Morrison.","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"id":"open-conway_knot_topologically_slice","name":"The Conway knot is topologically slice","description":"Unsolved. Group: formalization-evaluation. Source: Freedman, *The topology of four-dimensional manifolds*, J. Diff. Geom. 17 (1982). See also Freedman-Quinn, *Topology of 4-Manifolds*, Princeton 1990. Problem by Kim Morrison.","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"id":"open-derived_solidification_free_CW_homology","name":"Derived solidification of free CW complexes (light condensed mathematics)","description":"Unsolved. Group: formalization-evaluation. Source: https://github.com/dagurtomas/LeanCondensed (LeanCondensed/Projects/DerivedSolidCWHomology.lean); D. Clausen and P. Scholze, lectures on analytic stacks and lig Problem by Dagur Asgeirsson.","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"id":"open-direct_summand","name":"Direct summand theorem and derived variant","description":"Unsolved. Group: formalization-evaluation. Problem by Junyan Xu.","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"id":"open-einsiedler_katok_lindenstrauss","name":"Smallness of exceptional set to Littlewood's conjecture","description":"Unsolved. Group: formalization-evaluation. Problem by Junyan Xu.","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"id":"open-equichordal_point_unique","name":"Equichordal point theorem (convex curves have a unique equichordal point)","description":"Unsolved. Group: formalization-evaluation. Source: M. R. Rychlik, A complete solution to the equichordal point problem of Fujiwara, Blaschke, Rothe and Weizenböck, Invent. Math. 129 (1997). Listed as §205 in O.  Problem by Kim Morrison.","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"id":"open-exists_topologically_slice_not_smoothly_slice","name":"Existence of a topologically slice, not smoothly slice knot","description":"Unsolved. Group: formalization-evaluation. Source: Casson, 1980s (unpublished); Akbulut-Matveyev, *A convex decomposition theorem for 4-manifolds*, IMRN 1998. See also Hedden-Kirk-Livingston, *Non-slice linear c Problem by Kim Morrison.","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"id":"open-faltings","name":"Faltings' theorem (Mordell conjecture)","description":"Unsolved. Group: formalization-evaluation. Problem by Junyan Xu.","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"id":"open-fermat_last_theorem","name":"Fermat's Last Theorem","description":"Unsolved. Group: formalization-evaluation. Source: https://en.wikipedia.org/wiki/Fermat%27s_Last_Theorem Problem by Xuanji Li.","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"id":"open-five_transitive_card_classification","name":"Possible orders of 5-transitive finite permutation groups","description":"Unsolved. Group: formalization-evaluation. Source: Folklore via CFSG; classical work of Mathieu, Jordan; modern accounts in P. Cameron, Permutation Groups (1999). Problem by Kim Morrison.","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"id":"open-four_manifold_not_smooth","name":"Freedman's non-smoothability theorem","description":"Unsolved. Group: formalization-evaluation. Problem by Oliver Nash.","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"id":"open-friedlander_iwaniec","name":"Friedlander–Iwaniec theorem","description":"Unsolved. Group: formalization-evaluation. Source: Friedlander, John, and Henryk Iwaniec. “The Polynomial X² + Y⁴ Captures Its Primes.” Annals of Mathematics, vol. 148, no. 3, 1998, pp. 945–1040. JSTOR, https:// Problem by Bolton Bailey/Project Numina.","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"id":"open-hSpace_sphere_iff","name":"Adams: S^n is an H-space iff n = 0, 1, 3, 7","description":"Unsolved. Group: formalization-evaluation. Source: J. F. Adams, 'On the non-existence of elements of Hopf invariant one', Ann. of Math. 72 (1960), 20-104; J. F. Adams and M. F. Atiyah, 'K-theory and the Hopf inv Problem by Vasily Ilin.","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"id":"open-hilbert_smith_padic_dimension_three","name":"No continuous faithful ℤ_p action on a connected 3-manifold (Pardon 2013)","description":"Unsolved. Group: formalization-evaluation. Source: John Pardon, 'The Hilbert-Smith conjecture for three-manifolds', Journal of the American Mathematical Society 26 (2013), no. 3, 879-899, Theorem 1.5. https://do Problem by Jack McCarthy.","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"id":"open-jacobian_challenge_alggeo","name":"Jacobian of a smooth proper curve (Merten challenge)","description":"Unsolved. Group: formalization-evaluation. Source: https://leanprover.zulipchat.com/#narrow/stream/583336-Autoformalization/topic/Jacobian%20challenge/near/587802685 Problem by Christian Merten.","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"id":"open-kepler_conjecture","name":"Kepler conjecture (optimal sphere packing in ℝ³)","description":"Unsolved. Group: formalization-evaluation. Source: Kepler conjecture 1611 (J. Kepler, *Strena seu de Nive Sexangula*); proof T. Hales, 'A proof of the Kepler conjecture', Ann. of Math. (2) 162 (2005) 1065–1185;  Problem by Kim Morrison.","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"id":"open-kollar_lieblich_olsson_sawin","name":"Topological reconstruction theorems for varieties","description":"Unsolved. Group: formalization-evaluation. Source: János Kollár, Max Lieblich, Martin Olsson, and Will Sawin. The Zariski topology, linear systems, and algebraic varieties. Problem by Junyan Xu.","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"id":"open-linnik","name":"Linnik's theorem (L = 5.5)","description":"Unsolved. Group: formalization-evaluation. Source: Heath-Brown, D.R. (1992), Zero-Free Regions for Dirichlet L-Functions, and the Least Prime in an Arithmetic Progression. Proceedings of the London Mathematical  Problem by Bolton Bailey/Project Numina.","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"id":"open-mandelbar_not_path_connected","name":"Mandelbar (tricorn) is not path-connected (Hubbard–Schleicher)","description":"Unsolved. Group: formalization-evaluation. Source: John H. Hubbard and Dierk Schleicher, *Multicorns are not Path Connected*, arXiv:1209.1753 (2012); published in *Frontiers in Complex Dynamics* (Princeton Unive Problem by Kim Morrison.","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"id":"open-mandelbrot_boundary_dimh","name":"Hausdorff dimension of the Mandelbrot boundary (Shishikura)","description":"Unsolved. Group: formalization-evaluation. Source: M. Shishikura, The Hausdorff dimension of the boundary of the Mandelbrot set and Julia sets, Ann. of Math. 147 (1998). Listed as §260 in O. Knill, Some Fundamen Problem by Kim Morrison.","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"id":"open-manolescu_triangulation_disproof","name":"Manolescu's disproof of the triangulation conjecture","description":"Unsolved. Group: formalization-evaluation. Source: C. Manolescu, *Pin(2)-equivariant Seiberg–Witten Floer homology and the triangulation conjecture*, J. Amer. Math. Soc. 29 (2016), via the Galewski–Stern and Mat Problem by Kim Morrison.","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"id":"open-mazur_torsion","name":"Mazur's torsion theorem","description":"Unsolved. Group: formalization-evaluation. Source: B. Mazur, *Modular curves and the Eisenstein ideal*, Publ. IHÉS 47 (1977). Knill, *Some fundamental theorems in mathematics*, §139. Problem by Kim Morrison.","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"id":"open-milnor_exotic_sphere_seven","name":"Milnor's exotic 7-sphere","description":"Unsolved. Group: formalization-evaluation. Source: J. Milnor, On manifolds homeomorphic to the 7-sphere, Ann. of Math. 64 (1956), 399-405. Recorded as a `proof_wanted` in Mathlib/Geometry/Manifold/PoincareConjec Problem by Kim Morrison.","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"id":"open-mostow_rigidity","name":"Mostow rigidity","description":"Unsolved. Group: formalization-evaluation. Source: https://en.wikipedia.org/wiki/Mostow_rigidity_theorem#Algebraic_form and Gopal Prasad, *Strong rigidity of ℚ-rank 1 lattices*, Invent. Math. 21 (1973), 255–286. Problem by Junyan Xu.","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"id":"open-newlander_nirenberg","name":"Newlander–Nirenberg theorem","description":"Unsolved. Group: formalization-evaluation. Source: A. Newlander and L. Nirenberg, `Complex analytic coordinates in almost complex manifolds`, Annals of Math, 65 (3) 1957, 391-404. Problem by Kim Morrison.","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"id":"open-nikolov_segal","name":"Nikolov–Segal strong completeness theorem","description":"Unsolved. Group: formalization-evaluation. Source: Nikolay Nikolov and Dan Segal, *On finitely generated profinite groups, I: strong completeness and uniform bounds*, Annals of Mathematics 165 (2007), no. 1, 171 Problem by Adam Topaz.","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"id":"open-ore_conjecture","name":"The Ore conjecture: every element of a finite nonabelian simple group is a commutator","description":"Unsolved. Group: formalization-evaluation. Source: Martin W. Liebeck, E. A. O'Brien, Aner Shalev, and Pham Huu Tiep, *The Ore conjecture*, Journal of the European Mathematical Society 12 (2010), no. 4, 939–1008, Problem by Adam Topaz.","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"id":"open-pi6_sphere_three_mulEquiv_zmod_twelve","name":"pi_6 of the 3-sphere is Z/12","description":"Unsolved. Group: formalization-evaluation. Source: H. Toda, 'Composition Methods in Homotopy Groups of Spheres', Annals of Mathematics Studies 49, Princeton University Press, 1962. The value pi_6(S^3) = Z/12 als Problem by Vasily Ilin.","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"id":"open-pi_sphere_infinite_iff","name":"Serre finiteness for homotopy groups of spheres","description":"Unsolved. Group: formalization-evaluation. Source: J.-P. Serre, 'Homologie singuliere des espaces fibres. Applications', Ann. of Math. 54 (1951), 425-505; J.-P. Serre, 'Groupes d'homotopie et classes de groupes  Problem by Vasily Ilin.","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"id":"open-poincare_3d_smooth","name":"3D smooth Poincaré conjecture (Perelman)","description":"Unsolved. Group: formalization-evaluation. Source: G. Perelman, three arXiv preprints 2002-2003 (math/0211159, math/0303109, math/0307245). Recorded as a `proof_wanted` in Mathlib/Geometry/Manifold/PoincareConje Problem by Kim Morrison.","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"id":"open-poincare_3d_topological","name":"3D topological Poincaré conjecture (Perelman)","description":"Unsolved. Group: formalization-evaluation. Source: G. Perelman, The entropy formula for the Ricci flow and its geometric applications (arXiv:math/0211159, 2002); Ricci flow with surgery on three-manifolds (arXiv Problem by Kim Morrison.","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"id":"open-poincare_4d_topological","name":"4D topological Poincaré conjecture (Freedman)","description":"Unsolved. Group: formalization-evaluation. Source: M. H. Freedman, The topology of four-dimensional manifolds, J. Differential Geom. 17 (1982), 357-453. Specialization of `proof_wanted ContinuousMap.HomotopyEqui Problem by Kim Morrison.","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"id":"open-poincare_high_dim_topological","name":"Generalized topological Poincaré conjecture in dimensions ≥ 5 (Smale)","description":"Unsolved. Group: formalization-evaluation. Source: S. Smale, Generalized Poincaré's conjecture in dimensions greater than four, Ann. of Math. 74 (1961), 391-406. Topological case: M. H. A. Newman, The engulfing  Problem by Kim Morrison.","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"id":"open-ramanujan_petersson","name":"Ramanujan–Petersson conjecture for the τ-function (Deligne's theorem)","description":"Unsolved. Group: formalization-evaluation. Source: S. Ramanujan, 'On certain arithmetical functions', Trans. Cambridge Philos. Soc. 22 (1916) 159–184. H. Petersson, 'Theorie der automorphen Formen beliebiger ree Problem by Seewoo Lee.","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"id":"open-schreier_conjecture","name":"Schreier's conjecture: outer automorphism group of a finite simple group is solvable","description":"Unsolved. Group: formalization-evaluation. Source: O. Schreier, Über die Erweiterung von Gruppen II, Abh. Math. Sem. Univ. Hamburg 4 (1926); CFSG, completed c. 2004. Problem by Kim Morrison.","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"id":"open-shafarevich_solvable_galois","name":"Shafarevich's theorem on solvable Galois groups","description":"Unsolved. Group: formalization-evaluation. Source: I. R. Shafarevich, 'Construction of fields of algebraic numbers with given solvable Galois group', Izv. Akad. Nauk SSSR Ser. Mat. 18 (1954), no. 6, 525–578, htt Problem by Ryan Smith.","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"id":"open-smale_conjecture","name":"Smale conjecture (Hatcher) in relative parameterized form","description":"Unsolved. Group: formalization-evaluation. Source: A. Hatcher, A proof of the Smale conjecture, Diff(S3) = O(4), Ann. of Math. 117 (1983). Problem by Kim Morrison.","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"id":"open-space_groups_230","name":"230 space groups (Fedorov 1891 / Schoenflies 1891)","description":"Unsolved. Group: formalization-evaluation. Source: E. Fedorov, 'Симметрія правильныхъ системъ фигуръ' (Symmetry of regular systems of figures), Trans. Min. Soc. St. Petersburg 28 (1891) 1–146; A. Schoenflies, *K Problem by Kim Morrison.","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"id":"open-sphere_theorem_differentiable","name":"Differentiable sphere theorem (Brendle–Schoen)","description":"Unsolved. Group: formalization-evaluation. Source: S. Brendle & R. Schoen, *Manifolds with 1/4-pinched curvature are space forms*, J. AMS 22 (2009). Knill, *Some fundamental theorems in mathematics*, §121. Problem by Kim Morrison.","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"id":"open-ten_martini_problem","name":"Avila-Jitomirskaya Ten Martini Problem","description":"Unsolved. Group: formalization-evaluation. Source: A. Avila and S. Jitomirskaya, 'The Ten Martini Problem', Ann. of Math. 170 (2009), 303-342, https://doi.org/10.4007/annals.2009.170.303. Problem by Kim Morrison.","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"id":"open-two_ninety_theorem","name":"The 290 theorem","description":"Unsolved. Group: formalization-evaluation. Source: M. Bhargava, J. Hanke, Universal quadratic forms and the 290-theorem, preprint (2011). See also https://en.wikipedia.org/wiki/15_and_290_theorems Problem by Bolton Bailey/Project Numina.","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"id":"open-wang_zahl_kakeya_dimH","name":"Wang-Zahl: the three-dimensional Kakeya conjecture","description":"Unsolved. Group: formalization-evaluation. Source: H. Wang and J. Zahl, 'Volume estimates for unions of convex sets, and the Kakeya set conjecture in three dimensions', Theorem 1.1, arXiv:2502.17655 (2025), http Problem by Kim Morrison.","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"id":"open-watanabe_four_dim_smale_disproof","name":"Watanabe's disproof of the 4-dimensional Smale conjecture","description":"Unsolved. Group: formalization-evaluation. Source: T. Watanabe, *Some exotic nontrivial elements of the rational homotopy groups of Diff(S⁴)*, arXiv:1812.02448 (2018). Pairs with the positive S³ statement of Hat Problem by Kim Morrison.","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"id":"open-weak_goldbach","name":"Weak Goldbach theorem","description":"Unsolved. Group: formalization-evaluation. Source: H. A. Helfgott, The ternary Goldbach conjecture is true, arXiv:1312.7748 (2013), https://arxiv.org/abs/1312.7748; H. A. Helfgott and D. J. Platt, Numerical Veri Problem by Vasily Ilin.","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"id":"open-weil_conjectures","name":"Weil conjectures in terms of point counts","description":"Unsolved. Group: formalization-evaluation. Source: P. Deligne, 'La conjecture de Weil. I', Publications Mathématiques de l'Institut des Hautes Études Scientifiques, Volume 43, pages 273–307 (1974), https://www.n Problem by Junyan Xu.","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"id":"open-weinstein_conjecture_dim3","name":"Weinstein conjecture in dimension three (Taubes 2007)","description":"Unsolved. Group: formalization-evaluation. Source: C.H. Taubes, 'The Seiberg–Witten equations and the Weinstein conjecture', Geom. Topol. 11 (2007) 2117–2202; arXiv:math/0611007. Conjecture: A. Weinstein, 'On th Problem by Kim Morrison.","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"id":"open-whitney_embedding","name":"Whitney embedding theorem (strong form, dimension 2n)","description":"Unsolved. Group: formalization-evaluation. Source: H. Whitney, 'The self-intersections of a smooth n-manifold in 2n-space', Ann. of Math. (2) 45 (1944) 220–246. Earlier 2n+1 form: H. Whitney, 'Differentiable man Problem by Kim Morrison.","metricName":"solved","metricDirection":"maximize","baseline":0,"isOpen":true,"records":[],"contested":false},{"id":"models","name":"Models — problems solved","description":"Official leaderboard: each row is a model (or harness + model) ranked by the number of distinct benchmark problems it has solved; 301 problems in the catalog, 128 active.","metricName":"problems solved","metricDirection":"maximize","isOpen":false,"records":[{"value":210,"date":"2026-09-16","contributor":"Axiom Prover (Axiom Math)","contributorKind":"model","description":"1 submitter: Vilin97 (210)","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":true,"agent":{"family":"axiom","model":"Axiom Prover (Axiom Math)","evidence":"lean-lang.org/eval leaderboard.json: \"Axiom Prover (Axiom Math)\" solved 210","confidence":"high","role":"subject"}},{"value":204,"date":"2026-08-31","contributor":"Humanifa + GPT 5.6 sol","contributorKind":"model","description":"2 submitters: ZhengyangZhang06 (198), juihuichung (6)","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"family":"openai","model":"GPT 5.6 sol","tool":"Humanifa","evidence":"lean-lang.org/eval leaderboard.json: \"Humanifa + GPT 5.6 sol\" solved 204","confidence":"high","role":"author"}},{"value":200,"date":"2026-09-07","contributor":"Tau (caj.al)","contributorKind":"model","description":"1 submitter: lukerj00 (200)","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"family":"other-ai","model":"Tau (caj.al)","evidence":"lean-lang.org/eval leaderboard.json: \"Tau (caj.al)\" solved 200","confidence":"high","role":"subject"}},{"value":162,"date":"2026-08-20","contributor":"Aristotle (Harmonic)","contributorKind":"model","description":"7 submitters: LorenzoLuccioli (105), parabamoghv (42), JohnEdwardJennings (7), sqrt-of-2 (7), kim-em (6), adrianmartir (3), Parcly-Taxel (1)","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"family":"harmonic","model":"Aristotle (Harmonic)","evidence":"lean-lang.org/eval leaderboard.json: \"Aristotle (Harmonic)\" solved 162","confidence":"high","role":"subject"}},{"value":156,"date":"2026-08-13","contributor":"Seed Prover (ByteDance)","contributorKind":"model","description":"2 submitters: GanjinZero (135), hanwenzhu (21)","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"family":"bytedance","model":"Seed Prover (ByteDance)","evidence":"lean-lang.org/eval leaderboard.json: \"Seed Prover (ByteDance)\" solved 156","confidence":"high","role":"subject"}},{"value":112,"date":"2026-07-21","contributor":"GPT-5.6","contributorKind":"model","description":"2 submitters: Morgan-Griffiths (111), eohjelle (3)","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"family":"openai","model":"GPT-5.6","evidence":"lean-lang.org/eval leaderboard.json: \"GPT-5.6\" solved 112","confidence":"high","role":"subject"}},{"value":104,"date":"2026-08-20","contributor":"Stealth Model","contributorKind":"model","description":"1 submitter: rishistyping (104)","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"family":"other-ai","model":"Stealth Model","evidence":"lean-lang.org/eval leaderboard.json: \"Stealth Model\" solved 104","confidence":"high","role":"subject"}},{"value":79,"date":"2026-08-17","contributor":"Vasily-24-hour-gpt-5.6-speedrun","contributorKind":"model","description":"1 submitter: Vilin97 (79)","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"family":"openai","model":"Vasily-24-hour-gpt-5.6-speedrun","evidence":"lean-lang.org/eval leaderboard.json: \"Vasily-24-hour-gpt-5.6-speedrun\" solved 79","confidence":"high","role":"subject"}},{"value":56,"date":"2026-06-05","contributor":"Aleph Prover(logicalintelligence.com)","contributorKind":"model","description":"2 submitters: mayorov-m-a (52), antpavzhi (4)","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"family":"other-ai","model":"Aleph Prover(logicalintelligence.com)","evidence":"lean-lang.org/eval leaderboard.json: \"Aleph Prover(logicalintelligence.com)\" solved 56","confidence":"high","role":"subject"}},{"value":32,"date":"2026-09-17","contributor":"NEAR AI w/ DeepSeek V4","contributorKind":"model","description":"1 submitter: skidanovalex (32)","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"family":"deepseek","model":"DeepSeek V4","tool":"NEAR AI","evidence":"lean-lang.org/eval leaderboard.json: \"NEAR AI w/ DeepSeek V4\" solved 32","confidence":"high","role":"author"}},{"value":31,"date":"2026-08-29","contributor":"github.com/mlgraham (various models)","contributorKind":"model","description":"1 submitter: mlgraham (31)","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"family":"other-ai","model":"github.com/mlgraham (various models)","evidence":"lean-lang.org/eval leaderboard.json: \"github.com/mlgraham (various models)\" solved 31","confidence":"high","role":"subject"}},{"value":30,"date":"2026-06-25","contributor":"MerLean-Prover","contributorKind":"model","description":"1 submitter: doxtor6 (30)","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"family":"other-ai","model":"MerLean-Prover","evidence":"lean-lang.org/eval leaderboard.json: \"MerLean-Prover\" solved 30","confidence":"high","role":"subject"}},{"value":21,"date":"2026-05-22","contributor":"Antigravity (Multi-Model Ensemble: Gemini 3.1 Pro, Gemini 3 Flash, Claude 4.6 Sonnet/Opus)","contributorKind":"model","description":"1 submitter: daouid (21)","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"family":"anthropic","model":"Antigravity (Multi-Model Ensemble: Gemini 3.1 Pro, Gemini 3 Flash, Claude 4.6 Sonnet/Opus)","evidence":"lean-lang.org/eval leaderboard.json: \"Antigravity (Multi-Model Ensemble: Gemini 3.1 Pro, Gemini 3 Flash, Claude 4.6 Sonnet/Opus)\" solved 21","confidence":"high","role":"subject"}},{"value":21,"date":"2026-08-17","contributor":"savarin-6-hour-gpt-5.6-speedrun","contributorKind":"model","description":"1 submitter: savarin (21)","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"family":"openai","model":"savarin-6-hour-gpt-5.6-speedrun","evidence":"lean-lang.org/eval leaderboard.json: \"savarin-6-hour-gpt-5.6-speedrun\" solved 21","confidence":"high","role":"subject"}},{"value":17,"date":"2026-05-24","contributor":"Claude Opus 4.7 (1M context)","contributorKind":"model","description":"2 submitters: rkirov (15), jzuiddam (2)","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"family":"anthropic","model":"Claude Opus 4.7 (1M context)","evidence":"lean-lang.org/eval leaderboard.json: \"Claude Opus 4.7 (1M context)\" solved 17","confidence":"high","role":"subject"}},{"value":16,"date":"2026-06-08","contributor":"GPT-5.5","contributorKind":"model","description":"3 submitters: Morgan-Griffiths (10), sqrt-of-2 (9), A-M-Berns (4)","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"family":"openai","model":"GPT-5.5","evidence":"lean-lang.org/eval leaderboard.json: \"GPT-5.5\" solved 16","confidence":"high","role":"subject"}},{"value":16,"date":"2026-08-28","contributor":"github.com/mlgraham (Claude Fable 5)","contributorKind":"model","description":"1 submitter: mlgraham (16)","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"family":"anthropic","model":"github.com/mlgraham (Claude Fable 5)","evidence":"lean-lang.org/eval leaderboard.json: \"github.com/mlgraham (Claude Fable 5)\" solved 16","confidence":"high","role":"subject"}},{"value":8,"date":"2026-06-23","contributor":"EVO (deepthought.com.au)","contributorKind":"model","description":"1 submitter: test1-deepthought (8)","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"family":"other-ai","model":"EVO (deepthought.com.au)","evidence":"lean-lang.org/eval leaderboard.json: \"EVO (deepthought.com.au)\" solved 8","confidence":"high","role":"subject"}},{"value":7,"date":"2026-06-19","contributor":"Public accepted source + Codex packaging","contributorKind":"model","description":"1 submitter: rishistyping (7)","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"family":"openai","model":"Codex packaging","tool":"Public accepted source","evidence":"lean-lang.org/eval leaderboard.json: \"Public accepted source + Codex packaging\" solved 7","confidence":"high","role":"author"}},{"value":7,"date":"2026-08-21","contributor":"Humanize2 flame chase: GPT-5.6-sol + Claude Opus 5","contributorKind":"model","description":"1 submitter: ZhengyangZhang06 (7)","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"family":"anthropic","model":"Claude Opus 5","tool":"Humanize2 flame chase: GPT-5.6-sol","evidence":"lean-lang.org/eval leaderboard.json: \"Humanize2 flame chase: GPT-5.6-sol + Claude Opus 5\" solved 7","confidence":"high","role":"author"}},{"value":7,"date":"2026-09-08","contributor":"GPT-5.6 Sol","contributorKind":"model","description":"2 submitters: Vilin97 (5), n-yamaguchi-0729 (2)","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"family":"openai","model":"GPT-5.6 Sol","evidence":"lean-lang.org/eval leaderboard.json: \"GPT-5.6 Sol\" solved 7","confidence":"high","role":"subject"}},{"value":6,"date":"2026-08-13","contributor":"Opus-5","contributorKind":"model","description":"1 submitter: Vilin97 (6)","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"family":"anthropic","model":"Opus-5","evidence":"lean-lang.org/eval leaderboard.json: \"Opus-5\" solved 6","confidence":"high","role":"subject"}},{"value":5,"date":"2026-07-28","contributor":"Claude Fable 5","contributorKind":"model","description":"2 submitters: rkirov (4), will1491 (1)","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"family":"anthropic","model":"Claude Fable 5","evidence":"lean-lang.org/eval leaderboard.json: \"Claude Fable 5\" solved 5","confidence":"high","role":"subject"}},{"value":5,"date":"2026-09-15","contributor":"Forge","contributorKind":"model","description":"1 submitter: xiaoyangliu-sjtu (5)","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"family":"other-ai","model":"Forge","evidence":"lean-lang.org/eval leaderboard.json: \"Forge\" solved 5","confidence":"high","role":"subject"}},{"value":3,"date":"2026-07-12","contributor":"DeepSeek V4 Flash","contributorKind":"model","description":"1 submitter: rwst (3)","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"family":"deepseek","model":"DeepSeek V4 Flash","evidence":"lean-lang.org/eval leaderboard.json: \"DeepSeek V4 Flash\" solved 3","confidence":"high","role":"subject"}},{"value":3,"date":"2026-07-23","contributor":"Grok 4.5","contributorKind":"model","description":"1 submitter: rwst (3)","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"family":"xai","model":"Grok 4.5","evidence":"lean-lang.org/eval leaderboard.json: \"Grok 4.5\" solved 3","confidence":"high","role":"subject"}},{"value":3,"date":"2026-07-27","contributor":"deepseek-v4 and gpt-5.x","contributorKind":"model","description":"1 submitter: ivwumupy (3)","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"family":"openai","model":"deepseek-v4 and gpt-5.x","evidence":"lean-lang.org/eval leaderboard.json: \"deepseek-v4 and gpt-5.x\" solved 3","confidence":"high","role":"subject"}},{"value":3,"date":"2026-08-03","contributor":"mostly ChatGPT 5.6 Sol","contributorKind":"model","description":"1 submitter: mccorvie (3)","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"family":"openai","model":"mostly ChatGPT 5.6 Sol","evidence":"lean-lang.org/eval leaderboard.json: \"mostly ChatGPT 5.6 Sol\" solved 3","confidence":"high","role":"subject"}},{"value":2,"date":"2026-05-07","contributor":"GPT-5.5 Codex","contributorKind":"model","description":"1 submitter: A-M-Berns (2)","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"family":"openai","model":"GPT-5.5 Codex","evidence":"lean-lang.org/eval leaderboard.json: \"GPT-5.5 Codex\" solved 2","confidence":"high","role":"subject"}},{"value":2,"date":"2026-05-10","contributor":"Gemini 3.1 Pro","contributorKind":"model","description":"1 submitter: sqrt-of-2 (2)","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"family":"google","model":"Gemini 3.1 Pro","evidence":"lean-lang.org/eval leaderboard.json: \"Gemini 3.1 Pro\" solved 2","confidence":"high","role":"subject"}},{"value":2,"date":"2026-05-12","contributor":"[submission] aegis-of-the-unit-circle-logos","contributorKind":"model","description":"1 submitter: rishistyping (2)","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"family":"other-ai","model":"[submission] aegis-of-the-unit-circle-logos","evidence":"lean-lang.org/eval leaderboard.json: \"[submission] aegis-of-the-unit-circle-logos\" solved 2","confidence":"high","role":"subject"}},{"value":2,"date":"2026-05-30","contributor":"EVO","contributorKind":"model","description":"2 submitters: machinelearning2014 (2), test1-deepthought (2)","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"family":"other-ai","model":"EVO","evidence":"lean-lang.org/eval leaderboard.json: \"EVO\" solved 2","confidence":"high","role":"subject"}},{"value":2,"date":"2026-07-01","contributor":"Claude Opus 4.8 (1M context)","contributorKind":"model","description":"1 submitter: rkirov (2)","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"family":"anthropic","model":"Claude Opus 4.8 (1M context)","evidence":"lean-lang.org/eval leaderboard.json: \"Claude Opus 4.8 (1M context)\" solved 2","confidence":"high","role":"subject"}},{"value":2,"date":"2026-07-10","contributor":"Kimi K2.7","contributorKind":"model","description":"1 submitter: rwst (2)","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"family":"moonshot","model":"Kimi K2.7","evidence":"lean-lang.org/eval leaderboard.json: \"Kimi K2.7\" solved 2","confidence":"high","role":"subject"}},{"value":2,"date":"2026-08-06","contributor":"GPT-5.6 and Fable 5 (human-in-the-loop)","contributorKind":"model","description":"1 submitter: KitaKen1 (2)","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"family":"anthropic","model":"GPT-5.6 and Fable 5 (human-in-the-loop)","evidence":"lean-lang.org/eval leaderboard.json: \"GPT-5.6 and Fable 5 (human-in-the-loop)\" solved 2","confidence":"high","role":"subject"}},{"value":2,"date":"2026-08-22","contributor":"Humanize RLCR: GPT-5.6-sol (max) worker + reviewer","contributorKind":"model","description":"1 submitter: ZhengyangZhang06 (2)","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"family":"openai","model":"reviewer","tool":"Humanize RLCR: GPT-5.6-sol (max) worker","evidence":"lean-lang.org/eval leaderboard.json: \"Humanize RLCR: GPT-5.6-sol (max) worker + reviewer\" solved 2","confidence":"high","role":"author"}},{"value":2,"date":"2026-09-07","contributor":"Forall (Astrio)","contributorKind":"model","description":"1 submitter: nolanlwin (2)","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"family":"other-ai","model":"Forall (Astrio)","evidence":"lean-lang.org/eval leaderboard.json: \"Forall (Astrio)\" solved 2","confidence":"high","role":"subject"}},{"value":1,"date":"2026-04-30","contributor":"Claude Opus 4.7","contributorKind":"model","description":"1 submitter: rkirov (1)","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"family":"anthropic","model":"Claude Opus 4.7","evidence":"lean-lang.org/eval leaderboard.json: \"Claude Opus 4.7\" solved 1","confidence":"high","role":"subject"}},{"value":1,"date":"2026-05-29","contributor":"Claude Opus 4.7 + GPT-5.5 (human-in-the-loop)","contributorKind":"model","description":"1 submitter: yawara (1)","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"family":"anthropic","model":"GPT-5.5 (human-in-the-loop)","tool":"Claude Opus 4.7","evidence":"lean-lang.org/eval leaderboard.json: \"Claude Opus 4.7 + GPT-5.5 (human-in-the-loop)\" solved 1","confidence":"high","role":"author"}},{"value":1,"date":"2026-06-01","contributor":"GPT-5 Codex + Aristotle","contributorKind":"model","description":"1 submitter: rishistyping (1)","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"family":"openai","model":"Aristotle","tool":"GPT-5 Codex","evidence":"lean-lang.org/eval leaderboard.json: \"GPT-5 Codex + Aristotle\" solved 1","confidence":"high","role":"author"}},{"value":1,"date":"2026-06-02","contributor":"Autoform-Bot","contributorKind":"model","description":"1 submitter: niketp03 (1)","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"family":"other-ai","model":"Autoform-Bot","evidence":"lean-lang.org/eval leaderboard.json: \"Autoform-Bot\" solved 1","confidence":"high","role":"subject"}},{"value":1,"date":"2026-06-06","contributor":"Leanstral 1.4","contributorKind":"model","description":"1 submitter: albertqjiang (1)","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"family":"other-ai","model":"Leanstral 1.4","evidence":"lean-lang.org/eval leaderboard.json: \"Leanstral 1.4\" solved 1","confidence":"high","role":"subject"}},{"value":1,"date":"2026-06-11","contributor":"Claude Opus 4.7, 4.8 and Fable 5 + OSS contributions","contributorKind":"model","description":"1 submitter: rkirov (1)","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"family":"anthropic","model":"OSS contributions","tool":"Claude Opus 4.7, 4.8 and Fable 5","evidence":"lean-lang.org/eval leaderboard.json: \"Claude Opus 4.7, 4.8 and Fable 5 + OSS contributions\" solved 1","confidence":"high","role":"author"}},{"value":1,"date":"2026-06-15","contributor":"Community multi-model project","contributorKind":"model","description":"1 submitter: mrdouglasny (1)","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"family":"other-ai","model":"Community multi-model project","evidence":"lean-lang.org/eval leaderboard.json: \"Community multi-model project\" solved 1","confidence":"high","role":"subject"}},{"value":1,"date":"2026-06-26","contributor":"Codex (with human in the loop)","contributorKind":"model","description":"1 submitter: plby (1)","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"family":"openai","model":"Codex (with human in the loop)","evidence":"lean-lang.org/eval leaderboard.json: \"Codex (with human in the loop)\" solved 1","confidence":"high","role":"subject"}},{"value":1,"date":"2026-07-05","contributor":"Claude Opus 4.8 + Fable 5 (multi-agent)","contributorKind":"model","description":"1 submitter: ldct (1)","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"family":"anthropic","model":"Fable 5 (multi-agent)","tool":"Claude Opus 4.8","evidence":"lean-lang.org/eval leaderboard.json: \"Claude Opus 4.8 + Fable 5 (multi-agent)\" solved 1","confidence":"high","role":"author"}},{"value":1,"date":"2026-07-12","contributor":"Hy3","contributorKind":"model","description":"1 submitter: rwst (1)","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"family":"other-ai","model":"Hy3","evidence":"lean-lang.org/eval leaderboard.json: \"Hy3\" solved 1","confidence":"high","role":"subject"}},{"value":1,"date":"2026-07-13","contributor":"UNICO/NOUS pipeline - Claude (Anthropic)","contributorKind":"model","description":"1 submitter: Solarys431 (1)","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"family":"anthropic","model":"UNICO/NOUS pipeline - Claude (Anthropic)","evidence":"lean-lang.org/eval leaderboard.json: \"UNICO/NOUS pipeline - Claude (Anthropic)\" solved 1","confidence":"high","role":"subject"}},{"value":1,"date":"2026-07-14","contributor":"Claude Fable 5 (orchestrating Opus/Sonnet 4.8)","contributorKind":"model","description":"1 submitter: ianklatzco (1)","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"family":"anthropic","model":"Claude Fable 5 (orchestrating Opus/Sonnet 4.8)","evidence":"lean-lang.org/eval leaderboard.json: \"Claude Fable 5 (orchestrating Opus/Sonnet 4.8)\" solved 1","confidence":"high","role":"subject"}},{"value":1,"date":"2026-07-14","contributor":"GPT-5.6 (human-in-the-loop)","contributorKind":"model","description":"1 submitter: matthewjasper (1)","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"family":"openai","model":"GPT-5.6 (human-in-the-loop)","evidence":"lean-lang.org/eval leaderboard.json: \"GPT-5.6 (human-in-the-loop)\" solved 1","confidence":"high","role":"subject"}},{"value":1,"date":"2026-07-16","contributor":"Various models ( with human in the loop )","contributorKind":"model","description":"1 submitter: yawara (1)","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"family":"other-ai","model":"Various models ( with human in the loop )","evidence":"lean-lang.org/eval leaderboard.json: \"Various models ( with human in the loop )\" solved 1","confidence":"high","role":"subject"}},{"value":1,"date":"2026-07-20","contributor":"Trellis","contributorKind":"model","description":"1 submitter: wpegden (1)","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"family":"other-ai","model":"Trellis","evidence":"lean-lang.org/eval leaderboard.json: \"Trellis\" solved 1","confidence":"high","role":"subject"}},{"value":1,"date":"2026-07-21","contributor":"UNICO/NOUS: Claude (Fable 5 + Opus 4.8) + GPT-5.6-Sol","contributorKind":"model","description":"1 submitter: Solarys431 (1)","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"family":"anthropic","model":"Opus 4.8) + GPT-5.6-Sol","tool":"UNICO/NOUS: Claude (Fable 5","evidence":"lean-lang.org/eval leaderboard.json: \"UNICO/NOUS: Claude (Fable 5 + Opus 4.8) + GPT-5.6-Sol\" solved 1","confidence":"high","role":"author"}},{"value":1,"date":"2026-07-24","contributor":"various model (deepseek-v4, gpt-5.*)","contributorKind":"model","description":"1 submitter: ivwumupy (1)","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"family":"openai","model":"various model (deepseek-v4, gpt-5.*)","evidence":"lean-lang.org/eval leaderboard.json: \"various model (deepseek-v4, gpt-5.*)\" solved 1","confidence":"high","role":"subject"}},{"value":1,"date":"2026-07-25","contributor":"Fable 5 and GPT-5.6","contributorKind":"model","description":"1 submitter: adamtopaz (1)","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"family":"anthropic","model":"Fable 5 and GPT-5.6","evidence":"lean-lang.org/eval leaderboard.json: \"Fable 5 and GPT-5.6\" solved 1","confidence":"high","role":"subject"}},{"value":1,"date":"2026-08-04","contributor":"OpenAI Codex (GPT-5.6 Sol), Claude Code (Opus 5 and Opus 4.8)","contributorKind":"model","description":"1 submitter: rigwild (1)","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"family":"anthropic","model":"OpenAI Codex (GPT-5.6 Sol), Claude Code (Opus 5 and Opus 4.8)","evidence":"lean-lang.org/eval leaderboard.json: \"OpenAI Codex (GPT-5.6 Sol), Claude Code (Opus 5 and Opus 4.8)\" solved 1","confidence":"high","role":"subject"}},{"value":1,"date":"2026-08-07","contributor":"GPT-5.6 Codex (public-proof reproduction)","contributorKind":"model","description":"1 submitter: Vilin97 (1)","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"family":"openai","model":"GPT-5.6 Codex (public-proof reproduction)","evidence":"lean-lang.org/eval leaderboard.json: \"GPT-5.6 Codex (public-proof reproduction)\" solved 1","confidence":"high","role":"subject"}},{"value":1,"date":"2026-08-12","contributor":"GPT-5 Codex","contributorKind":"model","description":"1 submitter: Vilin97 (1)","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"family":"openai","model":"GPT-5 Codex","evidence":"lean-lang.org/eval leaderboard.json: \"GPT-5 Codex\" solved 1","confidence":"high","role":"subject"}},{"value":1,"date":"2026-08-13","contributor":"OpenAI Codex (GPT-5)","contributorKind":"model","description":"1 submitter: Vilin97 (1)","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"family":"openai","model":"OpenAI Codex (GPT-5)","evidence":"lean-lang.org/eval leaderboard.json: \"OpenAI Codex (GPT-5)\" solved 1","confidence":"high","role":"subject"}},{"value":1,"date":"2026-08-18","contributor":"Claude Fable 5 (Claude Code)","contributorKind":"model","description":"1 submitter: homeowmorphism (1)","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"family":"anthropic","model":"Claude Fable 5 (Claude Code)","evidence":"lean-lang.org/eval leaderboard.json: \"Claude Fable 5 (Claude Code)\" solved 1","confidence":"high","role":"subject"}},{"value":1,"date":"2026-08-20","contributor":"github.com/mlgraham — public accepted source, packaged (Claude Fable 5)","contributorKind":"model","description":"1 submitter: mlgraham (1)","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"family":"anthropic","model":"github.com/mlgraham — public accepted source, packaged (Claude Fable 5)","evidence":"lean-lang.org/eval leaderboard.json: \"github.com/mlgraham — public accepted source, packaged (Claude Fable 5)\" solved 1","confidence":"high","role":"subject"}},{"value":1,"date":"2026-08-22","contributor":"DeepScientist + GPT-5.6","contributorKind":"model","description":"1 submitter: LikC1606 (1)","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"family":"openai","model":"GPT-5.6","tool":"DeepScientist","evidence":"lean-lang.org/eval leaderboard.json: \"DeepScientist + GPT-5.6\" solved 1","confidence":"high","role":"author"}},{"value":1,"date":"2026-08-31","contributor":"OpenAI Codex (GPT-5.6 Sol)","contributorKind":"model","description":"1 submitter: Cond-t (1)","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"family":"openai","model":"OpenAI Codex (GPT-5.6 Sol)","evidence":"lean-lang.org/eval leaderboard.json: \"OpenAI Codex (GPT-5.6 Sol)\" solved 1","confidence":"high","role":"subject"}},{"value":1,"date":"2026-09-02","contributor":"LeanEval production launch canary 3dc17f41ccd0c86fa443687fac784a42f3798183","contributorKind":"model","description":"1 submitter: kim-em (1)","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"family":"other-ai","model":"LeanEval production launch canary 3dc17f41ccd0c86fa443687fac784a42f3798183","evidence":"lean-lang.org/eval leaderboard.json: \"LeanEval production launch canary 3dc17f41ccd0c86fa443687fac784a42f3798183\" solved 1","confidence":"high","role":"subject"}},{"value":1,"date":"2026-09-02","contributor":"BRILLiant + ChatGPT 5.6 Sol","contributorKind":"model","description":"1 submitter: LexBrill (1)","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"family":"openai","model":"ChatGPT 5.6 Sol","tool":"BRILLiant","evidence":"lean-lang.org/eval leaderboard.json: \"BRILLiant + ChatGPT 5.6 Sol\" solved 1","confidence":"high","role":"author"}},{"value":1,"date":"2026-09-07","contributor":"OpenAI Codex (GPT-6 + GPT-5.6 Sol)","contributorKind":"model","description":"1 submitter: lixiang90 (1)","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"family":"openai","model":"GPT-5.6 Sol)","tool":"OpenAI Codex (GPT-6","evidence":"lean-lang.org/eval leaderboard.json: \"OpenAI Codex (GPT-6 + GPT-5.6 Sol)\" solved 1","confidence":"high","role":"author"}},{"value":1,"date":"2026-09-13","contributor":"(various models)","contributorKind":"model","description":"1 submitter: mlgraham (1)","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"family":"other-ai","model":"(various models)","evidence":"lean-lang.org/eval leaderboard.json: \"(various models)\" solved 1","confidence":"high","role":"subject"}},{"value":1,"date":"2026-09-13","contributor":"LeanEval P1 launch canary","contributorKind":"model","description":"1 submitter: kim-em (1)","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"family":"other-ai","model":"LeanEval P1 launch canary","evidence":"lean-lang.org/eval leaderboard.json: \"LeanEval P1 launch canary\" solved 1","confidence":"high","role":"subject"}},{"value":1,"date":"2026-09-13","contributor":"Codex 5.6 Sol + HITL","contributorKind":"model","description":"1 submitter: kimihiro64 (1)","submissionUrls":["https://lean-lang.org/eval/"],"isCurrentBest":false,"agent":{"family":"openai","model":"HITL","tool":"Codex 5.6 Sol","evidence":"lean-lang.org/eval leaderboard.json: \"Codex 5.6 Sol + HITL\" solved 1","confidence":"high","role":"author"}}],"contested":true},{"id":"submitters","name":"Submitters — problems solved","description":"People who ran a model and got a proof accepted, ranked by distinct problems solved. Attribution is the declared model on their accepted results.","metricName":"problems solved","metricDirection":"maximize","isOpen":false,"records":[{"value":212,"date":"2026-09-16","contributor":"Vilin97","contributorKind":"person","contributorUrl":"https://github.com/Vilin97","description":"models: Axiom Prover (Axiom Math) ×210, Vasily-24-hour-gpt-5.6-speedrun ×79, Opus-5 ×6, gpt-5.6-Sol ×5, GPT-5 Codex ×1, GPT-5.6 Codex (public-proof reproduction) ×1, OpenAI Codex (GPT-5) ×1","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/Vilin97.json","https://github.com/Vilin97/lean-eval-speedrun","https://github.com/Vilin97/lean-eval-ado"],"agent":{"family":"axiom","model":"Axiom Prover (Axiom Math)","evidence":"results/Vilin97.json declared_model=\"Axiom Prover (Axiom Math)\" on 210 of 212 problems","confidence":"high","role":"author"},"isCurrentBest":true},{"value":200,"date":"2026-09-07","contributor":"lukerj00","contributorKind":"person","contributorUrl":"https://github.com/lukerj00","description":"models: Tau (caj.al) ×200","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/lukerj00.json","https://github.com/lukerj00/tau-lean-eval-submission"],"agent":{"family":"other-ai","model":"Tau (caj.al)","evidence":"results/lukerj00.json declared_model=\"Tau (caj.al)\" on 200 of 200 problems","confidence":"high","role":"author"}},{"value":198,"date":"2026-08-27","contributor":"ZhengyangZhang06","contributorKind":"person","contributorUrl":"https://github.com/ZhengyangZhang06","description":"models: Humanifa + GPT 5.6 sol ×198, Humanize2 flame chase: GPT-5.6-sol + Claude Opus 5 ×7, Humanize RLCR: GPT-5.6-sol (max) worker + reviewer ×2","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/ZhengyangZhang06.json","https://github.com/humanfia/lean-eval"],"agent":{"family":"openai","model":"GPT 5.6 sol","tool":"Humanifa","evidence":"results/ZhengyangZhang06.json declared_model=\"Humanifa + GPT 5.6 sol\" on 198 of 198 problems","confidence":"high","role":"author"}},{"value":135,"date":"2026-08-02","contributor":"GanjinZero","contributorKind":"person","contributorUrl":"https://github.com/GanjinZero","description":"models: Seed Prover (ByteDance) ×135","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/GanjinZero.json","https://github.com/GanjinZero/lidskii_last","https://github.com/GanjinZero/frobenius_group_determinant"],"agent":{"family":"bytedance","model":"Seed Prover (ByteDance)","evidence":"results/GanjinZero.json declared_model=\"Seed Prover (ByteDance)\" on 135 of 135 problems","confidence":"high","role":"author"}},{"value":111,"date":"2026-07-16","contributor":"Morgan-Griffiths","contributorKind":"person","contributorUrl":"https://github.com/Morgan-Griffiths","description":"models: GPT-5.6 ×111, GPT-5.5 ×10","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/Morgan-Griffiths.json","https://github.com/Morgan-Griffiths/4bdee1299a2cb3e5ad3ee11129659885","https://github.com/Morgan-Griffiths/540d938ce971aabb2511aae2ea81e911"],"agent":{"family":"openai","model":"GPT-5.6","evidence":"results/Morgan-Griffiths.json declared_model=\"GPT-5.6\" on 111 of 111 problems","confidence":"high","role":"author"}},{"value":105,"date":"2026-07-18","contributor":"LorenzoLuccioli","contributorKind":"person","contributorUrl":"https://github.com/LorenzoLuccioli","description":"models: Aristotle (Harmonic) ×105","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/LorenzoLuccioli.json","https://github.com/LorenzoLuccioli/lean-eval-private","https://github.com/LorenzoLuccioli/lean-eval"],"agent":{"family":"harmonic","model":"Aristotle (Harmonic)","evidence":"results/LorenzoLuccioli.json declared_model=\"Aristotle (Harmonic)\" on 105 of 105 problems","confidence":"high","role":"author"}},{"value":104,"date":"2026-08-20","contributor":"rishistyping","contributorKind":"person","contributorUrl":"https://github.com/rishistyping","description":"models: Stealth Model ×104, Public accepted source + Codex packaging ×7, [submission] aegis-of-the-unit-circle-logos ×2, GPT-5 Codex + Aristotle ×1","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/rishistyping.json","https://github.com/rishistyping/autoformalization-machine-intelligence","https://github.com/rishistyping/lean-eval-symplectic-matrix-det-submission"],"agent":{"family":"other-ai","model":"Stealth Model","evidence":"results/rishistyping.json declared_model=\"Stealth Model\" on 104 of 104 problems","confidence":"high","role":"author"}},{"value":52,"date":"2026-06-05","contributor":"mayorov-m-a","contributorKind":"person","contributorUrl":"https://github.com/mayorov-m-a","description":"models: Aleph Prover(logicalintelligence.com) ×52","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/mayorov-m-a.json","https://github.com/mayorov-m-a/aleph-lean-eval-subm-2026-may-08"],"agent":{"family":"other-ai","model":"Aleph Prover(logicalintelligence.com)","evidence":"results/mayorov-m-a.json declared_model=\"Aleph Prover(logicalintelligence.com)\" on 52 of 52 problems","confidence":"high","role":"author"}},{"value":42,"date":"2026-08-20","contributor":"parabamoghv","contributorKind":"person","contributorUrl":"https://github.com/parabamoghv","description":"models: Aristotle (Harmonic) ×42","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/parabamoghv.json","https://github.com/parabamoghv/lean-eval","https://github.com/parabamoghv/lean-eval-private"],"agent":{"family":"harmonic","model":"Aristotle (Harmonic)","evidence":"results/parabamoghv.json declared_model=\"Aristotle (Harmonic)\" on 42 of 42 problems","confidence":"high","role":"author"}},{"value":32,"date":"2026-09-13","contributor":"mlgraham","contributorKind":"person","contributorUrl":"https://github.com/mlgraham","description":"models: github.com/mlgraham (various models) ×31, github.com/mlgraham (Claude Fable 5) ×16, github.com/mlgraham — public accepted source, packaged (Claude Fable 5) ×1, (various models) ×1","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/mlgraham.json","https://github.com/mlgraham/lean-eval-submission","https://github.com/mlgraham/lean-eval-coc"],"agent":{"family":"other-ai","model":"github.com/mlgraham (various models)","evidence":"results/mlgraham.json declared_model=\"github.com/mlgraham (various models)\" on 31 of 32 problems","confidence":"high","role":"author"}},{"value":32,"date":"2026-09-17","contributor":"skidanovalex","contributorKind":"person","contributorUrl":"https://github.com/skidanovalex","description":"models: NEAR AI w/ DeepSeek V4 ×32","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/skidanovalex.json","https://github.com/SkidanovAlex/leaneval-deepseek"],"agent":{"family":"deepseek","model":"DeepSeek V4","tool":"NEAR AI","evidence":"results/skidanovalex.json declared_model=\"NEAR AI w/ DeepSeek V4\" on 32 of 32 problems","confidence":"high","role":"author"}},{"value":30,"date":"2026-06-25","contributor":"doxtor6","contributorKind":"person","contributorUrl":"https://github.com/doxtor6","description":"models: MerLean-Prover ×30","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/doxtor6.json","https://github.com/doxtor6/lean-eval"],"agent":{"family":"other-ai","model":"MerLean-Prover","evidence":"results/doxtor6.json declared_model=\"MerLean-Prover\" on 30 of 30 problems","confidence":"high","role":"author"}},{"value":21,"date":"2026-05-22","contributor":"daouid","contributorKind":"person","contributorUrl":"https://github.com/daouid","description":"models: Antigravity (Multi-Model Ensemble: Gemini 3.1 Pro, Gemini 3 Flash, Claude 4.6 Sonnet/Opus) ×21","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/daouid.json","https://github.com/daouid/lean-eval","https://github.com/daouid/73182095262a4e329bcec8579bdd573a"],"agent":{"family":"anthropic","model":"Antigravity (Multi-Model Ensemble: Gemini 3.1 Pro, Gemini 3 Flash, Claude 4.6 Sonnet/Opus)","evidence":"results/daouid.json declared_model=\"Antigravity (Multi-Model Ensemble: Gemini 3.1 Pro, Gemini 3 Flash, Claude 4.6 Sonnet/Opus)\" on 21 of 21 problems","confidence":"high","role":"author"}},{"value":21,"date":"2026-08-13","contributor":"hanwenzhu","contributorKind":"person","contributorUrl":"https://github.com/hanwenzhu","description":"models: Seed Prover (ByteDance) ×21","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/hanwenzhu.json","https://github.com/hanwenzhu/lean-eval-seedprover"],"agent":{"family":"bytedance","model":"Seed Prover (ByteDance)","evidence":"results/hanwenzhu.json declared_model=\"Seed Prover (ByteDance)\" on 21 of 21 problems","confidence":"high","role":"author"}},{"value":21,"date":"2026-08-17","contributor":"savarin","contributorKind":"person","contributorUrl":"https://github.com/savarin","description":"models: savarin-6-hour-gpt-5.6-speedrun ×21","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/savarin.json","https://github.com/savarin/lean-eval-speedrun"],"agent":{"family":"openai","model":"savarin-6-hour-gpt-5.6-speedrun","evidence":"results/savarin.json declared_model=\"savarin-6-hour-gpt-5.6-speedrun\" on 21 of 21 problems","confidence":"high","role":"author"}},{"value":19,"date":"2026-07-10","contributor":"rkirov","contributorKind":"person","contributorUrl":"https://github.com/rkirov","description":"models: Claude Opus 4.7 (1M context) ×15, Claude Fable 5 ×4, Claude Opus 4.8 (1M context) ×2, Claude Opus 4.7 ×1, Claude Opus 4.7, 4.8 and Fable 5 + OSS contributions ×1","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/rkirov.json","https://github.com/rkirov/lean-eval","https://github.com/rkirov/jordan_pick"],"agent":{"family":"anthropic","model":"Claude Opus 4.7 (1M context)","evidence":"results/rkirov.json declared_model=\"Claude Opus 4.7 (1M context)\" on 15 of 19 problems","confidence":"high","role":"author"}},{"value":11,"date":"2026-05-12","contributor":"sqrt-of-2","contributorKind":"person","contributorUrl":"https://github.com/sqrt-of-2","description":"models: Aristotle (Harmonic) ×7, GPT-5.5 ×7, GPT 5.5 ×2, Gemini 3.1 Pro ×2","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/sqrt-of-2.json","https://github.com/sqrt-of-2/931556b393de1dc23de63d1c5aefc109","https://github.com/sqrt-of-2/5d253653afb78ab2e57ae1446599c9eb"],"agent":{"family":"harmonic","model":"Aristotle (Harmonic)","evidence":"results/sqrt-of-2.json declared_model=\"Aristotle (Harmonic)\" on 7 of 11 problems","confidence":"high","role":"author"}},{"value":8,"date":"2026-06-23","contributor":"test1-deepthought","contributorKind":"person","contributorUrl":"https://github.com/test1-deepthought","description":"models: EVO (deepthought.com.au) ×8, EVO ×2","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/test1-deepthought.json","https://github.com/test1-deepthought/lean-eval-solutions","https://github.com/machinelearning2014/deepthought_lean_eval"],"agent":{"family":"other-ai","model":"EVO (deepthought.com.au)","evidence":"results/test1-deepthought.json declared_model=\"EVO (deepthought.com.au)\" on 8 of 8 problems","confidence":"high","role":"author"}},{"value":7,"date":"2026-06-14","contributor":"JohnEdwardJennings","contributorKind":"person","contributorUrl":"https://github.com/JohnEdwardJennings","description":"models: Aristotle (Harmonic) ×7","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/JohnEdwardJennings.json","https://github.com/JohnEdwardJennings/lean-eval-personal"],"agent":{"family":"harmonic","model":"Aristotle (Harmonic)","evidence":"results/JohnEdwardJennings.json declared_model=\"Aristotle (Harmonic)\" on 7 of 7 problems","confidence":"high","role":"author"}},{"value":7,"date":"2026-09-13","contributor":"kim-em","contributorKind":"person","contributorUrl":"https://github.com/kim-em","description":"models: Aristotle (Harmonic) ×6, LeanEval production launch canary 3dc17f41ccd0c86fa443687fac784a42f3798183 ×1, LeanEval P1 launch canary ×1","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/kim-em.json","https://github.com/kim-em/0e2146482105e90a2bb4b63f137975fb","https://github.com/kim-em/ba837b9520ea4ec64c7945bb1f0cf10f"],"agent":{"family":"harmonic","model":"Aristotle (Harmonic)","evidence":"results/kim-em.json declared_model=\"Aristotle (Harmonic)\" on 6 of 7 problems","confidence":"high","role":"author"}},{"value":6,"date":"2026-08-31","contributor":"juihuichung","contributorKind":"person","contributorUrl":"https://github.com/juihuichung","description":"models: Humanifa + GPT 5.6 sol ×6","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/juihuichung.json","https://github.com/juihuichung/lean-eval-submissions-private"],"agent":{"family":"openai","model":"GPT 5.6 sol","tool":"Humanifa","evidence":"results/juihuichung.json declared_model=\"Humanifa + GPT 5.6 sol\" on 6 of 6 problems","confidence":"high","role":"author"}},{"value":5,"date":"2026-09-15","contributor":"xiaoyangliu-sjtu","contributorKind":"person","contributorUrl":"https://github.com/xiaoyangliu-sjtu","description":"models: Forge ×5","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/xiaoyangliu-sjtu.json","https://github.com/XiaoyangLiu-sjtu/forge-leaneval-submissions"],"agent":{"family":"other-ai","model":"Forge","evidence":"results/xiaoyangliu-sjtu.json declared_model=\"Forge\" on 5 of 5 problems","confidence":"high","role":"author"}},{"value":4,"date":"2026-05-16","contributor":"A-M-Berns","contributorKind":"person","contributorUrl":"https://github.com/A-M-Berns","description":"models: GPT-5.5 ×4, GPT-5.5 Codex ×2","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/A-M-Berns.json","https://github.com/A-M-Berns/lean-eval-submissions"],"agent":{"family":"openai","model":"GPT-5.5","evidence":"results/A-M-Berns.json declared_model=\"GPT-5.5\" on 4 of 4 problems","confidence":"high","role":"author"}},{"value":4,"date":"2026-05-11","contributor":"antpavzhi","contributorKind":"person","contributorUrl":"https://github.com/antpavzhi","description":"models: Aleph Prover(logicalintelligence.com) ×4","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/antpavzhi.json","https://github.com/antpavzhi/lean-eval"],"agent":{"family":"other-ai","model":"Aleph Prover(logicalintelligence.com)","evidence":"results/antpavzhi.json declared_model=\"Aleph Prover(logicalintelligence.com)\" on 4 of 4 problems","confidence":"high","role":"author"}},{"value":3,"date":"2026-06-29","contributor":"adrianmartir","contributorKind":"person","contributorUrl":"https://github.com/adrianmartir","description":"models: Aristotle (Harmonic) ×3","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/adrianmartir.json","https://github.com/adrianmartir/lean-eval-private","https://github.com/LorenzoLuccioli/lean-eval-private"],"agent":{"family":"harmonic","model":"Aristotle (Harmonic)","evidence":"results/adrianmartir.json declared_model=\"Aristotle (Harmonic)\" on 3 of 3 problems","confidence":"high","role":"author"}},{"value":3,"date":"2026-07-21","contributor":"eohjelle","contributorKind":"person","contributorUrl":"https://github.com/eohjelle","description":"models: GPT-5.6 ×3","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/eohjelle.json","https://github.com/eohjelle/feit-thompson-odd-order-theorem"],"agent":{"family":"openai","model":"GPT-5.6","evidence":"results/eohjelle.json declared_model=\"GPT-5.6\" on 3 of 3 problems","confidence":"high","role":"author"}},{"value":3,"date":"2026-07-27","contributor":"ivwumupy","contributorKind":"person","contributorUrl":"https://github.com/ivwumupy","description":"models: deepseek-v4 and gpt-5.x ×3, various model (deepseek-v4, gpt-5.*) ×1","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/ivwumupy.json","https://github.com/Qiuzhen-CFSG/CFSG"],"agent":{"family":"openai","model":"deepseek-v4 and gpt-5.x","evidence":"results/ivwumupy.json declared_model=\"deepseek-v4 and gpt-5.x\" on 3 of 3 problems","confidence":"high","role":"author"}},{"value":3,"date":"2026-08-03","contributor":"mccorvie","contributorKind":"person","contributorUrl":"https://github.com/mccorvie","description":"models: mostly ChatGPT 5.6 Sol ×3","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/mccorvie.json","https://github.com/mccorvie/classification-of-surfaces","https://github.com/mccorvie/tcos-submission"],"agent":{"family":"openai","model":"mostly ChatGPT 5.6 Sol","evidence":"results/mccorvie.json declared_model=\"mostly ChatGPT 5.6 Sol\" on 3 of 3 problems","confidence":"high","role":"author"}},{"value":3,"date":"2026-07-23","contributor":"rwst","contributorKind":"person","contributorUrl":"https://github.com/rwst","description":"models: Grok 4.5 ×3, DeepSeek V4 Flash ×3, Kimi K2.7 ×2, Hy3 ×1","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/rwst.json","https://github.com/rwst/lean-eval"],"agent":{"family":"xai","model":"Grok 4.5","evidence":"results/rwst.json declared_model=\"Grok 4.5\" on 3 of 3 problems","confidence":"high","role":"author"}},{"value":2,"date":"2026-05-07","contributor":"jzuiddam","contributorKind":"person","contributorUrl":"https://github.com/jzuiddam","description":"models: Claude Opus 4.7 (1M context) ×2","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/jzuiddam.json","https://github.com/jzuiddam/symAction-eval-submission","https://github.com/jzuiddam/glAction-eval-submission"],"agent":{"family":"anthropic","model":"Claude Opus 4.7 (1M context)","evidence":"results/jzuiddam.json declared_model=\"Claude Opus 4.7 (1M context)\" on 2 of 2 problems","confidence":"high","role":"author"}},{"value":2,"date":"2026-08-06","contributor":"KitaKen1","contributorKind":"person","contributorUrl":"https://github.com/KitaKen1","description":"models: GPT-5.6 and Fable 5 (human-in-the-loop) ×2","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/KitaKen1.json","https://github.com/KitaKen1/lean-eval-pi-succ-sphere-n-mulequiv-zmod-two","https://github.com/KitaKen1/lean-eval-conway-schneeberger-fifteen"],"agent":{"family":"anthropic","model":"GPT-5.6 and Fable 5 (human-in-the-loop)","evidence":"results/KitaKen1.json declared_model=\"GPT-5.6 and Fable 5 (human-in-the-loop)\" on 2 of 2 problems","confidence":"high","role":"author"}},{"value":2,"date":"2026-05-30","contributor":"machinelearning2014","contributorKind":"person","contributorUrl":"https://github.com/machinelearning2014","description":"models: EVO ×2","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/machinelearning2014.json","https://github.com/machinelearning2014/deepthought_lean_eval"],"agent":{"family":"other-ai","model":"EVO","evidence":"results/machinelearning2014.json declared_model=\"EVO\" on 2 of 2 problems","confidence":"high","role":"author"}},{"value":2,"date":"2026-09-08","contributor":"n-yamaguchi-0729","contributorKind":"person","contributorUrl":"https://github.com/n-yamaguchi-0729","description":"models: GPT-5.6 Sol ×2","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/n-yamaguchi-0729.json","https://github.com/n-yamaguchi-0729/LeanEvalSubmissions"],"agent":{"family":"openai","model":"GPT-5.6 Sol","evidence":"results/n-yamaguchi-0729.json declared_model=\"GPT-5.6 Sol\" on 2 of 2 problems","confidence":"high","role":"author"}},{"value":2,"date":"2026-09-07","contributor":"nolanlwin","contributorKind":"person","contributorUrl":"https://github.com/nolanlwin","description":"models: Forall (Astrio) ×2","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/nolanlwin.json","https://github.com/nolanlwin/lean-eval"],"agent":{"family":"other-ai","model":"Forall (Astrio)","evidence":"results/nolanlwin.json declared_model=\"Forall (Astrio)\" on 2 of 2 problems","confidence":"high","role":"author"}},{"value":2,"date":"2026-07-21","contributor":"Solarys431","contributorKind":"person","contributorUrl":"https://github.com/Solarys431","description":"models: UNICO/NOUS: Claude (Fable 5 + Opus 4.8) + GPT-5.6-Sol ×1, UNICO/NOUS pipeline - Claude (Anthropic) ×1","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/Solarys431.json","https://github.com/Solarys431/lean-eval-platonic-classification","https://github.com/Solarys431/unico-lean-proofs"],"agent":{"family":"anthropic","model":"Opus 4.8) + GPT-5.6-Sol","tool":"UNICO/NOUS: Claude (Fable 5","evidence":"results/Solarys431.json declared_model=\"UNICO/NOUS: Claude (Fable 5 + Opus 4.8) + GPT-5.6-Sol\" on 1 of 2 problems","confidence":"high","role":"author"}},{"value":2,"date":"2026-07-16","contributor":"yawara","contributorKind":"person","contributorUrl":"https://github.com/yawara","description":"models: Various models ( with human in the loop ) ×1, Claude Opus 4.7 + GPT-5.5 (human-in-the-loop) ×1","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/yawara.json","https://github.com/yawara/odd-order-submission","https://github.com/yawara/lean-eval-submissions"],"agent":{"family":"other-ai","model":"Various models ( with human in the loop )","evidence":"results/yawara.json declared_model=\"Various models ( with human in the loop )\" on 1 of 2 problems","confidence":"high","role":"author"}},{"value":1,"date":"2026-07-25","contributor":"adamtopaz","contributorKind":"person","contributorUrl":"https://github.com/adamtopaz","description":"models: Fable 5 and GPT-5.6 ×1","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/adamtopaz.json","https://github.com/adamtopaz/NeukirchUchida"],"agent":{"family":"anthropic","model":"Fable 5 and GPT-5.6","evidence":"results/adamtopaz.json declared_model=\"Fable 5 and GPT-5.6\" on 1 of 1 problems","confidence":"high","role":"author"}},{"value":1,"date":"2026-06-06","contributor":"albertqjiang","contributorKind":"person","contributorUrl":"https://github.com/albertqjiang","description":"models: Leanstral 1.4 ×1","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/albertqjiang.json","https://github.com/albertqjiang/lean-eval-submissions"],"agent":{"family":"other-ai","model":"Leanstral 1.4","evidence":"results/albertqjiang.json declared_model=\"Leanstral 1.4\" on 1 of 1 problems","confidence":"high","role":"author"}},{"value":1,"date":"2026-08-31","contributor":"Cond-t","contributorKind":"person","contributorUrl":"https://github.com/Cond-t","description":"models: OpenAI Codex (GPT-5.6 Sol) ×1","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/Cond-t.json","https://github.com/Cond-t/thema-chen-theorem-lean-eval"],"agent":{"family":"openai","model":"OpenAI Codex (GPT-5.6 Sol)","evidence":"results/Cond-t.json declared_model=\"OpenAI Codex (GPT-5.6 Sol)\" on 1 of 1 problems","confidence":"high","role":"author"}},{"value":1,"date":"2026-08-18","contributor":"homeowmorphism","contributorKind":"person","contributorUrl":"https://github.com/homeowmorphism","description":"models: Claude Fable 5 (Claude Code) ×1","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/homeowmorphism.json","https://github.com/homeowmorphism/lean-eval-annals-marton"],"agent":{"family":"anthropic","model":"Claude Fable 5 (Claude Code)","evidence":"results/homeowmorphism.json declared_model=\"Claude Fable 5 (Claude Code)\" on 1 of 1 problems","confidence":"high","role":"author"}},{"value":1,"date":"2026-07-14","contributor":"ianklatzco","contributorKind":"person","contributorUrl":"https://github.com/ianklatzco","description":"models: Claude Fable 5 (orchestrating Opus/Sonnet 4.8) ×1","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/ianklatzco.json","https://github.com/ianklatzco/odd-order-lean"],"agent":{"family":"anthropic","model":"Claude Fable 5 (orchestrating Opus/Sonnet 4.8)","evidence":"results/ianklatzco.json declared_model=\"Claude Fable 5 (orchestrating Opus/Sonnet 4.8)\" on 1 of 1 problems","confidence":"high","role":"author"}},{"value":1,"date":"2026-09-13","contributor":"kimihiro64","contributorKind":"person","contributorUrl":"https://github.com/kimihiro64","description":"models: Codex 5.6 Sol + HITL ×1","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/kimihiro64.json","https://github.com/kimihiro64/Robin1984"],"agent":{"family":"openai","model":"HITL","tool":"Codex 5.6 Sol","evidence":"results/kimihiro64.json declared_model=\"Codex 5.6 Sol + HITL\" on 1 of 1 problems","confidence":"high","role":"author"}},{"value":1,"date":"2026-07-05","contributor":"ldct","contributorKind":"person","contributorUrl":"https://github.com/ldct","description":"models: Claude Opus 4.8 + Fable 5 (multi-agent) ×1","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/ldct.json","https://github.com/ldct/lean-eval-chudnovsky"],"agent":{"family":"anthropic","model":"Fable 5 (multi-agent)","tool":"Claude Opus 4.8","evidence":"results/ldct.json declared_model=\"Claude Opus 4.8 + Fable 5 (multi-agent)\" on 1 of 1 problems","confidence":"high","role":"author"}},{"value":1,"date":"2026-09-02","contributor":"LexBrill","contributorKind":"person","contributorUrl":"https://github.com/LexBrill","description":"models: BRILLiant + ChatGPT 5.6 Sol ×1","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/LexBrill.json","https://github.com/LexBrill/lean-eval-annals-unit-submission"],"agent":{"family":"openai","model":"ChatGPT 5.6 Sol","tool":"BRILLiant","evidence":"results/LexBrill.json declared_model=\"BRILLiant + ChatGPT 5.6 Sol\" on 1 of 1 problems","confidence":"high","role":"author"}},{"value":1,"date":"2026-08-22","contributor":"LikC1606","contributorKind":"person","contributorUrl":"https://github.com/LikC1606","description":"models: DeepScientist + GPT-5.6 ×1","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/LikC1606.json","https://github.com/LikC1606/7570e284ff8b474a4139db2092de6eec"],"agent":{"family":"openai","model":"GPT-5.6","tool":"DeepScientist","evidence":"results/LikC1606.json declared_model=\"DeepScientist + GPT-5.6\" on 1 of 1 problems","confidence":"high","role":"author"}},{"value":1,"date":"2026-09-07","contributor":"lixiang90","contributorKind":"person","contributorUrl":"https://github.com/lixiang90","description":"models: OpenAI Codex (GPT-6 + GPT-5.6 Sol) ×1","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/lixiang90.json","https://github.com/lixiang90/chen_theorem"],"agent":{"family":"openai","model":"GPT-5.6 Sol)","tool":"OpenAI Codex (GPT-6","evidence":"results/lixiang90.json declared_model=\"OpenAI Codex (GPT-6 + GPT-5.6 Sol)\" on 1 of 1 problems","confidence":"high","role":"author"}},{"value":1,"date":"2026-07-14","contributor":"matthewjasper","contributorKind":"person","contributorUrl":"https://github.com/matthewjasper","description":"models: GPT-5.6 (human-in-the-loop) ×1","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/matthewjasper.json","https://github.com/matthewjasper/m23_irrep_tensor_square_decomp"],"agent":{"family":"openai","model":"GPT-5.6 (human-in-the-loop)","evidence":"results/matthewjasper.json declared_model=\"GPT-5.6 (human-in-the-loop)\" on 1 of 1 problems","confidence":"high","role":"author"}},{"value":1,"date":"2026-06-15","contributor":"mrdouglasny","contributorKind":"person","contributorUrl":"https://github.com/mrdouglasny","description":"models: Community multi-model project ×1","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/mrdouglasny.json","https://github.com/mrdouglasny/jacobian-challenge"],"agent":{"family":"other-ai","model":"Community multi-model project","evidence":"results/mrdouglasny.json declared_model=\"Community multi-model project\" on 1 of 1 problems","confidence":"high","role":"author"}},{"value":1,"date":"2026-06-02","contributor":"niketp03","contributorKind":"person","contributorUrl":"https://github.com/niketp03","description":"models: Autoform-Bot ×1","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/niketp03.json","https://github.com/niketp03/fourier-dirichlet-fejer-submission"],"agent":{"family":"other-ai","model":"Autoform-Bot","evidence":"results/niketp03.json declared_model=\"Autoform-Bot\" on 1 of 1 problems","confidence":"high","role":"author"}},{"value":1,"date":"2026-05-23","contributor":"Parcly-Taxel","contributorKind":"person","contributorUrl":"https://github.com/Parcly-Taxel","description":"models: Aristotle (Harmonic) ×1","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/Parcly-Taxel.json","https://github.com/Parcly-Taxel/lean-eval-bbp"],"agent":{"family":"harmonic","model":"Aristotle (Harmonic)","evidence":"results/Parcly-Taxel.json declared_model=\"Aristotle (Harmonic)\" on 1 of 1 problems","confidence":"high","role":"author"}},{"value":1,"date":"2026-06-26","contributor":"plby","contributorKind":"person","contributorUrl":"https://github.com/plby","description":"models: Codex (with human in the loop) ×1","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/plby.json","https://github.com/plby/Erdos90"],"agent":{"family":"openai","model":"Codex (with human in the loop)","evidence":"results/plby.json declared_model=\"Codex (with human in the loop)\" on 1 of 1 problems","confidence":"high","role":"author"}},{"value":1,"date":"2026-08-04","contributor":"rigwild","contributorKind":"person","contributorUrl":"https://github.com/rigwild","description":"models: OpenAI Codex (GPT-5.6 Sol), Claude Code (Opus 5 and Opus 4.8) ×1","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/rigwild.json","https://github.com/rigwild/lean-math-problems"],"agent":{"family":"anthropic","model":"OpenAI Codex (GPT-5.6 Sol), Claude Code (Opus 5 and Opus 4.8)","evidence":"results/rigwild.json declared_model=\"OpenAI Codex (GPT-5.6 Sol), Claude Code (Opus 5 and Opus 4.8)\" on 1 of 1 problems","confidence":"high","role":"author"}},{"value":1,"date":"2026-07-28","contributor":"will1491","contributorKind":"person","contributorUrl":"https://github.com/will1491","description":"models: Claude Fable 5 ×1","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/will1491.json","https://github.com/will1491/uniformization"],"agent":{"family":"anthropic","model":"Claude Fable 5","evidence":"results/will1491.json declared_model=\"Claude Fable 5\" on 1 of 1 problems","confidence":"high","role":"author"}},{"value":1,"date":"2026-07-20","contributor":"wpegden","contributorKind":"person","contributorUrl":"https://github.com/wpegden","description":"models: Trellis ×1","submissionUrls":["https://github.com/leanprover/lean-eval-submissions/blob/main/results/wpegden.json","https://github.com/wpegden/unitdistance_challenge"],"agent":{"family":"other-ai","model":"Trellis","evidence":"results/wpegden.json declared_model=\"Trellis\" on 1 of 1 problems","confidence":"high","role":"author"}}],"contested":true}],"agentStats":[{"family":"axiom","records":1,"currentBests":1,"models":["Axiom Prover (Axiom Math)"],"contributors":["Vilin97"]},{"family":"openai","records":24,"currentBests":0,"models":["GPT 5.6 sol","Codex packaging","reviewer","Aristotle","GPT-5.6","ChatGPT 5.6 Sol","GPT-5.6 Sol)","HITL","savarin-6-hour-gpt-5.6-speedrun","GPT-5.5","deepseek-v4 and gpt-5.x","mostly ChatGPT 5.6 Sol","GPT-5.6 Sol","OpenAI Codex (GPT-5.6 Sol)","GPT-5.6 (human-in-the-loop)","Codex (with human in the loop)"],"contributors":["Humanifa + GPT 5.6 sol","Public accepted source + Codex packaging","Humanize RLCR: GPT-5.6-sol (max) worker + reviewer","GPT-5 Codex + Aristotle","DeepScientist + GPT-5.6","BRILLiant + ChatGPT 5.6 Sol","OpenAI Codex (GPT-6 + GPT-5.6 Sol)","Codex 5.6 Sol + HITL","ZhengyangZhang06","Morgan-Griffiths","savarin","juihuichung","A-M-Berns","eohjelle","ivwumupy","mccorvie","n-yamaguchi-0729","Cond-t","kimihiro64","LexBrill","LikC1606","lixiang90","matthewjasper","plby"]},{"family":"anthropic","records":16,"currentBests":0,"models":["Claude Opus 5","GPT-5.5 (human-in-the-loop)","OSS contributions","Fable 5 (multi-agent)","Opus 4.8) + GPT-5.6-Sol","Antigravity (Multi-Model Ensemble: Gemini 3.1 Pro, Gemini 3 Flash, Claude 4.6 Sonnet/Opus)","Claude Opus 4.7 (1M context)","GPT-5.6 and Fable 5 (human-in-the-loop)","Fable 5 and GPT-5.6","Claude Fable 5 (Claude Code)","Claude Fable 5 (orchestrating Opus/Sonnet 4.8)","OpenAI Codex (GPT-5.6 Sol), Claude Code (Opus 5 and Opus 4.8)","Claude Fable 5"],"contributors":["Humanize2 flame chase: GPT-5.6-sol + Claude Opus 5","Claude Opus 4.7 + GPT-5.5 (human-in-the-loop)","Claude Opus 4.7, 4.8 and Fable 5 + OSS contributions","Claude Opus 4.8 + Fable 5 (multi-agent)","UNICO/NOUS: Claude (Fable 5 + Opus 4.8) + GPT-5.6-Sol","daouid","rkirov","jzuiddam","KitaKen1","Solarys431","adamtopaz","homeowmorphism","ianklatzco","ldct","rigwild","will1491"]},{"family":"other-ai","records":15,"currentBests":0,"models":["Tau (caj.al)","Stealth Model","Aleph Prover(logicalintelligence.com)","github.com/mlgraham (various models)","MerLean-Prover","EVO (deepthought.com.au)","Forge","EVO","Forall (Astrio)","Various models ( with human in the loop )","Leanstral 1.4","Community multi-model project","Autoform-Bot","Trellis"],"contributors":["lukerj00","rishistyping","mayorov-m-a","mlgraham","doxtor6","test1-deepthought","xiaoyangliu-sjtu","antpavzhi","machinelearning2014","nolanlwin","yawara","albertqjiang","mrdouglasny","niketp03","wpegden"]},{"family":"harmonic","records":7,"currentBests":0,"models":["Aristotle (Harmonic)"],"contributors":["LorenzoLuccioli","parabamoghv","sqrt-of-2","JohnEdwardJennings","kim-em","adrianmartir","Parcly-Taxel"]},{"family":"deepseek","records":2,"currentBests":0,"models":["DeepSeek V4"],"contributors":["NEAR AI w/ DeepSeek V4","skidanovalex"]},{"family":"bytedance","records":2,"currentBests":0,"models":["Seed Prover (ByteDance)"],"contributors":["GanjinZero","hanwenzhu"]},{"family":"xai","records":1,"currentBests":0,"models":["Grok 4.5"],"contributors":["rwst"]}],"repo":{"owner":"leanprover","name":"lean-eval-submissions","branch":"main"},"participants":[{"name":"Vilin97","url":"https://github.com/Vilin97","submissions":1,"problems":["submitters"],"bestRank":1,"lastActive":"2026-09-16"},{"name":"lukerj00","url":"https://github.com/lukerj00","submissions":1,"problems":["submitters"],"lastActive":"2026-09-07"},{"name":"ZhengyangZhang06","url":"https://github.com/ZhengyangZhang06","submissions":1,"problems":["submitters"],"lastActive":"2026-08-27"},{"name":"GanjinZero","url":"https://github.com/GanjinZero","submissions":1,"problems":["submitters"],"lastActive":"2026-08-02"},{"name":"Morgan-Griffiths","url":"https://github.com/Morgan-Griffiths","submissions":1,"problems":["submitters"],"lastActive":"2026-07-16"},{"name":"LorenzoLuccioli","url":"https://github.com/LorenzoLuccioli","submissions":1,"problems":["submitters"],"lastActive":"2026-07-18"},{"name":"rishistyping","url":"https://github.com/rishistyping","submissions":1,"problems":["submitters"],"lastActive":"2026-08-20"},{"name":"mayorov-m-a","url":"https://github.com/mayorov-m-a","submissions":1,"problems":["submitters"],"lastActive":"2026-06-05"},{"name":"parabamoghv","url":"https://github.com/parabamoghv","submissions":1,"problems":["submitters"],"lastActive":"2026-08-20"},{"name":"mlgraham","url":"https://github.com/mlgraham","submissions":1,"problems":["submitters"],"lastActive":"2026-09-13"},{"name":"skidanovalex","url":"https://github.com/skidanovalex","submissions":1,"problems":["submitters"],"lastActive":"2026-09-17"},{"name":"doxtor6","url":"https://github.com/doxtor6","submissions":1,"problems":["submitters"],"lastActive":"2026-06-25"},{"name":"daouid","url":"https://github.com/daouid","submissions":1,"problems":["submitters"],"lastActive":"2026-05-22"},{"name":"hanwenzhu","url":"https://github.com/hanwenzhu","submissions":1,"problems":["submitters"],"lastActive":"2026-08-13"},{"name":"savarin","url":"https://github.com/savarin","submissions":1,"problems":["submitters"],"lastActive":"2026-08-17"},{"name":"rkirov","url":"https://github.com/rkirov","submissions":1,"problems":["submitters"],"lastActive":"2026-07-10"},{"name":"sqrt-of-2","url":"https://github.com/sqrt-of-2","submissions":1,"problems":["submitters"],"lastActive":"2026-05-12"},{"name":"test1-deepthought","url":"https://github.com/test1-deepthought","submissions":1,"problems":["submitters"],"lastActive":"2026-06-23"},{"name":"JohnEdwardJennings","url":"https://github.com/JohnEdwardJennings","submissions":1,"problems":["submitters"],"lastActive":"2026-06-14"},{"name":"kim-em","url":"https://github.com/kim-em","submissions":1,"problems":["submitters"],"lastActive":"2026-09-13"},{"name":"juihuichung","url":"https://github.com/juihuichung","submissions":1,"problems":["submitters"],"lastActive":"2026-08-31"},{"name":"xiaoyangliu-sjtu","url":"https://github.com/xiaoyangliu-sjtu","submissions":1,"problems":["submitters"],"lastActive":"2026-09-15"},{"name":"A-M-Berns","url":"https://github.com/A-M-Berns","submissions":1,"problems":["submitters"],"lastActive":"2026-05-16"},{"name":"antpavzhi","url":"https://github.com/antpavzhi","submissions":1,"problems":["submitters"],"lastActive":"2026-05-11"},{"name":"adrianmartir","url":"https://github.com/adrianmartir","submissions":1,"problems":["submitters"],"lastActive":"2026-06-29"},{"name":"eohjelle","url":"https://github.com/eohjelle","submissions":1,"problems":["submitters"],"lastActive":"2026-07-21"},{"name":"ivwumupy","url":"https://github.com/ivwumupy","submissions":1,"problems":["submitters"],"lastActive":"2026-07-27"},{"name":"mccorvie","url":"https://github.com/mccorvie","submissions":1,"problems":["submitters"],"lastActive":"2026-08-03"},{"name":"rwst","url":"https://github.com/rwst","submissions":1,"problems":["submitters"],"lastActive":"2026-07-23"},{"name":"jzuiddam","url":"https://github.com/jzuiddam","submissions":1,"problems":["submitters"],"lastActive":"2026-05-07"},{"name":"KitaKen1","url":"https://github.com/KitaKen1","submissions":1,"problems":["submitters"],"lastActive":"2026-08-06"},{"name":"machinelearning2014","url":"https://github.com/machinelearning2014","submissions":1,"problems":["submitters"],"lastActive":"2026-05-30"},{"name":"n-yamaguchi-0729","url":"https://github.com/n-yamaguchi-0729","submissions":1,"problems":["submitters"],"lastActive":"2026-09-08"},{"name":"nolanlwin","url":"https://github.com/nolanlwin","submissions":1,"problems":["submitters"],"lastActive":"2026-09-07"},{"name":"Solarys431","url":"https://github.com/Solarys431","submissions":1,"problems":["submitters"],"lastActive":"2026-07-21"},{"name":"yawara","url":"https://github.com/yawara","submissions":1,"problems":["submitters"],"lastActive":"2026-07-16"},{"name":"adamtopaz","url":"https://github.com/adamtopaz","submissions":1,"problems":["submitters"],"lastActive":"2026-07-25"},{"name":"albertqjiang","url":"https://github.com/albertqjiang","submissions":1,"problems":["submitters"],"lastActive":"2026-06-06"},{"name":"Cond-t","url":"https://github.com/Cond-t","submissions":1,"problems":["submitters"],"lastActive":"2026-08-31"},{"name":"homeowmorphism","url":"https://github.com/homeowmorphism","submissions":1,"problems":["submitters"],"lastActive":"2026-08-18"},{"name":"ianklatzco","url":"https://github.com/ianklatzco","submissions":1,"problems":["submitters"],"lastActive":"2026-07-14"},{"name":"kimihiro64","url":"https://github.com/kimihiro64","submissions":1,"problems":["submitters"],"lastActive":"2026-09-13"},{"name":"ldct","url":"https://github.com/ldct","submissions":1,"problems":["submitters"],"lastActive":"2026-07-05"},{"name":"LexBrill","url":"https://github.com/LexBrill","submissions":1,"problems":["submitters"],"lastActive":"2026-09-02"},{"name":"LikC1606","url":"https://github.com/LikC1606","submissions":1,"problems":["submitters"],"lastActive":"2026-08-22"},{"name":"lixiang90","url":"https://github.com/lixiang90","submissions":1,"problems":["submitters"],"lastActive":"2026-09-07"},{"name":"matthewjasper","url":"https://github.com/matthewjasper","submissions":1,"problems":["submitters"],"lastActive":"2026-07-14"},{"name":"mrdouglasny","url":"https://github.com/mrdouglasny","submissions":1,"problems":["submitters"],"lastActive":"2026-06-15"},{"name":"niketp03","url":"https://github.com/niketp03","submissions":1,"problems":["submitters"],"lastActive":"2026-06-02"},{"name":"Parcly-Taxel","url":"https://github.com/Parcly-Taxel","submissions":1,"problems":["submitters"],"lastActive":"2026-05-23"},{"name":"plby","url":"https://github.com/plby","submissions":1,"problems":["submitters"],"lastActive":"2026-06-26"},{"name":"rigwild","url":"https://github.com/rigwild","submissions":1,"problems":["submitters"],"lastActive":"2026-08-04"},{"name":"will1491","url":"https://github.com/will1491","submissions":1,"problems":["submitters"],"lastActive":"2026-07-28"},{"name":"wpegden","url":"https://github.com/wpegden","submissions":1,"problems":["submitters"],"lastActive":"2026-07-20"}],"stats":{"totalRecords":123,"uniqueParticipants":54,"firstSubmission":"2026-04-30","lastSubmission":"2026-09-17","recordsLast30d":39,"recordsLast90d":95},"lastUpdated":"2026-09-18T16:32:55.538Z"},{"id":"agent-anvil-leaderboard","name":"Agent Anvil Public Leaderboard","tagline":"Compare final-answer-only checks with trace-aware assertions across agent tool-use safety scenarios.","description":"Agent Anvil is an evaluation framework for comparing agent trace-aware assertions against final-answer-only checks. Authors run evaluations in their own CI, export a compact JSON submission, and open a pull request to join the leaderboard. It helps developers measure and verify agent safety and correctness.","url":"https://github.com/agent-axiom/agent-anvil-leaderboard","host":"github","status":"active","tags":["agent-evaluation","llm-agents","trace-evaluation","safety","assertions"],"domain":"llm-eval","images":[],"organizer":{"name":"Agent Axiom","url":"https://github.com/agent-axiom","type":"company"},"participation":{"howToSubmit":"Open a pull request adding your exported `leaderboard_submission.json` under `submissions/`.","submissionFormat":"JSON file generated via `anvil leaderboard export`","requirements":["Python","Agent Anvil","GitHub Actions"],"compute":"none","computeDetails":"The leaderboard does not execute arbitrary user agents; you run the benchmark in your own repository or CI and submit the attested JSON.","deadline":"rolling","verification":"CI validates the submitted JSON schema, checks GitHub API for run provenance, and verifies artifact attestations."},"quickstart":"uv run anvil paper reproduce\nuv run anvil leaderboard export docs/paper/results.json \\\n  --manifest experiments/paper.yaml \\\n  --out leaderboard_submission.json \\\n  --agent-name \"My Agent\"\nuv run anvil leaderboard validate leaderboard_submission.json","links":[{"label":"Hugging Face Space","url":"https://huggingface.co/spaces/ifif/agent-anvil-leaderboard"},{"label":"Hugging Face Dataset","url":"https://huggingface.co/datasets/ifif/agent-anvil-leaderboard-data"},{"label":"Demo Agent Repository","url":"https://github.com/agent-axiom/agent-anvil-demo-agent"}],"problems":[{"id":"agent_anvil_trace_eval_benchmark","name":"Agent Anvil Trace Eval Benchmark","description":"A compact offline benchmark for comparing final-answer-only checks with trace-aware Agent Anvil assertions across tool-use safety scenarios.","metricName":"trace_aware_pass_rate","metricUnit":"%","metricDirection":"maximize","records":[{"date":"2026-05-29","value":30,"contributor":"Agent Anvil Demo Agent","contributorKind":"method","contributorUrl":"https://github.com/agent-axiom/agent-anvil-demo-agent","description":"Reference demo agent submission","submissionUrls":["https://github.com/agent-axiom/agent-anvil-leaderboard/blob/main/submissions/agent-anvil-demo-agent.json"],"isCurrentBest":true,"isBaseline":true}],"isOpen":true,"contested":false}],"repo":{"owner":"agent-axiom","name":"agent-anvil-leaderboard","branch":"main"},"participants":[],"stats":{"totalRecords":1,"uniqueParticipants":0,"firstSubmission":"2026-05-29","lastSubmission":"2026-05-29","recordsLast30d":0,"recordsLast90d":0},"agentStats":[],"lastUpdated":"2026-09-18T16:32:55.538Z"},{"id":"modded-nanogpt-npu","name":"Modded-NanoGPT Ascend NPU Leaderboard","tagline":"Speedrun training NanoGPT to 3.28 validation loss on 16x Ascend 910C NPUs.","description":"A benchmark tracking NPU reproductions of the modded-nanogpt speedrun leaderboard on 16×Ascend 910C NPUs. It ports, reviews, and benchmarks upstream GPU records to maintain a comparable NPU leaderboard with auditable sources, logs, and metrics.","url":"https://github.com/zhaoyiyidan/modded-nanogpt-npu","host":"github","status":"active","tags":["nanogpt","npu","ascend","speedrun","llm-training","muon","pytorch"],"domain":"systems-perf","images":[],"organizer":{"name":"zhaoyiyidan","url":"https://github.com/zhaoyiyidan","type":"individual"},"participation":{"howToSubmit":"Port upstream GPU records to 16x Ascend 910C NPUs and submit via PR with successful run logs under records/track_1_short_npu/","submissionFormat":"NPU source code, successful run logs, and metrics","requirements":["Python 3.10+","PyTorch","torch_npu","Ascend 910C NPU"],"compute":"cluster","computeDetails":"16×Ascend 910C NPU","verification":"Reviewed by maintainers based on successful .train.log files"},"quickstart":"git clone https://github.com/zhaoyiyidan/modded-nanogpt-npu.git && cd modded-nanogpt-npu\ntorchrun --standalone --nproc_per_node=16 train_gpt.py","links":[{"label":"Upstream Repository","url":"https://github.com/KellerJordan/modded-nanogpt"},{"label":"Muon Optimizer Writeup","url":"https://kellerjordan.github.io/posts/muon/"}],"problems":[{"id":"track-1-short-npu","name":"Track 1 Short (16x Ascend 910C NPU)","description":"Train a language model to <= 3.28 validation loss on FineWeb using 16x Ascend 910C NPUs.","metricName":"time","metricUnit":"minutes","metricDirection":"minimize","baseline":104.72,"records":[{"value":104.72,"contributor":"llm.c baseline","contributorKind":"method","description":"llm.c baseline; baseline/reference port, no specific speed-up trick.","isBaseline":true},{"value":61.6,"contributor":"Tuned learning rate and rotary embeddings","contributorKind":"method"},{"value":42.18,"contributor":"Pad embeddings, ReLU2, zero-init projections, QK-norm","contributorKind":"method"},{"value":40.43,"contributor":"Muon optimizer improvements","contributorKind":"method"},{"value":40.33,"contributor":"Distributed Muon overhead","contributorKind":"method"},{"value":40.12,"contributor":"PyTorch 2.5 runtime upgrade","contributorKind":"method"},{"value":36.6,"contributor":"Untied token embedding and LM head","contributorKind":"method"},{"value":33.96,"contributor":"Cast activations to bfloat16","contributorKind":"method"},{"value":31.83,"contributor":"Value/embedding skip connections, momentum warmup, logit softcap","contributorKind":"method"},{"value":29.56,"contributor":"Reduced batch size","contributorKind":"method"},{"value":19.72,"contributor":"Sparsified value embeddings, improved rotary embeddings, dropped one attention layer","contributorKind":"method"},{"value":12.58,"contributor":"Transposed one MLP matrix and added a symmetric matmul kernel","contributorKind":"method"},{"value":11.65,"contributor":"Dropped the first MLP layer","contributorKind":"method"},{"value":11.06,"contributor":"Optimized distributed training, improved skip-connection gating, enhanced bfloat16 usage","contributorKind":"method"},{"value":11.05,"contributor":"Smeared token embeddings one position forward","contributorKind":"method"},{"value":10.92,"contributor":"Asynchronous data fetch/indexing and final-layer validation attention window extension","contributorKind":"method"},{"value":10.77,"contributor":"Dropped first attention layer, extended long validation windows, updated schedule","contributorKind":"method"},{"value":10.76,"contributor":"Polar Express replacement for Newton-Schulz","contributorKind":"method"},{"value":10.53,"contributor":"Updated Adam params every other step and reduced batch size","contributorKind":"method"},{"value":10.16,"contributor":"Backout changes, hyperparameter tuning, lambda padding optimization","contributorKind":"method"},{"value":9.98,"contributor":"Updated NorMuon learning rate and step logic","contributorKind":"method"},{"value":9.69,"contributor":"NorMuon optimizer","contributorKind":"method"},{"value":9.65,"contributor":"Refined skip architecture, updated exponential decay init","contributorKind":"method"},{"value":9.57,"contributor":"Batch size schedule","contributorKind":"method"},{"value":9.56,"contributor":"Sped up Muon, pre-multiplied lambda, reshaped matrices, updated lr and NorMuon axis","contributorKind":"method"},{"value":9.48,"contributor":"Partial Key Offset","contributorKind":"method"},{"value":9.38,"contributor":"Extended Cautious Weight Decay to Adam parameters","contributorKind":"method"},{"value":9.34,"contributor":"Computed cross entropy in BF16 during training","contributorKind":"method"},{"value":9.04,"contributor":"Multiplied attention lambda with weight instead of data, fixed warmup","contributorKind":"method"},{"value":5.78,"contributor":"NPU-optimized Cautious Weight Decay on Adam","contributorKind":"method","description":"fused RMSNorm/rotary, buffer reuse, bfloat16 weights, a 2070-step schedule, and reduced validation/metrics overhead.","isCurrentBest":true}],"isOpen":false,"contested":true},{"id":"track-1-short-gpu","name":"Track 1 Short (8x NVIDIA H100 GPU)","description":"Train a language model to <= 3.28 validation loss on FineWeb using 8x NVIDIA H100 GPUs.","metricName":"time","metricUnit":"minutes","metricDirection":"minimize","baseline":45,"records":[{"date":"2024-05-28","value":45,"contributor":"karpathy","contributorKind":"person","description":"llm.c baseline","isBaseline":true,"contributorUrl":"https://github.com/karpathy"},{"date":"2024-06-06","value":31.4,"contributor":"kellerjordan0","contributorKind":"person","description":"Tuned learning rate & rotary embeddings","contributorUrl":"https://github.com/kellerjordan0"},{"date":"2024-10-04","value":24.9,"contributor":"kellerjordan0","contributorKind":"person","description":"Introduced the Muon optimizer","contributorUrl":"https://github.com/kellerjordan0"},{"date":"2024-10-11","value":22.3,"contributor":"kellerjordan0","contributorKind":"person","description":"Muon improvements","contributorUrl":"https://github.com/kellerjordan0"},{"date":"2024-10-14","value":15.2,"contributor":"Grad62304977","contributorKind":"person","description":"Pad embeddings, ReLU², zero-init projections, QK-norm","contributorUrl":"https://github.com/Grad62304977"},{"date":"2024-10-18","value":13.1,"contributor":"kellerjordan0","contributorKind":"person","description":"Distributed the overhead of Muon","contributorUrl":"https://github.com/kellerjordan0"},{"date":"2024-10-18","value":12,"contributor":"kellerjordan0","contributorKind":"person","description":"Upgraded PyTorch 2.5.0","contributorUrl":"https://github.com/kellerjordan0"},{"date":"2024-11-03","value":10.8,"contributor":"Grad62304977","contributorKind":"person","description":"Untied embedding and head","contributorUrl":"https://github.com/Grad62304977"},{"date":"2024-11-06","value":8.2,"contributor":"Grad62304977","contributorKind":"person","description":"Value and embedding skip connections, momentum warmup, logit softcap","contributorUrl":"https://github.com/Grad62304977"},{"date":"2024-11-08","value":7.8,"contributor":"kellerjordan0","contributorKind":"person","description":"Bfloat16 activations","contributorUrl":"https://github.com/kellerjordan0"},{"date":"2024-11-10","value":7.2,"contributor":"brendanh0gan","contributorKind":"person","description":"U-net pattern skip connections & double lr","contributorUrl":"https://github.com/brendanh0gan"},{"date":"2024-11-19","value":5.03,"contributor":"KoszarskyB","contributorKind":"person","description":"1024-ctx dense causal attention → 64K-ctx FlexAttention","contributorUrl":"https://github.com/KoszarskyB"},{"date":"2024-11-24","value":4.66,"contributor":"fernbear.bsky.social","contributorKind":"person","description":"Attention window warmup"},{"date":"2024-12-04","value":4.41,"contributor":"KoszarskyB","contributorKind":"person","description":"Value Embeddings","contributorUrl":"https://github.com/KoszarskyB"},{"date":"2024-12-08","value":3.95,"contributor":"leloykun","contributorKind":"person","description":"U-net pattern value embeddings, assorted code optimizations","contributorUrl":"https://github.com/leloykun"},{"date":"2024-12-10","value":3.8,"contributor":"YouJiacheng","contributorKind":"person","description":"Split value embeddings, block sliding window, separate block mask","contributorUrl":"https://github.com/YouJiacheng"},{"date":"2024-12-17","value":3.57,"contributor":"YouJiacheng","contributorKind":"person","description":"Sparsify value embeddings, improve rotary embeddings, drop an attn layer","contributorUrl":"https://github.com/YouJiacheng"},{"date":"2025-01-04","value":3.4,"contributor":"KoszarskyB","contributorKind":"person","description":"Lower logit softcap from 30 to 15","contributorUrl":"https://github.com/KoszarskyB"},{"date":"2025-01-13","value":3.142,"contributor":"YouJiacheng","contributorKind":"person","description":"FP8 head, offset logits, lr decay to 0.1 instead of 0.0","contributorUrl":"https://github.com/YouJiacheng"},{"date":"2025-01-16","value":2.992,"contributor":"leloykun","contributorKind":"person","description":"Merged QKV weights, long-short attention, attention scale, lower Adam epsilon, batched Muon","contributorUrl":"https://github.com/leloykun"},{"date":"2025-01-26","value":2.933,"contributor":"leloykun","contributorKind":"person","description":"Reduced batch size","contributorUrl":"https://github.com/leloykun"},{"date":"2025-02-01","value":2.997,"contributor":"leloykun","contributorKind":"person","description":"21st record with new timing","contributorUrl":"https://github.com/leloykun"},{"date":"2025-05-24","value":3.014,"contributor":"leloykun","contributorKind":"person","description":"21st record with latest torch","contributorUrl":"https://github.com/leloykun"},{"date":"2025-05-24","value":2.99,"contributor":"KonstantinWilleke","contributorKind":"person","description":"Faster gradient all-reduce","contributorUrl":"https://github.com/KonstantinWilleke"},{"date":"2025-05-25","value":2.979,"contributor":"ryanyang0","contributorKind":"person","description":"Overlap computation and gradient communication","contributorUrl":"https://github.com/ryanyang0"},{"date":"2025-05-30","value":2.966,"contributor":"vagrawal","contributorKind":"person","description":"Replace gradient all_reduce with reduce_scatter","contributorUrl":"https://github.com/vagrawal"},{"date":"2025-07-13","value":2.896,"contributor":"kellerjordan0","contributorKind":"person","description":"Upgrade PyTorch to 2.9.0.dev20250713+cu126","contributorUrl":"https://github.com/kellerjordan0"},{"date":"2025-07-13","value":2.863,"contributor":"classiclarryd","contributorKind":"person","description":"Align training batch starts with EoS, increase cooldown fraction to 0.45","contributorUrl":"https://github.com/classiclarryd"},{"date":"2025-07-18","value":2.817,"contributor":"byronxu99","contributorKind":"person","description":"Transpose one of the MLP matrices + add Triton kernel for symmetric matmul","contributorUrl":"https://github.com/byronxu99"},{"date":"2025-08-23","value":2.812,"contributor":"classiclarryd","contributorKind":"person","description":"Sparse attention gate","contributorUrl":"https://github.com/classiclarryd"},{"date":"2025-09-03","value":2.731,"contributor":"varunneal","contributorKind":"person","description":"Flash Attention 3, 2048 max_doc_len, update ws schedule","contributorUrl":"https://github.com/varunneal"},{"date":"2025-09-05","value":2.717,"contributor":"EmelyanenkoK","contributorKind":"person","description":"Drop first MLP layer","contributorUrl":"https://github.com/EmelyanenkoK"},{"date":"2025-09-10","value":2.656,"contributor":"classiclarryd","contributorKind":"person","description":"Dynamically incorporate YaRN during training and validation","contributorUrl":"https://github.com/classiclarryd"},{"date":"2025-09-11","value":2.625,"contributor":"bernard24","contributorKind":"person","description":"Optimize distributed training, improve skip connection gating, and enhance bfloat16 usage","contributorUrl":"https://github.com/bernard24"},{"date":"2025-09-15","value":2.565,"contributor":"classiclarryd","contributorKind":"person","description":"Asynchronously fetch and index data batches, extend final layer attention window for validation","contributorUrl":"https://github.com/classiclarryd"},{"date":"2025-09-18","value":2.547,"contributor":"classiclarryd","contributorKind":"person","description":"Smear token embeddings 1 position forward","contributorUrl":"https://github.com/classiclarryd"},{"date":"2025-09-21","value":2.527,"contributor":"classiclarryd","contributorKind":"person","description":"Drop first attn layer, extend all long windows for validation, update schedule","contributorUrl":"https://github.com/classiclarryd"},{"date":"2025-09-23","value":2.495,"contributor":"classiclarryd","contributorKind":"person","description":"MuonCustomSizing, perform mlp and attn reduce scatter in shared call","contributorUrl":"https://github.com/classiclarryd"},{"date":"2025-09-27","value":2.483,"contributor":"Gusarich","contributorKind":"person","description":"Compute cross entropy in BF16 during training","contributorUrl":"https://github.com/Gusarich"},{"date":"2025-09-29","value":2.476,"contributor":"varunneal","contributorKind":"person","description":"Polar Express, replacement for Newton-Schulz","contributorUrl":"https://github.com/varunneal"},{"date":"2025-09-30","value":2.447,"contributor":"classiclarryd","contributorKind":"person","description":"Only update Adam params every other step, reduce batch size","contributorUrl":"https://github.com/classiclarryd"},{"date":"2025-10-04","value":2.358,"contributor":"classiclarryd","contributorKind":"person","description":"Backout, misc hyperparameter tuning, optimize lambda padding","contributorUrl":"https://github.com/classiclarryd"},{"date":"2025-10-24","value":2.345,"contributor":"li_zichong","contributorKind":"person","description":"NorMuon","contributorUrl":"https://github.com/li_zichong"},{"date":"2025-10-27","value":2.313,"contributor":"varunneal","contributorKind":"person","description":"Update NorMuon LR, Step Logic","contributorUrl":"https://github.com/varunneal"},{"date":"2025-11-10","value":2.284,"contributor":"varunneal","contributorKind":"person","description":"Cautious Weight Decay w/ schedule","contributorUrl":"https://github.com/varunneal"},{"date":"2025-11-16","value":2.269,"contributor":"akash5474","contributorKind":"person","description":"Backward hooks on Adam, Profiling 101","contributorUrl":"https://github.com/akash5474"},{"date":"2025-11-18","value":2.248,"contributor":"classiclarryd","contributorKind":"person","description":"Refine skip arch, update exponential decay init","contributorUrl":"https://github.com/classiclarryd"},{"date":"2025-11-29","value":2.203,"contributor":"varunneal","contributorKind":"person","description":"Batch size schedule","contributorUrl":"https://github.com/varunneal"},{"date":"2025-12-10","value":2.193,"contributor":"roeeshenberg","contributorKind":"person","description":"Multiply attn lambda with weight instead of data, fix warmup","contributorUrl":"https://github.com/roeeshenberg"},{"date":"2025-12-11","value":2.17,"contributor":"ChrisJMcCormick","contributorKind":"person","description":"Speed up Muon, additional pre-multiply lambda, reshape matrices, update lr, update NorMuon axis","contributorUrl":"https://github.com/ChrisJMcCormick"},{"date":"2025-12-14","value":2.146,"contributor":"classiclarryd","contributorKind":"person","description":"Partial Key Offset","isCurrentBest":true,"contributorUrl":"https://github.com/classiclarryd"}],"isOpen":false,"contested":true},{"id":"track-2-medium-gpu","name":"Track 2 Medium (8x NVIDIA H100 GPU)","description":"Train a language model to <= 2.92 validation loss on FineWeb using 8x NVIDIA H100 GPUs.","metricName":"time","metricUnit":"minutes","metricDirection":"minimize","baseline":348,"records":[{"date":"2024-05-28","value":348,"contributor":"karpathy","contributorKind":"person","description":"llm.c baseline (350M parameters)","isBaseline":true,"contributorUrl":"https://github.com/karpathy"},{"date":"2025-01-18","value":29.3,"contributor":"kellerjordan0","contributorKind":"person","description":"Initial record based on scaling up the GPT-2 small track speedrun","contributorUrl":"https://github.com/kellerjordan0"},{"date":"2025-02-08","value":28.1,"contributor":"kellerjordan0","contributorKind":"person","description":"Added standard weight decay","contributorUrl":"https://github.com/kellerjordan0"},{"date":"2025-02-14","value":27.7,"contributor":"leloykun","contributorKind":"person","description":"Tuned Muon Newton-Schulz coefficients","contributorUrl":"https://github.com/leloykun"},{"date":"2025-03-06","value":27.2,"contributor":"YouJiacheng","contributorKind":"person","description":"Increased learning rate cooldown phase duration","contributorUrl":"https://github.com/YouJiacheng"},{"date":"2025-03-25","value":25.95,"contributor":"YouJiacheng","contributorKind":"person","description":"2x MLP wd, qkv norm, all_reduce/opt.step() overlap, optimized skip pattern","contributorUrl":"https://github.com/YouJiacheng"},{"date":"2025-04-16","value":25.29,"contributor":"YouJiacheng","contributorKind":"person","description":"Remove FP8 head; ISRU logits softcap; New sharded mixed precision Muon; merge weights","contributorUrl":"https://github.com/YouJiacheng"},{"date":"2025-04-22","value":24.5,"contributor":"jadenj3o","contributorKind":"person","description":"Cubic sliding window size schedule, 2× max window size","contributorUrl":"https://github.com/jadenj3o"},{"date":"2025-08-28","value":24.12,"contributor":"snimu","contributorKind":"person","description":"Add two value embeddings","contributorUrl":"https://github.com/snimu"},{"date":"2025-09-11","value":24.07,"contributor":"snimu","contributorKind":"person","description":"Second input embedding","isCurrentBest":true,"contributorUrl":"https://github.com/snimu"}],"isOpen":false,"contested":true}],"repo":{"owner":"zhaoyiyidan","name":"modded-nanogpt-npu","branch":"master"},"participants":[{"name":"classiclarryd","url":"https://github.com/classiclarryd","submissions":11,"problems":["track-1-short-gpu"],"lastActive":"2025-12-14","bestRank":1},{"name":"snimu","url":"https://github.com/snimu","submissions":2,"problems":["track-2-medium-gpu"],"lastActive":"2025-09-11","bestRank":1},{"name":"kellerjordan0","url":"https://github.com/kellerjordan0","submissions":9,"problems":["track-1-short-gpu","track-2-medium-gpu"],"lastActive":"2025-07-13"},{"name":"leloykun","url":"https://github.com/leloykun","submissions":6,"problems":["track-1-short-gpu","track-2-medium-gpu"],"lastActive":"2025-05-24"},{"name":"YouJiacheng","url":"https://github.com/YouJiacheng","submissions":6,"problems":["track-1-short-gpu","track-2-medium-gpu"],"lastActive":"2025-04-16"},{"name":"varunneal","url":"https://github.com/varunneal","submissions":5,"problems":["track-1-short-gpu"],"lastActive":"2025-11-29"},{"name":"Grad62304977","url":"https://github.com/Grad62304977","submissions":3,"problems":["track-1-short-gpu"],"lastActive":"2024-11-06"},{"name":"KoszarskyB","url":"https://github.com/KoszarskyB","submissions":3,"problems":["track-1-short-gpu"],"lastActive":"2025-01-04"},{"name":"karpathy","url":"https://github.com/karpathy","submissions":2,"problems":["track-1-short-gpu","track-2-medium-gpu"],"lastActive":"2024-05-28"},{"name":"brendanh0gan","url":"https://github.com/brendanh0gan","submissions":1,"problems":["track-1-short-gpu"],"lastActive":"2024-11-10"},{"name":"fernbear.bsky.social","submissions":1,"problems":["track-1-short-gpu"],"lastActive":"2024-11-24"},{"name":"KonstantinWilleke","url":"https://github.com/KonstantinWilleke","submissions":1,"problems":["track-1-short-gpu"],"lastActive":"2025-05-24"},{"name":"ryanyang0","url":"https://github.com/ryanyang0","submissions":1,"problems":["track-1-short-gpu"],"lastActive":"2025-05-25"},{"name":"vagrawal","url":"https://github.com/vagrawal","submissions":1,"problems":["track-1-short-gpu"],"lastActive":"2025-05-30"},{"name":"byronxu99","url":"https://github.com/byronxu99","submissions":1,"problems":["track-1-short-gpu"],"lastActive":"2025-07-18"},{"name":"EmelyanenkoK","url":"https://github.com/EmelyanenkoK","submissions":1,"problems":["track-1-short-gpu"],"lastActive":"2025-09-05"},{"name":"bernard24","url":"https://github.com/bernard24","submissions":1,"problems":["track-1-short-gpu"],"lastActive":"2025-09-11"},{"name":"Gusarich","url":"https://github.com/Gusarich","submissions":1,"problems":["track-1-short-gpu"],"lastActive":"2025-09-27"},{"name":"li_zichong","url":"https://github.com/li_zichong","submissions":1,"problems":["track-1-short-gpu"],"lastActive":"2025-10-24"},{"name":"akash5474","url":"https://github.com/akash5474","submissions":1,"problems":["track-1-short-gpu"],"lastActive":"2025-11-16"},{"name":"roeeshenberg","url":"https://github.com/roeeshenberg","submissions":1,"problems":["track-1-short-gpu"],"lastActive":"2025-12-10"},{"name":"ChrisJMcCormick","url":"https://github.com/ChrisJMcCormick","submissions":1,"problems":["track-1-short-gpu"],"lastActive":"2025-12-11"},{"name":"jadenj3o","url":"https://github.com/jadenj3o","submissions":1,"problems":["track-2-medium-gpu"],"lastActive":"2025-04-22"}],"stats":{"totalRecords":91,"uniqueParticipants":23,"firstSubmission":"2024-05-28","lastSubmission":"2025-12-14","recordsLast30d":0,"recordsLast90d":0},"agentStats":[],"lastUpdated":"2026-09-18T16:32:55.538Z"},{"id":"swe-bench-pro-leaderboard","name":"SWE-bench Pro Leaderboard","tagline":"Evaluate coding agents on 731 real-world software engineering tasks from SWE-bench Pro.","description":"SWE-bench Pro Leaderboard evaluates coding agents on SWE-bench Pro, which consists of 731 real-world software engineering tasks. Submissions are evaluated by running the project's test suite in Docker containers to check if bugs are fixed without regressions. It is run by the RDI Foundation to track state-of-the-art coding agents.","url":"https://github.com/RDI-Foundation/swe-bench-leaderboard","host":"github","status":"active","tags":["swe-bench","coding-agents","software-engineering","docker"],"domain":"coding-agents","images":[],"organizer":{"name":"RDI Foundation","url":"https://github.com/RDI-Foundation","type":"foundation"},"participation":{"howToSubmit":"Fork the repository, edit scenario.json5 with your Agentbeats ID, add API keys as secrets, and open a PR to main, or submit via agentbeats.dev.","submissionFormat":"scenario.json5 configuration and Agentbeats ID","requirements":["Docker","Python","Amber CLI"],"compute":"unknown","cost":"free","verification":"CI runs the evaluation stack using Docker to execute the project's test suite"},"quickstart":"amber compile scenario.json5 --docker-compose amber-out\ncp amber-out/env.example amber-out/.env\nAMBER_CONFIG_CODING_AGENT__MODEL_NAME=gpt-4o \\\nAMBER_CONFIG_CODING_AGENT__OPENAI_API_KEY=sk-xxx \\\ndocker compose -f amber-out/compose.yaml --env-file amber-out/.env up","links":[{"label":"SWE-bench Pro Dataset","url":"https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro"},{"label":"Amber CLI","url":"https://github.com/rdi-foundation/amber-cli"},{"label":"Agentbeats","url":"https://agentbeats.dev"},{"label":"Reference Coding Agent","url":"https://github.com/rdi-foundation/swe-bench-purple-agent"},{"label":"Evaluation Orchestrator","url":"https://github.com/rdi-foundation/swe-bench-green-agent"}],"problems":[{"id":"swe-bench-pro","name":"SWE-bench Pro","description":"Evaluating coding agents on 731 real-world software engineering tasks from SWE-bench Pro.","metricName":"accuracy","metricUnit":"%","metricDirection":"maximize","records":[{"date":"2026-04-16","value":14.29,"contributor":"SWE-bench Purple Agent (deepseek-chat)","contributorKind":"method","description":"A2A coding agent that solves SWE-bench Pro instances using mini-swe-agent (deepseek/deepseek-chat).","submissionUrls":["https://github.com/RDI-Foundation/swe-bench-leaderboard/blob/main/results/019d9602-bdf5-7b71-8185-1ba4e00fa994.json"],"isCurrentBest":true,"isBaseline":true,"agent":{"family":"deepseek","model":"deepseek-chat","tool":"mini-swe-agent","evidence":"A2A coding agent that solves SWE-bench Pro instances using mini-swe-agent (deepseek/deepseek-chat).","confidence":"high","role":"author"}},{"date":"2026-04-13","value":10,"contributor":"SWE-bench Purple Agent (deepseek-v3.2)","contributorKind":"method","description":"A2A coding agent that solves SWE-bench Pro instances using mini-swe-agent (openrouter/deepseek/deepseek-v3.2).","submissionUrls":["https://github.com/RDI-Foundation/swe-bench-leaderboard/blob/main/results/019d86f7-8639-7853-bbd1-b045cc625e16.json"],"isBaseline":true,"agent":{"family":"deepseek","model":"deepseek-v3.2","tool":"mini-swe-agent","evidence":"A2A coding agent that solves SWE-bench Pro instances using mini-swe-agent (openrouter/deepseek/deepseek-v3.2).","confidence":"high","role":"author"}},{"date":"2026-04-15","value":10,"contributor":"SWE-bench Purple Agent (deepseek-chat)","contributorKind":"method","description":"A2A coding agent that solves SWE-bench Pro instances using mini-swe-agent (deepseek/deepseek-chat).","submissionUrls":["https://github.com/RDI-Foundation/swe-bench-leaderboard/blob/main/results/019d9025-c7b0-7240-b0e0-069d053c12c7.json"],"isBaseline":true,"agent":{"family":"deepseek","model":"deepseek-chat","tool":"mini-swe-agent","evidence":"A2A coding agent that solves SWE-bench Pro instances using mini-swe-agent (deepseek/deepseek-chat).","confidence":"high","role":"author"}},{"date":"2026-04-16","value":3.33,"contributor":"SWE-bench Purple Agent (deepseek-chat)","contributorKind":"method","description":"A2A coding agent that solves SWE-bench Pro instances using mini-swe-agent (deepseek/deepseek-chat).","submissionUrls":["https://github.com/RDI-Foundation/swe-bench-leaderboard/blob/main/results/019d9480-9365-7273-b12a-bd76d7d604b4.json"],"isBaseline":true,"agent":{"family":"deepseek","model":"deepseek-chat","tool":"mini-swe-agent","evidence":"A2A coding agent that solves SWE-bench Pro instances using mini-swe-agent (deepseek/deepseek-chat).","confidence":"high","role":"author"}},{"date":"2026-04-13","value":0,"contributor":"SWE-bench Purple Agent (deepseek-v3.2)","contributorKind":"method","description":"A2A coding agent that solves SWE-bench Pro instances using mini-swe-agent (openrouter/deepseek/deepseek-v3.2).","submissionUrls":["https://github.com/RDI-Foundation/swe-bench-leaderboard/blob/main/results/019d8691-0a8c-7a32-a5e1-1b72cf211f3a.json"],"isBaseline":true,"agent":{"family":"deepseek","model":"deepseek-v3.2","tool":"mini-swe-agent","evidence":"A2A coding agent that solves SWE-bench Pro instances using mini-swe-agent (openrouter/deepseek/deepseek-v3.2).","confidence":"high","role":"author"}},{"date":"2026-04-18","value":0,"contributor":"AgentX-Spring0123","contributorKind":"team","description":"Purple agent using official A2A Python server template.","submissionUrls":["https://github.com/RDI-Foundation/swe-bench-leaderboard/blob/main/results/019da114-0d54-7310-b6be-58c57d201617.json"],"agent":{"family":"other-ai","evidence":"Purple agent using official A2A Python server template.","confidence":"high","role":"author"}}],"isOpen":false,"contested":false}],"repo":{"owner":"RDI-Foundation","name":"swe-bench-leaderboard","branch":"main"},"participants":[{"name":"AgentX-Spring0123","submissions":1,"problems":["swe-bench-pro"],"lastActive":"2026-04-18"}],"stats":{"totalRecords":6,"uniqueParticipants":1,"firstSubmission":"2026-04-13","lastSubmission":"2026-04-18","recordsLast30d":0,"recordsLast90d":0},"agentStats":[{"family":"other-ai","records":1,"currentBests":0,"models":[],"contributors":["AgentX-Spring0123"]}],"lastUpdated":"2026-09-18T16:32:55.538Z"},{"id":"terminal-bench-leaderboard","name":"Terminal Bench 2.0 Leaderboard","tagline":"Evaluate AI agents on terminal-based coding and system tasks.","description":"Terminal Bench 2.0 runs a suite of terminal tasks against your agent and scores it by task pass rate. Submissions are automated via GitHub Actions and Docker Compose, allowing developers to test their agents on complex terminal environments.","url":"https://github.com/RDI-Foundation/terminal-bench-leaderboard","host":"github","status":"active","tags":["terminal-tasks","coding-agents","system-tasks","agent-evaluation","docker"],"domain":"coding-agents","images":[],"organizer":{"name":"RDI Foundation","url":"https://github.com/RDI-Foundation","type":"foundation"},"participation":{"howToSubmit":"Fork the repository, configure your agent in scenario.toml with your agentbeats_id and AGENT_LLM, add API keys as GitHub Secrets, and push to trigger the scenario runner.","submissionFormat":"Configuration in scenario.toml and GitHub Secrets.","requirements":["Docker","Docker Compose","GitHub Actions","LiteLLM"],"compute":"unknown","verification":"CI"},"links":[{"label":"Terminal Bench 2.0 Repository","url":"https://github.com/laude-ntt/terminal-bench"},{"label":"AgentBeats","url":"https://agentbeats.dev"}],"problems":[{"id":"terminal-bench-2.0","name":"Terminal Bench 2.0","description":"Overall task pass rate across all terminal-based coding and system tasks.","metricName":"pass_rate","metricUnit":"%","metricDirection":"maximize","records":[{"date":"2026-04-10","value":0,"contributor":"019d76c4-59a1-70f3-a147-643ce7091b40","contributorKind":"unknown","submissionUrls":["https://github.com/RDI-Foundation/terminal-bench-leaderboard/blob/main/results/019d76c7-6d7e-7bc3-aa66-0e1d80fc0067.json"],"isCurrentBest":true,"agent":{"family":"other-ai","model":"Green Agent","evidence":"Green Agent ID: 019d76c2-3bb7-7652-b932-76ad868837b0","confidence":"high","role":"author"}},{"date":"2026-04-10","value":0,"contributor":"019d76c4-59a1-70f3-a147-643ce7091b40","contributorKind":"unknown","submissionUrls":["https://github.com/RDI-Foundation/terminal-bench-leaderboard/blob/main/results/019d76d0-5e6e-7f71-ac4a-39996fef99c8.json"],"isCurrentBest":false,"agent":{"family":"other-ai","model":"Green Agent","evidence":"Green Agent ID: 019d76c2-3bb7-7652-b932-76ad868837b0","confidence":"high","role":"author"}},{"date":"2026-04-10","value":0,"contributor":"019d76c4-59a1-70f3-a147-643ce7091b40","contributorKind":"unknown","submissionUrls":["https://github.com/RDI-Foundation/terminal-bench-leaderboard/blob/main/results/019d76da-437f-7c33-a037-569015b1272f.json"],"isCurrentBest":false,"agent":{"family":"other-ai","model":"Green Agent","evidence":"Green Agent ID: 019d76c2-3bb7-7652-b932-76ad868837b0","confidence":"high","role":"author"}},{"date":"2026-04-10","value":0,"contributor":"019d76c4-59a1-70f3-a147-643ce7091b40","contributorKind":"unknown","submissionUrls":["https://github.com/RDI-Foundation/terminal-bench-leaderboard/blob/main/results/019d76ea-e6cd-7041-aef5-f7ee9d57ce58.json"],"isCurrentBest":false,"agent":{"family":"other-ai","model":"Green Agent","evidence":"Green Agent ID: 019d76c2-3bb7-7652-b932-76ad868837b0","confidence":"high","role":"author"}},{"date":"2026-04-29","value":0,"contributor":"019dd8de-40f9-74c3-ae33-91a250850feb","contributorKind":"unknown","submissionUrls":["https://github.com/RDI-Foundation/terminal-bench-leaderboard/blob/main/results/019dd928-6d99-7973-8d67-ff55b1f69e7f.json"],"isCurrentBest":false,"agent":{"family":"other-ai","model":"Green Agent","evidence":"Green Agent ID: 019d87ef-2c6d-7091-8d93-827799c2ade3","confidence":"high","role":"author"}},{"date":"2026-04-29","value":0,"contributor":"019dd8de-40f9-74c3-ae33-91a250850feb","contributorKind":"unknown","submissionUrls":["https://github.com/RDI-Foundation/terminal-bench-leaderboard/blob/main/results/019dd953-c6d9-7da0-8171-b4361369bc05.json"],"isCurrentBest":false,"agent":{"family":"other-ai","model":"Green Agent","evidence":"Green Agent ID: 019d87ef-2c6d-7091-8d93-827799c2ade3","confidence":"high","role":"author"}},{"date":"2026-04-29","value":0,"contributor":"019dd8de-40f9-74c3-ae33-91a250850feb","contributorKind":"unknown","submissionUrls":["https://github.com/RDI-Foundation/terminal-bench-leaderboard/blob/main/results/019dd98c-29c2-7621-98a6-bd6622066879.json"],"isCurrentBest":false,"agent":{"family":"other-ai","model":"Green Agent","evidence":"Green Agent ID: 019d87ef-2c6d-7091-8d93-827799c2ade3","confidence":"high","role":"author"}}],"isOpen":false,"contested":true}],"repo":{"owner":"RDI-Foundation","name":"terminal-bench-leaderboard","branch":"main"},"participants":[{"name":"019d76c4-59a1-70f3-a147-643ce7091b40","submissions":4,"problems":["terminal-bench-2.0"],"bestRank":1,"lastActive":"2026-04-10"},{"name":"019dd8de-40f9-74c3-ae33-91a250850feb","submissions":3,"problems":["terminal-bench-2.0"],"lastActive":"2026-04-29"}],"stats":{"totalRecords":7,"uniqueParticipants":2,"firstSubmission":"2026-04-10","lastSubmission":"2026-04-29","recordsLast30d":0,"recordsLast90d":0},"agentStats":[{"family":"other-ai","records":7,"currentBests":1,"models":["Green Agent"],"contributors":["019d76c4-59a1-70f3-a147-643ce7091b40","019dd8de-40f9-74c3-ae33-91a250850feb"]}],"lastUpdated":"2026-09-18T16:32:55.538Z"},{"id":"alpaca-eval","name":"AlpacaEval","tagline":"An automatic evaluator for instruction-following language models.","description":"AlpacaEval is a fast, cheap, and highly reliable automatic evaluation pipeline for chat LLMs. It measures model win rates against a reference baseline using LLM-based judges, achieving a 0.98 Spearman correlation with ChatBot Arena.","url":"https://github.com/tatsu-lab/alpaca_eval","host":"github","status":"active","tags":["llm-eval","nlp","instruction-following","benchmark","automated-evaluation"],"domain":"llm-eval","images":["https://raw.githubusercontent.com/tatsu-lab/alpaca_eval/main/figures/chat_correlations.png","https://raw.githubusercontent.com/tatsu-lab/alpaca_eval/main/figures/length_gameability.png","https://raw.githubusercontent.com/tatsu-lab/alpaca_eval/main/figures/different_baselines.png"],"organizer":{"name":"Tatsu Lab","url":"https://github.com/tatsu-lab","type":"academic"},"participation":{"howToSubmit":"Fork the repository, add your model config under src/alpaca_eval/models_configs/<model_name>, run evaluate_from_model to generate outputs and score, and open a PR with the config, outputs, and updated leaderboard.","submissionFormat":"Model configuration YAML, model outputs JSON, and updated leaderboard CSV.","requirements":["Python 3.10+","OpenAI API Key"],"compute":"none","computeDetails":"Submitters generate their own model outputs (locally or via API) and run the evaluation judge using their own OpenAI/Anthropic API keys.","deadline":"rolling","prizes":"recognition only","cost":"free","verification":"Community submissions are self-reported. Top-5 models can be verified by core maintainers who run the decoding and evaluation themselves."},"quickstart":"pip install alpaca-eval\nexport OPENAI_API_KEY=<your_api_key>\nalpaca_eval --model_outputs 'example/outputs.json'","links":[{"label":"Website","url":"https://tatsu-lab.github.io/alpaca_eval/"},{"label":"Paper (Length-Controlled)","url":"https://arxiv.org/abs/2404.04475"},{"label":"Paper (AlpacaFarm)","url":"https://arxiv.org/abs/2305.14387"},{"label":"Dataset","url":"https://huggingface.co/datasets/tatsu-lab/alpaca_eval"},{"label":"Discord","url":"https://discord.gg/GJMxJMVZZM"}],"problems":[{"id":"alpaca-eval-2","name":"AlpacaEval 2.0 (Length-Controlled Win Rate)","description":"Evaluates instruction-following models using weighted_alpaca_eval_gpt4_turbo as the annotator and gpt4_turbo as the baseline, controlling for length bias.","metricName":"length_controlled_winrate","metricUnit":"%","metricDirection":"maximize","baseline":50,"records":[{"value":77.45,"contributor":"Shopee-SlimMoA-v1","contributorKind":"method","isCurrentBest":true,"agent":{"family":"other-ai","model":"Shopee-SlimMoA-v1","evidence":"Add Shopee-SlimMoA-v1 to AlpacaEval","confidence":"high","role":"subject"}},{"value":72.35,"contributor":"gemma-2-9b-it-SimPO","contributorKind":"model","agent":{"family":"google","model":"gemma-2-9b-it","confidence":"high","role":"subject"}},{"value":67.66,"contributor":"gemma-2-9b-it-DPO","contributorKind":"model","agent":{"family":"google","model":"gemma-2-9b-it","confidence":"high","role":"subject"}},{"value":57.46,"contributor":"gpt-4o-2024-05-13","contributorKind":"model","agent":{"family":"openai","model":"gpt-4o-2024-05-13","confidence":"high","role":"subject"}},{"value":56.76,"contributor":"higgs-llama-3-70b-v2","contributorKind":"model","agent":{"family":"meta","model":"Llama-3-70B","confidence":"high","role":"subject"}},{"value":55.02,"contributor":"gpt-4-turbo-2024-04-09","contributorKind":"model","agent":{"family":"openai","model":"gpt-4-turbo-2024-04-09","confidence":"high","role":"subject"}},{"value":50.73,"contributor":"gpt-4o-mini-2024-07-18","contributorKind":"model","agent":{"family":"openai","model":"gpt-4o-mini-2024-07-18","confidence":"high","role":"subject"}},{"value":50,"contributor":"gpt4_1106_preview","contributorKind":"model","isBaseline":true,"agent":{"family":"openai","model":"gpt-4-1106-preview","confidence":"high","role":"subject"}}],"isOpen":false,"contested":true},{"id":"alpaca-eval-1","name":"AlpacaEval 1.0 (text-davinci-003 baseline)","description":"Evaluates instruction-following models using alpaca_eval_gpt4 as the annotator and text_davinci_003 as the baseline.","metricName":"win_rate","metricUnit":"%","metricDirection":"maximize","baseline":50,"records":[{"value":95.3,"contributor":"gpt4","contributorKind":"model","isCurrentBest":true,"agent":{"family":"openai","model":"gpt-4","confidence":"high","role":"subject"}},{"value":88.4,"contributor":"claude","contributorKind":"model","agent":{"family":"anthropic","model":"claude","confidence":"high","role":"subject"}},{"value":86.1,"contributor":"chatgpt","contributorKind":"model","agent":{"family":"openai","model":"gpt-3.5-turbo","confidence":"high","role":"subject"}},{"value":71.8,"contributor":"guanaco-65b","contributorKind":"model","agent":{"family":"meta","model":"llama-65b","confidence":"high","role":"author"}},{"value":70.4,"contributor":"vicuna-13b","contributorKind":"model","agent":{"family":"meta","model":"llama-13b","confidence":"high","role":"author"}},{"value":50,"contributor":"text_davinci_003","contributorKind":"model","isBaseline":true,"agent":{"family":"openai","model":"text-davinci-003","confidence":"high","role":"subject"}},{"value":41.2,"contributor":"alpaca-farm-ppo-human","contributorKind":"method"},{"value":26.5,"contributor":"alpaca-7b","contributorKind":"model","agent":{"family":"meta","model":"llama-7b","confidence":"high","role":"author"}},{"value":15.2,"contributor":"text_davinci_001","contributorKind":"model","agent":{"family":"openai","model":"text-davinci-001","confidence":"high","role":"subject"}}],"isOpen":false,"contested":true},{"id":"alpaca-eval-evaluators","name":"Evaluator Meta-Evaluation (Human Agreement)","description":"Evaluates different automatic annotators on the AlpacaEval set by comparing their preferences to 2.5K human annotations.","metricName":"accuracy","metricUnit":"%","metricDirection":"maximize","records":[{"value":69.2,"contributor":"alpaca_eval_gpt4","contributorKind":"method","isCurrentBest":true,"agent":{"family":"openai","model":"gpt-4","confidence":"high","role":"author"}},{"value":68.6,"contributor":"alpaca_eval_cot_gpt4_turbo_fn","contributorKind":"method","agent":{"family":"openai","model":"gpt-4-turbo","confidence":"high","role":"author"}},{"value":68.1,"contributor":"alpaca_eval_gpt4_turbo_fn","contributorKind":"method","agent":{"family":"openai","model":"gpt-4-turbo","confidence":"high","role":"author"}},{"value":67.5,"contributor":"alpaca_eval_llama3_70b_fn","contributorKind":"method","agent":{"family":"meta","model":"llama3-70b","confidence":"high","role":"author"}},{"value":66.9,"contributor":"gpt4","contributorKind":"model","agent":{"family":"openai","model":"gpt-4","confidence":"high","role":"subject"}},{"value":66.4,"contributor":"alpaca_farm_greedy_gpt4","contributorKind":"method","agent":{"family":"openai","model":"gpt-4","confidence":"high","role":"author"}},{"value":65.7,"contributor":"alpaca_eval_cot_gpt4_turbo_fn (low cost)","contributorKind":"method","agent":{"family":"openai","model":"gpt-4-turbo","confidence":"high","role":"author"}},{"value":65.7,"contributor":"humans","contributorKind":"person","agent":{"family":"human","confidence":"high","role":"author"},"contributorUrl":"https://github.com/humans"},{"value":65.3,"contributor":"claude","contributorKind":"model","agent":{"family":"anthropic","model":"claude","confidence":"high","role":"subject"}},{"value":65.3,"contributor":"lmsys_gpt4","contributorKind":"method","agent":{"family":"openai","model":"gpt-4","confidence":"high","role":"author"}},{"value":64.1,"contributor":"text_davinci_003","contributorKind":"model","agent":{"family":"openai","model":"text-davinci-003","confidence":"high","role":"subject"}},{"value":62.2,"contributor":"longest","contributorKind":"method"},{"value":57.3,"contributor":"chatgpt","contributorKind":"model","agent":{"family":"openai","model":"gpt-3.5-turbo","confidence":"high","role":"subject"}}],"isOpen":false,"contested":true}],"repo":{"owner":"tatsu-lab","name":"alpaca_eval","branch":"main"},"participants":[{"name":"humans","url":"https://github.com/humans","submissions":1,"problems":["alpaca-eval-evaluators"]}],"stats":{"totalRecords":30,"uniqueParticipants":1,"recordsLast30d":0,"recordsLast90d":0},"agentStats":[{"family":"openai","records":6,"currentBests":1,"models":["gpt-4","gpt-4-turbo"],"contributors":["alpaca_eval_gpt4","alpaca_eval_cot_gpt4_turbo_fn","alpaca_eval_gpt4_turbo_fn","alpaca_farm_greedy_gpt4","alpaca_eval_cot_gpt4_turbo_fn (low cost)","lmsys_gpt4"]},{"family":"meta","records":4,"currentBests":0,"models":["llama-65b","llama-13b","llama-7b","llama3-70b"],"contributors":["guanaco-65b","vicuna-13b","alpaca-7b","alpaca_eval_llama3_70b_fn"]},{"family":"human","records":1,"currentBests":0,"models":[],"contributors":["humans"]}],"lastUpdated":"2026-09-18T16:32:55.538Z"},{"id":"clawprobench","name":"ClawProBench","tagline":"Trace-aware evaluation of AI agents with runtime coverage and frozen workplace-style holdouts.","description":"ClawProBench is a transparent, live-first benchmark harness for evaluating model capabilities inside the OpenClaw runtime. It features deterministic grading across up to 102 active scenarios and supports multiple execution profiles. The benchmark is designed to measure agent stability, quality, and success rates under real-world runtime conditions.","url":"https://github.com/suyoumo/ClawProBench","host":"github","status":"active","tags":["agent-evaluation","benchmark","openclaw","llm-eval","runtime-coverage","ai-agents"],"domain":"coding-agents","images":["https://raw.githubusercontent.com/suyoumo/ClawProBench/main/docs/assets/leaderboard-preview-20260426.png","https://raw.githubusercontent.com/suyoumo/ClawProBench/main/docs/assets/modelpk.png"],"organizer":{"name":"Xiao YuanHang","type":"individual"},"participation":{"howToSubmit":"Run the benchmark locally using the provided harness and email the generated reports to xyh920691910@outlook.com.","submissionFormat":"Generated benchmark reports from the results/ directory containing avg_score, max_score, coverage-aware summaries, cost, latency, and resume metadata.","requirements":["Python 3.11","uv","OpenClaw CLI","Docker"],"compute":"consumer-gpu","computeDetails":"Requires a local OpenClaw runtime. Models can be evaluated via cloud APIs or run locally using Ollama.","deadline":"rolling","verification":"Deterministic grading using custom checkers with order-insensitive matchers."},"quickstart":"pip install uv\nuv venv --python 3.11\nsource .venv/bin/activate\nuv pip install -r requirements.txt\npython3 run.py run --model '<MODEL>' --execution-mode live --benchmark-profile core --trials 3 --cleanup-agents","links":[{"label":"Website","url":"https://suyoumo.github.io/bench"},{"label":"Paper","url":"https://arxiv.org/abs/2608.22510"},{"label":"Dataset","url":"https://huggingface.co/datasets/xyh110sym/clawprobench"}],"problems":[{"id":"core","name":"Core Profile","description":"The default ranking suite consisting of 26 active scenarios.","metricName":"FinalScore","metricDirection":"maximize","records":[],"isOpen":true,"contested":false},{"id":"intelligence","name":"Intelligence Profile","description":"Extended active capability benchmark consisting of 95 active scenarios.","metricName":"FinalScore","metricDirection":"maximize","records":[],"isOpen":true,"contested":false},{"id":"coverage","name":"Coverage Profile","description":"Lower-stakes breadth and regression slice consisting of 7 active scenarios.","metricName":"FinalScore","metricDirection":"maximize","records":[],"isOpen":true,"contested":false},{"id":"native","name":"Native Profile","description":"Active OpenClaw-native slice only consisting of 36 active scenarios.","metricName":"FinalScore","metricDirection":"maximize","records":[],"isOpen":true,"contested":false},{"id":"full","name":"Full Profile","description":"Union of all active scenarios consisting of 102 active scenarios.","metricName":"FinalScore","metricDirection":"maximize","records":[],"isOpen":true,"contested":false}],"repo":{"owner":"suyoumo","name":"ClawProBench","branch":"main"},"participants":[],"stats":{"totalRecords":0,"uniqueParticipants":0,"recordsLast30d":0,"recordsLast90d":0},"agentStats":[],"lastUpdated":"2026-09-18T16:32:55.538Z"},{"id":"gpu-mode-reference-kernels","name":"GPU Mode Reference Kernels","tagline":"Optimize CUDA and Triton kernels for deep learning and linear algebra in community GPU challenges.","description":"A collection of reference kernels and evaluators for GPU Mode's KernelBot competitions. Participants optimize operators like attention, linear algebra, and quantization for speed and correctness on modern GPUs. Submissions are profiled using Nsight Compute and ranked on community leaderboards.","url":"https://github.com/gpu-mode/reference-kernels","host":"github","status":"active","tags":["cuda","triton","gpu-kernels","performance","deep-learning","profiling"],"domain":"hardware-efficiency","images":[],"organizer":{"name":"GPU Mode","url":"https://www.gpumode.com/","type":"community"},"participation":{"howToSubmit":"Submit kernels via the KernelBot on the GPU Mode Discord server, or contribute new problems by opening a PR to the problems/ directory.","submissionFormat":"Python files containing Triton or CUDA kernel implementations (e.g., submission.py).","requirements":["Python","PyTorch","Triton","CUDA","Modal"],"compute":"datacenter-gpu","computeDetails":"Evaluated on high-end GPUs such as NVIDIA B200, A100, and AMD hardware depending on the specific competition track.","prizes":"Up to $1.1M for specific AMD kernel competitions; other tracks offer recognition and smaller cash prizes.","verification":"Automated correctness checks and hardware profiling using Nsight Compute (NCU) or AMD profilers."},"links":[{"label":"Website","url":"https://www.gpumode.com/"},{"label":"Discord","url":"https://discord.gg/gpumode"},{"label":"Popcorn CLI","url":"https://github.com/gpu-mode/popcorn-cli"}],"problems":[{"id":"causal-conv1d","name":"Causal Conv1d","description":"Optimize the causal_conv1d kernel from the Helion Kernel Challenge.","metricName":"time","metricDirection":"minimize","records":[],"isOpen":true,"contested":false},{"id":"qr-v2","name":"QR Decomposition v2","description":"Optimize QR decomposition with Nsight Compute profiling support.","metricName":"time","metricDirection":"minimize","records":[],"isOpen":true,"contested":false},{"id":"cholesky","name":"Cholesky Validation","description":"Optimize Cholesky factorization evaluated over natural-gradient logistic-regression loops.","metricName":"time","metricDirection":"minimize","records":[],"isOpen":true,"contested":false},{"id":"eigh","name":"Eigh Validation","description":"Optimize symmetric eigenvalue decomposition (eigh) kernels.","metricName":"time","metricDirection":"minimize","records":[],"isOpen":true,"contested":false},{"id":"gated-deltanet-chunk-fwd-o","name":"Gated Deltanet chunk_fwd_o","description":"Optimize the chunk_fwd_o kernel from the Helion Kernel Challenge.","metricName":"time","metricUnit":"ms","metricDirection":"minimize","baseline":2.08,"records":[{"value":1.19,"contributor":"yf225","contributorKind":"person","description":"Remove redundant duplicate dot products and increase warps from 1 to 8","isCurrentBest":true,"agent":{"family":"anthropic","model":"Claude Opus 4.6 (1M context)","tool":"Claude Code","evidence":"Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com> ... Generated with Claude Code","confidence":"high","role":"author"},"contributorUrl":"https://github.com/yf225"},{"value":2.08,"contributor":"baseline","contributorKind":"method","isBaseline":true}],"isOpen":false,"contested":false}],"repo":{"owner":"gpu-mode","name":"reference-kernels","branch":"main"},"participants":[{"name":"yf225","url":"https://github.com/yf225","submissions":1,"problems":["gated-deltanet-chunk-fwd-o"]}],"stats":{"totalRecords":2,"uniqueParticipants":1,"recordsLast30d":0,"recordsLast90d":0},"agentStats":[{"family":"anthropic","records":1,"currentBests":0,"models":["Claude Opus 4.6 (1M context)"],"contributors":["yf225"],"uncontestedBests":1}],"lastUpdated":"2026-09-18T16:32:55.538Z"},{"id":"equational-theories-lean-stage2","name":"Mathematics Distillation Challenge — Equational Theories — Stage 2","tagline":"Generate machine-verifiable Lean 4 proofs or countermodels for equational implications over magmas.","description":"A formal mathematics competition based on Terence Tao's Equational Theories project. Solvers must decide if one equational law implies another over any magma, providing a Lean 4 proof or a countermodel. Evaluated deterministically across Solo and Marathon tracks.","url":"https://github.com/SAIRcompetition/equational-theories-lean-stage2","host":"github","status":"active","tags":["lean4","theorem-proving","magmas","equational-theories","formal-methods"],"domain":"formal-methods","images":[],"organizer":{"name":"SAIR Foundation","url":"https://competition.sair.foundation","type":"foundation"},"participation":{"howToSubmit":"Submit a single solver.py file (≤ 500 KB) to the official evaluation platform.","submissionFormat":"A single Python file solver.py implementing the stdin/stdout JSON protocol (Solo) or file-based manifest protocol (Marathon).","requirements":["Python 3.8+","Lean 4","elan","Mathlib","Docker"],"compute":"unknown","computeDetails":"Solvers run in a sandboxed Docker container (python:3.11-slim with sympy) with 2048 MB RAM, 2 CPUs, and no network access. LLM calls are routed through the organizer's proxy to gpt-oss-120b.","cost":"free","verification":"Deterministic Lean 4 judge verifier"},"venue":{"type":"online","event":"Mathematics Distillation Challenge"},"quickstart":"bash scripts/setup.sh\nsource .env.judge\npython3 scripts/run_harness.py\npython3 -m pipeline.runner --submission examples/solo/demos/baseline --problems examples/problems/sample_20.json","links":[{"label":"Official Competition Page","url":"https://competition.sair.foundation/competitions/mathematics-distillation-challenge-equational-theories-stage2/overview"},{"label":"Equational Theories Project","url":"https://teorth.github.io/equational_theories/"},{"label":"HuggingFace Dataset","url":"https://huggingface.co/datasets/SAIRfoundation/equational-theories-selected-problems"}],"problems":[{"id":"solo-track","name":"Solo Track","description":"One problem per solver subprocess with a fixed per-problem budget of 3600s wall-clock and 65,536 output tokens per LLM call.","metricName":"score","metricDirection":"maximize","records":[],"isOpen":true,"contested":false},{"id":"marathon-track","name":"Marathon Track","description":"N problems per solver subprocess under a compressed global budget (N x 5 minutes wall-clock and N x 32,768 tokens).","metricName":"score","metricDirection":"maximize","records":[],"isOpen":true,"contested":false}],"repo":{"owner":"SAIRcompetition","name":"equational-theories-lean-stage2","branch":"main"},"participants":[],"stats":{"totalRecords":0,"uniqueParticipants":0,"recordsLast30d":0,"recordsLast90d":0},"agentStats":[],"lastUpdated":"2026-09-18T16:32:55.538Z"},{"id":"chi-bench","name":"CHI-Bench","tagline":"Benchmark healthcare AI agents on clinical operations, prior authorization, and care management tasks.","description":"CHI-Bench evaluates AI agents on healthcare administrative tasks across provider prior authorization, payer utilization management, and care management. Submissions are verified via a git-based audit packet containing full execution trajectories. Run by Actava AI, it aims to establish rigorous standards for clinical operations agents.","url":"https://github.com/actava-ai/leaderboard","host":"github","status":"active","tags":["healthcare","clinical-operations","ai-agents","prior-authorization","care-management","utilization-management"],"domain":"other","images":[],"organizer":{"name":"Actava AI","url":"https://actava.ai","type":"company"},"participation":{"howToSubmit":"Fork the repository, prepare your submission packet using the chi-bench CLI, and open a pull request adding the packet under benchmarks/chi-bench/submissions/<YYYY-MM-DD>-<slug>/","submissionFormat":"A submission packet containing submission.json, results.csv, sub.yaml, provenance.json, README.md, and compressed trial trajectories (.jsonl.zst)","requirements":["Python","uv","zstd","GitHub CLI"],"compute":"unknown","deadline":"rolling","prizes":"recognition only","verification":"CI validation of schema, integrity, and trial counts, followed by maintainer review"},"quickstart":"git clone https://github.com/<you>/leaderboard && cd leaderboard\npython scripts/submit.py /path/to/packet/2026-05-12-<slug>/","links":[{"label":"Live Leaderboard","url":"https://actava.ai/benchmarks/leaderboards"},{"label":"Submission Guide","url":"https://actava.ai/benchmarks/submit"},{"label":"Producer Repo","url":"https://github.com/actava-ai/chi-bench"},{"label":"Task Explorer","url":"https://actava.ai/benchmarks/tasks"}],"problems":[{"id":"chi-bench-overall","name":"Overall","description":"Overall pass@1 accuracy across all 75 tasks in CHI-Bench.","metricName":"accuracy","metricUnit":"%","metricDirection":"maximize","records":[{"date":"2026-07-27","value":54.7,"contributor":"Michael Johnson (MJ)","contributorKind":"person","description":"Erius agent on Claude Opus 5","isCurrentBest":true,"agent":{"family":"anthropic","model":"anthropic/claude-opus-5","confidence":"high","role":"author"}},{"date":"2026-07-24","value":37.3,"contributor":"Actava","contributorKind":"team","description":"Claude Opus 5 on stock claude-code harness","agent":{"family":"anthropic","model":"anthropic/claude-opus-5","confidence":"high","role":"author"}},{"date":"2026-06-05","value":37.3,"contributor":"Michael Johnson (MJ)","contributorKind":"person","description":"Erius agent on Claude Opus 4.8 (with CM simulator fix)","agent":{"family":"anthropic","model":"anthropic/claude-opus-4-8","confidence":"high","role":"author"}},{"date":"2026-05-28","value":33.3,"contributor":"Actava","contributorKind":"team","description":"Claude Opus 4.8 on claude-code harness","agent":{"family":"anthropic","model":"anthropic/claude-opus-4-8","confidence":"high","role":"author"}},{"date":"2026-05-12","value":28,"contributor":"My Team","contributorKind":"team","description":"Claude Opus 4.6 on claude-code harness","agent":{"family":"anthropic","model":"anthropic/claude-opus-4-6","confidence":"high","role":"author"}},{"date":"2026-07-24","value":25.3,"contributor":"Actava","contributorKind":"team","description":"Kimi K3 on OpenAI Agents SDK harness","agent":{"family":"moonshot","model":"moonshotai/kimi-k3","confidence":"high","role":"author"}},{"date":"2026-07-24","value":25.3,"contributor":"Actava","contributorKind":"team","description":"GPT-5.6 Sol on Codex harness","agent":{"family":"openai","model":"openai/gpt-5.6-sol","confidence":"high","role":"author"}},{"date":"2026-07-22","value":24,"contributor":"Actava","contributorKind":"team","description":"Claude Fable 5 on claude-code harness","agent":{"family":"anthropic","model":"anthropic/claude-fable-5","confidence":"high","role":"author"}},{"date":"2026-07-06","value":22.7,"contributor":"cuilinke","contributorKind":"person","description":"MedGuard model on Hermes agent","agent":{"family":"other-ai","model":"MedGuard","confidence":"high","role":"author"},"contributorUrl":"https://github.com/cuilinke"},{"date":"2026-07-06","value":20,"contributor":"Actava","contributorKind":"team","description":"Claude Sonnet 5 on claude-code harness","agent":{"family":"anthropic","model":"anthropic/claude-sonnet-5","confidence":"high","role":"author"}},{"date":"2026-07-06","value":18.7,"contributor":"Actava","contributorKind":"team","description":"GLM-5.2 on OpenAI Agents SDK harness","agent":{"family":"zhipu","model":"z-ai/glm-5.2","confidence":"high","role":"author"}},{"date":"2026-06-02","value":18.7,"contributor":"Michael Johnson (MJ)","contributorKind":"person","description":"Erius agent on Claude Opus 4.8 (unfixed CM)","agent":{"family":"anthropic","model":"anthropic/claude-opus-4-8","confidence":"high","role":"author"}},{"date":"2026-07-24","value":13.3,"contributor":"Actava","contributorKind":"team","description":"GPT-5.6 Luna on Codex harness","agent":{"family":"openai","model":"openai/gpt-5.6-luna","confidence":"high","role":"author"}},{"date":"2026-07-24","value":13.3,"contributor":"Actava","contributorKind":"team","description":"GPT-5.6 Terra on Codex harness","agent":{"family":"openai","model":"openai/gpt-5.6-terra","confidence":"high","role":"author"}},{"date":"2026-07-24","value":8,"contributor":"Actava","contributorKind":"team","description":"Inkling on OpenAI Agents SDK harness","agent":{"family":"other-ai","model":"thinkingmachines/Inkling:peft:262144","confidence":"high","role":"author"}},{"date":"2026-07-24","value":0,"contributor":"Actava","contributorKind":"team","description":"Nemotron 3 Ultra on OpenAI Agents SDK harness","agent":{"family":"other-ai","model":"nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-BF16:peft:262144","confidence":"high","role":"author"}}],"isOpen":false,"contested":true},{"id":"chi-bench-pa-provider","name":"Provider Prior Authorization","description":"Pass@1 accuracy on the Provider Prior Authorization (PA) domain tasks.","metricName":"accuracy","metricUnit":"%","metricDirection":"maximize","records":[{"date":"2026-07-27","value":72,"contributor":"Michael Johnson (MJ)","contributorKind":"person","description":"Erius agent on Claude Opus 5","isCurrentBest":true,"agent":{"family":"anthropic","model":"anthropic/claude-opus-5","confidence":"high","role":"author"}},{"date":"2026-06-05","value":68,"contributor":"MedArise","contributorKind":"team","description":"GPT-5.5 on Codex harness (PA only)","agent":{"family":"openai","model":"openai/gpt-5.5","confidence":"high","role":"author"}},{"date":"2026-06-05","value":40,"contributor":"Michael Johnson (MJ)","contributorKind":"person","description":"Erius agent on Claude Opus 4.8","agent":{"family":"anthropic","model":"anthropic/claude-opus-4-8","confidence":"high","role":"author"}},{"date":"2026-06-02","value":40,"contributor":"Michael Johnson (MJ)","contributorKind":"person","description":"Erius agent on Claude Opus 4.8","agent":{"family":"anthropic","model":"anthropic/claude-opus-4-8","confidence":"high","role":"author"}},{"date":"2026-07-24","value":36,"contributor":"Actava","contributorKind":"team","description":"GPT-5.6 Sol on Codex harness","agent":{"family":"openai","model":"openai/gpt-5.6-sol","confidence":"high","role":"author"}},{"date":"2026-05-28","value":32,"contributor":"Actava","contributorKind":"team","description":"Claude Opus 4.8 on claude-code harness","agent":{"family":"anthropic","model":"anthropic/claude-opus-4-8","confidence":"high","role":"author"}},{"date":"2026-07-24","value":28,"contributor":"Actava","contributorKind":"team","description":"Kimi K3 on OpenAI Agents SDK harness","agent":{"family":"moonshot","model":"moonshotai/kimi-k3","confidence":"high","role":"author"}},{"date":"2026-07-22","value":24,"contributor":"Actava","contributorKind":"team","description":"Claude Fable 5 on claude-code harness","agent":{"family":"anthropic","model":"anthropic/claude-fable-5","confidence":"high","role":"author"}},{"date":"2026-07-06","value":24,"contributor":"Actava","contributorKind":"team","description":"Claude Sonnet 5 on claude-code harness","agent":{"family":"anthropic","model":"anthropic/claude-sonnet-5","confidence":"high","role":"author"}},{"date":"2026-07-24","value":20,"contributor":"Actava","contributorKind":"team","description":"Claude Opus 5 on stock claude-code harness","agent":{"family":"anthropic","model":"anthropic/claude-opus-5","confidence":"high","role":"author"}},{"date":"2026-05-12","value":20,"contributor":"My Team","contributorKind":"team","description":"Claude Opus 4.6 on claude-code harness","agent":{"family":"anthropic","model":"anthropic/claude-opus-4-6","confidence":"high","role":"author"}},{"date":"2026-07-06","value":20,"contributor":"Actava","contributorKind":"team","description":"GLM-5.2 on OpenAI Agents SDK harness","agent":{"family":"zhipu","model":"z-ai/glm-5.2","confidence":"high","role":"author"}},{"date":"2026-07-24","value":20,"contributor":"Actava","contributorKind":"team","description":"GPT-5.6 Luna on Codex harness","agent":{"family":"openai","model":"openai/gpt-5.6-luna","confidence":"high","role":"author"}},{"date":"2026-07-24","value":12,"contributor":"Actava","contributorKind":"team","description":"GPT-5.6 Terra on Codex harness","agent":{"family":"openai","model":"openai/gpt-5.6-terra","confidence":"high","role":"author"}},{"date":"2026-07-24","value":4,"contributor":"Actava","contributorKind":"team","description":"Inkling on OpenAI Agents SDK harness","agent":{"family":"other-ai","model":"thinkingmachines/Inkling:peft:262144","confidence":"high","role":"author"}},{"date":"2026-07-06","value":4,"contributor":"cuilinke","contributorKind":"person","description":"MedGuard model on Hermes agent","agent":{"family":"other-ai","model":"MedGuard","confidence":"high","role":"author"},"contributorUrl":"https://github.com/cuilinke"},{"date":"2026-07-24","value":0,"contributor":"Actava","contributorKind":"team","description":"Nemotron 3 Ultra on OpenAI Agents SDK harness","agent":{"family":"other-ai","model":"nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-BF16:peft:262144","confidence":"high","role":"author"}}],"isOpen":false,"contested":true},{"id":"chi-bench-pa-um","name":"Payer Utilization Management","description":"Pass@1 accuracy on the Payer Utilization Management (UM) domain tasks.","metricName":"accuracy","metricUnit":"%","metricDirection":"maximize","records":[{"date":"2026-07-27","value":36,"contributor":"Michael Johnson (MJ)","contributorKind":"person","description":"Erius agent on Claude Opus 5","isCurrentBest":true,"agent":{"family":"anthropic","model":"anthropic/claude-opus-5","confidence":"high","role":"author"}},{"date":"2026-05-12","value":36,"contributor":"My Team","contributorKind":"team","description":"Claude Opus 4.6 on claude-code harness","agent":{"family":"anthropic","model":"anthropic/claude-opus-4-6","confidence":"high","role":"author"}},{"date":"2026-07-24","value":32,"contributor":"Actava","contributorKind":"team","description":"Claude Opus 5 on stock claude-code harness","agent":{"family":"anthropic","model":"anthropic/claude-opus-5","confidence":"high","role":"author"}},{"date":"2026-07-24","value":32,"contributor":"Actava","contributorKind":"team","description":"Kimi K3 on OpenAI Agents SDK harness","agent":{"family":"moonshot","model":"moonshotai/kimi-k3","confidence":"high","role":"author"}},{"date":"2026-07-06","value":32,"contributor":"Actava","contributorKind":"team","description":"GLM-5.2 on OpenAI Agents SDK harness","agent":{"family":"zhipu","model":"z-ai/glm-5.2","confidence":"high","role":"author"}},{"date":"2026-07-24","value":28,"contributor":"Actava","contributorKind":"team","description":"GPT-5.6 Sol on Codex harness","agent":{"family":"openai","model":"openai/gpt-5.6-sol","confidence":"high","role":"author"}},{"date":"2026-05-28","value":28,"contributor":"Actava","contributorKind":"team","description":"Claude Opus 4.8 on claude-code harness","agent":{"family":"anthropic","model":"anthropic/claude-opus-4-8","confidence":"high","role":"author"}},{"date":"2026-07-22","value":24,"contributor":"Actava","contributorKind":"team","description":"Claude Fable 5 on claude-code harness","agent":{"family":"anthropic","model":"anthropic/claude-fable-5","confidence":"high","role":"author"}},{"date":"2026-07-06","value":24,"contributor":"Actava","contributorKind":"team","description":"Claude Sonnet 5 on claude-code harness","agent":{"family":"anthropic","model":"anthropic/claude-sonnet-5","confidence":"high","role":"author"}},{"date":"2026-07-24","value":20,"contributor":"Actava","contributorKind":"team","description":"GPT-5.6 Terra on Codex harness","agent":{"family":"openai","model":"openai/gpt-5.6-terra","confidence":"high","role":"author"}},{"date":"2026-06-05","value":16,"contributor":"Michael Johnson (MJ)","contributorKind":"person","description":"Erius agent on Claude Opus 4.8","agent":{"family":"anthropic","model":"anthropic/claude-opus-4-8","confidence":"high","role":"author"}},{"date":"2026-06-02","value":16,"contributor":"Michael Johnson (MJ)","contributorKind":"person","description":"Erius agent on Claude Opus 4.8","agent":{"family":"anthropic","model":"anthropic/claude-opus-4-8","confidence":"high","role":"author"}},{"date":"2026-07-24","value":16,"contributor":"Actava","contributorKind":"team","description":"GPT-5.6 Luna on Codex harness","agent":{"family":"openai","model":"openai/gpt-5.6-luna","confidence":"high","role":"author"}},{"date":"2026-07-24","value":16,"contributor":"Actava","contributorKind":"team","description":"Inkling on OpenAI Agents SDK harness","agent":{"family":"other-ai","model":"thinkingmachines/Inkling:peft:262144","confidence":"high","role":"author"}},{"date":"2026-07-06","value":4,"contributor":"cuilinke","contributorKind":"person","description":"MedGuard model on Hermes agent","agent":{"family":"other-ai","model":"MedGuard","confidence":"high","role":"author"},"contributorUrl":"https://github.com/cuilinke"},{"date":"2026-07-24","value":0,"contributor":"Actava","contributorKind":"team","description":"Nemotron 3 Ultra on OpenAI Agents SDK harness","agent":{"family":"other-ai","model":"nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-BF16:peft:262144","confidence":"high","role":"author"}}],"isOpen":false,"contested":true},{"id":"chi-bench-cm","name":"Care Management","description":"Pass@1 accuracy on the Care Management (CM) domain tasks.","metricName":"accuracy","metricUnit":"%","metricDirection":"maximize","records":[{"date":"2026-07-24","value":60,"contributor":"Actava","contributorKind":"team","description":"Claude Opus 5 on stock claude-code harness","isCurrentBest":true,"agent":{"family":"anthropic","model":"anthropic/claude-opus-5","confidence":"high","role":"author"}},{"date":"2026-07-06","value":60,"contributor":"cuilinke","contributorKind":"person","description":"MedGuard model on Hermes agent","agent":{"family":"other-ai","model":"MedGuard","confidence":"high","role":"author"},"contributorUrl":"https://github.com/cuilinke"},{"date":"2026-07-27","value":56,"contributor":"Michael Johnson (MJ)","contributorKind":"person","description":"Erius agent on Claude Opus 5","agent":{"family":"anthropic","model":"anthropic/claude-opus-5","confidence":"high","role":"author"}},{"date":"2026-06-05","value":56,"contributor":"Michael Johnson (MJ)","contributorKind":"person","description":"Erius agent on Claude Opus 4.8 (with CM simulator fix)","agent":{"family":"anthropic","model":"anthropic/claude-opus-4-8","confidence":"high","role":"author"}},{"date":"2026-05-28","value":40,"contributor":"Actava","contributorKind":"team","description":"Claude Opus 4.8 on claude-code harness","agent":{"family":"anthropic","model":"anthropic/claude-opus-4-8","confidence":"high","role":"author"}},{"date":"2026-05-12","value":28,"contributor":"My Team","contributorKind":"team","description":"Claude Opus 4.6 on claude-code harness","agent":{"family":"anthropic","model":"anthropic/claude-opus-4-6","confidence":"high","role":"author"}},{"date":"2026-07-22","value":24,"contributor":"Actava","contributorKind":"team","description":"Claude Fable 5 on claude-code harness","agent":{"family":"anthropic","model":"anthropic/claude-fable-5","confidence":"high","role":"author"}},{"date":"2026-07-24","value":16,"contributor":"Actava","contributorKind":"team","description":"Kimi K3 on OpenAI Agents SDK harness","agent":{"family":"moonshot","model":"moonshotai/kimi-k3","confidence":"high","role":"author"}},{"date":"2026-07-24","value":12,"contributor":"Actava","contributorKind":"team","description":"GPT-5.6 Sol on Codex harness","agent":{"family":"openai","model":"openai/gpt-5.6-sol","confidence":"high","role":"author"}},{"date":"2026-07-06","value":12,"contributor":"Actava","contributorKind":"team","description":"Claude Sonnet 5 on claude-code harness","agent":{"family":"anthropic","model":"anthropic/claude-sonnet-5","confidence":"high","role":"author"}},{"date":"2026-07-24","value":8,"contributor":"Actava","contributorKind":"team","description":"GPT-5.6 Terra on Codex harness","agent":{"family":"openai","model":"openai/gpt-5.6-terra","confidence":"high","role":"author"}},{"date":"2026-07-06","value":4,"contributor":"Actava","contributorKind":"team","description":"GLM-5.2 on OpenAI Agents SDK harness","agent":{"family":"zhipu","model":"z-ai/glm-5.2","confidence":"high","role":"author"}},{"date":"2026-07-24","value":4,"contributor":"Actava","contributorKind":"team","description":"GPT-5.6 Luna on Codex harness","agent":{"family":"openai","model":"openai/gpt-5.6-luna","confidence":"high","role":"author"}},{"date":"2026-07-24","value":4,"contributor":"Actava","contributorKind":"team","description":"Inkling on OpenAI Agents SDK harness","agent":{"family":"other-ai","model":"thinkingmachines/Inkling:peft:262144","confidence":"high","role":"author"}},{"date":"2026-06-02","value":0,"contributor":"Michael Johnson (MJ)","contributorKind":"person","description":"Erius agent on Claude Opus 4.8 (unfixed CM)","agent":{"family":"anthropic","model":"anthropic/claude-opus-4-8","confidence":"high","role":"author"}},{"date":"2026-07-24","value":0,"contributor":"Actava","contributorKind":"team","description":"Nemotron 3 Ultra on OpenAI Agents SDK harness","agent":{"family":"other-ai","model":"nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-BF16:peft:262144","confidence":"high","role":"author"}}],"isOpen":false,"contested":true}],"repo":{"owner":"actava-ai","name":"leaderboard","branch":"main"},"participants":[{"name":"Actava","submissions":44,"problems":["chi-bench-overall","chi-bench-pa-provider","chi-bench-pa-um","chi-bench-cm"],"lastActive":"2026-07-24","bestRank":1},{"name":"Michael Johnson (MJ)","submissions":12,"problems":["chi-bench-overall","chi-bench-pa-provider","chi-bench-pa-um","chi-bench-cm"],"bestRank":1,"lastActive":"2026-07-27"},{"name":"My Team","submissions":4,"problems":["chi-bench-overall","chi-bench-pa-provider","chi-bench-pa-um","chi-bench-cm"],"lastActive":"2026-05-12"},{"name":"cuilinke","url":"https://github.com/cuilinke","submissions":4,"problems":["chi-bench-overall","chi-bench-pa-provider","chi-bench-pa-um","chi-bench-cm"],"lastActive":"2026-07-06"},{"name":"MedArise","submissions":1,"problems":["chi-bench-pa-provider"],"lastActive":"2026-06-05"}],"stats":{"totalRecords":65,"uniqueParticipants":5,"firstSubmission":"2026-05-12","lastSubmission":"2026-07-27","recordsLast30d":0,"recordsLast90d":48},"agentStats":[{"family":"anthropic","records":32,"currentBests":4,"models":["anthropic/claude-opus-5","anthropic/claude-opus-4-8","anthropic/claude-opus-4-6","anthropic/claude-fable-5","anthropic/claude-sonnet-5"],"contributors":["Michael Johnson (MJ)","Actava","My Team"]},{"family":"openai","records":13,"currentBests":0,"models":["openai/gpt-5.6-sol","openai/gpt-5.6-luna","openai/gpt-5.6-terra","openai/gpt-5.5"],"contributors":["Actava","MedArise"]},{"family":"other-ai","records":12,"currentBests":0,"models":["MedGuard","thinkingmachines/Inkling:peft:262144","nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-BF16:peft:262144"],"contributors":["cuilinke","Actava"]},{"family":"moonshot","records":4,"currentBests":0,"models":["moonshotai/kimi-k3"],"contributors":["Actava"]},{"family":"zhipu","records":4,"currentBests":0,"models":["z-ai/glm-5.2"],"contributors":["Actava"]}],"lastUpdated":"2026-09-18T16:32:55.538Z"},{"id":"geo-bench-2","name":"GEO-Bench 2 Leaderboard","tagline":"Benchmark geospatial foundation models on diverse Earth observation datasets.","description":"GEO-Bench tracks the performance of geospatial foundation models across various benchmark datasets. Submissions are evaluated using metrics like Overall Accuracy, Multilabel F1 Score, and Multiclass Jaccard Index, with benchmarking facilitated by the TerraTorch-Iterate framework.","url":"https://github.com/The-AI-Alliance/GEO-Bench-2-Leaderboard","host":"github","status":"active","tags":["geospatial","earth-observation","foundation-models","remote-sensing","terratorch"],"domain":"ml-research","images":[],"organizer":{"name":"The AI Alliance","url":"https://thealliance.ai","type":"community"},"participation":{"howToSubmit":"Fork the repository, create a new folder under `new_submission/` containing `results_and_parameters.csv` and `additional_info.json`, and open a pull request to main.","submissionFormat":"CSV file with results and parameters, and a JSON file with metadata.","requirements":["Python 3.10+","TerraTorch-Iterate","MLFlow"],"compute":"unknown","deadline":"rolling","prizes":"recognition only","cost":"free","verification":"CI and manual review of pull requests"},"venue":{"type":"online","event":"GEO-Bench Leaderboard"},"quickstart":"terratorch iterate --hpo --repeat --config <config-file>\nterratorch iterate --summarize --config <summarize-config-file>","links":[{"label":"Hugging Face Leaderboard","url":"https://huggingface.co/spaces/aialliance/GEO-Bench-2-Leaderboard"},{"label":"TerraTorch-Iterate GitHub","url":"https://github.com/IBM/terratorch-iterate"},{"label":"TerraTorch GitHub","url":"https://github.com/IBM/terratorch"}],"problems":[{"id":"overall-accuracy","name":"Overall Accuracy Track","description":"Evaluation of geospatial models on datasets using Overall Accuracy.","metricName":"accuracy","metricUnit":"%","metricDirection":"maximize","records":[],"isOpen":true,"contested":false},{"id":"multilabel-f1","name":"Multilabel F1 Score Track","description":"Evaluation of geospatial models on datasets using Multilabel F1 Score.","metricName":"F1-score","metricDirection":"maximize","records":[],"isOpen":true,"contested":false},{"id":"multiclass-jaccard","name":"Multiclass Jaccard Index Track","description":"Evaluation of geospatial models on datasets using Multiclass Jaccard Index (IoU).","metricName":"Jaccard Index","metricDirection":"maximize","records":[],"isOpen":true,"contested":false},{"id":"biomassters-rmse","name":"Biomassters RMSE Track","description":"Evaluation of geospatial models on the Biomassters dataset using Root Mean Squared Error.","metricName":"RMSE","metricDirection":"minimize","records":[],"isOpen":true,"contested":false}],"repo":{"owner":"The-AI-Alliance","name":"GEO-Bench-2-Leaderboard","branch":"main"},"participants":[],"stats":{"totalRecords":0,"uniqueParticipants":0,"recordsLast30d":0,"recordsLast90d":0},"agentStats":[],"lastUpdated":"2026-09-18T16:32:55.538Z"},{"id":"programbench","name":"ProgramBench","tagline":"Evaluate and track the performance of AI coding agents on software engineering tasks.","description":"ProgramBench is a benchmark for evaluating AI coding agents. Submissions are registered via pull requests containing execution manifests and per-instance scores, which are then verified and compiled into the public leaderboard.","url":"https://github.com/programbench/submissions","host":"github","status":"active","tags":["coding-agents","benchmark","software-engineering","swe-agent","llm-eval"],"domain":"coding-agents","images":[],"organizer":{"name":"Meta","url":"https://github.com/facebookresearch/ProgramBench","type":"company"},"participation":{"howToSubmit":"Open a PR to main adding a folder under submissions/<submission_id>/ containing pointer.yaml, submission.yaml, and _stats/.","submissionFormat":"A folder containing pointer.yaml (fork URL + pinned commit SHA), submission.yaml (manifest), and _stats/ (score.json, cost.json, calls.json).","requirements":["Python","Docker","uv"],"compute":"unknown","deadline":"rolling","verification":"CI verification using programbench submit verify (Tier-0 recomputes headline, Tier-1 re-runs eval in Docker)"},"quickstart":"git clone <your-submission-fork> && cd <fork>\nuvx programbench submit verify .","links":[{"label":"Leaderboard","url":"https://programbench.com/"},{"label":"Main Repository","url":"https://github.com/facebookresearch/ProgramBench"},{"label":"Submission Guide","url":"https://programbench.com/blog/submission-guide"}],"problems":[{"id":"programbench-mini","name":"ProgramBench Mini","description":"Evaluation on the ProgramBench Mini dataset.","metricName":"mean score","metricUnit":"%","metricDirection":"maximize","records":[{"date":"2026-05-08","value":69.8,"contributor":"GPT-5.5 (xhigh reasoning)","contributorKind":"model","contributorUrl":"https://github.com/ProgramBench/20260508_mini-v2.2.6_gpt-5-5-xhigh","description":"resolved: 0.5% / near-resolved: 13.5%","isCurrentBest":true,"agent":{"family":"openai","model":"GPT-5.5 (xhigh reasoning)","tool":"mini-SWE-agent","evidence":"Registers GPT-5.5 (xhigh reasoning) (OpenAI) + mini-SWE-agent.","confidence":"high","role":"author"}},{"date":"2026-05-07","value":67,"contributor":"GPT-5.5 (high reasoning)","contributorKind":"model","contributorUrl":"https://github.com/ProgramBench/20260507_mini-v2.2.6_gpt-5-5-high","description":"resolved: 0.5% / near-resolved: 5.0%","agent":{"family":"openai","model":"GPT-5.5 (high reasoning)","tool":"mini-SWE-agent","evidence":"Registers GPT-5.5 (high reasoning) (OpenAI) + mini-SWE-agent.","confidence":"high","role":"author"}},{"date":"2026-05-07","value":56.8,"contributor":"GPT-5.5","contributorKind":"model","contributorUrl":"https://github.com/ProgramBench/20260507_mini-v2.2.6_gpt-5-5","description":"resolved: 0.0% / near-resolved: 1.5%","agent":{"family":"openai","model":"GPT-5.5","tool":"mini-SWE-agent","evidence":"Registers GPT-5.5 (OpenAI) + mini-SWE-agent.","confidence":"high","role":"author"}},{"date":"2026-04-30","value":52.1,"contributor":"Claude Opus 4.6","contributorKind":"model","contributorUrl":"https://github.com/ProgramBench/20260430_mini-v2.2.6_opus-4-6","description":"resolved: 0.0% / near-resolved: 2.5%","agent":{"family":"anthropic","model":"Claude Opus 4.6","tool":"mini-SWE-agent","evidence":"Registers Claude Opus 4.6 (Anthropic) + mini-SWE-agent.","confidence":"high","role":"author"}},{"date":"2026-04-29","value":48.6,"contributor":"Claude Sonnet 4.6","contributorKind":"model","contributorUrl":"https://github.com/ProgramBench/20260429_mini-v2.2.6_sonnet-4-6","description":"resolved: 0.0% / near-resolved: 1.5%","agent":{"family":"anthropic","model":"Claude Sonnet 4.6","tool":"mini-SWE-agent","evidence":"Registers Claude Sonnet 4.6 (Anthropic) + mini-SWE-agent.","confidence":"high","role":"author"}},{"date":"2026-04-29","value":38.7,"contributor":"GPT-5.4","contributorKind":"model","contributorUrl":"https://github.com/ProgramBench/20260429_mini-v2.2.6_gpt-5-4","description":"resolved: 0.0% / near-resolved: 0.0%","agent":{"family":"openai","model":"GPT-5.4","tool":"mini-SWE-agent","evidence":"Registers GPT-5.4 (OpenAI) + mini-SWE-agent.","confidence":"high","role":"author"}},{"date":"2026-04-29","value":32.6,"contributor":"Gemini 3 Flash","contributorKind":"model","contributorUrl":"https://github.com/ProgramBench/20260429_mini-v2.2.6_gemini-3-flash","description":"resolved: 0.0% / near-resolved: 0.0%","agent":{"family":"google","model":"Gemini 3 Flash","tool":"mini-SWE-agent","evidence":"Registers Gemini 3 Flash (Google) + mini-SWE-agent.","confidence":"high","role":"author"}},{"date":"2026-04-29","value":30.4,"contributor":"Claude Haiku 4.5","contributorKind":"model","contributorUrl":"https://github.com/ProgramBench/20260429_mini-v2.2.6_haiku-4-5","description":"resolved: 0.0% / near-resolved: 0.0%","agent":{"family":"anthropic","model":"Claude Haiku 4.5","tool":"mini-SWE-agent","evidence":"Registers Claude Haiku 4.5 (Anthropic) + mini-SWE-agent.","confidence":"high","role":"author"}},{"date":"2026-04-29","value":17.1,"contributor":"GPT-5.4 mini","contributorKind":"model","contributorUrl":"https://github.com/ProgramBench/20260429_mini-v2.2.6_gpt-5-4-mini","description":"resolved: 0.0% / near-resolved: 0.0%","agent":{"family":"openai","model":"GPT-5.4 mini","tool":"mini-SWE-agent","evidence":"Registers GPT-5.4 mini (OpenAI) + mini-SWE-agent.","confidence":"high","role":"author"}},{"date":"2026-04-29","value":16.2,"contributor":"GPT-5 mini","contributorKind":"model","contributorUrl":"https://github.com/ProgramBench/20260429_mini-v2.2.6_gpt-5-mini","description":"resolved: 0.0% / near-resolved: 0.0%","agent":{"family":"openai","model":"GPT-5 mini","tool":"mini-SWE-agent","evidence":"Registers GPT-5 mini (OpenAI) + mini-SWE-agent.","confidence":"high","role":"author"}}],"isOpen":false,"contested":true}],"repo":{"owner":"programbench","name":"submissions","branch":"main"},"participants":[],"stats":{"totalRecords":10,"uniqueParticipants":0,"firstSubmission":"2026-04-29","lastSubmission":"2026-05-08","recordsLast30d":0,"recordsLast90d":0},"agentStats":[{"family":"openai","records":6,"currentBests":1,"models":["GPT-5.5 (xhigh reasoning)","GPT-5.5 (high reasoning)","GPT-5.5","GPT-5.4","GPT-5.4 mini","GPT-5 mini"],"contributors":["GPT-5.5 (xhigh reasoning)","GPT-5.5 (high reasoning)","GPT-5.5","GPT-5.4","GPT-5.4 mini","GPT-5 mini"]},{"family":"anthropic","records":3,"currentBests":0,"models":["Claude Opus 4.6","Claude Sonnet 4.6","Claude Haiku 4.5"],"contributors":["Claude Opus 4.6","Claude Sonnet 4.6","Claude Haiku 4.5"]},{"family":"google","records":1,"currentBests":0,"models":["Gemini 3 Flash"],"contributors":["Gemini 3 Flash"]}],"lastUpdated":"2026-09-18T16:32:55.538Z"},{"id":"s2n-bignum-bench","name":"s2n-bignum-bench","tagline":"Evaluate LLM low-level code reasoning through HOL Light tactic synthesis on cryptographic proofs.","description":"A practical benchmark for evaluating low-level code reasoning of LLMs through HOL Light tactic synthesis on cryptographic proofs from AWS s2n-bignum. Participants generate tactic proofs for HOL Light goals, which are then evaluated for correctness. Submissions are verified by running the proofs in HOL Light.","url":"https://github.com/kings-crown/s2n-bignum-bench","host":"github","status":"active","tags":["hol-light","theorem-proving","cryptography","formal-verification","tactic-synthesis","llm-eval"],"domain":"formal-methods","images":[],"organizer":{"name":"Balaji Rao & Juneyoung Lee","type":"academic"},"participation":{"howToSubmit":"Zip your answers CSV and submit via a GitHub issue on the leaderboard repository.","submissionFormat":"CSV with columns problem_id, category, query, answer","requirements":["Python 3.9+","HOL Light","OCaml","opam","bash"],"compute":"cpu","computeDetails":"HOL Light proof checking requires ~5 GB RAM per core; setup builds HOL Light and s2n-bignum.","deadline":"rolling","prizes":"recognition only","cost":"free","verification":"HOL Light execution"},"quickstart":"git clone https://github.com/kings-crown/s2n-bignum-bench\ncd s2n-bignum-bench\n./setup.sh arm 4\npython3 collect-problems.py toplevel-thms/ problems.json ml_files\npython3 retrieve-problem.py retrieve --outputdir workdir --csv problems.csv --csv-only","links":[{"label":"Leaderboard","url":"https://kings-crown.github.io/s2n-bignum-leaderboard/"},{"label":"Paper","url":"https://arxiv.org/pdf/2603.14628"},{"label":"Dataset on Hugging Face","url":"https://huggingface.co/datasets/kings-crown/s2n-bignum-bench"},{"label":"Submission Template","url":"https://github.com/kings-crown/s2n-bignum-leaderboard/issues/new?template=leaderboard-submission.yml"}],"problems":[{"id":"tactic-synthesis","name":"HOL Light Tactic Synthesis","description":"Synthesize HOL Light tactic proofs for cryptographic theorems from AWS s2n-bignum.","metricName":"accuracy","metricUnit":"%","metricDirection":"maximize","records":[],"isOpen":true,"contested":false}],"repo":{"owner":"kings-crown","name":"s2n-bignum-bench","branch":"main"},"participants":[],"stats":{"totalRecords":0,"uniqueParticipants":0,"recordsLast30d":0,"recordsLast90d":0},"agentStats":[],"lastUpdated":"2026-09-18T16:32:55.538Z"},{"id":"humanoid-parkour","name":"Humanoid Parkour","tagline":"Optimize an ONNX policy to drive a Unitree G1 humanoid through a 51m parkour course.","description":"Miners submit an ONNX policy to control a 12-DoF Unitree G1 humanoid robot through a challenging 51-meter parkour course. Submissions are evaluated across 24 randomized instances with varying wind and friction conditions. It is run as an Apex competition on Bittensor Subnet 1.","url":"https://github.com/macrocosm-os/apex-competition-humanoid-parkour","host":"github","status":"active","tags":["humanoid","parkour","reinforcement-learning","mujoco","onnx","bittensor","robotics"],"domain":"robotics","images":["https://github.com/user-attachments/assets/17b9f4ca-39ef-4d15-a216-b3eee1888a43"],"organizer":{"name":"Macrocosm","type":"community"},"participation":{"howToSubmit":"Submit an ONNX policy (≤ 15 MB) to the Bittensor Subnet 1 platform.","submissionFormat":"ONNX graph, ≤ 15 MB","requirements":["Python","MuJoCo","ONNX","Docker"],"compute":"cpu","computeDetails":"Referee peaks at 560 MiB, player at 30 MiB. Evaluated under limits: --cpus 1 --memory 1.5g.","deadline":"rolling","prizes":"Bittensor Subnet 1 mining rewards","cost":"free","verification":"Evaluated on 24 randomized instances in a Docker sandbox by the referee."},"quickstart":"docker build -f referee/Dockerfile -t hp-referee .\ndocker build -f player/Dockerfile  -t hp-player  .\ndocker network create hpnet\ndocker run -d --name hp-p --network hpnet -v \"$PWD/baseline/baseline.onnx:/app/submission.onnx:ro\" hp-player","links":[{"label":"GitHub Repository","url":"https://github.com/macrocosm-os/apex-competition-humanoid-parkour"},{"label":"Unitree RL Gym","url":"https://github.com/unitreerobotics/unitree_rl_gym"}],"problems":[{"id":"humanoid-parkour-course","name":"Humanoid Parkour Course","description":"Drive the Unitree G1 humanoid through the 51m parkour course under randomized wind and friction.","metricName":"raw_score","metricUnit":"score","metricDirection":"maximize","baseline":0.21,"records":[{"value":0.21,"contributor":"Unitree G1 Stock Walker","contributorKind":"method","description":"Unitree's stock G1 walking policy (deploy/pre_train/g1/motion.pt) wrapped for the interface.","isBaseline":true,"isCurrentBest":true,"agent":{"family":"other-ai","model":"Unitree G1 Stock Walker","evidence":"Unitree's stock G1 walking policy (deploy/pre_train/g1/motion.pt) wrapped for the interface.","confidence":"high","role":"subject"}}],"isOpen":true,"contested":false}],"repo":{"owner":"macrocosm-os","name":"apex-competition-humanoid-parkour","branch":"main"},"participants":[],"stats":{"totalRecords":1,"uniqueParticipants":0,"recordsLast30d":0,"recordsLast90d":0},"agentStats":[],"lastUpdated":"2026-09-18T16:32:55.538Z"},{"id":"llm-leaderboard","name":"LLM-Leaderboard","tagline":"A joint community effort to create one central leaderboard for Large Language Models.","description":"A crowdsourced leaderboard compiling evaluation results of open and closed large language models across multiple standard benchmarks. It tracks performance on Chatbot Arena Elo, HellaSwag, HumanEval, MMLU, and more, allowing the community to submit new models and benchmarks via pull requests.","url":"https://github.com/LudwigStumpp/llm-leaderboard","host":"github","status":"ended","tags":["llm","leaderboard","nlp","evaluation","benchmark"],"domain":"llm-eval","images":[],"organizer":{"name":"Ludwig Stumpp","url":"https://github.com/LudwigStumpp","type":"individual"},"participation":{"howToSubmit":"Open a PR to main adding a new model as a new row to the leaderboard table in README.md, keeping alphabetical order.","submissionFormat":"Markdown table row with model details and benchmark links","requirements":["Markdown"],"compute":"none","computeDetails":"No compute is provided; results are self-reported from papers or external benchmarks.","cost":"free","verification":"Human review of the pull request and linked sources"},"links":[{"label":"Interactive Dashboard","url":"https://llm-leaderboard.streamlit.app/"},{"label":"Hugging Face Space","url":"https://huggingface.co/spaces/ludwigstumpp/llm-leaderboard"}],"problems":[{"id":"chatbot-arena-elo","name":"Chatbot Arena Elo","description":"LMSYS Chatbot Arena Elo rating based on crowdsourced anonymous randomized battles.","metricName":"Elo","metricDirection":"maximize","records":[{"value":1008,"contributor":"alpaca-13b","contributorKind":"model","submissionUrls":["https://lmsys.org/blog/2023-05-03-arena/"],"agent":{"family":"other-ai","model":"alpaca-13b","confidence":"high","role":"subject"}},{"value":985,"contributor":"chatglm-6b","contributorKind":"model","submissionUrls":["https://lmsys.org/blog/2023-05-03-arena/"],"agent":{"family":"other-ai","model":"chatglm-6b","confidence":"high","role":"subject"}},{"value":944,"contributor":"dolly-v2-12b","contributorKind":"model","submissionUrls":["https://lmsys.org/blog/2023-05-03-arena/"],"agent":{"family":"other-ai","model":"dolly-v2-12b","confidence":"high","role":"subject"}},{"value":951,"contributor":"fastchat-t5-3b","contributorKind":"model","submissionUrls":["https://lmsys.org/blog/2023-05-03-arena/"],"agent":{"family":"other-ai","model":"fastchat-t5-3b","confidence":"high","role":"subject"}},{"value":1082,"contributor":"koala-13b","contributorKind":"model","submissionUrls":["https://lmsys.org/blog/2023-05-03-arena/"],"agent":{"family":"other-ai","model":"koala-13b","confidence":"high","role":"subject"}},{"value":932,"contributor":"llama-13b","contributorKind":"model","submissionUrls":["https://lmsys.org/blog/2023-05-03-arena/"],"agent":{"family":"meta","model":"llama-13b","confidence":"high","role":"subject"}},{"value":1065,"contributor":"oasst-pythia-12b","contributorKind":"model","submissionUrls":["https://lmsys.org/blog/2023-05-03-arena/"],"agent":{"family":"other-ai","model":"oasst-pythia-12b","confidence":"high","role":"subject"}},{"value":858,"contributor":"stablelm-tuned-alpha-7b","contributorKind":"model","submissionUrls":["https://lmsys.org/blog/2023-05-03-arena/"],"agent":{"family":"other-ai","model":"stablelm-tuned-alpha-7b","confidence":"high","role":"subject"}},{"value":1169,"contributor":"vicuna-13b","contributorKind":"model","submissionUrls":["https://lmsys.org/blog/2023-05-03-arena/"],"isCurrentBest":true,"agent":{"family":"other-ai","model":"vicuna-13b","confidence":"high","role":"subject"}}],"isOpen":false,"contested":true},{"id":"hellaswag-few-shot","name":"HellaSwag (few-shot)","description":"HellaSwag commonsense reasoning benchmark evaluated in a few-shot setting.","metricName":"accuracy","metricDirection":"maximize","records":[{"value":0.744,"contributor":"bloom-176b","contributorKind":"model","submissionUrls":["https://crfm.stanford.edu/helm/latest/?group=core_scenarios"],"agent":{"family":"other-ai","model":"bloom-176b","confidence":"high","role":"subject"}},{"value":0.781,"contributor":"falcon-7b","contributorKind":"model","submissionUrls":["https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard"],"agent":{"family":"other-ai","model":"falcon-7b","confidence":"high","role":"subject"}},{"value":0.853,"contributor":"falcon-40b","contributorKind":"model","submissionUrls":["https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard"],"agent":{"family":"other-ai","model":"falcon-40b","confidence":"high","role":"subject"}},{"value":0.682,"contributor":"gpt-3-7b / curie","contributorKind":"model","submissionUrls":["https://crfm.stanford.edu/helm/latest/?group=core_scenarios"],"agent":{"family":"openai","model":"curie","confidence":"high","role":"subject"}},{"value":0.793,"contributor":"gpt-3-175b / davinci","contributorKind":"model","submissionUrls":["https://arxiv.org/abs/2005.14165"],"agent":{"family":"openai","model":"davinci","confidence":"high","role":"subject"}},{"value":0.822,"contributor":"gpt-3.5-175b / text-davinci-003","contributorKind":"model","submissionUrls":["https://crfm.stanford.edu/helm/latest/?group=core_scenarios"],"agent":{"family":"openai","model":"text-davinci-003","confidence":"high","role":"subject"}},{"value":0.953,"contributor":"gpt-4","contributorKind":"model","submissionUrls":["https://arxiv.org/abs/2303.08774v3"],"isCurrentBest":true,"agent":{"family":"openai","model":"gpt-4","confidence":"high","role":"subject"}},{"value":0.718,"contributor":"gpt-neox-20b","contributorKind":"model","submissionUrls":["https://crfm.stanford.edu/helm/latest/?group=core_scenarios"],"agent":{"family":"other-ai","model":"gpt-neox-20b","confidence":"high","role":"subject"}},{"value":0.663,"contributor":"gpt-j-6b","contributorKind":"model","submissionUrls":["https://crfm.stanford.edu/helm/latest/?group=core_scenarios"],"agent":{"family":"other-ai","model":"gpt-j-6b","confidence":"high","role":"subject"}},{"value":0.873,"contributor":"llama-2-70b","contributorKind":"model","submissionUrls":["https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard"],"agent":{"family":"meta","model":"llama-2-70b","confidence":"high","role":"subject"}},{"value":0.745,"contributor":"opt-66b","contributorKind":"model","submissionUrls":["https://crfm.stanford.edu/helm/latest/?group=core_scenarios"],"agent":{"family":"meta","model":"opt-66b","confidence":"high","role":"subject"}},{"value":0.791,"contributor":"opt-175b","contributorKind":"model","submissionUrls":["https://crfm.stanford.edu/helm/latest/?group=core_scenarios"],"agent":{"family":"meta","model":"opt-175b","confidence":"high","role":"subject"}},{"value":0.838,"contributor":"palm-540b","contributorKind":"model","submissionUrls":["https://arxiv.org/abs/2204.02311v5"],"agent":{"family":"google","model":"palm-540b","confidence":"high","role":"subject"}}],"isOpen":false,"contested":true},{"id":"hellaswag-zero-shot","name":"HellaSwag (zero-shot)","description":"HellaSwag commonsense reasoning benchmark evaluated in a zero-shot setting.","metricName":"accuracy","metricDirection":"maximize","records":[{"value":0.739,"contributor":"alpaca-7b","contributorKind":"model","submissionUrls":["https://gpt4all.io/reports/GPT4All_Technical_Report_3.pdf"],"agent":{"family":"other-ai","model":"alpaca-7b","confidence":"high","role":"subject"}},{"value":0.636,"contributor":"cerebras-gpt-7b","contributorKind":"model","submissionUrls":["https://www.mosaicml.com/blog/mpt-7b"],"agent":{"family":"other-ai","model":"cerebras-gpt-7b","confidence":"high","role":"subject"}},{"value":0.635,"contributor":"cerebras-gpt-13b","contributorKind":"model","submissionUrls":["https://www.mosaicml.com/blog/mpt-7b"],"agent":{"family":"other-ai","model":"cerebras-gpt-13b","confidence":"high","role":"subject"}},{"value":0.808,"contributor":"chinchilla-70b","contributorKind":"model","submissionUrls":["https://arxiv.org/abs/2203.15556v1"],"agent":{"family":"google","model":"chinchilla-70b","confidence":"high","role":"subject"}},{"value":0.71,"contributor":"dolly-v2-12b","contributorKind":"model","submissionUrls":["https://gpt4all.io/reports/GPT4All_Technical_Report_3.pdf"],"agent":{"family":"other-ai","model":"dolly-v2-12b","confidence":"high","role":"subject"}},{"value":0.667,"contributor":"eleuther-pythia-7b","contributorKind":"model","submissionUrls":["https://www.mosaicml.com/blog/mpt-7b"],"agent":{"family":"other-ai","model":"pythia-7b","confidence":"high","role":"author"}},{"value":0.704,"contributor":"eleuther-pythia-12b","contributorKind":"model","submissionUrls":["https://www.mosaicml.com/blog/mpt-7b"],"agent":{"family":"other-ai","model":"pythia-12b","confidence":"high","role":"author"}},{"value":0.789,"contributor":"gpt-3-175b / davinci","contributorKind":"model","submissionUrls":["https://arxiv.org/abs/2005.14165"],"agent":{"family":"openai","model":"davinci","confidence":"high","role":"subject"}},{"value":0.834,"contributor":"gpt-3.5-175b / text-davinci-003","contributorKind":"model","submissionUrls":["https://gpt4all.io/reports/GPT4All_Technical_Report_3.pdf"],"agent":{"family":"openai","model":"text-davinci-003","confidence":"high","role":"subject"}},{"value":0.75,"contributor":"gpt4all-13b-snoozy","contributorKind":"model","submissionUrls":["https://gpt4all.io/reports/GPT4All_Technical_Report_3.pdf"],"agent":{"family":"other-ai","model":"gpt4all-13b-snoozy","confidence":"high","role":"subject"}},{"value":0.719,"contributor":"gpt-neox-20b","contributorKind":"model","submissionUrls":["https://www.mosaicml.com/blog/mpt-7b"],"agent":{"family":"other-ai","model":"gpt-neox-20b","confidence":"high","role":"subject"}},{"value":0.683,"contributor":"gpt-j-6b","contributorKind":"model","submissionUrls":["https://www.mosaicml.com/blog/mpt-7b"],"agent":{"family":"other-ai","model":"gpt-j-6b","confidence":"high","role":"subject"}},{"value":0.726,"contributor":"koala-13b","contributorKind":"model","submissionUrls":["https://gpt4all.io/reports/GPT4All_Technical_Report_3.pdf"],"agent":{"family":"other-ai","model":"koala-13b","confidence":"high","role":"subject"}},{"value":0.738,"contributor":"llama-7b","contributorKind":"model","submissionUrls":["https://www.mosaicml.com/blog/mpt-7b"],"agent":{"family":"meta","model":"llama-7b","confidence":"high","role":"subject"}},{"value":0.792,"contributor":"llama-13b","contributorKind":"model","submissionUrls":["https://arxiv.org/abs/2302.13971"],"agent":{"family":"meta","model":"llama-13b","confidence":"high","role":"subject"}},{"value":0.828,"contributor":"llama-33b","contributorKind":"model","submissionUrls":["https://arxiv.org/abs/2302.13971"],"agent":{"family":"meta","model":"llama-33b","confidence":"high","role":"subject"}},{"value":0.842,"contributor":"llama-65b","contributorKind":"model","submissionUrls":["https://arxiv.org/abs/2302.13971"],"isCurrentBest":true,"agent":{"family":"meta","model":"llama-65b","confidence":"high","role":"subject"}},{"value":0.761,"contributor":"mpt-7b","contributorKind":"model","submissionUrls":["https://www.mosaicml.com/blog/mpt-7b"],"agent":{"family":"other-ai","model":"mpt-7b","confidence":"high","role":"subject"}},{"value":0.681,"contributor":"oasst-pythia-12b","contributorKind":"model","submissionUrls":["https://gpt4all.io/reports/GPT4All_Technical_Report_3.pdf"],"agent":{"family":"other-ai","model":"oasst-pythia-12b","confidence":"high","role":"subject"}},{"value":0.677,"contributor":"opt-7b","contributorKind":"model","submissionUrls":["https://www.mosaicml.com/blog/mpt-7b"],"agent":{"family":"meta","model":"opt-7b","confidence":"high","role":"subject"}},{"value":0.692,"contributor":"opt-13b","contributorKind":"model","submissionUrls":["https://www.mosaicml.com/blog/mpt-7b"],"agent":{"family":"meta","model":"opt-13b","confidence":"high","role":"subject"}},{"value":0.834,"contributor":"palm-540b","contributorKind":"model","submissionUrls":["https://arxiv.org/abs/2204.02311v5"],"agent":{"family":"google","model":"palm-540b","confidence":"high","role":"subject"}},{"value":0.412,"contributor":"stablelm-base-alpha-7b","contributorKind":"model","submissionUrls":["https://gpt4all.io/reports/GPT4All_Technical_Report_3.pdf"],"agent":{"family":"other-ai","model":"stablelm-base-alpha-7b","confidence":"high","role":"subject"}},{"value":0.536,"contributor":"stablelm-tuned-alpha-7b","contributorKind":"model","submissionUrls":["https://gpt4all.io/reports/GPT4All_Technical_Report_3.pdf"],"agent":{"family":"other-ai","model":"stablelm-tuned-alpha-7b","confidence":"high","role":"subject"}}],"isOpen":false,"contested":true},{"id":"hellaswag-one-shot","name":"HellaSwag (one-shot)","description":"HellaSwag commonsense reasoning benchmark evaluated in a one-shot setting.","metricName":"accuracy","metricDirection":"maximize","records":[{"value":0.836,"contributor":"palm-540b","contributorKind":"model","submissionUrls":["https://ai.google/static/documents/palm2techreport.pdf"],"agent":{"family":"google","model":"palm-540b","confidence":"high","role":"subject"}},{"value":0.82,"contributor":"palm-2-s","contributorKind":"model","submissionUrls":["https://ai.google/static/documents/palm2techreport.pdf"],"agent":{"family":"google","model":"palm-2-s","confidence":"high","role":"subject"}},{"value":0.84,"contributor":"palm-2-m","contributorKind":"model","submissionUrls":["https://ai.google/static/documents/palm2techreport.pdf"],"agent":{"family":"google","model":"palm-2-m","confidence":"high","role":"subject"}},{"value":0.868,"contributor":"palm-2-l","contributorKind":"model","submissionUrls":["https://ai.google/static/documents/palm2techreport.pdf"],"isCurrentBest":true,"agent":{"family":"google","model":"palm-2-l","confidence":"high","role":"subject"}}],"isOpen":false,"contested":true},{"id":"humaneval-python-pass1","name":"HumanEval-Python (pass@1)","description":"Measures functional correctness for synthesizing Python programs from docstrings.","metricName":"pass@1","metricDirection":"maximize","records":[{"value":0.155,"contributor":"bloom-176b","contributorKind":"model","submissionUrls":["https://huggingface.co/bigscience/bloom#results"],"agent":{"family":"other-ai","model":"bloom-176b","confidence":"high","role":"subject"}},{"value":0.317,"contributor":"codex-12b / code-cushman-001","contributorKind":"model","submissionUrls":["https://crfm.stanford.edu/helm/latest/?group=targeted_evaluations"],"agent":{"family":"openai","model":"code-cushman-001","confidence":"high","role":"subject"}},{"value":0.293,"contributor":"codegen-16B-mono","contributorKind":"model","submissionUrls":["https://drive.google.com/file/d/1cN-b9GnWtHzQRoE7M7gAEyivY0kl4BYs/view"],"agent":{"family":"other-ai","model":"codegen-16B-mono","confidence":"high","role":"subject"}},{"value":0.183,"contributor":"codegen-16B-multi","contributorKind":"model","submissionUrls":["https://drive.google.com/file/d/1cN-b9GnWtHzQRoE7M7gAEyivY0kl4BYs/view"],"agent":{"family":"other-ai","model":"codegen-16B-multi","confidence":"high","role":"subject"}},{"value":0.229,"contributor":"codegx-13b","contributorKind":"model","submissionUrls":["https://drive.google.com/file/d/1cN-b9GnWtHzQRoE7M7gAEyivY0kl4BYs/view"],"agent":{"family":"other-ai","model":"codegx-13b","confidence":"high","role":"subject"}},{"value":0.481,"contributor":"gpt-3.5-175b / text-davinci-003","contributorKind":"model","submissionUrls":["https://arxiv.org/abs/2303.08774v3"],"agent":{"family":"openai","model":"text-davinci-003","confidence":"high","role":"subject"}},{"value":0.463,"contributor":"gpt-3.5-175b / code-davinci-002","contributorKind":"model","submissionUrls":["https://crfm.stanford.edu/helm/latest/?group=targeted_evaluations"],"agent":{"family":"openai","model":"code-davinci-002","confidence":"high","role":"subject"}},{"value":0.67,"contributor":"gpt-4","contributorKind":"model","submissionUrls":["https://arxiv.org/abs/2303.08774v3"],"isCurrentBest":true,"agent":{"family":"openai","model":"gpt-4","confidence":"high","role":"subject"}},{"value":0.105,"contributor":"llama-7b","contributorKind":"model","submissionUrls":["https://drive.google.com/file/d/1cN-b9GnWtHzQRoE7M7gAEyivY0kl4BYs/view"],"agent":{"family":"meta","model":"llama-7b","confidence":"high","role":"subject"}},{"value":0.158,"contributor":"llama-13b","contributorKind":"model","submissionUrls":["https://drive.google.com/file/d/1cN-b9GnWtHzQRoE7M7gAEyivY0kl4BYs/view"],"agent":{"family":"meta","model":"llama-13b","confidence":"high","role":"subject"}},{"value":0.217,"contributor":"llama-33b","contributorKind":"model","submissionUrls":["https://drive.google.com/file/d/1cN-b9GnWtHzQRoE7M7gAEyivY0kl4BYs/view"],"agent":{"family":"meta","model":"llama-33b","confidence":"high","role":"subject"}},{"value":0.237,"contributor":"llama-65b","contributorKind":"model","submissionUrls":["https://drive.google.com/file/d/1cN-b9GnWtHzQRoE7M7gAEyivY0kl4BYs/view"],"agent":{"family":"meta","model":"llama-65b","confidence":"high","role":"subject"}},{"value":0.262,"contributor":"palm-540b","contributorKind":"model","submissionUrls":["https://drive.google.com/file/d/1cN-b9GnWtHzQRoE7M7gAEyivY0kl4BYs/view"],"agent":{"family":"google","model":"palm-540b","confidence":"high","role":"subject"}},{"value":0.359,"contributor":"palm-coder-540b","contributorKind":"model","submissionUrls":["https://ai.google/static/documents/palm2techreport.pdf"],"agent":{"family":"google","model":"palm-coder-540b","confidence":"high","role":"subject"}},{"value":0.376,"contributor":"palm-2-s*","contributorKind":"model","submissionUrls":["https://ai.google/static/documents/palm2techreport.pdf"],"agent":{"family":"google","model":"palm-2-s*","confidence":"high","role":"subject"}},{"value":0.219,"contributor":"replit-code-v1-3b","contributorKind":"model","submissionUrls":["https://twitter.com/amasad/status/1651019556423598081/photo/2"],"agent":{"family":"other-ai","model":"replit-code-v1-3b","confidence":"high","role":"subject"}},{"value":0.304,"contributor":"starcoder-base-16b","contributorKind":"model","submissionUrls":["https://drive.google.com/file/d/1cN-b9GnWtHzQRoE7M7gAEyivY0kl4BYs/view"],"agent":{"family":"other-ai","model":"starcoder-base-16b","confidence":"high","role":"subject"}},{"value":0.336,"contributor":"starcoder-16b","contributorKind":"model","submissionUrls":["https://drive.google.com/file/d/1cN-b9GnWtHzQRoE7M7gAEyivY0kl4BYs/view"],"agent":{"family":"other-ai","model":"starcoder-16b","confidence":"high","role":"subject"}}],"isOpen":false,"contested":true},{"id":"lambada-zero-shot","name":"LAMBADA (zero-shot)","description":"Evaluates capabilities for text understanding by means of a word prediction task in a zero-shot setting.","metricName":"accuracy","metricDirection":"maximize","records":[{"value":0.636,"contributor":"cerebras-gpt-7b","contributorKind":"model","submissionUrls":["https://www.mosaicml.com/blog/mpt-7b"],"agent":{"family":"other-ai","model":"cerebras-gpt-7b","confidence":"high","role":"subject"}},{"value":0.635,"contributor":"cerebras-gpt-13b","contributorKind":"model","submissionUrls":["https://www.mosaicml.com/blog/mpt-7b"],"agent":{"family":"other-ai","model":"cerebras-gpt-13b","confidence":"high","role":"subject"}},{"value":0.774,"contributor":"chinchilla-70b","contributorKind":"model","submissionUrls":["https://arxiv.org/abs/2203.15556v1"],"agent":{"family":"google","model":"chinchilla-70b","confidence":"high","role":"subject"}},{"value":0.667,"contributor":"eleuther-pythia-7b","contributorKind":"model","submissionUrls":["https://www.mosaicml.com/blog/mpt-7b"],"agent":{"family":"other-ai","model":"pythia-7b","confidence":"high","role":"author"}},{"value":0.704,"contributor":"eleuther-pythia-12b","contributorKind":"model","submissionUrls":["https://www.mosaicml.com/blog/mpt-7b"],"agent":{"family":"other-ai","model":"pythia-12b","confidence":"high","role":"author"}},{"value":0.762,"contributor":"gpt-3.5-175b / text-davinci-003","contributorKind":"model","submissionUrls":["https://arxiv.org/abs/2303.08774v3"],"agent":{"family":"openai","model":"text-davinci-003","confidence":"high","role":"subject"}},{"value":0.719,"contributor":"gpt-neox-20b","contributorKind":"model","submissionUrls":["https://www.mosaicml.com/blog/mpt-7b"],"agent":{"family":"other-ai","model":"gpt-neox-20b","confidence":"high","role":"subject"}},{"value":0.683,"contributor":"gpt-j-6b","contributorKind":"model","submissionUrls":["https://www.mosaicml.com/blog/mpt-7b"],"agent":{"family":"other-ai","model":"gpt-j-6b","confidence":"high","role":"subject"}},{"value":0.738,"contributor":"llama-7b","contributorKind":"model","submissionUrls":["https://www.mosaicml.com/blog/mpt-7b"],"agent":{"family":"meta","model":"llama-7b","confidence":"high","role":"subject"}},{"value":0.677,"contributor":"opt-7b","contributorKind":"model","submissionUrls":["https://www.mosaicml.com/blog/mpt-7b"],"agent":{"family":"meta","model":"opt-7b","confidence":"high","role":"subject"}},{"value":0.692,"contributor":"opt-13b","contributorKind":"model","submissionUrls":["https://www.mosaicml.com/blog/mpt-7b"],"agent":{"family":"meta","model":"opt-13b","confidence":"high","role":"subject"}},{"value":0.779,"contributor":"palm-540b","contributorKind":"model","submissionUrls":["https://arxiv.org/abs/2204.02311v5"],"isCurrentBest":true,"agent":{"family":"google","model":"palm-540b","confidence":"high","role":"subject"}},{"value":0.533,"contributor":"stablelm-base-alpha-7b","contributorKind":"model","submissionUrls":["https://www.mosaicml.com/blog/mpt-7b"],"agent":{"family":"other-ai","model":"stablelm-base-alpha-7b","confidence":"high","role":"subject"}}],"isOpen":false,"contested":true},{"id":"lambada-one-shot","name":"LAMBADA (one-shot)","description":"Evaluates capabilities for text understanding by means of a word prediction task in a one-shot setting.","metricName":"accuracy","metricDirection":"maximize","records":[{"value":0.818,"contributor":"palm-540b","contributorKind":"model","submissionUrls":["https://ai.google/static/documents/palm2techreport.pdf"],"agent":{"family":"google","model":"palm-540b","confidence":"high","role":"subject"}},{"value":0.807,"contributor":"palm-2-s","contributorKind":"model","submissionUrls":["https://ai.google/static/documents/palm2techreport.pdf"],"agent":{"family":"google","model":"palm-2-s","confidence":"high","role":"subject"}},{"value":0.837,"contributor":"palm-2-m","contributorKind":"model","submissionUrls":["https://ai.google/static/documents/palm2techreport.pdf"],"agent":{"family":"google","model":"palm-2-m","confidence":"high","role":"subject"}},{"value":0.869,"contributor":"palm-2-l","contributorKind":"model","submissionUrls":["https://ai.google/static/documents/palm2techreport.pdf"],"isCurrentBest":true,"agent":{"family":"google","model":"palm-2-l","confidence":"high","role":"subject"}}],"isOpen":false,"contested":true},{"id":"mmlu-zero-shot","name":"MMLU (zero-shot)","description":"Multi-task Language Understanding benchmark covering 57 subjects in a zero-shot setting.","metricName":"accuracy","metricDirection":"maximize","records":[{"value":0.259,"contributor":"cerebras-gpt-7b","contributorKind":"model","submissionUrls":["https://www.mosaicml.com/blog/mpt-7b"],"agent":{"family":"other-ai","model":"cerebras-gpt-7b","confidence":"high","role":"subject"}},{"value":0.258,"contributor":"cerebras-gpt-13b","contributorKind":"model","submissionUrls":["https://www.mosaicml.com/blog/mpt-7b"],"agent":{"family":"other-ai","model":"cerebras-gpt-13b","confidence":"high","role":"subject"}},{"value":0.265,"contributor":"eleuther-pythia-7b","contributorKind":"model","submissionUrls":["https://www.mosaicml.com/blog/mpt-7b"],"agent":{"family":"other-ai","model":"pythia-7b","confidence":"high","role":"author"}},{"value":0.253,"contributor":"eleuther-pythia-12b","contributorKind":"model","submissionUrls":["https://www.mosaicml.com/blog/mpt-7b"],"agent":{"family":"other-ai","model":"pythia-12b","confidence":"high","role":"author"}},{"value":0.526,"contributor":"gal-120b","contributorKind":"model","submissionUrls":["https://paperswithcode.com/paper/galactica-a-large-language-model-for-science-1"],"isCurrentBest":true,"agent":{"family":"meta","model":"galactica-120b","confidence":"high","role":"author"}},{"value":0.269,"contributor":"gpt-neox-20b","contributorKind":"model","submissionUrls":["https://www.mosaicml.com/blog/mpt-7b"],"agent":{"family":"other-ai","model":"gpt-neox-20b","confidence":"high","role":"subject"}},{"value":0.261,"contributor":"gpt-j-6b","contributorKind":"model","submissionUrls":["https://www.mosaicml.com/blog/mpt-7b"],"agent":{"family":"other-ai","model":"gpt-j-6b","confidence":"high","role":"subject"}},{"value":0.302,"contributor":"llama-7b","contributorKind":"model","submissionUrls":["https://www.mosaicml.com/blog/mpt-7b"],"agent":{"family":"meta","model":"llama-7b","confidence":"high","role":"subject"}},{"value":0.296,"contributor":"mpt-7b","contributorKind":"model","submissionUrls":["https://www.mosaicml.com/blog/mpt-7b"],"agent":{"family":"other-ai","model":"mpt-7b","confidence":"high","role":"subject"}},{"value":0.251,"contributor":"opt-7b","contributorKind":"model","submissionUrls":["https://www.mosaicml.com/blog/mpt-7b"],"agent":{"family":"meta","model":"opt-7b","confidence":"high","role":"subject"}},{"value":0.257,"contributor":"opt-13b","contributorKind":"model","submissionUrls":["https://www.mosaicml.com/blog/mpt-7b"],"agent":{"family":"meta","model":"opt-13b","confidence":"high","role":"subject"}},{"value":0.251,"contributor":"stablelm-base-alpha-7b","contributorKind":"model","submissionUrls":["https://www.mosaicml.com/blog/mpt-7b"],"agent":{"family":"other-ai","model":"stablelm-base-alpha-7b","confidence":"high","role":"subject"}}],"isOpen":false,"contested":true},{"id":"mmlu-few-shot","name":"MMLU (few-shot)","description":"Multi-task Language Understanding benchmark covering 57 subjects in a few-shot setting.","metricName":"accuracy","metricDirection":"maximize","records":[{"value":0.675,"contributor":"chinchilla-70b","contributorKind":"model","submissionUrls":["https://arxiv.org/abs/2203.15556v1"],"agent":{"family":"google","model":"chinchilla-70b","confidence":"high","role":"subject"}},{"value":0.35,"contributor":"falcon-7b","contributorKind":"model","submissionUrls":["https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard"],"agent":{"family":"other-ai","model":"falcon-7b","confidence":"high","role":"subject"}},{"value":0.527,"contributor":"falcon-40b","contributorKind":"model","submissionUrls":["https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard"],"agent":{"family":"other-ai","model":"falcon-40b","confidence":"high","role":"subject"}},{"value":0.243,"contributor":"gpt-3-7b / curie","contributorKind":"model","submissionUrls":["https://crfm.stanford.edu/helm/latest/?group=core_scenarios"],"agent":{"family":"openai","model":"curie","confidence":"high","role":"subject"}},{"value":0.439,"contributor":"gpt-3-175b / davinci","contributorKind":"model","submissionUrls":["https://arxiv.org/abs/2005.14165"],"agent":{"family":"openai","model":"davinci","confidence":"high","role":"subject"}},{"value":0.569,"contributor":"gpt-3.5-175b / text-davinci-003","contributorKind":"model","submissionUrls":["https://crfm.stanford.edu/helm/latest/?group=core_scenarios"],"agent":{"family":"openai","model":"text-davinci-003","confidence":"high","role":"subject"}},{"value":0.864,"contributor":"gpt-4","contributorKind":"model","submissionUrls":["https://arxiv.org/abs/2303.08774v3"],"isCurrentBest":true,"agent":{"family":"openai","model":"gpt-4","confidence":"high","role":"subject"}},{"value":0.634,"contributor":"llama-65b","contributorKind":"model","submissionUrls":["https://arxiv.org/abs/2302.13971v1"],"agent":{"family":"meta","model":"llama-65b","confidence":"high","role":"subject"}},{"value":0.698,"contributor":"llama-2-70b","contributorKind":"model","submissionUrls":["https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard"],"agent":{"family":"meta","model":"llama-2-70b","confidence":"high","role":"subject"}},{"value":0.276,"contributor":"opt-66b","contributorKind":"model","submissionUrls":["https://crfm.stanford.edu/helm/latest/?group=core_scenarios"],"agent":{"family":"meta","model":"opt-66b","confidence":"high","role":"subject"}},{"value":0.318,"contributor":"opt-175b","contributorKind":"model","submissionUrls":["https://crfm.stanford.edu/helm/latest/?group=core_scenarios"],"agent":{"family":"meta","model":"opt-175b","confidence":"high","role":"subject"}},{"value":0.693,"contributor":"palm-540b","contributorKind":"model","submissionUrls":["https://arxiv.org/abs/2204.02311v5"],"agent":{"family":"google","model":"palm-540b","confidence":"high","role":"subject"}}],"isOpen":false,"contested":true},{"id":"triviaqa-zero-shot","name":"TriviaQA (zero-shot)","description":"Reading comprehension dataset containing question-answer-evidence triples evaluated in a zero-shot setting.","metricName":"accuracy","metricDirection":"maximize","records":[{"value":0.276,"contributor":"gpt-neox-20b","contributorKind":"model","submissionUrls":["https://crfm.stanford.edu/helm/latest/?group=core_scenarios"],"isCurrentBest":true,"agent":{"family":"other-ai","model":"gpt-neox-20b","confidence":"high","role":"subject"}},{"value":0.249,"contributor":"gpt-j-6b","contributorKind":"model","submissionUrls":["https://crfm.stanford.edu/helm/latest/?group=core_scenarios"],"agent":{"family":"other-ai","model":"gpt-j-6b","confidence":"high","role":"subject"}}],"isOpen":false,"contested":true},{"id":"triviaqa-one-shot","name":"TriviaQA (one-shot)","description":"Reading comprehension dataset containing question-answer-evidence triples evaluated in a one-shot setting.","metricName":"accuracy","metricDirection":"maximize","records":[{"value":0.141,"contributor":"cerebras-gpt-7b","contributorKind":"model","submissionUrls":["https://www.mosaicml.com/blog/mpt-7b"],"agent":{"family":"other-ai","model":"cerebras-gpt-7b","confidence":"high","role":"subject"}},{"value":0.146,"contributor":"cerebras-gpt-13b","contributorKind":"model","submissionUrls":["https://www.mosaicml.com/blog/mpt-7b"],"agent":{"family":"other-ai","model":"cerebras-gpt-13b","confidence":"high","role":"subject"}},{"value":0.198,"contributor":"eleuther-pythia-7b","contributorKind":"model","submissionUrls":["https://www.mosaicml.com/blog/mpt-7b"],"agent":{"family":"other-ai","model":"pythia-7b","confidence":"high","role":"author"}},{"value":0.233,"contributor":"eleuther-pythia-12b","contributorKind":"model","submissionUrls":["https://www.mosaicml.com/blog/mpt-7b"],"agent":{"family":"other-ai","model":"pythia-12b","confidence":"high","role":"author"}},{"value":0.347,"contributor":"gpt-neox-20b","contributorKind":"model","submissionUrls":["https://www.mosaicml.com/blog/mpt-7b"],"agent":{"family":"other-ai","model":"gpt-neox-20b","confidence":"high","role":"subject"}},{"value":0.234,"contributor":"gpt-j-6b","contributorKind":"model","submissionUrls":["https://www.mosaicml.com/blog/mpt-7b"],"agent":{"family":"other-ai","model":"gpt-j-6b","confidence":"high","role":"subject"}},{"value":0.443,"contributor":"llama-7b","contributorKind":"model","submissionUrls":["https://www.mosaicml.com/blog/mpt-7b"],"agent":{"family":"meta","model":"llama-7b","confidence":"high","role":"subject"}},{"value":0.343,"contributor":"mpt-7b","contributorKind":"model","submissionUrls":["https://www.mosaicml.com/blog/mpt-7b"],"agent":{"family":"other-ai","model":"mpt-7b","confidence":"high","role":"subject"}},{"value":0.227,"contributor":"opt-7b","contributorKind":"model","submissionUrls":["https://www.mosaicml.com/blog/mpt-7b"],"agent":{"family":"meta","model":"opt-7b","confidence":"high","role":"subject"}},{"value":0.282,"contributor":"opt-13b","contributorKind":"model","submissionUrls":["https://www.mosaicml.com/blog/mpt-7b"],"agent":{"family":"meta","model":"opt-13b","confidence":"high","role":"subject"}},{"value":0.752,"contributor":"palm-2-s","contributorKind":"model","submissionUrls":["https://ai.google/static/documents/palm2techreport.pdf"],"agent":{"family":"google","model":"palm-2-s","confidence":"high","role":"subject"}},{"value":0.817,"contributor":"palm-2-m","contributorKind":"model","submissionUrls":["https://ai.google/static/documents/palm2techreport.pdf"],"agent":{"family":"google","model":"palm-2-m","confidence":"high","role":"subject"}},{"value":0.861,"contributor":"palm-2-l","contributorKind":"model","submissionUrls":["https://ai.google/static/documents/palm2techreport.pdf"],"isCurrentBest":true,"agent":{"family":"google","model":"palm-2-l","confidence":"high","role":"subject"}},{"value":0.049,"contributor":"stablelm-base-alpha-7b","contributorKind":"model","submissionUrls":["https://www.mosaicml.com/blog/mpt-7b"],"agent":{"family":"other-ai","model":"stablelm-base-alpha-7b","confidence":"high","role":"subject"}}],"isOpen":false,"contested":true},{"id":"winogrande-zero-shot","name":"WinoGrande (zero-shot)","description":"WinoGrande pronoun resolution benchmark evaluated in a zero-shot setting.","metricName":"accuracy","metricDirection":"maximize","records":[{"value":0.661,"contributor":"alpaca-7b","contributorKind":"model","submissionUrls":["https://gpt4all.io/reports/GPT4All_Technical_Report_3.pdf"],"agent":{"family":"other-ai","model":"alpaca-7b","confidence":"high","role":"subject"}},{"value":0.749,"contributor":"chinchilla-70b","contributorKind":"model","submissionUrls":["https://arxiv.org/abs/2203.15556v1"],"agent":{"family":"google","model":"chinchilla-70b","confidence":"high","role":"subject"}},{"value":0.622,"contributor":"dolly-v2-12b","contributorKind":"model","submissionUrls":["https://gpt4all.io/reports/GPT4All_Technical_Report_3.pdf"],"agent":{"family":"other-ai","model":"dolly-v2-12b","confidence":"high","role":"subject"}},{"value":0.661,"contributor":"eleuther-pythia-7b","contributorKind":"model","submissionUrls":["https://gpt4all.io/reports/GPT4All_Technical_Report_3.pdf"],"agent":{"family":"other-ai","model":"pythia-7b","confidence":"high","role":"author"}},{"value":0.638,"contributor":"eleuther-pythia-12b","contributorKind":"model","submissionUrls":["https://gpt4all.io/reports/GPT4All_Technical_Report_3.pdf"],"agent":{"family":"other-ai","model":"pythia-12b","confidence":"high","role":"author"}},{"value":0.702,"contributor":"gpt-3-175b / davinci","contributorKind":"model","submissionUrls":["https://arxiv.org/abs/2005.14165v4"],"agent":{"family":"openai","model":"davinci","confidence":"high","role":"subject"}},{"value":0.758,"contributor":"gpt-3.5-175b / text-davinci-003","contributorKind":"model","submissionUrls":["https://gpt4all.io/reports/GPT4All_Technical_Report_3.pdf"],"agent":{"family":"openai","model":"text-davinci-003","confidence":"high","role":"subject"}},{"value":0.713,"contributor":"gpt4all-13b-snoozy","contributorKind":"model","submissionUrls":["https://gpt4all.io/reports/GPT4All_Technical_Report_3.pdf"],"agent":{"family":"other-ai","model":"gpt4all-13b-snoozy","confidence":"high","role":"subject"}},{"value":0.688,"contributor":"koala-13b","contributorKind":"model","submissionUrls":["https://gpt4all.io/reports/GPT4All_Technical_Report_3.pdf"],"agent":{"family":"other-ai","model":"koala-13b","confidence":"high","role":"subject"}},{"value":0.701,"contributor":"llama-7b","contributorKind":"model","submissionUrls":["https://arxiv.org/abs/2302.13971v1"],"agent":{"family":"meta","model":"llama-7b","confidence":"high","role":"subject"}},{"value":0.73,"contributor":"llama-13b","contributorKind":"model","submissionUrls":["https://arxiv.org/abs/2302.13971v1"],"agent":{"family":"meta","model":"llama-13b","confidence":"high","role":"subject"}},{"value":0.76,"contributor":"llama-33b","contributorKind":"model","submissionUrls":["https://arxiv.org/abs/2302.13971v1"],"agent":{"family":"meta","model":"llama-33b","confidence":"high","role":"subject"}},{"value":0.77,"contributor":"llama-65b","contributorKind":"model","submissionUrls":["https://arxiv.org/abs/2302.13971v1"],"agent":{"family":"meta","model":"llama-65b","confidence":"high","role":"subject"}},{"value":0.65,"contributor":"oasst-pythia-12b","contributorKind":"model","submissionUrls":["https://gpt4all.io/reports/GPT4All_Technical_Report_3.pdf"],"agent":{"family":"other-ai","model":"oasst-pythia-12b","confidence":"high","role":"subject"}},{"value":0.77,"contributor":"palm-62b","contributorKind":"model","submissionUrls":["https://arxiv.org/abs/2204.02311"],"agent":{"family":"google","model":"palm-62b","confidence":"high","role":"subject"}},{"value":0.811,"contributor":"palm-540b","contributorKind":"model","submissionUrls":["https://arxiv.org/abs/2204.02311"],"isCurrentBest":true,"agent":{"family":"google","model":"palm-540b","confidence":"high","role":"subject"}},{"value":0.501,"contributor":"stablelm-base-alpha-7b","contributorKind":"model","submissionUrls":["https://gpt4all.io/reports/GPT4All_Technical_Report_3.pdf"],"agent":{"family":"other-ai","model":"stablelm-base-alpha-7b","confidence":"high","role":"subject"}},{"value":0.548,"contributor":"stablelm-tuned-alpha-7b","contributorKind":"model","submissionUrls":["https://gpt4all.io/reports/GPT4All_Technical_Report_3.pdf"],"agent":{"family":"other-ai","model":"stablelm-tuned-alpha-7b","confidence":"high","role":"subject"}}],"isOpen":false,"contested":true},{"id":"winogrande-one-shot","name":"WinoGrande (one-shot)","description":"WinoGrande pronoun resolution benchmark evaluated in a one-shot setting.","metricName":"accuracy","metricDirection":"maximize","records":[{"value":0.837,"contributor":"palm-540b","contributorKind":"model","submissionUrls":["https://ai.google/static/documents/palm2techreport.pdf"],"isCurrentBest":true,"agent":{"family":"google","model":"palm-540b","confidence":"high","role":"subject"}},{"value":0.779,"contributor":"palm-2-s","contributorKind":"model","submissionUrls":["https://ai.google/static/documents/palm2techreport.pdf"],"agent":{"family":"google","model":"palm-2-s","confidence":"high","role":"subject"}},{"value":0.792,"contributor":"palm-2-m","contributorKind":"model","submissionUrls":["https://ai.google/static/documents/palm2techreport.pdf"],"agent":{"family":"google","model":"palm-2-m","confidence":"high","role":"subject"}},{"value":0.83,"contributor":"palm-2-l","contributorKind":"model","submissionUrls":["https://ai.google/static/documents/palm2techreport.pdf"],"agent":{"family":"google","model":"palm-2-l","confidence":"high","role":"subject"}}],"isOpen":false,"contested":true},{"id":"winogrande-few-shot","name":"WinoGrande (few-shot)","description":"WinoGrande pronoun resolution benchmark evaluated in a few-shot setting.","metricName":"accuracy","metricDirection":"maximize","records":[{"value":0.816,"contributor":"gpt-3.5-175b / text-davinci-003","contributorKind":"model","submissionUrls":["https://arxiv.org/abs/2303.08774v3"],"agent":{"family":"openai","model":"text-davinci-003","confidence":"high","role":"subject"}},{"value":0.875,"contributor":"gpt-4","contributorKind":"model","submissionUrls":["https://arxiv.org/abs/2303.08774v3"],"agent":{"family":"openai","model":"gpt-4","confidence":"high","role":"subject"}},{"value":0.851,"contributor":"palm-540b","contributorKind":"model","submissionUrls":["https://arxiv.org/abs/2204.02311"],"agent":{"family":"google","model":"palm-540b","confidence":"high","role":"subject"}},{"value":0.909,"contributor":"palm-2-l-instruct","contributorKind":"model","submissionUrls":["https://ai.google/static/documents/palm2techreport.pdf"],"isCurrentBest":true,"agent":{"family":"google","model":"palm-2-l-instruct","confidence":"high","role":"subject"}}],"isOpen":false,"contested":true}],"repo":{"owner":"LudwigStumpp","name":"llm-leaderboard","branch":"main"},"participants":[],"stats":{"totalRecords":151,"uniqueParticipants":0,"recordsLast30d":0,"recordsLast90d":0},"agentStats":[{"family":"meta","records":1,"currentBests":1,"models":["galactica-120b"],"contributors":["gal-120b"]},{"family":"other-ai","records":10,"currentBests":0,"models":["pythia-7b","pythia-12b"],"contributors":["eleuther-pythia-7b","eleuther-pythia-12b"]}],"lastUpdated":"2026-09-18T16:32:55.539Z"},{"id":"sutro-problems","name":"Sutro Problems","tagline":"Energy-efficient learning and computation benchmarks scored on physical data movement and hardware energy.","description":"A collection of small, self-contained problems used as benchmarks for energy-efficient learning research. Submissions are scored on physical data movement using a simplified Bill Dally model or measured on A100 GPUs. Run by the Sutro Group, these benchmarks aim to address the memory wall in modern AI.","url":"https://github.com/cybertronai/sutro-problems","host":"github","status":"active","tags":["energy-efficient","hardware-efficiency","matrix-multiplication","sparse-parity","mnist","dally-model"],"domain":"hardware-efficiency","images":["https://github.com/user-attachments/assets/dfab8831-876a-4cdd-bda4-79a7ef869152","https://github.com/user-attachments/assets/e4610f22-00e6-4d70-adee-1948f219cf51","https://github.com/user-attachments/assets/0d71bd80-7da8-4e0c-942d-7bd6b741a193","https://raw.githubusercontent.com/cybertronai/sutro-problems/main/sparse-parity/doc/task.svg"],"organizer":{"name":"Sutro Group","url":"https://docs.google.com/document/d/1B9867EN6Bg4ZVQK9vI_ZqykZ5HEtMAHJ7zBGGas4szQ/edit?tab=t.0#heading=h.j6rssh3enbtd","type":"community"},"participation":{"howToSubmit":"Open a pull request against main adding your source/generator, reproduction commands, and a standalone report under the problem's submissions/ directory, and update the leaderboard table.","submissionFormat":"An IR file (straight-line program), a Python generator script, and a markdown report.","requirements":["Python 3.10+","NumPy","SciPy"],"compute":"datacenter-gpu","computeDetails":"Dally model tracks require no compute (scored symbolically/analytically); MNIST tracks are evaluated on NVIDIA A100 GPUs.","deadline":"rolling","cost":"free","verification":"CI, symbolic check, or hardware energy measurement via NVML"},"quickstart":"import matmul\n\n# Verify your IR computes A @ B correctly and return its read-cost.\ncost = matmul.score_1x1(\"1,2;mul 3,1,2;3\")\n\nir = matmul.generate_baseline_4x4()\ncost = matmul.score_4x4(ir)","links":[{"label":"Sutro Group Document","url":"https://docs.google.com/document/d/1B9867EN6Bg4ZVQK9vI_ZqykZ5HEtMAHJ7zBGGas4szQ/edit?tab=t.0#heading=h.j6rssh3enbtd"},{"label":"Bill Dally's Model","url":"https://github.com/cybertronai/simplified-dally-model"}],"problems":[{"id":"mnist-medium-3","name":"MNIST-medium (3% error target)","group":"mnist","description":"Learn from 9x9 images and predict test digits with at most 3% error at the lowest A100 energy.","metricName":"energy","metricUnit":"mJ","metricDirection":"minimize","records":[],"isOpen":true,"contested":false},{"id":"mnist-medium-8","name":"MNIST-medium (8% error target)","group":"mnist","description":"Learn from 9x9 images and predict test digits with at most 8% error at the lowest A100 energy.","metricName":"energy","metricUnit":"mJ","metricDirection":"minimize","records":[],"isOpen":true,"contested":false},{"id":"mnist-original","name":"MNIST-original (1% test error target)","group":"mnist","description":"Learn from original 28x28 images and predict test digits with at most 1% error at the lowest A100 energy.","metricName":"energy","metricUnit":"mJ","metricDirection":"minimize","records":[],"isOpen":true,"contested":false},{"id":"mnist-small-67","name":"MNIST-small (67% accuracy target)","group":"mnist","description":"Learn from 3x3 images and predict test digits with at least 67% accuracy at the lowest A100 energy.","metricName":"energy","metricUnit":"mJ","metricDirection":"minimize","records":[{"date":"2026-09-11","value":3300,"contributor":"H32 MLP","contributorKind":"method","description":"Accuracy: 67.08% ± 1.54 pp, Time: 130 ms","agent":{"family":"openai","tool":"Codex","evidence":"Merge pull request #65 from cybertronai/codex/mnist-small-60-submission","confidence":"high","role":"author"}},{"date":"2026-09-14","value":2600,"contributor":"NR-K8 Adam MLP","contributorKind":"method","description":"Accuracy: 67.35% ± 1.76 pp, Time: 110 ms"},{"date":"2026-09-15","value":3200,"contributor":"Panel-cached H32 MLP (1,000/1,000)","contributorKind":"method","description":"Accuracy: 67.11% ± 1.63 pp, Time: 120 ms"},{"date":"2026-09-15","value":0.59,"contributor":"QDA","contributorKind":"method","description":"Accuracy: 67.95% ± 1.64 pp, Time: 0.017 ms","isCurrentBest":true,"agent":{"family":"anthropic","model":"Claude Fable 5.1","confidence":"high","evidence":"Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>","role":"author"}}],"isOpen":false,"contested":true},{"id":"mnist-medium-2","name":"MNIST-medium (2% error target)","group":"mnist","description":"Learn from 9x9 images and predict test digits with at most 2% error at the lowest A100 energy.","metricName":"energy","metricUnit":"mJ","metricDirection":"minimize","records":[{"date":"2026-09-16","value":65000,"contributor":"512-filter CG pair","contributorKind":"method","description":"Accuracy: 98.12% ± 0.15 pp (A100); 98.11% ± 0.14 pp (grid), Time: 260 ms","isCurrentBest":true}],"isOpen":false,"contested":false},{"id":"mnist-medium-5","name":"MNIST-medium (5% error target)","group":"mnist","description":"Learn from 9x9 images and predict test digits with at most 5% error at the lowest A100 energy.","metricName":"energy","metricUnit":"mJ","metricDirection":"minimize","records":[{"date":"2026-09-11","value":290000,"contributor":"512-unit MLP (96% target)","contributorKind":"method","description":"Accuracy: 96.41% ± 0.13 pp, Time: 9,300 ms","agent":{"family":"openai","tool":"Codex","evidence":"Merge pull request #66 from cybertronai/codex/mnist-medium-submission","confidence":"high","role":"author"}},{"date":"2026-09-15","value":174,"contributor":"PCA-QDA","contributorKind":"method","description":"Accuracy: 95.57% ± 0.17 pp, Time: 3.3 ms","isCurrentBest":true,"agent":{"family":"openai","tool":"Codex","confidence":"medium","evidence":"PR branch codex/pca-qda-energy-correction","role":"author"}}],"isOpen":false,"contested":true},{"id":"mnist-medium-12","name":"MNIST-medium (12% error target)","group":"mnist","description":"Learn from 9x9 images and predict test digits with at most 12% error at the lowest A100 energy.","metricName":"energy","metricUnit":"mJ","metricDirection":"minimize","records":[{"date":"2026-09-14","value":1000,"contributor":"Reversible MLP, smaller workspace (12% error target)","contributorKind":"method","description":"Accuracy: 89.28% ± 0.36 pp, Time: 39 ms","isCurrentBest":true,"agent":{"family":"openai","tool":"Codex","confidence":"medium","evidence":"PR branch codex/rev88-20260914","role":"author"}}],"isOpen":false,"contested":false},{"id":"mnist-small-historical","name":"MNIST-small (Historical)","group":"mnist","description":"Historical submissions for MNIST-small under the previous specification (600/600 sizes).","metricName":"energy","metricUnit":"mJ","metricDirection":"minimize","records":[{"date":"2026-09-12","value":1800,"contributor":"Panel-cached MLP (600/600)","contributorKind":"method","description":"Accuracy: 65.0% ± 2.1 pp, Time: 57 ms"},{"date":"2026-09-10","value":2000,"contributor":"32-unit MLP","contributorKind":"method","description":"Accuracy: 62%, Time: 71 ms","agent":{"family":"openai","tool":"Codex","confidence":"medium","evidence":"PR branch codex/mnist-small-60-submission","role":"author"}},{"date":"2026-09-10","value":0.52,"contributor":"1NN","contributorKind":"method","description":"Accuracy: 51%, Time: 0.0069 ms","isCurrentBest":true,"agent":{"family":"openai","tool":"Codex","confidence":"medium","evidence":"PR branch codex/mnist-small-1nn-submission","role":"author"}}],"isOpen":false,"contested":true},{"id":"mnist-medium-historical","name":"MNIST-medium (Historical)","group":"mnist","description":"Historical submissions for MNIST-medium under the previous specification (6,000/6,000 sizes).","metricName":"energy","metricUnit":"mJ","metricDirection":"minimize","records":[{"date":"2026-09-11","value":780000,"contributor":"Ordered ConvNets (3% error target)","contributorKind":"method","description":"Accuracy: 97.8% ± 0.1 pp, Time: 13,000 ms","agent":{"family":"openai","tool":"Codex","confidence":"medium","evidence":"PR branch codex/mnist-medium-four-percent-scored","role":"author"}},{"date":"2026-09-10","value":1700000,"contributor":"Three ConvNets","contributorKind":"method","description":"Accuracy: 98.1% ± 0.1 pp, Time: 5.9 × 10⁴ ms","agent":{"family":"openai","tool":"Codex","confidence":"medium","evidence":"PR branch codex/mnist-medium-convnet-submission","role":"author"}},{"date":"2026-09-10","value":150000,"contributor":"512-unit MLP","contributorKind":"method","description":"Accuracy: 96%, Time: 4,700 ms","isCurrentBest":true,"agent":{"family":"openai","tool":"Codex","confidence":"medium","evidence":"PR branch codex/mnist-medium-submission","role":"author"}}],"isOpen":false,"contested":true},{"id":"sparse-parity-20","name":"Sparse Parity (20% target)","group":"sparse-parity","description":"Recover the k secret bit positions at the lowest energy with at least 20% accuracy.","metricName":"cost","metricDirection":"minimize","records":[{"date":"2026-09-02","value":86753,"contributor":"b0nce","contributorKind":"person","description":"`generate_packed_static20()` (packed static information set)","isCurrentBest":true,"contributorUrl":"https://github.com/b0nce"},{"date":"2026-09-01","value":135348,"contributor":"jurajselep","contributorKind":"person","description":"`generate_packed_scan(1)` (packed-column scan + SSA layout)","contributorUrl":"https://github.com/jurajselep"},{"date":"2026-08-31","value":151117,"contributor":"npow","contributorKind":"person","description":"`generate_packed_sis(cap=2, seed=13, g2=8)` (packed SIS, partial cap-2 walk, exhaustive information-set tuning)","contributorUrl":"https://github.com/npow"},{"date":"2026-08-30","value":1317480,"contributor":"zh4ngx","contributorKind":"person","description":"`optimize_layout(generate_sis_mask(1, 2))` (static-IS walk, staged layout)","contributorUrl":"https://github.com/zh4ngx"},{"date":"2026-08-27","value":17331683,"contributor":"yaroslavvb","contributorKind":"person","description":"`generate_scan(127)` (Gray scan)","contributorUrl":"https://github.com/yaroslavvb"},{"date":"2026-08-26","value":12042480,"contributor":"yaroslavvb","contributorKind":"person","description":"`generate_isd_mask(8)` (ISD restarts)","contributorUrl":"https://github.com/yaroslavvb"}],"isOpen":false,"contested":true},{"id":"sparse-parity-40","name":"Sparse Parity (40% target)","group":"sparse-parity","description":"Recover the k secret bit positions at the lowest energy with at least 40% accuracy.","metricName":"cost","metricDirection":"minimize","records":[{"date":"2026-09-01","value":141218,"contributor":"npow","contributorKind":"person","description":"55 audited weight-2 states + compact flow + reverse route ★ best","isCurrentBest":true,"contributorUrl":"https://github.com/npow"},{"date":"2026-09-02","value":147000,"contributor":"b0nce","contributorKind":"person","description":"`generate_packed_route40()` (cap-2 Gray prefix)","contributorUrl":"https://github.com/b0nce"},{"date":"2026-09-01","value":151943,"contributor":"jurajselep","contributorKind":"person","description":"`generate_packed_scan(2)` (packed-column scan + SSA layout)","contributorUrl":"https://github.com/jurajselep"},{"date":"2026-08-31","value":163378,"contributor":"npow","contributorKind":"person","description":"`generate(1, 2, seed=5)` (bit-packed SIS walk, 3-phase layout, higher dev recovery)","contributorUrl":"https://github.com/npow"},{"date":"2026-08-30","value":1317480,"contributor":"zh4ngx","contributorKind":"person","description":"`optimize_layout(generate_sis_mask(1, 2))` (static-IS walk, staged layout)","contributorUrl":"https://github.com/zh4ngx"},{"date":"2026-08-28","value":17418235,"contributor":"zh4ngx","contributorKind":"person","description":"`generate_scan(0, walk=\"weight\", weight_cap=2)` (weight-ordered scan)","contributorUrl":"https://github.com/zh4ngx"},{"date":"2026-08-27","value":18764343,"contributor":"yaroslavvb","contributorKind":"person","description":"`generate_scan(1023)` (Gray scan)","contributorUrl":"https://github.com/yaroslavvb"},{"date":"2026-08-27","value":17945660,"contributor":"yaroslavvb","contributorKind":"person","description":"`generate_scan(511)` (Gray scan)","contributorUrl":"https://github.com/yaroslavvb"}],"isOpen":false,"contested":true},{"id":"sparse-parity-60","name":"Sparse Parity (60% target)","group":"sparse-parity","description":"Recover the k secret bit positions at the lowest energy with at least 60% accuracy.","metricName":"cost","metricDirection":"minimize","records":[{"date":"2026-09-01","value":149665,"contributor":"npow","contributorKind":"person","description":"19 audited weight-3 states + compact flow + reversed 2-opt walk ★ best","isCurrentBest":true,"contributorUrl":"https://github.com/npow"},{"date":"2026-09-02","value":176331,"contributor":"b0nce","contributorKind":"person","description":"`generate_packed_route60()` (cap-3 Gray prefix)","contributorUrl":"https://github.com/b0nce"},{"date":"2026-09-01","value":200937,"contributor":"jurajselep","contributorKind":"person","description":"`generate_packed_scan(3)` (packed-column scan + SSA layout)","contributorUrl":"https://github.com/jurajselep"},{"date":"2026-08-31","value":284049,"contributor":"npow","contributorKind":"person","description":"`generate_packed_sis(cap=3, seed=13)` (bit-packed SIS walk, full cap-3, tuned seed)","contributorUrl":"https://github.com/npow"},{"date":"2026-08-30","value":2137725,"contributor":"zh4ngx","contributorKind":"person","description":"`optimize_layout(generate_sis_mask(1, 3))` (static-IS walk, staged layout)","contributorUrl":"https://github.com/zh4ngx"},{"date":"2026-08-28","value":18509753,"contributor":"zh4ngx","contributorKind":"person","description":"`generate_scan(0, walk=\"weight\", weight_cap=3)` (weight-ordered scan)","contributorUrl":"https://github.com/zh4ngx"},{"date":"2026-08-27","value":26951367,"contributor":"yaroslavvb","contributorKind":"person","description":"`generate_scan(6143)` (Gray scan)","contributorUrl":"https://github.com/yaroslavvb"},{"date":"2026-08-26","value":23676539,"contributor":"yaroslavvb","contributorKind":"person","description":"`generate_scan(4095)` (Gray scan)","contributorUrl":"https://github.com/yaroslavvb"}],"isOpen":false,"contested":true},{"id":"sparse-parity-80","name":"Sparse Parity (80% target)","group":"sparse-parity","description":"Recover the k secret bit positions at the lowest energy with at least 80% accuracy.","metricName":"cost","metricDirection":"minimize","records":[{"date":"2026-09-01","value":182744,"contributor":"npow","contributorKind":"person","description":"269 audited weight-3 states + compact flow + reversed 2-opt walk ★ best","isCurrentBest":true,"contributorUrl":"https://github.com/npow"},{"date":"2026-09-02","value":196139,"contributor":"b0nce","contributorKind":"person","description":"`generate_packed_route80()` (cap-3 Gray prefix)","contributorUrl":"https://github.com/b0nce"},{"date":"2026-09-01","value":200937,"contributor":"jurajselep","contributorKind":"person","description":"`generate_packed_scan(3)` (packed-column scan + SSA layout)","contributorUrl":"https://github.com/jurajselep"},{"date":"2026-08-31","value":493193,"contributor":"npow","contributorKind":"person","description":"`generate_staged(weight_cap=3)` (septet-packed dynamic RREF + row-coordinate walk)","contributorUrl":"https://github.com/npow"},{"date":"2026-08-30","value":5593997,"contributor":"zh4ngx","contributorKind":"person","description":"`optimize_layout(generate_scan(0, walk=\"weight\", weight_cap=3))` (weight-ordered scan, staged layout)","contributorUrl":"https://github.com/zh4ngx"},{"date":"2026-08-27","value":33501030,"contributor":"yaroslavvb","contributorKind":"person","description":"`generate_scan(10239)` (Gray scan)","contributorUrl":"https://github.com/yaroslavvb"},{"date":"2026-08-27","value":30226172,"contributor":"yaroslavvb","contributorKind":"person","description":"`generate_scan(8191)` (Gray scan)","contributorUrl":"https://github.com/yaroslavvb"}],"isOpen":false,"contested":true},{"id":"sparse-parity-100","name":"Sparse Parity (100% target)","group":"sparse-parity","description":"Recover the k secret bit positions at the lowest energy with 100% accuracy.","metricName":"cost","metricDirection":"minimize","records":[{"date":"2026-09-02","value":392666,"contributor":"jurajselep, b0nce","contributorKind":"team","description":"`generate_packed_scan(5)` (packed RREF + specialized capture)","isCurrentBest":true},{"date":"2026-08-31","value":938331,"contributor":"npow","contributorKind":"person","description":"`generate_staged()` (septet-packed RREF + row-coordinate walk)","contributorUrl":"https://github.com/npow"},{"date":"2026-08-30","value":12461610,"contributor":"zh4ngx","contributorKind":"person","description":"`optimize_layout(generate_scan(0, walk=\"weight\", weight_cap=5))` (weight-ordered scan, staged layout)","contributorUrl":"https://github.com/zh4ngx"},{"date":"2026-08-26","value":43325468,"contributor":"yaroslavvb","contributorKind":"person","description":"`generate_scan(16383)` (Gray scan, full walk)","contributorUrl":"https://github.com/yaroslavvb"}],"isOpen":false,"contested":true},{"id":"symmetry-6bit","name":"Symmetry (6-bit, 100% target)","group":"symmetry","description":"Classify a 6-bit pattern as a palindrome (1) or not (0) at the lowest energy.","metricName":"cost","metricDirection":"minimize","baseline":20,"records":[{"date":"2026-05-11","value":20,"contributor":"baseline","contributorKind":"method","description":"`generate_baseline_six` (3× cmp eq + 2× and)","isBaseline":true,"isCurrentBest":true}],"isOpen":true,"contested":false},{"id":"symmetry-8bit","name":"Symmetry (8-bit, 100% target)","group":"symmetry","description":"Classify an 8-bit pattern as a palindrome (1) or not (0) at the lowest energy.","metricName":"cost","metricDirection":"minimize","baseline":29,"records":[{"date":"2026-05-11","value":29,"contributor":"baseline","contributorKind":"method","description":"`generate_baseline_eight` (4× cmp eq + 3× and)","isBaseline":true,"isCurrentBest":true}],"isOpen":true,"contested":false},{"id":"matmul-4x4","name":"4x4 Matrix Multiplication","group":"matmul","description":"Optimize 4x4 matrix multiplication read-cost under the simplified Bill Dally model.","metricName":"cost","metricDirection":"minimize","baseline":1316,"records":[{"date":"2026-04-29","value":1316,"contributor":"yaroslavvb","contributorKind":"person","description":"`generate_baseline_4x4` (naive)","isBaseline":true,"contributorUrl":"https://github.com/yaroslavvb"},{"date":"2026-04-30","value":800,"contributor":"sjbaebae","contributorKind":"person","description":"`generate_outer_product_4x4` (size-1 sA)","contributorUrl":"https://github.com/sjbaebae"},{"date":"2026-09-01","value":689,"contributor":"jurajselep","contributorKind":"person","description":"row-wise lifetime fusion + dead-A output reuse + exact tier allocation","contributorUrl":"https://github.com/jurajselep"},{"date":"2026-09-01","value":683,"contributor":"npow","contributorKind":"person","description":"HKKK + one deferred output add + exact fixed-schedule allocation","contributorUrl":"https://github.com/npow"},{"date":"2026-09-01","value":681,"contributor":"jurajselep","contributorKind":"person","description":"hybrid row-0 schedule + JIT A staging + interleaved final adds","contributorUrl":"https://github.com/jurajselep"},{"date":"2026-09-04","value":675,"contributor":"jurajselep","contributorKind":"person","description":"selective A/B lifetime splits + dependency-safe scheduling + exact lifetime coloring ★ best","isCurrentBest":true,"contributorUrl":"https://github.com/jurajselep"}],"isOpen":false,"contested":true},{"id":"matmul-16x16","name":"16x16 Matrix Multiplication","group":"matmul","description":"Optimize 16x16 matrix multiplication read-cost under the simplified Bill Dally model.","metricName":"cost","metricDirection":"minimize","baseline":340704,"records":[{"date":"2026-04-29","value":340704,"contributor":"yaroslavvb","contributorKind":"person","description":"`generate_baseline_16x16` (naive)","isBaseline":true,"contributorUrl":"https://github.com/yaroslavvb"},{"date":"2026-05-08","value":237456,"contributor":"yaroslavvb","contributorKind":"person","description":"`generate_recursive_16x16` (1×1-leaf D&C, Z-order)","contributorUrl":"https://github.com/yaroslavvb"},{"date":"2026-04-29","value":133783,"contributor":"yaroslavvb","contributorKind":"person","description":"`generate_tiled_16x16` (4×4 tiles)","contributorUrl":"https://github.com/yaroslavvb"},{"date":"2026-04-30","value":110743,"contributor":"SethTS","contributorKind":"person","description":"`generate_tiled_16x16_opt1` (tmp@1)","agent":{"family":"anthropic","model":"Claude Sonnet 4.6","evidence":"Co-authored-by: Claude Sonnet 4.6 <noreply@anthropic.com>","confidence":"high","role":"author"},"contributorUrl":"https://github.com/SethTS"},{"date":"2026-04-30","value":80217,"contributor":"sjbaebae","contributorKind":"person","description":"`generate_hierarchical_16x16` (asym. reload)","contributorUrl":"https://github.com/sjbaebae"},{"date":"2026-04-30","value":73602,"contributor":"adotzh","contributorKind":"person","description":"sA-cache + sB scratchpad (rank2)","contributorUrl":"https://github.com/adotzh"},{"date":"2026-05-01","value":72642,"contributor":"sjbaebae","contributorKind":"person","description":"+ redirect last-mul to addr 1","contributorUrl":"https://github.com/sjbaebae"},{"date":"2026-05-01","value":71724,"contributor":"sjbaebae","contributorKind":"person","description":"+ last-super-block outputs in sC","contributorUrl":"https://github.com/sjbaebae"},{"date":"2026-05-01","value":70053,"contributor":"sjbaebae","contributorKind":"person","description":"+ dead-input output reuse + B packing","contributorUrl":"https://github.com/sjbaebae"},{"date":"2026-05-06","value":69697,"contributor":"yaroslavvb","contributorKind":"person","description":"C↔A address aliasing + final-add fusion","contributorUrl":"https://github.com/yaroslavvb"},{"date":"2026-05-05","value":68452,"contributor":"zh4ngx","contributorKind":"person","description":"+ column-major order + fused final copy-out","contributorUrl":"https://github.com/zh4ngx"},{"date":"2026-05-13","value":68390,"contributor":"cosminscn","contributorKind":"person","description":"+ liveness order + output-read-aware packing + five-output scratch tail","agent":{"family":"openai","tool":"Codex","confidence":"medium","evidence":"PR branch codex/matmul-68390-five-direct-tail","role":"author"},"contributorUrl":"https://github.com/cosminscn"},{"date":"2026-05-13","value":67834,"contributor":"cosminscn","contributorKind":"person","description":"+ live-B evacuation + output deferral + A staging + value-lifetime coloring","agent":{"family":"openai","tool":"Codex","confidence":"medium","evidence":"PR branch codex/matmul-67834-clean-submission","role":"author"},"contributorUrl":"https://github.com/cosminscn"},{"date":"2026-05-14","value":67821,"contributor":"cosminscn","contributorKind":"person","description":"+ live-B evacuation + output deferral + tiny A-staging mask + staged-reload endpoint lift + value-lifetime coloring","agent":{"family":"openai","tool":"Codex","confidence":"medium","evidence":"PR branch codex/matmul-wave15-67821","role":"author"},"contributorUrl":"https://github.com/cosminscn"},{"date":"2026-05-08","value":66707,"contributor":"sjbaebae","contributorKind":"person","description":"weighted-lifetime pressure search + copy elimination","agent":{"family":"openai","tool":"Codex","confidence":"medium","evidence":"PR branch codex/weighted-lifetime-matmul","role":"author"},"contributorUrl":"https://github.com/sjbaebae"},{"date":"2026-05-25","value":66633,"contributor":"cosminscn","contributorKind":"person","description":"macro B-staging + row-7 later-panel prestaging from addr 1","agent":{"family":"openai","tool":"Codex","confidence":"medium","evidence":"PR branch codex/matmul-66633-macro-b-staging","role":"author"},"contributorUrl":"https://github.com/cosminscn"},{"date":"2026-05-25","value":66524,"contributor":"cosminscn","contributorKind":"person","description":"late B-block cheap capture from addr 1 + value-lifetime coloring","agent":{"family":"openai","tool":"Codex","evidence":"Author: Codex and Cosmin","confidence":"high","role":"author"},"contributorUrl":"https://github.com/cosminscn"},{"date":"2026-05-26","value":66400,"contributor":"cosminscn","contributorKind":"person","description":"late copy-schedule motif bundle + value-lifetime coloring","agent":{"family":"openai","tool":"Codex","confidence":"medium","evidence":"PR branch codex/matmul-66400-motif-bundle","role":"author"},"contributorUrl":"https://github.com/cosminscn"},{"date":"2026-05-28","value":66300,"contributor":"cosminscn","contributorKind":"person","description":"Claude-assisted simulated annealing over a prior leaderboard physical-address IR","agent":{"family":"openai","tool":"Codex","confidence":"medium","evidence":"PR branch codex/matmul-66300-best-submission","role":"author"},"contributorUrl":"https://github.com/cosminscn"},{"date":"2026-08-29","value":66199,"contributor":"sigkillme0","contributorKind":"person","description":"dependency-safe rescheduling + exact tier allocation","contributorUrl":"https://github.com/sigkillme0"},{"date":"2026-08-30","value":66178,"contributor":"sigkillme0","contributorKind":"person","description":"exact LP-optimal address assignment (provably optimal for this operation order)","contributorUrl":"https://github.com/sigkillme0"},{"date":"2026-09-04","value":65084,"contributor":"jurajselep","contributorKind":"person","description":"newest-first snake passes + JIT A staging + local exact tier allocation","contributorUrl":"https://github.com/jurajselep"},{"date":"2026-09-04","value":64431,"contributor":"cosminscn","contributorKind":"person","description":"asymmetric panel schedule + persistent B captures + dependency-safe order search","agent":{"family":"openai","tool":"Codex","confidence":"medium","evidence":"PR branch codex/matmul-record-64431-clean","role":"author"},"contributorUrl":"https://github.com/cosminscn"},{"date":"2026-09-07","value":64074,"contributor":"SecurityQQ","contributorKind":"person","description":"temporary input captures + cheapest surviving replica reads + redundant-copy deletion","contributorUrl":"https://github.com/SecurityQQ"},{"date":"2026-09-07","value":63819,"contributor":"SecurityQQ","contributorKind":"person","description":"6+10 asymmetric panels + dependency-safe scheduling + input captures and address allocation (frozen-artifact verifier)","contributorUrl":"https://github.com/SecurityQQ"},{"date":"2026-09-15","value":63639,"contributor":"jurajselep","contributorKind":"person","description":"input copy-chain and neutral-plan optimization + whole-program allocation with an exact rational certificate","contributorUrl":"https://github.com/jurajselep"},{"date":"2026-09-15","value":63354,"contributor":"jurajselep","contributorKind":"person","description":"exact block reordering + joint reduction-tree and storage allocation","contributorUrl":"https://github.com/jurajselep"},{"date":"2026-09-18","value":63350,"contributor":"jurajselep","contributorKind":"person","description":"joint reduction-tree/storage repair + redundant-copy elimination ★ best","isCurrentBest":true,"contributorUrl":"https://github.com/jurajselep"}],"isOpen":false,"contested":true}],"repo":{"owner":"cybertronai","name":"sutro-problems","branch":"main"},"participants":[{"name":"jurajselep","url":"https://github.com/jurajselep","submissions":12,"problems":["sparse-parity-20","sparse-parity-40","sparse-parity-60","sparse-parity-80","sparse-parity-100","matmul-4x4","matmul-16x16"],"lastActive":"2026-09-18","bestRank":1},{"name":"npow","url":"https://github.com/npow","submissions":9,"problems":["sparse-parity-20","sparse-parity-40","sparse-parity-60","sparse-parity-80","sparse-parity-100","matmul-4x4"],"lastActive":"2026-09-01","bestRank":1},{"name":"b0nce","url":"https://github.com/b0nce","submissions":5,"problems":["sparse-parity-20","sparse-parity-40","sparse-parity-60","sparse-parity-80","sparse-parity-100"],"bestRank":1,"lastActive":"2026-09-02"},{"name":"yaroslavvb","url":"https://github.com/yaroslavvb","submissions":14,"problems":["sparse-parity-20","sparse-parity-40","sparse-parity-60","sparse-parity-80","sparse-parity-100","matmul-4x4","matmul-16x16"],"lastActive":"2026-08-27"},{"name":"zh4ngx","url":"https://github.com/zh4ngx","submissions":8,"problems":["sparse-parity-20","sparse-parity-40","sparse-parity-60","sparse-parity-80","sparse-parity-100","matmul-16x16"],"lastActive":"2026-08-30"},{"name":"cosminscn","url":"https://github.com/cosminscn","submissions":8,"problems":["matmul-16x16"],"lastActive":"2026-09-04"},{"name":"sjbaebae","url":"https://github.com/sjbaebae","submissions":6,"problems":["matmul-4x4","matmul-16x16"],"lastActive":"2026-05-08"},{"name":"sigkillme0","url":"https://github.com/sigkillme0","submissions":2,"problems":["matmul-16x16"],"lastActive":"2026-08-30"},{"name":"SecurityQQ","url":"https://github.com/SecurityQQ","submissions":2,"problems":["matmul-16x16"],"lastActive":"2026-09-07"},{"name":"SethTS","url":"https://github.com/SethTS","submissions":1,"problems":["matmul-16x16"],"lastActive":"2026-04-30"},{"name":"adotzh","url":"https://github.com/adotzh","submissions":1,"problems":["matmul-16x16"],"lastActive":"2026-04-30"}],"stats":{"totalRecords":83,"uniqueParticipants":11,"firstSubmission":"2026-04-29","lastSubmission":"2026-09-18","recordsLast30d":60,"recordsLast90d":60},"agentStats":[{"family":"openai","records":18,"currentBests":3,"models":[],"contributors":["H32 MLP","512-unit MLP (96% target)","PCA-QDA","Reversible MLP, smaller workspace (12% error target)","32-unit MLP","1NN","Ordered ConvNets (3% error target)","Three ConvNets","512-unit MLP","cosminscn","sjbaebae"],"uncontestedBests":1},{"family":"anthropic","records":2,"currentBests":1,"models":["Claude Fable 5.1","Claude Sonnet 4.6"],"contributors":["QDA","SethTS"]}],"lastUpdated":"2026-09-18T16:32:55.539Z"}],"errors":[]}